/* * (C) Copyright 2013 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "atlas/linalg/sparse/SparseMatrixMultiply_OpenMP.h" #include "atlas/parallel/omp/omp.h" #include "atlas/runtime/Exception.h" namespace atlas { namespace linalg { namespace sparse { template void spmv_layout_left(const SparseMatrixView& W, const View& src, View& tgt) { using Value = TargetValue; const auto outer = W.outer(); const auto index = W.inner(); const auto weight = W.value(); const idx_t rows = static_cast(W.rows()); ATLAS_ASSERT(src.shape(0) >= W.cols()); ATLAS_ASSERT(tgt.shape(0) >= W.rows()); atlas_omp_parallel_for(idx_t r = 0; r < rows; ++r) { if constexpr (SetZero) { tgt[r] = 0.; } for (idx_t c = outer[r]; c < outer[r + 1]; ++c) { idx_t n = index[c]; Value w = static_cast(weight[c]); tgt[r] += w * src[n]; } } } template void SparseMatrixMultiply::multiply( const SparseMatrixView& W, const View& src, View& tgt, const Configuration&) { spmv_layout_left(W, src, tgt); } template void SparseMatrixMultiply::multiply_add( const SparseMatrixView& W, const View& src, View& tgt, const Configuration&) { spmv_layout_left(W, src, tgt); } template void spmm_layout_left(const SparseMatrixView& W, const View& src, View& tgt) { using Value = TargetValue; const auto outer = W.outer(); const auto index = W.inner(); const auto weight = W.value(); const idx_t rows = static_cast(W.rows()); const idx_t Nk = src.shape(1); ATLAS_ASSERT(src.shape(0) >= W.cols()); ATLAS_ASSERT(tgt.shape(0) >= W.rows()); atlas_omp_parallel_for(idx_t r = 0; r < rows; ++r) { if constexpr (SetZero) { for (idx_t k = 0; k < Nk; ++k) { tgt(r, k) = 0.; } } for (idx_t c = outer[r]; c < outer[r + 1]; ++c) { idx_t n = index[c]; Value w = static_cast(weight[c]); for (idx_t k = 0; k < Nk; ++k) { tgt(r, k) += w * src(n, k); } } } } template void SparseMatrixMultiply::multiply( const SparseMatrixView& W, const View& src, View& tgt, const Configuration&) { spmm_layout_left(W, src, tgt); } template void SparseMatrixMultiply::multiply_add( const SparseMatrixView& W, const View& src, View& tgt, const Configuration&) { spmm_layout_left(W, src, tgt); } template void spmt_layout_left(const SparseMatrixView& W, const View& src, View& tgt) { if (src.contiguous() && tgt.contiguous()) { // We can take a more optimized route by reducing rank auto src_v = View(src.data(), array::make_shape(src.shape(0), src.stride(0))); auto tgt_v = View(tgt.data(), array::make_shape(tgt.shape(0), tgt.stride(0))); spmm_layout_left(W, src_v, tgt_v); return; } using Value = TargetValue; const auto outer = W.outer(); const auto index = W.inner(); const auto weight = W.value(); const idx_t rows = static_cast(W.rows()); const idx_t Nk = src.shape(1); const idx_t Nl = src.shape(2); atlas_omp_parallel_for(idx_t r = 0; r < rows; ++r) { if constexpr (SetZero) { for (idx_t k = 0; k < Nk; ++k) { for (idx_t l = 0; l < Nl; ++l) { tgt(r, k, l) = 0.; } } } for (idx_t c = outer[r]; c < outer[r + 1]; ++c) { idx_t n = index[c]; const Value w = static_cast(weight[c]); for (idx_t k = 0; k < Nk; ++k) { for (idx_t l = 0; l < Nl; ++l) { tgt(r, k, l) += w * src(n, k, l); } } } } } template void SparseMatrixMultiply::multiply( const SparseMatrixView& W, const View& src, View& tgt, const Configuration& config) { spmt_layout_left(W, src, tgt); } template void SparseMatrixMultiply::multiply_add( const SparseMatrixView& W, const View& src, View& tgt, const Configuration& config) { spmt_layout_left(W, src, tgt); } template void SparseMatrixMultiply::multiply( const SparseMatrixView& W, const View& src, View& tgt, const Configuration& config) { return SparseMatrixMultiply::multiply(W, src, tgt, config); } template void SparseMatrixMultiply::multiply_add( const SparseMatrixView& W, const View& src, View& tgt, const Configuration& config) { return SparseMatrixMultiply::multiply_add(W, src, tgt, config); } template void spmm_layout_right(const SparseMatrixView& W, const View& src, View& tgt) { using Value = TargetValue; const auto outer = W.outer(); const auto index = W.inner(); const auto weight = W.value(); const idx_t rows = static_cast(W.rows()); const idx_t Nk = src.shape(0); ATLAS_ASSERT(src.shape(1) >= W.cols()); ATLAS_ASSERT(tgt.shape(1) >= W.rows()); atlas_omp_parallel_for(idx_t r = 0; r < rows; ++r) { if constexpr (SetZero) { for (idx_t k = 0; k < Nk; ++k) { tgt(k, r) = 0.; } } for (idx_t c = outer[r]; c < outer[r + 1]; ++c) { idx_t n = index[c]; Value w = static_cast(weight[c]); for (idx_t k = 0; k < Nk; ++k) { tgt(k, r) += w * src(k, n); } } } } template void SparseMatrixMultiply::multiply( const SparseMatrixView& W, const View& src, View& tgt, const Configuration&) { spmm_layout_right(W, src, tgt); } template void SparseMatrixMultiply::multiply_add( const SparseMatrixView& W, const View& src, View& tgt, const Configuration&) { spmm_layout_right(W, src, tgt); } template void spmt_layout_right(const SparseMatrixView& W, const View& src, View& tgt) { if (src.contiguous() && tgt.contiguous()) { // We can take a more optimized route by reducing rank auto src_v = View(src.data(), array::make_shape(src.shape(0), src.stride(0))); auto tgt_v = View(tgt.data(), array::make_shape(tgt.shape(0), tgt.stride(0))); spmm_layout_right(W, src_v, tgt_v); return; } using Value = TargetValue; const auto outer = W.outer(); const auto index = W.inner(); const auto weight = W.value(); const idx_t rows = static_cast(W.rows()); const idx_t Nk = src.shape(1); const idx_t Nl = src.shape(0); atlas_omp_parallel_for(idx_t r = 0; r < rows; ++r) { if constexpr (SetZero){ for (idx_t k = 0; k < Nk; ++k) { for (idx_t l = 0; l < Nl; ++l) { tgt(l, k, r) = 0.; } } } for (idx_t c = outer[r]; c < outer[r + 1]; ++c) { idx_t n = index[c]; const Value w = static_cast(weight[c]); for (idx_t k = 0; k < Nk; ++k) { for (idx_t l = 0; l < Nl; ++l) { tgt(l, k, r) += w * src(l, k, n); } } } } } template void SparseMatrixMultiply::multiply( const SparseMatrixView& W, const View& src, View& tgt, const Configuration& config) { spmt_layout_right(W, src, tgt); } template void SparseMatrixMultiply::multiply_add( const SparseMatrixView& W, const View& src, View& tgt, const Configuration& config) { spmt_layout_right(W, src, tgt); } #define EXPLICIT_TEMPLATE_INSTANTIATION(TYPE) \ template struct SparseMatrixMultiply; \ template struct SparseMatrixMultiply; \ template struct SparseMatrixMultiply; \ template struct SparseMatrixMultiply; \ template struct SparseMatrixMultiply; \ template struct SparseMatrixMultiply; \ template struct SparseMatrixMultiply; \ template struct SparseMatrixMultiply; \ template struct SparseMatrixMultiply; \ template struct SparseMatrixMultiply; \ template struct SparseMatrixMultiply; \ template struct SparseMatrixMultiply; EXPLICIT_TEMPLATE_INSTANTIATION(double); EXPLICIT_TEMPLATE_INSTANTIATION(float); } // namespace sparse } // namespace linalg } // namespace atlas