/* * (C) Copyright 2013 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "SparseMatrixMultiply.h" #include "eckit/linalg/SparseMatrix.h" #include "atlas/linalg/Indexing.h" #include "atlas/linalg/Introspection.h" #include "atlas/linalg/View.h" #include "atlas/linalg/sparse/Backend.h" #include "atlas/linalg/sparse/SparseMatrixView.h" #include "atlas/runtime/Exception.h" #if ATLAS_ECKIT_HAVE_ECKIT_585 #include "eckit/linalg/LinearAlgebraSparse.h" #else #include "eckit/linalg/LinearAlgebra.h" #endif namespace atlas { namespace linalg { namespace sparse { namespace { inline SparseMatrixView make_view_from_eckit_sparse_matrix(const eckit::linalg::SparseMatrix& m) { return SparseMatrixView{ m.rows(), m.cols(), m.nonZeros(), m.data(), m.inner(), m.outer() }; } template struct SparseMatrixMultiplyHelper { template static void multiply( const Matrix& W, const SourceView& src, TargetView& tgt, const eckit::Configuration& config ) { using MatrixValue = typename std::remove_const::type; using SourceValue = const typename std::remove_const::type; using TargetValue = typename std::remove_const::type; constexpr int src_rank = introspection::rank(); constexpr int tgt_rank = introspection::rank(); static_assert( src_rank == tgt_rank, "src and tgt need same rank" ); SparseMatrixMultiply::multiply( W, src, tgt, config ); } template static void multiply_add( const Matrix& W, const SourceView& src, TargetView& tgt, const eckit::Configuration& config ) { using MatrixValue = typename std::remove_const::type; using SourceValue = const typename std::remove_const::type; using TargetValue = typename std::remove_const::type; constexpr int src_rank = introspection::rank(); constexpr int tgt_rank = introspection::rank(); static_assert( src_rank == tgt_rank, "src and tgt need same rank" ); SparseMatrixMultiply::multiply_add( W, src, tgt, config ); } template static void multiply( const eckit::linalg::SparseMatrix& W, const SourceView& src, TargetView& tgt, const eckit::Configuration& config ) { using MatrixValue = double; // using MatrixIndex = int; using SourceValue = const typename std::remove_const::type; using TargetValue = typename std::remove_const::type; constexpr int src_rank = introspection::rank(); constexpr int tgt_rank = introspection::rank(); static_assert( src_rank == tgt_rank, "src and tgt need same rank" ); SparseMatrixMultiply::multiply( make_view_from_eckit_sparse_matrix(W), src, tgt, config ); } template static void multiply_add( const eckit::linalg::SparseMatrix& W, const SourceView& src, TargetView& tgt, const eckit::Configuration& config ) { using MatrixValue = double; // using MatrixIndex = int; using SourceValue = const typename std::remove_const::type; using TargetValue = typename std::remove_const::type; constexpr int src_rank = introspection::rank(); constexpr int tgt_rank = introspection::rank(); static_assert( src_rank == tgt_rank, "src and tgt need same rank" ); SparseMatrixMultiply::multiply_add( make_view_from_eckit_sparse_matrix(W), src, tgt, config ); } }; template void dispatch_sparse_matrix_multiply( const Matrix& matrix, const SourceView& src, TargetView& tgt, Indexing indexing, const eckit::Configuration& config ) { auto src_v = make_view( src ); auto tgt_v = make_view( tgt ); if ( introspection::layout_right( src ) || introspection::layout_right( tgt ) ) { ATLAS_ASSERT( introspection::layout_right( src ) && introspection::layout_right( tgt ) ); // Override layout with known layout given by introspection SparseMatrixMultiplyHelper::multiply( matrix, src_v, tgt_v, config ); } else { if( indexing == Indexing::layout_left ) { SparseMatrixMultiplyHelper::multiply( matrix, src_v, tgt_v, config ); } else if( indexing == Indexing::layout_right ) { SparseMatrixMultiplyHelper::multiply( matrix, src_v, tgt_v, config ); } else { throw_NotImplemented( "indexing not implemented", Here() ); } } } template void dispatch_sparse_matrix_multiply_add( const Matrix& matrix, const SourceView& src, TargetView& tgt, Indexing indexing, const eckit::Configuration& config ) { auto src_v = make_view( src ); auto tgt_v = make_view( tgt ); if ( introspection::layout_right( src ) || introspection::layout_right( tgt ) ) { ATLAS_ASSERT( introspection::layout_right( src ) && introspection::layout_right( tgt ) ); // Override layout with known layout given by introspection SparseMatrixMultiplyHelper::multiply_add( matrix, src_v, tgt_v, config ); } else { if( indexing == Indexing::layout_left ) { SparseMatrixMultiplyHelper::multiply_add( matrix, src_v, tgt_v, config ); } else if( indexing == Indexing::layout_right ) { SparseMatrixMultiplyHelper::multiply_add( matrix, src_v, tgt_v, config ); } else { throw_NotImplemented( "indexing not implemented", Here() ); } } } } } template void sparse_matrix_multiply( const Matrix& matrix, const SourceView& src, TargetView& tgt, Indexing indexing, const eckit::Configuration& config ) { std::string type = config.getString( "type", sparse::current_backend() ); if ( type == sparse::backend::openmp::type() ) { sparse::dispatch_sparse_matrix_multiply( matrix, src, tgt, indexing, config ); } else if ( type == sparse::backend::eckit_linalg::type() ) { sparse::dispatch_sparse_matrix_multiply( matrix, src, tgt, indexing, config ); } #if ATLAS_ECKIT_HAVE_ECKIT_585 else if( eckit::linalg::LinearAlgebraSparse::hasBackend(type) ) { #else else if( eckit::linalg::LinearAlgebra::hasBackend(type) ) { #endif sparse::dispatch_sparse_matrix_multiply( matrix, src, tgt, indexing, util::Config("backend",type) ); } else if ( type == sparse::backend::hicsparse::type() ) { sparse::dispatch_sparse_matrix_multiply( matrix, src, tgt, indexing, config ); } else { throw_NotImplemented( "sparse_matrix_multiply cannot be performed with unsupported backend [" + type + "]", Here() ); } } template void sparse_matrix_multiply( const Matrix& matrix, const SourceView& src, TargetView& tgt, const eckit::Configuration& config ) { sparse_matrix_multiply( matrix, src, tgt, Indexing::layout_left, config ); } template void sparse_matrix_multiply( const Matrix& matrix, const SourceView& src, TargetView& tgt, Indexing indexing ) { sparse_matrix_multiply( matrix, src, tgt, indexing, sparse::Backend() ); } template void sparse_matrix_multiply( const Matrix& matrix, const SourceView& src, TargetView& tgt ) { sparse_matrix_multiply( matrix, src, tgt, Indexing::layout_left ); } template void sparse_matrix_multiply(const Matrix& matrix, const SourceView& src, TargetView&& tgt) { auto& tgt_lvalue = tgt; sparse_matrix_multiply( matrix, src, tgt_lvalue, Indexing::layout_left ); } template void sparse_matrix_multiply_add( const Matrix& matrix, const SourceView& src, TargetView& tgt, Indexing indexing, const eckit::Configuration& config ) { std::string type = config.getString( "type", sparse::current_backend() ); if ( type == sparse::backend::openmp::type() ) { sparse::dispatch_sparse_matrix_multiply_add( matrix, src, tgt, indexing, config ); } else if ( type == sparse::backend::eckit_linalg::type() ) { sparse::dispatch_sparse_matrix_multiply_add( matrix, src, tgt, indexing, config ); } #if ATLAS_ECKIT_HAVE_ECKIT_585 else if( eckit::linalg::LinearAlgebraSparse::hasBackend(type) ) { #else else if( eckit::linalg::LinearAlgebra::hasBackend(type) ) { #endif sparse::dispatch_sparse_matrix_multiply_add( matrix, src, tgt, indexing, util::Config("backend",type) ); } else if ( type == sparse::backend::hicsparse::type() ) { sparse::dispatch_sparse_matrix_multiply_add( matrix, src, tgt, indexing, config ); } else { throw_NotImplemented( "sparse_matrix_multiply_add cannot be performed with unsupported backend [" + type + "]", Here() ); } } template void sparse_matrix_multiply_add( const Matrix& matrix, const SourceView& src, TargetView& tgt, const eckit::Configuration& config ) { sparse_matrix_multiply_add( matrix, src, tgt, Indexing::layout_left, config ); } template void sparse_matrix_multiply_add( const Matrix& matrix, const SourceView& src, TargetView& tgt, Indexing indexing ) { sparse_matrix_multiply_add( matrix, src, tgt, indexing, sparse::Backend() ); } template void sparse_matrix_multiply_add( const Matrix& matrix, const SourceView& src, TargetView& tgt ) { sparse_matrix_multiply_add( matrix, src, tgt, Indexing::layout_left ); } } // namespace linalg } // namespace atlas #undef ATLAS_ENABLE_IF