Debian Codegraph @

src/atlas/linalg/sparse/SparseMatrixMultiply.tcc

raw
1 /*
2 * (C) Copyright 2013 ECMWF.
3 *
4 * This software is licensed under the terms of the Apache Licence Version 2.0
5 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6 * In applying this licence, ECMWF does not waive the privileges and immunities
7 * granted to it by virtue of its status as an intergovernmental organisation
8 * nor does it submit to any jurisdiction.
9 */
10
11 #pragma once
12
13 #include "SparseMatrixMultiply.h"
14
15 #include "eckit/linalg/SparseMatrix.h"
16
17 #include "atlas/linalg/Indexing.h"
18 #include "atlas/linalg/Introspection.h"
19 #include "atlas/linalg/View.h"
20 #include "atlas/linalg/sparse/Backend.h"
21 #include "atlas/linalg/sparse/SparseMatrixView.h"
22 #include "atlas/runtime/Exception.h"
23
24 #if ATLAS_ECKIT_HAVE_ECKIT_585
25 #include "eckit/linalg/LinearAlgebraSparse.h"
26 #else
27 #include "eckit/linalg/LinearAlgebra.h"
28 #endif
29
30
31 namespace atlas {
32 namespace linalg {
33
34 namespace sparse {
35 namespace {
36
37 inline SparseMatrixView<eckit::linalg::Scalar,eckit::linalg::Index> make_view_from_eckit_sparse_matrix(const eckit::linalg::SparseMatrix& m) {
38 return SparseMatrixView<eckit::linalg::Scalar,eckit::linalg::Index>{
39 m.rows(),
40 m.cols(),
41 m.nonZeros(),
42 m.data(),
43 m.inner(),
44 m.outer()
45 };
46 }
47
48 template <typename Backend, Indexing indexing>
49 struct SparseMatrixMultiplyHelper {
50 template <typename Matrix, typename SourceView, typename TargetView>
51 static void multiply( const Matrix& W, const SourceView& src, TargetView& tgt,
52 const eckit::Configuration& config ) {
53 using MatrixValue = typename std::remove_const<typename Matrix::value_type>::type;
54 using SourceValue = const typename std::remove_const<typename SourceView::value_type>::type;
55 using TargetValue = typename std::remove_const<typename TargetView::value_type>::type;
56 constexpr int src_rank = introspection::rank<SourceView>();
57 constexpr int tgt_rank = introspection::rank<TargetView>();
58 static_assert( src_rank == tgt_rank, "src and tgt need same rank" );
59 SparseMatrixMultiply<Backend, indexing, src_rank, MatrixValue, SourceValue, TargetValue>::multiply( W, src, tgt, config );
60 }
61
62 template <typename Matrix, typename SourceView, typename TargetView>
63 static void multiply_add( const Matrix& W, const SourceView& src, TargetView& tgt,
64 const eckit::Configuration& config ) {
65 using MatrixValue = typename std::remove_const<typename Matrix::value_type>::type;
66 using SourceValue = const typename std::remove_const<typename SourceView::value_type>::type;
67 using TargetValue = typename std::remove_const<typename TargetView::value_type>::type;
68 constexpr int src_rank = introspection::rank<SourceView>();
69 constexpr int tgt_rank = introspection::rank<TargetView>();
70 static_assert( src_rank == tgt_rank, "src and tgt need same rank" );
71 SparseMatrixMultiply<Backend, indexing, src_rank, MatrixValue, SourceValue, TargetValue>::multiply_add( W, src, tgt, config );
72 }
73 template <typename SourceView, typename TargetView>
74 static void multiply( const eckit::linalg::SparseMatrix& W, const SourceView& src, TargetView& tgt,
75 const eckit::Configuration& config ) {
76 using MatrixValue = double;
77 // using MatrixIndex = int;
78 using SourceValue = const typename std::remove_const<typename SourceView::value_type>::type;
79 using TargetValue = typename std::remove_const<typename TargetView::value_type>::type;
80 constexpr int src_rank = introspection::rank<SourceView>();
81 constexpr int tgt_rank = introspection::rank<TargetView>();
82 static_assert( src_rank == tgt_rank, "src and tgt need same rank" );
83 SparseMatrixMultiply<Backend, indexing, src_rank, MatrixValue, SourceValue, TargetValue>::multiply( make_view_from_eckit_sparse_matrix(W), src, tgt, config );
84 }
85
86 template <typename SourceView, typename TargetView>
87 static void multiply_add( const eckit::linalg::SparseMatrix& W, const SourceView& src, TargetView& tgt,
88 const eckit::Configuration& config ) {
89 using MatrixValue = double;
90 // using MatrixIndex = int;
91 using SourceValue = const typename std::remove_const<typename SourceView::value_type>::type;
92 using TargetValue = typename std::remove_const<typename TargetView::value_type>::type;
93 constexpr int src_rank = introspection::rank<SourceView>();
94 constexpr int tgt_rank = introspection::rank<TargetView>();
95 static_assert( src_rank == tgt_rank, "src and tgt need same rank" );
96 SparseMatrixMultiply<Backend, indexing, src_rank, MatrixValue, SourceValue, TargetValue>::multiply_add( make_view_from_eckit_sparse_matrix(W), src, tgt, config );
97 }
98 };
99
100 template <typename Backend, typename Matrix, typename SourceView, typename TargetView>
101 void dispatch_sparse_matrix_multiply( const Matrix& matrix, const SourceView& src, TargetView& tgt, Indexing indexing,
102 const eckit::Configuration& config ) {
103 auto src_v = make_view( src );
104 auto tgt_v = make_view( tgt );
105
106 if ( introspection::layout_right( src ) || introspection::layout_right( tgt ) ) {
107 ATLAS_ASSERT( introspection::layout_right( src ) && introspection::layout_right( tgt ) );
108 // Override layout with known layout given by introspection
109 SparseMatrixMultiplyHelper<Backend, linalg::Indexing::layout_right>::multiply( matrix, src_v, tgt_v, config );
110 }
111 else {
112 if( indexing == Indexing::layout_left ) {
113 SparseMatrixMultiplyHelper<Backend, Indexing::layout_left>::multiply( matrix, src_v, tgt_v, config );
114 }
115 else if( indexing == Indexing::layout_right ) {
116 SparseMatrixMultiplyHelper<Backend, Indexing::layout_right>::multiply( matrix, src_v, tgt_v, config );
117 }
118 else {
119 throw_NotImplemented( "indexing not implemented", Here() );
120 }
121 }
122 }
123
124 template <typename Backend, typename Matrix, typename SourceView, typename TargetView>
125 void dispatch_sparse_matrix_multiply_add( const Matrix& matrix, const SourceView& src, TargetView& tgt, Indexing indexing,
126 const eckit::Configuration& config ) {
127 auto src_v = make_view( src );
128 auto tgt_v = make_view( tgt );
129
130 if ( introspection::layout_right( src ) || introspection::layout_right( tgt ) ) {
131 ATLAS_ASSERT( introspection::layout_right( src ) && introspection::layout_right( tgt ) );
132 // Override layout with known layout given by introspection
133 SparseMatrixMultiplyHelper<Backend, linalg::Indexing::layout_right>::multiply_add( matrix, src_v, tgt_v, config );
134 }
135 else {
136 if( indexing == Indexing::layout_left ) {
137 SparseMatrixMultiplyHelper<Backend, Indexing::layout_left>::multiply_add( matrix, src_v, tgt_v, config );
138 }
139 else if( indexing == Indexing::layout_right ) {
140 SparseMatrixMultiplyHelper<Backend, Indexing::layout_right>::multiply_add( matrix, src_v, tgt_v, config );
141 }
142 else {
143 throw_NotImplemented( "indexing not implemented", Here() );
144 }
145 }
146 }
147 }
148 }
149
150 template <typename Matrix, typename SourceView, typename TargetView>
151 void sparse_matrix_multiply( const Matrix& matrix, const SourceView& src, TargetView& tgt, Indexing indexing,
152 const eckit::Configuration& config ) {
153 std::string type = config.getString( "type", sparse::current_backend() );
154 if ( type == sparse::backend::openmp::type() ) {
155 sparse::dispatch_sparse_matrix_multiply<sparse::backend::openmp>( matrix, src, tgt, indexing, config );
156 }
157 else if ( type == sparse::backend::eckit_linalg::type() ) {
158 sparse::dispatch_sparse_matrix_multiply<sparse::backend::eckit_linalg>( matrix, src, tgt, indexing, config );
159 }
160 #if ATLAS_ECKIT_HAVE_ECKIT_585
161 else if( eckit::linalg::LinearAlgebraSparse::hasBackend(type) ) {
162 #else
163 else if( eckit::linalg::LinearAlgebra::hasBackend(type) ) {
164 #endif
165 sparse::dispatch_sparse_matrix_multiply<sparse::backend::eckit_linalg>( matrix, src, tgt, indexing, util::Config("backend",type) );
166 }
167 else if ( type == sparse::backend::hicsparse::type() ) {
168 sparse::dispatch_sparse_matrix_multiply<sparse::backend::hicsparse>( matrix, src, tgt, indexing, config );
169 }
170 else {
171 throw_NotImplemented( "sparse_matrix_multiply cannot be performed with unsupported backend [" + type + "]",
172 Here() );
173 }
174 }
175
176 template <typename Matrix, typename SourceView, typename TargetView>
177 void sparse_matrix_multiply( const Matrix& matrix, const SourceView& src, TargetView& tgt, const eckit::Configuration& config ) {
178 sparse_matrix_multiply( matrix, src, tgt, Indexing::layout_left, config );
179 }
180
181 template <typename Matrix, typename SourceView, typename TargetView>
182 void sparse_matrix_multiply( const Matrix& matrix, const SourceView& src, TargetView& tgt, Indexing indexing ) {
183 sparse_matrix_multiply( matrix, src, tgt, indexing, sparse::Backend() );
184 }
185
186 template <typename Matrix, typename SourceView, typename TargetView>
187 void sparse_matrix_multiply( const Matrix& matrix, const SourceView& src, TargetView& tgt ) {
188 sparse_matrix_multiply( matrix, src, tgt, Indexing::layout_left );
189 }
190
191 template <typename Matrix, typename SourceView, typename TargetView>
192 void sparse_matrix_multiply(const Matrix& matrix, const SourceView& src, TargetView&& tgt) {
193 auto& tgt_lvalue = tgt;
194 sparse_matrix_multiply( matrix, src, tgt_lvalue, Indexing::layout_left );
195 }
196
197
198 template <typename Matrix, typename SourceView, typename TargetView>
199 void sparse_matrix_multiply_add( const Matrix& matrix, const SourceView& src, TargetView& tgt, Indexing indexing,
200 const eckit::Configuration& config ) {
201 std::string type = config.getString( "type", sparse::current_backend() );
202 if ( type == sparse::backend::openmp::type() ) {
203 sparse::dispatch_sparse_matrix_multiply_add<sparse::backend::openmp>( matrix, src, tgt, indexing, config );
204 }
205 else if ( type == sparse::backend::eckit_linalg::type() ) {
206 sparse::dispatch_sparse_matrix_multiply_add<sparse::backend::eckit_linalg>( matrix, src, tgt, indexing, config );
207 }
208 #if ATLAS_ECKIT_HAVE_ECKIT_585
209 else if( eckit::linalg::LinearAlgebraSparse::hasBackend(type) ) {
210 #else
211 else if( eckit::linalg::LinearAlgebra::hasBackend(type) ) {
212 #endif
213 sparse::dispatch_sparse_matrix_multiply_add<sparse::backend::eckit_linalg>( matrix, src, tgt, indexing, util::Config("backend",type) );
214 }
215 else if ( type == sparse::backend::hicsparse::type() ) {
216 sparse::dispatch_sparse_matrix_multiply_add<sparse::backend::hicsparse>( matrix, src, tgt, indexing, config );
217 }
218 else {
219 throw_NotImplemented( "sparse_matrix_multiply_add cannot be performed with unsupported backend [" + type + "]",
220 Here() );
221 }
222 }
223
224 template <typename Matrix, typename SourceView, typename TargetView>
225 void sparse_matrix_multiply_add( const Matrix& matrix, const SourceView& src, TargetView& tgt, const eckit::Configuration& config ) {
226 sparse_matrix_multiply_add( matrix, src, tgt, Indexing::layout_left, config );
227 }
228
229 template <typename Matrix, typename SourceView, typename TargetView>
230 void sparse_matrix_multiply_add( const Matrix& matrix, const SourceView& src, TargetView& tgt, Indexing indexing ) {
231 sparse_matrix_multiply_add( matrix, src, tgt, indexing, sparse::Backend() );
232 }
233
234 template <typename Matrix, typename SourceView, typename TargetView>
235 void sparse_matrix_multiply_add( const Matrix& matrix, const SourceView& src, TargetView& tgt ) {
236 sparse_matrix_multiply_add( matrix, src, tgt, Indexing::layout_left );
237 }
238
239 } // namespace linalg
240 } // namespace atlas
241
242 #undef ATLAS_ENABLE_IF