Debian Codegraph @

src/atlas/linalg/sparse/SparseMatrixMultiply_OpenMP.cc

raw
1 /*
2 * (C) Copyright 2013 ECMWF.
3 *
4 * This software is licensed under the terms of the Apache Licence Version 2.0
5 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6 * In applying this licence, ECMWF does not waive the privileges and immunities
7 * granted to it by virtue of its status as an intergovernmental organisation
8 * nor does it submit to any jurisdiction.
9 */
10
11 #include "atlas/linalg/sparse/SparseMatrixMultiply_OpenMP.h"
12
13 #include "atlas/parallel/omp/omp.h"
14 #include "atlas/runtime/Exception.h"
15
16 namespace atlas {
17 namespace linalg {
18 namespace sparse {
19
20 template <bool SetZero, typename MatrixValue, typename SourceValue, typename TargetValue>
21 void spmv_layout_left(const SparseMatrixView<MatrixValue>& W, const View<SourceValue, 1>& src, View<TargetValue, 1>& tgt) {
22 using Value = TargetValue;
23 const auto outer = W.outer();
24 const auto index = W.inner();
25 const auto weight = W.value();
26 const idx_t rows = static_cast<idx_t>(W.rows());
27
28 ATLAS_ASSERT(src.shape(0) >= W.cols());
29 ATLAS_ASSERT(tgt.shape(0) >= W.rows());
30
31 atlas_omp_parallel_for(idx_t r = 0; r < rows; ++r) {
32 if constexpr (SetZero) {
33 tgt[r] = 0.;
34 }
35 for (idx_t c = outer[r]; c < outer[r + 1]; ++c) {
36 idx_t n = index[c];
37 Value w = static_cast<Value>(weight[c]);
38 tgt[r] += w * src[n];
39 }
40 }
41 }
42
43 template <typename MatrixValue, typename SourceValue, typename TargetValue>
44 void SparseMatrixMultiply<backend::openmp, Indexing::layout_left, 1, MatrixValue, SourceValue, TargetValue>::multiply(
45 const SparseMatrixView<MatrixValue>& W, const View<SourceValue, 1>& src, View<TargetValue, 1>& tgt, const Configuration&) {
46 spmv_layout_left<true>(W, src, tgt);
47 }
48
49 template <typename MatrixValue, typename SourceValue, typename TargetValue>
50 void SparseMatrixMultiply<backend::openmp, Indexing::layout_left, 1, MatrixValue, SourceValue, TargetValue>::multiply_add(
51 const SparseMatrixView<MatrixValue>& W, const View<SourceValue, 1>& src, View<TargetValue, 1>& tgt, const Configuration&) {
52 spmv_layout_left<false>(W, src, tgt);
53 }
54
55 template <bool SetZero, typename MatrixValue, typename SourceValue, typename TargetValue>
56 void spmm_layout_left(const SparseMatrixView<MatrixValue>& W, const View<SourceValue, 2>& src, View<TargetValue, 2>& tgt) {
57 using Value = TargetValue;
58 const auto outer = W.outer();
59 const auto index = W.inner();
60 const auto weight = W.value();
61 const idx_t rows = static_cast<idx_t>(W.rows());
62 const idx_t Nk = src.shape(1);
63
64 ATLAS_ASSERT(src.shape(0) >= W.cols());
65 ATLAS_ASSERT(tgt.shape(0) >= W.rows());
66
67 atlas_omp_parallel_for(idx_t r = 0; r < rows; ++r) {
68 if constexpr (SetZero) {
69 for (idx_t k = 0; k < Nk; ++k) {
70 tgt(r, k) = 0.;
71 }
72 }
73 for (idx_t c = outer[r]; c < outer[r + 1]; ++c) {
74 idx_t n = index[c];
75 Value w = static_cast<Value>(weight[c]);
76 for (idx_t k = 0; k < Nk; ++k) {
77 tgt(r, k) += w * src(n, k);
78 }
79 }
80 }
81 }
82
83 template <typename MatrixValue, typename SourceValue, typename TargetValue>
84 void SparseMatrixMultiply<backend::openmp, Indexing::layout_left, 2, MatrixValue, SourceValue, TargetValue>::multiply(
85 const SparseMatrixView<MatrixValue>& W, const View<SourceValue, 2>& src, View<TargetValue, 2>& tgt, const Configuration&) {
86 spmm_layout_left<true>(W, src, tgt);
87 }
88
89 template <typename MatrixValue, typename SourceValue, typename TargetValue>
90 void SparseMatrixMultiply<backend::openmp, Indexing::layout_left, 2, MatrixValue, SourceValue, TargetValue>::multiply_add(
91 const SparseMatrixView<MatrixValue>& W, const View<SourceValue, 2>& src, View<TargetValue, 2>& tgt, const Configuration&) {
92 spmm_layout_left<false>(W, src, tgt);
93 }
94
95 template <bool SetZero,typename MatrixValue, typename SourceValue, typename TargetValue>
96 void spmt_layout_left(const SparseMatrixView<MatrixValue>& W, const View<SourceValue, 3>& src, View<TargetValue, 3>& tgt) {
97 if (src.contiguous() && tgt.contiguous()) {
98 // We can take a more optimized route by reducing rank
99 auto src_v = View<SourceValue, 2>(src.data(), array::make_shape(src.shape(0), src.stride(0)));
100 auto tgt_v = View<TargetValue, 2>(tgt.data(), array::make_shape(tgt.shape(0), tgt.stride(0)));
101 spmm_layout_left<SetZero>(W, src_v, tgt_v);
102 return;
103 }
104 using Value = TargetValue;
105 const auto outer = W.outer();
106 const auto index = W.inner();
107 const auto weight = W.value();
108 const idx_t rows = static_cast<idx_t>(W.rows());
109 const idx_t Nk = src.shape(1);
110 const idx_t Nl = src.shape(2);
111
112 atlas_omp_parallel_for(idx_t r = 0; r < rows; ++r) {
113 if constexpr (SetZero) {
114 for (idx_t k = 0; k < Nk; ++k) {
115 for (idx_t l = 0; l < Nl; ++l) {
116 tgt(r, k, l) = 0.;
117 }
118 }
119 }
120 for (idx_t c = outer[r]; c < outer[r + 1]; ++c) {
121 idx_t n = index[c];
122 const Value w = static_cast<Value>(weight[c]);
123 for (idx_t k = 0; k < Nk; ++k) {
124 for (idx_t l = 0; l < Nl; ++l) {
125 tgt(r, k, l) += w * src(n, k, l);
126 }
127 }
128 }
129 }
130 }
131
132 template <typename MatrixValue, typename SourceValue, typename TargetValue>
133 void SparseMatrixMultiply<backend::openmp, Indexing::layout_left, 3, MatrixValue, SourceValue, TargetValue>::multiply(
134 const SparseMatrixView<MatrixValue>& W, const View<SourceValue, 3>& src, View<TargetValue, 3>& tgt, const Configuration& config) {
135 spmt_layout_left<true>(W, src, tgt);
136 }
137
138 template <typename MatrixValue, typename SourceValue, typename TargetValue>
139 void SparseMatrixMultiply<backend::openmp, Indexing::layout_left, 3, MatrixValue, SourceValue, TargetValue>::multiply_add(
140 const SparseMatrixView<MatrixValue>& W, const View<SourceValue, 3>& src, View<TargetValue, 3>& tgt, const Configuration& config) {
141 spmt_layout_left<false>(W, src, tgt);
142 }
143
144 template <typename MatrixValue, typename SourceValue, typename TargetValue>
145 void SparseMatrixMultiply<backend::openmp, Indexing::layout_right, 1, MatrixValue, SourceValue, TargetValue>::multiply(
146 const SparseMatrixView<MatrixValue>& W, const View<SourceValue, 1>& src, View<TargetValue, 1>& tgt, const Configuration& config) {
147 return SparseMatrixMultiply<backend::openmp, Indexing::layout_left, 1, MatrixValue, SourceValue, TargetValue>::multiply(W, src, tgt, config);
148 }
149
150 template <typename MatrixValue, typename SourceValue, typename TargetValue>
151 void SparseMatrixMultiply<backend::openmp, Indexing::layout_right, 1, MatrixValue, SourceValue, TargetValue>::multiply_add(
152 const SparseMatrixView<MatrixValue>& W, const View<SourceValue, 1>& src, View<TargetValue, 1>& tgt, const Configuration& config) {
153 return SparseMatrixMultiply<backend::openmp, Indexing::layout_left, 1, MatrixValue, SourceValue, TargetValue>::multiply_add(W, src, tgt, config);
154 }
155
156 template <bool SetZero, typename MatrixValue, typename SourceValue, typename TargetValue>
157 void spmm_layout_right(const SparseMatrixView<MatrixValue>& W, const View<SourceValue, 2>& src, View<TargetValue, 2>& tgt) {
158 using Value = TargetValue;
159 const auto outer = W.outer();
160 const auto index = W.inner();
161 const auto weight = W.value();
162 const idx_t rows = static_cast<idx_t>(W.rows());
163 const idx_t Nk = src.shape(0);
164
165 ATLAS_ASSERT(src.shape(1) >= W.cols());
166 ATLAS_ASSERT(tgt.shape(1) >= W.rows());
167
168 atlas_omp_parallel_for(idx_t r = 0; r < rows; ++r) {
169 if constexpr (SetZero) {
170 for (idx_t k = 0; k < Nk; ++k) {
171 tgt(k, r) = 0.;
172 }
173 }
174 for (idx_t c = outer[r]; c < outer[r + 1]; ++c) {
175 idx_t n = index[c];
176 Value w = static_cast<Value>(weight[c]);
177 for (idx_t k = 0; k < Nk; ++k) {
178 tgt(k, r) += w * src(k, n);
179 }
180 }
181 }
182 }
183
184 template <typename MatrixValue, typename SourceValue, typename TargetValue>
185 void SparseMatrixMultiply<backend::openmp, Indexing::layout_right, 2, MatrixValue, SourceValue, TargetValue>::multiply(
186 const SparseMatrixView<MatrixValue>& W, const View<SourceValue, 2>& src, View<TargetValue, 2>& tgt, const Configuration&) {
187 spmm_layout_right<true>(W, src, tgt);
188 }
189
190 template <typename MatrixValue, typename SourceValue, typename TargetValue>
191 void SparseMatrixMultiply<backend::openmp, Indexing::layout_right, 2, MatrixValue, SourceValue, TargetValue>::multiply_add(
192 const SparseMatrixView<MatrixValue>& W, const View<SourceValue, 2>& src, View<TargetValue, 2>& tgt, const Configuration&) {
193 spmm_layout_right<false>(W, src, tgt);
194 }
195
196 template <bool SetZero, typename MatrixValue, typename SourceValue, typename TargetValue>
197 void spmt_layout_right(const SparseMatrixView<MatrixValue>& W, const View<SourceValue, 3>& src, View<TargetValue, 3>& tgt) {
198 if (src.contiguous() && tgt.contiguous()) {
199 // We can take a more optimized route by reducing rank
200 auto src_v = View<SourceValue, 2>(src.data(), array::make_shape(src.shape(0), src.stride(0)));
201 auto tgt_v = View<TargetValue, 2>(tgt.data(), array::make_shape(tgt.shape(0), tgt.stride(0)));
202 spmm_layout_right<SetZero>(W, src_v, tgt_v);
203 return;
204 }
205 using Value = TargetValue;
206 const auto outer = W.outer();
207 const auto index = W.inner();
208 const auto weight = W.value();
209 const idx_t rows = static_cast<idx_t>(W.rows());
210 const idx_t Nk = src.shape(1);
211 const idx_t Nl = src.shape(0);
212
213 atlas_omp_parallel_for(idx_t r = 0; r < rows; ++r) {
214 if constexpr (SetZero){
215 for (idx_t k = 0; k < Nk; ++k) {
216 for (idx_t l = 0; l < Nl; ++l) {
217 tgt(l, k, r) = 0.;
218 }
219 }
220 }
221 for (idx_t c = outer[r]; c < outer[r + 1]; ++c) {
222 idx_t n = index[c];
223 const Value w = static_cast<Value>(weight[c]);
224 for (idx_t k = 0; k < Nk; ++k) {
225 for (idx_t l = 0; l < Nl; ++l) {
226 tgt(l, k, r) += w * src(l, k, n);
227 }
228 }
229 }
230 }
231 }
232
233 template <typename MatrixValue, typename SourceValue, typename TargetValue>
234 void SparseMatrixMultiply<backend::openmp, Indexing::layout_right, 3, MatrixValue, SourceValue, TargetValue>::multiply(
235 const SparseMatrixView<MatrixValue>& W, const View<SourceValue, 3>& src, View<TargetValue, 3>& tgt, const Configuration& config) {
236 spmt_layout_right<true>(W, src, tgt);
237 }
238
239 template <typename MatrixValue, typename SourceValue, typename TargetValue>
240 void SparseMatrixMultiply<backend::openmp, Indexing::layout_right, 3, MatrixValue, SourceValue, TargetValue>::multiply_add(
241 const SparseMatrixView<MatrixValue>& W, const View<SourceValue, 3>& src, View<TargetValue, 3>& tgt, const Configuration& config) {
242 spmt_layout_right<false>(W, src, tgt);
243 }
244
245 #define EXPLICIT_TEMPLATE_INSTANTIATION(TYPE) \
246 template struct SparseMatrixMultiply<backend::openmp, Indexing::layout_left, 1, double, TYPE const, TYPE>; \
247 template struct SparseMatrixMultiply<backend::openmp, Indexing::layout_left, 2, double, TYPE const, TYPE>; \
248 template struct SparseMatrixMultiply<backend::openmp, Indexing::layout_left, 3, double, TYPE const, TYPE>; \
249 template struct SparseMatrixMultiply<backend::openmp, Indexing::layout_right, 1, double, TYPE const, TYPE>; \
250 template struct SparseMatrixMultiply<backend::openmp, Indexing::layout_right, 2, double, TYPE const, TYPE>; \
251 template struct SparseMatrixMultiply<backend::openmp, Indexing::layout_right, 3, double, TYPE const, TYPE>; \
252 template struct SparseMatrixMultiply<backend::openmp, Indexing::layout_left, 1, float, TYPE const, TYPE>; \
253 template struct SparseMatrixMultiply<backend::openmp, Indexing::layout_left, 2, float, TYPE const, TYPE>; \
254 template struct SparseMatrixMultiply<backend::openmp, Indexing::layout_left, 3, float, TYPE const, TYPE>; \
255 template struct SparseMatrixMultiply<backend::openmp, Indexing::layout_right, 1, float, TYPE const, TYPE>; \
256 template struct SparseMatrixMultiply<backend::openmp, Indexing::layout_right, 2, float, TYPE const, TYPE>; \
257 template struct SparseMatrixMultiply<backend::openmp, Indexing::layout_right, 3, float, TYPE const, TYPE>;
258
259 EXPLICIT_TEMPLATE_INSTANTIATION(double);
260 EXPLICIT_TEMPLATE_INSTANTIATION(float);
261
262 } // namespace sparse
263 } // namespace linalg
264 } // namespace atlas