Debian Codegraph @

src/atlas/linalg/sparse/SparseMatrixMultiply_HicSparse.cc

raw
1 /*
2 * (C) Copyright 2024 ECMWF.
3 *
4 * This software is licensed under the terms of the Apache Licence Version 2.0
5 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6 * In applying this licence, ECMWF does not waive the privileges and immunities
7 * granted to it by virtue of its status as an intergovernmental organisation
8 * nor does it submit to any jurisdiction.
9 */
10
11 #include "atlas/linalg/sparse/SparseMatrixMultiply_HicSparse.h"
12
13 #include "atlas/parallel/omp/omp.h"
14 #include "atlas/runtime/Exception.h"
15
16 #include "hic/hic.h"
17 #include "hic/hic_library_types.h"
18 #include "hic/hicsparse.h"
19
20 namespace {
21
22 class HicSparseHandleRAIIWrapper {
23 public:
24 HicSparseHandleRAIIWrapper() { hicsparseCreate(&handle_); };
25 ~HicSparseHandleRAIIWrapper() { hicsparseDestroy(handle_); }
26 hicsparseHandle_t value() { return handle_; }
27 private:
28 hicsparseHandle_t handle_;
29 };
30
31 hicsparseHandle_t getDefaultHicSparseHandle() {
32 static auto handle = HicSparseHandleRAIIWrapper();
33 return handle.value();
34 }
35
36 template<typename T>
37 constexpr hicsparseIndexType_t getHicsparseIndexType() {
38 using base_type = std::remove_const_t<T>;
39 if constexpr (std::is_same_v<base_type, int>) {
40 return HICSPARSE_INDEX_32I;
41 } else {
42 static_assert(std::is_same_v<base_type, long>, "Unsupported index type");
43 return HICSPARSE_INDEX_64I;
44 }
45 }
46
47 template<typename T>
48 constexpr auto getHicsparseValueType() {
49 using base_type = std::remove_const_t<T>;
50 if constexpr (std::is_same_v<base_type, float>) {
51 return HIC_R_32F;
52 } else {
53 static_assert(std::is_same_v<base_type, double>, "Unsupported value type");\
54 return HIC_R_64F;
55 }
56 }
57
58 template<atlas::linalg::Indexing IndexLayout, typename T>
59 hicsparseOrder_t getHicsparseOrder(const atlas::linalg::View<T, 2>& v) {
60 constexpr int row_idx = (IndexLayout == atlas::linalg::Indexing::layout_left) ? 0 : 1;
61 constexpr int col_idx = (IndexLayout == atlas::linalg::Indexing::layout_left) ? 1 : 0;
62
63 if (v.stride(row_idx) == 1) {
64 return HICSPARSE_ORDER_COL;
65 } else if (v.stride(col_idx) == 1) {
66 return HICSPARSE_ORDER_ROW;
67 } else {
68 atlas::throw_Exception("Unsupported dense matrix memory order", Here());
69 return HICSPARSE_ORDER_COL;
70 }
71 }
72
73 template<typename T>
74 int64_t getLeadingDimension(const atlas::linalg::View<T, 2>& v) {
75 if (v.stride(0) == 1) {
76 return v.stride(1);
77 } else if (v.stride(1) == 1) {
78 return v.stride(0);
79 } else {
80 atlas::throw_Exception("Unsupported dense matrix memory order", Here());
81 return 0;
82 }
83 }
84
85 }
86
87 namespace atlas {
88 namespace linalg {
89 namespace sparse {
90
91 template <typename Value>
92 void hsSpMV(const SparseMatrixView<Value>& W, const View<const Value, 1>& src, Value beta, View<Value, 1>& tgt) {
93 // Assume that W, src, and tgt are all device views
94
95 using Index [[maybe_unused]] = typename SparseMatrixView<Value>::index_type;
96
97 ATLAS_ASSERT(src.shape(0) >= W.cols());
98 ATLAS_ASSERT(tgt.shape(0) >= W.rows());
99
100 [[maybe_unused]] auto handle = getDefaultHicSparseHandle();
101
102 // Create a sparse matrix descriptor
103 [[maybe_unused]] hicsparseConstSpMatDescr_t matA;
104 HICSPARSE_CALL(hicsparseCreateConstCsr(
105 &matA,
106 W.rows(), W.cols(), W.nnz(),
107 W.outer(), // row_offsets
108 W.inner(), // column_indices
109 W.value(), // values
110 getHicsparseIndexType<Index>(),
111 getHicsparseIndexType<Index>(),
112 HICSPARSE_INDEX_BASE_ZERO,
113 getHicsparseValueType<Value>()));
114
115 // Create dense matrix descriptors
116 [[maybe_unused]] hicsparseConstDnVecDescr_t vecX;
117 HICSPARSE_CALL(hicsparseCreateConstDnVec(
118 &vecX,
119 static_cast<int64_t>(W.cols()),
120 src.data(),
121 getHicsparseValueType<Value>()));
122
123 [[maybe_unused]] hicsparseDnVecDescr_t vecY;
124 HICSPARSE_CALL(hicsparseCreateDnVec(
125 &vecY,
126 W.rows(),
127 tgt.data(),
128 getHicsparseValueType<Value>()));
129
130 [[maybe_unused]] const Value alpha = 1;
131
132 // Determine buffer size
133 [[maybe_unused]] size_t bufferSize = 0;
134 HICSPARSE_CALL(hicsparseSpMV_bufferSize(
135 handle,
136 HICSPARSE_OPERATION_NON_TRANSPOSE,
137 &alpha,
138 matA,
139 vecX,
140 &beta,
141 vecY,
142 getHicsparseValueType<Value>(),
143 HICSPARSE_SPMV_ALG_DEFAULT,
144 &bufferSize));
145
146 // Allocate buffer
147 [[maybe_unused]] char* buffer;
148 HIC_CALL(hicMalloc(&buffer, bufferSize));
149
150 // Perform SpMV
151 HICSPARSE_CALL(hicsparseSpMV(
152 handle,
153 HICSPARSE_OPERATION_NON_TRANSPOSE,
154 &alpha,
155 matA,
156 vecX,
157 &beta,
158 vecY,
159 getHicsparseValueType<Value>(),
160 HICSPARSE_SPMV_ALG_DEFAULT,
161 buffer));
162
163 HIC_CALL(hicFree(buffer));
164 HICSPARSE_CALL(hicsparseDestroyDnVec(vecX));
165 HICSPARSE_CALL(hicsparseDestroyDnVec(vecY));
166 HICSPARSE_CALL(hicsparseDestroySpMat(matA));
167
168 HIC_CALL(hicDeviceSynchronize());
169 }
170
171
172 template <Indexing IndexLayout, typename Value>
173 void hsSpMM(const SparseMatrixView<Value>& W, const View<const Value, 2>& src, Value beta, View<Value, 2>& tgt) {
174 // Assume that W, src, and tgt are all device views
175
176 using Index [[maybe_unused]] = typename SparseMatrixView<Value>::index_type;
177
178 constexpr int row_idx = (IndexLayout == Indexing::layout_left) ? 0 : 1;
179 constexpr int col_idx = (IndexLayout == Indexing::layout_left) ? 1 : 0;
180
181 ATLAS_ASSERT(src.shape(row_idx) >= W.cols());
182 ATLAS_ASSERT(tgt.shape(row_idx) >= W.rows());
183 ATLAS_ASSERT(src.shape(col_idx) == tgt.shape(col_idx));
184
185 [[maybe_unused]] auto handle = getDefaultHicSparseHandle();
186
187 // Create a sparse matrix descriptor
188 [[maybe_unused]] hicsparseConstSpMatDescr_t matA;
189 HICSPARSE_CALL(hicsparseCreateConstCsr(
190 &matA,
191 W.rows(), W.cols(), W.nnz(),
192 W.outer(), // row_offsets
193 W.inner(), // column_indices
194 W.value(), // values
195 getHicsparseIndexType<Index>(),
196 getHicsparseIndexType<Index>(),
197 HICSPARSE_INDEX_BASE_ZERO,
198 getHicsparseValueType<Value>()));
199
200 // Create dense matrix descriptors
201 [[maybe_unused]] hicsparseConstDnMatDescr_t matB;
202 HICSPARSE_CALL(hicsparseCreateConstDnMat(
203 &matB,
204 W.cols(), src.shape(col_idx),
205 getLeadingDimension(src),
206 src.data(),
207 getHicsparseValueType<Value>(),
208 getHicsparseOrder<IndexLayout>(src)));
209
210 [[maybe_unused]] hicsparseDnMatDescr_t matC;
211 HICSPARSE_CALL(hicsparseCreateDnMat(
212 &matC,
213 W.rows(), tgt.shape(col_idx),
214 getLeadingDimension(tgt),
215 tgt.data(),
216 getHicsparseValueType<Value>(),
217 getHicsparseOrder<IndexLayout>(tgt)));
218
219 [[maybe_unused]] const Value alpha = 1;
220
221 // Determine buffer size
222 [[maybe_unused]] size_t bufferSize = 0;
223 HICSPARSE_CALL(hicsparseSpMM_bufferSize(
224 handle,
225 HICSPARSE_OPERATION_NON_TRANSPOSE,
226 HICSPARSE_OPERATION_NON_TRANSPOSE,
227 &alpha,
228 matA,
229 matB,
230 &beta,
231 matC,
232 getHicsparseValueType<Value>(),
233 HICSPARSE_SPMM_ALG_DEFAULT,
234 &bufferSize));
235
236 // Allocate buffer
237 [[maybe_unused]] char* buffer;
238 HIC_CALL(hicMalloc(&buffer, bufferSize));
239
240 // Perform SpMM
241 HICSPARSE_CALL(hicsparseSpMM(
242 handle,
243 HICSPARSE_OPERATION_NON_TRANSPOSE,
244 HICSPARSE_OPERATION_NON_TRANSPOSE,
245 &alpha,
246 matA,
247 matB,
248 &beta,
249 matC,
250 getHicsparseValueType<Value>(),
251 HICSPARSE_SPMM_ALG_DEFAULT,
252 buffer));
253
254 HIC_CALL(hicFree(buffer));
255 HICSPARSE_CALL(hicsparseDestroyDnMat(matC));
256 HICSPARSE_CALL(hicsparseDestroyDnMat(matB));
257 HICSPARSE_CALL(hicsparseDestroySpMat(matA));
258
259 HIC_CALL(hicDeviceSynchronize());
260 }
261
262 template <typename ScalarValue>
263 void SparseMatrixMultiply<backend::hicsparse, Indexing::layout_left, 1, ScalarValue, const ScalarValue, ScalarValue>::multiply(
264 const SparseMatrixView<ScalarValue>& W, const View<ScalarValue const, 1>& src, View<ScalarValue, 1>& tgt, const Configuration&) {
265 ScalarValue beta = 0;
266 hsSpMV(W, src, beta, tgt);
267 }
268
269 template <typename ScalarValue>
270 void SparseMatrixMultiply<backend::hicsparse, Indexing::layout_left, 1, ScalarValue, const ScalarValue, ScalarValue>::multiply_add(
271 const SparseMatrixView<ScalarValue>& W, const View<ScalarValue const, 1>& src, View<ScalarValue, 1>& tgt, const Configuration&) {
272 ScalarValue beta = 1;
273 hsSpMV(W, src, beta, tgt);
274 }
275
276 template <typename ScalarValue>
277 void SparseMatrixMultiply<backend::hicsparse, Indexing::layout_left, 2, ScalarValue, const ScalarValue, ScalarValue>::multiply(
278 const SparseMatrixView<ScalarValue>& W, const View<ScalarValue const, 2>& src, View<ScalarValue, 2>& tgt, const Configuration&) {
279 ScalarValue beta = 0;
280 hsSpMM<Indexing::layout_left>(W, src, beta, tgt);
281 }
282
283 template <typename ScalarValue>
284 void SparseMatrixMultiply<backend::hicsparse, Indexing::layout_left, 2, ScalarValue, const ScalarValue, ScalarValue>::multiply_add(
285 const SparseMatrixView<ScalarValue>& W, const View<ScalarValue const, 2>& src, View<ScalarValue, 2>& tgt, const Configuration&) {
286 ScalarValue beta = 1;
287 hsSpMM<Indexing::layout_left>(W, src, beta, tgt);
288 }
289
290 template <typename ScalarValue>
291 void SparseMatrixMultiply<backend::hicsparse, Indexing::layout_right, 1, ScalarValue, const ScalarValue, ScalarValue>::multiply(
292 const SparseMatrixView<ScalarValue>& W, const View<ScalarValue const, 1>& src, View<ScalarValue, 1>& tgt, const Configuration&) {
293 ScalarValue beta = 0;
294 hsSpMV(W, src, beta, tgt);
295 }
296
297 template <typename ScalarValue>
298 void SparseMatrixMultiply<backend::hicsparse, Indexing::layout_right, 1, ScalarValue, const ScalarValue, ScalarValue>::multiply_add(
299 const SparseMatrixView<ScalarValue>& W, const View<ScalarValue const, 1>& src, View<ScalarValue, 1>& tgt, const Configuration&) {
300 ScalarValue beta = 1;
301 hsSpMV(W, src, beta, tgt);
302 }
303
304 template <typename ScalarValue>
305 void SparseMatrixMultiply<backend::hicsparse, Indexing::layout_right, 2, ScalarValue, const ScalarValue, ScalarValue>::multiply(
306 const SparseMatrixView<ScalarValue>& W, const View<ScalarValue const, 2>& src, View<ScalarValue, 2>& tgt, const Configuration&) {
307 ScalarValue beta = 0;
308 hsSpMM<Indexing::layout_right>(W, src, beta, tgt);
309 }
310
311 template <typename ScalarValue>
312 void SparseMatrixMultiply<backend::hicsparse, Indexing::layout_right, 2, ScalarValue, const ScalarValue, ScalarValue>::multiply_add(
313 const SparseMatrixView<ScalarValue>& W, const View<ScalarValue const, 2>& src, View<ScalarValue, 2>& tgt, const Configuration&) {
314 ScalarValue beta = 1;
315 hsSpMM<Indexing::layout_right>(W, src, beta, tgt);
316 }
317
318 #define EXPLICIT_TEMPLATE_INSTANTIATION(TYPE) \
319 template struct SparseMatrixMultiply<backend::hicsparse, Indexing::layout_left, 1, TYPE, TYPE const, TYPE>; \
320 template struct SparseMatrixMultiply<backend::hicsparse, Indexing::layout_left, 2, TYPE, TYPE const, TYPE>; \
321 template struct SparseMatrixMultiply<backend::hicsparse, Indexing::layout_right, 1, TYPE, TYPE const, TYPE>; \
322 template struct SparseMatrixMultiply<backend::hicsparse, Indexing::layout_right, 2, TYPE, TYPE const, TYPE>;
323
324 EXPLICIT_TEMPLATE_INSTANTIATION(double);
325 EXPLICIT_TEMPLATE_INSTANTIATION(float);
326
327 } // namespace sparse
328 } // namespace linalg
329 } // namespace atlas