Debian Codegraph @

src/atlas/linalg/sparse/SparseMatrixStorage.h

raw
1 /*
2 * (C) Copyright 2024- ECMWF.
3 *
4 * This software is licensed under the terms of the Apache Licence Version 2.0
5 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6 * In applying this licence, ECMWF does not waive the privileges and immunities
7 * granted to it by virtue of its status as an intergovernmental organisation
8 * nor does it submit to any jurisdiction.
9 */
10
11 #pragma once
12
13 #include <algorithm>
14 #include <any>
15 #include <memory>
16 #include <utility>
17
18 #include "eckit/linalg/SparseMatrix.h"
19
20 #include "atlas/array.h"
21 #include "atlas/runtime/Exception.h"
22 #include "atlas/linalg/sparse/SparseMatrixView.h"
23
24 namespace atlas {
25 namespace linalg {
26
27 //----------------------------------------------------------------------------------------------------------------------
28
29 /// @brief SparseMatrixStorage
30 /// Storage class based on atlas::array::Array with GPU offload capability
31 /// This class contains only the data and no operators, iterators, or special
32 /// sparse matrix construction from triplets etc.
33 ///
34 /// The construction is handled via SparseMatrixConvertor class.
35 /// Examples:
36 ///
37 /// To construct it, taking ownership of a constructed Eigen::SparseMatrix
38 ///
39 /// SparseMatrixStorage s = make_sparse_matrix_storage(std::move(eigen_matrix));
40 ///
41 /// To construct it, taking a copy of a constructed Eigen::SparseMatrix
42 ///
43 /// SparseMatrixStorage s = make_sparse_matrix_storage(eigen_matrix);
44 ///
45 /// To construct it, taking ownership of a constructed eckit::linalg::SparseMatrix, avoiding copies if data types match
46 ///
47 /// SparseMatrixStorage s = make_sparse_matrix_storage(std::move(eckit_matrix));
48 ///
49 ///
50 /// To construct it, taking a copy of a constructed eckit::linalg::SparseMatrix (no std::move)
51 ///
52 /// SparseMatrixStorage s = make_sparse_matrix_storage(eckit_matrix);
53 ///
54 ///
55 /// It is also possible to initialise empty and move into it at later stage:
56 ///
57 /// SparseMatrixStorage s;
58 /// s = make_sparse_matrix_storage(eckit_matrix);
59 ///
60 ///
61 /// To construct it, taking a single precision copy of a constructed eckit::linalg::SparseMatrix
62 ///
63 /// s = make_sparse_matrix_storage<float>(eckit_matrix);
64 ///
65
66 class SparseMatrixStorage {
67 public:
68
69 // Virtual destructor
70 virtual ~SparseMatrixStorage() = default;
71
72 /// Default constructor
73 SparseMatrixStorage() = default;
74
75 /// Move constructor, takes ownership!
76 SparseMatrixStorage(SparseMatrixStorage&& other);
77
78 /// Copy constructor, makes copy!
79 SparseMatrixStorage(const SparseMatrixStorage& other);
80
81 /// Copy from a SparseMatrixView
82 template<typename Value, typename Index>
83 SparseMatrixStorage(const SparseMatrixView<Value,Index>& host_view);
84
85 /// Move assign from other SparseMatrixStorage, takes ownership!
86 SparseMatrixStorage& operator=(SparseMatrixStorage&& other);
87
88 /// Copy assign from other SparseMatrixStorage, takes ownership!
89 SparseMatrixStorage& operator=(const SparseMatrixStorage& other);
90
91 // Remove all storage, make empty
92 void clear();
93
94 /// Empty if rows and cols are zero.
95 bool empty() const { return rows_ == 0 && cols_ == 0; }
96
97 /// Footprint in bytes in host memory space
98 std::size_t footprint() const { return value_->footprint() + outer_->footprint() + inner_->footprint(); }
99
100 const atlas::array::Array& value() const { return *value_; }
101 const atlas::array::Array& outer() const { return *outer_; }
102 const atlas::array::Array& inner() const { return *inner_; }
103 std::size_t rows() const { return rows_;}
104 std::size_t cols() const { return cols_;}
105 std::size_t nnz() const { return nnz_;}
106
107 void updateDevice() const;
108
109 void updateHost() const;
110
111 bool hostNeedsUpdate() const;
112
113 bool deviceNeedsUpdate() const;
114
115 void setHostNeedsUpdate(bool) const;
116
117 void setDeviceNeedsUpdate(bool) const;
118
119 bool deviceAllocated() const;
120
121 void allocateDevice() const;
122
123 void deallocateDevice() const;
124
125 void syncHost() const;
126
127 void syncDevice() const;
128
129 static SparseMatrixStorage make(
130 std::size_t rows,
131 std::size_t cols,
132 std::size_t nnz,
133 std::unique_ptr<atlas::array::Array>&& value,
134 std::unique_ptr<atlas::array::Array>&& inner,
135 std::unique_ptr<atlas::array::Array>&& outer,
136 std::any&& storage) {
137 SparseMatrixStorage S;
138 S.rows_ = rows;
139 S.cols_ = cols;
140 S.nnz_ = nnz;
141 S.outer_ = std::move(outer);
142 S.inner_ = std::move(inner);
143 S.value_ = std::move(value);
144 S.storage_ = std::move(storage);
145 return S;
146 }
147
148 void swap(SparseMatrixStorage& other) {
149 std::swap(other.rows_, rows_);
150 std::swap(other.cols_, cols_);
151 std::swap(other.nnz_, nnz_);
152 std::swap(other.value_, value_);
153 std::swap(other.inner_, inner_);
154 std::swap(other.outer_, outer_);
155 std::swap(other.storage_, storage_);
156 }
157
158 bool contains(DataType value_type, DataType index_type) {
159 if (empty()) {
160 return false;
161 }
162 return value_->datatype() == value_type && outer_->datatype() == index_type;
163 }
164
165 template <typename value_type, typename index_type>
166 bool contains() {
167 return contains(make_datatype<value_type>(), make_datatype<index_type>());
168 }
169
170 protected:
171 std::size_t nnz_{0};
172 std::size_t rows_{0};
173 std::size_t cols_{0};
174 std::unique_ptr<atlas::array::Array> outer_;
175 std::unique_ptr<atlas::array::Array> inner_;
176 std::unique_ptr<atlas::array::Array> value_;
177
178 std::any storage_;
179 // This storage is usually empty.
180 // It is used to allow to move alternative sparse matrix formats into it,
181 // and then wrap this data using the atlas Arrays
182
183 private:
184 template<typename ValueT>
185 void host_copy(const ValueT* input_data, atlas::array::Array& output) {
186 auto size = output.size();
187 ValueT* output_data = output.host_data<ValueT>();
188 std::copy( input_data, input_data + size, output_data );
189 }
190 };
191
192 //----------------------------------------------------------------------------------------------------------------------
193
194 template<typename Value, typename Index>
195 SparseMatrixStorage::SparseMatrixStorage(const SparseMatrixView<Value,Index>& host_view) {
196 nnz_ = host_view.nnz();
197 rows_ = host_view.rows();
198 cols_ = host_view.cols();
199 outer_.reset(atlas::array::Array::create<Index>(host_view.outer_size()));
200 inner_.reset(atlas::array::Array::create<Index>(host_view.inner_size()));
201 value_.reset(atlas::array::Array::create<Value>(host_view.value_size()));
202 host_copy(host_view.outer(),*outer_);
203 host_copy(host_view.inner(),*inner_);
204 host_copy(host_view.value(),*value_);
205 }
206
207 //----------------------------------------------------------------------------------------------------------------------
208
209 namespace detail {
210 void host_copy(const array::Array& input, array::Array& output);
211 }
212
213 template<typename value_type, typename index_type = eckit::linalg::Index>
214 SparseMatrixStorage make_sparse_matrix_storage(const SparseMatrixStorage& other) {
215 auto rows = other.rows();
216 auto cols = other.cols();
217 auto nnz = other.nnz();
218 std::unique_ptr<array::Array> value(array::Array::create<value_type>(nnz));
219 std::unique_ptr<array::Array> inner(array::Array::create<index_type>(nnz));
220 std::unique_ptr<array::Array> outer(array::Array::create<index_type>(rows+1));
221 detail::host_copy(other.value(), *value);
222 detail::host_copy(other.inner(), *inner);
223 detail::host_copy(other.outer(), *outer);
224 return SparseMatrixStorage::make(rows,cols,nnz,std::move(value), std::move(inner), std::move(outer), std::any());
225 }
226
227 template<typename value_type, typename index_type = eckit::linalg::Index>
228 SparseMatrixStorage make_sparse_matrix_storage(SparseMatrixStorage&& other) {
229 SparseMatrixStorage S;
230
231 if (other.contains<value_type,index_type>()) {
232 S = std::move(other);
233 }
234 else {
235 auto rows = other.rows();
236 auto cols = other.cols();
237 auto nnz = other.nnz();
238 std::unique_ptr<array::Array> value(array::Array::create<value_type>(nnz));
239 std::unique_ptr<array::Array> inner(array::Array::create<index_type>(nnz));
240 std::unique_ptr<array::Array> outer(array::Array::create<index_type>(rows+1));
241 detail::host_copy(other.value(), *value);
242 detail::host_copy(other.inner(), *inner);
243 detail::host_copy(other.outer(), *outer);
244 S = SparseMatrixStorage::make(rows,cols,nnz,std::move(value), std::move(inner), std::move(outer), std::any());
245 }
246 return S;
247 }
248
249 //----------------------------------------------------------------------------------------------------------------------
250
251 template<typename Value, typename Index = eckit::linalg::Index>
252 inline SparseMatrixView<Value,Index> make_host_view(const SparseMatrixStorage& m) {
253 if(m.rows() == 0 && m.cols() == 0) {
254 return SparseMatrixView<Value,Index>();
255 }
256 if( m.value().datatype().kind() != DataType::kind<Value>() || m.outer().datatype().kind() != DataType::kind<Index>() ) {
257 ATLAS_THROW_EXCEPTION("Cannot make_host_view<" + DataType::str<Value>() + "," << DataType::str<Index>() +
258 ">(const SparseMatrixStorage&) from SparseMatrixStorage containing values of type <" + m.value().datatype().str() + "> and indices of type <" + m.outer().datatype().str() +">" );
259 }
260 return SparseMatrixView<Value,Index> {
261 m.rows(),
262 m.cols(),
263 m.nnz(),
264 m.value().host_data<Value>(),
265 m.inner().host_data<Index>(),
266 m.outer().host_data<Index>()
267 };
268 }
269
270 //----------------------------------------------------------------------------------------------------------------------
271
272 template<typename Value, typename Index = eckit::linalg::Index>
273 inline SparseMatrixView<Value,Index> make_device_view(const SparseMatrixStorage& m) {
274 if(m.rows() == 0 && m.cols() == 0) {
275 return SparseMatrixView<Value,Index>();
276 }
277 if( m.value().datatype().kind() != DataType::kind<Value>() || m.outer().datatype().kind() != DataType::kind<Index>() ) {
278 ATLAS_THROW_EXCEPTION("Cannot make_device_view<" + DataType::str<Value>() + "," << DataType::str<Index>() +
279 ">(const SparseMatrixStorage&) from SparseMatrixStorage containing values of type <" + m.value().datatype().str() + "> and indices of type <" + m.outer().datatype().str() +">" );
280 }
281 if( ! m.deviceAllocated() ) {
282 ATLAS_THROW_EXCEPTION("Cannot make_device_view(const SparseMatrixStorage&) as the device data is not allocated");
283 }
284 return SparseMatrixView<Value,Index>{
285 m.rows(),
286 m.cols(),
287 m.nnz(),
288 m.value().device_data<Value>(),
289 m.inner().device_data<Index>(),
290 m.outer().device_data<Index>()
291 };
292 }
293
294 //----------------------------------------------------------------------------------------------------------------------
295
296 } // namespace linalg
297 } // namespace atlas