| 1 |
/* |
| 2 |
* (C) Copyright 2024- ECMWF. |
| 3 |
* |
| 4 |
* This software is licensed under the terms of the Apache Licence Version 2.0 |
| 5 |
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. |
| 6 |
* In applying this licence, ECMWF does not waive the privileges and immunities |
| 7 |
* granted to it by virtue of its status as an intergovernmental organisation |
| 8 |
* nor does it submit to any jurisdiction. |
| 9 |
*/ |
| 10 |
|
| 11 |
#pragma once |
| 12 |
|
| 13 |
#include <algorithm> |
| 14 |
#include <any> |
| 15 |
#include <memory> |
| 16 |
#include <utility> |
| 17 |
|
| 18 |
#include "eckit/linalg/SparseMatrix.h" |
| 19 |
|
| 20 |
#include "atlas/array.h" |
| 21 |
#include "atlas/runtime/Exception.h" |
| 22 |
#include "atlas/linalg/sparse/SparseMatrixView.h" |
| 23 |
|
| 24 |
namespace atlas { |
| 25 |
namespace linalg { |
| 26 |
|
| 27 |
//---------------------------------------------------------------------------------------------------------------------- |
| 28 |
|
| 29 |
/// @brief SparseMatrixStorage |
| 30 |
/// Storage class based on atlas::array::Array with GPU offload capability |
| 31 |
/// This class contains only the data and no operators, iterators, or special |
| 32 |
/// sparse matrix construction from triplets etc. |
| 33 |
/// |
| 34 |
/// The construction is handled via SparseMatrixConvertor class. |
| 35 |
/// Examples: |
| 36 |
/// |
| 37 |
/// To construct it, taking ownership of a constructed Eigen::SparseMatrix |
| 38 |
/// |
| 39 |
/// SparseMatrixStorage s = make_sparse_matrix_storage(std::move(eigen_matrix)); |
| 40 |
/// |
| 41 |
/// To construct it, taking a copy of a constructed Eigen::SparseMatrix |
| 42 |
/// |
| 43 |
/// SparseMatrixStorage s = make_sparse_matrix_storage(eigen_matrix); |
| 44 |
/// |
| 45 |
/// To construct it, taking ownership of a constructed eckit::linalg::SparseMatrix, avoiding copies if data types match |
| 46 |
/// |
| 47 |
/// SparseMatrixStorage s = make_sparse_matrix_storage(std::move(eckit_matrix)); |
| 48 |
/// |
| 49 |
/// |
| 50 |
/// To construct it, taking a copy of a constructed eckit::linalg::SparseMatrix (no std::move) |
| 51 |
/// |
| 52 |
/// SparseMatrixStorage s = make_sparse_matrix_storage(eckit_matrix); |
| 53 |
/// |
| 54 |
/// |
| 55 |
/// It is also possible to initialise empty and move into it at later stage: |
| 56 |
/// |
| 57 |
/// SparseMatrixStorage s; |
| 58 |
/// s = make_sparse_matrix_storage(eckit_matrix); |
| 59 |
/// |
| 60 |
/// |
| 61 |
/// To construct it, taking a single precision copy of a constructed eckit::linalg::SparseMatrix |
| 62 |
/// |
| 63 |
/// s = make_sparse_matrix_storage<float>(eckit_matrix); |
| 64 |
/// |
| 65 |
|
| 66 |
class SparseMatrixStorage { |
| 67 |
public: |
| 68 |
|
| 69 |
// Virtual destructor |
| 70 |
virtual ~SparseMatrixStorage() = default; |
| 71 |
|
| 72 |
/// Default constructor |
| 73 |
SparseMatrixStorage() = default; |
| 74 |
|
| 75 |
/// Move constructor, takes ownership! |
| 76 |
SparseMatrixStorage(SparseMatrixStorage&& other); |
| 77 |
|
| 78 |
/// Copy constructor, makes copy! |
| 79 |
SparseMatrixStorage(const SparseMatrixStorage& other); |
| 80 |
|
| 81 |
/// Copy from a SparseMatrixView |
| 82 |
template<typename Value, typename Index> |
| 83 |
SparseMatrixStorage(const SparseMatrixView<Value,Index>& host_view); |
| 84 |
|
| 85 |
/// Move assign from other SparseMatrixStorage, takes ownership! |
| 86 |
SparseMatrixStorage& operator=(SparseMatrixStorage&& other); |
| 87 |
|
| 88 |
/// Copy assign from other SparseMatrixStorage, takes ownership! |
| 89 |
SparseMatrixStorage& operator=(const SparseMatrixStorage& other); |
| 90 |
|
| 91 |
// Remove all storage, make empty |
| 92 |
void clear(); |
| 93 |
|
| 94 |
/// Empty if rows and cols are zero. |
| 95 |
bool empty() const { return rows_ == 0 && cols_ == 0; } |
| 96 |
|
| 97 |
/// Footprint in bytes in host memory space |
| 98 |
std::size_t footprint() const { return value_->footprint() + outer_->footprint() + inner_->footprint(); } |
| 99 |
|
| 100 |
const atlas::array::Array& value() const { return *value_; } |
| 101 |
const atlas::array::Array& outer() const { return *outer_; } |
| 102 |
const atlas::array::Array& inner() const { return *inner_; } |
| 103 |
std::size_t rows() const { return rows_;} |
| 104 |
std::size_t cols() const { return cols_;} |
| 105 |
std::size_t nnz() const { return nnz_;} |
| 106 |
|
| 107 |
void updateDevice() const; |
| 108 |
|
| 109 |
void updateHost() const; |
| 110 |
|
| 111 |
bool hostNeedsUpdate() const; |
| 112 |
|
| 113 |
bool deviceNeedsUpdate() const; |
| 114 |
|
| 115 |
void setHostNeedsUpdate(bool) const; |
| 116 |
|
| 117 |
void setDeviceNeedsUpdate(bool) const; |
| 118 |
|
| 119 |
bool deviceAllocated() const; |
| 120 |
|
| 121 |
void allocateDevice() const; |
| 122 |
|
| 123 |
void deallocateDevice() const; |
| 124 |
|
| 125 |
void syncHost() const; |
| 126 |
|
| 127 |
void syncDevice() const; |
| 128 |
|
| 129 |
static SparseMatrixStorage make( |
| 130 |
std::size_t rows, |
| 131 |
std::size_t cols, |
| 132 |
std::size_t nnz, |
| 133 |
std::unique_ptr<atlas::array::Array>&& value, |
| 134 |
std::unique_ptr<atlas::array::Array>&& inner, |
| 135 |
std::unique_ptr<atlas::array::Array>&& outer, |
| 136 |
std::any&& storage) { |
| 137 |
SparseMatrixStorage S; |
| 138 |
S.rows_ = rows; |
| 139 |
S.cols_ = cols; |
| 140 |
S.nnz_ = nnz; |
| 141 |
S.outer_ = std::move(outer); |
| 142 |
S.inner_ = std::move(inner); |
| 143 |
S.value_ = std::move(value); |
| 144 |
S.storage_ = std::move(storage); |
| 145 |
return S; |
| 146 |
} |
| 147 |
|
| 148 |
void swap(SparseMatrixStorage& other) { |
| 149 |
std::swap(other.rows_, rows_); |
| 150 |
std::swap(other.cols_, cols_); |
| 151 |
std::swap(other.nnz_, nnz_); |
| 152 |
std::swap(other.value_, value_); |
| 153 |
std::swap(other.inner_, inner_); |
| 154 |
std::swap(other.outer_, outer_); |
| 155 |
std::swap(other.storage_, storage_); |
| 156 |
} |
| 157 |
|
| 158 |
bool contains(DataType value_type, DataType index_type) { |
| 159 |
if (empty()) { |
| 160 |
return false; |
| 161 |
} |
| 162 |
return value_->datatype() == value_type && outer_->datatype() == index_type; |
| 163 |
} |
| 164 |
|
| 165 |
template <typename value_type, typename index_type> |
| 166 |
bool contains() { |
| 167 |
return contains(make_datatype<value_type>(), make_datatype<index_type>()); |
| 168 |
} |
| 169 |
|
| 170 |
protected: |
| 171 |
std::size_t nnz_{0}; |
| 172 |
std::size_t rows_{0}; |
| 173 |
std::size_t cols_{0}; |
| 174 |
std::unique_ptr<atlas::array::Array> outer_; |
| 175 |
std::unique_ptr<atlas::array::Array> inner_; |
| 176 |
std::unique_ptr<atlas::array::Array> value_; |
| 177 |
|
| 178 |
std::any storage_; |
| 179 |
// This storage is usually empty. |
| 180 |
// It is used to allow to move alternative sparse matrix formats into it, |
| 181 |
// and then wrap this data using the atlas Arrays |
| 182 |
|
| 183 |
private: |
| 184 |
template<typename ValueT> |
| 185 |
void host_copy(const ValueT* input_data, atlas::array::Array& output) { |
| 186 |
auto size = output.size(); |
| 187 |
ValueT* output_data = output.host_data<ValueT>(); |
| 188 |
std::copy( input_data, input_data + size, output_data ); |
| 189 |
} |
| 190 |
}; |
| 191 |
|
| 192 |
//---------------------------------------------------------------------------------------------------------------------- |
| 193 |
|
| 194 |
template<typename Value, typename Index> |
| 195 |
SparseMatrixStorage::SparseMatrixStorage(const SparseMatrixView<Value,Index>& host_view) { |
| 196 |
nnz_ = host_view.nnz(); |
| 197 |
rows_ = host_view.rows(); |
| 198 |
cols_ = host_view.cols(); |
| 199 |
outer_.reset(atlas::array::Array::create<Index>(host_view.outer_size())); |
| 200 |
inner_.reset(atlas::array::Array::create<Index>(host_view.inner_size())); |
| 201 |
value_.reset(atlas::array::Array::create<Value>(host_view.value_size())); |
| 202 |
host_copy(host_view.outer(),*outer_); |
| 203 |
host_copy(host_view.inner(),*inner_); |
| 204 |
host_copy(host_view.value(),*value_); |
| 205 |
} |
| 206 |
|
| 207 |
//---------------------------------------------------------------------------------------------------------------------- |
| 208 |
|
| 209 |
namespace detail { |
| 210 |
void host_copy(const array::Array& input, array::Array& output); |
| 211 |
} |
| 212 |
|
| 213 |
template<typename value_type, typename index_type = eckit::linalg::Index> |
| 214 |
SparseMatrixStorage make_sparse_matrix_storage(const SparseMatrixStorage& other) { |
| 215 |
auto rows = other.rows(); |
| 216 |
auto cols = other.cols(); |
| 217 |
auto nnz = other.nnz(); |
| 218 |
std::unique_ptr<array::Array> value(array::Array::create<value_type>(nnz)); |
| 219 |
std::unique_ptr<array::Array> inner(array::Array::create<index_type>(nnz)); |
| 220 |
std::unique_ptr<array::Array> outer(array::Array::create<index_type>(rows+1)); |
| 221 |
detail::host_copy(other.value(), *value); |
| 222 |
detail::host_copy(other.inner(), *inner); |
| 223 |
detail::host_copy(other.outer(), *outer); |
| 224 |
return SparseMatrixStorage::make(rows,cols,nnz,std::move(value), std::move(inner), std::move(outer), std::any()); |
| 225 |
} |
| 226 |
|
| 227 |
template<typename value_type, typename index_type = eckit::linalg::Index> |
| 228 |
SparseMatrixStorage make_sparse_matrix_storage(SparseMatrixStorage&& other) { |
| 229 |
SparseMatrixStorage S; |
| 230 |
|
| 231 |
if (other.contains<value_type,index_type>()) { |
| 232 |
S = std::move(other); |
| 233 |
} |
| 234 |
else { |
| 235 |
auto rows = other.rows(); |
| 236 |
auto cols = other.cols(); |
| 237 |
auto nnz = other.nnz(); |
| 238 |
std::unique_ptr<array::Array> value(array::Array::create<value_type>(nnz)); |
| 239 |
std::unique_ptr<array::Array> inner(array::Array::create<index_type>(nnz)); |
| 240 |
std::unique_ptr<array::Array> outer(array::Array::create<index_type>(rows+1)); |
| 241 |
detail::host_copy(other.value(), *value); |
| 242 |
detail::host_copy(other.inner(), *inner); |
| 243 |
detail::host_copy(other.outer(), *outer); |
| 244 |
S = SparseMatrixStorage::make(rows,cols,nnz,std::move(value), std::move(inner), std::move(outer), std::any()); |
| 245 |
} |
| 246 |
return S; |
| 247 |
} |
| 248 |
|
| 249 |
//---------------------------------------------------------------------------------------------------------------------- |
| 250 |
|
| 251 |
template<typename Value, typename Index = eckit::linalg::Index> |
| 252 |
inline SparseMatrixView<Value,Index> make_host_view(const SparseMatrixStorage& m) { |
| 253 |
if(m.rows() == 0 && m.cols() == 0) { |
| 254 |
return SparseMatrixView<Value,Index>(); |
| 255 |
} |
| 256 |
if( m.value().datatype().kind() != DataType::kind<Value>() || m.outer().datatype().kind() != DataType::kind<Index>() ) { |
| 257 |
ATLAS_THROW_EXCEPTION("Cannot make_host_view<" + DataType::str<Value>() + "," << DataType::str<Index>() + |
| 258 |
">(const SparseMatrixStorage&) from SparseMatrixStorage containing values of type <" + m.value().datatype().str() + "> and indices of type <" + m.outer().datatype().str() +">" ); |
| 259 |
} |
| 260 |
return SparseMatrixView<Value,Index> { |
| 261 |
m.rows(), |
| 262 |
m.cols(), |
| 263 |
m.nnz(), |
| 264 |
m.value().host_data<Value>(), |
| 265 |
m.inner().host_data<Index>(), |
| 266 |
m.outer().host_data<Index>() |
| 267 |
}; |
| 268 |
} |
| 269 |
|
| 270 |
//---------------------------------------------------------------------------------------------------------------------- |
| 271 |
|
| 272 |
template<typename Value, typename Index = eckit::linalg::Index> |
| 273 |
inline SparseMatrixView<Value,Index> make_device_view(const SparseMatrixStorage& m) { |
| 274 |
if(m.rows() == 0 && m.cols() == 0) { |
| 275 |
return SparseMatrixView<Value,Index>(); |
| 276 |
} |
| 277 |
if( m.value().datatype().kind() != DataType::kind<Value>() || m.outer().datatype().kind() != DataType::kind<Index>() ) { |
| 278 |
ATLAS_THROW_EXCEPTION("Cannot make_device_view<" + DataType::str<Value>() + "," << DataType::str<Index>() + |
| 279 |
">(const SparseMatrixStorage&) from SparseMatrixStorage containing values of type <" + m.value().datatype().str() + "> and indices of type <" + m.outer().datatype().str() +">" ); |
| 280 |
} |
| 281 |
if( ! m.deviceAllocated() ) { |
| 282 |
ATLAS_THROW_EXCEPTION("Cannot make_device_view(const SparseMatrixStorage&) as the device data is not allocated"); |
| 283 |
} |
| 284 |
return SparseMatrixView<Value,Index>{ |
| 285 |
m.rows(), |
| 286 |
m.cols(), |
| 287 |
m.nnz(), |
| 288 |
m.value().device_data<Value>(), |
| 289 |
m.inner().device_data<Index>(), |
| 290 |
m.outer().device_data<Index>() |
| 291 |
}; |
| 292 |
} |
| 293 |
|
| 294 |
//---------------------------------------------------------------------------------------------------------------------- |
| 295 |
|
| 296 |
} // namespace linalg |
| 297 |
} // namespace atlas |