| 1 |
/* |
| 2 |
* (C) Crown Copyright 2024 Met Office |
| 3 |
* |
| 4 |
* This software is licensed under the terms of the Apache Licence Version 2.0 |
| 5 |
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. |
| 6 |
*/ |
| 7 |
|
| 8 |
#pragma once |
| 9 |
|
| 10 |
#include <array> |
| 11 |
#include <iomanip> |
| 12 |
#include <limits> |
| 13 |
#include <sstream> |
| 14 |
#include <tuple> |
| 15 |
#include <type_traits> |
| 16 |
|
| 17 |
#include "atlas/array/ArrayView.h" |
| 18 |
#include "atlas/array/LocalView.h" |
| 19 |
#include "atlas/array/Range.h" |
| 20 |
#include "atlas/array/helpers/ArrayForEach.h" |
| 21 |
#include "atlas/interpolation/method/sphericalvector/Types.h" |
| 22 |
#include "atlas/parallel/omp/omp.h" |
| 23 |
|
| 24 |
namespace atlas { |
| 25 |
namespace interpolation { |
| 26 |
namespace method { |
| 27 |
namespace detail { |
| 28 |
|
| 29 |
struct TwoVectorTag {}; |
| 30 |
struct ThreeVectorTag {}; |
| 31 |
constexpr auto twoVector = TwoVectorTag{}; |
| 32 |
constexpr auto threeVector = ThreeVectorTag{}; |
| 33 |
|
| 34 |
// Permits the equivalent of static_assert(false, msg). Will be addressed in |
| 35 |
// C++26. |
| 36 |
template <typename> |
| 37 |
constexpr bool always_false_v = false; |
| 38 |
|
| 39 |
/// @brief Helper class to perform complex matrix multiplications |
| 40 |
/// |
| 41 |
/// @details Performs matrix multiplication between fields of 2-vectors or |
| 42 |
/// 3-vectors. Fields must have Rank >= 2. Here, the assumption is |
| 43 |
/// that Dim = 0 is the horizontal dimension, and Dim = (Rank - 1) is |
| 44 |
/// the vector element dimension. |
| 45 |
template <bool InitialiseTarget> |
| 46 |
class ComplexMatrixMultiply { |
| 47 |
public: |
| 48 |
ComplexMatrixMultiply() = default; |
| 49 |
|
| 50 |
/// @brief Construct object from sparse matrices. |
| 51 |
/// |
| 52 |
/// @details complexWeights is a SparseMatrix of weights. realWeights is a |
| 53 |
/// SparseMatrix containing the magnitudes of the elements of |
| 54 |
/// complexWeights. |
| 55 |
ComplexMatrixMultiply(const ComplexMatrix& complexWeights, |
| 56 |
const RealMatrix& realWeights) |
| 57 |
: complexWeightsPtr_{&complexWeights}, realWeightsPtr_{&realWeights} { |
| 58 |
if constexpr (ATLAS_BUILD_TYPE_DEBUG) { |
| 59 |
ATLAS_ASSERT(complexWeightsPtr_->rows() == realWeightsPtr_->rows()); |
| 60 |
ATLAS_ASSERT(complexWeightsPtr_->cols() == realWeightsPtr_->cols()); |
| 61 |
ATLAS_ASSERT(complexWeightsPtr_->nonZeros() == |
| 62 |
realWeightsPtr_->nonZeros()); |
| 63 |
for (auto rowIndex = Index{0}; rowIndex < complexWeightsPtr_->rows(); |
| 64 |
++rowIndex) { |
| 65 |
for (auto [complexRowIter, realRowIter] = rowIters(rowIndex); |
| 66 |
complexRowIter; ++complexRowIter, ++realRowIter) { |
| 67 |
ATLAS_ASSERT(realRowIter); |
| 68 |
ATLAS_ASSERT(realRowIter.row() == complexRowIter.row()); |
| 69 |
ATLAS_ASSERT(realRowIter.col() == complexRowIter.col()); |
| 70 |
// tinyNum ~= 2.3e-13 for double. |
| 71 |
constexpr auto tinyNum = 1024 * std::numeric_limits<Real>::epsilon(); |
| 72 |
const auto complexMagnitude = std::abs(complexRowIter.value()); |
| 73 |
const auto realValue = std::abs(realRowIter.value()); |
| 74 |
const auto error = std::abs(complexMagnitude - realValue); |
| 75 |
|
| 76 |
const auto printError = [&]() { |
| 77 |
auto strStream = std::ostringstream{}; |
| 78 |
strStream << "Error complex components: "; |
| 79 |
strStream << std::setprecision(19) << error; |
| 80 |
return strStream.str(); |
| 81 |
}; |
| 82 |
ATLAS_ASSERT(error < tinyNum, printError()); |
| 83 |
} |
| 84 |
} |
| 85 |
} |
| 86 |
} |
| 87 |
|
| 88 |
/// @brief Apply complex matrix vector multiplication. |
| 89 |
/// |
| 90 |
/// @details Multiply weights by the elements in sourceView to give |
| 91 |
/// elements in targetView. If VectorType == TwoVectorTag, |
| 92 |
/// complexWeights are applied to the horizontal elements of |
| 93 |
/// sourceView. If VectorType == ThreeVectorTag, then realWeights |
| 94 |
/// are additionally applied to the vertical elements of sourceView. |
| 95 |
template <typename Value, int Rank, typename VectorType> |
| 96 |
void apply(const array::ArrayView<const Value, Rank>& sourceView, |
| 97 |
array::ArrayView<Value, Rank>& targetView, VectorType) const { |
| 98 |
if constexpr (std::is_same_v<VectorType, TwoVectorTag>) { |
| 99 |
applyTwoVector(sourceView, targetView); |
| 100 |
} else if constexpr (std::is_same_v<VectorType, ThreeVectorTag>) { |
| 101 |
applyThreeVector(sourceView, targetView); |
| 102 |
} else { |
| 103 |
static_assert(always_false_v<VectorType>, "Unknown vector type"); |
| 104 |
} |
| 105 |
} |
| 106 |
|
| 107 |
private: |
| 108 |
/// @brief Apply 2-vector MatMul. |
| 109 |
template <typename Value, int Rank> |
| 110 |
void applyTwoVector(const array::ArrayView<const Value, Rank>& sourceView, |
| 111 |
array::ArrayView<Value, Rank>& targetView) const { |
| 112 |
// We could probably optimise contiguous arrays using |
| 113 |
// reinterpret_cast<std::complex<double>*>(view.data()). This is fine |
| 114 |
// according to the C++ standard! |
| 115 |
atlas_omp_parallel_for(auto rowIndex = Index{0}; |
| 116 |
rowIndex < complexWeightsPtr_->rows(); ++rowIndex) { |
| 117 |
auto targetSlice = sliceColumn(targetView, rowIndex); |
| 118 |
if constexpr (InitialiseTarget) { |
| 119 |
targetSlice.assign(0.); |
| 120 |
} |
| 121 |
|
| 122 |
for (auto complexRowIter = complexWeightsPtr_->rowIter(rowIndex); |
| 123 |
complexRowIter; ++complexRowIter) { |
| 124 |
const auto colIndex = complexRowIter.col(); |
| 125 |
const auto complexWeight = complexRowIter.value(); |
| 126 |
const auto sourceSlice = sliceColumn(sourceView, colIndex); |
| 127 |
multiplyAdd(sourceSlice, targetSlice, complexWeight); |
| 128 |
} |
| 129 |
} |
| 130 |
} |
| 131 |
|
| 132 |
/// @brief Apply 3-vector MatMul. |
| 133 |
template <typename Value, int Rank> |
| 134 |
void applyThreeVector(const array::ArrayView<const Value, Rank>& sourceView, |
| 135 |
array::ArrayView<Value, Rank>& targetView) const { |
| 136 |
atlas_omp_parallel_for(auto rowIndex = Index{0}; |
| 137 |
rowIndex < complexWeightsPtr_->rows(); ++rowIndex) { |
| 138 |
auto targetSlice = sliceColumn(targetView, rowIndex); |
| 139 |
if constexpr (InitialiseTarget) { |
| 140 |
targetSlice.assign(0.); |
| 141 |
} |
| 142 |
|
| 143 |
for (auto [complexRowIter, realRowIter] = rowIters(rowIndex); |
| 144 |
complexRowIter; ++complexRowIter, ++realRowIter) { |
| 145 |
const auto colIndex = complexRowIter.col(); |
| 146 |
const auto complexWeight = complexRowIter.value(); |
| 147 |
const auto realWeight = realRowIter.value(); |
| 148 |
const auto sourceSlice = sliceColumn(sourceView, colIndex); |
| 149 |
multiplyAdd(sourceSlice, targetSlice, complexWeight, realWeight); |
| 150 |
} |
| 151 |
} |
| 152 |
} |
| 153 |
|
| 154 |
/// @brief Multiply source column by weight(s) and add to target column. |
| 155 |
template <typename Value, typename... Weights> |
| 156 |
void multiplyAdd(const array::LocalView<const Value, 2>& sourceColumn, |
| 157 |
array::LocalView<Value, 2>& targetColumn, |
| 158 |
Weights... weights) const { |
| 159 |
for (auto levelIdx = 0; levelIdx < sourceColumn.shape(0); ++levelIdx) { |
| 160 |
const auto sourceElem = sliceColumn(sourceColumn, levelIdx); |
| 161 |
auto targetElem = sliceColumn(targetColumn, levelIdx); |
| 162 |
multiplyAdd(sourceElem, targetElem, weights...); |
| 163 |
} |
| 164 |
} |
| 165 |
|
| 166 |
/// @brief Multiply source element by complex weight and add to target |
| 167 |
/// element. |
| 168 |
template <typename Value> |
| 169 |
void multiplyAdd(const array::LocalView<const Value, 1>& sourceElem, |
| 170 |
array::LocalView<Value, 1>& targetElem, |
| 171 |
Complex complexWeight) const { |
| 172 |
const auto targetVector = |
| 173 |
complexWeight * Complex(sourceElem(0), sourceElem(1)); |
| 174 |
targetElem(0) += targetVector.real(); |
| 175 |
targetElem(1) += targetVector.imag(); |
| 176 |
} |
| 177 |
|
| 178 |
/// @brief Multiply source element by complex and real weights and add to |
| 179 |
/// target element. |
| 180 |
template <typename Value> |
| 181 |
void multiplyAdd(const array::LocalView<const Value, 1>& sourceElem, |
| 182 |
array::LocalView<Value, 1>& targetElem, |
| 183 |
Complex complexWeight, Real realWeight) const { |
| 184 |
multiplyAdd(sourceElem, targetElem, complexWeight); |
| 185 |
targetElem(2) += realWeight * sourceElem(2); |
| 186 |
} |
| 187 |
|
| 188 |
/// @brief Return a pair of complex and real row iterators |
| 189 |
std::pair<ComplexMatrix::RowIter, RealMatrix::RowIter> rowIters( |
| 190 |
Index rowIndex) const { |
| 191 |
return std::make_pair(complexWeightsPtr_->rowIter(rowIndex), |
| 192 |
realWeightsPtr_->rowIter(rowIndex)); |
| 193 |
} |
| 194 |
|
| 195 |
/// @brief Makes the slice arrayView.slice(index, Range::all()...). |
| 196 |
template <typename View, typename Index> |
| 197 |
static auto sliceColumn(View& arrayView, Index index) { |
| 198 |
constexpr auto Rank = std::decay_t<View>::rank(); |
| 199 |
using RangeAll = decltype(array::Range::all()); |
| 200 |
|
| 201 |
const auto slicerArgs = std::tuple_cat(std::make_tuple(index), |
| 202 |
std::array<RangeAll, Rank - 1>{}); |
| 203 |
const auto slicer = [&](auto&&... args) { |
| 204 |
return arrayView.slice(args...); |
| 205 |
}; |
| 206 |
|
| 207 |
return std::apply(slicer, slicerArgs); |
| 208 |
} |
| 209 |
|
| 210 |
const ComplexMatrix* complexWeightsPtr_{}; |
| 211 |
const RealMatrix* realWeightsPtr_{}; |
| 212 |
}; |
| 213 |
|
| 214 |
} // namespace detail |
| 215 |
} // namespace method |
| 216 |
} // namespace interpolation |
| 217 |
} // namespace atlas |