| 1 |
/* |
| 2 |
* (C) Copyright 2025- ECMWF. |
| 3 |
* |
| 4 |
* This software is licensed under the terms of the Apache Licence Version 2.0 |
| 5 |
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. |
| 6 |
* In applying this licence, ECMWF does not waive the privileges and immunities |
| 7 |
* granted to it by virtue of its status as an intergovernmental organisation |
| 8 |
* nor does it submit to any jurisdiction. |
| 9 |
*/ |
| 10 |
|
| 11 |
#include "FFT.h" |
| 12 |
|
| 13 |
#include <cstdlib> |
| 14 |
#include <mutex> |
| 15 |
#include <sstream> |
| 16 |
|
| 17 |
#include "eckit/log/Bytes.h" |
| 18 |
|
| 19 |
#include "atlas/runtime/Exception.h" |
| 20 |
#include "atlas/runtime/Log.h" |
| 21 |
#include "atlas/runtime/Trace.h" |
| 22 |
|
| 23 |
#include "atlas/library/defines.h" |
| 24 |
|
| 25 |
#if ATLAS_HAVE_FFTW |
| 26 |
#include "fftw3.h" |
| 27 |
#endif |
| 28 |
|
| 29 |
namespace atlas::linalg { |
| 30 |
|
| 31 |
#define THROW_ATLAS_FFTW_NOT_SUPPORTED() throw_Exception("Atlas was not compiled with FFTW support", Here()) |
| 32 |
|
| 33 |
struct FFTW_Plan { |
| 34 |
#if ATLAS_HAVE_FFTW |
| 35 |
~FFTW_Plan() { |
| 36 |
destroy(); |
| 37 |
} |
| 38 |
void create_inverse_c2r(size_t size_out, std::complex<double>* in, double* out) { |
| 39 |
ATLAS_TRACE("FFTW::create_plan_inverse_c2r"); |
| 40 |
plan = fftw_plan_dft_c2r_1d(size_out, reinterpret_cast<fftw_complex*>(in), out, fftw_plan_flags); |
| 41 |
constructed_ = true; |
| 42 |
} |
| 43 |
void create_inverse_c2r_many(size_t howmany, size_t size_in, size_t size_out, size_t dist_in, size_t dist_out, std::complex<double>* in, double* out) { |
| 44 |
ATLAS_TRACE("FFTW::create_plan_inverse_c2r"); |
| 45 |
constexpr int rank = 1; /* 1d transforms */ |
| 46 |
const int n[] = {static_cast<int>(size_out)}; /* each transform of length size_out */ |
| 47 |
constexpr int istride = 1; |
| 48 |
constexpr int ostride = 1; |
| 49 |
const int idist = dist_in; |
| 50 |
const int odist = dist_out; |
| 51 |
const int* inembed = nullptr; |
| 52 |
const int* onembed = nullptr; |
| 53 |
plan = fftw_plan_many_dft_c2r(rank, n, howmany, reinterpret_cast<fftw_complex*>(in), |
| 54 |
inembed, istride, idist, out, onembed, ostride, odist, fftw_plan_flags); |
| 55 |
constructed_ = true; |
| 56 |
} |
| 57 |
|
| 58 |
void inverse_c2r(std::complex<double>* in, double* out) { |
| 59 |
ATLAS_TRACE("FFTW::inverse_c2r"); |
| 60 |
ATLAS_ASSERT(constructed_); |
| 61 |
fftw_execute_dft_c2r(plan, reinterpret_cast<fftw_complex*>(in), out); |
| 62 |
} |
| 63 |
|
| 64 |
void destroy() { |
| 65 |
if (constructed_) { |
| 66 |
fftw_destroy_plan(plan); |
| 67 |
} |
| 68 |
constructed_ = false; |
| 69 |
} |
| 70 |
fftw_plan plan; |
| 71 |
bool constructed_{false}; |
| 72 |
int fftw_plan_flags{FFTW_ESTIMATE}; |
| 73 |
#else |
| 74 |
void inverse_c2r(std::complex<double>* in, double* out) { |
| 75 |
THROW_ATLAS_FFTW_NOT_SUPPORTED(); |
| 76 |
} |
| 77 |
void create_inverse_c2r(size_t size_out, std::complex<double>* in, double* out) { |
| 78 |
THROW_ATLAS_FFTW_NOT_SUPPORTED(); |
| 79 |
} |
| 80 |
void create_inverse_c2r_many(size_t howmany, size_t size_in, size_t size_out, size_t dist_in, size_t dist_out, std::complex<double>* in, double* out) { |
| 81 |
THROW_ATLAS_FFTW_NOT_SUPPORTED(); |
| 82 |
} |
| 83 |
#endif |
| 84 |
}; |
| 85 |
|
| 86 |
struct FFTW::FFTW_Plans { |
| 87 |
mutable std::map<std::int64_t, std::unique_ptr<FFTW_Plan>> fftw_plans_; |
| 88 |
mutable std::mutex fftw_plans_mutex; // protects fftw_plans_ |
| 89 |
|
| 90 |
FFTW_Plan& get_plan(size_t size_out, std::complex<double>* in, double* out) const { |
| 91 |
std::lock_guard lock(fftw_plans_mutex); |
| 92 |
|
| 93 |
bool inverse = true; |
| 94 |
bool inplace = (static_cast<void*>(in) == static_cast<void*>(out)); |
| 95 |
bool aligned = ((reinterpret_cast<size_t>(in) & 15) | (reinterpret_cast<size_t>(out) & 15)) == 0; |
| 96 |
int n1 = size_out; |
| 97 |
std::int64_t key = ((n1 << 3) | (inverse << 2) | (inplace << 1) | aligned) << 1; |
| 98 |
auto it = fftw_plans_.find(key); |
| 99 |
if (it == fftw_plans_.end()) { |
| 100 |
it = fftw_plans_.emplace(key, new FFTW_Plan()).first; |
| 101 |
it->second->create_inverse_c2r(size_out, in, out); |
| 102 |
} |
| 103 |
return *(it->second); |
| 104 |
} |
| 105 |
|
| 106 |
FFTW_Plan& get_plan_many(size_t howmany, size_t size_in, size_t size_out, size_t dist_in, size_t dist_out, std::complex<double>* in, double* out) const { |
| 107 |
std::lock_guard lock(fftw_plans_mutex); |
| 108 |
|
| 109 |
bool inverse = true; |
| 110 |
bool inplace = (static_cast<void*>(in) == static_cast<void*>(out)); |
| 111 |
bool aligned = ((reinterpret_cast<size_t>(in) & 15) | (reinterpret_cast<size_t>(out) & 15)) == 0; |
| 112 |
int n0 = howmany; |
| 113 |
int n1 = size_out; |
| 114 |
std::int64_t key = (((((std::int64_t)n0) << 30) | (n1 << 3) | (inverse << 2) | (inplace << 1) | aligned) << 1) + 1; |
| 115 |
auto it = fftw_plans_.find(key); |
| 116 |
if (it == fftw_plans_.end()) { |
| 117 |
it = fftw_plans_.emplace(key, new FFTW_Plan()).first; |
| 118 |
it->second->create_inverse_c2r_many(howmany, size_in, size_out, dist_in, dist_out, in, out); |
| 119 |
} |
| 120 |
return *(it->second); |
| 121 |
} |
| 122 |
}; |
| 123 |
|
| 124 |
FFTW::FFTW() { |
| 125 |
plans_ = new FFTW_Plans(); |
| 126 |
} |
| 127 |
|
| 128 |
FFTW::~FFTW() { |
| 129 |
delete plans_; |
| 130 |
} |
| 131 |
|
| 132 |
void FFTW::do_plan_inverse_c2r(size_t size_out, std::complex<double>* in, double* out) const { |
| 133 |
plans_->get_plan(size_out, in, out); |
| 134 |
} |
| 135 |
void FFTW::do_plan_inverse_c2r_many(size_t howmany, size_t size_in, size_t size_out, size_t dist_in, size_t dist_out, std::complex<double>* in, double* out) const { |
| 136 |
plans_->get_plan_many(howmany, size_in, size_out, dist_in, dist_out, in, out); |
| 137 |
} |
| 138 |
void FFTW::do_inverse_c2r(size_t size_out, std::complex<double>* in, double* out) const { |
| 139 |
plans_->get_plan(size_out, in, out).inverse_c2r(in, out); |
| 140 |
} |
| 141 |
void FFTW::do_inverse_c2r_many(size_t howmany, size_t size_in, size_t size_out, size_t dist_in, size_t dist_out, std::complex<double>* in, double* out) const { |
| 142 |
plans_->get_plan_many(howmany, size_in, size_out, dist_in, dist_out, in, out).inverse_c2r(in, out); |
| 143 |
} |
| 144 |
|
| 145 |
// -------------------------------------------------------------------------------------------------------------------- |
| 146 |
|
| 147 |
} // namespace atlas::linalg |