Debian Codegraph @

src/atlas/linalg/fft/FFTW.cc

raw
1 /*
2 * (C) Copyright 2025- ECMWF.
3 *
4 * This software is licensed under the terms of the Apache Licence Version 2.0
5 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6 * In applying this licence, ECMWF does not waive the privileges and immunities
7 * granted to it by virtue of its status as an intergovernmental organisation
8 * nor does it submit to any jurisdiction.
9 */
10
11 #include "FFT.h"
12
13 #include <cstdlib>
14 #include <mutex>
15 #include <sstream>
16
17 #include "eckit/log/Bytes.h"
18
19 #include "atlas/runtime/Exception.h"
20 #include "atlas/runtime/Log.h"
21 #include "atlas/runtime/Trace.h"
22
23 #include "atlas/library/defines.h"
24
25 #if ATLAS_HAVE_FFTW
26 #include "fftw3.h"
27 #endif
28
29 namespace atlas::linalg {
30
31 #define THROW_ATLAS_FFTW_NOT_SUPPORTED() throw_Exception("Atlas was not compiled with FFTW support", Here())
32
33 struct FFTW_Plan {
34 #if ATLAS_HAVE_FFTW
35 ~FFTW_Plan() {
36 destroy();
37 }
38 void create_inverse_c2r(size_t size_out, std::complex<double>* in, double* out) {
39 ATLAS_TRACE("FFTW::create_plan_inverse_c2r");
40 plan = fftw_plan_dft_c2r_1d(size_out, reinterpret_cast<fftw_complex*>(in), out, fftw_plan_flags);
41 constructed_ = true;
42 }
43 void create_inverse_c2r_many(size_t howmany, size_t size_in, size_t size_out, size_t dist_in, size_t dist_out, std::complex<double>* in, double* out) {
44 ATLAS_TRACE("FFTW::create_plan_inverse_c2r");
45 constexpr int rank = 1; /* 1d transforms */
46 const int n[] = {static_cast<int>(size_out)}; /* each transform of length size_out */
47 constexpr int istride = 1;
48 constexpr int ostride = 1;
49 const int idist = dist_in;
50 const int odist = dist_out;
51 const int* inembed = nullptr;
52 const int* onembed = nullptr;
53 plan = fftw_plan_many_dft_c2r(rank, n, howmany, reinterpret_cast<fftw_complex*>(in),
54 inembed, istride, idist, out, onembed, ostride, odist, fftw_plan_flags);
55 constructed_ = true;
56 }
57
58 void inverse_c2r(std::complex<double>* in, double* out) {
59 ATLAS_TRACE("FFTW::inverse_c2r");
60 ATLAS_ASSERT(constructed_);
61 fftw_execute_dft_c2r(plan, reinterpret_cast<fftw_complex*>(in), out);
62 }
63
64 void destroy() {
65 if (constructed_) {
66 fftw_destroy_plan(plan);
67 }
68 constructed_ = false;
69 }
70 fftw_plan plan;
71 bool constructed_{false};
72 int fftw_plan_flags{FFTW_ESTIMATE};
73 #else
74 void inverse_c2r(std::complex<double>* in, double* out) {
75 THROW_ATLAS_FFTW_NOT_SUPPORTED();
76 }
77 void create_inverse_c2r(size_t size_out, std::complex<double>* in, double* out) {
78 THROW_ATLAS_FFTW_NOT_SUPPORTED();
79 }
80 void create_inverse_c2r_many(size_t howmany, size_t size_in, size_t size_out, size_t dist_in, size_t dist_out, std::complex<double>* in, double* out) {
81 THROW_ATLAS_FFTW_NOT_SUPPORTED();
82 }
83 #endif
84 };
85
86 struct FFTW::FFTW_Plans {
87 mutable std::map<std::int64_t, std::unique_ptr<FFTW_Plan>> fftw_plans_;
88 mutable std::mutex fftw_plans_mutex; // protects fftw_plans_
89
90 FFTW_Plan& get_plan(size_t size_out, std::complex<double>* in, double* out) const {
91 std::lock_guard lock(fftw_plans_mutex);
92
93 bool inverse = true;
94 bool inplace = (static_cast<void*>(in) == static_cast<void*>(out));
95 bool aligned = ((reinterpret_cast<size_t>(in) & 15) | (reinterpret_cast<size_t>(out) & 15)) == 0;
96 int n1 = size_out;
97 std::int64_t key = ((n1 << 3) | (inverse << 2) | (inplace << 1) | aligned) << 1;
98 auto it = fftw_plans_.find(key);
99 if (it == fftw_plans_.end()) {
100 it = fftw_plans_.emplace(key, new FFTW_Plan()).first;
101 it->second->create_inverse_c2r(size_out, in, out);
102 }
103 return *(it->second);
104 }
105
106 FFTW_Plan& get_plan_many(size_t howmany, size_t size_in, size_t size_out, size_t dist_in, size_t dist_out, std::complex<double>* in, double* out) const {
107 std::lock_guard lock(fftw_plans_mutex);
108
109 bool inverse = true;
110 bool inplace = (static_cast<void*>(in) == static_cast<void*>(out));
111 bool aligned = ((reinterpret_cast<size_t>(in) & 15) | (reinterpret_cast<size_t>(out) & 15)) == 0;
112 int n0 = howmany;
113 int n1 = size_out;
114 std::int64_t key = (((((std::int64_t)n0) << 30) | (n1 << 3) | (inverse << 2) | (inplace << 1) | aligned) << 1) + 1;
115 auto it = fftw_plans_.find(key);
116 if (it == fftw_plans_.end()) {
117 it = fftw_plans_.emplace(key, new FFTW_Plan()).first;
118 it->second->create_inverse_c2r_many(howmany, size_in, size_out, dist_in, dist_out, in, out);
119 }
120 return *(it->second);
121 }
122 };
123
124 FFTW::FFTW() {
125 plans_ = new FFTW_Plans();
126 }
127
128 FFTW::~FFTW() {
129 delete plans_;
130 }
131
132 void FFTW::do_plan_inverse_c2r(size_t size_out, std::complex<double>* in, double* out) const {
133 plans_->get_plan(size_out, in, out);
134 }
135 void FFTW::do_plan_inverse_c2r_many(size_t howmany, size_t size_in, size_t size_out, size_t dist_in, size_t dist_out, std::complex<double>* in, double* out) const {
136 plans_->get_plan_many(howmany, size_in, size_out, dist_in, dist_out, in, out);
137 }
138 void FFTW::do_inverse_c2r(size_t size_out, std::complex<double>* in, double* out) const {
139 plans_->get_plan(size_out, in, out).inverse_c2r(in, out);
140 }
141 void FFTW::do_inverse_c2r_many(size_t howmany, size_t size_in, size_t size_out, size_t dist_in, size_t dist_out, std::complex<double>* in, double* out) const {
142 plans_->get_plan_many(howmany, size_in, size_out, dist_in, dist_out, in, out).inverse_c2r(in, out);
143 }
144
145 // --------------------------------------------------------------------------------------------------------------------
146
147 } // namespace atlas::linalg