| 1 |
/* |
| 2 |
* SPDX-FileCopyrightText: 2026 Oak Ridge National Laboratory and Contributors |
| 3 |
* |
| 4 |
* SPDX-License-Identifier: Apache-2.0 |
| 5 |
*/ |
| 6 |
|
| 7 |
#include <algorithm> |
| 8 |
#include <chrono> |
| 9 |
#include <cmath> |
| 10 |
#include <cstdint> |
| 11 |
#include <iostream> |
| 12 |
#include <stdexcept> |
| 13 |
#include <string> |
| 14 |
#include <thread> |
| 15 |
|
| 16 |
#include "adios2.h" |
| 17 |
|
| 18 |
bool epsilon(double d) { return (d < 1.0e-20); } |
| 19 |
bool epsilon(float d) { return (d < 1.0e-20); } |
| 20 |
|
| 21 |
/* |
| 22 |
* Function to compute the PDF of a 2D slice |
| 23 |
*/ |
| 24 |
template <class T> |
| 25 |
void compute_pdf(const std::vector<T> &data, const std::vector<std::size_t> &shape, |
| 26 |
const size_t start, const size_t count, const size_t nbins, const T min, |
| 27 |
const T max, std::vector<T> &pdf, std::vector<T> &bins) |
| 28 |
{ |
| 29 |
if (shape.size() != 3) |
| 30 |
throw std::invalid_argument("ERROR: shape is expected to be 3D\n"); |
| 31 |
|
| 32 |
size_t slice_size = shape[1] * shape[2]; |
| 33 |
pdf.resize(count * nbins); |
| 34 |
bins.resize(nbins); |
| 35 |
|
| 36 |
size_t start_data = 0; |
| 37 |
size_t start_pdf = 0; |
| 38 |
|
| 39 |
T binWidth = static_cast<T>((max - min) / static_cast<double>(nbins)); |
| 40 |
for (size_t i = 0; i < nbins; ++i) |
| 41 |
{ |
| 42 |
bins[i] = static_cast<T>(min + (static_cast<T>(i) * binWidth)); |
| 43 |
} |
| 44 |
|
| 45 |
if (nbins == 1) |
| 46 |
{ |
| 47 |
// special case: only one bin |
| 48 |
for (size_t i = 0; i < count; ++i) |
| 49 |
{ |
| 50 |
pdf[i] = static_cast<T>(slice_size); |
| 51 |
} |
| 52 |
return; |
| 53 |
} |
| 54 |
|
| 55 |
if (epsilon(max - min) || epsilon(binWidth)) |
| 56 |
{ |
| 57 |
// special case: constant array |
| 58 |
for (size_t i = 0; i < count; ++i) |
| 59 |
{ |
| 60 |
pdf[i * nbins + (nbins / 2)] = static_cast<T>(slice_size); |
| 61 |
} |
| 62 |
return; |
| 63 |
} |
| 64 |
|
| 65 |
for (size_t i = 0; i < count; ++i) |
| 66 |
{ |
| 67 |
// Calculate a PDF for 'nbins' bins for values between 'min' and 'max' |
| 68 |
// from data[ start_data .. start_data+slice_size-1 ] |
| 69 |
// into pdf[ start_pdf .. start_pdf+nbins-1 ] |
| 70 |
for (size_t j = 0; j < slice_size; ++j) |
| 71 |
{ |
| 72 |
if (data[start_data + j] > max || data[start_data + j] < min) |
| 73 |
{ |
| 74 |
std::cout << " data[" << start * slice_size + start_data + j |
| 75 |
<< "] = " << data[start_data + j] << " is out of [min,max] = [" << min |
| 76 |
<< "," << max << "]" << std::endl; |
| 77 |
} |
| 78 |
size_t bin = static_cast<size_t>(std::floor((data[start_data + j] - min) / binWidth)); |
| 79 |
if (bin == nbins) |
| 80 |
{ |
| 81 |
bin = nbins - 1; |
| 82 |
} |
| 83 |
++pdf[start_pdf + bin]; |
| 84 |
} |
| 85 |
start_pdf += nbins; |
| 86 |
start_data += slice_size; |
| 87 |
} |
| 88 |
return; |
| 89 |
} |
| 90 |
|
| 91 |
/* |
| 92 |
* Print info to the user on how to invoke the application |
| 93 |
*/ |
| 94 |
void printUsage() |
| 95 |
{ |
| 96 |
std::cout << "Usage: pdf_calc input output [N] [output_inputdata]\n" |
| 97 |
<< " input: Name of the input file handle for reading data\n" |
| 98 |
<< " output: Name of the output file to which data must be written\n" |
| 99 |
<< " N: Number of bins for the PDF calculation, default = 1000\n" |
| 100 |
<< " output_inputdata: YES will write the original variables besides " |
| 101 |
"the analysis results\n\n"; |
| 102 |
} |
| 103 |
|
| 104 |
/* |
| 105 |
* MAIN |
| 106 |
*/ |
| 107 |
int main(int argc, char *argv[]) |
| 108 |
{ |
| 109 |
int provided; |
| 110 |
MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); |
| 111 |
int rank, comm_size, wrank; |
| 112 |
|
| 113 |
MPI_Comm_rank(MPI_COMM_WORLD, &wrank); |
| 114 |
|
| 115 |
const unsigned int color = 2; |
| 116 |
MPI_Comm comm; |
| 117 |
MPI_Comm_split(MPI_COMM_WORLD, color, wrank, &comm); |
| 118 |
|
| 119 |
MPI_Comm_rank(comm, &rank); |
| 120 |
MPI_Comm_size(comm, &comm_size); |
| 121 |
|
| 122 |
if (argc < 3) |
| 123 |
{ |
| 124 |
std::cout << "Not enough arguments\n"; |
| 125 |
if (rank == 0) |
| 126 |
printUsage(); |
| 127 |
MPI_Finalize(); |
| 128 |
return 0; |
| 129 |
} |
| 130 |
|
| 131 |
std::string in_filename; |
| 132 |
std::string out_filename; |
| 133 |
size_t nbins = 1000; |
| 134 |
bool write_inputvars = false; |
| 135 |
in_filename = argv[1]; |
| 136 |
out_filename = argv[2]; |
| 137 |
|
| 138 |
if (argc >= 4) |
| 139 |
{ |
| 140 |
int value = std::stoi(argv[3]); |
| 141 |
if (value > 0) |
| 142 |
nbins = static_cast<size_t>(value); |
| 143 |
} |
| 144 |
|
| 145 |
if (argc >= 5) |
| 146 |
{ |
| 147 |
std::string value = argv[4]; |
| 148 |
std::transform(value.begin(), value.end(), value.begin(), ::tolower); |
| 149 |
if (value == "yes") |
| 150 |
write_inputvars = true; |
| 151 |
} |
| 152 |
|
| 153 |
// std::size_t u_global_size, v_global_size; |
| 154 |
// std::size_t u_local_size, v_local_size; |
| 155 |
|
| 156 |
bool firstStep = true; |
| 157 |
|
| 158 |
std::vector<std::size_t> shape; |
| 159 |
|
| 160 |
std::vector<double> u; |
| 161 |
std::vector<double> v; |
| 162 |
int simStep = -5; |
| 163 |
|
| 164 |
std::vector<double> pdf_u; |
| 165 |
std::vector<double> pdf_v; |
| 166 |
std::vector<double> bins_u; |
| 167 |
std::vector<double> bins_v; |
| 168 |
|
| 169 |
// adios2 variable declarations |
| 170 |
adios2::Variable<double> var_u_in, var_v_in; |
| 171 |
adios2::Variable<int> var_step_in; |
| 172 |
adios2::Variable<double> var_u_pdf, var_v_pdf; |
| 173 |
adios2::Variable<double> var_u_bins, var_v_bins; |
| 174 |
adios2::Variable<int> var_step_out; |
| 175 |
adios2::Variable<double> var_u_out, var_v_out; |
| 176 |
|
| 177 |
{ |
| 178 |
// adios2 io object and engine init |
| 179 |
adios2::ADIOS ad("adios2.xml", comm); |
| 180 |
|
| 181 |
// IO objects for reading and writing |
| 182 |
adios2::IO reader_io = ad.DeclareIO("SimulationOutput"); |
| 183 |
adios2::IO writer_io = ad.DeclareIO("PDFAnalysisOutput"); |
| 184 |
if (!rank) |
| 185 |
{ |
| 186 |
std::cout << "PDF analysis reads from Simulation using engine type: " |
| 187 |
<< reader_io.EngineType() << std::endl; |
| 188 |
std::cout << "PDF analysis writes using engine type: " |
| 189 |
<< writer_io.EngineType() << std::endl; |
| 190 |
} |
| 191 |
|
| 192 |
// Engines for reading and writing |
| 193 |
adios2::Engine reader = reader_io.Open(in_filename, adios2::Mode::Read, comm); |
| 194 |
adios2::Engine writer = writer_io.Open(out_filename, adios2::Mode::Write, comm); |
| 195 |
|
| 196 |
bool shouldIWrite = (!rank || reader_io.EngineType() == "HDF5"); |
| 197 |
|
| 198 |
// read data per timestep |
| 199 |
int stepAnalysis = 0; |
| 200 |
while (true) |
| 201 |
{ |
| 202 |
|
| 203 |
// Begin step |
| 204 |
adios2::StepStatus read_status = reader.BeginStep(adios2::StepMode::Read, 10.0f); |
| 205 |
if (read_status == adios2::StepStatus::NotReady) |
| 206 |
{ |
| 207 |
// std::cout << "Stream not ready yet. Waiting...\n"; |
| 208 |
std::this_thread::sleep_for(std::chrono::milliseconds(1000)); |
| 209 |
continue; |
| 210 |
} |
| 211 |
else if (read_status != adios2::StepStatus::OK) |
| 212 |
{ |
| 213 |
break; |
| 214 |
} |
| 215 |
|
| 216 |
int stepSimOut = static_cast<int>(reader.CurrentStep()); |
| 217 |
|
| 218 |
// Inquire variable and set the selection at the first step only |
| 219 |
// This assumes that the variable dimensions do not change across |
| 220 |
// timesteps |
| 221 |
|
| 222 |
// Inquire variable |
| 223 |
var_u_in = reader_io.InquireVariable<double>("U"); |
| 224 |
var_v_in = reader_io.InquireVariable<double>("V"); |
| 225 |
var_step_in = reader_io.InquireVariable<int>("step"); |
| 226 |
|
| 227 |
std::pair<double, double> minmax_u = var_u_in.MinMax(); |
| 228 |
std::pair<double, double> minmax_v = var_v_in.MinMax(); |
| 229 |
|
| 230 |
shape = var_u_in.Shape(); |
| 231 |
|
| 232 |
// Calculate global and local sizes of U and V |
| 233 |
// u_global_size = shape[0] * shape[1] * shape[2]; |
| 234 |
// u_local_size = u_global_size / comm_size; |
| 235 |
// v_global_size = shape[0] * shape[1] * shape[2]; |
| 236 |
// v_local_size = v_global_size / comm_size; |
| 237 |
|
| 238 |
size_t count1 = shape[0] / comm_size; |
| 239 |
size_t start1 = count1 * rank; |
| 240 |
if (rank == comm_size - 1) |
| 241 |
{ |
| 242 |
// last process need to read all the rest of slices |
| 243 |
count1 = shape[0] - count1 * (comm_size - 1); |
| 244 |
} |
| 245 |
|
| 246 |
/*std::cout << " rank " << rank << " slice start={" << start1 |
| 247 |
<< ",0,0} count={" << count1 << "," << shape[1] << "," << |
| 248 |
shape[2] |
| 249 |
<< "}" << std::endl;*/ |
| 250 |
|
| 251 |
// Set selection |
| 252 |
var_u_in.SetSelection( |
| 253 |
adios2::Box<adios2::Dims>({start1, 0, 0}, {count1, shape[1], shape[2]})); |
| 254 |
var_v_in.SetSelection( |
| 255 |
adios2::Box<adios2::Dims>({start1, 0, 0}, {count1, shape[1], shape[2]})); |
| 256 |
|
| 257 |
// Declare variables to output |
| 258 |
if (firstStep) |
| 259 |
{ |
| 260 |
var_u_pdf = writer_io.DefineVariable<double>("U/pdf", {shape[0], nbins}, |
| 261 |
{start1, 0}, {count1, nbins}); |
| 262 |
var_v_pdf = writer_io.DefineVariable<double>("V/pdf", {shape[0], nbins}, |
| 263 |
{start1, 0}, {count1, nbins}); |
| 264 |
|
| 265 |
if (shouldIWrite) |
| 266 |
{ |
| 267 |
var_u_bins = writer_io.DefineVariable<double>("U/bins", {nbins}, {0}, {nbins}); |
| 268 |
var_v_bins = writer_io.DefineVariable<double>("V/bins", {nbins}, {0}, {nbins}); |
| 269 |
var_step_out = writer_io.DefineVariable<int>("step"); |
| 270 |
} |
| 271 |
|
| 272 |
if (write_inputvars) |
| 273 |
{ |
| 274 |
var_u_out = writer_io.DefineVariable<double>( |
| 275 |
"U", {shape[0], shape[1], shape[2]}, {start1, 0, 0}, |
| 276 |
{count1, shape[1], shape[2]}); |
| 277 |
var_v_out = writer_io.DefineVariable<double>( |
| 278 |
"V", {shape[0], shape[1], shape[2]}, {start1, 0, 0}, |
| 279 |
{count1, shape[1], shape[2]}); |
| 280 |
} |
| 281 |
firstStep = false; |
| 282 |
} |
| 283 |
|
| 284 |
// Read adios2 data |
| 285 |
reader.Get<double>(var_u_in, u); |
| 286 |
reader.Get<double>(var_v_in, v); |
| 287 |
if (shouldIWrite) |
| 288 |
{ |
| 289 |
reader.Get<int>(var_step_in, &simStep); |
| 290 |
} |
| 291 |
|
| 292 |
// End adios2 step |
| 293 |
reader.EndStep(); |
| 294 |
|
| 295 |
if (!rank) |
| 296 |
{ |
| 297 |
std::cout << "PDF Analysis step " << stepAnalysis << " processing sim output step " |
| 298 |
<< stepSimOut << " sim compute step " << simStep << std::endl; |
| 299 |
} |
| 300 |
|
| 301 |
// HDF5 engine does not provide min/max. Let's calculate it |
| 302 |
// if (reader_io.EngineType() == "HDF5") |
| 303 |
{ |
| 304 |
auto mmu = std::minmax_element(u.begin(), u.end()); |
| 305 |
minmax_u = std::make_pair(*mmu.first, *mmu.second); |
| 306 |
auto mmv = std::minmax_element(v.begin(), v.end()); |
| 307 |
minmax_v = std::make_pair(*mmv.first, *mmv.second); |
| 308 |
} |
| 309 |
|
| 310 |
// Compute PDF |
| 311 |
std::vector<double> pdf_u; |
| 312 |
std::vector<double> bins_u; |
| 313 |
compute_pdf(u, shape, start1, count1, nbins, minmax_u.first, minmax_u.second, pdf_u, |
| 314 |
bins_u); |
| 315 |
|
| 316 |
std::vector<double> pdf_v; |
| 317 |
std::vector<double> bins_v; |
| 318 |
compute_pdf(v, shape, start1, count1, nbins, minmax_v.first, minmax_v.second, pdf_v, |
| 319 |
bins_v); |
| 320 |
|
| 321 |
// write U, V, and their norms out |
| 322 |
writer.BeginStep(); |
| 323 |
writer.Put<double>(var_u_pdf, pdf_u.data()); |
| 324 |
writer.Put<double>(var_v_pdf, pdf_v.data()); |
| 325 |
if (shouldIWrite) |
| 326 |
{ |
| 327 |
writer.Put<double>(var_u_bins, bins_u.data()); |
| 328 |
writer.Put<double>(var_v_bins, bins_v.data()); |
| 329 |
writer.Put<int>(var_step_out, simStep); |
| 330 |
} |
| 331 |
if (write_inputvars) |
| 332 |
{ |
| 333 |
writer.Put<double>(var_u_out, u.data()); |
| 334 |
writer.Put<double>(var_v_out, v.data()); |
| 335 |
} |
| 336 |
writer.EndStep(); |
| 337 |
++stepAnalysis; |
| 338 |
} |
| 339 |
|
| 340 |
// cleanup |
| 341 |
reader.Close(); |
| 342 |
writer.Close(); |
| 343 |
} |
| 344 |
|
| 345 |
MPI_Barrier(comm); |
| 346 |
MPI_Finalize(); |
| 347 |
return 0; |
| 348 |
} |