| 1 |
/* |
| 2 |
* SPDX-FileCopyrightText: 2026 Oak Ridge National Laboratory and Contributors |
| 3 |
* |
| 4 |
* SPDX-License-Identifier: Apache-2.0 |
| 5 |
*/ |
| 6 |
|
| 7 |
#include <fstream> |
| 8 |
#include <iostream> |
| 9 |
|
| 10 |
#include <adios2.h> |
| 11 |
#include <mpi.h> |
| 12 |
|
| 13 |
#include "../common/timer.hpp" |
| 14 |
#include "gray-scott.h" |
| 15 |
#include "restart.h" |
| 16 |
#include "writer.h" |
| 17 |
|
| 18 |
void print_io_settings(const adios2::IO &io) |
| 19 |
{ |
| 20 |
std::cout << "Simulation writes data using engine type: " << io.EngineType() |
| 21 |
<< std::endl; |
| 22 |
auto ioparams = io.Parameters(); |
| 23 |
std::cout << "IO parameters: " << std::endl; |
| 24 |
for (const auto &p : ioparams) |
| 25 |
{ |
| 26 |
std::cout << " " << p.first << " = " << p.second << std::endl; |
| 27 |
} |
| 28 |
} |
| 29 |
|
| 30 |
void print_settings(const Settings &s, int restart_step) |
| 31 |
{ |
| 32 |
std::cout << "grid: " << s.L << "x" << s.L << "x" << s.L << std::endl; |
| 33 |
if (restart_step > 0) |
| 34 |
{ |
| 35 |
std::cout << "restart: from step " << restart_step << std::endl; |
| 36 |
} |
| 37 |
else |
| 38 |
{ |
| 39 |
std::cout << "restart: no" << std::endl; |
| 40 |
} |
| 41 |
std::cout << "steps: " << s.steps << std::endl; |
| 42 |
std::cout << "plotgap: " << s.plotgap << std::endl; |
| 43 |
std::cout << "F: " << s.F << std::endl; |
| 44 |
std::cout << "k: " << s.k << std::endl; |
| 45 |
std::cout << "dt: " << s.dt << std::endl; |
| 46 |
std::cout << "Du: " << s.Du << std::endl; |
| 47 |
std::cout << "Dv: " << s.Dv << std::endl; |
| 48 |
std::cout << "noise: " << s.noise << std::endl; |
| 49 |
std::cout << "output: " << s.output << std::endl; |
| 50 |
std::cout << "adios_config: " << s.adios_config << std::endl; |
| 51 |
} |
| 52 |
|
| 53 |
void print_simulator_settings(const GrayScott &s) |
| 54 |
{ |
| 55 |
std::cout << "process layout: " << s.npx << "x" << s.npy << "x" << s.npz << std::endl; |
| 56 |
std::cout << "local grid size: " << s.size_x << "x" << s.size_y << "x" << s.size_z |
| 57 |
<< std::endl; |
| 58 |
} |
| 59 |
|
| 60 |
int main(int argc, char **argv) |
| 61 |
{ |
| 62 |
int provided; |
| 63 |
MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); |
| 64 |
int rank, procs, wrank; |
| 65 |
|
| 66 |
MPI_Comm_rank(MPI_COMM_WORLD, &wrank); |
| 67 |
|
| 68 |
const unsigned int color = 1; |
| 69 |
MPI_Comm comm; |
| 70 |
MPI_Comm_split(MPI_COMM_WORLD, color, wrank, &comm); |
| 71 |
|
| 72 |
MPI_Comm_rank(comm, &rank); |
| 73 |
MPI_Comm_size(comm, &procs); |
| 74 |
|
| 75 |
if (argc < 2) |
| 76 |
{ |
| 77 |
if (rank == 0) |
| 78 |
{ |
| 79 |
std::cerr << "Too few arguments" << std::endl; |
| 80 |
std::cerr << "Usage: gray-scott settings.json" << std::endl; |
| 81 |
} |
| 82 |
MPI_Abort(MPI_COMM_WORLD, -1); |
| 83 |
} |
| 84 |
|
| 85 |
Settings settings = Settings::from_json(argv[1]); |
| 86 |
{ |
| 87 |
|
| 88 |
GrayScott sim(settings, comm); |
| 89 |
sim.init(); |
| 90 |
|
| 91 |
adios2::ADIOS adios(settings.adios_config, comm); |
| 92 |
adios2::IO io_main = adios.DeclareIO("SimulationOutput"); |
| 93 |
adios2::IO io_ckpt = adios.DeclareIO("SimulationCheckpoint"); |
| 94 |
|
| 95 |
int restart_step = 0; |
| 96 |
if (settings.restart) |
| 97 |
{ |
| 98 |
restart_step = ReadRestart(comm, settings, sim, io_ckpt); |
| 99 |
io_main.SetParameter("AppendAfterSteps", |
| 100 |
std::to_string(restart_step / settings.plotgap)); |
| 101 |
} |
| 102 |
|
| 103 |
Writer writer_main(settings, sim, io_main); |
| 104 |
writer_main.open(settings.output, (restart_step > 0)); |
| 105 |
|
| 106 |
if (rank == 0) |
| 107 |
{ |
| 108 |
print_io_settings(io_main); |
| 109 |
std::cout << "========================================" << std::endl; |
| 110 |
print_settings(settings, restart_step); |
| 111 |
print_simulator_settings(sim); |
| 112 |
std::cout << "========================================" << std::endl; |
| 113 |
} |
| 114 |
|
| 115 |
#ifdef ENABLE_TIMERS |
| 116 |
Timer timer_total; |
| 117 |
Timer timer_compute; |
| 118 |
Timer timer_write; |
| 119 |
|
| 120 |
std::ostringstream log_fname; |
| 121 |
log_fname << "gray_scott_pe_" << rank << ".log"; |
| 122 |
|
| 123 |
std::ofstream log(log_fname.str()); |
| 124 |
log << "step\ttotal_gs\tcompute_gs\twrite_gs" << std::endl; |
| 125 |
#endif |
| 126 |
|
| 127 |
for (int it = restart_step; it < settings.steps;) |
| 128 |
{ |
| 129 |
#ifdef ENABLE_TIMERS |
| 130 |
MPI_Barrier(comm); |
| 131 |
timer_total.start(); |
| 132 |
timer_compute.start(); |
| 133 |
#endif |
| 134 |
|
| 135 |
sim.iterate(); |
| 136 |
it++; |
| 137 |
|
| 138 |
#ifdef ENABLE_TIMERS |
| 139 |
timer_compute.stop(); |
| 140 |
MPI_Barrier(comm); |
| 141 |
timer_write.start(); |
| 142 |
#endif |
| 143 |
|
| 144 |
if (it % settings.plotgap == 0) |
| 145 |
{ |
| 146 |
if (rank == 0) |
| 147 |
{ |
| 148 |
std::cout << "Simulation at step " << it << " writing output step " |
| 149 |
<< it / settings.plotgap << std::endl; |
| 150 |
} |
| 151 |
|
| 152 |
writer_main.write(it, sim); |
| 153 |
} |
| 154 |
|
| 155 |
if (settings.checkpoint && (it % settings.checkpoint_freq) == 0) |
| 156 |
{ |
| 157 |
WriteCkpt(comm, it, settings, sim, io_ckpt); |
| 158 |
} |
| 159 |
|
| 160 |
#ifdef ENABLE_TIMERS |
| 161 |
double time_write = timer_write.stop(); |
| 162 |
double time_step = timer_total.stop(); |
| 163 |
MPI_Barrier(comm); |
| 164 |
|
| 165 |
log << it << "\t" << timer_total.elapsed() << "\t" << timer_compute.elapsed() << "\t" |
| 166 |
<< timer_write.elapsed() << std::endl; |
| 167 |
#endif |
| 168 |
} |
| 169 |
|
| 170 |
writer_main.close(); |
| 171 |
} |
| 172 |
#ifdef ENABLE_TIMERS |
| 173 |
log << "total\t" << timer_total.elapsed() << "\t" << timer_compute.elapsed() << "\t" |
| 174 |
<< timer_write.elapsed() << std::endl; |
| 175 |
|
| 176 |
log.close(); |
| 177 |
#endif |
| 178 |
|
| 179 |
MPI_Finalize(); |
| 180 |
} |