| 1 |
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced |
| 2 |
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files |
| 3 |
// LICENSE and NOTICE for details. LLNL-CODE-806117. |
| 4 |
// |
| 5 |
// This file is part of the MFEM library. For more information and source code |
| 6 |
// availability visit https://mfem.org. |
| 7 |
// |
| 8 |
// MFEM is free software; you can redistribute it and/or modify it under the |
| 9 |
// terms of the BSD-3 license. We welcome feedback and contributions, see file |
| 10 |
// CONTRIBUTING.md for details. |
| 11 |
// |
| 12 |
// -------------------------------------- |
| 13 |
// MG Abs L(1)-Jacobi smoothers miniapp |
| 14 |
// -------------------------------------- |
| 15 |
// |
| 16 |
// (See abs-l1-jacobi.cpp first) |
| 17 |
// |
| 18 |
// This miniapp illustrates the use of an absolute value L(1)-Jacobi smoother. |
| 19 |
// We use a multigrid approach (cf. ex26(p)). The global solver and the coarse |
| 20 |
// level solver are user-selected. The current options are SLI and PCG. The |
| 21 |
// intermediate levels are directy smoothed with the absolute value L(1)-Jacobi |
| 22 |
// preconditioner. The systems to solve correspond to a mass matrix, and a |
| 23 |
// difussion system. |
| 24 |
// |
| 25 |
// The preconditioner can be defined at run-time. Similarly, the mesh can be |
| 26 |
// modified by a Kershaw transformation at run-time. Relative tolerance and |
| 27 |
// maximum number of iterations can be modified as well. |
| 28 |
// |
| 29 |
// Compile with: make mg-abs-l1-jacobi |
| 30 |
// |
| 31 |
// Sample runs: |
| 32 |
// mpirun -np 4 mg-abs-l1-jacobi |
| 33 |
// mpirun -np 4 mg-abs-l1-jacobi -s 0 -i 0 |
| 34 |
// mpirun -np 4 mg-abs-l1-jacobi -m ../meshing/icf.mesh -f 0.5 |
| 35 |
// mpirun -np 4 mg-abs-l1-jacobi -rs 2 -rp 1 |
| 36 |
// mpirun -np 4 mg-abs-l1-jacobi -t 1e-5 -ni 100 |
| 37 |
// mpirun -np 4 mg-abs-l1-jacobi -m ../../data/beam-quad.mesh -a 3 -Ky 0.5 -Kz 0.5 |
| 38 |
// mpirun -np 4 mg-abs-l1-jacobi --device cuda |
| 39 |
|
| 40 |
#include "ds-common.hpp" |
| 41 |
|
| 42 |
using namespace std; |
| 43 |
using namespace mfem; |
| 44 |
using namespace ds_common; |
| 45 |
|
| 46 |
int main(int argc, char *argv[]) |
| 47 |
{ |
| 48 |
// 1. Initialize MPI and HYPRE. |
| 49 |
Mpi::Init(argc, argv); |
| 50 |
Hypre::Init(); |
| 51 |
|
| 52 |
// 2. Parse command line options. |
| 53 |
string mesh_file = "../../data/ref-cube.mesh"; |
| 54 |
// System properties |
| 55 |
int order = 1; |
| 56 |
SolverType solver_type = cg; |
| 57 |
IntegratorType integrator_type = diffusion; |
| 58 |
int assembly_type_int = 3; // Default is PARTIAL |
| 59 |
AssemblyLevel assembly_type; |
| 60 |
// Number of refinements |
| 61 |
int refine_serial = 3; |
| 62 |
int refine_parallel = 0; |
| 63 |
// Number of geometric and order levels |
| 64 |
int geometric_levels = 1; |
| 65 |
int order_levels = 1; |
| 66 |
// Solver parameters |
| 67 |
real_t rel_tol = 1e-10; |
| 68 |
real_t max_iter = 3000; |
| 69 |
// Kershaw Transformation |
| 70 |
real_t eps_y = 0.0; |
| 71 |
real_t eps_z = 0.0; |
| 72 |
// Other options |
| 73 |
string device_config = "cpu"; |
| 74 |
bool use_monitor = false; |
| 75 |
bool visualization = true; |
| 76 |
|
| 77 |
OptionsParser args(argc, argv); |
| 78 |
args.AddOption(&mesh_file, "-m", "--mesh", |
| 79 |
"Mesh file to use."); |
| 80 |
args.AddOption(&order, "-o", "--order", |
| 81 |
"Finite element order (polynomial degree)"); |
| 82 |
args.AddOption(&geometric_levels, "-gl", "--geometric-levels", |
| 83 |
"Number of geometric refinements (levels) done prior to order" |
| 84 |
" refinements."); |
| 85 |
args.AddOption(&order_levels, "-ol", "--order-levels", |
| 86 |
"Number of order refinements (levels). " |
| 87 |
"Finest level in the hierarchy has order 2^{or}."); |
| 88 |
args.AddOption((int*)&solver_type, "-s", "--solver", |
| 89 |
"Solvers to be considered:" |
| 90 |
"\n\t0: Stationary Linear Iteration" |
| 91 |
"\n\t1: Preconditioned Conjugate Gradient"); |
| 92 |
args.AddOption((int*)&integrator_type, "-i", "--integrator", |
| 93 |
"Integrators to be considered:" |
| 94 |
"\n\t0: MassIntegrator" |
| 95 |
"\n\t1: DiffusionIntegrator"); |
| 96 |
args.AddOption(&assembly_type_int, "-a", "--assembly", |
| 97 |
"Assembly level to be considered:" |
| 98 |
"\n\t0: LEGACY" |
| 99 |
"\n\t1: FULL" |
| 100 |
"\n\t2: ELEMENT" |
| 101 |
"\n\t3: PARTIAL" |
| 102 |
"\n\t4: NONE"); |
| 103 |
args.AddOption(&refine_serial, "-rs", "--refine-serial", |
| 104 |
"Number of serial refinements"); |
| 105 |
args.AddOption(&refine_parallel, "-rp", "--refine-parallel", |
| 106 |
"Number of parallel refinements"); |
| 107 |
args.AddOption(&rel_tol, "-t", "--tolerance", |
| 108 |
"Relative tolerance for the iterative solver"); |
| 109 |
args.AddOption(&max_iter, "-ni", "--iterations", |
| 110 |
"Maximum number of iterations"); |
| 111 |
args.AddOption(&eps_y, "-Ky", "--Kershaw-y", |
| 112 |
"Kershaw transform factor, eps_y in (0,1]"); |
| 113 |
args.AddOption(&eps_z, "-Kz", "--Kershaw-z", |
| 114 |
"Kershaw transform factor, eps_z in (0,1]"); |
| 115 |
args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact" |
| 116 |
" solution."); |
| 117 |
args.AddOption(&device_config, "-d", "--device", |
| 118 |
"Device configuration string, see Device::Configure()."); |
| 119 |
args.AddOption(&use_monitor, "-mon", "--monitor", "-no-mon", |
| 120 |
"--no-monitor", |
| 121 |
"Enable or disable Data Monitor."); |
| 122 |
args.AddOption(&visualization, "-vis", "--visualization", "-no-vis", |
| 123 |
"--no-visualization", |
| 124 |
"Enable or disable GLVis visualization."); |
| 125 |
args.ParseCheck(); |
| 126 |
|
| 127 |
MFEM_VERIFY((0 <= solver_type) && (solver_type < num_solvers), |
| 128 |
"invalid solver type: " << solver_type); |
| 129 |
MFEM_VERIFY((0 <= integrator_type) && (integrator_type < num_integrators), |
| 130 |
"invalid integrator type: " << integrator_type); |
| 131 |
MFEM_VERIFY((0 <= assembly_type_int) && (assembly_type_int < 6), |
| 132 |
"invalid assembly type: " << assembly_type_int); |
| 133 |
MFEM_VERIFY(geometric_levels >= 0, |
| 134 |
"geometric_levels needs to be non-negative"); |
| 135 |
MFEM_VERIFY(order_levels >= 0, "order_levels needs to be non-negative"); |
| 136 |
MFEM_VERIFY((0.0 <= eps_y) && (eps_y <= 1.0), "eps_y must be in [0,1]"); |
| 137 |
MFEM_VERIFY((0.0 <= eps_z) && (eps_z <= 1.0), "eps_z must be in [0,1]"); |
| 138 |
|
| 139 |
kappa = freq * M_PI; |
| 140 |
|
| 141 |
ostringstream file_name; |
| 142 |
if (use_monitor) |
| 143 |
{ |
| 144 |
file_name << "MGABS-" |
| 145 |
<< "G" << geometric_levels |
| 146 |
<< "O" << order_levels |
| 147 |
<< "O" << order |
| 148 |
<< "I" << (int) integrator_type |
| 149 |
<< "S" << (int) solver_type |
| 150 |
<< "A" << assembly_type_int |
| 151 |
<< ".csv"; |
| 152 |
} |
| 153 |
|
| 154 |
string assembly_description; |
| 155 |
switch (assembly_type_int) |
| 156 |
{ |
| 157 |
case 0: |
| 158 |
assembly_type = AssemblyLevel::LEGACY; |
| 159 |
assembly_description = "Using Legacy type of assembly level..."; |
| 160 |
break; |
| 161 |
case 1: |
| 162 |
assembly_type = AssemblyLevel::FULL; |
| 163 |
assembly_description = "Using Full type of assembly level..."; |
| 164 |
break; |
| 165 |
case 2: |
| 166 |
assembly_type = AssemblyLevel::ELEMENT; |
| 167 |
assembly_description = "Using Element type of assembly level..."; |
| 168 |
break; |
| 169 |
case 3: |
| 170 |
assembly_type = AssemblyLevel::PARTIAL; |
| 171 |
assembly_description = "Using Partial type of assembly level..."; |
| 172 |
break; |
| 173 |
case 4: |
| 174 |
assembly_type = AssemblyLevel::NONE; |
| 175 |
assembly_description = "Using matrix-free type of assembly level..."; |
| 176 |
break; |
| 177 |
default: |
| 178 |
MFEM_ABORT("Unsupported option!"); |
| 179 |
} |
| 180 |
|
| 181 |
Device device(device_config); |
| 182 |
if (Mpi::Root()) { device.Print(); } |
| 183 |
|
| 184 |
// 3. Read the serial mesh from the given mesh file. The number of serial and |
| 185 |
// parallel refinements can be set by the user on the command line. |
| 186 |
Mesh *serial_mesh = new Mesh(mesh_file); |
| 187 |
for (int ls = 0; ls < refine_serial; ls++) |
| 188 |
{ |
| 189 |
serial_mesh->UniformRefinement(); |
| 190 |
} |
| 191 |
|
| 192 |
// 4. Define a parallel mesh by a partitioning of the serial mesh. The number |
| 193 |
// of parallel refinements can be set by the user. If defined, apply |
| 194 |
// Kershaw transformation. |
| 195 |
ParMesh *mesh = new ParMesh(MPI_COMM_WORLD, *serial_mesh); |
| 196 |
delete serial_mesh; |
| 197 |
for (int lp = 0; lp < refine_parallel; lp++) |
| 198 |
{ |
| 199 |
mesh->UniformRefinement(); |
| 200 |
} |
| 201 |
|
| 202 |
dim = mesh->Dimension(); |
| 203 |
space_dim = mesh->SpaceDimension(); |
| 204 |
|
| 205 |
bool cond_z = (dim < 3) ? true : (eps_z != 0.0); // lazy check |
| 206 |
if (eps_y != 0.0 && cond_z) |
| 207 |
{ |
| 208 |
if (dim < 3) { eps_z = 0.0; } |
| 209 |
common::KershawTransformation kershawT(dim, eps_y, eps_z); |
| 210 |
mesh->Transform(kershawT); |
| 211 |
} |
| 212 |
|
| 213 |
// 5. Define a finite element space on the mesh. We use different spaces and |
| 214 |
// collections for different systems. |
| 215 |
// - H1-conforming Lagrange elements for the H1-mass matrix and the |
| 216 |
// diffusion problem. |
| 217 |
FiniteElementCollection *fec; |
| 218 |
ParFiniteElementSpace *coarse_fes; |
| 219 |
switch (integrator_type) |
| 220 |
{ |
| 221 |
case mass: |
| 222 |
case diffusion: |
| 223 |
fec = new H1_FECollection(order, dim); |
| 224 |
coarse_fes = new ParFiniteElementSpace(mesh, fec); |
| 225 |
break; |
| 226 |
case maxwell: |
| 227 |
mfem_error("Maxwell integrator not supported in this miniapp!"); |
| 228 |
default: |
| 229 |
mfem_error("Invalid integrator type! Check FiniteElementCollection"); |
| 230 |
} |
| 231 |
|
| 232 |
if (order > 1) |
| 233 |
{ |
| 234 |
if (Mpi::Root()) |
| 235 |
{ |
| 236 |
mfem::out << "Warning! Polynomial order provided. " |
| 237 |
<< "Ignoring order level..." << endl; |
| 238 |
} |
| 239 |
order_levels = 0; |
| 240 |
} |
| 241 |
|
| 242 |
// 6. Define a finite element space hierarchy for the multigrid solver. |
| 243 |
// Define a FEC array for the order-refinement levels. Add the refinements |
| 244 |
// to the hierarchy. |
| 245 |
Array<FiniteElementCollection*> fec_array; |
| 246 |
fec_array.Append(fec); |
| 247 |
// Transfer ownership of mesh and coarse_fes to fes_hierarchy |
| 248 |
ParFiniteElementSpaceHierarchy* fes_hierarchy = new |
| 249 |
ParFiniteElementSpaceHierarchy(mesh, coarse_fes, true, true); |
| 250 |
|
| 251 |
for (int lg = 0; lg < geometric_levels; ++lg) |
| 252 |
{ |
| 253 |
fes_hierarchy->AddUniformlyRefinedLevel(); |
| 254 |
} |
| 255 |
for (int lo = 0; lo < order_levels; ++lo) |
| 256 |
{ |
| 257 |
switch (integrator_type) |
| 258 |
{ |
| 259 |
case mass: |
| 260 |
case diffusion: |
| 261 |
fec_array.Append(new H1_FECollection(std::pow(2, lo + 1), dim)); |
| 262 |
break; |
| 263 |
default: |
| 264 |
mfem_error("Invalid integrator type! Check " |
| 265 |
"FiniteElementCollection for order refinements..."); |
| 266 |
} |
| 267 |
fes_hierarchy->AddOrderRefinedLevel(fec_array.Last()); |
| 268 |
} |
| 269 |
|
| 270 |
HYPRE_BigInt sys_size = fes_hierarchy->GetFinestFESpace().GlobalTrueVSize(); |
| 271 |
if (Mpi::Root()) |
| 272 |
{ |
| 273 |
mfem::out << "Number of unknowns: " << sys_size << endl; |
| 274 |
mfem::out << assembly_description << endl; |
| 275 |
} |
| 276 |
|
| 277 |
// 7. Extract the list of the essential boundary DoFs. We mark all boundary |
| 278 |
// attibutes as essential. AbsL1GeometricMultigrid will determine the |
| 279 |
// DoFs per level. |
| 280 |
Array<int> ess_bdr(mesh->bdr_attributes.Max()); |
| 281 |
if (mesh->bdr_attributes.Size()) { ess_bdr = 1; } |
| 282 |
|
| 283 |
// 8. Define the linear system. Set up the linear form b(.) which has the |
| 284 |
// standard form (f,v). |
| 285 |
ParLinearForm *b = new ParLinearForm(&fes_hierarchy->GetFinestFESpace()); |
| 286 |
LinearFormIntegrator *lfi = nullptr; |
| 287 |
|
| 288 |
// These pointers are not owned by the integrators |
| 289 |
FunctionCoefficient *scalar_u = nullptr; |
| 290 |
FunctionCoefficient *scalar_f = nullptr; |
| 291 |
|
| 292 |
ConstantCoefficient one(1.0); |
| 293 |
|
| 294 |
// These variables will define the linear system |
| 295 |
ParGridFunction x(&fes_hierarchy->GetFinestFESpace()); |
| 296 |
OperatorPtr A; |
| 297 |
Vector B, X; |
| 298 |
|
| 299 |
x = 0.0; |
| 300 |
|
| 301 |
switch (integrator_type) |
| 302 |
{ |
| 303 |
case mass: |
| 304 |
scalar_u = new FunctionCoefficient(diffusion_solution); |
| 305 |
lfi = new DomainLFIntegrator(*scalar_u); |
| 306 |
x.ProjectBdrCoefficient(*scalar_u, ess_bdr); |
| 307 |
break; |
| 308 |
case diffusion: |
| 309 |
scalar_u = new FunctionCoefficient(diffusion_solution); |
| 310 |
scalar_f = new FunctionCoefficient(diffusion_source); |
| 311 |
lfi = new DomainLFIntegrator(*scalar_f); |
| 312 |
x.ProjectBdrCoefficient(*scalar_u, ess_bdr); |
| 313 |
break; |
| 314 |
default: |
| 315 |
mfem_error("Invalid integrator type! Check ParLinearForm"); |
| 316 |
} |
| 317 |
b->AddDomainIntegrator(lfi); |
| 318 |
b->Assemble(); |
| 319 |
|
| 320 |
// 9. Define a geometric multigrid solver. The bilinear form a(.,.) is |
| 321 |
// assembled internally. Set up the type of cycles and form the linear |
| 322 |
// system. |
| 323 |
auto mg = new AbsL1GeometricMultigrid(*fes_hierarchy, |
| 324 |
ess_bdr, |
| 325 |
integrator_type, |
| 326 |
solver_type, |
| 327 |
assembly_type); |
| 328 |
mg->SetCycleType(Multigrid::CycleType::VCYCLE, 1, 1); |
| 329 |
mg->FormFineLinearSystem(x, *b, A, X, B); |
| 330 |
|
| 331 |
A.SetOperatorOwner(mg->GetOwnershipLevelOperators()); |
| 332 |
|
| 333 |
Solver *solver = nullptr; |
| 334 |
DataMonitor *monitor = nullptr; |
| 335 |
|
| 336 |
switch (solver_type) |
| 337 |
{ |
| 338 |
case sli: |
| 339 |
solver = new SLISolver(MPI_COMM_WORLD); |
| 340 |
break; |
| 341 |
case cg: |
| 342 |
solver = new CGSolver(MPI_COMM_WORLD); |
| 343 |
break; |
| 344 |
default: |
| 345 |
mfem_error("Invalid solver type!"); |
| 346 |
} |
| 347 |
solver->SetOperator(*A.Ptr()); |
| 348 |
|
| 349 |
IterativeSolver *it_solver = dynamic_cast<IterativeSolver*>(solver); |
| 350 |
if (it_solver) |
| 351 |
{ |
| 352 |
it_solver->SetRelTol(rel_tol); |
| 353 |
it_solver->SetMaxIter(max_iter); |
| 354 |
it_solver->SetPrintLevel(1); |
| 355 |
it_solver->SetPreconditioner(*mg); |
| 356 |
if (use_monitor) |
| 357 |
{ |
| 358 |
monitor = new DataMonitor(file_name.str(), MONITOR_DIGITS); |
| 359 |
it_solver->SetMonitor(*monitor); |
| 360 |
} |
| 361 |
} |
| 362 |
|
| 363 |
solver->Mult(B, X); |
| 364 |
|
| 365 |
// 10. Recover the solution x as a grid function. Send the data by socket to |
| 366 |
// a GLVis server. |
| 367 |
mg->RecoverFineFEMSolution(X, *b, x); |
| 368 |
|
| 369 |
if (visualization) |
| 370 |
{ |
| 371 |
char vishost[] = "localhost"; |
| 372 |
int visport = 19916; |
| 373 |
socketstream sol_sock(vishost, visport); |
| 374 |
sol_sock << "parallel " << Mpi::WorldSize() << " " << Mpi::WorldRank() |
| 375 |
<< "\n"; |
| 376 |
sol_sock.precision(8); |
| 377 |
sol_sock << "solution\n" |
| 378 |
<< *fes_hierarchy->GetFinestFESpace().GetParMesh() |
| 379 |
<< x << flush; |
| 380 |
} |
| 381 |
|
| 382 |
// 11. Compute and print the L^2 norm of the error. |
| 383 |
{ |
| 384 |
real_t error = 0.0; |
| 385 |
switch (integrator_type) |
| 386 |
{ |
| 387 |
case mass: |
| 388 |
case diffusion: |
| 389 |
error = x.ComputeL2Error(*scalar_u); |
| 390 |
break; |
| 391 |
default: |
| 392 |
mfem_error("Invalid integrator type! Check ComputeL2Error"); |
| 393 |
} |
| 394 |
if (Mpi::Root()) |
| 395 |
{ |
| 396 |
mfem::out << "\n|| u_h - u ||_{L^2} = " << error << "\n" << endl; |
| 397 |
} |
| 398 |
} |
| 399 |
|
| 400 |
// 12. Free the memory used. |
| 401 |
delete mg; |
| 402 |
delete solver; |
| 403 |
delete b; |
| 404 |
if (monitor) { delete monitor; } |
| 405 |
if (scalar_u) { delete scalar_u; } |
| 406 |
if (scalar_f) { delete scalar_f; } |
| 407 |
for (int level = 0; level < fec_array.Size(); ++level) |
| 408 |
{ |
| 409 |
delete fec_array[level]; |
| 410 |
} |
| 411 |
delete fes_hierarchy; |
| 412 |
|
| 413 |
return 0; |
| 414 |
} |