| 1 |
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced |
| 2 |
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files |
| 3 |
// LICENSE and NOTICE for details. LLNL-CODE-806117. |
| 4 |
// |
| 5 |
// This file is part of the MFEM library. For more information and source code |
| 6 |
// availability visit https://mfem.org. |
| 7 |
// |
| 8 |
// MFEM is free software; you can redistribute it and/or modify it under the |
| 9 |
// terms of the BSD-3 license. We welcome feedback and contributions, see file |
| 10 |
// CONTRIBUTING.md for details. |
| 11 |
// |
| 12 |
// ----------------------------------------- |
| 13 |
// Absolute L(1)-Jacobi smoothers miniapp |
| 14 |
// ----------------------------------------- |
| 15 |
// |
| 16 |
// This miniapp illustrates the implementation of an (slightly generalized) |
| 17 |
// absolute-L(1) Jacobi preconditioner. This preconditioner is tested in |
| 18 |
// different settings. We use Stationary Linear Iterations and Preconditioned |
| 19 |
// Conjugate Gradient as the main solvers. |
| 20 |
// We consider a H1-mass matrix, a diffusion matrix, and a definite Maxwell system. |
| 21 |
// |
| 22 |
// The preconditioner can be defined at run-time. Similarly, the mesh can be |
| 23 |
// modified by a Kershaw transformation at run-time. Relative tolerance and |
| 24 |
// maximum number of iterations can be modified as well. |
| 25 |
// |
| 26 |
// Compile with: make abs-l1-jacobi |
| 27 |
// |
| 28 |
// Sample runs: |
| 29 |
// mpirun -np 4 abs-l1-jacobi |
| 30 |
// mpirun -np 4 abs-l1-jacobi -s 0 -i 0 |
| 31 |
// mpirun -np 4 abs-l1-jacobi -m ../meshing/icf.mesh -f 0.5 |
| 32 |
// mpirun -np 4 abs-l1-jacobi -rs 3 -rp 1 |
| 33 |
// mpirun -np 4 abs-l1-jacobi -t 1e-5 -ni 100 |
| 34 |
// mpirun -np 4 abs-l1-jacobi -m ../../data/beam-quad.mesh -a 3 -Ky 0.5 -Kz 0.5 |
| 35 |
// mpirun -np 4 abs-l1-jacobi --device cuda |
| 36 |
|
| 37 |
#include "ds-common.hpp" |
| 38 |
|
| 39 |
using namespace std; |
| 40 |
using namespace mfem; |
| 41 |
using namespace ds_common; |
| 42 |
|
| 43 |
int main(int argc, char *argv[]) |
| 44 |
{ |
| 45 |
// 1. Initialize MPI and HYPRE. |
| 46 |
Mpi::Init(argc, argv); |
| 47 |
Hypre::Init(); |
| 48 |
|
| 49 |
// 2. Parse command line options. |
| 50 |
string mesh_file = "../../data/ref-cube.mesh"; |
| 51 |
// System properties |
| 52 |
int order = 1; |
| 53 |
SolverType solver_type = cg; |
| 54 |
IntegratorType integrator_type = diffusion; |
| 55 |
PCType pc_type = abs_global; |
| 56 |
int assembly_type_int = 3; // Default is PARTIAL |
| 57 |
AssemblyLevel assembly_type; |
| 58 |
// Number of refinements |
| 59 |
int refine_serial = 4; |
| 60 |
int refine_parallel = 0; |
| 61 |
// Preconditioner parameters, only for L(p,q)-Jacobi |
| 62 |
real_t p_order = 1.0; |
| 63 |
real_t q_order = 0.0; |
| 64 |
// Solver parameters |
| 65 |
real_t rel_tol = 1e-10; |
| 66 |
real_t max_iter = 3000; |
| 67 |
// Kershaw Transformation |
| 68 |
real_t eps_y = 0.0; |
| 69 |
real_t eps_z = 0.0; |
| 70 |
// Other options |
| 71 |
string device_config = "cpu"; |
| 72 |
bool use_monitor = false; |
| 73 |
bool visualization = true; |
| 74 |
|
| 75 |
// Construct argument parser |
| 76 |
OptionsParser args(argc, argv); |
| 77 |
args.AddOption(&mesh_file, "-m", "--mesh", |
| 78 |
"Mesh file to use."); |
| 79 |
args.AddOption(&order, "-o", "--order", |
| 80 |
"Finite element order (polynomial degree)"); |
| 81 |
args.AddOption((int*)&solver_type, "-s", "--solver", |
| 82 |
"Solvers to be considered:" |
| 83 |
"\n\t0: Stationary Linear Iteration" |
| 84 |
"\n\t1: Preconditioned Conjugate Gradient"); |
| 85 |
args.AddOption((int*)&integrator_type, "-i", "--integrator", |
| 86 |
"Integrators to be considered:" |
| 87 |
"\n\t0: MassIntegrator" |
| 88 |
"\n\t1: DiffusionIntegrator" |
| 89 |
"\n\t2: CurlCurlIntegrator + VectorFEMassIntegrator"); |
| 90 |
args.AddOption(&assembly_type_int, "-a", "--assembly", |
| 91 |
"Assembly level to be considered:" |
| 92 |
"\n\t0: LEGACY" |
| 93 |
"\n\t1: FULL" |
| 94 |
"\n\t2: ELEMENT" |
| 95 |
"\n\t3: PARTIAL" |
| 96 |
"\n\t4: NONE"); |
| 97 |
args.AddOption((int*)&pc_type, "-pc", "--preconditioner", |
| 98 |
"Preconditioners to be considered:" |
| 99 |
"\n\t0: No preconditioner" |
| 100 |
"\n\t1: Absolute L(1)-Jacobi preconditioner" |
| 101 |
"\n\t2: Element L(p,q)-Jacobi preconditioner"); |
| 102 |
args.AddOption(&refine_serial, "-rs", "--refine-serial", |
| 103 |
"Number of serial refinements"); |
| 104 |
args.AddOption(&refine_parallel, "-rp", "--refine-parallel", |
| 105 |
"Number of parallel refinements"); |
| 106 |
args.AddOption(&p_order, "-p", "--p-order", |
| 107 |
"P-order for L(p,q)-Jacobi preconditioner"); |
| 108 |
args.AddOption(&q_order, "-q", "--q-order", |
| 109 |
"Q-order for L(p,q)-Jacobi preconditioner"); |
| 110 |
args.AddOption(&rel_tol, "-t", "--tolerance", |
| 111 |
"Relative tolerance for the iterative solver"); |
| 112 |
args.AddOption(&max_iter, "-ni", "--iterations", |
| 113 |
"Maximum number of iterations"); |
| 114 |
args.AddOption(&eps_y, "-Ky", "--Kershaw-y", |
| 115 |
"Kershaw transform factor, eps_y in (0,1]"); |
| 116 |
args.AddOption(&eps_z, "-Kz", "--Kershaw-z", |
| 117 |
"Kershaw transform factor, eps_z in (0,1]"); |
| 118 |
args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact" |
| 119 |
" solution."); |
| 120 |
args.AddOption(&device_config, "-d", "--device", |
| 121 |
"Device configuration string, see Device::Configure()."); |
| 122 |
args.AddOption(&use_monitor, "-mon", "--monitor", "-no-mon", |
| 123 |
"--no-monitor", |
| 124 |
"Enable or disable Data Monitor."); |
| 125 |
args.AddOption(&visualization, "-vis", "--visualization", "-no-vis", |
| 126 |
"--no-visualization", |
| 127 |
"Enable or disable GLVis visualization."); |
| 128 |
args.ParseCheck(); |
| 129 |
|
| 130 |
MFEM_VERIFY(p_order > 0.0, "p needs to be positive"); |
| 131 |
MFEM_VERIFY((0 <= solver_type) && (solver_type < num_solvers), |
| 132 |
"invalid solver type: " << solver_type); |
| 133 |
MFEM_VERIFY((0 <= integrator_type) && (integrator_type < num_integrators), |
| 134 |
"invalid integrator type: " << integrator_type); |
| 135 |
MFEM_VERIFY((0 <= assembly_type_int) && (assembly_type_int < 5), |
| 136 |
"invalid assembly type: " << assembly_type_int); |
| 137 |
MFEM_VERIFY((0 <= pc_type) && (pc_type < num_pc), |
| 138 |
"invalid preconditioner type: " << pc_type); |
| 139 |
MFEM_VERIFY((0.0 <= eps_y) && (eps_y <= 1.0), "eps_y must be in [0,1]"); |
| 140 |
MFEM_VERIFY((0.0 <= eps_z) && (eps_z <= 1.0), "eps_z must be in [0,1]"); |
| 141 |
|
| 142 |
kappa = freq * M_PI; |
| 143 |
|
| 144 |
ostringstream file_name; |
| 145 |
if (use_monitor) |
| 146 |
{ |
| 147 |
file_name << "ABS-" |
| 148 |
<< "O" << order |
| 149 |
<< "I" << (int) integrator_type |
| 150 |
<< "S" << (int) solver_type |
| 151 |
<< "A" << assembly_type_int |
| 152 |
<< ".csv"; |
| 153 |
} |
| 154 |
|
| 155 |
switch (assembly_type_int) |
| 156 |
{ |
| 157 |
case 0: |
| 158 |
assembly_type = AssemblyLevel::LEGACY; |
| 159 |
break; |
| 160 |
case 1: |
| 161 |
assembly_type = AssemblyLevel::FULL; |
| 162 |
break; |
| 163 |
case 2: |
| 164 |
assembly_type = AssemblyLevel::ELEMENT; |
| 165 |
break; |
| 166 |
case 3: |
| 167 |
assembly_type = AssemblyLevel::PARTIAL; |
| 168 |
break; |
| 169 |
case 4: |
| 170 |
assembly_type = AssemblyLevel::NONE; |
| 171 |
break; |
| 172 |
default: |
| 173 |
MFEM_ABORT("Unsupported option!"); |
| 174 |
} |
| 175 |
|
| 176 |
Device device(device_config); |
| 177 |
if (Mpi::Root()) { device.Print(); } |
| 178 |
|
| 179 |
// 3. Read the serial mesh from the given mesh file. The number of serial and |
| 180 |
// parallel refinements can be set by the user on the command line. |
| 181 |
Mesh *serial_mesh = new Mesh(mesh_file); |
| 182 |
for (int ls = 0; ls < refine_serial; ls++) |
| 183 |
{ |
| 184 |
serial_mesh->UniformRefinement(); |
| 185 |
} |
| 186 |
|
| 187 |
// 4. Define a parallel mesh by a partitioning of the serial mesh. The number |
| 188 |
// of parallel refinements can be set by the user. If defined, apply |
| 189 |
// Kershaw transformation. |
| 190 |
ParMesh *mesh = new ParMesh(MPI_COMM_WORLD, *serial_mesh); |
| 191 |
delete serial_mesh; |
| 192 |
for (int lp = 0; lp < refine_parallel; lp++) |
| 193 |
{ |
| 194 |
mesh->UniformRefinement(); |
| 195 |
} |
| 196 |
|
| 197 |
dim = mesh->Dimension(); |
| 198 |
space_dim = mesh->SpaceDimension(); |
| 199 |
|
| 200 |
bool cond_z = (dim < 3) ? true : (eps_z != 0); // lazy check |
| 201 |
if (eps_y != 0.0 && cond_z) |
| 202 |
{ |
| 203 |
if (dim < 3) { eps_z = 0.0; } |
| 204 |
common::KershawTransformation kershawT(dim, eps_y, eps_z); |
| 205 |
mesh->Transform(kershawT); |
| 206 |
} |
| 207 |
|
| 208 |
// 5. Define a finite element space on the mesh. We use different spaces and |
| 209 |
// collections for different systems. |
| 210 |
// - H1-conforming Lagrange elements for the H1-mass matrix and the |
| 211 |
// diffusion problem. |
| 212 |
// - H(curl)-conforming Nedelec elements for the definite Maxwell problem. |
| 213 |
FiniteElementCollection *fec; |
| 214 |
ParFiniteElementSpace *fespace; |
| 215 |
|
| 216 |
switch (integrator_type) |
| 217 |
{ |
| 218 |
case mass: |
| 219 |
case diffusion: |
| 220 |
fec = new H1_FECollection(order, dim); |
| 221 |
fespace = new ParFiniteElementSpace(mesh, fec); |
| 222 |
break; |
| 223 |
case maxwell: |
| 224 |
fec = new ND_FECollection(order, dim); |
| 225 |
fespace = new ParFiniteElementSpace(mesh, fec); |
| 226 |
break; |
| 227 |
default: |
| 228 |
mfem_error("Invalid integrator type! Check FiniteElementCollection"); |
| 229 |
} |
| 230 |
|
| 231 |
HYPRE_BigInt sys_size = fespace->GlobalTrueVSize(); |
| 232 |
if (Mpi::Root()) |
| 233 |
{ |
| 234 |
mfem::out << "Number of unknowns: " << sys_size << endl; |
| 235 |
} |
| 236 |
|
| 237 |
// 6. Extract the list of the essential boundary DoFs. We mark all boundary |
| 238 |
// attibutes as essential. Then we get the list of essential DoFs. |
| 239 |
Array<int> ess_tdof_list; |
| 240 |
Array<int> ess_bdr(mesh->bdr_attributes.Max()); |
| 241 |
if (mesh->bdr_attributes.Size()) |
| 242 |
{ |
| 243 |
ess_bdr = 1; |
| 244 |
fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list); |
| 245 |
} |
| 246 |
|
| 247 |
// 7. Define the linear system. Set up the bilinear form a(.,.) and the |
| 248 |
// linear form b(.). The currently implemented systems are the following: |
| 249 |
// - (u,v), i.e., L2-projection. |
| 250 |
// - (grad(u), grad(v)), i.e., diffusion operator. |
| 251 |
// - (curl(u), curl(v)) + (u,v), i.e., definite Maxwell operator. |
| 252 |
// The linear form has the standard form (f,v). |
| 253 |
// Also, define the matrices and vectors associated with the forms, and |
| 254 |
// project the required boundary data into the GridFunction solution. |
| 255 |
ParBilinearForm *a = new ParBilinearForm(fespace); |
| 256 |
ParLinearForm *b = new ParLinearForm(fespace); |
| 257 |
|
| 258 |
// These pointers are owned by the forms |
| 259 |
LinearFormIntegrator *lfi = nullptr; |
| 260 |
BilinearFormIntegrator *bfi = nullptr; |
| 261 |
// Required for a static_cast |
| 262 |
SumIntegrator *sum_bfi = nullptr; |
| 263 |
|
| 264 |
// These pointers are not owned by the integrators |
| 265 |
FunctionCoefficient *scalar_u = nullptr; |
| 266 |
FunctionCoefficient *scalar_f = nullptr; |
| 267 |
VectorFunctionCoefficient *vector_u = nullptr; |
| 268 |
VectorFunctionCoefficient *vector_f = nullptr; |
| 269 |
|
| 270 |
ConstantCoefficient one(1.0); |
| 271 |
|
| 272 |
// These variables will define the linear system |
| 273 |
ParGridFunction x(fespace), y(fespace); |
| 274 |
OperatorPtr A; |
| 275 |
Vector B, X; |
| 276 |
|
| 277 |
x = 0.0; |
| 278 |
|
| 279 |
switch (integrator_type) |
| 280 |
{ |
| 281 |
case mass: |
| 282 |
scalar_u = new FunctionCoefficient(diffusion_solution); |
| 283 |
lfi = new DomainLFIntegrator(*scalar_u); |
| 284 |
bfi = new MassIntegrator(one); |
| 285 |
x.ProjectBdrCoefficient(*scalar_u, ess_bdr); |
| 286 |
break; |
| 287 |
case diffusion: |
| 288 |
scalar_u = new FunctionCoefficient(diffusion_solution); |
| 289 |
scalar_f = new FunctionCoefficient(diffusion_source); |
| 290 |
lfi = new DomainLFIntegrator(*scalar_f); |
| 291 |
bfi = new DiffusionIntegrator(one); |
| 292 |
x.ProjectBdrCoefficient(*scalar_u, ess_bdr); |
| 293 |
break; |
| 294 |
case maxwell: |
| 295 |
vector_u = new VectorFunctionCoefficient(space_dim, maxwell_solution); |
| 296 |
vector_f = new VectorFunctionCoefficient(space_dim, maxwell_source); |
| 297 |
lfi = new VectorFEDomainLFIntegrator(*vector_f); |
| 298 |
bfi = new SumIntegrator(); |
| 299 |
sum_bfi = static_cast<SumIntegrator*>(bfi); |
| 300 |
sum_bfi->AddIntegrator(new CurlCurlIntegrator(one)); |
| 301 |
sum_bfi->AddIntegrator(new VectorFEMassIntegrator(one)); |
| 302 |
x.ProjectBdrCoefficientTangent(*vector_u, ess_bdr); |
| 303 |
break; |
| 304 |
default: |
| 305 |
mfem_error("Invalid integrator type! Check ParLinearForm"); |
| 306 |
} |
| 307 |
|
| 308 |
a->SetAssemblyLevel(assembly_type); |
| 309 |
a->AddDomainIntegrator(bfi); |
| 310 |
a->Assemble(); |
| 311 |
|
| 312 |
b->AddDomainIntegrator(lfi); |
| 313 |
b->Assemble(); |
| 314 |
|
| 315 |
a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B); |
| 316 |
|
| 317 |
// 8. Construct the preconditioner. Uses AbsMult to construct an appoximation |
| 318 |
// of the diagonal of the matrix. |
| 319 |
|
| 320 |
Solver *jacobi = nullptr; |
| 321 |
Vector ones(fespace->GetTrueVSize()); |
| 322 |
Vector diag(fespace->GetTrueVSize()); |
| 323 |
|
| 324 |
switch (pc_type) |
| 325 |
{ |
| 326 |
case none: |
| 327 |
break; |
| 328 |
case abs_global: |
| 329 |
ones = 1.0; |
| 330 |
A->AbsMult(ones, diag); |
| 331 |
jacobi = new OperatorJacobiSmoother(diag, ess_tdof_list); |
| 332 |
break; |
| 333 |
case pq_element: |
| 334 |
AssembleElementLpqJacobiDiag(*a, p_order, q_order, diag); |
| 335 |
jacobi = new OperatorJacobiSmoother(diag, ess_tdof_list); |
| 336 |
break; |
| 337 |
default: |
| 338 |
mfem_error("Invalid preconditioner type!"); |
| 339 |
} |
| 340 |
|
| 341 |
// 9. Construct the solver. The implemented solvers are the following: |
| 342 |
// - Stationary Linear Iteration |
| 343 |
// - Preconditioned Conjugate Gradient |
| 344 |
// Then, solve the system with the used-selected solver. |
| 345 |
Solver *solver = nullptr; |
| 346 |
DataMonitor *monitor = nullptr; |
| 347 |
|
| 348 |
switch (solver_type) |
| 349 |
{ |
| 350 |
case sli: |
| 351 |
solver = new SLISolver(MPI_COMM_WORLD); |
| 352 |
break; |
| 353 |
case cg: |
| 354 |
solver = new CGSolver(MPI_COMM_WORLD); |
| 355 |
break; |
| 356 |
default: |
| 357 |
mfem_error("Invalid solver type!"); |
| 358 |
} |
| 359 |
solver->SetOperator(*A); |
| 360 |
|
| 361 |
IterativeSolver *it_solver = dynamic_cast<IterativeSolver *>(solver); |
| 362 |
if (it_solver) |
| 363 |
{ |
| 364 |
it_solver->SetRelTol(rel_tol); |
| 365 |
it_solver->SetMaxIter(max_iter); |
| 366 |
it_solver->SetPrintLevel(1); |
| 367 |
if (use_monitor) |
| 368 |
{ |
| 369 |
monitor = new DataMonitor(file_name.str(), MONITOR_DIGITS); |
| 370 |
it_solver->SetMonitor(*monitor); |
| 371 |
} |
| 372 |
if (jacobi) |
| 373 |
{ |
| 374 |
it_solver->SetPreconditioner(*jacobi); |
| 375 |
} |
| 376 |
} |
| 377 |
|
| 378 |
solver->Mult(B, X); |
| 379 |
|
| 380 |
// 10. Recover the solution x as a grid function. Send the data by socket to |
| 381 |
// a GLVis server. |
| 382 |
a->RecoverFEMSolution(X, *b, x); |
| 383 |
|
| 384 |
if (visualization) |
| 385 |
{ |
| 386 |
char vishost[] = "localhost"; |
| 387 |
int visport = 19916; |
| 388 |
socketstream sol_sock(vishost, visport); |
| 389 |
|
| 390 |
sol_sock << "parallel " << Mpi::WorldSize() << " " << Mpi::WorldRank() |
| 391 |
<< "\n"; |
| 392 |
sol_sock.precision(8); |
| 393 |
sol_sock << "solution\n" << *mesh << x << flush; |
| 394 |
} |
| 395 |
|
| 396 |
// 11. Compute and print the L^2 norm of the error. |
| 397 |
{ |
| 398 |
real_t error = 0.0; |
| 399 |
switch (integrator_type) |
| 400 |
{ |
| 401 |
case mass: |
| 402 |
case diffusion: |
| 403 |
error = x.ComputeL2Error(*scalar_u); |
| 404 |
break; |
| 405 |
case maxwell: |
| 406 |
error = x.ComputeL2Error(*vector_u); |
| 407 |
break; |
| 408 |
default: |
| 409 |
mfem_error("Invalid integrator type! Check ComputeL2Error"); |
| 410 |
} |
| 411 |
if (Mpi::Root()) |
| 412 |
{ |
| 413 |
mfem::out << "\n|| u_h - u ||_{L^2} = " << error << "\n" << endl; |
| 414 |
} |
| 415 |
} |
| 416 |
|
| 417 |
// 12. Free the memory used. |
| 418 |
delete solver; |
| 419 |
if (jacobi) { delete jacobi; } |
| 420 |
delete a; |
| 421 |
delete b; |
| 422 |
delete fespace; |
| 423 |
delete fec; |
| 424 |
delete mesh; |
| 425 |
if (monitor) { delete monitor; } |
| 426 |
if (scalar_u) { delete scalar_u; } |
| 427 |
if (scalar_f) { delete scalar_f; } |
| 428 |
if (vector_u) { delete vector_u; } |
| 429 |
if (vector_f) { delete vector_f; } |
| 430 |
|
| 431 |
return 0; |
| 432 |
} |