Debian Codegraph @

demo/undocumented/matnest/cpp/main.cpp

raw
1 // Copyright (C) 2016 Garth N. Wells and Chris N. Richardson
2 //
3 // This file is part of DOLFIN.
4 //
5 // DOLFIN is free software: you can redistribute it and/or modify
6 // it under the terms of the GNU Lesser General Public License as published by
7 // the Free Software Foundation, either version 3 of the License, or
8 // (at your option) any later version.
9 //
10 // DOLFIN is distributed in the hope that it will be useful,
11 // but WITHOUT ANY WARRANTY; without even the implied warranty of
12 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 // GNU Lesser General Public License for more details.
14 //
15 // You should have received a copy of the GNU Lesser General Public License
16 // along with DOLFIN. If not, see <http://www.gnu.org/licenses/>.
17 //
18 // This demo program solves Stokes's equations
19 //
20 // mu*div(grad(u)) - grad(p) = f
21 // div(u) = 0
22 //
23 // on a unit cube, using the MatNest facility of PETSc. By doing this, it is possible
24 // to save memory when creating preconditioners, an essential requirement for computation
25 // at large scale (over 100 million degrees of freedom).
26
27 #include <array>
28 #include <cmath>
29 #include <iomanip>
30 #include <iostream>
31 #include <memory>
32 #include <string>
33
34 #include <dolfin.h>
35
36 #include "Stokes.h"
37
38 using namespace dolfin;
39
40 #ifdef HAS_PETSC
41
42 // Function to compute the near nullspace A00 operator
43 VectorSpaceBasis build_nullspace_nested(const FunctionSpace& V,
44 const GenericVector& x)
45 {
46 // Get subspaces
47 auto V0 = V.sub(0);
48 auto V1 = V.sub(1);
49 auto V2 = V.sub(2);
50
51 // Create vectors for nullspace basis
52 std::vector<std::shared_ptr<GenericVector>> basis(3);
53 for (std::size_t i = 0; i < basis.size(); ++i)
54 basis[i] = x.copy();
55
56 // x0, x1, x2 translations
57 V0->dofmap()->set(*basis[0], 1.0);
58 V1->dofmap()->set(*basis[1], 1.0);
59 V2->dofmap()->set(*basis[2], 1.0);
60
61 // Rotations
62 /*
63 V0->set_x(*basis[3], -1.0, 1);
64 V1->set_x(*basis[3], 1.0, 0);
65
66 V0->set_x(*basis[4], 1.0, 2);
67 V2->set_x(*basis[4], -1.0, 0);
68
69 V2->set_x(*basis[5], 1.0, 1);
70 V1->set_x(*basis[5], -1.0, 2);
71 */
72
73 // Apply
74 for (std::size_t i = 0; i < basis.size(); ++i)
75 basis[i]->apply("add");
76
77 // Create vector space and orthonormalize
78 VectorSpaceBasis vector_space(basis);
79 vector_space.orthonormalize();
80 return vector_space;
81 }
82
83 // Function for no-slip boundary condition for velocity
84 class Source : public Expression
85 {
86 public:
87
88 Source() : Expression(3) {}
89
90 void eval(Array<double>& values, const Array<double>& x) const
91 {
92 values[0] = x[1];
93 values[1] = x[1]*x[2];
94 values[2] = 0.0;
95 }
96 };
97
98 // Sub domain for top and bottom
99 class TopBottom : public SubDomain
100 {
101 bool inside(const Array<double>& x, bool on_boundary) const
102 {
103 return on_boundary;
104 // return std::abs(1.0 - x[1]) < DOLFIN_EPS || std::abs(x[1]) < DOLFIN_EPS;
105 }
106 };
107
108 class LeftEdge : public SubDomain
109 {
110 bool inside(const Array<double>& x, bool on_boundary) const
111 {
112 return x[0] < DOLFIN_EPS;
113 }
114 };
115
116 int main(int argc, char *argv[])
117 {
118 // Application parameters
119 Parameters application_parameters("application_parameters");
120 application_parameters.add("size", 12);
121
122 // Parse application parameters
123 application_parameters.parse(argc, argv);
124 const int size = application_parameters["size"];
125
126 // Parse PETSc parameters
127 parameters.parse(argc, argv);
128
129 PETScOptions::set("ksp_monitor_true_residual");
130
131 // Create mesh
132 auto mesh = std::make_shared<UnitCubeMesh>(size, size, size);
133
134 // Velocity BC
135 auto top_bottom = std::make_shared<TopBottom>();
136 auto zero_vec = std::make_shared<Constant>(0.0, 0.0, 0.0);
137
138 auto f = std::make_shared<Source>();
139 auto zero = std::make_shared<Constant>(0.0);
140
141 // Common fieldsplit options
142 PETScOptions::set("ksp_type", "minres");
143 PETScOptions::set("pc_type", "fieldsplit");
144 PETScOptions::set("pc_fieldsplit_type", "additive");
145
146 PETScOptions::set("ksp_view");
147
148 // AMG
149 PETScOptions::set("fieldsplit_0_ksp_type", "preonly");
150
151 // "gamg" or "hypre"
152 std::string mg = "gamg";
153
154 if (mg == "gamg")
155 {
156 PETScOptions::set("fieldsplit_0_pc_type", "gamg");
157 PETScOptions::set("fieldsplit_0_pc_gamg_coarse_eq_limit", 10000);
158
159 PETScOptions::set("fieldsplit_0_pc_gamg_threshold", 0.05);
160 //PETScOptions::set("fieldsplit_0_pc_gamg_square_graph", 2);
161 PETScOptions::set("fieldsplit_0_mg_levels_ksp_type", "chebyshev");
162 PETScOptions::set("fieldsplit_0_mg_levels_pc_type", "sor");
163 PETScOptions::set("fieldsplit_0_mg_levels_ksp_max_it", 3);
164
165 PETScOptions::set("fieldsplit_0_mg_levels_esteig_ksp_type", "cg");
166 PETScOptions::set("fieldsplit_0_mg_levels_ksp_chebyshev_esteig_steps", 20);
167 PETScOptions::set("fieldsplit_0_mg_levels_ksp_chebyshev_esteig_random");
168
169 PETScOptions::set("fieldsplit_0_mg_coarse_ksp_type", "preonly");
170 PETScOptions::set("fieldsplit_0_mg_coarse_pc_type", "lu");
171 PETScOptions::set("fieldsplit_0_mg_coarse_pc_factor_mat_solver_package", "mumps");
172 }
173 else if (mg == "hypre")
174 {
175 PETScOptions::set("fieldsplit_0_pc_type", "hypre");
176 PETScOptions::set("fieldsplit_0_hypre_type", "boomeramg");
177 PETScOptions::set("fieldsplit_1_pc_hypre_boomeramg_strong_threshold", 0.5);
178 }
179 else
180 error("Unknown A00 pc");
181
182 PETScOptions::set("fieldsplit_1_ksp_type", "preonly");
183 PETScOptions::set("fieldsplit_1_pc_type", "hypre");
184 PETScOptions::set("fieldsplit_1_hypre_type", "boomeramg");
185 PETScOptions::set("fieldsplit_1_pc_hypre_boomeramg_strong_threshold", 0.5);
186
187 // Create function spaces
188 auto V = std::make_shared<Stokes::Form_a00::TestSpace>(mesh);
189 auto Q = std::make_shared<Stokes::Form_a11::TestSpace>(mesh);
190
191 // Define variational problem
192 auto a00 = std::make_shared<Stokes::Form_a00>(V, V);
193 auto L0 = std::make_shared<Stokes::Form_L0>(V);
194 L0->f = f;
195
196 auto a01 = std::make_shared<Stokes::Form_a01>(Q, V);
197 auto a10 = std::make_shared<Stokes::Form_a10>(V, Q);
198 auto a11 = std::make_shared<Stokes::Form_a11>(Q, Q);
199 a11->c = zero;
200
201 auto L1 = std::make_shared<Stokes::Form_L1>(Q);
202 L1->g = zero;
203
204 // Velocity BC
205 auto bcV = std::make_shared<DirichletBC>(V, zero_vec,
206 top_bottom);
207
208 // Assemble all blocks with BCs
209 SystemAssembler assemblerA0({a00, a01, a10, a11}, {L0, L1},
210 {{bcV}, {}});
211 auto A00 = std::make_shared<PETScMatrix>();
212 auto A01 = std::make_shared<PETScMatrix>();
213 auto b0 = std::make_shared<PETScVector>();
214 auto A10 = std::make_shared<PETScMatrix>();
215 auto b1 = std::make_shared<PETScVector>();
216
217 auto A11 = std::make_shared<PETScMatrix>();
218 std::cout << "Assemble A00, A01, A10, A11\n";
219 assemblerA0.assemble({A00, A01, A10, A11}, {b0, b1});
220
221 // Assemble a mass matrix (QxQ) for the preconditioner
222 auto one = std::make_shared<Constant>(1.0);
223 a11->c = one;
224 SystemAssembler assemblerP(a11, L1, {});
225 auto P11 = std::make_shared<PETScMatrix>();
226 assemblerP.assemble(*P11);
227
228 std::cout << "A00:" << A00->size(0) << "x" << A00->size(1)
229 << " : " << std::setprecision(10)
230 << std::pow(A00->norm("frobenius"), 2) << "\n";
231 std::cout << "A01:" << A01->size(0) << "x" << A01->size(1)
232 << " : " << std::setprecision(10)
233 << std::pow(A01->norm("frobenius"), 2) << "\n";
234 std::cout << "A10:" << A10->size(0) << "x" << A10->size(1)
235 << " : " << std::setprecision(10)
236 << std::pow(A10->norm("frobenius"), 2) << "\n";
237 std::cout << "P11:" << P11->size(0) << "x" << P11->size(1)
238 << " : " << std::setprecision(10)
239 << std::pow(P11->norm("frobenius"), 2) << "\n";
240
241 std::cout << "b0:" << b0->local_range().second - b0->local_range().first
242 << " : " << std::setprecision(10)
243 << std::pow(b0->norm("l2"), 2) << "\n";
244 std::cout << "b1:" << b1->local_range().second - b1->local_range().first
245 << " : " << std::setprecision(10)
246 << std::pow(b1->norm("l2"), 2) << "\n";
247
248 // Create nested A and P
249
250 // NOTE: We need an empty bottom-right block when using PETSc Schur
251 // complement preconditioning.
252 std::vector<std::shared_ptr<const GenericMatrix>> Amats, Pmats;
253 std::shared_ptr<PETScNestMatrix> Anest, Pnest;
254
255 Amats = {A00, A01, A10, NULL};
256 Anest = std::make_shared<PETScNestMatrix>(Amats);
257
258 Pmats = {A00, NULL, NULL, P11};
259 Pnest = std::make_shared<PETScNestMatrix>(Pmats);
260
261 // Create VecNest vectors for RHS and solution vector
262 auto u0 = std::make_shared<Function>(V);
263 auto p0 = std::make_shared<Function>(Q);
264 PETScVector xvec, bvec;
265 Anest->init_vectors(xvec, {u0->vector(), p0->vector()});
266 Anest->init_vectors(bvec, {b0, b1});
267
268 // Create nullspace vector for p
269 auto u0tmp = std::make_shared<Function>(V);
270 auto p0tmp = std::make_shared<Function>(Q);
271 p0tmp->interpolate(*one);
272 u0tmp->interpolate(*zero_vec);
273
274 auto xvec_nullspace = std::make_shared<PETScVector>();
275 Anest->init_vectors(*xvec_nullspace, {u0tmp->vector(), p0tmp->vector()});
276 *xvec_nullspace /= xvec_nullspace->norm("l2");
277
278 VectorSpaceBasis basis({xvec_nullspace});
279 Anest->set_nullspace(basis);
280
281 // Create near null space basis for u
282 VectorSpaceBasis null_space
283 = build_nullspace_nested(*V, *u0->vector());
284 A00->set_near_nullspace(null_space);
285
286 // Solve
287 PETScKrylovSolver solver;
288 solver.set_from_options();
289 solver.set_operators(Anest, Pnest);
290
291 // Set field-split blocks for preconditioner
292 std::vector<la_index> u_dofs, p_dofs;
293 Anest->get_block_dofs(u_dofs, 0);
294 Anest->get_block_dofs(p_dofs, 1);
295 PETScPreconditioner::set_fieldsplit(solver, {u_dofs, p_dofs},
296 {"0", "1"});
297 solver.solve(xvec, bvec);
298
299 if (has_hdf5_parallel())
300 {
301 XDMFFile xdmf_u(mesh->mpi_comm(), "u0.xdmf");
302 xdmf_u.write(*u0);
303
304 XDMFFile xdmf_p(mesh->mpi_comm(), "p0.xdmf");
305 xdmf_p.write(*p0);
306 }
307
308 double unorm = u0->vector()->norm("l2");
309 if (MPI::rank(mesh->mpi_comm()) == 0)
310 {
311 std::cout << "u norm:" << unorm << std::endl;
312 }
313
314 list_timings(TimingClear::clear, {TimingType::wall});
315
316 return 0;
317 }
318
319 #else
320
321 int main()
322 {
323 info("DOLFIN has not been configured with PETSc. Exiting.");
324 return 0;
325 }
326
327 #endif