// Copyright (C) 2016 Garth N. Wells and Chris N. Richardson // // This file is part of DOLFIN. // // DOLFIN is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // DOLFIN is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with DOLFIN. If not, see . // // This demo program solves Stokes's equations // // mu*div(grad(u)) - grad(p) = f // div(u) = 0 // // on a unit cube, using the MatNest facility of PETSc. By doing this, it is possible // to save memory when creating preconditioners, an essential requirement for computation // at large scale (over 100 million degrees of freedom). #include #include #include #include #include #include #include #include "Stokes.h" using namespace dolfin; #ifdef HAS_PETSC // Function to compute the near nullspace A00 operator VectorSpaceBasis build_nullspace_nested(const FunctionSpace& V, const GenericVector& x) { // Get subspaces auto V0 = V.sub(0); auto V1 = V.sub(1); auto V2 = V.sub(2); // Create vectors for nullspace basis std::vector> basis(3); for (std::size_t i = 0; i < basis.size(); ++i) basis[i] = x.copy(); // x0, x1, x2 translations V0->dofmap()->set(*basis[0], 1.0); V1->dofmap()->set(*basis[1], 1.0); V2->dofmap()->set(*basis[2], 1.0); // Rotations /* V0->set_x(*basis[3], -1.0, 1); V1->set_x(*basis[3], 1.0, 0); V0->set_x(*basis[4], 1.0, 2); V2->set_x(*basis[4], -1.0, 0); V2->set_x(*basis[5], 1.0, 1); V1->set_x(*basis[5], -1.0, 2); */ // Apply for (std::size_t i = 0; i < basis.size(); ++i) basis[i]->apply("add"); // Create vector space and orthonormalize VectorSpaceBasis vector_space(basis); vector_space.orthonormalize(); return vector_space; } // Function for no-slip boundary condition for velocity class Source : public Expression { public: Source() : Expression(3) {} void eval(Array& values, const Array& x) const { values[0] = x[1]; values[1] = x[1]*x[2]; values[2] = 0.0; } }; // Sub domain for top and bottom class TopBottom : public SubDomain { bool inside(const Array& x, bool on_boundary) const { return on_boundary; // return std::abs(1.0 - x[1]) < DOLFIN_EPS || std::abs(x[1]) < DOLFIN_EPS; } }; class LeftEdge : public SubDomain { bool inside(const Array& x, bool on_boundary) const { return x[0] < DOLFIN_EPS; } }; int main(int argc, char *argv[]) { // Application parameters Parameters application_parameters("application_parameters"); application_parameters.add("size", 12); // Parse application parameters application_parameters.parse(argc, argv); const int size = application_parameters["size"]; // Parse PETSc parameters parameters.parse(argc, argv); PETScOptions::set("ksp_monitor_true_residual"); // Create mesh auto mesh = std::make_shared(size, size, size); // Velocity BC auto top_bottom = std::make_shared(); auto zero_vec = std::make_shared(0.0, 0.0, 0.0); auto f = std::make_shared(); auto zero = std::make_shared(0.0); // Common fieldsplit options PETScOptions::set("ksp_type", "minres"); PETScOptions::set("pc_type", "fieldsplit"); PETScOptions::set("pc_fieldsplit_type", "additive"); PETScOptions::set("ksp_view"); // AMG PETScOptions::set("fieldsplit_0_ksp_type", "preonly"); // "gamg" or "hypre" std::string mg = "gamg"; if (mg == "gamg") { PETScOptions::set("fieldsplit_0_pc_type", "gamg"); PETScOptions::set("fieldsplit_0_pc_gamg_coarse_eq_limit", 10000); PETScOptions::set("fieldsplit_0_pc_gamg_threshold", 0.05); //PETScOptions::set("fieldsplit_0_pc_gamg_square_graph", 2); PETScOptions::set("fieldsplit_0_mg_levels_ksp_type", "chebyshev"); PETScOptions::set("fieldsplit_0_mg_levels_pc_type", "sor"); PETScOptions::set("fieldsplit_0_mg_levels_ksp_max_it", 3); PETScOptions::set("fieldsplit_0_mg_levels_esteig_ksp_type", "cg"); PETScOptions::set("fieldsplit_0_mg_levels_ksp_chebyshev_esteig_steps", 20); PETScOptions::set("fieldsplit_0_mg_levels_ksp_chebyshev_esteig_random"); PETScOptions::set("fieldsplit_0_mg_coarse_ksp_type", "preonly"); PETScOptions::set("fieldsplit_0_mg_coarse_pc_type", "lu"); PETScOptions::set("fieldsplit_0_mg_coarse_pc_factor_mat_solver_package", "mumps"); } else if (mg == "hypre") { PETScOptions::set("fieldsplit_0_pc_type", "hypre"); PETScOptions::set("fieldsplit_0_hypre_type", "boomeramg"); PETScOptions::set("fieldsplit_1_pc_hypre_boomeramg_strong_threshold", 0.5); } else error("Unknown A00 pc"); PETScOptions::set("fieldsplit_1_ksp_type", "preonly"); PETScOptions::set("fieldsplit_1_pc_type", "hypre"); PETScOptions::set("fieldsplit_1_hypre_type", "boomeramg"); PETScOptions::set("fieldsplit_1_pc_hypre_boomeramg_strong_threshold", 0.5); // Create function spaces auto V = std::make_shared(mesh); auto Q = std::make_shared(mesh); // Define variational problem auto a00 = std::make_shared(V, V); auto L0 = std::make_shared(V); L0->f = f; auto a01 = std::make_shared(Q, V); auto a10 = std::make_shared(V, Q); auto a11 = std::make_shared(Q, Q); a11->c = zero; auto L1 = std::make_shared(Q); L1->g = zero; // Velocity BC auto bcV = std::make_shared(V, zero_vec, top_bottom); // Assemble all blocks with BCs SystemAssembler assemblerA0({a00, a01, a10, a11}, {L0, L1}, {{bcV}, {}}); auto A00 = std::make_shared(); auto A01 = std::make_shared(); auto b0 = std::make_shared(); auto A10 = std::make_shared(); auto b1 = std::make_shared(); auto A11 = std::make_shared(); std::cout << "Assemble A00, A01, A10, A11\n"; assemblerA0.assemble({A00, A01, A10, A11}, {b0, b1}); // Assemble a mass matrix (QxQ) for the preconditioner auto one = std::make_shared(1.0); a11->c = one; SystemAssembler assemblerP(a11, L1, {}); auto P11 = std::make_shared(); assemblerP.assemble(*P11); std::cout << "A00:" << A00->size(0) << "x" << A00->size(1) << " : " << std::setprecision(10) << std::pow(A00->norm("frobenius"), 2) << "\n"; std::cout << "A01:" << A01->size(0) << "x" << A01->size(1) << " : " << std::setprecision(10) << std::pow(A01->norm("frobenius"), 2) << "\n"; std::cout << "A10:" << A10->size(0) << "x" << A10->size(1) << " : " << std::setprecision(10) << std::pow(A10->norm("frobenius"), 2) << "\n"; std::cout << "P11:" << P11->size(0) << "x" << P11->size(1) << " : " << std::setprecision(10) << std::pow(P11->norm("frobenius"), 2) << "\n"; std::cout << "b0:" << b0->local_range().second - b0->local_range().first << " : " << std::setprecision(10) << std::pow(b0->norm("l2"), 2) << "\n"; std::cout << "b1:" << b1->local_range().second - b1->local_range().first << " : " << std::setprecision(10) << std::pow(b1->norm("l2"), 2) << "\n"; // Create nested A and P // NOTE: We need an empty bottom-right block when using PETSc Schur // complement preconditioning. std::vector> Amats, Pmats; std::shared_ptr Anest, Pnest; Amats = {A00, A01, A10, NULL}; Anest = std::make_shared(Amats); Pmats = {A00, NULL, NULL, P11}; Pnest = std::make_shared(Pmats); // Create VecNest vectors for RHS and solution vector auto u0 = std::make_shared(V); auto p0 = std::make_shared(Q); PETScVector xvec, bvec; Anest->init_vectors(xvec, {u0->vector(), p0->vector()}); Anest->init_vectors(bvec, {b0, b1}); // Create nullspace vector for p auto u0tmp = std::make_shared(V); auto p0tmp = std::make_shared(Q); p0tmp->interpolate(*one); u0tmp->interpolate(*zero_vec); auto xvec_nullspace = std::make_shared(); Anest->init_vectors(*xvec_nullspace, {u0tmp->vector(), p0tmp->vector()}); *xvec_nullspace /= xvec_nullspace->norm("l2"); VectorSpaceBasis basis({xvec_nullspace}); Anest->set_nullspace(basis); // Create near null space basis for u VectorSpaceBasis null_space = build_nullspace_nested(*V, *u0->vector()); A00->set_near_nullspace(null_space); // Solve PETScKrylovSolver solver; solver.set_from_options(); solver.set_operators(Anest, Pnest); // Set field-split blocks for preconditioner std::vector u_dofs, p_dofs; Anest->get_block_dofs(u_dofs, 0); Anest->get_block_dofs(p_dofs, 1); PETScPreconditioner::set_fieldsplit(solver, {u_dofs, p_dofs}, {"0", "1"}); solver.solve(xvec, bvec); if (has_hdf5_parallel()) { XDMFFile xdmf_u(mesh->mpi_comm(), "u0.xdmf"); xdmf_u.write(*u0); XDMFFile xdmf_p(mesh->mpi_comm(), "p0.xdmf"); xdmf_p.write(*p0); } double unorm = u0->vector()->norm("l2"); if (MPI::rank(mesh->mpi_comm()) == 0) { std::cout << "u norm:" << unorm << std::endl; } list_timings(TimingClear::clear, {TimingType::wall}); return 0; } #else int main() { info("DOLFIN has not been configured with PETSc. Exiting."); return 0; } #endif