// @HEADER // ***************************************************************************** // MueLu: A package for multigrid based preconditioning // // Copyright 2012 NTESS and the MueLu contributors. // SPDX-License-Identifier: BSD-3-Clause // ***************************************************************************** // @HEADER #include #include // Teuchos #include #include #include // Xpetra #include #include #include #include #include #include #include #include #include // MueLu #include #include #include #include #include using Teuchos::RCP; using Teuchos::rcp; using Teuchos::TimeMonitor; #include // Belos #ifdef HAVE_MUELU_BELOS #include #include #include #include #endif #include // => This header defines Belos::XpetraOp // Stratimikos #if defined(HAVE_MUELU_STRATIMIKOS) && defined(HAVE_MUELU_THYRA) // Thyra includes #include #include #include // Stratimikos includes #include #include #endif // Support for ML interface #if defined(HAVE_MUELU_ML) and defined(HAVE_MUELU_EPETRA) #include #include "ml_MultiLevelPreconditioner.h" #include "ml_MultiLevelOperator.h" #include "ml_RefMaxwell.h" #endif // Helper functions for compilation purposes template struct EpetraSolvers_Wrapper { static void Generate_ML_MaxwellPreconditioner(Teuchos::RCP >& SM, Teuchos::RCP >& D0, Teuchos::RCP >& Kn, Teuchos::RCP >& nullspace, Teuchos::RCP::coordinateType, LocalOrdinal, GlobalOrdinal, Node> >& coords, Teuchos::ParameterList& mueluList, Teuchos::RCP >& mlopX) { throw std::runtime_error("Template parameter mismatch"); } static void Generate_ML_RefMaxwellPreconditioner(Teuchos::RCP >& SM, Teuchos::RCP >& D0, Teuchos::RCP >& Ms, Teuchos::RCP >& M0inv, Teuchos::RCP >& M1, Teuchos::RCP >& nullspace, Teuchos::RCP >& node_material, Teuchos::RCP::coordinateType, LocalOrdinal, GlobalOrdinal, Node> >& coords, Teuchos::ParameterList& mueluList, Teuchos::RCP >& mlopX) { throw std::runtime_error("Template parameter mismatch"); } }; #if defined(HAVE_MUELU_EPETRA) template struct EpetraSolvers_Wrapper { static void Generate_ML_MaxwellPreconditioner(Teuchos::RCP >& SM, Teuchos::RCP >& D0, Teuchos::RCP >& Kn, Teuchos::RCP >& nullspace, Teuchos::RCP::coordinateType, int, GlobalOrdinal, Tpetra::KokkosCompat::KokkosSerialWrapperNode> >& coords, Teuchos::ParameterList& mueluList, Teuchos::RCP >& mlopX) { #if defined(HAVE_MUELU_ML) typedef double SC; typedef int LO; typedef GlobalOrdinal GO; typedef Tpetra::KokkosCompat::KokkosSerialWrapperNode NO; typedef typename Teuchos::ScalarTraits::coordinateType coordinate_type; typedef typename Xpetra::Matrix Matrix; RCP epetraSM = Xpetra::Helpers::Op2EpetraCrs(SM); RCP epetraD0 = Xpetra::Helpers::Op2EpetraCrs(D0); if (!coords.is_null()) { RCP epetraCoord = MueLu::Utilities::MV2EpetraMV(coords); if (epetraCoord->NumVectors() > 0) mueluList.set("x-coordinates", (*epetraCoord)[0]); if (epetraCoord->NumVectors() > 1) mueluList.set("y-coordinates", (*epetraCoord)[1]); if (epetraCoord->NumVectors() > 2) mueluList.set("z-coordinates", (*epetraCoord)[2]); } if (!nullspace.is_null()) { RCP epetraNullspace = MueLu::Utilities::MV2EpetraMV(nullspace); mueluList.set("null space: dimension", epetraNullspace->NumVectors()); mueluList.set("null space: vectors", (*epetraNullspace)[0]); mueluList.set("null space: type", "pre-computed"); } RCP epetraKn; if (Kn.is_null()) { RCP temp = Xpetra::MatrixFactory::Build(SM->getRangeMap()); Xpetra::MatrixMatrix::Multiply(*SM, false, *D0, false, *temp, true, true); RCP Kn2 = Xpetra::MatrixFactory::Build(D0->getDomainMap()); Xpetra::MatrixMatrix::Multiply(*D0, true, *temp, false, *Kn2, true, true); epetraKn = Xpetra::Helpers::Op2EpetraCrs(Kn2); } else epetraKn = Xpetra::Helpers::Op2EpetraCrs(Kn); RCP mlop = rcp(new ML_Epetra::MultiLevelPreconditioner(*epetraSM, *epetraD0, *epetraKn, mueluList, true)); #if defined(HAVE_MUELU_BELOS) // NOTE: Belos needs the Apply() and AppleInverse() routines of ML swapped. So... mlop = rcp(new Belos::EpetraPrecOp(mlop)); #endif mlopX = rcp(new Xpetra::EpetraOperator(mlop)); #else TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error, "Need ML & Epetra support"); #endif } static void Generate_ML_RefMaxwellPreconditioner(Teuchos::RCP >& SM, Teuchos::RCP >& D0, Teuchos::RCP >& Ms, Teuchos::RCP >& M0inv, Teuchos::RCP >& M1, Teuchos::RCP >& nullspace, Teuchos::RCP >& node_material, Teuchos::RCP::coordinateType, int, GlobalOrdinal, Tpetra::KokkosCompat::KokkosSerialWrapperNode> >& coords, Teuchos::ParameterList& mueluList, Teuchos::RCP >& mlopX) { #if defined(HAVE_MUELU_ML) typedef double SC; typedef int LO; typedef GlobalOrdinal GO; typedef Tpetra::KokkosCompat::KokkosSerialWrapperNode NO; typedef typename Teuchos::ScalarTraits::coordinateType coordinate_type; RCP epetraSM = Xpetra::Helpers::Op2EpetraCrs(SM); RCP epetraD0 = Xpetra::Helpers::Op2EpetraCrs(D0); RCP epetraM0inv = Xpetra::Helpers::Op2EpetraCrs(M0inv); RCP epetraMs; RCP epetraM1 = Xpetra::Helpers::Op2EpetraCrs(M1); if (!Ms.is_null()) epetraMs = Xpetra::Helpers::Op2EpetraCrs(Ms); else epetraMs = epetraM1; mueluList.set("D0", epetraD0); mueluList.set("Ms", epetraMs); mueluList.set("M0inv", epetraM0inv); mueluList.set("M1", epetraM1); if (!coords.is_null()) { RCP epetraCoord = MueLu::Utilities::MV2EpetraMV(coords); if (epetraCoord->NumVectors() > 0) mueluList.sublist("refmaxwell: 11list").set("x-coordinates", (*epetraCoord)[0]); if (epetraCoord->NumVectors() > 1) mueluList.sublist("refmaxwell: 11list").set("y-coordinates", (*epetraCoord)[1]); if (epetraCoord->NumVectors() > 2) mueluList.sublist("refmaxwell: 11list").set("z-coordinates", (*epetraCoord)[2]); } if (!node_material.is_null()) { RCP epetraMaterial = MueLu::Utilities::MV2EpetraMV(node_material); mueluList.sublist("refmaxwell: 11list").set("material coordinates", (*epetraMaterial)[0]); } if (!nullspace.is_null()) { RCP epetraNullspace = MueLu::Utilities::MV2EpetraMV(nullspace); mueluList.sublist("refmaxwell: 11list").set("null space: dimension", epetraNullspace->NumVectors()); mueluList.sublist("refmaxwell: 11list").set("null space: vectors", (*epetraNullspace)[0]); mueluList.sublist("refmaxwell: 11list").set("null space: type", "pre-computed"); } RCP mlop = rcp(new ML_Epetra::RefMaxwellPreconditioner(*epetraSM, mueluList, true)); #if defined(HAVE_MUELU_BELOS) // NOTE: Belos needs the Apply() and AppleInverse() routines of ML swapped. So... mlop = rcp(new Belos::EpetraPrecOp(mlop)); #endif mlopX = rcp(new Xpetra::EpetraOperator(mlop)); #else TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error, "Need ML & Epetra support"); #endif } }; #endif // HAVE_MUELU_EPETRA template bool SetupSolve(std::map inputs) { #include typedef Xpetra::MultiVector::magnitudeType, LO, GO, NO> coordMV; RCP SM_Matrix = *static_cast*>(inputs["SM"]); RCP GmhdA_Matrix = *static_cast*>(inputs["GmhdA"]); RCP D0_Matrix = *static_cast*>(inputs["D0"]); RCP M1_Matrix = *static_cast*>(inputs["M1"]); RCP Ms_Matrix = *static_cast*>(inputs["Ms"]); RCP M0inv_Matrix = *static_cast*>(inputs["M0inv"]); RCP Kn_Matrix = *static_cast*>(inputs["Kn"]); RCP coords = *static_cast*>(inputs["coordinates"]); RCP nullspace = *static_cast*>(inputs["nullspace"]); RCP material = *static_cast*>(inputs["material"]); RCP B = *static_cast*>(inputs["B"]); RCP X = *static_cast*>(inputs["X"]); RCP X0 = *static_cast*>(inputs["X0"]); Teuchos::ParameterList params = *static_cast(inputs["params"]); Teuchos::ParameterList belosParams = *static_cast(inputs["belosParams"]); Teuchos::ParameterList stratimikosParams = *static_cast(inputs["stratimikosParams"]); std::string solverName = *static_cast(inputs["solverName"]); std::string belosSolverType = *static_cast(inputs["belosSolverType"]); std::string precType = *static_cast(inputs["precType"]); int numResolves = *static_cast(inputs["numResolves"]); bool reuse = *static_cast(inputs["reuse"]); RCP > comm = *static_cast >*>(inputs["comm"]); RCP out = *static_cast*>(inputs["out"]); bool success = false; auto tm2 = TimeMonitor::getNewTimer("Maxwell: 2 - Build solver and preconditioner"); #ifdef HAVE_MUELU_BELOS if (solverName == "Belos") { // construct preconditioner RCP preconditioner; if (precType == "MueLu-RefMaxwell") { preconditioner = rcp(new MueLu::RefMaxwell(SM_Matrix, D0_Matrix, Ms_Matrix, M0inv_Matrix, M1_Matrix, nullspace, coords, material, params)); } else if (precType == "MueLu-Maxwell1" || precType == "MueLu-Reitzinger") { if (GmhdA_Matrix.is_null()) // are we doing MHD as opposed to GMHD? preconditioner = rcp(new MueLu::Maxwell1(SM_Matrix, D0_Matrix, Kn_Matrix, nullspace, coords, material, params)); else preconditioner = rcp(new MueLu::Maxwell1(SM_Matrix, D0_Matrix, Kn_Matrix, nullspace, coords, params, GmhdA_Matrix)); } #ifdef HAVE_MUELU_EPETRA else if (precType == "ML-RefMaxwell") { Xpetra::UnderlyingLib lib = *static_cast(inputs["lib"]); TEUCHOS_ASSERT(lib == Xpetra::UseEpetra); EpetraSolvers_Wrapper::Generate_ML_RefMaxwellPreconditioner(SM_Matrix, D0_Matrix, Ms_Matrix, M0inv_Matrix, M1_Matrix, nullspace, material, coords, params, preconditioner); } else if (precType == "ML-Maxwell") { Xpetra::UnderlyingLib lib = *static_cast(inputs["lib"]); TEUCHOS_ASSERT(lib == Xpetra::UseEpetra); EpetraSolvers_Wrapper::Generate_ML_MaxwellPreconditioner(SM_Matrix, D0_Matrix, Kn_Matrix, nullspace, coords, params, preconditioner); } #endif { // A test to make sure we can wrap this guy as a MueLu::TpetraOperator RCP precOp = Teuchos::rcp_dynamic_cast(preconditioner); MueLu::TpetraOperator OpT(precOp); } // Belos linear problem typedef MultiVector MV; typedef Belos::OperatorT OP; RCP belosOp; if (GmhdA_Matrix.is_null()) // are we doing MHD as opposed to GMHD? belosOp = Teuchos::rcp(new Belos::XpetraOp(SM_Matrix)); // Turns a Xpetra::Matrix object into a Belos operator else belosOp = Teuchos::rcp(new Belos::XpetraOp(GmhdA_Matrix)); // Turns a Xpetra::Matrix object into a Belos operator RCP > problem = rcp(new Belos::LinearProblem()); problem->setOperator(belosOp); RCP belosPrecOp; if (precType != "none") { belosPrecOp = Teuchos::rcp(new Belos::XpetraOp(preconditioner)); // Turns a Xpetra::Matrix object into a Belos operator problem->setRightPrec(belosPrecOp); } problem->setProblem(X, B); bool set = problem->setProblem(); if (set == false) { *out << "\nERROR: Belos::LinearProblem failed to set up correctly!" << std::endl; return false; } // Belos solver RCP > solver; RCP > factory = rcp(new Belos::SolverFactory()); solver = factory->create(belosSolverType, Teuchos::rcpFromRef(belosParams.sublist(belosSolverType))); comm->barrier(); tm2 = Teuchos::null; auto tm3 = TimeMonitor::getNewTimer("Maxwell: 3 - Solve"); // set problem and solve solver->setProblem(problem); for (int solveno = 0; solveno <= numResolves; solveno++) { if (X0.is_null()) X->putScalar(Teuchos::ScalarTraits::zero()); else X = X0; Belos::ReturnType status = solver->solve(); int iters = solver->getNumIters(); success = (iters < 50 && status == Belos::Converged); if (status == Belos::Converged) *out << "Belos converged in " << iters << " iterations." << std::endl; if (success) *out << "SUCCESS!" << std::endl; else *out << "FAILURE! Belos did not converge fast enough." << std::endl; } if (reuse) { TEUCHOS_ASSERT(precType == "MueLu-RefMaxwell"); for (int solveno = 0; solveno < 2; solveno++) { // SM_Matrix->resumeFill(); // SM_Matrix->fillComplete(); if (X0.is_null()) X->putScalar(Teuchos::ScalarTraits::zero()); else X = X0; problem->setProblem(X, B); if (GmhdA_Matrix.is_null()) // are we doing MHD as opposed to GMHD? Teuchos::rcp_dynamic_cast >(preconditioner)->resetMatrix(SM_Matrix); else Teuchos::rcp_dynamic_cast >(preconditioner)->resetMatrix(GmhdA_Matrix); Belos::ReturnType status = solver->solve(); int iters = solver->getNumIters(); success = (iters < 50 && status == Belos::Converged); if (status == Belos::Converged) *out << "Belos converged in " << iters << " iterations." << std::endl; if (success) *out << "SUCCESS!" << std::endl; else *out << "FAILURE! Belos did not converge fast enough." << std::endl; } } } #endif // HAVE_MUELU_BELOS #if defined(HAVE_MUELU_STRATIMIKOS) && defined(HAVE_MUELU_THYRA) if (solverName == "Stratimikos") { using Teuchos::ParameterList; using Teuchos::rcp_dynamic_cast; // Add matrices to parameterlist Teuchos::ParameterList& precTypesPL = stratimikosParams.sublist("Preconditioner Types"); std::list sublists; sublists.push_back(&precTypesPL); while (sublists.size() > 0) { Teuchos::ParameterList* sublist = sublists.front(); sublists.pop_front(); std::list keys; for (Teuchos::ParameterList::ConstIterator it = sublist->begin(); it != sublist->end(); ++it) { const std::string& entryName = sublist->name(it); const Teuchos::ParameterEntry& theEntry = sublist->entry(it); if (theEntry.isList()) { Teuchos::ParameterList& sl = sublist->sublist(entryName); sublists.push_back(&sl); } else if (theEntry.isType() && Teuchos::getValue(theEntry).find("substitute ") != std::string::npos) { keys.push_back(entryName); } } for (auto key_it = keys.begin(); key_it != keys.end(); ++key_it) { std::string value = sublist->get(*key_it).substr(11, std::string::npos); if (value == "D0") sublist->set(*key_it, D0_Matrix); else if (value == "M0inv") sublist->set(*key_it, M0inv_Matrix); else if (value == "M1") sublist->set(*key_it, M1_Matrix); else if (value == "Ms") sublist->set(*key_it, Ms_Matrix); else if (value == "Kn") sublist->set(*key_it, Kn_Matrix); else if (value == "Coordinates") sublist->set(*key_it, coords); // else if (*key_it == "Nullspace") // sublist->set(*key_it, Nullspace); #ifdef HAVE_MUELU_EPETRA else if (value == "eD0") { auto eD0 = Teuchos::rcp_dynamic_cast(Teuchos::rcp_dynamic_cast(D0_Matrix, true)->getCrsMatrix(), true)->getEpetra_CrsMatrix(); sublist->set(*key_it, eD0); } else if (value == "eCoordinates") sublist->set(*key_it, Teuchos::rcp_dynamic_cast >(coords, true)->getEpetra_MultiVector()); #endif else if (value == "tD0") { auto tD0 = toTpetra(D0_Matrix); sublist->set(*key_it, tD0); } else if (value == "tCoordinates") { sublist->set(*key_it, toTpetra(coords)); } } } // Build Thyra linear algebra objects RCP > thyraA; if (GmhdA_Matrix.is_null()) // are we doing MHD as opposed to GMHD? thyraA = Xpetra::ThyraUtils::toThyra(Teuchos::rcp_dynamic_cast(SM_Matrix)->getCrsMatrix()); else thyraA = Xpetra::ThyraUtils::toThyra(Teuchos::rcp_dynamic_cast(GmhdA_Matrix)->getCrsMatrix()); RCP > thyraX = Teuchos::rcp_const_cast >(Xpetra::ThyraUtils::toThyraVector(X->getVectorNonConst(0))); // TODO: Why do we loose a reference when running this with Epetra? RCP > thyraB = Xpetra::ThyraUtils::toThyraVector(B->getVector(0)); // Build Stratimikos solver Stratimikos::LinearSolverBuilder linearSolverBuilder; // This is the Stratimikos main class (= factory of solver factory). Stratimikos::enableMueLuRefMaxwell(linearSolverBuilder); // Register MueLu as a Stratimikos preconditioner strategy. Stratimikos::enableMueLuMaxwell1(linearSolverBuilder); linearSolverBuilder.setParameterList(rcp(&stratimikosParams, false)); // Setup solver parameters using a Stratimikos parameter list. // Build a new "solver factory" according to the previously specified parameter list. RCP > solverFactory = Thyra::createLinearSolveStrategy(linearSolverBuilder); auto precFactory = solverFactory->getPreconditionerFactory(); auto prec = precFactory->createPrec(); // Build a Thyra operator corresponding to A^{-1} computed using the Stratimikos solver. Thyra::initializePrec(*precFactory, thyraA, prec.ptr()); Teuchos::RCP > thyraInverseA = solverFactory->createOp(); Thyra::initializePreconditionedOp(*solverFactory, thyraA, prec, thyraInverseA.ptr()); comm->barrier(); tm2 = Teuchos::null; auto tm5 = TimeMonitor::getNewTimer("Maxwell: 3 - Solve"); // Solve Ax = b. Thyra::SolveStatus status = Thyra::solve(*thyraInverseA, Thyra::NOTRANS, *thyraB, thyraX.ptr()); comm->barrier(); tm5 = Teuchos::null; std::cout << status << std::endl; success = (status.solveStatus == Thyra::SOLVE_STATUS_CONVERGED); if (reuse) { for (int solveno = 0; solveno < 2; solveno++) { auto tm6 = TimeMonitor::getNewTimer("Maxwell: 4 - Setup Re"); Thyra::initializePrec(*precFactory, thyraA, prec.ptr()); comm->barrier(); tm6 = Teuchos::null; auto tm7 = TimeMonitor::getNewTimer("Maxwell: 5 - Solve Re"); status = Thyra::solve(*thyraInverseA, Thyra::NOTRANS, *thyraB, thyraX.ptr()); comm->barrier(); tm7 = Teuchos::null; } } } // HAVE_MUELU_STRATIMIKOS && HAVE_MUELU_THYRA #endif comm->barrier(); return success; } // SetupSolve template int main_(Teuchos::CommandLineProcessor& clp, Xpetra::UnderlyingLib lib, int argc, char* argv[]) { #include RCP > comm = Teuchos::DefaultComm::getComm(); RCP out = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout)); out->setOutputToRootOnly(0); bool printTimings = true; clp.setOption("timings", "notimings", &printTimings, "print timings to screen"); std::string timingsFormat = "table-fixed"; clp.setOption("time-format", &timingsFormat, "timings format (table-fixed | table-scientific | yaml)"); double scaling = 1.0; clp.setOption("scaling", &scaling, "scale mass term"); std::string solverName = "Belos"; clp.setOption("solverName", &solverName, "Name of iterative linear solver " "to use for solving the linear system. " "(\"Belos\" or \"Stratimikos\")"); std::string belosSolverType = "Block CG"; clp.setOption("belosSolverType", &belosSolverType, "Name of the Belos linear solver"); std::string precType = "MueLu-RefMaxwell"; clp.setOption("precType", &precType, "preconditioner to use (MueLu-RefMaxwell|MueLu-Maxwell1|ML-RefMaxwell|none)"); std::string xml = ""; clp.setOption("xml", &xml, "xml file with solver parameters (default: \"Maxwell.xml\")"); std::string belos_xml = "Belos.xml"; clp.setOption("belos_xml", &belos_xml, "xml file with Belos solver parameters (default: \"Belos.xml\")"); std::string stratimikos_xml = "Stratimikos.xml"; clp.setOption("stratimikos_xml", &stratimikos_xml, "xml file with Stratimikos solver parameters (default: \"Stratimikos.xml\")"); int numResolves = 0; clp.setOption("resolve", &numResolves, "#times to redo solve"); bool reuse = false; clp.setOption("reuse", "no-reuse", &reuse, "test reuse"); double tol = 1e-10; clp.setOption("tol", &tol, "solver convergence tolerance"); int maxIts = 200; clp.setOption("its", &maxIts, "maximum number of solver iterations"); bool use_stacked_timer = false; clp.setOption("stacked-timer", "no-stacked-timer", &use_stacked_timer, "use stacked timer"); bool ensure_kn = false; clp.setOption("ensure-kn", "no-ensure-kn", &ensure_kn, "generate a kn matrix if the user doesn't provide one"); std::string S_file, D0_file, M1_file, M0_file; if (!Teuchos::ScalarTraits::isComplex) { S_file = "S.mat"; D0_file = "D0.mat"; M1_file = "M1.mat"; M0_file = "M0.mat"; } else { S_file = "S_complex.mat"; D0_file = "D0_complex.mat"; M1_file = "M1_complex.mat"; M0_file = "M0_complex.mat"; } clp.setOption("S", &S_file); std::string SM_file = ""; clp.setOption("SM", &SM_file); std::string Kn_file = ""; clp.setOption("Kn", &Kn_file); clp.setOption("D0", &D0_file); clp.setOption("M1", &M1_file); std::string Ms_file = ""; clp.setOption("Ms", &Ms_file); clp.setOption("M0", &M0_file); std::string M0inv_file = ""; clp.setOption("M0inv", &M0inv_file); std::string coords_file = "coords.mat"; clp.setOption("coords", &coords_file); std::string nullspace_file = ""; clp.setOption("nullspace", &nullspace_file); std::string material_file = ""; clp.setOption("material", &material_file); std::string rhs_file = ""; clp.setOption("rhs", &rhs_file); std::string x0_file = ""; clp.setOption("x0", &x0_file); std::string GmhdA_file = ""; clp.setOption("GmhdA", &GmhdA_file); clp.recogniseAllOptions(true); switch (clp.parse(argc, argv)) { case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; case Teuchos::CommandLineProcessor::PARSE_ERROR: case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break; } if (precType == "MueLu-Reitzinger" || precType == "MueLu-Maxwell1" || precType == "ML-Maxwell") { if (SM_file != "") M1_file = ""; M0_file = ""; } if (xml == "") { if (precType == "MueLu-RefMaxwell") xml = "Maxwell.xml"; else if (precType == "MueLu-Reitzinger" || precType == "MueLu-Maxwell1") xml = "Maxwell_Reitzinger.xml"; else if (precType == "ML-RefMaxwell") xml = "Maxwell_ML.xml"; else if (precType == "ML-Maxwell") xml = "Maxwell_ML1.xml"; else if (precType == "Hypre") xml = "Hypre.xml"; } RCP stacked_timer; if (use_stacked_timer) stacked_timer = rcp(new Teuchos::StackedTimer("Maxwell Driver")); TimeMonitor::setStackedTimer(stacked_timer); auto globalTimeMonitor = TimeMonitor::getNewTimer("Maxwell: S - Global Time"); auto tm = TimeMonitor::getNewTimer("Maxwell: 1 - Read and Build Matrices"); // Read matrices in from files RCP D0_Matrix, SM_Matrix, M1_Matrix, Ms_Matrix, M0inv_Matrix, Kn_Matrix, GmhdA_Matrix; // maps for nodal and edge matrices RCP node_map; RCP edge_map; RCP Gmhd_map; RCP edgeOrGmhdMap; // gradient matrix try { std::string base = D0_file.substr(0, D0_file.find_last_of('/') + 1); std::string D0_filename = D0_file.substr(D0_file.find_last_of('/') + 1, std::string::npos); std::string edgeMap_file = base + "rowmap_" + D0_filename; std::string nodeMap_file = base + "domainmap_" + D0_filename; std::string colMap_file = base + "colmap_" + D0_filename; node_map = Xpetra::IO::ReadMap(nodeMap_file, lib, comm); edge_map = Xpetra::IO::ReadMap(edgeMap_file, lib, comm); RCP colMap; if (comm->getSize() > 1) colMap = Xpetra::IO::ReadMap(colMap_file, lib, comm); D0_Matrix = Xpetra::IO::Read(D0_file, edge_map, colMap, node_map, edge_map); } catch (const std::exception&) { // *out << "Skipping D0 maps, because: " << e.what() << std::endl; D0_Matrix = Xpetra::IO::Read(D0_file, lib, comm); node_map = D0_Matrix->getDomainMap(); edge_map = D0_Matrix->getRangeMap(); } // build stiffness plus mass matrix (SM_Matrix) if (SM_file == "") { // edge stiffness matrix RCP S_Matrix = Xpetra::IO::Read(S_file, edge_map); M1_Matrix = Xpetra::IO::Read(M1_file, edge_map); Xpetra::MatrixMatrix::TwoMatrixAdd(*S_Matrix, false, (SC)1.0, *M1_Matrix, false, scaling, SM_Matrix, *out); SM_Matrix->fillComplete(); } else { SM_Matrix = Xpetra::IO::Read(SM_file, edge_map); if (M1_file != "") M1_Matrix = Xpetra::IO::Read(M1_file, edge_map); } if (GmhdA_file != "") { Teuchos::ArrayView edgeMapEleList = edge_map->getLocalElementList(); Teuchos::Array elementList(edgeMapEleList.size() * 2); for (int i = 0; i < edgeMapEleList.size(); i++) { elementList[2 * i] = 2 * edgeMapEleList[i]; elementList[2 * i + 1] = 2 * edgeMapEleList[i] + 1; } edgeOrGmhdMap = MapFactory::Build(edge_map->lib(), edge_map->getGlobalNumElements() * 2, elementList, edge_map->getIndexBase(), edge_map->getComm()); GmhdA_Matrix = Xpetra::IO::Read(GmhdA_file, edgeOrGmhdMap); } else edgeOrGmhdMap = edge_map; if (Ms_file != "") Ms_Matrix = Xpetra::IO::Read(Ms_file, edge_map); else Ms_Matrix = M1_Matrix; if (Kn_file != "") { *out << "User provided Kn matrix" << std::endl; Kn_Matrix = Xpetra::IO::Read(Kn_file, node_map); } else if (ensure_kn) { // The user requested we generate Kn_Matrix with an SpGEMM Teuchos::ParameterList params; *out << "Making Kn Matrix as requested by yser" << std::endl; Kn_Matrix = MueLu::Maxwell_Utils::PtAPWrapper(SM_Matrix, D0_Matrix, params, "User Kn"); } else { *out << "NOT using a Kn matrix" << std::endl; } if ((M0inv_file == "") && (M0_file != "")) { // nodal mass matrix RCP M0_Matrix = Xpetra::IO::Read(M0_file, node_map); // build lumped mass matrix inverse (M0inv_Matrix) RCP diag = Utilities::GetLumpedMatrixDiagonal(*M0_Matrix); RCP M0inv_MatrixWrap = rcp(new CrsMatrixWrap(node_map, node_map, 0)); RCP M0inv_CrsMatrix = M0inv_MatrixWrap->getCrsMatrix(); Teuchos::ArrayRCP rowPtr; Teuchos::ArrayRCP colInd; Teuchos::ArrayRCP values; Teuchos::ArrayRCP diags = diag->getData(0); size_t nodeNumElements = node_map->getLocalNumElements(); M0inv_CrsMatrix->allocateAllValues(nodeNumElements, rowPtr, colInd, values); SC ONE = Teuchos::ScalarTraits::one(); for (size_t i = 0; i < nodeNumElements; i++) { rowPtr[i] = i; colInd[i] = i; values[i] = ONE / diags[i]; } rowPtr[nodeNumElements] = nodeNumElements; M0inv_CrsMatrix->setAllValues(rowPtr, colInd, values); M0inv_CrsMatrix->expertStaticFillComplete(node_map, node_map); M0inv_Matrix = Teuchos::rcp_dynamic_cast(M0inv_MatrixWrap); } else if (M0inv_file != "") { M0inv_Matrix = Xpetra::IO::Read(M0inv_file, node_map); } // coordinates RCP::magnitudeType, LO, GO, NO> > coords = Xpetra::IO::magnitudeType, LO, GO, NO>::ReadMultiVector(coords_file, node_map); RCP nullspace = Teuchos::null; if (nullspace_file != "") nullspace = Xpetra::IO::ReadMultiVector(nullspace_file, edge_map); RCP material = Teuchos::null; if (material_file != "") { material = Xpetra::IO::ReadMultiVector(material_file, node_map); } // set parameters Teuchos::ParameterList params, belosParams, stratimikosParams; Teuchos::updateParametersFromXmlFileAndBroadcast(xml, Teuchos::Ptr(¶ms), *comm); Teuchos::updateParametersFromXmlFileAndBroadcast(belos_xml, Teuchos::Ptr(&belosParams), *comm); Teuchos::updateParametersFromXmlFileAndBroadcast(stratimikos_xml, Teuchos::Ptr(&stratimikosParams), *comm); belosParams.sublist(belosSolverType).set("Maximum Iterations", maxIts); belosParams.sublist(belosSolverType).set("Convergence Tolerance", tol); // setup LHS, RHS RCP X, X0, B; if (rhs_file == "") { B = MultiVectorFactory::Build(edgeOrGmhdMap, 1); RCP vec = MultiVectorFactory::Build(edgeOrGmhdMap, 1); vec->putScalar(Teuchos::ScalarTraits::one()); if (GmhdA_Matrix.is_null()) // are we doing MHD as opposed to GMHD? SM_Matrix->apply(*vec, *B); else GmhdA_Matrix->apply(*vec, *B); } else B = Xpetra::IO::ReadMultiVector(rhs_file, edgeOrGmhdMap); X = MultiVectorFactory::Build(edgeOrGmhdMap, 1); X->putScalar(Teuchos::ScalarTraits::zero()); if (x0_file != "") X0 = Xpetra::IO::ReadMultiVector(x0_file, edgeOrGmhdMap); comm->barrier(); tm = Teuchos::null; std::map inputs; inputs["SM"] = &SM_Matrix; inputs["GmhdA"] = &GmhdA_Matrix; inputs["D0"] = &D0_Matrix; inputs["M1"] = &M1_Matrix; inputs["Ms"] = &Ms_Matrix; inputs["M0inv"] = &M0inv_Matrix; inputs["Kn"] = &Kn_Matrix; inputs["coordinates"] = &coords; inputs["nullspace"] = &nullspace; inputs["material"] = &material; inputs["B"] = &B; inputs["X"] = &X; inputs["X0"] = &X0; inputs["params"] = ¶ms; inputs["solverName"] = &solverName; inputs["belosSolverType"] = &belosSolverType; inputs["precType"] = &precType; inputs["belosParams"] = &belosParams; inputs["stratimikosParams"] = &stratimikosParams; inputs["numResolves"] = &numResolves; inputs["reuse"] = &reuse; inputs["lib"] = &lib; inputs["comm"] = &comm; inputs["out"] = &out; bool success = SetupSolve(inputs); globalTimeMonitor = Teuchos::null; if (printTimings) { if (use_stacked_timer) { stacked_timer->stop("Maxwell Driver"); Teuchos::StackedTimer::OutputOptions options; options.output_fraction = options.output_histogram = options.output_minmax = true; stacked_timer->report(*out, comm, options); } else { RCP reportParams = rcp(new Teuchos::ParameterList); if (timingsFormat == "yaml") { reportParams->set("Report format", "YAML"); // "Table" or "YAML" reportParams->set("YAML style", "compact"); // "spacious" or "compact" } reportParams->set("How to merge timer sets", "Union"); reportParams->set("alwaysWriteLocal", false); reportParams->set("writeGlobalStats", true); reportParams->set("writeZeroTimers", false); // FIXME: no "ignoreZeroTimers" const std::string filter = ""; std::ios_base::fmtflags ff(out->flags()); if (timingsFormat == "table-fixed") *out << std::fixed; else *out << std::scientific; TimeMonitor::report(comm.ptr(), *out, filter, reportParams); *out << std::setiosflags(ff); } } TimeMonitor::clearCounters(); return (success ? EXIT_SUCCESS : EXIT_FAILURE); } //- -- -------------------------------------------------------- #define MUELU_AUTOMATIC_TEST_ETI_NAME main_ #include "MueLu_Test_ETI.hpp" int main(int argc, char* argv[]) { return Automatic_Test_ETI(argc, argv); }