// @HEADER // ***************************************************************************** // Panzer: A partial differential equation assembly // engine for strongly coupled complex multiphysics systems // // Copyright 2011 NTESS and the Panzer contributors. // SPDX-License-Identifier: BSD-3-Clause // ***************************************************************************** // @HEADER /////////////////////////////////////////////////////////////////////////////// // // Include Files // /////////////////////////////////////////////////////////////////////////////// // C++ #include #include #include // Kokkos #include "Kokkos_View_Fad.hpp" // Panzer #include "PanzerAdaptersSTK_config.hpp" #include "Panzer_BasisIRLayout.hpp" #include "Panzer_TpetraLinearObjFactory.hpp" #include "Panzer_DOFManager.hpp" #include "Panzer_Evaluator_WithBaseImpl.hpp" #include "Panzer_FieldManagerBuilder.hpp" #include "Panzer_GatherOrientation.hpp" #include "Panzer_PureBasis.hpp" #include "Panzer_STKConnManager.hpp" #include "Panzer_STK_Interface.hpp" #include "Panzer_STK_SetupUtilities.hpp" #include "Panzer_STK_SquareQuadMeshFactory.hpp" #include "Panzer_STK_Version.hpp" #include "Panzer_Workset.hpp" #include "Panzer_LOCPair_GlobalEvaluationData.hpp" #include "Panzer_GlobalEvaluationDataContainer.hpp" // Teuchos #include "Teuchos_DefaultMpiComm.hpp" #include "Teuchos_GlobalMPISession.hpp" #include "Teuchos_OpaqueWrapper.hpp" #include "Teuchos_RCP.hpp" #include "Teuchos_TimeMonitor.hpp" #include "Teuchos_UnitTestHarness.hpp" // Thyra #include "Thyra_ProductVectorBase.hpp" #include "Thyra_VectorStdOps.hpp" // Tpetra #include "Tpetra_Vector.hpp" // user_app #include "user_app_EquationSetFactory.hpp" typedef double ScalarT; using LocalOrdinalT = panzer::LocalOrdinal; using GlobalOrdinalT = panzer::GlobalOrdinal; typedef Tpetra::Vector VectorType; typedef Tpetra::Operator OperatorType; typedef Tpetra::CrsMatrix CrsMatrixType; typedef Tpetra::CrsGraph CrsGraphType; typedef Tpetra::Map MapType; typedef Tpetra::Import ImportType; typedef Tpetra::Export ExportType; typedef Thyra::TpetraLinearOp ThyraLinearOp; typedef panzer::TpetraLinearObjFactory TpetraLinObjFactoryType; typedef panzer::TpetraLinearObjContainer TpetraLinObjContainerType; namespace panzer { Teuchos::RCP buildBasis(std::size_t worksetSize, const std::string &basisName); void testInitialization(const Teuchos::RCP &ipb); Teuchos::RCP buildMesh(int elemX, int elemY); void testGatherScatter(const bool enable_tangents, Teuchos::FancyOStream &out, bool &success); // Test without tangent fields in gather evaluator TEUCHOS_UNIT_TEST(tpetra_assembly, gather_solution_no_tangents) { testGatherScatter(false, out, success); } // Test with tangent fields in gather evaluator TEUCHOS_UNIT_TEST(tpetra_assembly, gather_solution_tangents) { testGatherScatter(true, out, success); } // enable_tangents determines whether tangent fields dx/dp are added to gather evaluator. // These are used when computing df/dx*dx/dp with the tangent evaluation type void testGatherScatter(const bool enable_tangents, Teuchos::FancyOStream &out, bool &success) { #ifdef HAVE_MPI Teuchos::RCP> tComm = Teuchos::rcp(new Teuchos::MpiComm(MPI_COMM_WORLD)); #else Teuchos::RCP> tComm = Teuchos::rcp(new Teuchos::SerialComm(MPI_COMM_WORLD)); #endif int myRank = tComm->getRank(); int numProcs = tComm->getSize(); const std::size_t workset_size = 4 / numProcs; const std::string fieldName1_q1 = "U"; const std::string fieldName2_q1 = "V"; const std::string fieldName_qedge1 = "B"; const int num_tangent = enable_tangents ? 5 : 0; Teuchos::RCP mesh = buildMesh(2, 2); // build input physics block Teuchos::RCP basis_q1 = buildBasis(workset_size, "Q1"); Teuchos::RCP basis_qedge1 = buildBasis(workset_size, "QEdge1"); Teuchos::RCP ipb = Teuchos::parameterList(); testInitialization(ipb); const int default_int_order = 1; std::string eBlockID = "eblock-0_0"; Teuchos::RCP eqset_factory = Teuchos::rcp(new user_app::MyFactory); panzer::CellData cellData(workset_size, mesh->getCellTopology("eblock-0_0")); Teuchos::RCP gd = panzer::createGlobalData(); Teuchos::RCP physicsBlock = Teuchos::rcp(new PhysicsBlock(ipb, eBlockID, default_int_order, cellData, eqset_factory, gd, false)); Teuchos::RCP> work_sets = panzer_stk::buildWorksets(*mesh, physicsBlock->elementBlockID(), physicsBlock->getWorksetNeeds()); TEST_EQUALITY(work_sets->size(), 1); // build connection manager and field manager const Teuchos::RCP conn_manager = Teuchos::rcp(new panzer_stk::STKConnManager(mesh)); Teuchos::RCP dofManager = Teuchos::rcp(new panzer::DOFManager(conn_manager, MPI_COMM_WORLD)); dofManager->addField(fieldName1_q1, Teuchos::rcp(new panzer::Intrepid2FieldPattern(basis_q1->getIntrepid2Basis()))); dofManager->addField(fieldName2_q1, Teuchos::rcp(new panzer::Intrepid2FieldPattern(basis_q1->getIntrepid2Basis()))); dofManager->addField(fieldName_qedge1, Teuchos::rcp(new panzer::Intrepid2FieldPattern(basis_qedge1->getIntrepid2Basis()))); std::vector fieldOrder; fieldOrder.push_back(fieldName1_q1); fieldOrder.push_back(fieldName_qedge1); fieldOrder.push_back(fieldName2_q1); TEST_ASSERT(true == dofManager->validFieldOrder(fieldOrder)); dofManager->setFieldOrder(fieldOrder); dofManager->buildGlobalUnknowns(); // setup linear object factory ///////////////////////////////////////////////////////////// Teuchos::RCP t_lof = Teuchos::rcp(new TpetraLinObjFactoryType(tComm.getConst(), dofManager)); Teuchos::RCP> lof = t_lof; Teuchos::RCP loc = t_lof->buildGhostedLinearObjContainer(); t_lof->initializeGhostedContainer(LinearObjContainer::X, *loc); loc->initialize(); Teuchos::RCP t_loc = Teuchos::rcp_dynamic_cast(loc); Teuchos::RCP> x_vec = t_loc->get_x_th(); Thyra::assign(x_vec.ptr(), 123.0 + myRank); std::vector> tangentContainers; if (enable_tangents) { using Teuchos::RCP; using Teuchos::rcp; using Teuchos::rcp_dynamic_cast; using Thyra::VectorBase; using LOCPair = panzer::LOCPair_GlobalEvaluationData; for (int i(0); i < num_tangent; ++i) { auto locPair = Teuchos::rcp(new LOCPair(t_lof, panzer::LinearObjContainer::X)); auto global_t_loc = rcp_dynamic_cast(locPair->getGlobalLOC()); Teuchos::RCP> global_x_vec = global_t_loc->get_x_th(); Thyra::assign(global_x_vec.ptr(), 0.123 + myRank + i); auto ghosted_t_loc = rcp_dynamic_cast(locPair->getGhostedLOC()); Teuchos::RCP> ghosted_x_vec = ghosted_t_loc->get_x_th(); Thyra::assign(ghosted_x_vec.ptr(), 0.123 + myRank + i); tangentContainers.push_back(locPair); } // end loop over the tangents } // end if (enable_tangents) // setup field manager, add evaluator under test ///////////////////////////////////////////////////////////// PHX::FieldManager fm; std::vector derivative_dimensions; derivative_dimensions.push_back(12); fm.setKokkosExtendedDataTypeDimensions(derivative_dimensions); std::vector tan_derivative_dimensions; if (enable_tangents) tan_derivative_dimensions.push_back(num_tangent); else tan_derivative_dimensions.push_back(0); fm.setKokkosExtendedDataTypeDimensions(tan_derivative_dimensions); Teuchos::RCP evalField_q1, evalField_qedge1; { using Teuchos::RCP; using Teuchos::rcp; RCP> names = rcp(new std::vector); names->push_back(fieldName1_q1); names->push_back(fieldName2_q1); Teuchos::ParameterList pl; pl.set("Basis", basis_q1); pl.set("DOF Names", names); pl.set("Indexer Names", names); Teuchos::RCP> evaluator = lof->buildGather(pl); TEST_EQUALITY(evaluator->evaluatedFields().size(), 2); fm.registerEvaluator(evaluator); fm.requireField(*evaluator->evaluatedFields()[0]); } { using Teuchos::RCP; using Teuchos::rcp; RCP> names = rcp(new std::vector); names->push_back(fieldName_qedge1); Teuchos::ParameterList pl; pl.set("Basis", basis_qedge1); pl.set("DOF Names", names); pl.set("Indexer Names", names); Teuchos::RCP> evaluator = lof->buildGather(pl); TEST_EQUALITY(evaluator->evaluatedFields().size(), 1); fm.registerEvaluator(evaluator); fm.requireField(*evaluator->evaluatedFields()[0]); } { using Teuchos::RCP; using Teuchos::rcp; RCP> names = rcp(new std::vector); names->push_back(fieldName1_q1); names->push_back(fieldName2_q1); Teuchos::ParameterList pl; pl.set("Basis", basis_q1); pl.set("DOF Names", names); pl.set("Indexer Names", names); Teuchos::RCP> evaluator = lof->buildGather(pl); TEST_EQUALITY(evaluator->evaluatedFields().size(), 2); fm.registerEvaluator(evaluator); fm.requireField(*evaluator->evaluatedFields()[0]); } { using Teuchos::RCP; using Teuchos::rcp; RCP> names = rcp(new std::vector); names->push_back(fieldName_qedge1); Teuchos::ParameterList pl; pl.set("Basis", basis_qedge1); pl.set("DOF Names", names); pl.set("Indexer Names", names); Teuchos::RCP> evaluator = lof->buildGather(pl); TEST_EQUALITY(evaluator->evaluatedFields().size(), 1); fm.registerEvaluator(evaluator); fm.requireField(*evaluator->evaluatedFields()[0]); } { using Teuchos::RCP; using Teuchos::rcp; RCP> names = rcp(new std::vector); names->push_back(fieldName1_q1); names->push_back(fieldName2_q1); Teuchos::ParameterList pl; pl.set("Basis", basis_q1); pl.set("DOF Names", names); pl.set("Indexer Names", names); if (enable_tangents) { RCP>> tangent_names = rcp(new std::vector>(2)); for (int i = 0; i < num_tangent; ++i) { std::stringstream ss1, ss2; ss1 << fieldName1_q1 << " Tangent " << i; ss2 << fieldName2_q1 << " Tangent " << i; (*tangent_names)[0].push_back(ss1.str()); (*tangent_names)[1].push_back(ss2.str()); } pl.set("Tangent Names", tangent_names); } Teuchos::RCP> evaluator = lof->buildGather(pl); TEST_EQUALITY(evaluator->evaluatedFields().size(), 2); fm.registerEvaluator(evaluator); fm.requireField(*evaluator->evaluatedFields()[0]); } { using Teuchos::RCP; using Teuchos::rcp; RCP> names = rcp(new std::vector); names->push_back(fieldName_qedge1); Teuchos::ParameterList pl; pl.set("Basis", basis_qedge1); pl.set("DOF Names", names); pl.set("Indexer Names", names); if (enable_tangents) { RCP>> tangent_names = rcp(new std::vector>(1)); for (int i = 0; i < num_tangent; ++i) { std::stringstream ss; ss << fieldName_qedge1 << " Tangent " << i; (*tangent_names)[0].push_back(ss.str()); } pl.set("Tangent Names", tangent_names); } Teuchos::RCP> evaluator = lof->buildGather(pl); TEST_EQUALITY(evaluator->evaluatedFields().size(), 1); fm.registerEvaluator(evaluator); fm.requireField(*evaluator->evaluatedFields()[0]); } if (enable_tangents) { for (int i = 0; i < num_tangent; ++i) { using Teuchos::RCP; using Teuchos::rcp; RCP> names = rcp(new std::vector); RCP> tangent_names = rcp(new std::vector); names->push_back(fieldName1_q1); names->push_back(fieldName2_q1); { std::stringstream ss1, ss2; ss1 << fieldName1_q1 << " Tangent " << i; ss2 << fieldName2_q1 << " Tangent " << i; tangent_names->push_back(ss1.str()); tangent_names->push_back(ss2.str()); } Teuchos::ParameterList pl; pl.set("Basis", basis_q1); pl.set("DOF Names", tangent_names); pl.set("Indexer Names", names); { std::stringstream ss; ss << "Tangent Container " << i; pl.set("Global Data Key", ss.str()); } Teuchos::RCP> evaluator = lof->buildGatherTangent(pl); TEST_EQUALITY(evaluator->evaluatedFields().size(), 2); fm.registerEvaluator(evaluator); } for (int i = 0; i < num_tangent; ++i) { using Teuchos::RCP; using Teuchos::rcp; RCP> names = rcp(new std::vector); RCP> tangent_names = rcp(new std::vector); names->push_back(fieldName_qedge1); { std::stringstream ss; ss << fieldName_qedge1 << " Tangent " << i; tangent_names->push_back(ss.str()); } Teuchos::ParameterList pl; pl.set("Basis", basis_qedge1); pl.set("DOF Names", tangent_names); pl.set("Indexer Names", names); { std::stringstream ss; ss << "Tangent Container " << i; pl.set("Global Data Key", ss.str()); } Teuchos::RCP> evaluator = lof->buildGatherTangent(pl); TEST_EQUALITY(evaluator->evaluatedFields().size(), 1); fm.registerEvaluator(evaluator); } } panzer::Traits::SD sd; panzer::Workset &workset = (*work_sets)[0]; workset.alpha = 0.0; workset.beta = 2.0; // derivatives multiplied by 2 workset.time = 0.0; workset.evaluate_transient_terms = false; sd.worksets_ = work_sets; fm.postRegistrationSetup(sd); panzer::Traits::PED ped; ped.gedc->addDataObject("Solution Gather Container", loc); if (enable_tangents) { for (int i(0); i < num_tangent; ++i) { std::stringstream ss; ss << "Tangent Container " << i; ped.gedc->addDataObject(ss.str(), tangentContainers[i]); } } fm.preEvaluate(ped); fm.evaluateFields(workset); fm.postEvaluate(0); fm.preEvaluate(ped); fm.evaluateFields(workset); fm.postEvaluate(0); fm.preEvaluate(ped); fm.evaluateFields(workset); fm.postEvaluate(0); // test Residual fields { PHX::MDField fieldData1_q1(fieldName1_q1, basis_q1->functional); PHX::MDField fieldData2_q1(fieldName2_q1, basis_qedge1->functional); fm.getFieldData(fieldData1_q1); fm.getFieldData(fieldData2_q1); TEST_EQUALITY(fieldData1_q1.extent(0), Teuchos::as(4 / numProcs)); TEST_EQUALITY(fieldData1_q1.extent(1), 4); TEST_EQUALITY(fieldData2_q1.extent(0), Teuchos::as(4 / numProcs)); TEST_EQUALITY(fieldData2_q1.extent(1), 4); TEST_EQUALITY(fieldData1_q1.size(), Teuchos::as(4 * 4 / numProcs)); TEST_EQUALITY(fieldData2_q1.size(), Teuchos::as(4 * 4 / numProcs)); auto fieldData1_q1_h = Kokkos::create_mirror_view(fieldData1_q1.get_static_view()); auto fieldData2_q1_h = Kokkos::create_mirror_view(fieldData2_q1.get_static_view()); Kokkos::deep_copy(fieldData1_q1_h, fieldData1_q1.get_static_view()); Kokkos::deep_copy(fieldData2_q1_h, fieldData2_q1.get_static_view()); for (unsigned int i = 0; i < fieldData1_q1.extent(0); i++) for (unsigned int j = 0; j < fieldData1_q1.extent(1); j++) TEST_EQUALITY(fieldData1_q1_h(i, j), 123.0 + myRank); for (unsigned int i = 0; i < fieldData2_q1.extent(0); i++) for (unsigned int j = 0; j < fieldData2_q1.extent(1); j++) TEST_EQUALITY(fieldData2_q1_h(i, j), 123.0 + myRank); } { PHX::MDField fieldData_qedge1(fieldName_qedge1, basis_qedge1->functional); fm.getFieldData(fieldData_qedge1); auto fieldData_qedge1_h = Kokkos::create_mirror_view(fieldData_qedge1.get_static_view()); Kokkos::deep_copy(fieldData_qedge1_h, fieldData_qedge1.get_static_view()); TEST_EQUALITY(fieldData_qedge1.extent(0), Teuchos::as(4 / numProcs)); TEST_EQUALITY(fieldData_qedge1.extent(1), 4); TEST_EQUALITY(fieldData_qedge1.size(), Teuchos::as(4 * 4 / numProcs)); for (unsigned int cell = 0; cell < fieldData_qedge1.extent(0); ++cell) for (unsigned int pt = 0; pt < fieldData_qedge1.extent(1); pt++) TEST_EQUALITY(fieldData_qedge1_h(cell, pt), 123.0 + myRank); } // test Jacobian fields { PHX::MDField fieldData1_q1(fieldName1_q1, basis_q1->functional); PHX::MDField fieldData2_q1(fieldName2_q1, basis_qedge1->functional); fm.getFieldData(fieldData1_q1); fm.getFieldData(fieldData2_q1); auto fieldData1_q1_h = Kokkos::create_mirror_view(fieldData1_q1.get_static_view()); auto fieldData2_q1_h = Kokkos::create_mirror_view(fieldData2_q1.get_static_view()); Kokkos::deep_copy(fieldData1_q1_h, fieldData1_q1.get_static_view()); Kokkos::deep_copy(fieldData2_q1_h, fieldData2_q1.get_static_view()); for (unsigned int cell = 0; cell < fieldData1_q1.extent(0); ++cell) { for (unsigned int pt = 0; pt < fieldData1_q1.extent(1); pt++) { TEST_EQUALITY(fieldData1_q1_h(cell, pt), 123.0 + myRank); TEST_EQUALITY(fieldData1_q1_h(cell, pt).availableSize(), 12); } } for (unsigned int cell = 0; cell < fieldData2_q1.extent(0); ++cell) { for (unsigned int pt = 0; pt < fieldData2_q1.extent(1); pt++) { TEST_EQUALITY(fieldData2_q1_h(cell, pt), 123.0 + myRank); TEST_EQUALITY(fieldData2_q1_h(cell, pt).availableSize(), 12); } } } { PHX::MDField fieldData_qedge1(fieldName_qedge1, basis_qedge1->functional); fm.getFieldData(fieldData_qedge1); auto fieldData_qedge1_h = Kokkos::create_mirror_view(fieldData_qedge1.get_static_view()); Kokkos::deep_copy(fieldData_qedge1_h, fieldData_qedge1.get_static_view()); for (unsigned int cell = 0; cell < fieldData_qedge1.extent(0); ++cell) { for (unsigned int pt = 0; pt < fieldData_qedge1.extent(1); ++pt) { TEST_EQUALITY(fieldData_qedge1_h(cell, pt), 123.0 + myRank); TEST_EQUALITY(fieldData_qedge1_h(cell, pt).availableSize(), 12); } } } // test Tangent fields { PHX::MDField fieldData1_q1(fieldName1_q1, basis_q1->functional); PHX::MDField fieldData2_q1(fieldName2_q1, basis_qedge1->functional); fm.getFieldData(fieldData1_q1); fm.getFieldData(fieldData2_q1); auto fieldData1_q1_h = Kokkos::create_mirror_view(fieldData1_q1.get_static_view()); auto fieldData2_q1_h = Kokkos::create_mirror_view(fieldData2_q1.get_static_view()); Kokkos::deep_copy(fieldData1_q1_h, fieldData1_q1.get_static_view()); Kokkos::deep_copy(fieldData2_q1_h, fieldData2_q1.get_static_view()); for (unsigned int cell = 0; cell < fieldData1_q1.extent(0); ++cell) { for (unsigned int pt = 0; pt < fieldData1_q1.extent(1); pt++) { if (enable_tangents) { TEST_EQUALITY(fieldData1_q1_h(cell, pt).val(), 123.0 + myRank); TEST_EQUALITY(fieldData1_q1_h(cell, pt).availableSize(), num_tangent); for (int i = 0; i < num_tangent; ++i) TEST_EQUALITY(fieldData1_q1_h(cell, pt).dx(i), 0.123 + myRank + i); } else { TEST_EQUALITY(fieldData1_q1_h(cell, pt), 123.0 + myRank); TEST_EQUALITY(fieldData1_q1_h(cell, pt).availableSize(), 0); } } } for (unsigned int cell = 0; cell < fieldData2_q1.extent(0); ++cell) { for (unsigned int pt = 0; pt < fieldData2_q1.extent(1); pt++) { if (enable_tangents) { TEST_EQUALITY(fieldData2_q1_h(cell, pt).val(), 123.0 + myRank); TEST_EQUALITY(fieldData2_q1_h(cell, pt).availableSize(), num_tangent); for (int i = 0; i < num_tangent; ++i) { TEST_EQUALITY(fieldData2_q1_h(cell, pt).dx(i), 0.123 + myRank + i); TEST_EQUALITY(fieldData2_q1_h(cell, pt).dx(i), 0.123 + myRank + i); } } else { TEST_EQUALITY(fieldData2_q1_h(cell, pt), 123.0 + myRank); TEST_EQUALITY(fieldData2_q1_h(cell, pt).availableSize(), 0); } } } } { PHX::MDField fieldData_qedge1(fieldName_qedge1, basis_qedge1->functional); fm.getFieldData(fieldData_qedge1); auto fieldData_qedge1_h = Kokkos::create_mirror_view(fieldData_qedge1.get_static_view()); Kokkos::deep_copy(fieldData_qedge1_h, fieldData_qedge1.get_static_view()); for (unsigned int cell = 0; cell < fieldData_qedge1.extent(0); ++cell) { for (unsigned int pt = 0; pt < fieldData_qedge1.extent(1); ++pt) { if (enable_tangents) { TEST_EQUALITY(fieldData_qedge1_h(cell, pt).val(), 123.0 + myRank); TEST_EQUALITY(fieldData_qedge1_h(cell, pt).availableSize(), num_tangent); for (int i = 0; i < num_tangent; ++i) TEST_EQUALITY(fieldData_qedge1_h(cell, pt).dx(i), 0.123 + myRank + i); } else { TEST_EQUALITY(fieldData_qedge1_h(cell, pt), 123.0 + myRank); TEST_EQUALITY(fieldData_qedge1_h(cell, pt).availableSize(), 0); } } } } } Teuchos::RCP buildBasis(std::size_t worksetSize, const std::string &basisName) { Teuchos::RCP topo = Teuchos::rcp(new shards::CellTopology(shards::getCellTopologyData>())); panzer::CellData cellData(worksetSize, topo); return Teuchos::rcp(new panzer::PureBasis(basisName, 1, cellData)); } Teuchos::RCP buildMesh(int elemX, int elemY) { Teuchos::RCP pl = rcp(new Teuchos::ParameterList); pl->set("X Blocks", 1); pl->set("Y Blocks", 1); pl->set("X Elements", elemX); pl->set("Y Elements", elemY); panzer_stk::SquareQuadMeshFactory factory; factory.setParameterList(pl); Teuchos::RCP mesh = factory.buildUncommitedMesh(MPI_COMM_WORLD); factory.completeMeshConstruction(*mesh, MPI_COMM_WORLD); return mesh; } void testInitialization(const Teuchos::RCP &ipb) { // Physics block ipb->setName("test physics"); { Teuchos::ParameterList &p = ipb->sublist("a"); p.set("Type", "Energy"); p.set("Prefix", ""); p.set("Model ID", "solid"); p.set("Basis Type", "HGrad"); p.set("Basis Order", 1); p.set("Integration Order", 1); } { Teuchos::ParameterList &p = ipb->sublist("b"); p.set("Type", "Energy"); p.set("Prefix", "ION_"); p.set("Model ID", "solid"); p.set("Basis Type", "HCurl"); p.set("Basis Order", 1); p.set("Integration Order", 1); } } }