// @HEADER // ***************************************************************************** // Panzer: A partial differential equation assembly // engine for strongly coupled complex multiphysics systems // // Copyright 2011 NTESS and the Panzer contributors. // SPDX-License-Identifier: BSD-3-Clause // ***************************************************************************** // @HEADER /////////////////////////////////////////////////////////////////////////////// // // Include Files // /////////////////////////////////////////////////////////////////////////////// // C++ #include #include #include // Epetra #include "Epetra_MpiComm.h" // Kokkos #include "Kokkos_View_Fad.hpp" // Panzer #include "PanzerAdaptersSTK_config.hpp" #include "Panzer_BasisIRLayout.hpp" #include "Panzer_BlockedDOFManager.hpp" #include "Panzer_BlockedEpetraLinearObjFactory.hpp" #include "Panzer_BlockedTpetraLinearObjFactory.hpp" #include "Panzer_DOFManager.hpp" #include "Panzer_EpetraVector_ReadOnly_GlobalEvaluationData.hpp" #include "Panzer_Evaluator_WithBaseImpl.hpp" #include "Panzer_FieldManagerBuilder.hpp" #include "Panzer_GatherOrientation.hpp" #include "Panzer_PureBasis.hpp" #include "Panzer_STKConnManager.hpp" #include "Panzer_STK_Interface.hpp" #include "Panzer_STK_SetupUtilities.hpp" #include "Panzer_STK_SquareQuadMeshFactory.hpp" #include "Panzer_STK_Version.hpp" #include "Panzer_Workset.hpp" // Teuchos #include "Teuchos_DefaultMpiComm.hpp" #include "Teuchos_GlobalMPISession.hpp" #include "Teuchos_OpaqueWrapper.hpp" #include "Teuchos_RCP.hpp" #include "Teuchos_TimeMonitor.hpp" #include "Teuchos_UnitTestHarness.hpp" // Thyra #include "Thyra_ProductVectorBase.hpp" #include "Thyra_VectorStdOps.hpp" // user_app #include "user_app_EquationSetFactory.hpp" typedef double ScalarT; using LocalOrdinalT = panzer::LocalOrdinal; using GlobalOrdinalT = panzer::GlobalOrdinal; typedef panzer::TpetraNodeType NodeT; typedef Tpetra::Vector VectorType; typedef Tpetra::Operator OperatorType; typedef Tpetra::CrsMatrix CrsMatrixType; typedef Tpetra::CrsGraph CrsGraphType; typedef Tpetra::Map MapType; typedef Tpetra::Import ImportType; typedef Tpetra::Export ExportType; typedef Thyra::TpetraLinearOp ThyraLinearOp; typedef panzer::BlockedTpetraLinearObjContainer BLOC; typedef panzer::BlockedTpetraLinearObjFactory BLOFact; namespace panzer { Teuchos::RCP buildBasis(std::size_t worksetSize,const std::string & basisName); void testInitialization(const Teuchos::RCP& ipb); Teuchos::RCP buildMesh(int elemX,int elemY); void testGatherScatter(const bool enable_tangents, Teuchos::FancyOStream& out, bool& success); // Test without tangent fields in gather evaluator TEUCHOS_UNIT_TEST(epetra_assembly, gather_solution_no_tangents) { testGatherScatter(false,out,success); } // Test with tangent fields in gather evaluator TEUCHOS_UNIT_TEST(epetra_assembly, gather_solution_tangents) { testGatherScatter(true,out,success); } // enable_tangents determines whether tangent fields dx/dp are added to gather evaluator. // These are used when computing df/dx*dx/dp with the tangent evaluation type void testGatherScatter(const bool enable_tangents, Teuchos::FancyOStream& out, bool& success) { #ifdef HAVE_MPI Teuchos::RCP eComm = Teuchos::rcp(new Epetra_MpiComm(MPI_COMM_WORLD)); #else Teuchos::RCP eComm = Teuchos::rcp(new Epetra_SerialComm()); #endif int myRank = eComm->MyPID(); int numProcs = eComm->NumProc(); const std::size_t workset_size = 4/numProcs; const std::string fieldName1_q1 = "U"; const std::string fieldName2_q1 = "V"; const std::string fieldName_qedge1 = "B"; const int num_tangent = enable_tangents ? 5 : 0; Teuchos::RCP mesh = buildMesh(2,2); // build input physics block Teuchos::RCP basis_q1 = buildBasis(workset_size,"Q1"); Teuchos::RCP basis_qedge1 = buildBasis(workset_size,"QEdge1"); Teuchos::RCP ipb = Teuchos::parameterList(); testInitialization(ipb); const int default_int_order = 1; std::string eBlockID = "eblock-0_0"; Teuchos::RCP eqset_factory = Teuchos::rcp(new user_app::MyFactory); panzer::CellData cellData(workset_size,mesh->getCellTopology("eblock-0_0")); Teuchos::RCP gd = panzer::createGlobalData(); Teuchos::RCP physicsBlock = Teuchos::rcp(new PhysicsBlock(ipb,eBlockID,default_int_order,cellData,eqset_factory,gd,false)); Teuchos::RCP > work_sets = panzer_stk::buildWorksets(*mesh,physicsBlock->elementBlockID(), physicsBlock->getWorksetNeeds()); TEST_EQUALITY(work_sets->size(),1); // build connection manager and field manager const Teuchos::RCP conn_manager = Teuchos::rcp(new panzer_stk::STKConnManager(mesh)); Teuchos::RCP dofManager = Teuchos::rcp(new panzer::DOFManager(conn_manager,MPI_COMM_WORLD)); dofManager->addField(fieldName1_q1,Teuchos::rcp(new panzer::Intrepid2FieldPattern(basis_q1->getIntrepid2Basis()))); dofManager->addField(fieldName2_q1,Teuchos::rcp(new panzer::Intrepid2FieldPattern(basis_q1->getIntrepid2Basis()))); dofManager->addField(fieldName_qedge1,Teuchos::rcp(new panzer::Intrepid2FieldPattern(basis_qedge1->getIntrepid2Basis()))); std::vector fieldOrder; fieldOrder.push_back(fieldName1_q1); fieldOrder.push_back(fieldName_qedge1); fieldOrder.push_back(fieldName2_q1); dofManager->setFieldOrder(fieldOrder); // dofManager->setOrientationsRequired(true); dofManager->buildGlobalUnknowns(); // setup linear object factory ///////////////////////////////////////////////////////////// Teuchos::RCP > tComm = Teuchos::rcp(new Teuchos::MpiComm(MPI_COMM_WORLD)); Teuchos::RCP > e_lof = Teuchos::rcp(new BlockedEpetraLinearObjFactory(tComm.getConst(),dofManager)); Teuchos::RCP > lof = e_lof; Teuchos::RCP loc = e_lof->buildGhostedLinearObjContainer(); e_lof->initializeGhostedContainer(LinearObjContainer::X,*loc); Teuchos::RCP e_loc = Teuchos::rcp_dynamic_cast(loc); Teuchos::RCP > x_vec = e_loc->get_x_th(); Thyra::assign(x_vec.ptr(),123.0+myRank); // Set up the tangent data containers. std::vector> tangentContainers; if (enable_tangents) { using Teuchos::RCP; using Teuchos::rcp_dynamic_cast; using Thyra::VectorBase; using EVROGED = panzer::EpetraVector_ReadOnly_GlobalEvaluationData; using ROVGED = panzer::ReadOnlyVector_GlobalEvaluationData; for (int i(0); i < num_tangent; ++i) { RCP tangentContainer = e_lof->buildReadOnlyDomainContainer(); auto tanContainerEpetra = rcp_dynamic_cast(tangentContainer); RCP> tanVecOwned = tanContainerEpetra->getOwnedVector()->clone_v(); assign(tanVecOwned.ptr(), 0.123 + myRank + i); tanContainerEpetra->setOwnedVector(tanVecOwned); RCP> tanVecGhosted = tanContainerEpetra->getGhostedVector(); assign(tanVecGhosted.ptr(), 0.123 + myRank + i); tangentContainers.push_back(tangentContainer); } // end loop over the tangents } // end if (enable_tangents) // setup field manager, add evaluator under test ///////////////////////////////////////////////////////////// PHX::FieldManager fm; std::vector derivative_dimensions; derivative_dimensions.push_back(12); fm.setKokkosExtendedDataTypeDimensions(derivative_dimensions); std::vector tan_derivative_dimensions; if (enable_tangents) tan_derivative_dimensions.push_back(num_tangent); else tan_derivative_dimensions.push_back(0); fm.setKokkosExtendedDataTypeDimensions(tan_derivative_dimensions); Teuchos::RCP evalField_q1, evalField_qedge1; { using Teuchos::RCP; using Teuchos::rcp; RCP > names = rcp(new std::vector); names->push_back(fieldName1_q1); names->push_back(fieldName2_q1); Teuchos::ParameterList pl; pl.set("Basis", basis_q1); pl.set("DOF Names",names); pl.set("Indexer Names",names); Teuchos::RCP > evaluator = lof->buildGather(pl); TEST_EQUALITY(evaluator->evaluatedFields().size(),2); fm.registerEvaluator(evaluator); fm.requireField(*evaluator->evaluatedFields()[0]); } { using Teuchos::RCP; using Teuchos::rcp; RCP > names = rcp(new std::vector); names->push_back(fieldName_qedge1); Teuchos::ParameterList pl; pl.set("Basis", basis_qedge1); pl.set("DOF Names",names); pl.set("Indexer Names",names); Teuchos::RCP > evaluator = lof->buildGather(pl); TEST_EQUALITY(evaluator->evaluatedFields().size(),1); fm.registerEvaluator(evaluator); fm.requireField(*evaluator->evaluatedFields()[0]); } { using Teuchos::RCP; using Teuchos::rcp; RCP > names = rcp(new std::vector); names->push_back(fieldName1_q1); names->push_back(fieldName2_q1); Teuchos::ParameterList pl; pl.set("Basis", basis_q1); pl.set("DOF Names",names); pl.set("Indexer Names",names); Teuchos::RCP > evaluator = lof->buildGather(pl); TEST_EQUALITY(evaluator->evaluatedFields().size(),2); fm.registerEvaluator(evaluator); fm.requireField(*evaluator->evaluatedFields()[0]); } { using Teuchos::RCP; using Teuchos::rcp; RCP > names = rcp(new std::vector); names->push_back(fieldName_qedge1); Teuchos::ParameterList pl; pl.set("Basis", basis_qedge1); pl.set("DOF Names",names); pl.set("Indexer Names",names); Teuchos::RCP > evaluator = lof->buildGather(pl); TEST_EQUALITY(evaluator->evaluatedFields().size(),1); fm.registerEvaluator(evaluator); fm.requireField(*evaluator->evaluatedFields()[0]); } { using Teuchos::RCP; using Teuchos::rcp; RCP > names = rcp(new std::vector); names->push_back(fieldName1_q1); names->push_back(fieldName2_q1); Teuchos::ParameterList pl; pl.set("Basis", basis_q1); pl.set("DOF Names",names); pl.set("Indexer Names",names); if (enable_tangents) { RCP > > tangent_names = rcp(new std::vector< std::vector >(2)); for (int i=0; i > evaluator = lof->buildGather(pl); TEST_EQUALITY(evaluator->evaluatedFields().size(),2); fm.registerEvaluator(evaluator); fm.requireField(*evaluator->evaluatedFields()[0]); } { using Teuchos::RCP; using Teuchos::rcp; RCP > names = rcp(new std::vector); names->push_back(fieldName_qedge1); Teuchos::ParameterList pl; pl.set("Basis", basis_qedge1); pl.set("DOF Names",names); pl.set("Indexer Names",names); if (enable_tangents) { RCP > > tangent_names = rcp(new std::vector< std::vector >(1)); for (int i=0; i > evaluator = lof->buildGather(pl); TEST_EQUALITY(evaluator->evaluatedFields().size(),1); fm.registerEvaluator(evaluator); fm.requireField(*evaluator->evaluatedFields()[0]); } if (enable_tangents) { for (int i=0; i > names = rcp(new std::vector); RCP > tangent_names = rcp(new std::vector); names->push_back(fieldName1_q1); names->push_back(fieldName2_q1); { std::stringstream ss1, ss2; ss1 << fieldName1_q1 << " Tangent " << i; ss2 << fieldName2_q1 << " Tangent " << i; tangent_names->push_back(ss1.str()); tangent_names->push_back(ss2.str()); } Teuchos::ParameterList pl; pl.set("Basis", basis_q1); pl.set("DOF Names",tangent_names); pl.set("Indexer Names",names); { std::stringstream ss; ss << "Tangent Container " << i; pl.set("Global Data Key", ss.str()); } Teuchos::RCP > evaluator = lof->buildGatherTangent(pl); TEST_EQUALITY(evaluator->evaluatedFields().size(),2); fm.registerEvaluator(evaluator); } for (int i=0; i > names = rcp(new std::vector); RCP > tangent_names = rcp(new std::vector); names->push_back(fieldName_qedge1); { std::stringstream ss; ss << fieldName_qedge1 << " Tangent " << i; tangent_names->push_back(ss.str()); } Teuchos::ParameterList pl; pl.set("Basis", basis_qedge1); pl.set("DOF Names",tangent_names); pl.set("Indexer Names",names); { std::stringstream ss; ss << "Tangent Container " << i; pl.set("Global Data Key", ss.str()); } Teuchos::RCP > evaluator = lof->buildGatherTangent(pl); TEST_EQUALITY(evaluator->evaluatedFields().size(),1); fm.registerEvaluator(evaluator); } } panzer::Traits::SD sd; fm.postRegistrationSetup(sd); panzer::Traits::PED ped; ped.gedc->addDataObject("Solution Gather Container", loc); if (enable_tangents) { for (int i(0); i < num_tangent; ++i) { std::stringstream ss; ss << "Tangent Container " << i; ped.gedc->addDataObject(ss.str(), tangentContainers[i]); } } fm.preEvaluate(ped); fm.preEvaluate(ped); fm.preEvaluate(ped); // run tests ///////////////////////////////////////////////////////////// panzer::Workset & workset = (*work_sets)[0]; workset.alpha = 0.0; workset.beta = 2.0; // derivatives multiplied by 2 workset.time = 0.0; workset.evaluate_transient_terms = false; fm.evaluateFields(workset); fm.evaluateFields(workset); fm.evaluateFields(workset); // test Residual fields { PHX::MDField fieldData1_q1(fieldName1_q1,basis_q1->functional); PHX::MDField fieldData2_q1(fieldName2_q1,basis_qedge1->functional); fm.getFieldData(fieldData1_q1); fm.getFieldData(fieldData2_q1); TEST_EQUALITY(fieldData1_q1.extent(0),Teuchos::as(4/numProcs)); TEST_EQUALITY(fieldData1_q1.extent(1),4); TEST_EQUALITY(fieldData2_q1.extent(0),Teuchos::as(4/numProcs)); TEST_EQUALITY(fieldData2_q1.extent(1),4); TEST_EQUALITY(fieldData1_q1.size(),Teuchos::as(4*4/numProcs)); TEST_EQUALITY(fieldData2_q1.size(),Teuchos::as(4*4/numProcs)); auto fieldData1_q1_h = Kokkos::create_mirror_view(fieldData1_q1.get_static_view()); auto fieldData2_q1_h = Kokkos::create_mirror_view(fieldData2_q1.get_static_view()); Kokkos::deep_copy(fieldData1_q1_h, fieldData1_q1.get_static_view()); Kokkos::deep_copy(fieldData2_q1_h, fieldData2_q1.get_static_view()); for(unsigned int i=0;i fieldData_qedge1(fieldName_qedge1,basis_qedge1->functional); fm.getFieldData(fieldData_qedge1); auto fieldData_qedge1_h = Kokkos::create_mirror_view(fieldData_qedge1.get_static_view()); Kokkos::deep_copy(fieldData_qedge1_h, fieldData_qedge1.get_static_view()); TEST_EQUALITY(fieldData_qedge1.extent(0),Teuchos::as(4/numProcs)); TEST_EQUALITY(fieldData_qedge1.extent(1),4); TEST_EQUALITY(fieldData_qedge1.size(),Teuchos::as(4*4/numProcs)); for(unsigned int cell=0;cell fieldData1_q1(fieldName1_q1,basis_q1->functional); PHX::MDField fieldData2_q1(fieldName2_q1,basis_qedge1->functional); fm.getFieldData(fieldData1_q1); fm.getFieldData(fieldData2_q1); auto fieldData1_q1_h = Kokkos::create_mirror_view(fieldData1_q1.get_static_view()); auto fieldData2_q1_h = Kokkos::create_mirror_view(fieldData2_q1.get_static_view()); Kokkos::deep_copy(fieldData1_q1_h, fieldData1_q1.get_static_view()); Kokkos::deep_copy(fieldData2_q1_h, fieldData2_q1.get_static_view()); for(unsigned int cell=0;cell fieldData_qedge1(fieldName_qedge1,basis_qedge1->functional); fm.getFieldData(fieldData_qedge1); auto fieldData_qedge1_h = Kokkos::create_mirror_view(fieldData_qedge1.get_static_view()); Kokkos::deep_copy(fieldData_qedge1_h, fieldData_qedge1.get_static_view()); for(unsigned int cell=0;cell fieldData1_q1(fieldName1_q1,basis_q1->functional); PHX::MDField fieldData2_q1(fieldName2_q1,basis_qedge1->functional); fm.getFieldData(fieldData1_q1); fm.getFieldData(fieldData2_q1); auto fieldData1_q1_h = Kokkos::create_mirror_view(fieldData1_q1.get_static_view()); auto fieldData2_q1_h = Kokkos::create_mirror_view(fieldData2_q1.get_static_view()); Kokkos::deep_copy(fieldData1_q1_h, fieldData1_q1.get_static_view()); Kokkos::deep_copy(fieldData2_q1_h, fieldData2_q1.get_static_view()); for(unsigned int cell=0;cell fieldData_qedge1(fieldName_qedge1,basis_qedge1->functional); fm.getFieldData(fieldData_qedge1); auto fieldData_qedge1_h = Kokkos::create_mirror_view(fieldData_qedge1.get_static_view()); Kokkos::deep_copy(fieldData_qedge1_h, fieldData_qedge1.get_static_view()); for(unsigned int cell=0;cell buildBasis(std::size_t worksetSize,const std::string & basisName) { Teuchos::RCP topo = Teuchos::rcp(new shards::CellTopology(shards::getCellTopologyData< shards::Quadrilateral<4> >())); panzer::CellData cellData(worksetSize,topo); return Teuchos::rcp(new panzer::PureBasis(basisName,1,cellData)); } Teuchos::RCP buildMesh(int elemX,int elemY) { Teuchos::RCP pl = rcp(new Teuchos::ParameterList); pl->set("X Blocks",1); pl->set("Y Blocks",1); pl->set("X Elements",elemX); pl->set("Y Elements",elemY); panzer_stk::SquareQuadMeshFactory factory; factory.setParameterList(pl); Teuchos::RCP mesh = factory.buildUncommitedMesh(MPI_COMM_WORLD); factory.completeMeshConstruction(*mesh,MPI_COMM_WORLD); return mesh; } void testInitialization(const Teuchos::RCP& ipb) { // Physics block ipb->setName("test physics"); { Teuchos::ParameterList& p = ipb->sublist("a"); p.set("Type","Energy"); p.set("Prefix",""); p.set("Model ID","solid"); p.set("Basis Type","HGrad"); p.set("Basis Order",1); p.set("Integration Order",1); } { Teuchos::ParameterList& p = ipb->sublist("b"); p.set("Type","Energy"); p.set("Prefix","ION_"); p.set("Model ID","solid"); p.set("Basis Type","HCurl"); p.set("Basis Order",1); p.set("Integration Order",1); } } }