Debian Codegraph @

packages/intrepid2/perf-test/Sierra/SierraGradProjection.cpp

raw
1 // @HEADER
2 // *****************************************************************************
3 // Intrepid2 Package
4 //
5 // Copyright 2007 NTESS and the Intrepid2 contributors.
6 // SPDX-License-Identifier: BSD-3-Clause
7 // *****************************************************************************
8 // @HEADER
9 #include "Teuchos_DefaultComm.hpp"
10 #include "Teuchos_GlobalMPISession.hpp"
11 #include "Teuchos_StackedTimer.hpp"
12 #include "Teuchos_TimeMonitor.hpp"
13
14 #include "Intrepid2_ArrayTools.hpp"
15 #include "Intrepid2_CellGeometry.hpp"
16 #include "Intrepid2_CellTools.hpp"
17 #include "Intrepid2_FunctionSpaceTools.hpp"
18
19 #include "Intrepid2_HGRAD_HEX_C1_FEM.hpp"
20
21 #include "Teuchos_Array.hpp"
22 #include "Teuchos_UnitTestRepository.hpp"
23
24 #include "Kokkos_Core.hpp"
25
26 #include <fstream>
27
28 const int numFields = 8;
29 const int numNodes = 8;
30 const int numDims = 3;
31 const int numIntg = 8;
32
33 using DeviceType = Kokkos::DefaultExecutionSpace::device_type;
34
35 int worksetSize = 512;
36
37 Kokkos::DynRankView<double, DeviceType> points; // (P,D) or (C,P,D), allocated/initialized in main below
38 Kokkos::DynRankView<double, DeviceType> gradient; // (C,F,P,D), allocated/initialized in main below
39
40 Kokkos::DynRankView<double, DeviceType> jacobian;
41 Kokkos::DynRankView<double, DeviceType> jacobian_inv;
42 Kokkos::DynRankView<double, DeviceType> tempGrad;
43
44 shards::CellTopology cell_topology = shards::getCellTopologyData<shards::Hexahedron<8> >();
45
46 using BasisFamily = Intrepid2::DerivedNodalBasisFamily<DeviceType>;
47 BasisFamily::BasisPtr basis;
48
49 /** \brief Create a uniform Cartesian mesh, with origin at 0, and domain extents and mesh widths that can be different in different coordinate dimensions.
50 \param [in] domainExtents - array specifying the extent of the domain in each coordinate dimension.
51 \param [in] gridCellCounts - array specifying the number of cells in each coordinate dimension.
52 \return a uniform Cartesion mesh, with origin at 0, with the specified domain extents and grid cell counts.
53 */
54 template<class PointScalar, int spaceDim, typename DeviceType>
55 inline Intrepid2::CellGeometry<PointScalar,spaceDim,DeviceType> uniformCartesianMesh(const Kokkos::Array<PointScalar,spaceDim> &domainExtents,
56 const Kokkos::Array<int,spaceDim> &gridCellCounts)
57 {
58 Kokkos::Array<PointScalar,spaceDim> origin;
59 for (int d=0; d<spaceDim; d++)
60 {
61 origin[d] = 0.0;
62 }
63
64 using CellGeometry = ::Intrepid2::CellGeometry<PointScalar, spaceDim, DeviceType>;
65 const auto NO_SUBDIVISION = CellGeometry::NO_SUBDIVISION;
66 const auto HYPERCUBE_NODE_ORDER_CLASSIC_SHARDS = CellGeometry::HYPERCUBE_NODE_ORDER_CLASSIC_SHARDS;
67
68 return CellGeometry(origin, domainExtents, gridCellCounts, NO_SUBDIVISION, HYPERCUBE_NODE_ORDER_CLASSIC_SHARDS);
69 }
70
71 void intrepid2_gradient_operator(const int n, const double* coords, double* grad, double* det)
72 {
73 using FunctionSpaceTools = Intrepid2::FunctionSpaceTools<DeviceType>;
74 using CellToolsReal2 = Intrepid2::CellTools<DeviceType>;
75
76 using FieldContainer = Kokkos::DynRankView<double, DeviceType>;
77
78 const FieldContainer coordVec(const_cast<double*>(coords), n, numNodes, numDims);
79
80 FieldContainer jacobian_det(det, n, numIntg);
81
82 // compute the jacobian, its inverse, and its determinant
83 CellToolsReal2::setJacobian(jacobian, points, coordVec, basis);
84 CellToolsReal2::setJacobianInv(jacobian_inv, jacobian);
85 CellToolsReal2::setJacobianDet(jacobian_det, jacobian);
86
87 // compute the Basis Function Gradients
88 FunctionSpaceTools::HGRADtransformGRAD(tempGrad, jacobian_inv, gradient);
89
90 //
91 // Transform gradient operator from the intrepid order: ( Element, Node Number, Intg Point, Component)
92 // to index order consistent with the rest of Sierra: ( Element, Intg Point, Node Number, Component)
93 //
94 FieldContainer gradOpContainer(grad, n, numIntg, numNodes, numDims);
95 FieldContainer tempGradLocal = tempGrad; // local View to avoid CUDA complaints.
96 Kokkos::parallel_for("reorder gradients", n,
97 KOKKOS_LAMBDA(const int &ielem)
98 {
99 for (int ip = 0; ip < numIntg; ++ip) {
100 for (int inode = 0; inode < numNodes; ++inode) {
101 for (int d = 0; d<numDims; ++d) {
102 gradOpContainer(ielem, ip, inode, d) = tempGradLocal(ielem, inode, ip, d);
103 }
104 }
105 }
106 });
107 }
108
109 int main(int argc, char *argv[])
110 {
111 int numInvocations = 10000;
112
113 Teuchos::RCP<Teuchos::StackedTimer> stacked_timer;
114
115 std::string test_name = "Intrepid2 Sierra Test";
116
117 {
118 // Note that the dtor for GlobalMPISession will call Kokkos::finalize_all() but ctor does not call Kokkos::initialize()...
119 Teuchos::GlobalMPISession mpiSession(&argc, &argv);
120 Kokkos::initialize(argc,argv);
121 Teuchos::UnitTestRepository::setGloballyReduceTestResult(true);
122
123 Teuchos::RCP<Teuchos::FancyOStream> out = Teuchos::rcp(new Teuchos::FancyOStream(Teuchos::rcpFromRef(std::cout)));
124 auto comm = Teuchos::DefaultComm<int>::getComm();
125
126 {
127 Teuchos::CommandLineProcessor cmdp(false,true); // false: don't throw exceptions; true: do return errors for unrecognized options
128
129 cmdp.setOption("numInvocations", &numInvocations, "How many times to run the main loop");
130 cmdp.setOption("worksetSize", &worksetSize, "The size of the workset - how many cells to process at a time");
131
132 if (cmdp.parse(argc,argv) != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL)
133 {
134 return -1;
135 }
136 }
137
138 {
139 stacked_timer = rcp(new Teuchos::StackedTimer(test_name.c_str()));
140 Teuchos::RCP<Teuchos::FancyOStream> verbose_out = Teuchos::rcp(new Teuchos::FancyOStream(Teuchos::rcpFromRef(std::cout)));
141 verbose_out->setShowProcRank(true);
142 stacked_timer->setVerboseOstream(verbose_out);
143 Teuchos::TimeMonitor::setStackedTimer(stacked_timer);
144 }
145
146 Kokkos::DynRankView<double, DeviceType> coords, det;
147 {
148 Teuchos::TimeMonitor setupTimer(*Teuchos::TimeMonitor::getNewTimer(std::string("allocations and other one-time setup")));
149
150 gradient = Kokkos::DynRankView<double, DeviceType>("gradient", worksetSize, numFields, numIntg, numDims);
151 points = Kokkos::DynRankView<double, DeviceType>("points", numIntg, numDims); // (P,D)
152
153 jacobian = Kokkos::DynRankView<double, DeviceType>("jacobian", worksetSize, numIntg, numDims, numDims);
154 jacobian_inv = Kokkos::DynRankView<double, DeviceType>("jacobian_inv", worksetSize, numIntg, numDims, numDims);
155 tempGrad = Kokkos::DynRankView<double, DeviceType>("tempGrad", worksetSize, numNodes, numIntg, numDims);
156
157 basis = Teuchos::rcp( new Intrepid2::Basis_HGRAD_HEX_C1_FEM<DeviceType>() );
158
159 const int cellCount = 25000;
160
161 Kokkos::Array<double,numDims> domainExtents{1.,1.,1.};
162 Kokkos::Array<int,numDims> gridCellCounts{50,50,10}; // 25000 total cells
163
164 auto geometry = uniformCartesianMesh<double, numDims, DeviceType>(domainExtents, gridCellCounts);
165
166 det = Kokkos::DynRankView<double, DeviceType>("det", worksetSize, numIntg);
167 coords = Kokkos::DynRankView<double, DeviceType>("coords", cellCount, numNodes, numDims);
168 Kokkos::parallel_for("initialize coords", cellCount,
169 KOKKOS_LAMBDA(const int &cellOrdinal)
170 {
171 for (int nodeOrdinal=0; nodeOrdinal<numNodes; nodeOrdinal++)
172 {
173 for (int d=0; d<numDims; d++)
174 {
175 coords(cellOrdinal,nodeOrdinal,d) = geometry(cellOrdinal,nodeOrdinal,d);
176 }
177 }
178 });
179 Kokkos::fence();
180 }
181
182 {
183 Teuchos::TimeMonitor intrepid2Timer(*Teuchos::TimeMonitor::getNewTimer(std::string("intrepid2_gradient_operator")));
184 for (int i=0; i<numInvocations; i++)
185 {
186 // this is only approximately what will be happening in Sierra; there, the pointers here will move from one call to the next
187 // corresponding to the cell ordinal. But we start with this; probably that difference will not be significant.
188 intrepid2_gradient_operator(worksetSize, coords.data(), gradient.data(), det.data());
189 }
190 }
191
192 {
193 Teuchos::TimeMonitor dtorTimer(*Teuchos::TimeMonitor::getNewTimer(std::string("post-run cleanup")));
194 gradient = Kokkos::DynRankView<double, DeviceType>();
195 points = Kokkos::DynRankView<double, DeviceType>();
196 basis = Teuchos::null;
197 jacobian = Kokkos::DynRankView<double, DeviceType>();
198 jacobian_inv = Kokkos::DynRankView<double, DeviceType>();
199 tempGrad = Kokkos::DynRankView<double, DeviceType>();
200 }
201
202 stacked_timer->stop(test_name);
203 Teuchos::StackedTimer::OutputOptions options;
204 options.output_fraction = options.output_histogram = options.output_minmax = true;
205 stacked_timer->report(*out, comm, options);
206
207 auto xmlOut = stacked_timer->reportWatchrXML(test_name + ' ' + std::to_string(comm->getSize()) + " ranks", comm);
208 if(xmlOut.length())
209 std::cout << "\nAlso created Watchr performance report " << xmlOut << '\n';
210 }
211
212 return 0;
213 }
214