Debian Codegraph @

packages/muelu/test/longlong/TwoBillion.cpp

raw
1 // @HEADER
2 // *****************************************************************************
3 // MueLu: A package for multigrid based preconditioning
4 //
5 // Copyright 2012 NTESS and the MueLu contributors.
6 // SPDX-License-Identifier: BSD-3-Clause
7 // *****************************************************************************
8 // @HEADER
9
10 #include <unistd.h>
11 #include <iostream>
12
13 // Teuchos
14 #include <Teuchos_ConfigDefs.hpp>
15 #include <Teuchos_RCP.hpp>
16 #include <Teuchos_ParameterList.hpp>
17 #include <Teuchos_CommandLineProcessor.hpp>
18 #include <Teuchos_GlobalMPISession.hpp>
19 #include <Teuchos_DefaultComm.hpp>
20 #include <Teuchos_XMLParameterListHelpers.hpp>
21 #include <Teuchos_TestingHelpers.hpp>
22 #include <Teuchos_StandardCatchMacros.hpp>
23
24 // Xpetra
25 #include "Xpetra_ConfigDefs.hpp"
26 #include <Xpetra_Map.hpp>
27 #include <Xpetra_MapFactory.hpp>
28 #include <Xpetra_CrsMatrixWrap.hpp>
29 #include <Xpetra_MultiVectorFactory.hpp>
30 #include <Xpetra_VectorFactory.hpp>
31 #include <Xpetra_Parameters.hpp>
32
33 // Galeri
34 #include <Galeri_XpetraParameters.hpp>
35 #include <Galeri_XpetraProblemFactory.hpp>
36 #include <Galeri_XpetraUtils.hpp>
37
38 // MueLu
39 #include <MueLu_ConfigDefs.hpp>
40 #include <MueLu_ParameterListInterpreter.hpp>
41
42 // Belos
43 #ifdef HAVE_MUELU_BELOS
44 #include "BelosConfigDefs.hpp"
45 #include "BelosLinearProblem.hpp"
46 #include "BelosBlockCGSolMgr.hpp"
47 #include "BelosBlockGmresSolMgr.hpp"
48 #include "BelosXpetraAdapter.hpp" // this header defines Belos::XpetraOp()
49 #include "BelosMueLuAdapter.hpp" // this header defines Belos::MueLuOp()
50 #endif
51
52 typedef double Scalar;
53 typedef int LocalOrdinal;
54 typedef long long GlobalOrdinal;
55 typedef Tpetra::KokkosClassic::DefaultNode::DefaultNodeType Node;
56
57 using Xpetra::global_size_t;
58
59 int main(int argc, char *argv[]) {
60 #include "MueLu_UseShortNames.hpp"
61
62 using Teuchos::RCP;
63 using Teuchos::rcp;
64 using Teuchos::TimeMonitor;
65
66 Teuchos::oblackholestream blackhole;
67 Teuchos::GlobalMPISession mpiSession(&argc, &argv, &blackhole);
68
69 bool success = false;
70 bool verbose = true;
71 try {
72 RCP<const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
73 RCP<Teuchos::FancyOStream> out = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
74 out->setOutputToRootOnly(0);
75
76 Teuchos::CommandLineProcessor clp(false);
77 std::string xmlFileName = "TwoBillion.xml";
78 clp.setOption("xml", &xmlFileName, "read parameters from a file");
79 int num_per_proc = 1000;
80 clp.setOption("dpc", &num_per_proc, "DOFs per core");
81
82 switch (clp.parse(argc, argv)) {
83 case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS;
84 case Teuchos::CommandLineProcessor::PARSE_ERROR:
85 case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE;
86 case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
87 }
88
89 int NumProcs = comm->getSize();
90 int MyPID = comm->getRank();
91
92 if (!MyPID) printf("TwoBillion: Running Test\n");
93
94 const long long FIRST_GID = 3000000000L;
95 // const long long FIRST_GID = 0L;
96
97 const long long IndexBase = 0L;
98 // const long long IndexBase = 3000000000L;
99
100 global_size_t NumGlobalElements = NumProcs * num_per_proc;
101
102 // Create Map w/ GIDs starting at > 2 billion
103 RCP<const Map> map;
104 RCP<CrsMatrix> Acrs;
105 Teuchos::Array<GlobalOrdinal> mygids(num_per_proc);
106
107 for (int i = 0; i < num_per_proc; i++)
108 mygids[i] = FIRST_GID + MyPID * num_per_proc + i;
109
110 for (int i = 0; i < NumProcs; ++i) {
111 if (i == MyPID)
112 std::cout << "pid " << i << " : 1st GID = " << mygids[0] << std::endl;
113 }
114
115 // for(int i=0;i<num_per_proc;i++)
116 // printf("[%d] mygids[%d] = %lld\n",MyPID,i,mygids[i]);
117
118 map = MapFactory::Build(Xpetra::UseTpetra, Teuchos::OrdinalTraits<global_size_t>::invalid(), mygids(), IndexBase, comm);
119
120 // RCP<Teuchos::FancyOStream> fox = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
121 // fox->setOutputToRootOnly(-1);
122 // map->describe(*fox,Teuchos::VERB_EXTREME);
123
124 // Create 1D Laplacian w/ GIDs starting at > 2 billion
125 Teuchos::Array<Scalar> myvals(3);
126 Teuchos::Array<GlobalOrdinal> mycols(3);
127 Teuchos::ArrayView<Scalar> ValView;
128 Teuchos::ArrayView<GlobalOrdinal> ColView;
129
130 Acrs = CrsMatrixFactory::Build(map, 3);
131 for (int i = 0; i < num_per_proc; i++) {
132 if (mygids[i] == FIRST_GID) {
133 mycols[0] = mygids[i];
134 myvals[0] = 2;
135 mycols[1] = mygids[i] + 1;
136 myvals[1] = -1;
137 ValView = myvals.view(0, 2);
138 ColView = mycols.view(0, 2);
139 // printf("[%d %lld] cols %lld %lld\n",MyPID,mygids[i],mycols[0],mycols[1]);
140 } else if (mygids[i] == FIRST_GID + (long long)NumGlobalElements - 1) {
141 mycols[0] = mygids[i] - 1;
142 myvals[0] = -1;
143 mycols[1] = mygids[i];
144 myvals[1] = 2;
145 ValView = myvals.view(0, 2);
146 ColView = mycols.view(0, 2);
147 // printf("[%d %lld] cols %lld %lld\n",MyPID,mygids[i],mycols[0],mycols[1]);
148 } else {
149 mycols[0] = mygids[i] - 1;
150 myvals[0] = -1;
151 mycols[1] = mygids[i];
152 myvals[1] = -2;
153 mycols[2] = mygids[i] + 1;
154 myvals[1] = -1;
155 ValView = myvals();
156 ColView = mycols();
157 // printf("[%d %lld] cols %lld %lld %lld\n",MyPID,mygids[i],mycols[0],mycols[1],mycols[2]);
158 }
159 Acrs->insertGlobalValues(mygids[i], ColView, ValView);
160 }
161 Acrs->fillComplete();
162
163 RCP<Matrix> A = rcp(new CrsMatrixWrap(Acrs));
164
165 RCP<MultiVector> nullspace = MultiVectorFactory::Build(map, 1);
166 nullspace->putScalar(Teuchos::ScalarTraits<SC>::one());
167 RCP<MultiVector> coordinates = MultiVectorFactory::Build(map, 1);
168 Teuchos::ArrayRCP<Scalar> coordVals = coordinates->getDataNonConst(0);
169 double h = 1.0 / NumGlobalElements;
170 for (LocalOrdinal i = 0; i < num_per_proc; ++i)
171 coordVals[i] = MyPID * num_per_proc * h + i * h;
172 coordVals = Teuchos::null;
173
174 Teuchos::ParameterList paramList;
175 Teuchos::updateParametersFromXmlFileAndBroadcast(xmlFileName, Teuchos::Ptr<Teuchos::ParameterList>(&paramList), *comm);
176
177 RCP<HierarchyManager> mueLuFactory;
178 mueLuFactory = rcp(new ParameterListInterpreter(xmlFileName, *comm));
179
180 RCP<Hierarchy> H;
181 H = mueLuFactory->CreateHierarchy();
182 H->GetLevel(0)->Set("A", A);
183 H->GetLevel(0)->Set("Nullspace", nullspace);
184 H->GetLevel(0)->Set("Coordinates", coordinates);
185 mueLuFactory->SetupHierarchy(*H);
186
187 //
188 //
189 // SOLVE
190 //
191 //
192
193 // Define X, B
194 RCP<MultiVector> X = MultiVectorFactory::Build(map, 1);
195 RCP<MultiVector> B = MultiVectorFactory::Build(map, 1);
196 Teuchos::Array<Teuchos::ScalarTraits<SC>::magnitudeType> norms(1);
197
198 X->setSeed(846930886);
199 X->randomize();
200 A->apply(*X, *B, Teuchos::NO_TRANS, (SC)1.0, (SC)0.0);
201 B->norm2(norms);
202 B->scale(1.0 / norms[0]);
203
204 //
205 // Use AMG as a preconditioner in Belos
206 //
207 #ifdef HAVE_MUELU_BELOS
208 // Operator and Multivector type that will be used with Belos
209 typedef MultiVector MV;
210 typedef Belos::OperatorT<MV> OP;
211 H->IsPreconditioner(true);
212
213 // Define Operator and Preconditioner
214 Teuchos::RCP<OP> belosOp = Teuchos::rcp(new Belos::XpetraOp<SC, LO, GO, NO>(A)); // Turns a Xpetra::Operator object into a Belos operator
215 Teuchos::RCP<OP> belosPrec = Teuchos::rcp(new Belos::MueLuOp<SC, LO, GO, NO>(H)); // Turns a MueLu::Hierarchy object into a Belos operator
216
217 // Construct a Belos LinearProblem object
218 RCP<Belos::LinearProblem<SC, MV, OP> > belosProblem = rcp(new Belos::LinearProblem<SC, MV, OP>(belosOp, X, B));
219 belosProblem->setLeftPrec(belosPrec);
220
221 bool set = belosProblem->setProblem();
222 if (set == false) {
223 if (comm->getRank() == 0)
224 std::cout << std::endl
225 << "ERROR: Belos::LinearProblem failed to set up correctly!" << std::endl;
226 return EXIT_FAILURE;
227 }
228
229 // Belos parameter list
230 int maxIts = 100;
231 double optTol = 1e-8;
232 Teuchos::ParameterList belosList;
233 belosList.set("Maximum Iterations", maxIts); // Maximum number of iterations allowed
234 belosList.set("Convergence Tolerance", optTol); // Relative convergence tolerance requested
235 // belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::TimingDetails + Belos::StatusTestDetails);
236 belosList.set("Verbosity", Belos::Errors + Belos::Warnings + Belos::StatusTestDetails);
237 belosList.set("Output Frequency", 1);
238 belosList.set("Output Style", Belos::Brief);
239
240 // Create an iterative solver manager
241 RCP<Belos::SolverManager<SC, MV, OP> > solver = rcp(new Belos::BlockCGSolMgr<SC, MV, OP>(belosProblem, rcp(&belosList, false)));
242
243 // Perform solve
244 Belos::ReturnType ret = Belos::Unconverged;
245 try {
246 ret = solver->solve();
247
248 // Get the number of iterations for this solve.
249 if (comm->getRank() == 0)
250 std::cout << "Number of iterations performed for this solve: " << solver->getNumIters() << std::endl;
251
252 if (solver->getNumIters() > 6) {
253 if (comm->getRank() == 0) std::cout << std::endl
254 << "ERROR: Belos did not converge! " << std::endl;
255 return (EXIT_FAILURE);
256 }
257
258 // Compute actual residuals.
259 int numrhs = 1;
260 std::vector<double> actual_resids(numrhs); // TODO: double?
261 std::vector<double> rhs_norm(numrhs);
262 RCP<MultiVector> resid = MultiVectorFactory::Build(map, numrhs);
263
264 typedef Belos::OperatorTraits<SC, MV, OP> OPT;
265 typedef Belos::MultiVecTraits<SC, MV> MVT;
266
267 OPT::Apply(*belosOp, *X, *resid);
268 MVT::MvAddMv(-1.0, *resid, 1.0, *B, *resid);
269 MVT::MvNorm(*resid, actual_resids);
270 MVT::MvNorm(*B, rhs_norm);
271 *out << "---------- Actual Residuals (normalized) ----------" << std::endl
272 << std::endl;
273 for (int i = 0; i < numrhs; i++) {
274 double actRes = actual_resids[i] / rhs_norm[i];
275 *out << "Problem " << i << " : \t" << actRes << std::endl;
276 // if (actRes > tol) { badRes = true; }
277 }
278
279 } // try
280
281 catch (...) {
282 if (comm->getRank() == 0)
283 std::cout << std::endl
284 << "ERROR: Belos threw an error! " << std::endl;
285 }
286
287 success = (ret == Belos::Converged);
288 // Check convergence
289 if (success) {
290 if (comm->getRank() == 0) std::cout << std::endl
291 << "SUCCESS: Belos converged!" << std::endl;
292 } else {
293 if (comm->getRank() == 0) std::cout << std::endl
294 << "ERROR: Belos did not converge! " << std::endl;
295 }
296 #endif // HAVE_MUELU_BELOS
297 }
298 TEUCHOS_STANDARD_CATCH_STATEMENTS(verbose, std::cerr, success);
299 return (success ? EXIT_SUCCESS : EXIT_FAILURE);
300 }