Debian Codegraph @

packages/nox/test/epetra/MultiPhysics/Problem_Manager.C

raw
1 // @HEADER
2 // *****************************************************************************
3 // NOX: An Object-Oriented Nonlinear Solver Package
4 //
5 // Copyright 2002 NTESS and the NOX contributors.
6 // SPDX-License-Identifier: BSD-3-Clause
7 // *****************************************************************************
8 // @HEADER
9
10 #include "Problem_Manager.H"
11
12 #include "NOX.H"
13 #include "NOX_Epetra.H"
14 #include "NOX_Epetra_DebugTools.H"
15 #include "NOX_Epetra_SchurCoupler.H"
16
17 // Trilinos Objects
18 #ifdef HAVE_MPI
19 #include "Epetra_MpiComm.h"
20 #else
21 #include "Epetra_SerialComm.h"
22 #endif
23 #include "Epetra_Map.h"
24 #include "Epetra_Vector.h"
25 #include "Epetra_IntVector.h"
26 #include "Epetra_RowMatrix.h"
27 #include "Epetra_CrsMatrix.h"
28 #include "Epetra_Map.h"
29 #include "Epetra_LinearProblem.h"
30 #include "AztecOO.h"
31 #include "AztecOO_ConditionNumber.h"
32
33 #include "GenericEpetraProblem.H"
34 #include "Xfer_Operator.H"
35 #include "OffBlock_Manager.H"
36
37 // Headers needed for Coloring
38 #ifdef HAVE_NOX_EPETRAEXT // Use epetraext package in Trilinos
39 #include "Epetra_MapColoring.h"
40 #include "EpetraExt_MapColoring.h"
41 #include "EpetraExt_MapColoringIndex.h"
42 #endif
43
44 // Needed for block inverse operators
45 #include "Ifpack.h"
46
47 // Header for Timing info
48 #include "Epetra_Time.h"
49
50 #ifdef HAVE_MATLAB
51 #include "NOX_Multiphysics_Matlab_Interface.H"
52 #endif
53
54 // Hard-wired switch to turn on/off use of FD Coloring
55 #define USE_FD
56 //#undef USE_FD
57
58 // Simple macro for turning on debug code
59 #undef DEBUG_BLOCKGRAPH
60 #ifdef ENABLE_DEBUG_BLOCKGRAPH
61 #define DEBUG_BLOCKGRAPH(a) a
62 #else
63 #define DEBUG_BLOCKGRAPH(a)
64 #endif
65
66 Problem_Manager::Problem_Manager(Epetra_Comm& comm,
67 bool doOffBlocks_,
68 int numGlobalElements,
69 bool useMatlab_ ) :
70 GenericEpetraProblem(comm, numGlobalElements),
71 problemCount(0),
72 doOffBlocks(doOffBlocks_),
73 useMatlab(useMatlab_)
74 {
75 // Unset doOffBlocks flag if this build does not include the required
76 // EpetraExt library intreface
77 #ifndef HAVE_NOX_EPETRAEXT
78 doOffBlocks = false;
79 #endif
80
81 // Reset composite number of dofs (overwrites base constructor assignment)
82 NumMyNodes = 0;
83
84 // Do all setup in registerComplete after all registrations have been
85 // performed
86 }
87
88 //-----------------------------------------------------------------------------
89
90 Problem_Manager::~Problem_Manager()
91 {
92 }
93
94 //-----------------------------------------------------------------------------
95
96 void
97 Problem_Manager::addProblem(GenericEpetraProblem & problem)
98 {
99 Problems[++problemCount] = Teuchos::rcp( &problem, false );
100 problem.setId(problemCount);
101 problem.setManager(this);
102
103 // Give this problem a name if it doesn't already have one
104 if( problem.getName() == "" )
105 {
106 std::string name = "Problem_";
107 char id_str[4];
108 (void) sprintf(id_str, "%d",problemCount);
109 name += id_str;
110 problem.setName(name);
111 }
112
113 Names[problemCount] = problem.getName();
114 NameLookup[problem.getName()] = problemCount;
115
116 // Keep a running total of dofs for use in constructing composite objects
117 NumMyNodes += problem.NumMyNodes;
118
119 return;
120
121 }
122
123 //-----------------------------------------------------------------------------
124
125 GenericEpetraProblem &
126 Problem_Manager::getProblem( int id_ )
127 {
128 // Get a problem given its unique id
129 Teuchos::RCP<GenericEpetraProblem> problem = Problems[id_];
130
131 if( Teuchos::is_null(problem) )
132 {
133 std::cout << "ERROR: Problem with id --> " << id_ << " not registered with "
134 << "Problem_Manager !!" << std::endl;
135 outputStatus(std::cout);
136
137 throw "Problem_Manager ERROR";
138 }
139 else
140 return *problem;
141 }
142
143 //-----------------------------------------------------------------------------
144
145 GenericEpetraProblem &
146 Problem_Manager::getProblem( std::string name )
147 {
148 // Get a problem given its name
149 map<string, int>::iterator iter = NameLookup.find(name);
150
151 if( iter == NameLookup.end() )
152 {
153 std::cout << "ERROR: Could not find lookup id for Problem --> " << name
154 << std::endl;
155 outputStatus(std::cout);
156
157 throw "Problem_Manager ERROR";
158 }
159 else
160 return getProblem( (*iter).second );
161 }
162
163 //-----------------------------------------------------------------------------
164
165 const NOX::Epetra::Group &
166 Problem_Manager::getProblemSolutionGroup(int id)
167 {
168 // Get a group given its unique id
169 //if( Teuchos::is_null(Solvers[id]->rcpSolver) )
170 if( Teuchos::is_null(Solvers[id]) )
171 {
172 std::cout << "Could not get a valid NOX::Solver::Manager object for id = " << id << std::endl;
173 throw "Problem_Manager ERROR";
174 }
175
176 const NOX::Epetra::Group & const_epetraSolnGrp =
177 dynamic_cast<const NOX::Epetra::Group&>(Solvers[id]->getSolutionGroup());
178
179 return const_epetraSolnGrp;
180 }
181
182 //-----------------------------------------------------------------------------
183
184 const Epetra_Vector &
185 Problem_Manager::getSolutionVec(int id)
186 {
187 //if( Teuchos::is_null(noxEpetraSolvers[id]->rcpSolver) )
188 if( Teuchos::is_null(Solvers[id]) )
189 {
190 std::cout << "Could not get a valid NOX::Solver::Manager object for id = " << id << std::endl;
191 throw "Problem_Manager ERROR";
192 }
193
194 const Epetra_Vector & epetraSolnVec = dynamic_cast<const NOX::Epetra::Vector&>
195 (getProblemSolutionGroup(id).getX()).getEpetraVector();
196
197 return epetraSolnVec;
198 }
199
200 //-----------------------------------------------------------------------------
201
202 NOX::Epetra::Group &
203 Problem_Manager::getGroup(int id_)
204 {
205 // Get a group given its unique id
206 Teuchos::RCP<NOX::Epetra::Group> group = Groups[id_];
207
208 if( Teuchos::is_null(group) )
209 {
210 std::cout << "ERROR: Could not get Group for Problem with id --> " << id_
211 << " !!" << std::endl;
212 throw "Problem_Manager ERROR";
213 }
214 else
215 return *(group.get());
216 }
217
218 //-----------------------------------------------------------------------------
219
220 Teuchos::RCP<Epetra_Vector>
221 Problem_Manager::getCompositeSoln()
222 {
223 if( !compositeSoln.get() ) {
224 std::cout << "ERROR: No valid Composite Solution vector with Problem Manager !!"
225 << std::endl;
226 throw "Problem_Manager ERROR";
227 }
228 return compositeSoln;
229 }
230
231 //-----------------------------------------------------------------------------
232
233 void
234 Problem_Manager::createDependency( std::string nameA, std::string nameB, bool isInterfacial )
235 {
236 // Create a dependence of Problem A equations on Problem B variables
237 int probId_A = (*(NameLookup.find(nameA))).second;
238 int probId_B = (*(NameLookup.find(nameB))).second;
239
240 if( !probId_A || !probId_B )
241 {
242 std::cout << "ERROR: Could not create dependency of \"" << nameA << "\" on \""
243 << nameB << "\" !!" << std::endl;
244 throw "Problem_Manager ERROR";
245 }
246
247 GenericEpetraProblem &probA = *(*(Problems.find(probId_A))).second,
248 &probB = *(*(Problems.find(probId_B))).second;
249
250 createDependency( probA, probB, isInterfacial );
251
252 return;
253 }
254
255 //-----------------------------------------------------------------------------
256
257
258 void
259 Problem_Manager::createDependency( GenericEpetraProblem & problemA,
260 GenericEpetraProblem & problemB,
261 bool isInterfacial )
262 {
263 // Ensure that both problems already exist
264 if( !problemA.getId() || !problemB.getId() ) {
265 std::cout << "ERROR: Invalid dependency since at least one problem is "
266 << "not registered with Problem_Manager !!"
267 << std::endl;
268 throw "Problem_Manager ERROR";
269 }
270
271 if( !isInterfacial )
272 problemA.addTransferOp(problemB);
273
274 problemA.addProblemDependence(problemB);
275
276 return;
277 }
278
279 //-----------------------------------------------------------------------------
280
281 void
282 Problem_Manager::registerParameters(const Teuchos::RCP<Teuchos::ParameterList>& List)
283 {
284 nlParams = List;
285
286 return;
287 }
288
289 //-----------------------------------------------------------------------------
290
291 void
292 Problem_Manager::registerStatusTest(const Teuchos::RCP<NOX::StatusTest::Combo>& comboTest)
293 {
294 statusTest = comboTest;
295
296 return;
297 }
298
299 //-----------------------------------------------------------------------------
300
301 void
302 Problem_Manager::registerComplete()
303 {
304 if(Problems.empty())
305 {
306 std::cout << "ERROR: No problems registered with Problem_Manager !!"
307 << std::endl;
308 throw "Problem_Manager ERROR";
309 }
310
311 if(Teuchos::is_null(nlParams) || Teuchos::is_null(statusTest))
312 {
313 std::cout << "ERROR: No nlParams and/or statusTest registered with "
314 << "Problem_Manager !!" << std::endl;
315 throw "Problem_Manager ERROR";
316 }
317
318 // Make sure everything is starting clean
319 assert(Groups.empty() && Interfaces.empty() && Solvers.empty()
320 && ProblemToCompositeIndices.empty());
321
322 // Iterate over each problem and construct the necessary objects
323
324 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator iter = Problems.begin();
325 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator last = Problems.end();
326
327 iter = Problems.begin();
328
329 int icount = 0; // Problem counter
330 int runningProblemNodeCount = 0;
331 int runningMaxGlobalId = 0; // Accruing max index used for establishing
332 // each problem's mapping into the composite problem
333
334 // Create an index array for use in constructing a composite Epetra_Map
335 int* compositeGlobalNodes = new int[NumMyNodes];
336
337 // Do first pass over problems to allocate needed data.
338 // This is needed prior to calling computeF below.
339 while( iter != last)
340 {
341 // Get a convenient reference to the current problem
342 GenericEpetraProblem& problem = *(*iter).second;
343 int probId = problem.getId();
344
345 // Create dependent vectors for this problem
346 problem.createDependentVectors();
347
348 // Give each problem a chance to initialize based on registrations
349 problem.initialize();
350
351 // Create index mapping for this problem into the composite problem
352 ProblemToCompositeIndices[probId] = Teuchos::rcp( new Epetra_IntVector(*problem.StandardMap) );
353 Epetra_IntVector &indices = *(ProblemToCompositeIndices[probId]);
354
355 for (int i=0; i<problem.NumMyNodes; i++)
356 {
357 int compositeId = runningMaxGlobalId + problem.StandardMap->GID(i);
358 compositeGlobalNodes[i + runningProblemNodeCount] = compositeId;
359 indices[i] = compositeId;
360 }
361
362 runningProblemNodeCount += problem.NumMyNodes;
363 runningMaxGlobalId += problem.StandardMap->MaxAllGID() + 1;
364
365 iter++;
366 }
367
368 // Do second pass to setup each problem
369 iter = Problems.begin();
370
371 // Now that all problems have created vectors to receive fields from
372 // problems on which they depend, sync everyone so that the subsequent
373 // call to computeF below will use the correct values transferred from
374 // each dependent problem.
375 syncAllProblems();
376
377 while( iter != last)
378 {
379 // Get a convenient reference to the current problem
380 GenericEpetraProblem& problem = *(*iter).second;
381 int probId = problem.getId();
382
383 Interfaces[probId] = Teuchos::rcp(new Problem_Interface(problem));
384
385 NOX::Epetra::Vector nox_soln( problem.getSolution(), NOX::Epetra::Vector::CreateView);
386
387 // Use this for analytic Matrix Fills
388 #ifndef USE_FD
389 Teuchos::RCP<NOX::Epetra::Interface::Jacobian> jacInt = Interfaces[probId];
390 LinearSystems[probId] = Teuchos::rcp(new NOX::Epetra::LinearSystemAztecOO(
391 nlParams->sublist("Printing"),
392 nlParams->sublist("Direction").sublist("Newton").sublist("Linear Solver"),
393 Interfaces[probId],
394 jacInt,
395 problem.getJacobian(),
396 problem.getSolution() ));
397
398 Groups[probId] = Teuchos::rcp( new NOX::Epetra::Group(
399 nlParams->sublist("Printing"),
400 Interfaces[probId],
401 nox_soln,
402 LinearSystems[probId] ));
403
404 // OR use this to fill matrices using Finite-Differences with Coloring
405 #else
406 #ifdef HAVE_NOX_EPETRAEXT
407 // Create a timer for performance
408 Epetra_Time fillTime(*Comm);
409
410 EpetraExt::CrsGraph_MapColoring::ColoringAlgorithm algType =
411 EpetraExt::CrsGraph_MapColoring::JONES_PLASSMAN;
412 // NOTE: GREEDY causes a core dump????
413 int reordering = 0;
414 bool useParallel = true;
415 bool distance1 = false;
416 int verbose = 0;
417
418 TmpMapColorings[probId] = Teuchos::rcp(new EpetraExt::CrsGraph_MapColoring(algType, reordering, distance1, verbose));
419 ColorMaps[probId] = Teuchos::rcp(&( (*(TmpMapColorings[probId]))( *problem.getGraph() ) ));
420 ColorMapIndexSets[probId] = Teuchos::rcp(new EpetraExt::CrsGraph_MapColoringIndex( *(ColorMaps[probId]) ));
421 ColumnsSets[probId] = Teuchos::rcp(& (*(ColorMapIndexSets[probId]))( *problem.getGraph() ));
422
423 Teuchos::RCP<Epetra_CrsGraph> & problemGraph = problem.getGraph();
424
425 if (MyPID == 0)
426 printf("\n\tTime to color Jacobian # %d --> %e sec. \n\n",
427 icount++,fillTime.ElapsedTime());
428 MatrixOperators[probId] = Teuchos::rcp( new NOX::Epetra::FiniteDifferenceColoring(
429 nlParams->sublist("Printing"),
430 Interfaces[probId],
431 nox_soln,
432 problemGraph,
433 ColorMaps[probId],
434 ColumnsSets[probId],
435 useParallel,
436 distance1) );
437 //MatrixOperators[probId] = Teuchos::rcp(new NOX::Epetra::FiniteDifference(
438 // nlParams->sublist("Printing"),
439 // Interfaces[probId],
440 // nox_soln,
441 // problemGraph ));
442 //NOX::Epetra::Interface::Jacobian& jacInt =
443 NOX::Epetra::Interface::Jacobian * p_jacInt = dynamic_cast<NOX::Epetra::FiniteDifference*>(MatrixOperators[probId].get());
444 Teuchos::RCP<NOX::Epetra::Interface::Jacobian> jacInt = Teuchos::rcp(p_jacInt, false);
445 //dynamic_cast<NOX::Epetra::Interface::Jacobian&>(*(*(MatrixOperators[probId]);
446 LinearSystems[probId] = Teuchos::rcp(new NOX::Epetra::LinearSystemAztecOO(
447 nlParams->sublist("Printing"),
448 nlParams->sublist("Direction").sublist("Newton").sublist("Linear Solver"),
449 Interfaces[probId],
450 jacInt,
451 MatrixOperators[probId],
452 *problem.getSolution() ) );
453
454 Groups[probId] = Teuchos::rcp( new NOX::Epetra::Group(
455 nlParams->sublist("Printing"),
456 Interfaces[probId],
457 nox_soln,
458 LinearSystems[probId] ));
459
460 #else
461 if(MyPID==0)
462 std::cout << "ERROR: Cannot use EpetraExt with this build !!" << std::endl;
463 exit(0);
464 #endif
465 #endif
466
467 // Needed to establish initial convergence state
468 Groups[probId]->computeF();
469
470 Solvers[probId] =
471 NOX::Solver::buildSolver(Groups[probId], statusTest, nlParams);
472
473 ++iter;
474 }
475
476
477 compositeMap = Teuchos::rcp( new Epetra_Map(-1, NumMyNodes, compositeGlobalNodes, 0, *Comm) );
478
479 delete [] compositeGlobalNodes; compositeGlobalNodes = 0;
480
481 compositeSoln = Teuchos::rcp( new Epetra_Vector(*compositeMap) );
482
483 // --- moved from solveMF -------
484
485 // Sync all the problems and get initial convergence state
486 syncAllProblems();
487
488 setAllGroupX();
489
490 computeAllF();
491
492 double normSum = getNormSum();
493 std::cout << "Initial 2-Norm of composite Problem --> " << normSum;
494
495 // Fill initial composite solution with values from each problem
496 copyProblemsToComposite(*(compositeSoln.get()), SOLUTION);
497
498 // Set up a problem interface for the Problem Manager
499 Teuchos::RCP<Problem_Interface> interface = Teuchos::rcp(new Problem_Interface(*this));
500 // Now create a composite matrix graph needed for preconditioning
501 AA = Teuchos::rcp( new Epetra_CrsGraph(Copy, *compositeMap, 0) );
502 generateGraph();
503
504 /* -------------- Block for Coloring Preconditioner Operator ------
505
506 // NOT YET WORKING
507 // This needs more work to deal with the parallel-use coloring capability.
508
509 // We now attempt to use Coloring on the global preconditioning matrix
510 // Create the Epetra_RowMatrix using Finite Difference with Coloring
511 #ifdef HAVE_NOX_EPETRAEXT
512 bool verbose = false;
513 EpetraExt::CrsGraph_MapColoring tmpMapColoring( verbose );
514 Epetra_MapColoring* colorMap = &tmpMapColoring(*AA);
515 EpetraExt::CrsGraph_MapColoringIndex colorMapIndex(*colorMap);
516 std::vector<Epetra_IntVector>* columns = &colorMapIndex(*AA);
517 #else
518 if(MyPID==0)
519 std::cout << "ERROR: Cannot use EpetraExt with this build !!" << std::endl;
520 exit(0);
521 #endif
522
523 // Use this constructor to create the graph numerically as a means of timing
524 // the old way of looping without colors :
525 // NOX::Epetra::FiniteDifferenceColoring A(interface, soln,
526 // *colorMap, *columns);
527 // Or use this as the standard way of using finite differencing with coloring
528 // where the application is responsible for creating the matrix graph
529 // beforehand, ie as is done in Problem.
530 NOX::Epetra::FiniteDifferenceColoring* A =
531 new NOX::Epetra::FiniteDifferenceColoring(interface, compositeSoln, *AA,
532 *colorMap, *columns);
533 // -------- End of Block for Coloring Preconditioner Operator ------ */
534
535
536 // Create a preconditioning matrix using the graph just created - this
537 // creates a static graph so we can refill the new matirx after
538 // FillComplete() is called.
539 A = Teuchos::rcp( new Epetra_CrsMatrix(Copy, *AA) );
540 A->FillComplete();
541
542 //NOX::Epetra::Interface::Required& reqInt =
543 // dynamic_cast<NOX::Epetra::Interface::Required&>(interface);
544 Teuchos::RCP<NOX::Epetra::Vector> compositeNOXSoln =
545 Teuchos::rcp( new NOX::Epetra::Vector(*(compositeSoln.get())) );
546
547 Teuchos::ParameterList & printParams = nlParams->sublist("Printing");
548 Teuchos::ParameterList & lsParams = nlParams->sublist("Direction").sublist("Newton").sublist("Linear Solver");
549 Teuchos::RCP<NOX::Utils> utilsPtr = Teuchos::rcp( new NOX::Utils(printParams) );
550
551 // Setup appropriate operators and their use
552
553 Teuchos::RCP<NOX::Epetra::LinearSystemAztecOO> composite_linearSystem;
554
555 if( 1 ) // default for MultiPhyics tests
556 {
557 // Use Matrix-Free Jacobian Operator with FDC block preconditioner
558 jacOperator = Teuchos::rcp( new NOX::Epetra::MatrixFree( printParams, interface, *compositeSoln) );
559 NOX::Epetra::Interface::Jacobian * p_jacInt = dynamic_cast<NOX::Epetra::MatrixFree*>(jacOperator.get());
560 jacInterface = Teuchos::rcp(p_jacInt, false);
561 precOperator = Teuchos::rcp( A.get(), false );
562 precInterface = interface;
563
564 composite_linearSystem = Teuchos::rcp( new NOX::Epetra::LinearSystemAztecOO(
565 nlParams->sublist("Printing"),
566 lsParams,
567 jacInterface, jacOperator,
568 precInterface, precOperator,
569 compositeSoln) );
570
571 dynamic_cast<NOX::Epetra::MatrixFree*>(jacOperator.get())->computeJacobian(*compositeSoln, *jacOperator);
572 }
573
574 if( 0 ) // Matrix-Free Jacobian and Broyden preconditioner
575 {
576 evaluate( NOX::Epetra::Interface::Required::Jac, &(*compositeSoln), NULL );
577 jacOperator = Teuchos::rcp( new NOX::Epetra::MatrixFree( printParams, interface, *compositeSoln) );
578 NOX::Epetra::Interface::Jacobian * p_jacInt = dynamic_cast<NOX::Epetra::MatrixFree*>(jacOperator.get());
579 jacInterface = Teuchos::rcp(p_jacInt, false);
580
581 Teuchos::RCP<NOX::Epetra::BroydenOperator> broydenOp =
582 Teuchos::rcp(new NOX::Epetra::BroydenOperator( *nlParams, utilsPtr, *compositeSoln, A, true) );
583 precOperator = broydenOp;
584 precInterface = broydenOp;
585
586 composite_linearSystem = Teuchos::rcp( new NOX::Epetra::LinearSystemAztecOO(
587 nlParams->sublist("Printing"),
588 lsParams,
589 jacInterface, jacOperator,
590 precInterface, precOperator,
591 compositeSoln) );
592 }
593
594 if( 0 ) // Broyden Jacobian and preconditioner
595 {
596 // Turn on some debugging
597 Teuchos::ParameterList & broydenParams =
598 nlParams->sublist("Direction").sublist("Newton").sublist("Broyden Op");
599 broydenParams.set("Write Broyden Info", true);
600
601 // Create some temp objects needed for construction
602 Teuchos::RCP<Epetra_CrsGraph> blockAA = generateBlockDiagonalGraph( *compositeMap, false );
603 blockAA->FillComplete();
604 Teuchos::RCP<Epetra_CrsMatrix> blockA = Teuchos::rcp( new Epetra_CrsMatrix(Copy, *blockAA) );
605 blockA->FillComplete();
606 evaluate( NOX::Epetra::Interface::Required::Jac, &(*compositeSoln), NULL );
607 copyProblemJacobiansToComposite( *blockA );
608
609 std::cout << blockA->Graph() << std::endl;
610 std::cout << *blockA << std::endl;
611
612 Teuchos::RCP<NOX::Epetra::BroydenOperator> broydenOp =
613 Teuchos::rcp(new NOX::Epetra::BroydenOperator( *nlParams, utilsPtr, *compositeSoln, blockA, true) );
614 jacOperator = broydenOp;
615 jacInterface = broydenOp;
616
617 composite_linearSystem = Teuchos::rcp( new NOX::Epetra::LinearSystemAztecOO(
618 nlParams->sublist("Printing"),
619 lsParams,
620 interface,
621 jacInterface, jacOperator,
622 compositeSoln) );
623
624 // To allow replacement of diagonal block values, register ourself as the
625 // needed interface
626 //broydenOp->addReplacementInterface( this );
627 //broydenOp->removeEntriesFromBroydenUpdate( A->Graph() );
628 }
629
630 if( 0 )
631 {
632 // Use FDC Jacobian and itself as its own preconditioner
633
634 jacOperator = A;
635 jacInterface = interface;
636
637 composite_linearSystem = Teuchos::rcp( new NOX::Epetra::LinearSystemAztecOO(
638 nlParams->sublist("Printing"),
639 lsParams,
640 interface,
641 jacInterface, jacOperator,
642 compositeSoln) );
643 }
644
645 //lsParams.set("Preconditioning", "None");
646 lsParams.set("Preconditioner", "AztecOO");
647 compositeGroup = Teuchos::rcp(new NOX::Epetra::Group(nlParams->sublist("Printing"),
648 interface, *compositeNOXSoln,
649 composite_linearSystem));
650 compositeGroup->computeF();
651
652 compositeSolver =
653 NOX::Solver::buildSolver(compositeGroup, statusTest, nlParams);
654
655 #ifdef HAVE_MATLAB
656 // Just a test to see if the Matlab engine is working - RWH
657 if( useMatlab )
658 {
659 //Matlab_Interface testMatlab(*compositeSolver);
660 Coupling_Matlab_Interface testMatlab(*this);
661
662 testMatlab.interact();
663 }
664 #endif
665
666 return;
667 }
668
669 //-----------------------------------------------------------------------------
670
671 void
672 Problem_Manager::syncAllProblems()
673 {
674 if(Problems.empty())
675 {
676 std::cout << "ERROR: No problems registered with Problem_Manager !!"
677 << std::endl;
678 throw "Problem_Manager ERROR";
679 }
680
681 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemIter;
682 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemBegin = Problems.begin();
683 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemLast = Problems.end();
684
685 // Give each problem an opportunity to do things before transferring data, eg compute fluxes
686 for( problemIter = problemBegin ; problemIter != problemLast; ++problemIter )
687 (*problemIter).second->prepare_data_for_transfer();
688
689 // Loop over each problem being managed and invoke its transfer requests
690 for( problemIter = problemBegin ; problemIter != problemLast; ++problemIter )
691 (*problemIter).second->doTransfer();
692
693 // Give each problem an opportunity to do things after transferring data
694 for( problemIter = problemBegin ; problemIter != problemLast; ++problemIter )
695 (*problemIter).second->process_transferred_data();
696
697 return;
698 }
699
700 //-----------------------------------------------------------------------------
701
702 void
703 Problem_Manager::setAlldt( double dt )
704 {
705 if(Problems.empty()) {
706 std::cout << "ERROR: No problems registered with Problem_Manager !!"
707 << std::endl;
708 throw "Problem_Manager ERROR";
709 }
710
711 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemIter = Problems.begin();
712 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemLast = Problems.end();
713
714 for( ; problemIter != problemLast; ++problemIter)
715 (*problemIter).second->setdt(dt);
716
717 return;
718 }
719
720 //-----------------------------------------------------------------------------
721
722 void
723 Problem_Manager::setGroupX(int probId)
724 {
725 Teuchos::RCP<GenericEpetraProblem> problem = Problems[probId];
726
727 if( Teuchos::is_null(problem) )
728 {
729 std::cout << "ERROR: Could not get requested Problem to use with group.setX "
730 << std::endl;
731 throw "Problem_Manager ERROR";
732 }
733
734 Teuchos::RCP<NOX::Epetra::Group> grp = Groups[probId];
735
736 if( Teuchos::is_null(grp) ) {
737 std::cout << "ERROR: Could not get appropriate group for use in setX !!"
738 << std::endl;
739 throw "Problem_Manager ERROR";
740 }
741
742 grp->setX(*problem->getSolution());
743
744 return;
745 }
746
747 //-----------------------------------------------------------------------------
748
749 void
750 Problem_Manager::setGroupX(int probId, Epetra_Vector & vec)
751 {
752 Teuchos::RCP<GenericEpetraProblem> problem = Problems[probId];
753
754 if( Teuchos::is_null(problem) )
755 {
756 std::cout << "ERROR: Could not get requested Problem to use with group.setX " << std::endl;
757 throw "Problem_Manager ERROR";
758 }
759
760 Teuchos::RCP<NOX::Epetra::Group> grp = Groups[probId];
761
762 if( Teuchos::is_null(grp) )
763 {
764 std::cout << "ERROR: Could not get appropriate group for use in setX !!" << std::endl;
765 throw "Problem_Manager ERROR";
766 }
767
768 grp->setX( vec );
769 }
770
771 //-----------------------------------------------------------------------------
772
773 void
774 Problem_Manager::setAllGroupX()
775 {
776 if(Problems.empty()) {
777 std::cout << "ERROR: No problems registered with Problem_Manager !!"
778 << std::endl;
779 throw "Problem_Manager ERROR";
780 }
781
782 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemIter = Problems.begin();
783 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemLast = Problems.end();
784
785 // Loop over each problem being managed and set the corresponding group
786 // solution vector (used by NOX) with the problem's (used by application)
787 for( ; problemIter != problemLast; problemIter++)
788 {
789 int probId = (*problemIter).first;
790 setGroupX(probId);
791 }
792
793 return;
794 }
795
796 //-----------------------------------------------------------------------------
797
798 #ifdef HAVE_NOX_EPETRAEXT
799 void
800 Problem_Manager::setAllOffBlockGroupX(const Epetra_Vector &inVec)
801 {
802 map<int, std::vector<OffBlock_Manager*> >::iterator offBlockIter = OffBlock_Managers.begin();
803 map<int, std::vector<OffBlock_Manager*> >::iterator offBlockLast = OffBlock_Managers.end();
804
805 // Loop over each off-block manager and set the contained groups X-vector
806 // with the incoming vector
807 for( ; offBlockIter != offBlockLast; ++offBlockIter)
808 {
809 std::vector<OffBlock_Manager*> managerVec = (*offBlockIter).second;
810
811 // Need to extract block portion of incoming compositeVec
812 for( unsigned int i = 0; i < managerVec.size(); ++i )
813 {
814 // Note that we assign the group soln to be from the problemVar
815 int probVarId = managerVec[i]->getProblemVarId();
816 copyCompositeToVector(inVec, probVarId, *(managerVec[i]->getRowMapVec()) );
817 managerVec[i]->getGroup()->setX(managerVec[i]->getRowMapVec());
818 }
819 }
820
821 return;
822 }
823
824 #endif
825
826 //-----------------------------------------------------------------------------
827
828 void
829 Problem_Manager::resetProblems()
830 {
831
832 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemIter = Problems.begin();
833 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemLast = Problems.end();
834
835 // Loop over each problem and copy its solution into its old solution
836 for( ; problemIter != problemLast; problemIter++)
837 {
838 GenericEpetraProblem & problem = *(*problemIter).second;
839 problem.reset( *problem.getSolution() );
840 }
841
842 return;
843 }
844
845 //-----------------------------------------------------------------------------
846
847 void
848 Problem_Manager::computeAllF()
849 {
850 if(Problems.empty()) {
851 std::cout << "ERROR: No problems registered with Problem_Manager !!"
852 << std::endl;
853 throw "Problem_Manager ERROR";
854 }
855
856 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemIter = Problems.begin();
857 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemLast = Problems.end();
858
859 // Loop over each problem being managed and invoke the corresponding group's
860 // residual evaluation
861 for( ; problemIter != problemLast; problemIter++)
862 {
863 int probId = (*problemIter).first;
864
865 computeGroupF(probId);
866 }
867
868 return;
869 }
870
871 //-----------------------------------------------------------------------------
872
873 void
874 Problem_Manager::computeGroupF(int probId)
875 {
876 Teuchos::RCP<NOX::Epetra::Group> grp = Groups[ probId ];
877
878 if( Teuchos::is_null(grp) )
879 {
880 std::cout << "ERROR: Could not get a group for problem with id --> "
881 << probId << std::endl;
882 throw "Problem_Manager ERROR";
883 }
884 grp->computeF();
885
886 return;
887 }
888
889 //-----------------------------------------------------------------------------
890
891 void
892 Problem_Manager::computeAllJacobian()
893 {
894 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemIter = Problems.begin();
895 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemLast = Problems.end();
896
897 // Do diagoanl blocks
898 for( ; problemLast != problemIter; ++problemIter )
899 computeBlockJacobian( (*problemIter).first );
900
901 // Do off-diagoanl blocks if appropriate
902 if( doOffBlocks )
903 {
904 #ifdef HAVE_NOX_EPETRAEXT
905 problemIter = Problems.begin();
906
907 for( ; problemLast != problemIter; ++problemIter )
908 for( unsigned int k = 0; k < (*problemIter).second->depProblems.size(); ++k)
909 computeBlockJacobian( (*problemIter).first, (*problemIter).second->depProblems[k] );
910 #endif
911 }
912
913 if( 0 )
914 {
915 // Create a timer for performance
916 Epetra_Time fillTime(*Comm);
917
918 // Loop over each problem being managed and invoke its computeJacobian
919 // method
920 for( ; problemIter != problemLast; problemIter++)
921 {
922 int probId = (*problemIter).first;
923 Teuchos::RCP<NOX::Epetra::Group> grp = Groups[probId];
924 if( Teuchos::is_null(grp) )
925 {
926 std::cout << "ERROR: Could not find valid group for compouteJacobian !!"
927 << std::endl;
928 throw "Problem_Manager ERROR";
929 }
930
931 fillTime.ResetStartTime();
932
933 grp->computeJacobian();
934
935 if (MyPID == 0)
936 printf("\n\tTime to fill Jacobian %d --> %e sec. \n\n",
937 probId, fillTime.ElapsedTime());
938
939
940 if( doOffBlocks )
941 {
942 #ifdef HAVE_NOX_EPETRAEXT
943
944 fillTime.ResetStartTime();
945
946 std::vector<OffBlock_Manager*> & offBlocksVec = OffBlock_Managers[probId];
947
948 for( unsigned int i = 0; i < offBlocksVec.size(); ++i )
949 {
950 // Refresh all problem vectors
951 copyCompositeToProblems(*(compositeSoln.get()), SOLUTION);
952
953 DEBUG_BLOCKGRAPH( std::cout << "Doing computeJacobian for : " << offBlocksVec[i]->getName() << std::endl;)
954
955 offBlocksVec[i]->getGroup()->computeJacobian();
956
957 DEBUG_BLOCKGRAPH( std::cout << "For block : " << offBlocksVec[i]->getName() << std::endl;)
958 DEBUG_BLOCKGRAPH( offBlocksVec[i]->getMatrix().Print(cout);)
959
960 if (MyPID == 0)
961 printf("\n\tTime to fill Jacobian %d (%d) --> %e sec. \n\n",
962 probId, i, fillTime.ElapsedTime());
963 }
964 #endif
965 }
966 }
967 }
968
969 return;
970 }
971
972 //-----------------------------------------------------------------------------
973
974 void
975 Problem_Manager::computeBlockJacobian(int probId, int depId)
976 {
977
978 Epetra_Time fillTime(*Comm);
979
980 // If this is a self-dependence, simply call the corresponding Group's computeJacobian
981 if( (probId == depId) || (-1 == depId) ) // diagonal block contribution
982 {
983 fillTime.ResetStartTime();
984
985 getGroup(probId).computeJacobian();
986
987 if (MyPID == 0)
988 printf("\n\tTime to fill Jacobian %d --> %e sec. \n\n",
989 probId, fillTime.ElapsedTime());
990 }
991 else // off-diagonal block contribution
992 {
993 std::vector<OffBlock_Manager*> managerVec = OffBlock_Managers[probId];
994
995 OffBlock_Manager::idToFind = depId;
996
997 std::vector<OffBlock_Manager *>::iterator iter =
998 find_if( managerVec.begin(), managerVec.end(), mem_fn( &OffBlock_Manager::isMember) );
999
1000 if( managerVec.end() == iter )
1001 {
1002 std::string msg =
1003 "ERROR Problem_Manager::computeBlockJacobian : Problem \""
1004 + Problems[probId]->getName()
1005 + "\" does not have a registered dependence on Problem \""
1006 + Problems[depId]->getName() + "\"\n";
1007 throw msg;
1008 }
1009
1010 fillTime.ResetStartTime();
1011
1012 // Refresh all problem vectors
1013 copyCompositeToProblems(*(compositeSoln.get()), SOLUTION);
1014
1015 (*iter)->getGroup()->computeJacobian();
1016
1017 if (MyPID == 0)
1018 printf("\n\tTime to fill Jacobian %d (%d) --> %e sec. \n\n",
1019 probId, depId, fillTime.ElapsedTime());
1020 }
1021
1022 return;
1023 }
1024
1025 //-----------------------------------------------------------------------------
1026
1027 Teuchos::RCP<Epetra_CrsMatrix>
1028 Problem_Manager::getBlockJacobianMatrix(int probId, int depId)
1029 {
1030
1031 Epetra_CrsMatrix * p_problemMatrix = NULL;
1032
1033 if( (probId == depId) || (-1 == depId) ) // diagonal block
1034 {
1035 // Get objects holding our desired matrix
1036 GenericEpetraProblem & problem = *(Problems[probId]);
1037 NOX::Epetra::LinearSystemAztecOO & problemLinearSystem = *(LinearSystems[probId]);
1038
1039 // Use each group's operator test to determine the type of Jacobian
1040 // operator being used.
1041 const Epetra_Operator& jacOp = *(problemLinearSystem.getJacobianOperator().get());
1042
1043 if ( dynamic_cast<const NOX::Epetra::FiniteDifference*>(&jacOp) )
1044 p_problemMatrix = const_cast<Epetra_CrsMatrix*>(
1045 &dynamic_cast<const NOX::Epetra::FiniteDifference&>(jacOp).getUnderlyingMatrix());
1046 else if ( dynamic_cast<const Epetra_CrsMatrix*>(&jacOp) )
1047 // NOTE: We are getting the matrix from the problem. This SHOULD be
1048 // the same matrix wrapped in the group. A safer alternative would be
1049 // to get this matrix from the group as above for a more general
1050 // operator.
1051 p_problemMatrix = problem.getJacobian().get();
1052
1053 }
1054 else // off-diagonal block
1055 {
1056 std::vector<OffBlock_Manager*> offVec = OffBlock_Managers[probId];
1057
1058 OffBlock_Manager::idToFind = depId;
1059
1060 std::vector<OffBlock_Manager *>::iterator iter =
1061 find_if( offVec.begin(), offVec.end(), mem_fn( &OffBlock_Manager::isMember) );
1062
1063 if( offVec.end() == iter )
1064 {
1065 std::string msg =
1066 "ERROR Problem_Manager::getBlockJacobianMatrix : Problem \""
1067 + Problems[probId]->getName()
1068 + "\" does not have a registered dependence on Problem \""
1069 + Problems[depId]->getName() + "\"\n";
1070 throw msg;
1071 }
1072
1073 p_problemMatrix = &((*iter)->getMatrix());
1074 }
1075
1076 Teuchos::RCP<Epetra_CrsMatrix> mat = Teuchos::rcp( p_problemMatrix, false );
1077
1078 return mat;
1079 }
1080
1081 //-----------------------------------------------------------------------------
1082
1083 Teuchos::RCP<Epetra_Operator>
1084 Problem_Manager::getBlockInverseOperator(int probId)
1085 {
1086
1087 if( BlockInverseOperators.end() == BlockInverseOperators.find(probId) )
1088 {
1089 std::cout << "ERROR: No valid Block Inverse Operator exists for problem # "
1090 << probId << std::endl;
1091 throw "Problem_Manager ERROR";
1092 }
1093
1094 Teuchos::RCP<Epetra_Operator> pInvOp = BlockInverseOperators[probId];
1095
1096 return pInvOp;
1097 }
1098
1099 //-----------------------------------------------------------------------------
1100
1101 const Epetra_Vector &
1102 Problem_Manager::getResidualVec(int probId )
1103 {
1104
1105 const Epetra_Vector & vec = dynamic_cast<const NOX::Epetra::Vector&>
1106 (getProblemSolutionGroup(probId).getF()).getEpetraVector();
1107
1108 return vec;
1109 }
1110
1111 //-----------------------------------------------------------------------------
1112
1113 void
1114 Problem_Manager::copyGroupCurrentXtoProblemX(int probId)
1115 {
1116 // Copy Current solution X from NOX solver group into the problem's solution vector
1117
1118 Teuchos::RCP<GenericEpetraProblem> problem = Problems[probId];
1119
1120 if( Teuchos::is_null(problem) )
1121 {
1122 std::cout << "ERROR: Could not get requested Problem to update with final "
1123 << "solution" << std::endl;
1124 throw "Problem_Manager ERROR";
1125 }
1126
1127 Teuchos::RCP<NOX::Solver::Generic> solver = Solvers[probId];
1128
1129 if( Teuchos::is_null(solver) )
1130 {
1131 std::cout << "ERROR: Could not get appropriate Solver for use in update !!"
1132 << std::endl;
1133 throw "Problem_Manager ERROR";
1134 }
1135
1136 const NOX::Epetra::Group& finalGroup =
1137 dynamic_cast<const NOX::Epetra::Group&>(solver->getSolutionGroup());
1138 const Epetra_Vector& finalSolution =
1139 (dynamic_cast<const NOX::Epetra::Vector&>
1140 (finalGroup.getX())).getEpetraVector();
1141
1142 *problem->getSolution() = finalSolution;
1143
1144 return;
1145 }
1146
1147 //-----------------------------------------------------------------------------
1148
1149 void
1150 Problem_Manager::copyAllGroupXtoProblems()
1151 {
1152 // Copy final solution from NOX solvers into each problem's solution vector
1153
1154 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemIter = Problems.begin();
1155 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemLast = Problems.end();
1156
1157 for( ; problemIter != problemLast; ++problemIter )
1158 {
1159 int probId = (*problemIter).first;
1160 copyGroupCurrentXtoProblemX(probId);
1161 }
1162
1163 return;
1164 }
1165
1166 //-----------------------------------------------------------------------------
1167
1168 void
1169 Problem_Manager::resetCurrentGroupX(int probId)
1170 {
1171 // Reset each solvers current solution group X vector with itself, thereby resetting all valid flags
1172
1173 Teuchos::RCP<NOX::Solver::Generic> solver = Solvers[probId];
1174
1175 if( Teuchos::is_null(solver) )
1176 {
1177 std::cout << "ERROR: Could not get appropriate Solver for use in update !!"
1178 << std::endl;
1179 throw "Problem_Manager ERROR";
1180 }
1181
1182 const NOX::Epetra::Group& currentGroup =
1183 dynamic_cast<const NOX::Epetra::Group&>(solver->getSolutionGroup());
1184 const_cast<NOX::Epetra::Group&>(currentGroup).setX( currentGroup.getX() );
1185
1186 return;
1187 }
1188
1189 //-----------------------------------------------------------------------------
1190
1191 void
1192 Problem_Manager::resetAllCurrentGroupX()
1193 {
1194 // Reset each solvers current solution group X vector with itself, thereby resetting all valid flags
1195
1196 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemIter = Problems.begin();
1197 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemLast = Problems.end();
1198
1199 for( ; problemIter != problemLast; ++problemIter )
1200 {
1201 int probId = (*problemIter).first;
1202 resetCurrentGroupX(probId);
1203 }
1204
1205 return;
1206 }
1207
1208 //-----------------------------------------------------------------------------
1209
1210 void
1211 Problem_Manager::copyCompositeToProblems( const Epetra_Vector& compositeVec, Problem_Manager::VectorType vecType )
1212 {
1213 // Copy a composite problem vector to each problem's vector
1214
1215 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemIter = Problems.begin();
1216 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemLast = Problems.end();
1217
1218 Epetra_Vector* problemVec(0);
1219
1220 // Loop over each problem being managed and copy into the correct problem vector
1221 for( ; problemIter != problemLast; ++problemIter )
1222 {
1223 int probId = (*problemIter).first;
1224 switch (vecType)
1225 {
1226 case SOLUTION :
1227 problemVec = ((*problemIter).second->getSolution()).get();
1228 break;
1229
1230 case GROUP_F :
1231 default :
1232 std::cout << "ERROR: vecType not supported for copy FROM composite!!"
1233 << std::endl;
1234 throw "Problem_Manager ERROR";
1235 }
1236
1237 copyCompositeToVector(compositeVec, probId, *problemVec);
1238 }
1239
1240 return;
1241 }
1242
1243 //-----------------------------------------------------------------------------
1244
1245 void
1246 Problem_Manager::copyProblemsToComposite( Epetra_Vector& compositeVec, Problem_Manager::VectorType vecType )
1247 {
1248 // Copy vectors from each problem into a composite problem vector
1249
1250 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemIter = Problems.begin();
1251 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemLast = Problems.end();
1252
1253 const Epetra_Vector* problemVec(0);
1254
1255 // Loop over each problem being managed and copy from the correct
1256 // problem vector
1257 for( ; problemIter != problemLast; ++problemIter )
1258 {
1259 int probId = (*problemIter).first;
1260 switch (vecType) {
1261
1262 case SOLUTION :
1263 problemVec = ((*problemIter).second->getSolution()).get();
1264 break;
1265
1266 case GROUP_F :
1267 problemVec = &(dynamic_cast<const NOX::Epetra::Vector&>
1268 (Groups[probId]->getF()).getEpetraVector());
1269 break;
1270
1271 default :
1272 std::cout << "ERROR: vecType not supported for copy TO composite!!"
1273 << std::endl;
1274 throw "Problem_Manager ERROR";
1275 }
1276
1277 copyVectorToComposite(compositeVec, probId, *problemVec);
1278 }
1279
1280 return;
1281 }
1282
1283 //-----------------------------------------------------------------------------
1284
1285
1286 void
1287 Problem_Manager::copyCompositeToVector( const Epetra_Vector& compositeVec, int id, Epetra_Vector& problemVec )
1288 {
1289 // Copy part of a composite problem vector to a problem's vector
1290 Epetra_IntVector& indices = *(ProblemToCompositeIndices[id]);
1291 // This map is needed to get correct LID of indices
1292 const Epetra_BlockMap &map = compositeVec.Map();
1293
1294 for( int i = 0; i < problemVec.MyLength(); ++i )
1295 problemVec[i] = compositeVec[map.LID(indices[i])];
1296
1297 return;
1298 }
1299
1300 //-----------------------------------------------------------------------------
1301
1302 void
1303 Problem_Manager::copyVectorToComposite( Epetra_Vector& compositeVec, int id, const Epetra_Vector& problemVec)
1304 {
1305 // Copy a vector from a problem into part of a composite problem vector
1306 Epetra_IntVector& indices = *(ProblemToCompositeIndices[id]);
1307
1308 for( int i = 0; i < problemVec.MyLength(); ++i )
1309 compositeVec[indices[i]] = problemVec[i];
1310
1311 return;
1312 }
1313
1314 //-----------------------------------------------------------------------------
1315
1316 void
1317 Problem_Manager::copyProblemJacobiansToComposite()
1318 {
1319 copyProblemJacobiansToComposite(*A);
1320
1321 return;
1322 }
1323
1324 //-----------------------------------------------------------------------------
1325
1326 void
1327 Problem_Manager::copyProblemJacobiansToComposite( Epetra_CrsMatrix & mat )
1328 {
1329 // Copy problem Jacobians as block diagonal contributions to
1330 // composite Jacobian
1331
1332 Epetra_CrsMatrix & compositeMatrix = mat;
1333
1334 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemIter = Problems.begin();
1335 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemLast = Problems.end();
1336
1337 int problemMaxNodes = compositeSoln.get()->GlobalLength();
1338
1339 // Loop over each problem being managed and copy its Jacobian into
1340 // the composite diagonal blocks
1341 for( ; problemIter != problemLast; problemIter++)
1342 {
1343 int probId = (*problemIter).first;
1344
1345 // Get the problem, its Jacobian graph and its linear system
1346 GenericEpetraProblem & problem = *((*problemIter).second);
1347 Epetra_CrsGraph & problemGraph = *problem.getGraph();
1348 NOX::Epetra::LinearSystemAztecOO & problemLinearSystem = *(LinearSystems[probId]);
1349
1350 // Get the indices map for copying data from this problem into
1351 // the composite problem
1352 Epetra_IntVector & indices = *(ProblemToCompositeIndices[probId]);
1353
1354 // Each problem's Jacobian will be determined by type
1355 Epetra_CrsMatrix * p_problemMatrix(0);
1356
1357 // Use each group's operator test to determine the type of Jacobian
1358 // operator being used.
1359 const Epetra_Operator& jacOp = *(problemLinearSystem.getJacobianOperator().get());
1360
1361 if ( dynamic_cast<const NOX::Epetra::FiniteDifference*>(&jacOp) )
1362 p_problemMatrix = const_cast<Epetra_CrsMatrix*>(
1363 &dynamic_cast<const NOX::Epetra::FiniteDifference&>(jacOp).getUnderlyingMatrix());
1364 else if ( dynamic_cast<const Epetra_CrsMatrix*>(&jacOp) )
1365 // NOTE: We are getting the matrix from the problem. This SHOULD be
1366 // the same matrix wrapped in the group. A safer alternative would be
1367 // to get this matrix from the group as above for a more general
1368 // operator.
1369 p_problemMatrix = problem.getJacobian().get();
1370 else
1371 {
1372 if (MyPID==0)
1373 std::cout << "Jacobian operator for Problem not supported for "
1374 << "preconditioning Matrix-Free coupling solver." << std::endl;
1375 throw "Problem_Manager ERROR";
1376 }
1377
1378 // Create convenient reference for each Jacobian matrix
1379 Epetra_CrsMatrix &problemMatrix = *p_problemMatrix;
1380
1381
1382 // ROGER Block Norm
1383 if (1)
1384 {
1385 std::cout << "Block=" << probId << " Inf Norm=" << problemMatrix.NormInf()
1386 << " One Norm=" << problemMatrix.NormOne() << std::endl;
1387 }
1388
1389 // Temporary storage arrays for extracting/inserting matrix row data
1390 int* columnIndices = new int[problemMaxNodes];
1391 double* values = new double[problemMaxNodes];
1392
1393 int problemRow, compositeRow, numCols, numValues;
1394
1395 for (int i = 0; i<problemMatrix.NumMyRows(); i++)
1396 {
1397 problemRow = problemMatrix.Map().GID(i);
1398 problemGraph.ExtractGlobalRowCopy(problemRow, problemMaxNodes, numCols, columnIndices);
1399 problemMatrix.ExtractGlobalRowCopy(problemRow, problemMaxNodes, numValues, values);
1400 if( numCols != numValues )
1401 {
1402 if (MyPID==0)
1403 std::cout << "ERROR: Num Columns != Num Values from problem Matrix !!"
1404 << std::endl;
1405 throw "Problem_Manager ERROR";
1406 }
1407
1408 // Convert row/column indices to composite problem
1409 compositeRow = indices[problemRow];
1410
1411 for (int j = 0; j<numCols; j++)
1412 columnIndices[j] = indices[columnIndices[j]];
1413
1414 int ierr = compositeMatrix.ReplaceGlobalValues(compositeRow,
1415 numValues, values, columnIndices);
1416 if( ierr )
1417 {
1418 if (MyPID==0)
1419 std::cout << "ERROR: compositeMatrix.ReplaceGlobalValues(...)" << std::endl;
1420 throw "Problem_Manager ERROR";
1421 }
1422 }
1423 delete [] values; values = 0;
1424 delete [] columnIndices; columnIndices = 0;
1425
1426 // Sync up processors to be safe
1427 Comm->Barrier();
1428
1429 // Add off-diagonal FD block contributions if waranted
1430 if( doOffBlocks )
1431 {
1432 #ifdef HAVE_NOX_EPETRAEXT
1433 // Loop over each problem on which this one depends
1434 for( unsigned int k = 0; k < problem.depProblems.size(); ++k)
1435 {
1436 Teuchos::RCP<GenericEpetraProblem> p_depProblem = Problems[problem.depProblems[k]];
1437
1438 // Copy the off-block jacobian matrices for this
1439 // problem-problem coupling
1440 // !!! -------- THESE COMMENTS ARE NO LONGER VALID --------------!!!
1441 // *******************************************************************
1442 // *** NOTE: the map used for the off-block graph is the composite Map
1443 // *** to allow valid global indexing. This also allows us to
1444 // *** simply copy values directly from the off-block matrices to the
1445 // *** composite Jacobian matrix
1446 // *******************************************************************
1447 OffBlock_Manager * p_offBlockMgr = (OffBlock_Managers[probId])[k];
1448
1449 if( !p_offBlockMgr )
1450 {
1451 std::cout << "ERROR: Unable to get OffBlock_Manager for dependence of problem "
1452 << problem.getName() << " on problem " << p_depProblem->getName()
1453 << " !!" << std::endl;
1454 throw "Problem_Manager ERROR";
1455 }
1456 Epetra_CrsMatrix & offMatrix = p_offBlockMgr->getMatrix();
1457
1458 int * blockColIndices; // = new int[problemMaxNodes];
1459 int numCols = -1;
1460 double * values; // = new double[problemMaxNodes];
1461 std::vector<int> compositeColIndices;
1462 int compositeRow;
1463
1464 // Loop over each row and copy into composite matrix
1465 for (int i = 0; i < offMatrix.NumMyRows(); ++i)
1466 {
1467 offMatrix.ExtractMyRowView(i, numCols, values, blockColIndices);
1468
1469 compositeColIndices.resize(numCols);
1470
1471 p_offBlockMgr->convertBlockRowIndicesToComposite(1, &i, &compositeRow);
1472 p_offBlockMgr->convertBlockColIndicesToComposite( numCols, blockColIndices, &(compositeColIndices[0]) );
1473 compositeMatrix.ReplaceMyValues( compositeRow, numCols, values, &(compositeColIndices[0]) );
1474 }
1475
1476 // Sync up processors to be safe
1477 Comm->Barrier();
1478 }
1479 #endif
1480 }
1481 }
1482
1483 #ifdef DEBUG_PROBLEM_MANAGER
1484 compositeMatrix.Print(cout);
1485 #endif
1486
1487 return;
1488 }
1489
1490 //-----------------------------------------------------------------------------
1491
1492 double
1493 Problem_Manager::getNormSum()
1494 {
1495 // Get each problem's residual norm and sum into a total
1496 double normSum(0.0);
1497
1498 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemIter = Problems.begin();
1499 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemLast = Problems.end();
1500
1501 for( ; problemIter != problemLast; problemIter++) {
1502 int probId = (*problemIter).first;
1503
1504 //Teuchos::RCP<NOX::Epetra::Group> grp = Groups[ probId ];
1505 NOX::Epetra::Group grp = const_cast<NOX::Epetra::Group&>(dynamic_cast<const NOX::Epetra::Group&>(Solvers[ probId ]->getSolutionGroup()));
1506
1507 //if( Teuchos::is_null(grp) )
1508 //{
1509 // std::cout << "ERROR: Could not get appropriate group for use in NormSum !!"
1510 // << std::endl;
1511 // throw "Problem_Manager ERROR";
1512 //}
1513
1514 //double problemNorm = grp->getNormF();
1515 double problemNorm = grp.getNormF();
1516 std::cout << "2-Norm of Problem " << probId << " --> " << problemNorm
1517 << std::endl;
1518 normSum += problemNorm * problemNorm;
1519 }
1520 normSum = sqrt(normSum);
1521
1522 return normSum;
1523 }
1524
1525 //-----------------------------------------------------------------------------
1526
1527 bool
1528 Problem_Manager::solve()
1529 {
1530 if(Problems.empty())
1531 {
1532 std::cout << "ERROR: No problems registered with Problem_Manager !!" << std::endl;
1533 throw "Problem_Manager ERROR";
1534 }
1535
1536 if( Groups.empty() || Interfaces.empty() || Solvers.empty() )
1537 {
1538 std::cout << "ERROR: Groups, Interfaces and/or Solvers are emptry !!"
1539 << std::endl;
1540 throw "Problem_Manager ERROR";
1541 }
1542
1543 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemIter = Problems.begin();
1544 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemLast = Problems.end();
1545
1546 // Sync all the problems and get initial convergence state
1547 syncAllProblems();
1548 setAllGroupX();
1549 computeAllF();
1550
1551 double normSum = getNormSum();
1552 std::cout << "Initial 2-Norm of composite Problem --> " << normSum;
1553
1554 // Now do the decoupled solve
1555 int iter = 0;
1556 NOX::StatusTest::StatusType status;
1557
1558 // RPP: Inner status test
1559 Teuchos::RCP<NOX::StatusTest::NormF> absresid =
1560 Teuchos::rcp(new NOX::StatusTest::NormF(1.0e-10));
1561 Teuchos::RCP<NOX::StatusTest::NormUpdate> update =
1562 Teuchos::rcp(new NOX::StatusTest::NormUpdate(1.0e-5));
1563 Teuchos::RCP<NOX::StatusTest::Combo> converged =
1564 Teuchos::rcp(new NOX::StatusTest::Combo(NOX::StatusTest::Combo::AND));
1565 converged->addStatusTest(absresid);
1566 converged->addStatusTest(update);
1567 Teuchos::RCP<NOX::StatusTest::MaxIters> maxiters =
1568 Teuchos::rcp(new NOX::StatusTest::MaxIters(15));
1569 Teuchos::RCP<NOX::StatusTest::FiniteValue> finiteValue =
1570 Teuchos::rcp(new NOX::StatusTest::FiniteValue);
1571 Teuchos::RCP<NOX::StatusTest::Combo> combo =
1572 Teuchos::rcp(new NOX::StatusTest::Combo(NOX::StatusTest::Combo::OR));
1573 combo->addStatusTest(converged);
1574 combo->addStatusTest(maxiters);
1575 combo->addStatusTest(finiteValue);
1576
1577 while( iter < 50 && normSum > 1.0e-8 ) // Hard-coded convergence criterion for now.
1578 {
1579 ++iter;
1580
1581 problemIter = Problems.begin();
1582
1583 // Solve each problem in the order it was registered
1584 for( ; problemIter != problemLast; problemIter++)
1585 {
1586 GenericEpetraProblem & problem = *(*problemIter).second;
1587
1588 int probId = problem.getId();
1589
1590 Teuchos::RCP<NOX::Epetra::Group> problemGroup = Groups[probId];
1591 NOX::Solver::Generic & problemSolver = *(Solvers[probId]);
1592
1593 // Sync all dependent data with this problem
1594 problem.doTransfer();
1595 // Sync the problem solution with its solver group
1596 setGroupX(probId);
1597 // Reset the solver for this problem and solve
1598 //problemSolver.reset(problemGroup, *statusTest, *nlParams);
1599
1600 problemSolver.reset(problemGroup->getX(), combo);
1601 status = problemSolver.solve();
1602 if( status != NOX::StatusTest::Converged )
1603 {
1604 if (MyPID==0)
1605 std::cout << "\nRegistered Problem ## failed to converge !!" << std::endl;
1606 }
1607
1608 // Copy final solution from group into problem solution vector
1609 copyGroupCurrentXtoProblemX(probId);
1610 }
1611
1612 // Determine final residuals for use in testing convergence
1613 syncAllProblems();
1614 setAllGroupX();
1615 computeAllF();
1616
1617 normSum = getNormSum();
1618 std::cout << "iter #" << iter << ", 2-Norm of composite Problem --> "
1619 << normSum << std::endl;
1620 }
1621
1622 if (normSum > 1.0e-8)
1623 {
1624 std::cout << "Warning: composite problem failed to converge after "
1625 << iter << " fixed-point iterations." << std::endl;
1626
1627 return false;
1628 }
1629
1630 std::cout << "\nDecoupled solution required --> " << iter << " iterations.\n"
1631 << std::endl;
1632
1633 return true;
1634 }
1635
1636 //-----------------------------------------------------------------------------
1637
1638 bool
1639 Problem_Manager::solveMF()
1640 {
1641 if(Problems.empty())
1642 {
1643 std::cout << "ERROR: No problems registered with Problem_Manager !!"
1644 << std::endl;
1645 throw "Problem_Manager ERROR";
1646 }
1647
1648 compositeGroup->setX( compositeGroup->getX() );
1649
1650 compositeSolver->reset(compositeGroup->getX(), statusTest);
1651
1652 NOX::StatusTest::StatusType status = compositeSolver->solve();
1653 if( status != NOX::StatusTest::Converged )
1654 if (MyPID==0)
1655 std::cout << "\nMatrix-Free coupled Problem failed to converge !!" << std::endl;
1656
1657 // Update all problem's solutions with final solutions from solvers
1658 copyAllGroupXtoProblems();
1659
1660 return true;
1661 }
1662
1663 //-----------------------------------------------------------------------------
1664
1665 bool
1666 Problem_Manager::solveSchurBased()
1667 {
1668 if( 2 != Problems.size() )
1669 {
1670 std::cout << "ERROR: Schur-based coupling currently requires exactly two problems!" << std::endl;
1671 throw "Problem_Manager ERROR";
1672 }
1673
1674 Teuchos::RCP<Epetra_Vector> resA = Teuchos::rcp( new Epetra_Vector( getResidualVec(1)) );
1675 Teuchos::RCP<Epetra_Vector> resB = Teuchos::rcp( new Epetra_Vector( getResidualVec(2)) );
1676
1677 NOX::Epetra::SchurCoupler schurCoupler(*this);
1678 NOX::Epetra::SchurOp & schurCplOp = *schurCoupler.getSchurOperator(0, 0);
1679 //NOX::Epetra::SchurOp schurCplOp( 1, 2, *this );
1680
1681 Teuchos::ParameterList& lsParams = nlParams->sublist("Direction").sublist("Newton").sublist("Linear Solver");
1682
1683 setGroupX(1);
1684 setGroupX(2);
1685 getGroup(1).computeF();
1686 getGroup(2).computeF();
1687 getGroup(1).computeJacobian();
1688 getGroup(2).computeJacobian();
1689 createBlockInverseOperator(1, lsParams);
1690 createBlockInverseOperator(2, lsParams);
1691
1692 //schurCplOp.Apply( *tmpA1, *tmpA2 );
1693
1694 Epetra_Vector solutionA(getSolutionVec(1));
1695 solutionA.PutScalar(0.0);
1696 std::cout << "Solution A: \n" << solutionA << std::endl;
1697
1698 std::cout << "Original RHS :\n" << *resA << std::endl;
1699 schurCplOp.modifyRHS( resA, resB );
1700 std::cout << "Modified RHS :\n" << *resA << std::endl;
1701
1702 Epetra_LinearProblem * linear_problem = new Epetra_LinearProblem(&schurCplOp, &solutionA, &(*resA));
1703 AztecOO * aztecoo_solver = new AztecOO(*linear_problem);
1704
1705 aztecoo_solver->SetAztecOption(AZ_precond, AZ_none);
1706
1707 // Test use of a diagonal block preconditioner
1708 if( 1 )
1709 {
1710 double conditionNumberEstimate = -1.0;
1711 Teuchos::RCP<Epetra_CrsMatrix> pMatrix = getBlockJacobianMatrix(1);
1712 aztecoo_solver->SetPrecMatrix(pMatrix.get());
1713 aztecoo_solver->SetAztecOption(AZ_precond, AZ_dom_decomp);
1714 aztecoo_solver->SetAztecOption(AZ_overlap, 0);
1715 aztecoo_solver->SetAztecOption(AZ_subdomain_solve, AZ_ilu);
1716 aztecoo_solver->SetAztecOption(AZ_graph_fill, 0);
1717 aztecoo_solver->ConstructPreconditioner(conditionNumberEstimate);
1718 }
1719
1720 std::cout << "... Solving.\n";
1721 aztecoo_solver->Iterate(100, 1.0e-8);
1722 std::cout << "... Solved to a tolerance of " << aztecoo_solver->TrueResidual() << " in " << aztecoo_solver->NumIters() << " iterations.\n";
1723
1724 std::cout << "\n----- Solution for update vector ---- :\n" << solutionA << std::endl;
1725
1726 *resA = solutionA;
1727
1728 solutionA.Update(1.0, getSolutionVec(1), -1.0);
1729
1730 std::cout << "\n----- Solution vector ---- :\n" << solutionA << std::endl;
1731
1732 // Now back substitue for other problem solution
1733 applyBlockAction( 2, 1, *resA, *resB);
1734 resB->Update(-1.0, getResidualVec(2), 1.0);
1735 getBlockInverseOperator(2)->ApplyInverse(*resB, *resB);
1736
1737 std::cout << "\n----- Second Solution for update vector ---- :\n" << *resB << std::endl;
1738
1739 resB->Update(1.0, getSolutionVec(2), 1.0);
1740
1741 std::cout << "\n----- Second Solution vector ---- :\n" << *resB << std::endl;
1742
1743 exit(0);
1744 // Update all problem's solutions with final solutions from solvers
1745 copyAllGroupXtoProblems();
1746
1747 return true;
1748 }
1749
1750 //-----------------------------------------------------------------------------
1751
1752
1753 // These methods are needed to allow inheritance from GenericEpetraProblem base
1754
1755 bool
1756 Problem_Manager::evaluate(
1757 NOX::Epetra::Interface::Required::FillType flag,
1758 const Epetra_Vector *solnVector,
1759 Epetra_Vector *rhsVector)
1760 {
1761 //Determine what to fill (F or Jacobian)
1762 bool fillF = false;
1763 bool fillMatrix = false;
1764
1765 if (rhsVector != 0)
1766 fillF = true;
1767 else
1768 fillMatrix = true;
1769
1770 // Copy incoming vector from NOX solver into our composite solution Vector
1771 *(compositeSoln.get()) = *solnVector;
1772
1773 copyCompositeToProblems(*solnVector, SOLUTION);
1774
1775 // If used, give each off-block FDC manager a copy of the current total
1776 // solution vector
1777 #ifdef HAVE_NOX_EPETRAEXT
1778 if( fillMatrix && doOffBlocks )
1779 setAllOffBlockGroupX(*solnVector);
1780 #endif
1781
1782 // Do transfers from problem solution vectors into problem dependent vectors
1783 syncAllProblems();
1784
1785 // Set each problem group Xvec with its problem solution vector
1786 setAllGroupX();
1787
1788 if (fillF)
1789 {
1790 computeAllF();
1791 copyProblemsToComposite(*rhsVector, GROUP_F);
1792
1793 }
1794
1795 if (fillMatrix)
1796 {
1797 A->PutScalar(0.0);
1798
1799 computeAllJacobian();
1800
1801 copyProblemJacobiansToComposite();
1802
1803 A->FillComplete();
1804
1805 #ifdef DEBUG_PROBLEM_MANAGER
1806 A->Print(cout);
1807 #endif
1808 }
1809
1810 return true;
1811 }
1812
1813 //-----------------------------------------------------------------------------
1814
1815 Teuchos::RCP<Epetra_CrsGraph>
1816 Problem_Manager::generateBlockDiagonalGraph( const Epetra_Map & rowMap, bool includeProblemOffContribs )
1817 {
1818 Teuchos::RCP<Epetra_CrsGraph> graphPtr =
1819 Teuchos::rcp( new Epetra_CrsGraph(Copy, rowMap, 0) );
1820
1821 // First construct a graph for each problem's self-dependence
1822 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemIter = Problems.begin();
1823 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemLast = Problems.end();
1824
1825 // Loop over each problem being managed and ascertain its graph as well
1826 // as its graph from its dependencies
1827 for( ; problemIter != problemLast; ++problemIter )
1828 {
1829 GenericEpetraProblem & problem = *(*problemIter).second;
1830 int probId = problem.getId();
1831
1832 Epetra_CrsGraph & problemGraph = *problem.getGraph();
1833
1834 // Use max potential number of nonzero columns to dimension index array
1835 int problemMaxNodes = problemGraph.Map().MaxAllGID();
1836
1837 // Get the indices map for copying data from this problem into
1838 // the composite problem
1839 Epetra_IntVector& problemIndices = *(ProblemToCompositeIndices[probId]);
1840
1841 int problemRow, compositeRow, numCols;
1842
1843 // First fill composite graph for each problem's self-dependence. This
1844 // corresponds to diagonal blocks.
1845 int* columnIndices = new int[problemMaxNodes];
1846
1847 for (int i = 0; i < problemGraph.NumMyRows(); ++i)
1848 {
1849 problemRow = problemGraph.Map().GID(i);
1850 problemGraph.ExtractGlobalRowCopy(problemRow, problemMaxNodes,
1851 numCols, columnIndices);
1852
1853 // Convert row/column indices to composite problem
1854 compositeRow = problemIndices[problemRow];
1855 for (int j = 0; j<numCols; j++)
1856 columnIndices[j] = problemIndices[columnIndices[j]];
1857 int ierr = graphPtr->InsertGlobalIndices(compositeRow, numCols, columnIndices);
1858 if( ierr )
1859 {
1860 if (MyPID==0)
1861 std::cout << "ERROR: graphPtr->InsertGlobalIndices(...)" << std::endl;
1862 throw "Problem_Manager ERROR";
1863 }
1864 }
1865 delete [] columnIndices; columnIndices = 0;
1866
1867 // Now allow the problem to explicitly create known dependencies on other
1868 // problems
1869 if( includeProblemOffContribs )
1870 {
1871 for( unsigned int k = 0; k < problem.depProblems.size(); ++k)
1872 {
1873 // Get the needed objects for the depend problem
1874 GenericEpetraProblem & dependProblem = *(Problems[problem.depProblems[k]]);
1875 int dependId = dependProblem.getId();
1876 Epetra_IntVector & dependIndices = *(ProblemToCompositeIndices[dependId]);
1877
1878 if( problem.suppliesOffBlockStructure(k) )
1879 {
1880 map<int, std::vector<int> > offBlockIndices;
1881 problem.getOffBlockIndices( offBlockIndices );
1882
1883 map<int, std::vector<int> >::iterator indIter = offBlockIndices.begin(),
1884 indIter_end = offBlockIndices.end() ;
1885
1886 for( ; indIter != indIter_end; ++indIter )
1887 {
1888 int compositeRow = problemIndices[(*indIter).first];
1889 std::vector<int> & colIndices = (*indIter).second;
1890
1891 // Convert column indices to composite values
1892 for( unsigned int cols = 0; cols < colIndices.size(); ++cols )
1893 colIndices[cols] = dependIndices[ colIndices[cols] ];
1894
1895 graphPtr->InsertGlobalIndices( compositeRow, colIndices.size(), &colIndices[0] );
1896 }
1897 }
1898 }
1899 }
1900 }
1901
1902 return graphPtr;
1903 }
1904
1905 //-----------------------------------------------------------------------------
1906
1907 Teuchos::RCP<Epetra_CrsGraph>
1908 Problem_Manager::generateOffDiagonalBlockGraph( const Epetra_Map & rowMap )
1909 {
1910 Teuchos::RCP<Epetra_CrsGraph> graphPtr =
1911 Teuchos::rcp( new Epetra_CrsGraph(Copy, rowMap, 0) );
1912
1913 // Two things are achieved here: 1) The composite Graph is augmented to
1914 // accommodate off-diagonal blocks and 2) these blocks are packaged as
1915 // individual NOX::EpetreNew::Group's owned by the manager.
1916
1917 #ifdef HAVE_NOX_EPETRAEXT
1918
1919 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemIter = Problems.begin();
1920 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemLast = Problems.end();
1921
1922 problemIter = Problems.begin();
1923
1924 // Loop over each problem being managed and ascertain its graph as well
1925 // as its graph from its dependencies
1926 for( ; problemIter != problemLast; problemIter++)
1927 {
1928 GenericEpetraProblem & problem = *(*problemIter).second;
1929 int probId = problem.getId();
1930
1931 Epetra_CrsGraph & problemGraph = *problem.getGraph();
1932
1933 // Get the indices map for copying data from this problem into
1934 // the composite problem
1935 Epetra_IntVector& problemIndices = *(ProblemToCompositeIndices[probId]);
1936
1937 // Create containers for the off-block objects
1938 std::vector<OffBlock_Manager*> OffBlock_ManagersVec;
1939
1940 int problemMaxNodes = problemGraph.Map().NumGlobalElements();
1941
1942 int problemRow, compositeRow, numCols, numDepCols;
1943
1944 // Loop over each problem on which this one depends
1945 for( unsigned int k = 0; k < problem.depProblems.size(); ++k)
1946 {
1947 // Create the off-block graph to be constructed for this
1948 // problem-problem coupling
1949 // NOTE: the map used for the off-block graph is the composite Map
1950 // to allow valid global indexing
1951 Epetra_CrsGraph* offGraphPtr = new Epetra_CrsGraph(Copy, *compositeMap, 0);
1952
1953 Epetra_CrsGraph &offGraph = *offGraphPtr;
1954
1955 // Get the needed objects for the depend problem
1956 GenericEpetraProblem & dependProblem = *(Problems[problem.depProblems[k]]);
1957
1958 int dependId = dependProblem.getId();
1959
1960 XferOp * xferOpPtr = problem.xferOperators[dependId];
1961
1962 if( !xferOpPtr )
1963 {
1964 std::cout << "ERROR: Unable to get Xfer_Operator for dependence of "
1965 << "problem \"" << problem.getName() << "\" on problem "
1966 << "\"" << dependProblem.getName() << "\" !!" << std::endl;
1967 throw "Problem_Manager ERROR";
1968 }
1969 XferOp &xferOp = *xferOpPtr;
1970 multimap<int,int>& depNodesMap = xferOp.getDependentNodesMap();
1971
1972 // Get the indices map for copying data from the dependent problem into
1973 // the composite problem
1974 Epetra_IntVector& dependIndices = *(ProblemToCompositeIndices[dependId]);
1975
1976 // Dimension nonzero columns index array with upper bound which is
1977 // the previous definition * 2 since each node in problem could
1978 // depend at most on 2 nodes in dependProblem
1979 int* columnIndices = new int[problemMaxNodes];
1980 int maxDepNodes = 2 * problemGraph.Map().MaxAllGID();
1981 int* dependentColIndices = new int[maxDepNodes];
1982
1983 // We must loop over each dependent node of problem and then determine
1984 // the dependence of each on the nodes of dependProblem
1985
1986 // Loop over each row in problem and ascertain all dependencies on
1987 // dependProblem as determined by the xferOp map
1988 for (int i = 0; i<problemGraph.NumMyRows(); i++)
1989 {
1990 problemRow = problemGraph.Map().GID(i);
1991
1992 problemGraph.ExtractGlobalRowCopy(problemRow, problemMaxNodes,
1993 numCols, columnIndices);
1994
1995 // Convert row/column indices to composite problem
1996 compositeRow = problemIndices[problemRow];
1997 numDepCols = 0;
1998 for (int j = 0; j<numCols; j++)
1999 {
2000 pair< multimap<int, int>::iterator,
2001 multimap<int, int>::iterator > rangeN
2002 = depNodesMap.equal_range(columnIndices[j]);
2003 multimap<int, int>::iterator iterN;
2004 for( iterN = rangeN.first; iterN != rangeN.second; iterN++)
2005 dependentColIndices[numDepCols++] = dependIndices[(*iterN).second];
2006 }
2007 graphPtr->InsertGlobalIndices(compositeRow, numDepCols, dependentColIndices);
2008 offGraph.InsertGlobalIndices(compositeRow, numDepCols, dependentColIndices);
2009 }
2010 delete [] columnIndices; columnIndices = 0;
2011 delete [] dependentColIndices; dependentColIndices = 0;
2012
2013 offGraph.FillComplete();
2014 #ifdef DEBUG_PROBLEM_MANAGER
2015 offGraph.Print(cout);
2016 #endif
2017 // A new graph is created within the OffBlock_Manager; so we delete our temporary
2018 OffBlock_ManagersVec.push_back( new OffBlock_Manager(*this, offGraph,
2019 probId, dependId) );
2020 delete offGraphPtr;
2021 }
2022
2023 OffBlock_Managers[probId] = OffBlock_ManagersVec;
2024 }
2025 #endif
2026
2027 return graphPtr;
2028 }
2029
2030 //-----------------------------------------------------------------------------
2031
2032 Teuchos::RCP<Epetra_CrsGraph>
2033 Problem_Manager::composeGraphs( const Epetra_CrsGraph & graph1, const Epetra_CrsGraph & graph2 )
2034 {
2035 Teuchos::RCP<Epetra_CrsGraph> graphPtr =
2036 Teuchos::rcp( new Epetra_CrsGraph(Copy, graph1.RowMap(), 0) );
2037
2038 if( !graph1.Filled() || !graph2.Filled() )
2039 {
2040 std::cout << "ERROR: Problem_Manager::composeGraphs(...) One or both incoming graphs have not "
2041 << "been transformed to local index space by calling FillComplete()." << std::endl;
2042 throw "Problem_Manager ERROR";
2043 }
2044
2045 if( !graph1.RowMap().SameAs( graph2.RowMap() ) )
2046 {
2047 std::cout << "ERROR: Problem_Manager::composeGraphs(...) Incoming graphs do not have "
2048 << "the same row maps." << std::endl;
2049 throw "Problem_Manager ERROR";
2050 }
2051
2052 int * indices1 ;
2053 int * indices2 ;
2054 int * newIndices ;
2055 int numInd1 ;
2056 int numInd2 ;
2057 int numNewInds ;
2058 list<int> list1 ;
2059 list<int> list2 ;
2060
2061
2062 for( int row = 0; row < graph1.NumMyRows(); ++row )
2063 {
2064 list1.clear();
2065 list2.clear();
2066
2067 graph1.ExtractMyRowView( row, numInd1, indices1 );
2068 graph2.ExtractMyRowView( row, numInd2, indices2 );
2069
2070 for( int col = 0; col < numInd1; ++col )
2071 list1.push_back(graph1.ColMap().GID(indices1[col]));
2072
2073 for( int col = 0; col < numInd2; ++col )
2074 list2.push_back(graph2.ColMap().GID(indices2[col]));
2075
2076 list1.sort();
2077 list2.sort();
2078
2079 list1.merge(list2);
2080
2081 numNewInds = list1.size();
2082 newIndices = new int[ numNewInds ];
2083
2084 list<int>::iterator iter = list1.begin();
2085 for( int col = 0; col < numNewInds; )
2086 newIndices[col++] = *(iter++);
2087
2088 graphPtr->InsertGlobalIndices( graph1.RowMap().GID(row), numNewInds, newIndices );
2089
2090 delete [] newIndices;
2091 }
2092
2093 return graphPtr;
2094 }
2095
2096 //-----------------------------------------------------------------------------
2097
2098 void
2099 Problem_Manager::generateGraph()
2100 {
2101
2102 // First construct a graph for each problem's self-dependence
2103 Teuchos::RCP<Epetra_CrsGraph> diagBlkGraph = generateBlockDiagonalGraph( *compositeMap, false );
2104 diagBlkGraph->FillComplete();
2105
2106 // Next create inter-problem block graph contributions if desired;
2107 // default is false
2108 if( doOffBlocks )
2109 {
2110 Teuchos::RCP<Epetra_CrsGraph> offBlkGraph = generateOffDiagonalBlockGraph( *compositeMap );
2111 offBlkGraph->FillComplete();
2112
2113 AA = composeGraphs( *diagBlkGraph, *offBlkGraph );
2114 }
2115 else
2116 {
2117 AA = diagBlkGraph;
2118 }
2119
2120 AA->FillComplete();
2121
2122 return;
2123 }
2124
2125 //-----------------------------------------------------------------------------
2126
2127 string
2128 Problem_Manager::createIOname(GenericEpetraProblem & problem, int timeStep)
2129 {
2130
2131 int probId = problem.getId();
2132
2133 char file_name[25];
2134 (void) sprintf(file_name, "output.%03d.%03d_%05d", probId, MyPID, timeStep);
2135
2136 std::string name(file_name);
2137
2138 return name;
2139 }
2140
2141 //-----------------------------------------------------------------------------
2142
2143 void
2144 Problem_Manager::outputSolutions( const std::string outputDir, int timeStep )
2145 {
2146
2147 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemIter = Problems.begin();
2148 map<int, Teuchos::RCP<GenericEpetraProblem> >::iterator problemLast = Problems.end();
2149
2150 // Loop over each problem being managed and write its solution vector
2151 // to a file.
2152 for( ; problemIter != problemLast; problemIter++)
2153 {
2154 GenericEpetraProblem & problem = *(*problemIter).second;
2155
2156 Epetra_Vector& xMesh = problem.getMesh();
2157 Epetra_Vector& problemSoln = *problem.getSolution();
2158
2159 std::string baseName = createIOname( problem, timeStep );
2160 std::string fileName = outputDir + baseName;
2161
2162 #ifdef HAVE_NOX_EPETRAEXT
2163 NOX::Epetra::DebugTools::writeVector( fileName, problemSoln, NOX::Epetra::DebugTools::MATRIX_MARKET );
2164 fileName = outputDir + baseName + "_mesh";
2165 NOX::Epetra::DebugTools::writeVector( fileName, xMesh, NOX::Epetra::DebugTools::MATRIX_MARKET );
2166 #else
2167 int numNodes = xMesh.Map().NumMyElements();
2168 FILE *ifp;
2169 ifp = fopen(fileName.c_str(), "w");
2170 for (int i = 0; i < numNodes; i++)
2171 fprintf(ifp, "%d %E %E \n", i, xMesh[i], problemSoln[i]);
2172 fclose(ifp);
2173 #endif
2174 }
2175
2176 return;
2177 }
2178
2179 //-----------------------------------------------------------------------------
2180
2181 void
2182 Problem_Manager::outputStatus( std::ostream & os )
2183 {
2184
2185 map<int, Teuchos::RCP<GenericEpetraProblem> >::const_iterator problemIter = Problems.begin();
2186 map<int, Teuchos::RCP<GenericEpetraProblem> >::const_iterator problemLast = Problems.end();
2187
2188 os << std::endl << std::endl << "\t\t********************************" << std::endl;
2189 os << "\t\t******* PROBLEM SUMMARY ******" << std::endl;
2190 os << "\t\t********************************" << std::endl;
2191 os << std::endl << std::endl << "\t\tOff-diagonal contributions are "
2192 << ( doOffBlocks ? "Enabled" : "Disabled" ) << std::endl;
2193 os << std::endl << std::endl;
2194
2195 // Loop over each problem being managed and output its dependencies
2196 for( ; problemIter != problemLast; problemIter++)
2197 {
2198 GenericEpetraProblem & problem = *(*problemIter).second;
2199
2200 os << "\tProblem \"" << problem.getName() << "\" (" << (*problemIter).first
2201 << ")\t Depends on:" << std::endl;
2202
2203 for( unsigned int j = 0; j < problem.depProblems.size(); ++j )
2204 {
2205 GenericEpetraProblem & depProblem = *(Problems[ problem.depProblems[j] ]);
2206
2207 os << "\t\t-------------> \t\t\"" << depProblem.getName()
2208 << "\" (" << depProblem.getId() << ")" << std::endl;
2209 }
2210 os << std::endl;
2211
2212 // Allow problems to provide additional info if desired
2213 for( unsigned int j = 0; j < problem.depProblems.size(); ++j )
2214 {
2215 //GenericEpetraProblem & depProblem = *(Problems[ problem.depProblems[j] ]);
2216 problem.outputStatus(os);
2217 }
2218 os << std::endl;
2219 }
2220
2221 return;
2222 }
2223
2224 //-----------------------------------------------------------------------------
2225
2226 bool
2227 Problem_Manager::exchangeAllData()
2228 {
2229 std::cout << "Problem_Manager::exchangeAllData() .... called." << std::endl;
2230
2231 // Preceding this call, the solution vector for each problem has been placed
2232 // in each solver's group X vector. We need to copy this into each problem's
2233 // corresponding solution vector and then fire off the appropriate transfers
2234 // into auxiliary data vectors.
2235
2236 copyAllGroupXtoProblems();
2237
2238 syncAllProblems();
2239
2240 return true;
2241 }
2242
2243 //-----------------------------------------------------------------------------
2244
2245 bool
2246 Problem_Manager::exchangeDataTo(int solverId)
2247 {
2248 std::cout << "Problem_Manager::exchangeDataTo( " << solverId << " ) .... called." << std::endl;
2249
2250 // Note: the incoming solverId reflects the order in which this problem occurs
2251 // in the container of solvers passed to the coupling solver and is 0-based.
2252 // We need to convert this to 1-based nd account for any permutations of problem
2253 // ordering.
2254
2255 solverId++;
2256
2257 // Preceding this call, the solution vector for each problem has been placed
2258 // in each solver's group X vector. We need to copy this into each problem's
2259 // corresponding solution vector and then fire off the appropriate transfers
2260 // into auxiliary data vectors.
2261
2262 copyAllGroupXtoProblems();
2263
2264 Problems[solverId]->doTransfer();
2265
2266 return false;
2267 }
2268
2269 //-----------------------------------------------------------------------------
2270
2271 Teuchos::RCP<const Epetra_CrsMatrix>
2272 Problem_Manager::getReplacementValuesMatrix( const Epetra_Vector & x, FILL_TYPE )
2273 {
2274 std::cout << "Problem_Manager::getReplacementValuesMatrix(...) called." << std::endl;
2275
2276 bool ok = evaluate( NOX::Epetra::Interface::Required::Jac, &x, NULL );
2277
2278 if( !ok )
2279 {
2280 std::cout << "ERROR: Problem_Manager::getReplacementValuesMatrix call to evaluate failed."
2281 << std::endl;
2282
2283 throw "Problem_Manager ERROR";
2284 }
2285
2286 return A;
2287 }
2288
2289 //-----------------------------------------------------------------------------
2290
2291 bool
2292 Problem_Manager::applyBlockAction( int probId, int depId, const Epetra_Vector & x, Epetra_Vector & result )
2293 {
2294 // We currently rely on an existing MatrixFree jacobian operator for this action
2295 Teuchos::RCP<NOX::Epetra::MatrixFree> mfOp = Teuchos::rcp_dynamic_cast<NOX::Epetra::MatrixFree>( jacOperator );
2296 if( Teuchos::is_null(mfOp) )
2297 {
2298 std::cout << "ERROR: Problem_Manager::applyBlockAction : jacOperator is not of type NOX::Epetra::MatrixFree." << std::endl;
2299 throw "Problem_Manager ERROR";
2300 }
2301
2302 if( !x .Map().SameAs(getSolutionVec(depId).Map()) ||
2303 !result.Map().SameAs(getSolutionVec(probId).Map()) )
2304 {
2305 std::cout << "ERROR: Problem_Manager::applyBlockAction : vector sizes not compatible with domain and/or range of block dimensions." << std::endl;
2306 throw "Problem_Manager ERROR";
2307 }
2308 Teuchos::RCP<Epetra_Vector> tmpXcomposite = Teuchos::rcp( new Epetra_Vector(*compositeSoln) );
2309 Teuchos::RCP<Epetra_Vector> tmpYcomposite = Teuchos::rcp( new Epetra_Vector(*compositeSoln) );
2310
2311 tmpXcomposite->PutScalar(0.0);
2312 copyVectorToComposite( *tmpXcomposite, depId, x );
2313
2314 mfOp->Apply( *tmpXcomposite, *tmpYcomposite );
2315
2316 copyCompositeToVector( *tmpYcomposite, probId, result );
2317
2318 return true;
2319 }
2320
2321 //-----------------------------------------------------------------------------
2322
2323 bool
2324 Problem_Manager::applyBlockInverseAction( int probId, int depId, const Epetra_Vector & x, Epetra_Vector & result )
2325 {
2326 // We simply invoke the action of a previously created inverse
2327 getBlockInverseOperator(probId)->ApplyInverse(x, result);
2328
2329 return true;
2330 }
2331
2332 //-----------------------------------------------------------------------------
2333
2334 bool
2335 Problem_Manager::hasExplicitOperator( int rowBlock, int colBlock )
2336 {
2337 return ( rowBlock == colBlock );
2338 }
2339
2340 //-----------------------------------------------------------------------------
2341
2342 Teuchos::RCP<Epetra_Operator>
2343 Problem_Manager::getExplicitOperator( int rowBlock, int colBlock )
2344 {
2345 return getBlockJacobianMatrix(rowBlock);
2346 }
2347
2348 //-----------------------------------------------------------------------------
2349
2350 bool
2351 Problem_Manager::createBlockInverseOperator( int probId, Teuchos::ParameterList & pList )
2352 {
2353 // For now, we will hard-code use of Ifpack. Note that direct inverses via Amesos can be
2354 // used by propoer specification in the Teuchos::ParameterList.
2355
2356 Teuchos::RCP<Ifpack_Preconditioner> inverseOperator;
2357
2358 // Get any relevant parameters for the Inverting package using sublists
2359 Teuchos::ParameterList& teuchosParams = pList.sublist("Ifpack");
2360
2361 Ifpack Factory;
2362
2363 Teuchos::RCP<Epetra_CrsMatrix> pMatrix = getBlockJacobianMatrix(probId);
2364
2365 if( NULL == pMatrix.get() )
2366 {
2367 std::cout << "ERROR: Problem_Manager::createBlockInverseOperator : Cannot get an Epetra_CrsMatrix for diagonal block # " << probId << std::endl;
2368 throw "Problem_Manager ERROR";
2369 }
2370
2371 inverseOperator = Teuchos::rcp(Factory.Create( pList.get("Ifpack Preconditioner", "ILU"),
2372 &(*pMatrix),
2373 pList.get("Overlap", 0) ));
2374 inverseOperator->SetParameters(teuchosParams);
2375 inverseOperator->Initialize();
2376 inverseOperator->Compute();
2377
2378 BlockInverseOperators[probId] = inverseOperator;
2379
2380 return true;
2381 }
2382
2383 //-----------------------------------------------------------------------------