Debian Codegraph @

packages/shylu/shylu_node/fastilu/src/shylu_fastic.hpp

raw
1 // @HEADER
2 // *****************************************************************************
3 // ShyLU: Scalable Hybrid LU Preconditioner and Solver
4 //
5 // Copyright 2011 NTESS and the ShyLU contributors.
6 // SPDX-License-Identifier: BSD-3-Clause
7 // *****************************************************************************
8 // @HEADER
9
10 // The struct that iterates over all non-zeros for the FastIC
11 // Contact for bugs and complaints - Siva Rajamanickam ([email protected])
12 //
13 #ifndef __FAST_IC_HPP__
14 #define __FAST_IC_HPP__
15 #include <iostream>
16 #include <algorithm>
17 #include <vector>
18 #include <queue>
19 #include <random>
20 #include <cstdlib>
21 #include <cmath>
22 #include <ctime>
23 #include <string>
24
25 #include <assert.h>
26 #include <Kokkos_Core.hpp>
27 #include <Kokkos_Timer.hpp>
28
29 #include "shylu_fastilu.hpp"
30
31 //whether to print extra debug output at runtime to stdout
32 //comment out next line to disable
33 //#define FASTIC_DEBUG_OUTPUT
34
35 template<class Ordinal, class Scalar, class ExecSpace>
36 class FastICPrec
37 {
38
39 public:
40 typedef typename Teuchos::ScalarTraits<Scalar>::magnitudeType Real;
41 typedef Kokkos::View<Ordinal *, ExecSpace> OrdinalArray;
42 typedef Kokkos::View<Scalar *, ExecSpace> ScalarArray;
43 typedef Kokkos::View<Real *, ExecSpace> RealArray;
44 typedef Kokkos::View<Ordinal *, typename ExecSpace::array_layout, Kokkos::Serial,
45 Kokkos::MemoryUnmanaged> UMOrdinalArray;
46 typedef Kokkos::View<Scalar *, typename ExecSpace::array_layout, Kokkos::Serial,
47 Kokkos::MemoryUnmanaged> UMScalarArray;
48 typedef FastICPrec<Ordinal, Scalar, ExecSpace> FastPrec;
49
50 typedef Kokkos::View<Ordinal *, Kokkos::HostSpace> OrdinalArrayHost;
51 typedef Kokkos::View<Scalar *, Kokkos::HostSpace> ScalarArrayHost;
52 typedef typename OrdinalArray::host_mirror_type OrdinalArrayMirror;
53 typedef typename ScalarArray::host_mirror_type ScalarArrayMirror;
54
55 typedef Kokkos::RangePolicy<ExecSpace> RangePolicy;
56
57 using STS = Kokkos::ArithTraits<Scalar>;
58 using RTS = Kokkos::ArithTraits<Real>;
59
60 private:
61 double computeTime;
62 double applyTime;
63 double initTime;
64
65 Ordinal guessFlag;
66 Ordinal nRows;
67 Ordinal nFact;
68 Ordinal nTrisol;
69 Ordinal level;
70 Ordinal blkSzIC;
71 Ordinal blkSz;
72 Scalar omega;
73 Scalar shift;
74
75 //Lower triangular factor (CSR)
76 ScalarArray lVal;
77 OrdinalArray lColIdx;
78 OrdinalArray lRowMap;
79 // mirrors
80 ScalarArrayMirror lVal_;
81 OrdinalArrayMirror lColIdx_;
82 OrdinalArrayMirror lRowMap_;
83
84 //Upper triangular factor (CSR)
85 ScalarArray ltVal;
86 OrdinalArray ltColIdx;
87 OrdinalArray ltRowMap;
88
89 //Pointer to the original host copy of A.
90 ScalarArrayMirror aValHost;
91 OrdinalArrayMirror aRowMapHost;
92 OrdinalArrayMirror aColIdxHost;
93
94 //A matrix in COO format
95 ScalarArray aVal;
96 OrdinalArray aRowMap;
97 OrdinalArray aRowIdx;
98 OrdinalArray aColIdx;
99 // mirrors
100 ScalarArrayMirror aVal_;
101 OrdinalArrayMirror aRowMap_;
102 OrdinalArrayMirror aRowIdx_;
103 OrdinalArrayMirror aColIdx_;
104
105 //Diagonal scaling factors
106 RealArray diagFact;
107 ScalarArray diagElems;
108 ScalarArray diagElemsInv;
109 // mirrors
110 ScalarArrayMirror diagElems_;
111 ScalarArrayMirror diagElemsInv_;
112
113 //Temporary vectors for triangular solves
114 ScalarArray xOld;
115 ScalarArray xTemp;
116 ScalarArray onesVector;
117
118 //This will have the continuation initial
119 //guess if guessFlag=1
120 Teuchos::RCP<FastPrec> initGuessPrec;
121
122 // forward/backwar substitution for standard SpTrsv
123 using MemSpace = typename ExecSpace::memory_space;
124 using KernelHandle = KokkosKernels::Experimental::KokkosKernelsHandle <Ordinal, Ordinal, Scalar, ExecSpace, MemSpace, MemSpace >;
125 bool standard_sptrsv;
126 KernelHandle khL;
127 KernelHandle khLt;
128
129 public:
130 FastICPrec(OrdinalArray &aRowMapIn, OrdinalArray &aColIdxIn, ScalarArray &aValIn, Ordinal nRow_, bool standard_sptrsv_,
131 Ordinal nFact_, Ordinal nTrisol_, Ordinal level_, Scalar omega_, Scalar shift_, Ordinal guessFlag_, Ordinal blkSzIC_, Ordinal blkSz_)
132 {
133 nRows = nRow_;
134 standard_sptrsv = standard_sptrsv_;
135 nFact = nFact_;
136 nTrisol = nTrisol_;
137 computeTime = 0.0;
138 applyTime = 0.0;
139 initTime = 0.0;
140 level = level_;
141
142 // mirror & deep-copy the input matrix
143 aRowMapHost = Kokkos::create_mirror_view(aRowMapIn);
144 aColIdxHost = Kokkos::create_mirror_view(aColIdxIn);
145 aValHost = Kokkos::create_mirror_view(aValIn);
146 Kokkos::deep_copy(aRowMapHost, aRowMapIn);
147 Kokkos::deep_copy(aColIdxHost, aColIdxIn);
148 Kokkos::deep_copy(aValHost, aValIn);
149
150 omega = omega_;
151 guessFlag = guessFlag_;
152 shift = shift_;
153 blkSzIC = blkSzIC_;
154 blkSz = blkSz_;
155
156 const Scalar one = STS::one();
157 onesVector = ScalarArray("onesVector", nRow_);
158 Kokkos::deep_copy(onesVector, one);
159
160 diagFact = RealArray("diagFact", nRow_);
161 diagElems = ScalarArray("diagElems", nRow_);
162 diagElemsInv = ScalarArray("diagElems", nRow_);
163 xOld = ScalarArray("xOld", nRow_);
164 xTemp = ScalarArray("xTemp", nRow_);
165
166 if (level > 0)
167 {
168 initGuessPrec = Teuchos::rcp(new FastPrec(aRowMapIn, aColIdxIn, aValIn, nRow_, standard_sptrsv, 3, 5,
169 level_-1, omega_, shift_, guessFlag_, blkSzIC_, blkSz_));
170 }
171
172 }
173
174 void initialize()
175 {
176 Kokkos::Timer timer;
177 //Here we have to allocate memory for L, U.
178 if((level > 0) && (guessFlag != 0))
179 {
180 initGuessPrec->initialize();
181 }
182 //symbolicILUAndInit(); //Note the diagonal scaling is now embedded in this routine.
183 symbolicILU();
184 //initialize L, U, A patterns
185 #ifdef SHYLU_DEBUG
186 MemoryPrimeFunctorN<Ordinal, Scalar, ExecSpace> copyFunc1(aRowMap, lRowMap, lRowMap, diagElems);
187
188 //Make sure all memory resides on the device
189 ExecSpace().fence();
190 Kokkos::parallel_for(RangePolicy(0, nRows), copyFunc1);
191
192 //Note that the following is a temporary measure
193 //to ensure that memory resides on the device.
194 Ordinal nnzL = lRowMap[nRows];
195 Ordinal nnzA = aRowMap[nRows];
196
197 MemoryPrimeFunctorNnzCoo<Ordinal, Scalar, ExecSpace> copyFunc2(aColIdx, aRowIdx, aVal);
198 MemoryPrimeFunctorNnzCsr<Ordinal, Scalar, ExecSpace> copyFunc3(lColIdx, lVal);
199
200 ExecSpace().fence();
201 Kokkos::parallel_for(RangePolicy(0, nRows), copyFunc1);
202 Kokkos::parallel_for(RangePolicy(0, nnzA), copyFunc2);
203 Kokkos::parallel_for(RangePolicy(0, nnzL), copyFunc3);
204 #endif
205 double t = timer.seconds();
206
207 #ifdef FASTIC_DEBUG_OUTPUT
208 std::cout << "Symbolic phase complete." << std::endl;
209 std::cout << "Init time: "<< t << "s" << std::endl;
210 #endif
211 initTime = t;
212 }
213
214 void getL(OrdinalArray &lRowMapArg, OrdinalArray &lColIdxArg, ScalarArray &lValArg)
215 {
216 lRowMapArg = lRowMap;
217 lColIdxArg = lColIdx;
218 lValArg = lVal;
219 }
220
221 void getD(ScalarArray &diagElemsArg)
222 {
223 diagElemsArg = diagElems;
224 }
225
226 void transposeL()
227 {
228 auto ltRowMap_ = Kokkos::create_mirror_view(ltRowMap);
229 auto ltColIdx_ = Kokkos::create_mirror_view(ltColIdx);
230 auto ltVal_ = Kokkos::create_mirror_view(ltVal);
231
232 //Count the elements in each row of Lt
233 auto temp = OrdinalArrayHost("temp", nRows + 1);
234 auto rowPtrs = OrdinalArrayHost("rowPtrs", nRows);
235 for (Ordinal i = 0; i <= nRows; i++)
236 {
237 temp[i] = 0;
238 }
239 for (Ordinal i = 0; i < nRows; i++)
240 {
241 for (Ordinal k = lRowMap_[i]; k < lRowMap_[i+1]; k++)
242 {
243 temp[lColIdx_[k]+1]++;
244
245 }
246 }
247 //Perform an add scan to get the row map for
248 //the transpose
249 for (Ordinal i = 0; i <= nRows; i++)
250 {
251 ltRowMap_[i] = temp[i];
252 }
253 for (Ordinal i = 1; i <= nRows; i++)
254 {
255 ltRowMap_[i] += ltRowMap_[i-1];
256 }
257 //Set the row pointers to their initial places;
258 for (Ordinal i = 0; i < nRows; i++)
259 {
260 rowPtrs[i] = ltRowMap_[i];
261 }
262 //Copy the data
263 Kokkos::deep_copy(lVal_, lVal);
264 for (Ordinal i = 0; i < nRows; i++)
265 {
266 for (Ordinal k = lRowMap_[i]; k < lRowMap_[i+1]; k++)
267 {
268 Ordinal row = lColIdx_[k];
269 Scalar value = lVal_[k];
270 ltVal_[rowPtrs[row]] = value;
271 ltColIdx_[rowPtrs[row]] = i;
272 rowPtrs[row]++;
273 assert(rowPtrs[row] <= ltRowMap_[row + 1]);
274 }
275 }
276 Kokkos::deep_copy(ltRowMap, ltRowMap_);
277 Kokkos::deep_copy(ltColIdx, ltColIdx_);
278 Kokkos::deep_copy(ltVal,ltVal_);
279 }
280
281 void symbolicILU()
282 {
283 using std::vector;
284 using std::cout;
285 using std::stable_sort;
286 using std::sort;
287 OrdinalArrayMirror ia = aRowMapHost;
288 OrdinalArrayMirror ja = aColIdxHost;
289 int n = nRows;
290 int nzl = aRowMapHost[nRows] * (level + 2);
291 int nzu = aRowMapHost[nRows] * (level + 2);
292 Ordinal i;
293 int levfill = level;
294 vector<int> lnklst(n);
295 vector<int> curlev(n);
296 vector<int> levels(nzu);
297 vector<int> iwork(n);
298 vector<int> ial(nzl);
299 vector<int> jal(nzl);
300 vector<int> iau(nzu);
301 vector<int> jau(nzu);
302
303 int knzl = 0;
304 int knzu = 0;
305
306 ial[0] = 0;
307 iau[0] = 0;
308
309 for (i=0; i<n; i++)
310 {
311 int first, next, j;
312
313 /* copy column indices of row into workspace and sort them */
314
315 int len = ia[i+1] - ia[i];
316 next = 0;
317 for (j=ia[i]; j<ia[i+1]; j++)
318 iwork[next++] = ja[j];
319 // shell_sort(len, iwork);
320 //stable_sort(iwork.begin(), iwork.begin() + len);
321 sort(iwork.begin(), iwork.begin() + len);
322
323 /* construct implied linked list for row */
324
325 first = iwork[0];
326 curlev[first] = 0;
327
328 for (j=0; j<=len-2; j++)
329 {
330 lnklst[iwork[j]] = iwork[j+1];
331 curlev[iwork[j]] = 0;
332 }
333
334 lnklst[iwork[len-1]] = n;
335 curlev[iwork[len-1]] = 0;
336
337 /* merge with rows in U */
338
339 next = first;
340 while (next < i)
341 {
342 int oldlst = next;
343 int nxtlst = lnklst[next];
344 int row = next;
345 int ii;
346
347 /* scan row */
348
349 for (ii=iau[row]+1; ii<iau[row+1]; /*nop*/)
350 {
351 if (jau[ii] < nxtlst)
352 {
353 /* new fill-in */
354 int newlev = curlev[row] + levels[ii] + 1;
355 if (newlev <= levfill)
356 {
357 lnklst[oldlst] = jau[ii];
358 lnklst[jau[ii]] = nxtlst;
359 oldlst = jau[ii];
360 curlev[jau[ii]] = newlev;
361 }
362 ii++;
363 }
364 else if (jau[ii] == nxtlst)
365 {
366 int newlev;
367 oldlst = nxtlst;
368 nxtlst = lnklst[oldlst];
369 newlev = curlev[row] + levels[ii] + 1;
370 //curlev[jau[ii]] = MIN(curlev[jau[ii]], newlev);
371 if (curlev[jau[ii]] > newlev)
372 {
373 curlev[jau[ii]] = newlev;
374 }
375 ii++;
376 }
377 else /* (jau[ii] > nxtlst) */
378 {
379 oldlst = nxtlst;
380 nxtlst = lnklst[oldlst];
381 }
382 }
383 next = lnklst[next];
384 }
385
386 /* gather the pattern into L and U */
387
388 next = first;
389 while (next < i)
390 {
391 assert(knzl < nzl);
392 jal[knzl++] = next;
393 if (knzl >= nzl)
394 {
395 //(*nzl)++;
396 nzl += nRows;
397 jal.resize(nzl);
398 }
399 //jal.push_back(next);
400 // knzl++;
401 next = lnklst[next];
402 }
403 ial[i+1] = knzl;
404 assert(next == i);
405
406 while (next < n)
407 {
408 assert(knzu < nzu);
409 levels[knzu] = curlev[next];
410 jau[knzu++] = next;
411 if (knzu >= nzu)
412 {
413 //(*nzu)++;
414 nzu += nRows;
415 jau.resize(nzu);
416 levels.resize(nzu);
417 }
418 //jau.push_back(next);
419 //knzu++;
420 next = lnklst[next];
421 }
422 iau[i+1] = knzu;
423 }
424
425 nzl = knzl;
426 nzu = knzu;
427
428 #ifdef FASTIC_DEBUG_OUTPUT
429 std::cout << "knzl =" << knzl;
430 std::cout << "knzu =" << knzu;
431 std::cout << "ILU: nnz = "<< knzl + knzu << std::endl;
432 std::cout << "Actual nnz for ILU: " << nzl + nzu << std::endl;
433 #endif
434 //Initialize the A matrix that is to be used in the computation
435 aRowMap = OrdinalArray("aRowMap", nRows + 1);
436 aColIdx = OrdinalArray("aColIdx", knzl + knzu);
437 aRowIdx = OrdinalArray("aRowIds", knzl + knzu);
438 aRowMap_ = Kokkos::create_mirror_view(aRowMap);
439 aColIdx_ = Kokkos::create_mirror_view(aColIdx);
440 aRowIdx_ = Kokkos::create_mirror_view(aRowIdx);
441
442 Ordinal aRowPtr = 0;
443 aRowMap_[0] = 0;
444 for (i = 0; i < nRows; i++)
445 {
446 #ifdef FASTIC_DEBUG_OUTPUT
447 std::cout << "***row:" << i << std::endl;
448 #endif
449 for(Ordinal k = ial[i]; k < ial[i+1]; k++)
450 {
451 #ifdef FASTIC_DEBUG_OUTPUT
452 std::cout << "jal[k]=" << jal[k] << std::endl;
453 #endif
454 aColIdx_[aRowPtr] = jal[k];
455 aRowIdx_[aRowPtr] = i;
456 aRowPtr++;
457 }
458 for(Ordinal k = iau[i]; k < iau[i+1]; k++)
459 {
460 aColIdx_[aRowPtr] = jau[k];
461 aRowIdx_[aRowPtr] = i;
462 aRowPtr++;
463 }
464 aRowMap_[i+1] = aRowPtr;
465 }
466 Kokkos::deep_copy(aRowMap, aRowMap_);
467 Kokkos::deep_copy(aColIdx, aColIdx_);
468 Kokkos::deep_copy(aRowIdx, aRowIdx_);
469
470 //Now allocate memory for L and U.
471 //
472 lRowMap = OrdinalArray("lRowMap", nRows + 1);
473 ltRowMap = OrdinalArray("ltRowMap", nRows + 1);
474 countL();
475 //Allocate memory and initialize pattern for L, U (transpose).
476 lColIdx = OrdinalArray("lColIdx", lRowMap_[nRows]);
477 ltColIdx = OrdinalArray("ltColIdx", lRowMap_[nRows]);
478 #ifdef FASTIC_DEBUG_OUTPUT
479 std::cout << "nnz L = " << lRowMap[nRows] << std::endl;
480 #endif
481 }
482
483 void numericILU()
484 {
485 aVal = ScalarArray("aVal", aColIdx.extent(0));
486 aVal_ = Kokkos::create_mirror_view(aVal);
487
488 //Copy the host matrix into the initialized a;
489 for (Ordinal i = 0; i < nRows; i++)
490 {
491 for(Ordinal k = aRowMap_[i]; k < aRowMap_[i+1]; k++)
492 {
493 Ordinal col = aColIdx_[k];
494 #ifdef FASTIC_DEBUG_OUTPUT
495 std::cout << "col =" << col << std::endl;
496 #endif
497 for(Ordinal aHostPtr = aRowMapHost[i]; aHostPtr < aRowMapHost[i+1]; aHostPtr++)
498 {
499 if (col == aColIdxHost[aHostPtr])
500 {
501 aVal_[k] = aValHost[aHostPtr];
502 break;
503 }
504 }
505 }
506 }
507 lVal = ScalarArray("lVal", lRowMap_[nRows]);
508 ltVal = ScalarArray("ltVal", lRowMap_[nRows]);
509 applyDiagonalScaling();
510 applyManteuffelShift();
511 Kokkos::deep_copy(aVal, aVal_);
512
513 fillL();
514 }
515
516 void countL()
517 {
518 lRowMap_ = Kokkos::create_mirror_view(lRowMap);
519
520 lRowMap_[0] = 0;
521 for (Ordinal i = 0; i < nRows; i++)
522 {
523 Ordinal row_count = 0;
524 for (Ordinal k = aRowMap_[i]; k < aRowMap_[i+1]; k++)
525 {
526 Ordinal row = i;
527 Ordinal col = aColIdx_[k];
528
529 if (row >= col)
530 {
531 row_count++;
532 }
533 }
534 lRowMap_[i+1] = lRowMap_[i] + row_count;
535 }
536 Kokkos::deep_copy(lRowMap, lRowMap_);
537 }
538
539 void fillL()
540 {
541 lVal_ = Kokkos::create_mirror_view(lVal);
542 lColIdx_ = Kokkos::create_mirror_view(lColIdx);
543 diagElems_ = Kokkos::create_mirror_view(diagElems);
544
545 Ordinal lPtr = 0;
546 for (Ordinal i = 0; i < nRows; i++)
547 {
548 for (Ordinal k = aRowMap_[i]; k < aRowMap_[i+1]; k++)
549 {
550 Ordinal row = i;
551 Ordinal col = aColIdx_[k];
552
553 if (row >= col)
554 {
555 if (row == col)
556 {
557 diagElems_[row] = aVal_[k];
558 }
559 lVal_[lPtr] = aVal_[k];
560 lColIdx_[lPtr] = aColIdx_[k];
561 lPtr++;
562 }
563 }
564 }
565 Kokkos::deep_copy(diagElems, diagElems_);
566 assert(lPtr == lRowMap_[nRows]);
567
568 if ((level > 0) && (guessFlag !=0))
569 {
570 OrdinalArray lGRowMap;
571 OrdinalArray lGColIdx;
572 ScalarArray lGVal;
573 ScalarArray gD;
574 Ordinal lGPtr = 0;
575
576 initGuessPrec->getL(lGRowMap, lGColIdx, lGVal);
577 initGuessPrec->getD(gD);
578
579 Kokkos::deep_copy(diagElems, gD);
580
581 auto lGColIdx_ = Kokkos::create_mirror_view(lGColIdx);
582 auto lGVal_ = Kokkos::create_mirror_view(lGVal);
583 Kokkos::deep_copy(lGColIdx_, lGColIdx);
584 Kokkos::deep_copy(lGVal_, lGVal);
585 for (Ordinal i = 0; i < nRows; i++)
586 {
587 #ifdef SHYLU_DEBUG
588 Ordinal check = lGPtr;
589 #endif
590 for (Ordinal k = lRowMap_[i]; k < lRowMap_[i+1]; k++)
591 {
592 //unused: Ordinal row = i;
593 Ordinal col = lColIdx_[k];
594
595 if (col == lGColIdx_[lGPtr])
596 {
597 lVal_[k] = lGVal_[lGPtr];
598 lGPtr++;
599 }
600 }
601 #ifdef SHYLU_DEBUG
602 rowLen = lGPtr - check;
603 assert(rowLen == lGRowMap[i+1] - lGRowMap[i]);
604 #endif
605 }
606 }
607 Kokkos::deep_copy(lColIdx, lColIdx_);
608 Kokkos::deep_copy(lVal, lVal_);
609 }
610
611 void applyDiagonalScaling()
612 {
613 const Real one = RTS::one();
614 int anext = 0;
615 //First fill Aj and extract the diagonal scaling factors
616 //Use diag array to store scaling factors since
617 //it gets set to the correct value by findFactorPattern anyway.
618 auto diagFact_ = Kokkos::create_mirror_view(diagFact);
619 for (int i = 0; i < nRows; i++)
620 {
621 for(int k = aRowMap_[i]; k < aRowMap_[i+1]; k++)
622 {
623 aRowIdx_[anext++] = i;
624 if (aColIdx_[k] == i)
625 {
626 diagFact_[i] = one/RTS::sqrt(STS::abs(aVal_[k]));
627 #ifdef FASTIC_DEBUG_OUTPUT
628 std::cout << "diagFact["<<i<<"]="<<aVal[k]<<std::endl;
629 #endif
630 }
631 }
632 }
633
634 //Now go through each element of A and apply the scaling
635 int row;
636 int col;
637 Real sc1, sc2;
638 for (int i = 0; i < nRows; i++)
639 {
640 for (int k = aRowMap_[i]; k < aRowMap_[i+1]; k++)
641 {
642 row = aRowIdx_[k];
643 col = aColIdx_[k];
644
645 sc1 = diagFact_[row];
646 sc2 = diagFact_[col];
647 aVal_[k] = aVal_[k]*sc1*sc2;
648 }
649 }
650 Kokkos::deep_copy(diagFact, diagFact_);
651 }
652
653 void applyManteuffelShift()
654 {
655 const Scalar one = STS::one();
656 //Scalar shift = 0.05;
657 for (Ordinal i = 0; i < nRows; i++)
658 {
659 for (Ordinal k = aRowMap_[i]; k < aRowMap_[i+1]; k++)
660 {
661 Ordinal row = i;
662 Ordinal col = aColIdx_[k];
663 if (row != col)
664 {
665 aVal_[k] = (one/(one + shift))*aVal_[k];
666 }
667 }
668 }
669 }
670
671 void applyDD(ScalarArray &x, ScalarArray &y)
672 {
673 ParScalFunctor<Ordinal, Scalar, Scalar, ExecSpace> parScal(x, y, diagElemsInv);
674 ExecSpace().fence();
675 Kokkos::parallel_for(RangePolicy(0, nRows), parScal);
676 ExecSpace().fence();
677
678 }
679 void applyD(ScalarArray &x, ScalarArray &y)
680 {
681 ParScalFunctor<Ordinal, Scalar, Real, ExecSpace> parScal(x, y, diagFact);
682 ExecSpace().fence();
683 Kokkos::parallel_for(RangePolicy(0, nRows), parScal);
684 ExecSpace().fence();
685
686 }
687 void applyLIC(ScalarArray &x, ScalarArray &y)
688 {
689 ParInitZeroFunctor<Ordinal, Scalar, ExecSpace> parInitZero(xOld);
690 Kokkos::parallel_for(RangePolicy(0, nRows), parInitZero);
691 ExecSpace().fence();
692 #if 0
693 JacobiIterFunctor<Ordinal, Scalar, ExecSpace> jacIter(nRows, lRowMap, lColIdx, lVal,
694 x, y, xOld, diagElems);
695 #endif
696 BlockJacobiIterFunctorL<Ordinal, Scalar, ExecSpace> jacIter(nRows, blkSz, lRowMap,
697 lColIdx, lVal,
698 x, y, xOld, diagElems);
699 ParCopyFunctor<Ordinal, Scalar, ExecSpace> parCopy(xOld, y);
700 Ordinal extent = nRows/blkSz;
701 if(nRows%blkSz != 0)
702 {
703 extent++;
704 }
705 ExecSpace().fence();
706 for (Ordinal i = 0; i < nTrisol; i++)
707 {
708 Kokkos::parallel_for(RangePolicy(0, extent), jacIter);
709 ExecSpace().fence();
710 Kokkos::parallel_for(RangePolicy(0, nRows), parCopy);
711 ExecSpace().fence();
712 }
713 return;
714 }
715
716 void applyLT(ScalarArray &x, ScalarArray &y)
717 {
718 ParInitZeroFunctor<Ordinal, Scalar, ExecSpace> parInitZero(xOld);
719 Kokkos::parallel_for(RangePolicy(0, nRows), parInitZero);
720 ExecSpace().fence();
721 #if 0
722 JacobiIterFunctor<Ordinal, Scalar, ExecSpace> jacIter(nRows, ltRowMap, ltColIdx, ltVal,
723 x, y, xOld, diagElems);
724 #endif
725 BlockJacobiIterFunctorU<Ordinal, Scalar, ExecSpace> jacIter(nRows, blkSz, ltRowMap, ltColIdx,
726 ltVal, x, y, xOld, diagElems);
727 ParCopyFunctor<Ordinal, Scalar, ExecSpace> parCopy(xOld, y);
728 Ordinal extent = nRows/blkSz;
729 if (nRows%blkSz != 0)
730 {
731 extent++;
732 }
733 ExecSpace().fence();
734 for (Ordinal i = 0; i < nTrisol; i++)
735 {
736 Kokkos::parallel_for(RangePolicy(0, extent), jacIter);
737 ExecSpace().fence();
738 Kokkos::parallel_for(RangePolicy(0, nRows), parCopy);
739 ExecSpace().fence();
740 }
741 return;
742 }
743
744
745 #if 0
746 void applyLT(ScalarArray &x, ScalarArray &y)
747 {
748 ParInitZeroFunctor<Ordinal, Scalar, ExecSpace> parInitZero(nRows, xOld);
749 ParInitZeroFunctor<Ordinal, Scalar, ExecSpace> parInitZero1(nRows, y);
750 Kokkos::parallel_for(RangePolicy(0, nRows), parInitZero);
751 ExecSpace().fence();
752 //Transpose Jacobi implementation
753 JacobiIterFunctorT<Ordinal, Scalar, ExecSpace> jacIterT(nRows, lRowMap, lColIdx, lVal, x, y, xOld, onesVector);
754 ParCopyFunctor<Ordinal, Scalar, ExecSpace> parCopy(nRows, xOld, y);
755
756 ExecSpace().fence();
757
758 for (Ordinal i = 0; i < nTrisol; i++)
759 {
760 Kokkos::parallel_for(RangePolicy(0, nRows), parInitZero1);
761 ExecSpace().fence();
762 Kokkos::parallel_for(RangePolicy(0, nRows), jacIterT);
763 ExecSpace().fence();
764 Kokkos::parallel_for(RangePolicy(0, nRows), parCopy);
765 ExecSpace().fence();
766 }
767
768 return;
769 }
770 #endif
771
772 void setValues(ScalarArray& aValsIn)
773 {
774 this->aValHost = Kokkos::create_mirror_view(aValsIn);
775 Kokkos::deep_copy(this->aValHost, aValsIn);
776 if(!initGuessPrec.is_null())
777 {
778 initGuessPrec->setValues(aValsIn);
779 }
780 }
781
782 void compute()
783 {
784 const Scalar one = STS::one();
785 Kokkos::Timer timer;
786 if((level > 0) && (guessFlag != 0))
787 {
788 initGuessPrec->compute();
789 }
790 numericILU();
791 FastICFunctor<Ordinal, Scalar, ExecSpace> icFunctor(aRowMap_[nRows], blkSzIC,
792 aRowMap, aColIdx, aRowIdx, aVal, lRowMap, lColIdx, lVal, diagElems, omega);
793 ExecSpace().fence();
794 Ordinal extent = aRowMap_[nRows]/blkSzIC;
795 if (aRowMap_[nRows]%blkSzIC != 0)
796 {
797 extent++;
798 }
799 //Ordinal extent = aRowMap[nRows];
800 for (int i = 0; i < nFact; i++)
801 {
802 Kokkos::parallel_for(RangePolicy(0, extent), icFunctor);
803 }
804 ExecSpace().fence();
805
806 double t = timer.seconds();
807 computeTime = t;
808
809 Kokkos::deep_copy(diagElems_, diagElems);
810 diagElemsInv_ = Kokkos::create_mirror_view(diagElemsInv);
811 for (int i = 0; i < nRows; i++)
812 {
813 diagElemsInv_[i] = one/diagElems_[i];
814 }
815 Kokkos::deep_copy(diagElemsInv, diagElemsInv_);
816 transposeL();
817
818 if (standard_sptrsv) {
819 #if defined(KOKKOSKERNELS_ENABLE_TPL_CUSPARSE)
820 KokkosSparse::Experimental::SPTRSVAlgorithm algo = KokkosSparse::Experimental::SPTRSVAlgorithm::SPTRSV_CUSPARSE;
821 #else
822 KokkosSparse::Experimental::SPTRSVAlgorithm algo = KokkosSparse::Experimental::SPTRSVAlgorithm::SEQLVLSCHD_TP1;
823 #endif
824 // setup L solve
825 khL.create_sptrsv_handle(algo, nRows, true);
826 #if defined(KOKKOSKERNELS_ENABLE_TPL_CUSPARSE)
827 KokkosSparse::Experimental::sptrsv_symbolic(&khL, lRowMap, lColIdx, lVal);
828 #else
829 KokkosSparse::Experimental::sptrsv_symbolic(&khL, lRowMap, lColIdx);
830 #endif
831 // setup Lt solve
832 khLt.create_sptrsv_handle(algo, nRows, false);
833 #if defined(KOKKOSKERNELS_ENABLE_TPL_CUSPARSE)
834 KokkosSparse::Experimental::sptrsv_symbolic(&khLt, ltRowMap, ltColIdx, ltVal);
835 #else
836 KokkosSparse::Experimental::sptrsv_symbolic(&khLt, ltRowMap, ltColIdx);
837 #endif
838 }
839 }
840
841 void apply(ScalarArray &x, ScalarArray &y)
842 {
843 Kokkos::Timer timer;
844 ParCopyFunctor<Ordinal, Scalar, ExecSpace> parCopyFunctor(xTemp, x);
845 ExecSpace().fence();
846 Kokkos::parallel_for(RangePolicy(0, nRows), parCopyFunctor);
847 ExecSpace().fence();
848
849 applyD(x, xTemp);
850 if (standard_sptrsv) {
851 // solve with L
852 KokkosSparse::Experimental::sptrsv_solve(&khL, lRowMap, lColIdx, lVal, xTemp, y);
853 // solve with Lt
854 KokkosSparse::Experimental::sptrsv_solve(&khLt, ltRowMap, ltColIdx, ltVal, y, xTemp);
855 } else {
856 applyLIC(xTemp, y);
857 //applyDD(y, xTemp);
858 applyLT(y, xTemp);
859 }
860 applyD(xTemp, y);
861
862 // ParCopyFunctor<Ordinal, Scalar, ExecSpace> parCopyFunctor2(nRows, y, xTemp);
863 // ExecSpace().fence();
864 // Kokkos::parallel_for(RangePolicy(0, nRows), parCopyFunctor2);
865 // ExecSpace().fence();
866
867 double t = timer.seconds();
868 applyTime = t;
869 return;
870 }
871
872 Ordinal getNFact() const
873 {
874 return nFact;
875 }
876
877 std::string getSpTrsvType() const
878 {
879 return (standard_sptrsv ? "Standard" : "Fast");
880 }
881
882 Ordinal getNTrisol() const
883 {
884 return nTrisol;
885 }
886
887 Ordinal getNRows() const
888 {
889 return nRows;
890 }
891
892 double getComputeTime() const
893 {
894 return computeTime;
895 }
896
897 double getInitializeTime() const
898 {
899 return initTime;
900 }
901
902 double getApplyTime() const
903 {
904 return applyTime;
905 }
906 void checkIC() const
907 {
908 //Compute the L2 norm of the nonlinear residual (A - LLt) on sparsity pattern
909 //
910 Scalar sum = STS::zero();
911 for (int i = 0 ; i < nRows; i++)
912 {
913 int row = i;
914 // Each row in A matrix (result)
915 for (int k = aRowMap[i]; k < aRowMap[i+1]; k++)
916 {
917
918 int col = aColIdx[k];
919
920 if (row >= col) {
921 Scalar acc_val = aVal[k];
922 Ordinal lptr, uptr;
923 for ( lptr = lRowMap[i], uptr = lRowMap[aColIdx[k]] ;
924 lptr < lRowMap[i+1] && uptr < lRowMap[aColIdx[k]+1] ; )
925 {
926 if (lColIdx[lptr] == lColIdx[uptr])
927 {
928 acc_val -= lVal[lptr] * lVal[uptr];
929 lptr++;
930 uptr++;
931 }
932 else if (lColIdx[lptr] < lColIdx[uptr])
933 lptr++;
934 else
935 uptr++;
936 }
937
938 sum += acc_val * acc_val;
939 }
940 }
941 }
942 #ifdef FASTIC_DEBUG_OUTPUT
943 std::cout << "l2 norm of nonlinear residual = " << std::sqrt(sum) << std::endl;
944 #endif
945 }
946 friend class FastICFunctor<Ordinal, Scalar, ExecSpace>;
947 friend class JacobiIterFunctor<Ordinal, Scalar, ExecSpace>;
948 friend class ParCopyFunctor<Ordinal, Scalar, ExecSpace>;
949 friend class JacobiIterFunctorT<Ordinal, Scalar, ExecSpace>;
950 friend class ParScalFunctor<Ordinal, Scalar, Real, ExecSpace>;
951 friend class MemoryPrimeFunctorN<Ordinal, Scalar, ExecSpace>;
952 friend class MemoryPrimeFunctorNnzCoo<Ordinal, Scalar, ExecSpace>;
953 friend class MemoryPrimeFunctorNnzCsr<Ordinal, Scalar, ExecSpace>;
954 };
955
956
957 #endif