Debian Codegraph @

packages/shylu/shylu_node/fastilu/src/shylu_fastildl.hpp

raw
1 // @HEADER
2 // *****************************************************************************
3 // ShyLU: Scalable Hybrid LU Preconditioner and Solver
4 //
5 // Copyright 2011 NTESS and the ShyLU contributors.
6 // SPDX-License-Identifier: BSD-3-Clause
7 // *****************************************************************************
8 // @HEADER
9
10 // The struct that iterates over all non-zeros for the FastILDL
11 // Contact for bugs and complaints - Siva Rajamanickam ([email protected])
12 //
13 #ifndef __FAST_ILDL_HPP__
14 #define __FAST_ILDL_HPP__
15 #include <iostream>
16 #include <algorithm>
17 #include <vector>
18 #include <queue>
19 #include <random>
20 #include <cstdlib>
21 #include <cmath>
22 #include <ctime>
23 #include <string>
24
25 #include <assert.h>
26 #include <Kokkos_Core.hpp>
27 #include <Kokkos_Timer.hpp>
28
29 #include "shylu_fastilu.hpp"
30
31 //whether to print extra debug output at runtime to stdout
32 //comment out next line to disable
33 //#define FASTILDL_DEBUG_OUTPUT
34
35 template<class Ordinal, class Scalar, class ExecSpace>
36 class FastILDLFunctor;
37
38 template<class Ordinal, class Scalar, class ExecSpace>
39 class FastILDLPrec
40 {
41
42 public:
43 typedef typename Teuchos::ScalarTraits<Scalar>::magnitudeType Real;
44 typedef Kokkos::View<Ordinal *, ExecSpace> OrdinalArray;
45 typedef Kokkos::View<Scalar *, ExecSpace> ScalarArray;
46 typedef Kokkos::View<Real *, ExecSpace> RealArray;
47 typedef Kokkos::View<Ordinal *, typename ExecSpace::array_layout, Kokkos::Serial,
48 Kokkos::MemoryUnmanaged> UMOrdinalArray;
49 typedef Kokkos::View<Scalar *, typename ExecSpace::array_layout, Kokkos::Serial,
50 Kokkos::MemoryUnmanaged> UMScalarArray;
51 typedef FastILDLPrec<Ordinal, Scalar, ExecSpace> FastPrec;
52
53 typedef Kokkos::View<Ordinal *, Kokkos::HostSpace> OrdinalArrayHost;
54 typedef Kokkos::View<Scalar *, Kokkos::HostSpace> ScalarArrayHost;
55 typedef typename OrdinalArray::host_mirror_type OrdinalArrayMirror;
56 typedef typename ScalarArray::host_mirror_type ScalarArrayMirror;
57
58 typedef Kokkos::RangePolicy<ExecSpace> RangePolicy;
59
60 using STS = Kokkos::ArithTraits<Scalar>;
61 using RTS = Kokkos::ArithTraits<Real>;
62
63 private:
64 double computeTime;
65 double applyTime;
66 double initTime;
67
68 Ordinal guessFlag;
69 Ordinal nRows;
70 Ordinal nFact;
71 Ordinal nTrisol;
72 Ordinal level;
73 Ordinal blkSz;
74 Ordinal blkSzILDL;
75 Scalar omega;
76 Scalar shift;
77
78 //Lower triangular factor (CSR)
79 ScalarArray lVal;
80 OrdinalArray lColIdx;
81 OrdinalArray lRowMap;
82 // mirrors
83 ScalarArrayMirror lVal_;
84 OrdinalArrayMirror lColIdx_;
85 OrdinalArrayMirror lRowMap_;
86
87 //Upper triangular factor (CSR)
88 ScalarArray ltVal;
89 OrdinalArray ltColIdx;
90 OrdinalArray ltRowMap;
91
92 //Pointer to the original host copy of A.
93 ScalarArrayMirror aValHost;
94 OrdinalArrayMirror aRowMapHost;
95 OrdinalArrayMirror aColIdxHost;
96
97 //A matrix in COO format
98 ScalarArray aVal;
99 OrdinalArray aRowMap;
100 OrdinalArray aRowIdx;
101 OrdinalArray aColIdx;
102 // mirrors
103 ScalarArrayMirror aVal_;
104 OrdinalArrayMirror aRowMap_;
105 OrdinalArrayMirror aRowIdx_;
106 OrdinalArrayMirror aColIdx_;
107
108 //Diagonal scaling factors
109 RealArray diagFact;
110 ScalarArray diagElems;
111 ScalarArray diagElemsInv;
112 // mirrors
113 ScalarArrayMirror diagElems_;
114 ScalarArrayMirror diagElemsInv_;
115
116 //Temporary vectors for triangular solves
117 ScalarArray xOld;
118 ScalarArray xTemp;
119 ScalarArray onesVector;
120
121 //This will have the continuation initial
122 //guess if guessFlag=1
123 Teuchos::RCP<FastPrec> initGuessPrec;
124
125 // forward/backwar substitution for standard SpTrsv
126 using MemSpace = typename ExecSpace::memory_space;
127 using KernelHandle = KokkosKernels::Experimental::KokkosKernelsHandle <Ordinal, Ordinal, Scalar, ExecSpace, MemSpace, MemSpace >;
128 bool standard_sptrsv;
129 KernelHandle khL;
130 KernelHandle khLt;
131
132 public:
133 FastILDLPrec(OrdinalArray &aRowMapIn, OrdinalArray &aColIdxIn, ScalarArray &aValIn, Ordinal nRow_, bool standard_sptrsv_,
134 Ordinal nFact_, Ordinal nTrisol_, Ordinal level_, Scalar omega_, Scalar shift_, Ordinal guessFlag_, Ordinal blkSzILDL_, Ordinal blkSz_)
135 {
136 nRows = nRow_;
137 standard_sptrsv = standard_sptrsv_;
138 nFact = nFact_;
139 nTrisol = nTrisol_;
140 computeTime = 0.0;
141 applyTime = 0.0;
142 initTime = 0.0;
143 level = level_;
144
145 // mirror & deep-copy input matrix to host
146 aRowMapHost = Kokkos::create_mirror_view(aRowMapIn);
147 aColIdxHost = Kokkos::create_mirror_view(aColIdxIn);
148 aValHost = Kokkos::create_mirror_view(aValIn);
149 Kokkos::deep_copy(aRowMapHost, aRowMapIn);
150 Kokkos::deep_copy(aColIdxHost, aColIdxIn);
151 Kokkos::deep_copy(aValHost, aValIn);
152
153 omega = omega_;
154 guessFlag = guessFlag_;
155 shift = shift_;
156 blkSzILDL = blkSzILDL_;
157 blkSz = blkSz_;
158
159 const Scalar one = Kokkos::ArithTraits<Scalar>::one();
160 onesVector = ScalarArray("onesVector", nRow_);
161 Kokkos::deep_copy(onesVector, one);
162
163 diagFact = RealArray("diagFact", nRow_);
164 diagElems = ScalarArray("diagElems", nRow_);
165 diagElemsInv = ScalarArray("diagElems", nRow_);
166 xOld = ScalarArray("xOld", nRow_);
167 xTemp = ScalarArray("xTemp", nRow_);
168
169 if (level > 0)
170 {
171 initGuessPrec = Teuchos::rcp(new FastPrec(aRowMapIn, aColIdxIn, aValIn, nRow_, standard_sptrsv, 3, 5,
172 level_-1, omega_, shift_, guessFlag_, blkSzILDL_, blkSz_));
173 }
174
175 }
176
177 void initialize()
178 {
179 Kokkos::Timer timer;
180 //Here we have to allocate memory for L, U.
181 if((level > 0) && (guessFlag != 0))
182 {
183 initGuessPrec->initialize();
184 }
185
186 //symbolicILUAndInit(); //Note the diagonal scaling is now embedded in this routine.
187 symbolicILU();
188 #ifdef SHYLU_DEBUG
189 //initialize L, U, A patterns
190 MemoryPrimeFunctorN<Ordinal, Scalar, ExecSpace> copyFunc1(aRowMap, lRowMap, lRowMap, diagElems);
191
192 //Make sure all memory resides on the device
193 ExecSpace().fence();
194 Kokkos::parallel_for(RangePolicy(0, nRows), copyFunc1);
195
196 //Note that the following is a temporary measure
197 //to ensure that memory resides on the device.
198 Ordinal nnzL = lRowMap[nRows];
199 Ordinal nnzA = aRowMap[nRows];
200 MemoryPrimeFunctorNnzCoo<Ordinal, Scalar, ExecSpace> copyFunc2(aColIdx, aRowIdx, aVal);
201 MemoryPrimeFunctorNnzCsr<Ordinal, Scalar, ExecSpace> copyFunc3(lColIdx, lVal);
202
203 ExecSpace().fence();
204 Kokkos::parallel_for(RangePolicy(0, nRows), copyFunc1);
205 Kokkos::parallel_for(RangePolicy(0, nnzA), copyFunc2);
206 Kokkos::parallel_for(RangePolicy(0, nnzL), copyFunc3);
207 #endif
208 double t = timer.seconds();
209 #ifdef FASTILDL_DEBUG_OUTPUT
210 std::cout << "Symbolic phase complete." << std::endl;
211 std::cout << "Init time: "<< t << "s" << std::endl;
212 #endif
213 initTime = t;
214 return;
215
216 }
217
218 void getL(OrdinalArray &lRowMapOut, OrdinalArray &lColIdxOut, ScalarArray &lValOut)
219 {
220 lRowMapOut = lRowMap;
221 lColIdxOut = lColIdx;
222 lValOut = lVal;
223 }
224
225 void getD(ScalarArray &diagElemsOut)
226 {
227 diagElemsOut = diagElems;
228 }
229
230 void transposeL()
231 {
232 auto ltRowMap_ = Kokkos::create_mirror_view(ltRowMap);
233 auto ltColIdx_ = Kokkos::create_mirror_view(ltColIdx);
234 auto ltVal_ = Kokkos::create_mirror_view(ltVal);
235
236 //Count the elements in each row of Lt
237 auto temp = OrdinalArrayHost("temp", nRows + 1);
238 auto rowPtrs = OrdinalArrayHost("rowPtrs", nRows);
239 for (Ordinal i = 0; i <= nRows; i++)
240 {
241 temp[i] = 0;
242 }
243 for (Ordinal i = 0; i < nRows; i++)
244 {
245 for (Ordinal k = lRowMap_[i]; k < lRowMap_[i+1]; k++)
246 {
247 temp[lColIdx_[k]+1]++;
248
249 }
250 }
251 //Perform an add scan to get the row map for
252 //the transpose
253 for (Ordinal i = 0; i <= nRows; i++)
254 {
255 ltRowMap_[i] = temp[i];
256 }
257 for (Ordinal i = 1; i <= nRows; i++)
258 {
259 ltRowMap_[i] += ltRowMap_[i-1];
260 }
261 //Set the row pointers to their initial places;
262 for (Ordinal i = 0; i < nRows; i++)
263 {
264 rowPtrs[i] = ltRowMap_[i];
265 }
266 //Copy the data
267 Kokkos::deep_copy(lVal_, lVal);
268 for (Ordinal i = 0; i < nRows; i++)
269 {
270 for (Ordinal k = lRowMap_[i]; k < lRowMap_[i+1]; k++)
271 {
272 Ordinal row = lColIdx_[k];
273 Scalar value = lVal_[k];
274 ltVal_[rowPtrs[row]] = value;
275 ltColIdx_[rowPtrs[row]] = i;
276 rowPtrs[row]++;
277 assert(rowPtrs[row] <= ltRowMap_[row + 1]);
278 }
279 }
280 Kokkos::deep_copy(ltRowMap, ltRowMap_);
281 Kokkos::deep_copy(ltColIdx, ltColIdx_);
282 Kokkos::deep_copy(ltVal, ltVal_);
283 }
284
285 void symbolicILU()
286 {
287 using std::vector;
288 using std::cout;
289 using std::stable_sort;
290 using std::sort;
291 OrdinalArrayMirror ia = aRowMapHost;
292 OrdinalArrayMirror ja = aColIdxHost;
293 int n = nRows;
294 int nzl = aRowMapHost[nRows] * (level + 2);
295 int nzu = aRowMapHost[nRows] * (level + 2);
296 Ordinal i;
297 int levfill = level;
298 vector<int> lnklst(n);
299 vector<int> curlev(n);
300 vector<int> levels(nzu);
301 vector<int> iwork(n);
302 vector<int> ial(nzl);
303 vector<int> jal(nzl);
304 vector<int> iau(nzu);
305 vector<int> jau(nzu);
306
307 int knzl = 0;
308 int knzu = 0;
309
310 ial[0] = 0;
311 iau[0] = 0;
312
313 for (i=0; i<n; i++)
314 {
315 int first, next, j;
316
317 /* copy column indices of row into workspace and sort them */
318
319 int len = ia[i+1] - ia[i];
320 next = 0;
321 for (j=ia[i]; j<ia[i+1]; j++)
322 iwork[next++] = ja[j];
323 // shell_sort(len, iwork);
324 //stable_sort(iwork.begin(), iwork.begin() + len);
325 sort(iwork.begin(), iwork.begin() + len);
326
327 /* construct implied linked list for row */
328
329 first = iwork[0];
330 curlev[first] = 0;
331
332 for (j=0; j<=len-2; j++)
333 {
334 lnklst[iwork[j]] = iwork[j+1];
335 curlev[iwork[j]] = 0;
336 }
337
338 lnklst[iwork[len-1]] = n;
339 curlev[iwork[len-1]] = 0;
340
341 /* merge with rows in U */
342
343 next = first;
344 while (next < i)
345 {
346 int oldlst = next;
347 int nxtlst = lnklst[next];
348 int row = next;
349 int ii;
350
351 /* scan row */
352
353 for (ii=iau[row]+1; ii<iau[row+1]; /*nop*/)
354 {
355 if (jau[ii] < nxtlst)
356 {
357 /* new fill-in */
358 int newlev = curlev[row] + levels[ii] + 1;
359 if (newlev <= levfill)
360 {
361 lnklst[oldlst] = jau[ii];
362 lnklst[jau[ii]] = nxtlst;
363 oldlst = jau[ii];
364 curlev[jau[ii]] = newlev;
365 }
366 ii++;
367 }
368 else if (jau[ii] == nxtlst)
369 {
370 int newlev;
371 oldlst = nxtlst;
372 nxtlst = lnklst[oldlst];
373 newlev = curlev[row] + levels[ii] + 1;
374 //curlev[jau[ii]] = MIN(curlev[jau[ii]], newlev);
375 if (curlev[jau[ii]] > newlev)
376 {
377 curlev[jau[ii]] = newlev;
378 }
379 ii++;
380 }
381 else /* (jau[ii] > nxtlst) */
382 {
383 oldlst = nxtlst;
384 nxtlst = lnklst[oldlst];
385 }
386 }
387 next = lnklst[next];
388 }
389
390 /* gather the pattern into L and U */
391
392 next = first;
393 while (next < i)
394 {
395 assert(knzl < nzl);
396 jal[knzl++] = next;
397 if (knzl >= nzl)
398 {
399 //(*nzl)++;
400 nzl += nRows;
401 jal.resize(nzl);
402 }
403 //jal.push_back(next);
404 // knzl++;
405 next = lnklst[next];
406 }
407 ial[i+1] = knzl;
408 assert(next == i);
409
410 while (next < n)
411 {
412 assert(knzu < nzu);
413 levels[knzu] = curlev[next];
414 jau[knzu++] = next;
415 if (knzu >= nzu)
416 {
417 //(*nzu)++;
418 nzu += nzu + nRows;
419 jau.resize(nzu);
420 levels.resize(nzu);
421 }
422 //jau.push_back(next);
423 //knzu++;
424 next = lnklst[next];
425 }
426 iau[i+1] = knzu;
427 }
428
429 #ifdef FASTILDL_DEBUG_OUTPUT
430 std::cout << "knzl =" << knzl;
431 std::cout << "knzu =" << knzu;
432 #endif
433
434 nzl = knzl;
435 nzu = knzu;
436
437 #ifdef FASTILDL_DEBUG_OUTPUT
438 std::cout << "ILU: nnz = "<< knzl + knzu << std::endl;
439 std::cout << "Actual nnz for ILU: " << nzl + nzu << std::endl;
440 #endif
441 //Initialize the A matrix that is to be used in the computation
442 aRowMap = OrdinalArray("aRowMap", nRows + 1);
443 aColIdx = OrdinalArray("aColIdx", knzl + knzu);
444 aRowIdx = OrdinalArray("aRowIds", knzl + knzu);
445 aRowMap_ = Kokkos::create_mirror_view(aRowMap);
446 aColIdx_ = Kokkos::create_mirror_view(aColIdx);
447 aRowIdx_ = Kokkos::create_mirror_view(aRowIdx);
448
449 Ordinal aRowPtr = 0;
450 aRowMap_[0] = 0;
451 for (i = 0; i < nRows; i++)
452 {
453 #ifdef FASTILDL_DEBUG_OUTPUT
454 std::cout << "***row:" << i << std::endl;
455 #endif
456 for(Ordinal k = ial[i]; k < ial[i+1]; k++)
457 {
458 #ifdef FASTILDL_DEBUG_OUTPUT
459 std::cout << "jal[k]=" << jal[k] << std::endl;
460 #endif
461 aColIdx_[aRowPtr] = jal[k];
462 aRowIdx_[aRowPtr] = i;
463 aRowPtr++;
464 }
465 for(Ordinal k = iau[i]; k < iau[i+1]; k++)
466 {
467 aColIdx_[aRowPtr] = jau[k];
468 aRowIdx_[aRowPtr] = i;
469 aRowPtr++;
470 }
471 aRowMap_[i+1] = aRowPtr;
472 }
473 Kokkos::deep_copy(aRowMap, aRowMap_);
474 Kokkos::deep_copy(aColIdx, aColIdx_);
475 Kokkos::deep_copy(aRowIdx, aRowIdx_);
476
477 //Now allocate memory for L and U.
478 //
479 lRowMap = OrdinalArray("lRowMap", nRows + 1);
480 ltRowMap = OrdinalArray("ltRowMap", nRows + 1);
481 countL();
482 //Allocate memory and initialize pattern for L, U (transpose).
483 lColIdx = OrdinalArray("lColIdx", lRowMap_[nRows]);
484 ltColIdx = OrdinalArray("ltColIdx", lRowMap_[nRows]);
485 #ifdef FASTILDL_DEBUG_OUTPUT
486 std::cout << "nnz L = " << lRowMap[nRows] << std::endl;
487 #endif
488 }
489
490 void numericILU()
491 {
492 aVal = ScalarArray("aVal", aColIdx.extent(0));
493 aVal_ = Kokkos::create_mirror_view(aVal);
494
495 //Copy the host matrix into the initialized a;
496 for (Ordinal i = 0; i < nRows; i++)
497 {
498 for(Ordinal k = aRowMap_[i]; k < aRowMap_[i+1]; k++)
499 {
500 Ordinal col = aColIdx_[k];
501 #ifdef FASTILDL_DEBUG_OUTPUT
502 std::cout << "col =" << col << std::endl;
503 #endif
504 for(Ordinal aHostPtr = aRowMapHost[i]; aHostPtr < aRowMapHost[i+1]; aHostPtr++)
505 {
506 if (col == aColIdxHost[aHostPtr])
507 {
508 aVal_[k] = aValHost[aHostPtr];
509 break;
510 }
511 }
512 }
513 }
514 lVal = ScalarArray("lVal", lRowMap_[nRows]);
515 ltVal = ScalarArray("ltVal", lRowMap_[nRows]);
516 applyDiagonalScaling();
517 applyManteuffelShift();
518 Kokkos::deep_copy(aVal, aVal_);
519
520 fillL();
521 }
522
523 void countL()
524 {
525 lRowMap_ = Kokkos::create_mirror_view(lRowMap);
526
527 lRowMap_[0] = 0;
528 for (Ordinal i = 0; i < nRows; i++)
529 {
530 Ordinal row_count = 0;
531 for (Ordinal k = aRowMap_[i]; k < aRowMap_[i+1]; k++)
532 {
533 Ordinal row = i;
534 Ordinal col = aColIdx_[k];
535
536 if (row >= col)
537 {
538 row_count++;
539 }
540 }
541 lRowMap_[i+1] = lRowMap_[i] + row_count;
542 }
543 Kokkos::deep_copy(lRowMap, lRowMap_);
544 }
545
546 void fillL()
547 {
548 lVal_ = Kokkos::create_mirror_view(lVal);
549 lColIdx_ = Kokkos::create_mirror_view(lColIdx);
550 diagElems_ = Kokkos::create_mirror_view(diagElems);
551
552 Ordinal lPtr = 0;
553 for (Ordinal i = 0; i < nRows; i++)
554 {
555 for (Ordinal k = aRowMap_[i]; k < aRowMap_[i+1]; k++)
556 {
557 Ordinal row = i;
558 Ordinal col = aColIdx_[k];
559
560 if (row >= col)
561 {
562 if (row == col)
563 {
564 diagElems_[row] = aVal_[k];
565 }
566 lVal_[lPtr] = aVal_[k];
567 lColIdx_[lPtr] = aColIdx_[k];
568 lPtr++;
569 }
570 }
571 }
572 assert(lPtr == lRowMap_[nRows]);
573 Kokkos::deep_copy(diagElems, diagElems_);
574
575 if ((level > 0) && (guessFlag !=0))
576 {
577 OrdinalArray lGRowMap;
578 OrdinalArray lGColIdx;
579 ScalarArray lGVal;
580 ScalarArray gD;
581 Ordinal lGPtr = 0;
582
583 initGuessPrec->getL(lGRowMap, lGColIdx, lGVal);
584 initGuessPrec->getD(gD);
585
586 Kokkos::deep_copy(diagElems, gD);
587
588 auto lGColIdx_ = Kokkos::create_mirror_view(lGColIdx);
589 auto lGVal_ = Kokkos::create_mirror_view(lGVal);
590 Kokkos::deep_copy(lGColIdx_, lGColIdx);
591 Kokkos::deep_copy(lGVal_, lGVal);
592 for (Ordinal i = 0; i < nRows; i++)
593 {
594 #ifdef SHYLU_DEBUG
595 Ordinal check = lGPtr;
596 #endif
597 for (Ordinal k = lRowMap_[i]; k < lRowMap_[i+1]; k++)
598 {
599 //unused: Ordinal row = i;
600 Ordinal col = lColIdx_[k];
601
602 if (col == lGColIdx_[lGPtr])
603 {
604 lVal_[k] = lGVal_[lGPtr];
605 lGPtr++;
606 }
607 }
608 #ifdef SHYLU_DEBUG
609 Ordinal rowLen = lGPtr - check;
610 assert(rowLen == lGRowMap[i+1] - lGRowMap[i]);
611 #endif
612 }
613 }
614 Kokkos::deep_copy(lColIdx, lColIdx_);
615 Kokkos::deep_copy(lVal, lVal_);
616 }
617
618 void applyDiagonalScaling()
619 {
620 int anext = 0;
621 const Real one = Kokkos::ArithTraits<Real>::one();
622 //First fill Aj and extract the diagonal scaling factors
623 //Use diag array to store scaling factors since
624 //it gets set to the correct value by findFactorPattern anyway.
625 auto diagFact_ = Kokkos::create_mirror_view(diagFact);
626 for (int i = 0; i < nRows; i++)
627 {
628 for(int k = aRowMap_[i]; k < aRowMap_[i+1]; k++)
629 {
630 aRowIdx_[anext++] = i;
631 if (aColIdx_[k] == i)
632 {
633 diagFact_[i] = one/(RTS::sqrt(STS::abs(aVal_[k])));
634 #ifdef FASTILDL_DEBUG_OUTPUT
635 std::cout << "diagFact["<<i<<"]="<<aVal[k]<<std::endl;
636 #endif
637 }
638 }
639 }
640
641 //Now go through each element of A and apply the scaling
642 int row;
643 int col;
644 Real sc1, sc2;
645 for (int i = 0; i < nRows; i++)
646 {
647 for (int k = aRowMap_[i]; k < aRowMap_[i+1]; k++)
648 {
649 row = aRowIdx_[k];
650 col = aColIdx_[k];
651
652 sc1 = diagFact_[row];
653 sc2 = diagFact_[col];
654 aVal_[k] = aVal_[k]*sc1*sc2;
655 }
656 }
657 Kokkos::deep_copy(diagFact, diagFact_);
658 }
659
660 void applyManteuffelShift()
661 {
662 const Scalar one = Kokkos::ArithTraits<Scalar>::one();
663 //Scalar shift = 0.05;
664 for (Ordinal i = 0; i < nRows; i++)
665 {
666 for (Ordinal k = aRowMap_[i]; k < aRowMap_[i+1]; k++)
667 {
668 Ordinal row = i;
669 Ordinal col = aColIdx_[k];
670 if (row != col)
671 {
672 aVal_[k] = (one/(one + shift))*aVal_[k];
673 }
674 }
675 }
676 }
677
678 void applyDD(ScalarArray &x, ScalarArray &y)
679 {
680 ParScalFunctor<Ordinal, Scalar, Scalar, ExecSpace> parScal(x, y, diagElemsInv);
681 ExecSpace().fence();
682 Kokkos::parallel_for(RangePolicy(0, nRows), parScal);
683 ExecSpace().fence();
684
685 }
686 void applyD(ScalarArray &x, ScalarArray &y)
687 {
688 ParScalFunctor<Ordinal, Scalar, Real, ExecSpace> parScal(x, y, diagFact);
689 ExecSpace().fence();
690 Kokkos::parallel_for(RangePolicy(0, nRows), parScal);
691 ExecSpace().fence();
692
693 }
694 void applyLIC(ScalarArray &x, ScalarArray &y)
695 {
696 ParInitZeroFunctor<Ordinal, Scalar, ExecSpace> parInitZero(xOld);
697 Kokkos::parallel_for(RangePolicy(0, nRows), parInitZero);
698 ExecSpace().fence();
699 #if 0
700 JacobiIterFunctor<Ordinal, Scalar, ExecSpace> jacIter(nRows, lRowMap, lColIdx, lVal, x, y, xOld, onesVector);
701 #endif
702 BlockJacobiIterFunctorL<Ordinal, Scalar, ExecSpace> jacIter(nRows, blkSz, lRowMap, lColIdx, lVal,
703 x, y, xOld, onesVector);
704 ParCopyFunctor<Ordinal, Scalar, ExecSpace> parCopy(xOld, y);
705 Ordinal extent = nRows/blkSz;
706 if(nRows%blkSz != 0)
707 {
708 extent++;
709 }
710 ExecSpace().fence();
711 for (Ordinal i = 0; i < nTrisol; i++)
712 {
713 Kokkos::parallel_for(RangePolicy(0, extent), jacIter);
714 ExecSpace().fence();
715 Kokkos::parallel_for(RangePolicy(0, nRows), parCopy);
716 ExecSpace().fence();
717 }
718 return;
719 }
720
721 void applyLT(ScalarArray &x, ScalarArray &y)
722 {
723 ParInitZeroFunctor<Ordinal, Scalar, ExecSpace> parInitZero(xOld);
724 Kokkos::parallel_for(RangePolicy(0, nRows), parInitZero);
725 ExecSpace().fence();
726 #if 0
727 JacobiIterFunctor<Ordinal, Scalar, ExecSpace> jacIter(nRows, ltRowMap, ltColIdx, ltVal, x, y, xOld, onesVector);
728 #endif
729 BlockJacobiIterFunctorU<Ordinal, Scalar, ExecSpace> jacIter(nRows, blkSz, ltRowMap, ltColIdx, ltVal,
730 x, y, xOld, onesVector);
731 ParCopyFunctor<Ordinal, Scalar, ExecSpace> parCopy(xOld, y);
732 Ordinal extent = nRows/blkSz;
733 if (nRows%blkSz != 0)
734 {
735 extent++;
736 }
737 ExecSpace().fence();
738
739 for (Ordinal i = 0; i < nTrisol; i++)
740 {
741 Kokkos::parallel_for(RangePolicy(0, extent), jacIter);
742 ExecSpace().fence();
743 Kokkos::parallel_for(RangePolicy(0, nRows), parCopy);
744 ExecSpace().fence();
745 }
746
747 return;
748 }
749
750 void setValues(ScalarArray& aValsIn)
751 {
752 this->aValHost = Kokkos::create_mirror_view(aValsIn);
753 Kokkos::deep_copy(this->aValHost, aValsIn);
754 if(!initGuessPrec.is_null())
755 {
756 initGuessPrec->setValues(aValsIn);
757 }
758 }
759
760 void compute()
761 {
762 const Scalar one = Kokkos::ArithTraits<Scalar>::one();
763 if((level > 0) && (guessFlag != 0))
764 {
765 initGuessPrec->compute();
766 }
767 ExecSpace().fence();
768
769 Kokkos::Timer timer;
770 numericILU();
771 FastILDLFunctor<Ordinal, Scalar, ExecSpace> ildltFunctor(aRowMap_[nRows], blkSzILDL,
772 aRowMap, aColIdx, aRowIdx, aVal, lRowMap, lColIdx, lVal, diagElems, omega);
773
774 Ordinal extent = aRowMap_[nRows]/blkSzILDL;
775 if (aRowMap_[nRows]%blkSzILDL != 0)
776 {
777 extent++;
778 }
779 //Ordinal extent = aRowMap[nRows];
780 for (int i = 0; i < nFact; i++)
781 {
782 Kokkos::parallel_for(RangePolicy(0, extent), ildltFunctor);
783 }
784 ExecSpace().fence();
785
786 double t = timer.seconds();
787 computeTime = t;
788
789 Kokkos::deep_copy(diagElems_, diagElems);
790 diagElemsInv_ = Kokkos::create_mirror_view(diagElemsInv);
791 for (int i = 0; i < nRows; i++)
792 {
793 diagElemsInv_[i] = one/diagElems_[i];
794 }
795 Kokkos::deep_copy(diagElemsInv, diagElemsInv_);
796 transposeL();
797
798 if (standard_sptrsv) {
799 #if defined(KOKKOSKERNELS_ENABLE_TPL_CUSPARSE)
800 KokkosSparse::Experimental::SPTRSVAlgorithm algo = KokkosSparse::Experimental::SPTRSVAlgorithm::SPTRSV_CUSPARSE;
801 #else
802 KokkosSparse::Experimental::SPTRSVAlgorithm algo = KokkosSparse::Experimental::SPTRSVAlgorithm::SEQLVLSCHD_TP1;
803 #endif
804 // setup L solve
805 khL.create_sptrsv_handle(algo, nRows, true);
806 #if defined(KOKKOSKERNELS_ENABLE_TPL_CUSPARSE)
807 KokkosSparse::Experimental::sptrsv_symbolic(&khL, lRowMap, lColIdx, lVal);
808 #else
809 KokkosSparse::Experimental::sptrsv_symbolic(&khL, lRowMap, lColIdx);
810 #endif
811 // setup Lt solve
812 khLt.create_sptrsv_handle(algo, nRows, false);
813 #if defined(KOKKOSKERNELS_ENABLE_TPL_CUSPARSE)
814 KokkosSparse::Experimental::sptrsv_symbolic(&khLt, ltRowMap, ltColIdx, ltVal);
815 #else
816 KokkosSparse::Experimental::sptrsv_symbolic(&khLt, ltRowMap, ltColIdx);
817 #endif
818 }
819 }
820
821 void apply(ScalarArray &x, ScalarArray &y)
822 {
823
824 Kokkos::Timer timer;
825 ParCopyFunctor<Ordinal, Scalar, ExecSpace> parCopyFunctor(xTemp, x);
826 ExecSpace().fence();
827 Kokkos::parallel_for(RangePolicy(0, nRows), parCopyFunctor);
828 ExecSpace().fence();
829
830 applyD(x, xTemp);
831 if (standard_sptrsv) {
832 KokkosSparse::Experimental::sptrsv_solve(&khL, lRowMap, lColIdx, lVal, xTemp, y);
833 } else {
834 applyLIC(xTemp, y);
835 }
836 applyDD(y, xTemp);
837 if (standard_sptrsv) {
838 KokkosSparse::Experimental::sptrsv_solve(&khLt, ltRowMap, ltColIdx, ltVal, xTemp, y);
839 } else {
840 applyLT(xTemp, y);
841 }
842 applyD(y, xTemp);
843
844 ParCopyFunctor<Ordinal, Scalar, ExecSpace> parCopyFunctor2(y, xTemp);
845 ExecSpace().fence();
846 Kokkos::parallel_for(RangePolicy(0, nRows), parCopyFunctor2);
847 ExecSpace().fence();
848
849 double t = timer.seconds();
850 applyTime = t;
851 return;
852 }
853
854 Ordinal getNFact() const
855 {
856 return nFact;
857 }
858
859 std::string getSpTrsvType() const
860 {
861 return (standard_sptrsv ? "Standard" : "Fast");
862 }
863
864 Ordinal getNTrisol() const
865 {
866 return nTrisol;
867 }
868
869 Ordinal getNRows() const
870 {
871 return nRows;
872 }
873
874 double getComputeTime() const
875 {
876 return computeTime;
877 }
878
879 double getInitializeTime() const
880 {
881 return initTime;
882 }
883
884 double getApplyTime() const
885 {
886 return applyTime;
887 }
888 void checkIC() const
889 {
890 //Compute the L2 norm of the nonlinear residual (A - LLt) on sparsity pattern
891 //
892 Scalar sum = 0.0;
893 for (int i = 0 ; i < nRows; i++)
894 {
895 int row = i;
896 // Each row in A matrix (result)
897 for (int k = aRowMap[i]; k < aRowMap[i+1]; k++)
898 {
899
900 int col = aColIdx[k];
901
902 if (row >= col) {
903 Scalar acc_val = aVal[k];
904 Ordinal lptr, uptr;
905 for ( lptr = lRowMap[i], uptr = lRowMap[aColIdx[k]] ;
906 lptr < lRowMap[i+1] && uptr < lRowMap[aColIdx[k]+1] ; )
907 {
908 if (lColIdx[lptr] == lColIdx[uptr])
909 {
910 acc_val -= lVal[lptr] * diagElems[lColIdx[uptr]] * lVal[uptr];
911 lptr++;
912 uptr++;
913 }
914 else if (lColIdx[lptr] < lColIdx[uptr])
915 lptr++;
916 else
917 uptr++;
918 }
919
920 sum += acc_val * acc_val;
921 }
922 }
923 }
924 std::cout << "l2 norm of nonlinear residual = " << RTS::sqrt(STS::abs(sum)) << std::endl;
925 }
926 friend class FastILDLFunctor<Ordinal, Scalar, ExecSpace>;
927 friend class JacobiIterFunctor<Ordinal, Scalar, ExecSpace>;
928 friend class ParCopyFunctor<Ordinal, Scalar, ExecSpace>;
929 friend class JacobiIterFunctorT<Ordinal, Scalar, ExecSpace>;
930 friend class ParScalFunctor<Ordinal, Scalar, Real, ExecSpace>;
931 friend class MemoryPrimeFunctorN<Ordinal, Scalar, ExecSpace>;
932 friend class MemoryPrimeFunctorNnzCoo<Ordinal, Scalar, ExecSpace>;
933 friend class MemoryPrimeFunctorNnzCsr<Ordinal, Scalar, ExecSpace>;
934 };
935
936 template<class Ordinal, class Scalar, class ExecSpace>
937 class FastILDLFunctor
938 {
939 public:
940 typedef ExecSpace execution_space;
941 typedef Kokkos::View<Ordinal *, ExecSpace> ordinal_array_type;
942 typedef Kokkos::View<Scalar *, ExecSpace> scalar_array_type;
943
944 FastILDLFunctor (Ordinal nNZ, Ordinal bs, ordinal_array_type Ap, ordinal_array_type Ai,
945 ordinal_array_type Aj, scalar_array_type Ax, ordinal_array_type Lp,
946 ordinal_array_type Li, scalar_array_type Lx, scalar_array_type diag, Scalar omega)
947 :
948 nnz(nNZ), blk_size(bs),
949 _Ap(Ap), _Ai(Ai), _Aj(Aj), _Lp(Lp), _Li(Li), _Ax(Ax), _Lx(Lx), _diag(diag), _omega(omega)
950 {
951 }
952
953 KOKKOS_INLINE_FUNCTION
954 void operator()(const Ordinal blk_index) const
955 {
956 const Scalar zero = Kokkos::ArithTraits<Scalar>::zero();
957 const Scalar one = Kokkos::ArithTraits<Scalar>::one();
958
959 Ordinal start = blk_index * blk_size;
960 Ordinal end = start + blk_size;
961
962 Ordinal nz_index;
963
964 if (end > nnz)
965 {
966 end = nnz;
967 }
968
969 for (nz_index = start; nz_index < end && nz_index < nnz; nz_index++)
970 {
971 Ordinal i = _Ai[nz_index];
972 Ordinal j = _Aj[nz_index];
973 Scalar val = _Ax[nz_index];
974 Scalar acc_val = zero;
975 Scalar temp = zero;
976 Ordinal lptr = _Lp[i];
977 Ordinal ltptr = _Lp[j];
978 Ordinal endpt = j;
979 if (i >= j) {
980
981 for ( ; _Li[lptr] < endpt && _Li[ltptr] < endpt; )
982 {
983 if (_Li[lptr] == _Li[ltptr])
984 {
985 acc_val += _Lx[lptr] * _diag[_Li[ltptr]] * _Lx[ltptr];
986 lptr++;
987 ltptr++;
988 }
989 else if (_Li[lptr] < _Li[ltptr])
990 {
991 lptr++;
992 }
993 else
994 {
995 ltptr++;
996 }
997 }
998 if (i > j)
999 {
1000 val = (val-acc_val) / _diag[j];
1001 for ( ; _Li[lptr] < j ; lptr++) ; // dummy loop
1002 assert (_Li[lptr] == j);
1003 // _Lx[lptr] = val;
1004 _Lx[lptr] = ((one - _omega)*_Lx[lptr]) + (_omega * val);
1005 }
1006 else if (i == j)
1007 {
1008 //_diag[j] = val - acc_val;
1009 temp = val - acc_val;
1010 _diag[j] = ((one - _omega) * _diag[j]) + (_omega * temp);
1011 }
1012 }
1013 }
1014 }
1015
1016 Ordinal nnz, blk_size;
1017 ordinal_array_type _Ap, _Ai, _Aj, _Lp, _Li;
1018 scalar_array_type _Ax, _Lx, _diag;
1019 Scalar _omega;
1020 };
1021 #endif