Debian Codegraph @

packages/shylu/shylu_node/fastilu/src/shylu_fastilu.hpp

raw
1 // @HEADER
2 // *****************************************************************************
3 // ShyLU: Scalable Hybrid LU Preconditioner and Solver
4 //
5 // Copyright 2011 NTESS and the ShyLU contributors.
6 // SPDX-License-Identifier: BSD-3-Clause
7 // *****************************************************************************
8 // @HEADER
9
10 // The struct that iterates over all non-zeros for the FastILU
11 // Contact for bugs and complaints - Siva Rajamanickam ([email protected])
12 //
13 #ifndef __FAST_ILU_HPP__
14 #define __FAST_ILU_HPP__
15
16 #include <iostream>
17 #include <algorithm>
18 #include <vector>
19 #include <queue>
20 #include <random>
21 #include <cstdlib>
22 #include <cmath>
23 #include <ctime>
24 #include <string>
25
26 #include <assert.h>
27 #include <Kokkos_Core.hpp>
28 #include <Kokkos_Timer.hpp>
29 #include <KokkosKernels_Sorting.hpp>
30 #include <KokkosSparse_Utils.hpp>
31 #include <KokkosSparse_SortCrs.hpp>
32 #include <KokkosSparse_spmv.hpp>
33 #include <KokkosSparse_sptrsv.hpp>
34 #include <KokkosSparse_trsv.hpp>
35 #include <KokkosSparse_BsrMatrix.hpp>
36 #include <shylu_fastutil.hpp>
37
38 #include "Tpetra_BlockCrsMatrix_Helpers.hpp"
39
40 // FASTILU Preprocesser options:
41
42 // whether to print extra debug output at runtime to stdout
43 // comment out next line to disable
44 //#define FASTILU_DEBUG_OUTPUT
45
46 // whether to print timings
47 //#define FASTILU_TIMER
48
49 // Whether to try to maintain exact behavior with unblocked impl. Comes at
50 // a steep performance cost.
51 //#define FASTILU_ONE_TO_ONE_UNBLOCKED
52
53 template <typename Ordinal>
54 KOKKOS_INLINE_FUNCTION
55 Ordinal
56 unblock(const Ordinal block_idx, const Ordinal block_offset, const Ordinal bsize)
57 {
58 return block_idx * bsize + block_offset;
59 }
60
61 #ifdef FASTILU_ONE_TO_ONE_UNBLOCKED
62 template <typename View1, typename View2, typename View3>
63 void unfill_crs(View1& row_ptrs, View2& cols, View3& values)
64 {
65 using Scalar = typename View3::non_const_value_type;
66 using Ordinal = typename View1::non_const_value_type;
67 using STS = Kokkos::ArithTraits<Scalar>;
68 const Scalar zero = STS::zero();
69
70 const auto nrows = row_ptrs.size() - 1;
71 Ordinal real_nnzs = 0;
72 for (size_t row = 0; row < nrows; ++row) {
73 const Ordinal row_nnz_begin = row_ptrs[row];
74 row_ptrs[row] = real_nnzs;
75 for (Ordinal row_nnz = row_nnz_begin; row_nnz < row_ptrs[row+1]; ++row_nnz) {
76 const Ordinal col_idx = cols[row_nnz];
77 const Scalar value = values[row_nnz];
78 if (value != zero) {
79 cols[real_nnzs] = col_idx;
80 values[real_nnzs] = value;
81 ++real_nnzs;
82 }
83 }
84 }
85 row_ptrs[nrows] = real_nnzs;
86 Kokkos::resize(cols, real_nnzs);
87 Kokkos::resize(values, real_nnzs);
88 }
89
90 template <typename View1, typename View2, typename View3>
91 void unblock(View1& row_map_host, View2& col_inds_host, View3& values_host, const int block_size)
92 {
93 using tmap_type = Tpetra::Map<>;
94 using LO = tmap_type::local_ordinal_type;
95 using GO = tmap_type::global_ordinal_type;
96 using Node = Tpetra::KokkosCompat::KokkosSerialWrapperNode;
97 using Scalar = typename View3::non_const_value_type;
98 using BCrsMatrix = Tpetra::BlockCrsMatrix<Scalar, LO, GO, Node>;
99 using map_type = typename BCrsMatrix::map_type;
100 using graph_type = typename BCrsMatrix::crs_graph_type;
101 using row_vtype = typename graph_type::local_graph_device_type::row_map_type::non_const_type;
102
103 // Create new TCrsMatrix with the new filled data
104 row_vtype rv("temp", row_map_host.extent(0));
105 Kokkos::deep_copy(rv, row_map_host);
106 const auto nrows = row_map_host.size() - 1;
107
108 Teuchos::SerialComm<LO> SerialComm;
109 Teuchos::RCP<const Teuchos::Comm<LO> > comm_ptr = Teuchos::RCP(&SerialComm, false);
110 map_type proc_map(nrows, 0, comm_ptr, Tpetra::LocallyReplicated);
111 Teuchos::RCP<map_type> map_ptr = Teuchos::RCP(&proc_map, false);
112 graph_type graph(map_ptr, map_ptr, rv, col_inds_host);
113 graph.fillComplete();
114 BCrsMatrix bcrs_matrix(graph, values_host, block_size);
115
116 auto A = Tpetra::convertToCrsMatrix(bcrs_matrix);
117 auto localA = A->getLocalMatrixDevice();
118 Kokkos::resize(row_map_host, localA.graph.row_map.size());
119 Kokkos::deep_copy(row_map_host, localA.graph.row_map);
120 Kokkos::resize(col_inds_host, localA.graph.entries.size());
121 Kokkos::deep_copy(col_inds_host, localA.graph.entries);
122 Kokkos::resize(values_host, localA.values.size());
123 Kokkos::deep_copy(values_host, localA.values);
124 unfill_crs(row_map_host, col_inds_host, values_host);
125 }
126
127 template <typename View1, typename View2, typename View3, typename View4>
128 void reblock(View1& row_map, View2& row_idx, View3& col_inds, View4& values, const int block_size)
129 {
130 using LO = Tpetra::Map<>::local_ordinal_type;
131 using GO = Tpetra::Map<>::global_ordinal_type;
132 using Node = Tpetra::KokkosCompat::KokkosSerialWrapperNode;
133 using Scalar = typename View4::non_const_value_type;
134 using TCrsMatrix = Tpetra::CrsMatrix<Scalar, LO, GO, Node>;
135 using map_type = typename TCrsMatrix::map_type;
136 using graph_type = typename TCrsMatrix::crs_graph_type;
137 using row_vtype = typename graph_type::local_graph_device_type::row_map_type::non_const_type;
138
139 row_vtype rv("temp", row_map.extent(0));
140 Kokkos::deep_copy(rv, row_map);
141 const auto nrows = row_map.size() - 1;
142 Teuchos::SerialComm<LO> SerialComm;
143 Teuchos::RCP<const Teuchos::Comm<LO> > comm_ptr = Teuchos::RCP(&SerialComm, false);
144 map_type proc_map(nrows, 0, comm_ptr, Tpetra::LocallyReplicated);
145 Teuchos::RCP<map_type> map_ptr = Teuchos::RCP(&proc_map, false);
146 TCrsMatrix crs_matrix(map_ptr, map_ptr, rv, col_inds, values);
147 crs_matrix.fillComplete();
148 auto crs_matrix_block_filled = Tpetra::fillLogicalBlocks(crs_matrix, block_size);
149
150 auto bcrs_matrix = Tpetra::convertToBlockCrsMatrix(*crs_matrix_block_filled, block_size);
151 auto localA = bcrs_matrix->getLocalMatrixDevice();
152 auto rowptrs = localA.graph.row_map;
153 auto colinds = localA.graph.entries;
154 auto vals = localA.values;
155
156 Kokkos::resize(row_map, rowptrs.size());
157 Kokkos::deep_copy(row_map, rowptrs);
158 Kokkos::resize(col_inds, colinds.size());
159 Kokkos::deep_copy(col_inds, colinds);
160 Kokkos::resize(values, vals.size());
161 Kokkos::deep_copy(values, vals);
162
163 // Now do row idx
164 const auto nrows_blocked = row_map.size() -1;
165 Kokkos::resize(row_idx, colinds.size());
166 LO nnz = 0;
167 for (size_t row = 0; row < nrows_blocked; ++row) {
168 for (LO row_nnz = row_map(row); row_nnz < row_map(row+1); ++row_nnz) {
169 row_idx(nnz++) = row;
170 }
171 }
172 assert(nnz == row_map(nrows_blocked));
173 }
174 #endif
175
176 // some useful preprocessor functions
177 #ifdef FASTILU_TIMER
178 #define FASTILU_CREATE_TIMER(timer) Kokkos::Timer timer
179
180 #define FASTILU_REPORT_TIMER(timer, report) \
181 std::cout << report << " : " << timer.seconds() << std::endl; \
182 timer.reset()
183
184 #define FASTILU_FENCE_REPORT_TIMER(timer, fenceobj, report) \
185 fenceobj.fence(); \
186 FASTILU_REPORT_TIMER(timer, report)
187
188 #else
189 #define FASTILU_CREATE_TIMER(name) ((void) (0))
190 #define FASTILU_REPORT_TIMER(timer, report) ((void) (0))
191 #define FASTILU_FENCE_REPORT_TIMER(timer, fenceobj, report) ((void) (0))
192 #endif
193
194 #ifdef FASTILU_DEBUG_OUTPUT
195 #define FASTILU_DBG_COUT(args) std::cout << args << std::endl;
196 #else
197 #define FASTILU_DBG_COUT(args) ((void) (0))
198 #endif
199
200 // forward declarations
201 template<class Ordinal, class Scalar, class ExecSpace>
202 class FastILUFunctor;
203
204 template<class Ordinal, class Scalar, class ExecSpace>
205 class FastICFunctor;
206
207 template<class Ordinal, class Scalar, class ExecSpace>
208 class JacobiIterFunctor;
209
210 template<class Ordinal, class Scalar, class ExecSpace>
211 class BlockJacobiIterFunctorU;
212
213 template<class Ordinal, class Scalar, class ExecSpace>
214 class BlockJacobiIterFunctorL;
215
216 template<class Ordinal, class Scalar, class ExecSpace>
217 class ParCopyFunctor;
218
219 template<class Ordinal, class Scalar, class ExecSpace, bool BlockCrsEnabled>
220 class ParPermCopyFunctor;
221
222 template<class Ordinal, class Scalar, class Real, class ExecSpace>
223 class ParScalFunctor;
224
225 struct NonTranPermScalTag {};
226 struct TranPermScalTag {};
227 template<class Ordinal, class Scalar, class Real, class ExecSpace>
228 class PermScalFunctor;
229
230 template<class Ordinal, class Scalar, class ExecSpace>
231 class ParInitZeroFunctor;
232
233 template<class Ordinal, class Scalar, class ExecSpace>
234 class MemoryPrimeFunctorN;
235
236 template<class Ordinal, class Scalar, class ExecSpace>
237 class MemoryPrimeFunctorNnzCoo;
238
239 template<class Ordinal, class Scalar, class ExecSpace>
240 class MemoryPrimeFunctorNnzCsr;
241
242 template<class Ordinal, class Scalar, class ExecSpace, bool BlockCrsEnabled=false>
243 class FastILUPrec
244 {
245 public:
246 typedef typename Teuchos::ScalarTraits<Scalar>::magnitudeType Real;
247 typedef Kokkos::View<Ordinal *, ExecSpace> OrdinalArray;
248 typedef Kokkos::View<Scalar *, ExecSpace> ScalarArray;
249 typedef Kokkos::View<Scalar **, ExecSpace> Scalar2dArray;
250 typedef Kokkos::View<Real *, ExecSpace> RealArray;
251 typedef Kokkos::View<Ordinal *, typename ExecSpace::array_layout,
252 Kokkos::Serial, Kokkos::MemoryUnmanaged> UMOrdinalArray;
253 typedef Kokkos::View<Scalar *, typename ExecSpace::array_layout,
254 Kokkos::Serial, Kokkos::MemoryUnmanaged> UMScalarArray;
255 typedef FastILUPrec<Ordinal, Scalar, ExecSpace, BlockCrsEnabled> FastPrec;
256
257 using HostSpace = Kokkos::HostSpace;
258 using MirrorSpace = typename OrdinalArray::host_mirror_space;
259
260 typedef Kokkos::View<Ordinal *, HostSpace> OrdinalArrayHost;
261 typedef Kokkos::View<Scalar *, HostSpace> ScalarArrayHost;
262 typedef typename OrdinalArray::host_mirror_type OrdinalArrayMirror;
263 typedef typename ScalarArray::host_mirror_type ScalarArrayMirror;
264
265 using STS = Kokkos::ArithTraits<Scalar>;
266 using RTS = Kokkos::ArithTraits<Real>;
267
268 template <typename T>
269 struct IdentityFunctor
270 {
271 KOKKOS_INLINE_FUNCTION
272 T operator()(const T val) const {return val;}
273 };
274
275 struct ShiftFunctor
276 {
277 Scalar shift;
278 Scalar one;
279
280 KOKKOS_INLINE_FUNCTION
281 ShiftFunctor(const Scalar shift_arg) : shift(shift_arg), one(STS::one()) {}
282
283 KOKKOS_INLINE_FUNCTION
284 Scalar operator()(const Scalar val) const { return (one/(one + shift)) * val; }
285 };
286
287 struct InvFunctor
288 {
289 Scalar one;
290
291 KOKKOS_INLINE_FUNCTION
292 InvFunctor() : one(STS::one()) {}
293
294 KOKKOS_INLINE_FUNCTION
295 Scalar operator()(const Scalar val) const { return one/val; }
296 };
297
298 struct InvSqrtFunctor
299 {
300 Real one;
301
302 KOKKOS_INLINE_FUNCTION
303 InvSqrtFunctor() : one(RTS::one()) {}
304
305 KOKKOS_INLINE_FUNCTION
306 Real operator()(const Scalar val) const { return one/(RTS::sqrt(STS::abs(val))); };
307 };
308
309 struct NotDiagFunctor
310 {
311 KOKKOS_INLINE_FUNCTION
312 bool operator()(const Ordinal first, const Ordinal second) const {return first != second;}
313 };
314
315 struct LowerFunctor
316 {
317 KOKKOS_INLINE_FUNCTION
318 bool operator()(const Ordinal first, const Ordinal second) const {return first > second;}
319 };
320
321 struct UpperFunctor
322 {
323 KOKKOS_INLINE_FUNCTION
324 bool operator()(const Ordinal first, const Ordinal second) const {return first <= second;}
325 };
326
327 struct IsNotSentinelFunctor
328 {
329 Scalar zero;
330
331 KOKKOS_INLINE_FUNCTION
332 IsNotSentinelFunctor() : zero(STS::zero()) {}
333
334 KOKKOS_INLINE_FUNCTION
335 bool operator()(const Scalar val_dest, const Scalar val_src) const
336 { return !(val_dest != zero && val_src == zero) ; }
337 };
338
339 struct ProdThreeFunctor
340 {
341 KOKKOS_INLINE_FUNCTION
342 Scalar operator()(const Scalar val1, const Scalar val2, const Scalar val3) const
343 { return val1*val2*val3; }
344 };
345
346 template <bool Block, typename View1, typename View2, typename F = IdentityFunctor<typename View1::non_const_value_type> >
347 KOKKOS_INLINE_FUNCTION
348 static typename std::enable_if<Block, void>::type
349 assign_block(View1& vals_dest, const View2& vals_src, const Ordinal dest, const Ordinal src, const Ordinal blockCrsSize, const F& lam = F())
350 {
351 const Ordinal blockItems = blockCrsSize*blockCrsSize;
352 for (Ordinal itr = 0; itr < blockItems; ++itr) {
353 vals_dest[dest*blockItems + itr] = lam(vals_src[src*blockItems + itr]);
354 }
355 }
356
357 template <bool Block, typename View1, typename View2, typename F = IdentityFunctor<typename View1::non_const_value_type> >
358 KOKKOS_INLINE_FUNCTION
359 static typename std::enable_if<!Block, void>::type
360 assign_block(View1& vals_dest, const View2& vals_src, const Ordinal dest, const Ordinal src, const Ordinal blockCrsSize, const F& lam = F())
361 {
362 assert(blockCrsSize == 1);
363 vals_dest[dest] = lam(vals_src[src]);
364 }
365
366 template <bool Block, typename View1>
367 KOKKOS_INLINE_FUNCTION
368 static typename std::enable_if<Block, void>::type
369 assign_block(View1& vals_dest, const Ordinal dest, const Scalar value, const Ordinal blockCrsSize)
370 {
371 const Ordinal blockItems = blockCrsSize*blockCrsSize;
372 for (Ordinal itr = 0; itr < blockItems; ++itr) {
373 vals_dest[dest*blockItems + itr] = value;
374 }
375 }
376
377 template <bool Block, typename View1>
378 KOKKOS_INLINE_FUNCTION
379 static typename std::enable_if<!Block, void>::type
380 assign_block(View1& vals_dest, const Ordinal dest, const Scalar value, const Ordinal blockCrsSize)
381 {
382 assert(blockCrsSize == 1);
383 vals_dest[dest] = value;
384 }
385
386 template <bool Block, typename View1, typename View2, typename LO, typename F = IdentityFunctor<typename View1::non_const_value_type> >
387 KOKKOS_INLINE_FUNCTION
388 static typename std::enable_if<Block, void>::type
389 assign_block_cond(View1& vals_dest, const View2& vals_src, const Ordinal dest, const Ordinal src, const LO& ordinal_lam, const Ordinal blockCrsSize, const F& lam = F())
390 {
391 const Ordinal blockItems = blockCrsSize*blockCrsSize;
392 const Ordinal dest_offset = blockItems*dest;
393 const Ordinal src_offset = blockItems*src;
394 for (Ordinal i = 0; i < blockCrsSize; ++i) {
395 for (Ordinal j = 0; j < blockCrsSize; ++j) {
396 const Ordinal blockOffset = blockCrsSize*i + j;
397 if (ordinal_lam(i, j)) {
398 vals_dest(dest_offset + blockOffset) = lam(vals_src(src_offset + blockOffset));
399 }
400 }
401 }
402 }
403
404 template <bool Block, typename View1, typename View2, typename LO, typename F = IdentityFunctor<typename View1::non_const_value_type> >
405 KOKKOS_INLINE_FUNCTION
406 static typename std::enable_if<!Block, void>::type
407 assign_block_cond(View1& vals_dest, const View2& vals_src, const Ordinal dest, const Ordinal src, const LO& ordinal_lam, const Ordinal blockCrsSize, const F& lam = F())
408 {
409 vals_dest(dest) = lam(vals_src(src));
410 }
411
412 template <bool Block, typename View1, typename View2, typename LO>
413 KOKKOS_INLINE_FUNCTION
414 static typename std::enable_if<Block, void>::type
415 assign_block_cond_val(View1& vals_dest, const View2& vals_src, const Ordinal dest, const Ordinal src, const LO& value_lam, const Ordinal blockCrsSize)
416 {
417 const Ordinal blockItems = blockCrsSize*blockCrsSize;
418 const Ordinal dest_offset = blockItems*dest;
419 const Ordinal src_offset = blockItems*src;
420 for (Ordinal i = 0; i < blockCrsSize; ++i) {
421 for (Ordinal j = 0; j < blockCrsSize; ++j) {
422 const Ordinal blockOffset = blockCrsSize*i + j;
423 const Scalar val_dest = vals_dest(dest_offset + blockOffset);
424 const Scalar val_src = vals_src(src_offset + blockOffset);
425 if (value_lam(val_dest, val_src)) {
426 vals_dest(dest_offset + blockOffset) = val_src;
427 }
428 }
429 }
430 }
431
432 template <bool Block, typename View1, typename View2, typename LO>
433 KOKKOS_INLINE_FUNCTION
434 static typename std::enable_if<!Block, void>::type
435 assign_block_cond_val(View1& vals_dest, const View2& vals_src, const Ordinal dest, const Ordinal src, const LO& value_lam, const Ordinal blockCrsSize)
436 {
437 vals_dest[dest] = vals_src[src];
438 }
439
440 template <bool Block, typename View1, typename LO>
441 KOKKOS_INLINE_FUNCTION
442 static typename std::enable_if<Block, void>::type
443 assign_block_cond_val(View1& vals_dest, const Ordinal dest, const Scalar value, const LO& value_lam, const Ordinal blockCrsSize)
444 {
445 const Ordinal blockItems = blockCrsSize*blockCrsSize;
446 const Ordinal dest_offset = blockItems*dest;
447 for (Ordinal i = 0; i < blockCrsSize; ++i) {
448 for (Ordinal j = 0; j < blockCrsSize; ++j) {
449 const Ordinal blockOffset = blockCrsSize*i + j;
450 const Scalar val_dest = vals_dest(dest_offset + blockOffset);
451 if (value_lam(val_dest, value)) {
452 vals_dest(dest_offset + blockOffset) = value;
453 }
454 }
455 }
456 }
457
458 template <bool Block, typename View1, typename LO>
459 KOKKOS_INLINE_FUNCTION
460 static typename std::enable_if<!Block, void>::type
461 assign_block_cond_val(View1& vals_dest, const Ordinal dest, const Scalar value, const LO& value_lam, const Ordinal blockCrsSize)
462 {
463 vals_dest[dest] = value;
464 }
465
466 template <bool Block, typename View1, typename View2, typename LO>
467 KOKKOS_INLINE_FUNCTION
468 static typename std::enable_if<Block, void>::type
469 assign_block_cond_trans(View1& vals_dest, const View2& vals_src, const Ordinal dest, const Ordinal src, const LO& ordinal_lam, const Ordinal blockCrsSize)
470 {
471 const Ordinal blockItems = blockCrsSize*blockCrsSize;
472 const Ordinal dest_offset = blockItems*dest;
473 const Ordinal src_offset = blockItems*src;
474 for (Ordinal i = 0; i < blockCrsSize; ++i) {
475 for (Ordinal j = 0; j < blockCrsSize; ++j) {
476 const Ordinal blockOffset = blockCrsSize*i + j;
477 const Ordinal blockOffsetT = blockCrsSize*j + i;
478 if (ordinal_lam(i, j)) {
479 vals_dest(dest_offset + blockOffsetT) = vals_src(src_offset + blockOffset);
480 }
481 }
482 }
483 }
484
485 template <bool Block, typename View1, typename View2, typename LO>
486 KOKKOS_INLINE_FUNCTION
487 static typename std::enable_if<!Block, void>::type
488 assign_block_cond_trans(View1& vals_dest, const View2& vals_src, const Ordinal dest, const Ordinal src, const LO& ordinal_lam, const Ordinal blockCrsSize)
489 {
490 vals_dest(dest) = vals_src(src);
491 }
492
493 template <bool Block, typename View1, typename View2>
494 KOKKOS_INLINE_FUNCTION
495 static typename std::enable_if<Block, void>::type
496 assign_block_trans(View1& vals_dest, const View2& vals_src, const Ordinal dest, const Ordinal src, const Ordinal blockCrsSize)
497 {
498 const Ordinal blockItems = blockCrsSize*blockCrsSize;
499 const Ordinal dest_offset = blockItems*dest;
500 const Ordinal src_offset = blockItems*src;
501 for (Ordinal i = 0; i < blockCrsSize; ++i) {
502 for (Ordinal j = 0; j < blockCrsSize; ++j) {
503 const Ordinal blockOffset = blockCrsSize*i + j;
504 const Ordinal blockOffsetT = blockCrsSize*j + i;
505 vals_dest(dest_offset + blockOffsetT) = vals_src(src_offset + blockOffset);
506 }
507 }
508 }
509
510 template <bool Block, typename View1, typename View2>
511 KOKKOS_INLINE_FUNCTION
512 static typename std::enable_if<!Block, void>::type
513 assign_block_trans(View1& vals_dest, const View2& vals_src, const Ordinal dest, const Ordinal src, const Ordinal blockCrsSize)
514 {
515 vals_dest(dest) = vals_src(src);
516 }
517
518 template <bool Block, typename View1, typename View2, typename F = IdentityFunctor<typename View1::non_const_value_type> >
519 KOKKOS_INLINE_FUNCTION
520 static typename std::enable_if<Block, void>::type
521 assign_diag_from_block(View1& diag_dest, const View2& vals_src, const Ordinal dest, const Ordinal src, const Ordinal blockCrsSize, const F& lam = F())
522 {
523 const Ordinal blockItems = blockCrsSize*blockCrsSize;
524 for (Ordinal i = 0, j = blockItems*src; i < blockCrsSize; ++i, j+=(blockCrsSize+1)) {
525 diag_dest(i + blockCrsSize*dest) = lam(vals_src(j));
526 }
527 }
528
529 template <bool Block, typename View1, typename View2, typename F = IdentityFunctor<typename View1::non_const_value_type> >
530 KOKKOS_INLINE_FUNCTION
531 static typename std::enable_if<!Block, void>::type
532 assign_diag_from_block(View1& diag_dest, const View2& vals_src, const Ordinal dest, const Ordinal src, const Ordinal blockCrsSize, const F& lam = F())
533 {
534 diag_dest(dest) = lam(vals_src(src));
535 }
536
537 template <bool Block, typename View1>
538 KOKKOS_INLINE_FUNCTION
539 static typename std::enable_if<Block, void>::type
540 assign_block_diag_only(View1& vals_dest, const Ordinal dest, typename View1::const_value_type value, const Ordinal blockCrsSize)
541 {
542 const Ordinal blockItems = blockCrsSize*blockCrsSize;
543 for (Ordinal i = 0, j = blockItems*dest; i < blockCrsSize; ++i, j+=(blockCrsSize+1)) {
544 vals_dest(j) = value;
545 }
546 }
547
548 template <bool Block, typename View1>
549 KOKKOS_INLINE_FUNCTION
550 static typename std::enable_if<!Block, void>::type
551 assign_block_diag_only(View1& vals_dest, const Ordinal dest, typename View1::const_value_type value, const Ordinal blockCrsSize)
552 {
553 vals_dest[dest] = value;
554 }
555
556 template <bool Block, typename View1, typename View2, typename F = IdentityFunctor<typename View1::non_const_value_type> >
557 KOKKOS_INLINE_FUNCTION
558 static typename std::enable_if<Block, void>::type
559 assign_diag_from_diag(View1& diag_dest, const View2& diag_src, const Ordinal dest, const Ordinal src, const Ordinal blockCrsSize, const F& lam = F())
560 {
561 for (Ordinal i = 0; i < blockCrsSize; ++i) {
562 diag_dest(dest*blockCrsSize + i) = lam(diag_src(src*blockCrsSize + i));
563 }
564 }
565
566 template <bool Block, typename View1, typename View2, typename F = IdentityFunctor<typename View1::non_const_value_type> >
567 KOKKOS_INLINE_FUNCTION
568 static typename std::enable_if<!Block, void>::type
569 assign_diag_from_diag(View1& diag_dest, const View2& diag_src, const Ordinal dest, const Ordinal src, const Ordinal blockCrsSize, const F& lam = F())
570 {
571 diag_dest(dest) = lam(diag_src(src));
572 }
573
574 template <bool Block, typename View1, typename View2, typename View3, typename L>
575 KOKKOS_INLINE_FUNCTION
576 static typename std::enable_if<Block, void>::type
577 assign_block_from_2diags(View1& vals, View2& diag_src1, View3& diag_src2, const Ordinal dest, const Ordinal src1, const Ordinal src2, const Ordinal blockCrsSize, const L& lam)
578 {
579 const Ordinal blockItems = blockCrsSize*blockCrsSize;
580 const Ordinal dest_offset = blockItems*dest;
581 const Ordinal src1_offset = blockCrsSize*src1;
582 const Ordinal src2_offset = blockCrsSize*src2;
583 for (Ordinal i = 0; i < blockCrsSize; ++i) {
584 for (Ordinal j = 0; j < blockCrsSize; ++j) {
585 const Ordinal blockOffset = blockCrsSize*i + j;
586 vals(dest_offset + blockOffset) = lam(vals(dest_offset + blockOffset), diag_src1(src1_offset+i), diag_src2(src2_offset+j));
587 }
588 }
589 }
590
591 template <bool Block, typename View1, typename View2, typename View3, typename L>
592 KOKKOS_INLINE_FUNCTION
593 static typename std::enable_if<!Block, void>::type
594 assign_block_from_2diags(View1& vals, View2& diag_src1, View3& diag_src2, const Ordinal dest, const Ordinal src1, const Ordinal src2, const Ordinal blockCrsSize, const L& lam)
595 {
596 vals(dest) = lam(vals(dest), diag_src1(src1), diag_src2(src2));
597 }
598
599 // private: Set back to private once verification work is done
600 double m_computeTime;
601 double m_applyTime;
602 double m_initTime;
603
604 Ordinal m_nRows;
605 Ordinal m_guessFlag;
606 Ordinal m_nFact;
607 Ordinal m_nTrisol;
608 Ordinal m_level;
609 Ordinal m_blkSzILU;
610 Ordinal m_blkSz;
611 Ordinal m_blockCrsSize;
612 Scalar m_omega; //Underrelaxation parameter
613 Scalar m_shift; //Manteuffel Shift
614
615 // Metis
616 bool m_useMetis;
617 OrdinalArray m_permMetis;
618 OrdinalArray m_ipermMetis;
619 OrdinalArrayMirror m_permMetisHost;
620 OrdinalArrayMirror m_ipermMetisHost;
621
622 //Lower triangular factor (CSR)
623 bool m_sptrsv_KKSpMV; // use Kokkos-Kernels SpMV for Fast SpTRSV
624 ScalarArray m_lVal;
625 OrdinalArray m_lColIdx;
626 OrdinalArray m_lRowMap;
627 // mirrors
628 ScalarArrayMirror m_lValHost;
629 OrdinalArrayMirror m_lColIdxHost;
630 OrdinalArrayMirror m_lRowMapHost;
631
632 //Lower triangular factor, without (unit) diagonals,
633 // for TRSV (not SpTRSV)
634 ScalarArrayHost m_lVal_trsvHost;
635 OrdinalArrayHost m_lColIdx_trsvHost;
636 OrdinalArrayHost m_lRowMap_trsvHost;
637
638 //Upper triangular factor (CSC)
639 ScalarArray m_uVal;
640 OrdinalArray m_uColIdx;
641 OrdinalArray m_uRowMap;
642 OrdinalArray m_a2uMap;
643 // mirrors
644 ScalarArrayMirror m_uValHost;
645 OrdinalArrayMirror m_uColIdxHost;
646 OrdinalArrayMirror m_uRowMapHost;
647
648 //Upper triangular factor (CSR)
649 ScalarArray m_utVal;
650 OrdinalArray m_utColIdx;
651 OrdinalArray m_utRowMap;
652 // mirrors
653 ScalarArrayMirror m_utValHost;
654 OrdinalArrayMirror m_utColIdxHost;
655 OrdinalArrayMirror m_utRowMapHost;
656
657 //Upper triangular factor (CSR), with diagonal extracted out
658 // for TRSV (not SpTRSV)
659 bool m_doUnitDiag_TRSV; // perform TRSV with unit diagonals
660 ScalarArrayHost m_dVal_trsvHost;
661 ScalarArrayHost m_utVal_trsvHost;
662 OrdinalArrayHost m_utColIdx_trsvHost;
663 OrdinalArrayHost m_utRowMap_trsvHost;
664
665 //Pointer to the copy of input A.
666 bool m_skipSortMatrix;
667 // device
668 ScalarArray m_aValIn;
669 OrdinalArray m_aRowMapIn;
670 OrdinalArray m_aColIdxIn;
671 // host
672 ScalarArrayMirror m_aValInHost;
673 OrdinalArrayMirror m_aRowMapInHost;
674 OrdinalArrayMirror m_aColIdxInHost;
675
676 //A matrix in COO format
677 ScalarArray m_aVal;
678 OrdinalArray m_aRowMap;
679 OrdinalArray m_aRowIdx;
680 OrdinalArray m_aColIdx;
681 // mirrors
682 ScalarArrayMirror m_aValHost;
683 OrdinalArrayMirror m_aRowMapHost;
684 OrdinalArrayMirror m_aRowIdxHost;
685 OrdinalArrayMirror m_aColIdxHost;
686 OrdinalArrayHost m_aLvlIdxHost;
687
688 //Diagonal scaling factors
689 RealArray m_diagFact;
690 ScalarArray m_diagElems;
691
692 //Temporary vectors for triangular solves
693 ScalarArray m_xOld;
694 ScalarArray m_xTemp;
695 ScalarArray m_onesVector;
696
697 //This will have the continuation initial
698 //guess if guessFlag=1
699 Teuchos::RCP<FastPrec> m_initGuessPrec;
700
701 // forward/backwar substitution for standard SpTrsv
702 using MemSpace = typename ExecSpace::memory_space;
703 using KernelHandle = KokkosKernels::Experimental::KokkosKernelsHandle <Ordinal, Ordinal, Scalar, ExecSpace, MemSpace, MemSpace >;
704 FastILU::SpTRSV m_sptrsv_algo;
705
706 KernelHandle khL;
707 KernelHandle khU;
708
709 void findFills(int levfill, OrdinalArrayMirror aRowMap, OrdinalArrayMirror aColIdx,
710 int& nzl, std::vector<int> &lRowMap, std::vector<int> &lColIdx, std::vector<int> &lLevel,
711 int& nzu, std::vector<int> &uRowMap, std::vector<int> &uColIdx, std::vector<int> &uLevel) {
712 using std::vector;
713 using std::sort;
714
715 const Ordinal n = lRowMap.size() - 1;
716
717 Ordinal row = 0, i = 0;
718 vector<int> lnklst(n);
719 vector<int> curlev(n);
720 vector<int> iwork(n);
721
722 int knzl = 0;
723 int knzu = 0;
724
725 lRowMap[0] = 0;
726 uRowMap[0] = 0;
727
728 for (i=0; i<n; i++)
729 {
730 int first, next, j;
731 row = (m_useMetis ? m_permMetisHost(i) : i);
732
733 /* copy column indices of row into workspace and sort them */
734 int len = aRowMap[row+1] - aRowMap[row];
735 next = 0;
736 for (j=aRowMap[row]; j<aRowMap[row+1]; j++) {
737 iwork[next++] = (m_useMetis ? m_ipermMetisHost(aColIdx[j]) : aColIdx[j]);
738 }
739 // sort column indices in non-descending (ascending) order
740 sort(iwork.begin(), iwork.begin() + len);
741
742 /* construct implied linked list for row */
743 first = iwork[0];
744 curlev[first] = 0;
745
746 for (j=0; j<=len-2; j++)
747 {
748 lnklst[iwork[j]] = iwork[j+1];
749 curlev[iwork[j]] = 0;
750 }
751
752 lnklst[iwork[len-1]] = n;
753 curlev[iwork[len-1]] = 0;
754
755 /* merge with rows in U */
756 next = first;
757 while (next < i)
758 {
759 int oldlst = next;
760 int nxtlst = lnklst[next];
761 int inner_row = next;
762 int ii;
763
764 /* scan row */
765 for (ii=uRowMap[inner_row]+1; ii<uRowMap[inner_row+1]; /*nop*/)
766 {
767 if (uColIdx[ii] < nxtlst)
768 {
769 /* new fill-in */
770 int newlev = curlev[inner_row] + uLevel[ii] + 1;
771 if (newlev <= levfill)
772 {
773 lnklst[oldlst] = uColIdx[ii];
774 lnklst[uColIdx[ii]] = nxtlst;
775 oldlst = uColIdx[ii];
776 curlev[uColIdx[ii]] = newlev;
777 }
778 ii++;
779 }
780 else if (uColIdx[ii] == nxtlst)
781 {
782 int newlev;
783 oldlst = nxtlst;
784 nxtlst = lnklst[oldlst];
785 newlev = curlev[inner_row] + uLevel[ii] + 1;
786 //curlev[uColIdx[ii]] = MIN(curlev[uColIdx[ii]], newlev);
787 if (curlev[uColIdx[ii]] > newlev)
788 {
789 curlev[uColIdx[ii]] = newlev;
790 }
791 ii++;
792 }
793 else /* (jau[ii] > nxtlst) */
794 {
795 oldlst = nxtlst;
796 nxtlst = lnklst[oldlst];
797 }
798 }
799 next = lnklst[next];
800 }
801
802 /* gather the pattern into L and U */
803 /* L (no diagonal) */
804 next = first;
805 while (next < i)
806 {
807 assert(knzl < nzl);
808 lLevel[knzl] = curlev[next];
809 lColIdx[knzl++] = next;
810 if (knzl >= nzl)
811 {
812 nzl = nzl + n;
813 lColIdx.resize(nzl);
814 lLevel.resize(nzl);
815 }
816 next = lnklst[next];
817 }
818 lRowMap[i+1] = knzl;
819 assert(next == i);
820 /* U (with diagonal) */
821 while (next < n)
822 {
823 assert(knzu < nzu);
824 uLevel[knzu] = curlev[next];
825 uColIdx[knzu++] = next;
826 if (knzu >= nzu)
827 {
828 nzu = nzu + n;
829 uColIdx.resize(nzu);
830 uLevel.resize(nzu);
831 }
832 next = lnklst[next];
833 }
834 uRowMap[i+1] = knzu;
835 }
836
837 nzl = knzl;
838 nzu = knzu;
839 }
840
841 //Symbolic ILU code
842 //initializes the matrices L and U and readies them
843 //according to the level of fill
844 void symbolicILU(OrdinalArrayMirror ia, OrdinalArrayMirror ja)
845 {
846 using WithoutInit = Kokkos::ViewAllocateWithoutInitializing;
847 FASTILU_CREATE_TIMER(timer);
848 using std::vector;
849 using std::stable_sort;
850 using std::sort;
851 const Ordinal nRowsUnblocked = ia.extent(0) - 1;
852 int nzu = ia[nRowsUnblocked];
853 int nzl = ia[nRowsUnblocked];
854 Ordinal i;
855
856 //Compute sparsity structure of ILU
857 nzl *= (m_level + 2);
858 nzu *= (m_level + 2);
859 vector<int> ial(nRowsUnblocked+1);
860 vector<int> jal(nzl);
861 vector<int> levell(nzl);
862 vector<int> iau(nRowsUnblocked+1);
863 vector<int> jau(nzu);
864 vector<int> levelu(nzu);
865
866 // TODO: if (initGuess & level > 0), call this with (aRowMap_, aColIdx_) and level = 1
867 findFills(m_level, ia, ja, // input
868 nzl, ial, jal, levell, // output L in CSR
869 nzu, iau, jau, levelu // output U in CSR
870 );
871 FASTILU_REPORT_TIMER(timer, " findFills time");
872
873 FASTILU_DBG_COUT(
874 "nzl =" << nzl << "\n" <<
875 "nzu =" << nzu << "\n" <<
876 "ILU: nnz = "<< nzl + nzu << "\n" <<
877 "Actual nnz for ILU: " << nzl + nzu);
878
879 // Initialize the A host mirror matrix that is to be used in the computation
880 m_aRowMapHost = OrdinalArrayMirror(WithoutInit("aRowMap"), nRowsUnblocked + 1);
881 m_aColIdxHost = OrdinalArrayMirror(WithoutInit("aColIdx"), nzl + nzu);
882 m_aRowIdxHost = OrdinalArrayMirror(WithoutInit("aRowIdx"), nzl + nzu);
883 m_aLvlIdxHost = OrdinalArrayHost(WithoutInit("aLvlIdx"), nzl + nzu);
884
885 Ordinal aRowPtr = 0;
886 m_aRowMapHost[0] = aRowPtr;
887 for (i = 0; i < nRowsUnblocked; i++)
888 {
889 FASTILU_DBG_COUT("***row:" << i);
890 for(Ordinal k = ial[i]; k < ial[i+1]; k++)
891 {
892 FASTILU_DBG_COUT("jal[k]=" << jal[k]);
893 m_aColIdxHost[aRowPtr] = jal[k];
894 m_aRowIdxHost[aRowPtr] = i;
895 m_aLvlIdxHost[aRowPtr] = levell[k];
896 aRowPtr++;
897 }
898 for(Ordinal k = iau[i]; k < iau[i+1]; k++)
899 {
900 m_aColIdxHost[aRowPtr] = jau[k];
901 m_aRowIdxHost[aRowPtr] = i;
902 m_aLvlIdxHost[aRowPtr] = levelu[k];
903 aRowPtr++;
904 }
905 m_aRowMapHost[i+1] = aRowPtr;
906 }
907 FASTILU_REPORT_TIMER(timer, " Copy time");
908
909 // sort based on ColIdx, RowIdx stays the same (do we need this?)
910 using host_space = typename HostSpace::execution_space;
911 KokkosSparse::sort_crs_graph<host_space, OrdinalArrayMirror, OrdinalArrayMirror>
912 (m_aRowMapHost, m_aColIdxHost);
913 FASTILU_FENCE_REPORT_TIMER(timer, host_space(), " Sort time");
914 }
915
916 void symbolicILU(OrdinalArrayMirror pRowMap_, OrdinalArrayMirror pColIdx_, OrdinalArrayHost pLvlIdx_)
917 {
918 using WithoutInit = Kokkos::ViewAllocateWithoutInitializing;
919 const Ordinal nRowsUnblocked = pRowMap_.extent(0) - 1;
920 Ordinal nnzA = 0;
921 for (Ordinal k = 0; k < pRowMap_(nRowsUnblocked); k++) {
922 if(pLvlIdx_(k) <= m_level) {
923 nnzA++;
924 }
925 }
926 //Initialize the A matrix that is to be used in the computation
927 m_aRowMapHost = OrdinalArrayMirror(WithoutInit("aRowMap"), nRowsUnblocked + 1);
928 m_aColIdxHost = OrdinalArrayMirror(WithoutInit("aColIdx"), nnzA);
929 m_aRowIdxHost = OrdinalArrayMirror(WithoutInit("aRowIds"), nnzA);
930 m_aValHost = ScalarArrayMirror(WithoutInit("aVal"), nnzA);
931
932 Ordinal aRowPtr = 0;
933 m_aRowMapHost[0] = aRowPtr;
934 for (Ordinal i = 0; i < nRowsUnblocked; i++)
935 {
936 for(Ordinal k = pRowMap_(i); k < pRowMap_(i+1); k++)
937 {
938 if (pLvlIdx_(k) <= m_level) {
939 m_aColIdxHost[aRowPtr] = pColIdx_[k];
940 m_aRowIdxHost[aRowPtr] = i;
941 aRowPtr++;
942 }
943 }
944 m_aRowMapHost[i+1] = aRowPtr;
945 }
946 }
947
948 void symbolicILU_common()
949 {
950 using WithoutInit = Kokkos::ViewAllocateWithoutInitializing;
951
952 // Ensure all filled entries have the sentinel value
953 #ifdef FASTILU_ONE_TO_ONE_UNBLOCKED
954 aVal_ = ScalarArrayMirror("aVal", aColIdx_.extent(0));
955 #else
956 m_aValHost = ScalarArrayMirror("aVal", m_aColIdxHost.extent(0) * m_blockCrsSize * m_blockCrsSize);
957 #endif
958 Kokkos::deep_copy(m_aValHost, std::numeric_limits<Scalar>::min());
959
960 // Re-block A_. At this point, aHost and A_ are unblocked. The host stuff isn't
961 // used anymore after this, so just reblock A_
962 #ifdef FASTILU_ONE_TO_ONE_UNBLOCKED
963 if (m_blockCrsSize > 1) {
964 reblock(aRowMap_, aRowIdx_, aColIdx_, aVal_, m_blockCrsSize);
965 }
966 #endif
967
968 // Initialize A
969 m_aRowMap = OrdinalArray(WithoutInit("aRowMap"), m_aRowMapHost.extent(0));
970 m_aColIdx = OrdinalArray(WithoutInit("aColIdx"), m_aColIdxHost.extent(0));
971 m_aRowIdx = OrdinalArray(WithoutInit("aRowIdx"), m_aRowIdxHost.extent(0));
972 m_aVal = ScalarArray (WithoutInit("aVal"), m_aValHost.extent(0));
973
974 // Copy A_ to A
975 Kokkos::deep_copy(m_aRowMap, m_aRowMapHost);
976 Kokkos::deep_copy(m_aColIdx, m_aColIdxHost);
977 Kokkos::deep_copy(m_aRowIdx, m_aRowIdxHost);
978 Kokkos::deep_copy(m_aVal, m_aValHost);
979 FASTILU_DBG_COUT("**Finished initializing A");
980
981 //Compute RowMap for L and U.
982 // > form RowMap for L
983 m_lRowMap = OrdinalArray(WithoutInit("lRowMap"), m_nRows + 1);
984 m_lRowMapHost = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_lRowMap);
985 const Ordinal nnzL = countL();
986 FASTILU_DBG_COUT("**Finished counting L");
987
988 // > form RowMap for U and Ut
989 m_uRowMap = OrdinalArray(WithoutInit("uRowMap"), m_nRows + 1);
990 m_utRowMap = OrdinalArray(WithoutInit("utRowMap"), m_nRows + 1);
991 m_utRowMapHost = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_utRowMap);
992 m_uRowMapHost = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_uRowMap);
993 const Ordinal nnzU = countU();
994 FASTILU_DBG_COUT("**Finished counting U");
995
996 //Allocate memory and initialize pattern for L, U (transpose).
997 m_lColIdx = OrdinalArray(WithoutInit("lColIdx"), nnzL);
998 m_uColIdx = OrdinalArray(WithoutInit("uColIdx"), nnzU);
999 m_utColIdx = OrdinalArray(WithoutInit("utColIdx"), nnzU);
1000
1001 m_lVal = ScalarArray("lVal", nnzL * m_blockCrsSize * m_blockCrsSize);
1002 m_uVal = ScalarArray("uVal", nnzU * m_blockCrsSize * m_blockCrsSize);
1003 m_utVal = ScalarArray(WithoutInit("utVal"), nnzU * m_blockCrsSize * m_blockCrsSize);
1004
1005 //Create mirror
1006 m_lColIdxHost = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_lColIdx);
1007 m_uColIdxHost = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_uColIdx);
1008 m_utColIdxHost = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_utColIdx);
1009
1010 m_lValHost = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_lVal);
1011 m_uValHost = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_uVal);
1012 m_utValHost = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_utVal);
1013 }
1014
1015 void numericILU()
1016 {
1017 const Scalar zero = STS::zero();
1018 FASTILU_CREATE_TIMER(Timer);
1019 if (m_useMetis && (m_guessFlag == 0 || m_level == 0)) { // applied only at the first call (level 0)
1020 // apply column permutation before sorting it
1021 FastILUPrec_Functor perm_functor(m_aColIdxIn, m_ipermMetis);
1022 Kokkos::RangePolicy<ColPermTag, ExecSpace> perm_policy (0, m_aColIdxIn.size());
1023 Kokkos::parallel_for(
1024 "numericILU::colPerm", perm_policy, perm_functor);
1025 }
1026
1027 //Sort each row of AIn by ColIdx
1028 if (!m_skipSortMatrix || m_useMetis) {
1029 if (m_blockCrsSize > 1) {
1030 KokkosSparse::sort_bsr_matrix<ExecSpace, OrdinalArray, OrdinalArray, ScalarArray>(m_blockCrsSize, m_aRowMapIn, m_aColIdxIn, m_aValIn);
1031 }
1032 else {
1033 KokkosSparse::sort_crs_matrix<ExecSpace, OrdinalArray, OrdinalArray, ScalarArray>(m_aRowMapIn, m_aColIdxIn, m_aValIn);
1034 }
1035 }
1036
1037 //Copy the host matrix into the initialized a;
1038 //a contains the structure of ILU(k), values of original Ain is copied at level-0
1039 FastILUPrec_Functor functor(m_aValIn, m_aRowMapIn, m_aColIdxIn, m_aVal, m_diagFact, m_aRowMap, m_aColIdx, m_aRowIdx, m_blockCrsSize);
1040 if (m_useMetis) {
1041 assert(m_blockCrsSize == 1); // Not yet supported for block crs
1042 FastILUPrec_Functor functor_perm(m_aValIn, m_aRowMapIn, m_aColIdxIn, m_permMetis, m_aVal, m_diagFact, m_aRowMap, m_aColIdx, m_aRowIdx, m_blockCrsSize);
1043 Kokkos::RangePolicy<CopySortedValsPermTag, ExecSpace> copy_perm_policy (0, m_nRows);
1044 Kokkos::parallel_for(
1045 "numericILU::copyVals", copy_perm_policy, functor_perm);
1046 } else {
1047 Kokkos::RangePolicy<CopySortedValsKeepSentinelsTag, ExecSpace> copy_policy (0, m_nRows);
1048 Kokkos::parallel_for(
1049 "numericILU::copyVals", copy_policy, functor);
1050 }
1051 FASTILU_FENCE_REPORT_TIMER(Timer, ExecSpace(), " + sort/copy/permute values");
1052
1053 // obtain diagonal scaling factor
1054 Kokkos::RangePolicy<GetDiagsTag, ExecSpace> get_policy (0, m_nRows);
1055 Kokkos::parallel_for(
1056 "numericILU::getDiags", get_policy, functor);
1057
1058 // apply diagonal scaling
1059 Kokkos::RangePolicy<DiagScalTag, ExecSpace> scale_policy (0, m_nRows);
1060 Kokkos::parallel_for(
1061 "numericILU::diagScal", scale_policy, functor);
1062
1063 // applyShift
1064 if (m_shift != zero) {
1065 Kokkos::deep_copy(m_aValHost, m_aVal);
1066 applyManteuffelShift();
1067 Kokkos::deep_copy(m_aVal, m_aValHost);
1068 }
1069 else {
1070 Kokkos::deep_copy(m_aValHost, m_aVal); // keep in-sync
1071 }
1072 FASTILU_FENCE_REPORT_TIMER(Timer, ExecSpace(), " + apply shift/scale");
1073 FASTILU_DBG_COUT("**Finished diagonal scaling");
1074
1075 fillL();
1076 FASTILU_FENCE_REPORT_TIMER(Timer, ExecSpace(), " + fill L");
1077 FASTILU_DBG_COUT("**Finished copying L");
1078
1079 fillU();
1080 FASTILU_FENCE_REPORT_TIMER(Timer, ExecSpace(), " + fill U");
1081 FASTILU_DBG_COUT(
1082 "**Finished copying U\n" <<
1083 "nnz L = " << m_lRowMapHost[m_nRows] << "\n" <<
1084 "nnz U = " << m_uRowMapHost[m_nRows]);
1085 }
1086
1087 //Initialize the rowMap (rowPtr) for L
1088 int countL()
1089 {
1090 m_lRowMapHost[0] = 0;
1091 for (Ordinal i = 0; i < m_nRows; i++)
1092 {
1093 Ordinal row_count = 0;
1094 for (Ordinal k = m_aRowMapHost[i]; k < m_aRowMapHost[i+1]; k++)
1095 {
1096 Ordinal row = i;
1097 Ordinal col = m_aColIdxHost[k];
1098
1099 if (row >= col)
1100 {
1101 row_count++;
1102 }
1103 }
1104 m_lRowMapHost[i+1] = m_lRowMapHost[i] + row_count;
1105 }
1106 Kokkos::deep_copy(m_lRowMap, m_lRowMapHost);
1107 return m_lRowMapHost[m_nRows];
1108 }
1109
1110 //Put the initial guess into L.
1111 void fillL()
1112 {
1113 // extract L out of A, where A contains the structure of ILU(k), and original nonzero values at level-0
1114 FastILUPrec_Functor functor(m_aVal, m_aRowMap, m_aColIdx, m_lVal, m_lRowMap, m_lColIdx, m_diagElems, m_blockCrsSize);
1115 Kokkos::RangePolicy<GetLowerTag, ExecSpace> getL_policy (0, m_nRows);
1116 Kokkos::parallel_for(
1117 "numericILU::getLower", getL_policy, functor);
1118
1119 if ((m_level > 0) && (m_guessFlag !=0))
1120 {
1121 // overwrite initial values from warmup runs
1122 OrdinalArray lGRowMap;
1123 OrdinalArray lGColIdx;
1124 ScalarArray lGVal;
1125 ScalarArray gD;
1126 m_initGuessPrec->getL(lGRowMap, lGColIdx, lGVal);
1127 m_initGuessPrec->getD(gD);
1128 Kokkos::deep_copy(m_diagElems, gD);
1129
1130 // copy LG into L
1131 FastILUPrec_Functor functorG(lGVal, lGRowMap, lGColIdx, m_lVal, m_lRowMap, m_lColIdx, m_blockCrsSize);
1132 Kokkos::RangePolicy<CopySortedValsTag, ExecSpace> copy_policy (0, m_nRows);
1133 Kokkos::parallel_for(
1134 "numericILU::copyVals(G)", copy_policy, functorG);
1135 }
1136 }
1137
1138 //Initialize rowMap of U
1139 int countU()
1140 {
1141 // extract U out of A, where A contains the structure of ILU(k), and original nonzero values at level-0
1142 for (Ordinal i = 0; i <= m_nRows; i++)
1143 {
1144 m_uRowMapHost[i] = 0;
1145 }
1146 for(Ordinal i = 0; i < m_nRows; i++)
1147 {
1148 for(Ordinal k = m_aRowMapHost[i]; k < m_aRowMapHost[i+1]; k++)
1149 {
1150 Ordinal row = i;
1151 Ordinal col = m_aColIdxHost[k];
1152 if (row <= col)
1153 {
1154 m_uRowMapHost[col+1]++;
1155 }
1156 }
1157 }
1158 for (Ordinal i = 0; i < m_nRows; i++)
1159 {
1160 m_uRowMapHost[i+1] += m_uRowMapHost[i];
1161 }
1162 Kokkos::deep_copy(m_uRowMap, m_uRowMapHost);
1163
1164 // create a map from A to U (sorted)
1165 auto nnzU = m_uRowMapHost[m_nRows];
1166 m_a2uMap = OrdinalArray("a2uMap", nnzU);
1167 auto a2uMap = Kokkos::create_mirror_view(m_a2uMap);
1168 for (Ordinal i = 0; i < m_nRows; i++)
1169 {
1170 for (Ordinal k = m_aRowMapHost[i]; k < m_aRowMapHost[i+1]; k++)
1171 {
1172 Ordinal row = m_aRowIdxHost[k];
1173 Ordinal col = m_aColIdxHost[k];
1174 if (row <= col)
1175 {
1176 Ordinal pos = m_uRowMapHost[col];
1177 a2uMap(pos) = k;
1178 m_uRowMapHost[col]++;
1179 }
1180 }
1181 }
1182 Kokkos::deep_copy(m_a2uMap, a2uMap);
1183 // shift back pointer
1184 for (Ordinal i = m_nRows; i > 0; i--)
1185 {
1186 m_uRowMapHost[i] = m_uRowMapHost[i-1];
1187 }
1188 m_uRowMapHost[0] = 0;
1189
1190 return nnzU;
1191 }
1192
1193 //Put initial guess into U
1194 void fillU()
1195 {
1196 FASTILU_CREATE_TIMER(Timer);
1197 int nnzU = m_a2uMap.extent(0);
1198 ParPermCopyFunctor<Ordinal, Scalar, ExecSpace, BlockCrsEnabled> permCopy(m_a2uMap, m_aVal, m_aRowIdx, m_aColIdx, m_uVal, m_uColIdx, m_blockCrsSize);
1199 Kokkos::parallel_for(Kokkos::RangePolicy<ExecSpace>(0, nnzU), permCopy);
1200
1201 if ((m_level > 0) && (m_guessFlag !=0))
1202 {
1203 // overwrite initial values from warmup runs
1204 ScalarArray gD;
1205 m_initGuessPrec->getD(gD);
1206 Kokkos::deep_copy(m_diagElems, gD);
1207
1208 // copy UG into U
1209 OrdinalArray uGRowMap;
1210 OrdinalArray uGColIdx;
1211 ScalarArray uGVal;
1212 m_initGuessPrec->getU(uGRowMap, uGColIdx, uGVal);
1213
1214 Kokkos::RangePolicy<CopySortedValsTag, ExecSpace> copy_policy (0, m_nRows);
1215 FastILUPrec_Functor functorG(uGVal, uGRowMap, uGColIdx, m_uVal, m_uRowMap, m_uColIdx, m_blockCrsSize);
1216 Kokkos::parallel_for(
1217 "numericILU::copyVals(G)", copy_policy, functorG);
1218 FASTILU_FENCE_REPORT_TIMER(Timer, ExecSpace(), " + merge_sorted");
1219 }
1220 }
1221
1222 void getL(OrdinalArray &lRowMapOut, OrdinalArray &lColIdxOut, ScalarArray &lValOut)
1223 {
1224 lRowMapOut = m_lRowMap;
1225 lColIdxOut = m_lColIdx;
1226 lValOut = m_lVal;
1227 }
1228
1229 void getU(OrdinalArray &uRowMapOut, OrdinalArray &uColIdxOut, ScalarArray &uValOut)
1230 {
1231 uRowMapOut = m_uRowMap;
1232 uColIdxOut = m_uColIdx;
1233 uValOut = m_uVal;
1234 }
1235
1236 void getUt(OrdinalArray &utRowMapOut, OrdinalArray &utColIdxOut, ScalarArray &utValOut)
1237 {
1238 utRowMapOut = m_utRowMap;
1239 utColIdxOut = m_utColIdx;
1240 utValOut = m_utVal;
1241 }
1242
1243 void getD(ScalarArray &diagElemsOut)
1244 {
1245 diagElemsOut = m_diagElems;
1246 }
1247
1248 void applyManteuffelShift()
1249 {
1250 ShiftFunctor shift_lambda(m_shift);
1251 NotDiagFunctor not_diag_lamb;
1252 for (Ordinal i = 0; i < m_nRows; i++)
1253 {
1254 for (Ordinal k = m_aRowMapHost[i]; k < m_aRowMapHost[i+1]; k++)
1255 {
1256 Ordinal row = i;
1257 Ordinal col = m_aColIdxHost[k];
1258 if (row != col)
1259 {
1260 assign_block<BlockCrsEnabled>(m_aValHost, m_aValHost, k, k, m_blockCrsSize, shift_lambda);
1261 }
1262 else {
1263 assign_block_cond<BlockCrsEnabled>(m_aValHost, m_aValHost, k, k, not_diag_lamb, m_blockCrsSize, shift_lambda);
1264 }
1265 }
1266 }
1267 }
1268
1269 void applyD_Perm(ScalarArray &x, ScalarArray &y)
1270 {
1271 Kokkos::RangePolicy<NonTranPermScalTag, ExecSpace> scale_policy (0, x.extent(0));
1272 PermScalFunctor<Ordinal, Scalar, Real, ExecSpace> functor(x, y, m_diagFact, m_permMetis);
1273 Kokkos::parallel_for(
1274 "numericILU::applyD_iPerm", scale_policy, functor);
1275 }
1276
1277 void applyD_iPerm(ScalarArray &x, ScalarArray &y)
1278 {
1279 Kokkos::RangePolicy<TranPermScalTag, ExecSpace> scale_policy (0, x.extent(0));
1280 PermScalFunctor<Ordinal, Scalar, Real, ExecSpace> functor(x, y, m_diagFact, m_ipermMetis);
1281 Kokkos::parallel_for(
1282 "numericILU::applyD_iPerm", scale_policy, functor);
1283 }
1284
1285 void applyD(ScalarArray &x, ScalarArray &y)
1286 {
1287 ParScalFunctor<Ordinal, Scalar, Real, ExecSpace> parScal(x, y, m_diagFact);
1288 Kokkos::parallel_for(Kokkos::RangePolicy<ExecSpace>(0, x.extent(0)), parScal);
1289 }
1290
1291 void applyL(ScalarArray &x, ScalarArray &y)
1292 {
1293 ParInitZeroFunctor<Ordinal, Scalar, ExecSpace> parInitZero(m_xOld);
1294 Kokkos::parallel_for(Kokkos::RangePolicy<ExecSpace>(0, m_nRows), parInitZero);
1295 BlockJacobiIterFunctorL<Ordinal, Scalar, ExecSpace> jacIter(m_nRows, m_blkSz, m_lRowMap, m_lColIdx, m_lVal, x, y, m_xOld, m_onesVector);
1296 ParCopyFunctor<Ordinal, Scalar, ExecSpace> parCopy(m_xOld, y);
1297 Ordinal extent = m_nRows/m_blkSz;
1298 if (m_nRows%m_blkSz != 0)
1299 {
1300 extent++;
1301 }
1302 for (Ordinal i = 0; i < m_nTrisol; i++)
1303 {
1304 //Kokkos::parallel_for(Kokkos::RangePolicy<ExecSpace>(0, nRows), jacIter);
1305 Kokkos::parallel_for(Kokkos::RangePolicy<ExecSpace>(0, extent), jacIter);
1306 Kokkos::parallel_for(Kokkos::RangePolicy<ExecSpace>(0, m_nRows), parCopy);
1307 }
1308 }
1309
1310 void applyU(ScalarArray &x, ScalarArray &y)
1311 {
1312 ParInitZeroFunctor<Ordinal, Scalar, ExecSpace> parInitZero(m_xOld);
1313 Kokkos::parallel_for(Kokkos::RangePolicy<ExecSpace>(0, m_nRows), parInitZero);
1314 ExecSpace().fence();
1315 BlockJacobiIterFunctorU<Ordinal, Scalar, ExecSpace> jacIter(m_nRows, m_blkSz, m_utRowMap, m_utColIdx, m_utVal, x, y, m_xOld, m_diagElems);
1316 ParCopyFunctor<Ordinal, Scalar, ExecSpace> parCopy(m_xOld, y);
1317 Ordinal extent = m_nRows/m_blkSz;
1318 if (m_nRows%m_blkSz != 0)
1319 {
1320 extent++;
1321 }
1322 for (Ordinal i = 0; i < m_nTrisol; i++)
1323 {
1324 //Kokkos::parallel_for(Kokkos::RangePolicy<ExecSpace>(0, nRows), jacIter);
1325 Kokkos::parallel_for(Kokkos::RangePolicy<ExecSpace>(0, extent), jacIter);
1326 Kokkos::parallel_for(Kokkos::RangePolicy<ExecSpace>(0, m_nRows), parCopy);
1327 }
1328 }
1329
1330
1331 public:
1332 //Constructor
1333 //TODO: Use a Teuchos::ParameterList object
1334 FastILUPrec(bool skipSortMatrix, OrdinalArray &aRowMapIn, OrdinalArray &aColIdxIn, ScalarArray &aValIn, Ordinal nRows,
1335 FastILU::SpTRSV sptrsv_algo, Ordinal nFact, Ordinal nTrisol, Ordinal level, Scalar omega, Scalar shift,
1336 Ordinal guessFlag, Ordinal blkSzILU, Ordinal blkSz, Ordinal blockCrsSize = 1)
1337 {
1338 m_nRows = nRows;
1339 m_sptrsv_algo = sptrsv_algo;
1340 m_nFact = nFact;
1341 m_nTrisol = nTrisol;
1342
1343 m_useMetis = false;
1344
1345 m_computeTime = 0.0;
1346 m_applyTime = 0.0;
1347 m_initTime = 0.0;
1348 m_level = level;
1349
1350 // mirror & deep-copy the input matrix
1351 m_skipSortMatrix = skipSortMatrix;
1352 m_aRowMapIn = aRowMapIn;
1353 m_aColIdxIn = aColIdxIn;
1354 m_aValIn = aValIn;
1355
1356 m_omega = omega;
1357 m_guessFlag = guessFlag;
1358 m_shift = shift;
1359 m_blkSzILU = blkSzILU;
1360 m_blkSz = blkSz;
1361 m_blockCrsSize = blockCrsSize;
1362 m_doUnitDiag_TRSV = true; // perform TRSV with unit diagonals
1363 m_sptrsv_KKSpMV = true; // use Kokkos-Kernels SpMV for Fast SpTRSV
1364
1365 if (!BlockCrsEnabled) {
1366 assert(blockCrsSize == 1);
1367 }
1368
1369 const Scalar one = STS::one();
1370 m_onesVector = ScalarArray("onesVector", nRows);
1371 Kokkos::deep_copy(m_onesVector, one);
1372
1373 m_diagFact = RealArray("diagFact", nRows * blockCrsSize);
1374 m_diagElems = ScalarArray("diagElems", nRows * blockCrsSize);
1375 m_xOld = ScalarArray("xOld", nRows * blockCrsSize);
1376 m_xTemp = ScalarArray("xTemp", nRows * blockCrsSize);
1377
1378 m_aRowMapInHost = OrdinalArrayMirror("aRowMapHost", m_aRowMapIn.size());
1379 m_aColIdxInHost = OrdinalArrayMirror("aColIdxHost", m_aColIdxIn.size());
1380 m_aValInHost = ScalarArrayMirror("aValHost", m_aValIn.size());
1381 Kokkos::deep_copy(m_aRowMapInHost, aRowMapIn);
1382 Kokkos::deep_copy(m_aColIdxInHost, aColIdxIn);
1383 Kokkos::deep_copy(m_aValInHost, aValIn);
1384 #ifdef FASTILU_ONE_TO_ONE_UNBLOCKED
1385 if (m_blockCrsSize > 1) {
1386 unblock(m_aRowMapHost, m_aColIdxHost, m_aValHost, m_blockCrsSize);
1387 }
1388 #endif
1389
1390 if ((m_level > 0) && (m_guessFlag != 0))
1391 {
1392 m_initGuessPrec = Teuchos::rcp(new FastPrec(skipSortMatrix, aRowMapIn, aColIdxIn, aValIn, nRows, sptrsv_algo,
1393 3, 5, level-1, omega, shift, guessFlag, blkSzILU, blkSz, blockCrsSize));
1394 }
1395 }
1396
1397 // internal functors
1398 struct ColPermTag {};
1399 struct CopySortedValsTag {};
1400 struct CopySortedValsKeepSentinelsTag {};
1401 struct CopySortedValsPermTag {};
1402 struct GetDiagsTag {};
1403 struct DiagScalTag {};
1404 struct SwapDiagTag {};
1405 struct GetLowerTag{};
1406
1407 struct FastILUPrec_Functor
1408 {
1409 FastILUPrec_Functor(ScalarArray aValIn_, OrdinalArray aRowMapIn_, OrdinalArray aColIdxIn_,
1410 ScalarArray aVal_, RealArray diagFact_, OrdinalArray aRowMap_, OrdinalArray aColIdx_, OrdinalArray aRowIdx_, Ordinal blockCrsSize_) :
1411 aValIn (aValIn_),
1412 aRowMapIn (aRowMapIn_),
1413 aColIdxIn (aColIdxIn_),
1414 aVal (aVal_),
1415 diagFact (diagFact_),
1416 aRowMap (aRowMap_),
1417 aColIdx (aColIdx_),
1418 aRowIdx (aRowIdx_),
1419 blockCrsSize(blockCrsSize_)
1420 {}
1421
1422 // just calling CopySortedValsPermTag
1423 FastILUPrec_Functor(ScalarArray aValIn_, OrdinalArray aRowMapIn_, OrdinalArray aColIdxIn_, OrdinalArray perm_,
1424 ScalarArray aVal_, RealArray diagFact_, OrdinalArray aRowMap_, OrdinalArray aColIdx_, OrdinalArray aRowIdx_, Ordinal blockCrsSize_) :
1425 aValIn (aValIn_),
1426 aRowMapIn (aRowMapIn_),
1427 aColIdxIn (aColIdxIn_),
1428 aVal (aVal_),
1429 diagFact (diagFact_),
1430 aRowMap (aRowMap_),
1431 aColIdx (aColIdx_),
1432 aRowIdx (aRowIdx_),
1433 iperm (perm_),
1434 blockCrsSize(blockCrsSize_)
1435 {}
1436
1437 // just calling CopySortedValsTag, or GetUpperTag
1438 FastILUPrec_Functor(ScalarArray aValIn_, OrdinalArray aRowMapIn_, OrdinalArray aColIdxIn_,
1439 ScalarArray aVal_, OrdinalArray aRowMap_, OrdinalArray aColIdx_, Ordinal blockCrsSize_) :
1440 aValIn (aValIn_),
1441 aRowMapIn (aRowMapIn_),
1442 aColIdxIn (aColIdxIn_),
1443 aVal (aVal_),
1444 aRowMap (aRowMap_),
1445 aColIdx (aColIdx_),
1446 blockCrsSize(blockCrsSize_)
1447 {}
1448
1449 // just calling GetLowerTag
1450 FastILUPrec_Functor(ScalarArray aVal_, OrdinalArray aRowMap_, OrdinalArray aColIdx_,
1451 ScalarArray lVal_, OrdinalArray lRowMap_, OrdinalArray lColIdx_,
1452 ScalarArray diagElems_, Ordinal blockCrsSize_) :
1453 aVal (aVal_),
1454 diagElems (diagElems_),
1455 aRowMap (aRowMap_),
1456 aColIdx (aColIdx_),
1457 lVal (lVal_),
1458 lRowMap (lRowMap_),
1459 lColIdx (lColIdx_),
1460 blockCrsSize(blockCrsSize_)
1461 {}
1462
1463 // just calling SwapDiagTag
1464 FastILUPrec_Functor(const SwapDiagTag&, ScalarArray lVal_, OrdinalArray lRowMap_, OrdinalArray lColIdx_,
1465 ScalarArray utVal_, OrdinalArray utRowMap_, OrdinalArray utColIdx_,
1466 ScalarArray diagElems_, Ordinal blockCrsSize_) :
1467 diagElems (diagElems_),
1468 lVal (lVal_),
1469 lRowMap (lRowMap_),
1470 lColIdx (lColIdx_),
1471 utVal (utVal_),
1472 utRowMap (utRowMap_),
1473 utColIdx (utColIdx_),
1474 blockCrsSize(blockCrsSize_)
1475 {}
1476
1477 // just calling ColPerm
1478 FastILUPrec_Functor(OrdinalArray aColIdx_, OrdinalArray iperm_) :
1479 aColIdx (aColIdx_),
1480 iperm (iperm_),
1481 blockCrsSize(0)
1482 {}
1483
1484
1485 // ------------------------------------------------
1486 // functor to load values
1487 // both matrices are sorted and, "a" (with fills) contains "aIn" (original)
1488 KOKKOS_INLINE_FUNCTION
1489 void operator()(const CopySortedValsTag &, const int i) const {
1490 Ordinal aPtr = aRowMapIn[i];
1491 Ordinal aPtrEnd = aRowMapIn[i+1];
1492 for(Ordinal k = aRowMap[i]; k < aRowMap[i+1]; k++)
1493 {
1494 Ordinal col = aColIdx[k];
1495 if (aPtr < aPtrEnd && col == aColIdxIn[aPtr])
1496 {
1497 assign_block<BlockCrsEnabled>(aVal, aValIn, k, aPtr, blockCrsSize);
1498 aPtr++;
1499 } else
1500 {
1501 assign_block<BlockCrsEnabled>(aVal, k, STS::zero(), blockCrsSize);
1502 }
1503 }
1504 }
1505
1506 // ------------------------------------------------
1507 // functor to load values
1508 // both matrices are sorted and, "a" (with fills) contains "aIn" (original)
1509 KOKKOS_INLINE_FUNCTION
1510 void operator()(const CopySortedValsKeepSentinelsTag &, const int i) const {
1511 IsNotSentinelFunctor sentinel_lamb;
1512 Ordinal aPtr = aRowMapIn[i];
1513 Ordinal aPtrEnd = aRowMapIn[i+1];
1514 for(Ordinal k = aRowMap[i]; k < aRowMap[i+1]; k++)
1515 {
1516 Ordinal col = aColIdx[k];
1517 if (aPtr < aPtrEnd && col == aColIdxIn[aPtr])
1518 {
1519 assign_block_cond_val<BlockCrsEnabled>(aVal, aValIn, k, aPtr, sentinel_lamb, blockCrsSize);
1520 aPtr++;
1521 } else
1522 {
1523 assign_block_cond_val<BlockCrsEnabled>(aVal, k, STS::zero(), sentinel_lamb, blockCrsSize);
1524 }
1525 }
1526 }
1527
1528 // ------------------------------------------------
1529 // functor to load values with perm
1530 // both matrices are sorted and, "a" (with fills) contains "aIn" (original)
1531 KOKKOS_INLINE_FUNCTION
1532 void operator()(const CopySortedValsPermTag &, const int i) const {
1533 Ordinal aPtr = aRowMapIn[iperm[i]];
1534 Ordinal aPtrEnd = aRowMapIn[iperm[i]+1];
1535 for(Ordinal k = aRowMap[i]; k < aRowMap[i+1]; k++)
1536 {
1537 Ordinal col = aColIdx[k];
1538 if (aPtr < aPtrEnd && col == aColIdxIn[aPtr])
1539 {
1540 assign_block<BlockCrsEnabled>(aVal, aValIn, k, aPtr, blockCrsSize);
1541 aPtr++;
1542 } else
1543 {
1544 assign_block<BlockCrsEnabled>(aVal, k, STS::zero(), blockCrsSize);
1545 }
1546 }
1547 }
1548
1549 // functor to extract diagonals (inverted)
1550 KOKKOS_INLINE_FUNCTION
1551 void operator()(const GetDiagsTag &, const int i) const {
1552 InvSqrtFunctor dlambda;
1553 for(int k = aRowMap[i]; k < aRowMap[i+1]; k++)
1554 {
1555 aRowIdx[k] = i;
1556 if (aColIdx[k] == i)
1557 {
1558 assign_diag_from_block<BlockCrsEnabled>(diagFact, aVal, i, k, blockCrsSize, dlambda);
1559 }
1560 }
1561 }
1562
1563 // functor to swap diagonals
1564 KOKKOS_INLINE_FUNCTION
1565 void operator()(const SwapDiagTag &, const int i) const {
1566 const Scalar zero = STS::zero();
1567 // zero the diagonal of L. If sorted, this finds it on first iter.
1568 Ordinal lRowBegin = lRowMap(i);
1569 Ordinal lRowEnd = lRowMap(i + 1);
1570 for(Ordinal j = 0; j < lRowEnd - lRowBegin; j++) {
1571 Ordinal reversed = lRowEnd - j - 1;
1572 if(lColIdx(reversed) == i) {
1573 assign_block_diag_only<BlockCrsEnabled>(lVal, reversed, zero, blockCrsSize);
1574 break;
1575 }
1576 }
1577 // zero the diagonal of Ut. If sorted, this finds it on first iter.
1578 Ordinal utRowBegin = utRowMap(i);
1579 Ordinal utRowEnd = utRowMap(i + 1);
1580 for(Ordinal j = utRowBegin; j < utRowEnd; j++) {
1581 if(utColIdx(j) == i) {
1582 assign_block_diag_only<BlockCrsEnabled>(utVal, j, zero, blockCrsSize);
1583 break;
1584 }
1585 }
1586 // invert D
1587 InvFunctor dlambda;
1588 assign_diag_from_diag<BlockCrsEnabled>(diagElems, diagElems, i, i, blockCrsSize, dlambda);
1589 }
1590
1591 // functor to apply diagonal scaling
1592 KOKKOS_INLINE_FUNCTION
1593 void operator()(const DiagScalTag &, const int i) const {
1594 ProdThreeFunctor dlambda;
1595 for (int k = aRowMap[i]; k < aRowMap[i+1]; k++)
1596 {
1597 const Ordinal col = aColIdx[k];
1598 assign_block_from_2diags<BlockCrsEnabled>(aVal, diagFact, diagFact, k, i, col, blockCrsSize, dlambda);
1599 }
1600 }
1601
1602 // ----------------------------------------------------------
1603 // functor to extract L & diagongals
1604 KOKKOS_INLINE_FUNCTION
1605 void operator()(const GetLowerTag &, const int i) const {
1606 Ordinal lPtr = lRowMap[i];
1607 LowerFunctor lower_lamb;
1608 for (Ordinal k = aRowMap[i]; k < aRowMap[i+1]; k++)
1609 {
1610 Ordinal row = i;
1611 Ordinal col = aColIdx[k];
1612 if (row >= col)
1613 {
1614 if (row == col)
1615 {
1616 assign_diag_from_block<BlockCrsEnabled>(diagElems, aVal, row, k, blockCrsSize);
1617 assign_block_diag_only<BlockCrsEnabled>(lVal, lPtr, STS::one(), blockCrsSize);
1618 if (BlockCrsEnabled) {
1619 assign_block_cond<BlockCrsEnabled>(lVal, aVal, lPtr, k, lower_lamb, blockCrsSize);
1620 }
1621 } else {
1622 assign_block<BlockCrsEnabled>(lVal, aVal, lPtr, k, blockCrsSize);
1623 }
1624 lColIdx[lPtr] = col;
1625 lPtr++;
1626 }
1627 }
1628 }
1629
1630 // ----------------------------------------------------------
1631 // functor to apply column permutation
1632 KOKKOS_INLINE_FUNCTION
1633 void operator()(const ColPermTag &, const int i) const {
1634 aColIdx(i) = iperm(aColIdx(i));
1635 }
1636
1637 // member variables
1638 // + input matrix
1639 ScalarArray aValIn;
1640 OrdinalArray aRowMapIn;
1641 OrdinalArray aColIdxIn;
1642 // + output matrix
1643 ScalarArray aVal;
1644 ScalarArray diagElems;
1645 RealArray diagFact;
1646 OrdinalArray aRowMap;
1647 OrdinalArray aColIdx;
1648 OrdinalArray aRowIdx;
1649 // + output L matrix
1650 ScalarArray lVal;
1651 OrdinalArray lRowMap;
1652 OrdinalArray lColIdx;
1653 // + output U matrix
1654 ScalarArray utVal;
1655 OrdinalArray utRowMap;
1656 OrdinalArray utColIdx;
1657 // permutation
1658 OrdinalArray iperm;
1659 // blockCrs block size
1660 const Ordinal blockCrsSize;
1661 };
1662
1663 // set Metis pre-ordering
1664 template<class MetisArrayHost>
1665 void setMetisPerm(MetisArrayHost permMetis, MetisArrayHost ipermMetis)
1666 {
1667 Ordinal nRows_ = permMetis.size();
1668 if (m_nRows > 0) {
1669 m_permMetis = OrdinalArray("permMetis", nRows_);
1670 m_ipermMetis = OrdinalArray("ipermMetis", nRows_);
1671
1672 m_permMetisHost = Kokkos::create_mirror_view(m_permMetis);
1673 m_ipermMetisHost = Kokkos::create_mirror_view(m_ipermMetis);
1674 for (Ordinal i = 0; i < nRows_; i++) {
1675 m_permMetisHost(i) = permMetis(i);
1676 m_ipermMetisHost(i) = ipermMetis(i);
1677 }
1678 Kokkos::deep_copy(m_permMetis, m_permMetisHost);
1679 Kokkos::deep_copy(m_ipermMetis, m_ipermMetisHost);
1680 }
1681 if ((m_level > 0) && (m_guessFlag != 0))
1682 {
1683 m_initGuessPrec->setMetisPerm(permMetis, ipermMetis);
1684 }
1685 m_useMetis = true;
1686 }
1687
1688 //Symbolic Factorization Phase
1689 void initialize()
1690 {
1691 Kokkos::Timer timer;
1692 FASTILU_CREATE_TIMER(timer2);
1693 // call symbolic that generates A with level associated to each nonzero entry
1694 // then pass that to initialize the initGuessPrec
1695 symbolicILU(m_aRowMapInHost, m_aColIdxInHost);
1696 FASTILU_REPORT_TIMER(timer2, " + initial SymbolicILU (" << m_level << ") time");
1697 if ((m_level > 0) && (m_guessFlag != 0))
1698 {
1699 m_initGuessPrec->initialize(m_aRowMapHost, m_aColIdxHost, m_aLvlIdxHost);
1700 FASTILU_REPORT_TIMER(timer2, " > SymbolicILU (" << m_level << ") time");
1701 }
1702
1703 initialize_common(timer);
1704 FASTILU_REPORT_TIMER(timer, "Symbolic phase complete.\nInit time");
1705 }
1706
1707 //Symbolic Factorization Phase
1708 void initialize(OrdinalArrayMirror pRowMap_, OrdinalArrayMirror pColIdx_, OrdinalArrayHost pLvlIdx_)
1709 {
1710 Kokkos::Timer timer;
1711 FASTILU_CREATE_TIMER(timer2);
1712 // call symbolic that generates A with level associated to each nonzero entry
1713 // then pass that to initialize the initGuessPrec
1714 symbolicILU(pRowMap_, pColIdx_, pLvlIdx_);
1715 FASTILU_REPORT_TIMER(timer2, " - initial SymbolicILU (" << m_level << ") time");
1716 if ((m_level > 0) && (m_guessFlag != 0))
1717 {
1718 m_initGuessPrec->initialize(pRowMap_, pColIdx_, pLvlIdx_);
1719 FASTILU_REPORT_TIMER(timer2, " = SymbolicILU (" << m_level << ") time");
1720 }
1721
1722 initialize_common(timer);
1723 FASTILU_REPORT_TIMER(timer, " + Symbolic phase complete.\n + Init time");
1724 }
1725
1726 void initialize_common(Kokkos::Timer& timer)
1727 {
1728 //Allocate memory for the local A.
1729 //initialize L, U, A patterns
1730 symbolicILU_common();
1731
1732 ExecSpace().fence(); //Fence so that init time is accurate
1733 m_initTime = timer.seconds();
1734 }
1735
1736 void setValues(ScalarArray& aValIn_)
1737 {
1738 this->m_aValIn = aValIn_;
1739 this->m_aValInHost = Kokkos::create_mirror_view(aValIn_);
1740 Kokkos::deep_copy(this->m_aValInHost, aValIn_);
1741 if(!m_initGuessPrec.is_null())
1742 {
1743 m_initGuessPrec->setValues(aValIn_);
1744 }
1745 }
1746
1747 //Actual computation phase.
1748 //blkSzILU is the chunk size (hard coded).
1749 //1 gives the best performance on GPUs.
1750 //
1751 void compute()
1752 {
1753 Kokkos::Timer timer;
1754 FASTILU_CREATE_TIMER(Timer);
1755 if ((m_level > 0) && (m_guessFlag !=0))
1756 {
1757 m_initGuessPrec->compute();
1758 FASTILU_FENCE_REPORT_TIMER(Timer, ExecSpace(), " > initGuess");
1759 }
1760
1761 numericILU();
1762 FASTILU_FENCE_REPORT_TIMER(Timer, ExecSpace(), " > numericILU ");
1763
1764 FastILUFunctor<Ordinal, Scalar, ExecSpace> iluFunctor(m_aRowMapHost[m_nRows], m_blkSzILU,
1765 m_aRowMap, m_aRowIdx, m_aColIdx, m_aVal,
1766 m_lRowMap, m_lColIdx, m_lVal, m_uRowMap, m_uColIdx, m_uVal, m_diagElems, m_omega, m_blockCrsSize, m_level);
1767 Ordinal extent = m_aRowMapHost[m_nRows]/m_blkSzILU;
1768 if (m_aRowMapHost[m_nRows]%m_blkSzILU != 0)
1769 {
1770 extent++;
1771 }
1772
1773 for (int i = 0; i < m_nFact; i++)
1774 {
1775 #ifdef FASTILU_ONE_TO_ONE_UNBLOCKED
1776 // Force serialization to avoid races but still perform operation on device
1777 Kokkos::parallel_for(Kokkos::RangePolicy<ExecSpace>(0, 1), KOKKOS_LAMBDA(const int) {
1778 for (Ordinal j = 0; j < extent; ++j) {
1779 iluFunctor(j);
1780 }
1781 });
1782 #else
1783 Kokkos::parallel_for(Kokkos::RangePolicy<ExecSpace>(0, extent), iluFunctor);
1784 #endif
1785 }
1786 FASTILU_FENCE_REPORT_TIMER(Timer, ExecSpace(), " > iluFunctor (" << m_nFact << ")");
1787
1788 // transpose u
1789 Kokkos::deep_copy(m_utRowMap, 0);
1790 if (m_blockCrsSize == 1) {
1791 KokkosSparse::Impl::transpose_matrix<OrdinalArray, OrdinalArray, ScalarArray, OrdinalArray, OrdinalArray, ScalarArray, OrdinalArray, ExecSpace>
1792 (m_nRows, m_nRows, m_uRowMap, m_uColIdx, m_uVal, m_utRowMap, m_utColIdx, m_utVal);
1793 }
1794 else {
1795 KokkosSparse::Impl::transpose_bsr_matrix<OrdinalArray, OrdinalArray, ScalarArray, OrdinalArray, OrdinalArray, ScalarArray, ExecSpace>
1796 (m_nRows, m_nRows, m_blockCrsSize, m_uRowMap, m_uColIdx, m_uVal, m_utRowMap, m_utColIdx, m_utVal);
1797 }
1798
1799 // sort, if the triangular solve algorithm requires a sorted matrix.
1800 bool sortRequired = m_sptrsv_algo != FastILU::SpTRSV::Fast && m_sptrsv_algo != FastILU::SpTRSV::StandardHost;
1801 if (sortRequired) {
1802 if (m_blockCrsSize == 1) {
1803 KokkosSparse::sort_crs_matrix<ExecSpace, OrdinalArray, OrdinalArray, ScalarArray>
1804 (m_utRowMap, m_utColIdx, m_utVal);
1805 }
1806 else {
1807 KokkosSparse::sort_bsr_matrix<ExecSpace, OrdinalArray, OrdinalArray, ScalarArray>(
1808 m_blockCrsSize, m_utRowMap, m_utColIdx, m_utVal);
1809 }
1810 }
1811
1812 if (m_sptrsv_algo == FastILU::SpTRSV::StandardHost) {
1813 // deep-copy to host
1814 Kokkos::deep_copy(m_lColIdxHost, m_lColIdx);
1815 Kokkos::deep_copy(m_lValHost, m_lVal);
1816
1817 Kokkos::deep_copy(m_utRowMapHost, m_utRowMap);
1818 Kokkos::deep_copy(m_utColIdxHost, m_utColIdx);
1819 Kokkos::deep_copy(m_utValHost, m_utVal);
1820 }
1821 FASTILU_FENCE_REPORT_TIMER(Timer, ExecSpace(), " > transposeU");
1822
1823 if (m_sptrsv_algo == FastILU::SpTRSV::Standard) {
1824 assert(m_blockCrsSize == 1); // Not yet supported for block crs
1825 #if defined(KOKKOSKERNELS_ENABLE_TPL_CUSPARSE)
1826 KokkosSparse::Experimental::SPTRSVAlgorithm algo = KokkosSparse::Experimental::SPTRSVAlgorithm::SPTRSV_CUSPARSE;
1827 #else
1828 KokkosSparse::Experimental::SPTRSVAlgorithm algo = KokkosSparse::Experimental::SPTRSVAlgorithm::SEQLVLSCHD_TP1;
1829 #endif
1830 // setup L solve
1831 khL.create_sptrsv_handle(algo, m_nRows, true);
1832 #if defined(KOKKOSKERNELS_ENABLE_TPL_CUSPARSE)
1833 KokkosSparse::Experimental::sptrsv_symbolic(&khL, m_lRowMap, m_lColIdx, m_lVal);
1834 #else
1835 KokkosSparse::Experimental::sptrsv_symbolic(&khL, m_lRowMap, m_lColIdx);
1836 #endif
1837 // setup U solve
1838 khU.create_sptrsv_handle(algo, m_nRows, false);
1839 #if defined(KOKKOSKERNELS_ENABLE_TPL_CUSPARSE)
1840 KokkosSparse::Experimental::sptrsv_symbolic(&khU, m_utRowMap, m_utColIdx, m_utVal);
1841 #else
1842 KokkosSparse::Experimental::sptrsv_symbolic(&khU, m_utRowMap, m_utColIdx);
1843 #endif
1844 FASTILU_FENCE_REPORT_TIMER(Timer, ExecSpace(),
1845 " > sptrsv_symbolic : nnz(L)=" << m_lColIdx.extent(0) << " nnz(U)=" << m_utColIdx.extent(0));
1846 } else if (m_sptrsv_algo == FastILU::SpTRSV::StandardHost && m_doUnitDiag_TRSV) {
1847 // Prepare L for TRSV by removing unit-diagonals
1848 assert(m_blockCrsSize == 1); // Not yet supported for block crs
1849 m_lVal_trsvHost = ScalarArrayHost ("lVal_trsv", m_lRowMapHost[m_nRows]-m_nRows);
1850 m_lColIdx_trsvHost = OrdinalArrayHost("lColIdx_trsv", m_lRowMapHost[m_nRows]-m_nRows);
1851 m_lRowMap_trsvHost = OrdinalArrayHost("lRowMap_trsv", m_nRows+1);
1852
1853 size_t nnzL = 0;
1854 m_lRowMap_trsvHost(0) = 0;
1855 for (Ordinal i = 0; i < m_nRows; i++) {
1856 for (Ordinal k = m_lRowMapHost(i); k < m_lRowMapHost[i+1]; k++) {
1857 if (m_lColIdxHost(k) != i) {
1858 m_lVal_trsvHost(nnzL) = m_lValHost(k);
1859 m_lColIdx_trsvHost(nnzL) = m_lColIdxHost(k);
1860 nnzL++;
1861 }
1862 }
1863 m_lRowMap_trsvHost(i+1)=nnzL;
1864 }
1865
1866 // Prepare U by extracting and scaling D
1867 m_dVal_trsvHost = ScalarArrayHost ("dVal_trsv", m_nRows);
1868 m_utVal_trsvHost = ScalarArrayHost ("utVal_trsv", m_utRowMapHost[m_nRows]-m_nRows);
1869 m_utColIdx_trsvHost = OrdinalArrayHost("utColIdx_trsv", m_utRowMapHost[m_nRows]-m_nRows);
1870 m_utRowMap_trsvHost = OrdinalArrayHost("utRowMap_trsv", m_nRows+1);
1871
1872 size_t nnzU = 0;
1873 m_utRowMap_trsvHost(0) = 0;
1874 for (Ordinal i = 0; i < m_nRows; i++) {
1875 for (Ordinal k = m_utRowMapHost(i); k < m_utRowMapHost[i+1]; k++) {
1876 if (m_utColIdxHost(k) == i) {
1877 m_dVal_trsvHost(i) = m_utValHost(k);
1878 } else {
1879 m_utVal_trsvHost(nnzU) = m_utValHost(k);
1880 m_utColIdx_trsvHost(nnzU) = m_utColIdxHost(k);
1881 nnzU++;
1882 }
1883 }
1884 m_utRowMap_trsvHost(i+1)=nnzU;
1885 }
1886 for (Ordinal i = 0; i < m_nRows; i++) {
1887 for (Ordinal k = m_utRowMap_trsvHost(i); k < m_utRowMap_trsvHost[i+1]; k++) {
1888 m_utVal_trsvHost(k) = m_utVal_trsvHost(k) / m_dVal_trsvHost(i);
1889 }
1890 m_dVal_trsvHost(i) = STS::one() / m_dVal_trsvHost(i);
1891 }
1892 } else if (m_sptrsv_KKSpMV) {
1893 FastILUPrec_Functor functor(SwapDiagTag(), m_lVal, m_lRowMap, m_lColIdx, m_utVal, m_utRowMap, m_utColIdx, m_diagElems, m_blockCrsSize);
1894 Kokkos::RangePolicy<SwapDiagTag, ExecSpace> swap_policy (0, m_nRows);
1895 Kokkos::parallel_for(
1896 "numericILU::swapDiag", swap_policy, functor);
1897 }
1898 ExecSpace().fence(); // Fence so computeTime is accurate
1899 m_computeTime = timer.seconds();
1900 FASTILU_REPORT_TIMER(timer, " >> compute done\n");
1901 }
1902
1903 template <typename CRS>
1904 void sptrsv_impl(ScalarArray &x, ScalarArray &y, CRS& crsmatL, CRS& crsmatU)
1905 {
1906 const Scalar one(1.0);
1907 const Scalar minus_one(-1.0);
1908
1909 const auto nrows_unblocked = m_xOld.extent(0);
1910 Scalar2dArray x2d_old (const_cast<Scalar*>(m_xOld.data()), nrows_unblocked, 1);
1911
1912 Scalar2dArray x2d (const_cast<Scalar*>(m_xTemp.data()), nrows_unblocked, 1);
1913 Scalar2dArray y2d (const_cast<Scalar*>(y.data()), nrows_unblocked, 1);
1914
1915 // 1) approximately solve, y = L^{-1}*x
1916 // functor to copy RHS x into y (for even iteration)
1917 ParCopyFunctor<Ordinal, Scalar, ExecSpace> copy_x2y(y, m_xTemp);
1918 // functor to copy RHS x into xold (for odd iteration)
1919 ParCopyFunctor<Ordinal, Scalar, ExecSpace> copy_x2xold(m_xOld, m_xTemp);
1920
1921 // functor to copy x_old to y (final iteration)
1922 ParCopyFunctor<Ordinal, Scalar, ExecSpace> copy_xold2y(y, m_xOld);
1923
1924 // xold = zeros
1925 ParInitZeroFunctor<Ordinal, Scalar, ExecSpace> initZeroX(m_xOld);
1926 Kokkos::parallel_for(Kokkos::RangePolicy<ExecSpace>(0, nrows_unblocked), initZeroX);
1927 //Kokkos::deep_copy(x2d_old, zero);
1928 for (Ordinal i = 0; i < m_nTrisol; i++)
1929 {
1930 if (i%2 == 0) {
1931 // y = y - L*x_old
1932 // > y = x
1933 Kokkos::parallel_for(Kokkos::RangePolicy<ExecSpace>(0, nrows_unblocked), copy_x2y);
1934 // > y = y - L*x_old
1935 KokkosSparse::spmv("N", minus_one, crsmatL, x2d_old, one, y2d);
1936 } else {
1937 // x_old = x_old - L*y
1938 // > x_old = x
1939 Kokkos::parallel_for(Kokkos::RangePolicy<ExecSpace>(0, nrows_unblocked), copy_x2xold);
1940 // > x_old = x_old - L*y
1941 KokkosSparse::spmv("N", minus_one, crsmatL, y2d, one, x2d_old);
1942
1943 if (i == m_nTrisol-1) {
1944 // y = x_old
1945 Kokkos::parallel_for(Kokkos::RangePolicy<ExecSpace>(0, nrows_unblocked), copy_xold2y);
1946 }
1947 }
1948 }
1949
1950 // 2) approximately solve, x = U^{-1}*y
1951 // functor to copy y into x
1952 ParCopyFunctor<Ordinal, Scalar, ExecSpace> copy_y2x(m_xTemp, y);
1953 // functor to copy y into xold
1954 ParCopyFunctor<Ordinal, Scalar, ExecSpace> copy_y2xold(m_xOld, y);
1955
1956 // functor to scale x
1957 ParScalFunctor<Ordinal, Scalar, Scalar, ExecSpace> scal_x(m_xTemp, m_xTemp, m_diagElems);
1958 ParScalFunctor<Ordinal, Scalar, Scalar, ExecSpace> scal_xold(m_xOld, m_xOld, m_diagElems);
1959
1960 // functor to copy x_old to x (final iteration)
1961 ParCopyFunctor<Ordinal, Scalar, ExecSpace> copy_xold2x(m_xTemp, m_xOld);
1962
1963 // xold = zeros
1964 Kokkos::parallel_for(Kokkos::RangePolicy<ExecSpace>(0, nrows_unblocked), initZeroX);
1965 //Kokkos::deep_copy(x2d_old, zero);
1966 for (Ordinal i = 0; i < m_nTrisol; i++)
1967 {
1968 if (i%2 == 0) {
1969 // x = y - U*x_old
1970 // > x = y
1971 Kokkos::parallel_for(Kokkos::RangePolicy<ExecSpace>(0, nrows_unblocked), copy_y2x);
1972 // > x = x - U*x_old
1973 KokkosSparse::spmv("N", minus_one, crsmatU, x2d_old, one, x2d);
1974 // > scale x = inv(diag(U))*x
1975 Kokkos::parallel_for(Kokkos::RangePolicy<ExecSpace>(0, nrows_unblocked), scal_x);
1976 } else {
1977 // xold = y - U*x
1978 // > xold = y
1979 Kokkos::parallel_for(Kokkos::RangePolicy<ExecSpace>(0, nrows_unblocked), copy_y2xold);
1980 // > x = x - U*x_old
1981 KokkosSparse::spmv("N", minus_one, crsmatU, x2d, one, x2d_old);
1982 // > scale x = inv(diag(U))*x
1983 Kokkos::parallel_for(Kokkos::RangePolicy<ExecSpace>(0, nrows_unblocked), scal_xold);
1984
1985 if (i == m_nTrisol-1) {
1986 // x = x_old
1987 Kokkos::parallel_for(Kokkos::RangePolicy<ExecSpace>(0, nrows_unblocked), copy_xold2x);
1988 }
1989 }
1990 }
1991 }
1992
1993 //Preconditioner application. Note that this does
1994 //*not* support multiple right hand sizes.
1995 void apply(ScalarArray &x, ScalarArray &y)
1996 {
1997 Kokkos::Timer timer;
1998
1999 //required to prevent contamination of the input.
2000 ParCopyFunctor<Ordinal, Scalar, ExecSpace> parCopyFunctor(m_xTemp, x);
2001 Kokkos::parallel_for(Kokkos::RangePolicy<ExecSpace>(0, x.extent(0)), parCopyFunctor);
2002
2003 //apply D
2004 if (m_useMetis) {
2005 applyD_Perm(x, m_xTemp);
2006 } else {
2007 applyD(x, m_xTemp);
2008 }
2009
2010 if (m_sptrsv_algo == FastILU::SpTRSV::Standard) {
2011 assert(m_blockCrsSize == 1); // Not yet supported for block crs
2012 // solve with L
2013 KokkosSparse::Experimental::sptrsv_solve(&khL, m_lRowMap, m_lColIdx, m_lVal, m_xTemp, y);
2014 // solve with U
2015 KokkosSparse::Experimental::sptrsv_solve(&khU, m_utRowMap, m_utColIdx, m_utVal, y, m_xTemp);
2016 } else {
2017 // wrap x and y into 2D views
2018 Scalar2dArray x2d (const_cast<Scalar*>(m_xTemp.data()), m_nRows, 1);
2019 Scalar2dArray y2d (const_cast<Scalar*>(y.data()), m_nRows, 1);
2020
2021 if (m_sptrsv_algo == FastILU::SpTRSV::StandardHost) {
2022 assert(m_blockCrsSize == 1); // Not yet supported for block crs
2023
2024 // copy x to host
2025 auto x_ = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, x2d);
2026 // don't need to initialize y_ because KokkosSparse::trsv always overwrites the output vectors
2027 // (there is no alpha/beta, it's as if beta is always 0)
2028 auto y_ = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, y2d);
2029 Kokkos::deep_copy(x_, x2d);
2030
2031 if (m_doUnitDiag_TRSV) {
2032 using crsmat_host_t = KokkosSparse::CrsMatrix<Scalar, Ordinal, HostSpace, void, Ordinal>;
2033 using graph_host_t = typename crsmat_host_t::StaticCrsGraphType;
2034
2035 // wrap L into crsmat on host
2036 graph_host_t static_graphL(m_lColIdx_trsvHost, m_lRowMap_trsvHost);
2037 crsmat_host_t crsmatL("CrsMatrix", m_nRows, m_lVal_trsvHost, static_graphL);
2038
2039 // wrap U into crsmat on host
2040 graph_host_t static_graphU(m_utColIdx_trsvHost, m_utRowMap_trsvHost);
2041 crsmat_host_t crsmatU("CrsMatrix", m_nRows, m_utVal_trsvHost, static_graphU);
2042
2043 // solve with L, unit-diag
2044 KokkosSparse::trsv ("L", "N", "U", crsmatL, x_, y_);
2045 // solve with D
2046 for (Ordinal i = 0; i < m_nRows; i++) {
2047 y_(i, 0) = m_dVal_trsvHost(i) * y_(i, 0);
2048 }
2049 // solve with U, unit-diag
2050 KokkosSparse::trsv ("U", "N", "U", crsmatU, y_, x_);
2051 } else {
2052 using crsmat_mirror_t = KokkosSparse::CrsMatrix<Scalar, Ordinal, MirrorSpace, void, Ordinal>;
2053 using graph_mirror_t = typename crsmat_mirror_t::StaticCrsGraphType;
2054
2055 // wrap L into crsmat on host
2056 graph_mirror_t static_graphL(m_lColIdxHost, m_lRowMapHost);
2057 crsmat_mirror_t crsmatL("CrsMatrix", m_nRows, m_lValHost, static_graphL);
2058
2059 // wrap U into crsmat on host
2060 graph_mirror_t static_graphU(m_utColIdxHost, m_utRowMapHost);
2061 crsmat_mirror_t crsmatU("CrsMatrix", m_nRows, m_utValHost, static_graphU);
2062
2063 // solve with L
2064 KokkosSparse::trsv ("L", "N", "N", crsmatL, x_, y_);
2065 // solve with U
2066 KokkosSparse::trsv ("U", "N", "N", crsmatU, y_, x_);
2067 }
2068 // copy x to device
2069 Kokkos::deep_copy(x2d, x_);
2070 } else {
2071 if (m_sptrsv_KKSpMV) {
2072 if (m_blockCrsSize == 1) {
2073 using crsmat_t = KokkosSparse::CrsMatrix<Scalar, Ordinal, ExecSpace, void, Ordinal>;
2074 using graph_t = typename crsmat_t::StaticCrsGraphType;
2075
2076 graph_t static_graphL(m_lColIdx, m_lRowMap);
2077 crsmat_t crsmatL("CrsMatrix", m_nRows, m_lVal, static_graphL);
2078
2079 graph_t static_graphU(m_utColIdx, m_utRowMap);
2080 crsmat_t crsmatU("CrsMatrix", m_nRows, m_utVal, static_graphU);
2081
2082 sptrsv_impl(x, y, crsmatL, crsmatU);
2083 }
2084 else {
2085 using crsmat_t = KokkosSparse::Experimental::BsrMatrix<Scalar, Ordinal, ExecSpace, void, Ordinal>;
2086 using graph_t = typename crsmat_t::StaticCrsGraphType;
2087
2088 graph_t static_graphL(m_lColIdx, m_lRowMap);
2089 crsmat_t crsmatL("BsrMatrix", m_nRows, m_lVal, static_graphL, m_blockCrsSize);
2090
2091 graph_t static_graphU(m_utColIdx, m_utRowMap);
2092 crsmat_t crsmatU("BsrMatrix", m_nRows, m_utVal, static_graphU, m_blockCrsSize);
2093
2094 sptrsv_impl(x, y, crsmatL, crsmatU);
2095 }
2096 } else {
2097 assert(m_blockCrsSize == 1); // Not yet supported for block crs
2098
2099 //apply L^{-1} to xTemp
2100 applyL(m_xTemp, y);
2101 //apply U^{-1} to y
2102 applyU(y, m_xTemp);
2103 }
2104 }
2105 }
2106 //apply D again (we assume that the scaling is
2107 //symmetric for now).
2108 if (m_useMetis) {
2109 applyD_iPerm(m_xTemp, y);
2110 } else {
2111 applyD(m_xTemp, y);
2112 }
2113 //Only fencing here so that apply time is accurate
2114 ExecSpace().fence();
2115 m_applyTime = timer.seconds();
2116 }
2117
2118 Ordinal getNFact() const
2119 {
2120 return m_nFact;
2121 }
2122
2123 std::string getSpTrsvType() const
2124 {
2125 if (m_sptrsv_algo == FastILU::SpTRSV::StandardHost) {
2126 return "Standard Host";
2127 } else if (m_sptrsv_algo == FastILU::SpTRSV::Standard) {
2128 return "Standard";
2129 } else if (m_sptrsv_algo == FastILU::SpTRSV::Fast) {
2130 return "Fast";
2131 }
2132 return "Invalid";
2133 }
2134
2135 Ordinal getNTrisol() const
2136 {
2137 return m_nTrisol;
2138 }
2139
2140 Ordinal getNRows() const
2141 {
2142 return m_nRows;
2143 }
2144
2145 double getComputeTime() const
2146 {
2147 return m_computeTime;
2148 }
2149
2150 double getInitializeTime() const
2151 {
2152 return m_initTime;
2153 }
2154
2155 double getApplyTime() const
2156 {
2157 return m_applyTime;
2158 }
2159
2160 //Compute the L2 norm of the nonlinear residual (A - LU) on sparsity pattern
2161 void checkILU() const
2162 {
2163 assert(m_blockCrsSize == 1); // Not yet supported for block crs
2164 Scalar sum = 0.0;
2165 Scalar sum_diag = 0.0;
2166 for (int i = 0 ; i < m_nRows; i++)
2167 {
2168 // Each row in A matrix (result)
2169 for (int k = m_aRowMap[i]; k < m_aRowMap[i+1]; k++)
2170 {
2171 Scalar acc_val = m_aVal[k];
2172 Ordinal lptr, uptr;
2173 for ( lptr = m_lRowMap[i], uptr = m_uRowMap[m_aColIdx[k]] ;
2174 lptr < m_lRowMap[i+1] && uptr < m_uRowMap[m_aColIdx[k]+1] ; )
2175 {
2176 if (m_lColIdx[lptr] == m_uColIdx[uptr])
2177 {
2178 acc_val -= m_lVal[lptr] * m_uVal[uptr];
2179 lptr++;
2180 uptr++;
2181 }
2182 else if (m_lColIdx[lptr] < m_uColIdx[uptr])
2183 lptr++;
2184 else
2185 uptr++;
2186 }
2187 sum += acc_val * acc_val;
2188 }
2189 }
2190
2191 for (int i = 0; i < m_nRows; i++)
2192 {
2193 sum_diag += m_diagElems[i]*m_diagElems[i];
2194 }
2195
2196 FASTILU_DBG_COUT("l2 norm of nonlinear residual = " << RTS::sqrt(STS::abs(sum)));
2197 FASTILU_DBG_COUT("l2 norm of diag. of U = " << RTS::sqrt(STS::abs(sum_diag)));
2198 }
2199
2200 void checkIC() const
2201 {
2202 //Compute the L2 norm of the nonlinear residual (A - LLt) on sparsity pattern
2203 //
2204 assert(m_blockCrsSize == 1); // Not yet supported for block crs
2205 Scalar sum = 0.0;
2206 for (int i = 0 ; i < m_nRows; i++)
2207 {
2208 int row = i;
2209 // Each row in A matrix (result)
2210 for (int k = m_aRowMap[i]; k < m_aRowMap[i+1]; k++)
2211 {
2212
2213 int col = m_aColIdx[k];
2214
2215 if (row >= col) {
2216 Scalar acc_val = m_aVal[k];
2217 Ordinal lptr, uptr;
2218 for ( lptr = m_lRowMap[i], uptr = m_lRowMap[m_aColIdx[k]] ;
2219 lptr < m_lRowMap[i+1] && uptr < m_lRowMap[m_aColIdx[k]+1] ; )
2220 {
2221 if (m_lColIdx[lptr] == m_lColIdx[uptr])
2222 {
2223 acc_val -= m_lVal[lptr] * m_lVal[uptr];
2224 lptr++;
2225 uptr++;
2226 }
2227 else if (m_lColIdx[lptr] < m_lColIdx[uptr])
2228 lptr++;
2229 else
2230 uptr++;
2231 }
2232
2233 sum += acc_val * acc_val;
2234 }
2235 }
2236 }
2237 FASTILU_DBG_COUT("l2 norm of nonlinear residual = " << std::sqrt(sum));
2238 }
2239 friend class FastILUFunctor<Ordinal, Scalar, ExecSpace>;
2240 friend class FastICFunctor<Ordinal, Scalar, ExecSpace>;
2241 friend class JacobiIterFunctor<Ordinal, Scalar, ExecSpace>;
2242 friend class ParCopyFunctor<Ordinal, Scalar, ExecSpace>;
2243 friend class ParScalFunctor<Ordinal, Scalar, Real, ExecSpace>;
2244 friend class PermScalFunctor<Ordinal, Scalar, Real, ExecSpace>;
2245 friend class MemoryPrimeFunctorN<Ordinal, Scalar, ExecSpace>;
2246 friend class MemoryPrimeFunctorNnzCoo<Ordinal, Scalar, ExecSpace>;
2247 friend class MemoryPrimeFunctorNnzCsr<Ordinal, Scalar, ExecSpace>;
2248 };
2249
2250 //TODO: find a way to avoid the if condition (only store lower triangular part of A?)
2251 template<class Ordinal, class Scalar, class ExecSpace>
2252 class FastICFunctor
2253 {
2254 public:
2255 typedef ExecSpace execution_space;
2256 typedef Kokkos::View<Ordinal *, ExecSpace> ordinal_array_type;
2257 typedef Kokkos::View<Scalar *, ExecSpace> scalar_array_type;
2258
2259 using STS = Kokkos::ArithTraits<Scalar>;
2260
2261 FastICFunctor (Ordinal nNZ, Ordinal bs, ordinal_array_type Ap, ordinal_array_type Ai,
2262 ordinal_array_type Aj, scalar_array_type Ax, ordinal_array_type Lp,
2263 ordinal_array_type Li, scalar_array_type Lx, scalar_array_type diag, Scalar omega)
2264 :
2265 nnz(nNZ), blk_size(bs), _Ap(Ap), _Ai(Ai), _Aj(Aj), _Lp(Lp), _Li(Li), _Ax(Ax), _Lx(Lx), _diag(diag), _omega(omega)
2266 {}
2267
2268 KOKKOS_INLINE_FUNCTION
2269 void operator()(const Ordinal blk_index) const
2270 {
2271 Ordinal start = blk_index * blk_size;
2272 Ordinal end = start + blk_size;
2273
2274 Ordinal nz_index;
2275
2276 if (end > nnz)
2277 {
2278 end = nnz;
2279 }
2280
2281 for (nz_index = start; nz_index < end && nz_index < nnz; nz_index++)
2282 {
2283 Ordinal i = _Ai[nz_index];
2284 Ordinal j = _Aj[nz_index];
2285 //Ordinal temp = i;
2286
2287 Scalar val = _Ax[nz_index];
2288 Scalar acc_val = 0.0;
2289 Ordinal lptr = _Lp[i];
2290 Ordinal ltptr = _Lp[j];
2291 Ordinal endpt = j;
2292 if (i >= j) {
2293
2294 for ( ; _Li[lptr] < endpt && _Li[ltptr] < endpt; )
2295 {
2296 if (_Li[lptr] == _Li[ltptr])
2297 {
2298 acc_val += _Lx[lptr] * _Lx[ltptr];
2299 lptr++;
2300 ltptr++;
2301 }
2302 else if (_Li[lptr] < _Li[ltptr])
2303 {
2304 lptr++;
2305 }
2306 else
2307 {
2308 ltptr++;
2309 }
2310 }
2311 if (i > j)
2312 {
2313 val = (val-acc_val) / _diag[j];
2314 for ( ; _Li[lptr] < j ; lptr++) ; // dummy loop
2315 assert (_Li[lptr] == j);
2316 _Lx[lptr] = ((1.0 - _omega)*_Lx[lptr]) + (_omega*val);
2317 }
2318 else if (i == j)
2319 {
2320 //_diag[j] = std::sqrt(val - acc_val);
2321 val = STS::sqrt(val - acc_val);
2322 _diag[j] = ((1.0 - _omega) * _diag[j]) + (_omega*val);
2323 for ( ; _Li[lptr] < j ; lptr++) ; // dummy loop
2324 assert(_Li[lptr]==i);
2325 _Lx[lptr] = _diag[j];
2326 }
2327 }
2328 }
2329 }
2330
2331 Ordinal nnz, blk_size;
2332 ordinal_array_type _Ap, _Ai, _Aj, _Lp, _Li;
2333 scalar_array_type _Ax, _Lx, _diag;
2334 Scalar _omega;
2335 };
2336
2337 template<class Ordinal, class Scalar, class ExecSpace>
2338 class MemoryPrimeFunctorNnzCsr
2339 {
2340 public:
2341 typedef ExecSpace execution_space;
2342 typedef Kokkos::View<Ordinal *, ExecSpace> ordinal_array_type;
2343 typedef Kokkos::View<Scalar *, ExecSpace> scalar_array_type;
2344
2345 MemoryPrimeFunctorNnzCsr (ordinal_array_type Ai,
2346 scalar_array_type Ax)
2347 :
2348 _Ai(Ai), _Ax(Ax)
2349 {}
2350
2351 KOKKOS_INLINE_FUNCTION
2352 void operator()(const Ordinal index) const
2353 {
2354 _Ai[index];
2355 _Ax[index];
2356 }
2357
2358 ordinal_array_type _Ai;
2359 scalar_array_type _Ax;
2360 };
2361
2362 template<class Ordinal, class Scalar, class ExecSpace>
2363 class MemoryPrimeFunctorNnzCoo
2364 {
2365 public:
2366 typedef ExecSpace execution_space;
2367 typedef Kokkos::View<Ordinal *, ExecSpace> ordinal_array_type;
2368 typedef Kokkos::View<Scalar *, ExecSpace> scalar_array_type;
2369
2370 MemoryPrimeFunctorNnzCoo (ordinal_array_type Ai,
2371 ordinal_array_type Aj,
2372 scalar_array_type Ax)
2373 :
2374 _Ai(Ai), _Aj(Aj), _Ax(Ax)
2375 {}
2376
2377 KOKKOS_INLINE_FUNCTION
2378 void operator()(const Ordinal index) const
2379 {
2380 /*
2381 Ordinal v1, v2;
2382 Scalar v3;
2383 */
2384
2385 _Ai[index];
2386 _Aj[index];
2387 _Ax[index];
2388 }
2389
2390 ordinal_array_type _Ai, _Aj;
2391 scalar_array_type _Ax;
2392 };
2393
2394 template<class Ordinal, class Scalar, class ExecSpace>
2395 class MemoryPrimeFunctorN
2396 {
2397 public:
2398 typedef ExecSpace execution_space;
2399 typedef Kokkos::View<Ordinal *, ExecSpace> ordinal_array_type;
2400 typedef Kokkos::View<Scalar *, ExecSpace> scalar_array_type;
2401
2402 MemoryPrimeFunctorN (ordinal_array_type Ap,
2403 ordinal_array_type Lp,
2404 ordinal_array_type Up,
2405 scalar_array_type diag)
2406 :
2407 _Ap(Ap), _Lp(Lp), _Up(Up),
2408 _diag(diag)
2409 {}
2410
2411 KOKKOS_INLINE_FUNCTION
2412 void operator()(const Ordinal index) const
2413 {
2414 //bmk: fix unused warnings?
2415 //does this functor actually have any side effects, or is it just incomplete?
2416 /*
2417 Ordinal v1, v2, v3;
2418 Scalar v4;
2419 */
2420
2421 _Ap[index];
2422 _Lp[index];
2423 _Up[index];
2424 _diag[index];
2425 }
2426
2427 ordinal_array_type _Ap, _Lp, _Up;
2428 scalar_array_type _diag;
2429 };
2430
2431 template<class Ordinal, class Scalar, class ExecSpace>
2432 class FastILUFunctor
2433 {
2434 public:
2435 typedef ExecSpace execution_space;
2436 typedef Kokkos::View<Ordinal *, ExecSpace> ordinal_array_type;
2437 typedef Kokkos::View<Scalar *, ExecSpace> scalar_array_type;
2438
2439 using STS = Kokkos::ArithTraits<Scalar>;
2440
2441 FastILUFunctor (Ordinal nNZ, Ordinal bs,
2442 ordinal_array_type Ap, ordinal_array_type Ai, ordinal_array_type Aj, scalar_array_type Ax,
2443 ordinal_array_type Lp, ordinal_array_type Li, scalar_array_type Lx,
2444 ordinal_array_type Up, ordinal_array_type Ui, scalar_array_type Ux,
2445 scalar_array_type diag, Scalar omega, Ordinal blockCrsSize, Ordinal level)
2446 :
2447 nnz(nNZ), blk_size(bs), _Ap(Ap), _Ai(Ai), _Aj(Aj), _Lp(Lp), _Li(Li),_Up(Up),
2448 _Ui(Ui), _Ax(Ax), _Lx(Lx), _Ux(Ux), _diag(diag), _omega(omega), _blockCrsSize(blockCrsSize)
2449 {}
2450
2451 KOKKOS_INLINE_FUNCTION
2452 void operator()(const Ordinal blk_index) const
2453 {
2454 Ordinal start = blk_index * blk_size;
2455 Ordinal end = start + blk_size;
2456 end = (end > nnz) ? nnz : end;
2457 if (_blockCrsSize == 1) {
2458 functor_impl(start, end);
2459 }
2460 else {
2461 functor_bcrs_impl(start, end);
2462 }
2463 }
2464
2465 KOKKOS_INLINE_FUNCTION
2466 void functor_impl(const Ordinal start, const Ordinal end) const
2467 {
2468 const Scalar zero = STS::zero();
2469 const Scalar one = STS::one();
2470
2471 for (Ordinal nz_index = start; nz_index < end && nz_index < nnz; nz_index++)
2472 {
2473 Ordinal i = _Ai[nz_index];
2474 Ordinal j = _Aj[nz_index];
2475 Ordinal lCol;
2476 Ordinal uCol;
2477 Scalar val = _Ax[nz_index];
2478 Scalar acc_val = zero;
2479 Scalar lAdd = zero;
2480 Ordinal lptr = _Lp[i];
2481 Ordinal uptr = _Up[j];
2482
2483 while ( lptr < _Lp[i+1] && uptr < _Up[j+1] )
2484 {
2485 lCol = _Li[lptr];
2486 uCol = _Ui[uptr];
2487 lAdd = zero;
2488 if (lCol == uCol)
2489 {
2490 lAdd = _Lx[lptr] * _Ux[uptr];
2491 acc_val += lAdd;
2492 }
2493 if (lCol <= uCol)
2494 {
2495 lptr++;
2496 }
2497 if (lCol >= uCol)
2498 {
2499 uptr++;
2500 }
2501 }
2502
2503 acc_val -= lAdd;
2504
2505 // Place the value into L or U
2506 if (i > j)
2507 {
2508 val = (val-acc_val) / _Ux[_Up[j+1]-1];
2509 _Lx[lptr-1] = ((one - _omega) * _Lx[lptr-1]) + (_omega * val);
2510 }
2511 else
2512 {
2513 val = (val-acc_val);
2514 if (i == j) _diag[j] = val;
2515 _Ux[uptr-1] = ((one - _omega) * _Ux[uptr - 1]) + (_omega * val);
2516 }
2517 }
2518 }
2519
2520 KOKKOS_INLINE_FUNCTION
2521 void functor_bcrs_impl(const Ordinal start, const Ordinal end) const
2522 {
2523 const Scalar zero = STS::zero();
2524 const Scalar one = STS::one();
2525
2526 const Ordinal blockItems = _blockCrsSize*_blockCrsSize;
2527 for (Ordinal nz_index = start; nz_index < end && nz_index < nnz; nz_index++) {
2528 Ordinal i = _Ai[nz_index]; // row of this nnz block in A
2529 Ordinal j = _Aj[nz_index]; // col of this nnz block in A
2530
2531 const Ordinal a_offset = blockItems*nz_index;
2532 // A[i][j] has non-zero entries
2533 for (Ordinal bi = 0; bi < _blockCrsSize; ++bi) {
2534 for (Ordinal bj = 0; bj < _blockCrsSize; ++bj) {
2535 const Ordinal blockOffset = _blockCrsSize*bi + bj;
2536 Scalar val = _Ax[a_offset + blockOffset];
2537 if (val != zero) {
2538 Scalar acc_val = zero;
2539 Ordinal lptr = _Lp[i]; // lptr= curr col of row i
2540 Ordinal uptr = _Up[j]; // uptr= curr col of row j
2541 const Ordinal i_unblock = unblock(i, bi, _blockCrsSize);
2542 const Ordinal j_unblock = unblock(j, bj, _blockCrsSize);
2543
2544 // Iterate over bi row of L
2545 // Iterate over bj row of U
2546 while ( lptr < _Lp[i+1] && uptr < _Up[j+1] )
2547 {
2548 Ordinal lCol = _Li[lptr];
2549 Ordinal uCol = _Ui[uptr];
2550 if (lCol == uCol) {
2551 const Ordinal l_offset = blockItems*lptr;
2552 const Ordinal u_offset = blockItems*uptr;
2553 for (Ordinal bjj = 0; bjj < _blockCrsSize; ++bjj) {
2554 const Ordinal blockOffsetL = _blockCrsSize*bi + bjj;
2555 const Ordinal blockOffsetU = _blockCrsSize*bj + bjj;
2556 const Scalar lVal = _Lx[l_offset + blockOffsetL];
2557 const Scalar uVal = _Ux[u_offset + blockOffsetU];
2558 const Ordinal lCol_unblock = unblock(lCol, bjj, _blockCrsSize);
2559 const Ordinal uCol_unblock = unblock(uCol, bjj, _blockCrsSize);
2560
2561 const bool diag_item = (lCol_unblock == i_unblock || uCol_unblock == j_unblock);
2562 if (lVal != zero && uVal != zero && !diag_item) {
2563 const Scalar curr_val = lVal * uVal;
2564 acc_val += curr_val;
2565 }
2566 }
2567 }
2568 if (lCol <= uCol)
2569 {
2570 lptr++;
2571 }
2572 if (lCol >= uCol)
2573 {
2574 uptr++;
2575 }
2576 }
2577
2578 // The last item in the row of U will always be the diagonal
2579 Scalar lastU = _diag[j*_blockCrsSize + bj];
2580
2581 // Place the value into L or U
2582 const Ordinal l_offset = blockItems*(lptr-1);
2583 const Ordinal u_offset = blockItems*(uptr-1);
2584 const Ordinal blockOffsetT = _blockCrsSize*bj + bi;
2585 if ( (i == j && bi > bj) || i > j) {
2586 val = (val-acc_val) / lastU;
2587 _Lx[l_offset + blockOffset] = ((one - _omega) * _Lx[l_offset + blockOffset]) + (_omega * val);
2588 }
2589 else {
2590 val = (val-acc_val);
2591 if (i == j && bi == bj) {
2592 _diag[j*_blockCrsSize + bj] = val;
2593 }
2594 _Ux[u_offset + blockOffsetT] = ((one - _omega) * _Ux[u_offset + blockOffsetT]) + (_omega * val);
2595 }
2596 }
2597 }
2598 }
2599 }
2600 }
2601
2602 Ordinal nnz, blk_size;
2603 ordinal_array_type _Ap, _Ai, _Aj, _Lp, _Li, _Up, _Ui;
2604 scalar_array_type _Ax, _Lx, _Ux, _diag;
2605 Scalar _omega;
2606 const Ordinal _blockCrsSize;
2607 };
2608
2609
2610 template<class Ordinal, class Scalar, class ExecSpace>
2611 class BlockJacobiIterFunctorL
2612 {
2613 public:
2614 typedef ExecSpace ESpace;
2615 typedef Kokkos::View<Ordinal*, ExecSpace> OrdinalArray;
2616 typedef Kokkos::View<Scalar *, ExecSpace> ScalarArray;
2617
2618 BlockJacobiIterFunctorL (Ordinal n, Ordinal bs, OrdinalArray aI, OrdinalArray aJ,
2619 ScalarArray aVal, ScalarArray b, ScalarArray xNew,
2620 ScalarArray xOld, ScalarArray diag)
2621 :
2622 nRow(n), blkSize(bs), aRPtr(aI), aColIdx(aJ), aVal2(aVal), rhs(b), x2(xNew), x1(xOld),
2623 diagElems(diag)
2624 {}
2625
2626 KOKKOS_INLINE_FUNCTION
2627 void operator()(const Ordinal blkID) const
2628 {
2629 Ordinal idx1 = blkID * blkSize;
2630 Ordinal idx2 = idx1 + blkSize;
2631 Scalar val;
2632 Ordinal row;
2633 Ordinal col;
2634 Ordinal k;
2635
2636 if (idx2 > nRow)
2637 {
2638 idx2 = nRow;
2639 }
2640
2641 for (row = idx1; row < idx2; row++)
2642 {
2643 val = 0.0;
2644 val = rhs[row];
2645 for (k = aRPtr[row]; k < aRPtr[row+1]; k++)
2646 {
2647 col = aColIdx[k];
2648 if (col >= idx1 && col < row)
2649 {
2650 val -= aVal2[k]*x2[col];
2651 }
2652 else if (col < idx1 || col > row)
2653 {
2654 val -= aVal2[k]*x1[col];
2655 }
2656 }
2657 x2[row] = val/diagElems[row];
2658 }
2659 }
2660 Ordinal nRow;
2661 Ordinal blkSize;
2662 OrdinalArray aRPtr, aColIdx;
2663 ScalarArray aVal2, rhs, x2, x1, diagElems;
2664 };
2665
2666
2667 template<class Ordinal, class Scalar, class ExecSpace>
2668 class BlockJacobiIterFunctorU
2669 {
2670 public:
2671 typedef ExecSpace ESpace;
2672 typedef Kokkos::View<Ordinal*, ExecSpace> OrdinalArray;
2673 typedef Kokkos::View<Scalar *, ExecSpace> ScalarArray;
2674
2675 BlockJacobiIterFunctorU (Ordinal n, Ordinal bs, OrdinalArray aI, OrdinalArray aJ,
2676 ScalarArray aVal, ScalarArray b, ScalarArray xNew,
2677 ScalarArray xOld, ScalarArray diag)
2678 :
2679 nRow(n), blkSize(bs), aRPtr(aI), aColIdx(aJ), aVal2(aVal), rhs(b), x2(xNew), x1(xOld),
2680 diagElems(diag)
2681 {}
2682
2683 KOKKOS_INLINE_FUNCTION
2684 void operator()(const Ordinal blkID) const
2685 {
2686 Ordinal idx1 = blkID * blkSize;
2687 Ordinal idx2 = idx1 + blkSize;
2688 Scalar val;
2689 Ordinal row;
2690 Ordinal col;
2691 Ordinal k;
2692
2693 if (idx2 > nRow)
2694 {
2695 idx2 = nRow;
2696 }
2697
2698 for (row = idx2 - 1; row >= idx1; row--)
2699 {
2700 val = 0.0;
2701 val = rhs[row];
2702 for (k = aRPtr[row]; k < aRPtr[row+1]; k++)
2703 {
2704 col = aColIdx[k];
2705 if (col < idx2 && col > row)
2706 {
2707 val -= aVal2[k]*x2[col];
2708 }
2709 else if (col >= idx2 || col < row)
2710 {
2711 val -= aVal2[k]*x1[col];
2712 }
2713 }
2714 x2[row] = val/diagElems[row];
2715 }
2716 }
2717 Ordinal nRow;
2718 Ordinal blkSize;
2719 OrdinalArray aRPtr, aColIdx;
2720 ScalarArray aVal2, rhs, x2, x1, diagElems;
2721 };
2722
2723 template<class Ordinal, class Scalar, class ExecSpace>
2724 class JacobiIterFunctor
2725 {
2726 public:
2727 typedef ExecSpace execution_space;
2728 typedef Kokkos::View<Ordinal *, ExecSpace> ordinal_array_type;
2729 typedef Kokkos::View<Scalar *, ExecSpace> scalar_array_type;
2730
2731 JacobiIterFunctor (Ordinal n, ordinal_array_type aI, ordinal_array_type aJ,
2732 scalar_array_type aVal, scalar_array_type b, scalar_array_type xNew,
2733 scalar_array_type xOld, scalar_array_type diag)
2734 :
2735 aI_(aI), aJ_(aJ), aVal_(aVal), b_(b), xNew_(xNew), xOld_(xOld), diag_(diag)
2736 {}
2737
2738 KOKKOS_INLINE_FUNCTION
2739 void operator()(const Ordinal xId) const
2740 {
2741 Scalar rowDot = 0.0;
2742 Ordinal k;
2743
2744 //The equation is x_{k+1} = D^{-1}b + (I - D^{-1}A)x_{k}
2745 //The individual updates are x^{k+1}_{i} = b_{i}/d_{i} + x^{k}_{i} -
2746 // \sum_{j = 1}^{n} r_{ij} x_{j}^{k}
2747 xNew_[xId] = b_[xId]/diag_[xId];
2748 xNew_[xId] += xOld_[xId];
2749
2750 for (k = aI_[xId]; k < aI_[xId+1]; k++)
2751 {
2752 rowDot += aVal_[k]*xOld_[aJ_[k]];
2753 }
2754 xNew_[xId] -= rowDot/diag_[xId];
2755 }
2756
2757 ordinal_array_type aI_, aJ_;
2758 scalar_array_type aVal_, b_, xNew_, xOld_, diag_;
2759 };
2760
2761
2762 //Parallel copy operation
2763 template<class Ordinal, class Scalar, class ExecSpace>
2764 class ParCopyFunctor
2765 {
2766 public:
2767 typedef ExecSpace execution_space;
2768 typedef Kokkos::View<Scalar *, ExecSpace> scalar_array_type;
2769
2770 ParCopyFunctor (scalar_array_type xDestination, scalar_array_type xSource)
2771 :
2772 xDestination_(xDestination), xSource_(xSource)
2773 {}
2774
2775 KOKKOS_INLINE_FUNCTION
2776 void operator()(const Ordinal xId) const
2777 {
2778 xDestination_[xId] = xSource_[xId];
2779 }
2780
2781 scalar_array_type xDestination_, xSource_;
2782 };
2783
2784 //Parallel copy operation with permutation
2785 template<class Ordinal, class Scalar, class ExecSpace, bool BlockCrsEnabled>
2786 class ParPermCopyFunctor
2787 {
2788 public:
2789 typedef ExecSpace execution_space;
2790 typedef Kokkos::View<Ordinal *, ExecSpace> ordinal_array_type;
2791 typedef Kokkos::View<Scalar *, ExecSpace> scalar_array_type;
2792 using parent = FastILUPrec<Ordinal, Scalar, ExecSpace, BlockCrsEnabled>;
2793
2794 ParPermCopyFunctor (ordinal_array_type a2uMap, scalar_array_type aVal, ordinal_array_type aRowIdx,
2795 ordinal_array_type aColIdx, scalar_array_type uVal, ordinal_array_type uColIdx, const Ordinal blockCrsSize)
2796 :
2797 a2uMap_(a2uMap), aVal_(aVal), aRowIdx_(aRowIdx), aColIdx_(aColIdx), uVal_(uVal), uColIdx_(uColIdx), blockCrsSize_(blockCrsSize)
2798 {}
2799
2800 KOKKOS_INLINE_FUNCTION
2801 void operator()(const Ordinal k) const
2802 {
2803 typename parent::UpperFunctor upper_lam;
2804 auto pos = a2uMap_(k);
2805 auto row = aRowIdx_[pos];
2806 auto col = aColIdx_[pos];
2807 if (row == col) {
2808 parent::template assign_block_cond_trans<BlockCrsEnabled>(uVal_, aVal_, k, pos, upper_lam, blockCrsSize_);
2809 }
2810 else {
2811 parent::template assign_block_trans<BlockCrsEnabled>(uVal_, aVal_, k, pos, blockCrsSize_);
2812 }
2813 uColIdx_(k) = aRowIdx_[pos];
2814 }
2815
2816 ordinal_array_type a2uMap_;
2817 scalar_array_type aVal_;
2818 ordinal_array_type aRowIdx_;
2819 ordinal_array_type aColIdx_;
2820 scalar_array_type uVal_;
2821 ordinal_array_type uColIdx_;
2822 const Ordinal blockCrsSize_;
2823 };
2824
2825 template<class Ordinal, class Scalar, class ExecSpace>
2826 class JacobiIterFunctorT
2827 {
2828 public:
2829 typedef ExecSpace execution_space;
2830 typedef Kokkos::View<Ordinal *, ExecSpace> ordinal_array_type;
2831 typedef Kokkos::View<Scalar *, ExecSpace> scalar_array_type;
2832
2833 JacobiIterFunctorT (Ordinal n, ordinal_array_type aI, ordinal_array_type aJ,
2834 scalar_array_type aVal, scalar_array_type b, scalar_array_type xNew,
2835 scalar_array_type xOld, scalar_array_type diag)
2836 :
2837 aI_(aI), aJ_(aJ), aVal_(aVal), b_(b), xNew_(xNew), xOld_(xOld), diag_(diag), n_(n)
2838 {}
2839
2840 KOKKOS_INLINE_FUNCTION
2841 void operator()(const Ordinal xId) const
2842 {
2843 Ordinal k;
2844
2845 // xNew_[xId] += b_[xId]/diag_[xId];
2846 //xNew_[xId] += xOld_[xId];
2847
2848 Kokkos::atomic_add(&xNew_[xId], b_[xId]/diag_[xId]);
2849 Kokkos::atomic_add(&xNew_[xId], xOld_[xId]);
2850
2851 for (k = aI_[xId]; k < aI_[xId+1]; k++)
2852 {
2853
2854 //y[aJ_[k]] += (aVal_[k]*xOld_[aJ_[k]])/diag_[aJ_[k]];
2855 Kokkos::atomic_add(&xNew_[aJ_[k]], -(aVal_[k]*xOld_[xId])/diag_[aJ_[k]]);
2856 }
2857 }
2858
2859 ordinal_array_type aI_, aJ_;
2860 scalar_array_type aVal_, b_, xNew_, xOld_, diag_;
2861 Ordinal n_;
2862 };
2863
2864 template<class Ordinal, class Scalar, class Real, class ExecSpace>
2865 class ParScalFunctor
2866 {
2867 public:
2868 typedef ExecSpace execution_space;
2869 typedef Kokkos::View<Scalar *, ExecSpace> scalar_array_type;
2870 typedef Kokkos::View<Real *, ExecSpace> real_array_type;
2871
2872 ParScalFunctor (scalar_array_type x, scalar_array_type y, real_array_type scaleFactors)
2873 :
2874 x_(x), y_(y), scaleFactors_(scaleFactors)
2875 {
2876 }
2877
2878
2879 KOKKOS_INLINE_FUNCTION
2880 void operator()(const Ordinal xId) const
2881 {
2882 y_[xId] = x_[xId]*scaleFactors_[xId];
2883 }
2884
2885 scalar_array_type x_, y_;
2886 real_array_type scaleFactors_;
2887 };
2888
2889 template<class Ordinal, class Scalar, class Real, class ExecSpace>
2890 class PermScalFunctor
2891 {
2892 public:
2893 typedef ExecSpace execution_space;
2894 typedef Kokkos::View<Ordinal *, ExecSpace> ordinal_array_type;
2895 typedef Kokkos::View<Scalar *, ExecSpace> scalar_array_type;
2896 typedef Kokkos::View<Real *, ExecSpace> real_array_type;
2897
2898 PermScalFunctor (scalar_array_type x, scalar_array_type y, real_array_type scaleFactors, ordinal_array_type perm)
2899 :
2900 x_(x), y_(y), scaleFactors_(scaleFactors), perm_(perm)
2901 {
2902 }
2903
2904 KOKKOS_INLINE_FUNCTION
2905 void operator()(const NonTranPermScalTag &, const Ordinal xId) const
2906 {
2907 // y = D*P*x
2908 Ordinal row = perm_(xId);
2909 y_[xId] = scaleFactors_[xId]*x_[row];
2910 }
2911
2912 KOKKOS_INLINE_FUNCTION
2913 void operator()(const TranPermScalTag &, const Ordinal xId) const
2914 {
2915 // y = P'*D*x
2916 Ordinal row = perm_(xId);
2917 y_[xId] = x_[row]*scaleFactors_[row];
2918 }
2919
2920 scalar_array_type x_, y_;
2921 real_array_type scaleFactors_;
2922 ordinal_array_type perm_;
2923 };
2924
2925 template<class Ordinal, class Scalar, class ExecSpace>
2926 class ParInitZeroFunctor
2927 {
2928 public:
2929 typedef ExecSpace execution_space;
2930 typedef Kokkos::View<Scalar *, ExecSpace> scalar_array_type;
2931
2932 ParInitZeroFunctor(scalar_array_type x)
2933 :
2934 x_(x)
2935 {
2936 }
2937
2938
2939 KOKKOS_INLINE_FUNCTION
2940 void operator()(const Ordinal xId) const
2941 {
2942 x_[xId] = 0.0;
2943 }
2944
2945 scalar_array_type x_ ;
2946 };
2947
2948 #undef FASTILU_CREATE_TIMER
2949 #undef FASTILU_REPORT_TIMER
2950 #undef FASTILU_FENCE_REPORT_TIMER
2951 #undef FASTILU_DBG_COUT
2952 #undef FASTILU_ONE_TO_ONE_UNBLOCKED
2953
2954 #endif