| 1 |
// @HEADER |
| 2 |
// ***************************************************************************** |
| 3 |
// ShyLU: Scalable Hybrid LU Preconditioner and Solver |
| 4 |
// |
| 5 |
// Copyright 2011 NTESS and the ShyLU contributors. |
| 6 |
// SPDX-License-Identifier: BSD-3-Clause |
| 7 |
// ***************************************************************************** |
| 8 |
// @HEADER |
| 9 |
|
| 10 |
// The struct that iterates over all non-zeros for the FastIC |
| 11 |
// Contact for bugs and complaints - Siva Rajamanickam ([email protected]) |
| 12 |
// |
| 13 |
#ifndef __FAST_IC_HPP__ |
| 14 |
#define __FAST_IC_HPP__ |
| 15 |
#include <iostream> |
| 16 |
#include <algorithm> |
| 17 |
#include <vector> |
| 18 |
#include <queue> |
| 19 |
#include <random> |
| 20 |
#include <cstdlib> |
| 21 |
#include <cmath> |
| 22 |
#include <ctime> |
| 23 |
#include <string> |
| 24 |
|
| 25 |
#include <assert.h> |
| 26 |
#include <Kokkos_Core.hpp> |
| 27 |
#include <Kokkos_Timer.hpp> |
| 28 |
|
| 29 |
#include "shylu_fastilu.hpp" |
| 30 |
|
| 31 |
//whether to print extra debug output at runtime to stdout |
| 32 |
//comment out next line to disable |
| 33 |
//#define FASTIC_DEBUG_OUTPUT |
| 34 |
|
| 35 |
template<class Ordinal, class Scalar, class ExecSpace> |
| 36 |
class FastICPrec |
| 37 |
{ |
| 38 |
|
| 39 |
public: |
| 40 |
typedef typename Teuchos::ScalarTraits<Scalar>::magnitudeType Real; |
| 41 |
typedef Kokkos::View<Ordinal *, ExecSpace> OrdinalArray; |
| 42 |
typedef Kokkos::View<Scalar *, ExecSpace> ScalarArray; |
| 43 |
typedef Kokkos::View<Real *, ExecSpace> RealArray; |
| 44 |
typedef Kokkos::View<Ordinal *, typename ExecSpace::array_layout, Kokkos::Serial, |
| 45 |
Kokkos::MemoryUnmanaged> UMOrdinalArray; |
| 46 |
typedef Kokkos::View<Scalar *, typename ExecSpace::array_layout, Kokkos::Serial, |
| 47 |
Kokkos::MemoryUnmanaged> UMScalarArray; |
| 48 |
typedef FastICPrec<Ordinal, Scalar, ExecSpace> FastPrec; |
| 49 |
|
| 50 |
typedef Kokkos::View<Ordinal *, Kokkos::HostSpace> OrdinalArrayHost; |
| 51 |
typedef Kokkos::View<Scalar *, Kokkos::HostSpace> ScalarArrayHost; |
| 52 |
typedef typename OrdinalArray::host_mirror_type OrdinalArrayMirror; |
| 53 |
typedef typename ScalarArray::host_mirror_type ScalarArrayMirror; |
| 54 |
|
| 55 |
typedef Kokkos::RangePolicy<ExecSpace> RangePolicy; |
| 56 |
|
| 57 |
using STS = Kokkos::ArithTraits<Scalar>; |
| 58 |
using RTS = Kokkos::ArithTraits<Real>; |
| 59 |
|
| 60 |
private: |
| 61 |
double computeTime; |
| 62 |
double applyTime; |
| 63 |
double initTime; |
| 64 |
|
| 65 |
Ordinal guessFlag; |
| 66 |
Ordinal nRows; |
| 67 |
Ordinal nFact; |
| 68 |
Ordinal nTrisol; |
| 69 |
Ordinal level; |
| 70 |
Ordinal blkSzIC; |
| 71 |
Ordinal blkSz; |
| 72 |
Scalar omega; |
| 73 |
Scalar shift; |
| 74 |
|
| 75 |
//Lower triangular factor (CSR) |
| 76 |
ScalarArray lVal; |
| 77 |
OrdinalArray lColIdx; |
| 78 |
OrdinalArray lRowMap; |
| 79 |
// mirrors |
| 80 |
ScalarArrayMirror lVal_; |
| 81 |
OrdinalArrayMirror lColIdx_; |
| 82 |
OrdinalArrayMirror lRowMap_; |
| 83 |
|
| 84 |
//Upper triangular factor (CSR) |
| 85 |
ScalarArray ltVal; |
| 86 |
OrdinalArray ltColIdx; |
| 87 |
OrdinalArray ltRowMap; |
| 88 |
|
| 89 |
//Pointer to the original host copy of A. |
| 90 |
ScalarArrayMirror aValHost; |
| 91 |
OrdinalArrayMirror aRowMapHost; |
| 92 |
OrdinalArrayMirror aColIdxHost; |
| 93 |
|
| 94 |
//A matrix in COO format |
| 95 |
ScalarArray aVal; |
| 96 |
OrdinalArray aRowMap; |
| 97 |
OrdinalArray aRowIdx; |
| 98 |
OrdinalArray aColIdx; |
| 99 |
// mirrors |
| 100 |
ScalarArrayMirror aVal_; |
| 101 |
OrdinalArrayMirror aRowMap_; |
| 102 |
OrdinalArrayMirror aRowIdx_; |
| 103 |
OrdinalArrayMirror aColIdx_; |
| 104 |
|
| 105 |
//Diagonal scaling factors |
| 106 |
RealArray diagFact; |
| 107 |
ScalarArray diagElems; |
| 108 |
ScalarArray diagElemsInv; |
| 109 |
// mirrors |
| 110 |
ScalarArrayMirror diagElems_; |
| 111 |
ScalarArrayMirror diagElemsInv_; |
| 112 |
|
| 113 |
//Temporary vectors for triangular solves |
| 114 |
ScalarArray xOld; |
| 115 |
ScalarArray xTemp; |
| 116 |
ScalarArray onesVector; |
| 117 |
|
| 118 |
//This will have the continuation initial |
| 119 |
//guess if guessFlag=1 |
| 120 |
Teuchos::RCP<FastPrec> initGuessPrec; |
| 121 |
|
| 122 |
// forward/backwar substitution for standard SpTrsv |
| 123 |
using MemSpace = typename ExecSpace::memory_space; |
| 124 |
using KernelHandle = KokkosKernels::Experimental::KokkosKernelsHandle <Ordinal, Ordinal, Scalar, ExecSpace, MemSpace, MemSpace >; |
| 125 |
bool standard_sptrsv; |
| 126 |
KernelHandle khL; |
| 127 |
KernelHandle khLt; |
| 128 |
|
| 129 |
public: |
| 130 |
FastICPrec(OrdinalArray &aRowMapIn, OrdinalArray &aColIdxIn, ScalarArray &aValIn, Ordinal nRow_, bool standard_sptrsv_, |
| 131 |
Ordinal nFact_, Ordinal nTrisol_, Ordinal level_, Scalar omega_, Scalar shift_, Ordinal guessFlag_, Ordinal blkSzIC_, Ordinal blkSz_) |
| 132 |
{ |
| 133 |
nRows = nRow_; |
| 134 |
standard_sptrsv = standard_sptrsv_; |
| 135 |
nFact = nFact_; |
| 136 |
nTrisol = nTrisol_; |
| 137 |
computeTime = 0.0; |
| 138 |
applyTime = 0.0; |
| 139 |
initTime = 0.0; |
| 140 |
level = level_; |
| 141 |
|
| 142 |
// mirror & deep-copy the input matrix |
| 143 |
aRowMapHost = Kokkos::create_mirror_view(aRowMapIn); |
| 144 |
aColIdxHost = Kokkos::create_mirror_view(aColIdxIn); |
| 145 |
aValHost = Kokkos::create_mirror_view(aValIn); |
| 146 |
Kokkos::deep_copy(aRowMapHost, aRowMapIn); |
| 147 |
Kokkos::deep_copy(aColIdxHost, aColIdxIn); |
| 148 |
Kokkos::deep_copy(aValHost, aValIn); |
| 149 |
|
| 150 |
omega = omega_; |
| 151 |
guessFlag = guessFlag_; |
| 152 |
shift = shift_; |
| 153 |
blkSzIC = blkSzIC_; |
| 154 |
blkSz = blkSz_; |
| 155 |
|
| 156 |
const Scalar one = STS::one(); |
| 157 |
onesVector = ScalarArray("onesVector", nRow_); |
| 158 |
Kokkos::deep_copy(onesVector, one); |
| 159 |
|
| 160 |
diagFact = RealArray("diagFact", nRow_); |
| 161 |
diagElems = ScalarArray("diagElems", nRow_); |
| 162 |
diagElemsInv = ScalarArray("diagElems", nRow_); |
| 163 |
xOld = ScalarArray("xOld", nRow_); |
| 164 |
xTemp = ScalarArray("xTemp", nRow_); |
| 165 |
|
| 166 |
if (level > 0) |
| 167 |
{ |
| 168 |
initGuessPrec = Teuchos::rcp(new FastPrec(aRowMapIn, aColIdxIn, aValIn, nRow_, standard_sptrsv, 3, 5, |
| 169 |
level_-1, omega_, shift_, guessFlag_, blkSzIC_, blkSz_)); |
| 170 |
} |
| 171 |
|
| 172 |
} |
| 173 |
|
| 174 |
void initialize() |
| 175 |
{ |
| 176 |
Kokkos::Timer timer; |
| 177 |
//Here we have to allocate memory for L, U. |
| 178 |
if((level > 0) && (guessFlag != 0)) |
| 179 |
{ |
| 180 |
initGuessPrec->initialize(); |
| 181 |
} |
| 182 |
//symbolicILUAndInit(); //Note the diagonal scaling is now embedded in this routine. |
| 183 |
symbolicILU(); |
| 184 |
//initialize L, U, A patterns |
| 185 |
#ifdef SHYLU_DEBUG |
| 186 |
MemoryPrimeFunctorN<Ordinal, Scalar, ExecSpace> copyFunc1(aRowMap, lRowMap, lRowMap, diagElems); |
| 187 |
|
| 188 |
//Make sure all memory resides on the device |
| 189 |
ExecSpace().fence(); |
| 190 |
Kokkos::parallel_for(RangePolicy(0, nRows), copyFunc1); |
| 191 |
|
| 192 |
//Note that the following is a temporary measure |
| 193 |
//to ensure that memory resides on the device. |
| 194 |
Ordinal nnzL = lRowMap[nRows]; |
| 195 |
Ordinal nnzA = aRowMap[nRows]; |
| 196 |
|
| 197 |
MemoryPrimeFunctorNnzCoo<Ordinal, Scalar, ExecSpace> copyFunc2(aColIdx, aRowIdx, aVal); |
| 198 |
MemoryPrimeFunctorNnzCsr<Ordinal, Scalar, ExecSpace> copyFunc3(lColIdx, lVal); |
| 199 |
|
| 200 |
ExecSpace().fence(); |
| 201 |
Kokkos::parallel_for(RangePolicy(0, nRows), copyFunc1); |
| 202 |
Kokkos::parallel_for(RangePolicy(0, nnzA), copyFunc2); |
| 203 |
Kokkos::parallel_for(RangePolicy(0, nnzL), copyFunc3); |
| 204 |
#endif |
| 205 |
double t = timer.seconds(); |
| 206 |
|
| 207 |
#ifdef FASTIC_DEBUG_OUTPUT |
| 208 |
std::cout << "Symbolic phase complete." << std::endl; |
| 209 |
std::cout << "Init time: "<< t << "s" << std::endl; |
| 210 |
#endif |
| 211 |
initTime = t; |
| 212 |
} |
| 213 |
|
| 214 |
void getL(OrdinalArray &lRowMapArg, OrdinalArray &lColIdxArg, ScalarArray &lValArg) |
| 215 |
{ |
| 216 |
lRowMapArg = lRowMap; |
| 217 |
lColIdxArg = lColIdx; |
| 218 |
lValArg = lVal; |
| 219 |
} |
| 220 |
|
| 221 |
void getD(ScalarArray &diagElemsArg) |
| 222 |
{ |
| 223 |
diagElemsArg = diagElems; |
| 224 |
} |
| 225 |
|
| 226 |
void transposeL() |
| 227 |
{ |
| 228 |
auto ltRowMap_ = Kokkos::create_mirror_view(ltRowMap); |
| 229 |
auto ltColIdx_ = Kokkos::create_mirror_view(ltColIdx); |
| 230 |
auto ltVal_ = Kokkos::create_mirror_view(ltVal); |
| 231 |
|
| 232 |
//Count the elements in each row of Lt |
| 233 |
auto temp = OrdinalArrayHost("temp", nRows + 1); |
| 234 |
auto rowPtrs = OrdinalArrayHost("rowPtrs", nRows); |
| 235 |
for (Ordinal i = 0; i <= nRows; i++) |
| 236 |
{ |
| 237 |
temp[i] = 0; |
| 238 |
} |
| 239 |
for (Ordinal i = 0; i < nRows; i++) |
| 240 |
{ |
| 241 |
for (Ordinal k = lRowMap_[i]; k < lRowMap_[i+1]; k++) |
| 242 |
{ |
| 243 |
temp[lColIdx_[k]+1]++; |
| 244 |
|
| 245 |
} |
| 246 |
} |
| 247 |
//Perform an add scan to get the row map for |
| 248 |
//the transpose |
| 249 |
for (Ordinal i = 0; i <= nRows; i++) |
| 250 |
{ |
| 251 |
ltRowMap_[i] = temp[i]; |
| 252 |
} |
| 253 |
for (Ordinal i = 1; i <= nRows; i++) |
| 254 |
{ |
| 255 |
ltRowMap_[i] += ltRowMap_[i-1]; |
| 256 |
} |
| 257 |
//Set the row pointers to their initial places; |
| 258 |
for (Ordinal i = 0; i < nRows; i++) |
| 259 |
{ |
| 260 |
rowPtrs[i] = ltRowMap_[i]; |
| 261 |
} |
| 262 |
//Copy the data |
| 263 |
Kokkos::deep_copy(lVal_, lVal); |
| 264 |
for (Ordinal i = 0; i < nRows; i++) |
| 265 |
{ |
| 266 |
for (Ordinal k = lRowMap_[i]; k < lRowMap_[i+1]; k++) |
| 267 |
{ |
| 268 |
Ordinal row = lColIdx_[k]; |
| 269 |
Scalar value = lVal_[k]; |
| 270 |
ltVal_[rowPtrs[row]] = value; |
| 271 |
ltColIdx_[rowPtrs[row]] = i; |
| 272 |
rowPtrs[row]++; |
| 273 |
assert(rowPtrs[row] <= ltRowMap_[row + 1]); |
| 274 |
} |
| 275 |
} |
| 276 |
Kokkos::deep_copy(ltRowMap, ltRowMap_); |
| 277 |
Kokkos::deep_copy(ltColIdx, ltColIdx_); |
| 278 |
Kokkos::deep_copy(ltVal,ltVal_); |
| 279 |
} |
| 280 |
|
| 281 |
void symbolicILU() |
| 282 |
{ |
| 283 |
using std::vector; |
| 284 |
using std::cout; |
| 285 |
using std::stable_sort; |
| 286 |
using std::sort; |
| 287 |
OrdinalArrayMirror ia = aRowMapHost; |
| 288 |
OrdinalArrayMirror ja = aColIdxHost; |
| 289 |
int n = nRows; |
| 290 |
int nzl = aRowMapHost[nRows] * (level + 2); |
| 291 |
int nzu = aRowMapHost[nRows] * (level + 2); |
| 292 |
Ordinal i; |
| 293 |
int levfill = level; |
| 294 |
vector<int> lnklst(n); |
| 295 |
vector<int> curlev(n); |
| 296 |
vector<int> levels(nzu); |
| 297 |
vector<int> iwork(n); |
| 298 |
vector<int> ial(nzl); |
| 299 |
vector<int> jal(nzl); |
| 300 |
vector<int> iau(nzu); |
| 301 |
vector<int> jau(nzu); |
| 302 |
|
| 303 |
int knzl = 0; |
| 304 |
int knzu = 0; |
| 305 |
|
| 306 |
ial[0] = 0; |
| 307 |
iau[0] = 0; |
| 308 |
|
| 309 |
for (i=0; i<n; i++) |
| 310 |
{ |
| 311 |
int first, next, j; |
| 312 |
|
| 313 |
/* copy column indices of row into workspace and sort them */ |
| 314 |
|
| 315 |
int len = ia[i+1] - ia[i]; |
| 316 |
next = 0; |
| 317 |
for (j=ia[i]; j<ia[i+1]; j++) |
| 318 |
iwork[next++] = ja[j]; |
| 319 |
// shell_sort(len, iwork); |
| 320 |
//stable_sort(iwork.begin(), iwork.begin() + len); |
| 321 |
sort(iwork.begin(), iwork.begin() + len); |
| 322 |
|
| 323 |
/* construct implied linked list for row */ |
| 324 |
|
| 325 |
first = iwork[0]; |
| 326 |
curlev[first] = 0; |
| 327 |
|
| 328 |
for (j=0; j<=len-2; j++) |
| 329 |
{ |
| 330 |
lnklst[iwork[j]] = iwork[j+1]; |
| 331 |
curlev[iwork[j]] = 0; |
| 332 |
} |
| 333 |
|
| 334 |
lnklst[iwork[len-1]] = n; |
| 335 |
curlev[iwork[len-1]] = 0; |
| 336 |
|
| 337 |
/* merge with rows in U */ |
| 338 |
|
| 339 |
next = first; |
| 340 |
while (next < i) |
| 341 |
{ |
| 342 |
int oldlst = next; |
| 343 |
int nxtlst = lnklst[next]; |
| 344 |
int row = next; |
| 345 |
int ii; |
| 346 |
|
| 347 |
/* scan row */ |
| 348 |
|
| 349 |
for (ii=iau[row]+1; ii<iau[row+1]; /*nop*/) |
| 350 |
{ |
| 351 |
if (jau[ii] < nxtlst) |
| 352 |
{ |
| 353 |
/* new fill-in */ |
| 354 |
int newlev = curlev[row] + levels[ii] + 1; |
| 355 |
if (newlev <= levfill) |
| 356 |
{ |
| 357 |
lnklst[oldlst] = jau[ii]; |
| 358 |
lnklst[jau[ii]] = nxtlst; |
| 359 |
oldlst = jau[ii]; |
| 360 |
curlev[jau[ii]] = newlev; |
| 361 |
} |
| 362 |
ii++; |
| 363 |
} |
| 364 |
else if (jau[ii] == nxtlst) |
| 365 |
{ |
| 366 |
int newlev; |
| 367 |
oldlst = nxtlst; |
| 368 |
nxtlst = lnklst[oldlst]; |
| 369 |
newlev = curlev[row] + levels[ii] + 1; |
| 370 |
//curlev[jau[ii]] = MIN(curlev[jau[ii]], newlev); |
| 371 |
if (curlev[jau[ii]] > newlev) |
| 372 |
{ |
| 373 |
curlev[jau[ii]] = newlev; |
| 374 |
} |
| 375 |
ii++; |
| 376 |
} |
| 377 |
else /* (jau[ii] > nxtlst) */ |
| 378 |
{ |
| 379 |
oldlst = nxtlst; |
| 380 |
nxtlst = lnklst[oldlst]; |
| 381 |
} |
| 382 |
} |
| 383 |
next = lnklst[next]; |
| 384 |
} |
| 385 |
|
| 386 |
/* gather the pattern into L and U */ |
| 387 |
|
| 388 |
next = first; |
| 389 |
while (next < i) |
| 390 |
{ |
| 391 |
assert(knzl < nzl); |
| 392 |
jal[knzl++] = next; |
| 393 |
if (knzl >= nzl) |
| 394 |
{ |
| 395 |
//(*nzl)++; |
| 396 |
nzl += nRows; |
| 397 |
jal.resize(nzl); |
| 398 |
} |
| 399 |
//jal.push_back(next); |
| 400 |
// knzl++; |
| 401 |
next = lnklst[next]; |
| 402 |
} |
| 403 |
ial[i+1] = knzl; |
| 404 |
assert(next == i); |
| 405 |
|
| 406 |
while (next < n) |
| 407 |
{ |
| 408 |
assert(knzu < nzu); |
| 409 |
levels[knzu] = curlev[next]; |
| 410 |
jau[knzu++] = next; |
| 411 |
if (knzu >= nzu) |
| 412 |
{ |
| 413 |
//(*nzu)++; |
| 414 |
nzu += nRows; |
| 415 |
jau.resize(nzu); |
| 416 |
levels.resize(nzu); |
| 417 |
} |
| 418 |
//jau.push_back(next); |
| 419 |
//knzu++; |
| 420 |
next = lnklst[next]; |
| 421 |
} |
| 422 |
iau[i+1] = knzu; |
| 423 |
} |
| 424 |
|
| 425 |
nzl = knzl; |
| 426 |
nzu = knzu; |
| 427 |
|
| 428 |
#ifdef FASTIC_DEBUG_OUTPUT |
| 429 |
std::cout << "knzl =" << knzl; |
| 430 |
std::cout << "knzu =" << knzu; |
| 431 |
std::cout << "ILU: nnz = "<< knzl + knzu << std::endl; |
| 432 |
std::cout << "Actual nnz for ILU: " << nzl + nzu << std::endl; |
| 433 |
#endif |
| 434 |
//Initialize the A matrix that is to be used in the computation |
| 435 |
aRowMap = OrdinalArray("aRowMap", nRows + 1); |
| 436 |
aColIdx = OrdinalArray("aColIdx", knzl + knzu); |
| 437 |
aRowIdx = OrdinalArray("aRowIds", knzl + knzu); |
| 438 |
aRowMap_ = Kokkos::create_mirror_view(aRowMap); |
| 439 |
aColIdx_ = Kokkos::create_mirror_view(aColIdx); |
| 440 |
aRowIdx_ = Kokkos::create_mirror_view(aRowIdx); |
| 441 |
|
| 442 |
Ordinal aRowPtr = 0; |
| 443 |
aRowMap_[0] = 0; |
| 444 |
for (i = 0; i < nRows; i++) |
| 445 |
{ |
| 446 |
#ifdef FASTIC_DEBUG_OUTPUT |
| 447 |
std::cout << "***row:" << i << std::endl; |
| 448 |
#endif |
| 449 |
for(Ordinal k = ial[i]; k < ial[i+1]; k++) |
| 450 |
{ |
| 451 |
#ifdef FASTIC_DEBUG_OUTPUT |
| 452 |
std::cout << "jal[k]=" << jal[k] << std::endl; |
| 453 |
#endif |
| 454 |
aColIdx_[aRowPtr] = jal[k]; |
| 455 |
aRowIdx_[aRowPtr] = i; |
| 456 |
aRowPtr++; |
| 457 |
} |
| 458 |
for(Ordinal k = iau[i]; k < iau[i+1]; k++) |
| 459 |
{ |
| 460 |
aColIdx_[aRowPtr] = jau[k]; |
| 461 |
aRowIdx_[aRowPtr] = i; |
| 462 |
aRowPtr++; |
| 463 |
} |
| 464 |
aRowMap_[i+1] = aRowPtr; |
| 465 |
} |
| 466 |
Kokkos::deep_copy(aRowMap, aRowMap_); |
| 467 |
Kokkos::deep_copy(aColIdx, aColIdx_); |
| 468 |
Kokkos::deep_copy(aRowIdx, aRowIdx_); |
| 469 |
|
| 470 |
//Now allocate memory for L and U. |
| 471 |
// |
| 472 |
lRowMap = OrdinalArray("lRowMap", nRows + 1); |
| 473 |
ltRowMap = OrdinalArray("ltRowMap", nRows + 1); |
| 474 |
countL(); |
| 475 |
//Allocate memory and initialize pattern for L, U (transpose). |
| 476 |
lColIdx = OrdinalArray("lColIdx", lRowMap_[nRows]); |
| 477 |
ltColIdx = OrdinalArray("ltColIdx", lRowMap_[nRows]); |
| 478 |
#ifdef FASTIC_DEBUG_OUTPUT |
| 479 |
std::cout << "nnz L = " << lRowMap[nRows] << std::endl; |
| 480 |
#endif |
| 481 |
} |
| 482 |
|
| 483 |
void numericILU() |
| 484 |
{ |
| 485 |
aVal = ScalarArray("aVal", aColIdx.extent(0)); |
| 486 |
aVal_ = Kokkos::create_mirror_view(aVal); |
| 487 |
|
| 488 |
//Copy the host matrix into the initialized a; |
| 489 |
for (Ordinal i = 0; i < nRows; i++) |
| 490 |
{ |
| 491 |
for(Ordinal k = aRowMap_[i]; k < aRowMap_[i+1]; k++) |
| 492 |
{ |
| 493 |
Ordinal col = aColIdx_[k]; |
| 494 |
#ifdef FASTIC_DEBUG_OUTPUT |
| 495 |
std::cout << "col =" << col << std::endl; |
| 496 |
#endif |
| 497 |
for(Ordinal aHostPtr = aRowMapHost[i]; aHostPtr < aRowMapHost[i+1]; aHostPtr++) |
| 498 |
{ |
| 499 |
if (col == aColIdxHost[aHostPtr]) |
| 500 |
{ |
| 501 |
aVal_[k] = aValHost[aHostPtr]; |
| 502 |
break; |
| 503 |
} |
| 504 |
} |
| 505 |
} |
| 506 |
} |
| 507 |
lVal = ScalarArray("lVal", lRowMap_[nRows]); |
| 508 |
ltVal = ScalarArray("ltVal", lRowMap_[nRows]); |
| 509 |
applyDiagonalScaling(); |
| 510 |
applyManteuffelShift(); |
| 511 |
Kokkos::deep_copy(aVal, aVal_); |
| 512 |
|
| 513 |
fillL(); |
| 514 |
} |
| 515 |
|
| 516 |
void countL() |
| 517 |
{ |
| 518 |
lRowMap_ = Kokkos::create_mirror_view(lRowMap); |
| 519 |
|
| 520 |
lRowMap_[0] = 0; |
| 521 |
for (Ordinal i = 0; i < nRows; i++) |
| 522 |
{ |
| 523 |
Ordinal row_count = 0; |
| 524 |
for (Ordinal k = aRowMap_[i]; k < aRowMap_[i+1]; k++) |
| 525 |
{ |
| 526 |
Ordinal row = i; |
| 527 |
Ordinal col = aColIdx_[k]; |
| 528 |
|
| 529 |
if (row >= col) |
| 530 |
{ |
| 531 |
row_count++; |
| 532 |
} |
| 533 |
} |
| 534 |
lRowMap_[i+1] = lRowMap_[i] + row_count; |
| 535 |
} |
| 536 |
Kokkos::deep_copy(lRowMap, lRowMap_); |
| 537 |
} |
| 538 |
|
| 539 |
void fillL() |
| 540 |
{ |
| 541 |
lVal_ = Kokkos::create_mirror_view(lVal); |
| 542 |
lColIdx_ = Kokkos::create_mirror_view(lColIdx); |
| 543 |
diagElems_ = Kokkos::create_mirror_view(diagElems); |
| 544 |
|
| 545 |
Ordinal lPtr = 0; |
| 546 |
for (Ordinal i = 0; i < nRows; i++) |
| 547 |
{ |
| 548 |
for (Ordinal k = aRowMap_[i]; k < aRowMap_[i+1]; k++) |
| 549 |
{ |
| 550 |
Ordinal row = i; |
| 551 |
Ordinal col = aColIdx_[k]; |
| 552 |
|
| 553 |
if (row >= col) |
| 554 |
{ |
| 555 |
if (row == col) |
| 556 |
{ |
| 557 |
diagElems_[row] = aVal_[k]; |
| 558 |
} |
| 559 |
lVal_[lPtr] = aVal_[k]; |
| 560 |
lColIdx_[lPtr] = aColIdx_[k]; |
| 561 |
lPtr++; |
| 562 |
} |
| 563 |
} |
| 564 |
} |
| 565 |
Kokkos::deep_copy(diagElems, diagElems_); |
| 566 |
assert(lPtr == lRowMap_[nRows]); |
| 567 |
|
| 568 |
if ((level > 0) && (guessFlag !=0)) |
| 569 |
{ |
| 570 |
OrdinalArray lGRowMap; |
| 571 |
OrdinalArray lGColIdx; |
| 572 |
ScalarArray lGVal; |
| 573 |
ScalarArray gD; |
| 574 |
Ordinal lGPtr = 0; |
| 575 |
|
| 576 |
initGuessPrec->getL(lGRowMap, lGColIdx, lGVal); |
| 577 |
initGuessPrec->getD(gD); |
| 578 |
|
| 579 |
Kokkos::deep_copy(diagElems, gD); |
| 580 |
|
| 581 |
auto lGColIdx_ = Kokkos::create_mirror_view(lGColIdx); |
| 582 |
auto lGVal_ = Kokkos::create_mirror_view(lGVal); |
| 583 |
Kokkos::deep_copy(lGColIdx_, lGColIdx); |
| 584 |
Kokkos::deep_copy(lGVal_, lGVal); |
| 585 |
for (Ordinal i = 0; i < nRows; i++) |
| 586 |
{ |
| 587 |
#ifdef SHYLU_DEBUG |
| 588 |
Ordinal check = lGPtr; |
| 589 |
#endif |
| 590 |
for (Ordinal k = lRowMap_[i]; k < lRowMap_[i+1]; k++) |
| 591 |
{ |
| 592 |
//unused: Ordinal row = i; |
| 593 |
Ordinal col = lColIdx_[k]; |
| 594 |
|
| 595 |
if (col == lGColIdx_[lGPtr]) |
| 596 |
{ |
| 597 |
lVal_[k] = lGVal_[lGPtr]; |
| 598 |
lGPtr++; |
| 599 |
} |
| 600 |
} |
| 601 |
#ifdef SHYLU_DEBUG |
| 602 |
rowLen = lGPtr - check; |
| 603 |
assert(rowLen == lGRowMap[i+1] - lGRowMap[i]); |
| 604 |
#endif |
| 605 |
} |
| 606 |
} |
| 607 |
Kokkos::deep_copy(lColIdx, lColIdx_); |
| 608 |
Kokkos::deep_copy(lVal, lVal_); |
| 609 |
} |
| 610 |
|
| 611 |
void applyDiagonalScaling() |
| 612 |
{ |
| 613 |
const Real one = RTS::one(); |
| 614 |
int anext = 0; |
| 615 |
//First fill Aj and extract the diagonal scaling factors |
| 616 |
//Use diag array to store scaling factors since |
| 617 |
//it gets set to the correct value by findFactorPattern anyway. |
| 618 |
auto diagFact_ = Kokkos::create_mirror_view(diagFact); |
| 619 |
for (int i = 0; i < nRows; i++) |
| 620 |
{ |
| 621 |
for(int k = aRowMap_[i]; k < aRowMap_[i+1]; k++) |
| 622 |
{ |
| 623 |
aRowIdx_[anext++] = i; |
| 624 |
if (aColIdx_[k] == i) |
| 625 |
{ |
| 626 |
diagFact_[i] = one/RTS::sqrt(STS::abs(aVal_[k])); |
| 627 |
#ifdef FASTIC_DEBUG_OUTPUT |
| 628 |
std::cout << "diagFact["<<i<<"]="<<aVal[k]<<std::endl; |
| 629 |
#endif |
| 630 |
} |
| 631 |
} |
| 632 |
} |
| 633 |
|
| 634 |
//Now go through each element of A and apply the scaling |
| 635 |
int row; |
| 636 |
int col; |
| 637 |
Real sc1, sc2; |
| 638 |
for (int i = 0; i < nRows; i++) |
| 639 |
{ |
| 640 |
for (int k = aRowMap_[i]; k < aRowMap_[i+1]; k++) |
| 641 |
{ |
| 642 |
row = aRowIdx_[k]; |
| 643 |
col = aColIdx_[k]; |
| 644 |
|
| 645 |
sc1 = diagFact_[row]; |
| 646 |
sc2 = diagFact_[col]; |
| 647 |
aVal_[k] = aVal_[k]*sc1*sc2; |
| 648 |
} |
| 649 |
} |
| 650 |
Kokkos::deep_copy(diagFact, diagFact_); |
| 651 |
} |
| 652 |
|
| 653 |
void applyManteuffelShift() |
| 654 |
{ |
| 655 |
const Scalar one = STS::one(); |
| 656 |
//Scalar shift = 0.05; |
| 657 |
for (Ordinal i = 0; i < nRows; i++) |
| 658 |
{ |
| 659 |
for (Ordinal k = aRowMap_[i]; k < aRowMap_[i+1]; k++) |
| 660 |
{ |
| 661 |
Ordinal row = i; |
| 662 |
Ordinal col = aColIdx_[k]; |
| 663 |
if (row != col) |
| 664 |
{ |
| 665 |
aVal_[k] = (one/(one + shift))*aVal_[k]; |
| 666 |
} |
| 667 |
} |
| 668 |
} |
| 669 |
} |
| 670 |
|
| 671 |
void applyDD(ScalarArray &x, ScalarArray &y) |
| 672 |
{ |
| 673 |
ParScalFunctor<Ordinal, Scalar, Scalar, ExecSpace> parScal(x, y, diagElemsInv); |
| 674 |
ExecSpace().fence(); |
| 675 |
Kokkos::parallel_for(RangePolicy(0, nRows), parScal); |
| 676 |
ExecSpace().fence(); |
| 677 |
|
| 678 |
} |
| 679 |
void applyD(ScalarArray &x, ScalarArray &y) |
| 680 |
{ |
| 681 |
ParScalFunctor<Ordinal, Scalar, Real, ExecSpace> parScal(x, y, diagFact); |
| 682 |
ExecSpace().fence(); |
| 683 |
Kokkos::parallel_for(RangePolicy(0, nRows), parScal); |
| 684 |
ExecSpace().fence(); |
| 685 |
|
| 686 |
} |
| 687 |
void applyLIC(ScalarArray &x, ScalarArray &y) |
| 688 |
{ |
| 689 |
ParInitZeroFunctor<Ordinal, Scalar, ExecSpace> parInitZero(xOld); |
| 690 |
Kokkos::parallel_for(RangePolicy(0, nRows), parInitZero); |
| 691 |
ExecSpace().fence(); |
| 692 |
#if 0 |
| 693 |
JacobiIterFunctor<Ordinal, Scalar, ExecSpace> jacIter(nRows, lRowMap, lColIdx, lVal, |
| 694 |
x, y, xOld, diagElems); |
| 695 |
#endif |
| 696 |
BlockJacobiIterFunctorL<Ordinal, Scalar, ExecSpace> jacIter(nRows, blkSz, lRowMap, |
| 697 |
lColIdx, lVal, |
| 698 |
x, y, xOld, diagElems); |
| 699 |
ParCopyFunctor<Ordinal, Scalar, ExecSpace> parCopy(xOld, y); |
| 700 |
Ordinal extent = nRows/blkSz; |
| 701 |
if(nRows%blkSz != 0) |
| 702 |
{ |
| 703 |
extent++; |
| 704 |
} |
| 705 |
ExecSpace().fence(); |
| 706 |
for (Ordinal i = 0; i < nTrisol; i++) |
| 707 |
{ |
| 708 |
Kokkos::parallel_for(RangePolicy(0, extent), jacIter); |
| 709 |
ExecSpace().fence(); |
| 710 |
Kokkos::parallel_for(RangePolicy(0, nRows), parCopy); |
| 711 |
ExecSpace().fence(); |
| 712 |
} |
| 713 |
return; |
| 714 |
} |
| 715 |
|
| 716 |
void applyLT(ScalarArray &x, ScalarArray &y) |
| 717 |
{ |
| 718 |
ParInitZeroFunctor<Ordinal, Scalar, ExecSpace> parInitZero(xOld); |
| 719 |
Kokkos::parallel_for(RangePolicy(0, nRows), parInitZero); |
| 720 |
ExecSpace().fence(); |
| 721 |
#if 0 |
| 722 |
JacobiIterFunctor<Ordinal, Scalar, ExecSpace> jacIter(nRows, ltRowMap, ltColIdx, ltVal, |
| 723 |
x, y, xOld, diagElems); |
| 724 |
#endif |
| 725 |
BlockJacobiIterFunctorU<Ordinal, Scalar, ExecSpace> jacIter(nRows, blkSz, ltRowMap, ltColIdx, |
| 726 |
ltVal, x, y, xOld, diagElems); |
| 727 |
ParCopyFunctor<Ordinal, Scalar, ExecSpace> parCopy(xOld, y); |
| 728 |
Ordinal extent = nRows/blkSz; |
| 729 |
if (nRows%blkSz != 0) |
| 730 |
{ |
| 731 |
extent++; |
| 732 |
} |
| 733 |
ExecSpace().fence(); |
| 734 |
for (Ordinal i = 0; i < nTrisol; i++) |
| 735 |
{ |
| 736 |
Kokkos::parallel_for(RangePolicy(0, extent), jacIter); |
| 737 |
ExecSpace().fence(); |
| 738 |
Kokkos::parallel_for(RangePolicy(0, nRows), parCopy); |
| 739 |
ExecSpace().fence(); |
| 740 |
} |
| 741 |
return; |
| 742 |
} |
| 743 |
|
| 744 |
|
| 745 |
#if 0 |
| 746 |
void applyLT(ScalarArray &x, ScalarArray &y) |
| 747 |
{ |
| 748 |
ParInitZeroFunctor<Ordinal, Scalar, ExecSpace> parInitZero(nRows, xOld); |
| 749 |
ParInitZeroFunctor<Ordinal, Scalar, ExecSpace> parInitZero1(nRows, y); |
| 750 |
Kokkos::parallel_for(RangePolicy(0, nRows), parInitZero); |
| 751 |
ExecSpace().fence(); |
| 752 |
//Transpose Jacobi implementation |
| 753 |
JacobiIterFunctorT<Ordinal, Scalar, ExecSpace> jacIterT(nRows, lRowMap, lColIdx, lVal, x, y, xOld, onesVector); |
| 754 |
ParCopyFunctor<Ordinal, Scalar, ExecSpace> parCopy(nRows, xOld, y); |
| 755 |
|
| 756 |
ExecSpace().fence(); |
| 757 |
|
| 758 |
for (Ordinal i = 0; i < nTrisol; i++) |
| 759 |
{ |
| 760 |
Kokkos::parallel_for(RangePolicy(0, nRows), parInitZero1); |
| 761 |
ExecSpace().fence(); |
| 762 |
Kokkos::parallel_for(RangePolicy(0, nRows), jacIterT); |
| 763 |
ExecSpace().fence(); |
| 764 |
Kokkos::parallel_for(RangePolicy(0, nRows), parCopy); |
| 765 |
ExecSpace().fence(); |
| 766 |
} |
| 767 |
|
| 768 |
return; |
| 769 |
} |
| 770 |
#endif |
| 771 |
|
| 772 |
void setValues(ScalarArray& aValsIn) |
| 773 |
{ |
| 774 |
this->aValHost = Kokkos::create_mirror_view(aValsIn); |
| 775 |
Kokkos::deep_copy(this->aValHost, aValsIn); |
| 776 |
if(!initGuessPrec.is_null()) |
| 777 |
{ |
| 778 |
initGuessPrec->setValues(aValsIn); |
| 779 |
} |
| 780 |
} |
| 781 |
|
| 782 |
void compute() |
| 783 |
{ |
| 784 |
const Scalar one = STS::one(); |
| 785 |
Kokkos::Timer timer; |
| 786 |
if((level > 0) && (guessFlag != 0)) |
| 787 |
{ |
| 788 |
initGuessPrec->compute(); |
| 789 |
} |
| 790 |
numericILU(); |
| 791 |
FastICFunctor<Ordinal, Scalar, ExecSpace> icFunctor(aRowMap_[nRows], blkSzIC, |
| 792 |
aRowMap, aColIdx, aRowIdx, aVal, lRowMap, lColIdx, lVal, diagElems, omega); |
| 793 |
ExecSpace().fence(); |
| 794 |
Ordinal extent = aRowMap_[nRows]/blkSzIC; |
| 795 |
if (aRowMap_[nRows]%blkSzIC != 0) |
| 796 |
{ |
| 797 |
extent++; |
| 798 |
} |
| 799 |
//Ordinal extent = aRowMap[nRows]; |
| 800 |
for (int i = 0; i < nFact; i++) |
| 801 |
{ |
| 802 |
Kokkos::parallel_for(RangePolicy(0, extent), icFunctor); |
| 803 |
} |
| 804 |
ExecSpace().fence(); |
| 805 |
|
| 806 |
double t = timer.seconds(); |
| 807 |
computeTime = t; |
| 808 |
|
| 809 |
Kokkos::deep_copy(diagElems_, diagElems); |
| 810 |
diagElemsInv_ = Kokkos::create_mirror_view(diagElemsInv); |
| 811 |
for (int i = 0; i < nRows; i++) |
| 812 |
{ |
| 813 |
diagElemsInv_[i] = one/diagElems_[i]; |
| 814 |
} |
| 815 |
Kokkos::deep_copy(diagElemsInv, diagElemsInv_); |
| 816 |
transposeL(); |
| 817 |
|
| 818 |
if (standard_sptrsv) { |
| 819 |
#if defined(KOKKOSKERNELS_ENABLE_TPL_CUSPARSE) |
| 820 |
KokkosSparse::Experimental::SPTRSVAlgorithm algo = KokkosSparse::Experimental::SPTRSVAlgorithm::SPTRSV_CUSPARSE; |
| 821 |
#else |
| 822 |
KokkosSparse::Experimental::SPTRSVAlgorithm algo = KokkosSparse::Experimental::SPTRSVAlgorithm::SEQLVLSCHD_TP1; |
| 823 |
#endif |
| 824 |
// setup L solve |
| 825 |
khL.create_sptrsv_handle(algo, nRows, true); |
| 826 |
#if defined(KOKKOSKERNELS_ENABLE_TPL_CUSPARSE) |
| 827 |
KokkosSparse::Experimental::sptrsv_symbolic(&khL, lRowMap, lColIdx, lVal); |
| 828 |
#else |
| 829 |
KokkosSparse::Experimental::sptrsv_symbolic(&khL, lRowMap, lColIdx); |
| 830 |
#endif |
| 831 |
// setup Lt solve |
| 832 |
khLt.create_sptrsv_handle(algo, nRows, false); |
| 833 |
#if defined(KOKKOSKERNELS_ENABLE_TPL_CUSPARSE) |
| 834 |
KokkosSparse::Experimental::sptrsv_symbolic(&khLt, ltRowMap, ltColIdx, ltVal); |
| 835 |
#else |
| 836 |
KokkosSparse::Experimental::sptrsv_symbolic(&khLt, ltRowMap, ltColIdx); |
| 837 |
#endif |
| 838 |
} |
| 839 |
} |
| 840 |
|
| 841 |
void apply(ScalarArray &x, ScalarArray &y) |
| 842 |
{ |
| 843 |
Kokkos::Timer timer; |
| 844 |
ParCopyFunctor<Ordinal, Scalar, ExecSpace> parCopyFunctor(xTemp, x); |
| 845 |
ExecSpace().fence(); |
| 846 |
Kokkos::parallel_for(RangePolicy(0, nRows), parCopyFunctor); |
| 847 |
ExecSpace().fence(); |
| 848 |
|
| 849 |
applyD(x, xTemp); |
| 850 |
if (standard_sptrsv) { |
| 851 |
// solve with L |
| 852 |
KokkosSparse::Experimental::sptrsv_solve(&khL, lRowMap, lColIdx, lVal, xTemp, y); |
| 853 |
// solve with Lt |
| 854 |
KokkosSparse::Experimental::sptrsv_solve(&khLt, ltRowMap, ltColIdx, ltVal, y, xTemp); |
| 855 |
} else { |
| 856 |
applyLIC(xTemp, y); |
| 857 |
//applyDD(y, xTemp); |
| 858 |
applyLT(y, xTemp); |
| 859 |
} |
| 860 |
applyD(xTemp, y); |
| 861 |
|
| 862 |
// ParCopyFunctor<Ordinal, Scalar, ExecSpace> parCopyFunctor2(nRows, y, xTemp); |
| 863 |
// ExecSpace().fence(); |
| 864 |
// Kokkos::parallel_for(RangePolicy(0, nRows), parCopyFunctor2); |
| 865 |
// ExecSpace().fence(); |
| 866 |
|
| 867 |
double t = timer.seconds(); |
| 868 |
applyTime = t; |
| 869 |
return; |
| 870 |
} |
| 871 |
|
| 872 |
Ordinal getNFact() const |
| 873 |
{ |
| 874 |
return nFact; |
| 875 |
} |
| 876 |
|
| 877 |
std::string getSpTrsvType() const |
| 878 |
{ |
| 879 |
return (standard_sptrsv ? "Standard" : "Fast"); |
| 880 |
} |
| 881 |
|
| 882 |
Ordinal getNTrisol() const |
| 883 |
{ |
| 884 |
return nTrisol; |
| 885 |
} |
| 886 |
|
| 887 |
Ordinal getNRows() const |
| 888 |
{ |
| 889 |
return nRows; |
| 890 |
} |
| 891 |
|
| 892 |
double getComputeTime() const |
| 893 |
{ |
| 894 |
return computeTime; |
| 895 |
} |
| 896 |
|
| 897 |
double getInitializeTime() const |
| 898 |
{ |
| 899 |
return initTime; |
| 900 |
} |
| 901 |
|
| 902 |
double getApplyTime() const |
| 903 |
{ |
| 904 |
return applyTime; |
| 905 |
} |
| 906 |
void checkIC() const |
| 907 |
{ |
| 908 |
//Compute the L2 norm of the nonlinear residual (A - LLt) on sparsity pattern |
| 909 |
// |
| 910 |
Scalar sum = STS::zero(); |
| 911 |
for (int i = 0 ; i < nRows; i++) |
| 912 |
{ |
| 913 |
int row = i; |
| 914 |
// Each row in A matrix (result) |
| 915 |
for (int k = aRowMap[i]; k < aRowMap[i+1]; k++) |
| 916 |
{ |
| 917 |
|
| 918 |
int col = aColIdx[k]; |
| 919 |
|
| 920 |
if (row >= col) { |
| 921 |
Scalar acc_val = aVal[k]; |
| 922 |
Ordinal lptr, uptr; |
| 923 |
for ( lptr = lRowMap[i], uptr = lRowMap[aColIdx[k]] ; |
| 924 |
lptr < lRowMap[i+1] && uptr < lRowMap[aColIdx[k]+1] ; ) |
| 925 |
{ |
| 926 |
if (lColIdx[lptr] == lColIdx[uptr]) |
| 927 |
{ |
| 928 |
acc_val -= lVal[lptr] * lVal[uptr]; |
| 929 |
lptr++; |
| 930 |
uptr++; |
| 931 |
} |
| 932 |
else if (lColIdx[lptr] < lColIdx[uptr]) |
| 933 |
lptr++; |
| 934 |
else |
| 935 |
uptr++; |
| 936 |
} |
| 937 |
|
| 938 |
sum += acc_val * acc_val; |
| 939 |
} |
| 940 |
} |
| 941 |
} |
| 942 |
#ifdef FASTIC_DEBUG_OUTPUT |
| 943 |
std::cout << "l2 norm of nonlinear residual = " << std::sqrt(sum) << std::endl; |
| 944 |
#endif |
| 945 |
} |
| 946 |
friend class FastICFunctor<Ordinal, Scalar, ExecSpace>; |
| 947 |
friend class JacobiIterFunctor<Ordinal, Scalar, ExecSpace>; |
| 948 |
friend class ParCopyFunctor<Ordinal, Scalar, ExecSpace>; |
| 949 |
friend class JacobiIterFunctorT<Ordinal, Scalar, ExecSpace>; |
| 950 |
friend class ParScalFunctor<Ordinal, Scalar, Real, ExecSpace>; |
| 951 |
friend class MemoryPrimeFunctorN<Ordinal, Scalar, ExecSpace>; |
| 952 |
friend class MemoryPrimeFunctorNnzCoo<Ordinal, Scalar, ExecSpace>; |
| 953 |
friend class MemoryPrimeFunctorNnzCsr<Ordinal, Scalar, ExecSpace>; |
| 954 |
}; |
| 955 |
|
| 956 |
|
| 957 |
#endif |