| 1 |
// @HEADER |
| 2 |
// ***************************************************************************** |
| 3 |
// ShyLU: Scalable Hybrid LU Preconditioner and Solver |
| 4 |
// |
| 5 |
// Copyright 2011 NTESS and the ShyLU contributors. |
| 6 |
// SPDX-License-Identifier: BSD-3-Clause |
| 7 |
// ***************************************************************************** |
| 8 |
// @HEADER |
| 9 |
|
| 10 |
// The struct that iterates over all non-zeros for the FastILDL |
| 11 |
// Contact for bugs and complaints - Siva Rajamanickam ([email protected]) |
| 12 |
// |
| 13 |
#ifndef __FAST_ILDL_HPP__ |
| 14 |
#define __FAST_ILDL_HPP__ |
| 15 |
#include <iostream> |
| 16 |
#include <algorithm> |
| 17 |
#include <vector> |
| 18 |
#include <queue> |
| 19 |
#include <random> |
| 20 |
#include <cstdlib> |
| 21 |
#include <cmath> |
| 22 |
#include <ctime> |
| 23 |
#include <string> |
| 24 |
|
| 25 |
#include <assert.h> |
| 26 |
#include <Kokkos_Core.hpp> |
| 27 |
#include <Kokkos_Timer.hpp> |
| 28 |
|
| 29 |
#include "shylu_fastilu.hpp" |
| 30 |
|
| 31 |
//whether to print extra debug output at runtime to stdout |
| 32 |
//comment out next line to disable |
| 33 |
//#define FASTILDL_DEBUG_OUTPUT |
| 34 |
|
| 35 |
template<class Ordinal, class Scalar, class ExecSpace> |
| 36 |
class FastILDLFunctor; |
| 37 |
|
| 38 |
template<class Ordinal, class Scalar, class ExecSpace> |
| 39 |
class FastILDLPrec |
| 40 |
{ |
| 41 |
|
| 42 |
public: |
| 43 |
typedef typename Teuchos::ScalarTraits<Scalar>::magnitudeType Real; |
| 44 |
typedef Kokkos::View<Ordinal *, ExecSpace> OrdinalArray; |
| 45 |
typedef Kokkos::View<Scalar *, ExecSpace> ScalarArray; |
| 46 |
typedef Kokkos::View<Real *, ExecSpace> RealArray; |
| 47 |
typedef Kokkos::View<Ordinal *, typename ExecSpace::array_layout, Kokkos::Serial, |
| 48 |
Kokkos::MemoryUnmanaged> UMOrdinalArray; |
| 49 |
typedef Kokkos::View<Scalar *, typename ExecSpace::array_layout, Kokkos::Serial, |
| 50 |
Kokkos::MemoryUnmanaged> UMScalarArray; |
| 51 |
typedef FastILDLPrec<Ordinal, Scalar, ExecSpace> FastPrec; |
| 52 |
|
| 53 |
typedef Kokkos::View<Ordinal *, Kokkos::HostSpace> OrdinalArrayHost; |
| 54 |
typedef Kokkos::View<Scalar *, Kokkos::HostSpace> ScalarArrayHost; |
| 55 |
typedef typename OrdinalArray::host_mirror_type OrdinalArrayMirror; |
| 56 |
typedef typename ScalarArray::host_mirror_type ScalarArrayMirror; |
| 57 |
|
| 58 |
typedef Kokkos::RangePolicy<ExecSpace> RangePolicy; |
| 59 |
|
| 60 |
using STS = Kokkos::ArithTraits<Scalar>; |
| 61 |
using RTS = Kokkos::ArithTraits<Real>; |
| 62 |
|
| 63 |
private: |
| 64 |
double computeTime; |
| 65 |
double applyTime; |
| 66 |
double initTime; |
| 67 |
|
| 68 |
Ordinal guessFlag; |
| 69 |
Ordinal nRows; |
| 70 |
Ordinal nFact; |
| 71 |
Ordinal nTrisol; |
| 72 |
Ordinal level; |
| 73 |
Ordinal blkSz; |
| 74 |
Ordinal blkSzILDL; |
| 75 |
Scalar omega; |
| 76 |
Scalar shift; |
| 77 |
|
| 78 |
//Lower triangular factor (CSR) |
| 79 |
ScalarArray lVal; |
| 80 |
OrdinalArray lColIdx; |
| 81 |
OrdinalArray lRowMap; |
| 82 |
// mirrors |
| 83 |
ScalarArrayMirror lVal_; |
| 84 |
OrdinalArrayMirror lColIdx_; |
| 85 |
OrdinalArrayMirror lRowMap_; |
| 86 |
|
| 87 |
//Upper triangular factor (CSR) |
| 88 |
ScalarArray ltVal; |
| 89 |
OrdinalArray ltColIdx; |
| 90 |
OrdinalArray ltRowMap; |
| 91 |
|
| 92 |
//Pointer to the original host copy of A. |
| 93 |
ScalarArrayMirror aValHost; |
| 94 |
OrdinalArrayMirror aRowMapHost; |
| 95 |
OrdinalArrayMirror aColIdxHost; |
| 96 |
|
| 97 |
//A matrix in COO format |
| 98 |
ScalarArray aVal; |
| 99 |
OrdinalArray aRowMap; |
| 100 |
OrdinalArray aRowIdx; |
| 101 |
OrdinalArray aColIdx; |
| 102 |
// mirrors |
| 103 |
ScalarArrayMirror aVal_; |
| 104 |
OrdinalArrayMirror aRowMap_; |
| 105 |
OrdinalArrayMirror aRowIdx_; |
| 106 |
OrdinalArrayMirror aColIdx_; |
| 107 |
|
| 108 |
//Diagonal scaling factors |
| 109 |
RealArray diagFact; |
| 110 |
ScalarArray diagElems; |
| 111 |
ScalarArray diagElemsInv; |
| 112 |
// mirrors |
| 113 |
ScalarArrayMirror diagElems_; |
| 114 |
ScalarArrayMirror diagElemsInv_; |
| 115 |
|
| 116 |
//Temporary vectors for triangular solves |
| 117 |
ScalarArray xOld; |
| 118 |
ScalarArray xTemp; |
| 119 |
ScalarArray onesVector; |
| 120 |
|
| 121 |
//This will have the continuation initial |
| 122 |
//guess if guessFlag=1 |
| 123 |
Teuchos::RCP<FastPrec> initGuessPrec; |
| 124 |
|
| 125 |
// forward/backwar substitution for standard SpTrsv |
| 126 |
using MemSpace = typename ExecSpace::memory_space; |
| 127 |
using KernelHandle = KokkosKernels::Experimental::KokkosKernelsHandle <Ordinal, Ordinal, Scalar, ExecSpace, MemSpace, MemSpace >; |
| 128 |
bool standard_sptrsv; |
| 129 |
KernelHandle khL; |
| 130 |
KernelHandle khLt; |
| 131 |
|
| 132 |
public: |
| 133 |
FastILDLPrec(OrdinalArray &aRowMapIn, OrdinalArray &aColIdxIn, ScalarArray &aValIn, Ordinal nRow_, bool standard_sptrsv_, |
| 134 |
Ordinal nFact_, Ordinal nTrisol_, Ordinal level_, Scalar omega_, Scalar shift_, Ordinal guessFlag_, Ordinal blkSzILDL_, Ordinal blkSz_) |
| 135 |
{ |
| 136 |
nRows = nRow_; |
| 137 |
standard_sptrsv = standard_sptrsv_; |
| 138 |
nFact = nFact_; |
| 139 |
nTrisol = nTrisol_; |
| 140 |
computeTime = 0.0; |
| 141 |
applyTime = 0.0; |
| 142 |
initTime = 0.0; |
| 143 |
level = level_; |
| 144 |
|
| 145 |
// mirror & deep-copy input matrix to host |
| 146 |
aRowMapHost = Kokkos::create_mirror_view(aRowMapIn); |
| 147 |
aColIdxHost = Kokkos::create_mirror_view(aColIdxIn); |
| 148 |
aValHost = Kokkos::create_mirror_view(aValIn); |
| 149 |
Kokkos::deep_copy(aRowMapHost, aRowMapIn); |
| 150 |
Kokkos::deep_copy(aColIdxHost, aColIdxIn); |
| 151 |
Kokkos::deep_copy(aValHost, aValIn); |
| 152 |
|
| 153 |
omega = omega_; |
| 154 |
guessFlag = guessFlag_; |
| 155 |
shift = shift_; |
| 156 |
blkSzILDL = blkSzILDL_; |
| 157 |
blkSz = blkSz_; |
| 158 |
|
| 159 |
const Scalar one = Kokkos::ArithTraits<Scalar>::one(); |
| 160 |
onesVector = ScalarArray("onesVector", nRow_); |
| 161 |
Kokkos::deep_copy(onesVector, one); |
| 162 |
|
| 163 |
diagFact = RealArray("diagFact", nRow_); |
| 164 |
diagElems = ScalarArray("diagElems", nRow_); |
| 165 |
diagElemsInv = ScalarArray("diagElems", nRow_); |
| 166 |
xOld = ScalarArray("xOld", nRow_); |
| 167 |
xTemp = ScalarArray("xTemp", nRow_); |
| 168 |
|
| 169 |
if (level > 0) |
| 170 |
{ |
| 171 |
initGuessPrec = Teuchos::rcp(new FastPrec(aRowMapIn, aColIdxIn, aValIn, nRow_, standard_sptrsv, 3, 5, |
| 172 |
level_-1, omega_, shift_, guessFlag_, blkSzILDL_, blkSz_)); |
| 173 |
} |
| 174 |
|
| 175 |
} |
| 176 |
|
| 177 |
void initialize() |
| 178 |
{ |
| 179 |
Kokkos::Timer timer; |
| 180 |
//Here we have to allocate memory for L, U. |
| 181 |
if((level > 0) && (guessFlag != 0)) |
| 182 |
{ |
| 183 |
initGuessPrec->initialize(); |
| 184 |
} |
| 185 |
|
| 186 |
//symbolicILUAndInit(); //Note the diagonal scaling is now embedded in this routine. |
| 187 |
symbolicILU(); |
| 188 |
#ifdef SHYLU_DEBUG |
| 189 |
//initialize L, U, A patterns |
| 190 |
MemoryPrimeFunctorN<Ordinal, Scalar, ExecSpace> copyFunc1(aRowMap, lRowMap, lRowMap, diagElems); |
| 191 |
|
| 192 |
//Make sure all memory resides on the device |
| 193 |
ExecSpace().fence(); |
| 194 |
Kokkos::parallel_for(RangePolicy(0, nRows), copyFunc1); |
| 195 |
|
| 196 |
//Note that the following is a temporary measure |
| 197 |
//to ensure that memory resides on the device. |
| 198 |
Ordinal nnzL = lRowMap[nRows]; |
| 199 |
Ordinal nnzA = aRowMap[nRows]; |
| 200 |
MemoryPrimeFunctorNnzCoo<Ordinal, Scalar, ExecSpace> copyFunc2(aColIdx, aRowIdx, aVal); |
| 201 |
MemoryPrimeFunctorNnzCsr<Ordinal, Scalar, ExecSpace> copyFunc3(lColIdx, lVal); |
| 202 |
|
| 203 |
ExecSpace().fence(); |
| 204 |
Kokkos::parallel_for(RangePolicy(0, nRows), copyFunc1); |
| 205 |
Kokkos::parallel_for(RangePolicy(0, nnzA), copyFunc2); |
| 206 |
Kokkos::parallel_for(RangePolicy(0, nnzL), copyFunc3); |
| 207 |
#endif |
| 208 |
double t = timer.seconds(); |
| 209 |
#ifdef FASTILDL_DEBUG_OUTPUT |
| 210 |
std::cout << "Symbolic phase complete." << std::endl; |
| 211 |
std::cout << "Init time: "<< t << "s" << std::endl; |
| 212 |
#endif |
| 213 |
initTime = t; |
| 214 |
return; |
| 215 |
|
| 216 |
} |
| 217 |
|
| 218 |
void getL(OrdinalArray &lRowMapOut, OrdinalArray &lColIdxOut, ScalarArray &lValOut) |
| 219 |
{ |
| 220 |
lRowMapOut = lRowMap; |
| 221 |
lColIdxOut = lColIdx; |
| 222 |
lValOut = lVal; |
| 223 |
} |
| 224 |
|
| 225 |
void getD(ScalarArray &diagElemsOut) |
| 226 |
{ |
| 227 |
diagElemsOut = diagElems; |
| 228 |
} |
| 229 |
|
| 230 |
void transposeL() |
| 231 |
{ |
| 232 |
auto ltRowMap_ = Kokkos::create_mirror_view(ltRowMap); |
| 233 |
auto ltColIdx_ = Kokkos::create_mirror_view(ltColIdx); |
| 234 |
auto ltVal_ = Kokkos::create_mirror_view(ltVal); |
| 235 |
|
| 236 |
//Count the elements in each row of Lt |
| 237 |
auto temp = OrdinalArrayHost("temp", nRows + 1); |
| 238 |
auto rowPtrs = OrdinalArrayHost("rowPtrs", nRows); |
| 239 |
for (Ordinal i = 0; i <= nRows; i++) |
| 240 |
{ |
| 241 |
temp[i] = 0; |
| 242 |
} |
| 243 |
for (Ordinal i = 0; i < nRows; i++) |
| 244 |
{ |
| 245 |
for (Ordinal k = lRowMap_[i]; k < lRowMap_[i+1]; k++) |
| 246 |
{ |
| 247 |
temp[lColIdx_[k]+1]++; |
| 248 |
|
| 249 |
} |
| 250 |
} |
| 251 |
//Perform an add scan to get the row map for |
| 252 |
//the transpose |
| 253 |
for (Ordinal i = 0; i <= nRows; i++) |
| 254 |
{ |
| 255 |
ltRowMap_[i] = temp[i]; |
| 256 |
} |
| 257 |
for (Ordinal i = 1; i <= nRows; i++) |
| 258 |
{ |
| 259 |
ltRowMap_[i] += ltRowMap_[i-1]; |
| 260 |
} |
| 261 |
//Set the row pointers to their initial places; |
| 262 |
for (Ordinal i = 0; i < nRows; i++) |
| 263 |
{ |
| 264 |
rowPtrs[i] = ltRowMap_[i]; |
| 265 |
} |
| 266 |
//Copy the data |
| 267 |
Kokkos::deep_copy(lVal_, lVal); |
| 268 |
for (Ordinal i = 0; i < nRows; i++) |
| 269 |
{ |
| 270 |
for (Ordinal k = lRowMap_[i]; k < lRowMap_[i+1]; k++) |
| 271 |
{ |
| 272 |
Ordinal row = lColIdx_[k]; |
| 273 |
Scalar value = lVal_[k]; |
| 274 |
ltVal_[rowPtrs[row]] = value; |
| 275 |
ltColIdx_[rowPtrs[row]] = i; |
| 276 |
rowPtrs[row]++; |
| 277 |
assert(rowPtrs[row] <= ltRowMap_[row + 1]); |
| 278 |
} |
| 279 |
} |
| 280 |
Kokkos::deep_copy(ltRowMap, ltRowMap_); |
| 281 |
Kokkos::deep_copy(ltColIdx, ltColIdx_); |
| 282 |
Kokkos::deep_copy(ltVal, ltVal_); |
| 283 |
} |
| 284 |
|
| 285 |
void symbolicILU() |
| 286 |
{ |
| 287 |
using std::vector; |
| 288 |
using std::cout; |
| 289 |
using std::stable_sort; |
| 290 |
using std::sort; |
| 291 |
OrdinalArrayMirror ia = aRowMapHost; |
| 292 |
OrdinalArrayMirror ja = aColIdxHost; |
| 293 |
int n = nRows; |
| 294 |
int nzl = aRowMapHost[nRows] * (level + 2); |
| 295 |
int nzu = aRowMapHost[nRows] * (level + 2); |
| 296 |
Ordinal i; |
| 297 |
int levfill = level; |
| 298 |
vector<int> lnklst(n); |
| 299 |
vector<int> curlev(n); |
| 300 |
vector<int> levels(nzu); |
| 301 |
vector<int> iwork(n); |
| 302 |
vector<int> ial(nzl); |
| 303 |
vector<int> jal(nzl); |
| 304 |
vector<int> iau(nzu); |
| 305 |
vector<int> jau(nzu); |
| 306 |
|
| 307 |
int knzl = 0; |
| 308 |
int knzu = 0; |
| 309 |
|
| 310 |
ial[0] = 0; |
| 311 |
iau[0] = 0; |
| 312 |
|
| 313 |
for (i=0; i<n; i++) |
| 314 |
{ |
| 315 |
int first, next, j; |
| 316 |
|
| 317 |
/* copy column indices of row into workspace and sort them */ |
| 318 |
|
| 319 |
int len = ia[i+1] - ia[i]; |
| 320 |
next = 0; |
| 321 |
for (j=ia[i]; j<ia[i+1]; j++) |
| 322 |
iwork[next++] = ja[j]; |
| 323 |
// shell_sort(len, iwork); |
| 324 |
//stable_sort(iwork.begin(), iwork.begin() + len); |
| 325 |
sort(iwork.begin(), iwork.begin() + len); |
| 326 |
|
| 327 |
/* construct implied linked list for row */ |
| 328 |
|
| 329 |
first = iwork[0]; |
| 330 |
curlev[first] = 0; |
| 331 |
|
| 332 |
for (j=0; j<=len-2; j++) |
| 333 |
{ |
| 334 |
lnklst[iwork[j]] = iwork[j+1]; |
| 335 |
curlev[iwork[j]] = 0; |
| 336 |
} |
| 337 |
|
| 338 |
lnklst[iwork[len-1]] = n; |
| 339 |
curlev[iwork[len-1]] = 0; |
| 340 |
|
| 341 |
/* merge with rows in U */ |
| 342 |
|
| 343 |
next = first; |
| 344 |
while (next < i) |
| 345 |
{ |
| 346 |
int oldlst = next; |
| 347 |
int nxtlst = lnklst[next]; |
| 348 |
int row = next; |
| 349 |
int ii; |
| 350 |
|
| 351 |
/* scan row */ |
| 352 |
|
| 353 |
for (ii=iau[row]+1; ii<iau[row+1]; /*nop*/) |
| 354 |
{ |
| 355 |
if (jau[ii] < nxtlst) |
| 356 |
{ |
| 357 |
/* new fill-in */ |
| 358 |
int newlev = curlev[row] + levels[ii] + 1; |
| 359 |
if (newlev <= levfill) |
| 360 |
{ |
| 361 |
lnklst[oldlst] = jau[ii]; |
| 362 |
lnklst[jau[ii]] = nxtlst; |
| 363 |
oldlst = jau[ii]; |
| 364 |
curlev[jau[ii]] = newlev; |
| 365 |
} |
| 366 |
ii++; |
| 367 |
} |
| 368 |
else if (jau[ii] == nxtlst) |
| 369 |
{ |
| 370 |
int newlev; |
| 371 |
oldlst = nxtlst; |
| 372 |
nxtlst = lnklst[oldlst]; |
| 373 |
newlev = curlev[row] + levels[ii] + 1; |
| 374 |
//curlev[jau[ii]] = MIN(curlev[jau[ii]], newlev); |
| 375 |
if (curlev[jau[ii]] > newlev) |
| 376 |
{ |
| 377 |
curlev[jau[ii]] = newlev; |
| 378 |
} |
| 379 |
ii++; |
| 380 |
} |
| 381 |
else /* (jau[ii] > nxtlst) */ |
| 382 |
{ |
| 383 |
oldlst = nxtlst; |
| 384 |
nxtlst = lnklst[oldlst]; |
| 385 |
} |
| 386 |
} |
| 387 |
next = lnklst[next]; |
| 388 |
} |
| 389 |
|
| 390 |
/* gather the pattern into L and U */ |
| 391 |
|
| 392 |
next = first; |
| 393 |
while (next < i) |
| 394 |
{ |
| 395 |
assert(knzl < nzl); |
| 396 |
jal[knzl++] = next; |
| 397 |
if (knzl >= nzl) |
| 398 |
{ |
| 399 |
//(*nzl)++; |
| 400 |
nzl += nRows; |
| 401 |
jal.resize(nzl); |
| 402 |
} |
| 403 |
//jal.push_back(next); |
| 404 |
// knzl++; |
| 405 |
next = lnklst[next]; |
| 406 |
} |
| 407 |
ial[i+1] = knzl; |
| 408 |
assert(next == i); |
| 409 |
|
| 410 |
while (next < n) |
| 411 |
{ |
| 412 |
assert(knzu < nzu); |
| 413 |
levels[knzu] = curlev[next]; |
| 414 |
jau[knzu++] = next; |
| 415 |
if (knzu >= nzu) |
| 416 |
{ |
| 417 |
//(*nzu)++; |
| 418 |
nzu += nzu + nRows; |
| 419 |
jau.resize(nzu); |
| 420 |
levels.resize(nzu); |
| 421 |
} |
| 422 |
//jau.push_back(next); |
| 423 |
//knzu++; |
| 424 |
next = lnklst[next]; |
| 425 |
} |
| 426 |
iau[i+1] = knzu; |
| 427 |
} |
| 428 |
|
| 429 |
#ifdef FASTILDL_DEBUG_OUTPUT |
| 430 |
std::cout << "knzl =" << knzl; |
| 431 |
std::cout << "knzu =" << knzu; |
| 432 |
#endif |
| 433 |
|
| 434 |
nzl = knzl; |
| 435 |
nzu = knzu; |
| 436 |
|
| 437 |
#ifdef FASTILDL_DEBUG_OUTPUT |
| 438 |
std::cout << "ILU: nnz = "<< knzl + knzu << std::endl; |
| 439 |
std::cout << "Actual nnz for ILU: " << nzl + nzu << std::endl; |
| 440 |
#endif |
| 441 |
//Initialize the A matrix that is to be used in the computation |
| 442 |
aRowMap = OrdinalArray("aRowMap", nRows + 1); |
| 443 |
aColIdx = OrdinalArray("aColIdx", knzl + knzu); |
| 444 |
aRowIdx = OrdinalArray("aRowIds", knzl + knzu); |
| 445 |
aRowMap_ = Kokkos::create_mirror_view(aRowMap); |
| 446 |
aColIdx_ = Kokkos::create_mirror_view(aColIdx); |
| 447 |
aRowIdx_ = Kokkos::create_mirror_view(aRowIdx); |
| 448 |
|
| 449 |
Ordinal aRowPtr = 0; |
| 450 |
aRowMap_[0] = 0; |
| 451 |
for (i = 0; i < nRows; i++) |
| 452 |
{ |
| 453 |
#ifdef FASTILDL_DEBUG_OUTPUT |
| 454 |
std::cout << "***row:" << i << std::endl; |
| 455 |
#endif |
| 456 |
for(Ordinal k = ial[i]; k < ial[i+1]; k++) |
| 457 |
{ |
| 458 |
#ifdef FASTILDL_DEBUG_OUTPUT |
| 459 |
std::cout << "jal[k]=" << jal[k] << std::endl; |
| 460 |
#endif |
| 461 |
aColIdx_[aRowPtr] = jal[k]; |
| 462 |
aRowIdx_[aRowPtr] = i; |
| 463 |
aRowPtr++; |
| 464 |
} |
| 465 |
for(Ordinal k = iau[i]; k < iau[i+1]; k++) |
| 466 |
{ |
| 467 |
aColIdx_[aRowPtr] = jau[k]; |
| 468 |
aRowIdx_[aRowPtr] = i; |
| 469 |
aRowPtr++; |
| 470 |
} |
| 471 |
aRowMap_[i+1] = aRowPtr; |
| 472 |
} |
| 473 |
Kokkos::deep_copy(aRowMap, aRowMap_); |
| 474 |
Kokkos::deep_copy(aColIdx, aColIdx_); |
| 475 |
Kokkos::deep_copy(aRowIdx, aRowIdx_); |
| 476 |
|
| 477 |
//Now allocate memory for L and U. |
| 478 |
// |
| 479 |
lRowMap = OrdinalArray("lRowMap", nRows + 1); |
| 480 |
ltRowMap = OrdinalArray("ltRowMap", nRows + 1); |
| 481 |
countL(); |
| 482 |
//Allocate memory and initialize pattern for L, U (transpose). |
| 483 |
lColIdx = OrdinalArray("lColIdx", lRowMap_[nRows]); |
| 484 |
ltColIdx = OrdinalArray("ltColIdx", lRowMap_[nRows]); |
| 485 |
#ifdef FASTILDL_DEBUG_OUTPUT |
| 486 |
std::cout << "nnz L = " << lRowMap[nRows] << std::endl; |
| 487 |
#endif |
| 488 |
} |
| 489 |
|
| 490 |
void numericILU() |
| 491 |
{ |
| 492 |
aVal = ScalarArray("aVal", aColIdx.extent(0)); |
| 493 |
aVal_ = Kokkos::create_mirror_view(aVal); |
| 494 |
|
| 495 |
//Copy the host matrix into the initialized a; |
| 496 |
for (Ordinal i = 0; i < nRows; i++) |
| 497 |
{ |
| 498 |
for(Ordinal k = aRowMap_[i]; k < aRowMap_[i+1]; k++) |
| 499 |
{ |
| 500 |
Ordinal col = aColIdx_[k]; |
| 501 |
#ifdef FASTILDL_DEBUG_OUTPUT |
| 502 |
std::cout << "col =" << col << std::endl; |
| 503 |
#endif |
| 504 |
for(Ordinal aHostPtr = aRowMapHost[i]; aHostPtr < aRowMapHost[i+1]; aHostPtr++) |
| 505 |
{ |
| 506 |
if (col == aColIdxHost[aHostPtr]) |
| 507 |
{ |
| 508 |
aVal_[k] = aValHost[aHostPtr]; |
| 509 |
break; |
| 510 |
} |
| 511 |
} |
| 512 |
} |
| 513 |
} |
| 514 |
lVal = ScalarArray("lVal", lRowMap_[nRows]); |
| 515 |
ltVal = ScalarArray("ltVal", lRowMap_[nRows]); |
| 516 |
applyDiagonalScaling(); |
| 517 |
applyManteuffelShift(); |
| 518 |
Kokkos::deep_copy(aVal, aVal_); |
| 519 |
|
| 520 |
fillL(); |
| 521 |
} |
| 522 |
|
| 523 |
void countL() |
| 524 |
{ |
| 525 |
lRowMap_ = Kokkos::create_mirror_view(lRowMap); |
| 526 |
|
| 527 |
lRowMap_[0] = 0; |
| 528 |
for (Ordinal i = 0; i < nRows; i++) |
| 529 |
{ |
| 530 |
Ordinal row_count = 0; |
| 531 |
for (Ordinal k = aRowMap_[i]; k < aRowMap_[i+1]; k++) |
| 532 |
{ |
| 533 |
Ordinal row = i; |
| 534 |
Ordinal col = aColIdx_[k]; |
| 535 |
|
| 536 |
if (row >= col) |
| 537 |
{ |
| 538 |
row_count++; |
| 539 |
} |
| 540 |
} |
| 541 |
lRowMap_[i+1] = lRowMap_[i] + row_count; |
| 542 |
} |
| 543 |
Kokkos::deep_copy(lRowMap, lRowMap_); |
| 544 |
} |
| 545 |
|
| 546 |
void fillL() |
| 547 |
{ |
| 548 |
lVal_ = Kokkos::create_mirror_view(lVal); |
| 549 |
lColIdx_ = Kokkos::create_mirror_view(lColIdx); |
| 550 |
diagElems_ = Kokkos::create_mirror_view(diagElems); |
| 551 |
|
| 552 |
Ordinal lPtr = 0; |
| 553 |
for (Ordinal i = 0; i < nRows; i++) |
| 554 |
{ |
| 555 |
for (Ordinal k = aRowMap_[i]; k < aRowMap_[i+1]; k++) |
| 556 |
{ |
| 557 |
Ordinal row = i; |
| 558 |
Ordinal col = aColIdx_[k]; |
| 559 |
|
| 560 |
if (row >= col) |
| 561 |
{ |
| 562 |
if (row == col) |
| 563 |
{ |
| 564 |
diagElems_[row] = aVal_[k]; |
| 565 |
} |
| 566 |
lVal_[lPtr] = aVal_[k]; |
| 567 |
lColIdx_[lPtr] = aColIdx_[k]; |
| 568 |
lPtr++; |
| 569 |
} |
| 570 |
} |
| 571 |
} |
| 572 |
assert(lPtr == lRowMap_[nRows]); |
| 573 |
Kokkos::deep_copy(diagElems, diagElems_); |
| 574 |
|
| 575 |
if ((level > 0) && (guessFlag !=0)) |
| 576 |
{ |
| 577 |
OrdinalArray lGRowMap; |
| 578 |
OrdinalArray lGColIdx; |
| 579 |
ScalarArray lGVal; |
| 580 |
ScalarArray gD; |
| 581 |
Ordinal lGPtr = 0; |
| 582 |
|
| 583 |
initGuessPrec->getL(lGRowMap, lGColIdx, lGVal); |
| 584 |
initGuessPrec->getD(gD); |
| 585 |
|
| 586 |
Kokkos::deep_copy(diagElems, gD); |
| 587 |
|
| 588 |
auto lGColIdx_ = Kokkos::create_mirror_view(lGColIdx); |
| 589 |
auto lGVal_ = Kokkos::create_mirror_view(lGVal); |
| 590 |
Kokkos::deep_copy(lGColIdx_, lGColIdx); |
| 591 |
Kokkos::deep_copy(lGVal_, lGVal); |
| 592 |
for (Ordinal i = 0; i < nRows; i++) |
| 593 |
{ |
| 594 |
#ifdef SHYLU_DEBUG |
| 595 |
Ordinal check = lGPtr; |
| 596 |
#endif |
| 597 |
for (Ordinal k = lRowMap_[i]; k < lRowMap_[i+1]; k++) |
| 598 |
{ |
| 599 |
//unused: Ordinal row = i; |
| 600 |
Ordinal col = lColIdx_[k]; |
| 601 |
|
| 602 |
if (col == lGColIdx_[lGPtr]) |
| 603 |
{ |
| 604 |
lVal_[k] = lGVal_[lGPtr]; |
| 605 |
lGPtr++; |
| 606 |
} |
| 607 |
} |
| 608 |
#ifdef SHYLU_DEBUG |
| 609 |
Ordinal rowLen = lGPtr - check; |
| 610 |
assert(rowLen == lGRowMap[i+1] - lGRowMap[i]); |
| 611 |
#endif |
| 612 |
} |
| 613 |
} |
| 614 |
Kokkos::deep_copy(lColIdx, lColIdx_); |
| 615 |
Kokkos::deep_copy(lVal, lVal_); |
| 616 |
} |
| 617 |
|
| 618 |
void applyDiagonalScaling() |
| 619 |
{ |
| 620 |
int anext = 0; |
| 621 |
const Real one = Kokkos::ArithTraits<Real>::one(); |
| 622 |
//First fill Aj and extract the diagonal scaling factors |
| 623 |
//Use diag array to store scaling factors since |
| 624 |
//it gets set to the correct value by findFactorPattern anyway. |
| 625 |
auto diagFact_ = Kokkos::create_mirror_view(diagFact); |
| 626 |
for (int i = 0; i < nRows; i++) |
| 627 |
{ |
| 628 |
for(int k = aRowMap_[i]; k < aRowMap_[i+1]; k++) |
| 629 |
{ |
| 630 |
aRowIdx_[anext++] = i; |
| 631 |
if (aColIdx_[k] == i) |
| 632 |
{ |
| 633 |
diagFact_[i] = one/(RTS::sqrt(STS::abs(aVal_[k]))); |
| 634 |
#ifdef FASTILDL_DEBUG_OUTPUT |
| 635 |
std::cout << "diagFact["<<i<<"]="<<aVal[k]<<std::endl; |
| 636 |
#endif |
| 637 |
} |
| 638 |
} |
| 639 |
} |
| 640 |
|
| 641 |
//Now go through each element of A and apply the scaling |
| 642 |
int row; |
| 643 |
int col; |
| 644 |
Real sc1, sc2; |
| 645 |
for (int i = 0; i < nRows; i++) |
| 646 |
{ |
| 647 |
for (int k = aRowMap_[i]; k < aRowMap_[i+1]; k++) |
| 648 |
{ |
| 649 |
row = aRowIdx_[k]; |
| 650 |
col = aColIdx_[k]; |
| 651 |
|
| 652 |
sc1 = diagFact_[row]; |
| 653 |
sc2 = diagFact_[col]; |
| 654 |
aVal_[k] = aVal_[k]*sc1*sc2; |
| 655 |
} |
| 656 |
} |
| 657 |
Kokkos::deep_copy(diagFact, diagFact_); |
| 658 |
} |
| 659 |
|
| 660 |
void applyManteuffelShift() |
| 661 |
{ |
| 662 |
const Scalar one = Kokkos::ArithTraits<Scalar>::one(); |
| 663 |
//Scalar shift = 0.05; |
| 664 |
for (Ordinal i = 0; i < nRows; i++) |
| 665 |
{ |
| 666 |
for (Ordinal k = aRowMap_[i]; k < aRowMap_[i+1]; k++) |
| 667 |
{ |
| 668 |
Ordinal row = i; |
| 669 |
Ordinal col = aColIdx_[k]; |
| 670 |
if (row != col) |
| 671 |
{ |
| 672 |
aVal_[k] = (one/(one + shift))*aVal_[k]; |
| 673 |
} |
| 674 |
} |
| 675 |
} |
| 676 |
} |
| 677 |
|
| 678 |
void applyDD(ScalarArray &x, ScalarArray &y) |
| 679 |
{ |
| 680 |
ParScalFunctor<Ordinal, Scalar, Scalar, ExecSpace> parScal(x, y, diagElemsInv); |
| 681 |
ExecSpace().fence(); |
| 682 |
Kokkos::parallel_for(RangePolicy(0, nRows), parScal); |
| 683 |
ExecSpace().fence(); |
| 684 |
|
| 685 |
} |
| 686 |
void applyD(ScalarArray &x, ScalarArray &y) |
| 687 |
{ |
| 688 |
ParScalFunctor<Ordinal, Scalar, Real, ExecSpace> parScal(x, y, diagFact); |
| 689 |
ExecSpace().fence(); |
| 690 |
Kokkos::parallel_for(RangePolicy(0, nRows), parScal); |
| 691 |
ExecSpace().fence(); |
| 692 |
|
| 693 |
} |
| 694 |
void applyLIC(ScalarArray &x, ScalarArray &y) |
| 695 |
{ |
| 696 |
ParInitZeroFunctor<Ordinal, Scalar, ExecSpace> parInitZero(xOld); |
| 697 |
Kokkos::parallel_for(RangePolicy(0, nRows), parInitZero); |
| 698 |
ExecSpace().fence(); |
| 699 |
#if 0 |
| 700 |
JacobiIterFunctor<Ordinal, Scalar, ExecSpace> jacIter(nRows, lRowMap, lColIdx, lVal, x, y, xOld, onesVector); |
| 701 |
#endif |
| 702 |
BlockJacobiIterFunctorL<Ordinal, Scalar, ExecSpace> jacIter(nRows, blkSz, lRowMap, lColIdx, lVal, |
| 703 |
x, y, xOld, onesVector); |
| 704 |
ParCopyFunctor<Ordinal, Scalar, ExecSpace> parCopy(xOld, y); |
| 705 |
Ordinal extent = nRows/blkSz; |
| 706 |
if(nRows%blkSz != 0) |
| 707 |
{ |
| 708 |
extent++; |
| 709 |
} |
| 710 |
ExecSpace().fence(); |
| 711 |
for (Ordinal i = 0; i < nTrisol; i++) |
| 712 |
{ |
| 713 |
Kokkos::parallel_for(RangePolicy(0, extent), jacIter); |
| 714 |
ExecSpace().fence(); |
| 715 |
Kokkos::parallel_for(RangePolicy(0, nRows), parCopy); |
| 716 |
ExecSpace().fence(); |
| 717 |
} |
| 718 |
return; |
| 719 |
} |
| 720 |
|
| 721 |
void applyLT(ScalarArray &x, ScalarArray &y) |
| 722 |
{ |
| 723 |
ParInitZeroFunctor<Ordinal, Scalar, ExecSpace> parInitZero(xOld); |
| 724 |
Kokkos::parallel_for(RangePolicy(0, nRows), parInitZero); |
| 725 |
ExecSpace().fence(); |
| 726 |
#if 0 |
| 727 |
JacobiIterFunctor<Ordinal, Scalar, ExecSpace> jacIter(nRows, ltRowMap, ltColIdx, ltVal, x, y, xOld, onesVector); |
| 728 |
#endif |
| 729 |
BlockJacobiIterFunctorU<Ordinal, Scalar, ExecSpace> jacIter(nRows, blkSz, ltRowMap, ltColIdx, ltVal, |
| 730 |
x, y, xOld, onesVector); |
| 731 |
ParCopyFunctor<Ordinal, Scalar, ExecSpace> parCopy(xOld, y); |
| 732 |
Ordinal extent = nRows/blkSz; |
| 733 |
if (nRows%blkSz != 0) |
| 734 |
{ |
| 735 |
extent++; |
| 736 |
} |
| 737 |
ExecSpace().fence(); |
| 738 |
|
| 739 |
for (Ordinal i = 0; i < nTrisol; i++) |
| 740 |
{ |
| 741 |
Kokkos::parallel_for(RangePolicy(0, extent), jacIter); |
| 742 |
ExecSpace().fence(); |
| 743 |
Kokkos::parallel_for(RangePolicy(0, nRows), parCopy); |
| 744 |
ExecSpace().fence(); |
| 745 |
} |
| 746 |
|
| 747 |
return; |
| 748 |
} |
| 749 |
|
| 750 |
void setValues(ScalarArray& aValsIn) |
| 751 |
{ |
| 752 |
this->aValHost = Kokkos::create_mirror_view(aValsIn); |
| 753 |
Kokkos::deep_copy(this->aValHost, aValsIn); |
| 754 |
if(!initGuessPrec.is_null()) |
| 755 |
{ |
| 756 |
initGuessPrec->setValues(aValsIn); |
| 757 |
} |
| 758 |
} |
| 759 |
|
| 760 |
void compute() |
| 761 |
{ |
| 762 |
const Scalar one = Kokkos::ArithTraits<Scalar>::one(); |
| 763 |
if((level > 0) && (guessFlag != 0)) |
| 764 |
{ |
| 765 |
initGuessPrec->compute(); |
| 766 |
} |
| 767 |
ExecSpace().fence(); |
| 768 |
|
| 769 |
Kokkos::Timer timer; |
| 770 |
numericILU(); |
| 771 |
FastILDLFunctor<Ordinal, Scalar, ExecSpace> ildltFunctor(aRowMap_[nRows], blkSzILDL, |
| 772 |
aRowMap, aColIdx, aRowIdx, aVal, lRowMap, lColIdx, lVal, diagElems, omega); |
| 773 |
|
| 774 |
Ordinal extent = aRowMap_[nRows]/blkSzILDL; |
| 775 |
if (aRowMap_[nRows]%blkSzILDL != 0) |
| 776 |
{ |
| 777 |
extent++; |
| 778 |
} |
| 779 |
//Ordinal extent = aRowMap[nRows]; |
| 780 |
for (int i = 0; i < nFact; i++) |
| 781 |
{ |
| 782 |
Kokkos::parallel_for(RangePolicy(0, extent), ildltFunctor); |
| 783 |
} |
| 784 |
ExecSpace().fence(); |
| 785 |
|
| 786 |
double t = timer.seconds(); |
| 787 |
computeTime = t; |
| 788 |
|
| 789 |
Kokkos::deep_copy(diagElems_, diagElems); |
| 790 |
diagElemsInv_ = Kokkos::create_mirror_view(diagElemsInv); |
| 791 |
for (int i = 0; i < nRows; i++) |
| 792 |
{ |
| 793 |
diagElemsInv_[i] = one/diagElems_[i]; |
| 794 |
} |
| 795 |
Kokkos::deep_copy(diagElemsInv, diagElemsInv_); |
| 796 |
transposeL(); |
| 797 |
|
| 798 |
if (standard_sptrsv) { |
| 799 |
#if defined(KOKKOSKERNELS_ENABLE_TPL_CUSPARSE) |
| 800 |
KokkosSparse::Experimental::SPTRSVAlgorithm algo = KokkosSparse::Experimental::SPTRSVAlgorithm::SPTRSV_CUSPARSE; |
| 801 |
#else |
| 802 |
KokkosSparse::Experimental::SPTRSVAlgorithm algo = KokkosSparse::Experimental::SPTRSVAlgorithm::SEQLVLSCHD_TP1; |
| 803 |
#endif |
| 804 |
// setup L solve |
| 805 |
khL.create_sptrsv_handle(algo, nRows, true); |
| 806 |
#if defined(KOKKOSKERNELS_ENABLE_TPL_CUSPARSE) |
| 807 |
KokkosSparse::Experimental::sptrsv_symbolic(&khL, lRowMap, lColIdx, lVal); |
| 808 |
#else |
| 809 |
KokkosSparse::Experimental::sptrsv_symbolic(&khL, lRowMap, lColIdx); |
| 810 |
#endif |
| 811 |
// setup Lt solve |
| 812 |
khLt.create_sptrsv_handle(algo, nRows, false); |
| 813 |
#if defined(KOKKOSKERNELS_ENABLE_TPL_CUSPARSE) |
| 814 |
KokkosSparse::Experimental::sptrsv_symbolic(&khLt, ltRowMap, ltColIdx, ltVal); |
| 815 |
#else |
| 816 |
KokkosSparse::Experimental::sptrsv_symbolic(&khLt, ltRowMap, ltColIdx); |
| 817 |
#endif |
| 818 |
} |
| 819 |
} |
| 820 |
|
| 821 |
void apply(ScalarArray &x, ScalarArray &y) |
| 822 |
{ |
| 823 |
|
| 824 |
Kokkos::Timer timer; |
| 825 |
ParCopyFunctor<Ordinal, Scalar, ExecSpace> parCopyFunctor(xTemp, x); |
| 826 |
ExecSpace().fence(); |
| 827 |
Kokkos::parallel_for(RangePolicy(0, nRows), parCopyFunctor); |
| 828 |
ExecSpace().fence(); |
| 829 |
|
| 830 |
applyD(x, xTemp); |
| 831 |
if (standard_sptrsv) { |
| 832 |
KokkosSparse::Experimental::sptrsv_solve(&khL, lRowMap, lColIdx, lVal, xTemp, y); |
| 833 |
} else { |
| 834 |
applyLIC(xTemp, y); |
| 835 |
} |
| 836 |
applyDD(y, xTemp); |
| 837 |
if (standard_sptrsv) { |
| 838 |
KokkosSparse::Experimental::sptrsv_solve(&khLt, ltRowMap, ltColIdx, ltVal, xTemp, y); |
| 839 |
} else { |
| 840 |
applyLT(xTemp, y); |
| 841 |
} |
| 842 |
applyD(y, xTemp); |
| 843 |
|
| 844 |
ParCopyFunctor<Ordinal, Scalar, ExecSpace> parCopyFunctor2(y, xTemp); |
| 845 |
ExecSpace().fence(); |
| 846 |
Kokkos::parallel_for(RangePolicy(0, nRows), parCopyFunctor2); |
| 847 |
ExecSpace().fence(); |
| 848 |
|
| 849 |
double t = timer.seconds(); |
| 850 |
applyTime = t; |
| 851 |
return; |
| 852 |
} |
| 853 |
|
| 854 |
Ordinal getNFact() const |
| 855 |
{ |
| 856 |
return nFact; |
| 857 |
} |
| 858 |
|
| 859 |
std::string getSpTrsvType() const |
| 860 |
{ |
| 861 |
return (standard_sptrsv ? "Standard" : "Fast"); |
| 862 |
} |
| 863 |
|
| 864 |
Ordinal getNTrisol() const |
| 865 |
{ |
| 866 |
return nTrisol; |
| 867 |
} |
| 868 |
|
| 869 |
Ordinal getNRows() const |
| 870 |
{ |
| 871 |
return nRows; |
| 872 |
} |
| 873 |
|
| 874 |
double getComputeTime() const |
| 875 |
{ |
| 876 |
return computeTime; |
| 877 |
} |
| 878 |
|
| 879 |
double getInitializeTime() const |
| 880 |
{ |
| 881 |
return initTime; |
| 882 |
} |
| 883 |
|
| 884 |
double getApplyTime() const |
| 885 |
{ |
| 886 |
return applyTime; |
| 887 |
} |
| 888 |
void checkIC() const |
| 889 |
{ |
| 890 |
//Compute the L2 norm of the nonlinear residual (A - LLt) on sparsity pattern |
| 891 |
// |
| 892 |
Scalar sum = 0.0; |
| 893 |
for (int i = 0 ; i < nRows; i++) |
| 894 |
{ |
| 895 |
int row = i; |
| 896 |
// Each row in A matrix (result) |
| 897 |
for (int k = aRowMap[i]; k < aRowMap[i+1]; k++) |
| 898 |
{ |
| 899 |
|
| 900 |
int col = aColIdx[k]; |
| 901 |
|
| 902 |
if (row >= col) { |
| 903 |
Scalar acc_val = aVal[k]; |
| 904 |
Ordinal lptr, uptr; |
| 905 |
for ( lptr = lRowMap[i], uptr = lRowMap[aColIdx[k]] ; |
| 906 |
lptr < lRowMap[i+1] && uptr < lRowMap[aColIdx[k]+1] ; ) |
| 907 |
{ |
| 908 |
if (lColIdx[lptr] == lColIdx[uptr]) |
| 909 |
{ |
| 910 |
acc_val -= lVal[lptr] * diagElems[lColIdx[uptr]] * lVal[uptr]; |
| 911 |
lptr++; |
| 912 |
uptr++; |
| 913 |
} |
| 914 |
else if (lColIdx[lptr] < lColIdx[uptr]) |
| 915 |
lptr++; |
| 916 |
else |
| 917 |
uptr++; |
| 918 |
} |
| 919 |
|
| 920 |
sum += acc_val * acc_val; |
| 921 |
} |
| 922 |
} |
| 923 |
} |
| 924 |
std::cout << "l2 norm of nonlinear residual = " << RTS::sqrt(STS::abs(sum)) << std::endl; |
| 925 |
} |
| 926 |
friend class FastILDLFunctor<Ordinal, Scalar, ExecSpace>; |
| 927 |
friend class JacobiIterFunctor<Ordinal, Scalar, ExecSpace>; |
| 928 |
friend class ParCopyFunctor<Ordinal, Scalar, ExecSpace>; |
| 929 |
friend class JacobiIterFunctorT<Ordinal, Scalar, ExecSpace>; |
| 930 |
friend class ParScalFunctor<Ordinal, Scalar, Real, ExecSpace>; |
| 931 |
friend class MemoryPrimeFunctorN<Ordinal, Scalar, ExecSpace>; |
| 932 |
friend class MemoryPrimeFunctorNnzCoo<Ordinal, Scalar, ExecSpace>; |
| 933 |
friend class MemoryPrimeFunctorNnzCsr<Ordinal, Scalar, ExecSpace>; |
| 934 |
}; |
| 935 |
|
| 936 |
template<class Ordinal, class Scalar, class ExecSpace> |
| 937 |
class FastILDLFunctor |
| 938 |
{ |
| 939 |
public: |
| 940 |
typedef ExecSpace execution_space; |
| 941 |
typedef Kokkos::View<Ordinal *, ExecSpace> ordinal_array_type; |
| 942 |
typedef Kokkos::View<Scalar *, ExecSpace> scalar_array_type; |
| 943 |
|
| 944 |
FastILDLFunctor (Ordinal nNZ, Ordinal bs, ordinal_array_type Ap, ordinal_array_type Ai, |
| 945 |
ordinal_array_type Aj, scalar_array_type Ax, ordinal_array_type Lp, |
| 946 |
ordinal_array_type Li, scalar_array_type Lx, scalar_array_type diag, Scalar omega) |
| 947 |
: |
| 948 |
nnz(nNZ), blk_size(bs), |
| 949 |
_Ap(Ap), _Ai(Ai), _Aj(Aj), _Lp(Lp), _Li(Li), _Ax(Ax), _Lx(Lx), _diag(diag), _omega(omega) |
| 950 |
{ |
| 951 |
} |
| 952 |
|
| 953 |
KOKKOS_INLINE_FUNCTION |
| 954 |
void operator()(const Ordinal blk_index) const |
| 955 |
{ |
| 956 |
const Scalar zero = Kokkos::ArithTraits<Scalar>::zero(); |
| 957 |
const Scalar one = Kokkos::ArithTraits<Scalar>::one(); |
| 958 |
|
| 959 |
Ordinal start = blk_index * blk_size; |
| 960 |
Ordinal end = start + blk_size; |
| 961 |
|
| 962 |
Ordinal nz_index; |
| 963 |
|
| 964 |
if (end > nnz) |
| 965 |
{ |
| 966 |
end = nnz; |
| 967 |
} |
| 968 |
|
| 969 |
for (nz_index = start; nz_index < end && nz_index < nnz; nz_index++) |
| 970 |
{ |
| 971 |
Ordinal i = _Ai[nz_index]; |
| 972 |
Ordinal j = _Aj[nz_index]; |
| 973 |
Scalar val = _Ax[nz_index]; |
| 974 |
Scalar acc_val = zero; |
| 975 |
Scalar temp = zero; |
| 976 |
Ordinal lptr = _Lp[i]; |
| 977 |
Ordinal ltptr = _Lp[j]; |
| 978 |
Ordinal endpt = j; |
| 979 |
if (i >= j) { |
| 980 |
|
| 981 |
for ( ; _Li[lptr] < endpt && _Li[ltptr] < endpt; ) |
| 982 |
{ |
| 983 |
if (_Li[lptr] == _Li[ltptr]) |
| 984 |
{ |
| 985 |
acc_val += _Lx[lptr] * _diag[_Li[ltptr]] * _Lx[ltptr]; |
| 986 |
lptr++; |
| 987 |
ltptr++; |
| 988 |
} |
| 989 |
else if (_Li[lptr] < _Li[ltptr]) |
| 990 |
{ |
| 991 |
lptr++; |
| 992 |
} |
| 993 |
else |
| 994 |
{ |
| 995 |
ltptr++; |
| 996 |
} |
| 997 |
} |
| 998 |
if (i > j) |
| 999 |
{ |
| 1000 |
val = (val-acc_val) / _diag[j]; |
| 1001 |
for ( ; _Li[lptr] < j ; lptr++) ; // dummy loop |
| 1002 |
assert (_Li[lptr] == j); |
| 1003 |
// _Lx[lptr] = val; |
| 1004 |
_Lx[lptr] = ((one - _omega)*_Lx[lptr]) + (_omega * val); |
| 1005 |
} |
| 1006 |
else if (i == j) |
| 1007 |
{ |
| 1008 |
//_diag[j] = val - acc_val; |
| 1009 |
temp = val - acc_val; |
| 1010 |
_diag[j] = ((one - _omega) * _diag[j]) + (_omega * temp); |
| 1011 |
} |
| 1012 |
} |
| 1013 |
} |
| 1014 |
} |
| 1015 |
|
| 1016 |
Ordinal nnz, blk_size; |
| 1017 |
ordinal_array_type _Ap, _Ai, _Aj, _Lp, _Li; |
| 1018 |
scalar_array_type _Ax, _Lx, _diag; |
| 1019 |
Scalar _omega; |
| 1020 |
}; |
| 1021 |
#endif |