| 1 |
/*! \file MM-DissectionSolver.cpp |
| 2 |
\brief test rouinte of dissection solver reading Matrix Market format |
| 3 |
\author Atsushi Suzuki, Laboratoire Jacques-Louis Lions |
| 4 |
\date Jun. 20th 2014 |
| 5 |
\date Jul. 12th 2015 |
| 6 |
*/ |
| 7 |
|
| 8 |
// This file is part of Dissection |
| 9 |
// |
| 10 |
// Dissection is free software: you can redistribute it and/or modify |
| 11 |
// it under the terms of the GNU General Public License as published by |
| 12 |
// the Free Software Foundation, either version 3 of the License, or |
| 13 |
// (at your option) any later version. |
| 14 |
// |
| 15 |
// Linking Dissection statically or dynamically with other modules is making |
| 16 |
// a combined work based on Disssection. Thus, the terms and conditions of |
| 17 |
// the GNU General Public License cover the whole combination. |
| 18 |
// |
| 19 |
// As a special exception, the copyright holders of Dissection give you |
| 20 |
// permission to combine Dissection program with free software programs or |
| 21 |
// libraries that are released under the GNU LGPL and with independent modules |
| 22 |
// that communicate with Dissection solely through the Dissection-fortran |
| 23 |
// interface. You may copy and distribute such a system following the terms of |
| 24 |
// the GNU GPL for Dissection and the licenses of the other code concerned, |
| 25 |
// provided that you include the source code of that other code when and as |
| 26 |
// the GNU GPL requires distribution of source code and provided that you do |
| 27 |
// not modify the Dissection-fortran interface. |
| 28 |
// |
| 29 |
// Note that people who make modified versions of Dissection are not obligated |
| 30 |
// to grant this special exception for their modified versions; it is their |
| 31 |
// choice whether to do so. The GNU General Public License gives permission to |
| 32 |
// release a modified version without this exception; this exception also makes |
| 33 |
// it possible to release a modified version which carries forward this |
| 34 |
// exception. If you modify the Dissection-fortran interface, this exception |
| 35 |
// does not apply to your modified version of Dissection, and you must remove |
| 36 |
// this exception when you distribute your modified version. |
| 37 |
// |
| 38 |
// This exception is an additional permission under section 7 of the GNU |
| 39 |
// General Public License, version 3 ("GPLv3") |
| 40 |
// |
| 41 |
// Dissection is distributed in the hope that it will be useful, |
| 42 |
// but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 43 |
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 44 |
// GNU General Public License for more details. |
| 45 |
// |
| 46 |
// You should have received a copy of the GNU General Public License |
| 47 |
// along with Dissection. If not, see <http://www.gnu.org/licenses/>. |
| 48 |
// |
| 49 |
|
| 50 |
#include <cstdio> |
| 51 |
#include <cstdlib> |
| 52 |
#include <cstring> |
| 53 |
#include <fcntl.h> |
| 54 |
#include <unistd.h> |
| 55 |
#include <math.h> |
| 56 |
|
| 57 |
#include "Driver/DissectionSolver.hpp" |
| 58 |
#ifdef BLAS_MKL |
| 59 |
#include <mkl_service.h> |
| 60 |
#endif |
| 61 |
#include <pthread.h> |
| 62 |
#include <iostream> |
| 63 |
#include <sstream> |
| 64 |
#include <fstream> |
| 65 |
#include <complex> |
| 66 |
using namespace std; |
| 67 |
|
| 68 |
static int _stat = (-1); |
| 69 |
|
| 70 |
void *thread_child(void *arg) |
| 71 |
{ |
| 72 |
char buf[256]; |
| 73 |
int *pid = (int *)arg; |
| 74 |
unsigned int mem_tmp, mem_min, mem_max; |
| 75 |
double avg_mem; |
| 76 |
avg_mem = 0.0; |
| 77 |
mem_min = (1U << 31) - 1; |
| 78 |
mem_max = 0U; |
| 79 |
int stat0, stat1; |
| 80 |
stat0 = _stat; |
| 81 |
unsigned int count = 0U; |
| 82 |
// fprintf(stderr, "thread_child forked\n"); |
| 83 |
while(_stat != 0) { |
| 84 |
stat1 = _stat; |
| 85 |
if (stat1 == 1) { |
| 86 |
sprintf(buf, "/proc/%d/statm", *pid); |
| 87 |
ifstream fin(buf); |
| 88 |
fin >> mem_tmp; |
| 89 |
fin.close(); |
| 90 |
if (mem_tmp > mem_max) { |
| 91 |
mem_max = mem_tmp; |
| 92 |
} |
| 93 |
if (mem_tmp < mem_min) { |
| 94 |
mem_min = mem_tmp; |
| 95 |
} |
| 96 |
avg_mem += (double)mem_tmp; |
| 97 |
count++; |
| 98 |
} |
| 99 |
if ((stat1 == (-1)) && (stat0 == 1)) { |
| 100 |
fprintf(stderr, |
| 101 |
"used memory :min: %14.8e max: %14.8e avg: %14.8e count: %d\n", |
| 102 |
(double)mem_min * 4.0 / (1024.0 * 1024.0), |
| 103 |
(double)mem_max * 4.0 / (1024.0 * 1024.0), |
| 104 |
(avg_mem / (double)count) * 4.0 / (1024.0 * 1024.0), |
| 105 |
count); |
| 106 |
count = 0U; |
| 107 |
avg_mem = 0.0; |
| 108 |
mem_min = (1U << 31) - 1; |
| 109 |
mem_max = 0U; |
| 110 |
} |
| 111 |
stat0 = stat1; |
| 112 |
usleep(1000); |
| 113 |
} |
| 114 |
// fprintf(stderr, "thread_child join\n count = %ld\n", count); |
| 115 |
pthread_exit(arg); |
| 116 |
|
| 117 |
return (void *)NULL; |
| 118 |
} |
| 119 |
|
| 120 |
template<typename T> |
| 121 |
void generate_CSR(std::list<int>* ind_cols_tmp, std::list<T>* val_tmp, |
| 122 |
int nrow, int *nnz, int *mask, int *old2new, |
| 123 |
int *irow, int *jcol, T* val, bool symmetrize) |
| 124 |
{ |
| 125 |
const T zero(0.0); |
| 126 |
// ind_cols_tmp = new std::list<int>[nrow]; |
| 127 |
// val_tmp = new std::list<T>[nrow]; |
| 128 |
int nnz1 = *nnz; |
| 129 |
for (int i = 0; i < *nnz; i++) { |
| 130 |
const int i0 = irow[i]; |
| 131 |
const int j0 = jcol[i]; |
| 132 |
const int ii = old2new[i0]; |
| 133 |
const int jj = old2new[j0]; |
| 134 |
if ((mask[i0] != 1) || (mask[j0] != 1)) { |
| 135 |
// fprintf(stderr, "%d %d\n", i0, j0); |
| 136 |
nnz1--; |
| 137 |
continue; |
| 138 |
} |
| 139 |
// fprintf(stderr, "%d %d -> %d %d \n", i0, j0, ii, jj); |
| 140 |
if (ind_cols_tmp[ii].empty()) { |
| 141 |
ind_cols_tmp[ii].push_back(jj); |
| 142 |
val_tmp[ii].push_back(val[i]); |
| 143 |
} |
| 144 |
else { |
| 145 |
if (ind_cols_tmp[ii].back() < jj) { |
| 146 |
ind_cols_tmp[ii].push_back(jj); |
| 147 |
val_tmp[ii].push_back(val[i]); |
| 148 |
} |
| 149 |
else { |
| 150 |
typename std::list<T>::iterator iv = val_tmp[ii].begin(); |
| 151 |
std::list<int>::iterator it = ind_cols_tmp[ii].begin(); |
| 152 |
for ( ; it != ind_cols_tmp[ii].end(); ++it, ++iv) { |
| 153 |
if (*it == jj) { |
| 154 |
fprintf(stderr, "already exits? (%d %d)\n", ii, jj); |
| 155 |
break; |
| 156 |
} |
| 157 |
if (*it > jj) { |
| 158 |
ind_cols_tmp[ii].insert(it, jj); |
| 159 |
val_tmp[ii].insert(iv, val[i]); |
| 160 |
break; |
| 161 |
} |
| 162 |
} |
| 163 |
} |
| 164 |
} |
| 165 |
} |
| 166 |
// symmetrize |
| 167 |
if (symmetrize) { |
| 168 |
for (int i = 0; i < nrow; i++) { |
| 169 |
std::list<int>::iterator jt = ind_cols_tmp[i].begin(); |
| 170 |
for ( ; jt != ind_cols_tmp[i].end(); ++jt) { |
| 171 |
const int jj = (*jt); |
| 172 |
bool flag = false; |
| 173 |
std::list<int>::iterator it = ind_cols_tmp[jj].begin(); |
| 174 |
for ( ; it != ind_cols_tmp[jj].end(); ++it) { |
| 175 |
if ((*it) == i) { |
| 176 |
flag = true; |
| 177 |
// fprintf(stderr, "%d %d symmetric position found\n", i, jj); |
| 178 |
break; |
| 179 |
} |
| 180 |
} |
| 181 |
if (!flag) { |
| 182 |
if (ind_cols_tmp[jj].back() < i) { |
| 183 |
ind_cols_tmp[jj].push_back(i); |
| 184 |
val_tmp[jj].push_back(zero); |
| 185 |
fprintf(stderr, "%d %d added for symmetry\n", i, jj); |
| 186 |
nnz1++; |
| 187 |
} |
| 188 |
else { |
| 189 |
typename std::list<T>::iterator iv = val_tmp[jj].begin(); |
| 190 |
std::list<int>::iterator it = ind_cols_tmp[jj].begin(); |
| 191 |
for ( ; it != ind_cols_tmp[jj].end(); ++it, ++iv) { |
| 192 |
std::list<int>::iterator it1 = it; |
| 193 |
++it1; |
| 194 |
if (((*it)) < i && ((*it1) > i)) { |
| 195 |
ind_cols_tmp[jj].insert(it, i); |
| 196 |
val_tmp[jj].insert(iv, zero); |
| 197 |
nnz1++; |
| 198 |
// fprintf(stderr, "%d %d inserted for symmetry\n", i, jj); |
| 199 |
break; |
| 200 |
} |
| 201 |
} |
| 202 |
} |
| 203 |
} // if (!flag); |
| 204 |
} |
| 205 |
} |
| 206 |
} |
| 207 |
#if 0 |
| 208 |
{ |
| 209 |
for (int i = 0; i < nrow; i++) { |
| 210 |
std::list<int>::iterator jt = ind_cols_tmp[i].begin(); |
| 211 |
for ( ; jt != ind_cols_tmp[i].end(); ++jt) { |
| 212 |
const int jj = (*jt); |
| 213 |
bool flag = false; |
| 214 |
std::list<int>::iterator it = ind_cols_tmp[jj].begin(); |
| 215 |
for ( ; it != ind_cols_tmp[jj].end(); ++it) { |
| 216 |
if ((*it) == i) { |
| 217 |
flag = true; |
| 218 |
break; |
| 219 |
} |
| 220 |
} |
| 221 |
if (!flag) { |
| 222 |
fprintf(stderr, "%d %d position is not symmetric\n", i, jj); |
| 223 |
} |
| 224 |
} |
| 225 |
} |
| 226 |
} |
| 227 |
#endif |
| 228 |
*nnz = nnz1; |
| 229 |
} |
| 230 |
|
| 231 |
template |
| 232 |
void generate_CSR<double>(std::list<int>* ind_cols_tmp, std::list<double>* val_tmp, |
| 233 |
int nrow, int *nnz, int *mask, int *old2new, |
| 234 |
int *irow, int *jcol, double* val, bool symmetrize); |
| 235 |
template |
| 236 |
void generate_CSR<complex<double> >(std::list<int>* ind_cols_tmp, |
| 237 |
std::list<complex<double> >* val_tmp, |
| 238 |
int nrow, int *nnz, int *mask, int *old2new, |
| 239 |
int *irow, int *jcol, complex<double>* val, |
| 240 |
bool symmetrize); |
| 241 |
|
| 242 |
template<typename T> |
| 243 |
void copy_CSR(int *indcols, int *ptrows, T* coefs, int nrow, |
| 244 |
bool upper_flag, bool isSym, |
| 245 |
std::list<int>* ind_cols_tmp, std::list<T>* val_tmp) |
| 246 |
{ |
| 247 |
const T zero(0.0); |
| 248 |
ptrows[0] = 0; |
| 249 |
for (int i = 0; i < nrow; i++) { |
| 250 |
int k; |
| 251 |
int itmp = ind_cols_tmp[i].size(); |
| 252 |
if (upper_flag) { |
| 253 |
if (ind_cols_tmp[i].front() == i) { |
| 254 |
ptrows[i + 1] = ptrows[i] + itmp; |
| 255 |
k = ptrows[i]; |
| 256 |
} |
| 257 |
else { |
| 258 |
fprintf(stderr, "zero is added to diagonal : %d\n", i); |
| 259 |
ptrows[i + 1] = ptrows[i] + itmp + 1; |
| 260 |
indcols[ptrows[i]] = i; |
| 261 |
coefs[ptrows[i]] = zero; |
| 262 |
k = ptrows[i] + 1; |
| 263 |
} |
| 264 |
} |
| 265 |
else { |
| 266 |
k = ptrows[i]; |
| 267 |
if (ind_cols_tmp[i].back() == i || (!isSym)) { |
| 268 |
ptrows[i + 1] = ptrows[i] + itmp; |
| 269 |
} |
| 270 |
else { |
| 271 |
fprintf(stderr, "zero is added to diagonal : %d\n", i); |
| 272 |
ptrows[i + 1] = ptrows[i] + itmp + 1; |
| 273 |
indcols[ptrows[i + 1] - 1] = i; |
| 274 |
coefs[ptrows[i + 1] - 1] = zero; |
| 275 |
} |
| 276 |
} |
| 277 |
std::list<int>::iterator it = ind_cols_tmp[i].begin(); |
| 278 |
typename std::list<T>::iterator iv = val_tmp[i].begin(); |
| 279 |
for ( ; it != ind_cols_tmp[i].end(); ++it, ++iv, k++) { |
| 280 |
indcols[k] = *it; |
| 281 |
coefs[k] = *iv; |
| 282 |
} |
| 283 |
} // loop : i |
| 284 |
} |
| 285 |
|
| 286 |
template |
| 287 |
void copy_CSR<double>(int *indcols, int *ptrows, double* coefs, int nrow, |
| 288 |
bool upper_flag, bool isSym, |
| 289 |
std::list<int>* ind_cols_tmp, std::list<double>* val_tmp); |
| 290 |
|
| 291 |
template |
| 292 |
void copy_CSR<complex<double> >(int *indcols, int *ptrows, complex<double>* coefs, int nrow, |
| 293 |
bool upper_flag, bool isSym, |
| 294 |
std::list<int>* ind_cols_tmp, std::list<complex<double> >* val_tmp); |
| 295 |
|
| 296 |
int main(int argc, char **argv) |
| 297 |
{ |
| 298 |
int n, itmp, jtmp; |
| 299 |
char fname[256], fname1[256]; |
| 300 |
char buf[1024]; |
| 301 |
int nrow, nnz, flag; |
| 302 |
int *ptrows, *indcols; |
| 303 |
int *irow, *jcol; |
| 304 |
double *val, *coefs; |
| 305 |
complex<double> *valc, *ccoefs; |
| 306 |
int decomposer; |
| 307 |
int num_threads; |
| 308 |
int scaling = 1; |
| 309 |
double eps_pivot; |
| 310 |
int numlevels = -1; |
| 311 |
int minNodes = 128; |
| 312 |
std::list<int>* ind_cols_tmp; |
| 313 |
std::list<double>* val_tmp; |
| 314 |
std::list<complex<double> >* val_tmpc; |
| 315 |
FILE *fp; |
| 316 |
bool isSym, isComplex; |
| 317 |
bool upper_flag = true; |
| 318 |
bool isWhole = false; |
| 319 |
bool kernel_detection_all = false; |
| 320 |
int *indx_excl; |
| 321 |
int nexcl = 0; |
| 322 |
bool excl_flag = false; |
| 323 |
|
| 324 |
if (argc < 6) { |
| 325 |
fprintf(stderr, "MM-dissection [data file] [decomposer] [num_threads] [eps_pivot] [num_levels] [scaling] [kerner_detection_all] [upper_flag] [minNodes]\n"); |
| 326 |
exit(-1); |
| 327 |
} |
| 328 |
strcpy(fname, argv[1]); |
| 329 |
decomposer = atoi(argv[2]); |
| 330 |
num_threads = atoi(argv[3]); |
| 331 |
eps_pivot = atof(argv[4]); |
| 332 |
numlevels = atof(argv[5]); |
| 333 |
if (argc >= 7) { |
| 334 |
scaling = atoi(argv[6]); |
| 335 |
} |
| 336 |
if (argc >= 8) { |
| 337 |
kernel_detection_all = (atoi(argv[7]) == 1); |
| 338 |
} |
| 339 |
|
| 340 |
if (argc >= 9) { |
| 341 |
upper_flag = (atoi(argv[8]) == 1); |
| 342 |
isWhole = (atoi(argv[8]) == (-1)); |
| 343 |
} |
| 344 |
if (argc >= 10) { |
| 345 |
strcpy(fname1, argv[9]); |
| 346 |
excl_flag = true; |
| 347 |
} |
| 348 |
if (argc >= 11) { |
| 349 |
minNodes = atoi(argv[10]); |
| 350 |
} |
| 351 |
|
| 352 |
// read from the file |
| 353 |
if ((fp = fopen(fname, "r")) == NULL) { |
| 354 |
fprintf(stderr, "fail to open %s\n", fname); |
| 355 |
} |
| 356 |
fgets(buf, 256, fp); |
| 357 |
// |
| 358 |
if (strstr(buf, "symmetric") != NULL) { |
| 359 |
isSym = true; |
| 360 |
} |
| 361 |
else { |
| 362 |
isSym = false; |
| 363 |
upper_flag = false; |
| 364 |
} |
| 365 |
|
| 366 |
if (strstr(buf, "complex") != NULL) { |
| 367 |
isComplex = true; |
| 368 |
} |
| 369 |
else { |
| 370 |
isComplex = false; |
| 371 |
} |
| 372 |
|
| 373 |
fprintf(stderr, "symmetric = %s\n", isSym ? "true " : "false"); |
| 374 |
fprintf(stderr, "scaling = %d\n", scaling); |
| 375 |
fprintf(stderr, "upper = %s\n", upper_flag ? "true" : "false"); |
| 376 |
if (kernel_detection_all) { |
| 377 |
fprintf(stderr, "kernel detection is activated for all submatrices\n"); |
| 378 |
} |
| 379 |
if (excl_flag) { |
| 380 |
fprintf(stderr, "list of singular nodes %s\n", fname1); |
| 381 |
} |
| 382 |
while (1) { |
| 383 |
fgets(buf, 256, fp); |
| 384 |
if (buf[0] != '%') { |
| 385 |
sscanf(buf, "%d %d %d", &nrow, &itmp, &nnz); |
| 386 |
break; |
| 387 |
} |
| 388 |
} |
| 389 |
irow = new int[nnz]; |
| 390 |
jcol = new int[nnz]; |
| 391 |
|
| 392 |
if (isComplex) { |
| 393 |
double xreal, ximag; |
| 394 |
valc = new complex<double>[nnz]; |
| 395 |
if (upper_flag) { |
| 396 |
for (int i = 0; i < nnz; i++) { |
| 397 |
fscanf(fp, "%d %d %lf %lf", &jcol[i], &irow[i], &xreal, &ximag); |
| 398 |
valc[i] = complex<double>(xreal, ximag); |
| 399 |
irow[i]--; |
| 400 |
jcol[i]--; |
| 401 |
if (isSym && irow[i] > jcol[i]) { |
| 402 |
fprintf(stderr, "exchanged : %d > %d\n", irow[i], jcol[i]); |
| 403 |
itmp = irow[i]; |
| 404 |
irow[i] = jcol[i]; |
| 405 |
jcol[i] = itmp; |
| 406 |
} |
| 407 |
} |
| 408 |
} |
| 409 |
else { |
| 410 |
for (int i = 0; i < nnz; i++) { |
| 411 |
fscanf(fp, "%d %d %lf %lf", &irow[i], &jcol[i], &xreal, &ximag); |
| 412 |
valc[i] = complex<double>(xreal, ximag); |
| 413 |
irow[i]--; |
| 414 |
jcol[i]--; |
| 415 |
if (isSym && irow[i] < jcol[i]) { |
| 416 |
fprintf(stderr, "exchanged : %d > %d\n", irow[i], jcol[i]); |
| 417 |
itmp = irow[i]; |
| 418 |
irow[i] = jcol[i]; |
| 419 |
jcol[i] = itmp; |
| 420 |
} |
| 421 |
} |
| 422 |
} |
| 423 |
} |
| 424 |
else { |
| 425 |
val = new double[nnz]; |
| 426 |
if (upper_flag) { |
| 427 |
for (int i = 0; i < nnz; i++) { |
| 428 |
fscanf(fp, "%d %d %lf", &jcol[i], &irow[i], &val[i]); // read lower |
| 429 |
irow[i]--; |
| 430 |
jcol[i]--; |
| 431 |
if (isSym && irow[i] > jcol[i]) { |
| 432 |
fprintf(stderr, "exchanged : %d > %d\n", irow[i], jcol[i]); |
| 433 |
itmp = irow[i]; |
| 434 |
irow[i] = jcol[i]; |
| 435 |
jcol[i] = itmp; |
| 436 |
} |
| 437 |
} |
| 438 |
} |
| 439 |
else { |
| 440 |
for (int i = 0; i < nnz; i++) { |
| 441 |
fscanf(fp, "%d %d %lf", &irow[i], &jcol[i], &val[i]); // read lower |
| 442 |
irow[i]--; |
| 443 |
jcol[i]--; |
| 444 |
if (isSym && irow[i] < jcol[i]) { |
| 445 |
fprintf(stderr, "exchanged : %d > %d\n", irow[i], jcol[i]); |
| 446 |
itmp = irow[i]; |
| 447 |
irow[i] = jcol[i]; |
| 448 |
jcol[i] = itmp; |
| 449 |
} |
| 450 |
} |
| 451 |
} |
| 452 |
} |
| 453 |
fclose (fp); |
| 454 |
|
| 455 |
if (excl_flag) { |
| 456 |
if ((fp = fopen(fname1, "r")) == NULL) { |
| 457 |
fprintf(stderr, "fail to open %s\n", fname1); |
| 458 |
} |
| 459 |
fgets(buf, 256, fp); |
| 460 |
sscanf(buf, "# %d", &nexcl); |
| 461 |
indx_excl = new int[nexcl]; |
| 462 |
for (int i = 0; i < nexcl; i++) { |
| 463 |
fgets(buf, 256, fp); |
| 464 |
sscanf(buf, "%d", &itmp); |
| 465 |
indx_excl[i] = itmp; |
| 466 |
} |
| 467 |
fclose(fp); |
| 468 |
} |
| 469 |
|
| 470 |
int *mask = new int[nrow]; |
| 471 |
int *old2new = new int[nrow]; |
| 472 |
for (int i = 0; i < nrow; i++) { |
| 473 |
mask[i] = 1; |
| 474 |
} |
| 475 |
for (int i = 0; i < nexcl; i++) { |
| 476 |
mask[indx_excl[i]] = 0; |
| 477 |
} |
| 478 |
itmp = 0; |
| 479 |
jtmp = nrow - nexcl; |
| 480 |
for (int i = 0; i < nrow; i++) { |
| 481 |
if (mask[i] == 1) { |
| 482 |
old2new[i] = itmp++; |
| 483 |
} |
| 484 |
else { |
| 485 |
old2new[i] = jtmp++; |
| 486 |
} |
| 487 |
} |
| 488 |
#if 0 |
| 489 |
if ((fp = fopen("debug-index.data", "w")) != NULL) { |
| 490 |
for (int i = 0; i < nrow; i++) { |
| 491 |
fprintf(fp, "%d %d %d\n", i, old2new[i], mask[i]); |
| 492 |
} |
| 493 |
fclose(fp); |
| 494 |
} |
| 495 |
#endif |
| 496 |
nrow -= nexcl; |
| 497 |
|
| 498 |
ind_cols_tmp = new std::list<int>[nrow]; |
| 499 |
fprintf(stderr, "%s %d : getnerate_CSR\n", __FILE__, __LINE__); |
| 500 |
if (isComplex) { |
| 501 |
val_tmpc = new std::list<complex<double> >[nrow]; |
| 502 |
generate_CSR<complex<double> >(ind_cols_tmp, val_tmpc, |
| 503 |
nrow, &nnz, mask, old2new, |
| 504 |
irow, jcol, valc, (!isSym)); |
| 505 |
} |
| 506 |
else { |
| 507 |
val_tmp = new std::list<double>[nrow]; |
| 508 |
generate_CSR<double>(ind_cols_tmp, val_tmp, |
| 509 |
nrow, &nnz, mask, old2new, |
| 510 |
irow, jcol, val, (!isSym)); |
| 511 |
} |
| 512 |
delete [] irow; |
| 513 |
delete [] jcol; |
| 514 |
delete [] mask; |
| 515 |
delete [] old2new; |
| 516 |
if (isComplex) { |
| 517 |
delete [] valc; |
| 518 |
} |
| 519 |
else { |
| 520 |
delete [] val; |
| 521 |
} |
| 522 |
|
| 523 |
if (upper_flag) { |
| 524 |
for (int i = 0; i < nrow; i++) { |
| 525 |
if (ind_cols_tmp[i].front() != i) { |
| 526 |
nnz++; |
| 527 |
} |
| 528 |
} |
| 529 |
} |
| 530 |
else { |
| 531 |
for (int i = 0; i < nrow; i++) { |
| 532 |
if (ind_cols_tmp[i].back() != i) { |
| 533 |
nnz++; |
| 534 |
} |
| 535 |
} |
| 536 |
} |
| 537 |
fprintf(stderr, "%s %d : copy_CSR\n", __FILE__, __LINE__); |
| 538 |
ptrows = new int[nrow + 1]; |
| 539 |
indcols = new int[nnz]; |
| 540 |
if (isComplex) { |
| 541 |
ccoefs = new complex<double>[nnz]; |
| 542 |
copy_CSR<complex<double> >(indcols, ptrows, ccoefs, |
| 543 |
nrow, upper_flag, isSym, |
| 544 |
ind_cols_tmp, val_tmpc); |
| 545 |
} |
| 546 |
else { |
| 547 |
coefs = new double[nnz]; |
| 548 |
copy_CSR<double>(indcols, ptrows, coefs, |
| 549 |
nrow, upper_flag, isSym, |
| 550 |
ind_cols_tmp, val_tmp); |
| 551 |
} |
| 552 |
delete [] ind_cols_tmp; |
| 553 |
|
| 554 |
if (isComplex) { |
| 555 |
delete [] val_tmpc; |
| 556 |
} |
| 557 |
else { |
| 558 |
delete [] val_tmp; |
| 559 |
} |
| 560 |
#if 0 |
| 561 |
if ((fp = fopen("debug.matrix.data", "w")) != NULL) { |
| 562 |
for (int i = 0; i < nrow; i++) { |
| 563 |
fprintf(fp, "%d : %d :: ", i, (ptrows[i + 1] - ptrows[i])); |
| 564 |
for (int k = ptrows[i]; k < ptrows[i + 1]; k++) { |
| 565 |
fprintf(fp, "%d ", indcols[k]); |
| 566 |
} |
| 567 |
fprintf(fp, "\n"); |
| 568 |
} |
| 569 |
} |
| 570 |
fclose(fp); |
| 571 |
#endif |
| 572 |
int pid = (int)getpid(); |
| 573 |
#if 1 |
| 574 |
fprintf(stderr, "pid = %d\n", pid); |
| 575 |
sprintf(fname, "dissection.%04d.log", pid); |
| 576 |
fp = fopen(fname, "a"); |
| 577 |
#else |
| 578 |
fp = stderr; |
| 579 |
#endif |
| 580 |
for (int i = 0; i < argc; i++) { |
| 581 |
fprintf(fp, "%s ", argv[i]); |
| 582 |
} |
| 583 |
fprintf(fp, "\n"); |
| 584 |
fprintf(stderr, "%s %d : before pthread_create\n", __FILE__, __LINE__); |
| 585 |
void* results; |
| 586 |
pthread_attr_t th_attr; |
| 587 |
pthread_t thread; |
| 588 |
pthread_attr_init(&th_attr); |
| 589 |
pthread_attr_setdetachstate(&th_attr, PTHREAD_CREATE_JOINABLE); |
| 590 |
int pthid = pthread_create(&thread, &th_attr, |
| 591 |
&thread_child, |
| 592 |
(void *)&pid); |
| 593 |
if (pthid != 0) { |
| 594 |
cout << "bad thread creation ? " << pid << endl; |
| 595 |
exit(0); |
| 596 |
} |
| 597 |
fprintf(stderr, "%s %d : after pthread_create\n", __FILE__, __LINE__); |
| 598 |
|
| 599 |
if (isWhole) { |
| 600 |
isSym = true; |
| 601 |
upper_flag = false; |
| 602 |
} |
| 603 |
|
| 604 |
if (isComplex) { |
| 605 |
DissectionSolver<complex<double>, double>*dslv = |
| 606 |
new DissectionSolver<complex<double>, double>(num_threads, true, 0, fp); |
| 607 |
|
| 608 |
int called = 0; |
| 609 |
clock_t t0_cpu, t1_cpu, t2_cpu, t3_cpu, t4_cpu, t5_cpu; |
| 610 |
clock_t *tbegin_cpu, *tend_cpu; |
| 611 |
elapsed_t t0_elapsed, t1_elapsed, t2_elapsed, t3_elapsed, t4_elapsed, t5_elapsed; |
| 612 |
elapsed_t *tbegin_elapsed, *tend_elapsed; |
| 613 |
|
| 614 |
#ifdef BLAS_MKL |
| 615 |
mkl_set_num_threads(1); |
| 616 |
#endif |
| 617 |
#ifdef VECLIB |
| 618 |
setenv("VECLIB_MAXIMUM_THREADS", "1", true); |
| 619 |
#endif |
| 620 |
t0_cpu = clock(); |
| 621 |
get_realtime(&t0_elapsed); |
| 622 |
|
| 623 |
dslv->SymbolicFact(nrow, (int *)ptrows, (int *)indcols, |
| 624 |
isSym, |
| 625 |
upper_flag, |
| 626 |
isWhole, |
| 627 |
decomposer, numlevels, minNodes); |
| 628 |
|
| 629 |
t1_cpu = clock(); |
| 630 |
get_realtime(&t1_elapsed); |
| 631 |
|
| 632 |
t2_cpu = clock(); |
| 633 |
get_realtime(&t2_elapsed); |
| 634 |
_stat = 1; |
| 635 |
usleep(5000); |
| 636 |
dslv->NumericFact(0, (complex<double> *)ccoefs, scaling, |
| 637 |
eps_pivot, kernel_detection_all); |
| 638 |
_stat = (-1); |
| 639 |
t3_cpu = clock(); |
| 640 |
get_realtime(&t3_elapsed); |
| 641 |
usleep(5000); |
| 642 |
fprintf(stderr, "%s %d : NumericFact() done\n", __FILE__, __LINE__); |
| 643 |
|
| 644 |
int n0; |
| 645 |
n0 = dslv->kern_dimension(); |
| 646 |
fprintf(fp, "%s %d : ## kernel dimension = %d\n", __FILE__, __LINE__, n0); |
| 647 |
|
| 648 |
|
| 649 |
int nrhs = 40; |
| 650 |
complex<double> *x = new complex<double>[nrow]; |
| 651 |
complex<double> *y = new complex<double>[nrow * nrhs]; |
| 652 |
complex<double> *z = new complex<double>[nrow * nrhs]; |
| 653 |
|
| 654 |
int ntrial = 24; |
| 655 |
tbegin_cpu = new clock_t[ntrial]; |
| 656 |
tend_cpu = new clock_t[ntrial]; |
| 657 |
tbegin_elapsed = new elapsed_t[ntrial]; |
| 658 |
tend_elapsed = new elapsed_t[ntrial]; |
| 659 |
fprintf(stderr, "%s %d : RHS allocated\n", __FILE__, __LINE__); |
| 660 |
_stat = (-1); |
| 661 |
|
| 662 |
for (int k = 0; k < nrhs; k++) { |
| 663 |
const int kshft = k * nrow; |
| 664 |
for (int i = 0; i < nrow; i++) { |
| 665 |
y[kshft + i] = complex<double>((double)((i + k) % 11), 0.0); |
| 666 |
} |
| 667 |
dslv->SpMV(y + kshft, x); |
| 668 |
|
| 669 |
dslv->SpMV(x, y + kshft); |
| 670 |
for (int i = 0; i < nrow * nrhs; i++) { |
| 671 |
z[i] = y[i]; |
| 672 |
} |
| 673 |
} |
| 674 |
_stat = 1; |
| 675 |
for (int k = 1; k <= 20; k++) { |
| 676 |
for (int i = 0; i < nrow * nrhs; i++) { |
| 677 |
y[i] = z[i]; |
| 678 |
} |
| 679 |
tbegin_cpu[k - 1] = clock(); |
| 680 |
get_realtime(&tbegin_elapsed[k - 1]); |
| 681 |
if (k == 1) { |
| 682 |
fprintf(stderr, "%s %d : SolveSingle()\n", __FILE__, __LINE__); |
| 683 |
dslv->SolveSingle(y, false, false, true); |
| 684 |
fprintf(stderr, "%s %d : SolveSingle()\n", __FILE__, __LINE__); |
| 685 |
} |
| 686 |
else { |
| 687 |
const int nnrhs = k; |
| 688 |
fprintf(stderr, "%s %d : SolveMulti() %d\n", __FILE__, __LINE__, nnrhs); |
| 689 |
dslv->SolveMulti(y, nnrhs, false, false, true); |
| 690 |
fprintf(stderr, "%s %d : SolveMulti() %d\n", __FILE__, __LINE__, nnrhs); |
| 691 |
} |
| 692 |
tend_cpu[k - 1] = clock(); |
| 693 |
get_realtime(&tend_elapsed[k - 1]); |
| 694 |
} |
| 695 |
for (int k = 5; k < 9; k++) { |
| 696 |
for (int i = 0; i < nrow * nrhs; i++) { |
| 697 |
y[i] = z[i]; |
| 698 |
} |
| 699 |
tbegin_cpu[k + 15] = clock(); |
| 700 |
get_realtime(&tbegin_elapsed[k + 15]); |
| 701 |
const int nnrhs = k * 5; |
| 702 |
fprintf(stderr, "%s %d : SolveMulti() %d\n", __FILE__, __LINE__, nnrhs); |
| 703 |
dslv->SolveMulti(y, nnrhs, false, false, true); |
| 704 |
fprintf(stderr, "%s %d : SolveMulti() %d\n", __FILE__, __LINE__, nnrhs); |
| 705 |
tend_cpu[k + 15] = clock(); |
| 706 |
get_realtime(&tend_elapsed[k + 15]); |
| 707 |
} |
| 708 |
_stat = (-1); |
| 709 |
|
| 710 |
#if 1 |
| 711 |
for (int k = 0; k < nrhs; k++) { |
| 712 |
const int kshft = k * nrow; |
| 713 |
for (int i = 0; i < nrow; i++) { |
| 714 |
z[i] = complex<double>((double)((i + k) % 11), 0.0); |
| 715 |
} |
| 716 |
dslv->SpMV(z, x); |
| 717 |
dslv->SpMV(x, z); |
| 718 |
|
| 719 |
double norm0, norm1; |
| 720 |
norm0 = 0.0; |
| 721 |
norm1 = 0.0; |
| 722 |
for (int i = 0; i < nrow; i++) { |
| 723 |
norm0 += x[i].real() * x[i].real() + x[i].imag() * x[i].imag(); |
| 724 |
complex<double> ztmp; |
| 725 |
ztmp = y[i + kshft] - x[i]; |
| 726 |
norm1 += ztmp.real() * ztmp.real() + ztmp.imag() * ztmp.imag(); |
| 727 |
} |
| 728 |
fprintf(fp, "## %2d : error : %18.7e = %18.7e / %18.7e\n", k, |
| 729 |
sqrt(norm1 / norm0), sqrt(norm1), sqrt(norm0)); |
| 730 |
dslv->SpMV(y + kshft, x); |
| 731 |
|
| 732 |
norm0 = 0.0; |
| 733 |
norm1 = 0.0; |
| 734 |
for (int i = 0; i < nrow; i++) { |
| 735 |
norm0 += z[i].real() * z[i].real() + z[i].imag() * z[i].imag(); |
| 736 |
complex<double> ztmp; |
| 737 |
ztmp = z[i] - x[i]; |
| 738 |
norm1 += ztmp.real() * ztmp.real() + ztmp.imag() * ztmp.imag(); |
| 739 |
} |
| 740 |
fprintf(fp, "## %2d : residual : %18.7e = %18.7e / %18.7e\n", k, |
| 741 |
sqrt(norm1 / norm0), sqrt(norm1), sqrt(norm0)); |
| 742 |
} |
| 743 |
#endif |
| 744 |
|
| 745 |
delete dslv; |
| 746 |
delete [] ptrows; |
| 747 |
delete [] indcols; |
| 748 |
delete [] ccoefs; |
| 749 |
delete [] x; |
| 750 |
delete [] y; |
| 751 |
delete [] z; |
| 752 |
} // if (isComplex) |
| 753 |
else { |
| 754 |
DissectionSolver<double> *dslv = |
| 755 |
new DissectionSolver<double>(num_threads, true, 0, fp); |
| 756 |
|
| 757 |
int called = 0; |
| 758 |
clock_t t0_cpu, t1_cpu, t2_cpu, t3_cpu, t4_cpu, t5_cpu; |
| 759 |
clock_t *tbegin_cpu, *tend_cpu; |
| 760 |
elapsed_t t0_elapsed, t1_elapsed, t2_elapsed, t3_elapsed, t4_elapsed, t5_elapsed; |
| 761 |
elapsed_t *tbegin_elapsed, *tend_elapsed; |
| 762 |
#ifdef BLAS_MKL |
| 763 |
mkl_set_num_threads(1); |
| 764 |
#endif |
| 765 |
#ifdef VECLIB |
| 766 |
setenv("VECLIB_MAXIMUM_THREADS", "1", true); |
| 767 |
#endif |
| 768 |
t0_cpu = clock(); |
| 769 |
get_realtime(&t0_elapsed); |
| 770 |
|
| 771 |
dslv->SymbolicFact(nrow, (int *)ptrows, (int *)indcols, |
| 772 |
isSym, |
| 773 |
upper_flag, |
| 774 |
isWhole, |
| 775 |
decomposer, numlevels, minNodes); |
| 776 |
// sym, upper |
| 777 |
// dslv->SaveMMMatrix(0, coefs); |
| 778 |
// exit(-1); |
| 779 |
#if 0 |
| 780 |
fprintf(stderr, "%d %d\n", nnz, ptrows[nrow]); |
| 781 |
FORTRAN_DECL(csrmatrix_save)(0, |
| 782 |
0, |
| 783 |
0, // is_sym |
| 784 |
1, // real_or_cmplx |
| 785 |
nrow, |
| 786 |
ptrows[nrow], |
| 787 |
ptrows, |
| 788 |
indcols, |
| 789 |
coefs); |
| 790 |
exit(-1); |
| 791 |
#endif |
| 792 |
t1_cpu = clock(); |
| 793 |
get_realtime(&t1_elapsed); |
| 794 |
|
| 795 |
_stat = 1; |
| 796 |
usleep(5000); |
| 797 |
t2_cpu = clock(); |
| 798 |
get_realtime(&t2_elapsed); |
| 799 |
dslv->NumericFact(0, (double *)coefs, scaling, |
| 800 |
eps_pivot, kernel_detection_all); |
| 801 |
t3_cpu = clock(); |
| 802 |
get_realtime(&t3_elapsed); |
| 803 |
_stat = (-1); |
| 804 |
usleep(5000); |
| 805 |
fprintf(stderr, "%s %d : NumericFact() done\n", __FILE__, __LINE__); |
| 806 |
int n0; |
| 807 |
n0 = dslv->kern_dimension(); |
| 808 |
fprintf(fp, "%s %d : ## kernel dimension = %d\n", __FILE__, __LINE__, n0); |
| 809 |
|
| 810 |
_stat = 1; |
| 811 |
|
| 812 |
int nrhs = 40; |
| 813 |
double *x = new double[nrow]; |
| 814 |
double *y = new double[nrow * nrhs]; |
| 815 |
double *z = new double[nrow * nrhs]; |
| 816 |
int ntrial = 24; |
| 817 |
tbegin_cpu = new clock_t[ntrial]; |
| 818 |
tend_cpu = new clock_t[ntrial]; |
| 819 |
tbegin_elapsed = new elapsed_t[ntrial]; |
| 820 |
tend_elapsed = new elapsed_t[ntrial]; |
| 821 |
fprintf(stderr, "%s %d : RHS allocated\n", __FILE__, __LINE__); |
| 822 |
_stat = (-1); |
| 823 |
|
| 824 |
if (!isSym && (n0 > 0)) { |
| 825 |
_stat = 1; |
| 826 |
fprintf(stderr, "%s %d : ComputeTransposedKernels()\n", __FILE__, __LINE__); |
| 827 |
dslv->ComputeTransposedKernels(); |
| 828 |
fprintf(stderr, "%s %d : ComputeTransposedKernels()\n", __FILE__, __LINE__); |
| 829 |
_stat = (-1); |
| 830 |
} |
| 831 |
|
| 832 |
for (int k = 0; k < nrhs; k++) { |
| 833 |
const int kshft = k * nrow; |
| 834 |
for (int i = 0; i < nrow; i++) { |
| 835 |
y[kshft + i] = (double)((i + k) % 11); |
| 836 |
} |
| 837 |
dslv->SpMV(y + kshft, x); |
| 838 |
if (n0 > 0) { |
| 839 |
dslv->ProjectionImageSingle(x); |
| 840 |
} |
| 841 |
dslv->SpMV(x, y + kshft); |
| 842 |
for (int i = 0; i < nrow * nrhs; i++) { |
| 843 |
z[i] = y[i]; |
| 844 |
} |
| 845 |
} |
| 846 |
_stat = 1; |
| 847 |
for (int k = 1; k <= 20; k++) { |
| 848 |
for (int i = 0; i < nrow * nrhs; i++) { |
| 849 |
y[i] = z[i]; |
| 850 |
} |
| 851 |
tbegin_cpu[k - 1] = clock(); |
| 852 |
get_realtime(&tbegin_elapsed[k - 1]); |
| 853 |
if (k == 1) { |
| 854 |
fprintf(stderr, "%s %d : SolveSingle()\n", __FILE__, __LINE__); |
| 855 |
dslv->SolveSingle(y, false, false, true); |
| 856 |
fprintf(stderr, "%s %d : SolveSingle()\n", __FILE__, __LINE__); |
| 857 |
} |
| 858 |
else { |
| 859 |
const int nnrhs = k; |
| 860 |
fprintf(stderr, "%s %d : SolveMulti() %d\n", __FILE__, __LINE__, nnrhs); |
| 861 |
dslv->SolveMulti(y, nnrhs, false, false, true); |
| 862 |
fprintf(stderr, "%s %d : SolveMulti() %d\n", __FILE__, __LINE__, nnrhs); |
| 863 |
} |
| 864 |
if (n0 > 0) { |
| 865 |
dslv->ProjectionImageSingle(y); |
| 866 |
} |
| 867 |
tend_cpu[k - 1] = clock(); |
| 868 |
get_realtime(&tend_elapsed[k - 1]); |
| 869 |
} |
| 870 |
for (int k = 5; k < 9; k++) { |
| 871 |
for (int i = 0; i < nrow * nrhs; i++) { |
| 872 |
y[i] = z[i]; |
| 873 |
} |
| 874 |
tbegin_cpu[k + 15] = clock(); |
| 875 |
get_realtime(&tbegin_elapsed[k + 15]); |
| 876 |
const int nnrhs = k * 5; |
| 877 |
fprintf(stderr, "%s %d : SolveMulti() %d\n", __FILE__, __LINE__, nnrhs); |
| 878 |
dslv->SolveMulti(y, nnrhs, false, false, true); |
| 879 |
fprintf(stderr, "%s %d : SolveMulti() %d\n", __FILE__, __LINE__, nnrhs); |
| 880 |
if (n0 > 0) { |
| 881 |
dslv->ProjectionImageSingle(y); |
| 882 |
} |
| 883 |
tend_cpu[k + 15] = clock(); |
| 884 |
get_realtime(&tend_elapsed[k + 15]); |
| 885 |
|
| 886 |
} |
| 887 |
_stat = (-1); |
| 888 |
#if 1 |
| 889 |
for (int k = 0; k < nrhs; k++) { |
| 890 |
const int kshft = k * nrow; |
| 891 |
for (int i = 0; i < nrow; i++) { |
| 892 |
z[i] = (double)((i + k) % 11); |
| 893 |
} |
| 894 |
dslv->SpMV(z, x); |
| 895 |
dslv->SpMV(x, z); |
| 896 |
|
| 897 |
double norm0, norm1; |
| 898 |
norm0 = 0.0; |
| 899 |
norm1 = 0.0; |
| 900 |
for (int i = 0; i < nrow; i++) { |
| 901 |
norm0 += x[i] * x[i]; |
| 902 |
double ztmp; |
| 903 |
ztmp = y[i + kshft] - x[i]; |
| 904 |
norm1 += ztmp * ztmp; |
| 905 |
} |
| 906 |
fprintf(fp, "## %2d : error : %18.7e = %18.7e / %18.7e\n", k, |
| 907 |
sqrt(norm1 / norm0), sqrt(norm1), sqrt(norm0)); |
| 908 |
dslv->SpMV(y + kshft, x); |
| 909 |
|
| 910 |
norm0 = 0.0; |
| 911 |
norm1 = 0.0; |
| 912 |
for (int i = 0; i < nrow; i++) { |
| 913 |
norm0 += z[i] * z[i]; |
| 914 |
double ztmp; |
| 915 |
ztmp = z[i] - x[i]; |
| 916 |
norm1 += ztmp * ztmp; |
| 917 |
} |
| 918 |
fprintf(fp, "## %2d : residual : %18.7e = %18.7e / %18.7e\n", k, |
| 919 |
sqrt(norm1 / norm0), sqrt(norm1), sqrt(norm0)); |
| 920 |
} |
| 921 |
|
| 922 |
usleep(5000); |
| 923 |
t4_cpu = clock(); |
| 924 |
get_realtime(&t4_elapsed); |
| 925 |
dslv->SolveSingle(y, false, true, true); // with projection : true |
| 926 |
// dslv->ProjectionImageSingle(y); |
| 927 |
t5_cpu = clock(); |
| 928 |
get_realtime(&t5_elapsed); |
| 929 |
_stat = (-1); |
| 930 |
usleep(5000); |
| 931 |
fprintf(stderr, "%s %d : SolveSingle() done\n", __FILE__, __LINE__); |
| 932 |
#endif |
| 933 |
|
| 934 |
delete dslv; |
| 935 |
delete [] ptrows; |
| 936 |
delete [] indcols; |
| 937 |
delete [] coefs; |
| 938 |
delete [] x; |
| 939 |
delete [] y; |
| 940 |
delete [] z; |
| 941 |
} |
| 942 |
fclose(fp); |
| 943 |
|
| 944 |
} |