/*! \file C_BlasRoutines.cpp \brief factorization routines LDL^t, LDU, forward/backward substitution \author François-Xavier Roux, ONERA, Laboratoire Jacques-Louis Lions \author Atsushi Suzuki, Laboratoire Jacques-Louis Lions \date Jul. 12th 2015 \date Nov. 30th 2016 */ // This file is part of Dissection // // Dissection is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // Linking Dissection statically or dynamically with other modules is making // a combined work based on Disssection. Thus, the terms and conditions of // the GNU General Public License cover the whole combination. // // As a special exception, the copyright holders of Dissection give you // permission to combine Dissection program with free software programs or // libraries that are released under the GNU LGPL and with independent modules // that communicate with Dissection solely through the Dissection-fortran // interface. You may copy and distribute such a system following the terms of // the GNU GPL for Dissection and the licenses of the other code concerned, // provided that you include the source code of that other code when and as // the GNU GPL requires distribution of source code and provided that you do // not modify the Dissection-fortran interface. // // Note that people who make modified versions of Dissection are not obligated // to grant this special exception for their modified versions; it is their // choice whether to do so. The GNU General Public License gives permission to // release a modified version without this exception; this exception also makes // it possible to release a modified version which carries forward this // exception. If you modify the Dissection-fortran interface, this exception // does not apply to your modified version of Dissection, and you must remove // this exception when you distribute your modified version. // // This exception is an additional permission under section 7 of the GNU // General Public License, version 3 ("GPLv3") // // Dissection is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with Dissection. If not, see . // #include "Driver/C_BlasRoutines.hpp" #include "Driver/C_threads_tasks.hpp" #include "Algebra/VectorArray.hpp" // T may be complex of U that is higher precision than Z template Z matrix_infty_norm_(const int n, T *a, const int lda) { Z err(0.0); for (int i = 0; i < n; i++) { U err_tmp1(0.0); for (int j = 0; j < n; j++) { err_tmp1 += blas_abs(a[i + j * lda]); } Z err_tmp0 = conv_prec(err_tmp1); // accuracy conversion : U to Z err = err > err_tmp0 ? err : err_tmp0; } return err; } template double matrix_infty_norm_(const int n, quadruple *a, const int lda); #ifndef NO_OCTRUPLE template quadruple matrix_infty_norm_(const int n, octruple *a, const int lda); #endif template float matrix_infty_norm_(const int n, double *a, const int lda); template double matrix_infty_norm_, quadruple, double>(const int n, complex *a, const int lda); template float matrix_infty_norm_, double, float>(const int n, complex *a, const int lda); #ifndef NO_OCTRUPLE template quadruple matrix_infty_norm_, octruple, quadruple>(const int n, complex *a, const int lda); #endif // template U matrix_infty_norm(const int n, T *a, const int lda) { fprintf(stderr, "%s %d : general template is not implemented\n", __FILE__, __LINE__); return U(0.0); } template<> double matrix_infty_norm(const int n, quadruple *a, const int lda) { return matrix_infty_norm_(n, a, lda); } template<> float matrix_infty_norm(const int n, double *a, const int lda) { return matrix_infty_norm_(n, a, lda); } #ifndef NO_OCTRUPLE template<> quadruple matrix_infty_norm(const int n, octruple *a, const int lda) { return matrix_infty_norm_(n, a, lda); } #endif template<> double matrix_infty_norm, double>(const int n, complex *a, const int lda) { return matrix_infty_norm_, quadruple, double>(n, a, lda); } template<> float matrix_infty_norm, float>(const int n, complex *a, const int lda) { return matrix_infty_norm_, double, float>(n, a, lda); } #ifndef NO_OCTRUPLE template<> quadruple matrix_infty_norm, quadruple>(const int n, complex *a, const int lda) { return matrix_infty_norm_, octruple, quadruple>(n, a, lda); } #endif // #define LDLT_LOWER // *pivot is dealt as "double", which give no defect on accuracy, because // magnitude is important in the value of pivot template bool full_ldlt_permute(int *nn0, const int n0, const int n, T *a, const int lda, double *pivot, int *permute, const double eps, double *fop) { const int lda1 = lda + 1; T alpha; const T one(1.0); const T none(-1.0); bool flag = true; VectorArray col_k(n); VectorArray col_km(n); alpha = none; for (int i = 0; i < n; i++) { permute[i] = i; } const int n1 = n - n0; int k = 0; while (k < n1) { int km = k; double vmax = 0.0; km = blas_iamax((n - k), a + (k * lda1), lda1) + k; vmax = blas_abs(a[km * lda1]); if (vmax < (*pivot * eps)) { // enough in doule precision flag = false; break; } *pivot = vmax; if (km > k) { int itmp = permute[km]; permute[km] = permute[k]; permute[k] = itmp; // swap row/column #ifdef LDLT_LOWER swap_sym_lower(n, a, lda, k, km, col_k.addrCoefs(), col_km.addrCoefs()); #else swap_sym_upper(n, a, lda, k, km, col_k.addrCoefs(), col_km.addrCoefs()); #endif } const T d = one / a[k * lda1]; a[k * lda1] = d; if (k == (n1 - 1)) { k++; break; } alpha = (-d); #ifdef LDLT_LOWER blas_syr(CblasLower, (n - k - 1), alpha, &a[k * lda1 + 1], 1, &a[(k + 1) * lda1], lda); #else blas_syr(CblasUpper, (n - k - 1), alpha, &a[k * lda1 + lda], lda, &a[(k + 1) * lda1], lda); #endif *fop += 3.0 * (double)(n - k - 1) * (double)(n - k) / 2.0; #ifdef LDLT_LOWER blas_scal((n - k - 1), d, &a[k * lda1 + 1], 1); #else blas_scal((n - k - 1), d, &a[k * lda1 + lda], lda); #endif *fop += double(n - k - 1); k++; } // while (k < n1) // symmetrize the results : copy lower to upper #ifdef LDLT_LOWER for (int j = 1; j < n1; j++) { for (int i = 0; i < j; i++) { a[i + j * lda] = a[j + i * lda]; } } #else for (int j = 1; j < n1; j++) { for (int i = 0; i < j; i++) { a[j + i * lda] = a[i + j * lda]; } } #endif *nn0 = n - k; return flag; } template bool full_ldlt_permute(int *nn0, const int n0, const int n, double *a, const int lda, double *pivot, int *permute, const double eps, double *fop); template bool full_ldlt_permute, double>(int *nn0, const int n0, const int n, complex *a, const int lda, double *pivot, int *permute, const double eps, double *fop); template bool full_ldlt_permute(int *nn0, const int n0, const int n, float *a, const int lda, double *pivot, int *permute, const double eps, double *fop); template bool full_ldlt_permute, float>(int *nn0, const int n0, const int n, complex *a, const int lda, double *pivot, int *permute, const double eps, double *fop); template bool full_ldlt_permute(int *nn0, const int n0, const int n, quadruple *a, const int lda, double *pivot, int *permute, const double eps, double *fop); template bool full_ldlt_permute, quadruple>(int *nn0, const int n0, const int n, complex *a, const int lda, double *pivot, int *permute, const double eps, double *fop); template bool full_ldlt_permute(int *nn0, const int n0, const int n, octruple *a, const int lda, double *pivot, int *permute, const double eps, double *fop); template bool full_ldlt_permute, octruple>(int *nn0, const int n0, const int n, complex *a, const int lda, double *pivot, int *permute, const double eps, double *fop); // template bool full_ldu_permute(int *nn0, const int n0, const int n, T *a, const int lda, double *pivot, int *permute, const double eps, double *fop) { const int lda1 = lda + 1; // T *col_k, *col_km; const T one(1.0); const T none(-1.0); bool flag = true; VectorArray col_k(n); VectorArray col_km(n); for (int i = 0; i < n; i++) { permute[i] = i; } const int n1 = n - n0; int k = 0; while (k < n1) { int km = k; double vmax = 0.0; km = blas_iamax((n - k), a + (k * lda1), lda1) + k; vmax = blas_abs(a[km * lda1]); if (vmax < (*pivot * eps)) { flag = false; break; } *pivot = vmax; if (km > k) { int itmp = permute[km]; permute[km] = permute[k]; permute[k] = itmp; // swap row/column swap_unsym(n, a, lda, k, km, col_k.addrCoefs(), col_km.addrCoefs()); } const T d = one / a[k * lda1]; a[k * lda1] = d; if (k == (n1 - 1)) { k++; break; } // alpha = none; blas_scal((n - k - 1), d, &a[k * lda1 + lda], lda); blas_ger((n - k - 1), (n - k - 1), none, &a[k * lda1 + 1], 1, &a[k * lda1 + lda], lda, &a[(k + 1) * lda1], lda); *fop += 3.0 * (double)(n - k - 1) * (double)(n - k) / 2.0; blas_scal((n - k - 1), d, &a[k * lda1 + 1], 1); *fop += 2.0 * double(n - k - 1); k++; } // while (k < n1) *nn0 = n - k; return flag; } template bool full_ldu_permute(int *nn0, const int n0, const int n, double *a, const int lda, double *pivot, int *permute, const double eps, double *fop); template bool full_ldu_permute, double>(int *nn0, const int n0, const int n, complex *a, const int lda, double *pivot, int *permute, const double eps, double *fop); template bool full_ldu_permute(int *nn0, const int n0, const int n, float *a, const int lda, double *pivot, int *permute, const double eps, double *fop); template bool full_ldu_permute, float>(int *nn0, const int n0, const int n, complex *a, const int lda, double *pivot, int *permute, const double eps, double *fop); template bool full_ldu_permute(int *nn0, const int n0, const int n, quadruple *a, const int lda, double *pivot, int *permute, const double eps, double *fop); template bool full_ldu_permute, quadruple>(int *nn0, const int n0, const int n, complex *a, const int lda, double *pivot, int *permute, const double eps, double *fop); template bool full_ldu_permute(int *nn0, const int n0, const int n, octruple *a, const int lda, double *pivot, int *permute, const double eps, double *fop); template bool full_ldu_permute, octruple>(int *nn0, const int n0, const int n, complex *a, const int lda, double *pivot, int *permute, const double eps, double *fop); // template bool ldu_full_permute(int *nn0, const int n0, const int n, T *a, const int lda, double *pivot, int *permute_right, int *permute_left, const double eps, double *fop) { const int lda1 = lda + 1; // T *col_k, *col_km; const T one(1.0); const T none(-1.0); bool flag = true; VectorArray coli(n); VectorArray colj(n); for (int i = 0; i < n; i++) { permute_right[i] = i; permute_left[i] = i; } const int n1 = n - n0; int k = 0; while (k < n1) { int ki, kj, itmp; double vmax = 0.0; for (int i = k; i < n; i++) { for (int j = k; j < n; j++) { double tmp = blas_abs(a[i + j * lda]); if (tmp > vmax) { vmax = tmp; ki = i; kj = j; } } } if (vmax < (*pivot * eps)) { flag = false; break; } *pivot = vmax; if (kj > k) { itmp = permute_right[kj]; permute_right[kj] = permute_right[k]; permute_right[k] = itmp; } if (ki > k) { itmp = permute_left[ki]; permute_left[ki] = permute_left[k]; permute_left[k] = itmp; } // swap row/column swap_full(n, a, lda, k, ki, kj, coli.addrCoefs(), colj.addrCoefs()); const T d = one / a[k * lda1]; a[k * lda1] = d; if (k == (n1 - 1)) { k++; break; } // alpha = none; blas_scal((n - k - 1), d, &a[k * lda1 + lda], lda); blas_ger((n - k - 1), (n - k - 1), none, &a[k * lda1 + 1], 1, &a[k * lda1 + lda], lda, &a[(k + 1) * lda1], lda); *fop += 3.0 * (double)(n - k - 1) * (double)(n - k) / 2.0; blas_scal((n - k - 1), d, &a[k * lda1 + 1], 1); *fop += 2.0 * double(n - k - 1); k++; } // while (k < n1) *nn0 = n - k; return flag; } template bool ldu_full_permute(int *nn0, const int n0, const int n, double *a, const int lda, double *pivot, int *permute_right, int *permute_left, const double eps, double *fop); template bool ldu_full_permute, double>(int *nn0, const int n0, const int n, complex *a, const int lda, double *pivot, int *permute_right, int *permute_left, const double eps, double *fop); template bool ldu_full_permute(int *nn0, const int n0, const int n, quadruple *a, const int lda, double *pivot, int *permute_right, int *permute_left, const double eps, double *fop); template bool ldu_full_permute, quadruple>(int *nn0, const int n0, const int n, complex *a, const int lda, double *pivot, int *permute_right, int *permute_left, const double eps, double *fop); template bool ldu_full_permute(int *nn0, const int n0, const int n, octruple *a, const int lda, double *pivot, int *permute_right, int *permute_left, const double eps, double *fop); template bool ldu_full_permute, octruple>(int *nn0, const int n0, const int n, complex *a, const int lda, double *pivot, int *permute_right, int *permute_left, const double eps, double *fop); template bool ldu_full_permute(int *nn0, const int n0, const int n, float *a, const int lda, double *pivot, int *permute_right, int *permute_left, const double eps, double *fop); template bool ldu_full_permute, float>(int *nn0, const int n0, const int n, complex *a, const int lda, double *pivot, int *permute_right, int *permute_left, const double eps, double *fop); // template void swap_sym_lower(const int n, T *a, const int lda, const int k, const int km, T *col_k, T *col_km) { const int lda1 = lda + 1; for (int i = k; i < n; i++) { col_k[i] = a[i + k * lda]; // save lower column k } for (int j = 0; j < k; j++) { col_k[j] = a[k + j * lda]; // save lower row k } for (int i = km; i < n; i++) { col_km[i] = a[i + km * lda]; // save lower column km } for (int j = 0; j < km; j++) { col_km[j] = a[km + j * lda]; // save lower row km } for (int i = (k + 1); i < n; i++) { a[i + k * lda] = col_km[i]; // restore lower column k } for (int i = (km + 1); i < n; i++) { a[i + km * lda] = col_k[i]; // restore lower column km } for (int j = 0; j < k; j++) { a[k + j * lda] = col_km[j]; // restore lower row k } for (int j = 0; j < km; j++) { a[km + j * lda] = col_k[j]; // restore lower row km } a[k * lda1] = col_km[km]; a[km * lda1] = col_k[k]; a[km + k * lda] = col_k[km]; } template void swap_sym_lower(const int n, double *a, const int lda, const int k, const int km, double *col_k, double *col_km); template void swap_sym_lower >(const int n, complex *a, const int lda, const int k, const int km, complex *col_k, complex *col_km); template void swap_sym_lower(const int n, float *a, const int lda, const int k, const int km, float *col_k, float *col_km); template void swap_sym_lower >(const int n, complex *a, const int lda, const int k, const int km, complex *col_k, complex *col_km); template void swap_sym_lower(const int n, quadruple *a, const int lda, const int k, const int km, quadruple *col_k, quadruple *col_km); template void swap_sym_lower >(const int n, complex *a, const int lda, const int k, const int km, complex *col_k, complex *col_km); // template void swap_sym_upper(const int n, T *a, const int lda, const int k, const int km, T *col_k, T *col_km) { const int lda1 = lda + 1; for (int i = k; i < n; i++) { col_k[i] = a[k + i * lda]; // save lower column k } for (int j = 0; j < k; j++) { col_k[j] = a[j + k * lda]; // save lower row k } for (int i = km; i < n; i++) { col_km[i] = a[km + i * lda]; // save lower column km } for (int j = 0; j < km; j++) { col_km[j] = a[j + km * lda]; // save lower row km } for (int i = (k + 1); i < n; i++) { a[k + i * lda] = col_km[i]; // restore lower column k } for (int i = (km + 1); i < n; i++) { a[km + i * lda] = col_k[i]; // restore lower column km } for (int j = 0; j < k; j++) { a[j + k * lda] = col_km[j]; // restore lower row k } for (int j = 0; j < km; j++) { a[j + km * lda] = col_k[j]; // restore lower row km } a[k * lda1] = col_km[km]; a[km * lda1] = col_k[k]; a[k + km * lda] = col_k[km]; } template void swap_sym_upper(const int n, double *a, const int lda, const int k, const int km, double *col_k, double *col_km); template void swap_sym_upper >(const int n, complex *a, const int lda, const int k, const int km, complex *col_k, complex *col_km); template void swap_sym_upper(const int n, float *a, const int lda, const int k, const int km, float *col_k, float *col_km); template void swap_sym_upper >(const int n, complex *a, const int lda, const int k, const int km, complex *col_k, complex *col_km); template void swap_sym_upper(const int n, quadruple *a, const int lda, const int k, const int km, quadruple *col_k, quadruple *col_km); template void swap_sym_upper >(const int n, complex *a, const int lda, const int k, const int km, complex *col_k, complex *col_km); // template void swap_unsym(const int n, T *a, const int lda, const int k, const int km, T *col_k, T *col_km) { for (int i = 0; i < n; i++) { col_k[i] = a[i + k * lda]; // save lower column k col_km[i] = a[i + km * lda]; // save lower column km } for (int i = 0; i < n; i++) { a[i + km * lda] = col_k[i]; // save lower column km a[i + k * lda] = col_km[i]; } for (int i = 0; i < n; i++) { col_k[i] = a[k + i * lda]; // save lower column k col_km[i] = a[km + i * lda]; // save lower column km } for (int i = 0; i < n; i++) { a[km + i * lda] = col_k[i]; // save lower column km a[k + i * lda] = col_km[i]; } } template void swap_unsym(const int n, double *a, const int lda, const int k, const int km, double *col_k, double *col_km); template void swap_unsym >(const int n, complex *a, const int lda, const int k, const int km, complex *col_k, complex *col_km); template void swap_unsym(const int n, float *a, const int lda, const int k, const int km, float *col_k, float *col_km); template void swap_unsym >(const int n, complex *a, const int lda, const int k, const int km, complex *col_k, complex *col_km); template void swap_unsym(const int n, quadruple *a, const int lda, const int k, const int km, quadruple *col_k, quadruple *col_km); template void swap_unsym >(const int n, complex *a, const int lda, const int k, const int km, complex *col_k, complex *col_km); // template void swap_full(const int n, T *a, const int lda, const int k, const int ki, const int kj, T *coli, T *colj) { for (int i = 0; i < n; i++) { coli[i] = a[i + k * lda]; // save lower column k colj[i] = a[i + kj * lda]; // save lower column km } for (int i = 0; i < n; i++) { a[i + kj * lda] = coli[i]; // save lower column km a[i + k * lda] = colj[i]; } for (int i = 0; i < n; i++) { coli[i] = a[k + i * lda]; // save lower column k colj[i] = a[ki + i * lda]; // save lower column km } for (int i = 0; i < n; i++) { a[ki + i * lda] = coli[i]; // save lower column km a[k + i * lda] = colj[i]; } } template void swap_full(const int n, double *a, const int lda, const int k, const int ki, const int kj, double *coli, double *colj); template void swap_full >(const int n, complex *a, const int lda, const int k, const int ki, const int kj, complex *coli, complex *colj); template void swap_full(const int n, quadruple *a, const int lda, const int k, const int ki, const int kj, quadruple *coli, quadruple *colj); template void swap_full >(const int n, complex *a, const int lda, const int k, const int ki, const int kj, complex *coli, complex *colj); template void swap_full(const int n, float *a, const int lda, const int k, const int ki, const int kj, float *coli, float *colj); template void swap_full >(const int n, complex *a, const int lda, const int k, const int ki, const int kj, complex *coli, complex *colj); // template void full_ldlt(const int n, T *a, const int lda) { T alpha; const T one(1.0); const int lda1 = lda + 1; for (int k = 0; k < (n - 1); k++) { T d = one / a[k * lda1]; a[k * lda1] = d; alpha = (-d); blas_syr(CblasLower, (n - k - 1), alpha, &a[(k + 1) + k * lda], 1, &a[(k + 1) * lda1], lda); blas_scal((n - k - 1), d, &a[(k + 1) + k * lda], 1); } // symmetrize for (int j = 1; j < n; j++) { for (int i = 0; i < j; i++) { a[i + j * lda] = a[j + i * lda]; } } a[(n - 1) * lda1] = one / a[(n - 1) * lda1]; } template void full_ldlt(const int n, double *a, const int lda); template void full_ldlt(const int n, quadruple *a, const int lda); template void full_ldlt(const int n, float *a, const int lda); #ifndef NO_OCTRUPLE template void full_ldlt(const int n, octruple *a, const int lda); #endif template void full_ldlt >(const int n, complex *a, const int lda); template void full_ldlt >(const int n, complex *a, const int lda); template void full_ldlt >(const int n, complex *a, const int lda); #ifndef NO_OCTRUPLE template void full_ldlt >(const int n, complex *a, const int lda); #endif // template void full_ldlh(const int n, T *a, const int lda) { T alpha; const T one(1.0); const int lda1 = lda + 1; for (int k = 0; k < (n - 1); k++) { const T d = one / a[k * lda1]; alpha = (-d); a[k * lda1] = d; blas_syrc(CblasLower, (n - k - 1), alpha, &a[(k + 1) + k * lda], 1, &a[(k + 1) * lda1], lda); blas_scal((n - k - 1), d, &a[(k + 1) + k * lda], 1); } // symmetrize for (int j = 1; j < n; j++) { for (int i = 0; i < j; i++) { a[i + j * lda] = std::conj(a[j + i * lda]); } } a[(n - 1) * lda1] = one / a[(n - 1) * lda1]; } template<> void full_ldlh(const int n, double *a, const int lda) { full_ldlt(n, a, lda); } template<> void full_ldlh(const int n, quadruple *a, const int lda) { full_ldlt(n, a, lda); } template<> void full_ldlh(const int n, float *a, const int lda) { full_ldlt(n, a, lda); } #ifndef NO_OCTRUPLE template<> void full_ldlh(const int n, octruple *a, const int lda) { full_ldlt(n, a, lda); } #endif template void full_ldlh >(const int n, complex *a, const int lda); template void full_ldlh >(const int n, complex *a, const int lda); template void full_ldlh >(const int n, complex *a, const int lda); #ifndef NO_OCTRUPLE template void full_ldlh >(const int n, complex *a, const int lda); #endif // template void full_ldu(const int n, T *a, const int lda) { const int lda1 = lda + 1; const T one(1.0); const T none(-1.0); // alpha = none; for (int k = 0; k < (n - 1); k++) { const T d = one / a[k * lda1]; a[k * lda1] = d; blas_scal((n - k - 1), d, &a[k * lda1 + lda], lda); blas_ger((n - k - 1), (n - k - 1), none, &a[k * lda1 + 1], 1, &a[k * lda1 + lda], lda, &a[(k + 1) * lda1], lda); blas_scal((n - k - 1), d, &a[k * lda1 + 1], 1); } a[(n - 1) * lda1] = one / a[(n - 1) * lda1]; } template void full_ldu(const int n, double *a, const int lda); template void full_ldu(const int n, quadruple *a, const int lda); template void full_ldu(const int n, float *a, const int lda); #ifndef NO_OCTRUPLE template void full_ldu(const int n, octruple *a, const int lda); #endif template void full_ldu >(const int n, complex *a, const int lda); template void full_ldu >(const int n, complex *a, const int lda); template void full_ldu >(const int n, complex *a, const int lda); #ifndef NO_OCTRUPLE template void full_ldu >(const int n, complex *a, const int lda); #endif // template void FillUpperBlock(const int nrow, T *coef, const int *prow, const int *indcols, const int *indvals, const int *old2new_i, const int *old2new_j, ColumnMatrix &b) { for (int i = 0; i < nrow; i++) { const int ii = old2new_i[i]; for (int k = prow[ii]; k < prow[ii + 1]; k++) { const int jj = old2new_j[indcols[k]]; b(ii, jj) = coef[indvals[k]]; } } } template void FillUpperBlock(const int nnz, double *coef, const int *prow1, const int *indcols1, const int *indvals1, const int *new2old, const int *old_j, ColumnMatrix &b); template void FillUpperBlock >(const int nnz, complex *coef, const int *prow1, const int *indcols1, const int *indvals1, const int *new2old, const int *old_j, ColumnMatrix > &b); template void FillUpperBlock(const int nnz, quadruple *coef, const int *prow1, const int *indcols1, const int *indvals1, const int *new2old, const int *old_j, ColumnMatrix &b); template void FillUpperBlock >(const int nnz, complex *coef, const int *prow1, const int *indcols1, const int *indvals1, const int *new2old, const int *old_j, ColumnMatrix > &b); template void FillUpperBlock(const int nnz, float *coef, const int *prow1, const int *indcols1, const int *indvals1, const int *new2old, const int *old_j, ColumnMatrix &b); template void FillUpperBlock >(const int nnz, complex *coef, const int *prow1, const int *indcols1, const int *indvals1, const int *new2old, const int *old_j, ColumnMatrix > &b); // template void full_fw_multiprofile(bool isTransposed, const int nrow, const int n0, const int ncol, T *a, const int lda, T *y, const int ldy, vector &i0, double *fop) { const T one(1.0); const T zero(0.0); int jlast; const int n1 = nrow - n0; list j1; for (int j = 0; j < (ncol - 1); j++) { if (i0[j] < i0[j + 1]) { j1.push_back(j); } } j1.push_back(ncol - 1); jlast = (-1); for (list::const_iterator jt = j1.begin(); jt != j1.end(); ++jt) { const int width = (*jt) - jlast; const int ioffset = i0[(*jt)]; const int height = nrow - ioffset - n0; if (height > 0) { if (width > 1) { blas_trsm(CblasLeft, (isTransposed ? CblasUpper : CblasLower), (isTransposed ? CblasTrans : CblasNoTrans), CblasUnit, height, width, one, a + (ioffset * (nrow + 1)), nrow, y + (ioffset + (jlast + 1) * ldy), ldy); *fop += (double)height * (double)(height - 1) * (double)width; } else { blas_trsv((isTransposed ? CblasUpper : CblasLower), (isTransposed ? CblasTrans : CblasNoTrans), CblasUnit, height, a + (ioffset * (nrow + 1)), nrow, y + (ioffset + (jlast + 1) * ldy), 1); *fop += (double)height * (double)(height - 1); } } jlast = (*jt); } // loop : jt for (int j = 0; j < ncol; j++) { for (int i = n1; i < nrow; i++) { y[i + j * ldy] = zero; } } } template void full_fw_multiprofile(bool isTransposed, const int nrow, const int n0, const int ncol, double *a, const int lda, double *y, const int ldy, vector &i0, double *fop); template void full_fw_multiprofile >(bool isTransposed, const int nrow, const int n0, const int ncol, complex *a, const int lda, complex *y, const int ldy, vector &i0, double *fop); template void full_fw_multiprofile(bool isTransposed, const int nrow, const int n0, const int ncol, quadruple *a, const int lda, quadruple *y, const int ldy, vector &i0, double *fop); template void full_fw_multiprofile >(bool isTransposed, const int nrow, const int n0, const int ncol, complex *a, const int lda, complex *y, const int ldy, vector &i0, double *fop); template void full_fw_multiprofile(bool isTransposed, const int nrow, const int n0, const int ncol, float *a, const int lda, float *y, const int ldy, vector &i0, double *fop); template void full_fw_multiprofile >(bool isTransposed, const int nrow, const int n0, const int ncol, complex *a, const int lda, complex *y, const int ldy, vector &i0, double *fop); // template void full_fw_multi(bool isTransposed, const int nrow, const int n0, T *a, const int lda, const int ncol, T *y, const int ldy, double *fop) { const T one(1.0); const T zero(0.0); const int n1 = nrow - n0; // alpha = 1.0 // symmetric matrix should have upper part by symmeterization : LDL^t blas_trsm(CblasLeft, (isTransposed ? CblasUpper : CblasLower), (isTransposed ? CblasTrans : CblasNoTrans), CblasUnit, n1, ncol, one, a, lda, y, ldy); *fop += (double)nrow * (double)(nrow - 1) * (double)ncol; for (int j = 0; j < ncol; j++) { for (int i = n1; i < nrow; i++) { y[i + j * ldy] = zero; } } } template void full_fw_multi(bool isTransposed, const int nrow, const int n0, double *a, const int lda, const int ncol, double *y, const int ldy, double *fop); template void full_fw_multi >(bool isTransposed, const int nrow, const int n0, complex *a, const int lda, const int ncol, complex *y, const int ldy, double *fop); template void full_fw_multi(bool isTransposed, const int nrow, const int n0, quadruple *a, const int lda, const int ncol, quadruple *y, const int ldy, double *fop); template void full_fw_multi >(bool isTransposed, const int nrow, const int n0, complex *a, const int lda, const int ncol, complex *y, const int ldy, double *fop); template void full_fw_multi(bool isTransposed, const int nrow, const int n0, float *a, const int lda, const int ncol, float *y, const int ldy, double *fop); template void full_fw_multi >(bool isTransposed, const int nrow, const int n0, complex *a, const int lda, const int ncol, complex *y, const int ldy, double *fop); // template void full_fw_single(bool isTransposed, const int nrow, const int n0, T *a, const int lda, T *y, double *fop) { const T zero(0.0); const int n1 = nrow - n0; blas_trsv((isTransposed ? CblasUpper : CblasLower), (isTransposed ? CblasTrans : CblasNoTrans), CblasUnit, n1, a, lda, y, 1); *fop += (double)nrow * (double)(nrow - 1); for (int i = n1; i < nrow; i++) { y[i] = zero; } } template void full_fw_single(bool isTransposed, const int nrow, const int n0, double *a, const int lda, double *y, double *fop); template void full_fw_single >(bool isTransposed, const int nrow, const int n0, complex *a, const int lda, complex *y, double *fop); template void full_fw_single(bool isTransposed, const int nrow, const int n0, quadruple *a, const int lda, quadruple *y, double *fop); template void full_fw_single >(bool isTransposed, const int nrow, const int n0, complex *a, const int lda, complex *y, double *fop); template void full_fw_single(bool isTransposed, const int nrow, const int n0, float *a, const int lda, float *y, double *fop); template void full_fw_single >(bool isTransposed, const int nrow, const int n0, complex *a, const int lda, complex *y, double *fop); // template void full_bw_single(bool isTransposed, const int nrow, const int n0, T *a, const int lda, T *y, double *fop) { const T zero(0.0); const int n1 = nrow - n0; blas_trsv((isTransposed ? CblasLower : CblasUpper), (isTransposed ? CblasTrans : CblasNoTrans), CblasUnit, n1, a, lda, y, 1); *fop += (double)nrow * (double)(nrow - 1); for (int i = n1; i < nrow; i++) { y[i] = zero; } } template void full_bw_single(bool isTransposed, const int nrow, const int n0, double *a, const int lda, double *y, double *fop); template void full_bw_single >(bool isTransposed, const int nrow, const int n0, complex *a, const int lda, complex *y, double *fop); template void full_bw_single(bool isTransposed, const int nrow, const int n0, quadruple *a, const int lda, quadruple *y, double *fop); template void full_bw_single >(bool isTransposed, const int nrow, const int n0, complex *a, const int lda, complex *y, double *fop); template void full_bw_single(bool isTransposed, const int nrow, const int n0, float *a, const int lda, float *y, double *fop); template void full_bw_single >(bool isTransposed, const int nrow, const int n0, complex *a, const int lda, complex *y, double *fop); // template void full_bw_multi(bool isTransposed, const int nrow, const int n0, T *a, const int lda, const int ncol, T *y, const int ldy, double *fop) { const T one(1.0); const T zero(0.0); const int n1 = nrow - n0; // alpha = 1.0 // symmetric matrix should have upper part by symmeterization : LDL^t blas_trsm(CblasLeft, (isTransposed ? CblasLower : CblasUpper), (isTransposed ? CblasTrans : CblasNoTrans), CblasUnit, n1, ncol, one, a, lda, y, ldy); *fop += (double)nrow * (double)(nrow - 1) * (double)ncol; for (int j = 0; j < ncol; j++) { for (int i = n1; i < nrow; i++) { y[i + j * ldy] = zero; } } } template void full_bw_multi(bool isTransposed, const int nrow, const int n0, double *a, const int lda, const int ncol, double *y, const int ldy, double *fop); template void full_bw_multi >(bool isTransposed, const int nrow, const int n0, complex *a, const int lda, const int ncol, complex *y, const int ldy, double *fop); template void full_bw_multi(bool isTransposed, const int nrow, const int n0, quadruple *a, const int lda, const int ncol, quadruple *y, const int ldy, double *fop); template void full_bw_multi >(bool isTransposed, const int nrow, const int n0, complex *a, const int lda, const int ncol, complex *y, const int ldy, double *fop); template void full_bw_multi(bool isTransposed, const int nrow, const int n0, float *a, const int lda, const int ncol, float *y, const int ldy, double *fop); template void full_bw_multi >(bool isTransposed, const int nrow, const int n0, complex *a, const int lda, const int ncol, complex *y, const int ldy, double *fop); // template void SparseSchur(const bool isSym, const int dim2, const int dim1, vector& i0, ColumnMatrix &upper, ColumnMatrix &lower, ColumnMatrix &diag, double *fop) { const T zero(0.0); const T none(-1.0); // alpha = -1, beta = 0 if (isSym) { C_gemm_symm(dim2, dim1, none, lower.addrCoefs(), dim1, upper.addrCoefs(), dim1, zero, diag.addrCoefs(), dim2); // symmetrize : upper -> lower for (int i = 0; i < dim2; i++) { for (int j = 0; j < i; j++) { diag(i, j) = diag(j, i); } } } else { blas_gemm(CblasTrans, CblasNoTrans, dim2, dim2, dim1, none, lower.addrCoefs(), dim1, upper.addrCoefs(), dim1, zero, diag.addrCoefs(), dim2); *fop = (double)dim2 * (double)dim2 * (double)dim1; } } template void SparseSchur(const bool isSym, const int dim2, const int dim1, vector& i0, ColumnMatrix &upper, ColumnMatrix &lower, ColumnMatrix &diag, double *fop); template void SparseSchur >(const bool isSym, const int dim2, const int dim1, vector& i0, ColumnMatrix > &upper, ColumnMatrix > &lower, ColumnMatrix > &diag, double *fop); template void SparseSchur(const bool isSym, const int dim2, const int dim1, vector& i0, ColumnMatrix &upper, ColumnMatrix &lower, ColumnMatrix &diag, double *fop); template void SparseSchur >(const bool isSym, const int dim2, const int dim1, vector& i0, ColumnMatrix > &upper, ColumnMatrix > &lower, ColumnMatrix > &diag, double *fop); template void SparseSchur(const bool isSym, const int dim2, const int dim1, vector& i0, ColumnMatrix &upper, ColumnMatrix &lower, ColumnMatrix &diag, double *fop); template void SparseSchur >(const bool isSym, const int dim2, const int dim1, vector& i0, ColumnMatrix > &upper, ColumnMatrix > &lower, ColumnMatrix > &diag, double *fop); // template void full_fwbw_single(const bool isTrans, const int n, const int n0, T *a, const int lda, T *z) { const T zero(0.0); const int n1 = n - n0; for (int i = n1; i < n; i++) { z[i] = zero; } blas_trsv((isTrans ? CblasUpper : CblasLower), (isTrans ? CblasTrans : CblasNoTrans), CblasUnit, n1, a, lda, z, 1); for (int i = 0; i < n1; i++) { z[i] *= a[i * (n + 1)]; } blas_trsv((isTrans ? CblasLower : CblasUpper), (isTrans ? CblasTrans : CblasNoTrans), CblasUnit, n1, a, lda, z, 1); } template void full_fwbw_single(const bool isTrans, const int n, const int n0, double *a, const int lda, double *z); template void full_fwbw_single >(const bool isTrans, const int n, const int n0, complex *a, const int lda, complex *z); template void full_fwbw_single(const bool isTrans, const int n, const int n0, quadruple *a, const int lda, quadruple *z); template void full_fwbw_single >(const bool isTrans, const int n, const int n0, complex *a, const int lda, complex *z); template void full_fwbw_single(const bool isTrans, const int n, const int n0, float *a, const int lda, float *z); template void full_fwbw_single >(const bool isTrans, const int n, const int n0, complex *a, const int lda, complex *z); // template void full_fwbw_multi(const bool isTrans, const int n, const int n0, T *a, const int lda, const int m, T *x, const int ldx) { const T one(1.0); const T zero(0.0); const int n1 = n - n0; for (int j = 0; j < m; j++) { for (int i = n1; i < n; i++) { x[i + j * ldx] = zero; } } // alpha = 1.0 blas_trsm(CblasLeft, (isTrans ? CblasUpper : CblasLower), (isTrans ? CblasTrans : CblasNoTrans), CblasUnit, n1, m, one, a, lda, x, ldx); for (int j = 0; j < m; j++) { for (int i = 0; i < n1; i++) { // x[i + j * ldx] *= a[i * (n + 1)]; // 3 Nov.2015 x[i + j * ldx] *= a[i * (lda + 1)]; // 3 Nov.2015 } } blas_trsm(CblasLeft, (isTrans ? CblasLower : CblasUpper), (isTrans ? CblasTrans: CblasNoTrans), CblasUnit, n1, m, one, a, lda, x, ldx); } template void full_fwbw_multi(const bool isTrans, const int n, const int n0, double *a, const int lda, const int m, double *x, const int ldx); template void full_fwbw_multi >(const bool isTrans, const int n, const int n0, complex *a, const int lda, const int m, complex *x, const int ldx); template void full_fwbw_multi(const bool isTrans, const int n, const int n0, quadruple *a, const int lda, const int m, quadruple *x, const int ldx); template void full_fwbw_multi >(const bool isTrans, const int n, const int n0, complex *a, const int lda, const int m, complex *x, const int ldx); template void full_fwbw_multi(const bool isTrans, const int n, const int n0, float *a, const int lda, const int m, float *x, const int ldx); template void full_fwbw_multi >(const bool isTrans, const int n, const int n0, complex *a, const int lda, const int m, complex *x, const int ldx); // template void full_fwbw_part(const int n, T *a, const int lda, T*x) { int ifirst; const T zero(0.0); ifirst = n; for (int i = 0; i < n; i++) { if (x[i] != zero) { ifirst = i; break; } } if (ifirst == n) { return; } blas_trsv(CblasLower, CblasNoTrans, CblasUnit, (n - ifirst), &a[ifirst * (lda + 1)], lda, &x[ifirst], 1); for (int i = ifirst; i < n; i++) { x[i] *= a[i * (lda + 1)]; } blas_trsv(CblasUpper, CblasNoTrans, CblasUnit, n, a, lda, x, 1); } template void full_fwbw_part(const int n, double *a, const int lda, double *x); template void full_fwbw_part(const int n, quadruple *a, const int lda, quadruple *x); template void full_fwbw_part(const int n, float *a, const int lda, float *x); #ifndef NO_OCTRUPLE template void full_fwbw_part(const int n, octruple *a, const int lda, octruple *x); #endif template void full_fwbw_part >(const int n, complex *a, const int lda, complex *x); template void full_fwbw_part >(const int n, complex *a, const int lda, complex *x); template void full_fwbw_part >(const int n, complex *a, const int lda, complex *x); #ifndef NO_OCTRUPLE template void full_fwbw_part >(const int n, complex *a, const int lda, complex *x); #endif // template void SchurProfileSym(const int nrow, const int ncol, vector &i0, ColumnMatrix &b, ColumnMatrix &c, T *s, const int size_b1, double *fop) { const T one(1.0); const T zero(0.0); #if 1 const int num_block = ncol / size_b1 + ((ncol % size_b1) == 0 ? 0 : 1); for (int j = 0; j < num_block; j++) { const int jj = j * size_b1; for (int i = 0; i < j; i++) { const int ii = i * size_b1; const int nrowb = std::min(size_b1, ncol - ii); const int ncolb = std::min(size_b1, ncol - jj); const int ioffset = std::max(i0[ii], i0[jj]); //const int ioffset = 0; const int nnrow = nrow - ioffset; // alpha = 1 // beta = 0 blas_gemm(CblasTrans, CblasNoTrans, nrowb, ncolb, nnrow, one, c.addrCoefs() + (ioffset + ii * nrow), nrow, b.addrCoefs() + (ioffset + jj * nrow), nrow, zero, s + (ii + jj * ncol), ncol); *fop += (double)nrowb * (double)ncolb * (double) nnrow; } { const int nrowb = std::min(size_b1, ncol - jj); const int ioffset = i0[jj]; //const int ioffset = 0; const int nnrow = nrow - ioffset; // alpha = 1 // beta = 0 if (nnrow == 0) { // clear upper entries for (int j = 0; j < nrowb; j++) { for (int i = 0; i <= j; i++) { s[i + j * ncol + jj * (ncol + 1)] = zero; } } } else { C_gemm_symm(nrowb, nnrow, one, c.addrCoefs() + (ioffset + jj * nrow), nrow, b.addrCoefs() + (ioffset + jj * nrow), nrow, zero, s + (jj * (ncol + 1)), ncol); } *fop += (double)nrowb * (double)nrowb * (double) nnrow / 2.0; } } // loop : j #else // debug : compute all entries // alpha = 1 // beta = 0 // blas_gemm(CblasTrans, CblasNoTrans, // ncol, ncol, C_gemm_symm(ncol, nrow, one, c.addrCoefs(), nrow, b.addrCoefs(), nrow, zero, s, ncol); *fop += (double)ncol * (double)ncol * (double) nrow; #endif } template void SchurProfileSym(const int nrow, const int ncol, vector &i0, ColumnMatrix &b, ColumnMatrix &c, double *s, const int size_b1, double *fop); template void SchurProfileSym >(const int nrow, const int ncol, vector &i0, ColumnMatrix > &b, ColumnMatrix > &c, complex *s, const int size_b1, double *fop); template void SchurProfileSym(const int nrow, const int ncol, vector &i0, ColumnMatrix &b, ColumnMatrix &c, quadruple *s, const int size_b1, double *fop); template void SchurProfileSym >(const int nrow, const int ncol, vector &i0, ColumnMatrix > &b, ColumnMatrix > &c, complex *s, const int size_b1, double *fop); template void SchurProfileSym(const int nrow, const int ncol, vector &i0, ColumnMatrix &b, ColumnMatrix &c, float *s, const int size_b1, double *fop); template void SchurProfileSym >(const int nrow, const int ncol, vector &i0, ColumnMatrix > &b, ColumnMatrix > &c, complex *s, const int size_b1, double *fop); // template void SchurProfileUnSym(const int nrow, const int ncol, vector &i0, ColumnMatrix &b, ColumnMatrix &c, T* s, const int size_b1, double *fop) { const T one(1.0); const T zero(0.0); const int num_block = ncol / size_b1 + (ncol % size_b1 == 0 ? 0 : 1); for (int j = 0; j < num_block; j++) { const int jj = j * size_b1; for (int i = 0; i < num_block; i++) { const int ii = i * size_b1; const int nrowb = std::min(size_b1, ncol - ii); const int ncolb = std::min(size_b1, ncol - jj); const int ioffset = std::max(i0[ii], i0[jj]); const int nnrow = nrow - ioffset; // alpha=1 // beta=0 blas_gemm(CblasTrans, CblasNoTrans, nrowb, ncolb, nnrow, one, c.addrCoefs() + (ioffset + ii * nrow), nrow, b.addrCoefs() + (ioffset + jj * nrow), nrow, zero, s + (ii + jj * ncol), ncol); *fop += (double)nrowb * (double)ncolb * (double) nnrow; } } } template void SchurProfileUnSym(const int nrow, const int ncol, vector &i0, ColumnMatrix &b, ColumnMatrix &c, double *s, const int size_b1, double *fop); template void SchurProfileUnSym >(const int nrow, const int ncol, vector &i0, ColumnMatrix > &b, ColumnMatrix > &c, complex *s, const int size_b1, double *fop); template void SchurProfileUnSym(const int nrow, const int ncol, vector &i0, ColumnMatrix &b, ColumnMatrix &c, quadruple *s, const int size_b1, double *fop); template void SchurProfileUnSym >(const int nrow, const int ncol, vector &i0, ColumnMatrix > &b, ColumnMatrix > &c, complex *s, const int size_b1, double *fop); template void SchurProfileUnSym(const int nrow, const int ncol, vector &i0, ColumnMatrix &b, ColumnMatrix &c, float *s, const int size_b1, double *fop); template void SchurProfileUnSym >(const int nrow, const int ncol, vector &i0, ColumnMatrix > &b, ColumnMatrix > &c, complex *s, const int size_b1, double *fop); // // Z for perturbation has lower precision than T (real or complex valued) template void full_fwbw_perturb_single(const int n, T *a, const int lda, T *a_fact, T *x, const int dim_augkern, const Z &eps, bool flag_sym) { // T alpha, beta; // T *upper, *lower, *schur; const T one(1.0); const T none(-1.0); const T Teps = conv_prec(eps); int n1; if (dim_augkern < n) { n1 = n - dim_augkern; } else { n1 = 0; } if (n1 > 0) { ColumnMatrix upper(dim_augkern, n1); // new T[dim_augkern * n1]; ColumnMatrix lower(dim_augkern, n1); // lower = new T[dim_augkern * n1]; ColumnMatrix schur(n1, n1); // = new T[n1 * n1]; // factorization with perturbation should be shared with both sing and multi for (int j = 0; j < n1; j++) { for (int i = 0; i < dim_augkern; i++) { upper(i, j) = a[i + (j + dim_augkern) * lda]; } for (int i = 0; i < n1; i++) { schur(i, j) = a[i + dim_augkern + (j + dim_augkern) * lda]; } } { // alpha = one; blas_trsm(CblasLeft, CblasLower, CblasNoTrans, CblasUnit, dim_augkern, n1, one, a_fact, lda, upper.addrCoefs(), dim_augkern); for (int i = 0; i < dim_augkern; i++) { for (int j = 0; j < n1; j++) { upper(i, j) *= a_fact[i * (lda + 1)]; } } // alphe = one; blas_trsm(CblasLeft, CblasUpper, CblasNoTrans, CblasUnit, dim_augkern, n1, one, a_fact, lda, upper.addrCoefs(), dim_augkern); // emulating machine eplison in double for (int j = 0; j < n1; j++) { upper((dim_augkern - 1), j) += Teps; } } if (!flag_sym) { for (int j = 0; j < n1; j++) { for (int i = 0; i < dim_augkern; i++) { lower(i, j) = a[(j + dim_augkern) + i * lda]; } } // alpha = one; blas_trsm(CblasLeft, CblasUpper, CblasTrans, CblasUnit, dim_augkern, n1, one, a_fact, lda, lower.addrCoefs(), dim_augkern); for (int i = 0; i < dim_augkern; i++) { for (int j = 0; j < n1; j++) { lower(i, j) *= a_fact[i * (lda + 1)]; } } // alpha = one; blas_trsm(CblasLeft, CblasLower, CblasTrans, CblasUnit, dim_augkern, n1, one, a_fact, lda, lower.addrCoefs(), dim_augkern); for (int j = 0; j < n1; j++) { lower((dim_augkern - 1), j) += Teps; } } // alpha = none; // beta = one; blas_gemm(CblasNoTrans, CblasNoTrans, n1, n1, dim_augkern, none, &a[dim_augkern], lda, upper.addrCoefs(), dim_augkern, one, schur.addrCoefs(), n1); if (flag_sym) { full_ldlt(n1, schur.addrCoefs(), n1); } else { full_ldu(n1, schur.addrCoefs(), n1); } // alpha = none; // beta = one; if (flag_sym) { blas_gemv(CblasTrans, dim_augkern, n1, none, upper.addrCoefs(), dim_augkern, x, 1, one, &x[dim_augkern], 1); } else { blas_gemv(CblasTrans, dim_augkern, n1, none, lower.addrCoefs(), dim_augkern, x, 1, one, &x[dim_augkern], 1); } // forward blas_trsv(CblasLower, CblasNoTrans, CblasUnit, dim_augkern, a_fact, lda, x, 1); for (int j = 0; j < dim_augkern; j++) { x[j] *= a_fact[j * (lda + 1)]; } // backward blas_trsv(CblasUpper, CblasNoTrans, CblasUnit, dim_augkern, a_fact, lda, x, 1); // forward blas_trsv(CblasLower, CblasNoTrans, CblasUnit, dim_augkern, schur.addrCoefs(), n1, &x[dim_augkern], 1); // diagonal divide for (int j = 0; j < n1; j++) { x[j + dim_augkern] *= schur(j, j); //[j * (n1 + 1)]; } // backward blas_trsv(CblasUpper, CblasNoTrans, CblasUnit, dim_augkern, schur.addrCoefs(), n1, &x[dim_augkern], 1); // alpha = none; // beta = one; blas_gemv(CblasNoTrans, dim_augkern, n1, none, upper.addrCoefs(), dim_augkern, &x[dim_augkern], 1, one, x, 1); } else { // forward blas_trsv(CblasLower, CblasNoTrans, CblasUnit, n, a_fact, lda, x, 1); for (int j = 0; j < n; j++) { x[j] *= a_fact[j * (lda + 1)]; } x[(n - 1)] += Teps; // backward blas_trsv(CblasUpper, CblasNoTrans, CblasUnit, n, a_fact, lda, x, 1); } } template void full_fwbw_perturb_single(const int n, quadruple *a, const int lda, quadruple *a_fact, quadruple *x, const int dim_augkern, const double &eps, bool flag_sym); template void full_fwbw_perturb_single(const int n, double *a, const int lda, double *a_fact, double *x, const int dim_augkern, const float &eps, bool flag_sym); #ifndef NO_OCTRUPLE template void full_fwbw_perturb_single(const int n, octruple *a, const int lda, octruple *a_fact, octruple *x, const int dim_augkern, const quadruple &eps, bool flag_sym); #endif template void full_fwbw_perturb_single, double>(const int n, complex *a, const int lda, complex *a_fact, complex *x, const int dim_augkern, const double &eps, bool flag_sym); template void full_fwbw_perturb_single, float>(const int n, complex *a, const int lda, complex *a_fact, complex *x, const int dim_augkern, const float &eps, bool flag_sym); #ifndef NO_OCTRUPLE template void full_fwbw_perturb_single, quadruple>(const int n, complex *a, const int lda, complex *a_fact, complex *x, const int dim_augkern, const quadruple &eps, bool flag_sym); #endif // // Z for perturbation has lower precision than T (real or complex valued) template void full_fwbw_perturb_multi(const int n, const int m, T *a, const int lda, T *a_fact, T *x, const int dim_augkern, const Z &eps, bool flag_sym) { // T alpha, beta; // T *upper, *lower, *schur, *v; const T one(1.0); const T none(-1.0); const T Teps = conv_prec(eps); if (dim_augkern >= n) { // forward // alpha = one; blas_trsm(CblasLeft, CblasLower, CblasNoTrans, CblasUnit, n, m, one, a_fact, lda, x, lda); for (int j = 0; j < m; j++) { for (int i= 0; i < n; i++) { x[i + j * lda] *= a_fact[i * (lda + 1)]; } x[(n - 1) + j * lda] += Teps; } // backward blas_trsm(CblasLeft, CblasUpper, CblasNoTrans, CblasUnit, n, m, one, a_fact, lda, x, lda); } else { const int n1 = n - dim_augkern; ColumnMatrix upper(dim_augkern, n1); ColumnMatrix lower(dim_augkern, n1); ColumnMatrix schur(n1, n1); // factorization with perturbation should be shared with both sing and multi for (int j = 0; j < n1; j++) { for (int i = 0; i < dim_augkern; i++) { upper(i, j) = a[i + (j + dim_augkern) * lda]; } for (int i = 0; i < n1; i++) { schur(i, j) = a[i + dim_augkern + (j + dim_augkern) * lda]; } } { // alpha = one; blas_trsm(CblasLeft, CblasLower, CblasNoTrans, CblasUnit, dim_augkern, n1, one, a_fact, lda, upper.addrCoefs(), dim_augkern); for (int i = 0; i < dim_augkern; i++) { for (int j = 0; j < n1; j++) { upper(i, j) *= a_fact[i * (lda + 1)]; } } // alpha = one; blas_trsm(CblasLeft, CblasUpper, CblasNoTrans, CblasUnit, dim_augkern, n1, one, a_fact, lda, upper.addrCoefs(), dim_augkern); for (int j = 0; j < n1; j++) { upper((dim_augkern - 1), j) += Teps; } } if (!flag_sym) { for (int j = 0; j < n1; j++) { for (int i = 0; i < dim_augkern; i++) { lower(i, j) = a[(j + dim_augkern) + i * lda]; } } // alpha = one; blas_trsm(CblasLeft, CblasUpper, CblasTrans, CblasUnit, dim_augkern, n1, one, a_fact, lda, lower.addrCoefs(), dim_augkern); for (int i = 0; i < dim_augkern; i++) { for (int j = 0; j < n1; j++) { lower(i, j) *= a_fact[i * (lda + 1)]; } } // alpha = one; blas_trsm(CblasLeft, CblasLower, CblasTrans, CblasUnit, dim_augkern, n1, one, a_fact, lda, lower.addrCoefs(), dim_augkern); for (int j = 0; j < n1; j++) { lower((dim_augkern - 1), j) += Teps; } } // alpha = none; // beta = one; blas_gemm(CblasNoTrans, CblasNoTrans, n1, n1, dim_augkern, none, &a[dim_augkern], lda, upper.addrCoefs(), dim_augkern, one, schur.addrCoefs(), n1); if (flag_sym) { full_ldlt(n1, schur.addrCoefs(), n1); } else { full_ldu(n1, schur.addrCoefs(), n1); } // alpha = none; // beta = one; if (flag_sym) { blas_gemm(CblasTrans, CblasNoTrans, n1, m, dim_augkern, none, upper.addrCoefs(), dim_augkern, x, lda, one, &x[dim_augkern], lda); } else { blas_gemm(CblasTrans, CblasNoTrans, n1, m, dim_augkern, none, lower.addrCoefs(), dim_augkern, x, lda, one, &x[dim_augkern], lda); } // alpha = one; // forward blas_trsm(CblasLeft, CblasLower, CblasNoTrans, CblasUnit, dim_augkern, m, one, a_fact, lda, x, lda); // diagonal divide for (int j = 0; j < m; j++) { for (int i = 0; i < dim_augkern; i++) { x[i + j * lda] *= a_fact[i * (lda + 1)]; } } // backward blas_trsm(CblasLeft, CblasUpper, CblasNoTrans, CblasUnit, dim_augkern, m, one, a_fact, lda, x, lda); // forward blas_trsm(CblasLeft, CblasLower, CblasNoTrans, CblasUnit, n1, m, one, schur.addrCoefs(), n1, &x[dim_augkern], lda); for (int j = 0; j < m; j++) { for (int i = 0; i < n1; i++) { x[i + dim_augkern + j * lda] *= schur(i, i);// } } // backward blas_trsm(CblasLeft, CblasUpper, CblasNoTrans, CblasUnit, n1, m, one, schur.addrCoefs(), n1, &x[dim_augkern], lda); // alpha = none; // beta = one; blas_gemm(CblasNoTrans, CblasNoTrans, dim_augkern, m, n1, none, upper.addrCoefs(), dim_augkern, &x[dim_augkern], lda, one, x, lda); } } template void full_fwbw_perturb_multi(const int n, const int m, quadruple *a, const int lda, quadruple *a_fact, quadruple *x, const int dim_augkern, const double &eps, bool flag_sym); template void full_fwbw_perturb_multi(const int n, const int m, double *a, const int lda, double *a_fact, double *x, const int dim_augkern, const float &eps, bool flag_sym); #ifndef NO_OCTRUPLE template void full_fwbw_perturb_multi(const int n, const int m, octruple *a, const int lda, octruple *a_fact, octruple *x, const int dim_augkern, const quadruple &eps, bool flag_sym); #endif template void full_fwbw_perturb_multi, double>(const int n, const int m, complex *a, const int lda, complex *a_fact, complex *x, const int dim_augkern, const double &eps, bool flag_sym); template void full_fwbw_perturb_multi, float>(const int n, const int m, complex *a, const int lda, complex *a_fact, complex *x, const int dim_augkern, const float &eps, bool flag_sym); #ifndef NO_OCTRUPLE template void full_fwbw_perturb_multi, quadruple>(const int n, const int m, complex *a, const int lda, complex *a_fact, complex *x, const int dim_augkern, const quadruple &eps, bool flag_sym); #endif template void full_sym_2x2BK(int n, T *a, T *dd1, int *pivot_width, int *permute) { fprintf(stderr, "%s %d : general template is not implomented\n", __FILE__, __LINE__); } template void full_sym_2x2BK(int n, double *a, double *dd1, int *pivot_width, int *permute); template void full_sym_2x2BK >(int n, complex *a, complex *dd1, int *pivot_width, int *permute); template void full_sym_2x2BK(int n, quadruple *a, quadruple *dd1, int *pivot_width, int *permute); template void full_sym_2x2BK >(int n, complex *a, complex *dd1, int *pivot_width, int *permute); template void full_sym_2x2BK(int n, float *a, float *dd1, int *pivot_width, int *permute); template void full_sym_2x2BK >(int n, complex *a, complex *dd1, int *pivot_width, int *permute); // template void C_gemm_symm(const int ncol, const int nrow, const T &alpha, const T *a, const int lda, const T *b, const int ldb, const T &beta, T *c, const int ldc) { // #define DGEMM_FOR_SYMM #ifdef DGEMM_FOR_SYMM blas_gemm(CblasTrans, CblasNoTrans, ncol, ncol, nrow, alpha, a, lda, b, ldb, beta, c, ldc); #else int n1, n2, n11, n12, n21, n22, n3; n1 = ncol / 2; n2 = ncol - n1; n11 = n1 / 2; n12 = n1 - n11; n21 = n2 / 2; n22 = n2 - n21; n3 = n1 + n21; if (ncol < SIZE_DGEMM_SYMM_DTRSV) { for (int j = 0; j < ncol; j++) { blas_gemv(CblasTrans, nrow, (j + 1), alpha, a, lda, b + (ldb * j), 1, beta, c + (ldc * j), 1); } } else { C_gemm_symm(n11, nrow, alpha, a, lda, b, ldb, beta, c, ldc); C_gemm_symm(n12, nrow, alpha, a + (lda * n11), lda, b + (ldb * n11), ldb, beta, c + (ldc + 1) * n11, ldc); C_gemm_symm(n21, nrow, alpha, a + lda * n1, lda, b + ldb * n1, ldb, beta, c + (ldc + 1) * n1, ldc); C_gemm_symm(n22, nrow, alpha, a + lda * n3, lda, b + ldb * n3, ldb, beta, c + (ldc + 1) * n3, ldc); blas_gemm(CblasTrans, CblasNoTrans, n11, n12, nrow, alpha, a, lda, b + (ldb * n11), ldb, beta, c + (ldc * n11), ldc); blas_gemm(CblasTrans, CblasNoTrans, n21, n22, nrow, alpha, a + (lda * n1), lda, b + (ldb * n3), ldb, beta, c + (n1 + ldc * n3), ldc); blas_gemm(CblasTrans, CblasNoTrans, n1, n2, nrow, alpha, a, lda, b + (ldb * n1), ldb, beta, c + (ldc * n1), ldc); } #endif } template void C_gemm_symm(const int ncol, const int nrow, const double &alpha, const double *a, const int lda, const double *b, const int ldb, const double &beta, double *c, const int ldc); template void C_gemm_symm >(const int ncol, const int nrow, const complex &alpha, const complex *a, const int lda, const complex *b, const int ldb, const complex &beta, complex *c, const int ldc); template void C_gemm_symm(const int ncol, const int nrow, const quadruple &alpha, const quadruple *a, const int lda, const quadruple *b, const int ldb, const quadruple &beta, quadruple *c, const int ldc); template void C_gemm_symm >(const int ncol, const int nrow, const complex &alpha, const complex *a, const int lda, const complex *b, const int ldb, const complex &beta, complex *c, const int ldc); template void C_gemm_symm(const int ncol, const int nrow, const float &alpha, const float *a, const int lda, const float *b, const int ldb, const float &beta, float *c, const int ldc); template void C_gemm_symm >(const int ncol, const int nrow, const complex &alpha, const complex *a, const int lda, const complex *b, const int ldb, const complex &beta, complex *c, const int ldc); //