/******************************************************************************* * * * (C) 1997-2021 by Ernst W. Mayer. * * * * This program is free software; you can redistribute it and/or modify it * * under the terms of the GNU General Public License as published by the * * Free Software Foundation; either version 2 of the License, or (at your * * option) any later version. * * * * This program is distributed in the hope that it will be useful, but WITHOUT * * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * * more details. * * * * You should have received a copy of the GNU General Public License along * * with this program; see the file GPL.txt. If not, you may view one at * * http://www.fsf.org/licenses/licenses.html, or obtain one by writing to the * * Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA * * 02111-1307, USA. * * * *******************************************************************************/ #include "Mlucas.h" #include "radix16.h" #define RADIX 144 // Use #define rather than const int to ensure it's really a compile-time const in the C sense #define EPS 1e-10 #ifndef PFETCH_DIST #ifdef USE_AVX512 #define PFETCH_DIST 64 // Feb 2017: Test on KNL point to this as best #elif defined(USE_AVX) #define PFETCH_DIST 32 // This seems to work best on my Haswell, even though 64 bytes seems more logical in AVX mode #else #define PFETCH_DIST 32 #endif #endif #ifdef USE_SSE2 // For Mersenne-mod need (16 [SSE2] or 64 [AVX]) + (4 [HIACC] or 40 [LOACC]) added slots for half_arr lookup tables. // Max = (40 [SSE2]; 132 [AVX]), add to (half_arr_offset144 + RADIX) to get required value of radix144_creals_in_local_store: #ifdef USE_AVX512 // RADIX/8 = 0x12 fewer carry slots than AVX: const int half_arr_offset144 = 0x262; // + RADIX = 0x2f2; Used for thread local-storage-integrity checking const int radix144_creals_in_local_store = 0x378; // (half_arr_offset + RADIX) + 132(=0x84) and round up to nearest multiple of 4 #elif defined(USE_AVX) const int half_arr_offset144 = 0x274; // + RADIX = 0x304; Used for thread local-storage-integrity checking const int radix144_creals_in_local_store = 0x388; // (half_arr_offset + RADIX) + 132(=0x84) and round up to nearest multiple of 4 #else const int half_arr_offset144 = 0x298; // + RADIX = 0x328; Used for thread local-storage-integrity checking const int radix144_creals_in_local_store = 0x354; // (half_arr_offset + RADIX) = 40 and round up to nearest multiple of 4 #endif #include "sse2_macro_gcc64.h" #include "radix09_sse_macro.h" #endif // SSE2 #ifdef USE_PTHREAD // Use non-pooled simple spawn/rejoin thread-team model #include "threadpool.h" struct cy_thread_data_t{ // int data: int iter; int tid; int ndivr; int target_idx, target_set; // Jun 2018: Add support for residue shift. (Only LL-test needs intervention at carry-loop level). double target_cy; int khi; int i; int jstart; int jhi; int col; int co2; int co3; int sw; int nwt; // double data: double maxerr; double scale; double prp_mult; // pointer data: double *arrdat; /* Main data array */ double *wt0; double *wt1; double *wts_mult, *inv_mult; int *si; #ifdef USE_SSE2 vec_dbl *r00; vec_dbl *half_arr; #else double *r00; double *half_arr; #endif uint32 bjmodnini; int bjmodn0; // For large radix0 use thread-local arrays for DWT indices/carries - only caveat is these must be SIMD-aligned: #if GCC_EVER_GETS_ITS_ACT_TOGETHER_HERE /* Jan 2014: Bloody hell - turns out GCC uses __BIGGEST_ALIGNMENT__ = 16 on x86, which is too small to be useful for avx data! int bjmodn[RADIX] __attribute__ ((aligned (32))); double cy[RADIX] __attribute__ ((aligned (32))); */ #else // Thus, we are forced to resort to fugly hackage - add pad slots to a garbage-named struct-internal array along with // a pointer-to-be-inited-at-runtime, when we set ptr to the lowest-index array element having the desired alginment: double *cy; #ifdef USE_AVX512 double cy_dat[RADIX+8] __attribute__ ((__aligned__(8))); #else double cy_dat[RADIX+4] __attribute__ ((__aligned__(8))); // Enforce min-alignment of 8 bytes in 32-bit builds. #endif #endif }; #endif /****************/ int radix144_ditN_cy_dif1(double a[], int n, int nwt, int nwt_bits, double wt0[], double wt1[], int si[], double base[], double baseinv[], int iter, double *fracmax, uint64 p) { /* !...Acronym: DWT = Discrete Weighted Transform, DIT = Decimation In Time, DIF = Decimation In Frequency ! !...Performs a final radix-144 complex DIT pass, an inverse DWT weighting, a carry propagation, ! a forward DWT weighting, and an initial radix-144 complex DIF pass on the data in the length-N real vector A. ! ! Data enter and are returned in the A-array. ! ! See the documentation in mers_mod_square and radix16_dif_pass for further details on the array ! storage scheme, and radix8_ditN_cy_dif1 for details on the reduced-length weights array scheme. */ const char func[] = "radix144_ditN_cy_dif1"; const int pfetch_dist = PFETCH_DIST; const int stride = (int)RE_IM_STRIDE << 1; // main-array loop stride = 2*RE_IM_STRIDE static double wts_mult[2], inv_mult[2]; // Const wts-multiplier and 2*(its multiplicative inverse) double wt_re,wt_im, wi_re,wi_im; // Fermat-mod/LOACC weights stuff, used in both scalar and SIMD mode // Cleanup loop assumes carryins propagate at most 4 words up, but need at least 1 vec_cmplx // (2 vec_dbl)'s worth of doubles in wraparound step, hence AVX-512 needs value bumped up: #ifdef USE_AVX512 const int jhi_wrap = 15; #else const int jhi_wrap = 7; #endif int NDIVR,i,incr,j,j1,j2,jt,jp,jstart,jhi,full_pass,k,khi,l,ntmp,outer,nbytes; // incr = Carry-chain wts-multipliers recurrence length, which must divide // RADIX/[n-wayness of carry macro], e.g. RADIX/[16|8|4] = 9|18|36 for avx512,avx,sse, respectively: #ifdef USE_AVX512 // Oddly, FMA-based builds appear to need slightly shorter chains for radices 144,288 const int incr_long = 9,incr_med = 3,incr_short = 3; #elif defined(USE_AVX2) const int incr_long = 9,incr_med = 6,incr_short = 3; #else const int incr_long = 18,incr_med = 9,incr_short = 6; // Note: lump SSE2+AVX together for this radix, thus must divide 18! #endif // Have no specialized HIACC carry macro in USE_AVX512 and ARMv8 SIMD, so use "goes to 11" in LOACC mode via an incr_hiacc = 1: #if defined(USE_AVX512) || defined(USE_ARM_V8_SIMD) const int incr_hiacc = 1; #else const int incr_hiacc = 0; #endif // Allows cy-macro error data to be used to fiddle incr on the fly to a smaller, safer value if necessary if(USE_SHORT_CY_CHAIN == 0) incr = incr_long; else if(USE_SHORT_CY_CHAIN == 1) incr = incr_med; else if(USE_SHORT_CY_CHAIN == 2) incr = incr_short; else incr = incr_hiacc; // Jun 2018: Add support for residue shift. (Only LL-test needs intervention at carry-loop level). int target_idx = -1, target_set = 0,tidx_mod_stride; double target_cy = 0; static double ndivr_inv; uint64 itmp64; static uint64 psave = 0; static uint32 bw,sw,bjmodnini,p1,p2,p3,p4,p5,p6,p7,p8,p9,pa,pb,pc,pd,pe,pf ,p10,p20,p30,p40,p50,p60,p70,p80, nsave = 0; #ifndef MULTITHREAD // Shared DIF+DIT: const int *iptr; static int plo[16],phi[9]; uint64 i64; int k0,k1,k2,k3,k4,k5,k6,k7,k8,k9,ka,kb,kc,kd,ke,kf,po_kperm[16],*po_ptr = &(po_kperm[0]); double rt,it,re; // DIF: const uint64 dif16_oidx_lo[9] = { 0x01327654efcdab89ull, 0x10236745fedcba98ull, 0x89bafedc10236745ull, 0x5467102389bafedcull, 0xcdfe89ba54671023ull, 0x32105467cdfe89baull, 0xab89cdfe32105467ull, 0x76543210ab89cdfeull, 0xefcdab8976543210ull }; const int dif_phi[9] = {0,8,5,2,7,4,1,6,3}, // circ-shift count of basic array needed for stage 1: dif_ncshft[16] = {0x0,0x1,0x2,0x2,0x3,0x3,0x4,0x4,0x5,0x6,0x6,0x7,0x7,0x8,0x8,0x0}, // dif_pcshft has 9 base elts followed by repeat of first 8 of those to support circ-shift perms dif_pcshft[17] = {0,8,7,6,5,4,3,2,1,0,8,7,6,5,4,3,2}; // DIT: const uint64 dit16_iidx_lo[9] = { 0x01327654fedcba98ull, 0x5467102398abefcdull, 0xab89cdfe23014576ull, 0x76543210ba98dcefull, 0x10236745efcdab89ull, 0xcdfe89ba45760132ull, 0x32105467dcef98abull, 0xefcdab8967452301ull, 0x89bafedc01327654ull }; // circ-shift count of basic array needed for stage 2: const int dit_phi[9] = {0,2,1,6,8,7,4,3,5}, dit_ncshft[16] = {0,6,8,1,3,5,7,0,2,4,6,8,1,3,5,7}, // dit_pcshft has 9 base elts followed by repeat of first 8 of those to support circ-shift perms dit_pcshft[17] = {0,2,4,6,8,1,3,5,7,0,2,4,6,8,1,3,5}; #endif static int poff[RADIX>>2]; // Store mults of p4 offset for loop control static double radix_inv, n2inv; #if defined(USE_SSE2) || !defined(MULTITHREAD) const double c = 0.76604444311897803520, /* cos(2*pi/9) */ s = 0.64278760968653932631, /* sin(2*pi/9) */ c2 = 0.17364817766693034887, /* cos(2*u) */ s2 = 0.98480775301220805936, /* sin(2*u) */ c3m1= -1.50000000000000000000, /* cos(3*u)-1 */ s3 = 0.86602540378443864677, /* sin(3*u) */ c4 = -0.93969262078590838404, /* cos(4*u) */ s4 = 0.34202014332566873307; /* sin(4*u) */ #endif double scale, dtmp, maxerr = 0.0; // Local storage: We must use an array here because scalars have no guarantees about relative address offsets // [and even if those are contiguous-as-hoped-for, they may run in reverse]; Make array type (struct complex) // to allow us to use the same offset-indexing as in the original radix-32 in-place DFT macros: double *addr; struct complex t[RADIX], *tptr; int *itmp,*itm2; // Pointer into the bjmodn array int err; static int first_entry=TRUE; /*...stuff for the reduced-length DWT weights array is here: */ int n_div_nwt; int col,co2,co3; #ifdef USE_AVX512 double t0,t1,t2,t3; #ifdef CARRY_16_WAY static struct uint32x16 *n_minus_sil,*n_minus_silp1,*sinwt,*sinwtm1; #else static struct uint32x8 *n_minus_sil,*n_minus_silp1,*sinwt,*sinwtm1; #endif #elif defined(USE_AVX) static struct uint32x4 *n_minus_sil,*n_minus_silp1,*sinwt,*sinwtm1; #else int n_minus_sil,n_minus_silp1,sinwt,sinwtm1; double wtl,wtlp1,wtn,wtnm1; /* Mersenne-mod weights stuff */ #endif #ifdef USE_AVX2 // Due to GCC macro argc limit of 30, to enable 16-register data-doubled version of the radix-9 macros need 2 length-9 ptr arrays: vec_dbl *rad9_iptr[9], *rad9_optr[9]; #endif #ifdef USE_SSE2 static int cslots_in_local_store; static vec_dbl *sc_arr = 0x0, *sc_ptr; static uint64 *sm_ptr, *sign_mask, *sse_bw, *sse_sw, *sse_n; uint64 tmp64; #ifdef MULTITHREAD static vec_dbl *__r0; // Base address for discrete per-thread local stores #else double *add0,*add1,*add2,*add3; /* Addresses into array sections */ #endif // Uint64 bitmaps for alternate "rounded the other way" copies of sqrt2,isrt2. Default round-to-nearest versions // (SQRT2, ISRT2) end in ...3BCD. Since we round these down as ...3BCC90... --> ..3BCC, append _dn to varnames: const uint64 sqrt2_dn = 0x3FF6A09E667F3BCCull, isrt2_dn = 0x3FE6A09E667F3BCCull; static int *bjmodn; // Alloc mem for this along with other SIMD stuff const double crnd = 3.0*0x4000000*0x2000000; struct complex *ctmp; // Hybrid AVX-DFT/SSE2-carry scheme used for Mersenne-mod needs a 2-word-double pointer vec_dbl #ifndef MULTITHREAD *va0,*va1,*va2,*va3,*va4,*va5,*va6,*va7,*va8, *vb0,*vb1,*vb2,*vb3,*vb4,*vb5,*vb6,*vb7,*vb8, #endif *tmp,*tm0,*tm1,*tm2; // Non-static utility ptrs static vec_dbl *two,*one,*sqrt2,*isrt2,*cc0,*ss0, *cc1,*ss1,*cc2,*ss2,*cc3m1,*ss3,*cc4,*ss4, *max_err, *sse2_rnd, *half_arr, *r00, // Head of RADIX*vec_cmplx-sized local store #1 *s1p00, // Head of RADIX*vec_cmplx-sized local store #2 *cy; // Need RADIX/2 slots for sse2 carries, RADIX/4 for avx #else static int p0123[4]; #endif // USE_SSE2? #ifdef MULTITHREAD static struct cy_thread_data_t *tdat = 0x0; // Threadpool-based dispatch stuff: static int main_work_units = 0, pool_work_units = 0; static struct threadpool *tpool = 0x0; static int task_is_blocking = TRUE; static thread_control_t thread_control = {0,0,0}; // First 3 subfields same for all threads, 4th provides thread-specifc data, will be inited at thread dispatch: static task_control_t task_control = {NULL, (void*)cy144_process_chunk, NULL, 0x0}; #elif !defined(USE_SSE2) // Vars needed in scalar mode only: const double one_half[3] = {1.0, 0.5, 0.25}; /* Needed for small-weights-tables scheme */ int m,m2; double wt,wtinv,wtA,wtB,wtC; /* Mersenne-mod weights stuff */ int bjmodn[RADIX]; double temp,frac,cy[RADIX]; #endif /*...stuff for the multithreaded implementation is here: */ static uint32 CY_THREADS,pini; int ithread,j_jhi; uint32 ptr_prod; static int *_bjmodnini = 0x0,*_bjmodn[RADIX]; static int *_i, *_jstart = 0x0, *_jhi = 0x0, *_col = 0x0, *_co2 = 0x0, *_co3 = 0x0; static double *_cy[RADIX]; if(!_jhi) { _cy[0] = 0x0; // First of these used as an "already inited consts?" sentinel, must init = 0x0 at same time do so for non-array static ptrs } if(MODULUS_TYPE == MODULUS_TYPE_FERMAT) { ASSERT(HERE, 0, "Fermat-mod only available for radices 7,8,9,15 and their multiples!"); } // Init these to get rid of GCC "may be used uninitialized in this function" warnings: col=co2=co3=-1; // Jan 2018: To support PRP-testing, read the LR-modpow-scalar-multiply-needed bit for the current iteration from the global array: double prp_mult = 1.0; if((TEST_TYPE & 0xfffffffe) == TEST_TYPE_PRP) { // Mask off low bit to lump together PRP and PRP-C tests i = (iter-1) % ITERS_BETWEEN_CHECKPOINTS; // Bit we need to read...iter-counter is unit-offset w.r.to iter-interval, hence the -1 if((BASE_MULTIPLIER_BITS[i>>6] >> (i&63)) & 1) prp_mult = PRP_BASE; } /*...change NDIVR and n_div_wt to non-static to work around a gcc compiler bug. */ NDIVR = n/RADIX; ndivr_inv = (double)RADIX/n; n_div_nwt = NDIVR >> nwt_bits; if((n_div_nwt << nwt_bits) != NDIVR) { fprintf(stderr,"ERROR: iter = %10d; NWT_BITS does not divide N/RADIX in %s.\n",iter,func); err = ERR_SKIP_RADIX_SET; return(err); } if(p != psave || n != nsave #ifdef USE_PTHREAD // Oct 2021: cf. radix176_ditN_cy_dif1.c for why I added this || (tdat != 0x0 && tdat[0].wt1 != wt1) #endif ) { /* Exponent or array length change triggers re-init */ first_entry=TRUE; /* To-do: Support #thread change here! */ } /*...initialize things upon first entry: */ if(first_entry) { psave = p; nsave = n; radix_inv = qfdbl(qf_rational_quotient((int64)1, (int64)RADIX)); n2inv = qfdbl(qf_rational_quotient((int64)1, (int64)(n/2))); bw = p%n; /* Number of bigwords in the Crandall/Fagin mixed-radix representation = (Mersenne exponent) mod (vector length). */ sw = n - bw; /* Number of smallwords. */ #ifdef USE_AVX512 #ifdef CARRY_16_WAY i = 16; #else i = 8; #endif #elif defined(USE_AVX) // AVX LOACC: Make CARRY_8_WAY default here: i = 8; #elif defined(USE_SSE2) // AVX and SSE2 modes use 4-way carry macros i = 4; #else // Scalar-double mode: i = 1; #endif // For n a power of 2 don't need to worry about 32-bit integer overflow in the sw*NDIVR term, // but for non-power-of-2 n we must cast-to-uint64 to avoid such overflows fubaring the result: struct qfloat qt,qn; qt = i64_to_q(i*(uint64)sw*NDIVR % n); qn = i64_to_q((int64) n); qt = qfdiv(qt, qn); // x = (sw*NDIVR (mod n))/n qt = qfmul(qt, QLN2); // x*ln(2)... qt = qfexp(qt); // ...and get 2^x via exp[x*ln(2)]. wts_mult[0] = qfdbl(qt); // a = 2^(x/n), with x = sw inv_mult[0] = qfdbl(qfinv(qt)); // Double-based inversion (1.0 / wts_mult_a[0]) often gets LSB wrong ASSERT(HERE,fabs(wts_mult[0]*inv_mult[0] - 1.0) < EPS, "wts_mults fail accuracy check!"); //curr have w, 2/w, separate-mul-by-1-or-0.5 gives [w,w/2] and [1/w,2/w] for i = 0,1, resp: wts_mult[1] = 0.5*wts_mult[0]; inv_mult[1] = 2.0*inv_mult[0]; ASSERT(HERE,fabs(wts_mult[1]*inv_mult[1] - 1.0) < EPS, "wts_mults fail accuracy check!"); #ifdef MULTITHREAD /* #Chunks ||ized in carry step is ideally a power of 2, so use the largest power of 2 that is <= the value of the global NTHREADS (but still <= MAX_THREADS): */ if(isPow2(NTHREADS)) CY_THREADS = NTHREADS; else { i = leadz32(NTHREADS); CY_THREADS = (((uint32)NTHREADS << i) & 0x80000000) >> i; } if(CY_THREADS > MAX_THREADS) { // CY_THREADS = MAX_THREADS; fprintf(stderr,"WARN: CY_THREADS = %d exceeds number of cores = %d\n", CY_THREADS, MAX_THREADS); } if(!isPow2(CY_THREADS)) { WARN(HERE, "CY_THREADS not a power of 2!", "", 1); return(ERR_ASSERT); } if(CY_THREADS > 1) { if(NDIVR %CY_THREADS != 0) { WARN(HERE, "NDIVR %CY_THREADS != 0 ... likely more threads than this leading radix can handle.", "", 1); return(ERR_ASSERT); } if(n_div_nwt%CY_THREADS != 0) { WARN(HERE, "n_div_nwt%CY_THREADS != 0 ... likely more threads than this leading radix can handle.", "", 1); return(ERR_ASSERT); } } #ifdef USE_PTHREAD if(tdat == 0x0) { j = (uint32)sizeof(struct cy_thread_data_t); tdat = (struct cy_thread_data_t *)calloc(CY_THREADS, sizeof(struct cy_thread_data_t)); // MacOS does weird things with threading (e.g. Idle" main thread burning 100% of 1 CPU) // so on that platform try to be clever and interleave main-thread and threadpool-work processing #if 0//def OS_TYPE_MACOSX if(CY_THREADS > 1) { main_work_units = CY_THREADS/2; pool_work_units = CY_THREADS - main_work_units; ASSERT(HERE, 0x0 != (tpool = threadpool_init(pool_work_units, MAX_THREADS, pool_work_units, &thread_control)), "threadpool_init failed!"); printf("radix%d_ditN_cy_dif1: Init threadpool of %d threads\n", RADIX, pool_work_units); } else { main_work_units = 1; printf("radix%d_ditN_cy_dif1: CY_THREADS = 1: Using main execution thread, no threadpool needed.\n", RADIX); } #else pool_work_units = CY_THREADS; ASSERT(HERE, 0x0 != (tpool = threadpool_init(CY_THREADS, MAX_THREADS, CY_THREADS, &thread_control)), "threadpool_init failed!"); #endif fprintf(stderr,"Using %d threads in carry step\n", CY_THREADS); } #endif #else CY_THREADS = 1; #endif #ifdef USE_PTHREAD /* Populate the elements of the thread-specific data structs which don't change after init: */ for(ithread = 0; ithread < CY_THREADS; ithread++) { tdat[ithread].iter = iter; // int data: tdat[ithread].tid = ithread; tdat[ithread].ndivr = NDIVR; tdat[ithread].sw = sw; tdat[ithread].nwt = nwt; // pointer data: // tdat[ithread].arrdat = a; /* Main data array */ tdat[ithread].wt0 = wt0; tdat[ithread].wt1 = wt1; tdat[ithread].wts_mult = wts_mult; tdat[ithread].inv_mult = inv_mult; tdat[ithread].si = si; // This array pointer must be set based on vec_dbl-sized alignment at runtime for each thread: for(l = 0; l < RE_IM_STRIDE; l++) { if( ((long)&tdat[ithread].cy_dat[l] & SZ_VDM1) == 0 ) { tdat[ithread].cy = &tdat[ithread].cy_dat[l]; // fprintf(stderr,"%d-byte-align cy_dat array at element[%d]\n",SZ_VD,l); break; } } ASSERT(HERE, l < RE_IM_STRIDE, "Failed to align cy_dat array!"); } #endif #ifdef USE_SSE2 ASSERT(HERE, ((long)wt0 & 0x3f) == 0, "wt0[] not 64-byte aligned!"); ASSERT(HERE, ((long)wt1 & 0x3f) == 0, "wt1[] not 64-byte aligned!"); // Use double-complex type size (16 bytes) to alloc a block of local storage // consisting of radix144_creals_in_local_store dcomplex and (12+RADIX/2) uint64 element slots per thread // (Add as many padding elts to the latter as needed to make it a multiple of 4): cslots_in_local_store = radix144_creals_in_local_store + (((12+RADIX/2)/2 + 3) & ~0x3); sc_arr = ALLOC_VEC_DBL(sc_arr, cslots_in_local_store*CY_THREADS); if(!sc_arr){ sprintf(cbuf, "ERROR: unable to allocate sc_arr!.\n"); fprintf(stderr,"%s", cbuf); ASSERT(HERE, 0,cbuf); } sc_ptr = ALIGN_VEC_DBL(sc_arr); ASSERT(HERE, ((long)sc_ptr & 0x3f) == 0, "sc_ptr not 64-byte aligned!"); sm_ptr = (uint64*)(sc_ptr + radix144_creals_in_local_store); ASSERT(HERE, ((long)sm_ptr & 0x3f) == 0, "sm_ptr not 64-byte aligned!"); #ifdef USE_PTHREAD __r0 = sc_ptr; #endif tmp = sc_ptr; r00 = tmp; // Head of RADIX*vec_cmplx-sized local store #1 tmp += 0x120; s1p00 = tmp; // Head of RADIX*vec_cmplx-sized local store #2 tmp += 0x122; // Extra 2 slots here for one,sqrt2 below - added those late, too lazy to rejigger all the existing offsets following two = tmp - 2; // AVX+ versions of various DFT macros assume consts 2.0,1.0,sqrt2,isrt2 laid out thusly one = tmp - 1; sqrt2 = tmp + 0; // Roots for radix-16 DFTs: isrt2 = tmp + 0x1; cc0 = tmp + 0x2; ss0 = tmp + 0x3; // Roots for radix-9 DFTs: cc1 = tmp + 0x4; ss1 = tmp + 0x5; cc2 = tmp + 0x6; ss2 = tmp + 0x7; cc3m1 = tmp + 0x8; ss3 = tmp + 0x9; cc4 = tmp + 0xa; ss4 = tmp + 0xb; tmp += 0xc; // sc_ptr += 0x24e #ifdef USE_AVX512 cy = tmp; tmp += 0x12; // RADIX/8 vec_dbl slots for carry sub-array max_err = tmp + 0x00; sse2_rnd= tmp + 0x01; half_arr= tmp + 0x02; #elif defined(USE_AVX) cy = tmp; tmp += 0x24; // sc_ptr += 0x272 max_err = tmp + 0x00; sse2_rnd= tmp + 0x01; // sc_ptr += 2 = 0x274; This is where the value of half_arr_offset144 comes from half_arr= tmp + 0x02; // This table needs 96 vec_dbl in AVX mode #else cy = tmp; tmp += 0x48; // sc_ptr += 0x296 max_err = tmp + 0x00; sse2_rnd= tmp + 0x01; // sc_ptr += 2 = 0x296; This is where the value of half_arr_offset144 comes from half_arr= tmp + 0x02; // This table needs 32 x 16 bytes in SSE2 mode #endif ASSERT(HERE, (radix144_creals_in_local_store << L2_SZ_VD) >= ((long)half_arr - (long)r00) + (20 << L2_SZ_VD), "radix208_creals_in_local_store checksum failed!"); /* These remain fixed: */ VEC_DBL_INIT(two , 2.0 ); VEC_DBL_INIT(one, 1.0 ); #if 0 // Here this trick actually degrades accuracy ... must be interaction with the radix-9 DFTs of some kind // 2 unnamed slots for alternate "rounded the other way" copies of sqrt2,isrt2: dtmp = *(double *)&sqrt2_dn; VEC_DBL_INIT(sqrt2, dtmp); dtmp = *(double *)&isrt2_dn; VEC_DBL_INIT(isrt2, dtmp); #else VEC_DBL_INIT(sqrt2, SQRT2); VEC_DBL_INIT(isrt2, ISRT2); #endif VEC_DBL_INIT(cc0, c16 ); VEC_DBL_INIT(ss0, s16); // Radix-16 DFT macros assume [isrt2,cc0,ss0] memory ordering VEC_DBL_INIT(cc1 , c ); VEC_DBL_INIT(ss1, s ); VEC_DBL_INIT(cc2 , c2 ); VEC_DBL_INIT(ss2, s2); VEC_DBL_INIT(cc3m1, c3m1); VEC_DBL_INIT(ss3, s3); VEC_DBL_INIT(cc4 , c4 ); VEC_DBL_INIT(ss4, s4); /* SSE2 math = 53-mantissa-bit IEEE double-float: */ #ifdef USE_AVX512 // In AVX-512 mode, use VRNDSCALEPD for rounding and hijack this vector-data slot for the 4 base/baseinv-consts sse2_rnd->d0 = base[0]; sse2_rnd->d1 = baseinv[1]; sse2_rnd->d2 = wts_mult[1]; sse2_rnd->d3 = inv_mult[0]; #else VEC_DBL_INIT(sse2_rnd, crnd); #endif // Propagate the above consts to the remaining threads: nbytes = (long)cy - (long)two; // #bytes in 1st of above block of consts tmp = two; tm2 = tmp + cslots_in_local_store; for(ithread = 1; ithread < CY_THREADS; ++ithread) { memcpy(tm2, tmp, nbytes); tmp = tm2; tm2 += cslots_in_local_store; } nbytes = SZ_VD; // sse2_rnd is a solo (in the SIMD-vector) datum tmp = sse2_rnd; tm2 = tmp + cslots_in_local_store; for(ithread = 1; ithread < CY_THREADS; ++ithread) { memcpy(tm2, tmp, nbytes); tmp = tm2; tm2 += cslots_in_local_store; } /* SSE2 version of the one_half array - we have a 2-bit lookup, low bit is from the low word of the carry pair, high bit from the high, i.e. based on this lookup index [listed with LSB at right], we have: index half_lo half_hi 00 1.0 1.0 01 .50 1.0 10 1.0 .50 11 .50 .50 The inverse-weights computation uses a similar table, but with all entries multiplied by .50: index2 half_lo half_hi 00 .50 .50 01 .25 .50 10 .50 .25 11 .25 .25 We do similarly for the base[] and baseinv[] table lookups - each of these get 4 further slots in half_arr. We also allocate a further 4 16-byte slots [uninitialized] for storage of the wtl,wtn,wtlp1,wtnm1 locals. In 4-way SIMD (AVX) mode, we expand this from 2^2 2-vector table entries to 2^4 4-vector entries. */ tmp = half_arr; #ifdef USE_AVX512 // Each lookup-category in the 'mini-tables' used in AVX mode balloons from 16x32-bytes to 64x64-bytes, // so switch to an opmask-based scheme which starts with e.g. a broadcast constant and onditional doubling. // Here are the needed consts and opmasks: // [1] Fwd-wt multipliers: Init = 0.50 x 8, anytime AVX-style lookup into 1st table below would have bit = 0, double the corr. datum // [2] Inv-wt multipliers: Init = 0.25 x 8, anytime AVX-style lookup into 2nd table below would have bit = 0, double the corr. datum // [3] Fwd-base mults: Init = base[0] x 8, anytime AVX-style lookup into 3rd table below would have bit = 1, double the corr. datum // [4] Inv-base mults: Init = binv[1] x 8, anytime AVX-style lookup into 4th table below would have bit = 0, double the corr. datum // [5] [LOACC] Init = wts_mult[1] x 8, anytime AVX-style lookup into 5th table below would have bit = 0, double the corr. datum // [6] [LOACC] Init = inv_mult[0] x 8, anytime AVX-style lookup into 6th table below would have bit = 1, double the corr. datum nbytes = 0; #elif defined(USE_AVX) /* Forward-weight multipliers: */ tmp->d0 = 1.0; tmp->d1 = 1.0; tmp->d2 = 1.0; tmp->d3 = 1.0; ++tmp; tmp->d0 = .50; tmp->d1 = 1.0; tmp->d2 = 1.0; tmp->d3 = 1.0; ++tmp; tmp->d0 = 1.0; tmp->d1 = .50; tmp->d2 = 1.0; tmp->d3 = 1.0; ++tmp; tmp->d0 = .50; tmp->d1 = .50; tmp->d2 = 1.0; tmp->d3 = 1.0; ++tmp; tmp->d0 = 1.0; tmp->d1 = 1.0; tmp->d2 = .50; tmp->d3 = 1.0; ++tmp; tmp->d0 = .50; tmp->d1 = 1.0; tmp->d2 = .50; tmp->d3 = 1.0; ++tmp; tmp->d0 = 1.0; tmp->d1 = .50; tmp->d2 = .50; tmp->d3 = 1.0; ++tmp; tmp->d0 = .50; tmp->d1 = .50; tmp->d2 = .50; tmp->d3 = 1.0; ++tmp; tmp->d0 = 1.0; tmp->d1 = 1.0; tmp->d2 = 1.0; tmp->d3 = .50; ++tmp; tmp->d0 = .50; tmp->d1 = 1.0; tmp->d2 = 1.0; tmp->d3 = .50; ++tmp; tmp->d0 = 1.0; tmp->d1 = .50; tmp->d2 = 1.0; tmp->d3 = .50; ++tmp; tmp->d0 = .50; tmp->d1 = .50; tmp->d2 = 1.0; tmp->d3 = .50; ++tmp; tmp->d0 = 1.0; tmp->d1 = 1.0; tmp->d2 = .50; tmp->d3 = .50; ++tmp; tmp->d0 = .50; tmp->d1 = 1.0; tmp->d2 = .50; tmp->d3 = .50; ++tmp; tmp->d0 = 1.0; tmp->d1 = .50; tmp->d2 = .50; tmp->d3 = .50; ++tmp; tmp->d0 = .50; tmp->d1 = .50; tmp->d2 = .50; tmp->d3 = .50; ++tmp; /* Inverse-weight multipliers (only needed for mersenne-mod): */ tmp->d0 = .50; tmp->d1 = .50; tmp->d2 = .50; tmp->d3 = .50; ++tmp; tmp->d0 = .25; tmp->d1 = .50; tmp->d2 = .50; tmp->d3 = .50; ++tmp; tmp->d0 = .50; tmp->d1 = .25; tmp->d2 = .50; tmp->d3 = .50; ++tmp; tmp->d0 = .25; tmp->d1 = .25; tmp->d2 = .50; tmp->d3 = .50; ++tmp; tmp->d0 = .50; tmp->d1 = .50; tmp->d2 = .25; tmp->d3 = .50; ++tmp; tmp->d0 = .25; tmp->d1 = .50; tmp->d2 = .25; tmp->d3 = .50; ++tmp; tmp->d0 = .50; tmp->d1 = .25; tmp->d2 = .25; tmp->d3 = .50; ++tmp; tmp->d0 = .25; tmp->d1 = .25; tmp->d2 = .25; tmp->d3 = .50; ++tmp; tmp->d0 = .50; tmp->d1 = .50; tmp->d2 = .50; tmp->d3 = .25; ++tmp; tmp->d0 = .25; tmp->d1 = .50; tmp->d2 = .50; tmp->d3 = .25; ++tmp; tmp->d0 = .50; tmp->d1 = .25; tmp->d2 = .50; tmp->d3 = .25; ++tmp; tmp->d0 = .25; tmp->d1 = .25; tmp->d2 = .50; tmp->d3 = .25; ++tmp; tmp->d0 = .50; tmp->d1 = .50; tmp->d2 = .25; tmp->d3 = .25; ++tmp; tmp->d0 = .25; tmp->d1 = .50; tmp->d2 = .25; tmp->d3 = .25; ++tmp; tmp->d0 = .50; tmp->d1 = .25; tmp->d2 = .25; tmp->d3 = .25; ++tmp; tmp->d0 = .25; tmp->d1 = .25; tmp->d2 = .25; tmp->d3 = .25; ++tmp; /* Forward-base[] multipliers: */ tmp->d0 = base [0]; tmp->d1 = base [0]; tmp->d2 = base [0]; tmp->d3 = base [0]; ++tmp; tmp->d0 = base [1]; tmp->d1 = base [0]; tmp->d2 = base [0]; tmp->d3 = base [0]; ++tmp; tmp->d0 = base [0]; tmp->d1 = base [1]; tmp->d2 = base [0]; tmp->d3 = base [0]; ++tmp; tmp->d0 = base [1]; tmp->d1 = base [1]; tmp->d2 = base [0]; tmp->d3 = base [0]; ++tmp; tmp->d0 = base [0]; tmp->d1 = base [0]; tmp->d2 = base [1]; tmp->d3 = base [0]; ++tmp; tmp->d0 = base [1]; tmp->d1 = base [0]; tmp->d2 = base [1]; tmp->d3 = base [0]; ++tmp; tmp->d0 = base [0]; tmp->d1 = base [1]; tmp->d2 = base [1]; tmp->d3 = base [0]; ++tmp; tmp->d0 = base [1]; tmp->d1 = base [1]; tmp->d2 = base [1]; tmp->d3 = base [0]; ++tmp; tmp->d0 = base [0]; tmp->d1 = base [0]; tmp->d2 = base [0]; tmp->d3 = base [1]; ++tmp; tmp->d0 = base [1]; tmp->d1 = base [0]; tmp->d2 = base [0]; tmp->d3 = base [1]; ++tmp; tmp->d0 = base [0]; tmp->d1 = base [1]; tmp->d2 = base [0]; tmp->d3 = base [1]; ++tmp; tmp->d0 = base [1]; tmp->d1 = base [1]; tmp->d2 = base [0]; tmp->d3 = base [1]; ++tmp; tmp->d0 = base [0]; tmp->d1 = base [0]; tmp->d2 = base [1]; tmp->d3 = base [1]; ++tmp; tmp->d0 = base [1]; tmp->d1 = base [0]; tmp->d2 = base [1]; tmp->d3 = base [1]; ++tmp; tmp->d0 = base [0]; tmp->d1 = base [1]; tmp->d2 = base [1]; tmp->d3 = base [1]; ++tmp; tmp->d0 = base [1]; tmp->d1 = base [1]; tmp->d2 = base [1]; tmp->d3 = base [1]; ++tmp; /* Inverse-base[] multipliers: */ tmp->d0 = baseinv[0]; tmp->d1 = baseinv[0]; tmp->d2 = baseinv[0]; tmp->d3 = baseinv[0]; ++tmp; tmp->d0 = baseinv[1]; tmp->d1 = baseinv[0]; tmp->d2 = baseinv[0]; tmp->d3 = baseinv[0]; ++tmp; tmp->d0 = baseinv[0]; tmp->d1 = baseinv[1]; tmp->d2 = baseinv[0]; tmp->d3 = baseinv[0]; ++tmp; tmp->d0 = baseinv[1]; tmp->d1 = baseinv[1]; tmp->d2 = baseinv[0]; tmp->d3 = baseinv[0]; ++tmp; tmp->d0 = baseinv[0]; tmp->d1 = baseinv[0]; tmp->d2 = baseinv[1]; tmp->d3 = baseinv[0]; ++tmp; tmp->d0 = baseinv[1]; tmp->d1 = baseinv[0]; tmp->d2 = baseinv[1]; tmp->d3 = baseinv[0]; ++tmp; tmp->d0 = baseinv[0]; tmp->d1 = baseinv[1]; tmp->d2 = baseinv[1]; tmp->d3 = baseinv[0]; ++tmp; tmp->d0 = baseinv[1]; tmp->d1 = baseinv[1]; tmp->d2 = baseinv[1]; tmp->d3 = baseinv[0]; ++tmp; tmp->d0 = baseinv[0]; tmp->d1 = baseinv[0]; tmp->d2 = baseinv[0]; tmp->d3 = baseinv[1]; ++tmp; tmp->d0 = baseinv[1]; tmp->d1 = baseinv[0]; tmp->d2 = baseinv[0]; tmp->d3 = baseinv[1]; ++tmp; tmp->d0 = baseinv[0]; tmp->d1 = baseinv[1]; tmp->d2 = baseinv[0]; tmp->d3 = baseinv[1]; ++tmp; tmp->d0 = baseinv[1]; tmp->d1 = baseinv[1]; tmp->d2 = baseinv[0]; tmp->d3 = baseinv[1]; ++tmp; tmp->d0 = baseinv[0]; tmp->d1 = baseinv[0]; tmp->d2 = baseinv[1]; tmp->d3 = baseinv[1]; ++tmp; tmp->d0 = baseinv[1]; tmp->d1 = baseinv[0]; tmp->d2 = baseinv[1]; tmp->d3 = baseinv[1]; ++tmp; tmp->d0 = baseinv[0]; tmp->d1 = baseinv[1]; tmp->d2 = baseinv[1]; tmp->d3 = baseinv[1]; ++tmp; tmp->d0 = baseinv[1]; tmp->d1 = baseinv[1]; tmp->d2 = baseinv[1]; tmp->d3 = baseinv[1]; ++tmp; // In LOACC mode, put wts_mult and their inverses in the first 32 slots below in place of the 1/2-stuff: /* wts_mult:*/ tmp->d0 = wts_mult[0]; tmp->d1 = wts_mult[0]; tmp->d2 = wts_mult[0]; tmp->d3 = wts_mult[0]; ++tmp; tmp->d0 = wts_mult[1]; tmp->d1 = wts_mult[0]; tmp->d2 = wts_mult[0]; tmp->d3 = wts_mult[0]; ++tmp; tmp->d0 = wts_mult[0]; tmp->d1 = wts_mult[1]; tmp->d2 = wts_mult[0]; tmp->d3 = wts_mult[0]; ++tmp; tmp->d0 = wts_mult[1]; tmp->d1 = wts_mult[1]; tmp->d2 = wts_mult[0]; tmp->d3 = wts_mult[0]; ++tmp; tmp->d0 = wts_mult[0]; tmp->d1 = wts_mult[0]; tmp->d2 = wts_mult[1]; tmp->d3 = wts_mult[0]; ++tmp; tmp->d0 = wts_mult[1]; tmp->d1 = wts_mult[0]; tmp->d2 = wts_mult[1]; tmp->d3 = wts_mult[0]; ++tmp; tmp->d0 = wts_mult[0]; tmp->d1 = wts_mult[1]; tmp->d2 = wts_mult[1]; tmp->d3 = wts_mult[0]; ++tmp; tmp->d0 = wts_mult[1]; tmp->d1 = wts_mult[1]; tmp->d2 = wts_mult[1]; tmp->d3 = wts_mult[0]; ++tmp; tmp->d0 = wts_mult[0]; tmp->d1 = wts_mult[0]; tmp->d2 = wts_mult[0]; tmp->d3 = wts_mult[1]; ++tmp; tmp->d0 = wts_mult[1]; tmp->d1 = wts_mult[0]; tmp->d2 = wts_mult[0]; tmp->d3 = wts_mult[1]; ++tmp; tmp->d0 = wts_mult[0]; tmp->d1 = wts_mult[1]; tmp->d2 = wts_mult[0]; tmp->d3 = wts_mult[1]; ++tmp; tmp->d0 = wts_mult[1]; tmp->d1 = wts_mult[1]; tmp->d2 = wts_mult[0]; tmp->d3 = wts_mult[1]; ++tmp; tmp->d0 = wts_mult[0]; tmp->d1 = wts_mult[0]; tmp->d2 = wts_mult[1]; tmp->d3 = wts_mult[1]; ++tmp; tmp->d0 = wts_mult[1]; tmp->d1 = wts_mult[0]; tmp->d2 = wts_mult[1]; tmp->d3 = wts_mult[1]; ++tmp; tmp->d0 = wts_mult[0]; tmp->d1 = wts_mult[1]; tmp->d2 = wts_mult[1]; tmp->d3 = wts_mult[1]; ++tmp; tmp->d0 = wts_mult[1]; tmp->d1 = wts_mult[1]; tmp->d2 = wts_mult[1]; tmp->d3 = wts_mult[1]; ++tmp; /* inv_mult: */ tmp->d0 = inv_mult[0]; tmp->d1 = inv_mult[0]; tmp->d2 = inv_mult[0]; tmp->d3 = inv_mult[0]; ++tmp; tmp->d0 = inv_mult[1]; tmp->d1 = inv_mult[0]; tmp->d2 = inv_mult[0]; tmp->d3 = inv_mult[0]; ++tmp; tmp->d0 = inv_mult[0]; tmp->d1 = inv_mult[1]; tmp->d2 = inv_mult[0]; tmp->d3 = inv_mult[0]; ++tmp; tmp->d0 = inv_mult[1]; tmp->d1 = inv_mult[1]; tmp->d2 = inv_mult[0]; tmp->d3 = inv_mult[0]; ++tmp; tmp->d0 = inv_mult[0]; tmp->d1 = inv_mult[0]; tmp->d2 = inv_mult[1]; tmp->d3 = inv_mult[0]; ++tmp; tmp->d0 = inv_mult[1]; tmp->d1 = inv_mult[0]; tmp->d2 = inv_mult[1]; tmp->d3 = inv_mult[0]; ++tmp; tmp->d0 = inv_mult[0]; tmp->d1 = inv_mult[1]; tmp->d2 = inv_mult[1]; tmp->d3 = inv_mult[0]; ++tmp; tmp->d0 = inv_mult[1]; tmp->d1 = inv_mult[1]; tmp->d2 = inv_mult[1]; tmp->d3 = inv_mult[0]; ++tmp; tmp->d0 = inv_mult[0]; tmp->d1 = inv_mult[0]; tmp->d2 = inv_mult[0]; tmp->d3 = inv_mult[1]; ++tmp; tmp->d0 = inv_mult[1]; tmp->d1 = inv_mult[0]; tmp->d2 = inv_mult[0]; tmp->d3 = inv_mult[1]; ++tmp; tmp->d0 = inv_mult[0]; tmp->d1 = inv_mult[1]; tmp->d2 = inv_mult[0]; tmp->d3 = inv_mult[1]; ++tmp; tmp->d0 = inv_mult[1]; tmp->d1 = inv_mult[1]; tmp->d2 = inv_mult[0]; tmp->d3 = inv_mult[1]; ++tmp; tmp->d0 = inv_mult[0]; tmp->d1 = inv_mult[0]; tmp->d2 = inv_mult[1]; tmp->d3 = inv_mult[1]; ++tmp; tmp->d0 = inv_mult[1]; tmp->d1 = inv_mult[0]; tmp->d2 = inv_mult[1]; tmp->d3 = inv_mult[1]; ++tmp; tmp->d0 = inv_mult[0]; tmp->d1 = inv_mult[1]; tmp->d2 = inv_mult[1]; tmp->d3 = inv_mult[1]; ++tmp; tmp->d0 = inv_mult[1]; tmp->d1 = inv_mult[1]; tmp->d2 = inv_mult[1]; tmp->d3 = inv_mult[1]; ++tmp; nbytes = 96 << L2_SZ_VD; #elif defined(USE_SSE2) ctmp = (struct complex *)tmp; /* Forward-weight multipliers: */ ctmp->re = 1.0; ctmp->im = 1.0; ++ctmp; ctmp->re = .50; ctmp->im = 1.0; ++ctmp; ctmp->re = 1.0; ctmp->im = .50; ++ctmp; ctmp->re = .50; ctmp->im = .50; ++ctmp; /* Inverse-weight multipliers (only needed for mersenne-mod): */ ctmp->re = .50; ctmp->im = .50; ++ctmp; ctmp->re = .25; ctmp->im = .50; ++ctmp; ctmp->re = .50; ctmp->im = .25; ++ctmp; ctmp->re = .25; ctmp->im = .25; ++ctmp; /* Forward-base[] multipliers: */ ctmp->re = base [0]; ctmp->im = base [0]; ++ctmp; ctmp->re = base [1]; ctmp->im = base [0]; ++ctmp; ctmp->re = base [0]; ctmp->im = base [1]; ++ctmp; ctmp->re = base [1]; ctmp->im = base [1]; ++ctmp; /* Inverse-base[] multipliers: */ ctmp->re = baseinv[0]; ctmp->im = baseinv[0]; ++ctmp; ctmp->re = baseinv[1]; ctmp->im = baseinv[0]; ++ctmp; ctmp->re = baseinv[0]; ctmp->im = baseinv[1]; ++ctmp; ctmp->re = baseinv[1]; ctmp->im = baseinv[1]; ++ctmp; // In LOACC mode, put wts_mult and their inverses in the first 8 slots below in place of the 1/2-stuff: /* wts_mult:*/ ctmp->re = wts_mult[0]; ctmp->im = wts_mult[0]; ++ctmp; ctmp->re = wts_mult[1]; ctmp->im = wts_mult[0]; ++ctmp; ctmp->re = wts_mult[0]; ctmp->im = wts_mult[1]; ++ctmp; ctmp->re = wts_mult[1]; ctmp->im = wts_mult[1]; ++ctmp; /* inv_mult:*/ ctmp->re = inv_mult[0]; ctmp->im = inv_mult[0]; ++ctmp; ctmp->re = inv_mult[1]; ctmp->im = inv_mult[0]; ++ctmp; ctmp->re = inv_mult[0]; ctmp->im = inv_mult[1]; ++ctmp; ctmp->re = inv_mult[1]; ctmp->im = inv_mult[1]; ++ctmp; nbytes = 24 << L2_SZ_VD; #endif // Propagate the above consts to the remaining threads: tmp = half_arr; tm2 = tmp + cslots_in_local_store; for(ithread = 1; ithread < CY_THREADS; ++ithread) { memcpy(tm2, tmp, nbytes); tmp = tm2; tm2 += cslots_in_local_store; } /* Floating-point sign mask used for FABS on packed doubles: */ sign_mask = sm_ptr; for(i = 0; i < RE_IM_STRIDE; ++i) { *(sign_mask+i) = (uint64)0x7FFFFFFFFFFFFFFFull; } // Set up the SIMD-tupled-32-bit-int SSE constants used by the carry macros: sse_bw = sm_ptr + RE_IM_STRIDE; // (#doubles in a SIMD complex) x 32-bits = RE_IM_STRIDE x 64-bits tmp64 = (uint64)bw; tmp64 = tmp64 + (tmp64 << 32); for(i = 0; i < RE_IM_STRIDE; ++i) { *(sse_bw+i) = tmp64; } sse_sw = sse_bw + RE_IM_STRIDE; tmp64 = (uint64)sw; tmp64 = tmp64 + (tmp64 << 32); for(i = 0; i < RE_IM_STRIDE; ++i) { *(sse_sw+i) = tmp64; } sse_n = sse_sw + RE_IM_STRIDE; tmp64 = (uint64)n; tmp64 = tmp64 + (tmp64 << 32); for(i = 0; i < RE_IM_STRIDE; ++i) { *(sse_n +i) = tmp64; } nbytes = 4 << L2_SZ_VD; #ifdef USE_AVX512 #ifdef CARRY_16_WAY n_minus_sil = (struct uint32x16*)sse_n + 1; n_minus_silp1 = (struct uint32x16*)sse_n + 2; sinwt = (struct uint32x16*)sse_n + 3; sinwtm1 = (struct uint32x16*)sse_n + 4; nbytes += 256; #else n_minus_sil = (struct uint32x8 *)sse_n + 1; n_minus_silp1 = (struct uint32x8 *)sse_n + 2; sinwt = (struct uint32x8 *)sse_n + 3; sinwtm1 = (struct uint32x8 *)sse_n + 4; nbytes += 128; #endif #elif defined(USE_AVX) n_minus_sil = (struct uint32x4 *)sse_n + 1; n_minus_silp1 = (struct uint32x4 *)sse_n + 2; sinwt = (struct uint32x4 *)sse_n + 3; sinwtm1 = (struct uint32x4 *)sse_n + 4; nbytes += 64; #endif // Propagate the above consts to the remaining threads: tmp = (vec_dbl *)sm_ptr; tm2 = tmp + cslots_in_local_store; for(ithread = 1; ithread < CY_THREADS; ++ithread) { memcpy(tm2, tmp, nbytes); tmp = tm2; tm2 += cslots_in_local_store; } // For large radices, array-access to bjmodn means only init base-ptr here: #ifdef USE_AVX bjmodn = (int*)(sinwtm1 + RE_IM_STRIDE); #else bjmodn = (int*)(sse_n + RE_IM_STRIDE); #endif #endif // USE_SSE2 pini = NDIVR/CY_THREADS; /* constant index offsets for array load/stores are here. */ p1 = NDIVR; p2 = p1 + p1; p3 = p2 + p1; p4 = p3 + p1; p5 = p4 + p1; p6 = p5 + p1; p7 = p6 + p1; p8 = p7 + p1; p9 = p8 + p1; pa = p9 + p1; pb = pa + p1; pc = pb + p1; pd = pc + p1; pe = pd + p1; pf = pe + p1; p10 = pf + p1; p20 = p10 + p10; p30 = p20 + p10; p40 = p30 + p10; p50 = p40 + p10; p60 = p50 + p10; p70 = p60 + p10; p80 = p70 + p10; p1 += ( (p1 >> DAT_BITS) << PAD_BITS ); p2 += ( (p2 >> DAT_BITS) << PAD_BITS ); p3 += ( (p3 >> DAT_BITS) << PAD_BITS ); p4 += ( (p4 >> DAT_BITS) << PAD_BITS ); p5 += ( (p5 >> DAT_BITS) << PAD_BITS ); p6 += ( (p6 >> DAT_BITS) << PAD_BITS ); p7 += ( (p7 >> DAT_BITS) << PAD_BITS ); p8 += ( (p8 >> DAT_BITS) << PAD_BITS ); p9 += ( (p9 >> DAT_BITS) << PAD_BITS ); pa += ( (pa >> DAT_BITS) << PAD_BITS ); pb += ( (pb >> DAT_BITS) << PAD_BITS ); pc += ( (pc >> DAT_BITS) << PAD_BITS ); pd += ( (pd >> DAT_BITS) << PAD_BITS ); pe += ( (pe >> DAT_BITS) << PAD_BITS ); pf += ( (pf >> DAT_BITS) << PAD_BITS ); p10 += ( (p10 >> DAT_BITS) << PAD_BITS ); p20 += ( (p20 >> DAT_BITS) << PAD_BITS ); p30 += ( (p30 >> DAT_BITS) << PAD_BITS ); p40 += ( (p40 >> DAT_BITS) << PAD_BITS ); p50 += ( (p50 >> DAT_BITS) << PAD_BITS ); p60 += ( (p60 >> DAT_BITS) << PAD_BITS ); p70 += ( (p70 >> DAT_BITS) << PAD_BITS ); p80 += ( (p80 >> DAT_BITS) << PAD_BITS ); #ifndef USE_SSE2 p0123[0] = 0; p0123[1] = p1; p0123[2] = p2; p0123[3] = p3; #endif poff[ 0] = 0; poff[ 1] = p4; poff[ 2] = p8; poff[ 3] = pc; poff[0x04+0] = p10; poff[0x04+1] = p10+p4; poff[0x04+2] = p10+p8; poff[0x04+3] = p10+pc; poff[0x08+0] = p20; poff[0x08+1] = p20+p4; poff[0x08+2] = p20+p8; poff[0x08+3] = p20+pc; poff[0x0c+0] = p30; poff[0x0c+1] = p30+p4; poff[0x0c+2] = p30+p8; poff[0x0c+3] = p30+pc; poff[0x10+0] = p40; poff[0x10+1] = p40+p4; poff[0x10+2] = p40+p8; poff[0x10+3] = p40+pc; poff[0x14+0] = p50; poff[0x14+1] = p50+p4; poff[0x14+2] = p50+p8; poff[0x14+3] = p50+pc; poff[0x18+0] = p60; poff[0x18+1] = p60+p4; poff[0x18+2] = p60+p8; poff[0x18+3] = p60+pc; poff[0x1c+0] = p70; poff[0x1c+1] = p70+p4; poff[0x1c+2] = p70+p8; poff[0x1c+3] = p70+pc; poff[0x20+0] = p80; poff[0x20+1] = p80+p4; poff[0x20+2] = p80+p8; poff[0x20+3] = p80+pc; #ifndef MULTITHREAD // Shared: plo[0x0] = 0; plo[0x1] = p1; plo[0x2] = p2; plo[0x3] = p3; plo[0x4] = p4; plo[0x5] = p5; plo[0x6] = p6; plo[0x7] = p7; plo[0x8] = p8; plo[0x9] = p9; plo[0xa] = pa; plo[0xb] = pb; plo[0xc] = pc; plo[0xd] = pd; plo[0xe] = pe; plo[0xf] = pf; l = 0; phi[l++] = 0; phi[l++] = p10; phi[l++] = p20; phi[l++] = p30; phi[l++] = p40; phi[l++] = p50; phi[l++] = p60; phi[l++] = p70; phi[l++] = p80; #endif if(_cy[0]) /* If it's a new exponent of a range test, need to deallocate these. */ { free((void *)_i ); _i = 0x0; for(i = 0; i < RADIX; i++) { free((void *)_bjmodn[i]); _bjmodn[i] = 0x0; free((void *) _cy[i]); _cy[i] = 0x0; } free((void *)_jstart ); _jstart = 0x0; free((void *)_jhi ); _jhi = 0x0; free((void *)_col ); _col = 0x0; free((void *)_co2 ); _co2 = 0x0; free((void *)_co3 ); _co3 = 0x0; free((void *)_bjmodnini); _bjmodnini = 0x0; } ptr_prod = (uint32)0; /* Store bitmask for allocatable-array ptrs here, check vs 0 after all alloc calls finish */ j = CY_THREADS*sizeof(int); _i = (int *)malloc(j); ptr_prod += (uint32)(_i== 0x0); for(i = 0; i < RADIX; i++) { _bjmodn[i] = (int *)malloc(j); ptr_prod += (uint32)(_bjmodn[i]== 0x0); } _jstart = (int *)malloc(j); ptr_prod += (uint32)(_jstart == 0x0); _jhi = (int *)malloc(j); ptr_prod += (uint32)(_jhi == 0x0); _col = (int *)malloc(j); ptr_prod += (uint32)(_col == 0x0); _co2 = (int *)malloc(j); ptr_prod += (uint32)(_co2 == 0x0); _co3 = (int *)malloc(j); ptr_prod += (uint32)(_co3 == 0x0); j = CY_THREADS*sizeof(double); for(i = 0; i < RADIX; i++) { _cy[i] = (double *)malloc(j); ptr_prod += (uint32)(_cy[i]== 0x0); } ASSERT(HERE, ptr_prod == 0, "ERROR: unable to allocate one or more auxiliary arrays."); /* Create (THREADS + 1) copies of _bjmodnini and use the extra (uppermost) one to store the "master" increment, i.e. the one that n2/RADIX-separated FFT outputs need: */ _bjmodnini = (int *)malloc((CY_THREADS + 1)*sizeof(int)); if(!_bjmodnini){ sprintf(cbuf,"ERROR: unable to allocate array _bjmodnini in %s.\n",func); fprintf(stderr,"%s", cbuf); ASSERT(HERE, 0,cbuf); } _bjmodnini[0] = 0; _bjmodnini[1] = 0; jhi = NDIVR/CY_THREADS; for(j=0; j < jhi; j++) { _bjmodnini[1] -= sw; _bjmodnini[1] = _bjmodnini[1] + ( (-(int)((uint32)_bjmodnini[1] >> 31)) & n); } if(CY_THREADS > 1) { for(ithread = 2; ithread <= CY_THREADS; ithread++) { _bjmodnini[ithread] = _bjmodnini[ithread-1] + _bjmodnini[1] - n; _bjmodnini[ithread] = _bjmodnini[ithread] + ( (-(int)((uint32)_bjmodnini[ithread] >> 31)) & n); } } /* Check upper element against scalar value, as precomputed in single-thread mode: */ bjmodnini=0; for(j=0; j < jhi*CY_THREADS; j++) { bjmodnini -= sw; bjmodnini = bjmodnini + ( (-(int)((uint32)bjmodnini >> 31)) & n); } ASSERT(HERE, _bjmodnini[CY_THREADS] == bjmodnini,"_bjmodnini[CY_THREADS] != bjmodnini"); #ifdef USE_PTHREAD /* Populate the elements of the thread-specific data structs which don't change after init: */ for(ithread = 0; ithread < CY_THREADS; ithread++) { tdat[ithread].bjmodnini = _bjmodnini[CY_THREADS]; tdat[ithread].bjmodn0 = _bjmodnini[ithread]; #ifdef USE_SSE2 tdat[ithread].r00 = __r0 + ithread*cslots_in_local_store; tdat[ithread].half_arr = (vec_dbl *)((long)tdat[ithread].r00 + ((long)half_arr - (long)r00)); #else // In scalar mode use these 2 ptrs to pass the base & baseinv arrays: tdat[ithread].r00 = (double *)base; tdat[ithread].half_arr = (double *)baseinv; #endif // USE_SSE2 } #endif first_entry=FALSE; } /* endif(first_entry) */ // Jun 2018: If LL test and shift applied, compute target index for data-processing loop. // Note that only 1 thread of the carry-processing set will hit the target, but all need the same logic to check for a hit: if(MODULUS_TYPE == MODULUS_TYPE_MERSENNE && TEST_TYPE == TEST_TYPE_PRIMALITY) { if(RES_SHIFT) { itmp64 = shift_word(a, n, p, RES_SHIFT, 0.0); // Note return value (specifically high 7 bytes thereof) is an unpadded index target_idx = (int)(itmp64 >> 8); // This still needs to be (mod NDIVR)'ed, but first use unmodded form to compute needed DWT weights // Compute wt = 2^(target_idx*sw % n)/n and its reciprocal: uint32 sw_idx_modn = ((uint64)target_idx*sw) % n; // N is 32-bit, so only use 64-bit to hold intermediate product double target_wtfwd = pow(2.0, sw_idx_modn*0.5*n2inv); // 0.5*n2inv = 0.5/(n/2) = 1.0/n target_set = target_idx*ndivr_inv; // Which of the [RADIX] independent sub-carry-chains contains the target index? target_idx -= target_set*NDIVR; // Fast computation of target_idx = (target_idx % NDIVR) // Now compute the doubles-pointer offset of the target double w.r.to the SIMD s1p00-... data layout: tidx_mod_stride = target_idx & (stride-1); // Stride a power of 2, so can use AND-minus-1 for mod target_idx -= tidx_mod_stride; // printf("Iter %d: cy_shift = %d, target_idx,tidx_mod_stride,target_set = %d,%d,%d\n",iter,(itmp64 & 255),target_idx,tidx_mod_stride,target_set); #ifdef USE_AVX512 tidx_mod_stride = br16[tidx_mod_stride]; #elif defined(USE_AVX) tidx_mod_stride = br8[tidx_mod_stride]; #elif defined(USE_SSE2) tidx_mod_stride = br4[tidx_mod_stride]; #endif target_set = (target_set<<(L2_SZ_VD-2)) + tidx_mod_stride; target_cy = target_wtfwd * ((int)-2 << (itmp64 & 255)); } else { target_idx = target_set = 0; target_cy = -2.0; } } /*...The radix-144 final DIT pass is here. */ /* init carries */ for(ithread = 0; ithread < CY_THREADS; ithread++) { for(i = 0; i < RADIX; i++) { _cy[i][ithread] = 0; } } #if 0 //ndef USE_SSE2 *** v20: Non-SIMD builds now also support shifted-residue /* If an LL test, init the subtract-2: */ if(MODULUS_TYPE == MODULUS_TYPE_MERSENNE && TEST_TYPE == TEST_TYPE_PRIMALITY) { _cy[0][0] = -2; } #endif *fracmax=0; /* init max. fractional error */ full_pass = 1; /* set = 1 for normal carry pass, = 0 for wrapper pass */ scale = n2inv; // init inverse-weight scale factor = 2/n for normal carry pass, 1 for wrapper pass for(outer=0; outer <= 1; outer++) { _i[0] = 1; /* Pointer to the BASE and BASEINV arrays. If n does not divide p, lowest-order digit is always a bigword (_i[0] = 1). */ if(CY_THREADS > 1) { for(ithread = 1; ithread < CY_THREADS; ithread++) { _i[ithread] = ((uint32)(sw - _bjmodnini[ithread]) >> 31); } } /* Moved this inside the outer-loop, so on cleanup pass can use it to reset _col,_co2,_co3 starting values, then simply overwrite it with 1 prior to starting the k-loop. */ khi = n_div_nwt/CY_THREADS; j = _bjmodnini[CY_THREADS]; // Include 0-thread here ... bjmodn terms all 0 for that, but need jhi computed for all threads: for(ithread = 0; ithread < CY_THREADS; ithread++) { _bjmodn[0][ithread] = _bjmodnini[ithread]; for(i = 1; i < RADIX; i++) { MOD_ADD32(_bjmodn[i-1][ithread], j, n, _bjmodn[i][ithread]); } _jstart[ithread] = ithread*NDIVR/CY_THREADS; if(!full_pass) _jhi[ithread] = _jstart[ithread] + jhi_wrap; /* Cleanup loop assumes carryins propagate at most 4 words up. */ else _jhi[ithread] = _jstart[ithread] + nwt-1; _col[ithread] = ithread*(khi*RADIX); /* col gets incremented by RADIX_VEC[0] on every pass through the k-loop */ _co2[ithread] = (n>>nwt_bits)-1+RADIX - _col[ithread]; /* co2 gets decremented by RADIX_VEC[0] on every pass through the k-loop */ _co3[ithread] = _co2[ithread]-RADIX; /* At the start of each new j-loop, co3=co2-RADIX_VEC[0] */ } #ifdef USE_PTHREAD for(ithread = 0; ithread < CY_THREADS; ++ithread) { tdat[ithread].iter = iter; } if(MODULUS_TYPE == MODULUS_TYPE_MERSENNE) { // Carry-injection location for the shifted-residue -2 addend is only needed for full pass: if(full_pass) { tdat[0].target_idx = target_idx; tdat[0].target_set = target_set; tdat[0].target_cy = target_cy; } else { tdat[0].target_idx = -1; tdat[0].target_set = 0; tdat[0].target_cy = 0; } // Copy to the remaining threads: for(ithread = 1; ithread < CY_THREADS; ++ithread) { tdat[ithread].target_idx = tdat[0].target_idx; tdat[ithread].target_set = tdat[0].target_set; tdat[ithread].target_cy = tdat[0].target_cy; } } #endif #ifdef USE_SSE2 tmp = max_err; VEC_DBL_INIT(tmp, 0.0); tm2 = tmp + cslots_in_local_store; for(ithread = 1; ithread < CY_THREADS; ++ithread) { memcpy(tm2, tmp, SZ_VD); tmp = tm2; tm2 += cslots_in_local_store; } #endif // USE_PTHREAD /* Move this cleanup-pass-specific khi setting here, since need regular-pass khi value for above inits: */ if(!full_pass) { khi = 1; } #ifdef USE_PTHREAD /* Populate the thread-specific data structs - use the invariant terms as memchecks: */ for(ithread = 0; ithread < CY_THREADS; ithread++) { tdat[ithread].iter = iter; // int data: ASSERT(HERE, tdat[ithread].tid == ithread, "thread-local memcheck fail!"); ASSERT(HERE, tdat[ithread].ndivr == NDIVR, "thread-local memcheck fail!"); tdat[ithread].khi = khi; tdat[ithread].i = _i[ithread]; /* Pointer to the BASE and BASEINV arrays. */ tdat[ithread].jstart = _jstart[ithread]; tdat[ithread].jhi = _jhi[ithread]; tdat[ithread].col = _col[ithread]; tdat[ithread].co2 = _co2[ithread]; tdat[ithread].co3 = _co3[ithread]; ASSERT(HERE, tdat[ithread].sw == sw, "thread-local memcheck fail!"); ASSERT(HERE, tdat[ithread].nwt == nwt, "thread-local memcheck fail!"); // double data: tdat[ithread].maxerr = 0.0; tdat[ithread].scale = scale; tdat[ithread].prp_mult = prp_mult; // pointer data: tdat[ithread].arrdat = a; /* Main data array */ ASSERT(HERE, tdat[ithread].wt0 == wt0, "thread-local memcheck fail!"); ASSERT(HERE, tdat[ithread].wt1 == wt1, "thread-local memcheck fail!"); ASSERT(HERE, tdat[ithread].si == si, "thread-local memcheck fail!"); #ifdef USE_SSE2 ASSERT(HERE, tdat[ithread].r00 == __r0 + ithread*cslots_in_local_store, "thread-local memcheck fail!"); tmp = tdat[ithread].half_arr; #ifdef USE_AVX512 // In AVX-512 mode, use VRNDSCALEPD for rounding and hijack this vector-data slot for the 4 base/baseinv-consts ASSERT(HERE, ((tmp-1)->d0 == base[0] && (tmp-1)->d1 == baseinv[1] && (tmp-1)->d2 == wts_mult[1] && (tmp-1)->d3 == inv_mult[0]), "thread-local memcheck failed!"); #else ASSERT(HERE, ((tmp-1)->d0 == crnd && (tmp-1)->d1 == crnd), "thread-local memcheck failed!"); #endif #ifdef USE_AVX512 /* No-Op */ #elif defined(USE_AVX) // Grab some elt of base-data [offset by, say, +32] and mpy by its inverse [+16 further] dtmp = (tmp+40)->d0 * (tmp+56)->d0; ASSERT(HERE, fabs(dtmp - 1.0) < EPS, "thread-local memcheck failed!"); dtmp = (tmp+40)->d1 * (tmp+56)->d1; ASSERT(HERE, fabs(dtmp - 1.0) < EPS, "thread-local memcheck failed!"); #else // SSE2: dtmp = (tmp+10)->d0 * (tmp+14)->d0; ASSERT(HERE, fabs(dtmp - 1.0) < EPS, "thread-local memcheck failed!"); dtmp = (tmp+10)->d1 * (tmp+14)->d1; ASSERT(HERE, fabs(dtmp - 1.0) < EPS, "thread-local memcheck failed!"); #endif #endif /* init carries: */ for(i = 0; i < RADIX; i++) { tdat[ithread].cy[i] = _cy[i][ithread]; } } #endif #ifdef USE_PTHREAD // If also using main thread to do work units, that task-dispatch occurs after all the threadpool-task launches: for(ithread = 0; ithread < pool_work_units; ithread++) { task_control.data = (void*)(&tdat[ithread]); threadpool_add_task(tpool, &task_control, task_is_blocking); #else for(ithread = 0; ithread < CY_THREADS; ithread++) { if(full_pass) maxerr = 0.0; #ifdef USE_SSE2 // VEC_DBL_INIT(max_err, 0.0); *** must do this in conjunction with thread-local-data-copy #endif i = _i[ithread]; /* Pointer to the BASE and BASEINV arrays. */ jstart = _jstart[ithread]; jhi = _jhi[ithread]; col = _col[ithread]; co2 = _co2[ithread]; co3 = _co3[ithread]; for(l = 0; l < RADIX; l++) { bjmodn[l] = _bjmodn[l][ithread]; } /* init carries */ #ifdef USE_AVX512 tmp = cy; for(l = 0; l < RADIX; l += 8, ++tmp) { tmp->d0 = _cy[l ][ithread]; tmp->d1 = _cy[l+1][ithread]; tmp->d2 = _cy[l+2][ithread]; tmp->d3 = _cy[l+3][ithread]; tmp->d4 = _cy[l+4][ithread]; tmp->d5 = _cy[l+5][ithread]; tmp->d6 = _cy[l+6][ithread]; tmp->d7 = _cy[l+7][ithread]; } #elif defined(USE_AVX) // AVX and AVX2 both use 256-bit registers tmp = cy; for(l = 0; l < RADIX; l += 4, ++tmp) { tmp->d0 = _cy[l ][ithread]; tmp->d1 = _cy[l+1][ithread]; tmp->d2 = _cy[l+2][ithread]; tmp->d3 = _cy[l+3][ithread]; } #elif defined(USE_SSE2) tmp = cy; for(l = 0; l < RADIX; l += 2, ++tmp) { tmp->d0 = _cy[l ][ithread]; tmp->d1 = _cy[l+1][ithread]; } #else for(l = 0; l < RADIX; l++) { cy[l] = _cy[l][ithread]; } #endif /********************************************************************************/ /* This main loop is same for un-and-multithreaded, so stick into a header file */ /* (can't use a macro because of the #if-enclosed stuff). */ /********************************************************************************/ #include "radix144_main_carry_loop.h" /* At end of each thread-processed work chunk, dump the carryouts into their non-thread-private array slots: */ #ifdef USE_AVX512 tmp = cy; for(l = 0; l < RADIX; l += 8, ++tmp) { _cy[l ][ithread] = tmp->d0; _cy[l+1][ithread] = tmp->d1; _cy[l+2][ithread] = tmp->d2; _cy[l+3][ithread] = tmp->d3; _cy[l+4][ithread] = tmp->d4; _cy[l+5][ithread] = tmp->d5; _cy[l+6][ithread] = tmp->d6; _cy[l+7][ithread] = tmp->d7; } if(full_pass) { t0 = MAX(max_err->d0,max_err->d1); t1 = MAX(max_err->d2,max_err->d3); t2 = MAX(max_err->d4,max_err->d5); t3 = MAX(max_err->d6,max_err->d7); maxerr = MAX( MAX(t0,t1), MAX(t2,t3) ); } #elif defined(USE_AVX) // AVX and AVX2 both use 256-bit registers tmp = cy; for(l = 0; l < RADIX; l += 4, ++tmp) { _cy[l ][ithread] = tmp->d0; _cy[l+1][ithread] = tmp->d1; _cy[l+2][ithread] = tmp->d2; _cy[l+3][ithread] = tmp->d3; } if(full_pass) maxerr = MAX( MAX(max_err->d0,max_err->d1) , MAX(max_err->d2,max_err->d3) ); #elif defined(USE_SSE2) tmp = cy; for(l = 0; l < RADIX; l += 2, ++tmp) { _cy[l ][ithread] = tmp->d0; _cy[l+1][ithread] = tmp->d1; } if(full_pass) maxerr = MAX(max_err->d0,max_err->d1); #else for(l = 0; l < RADIX; l++) { _cy[l][ithread] = cy[l]; } #endif #endif // #ifdef USE_PTHREAD } /******* END OF PARALLEL FOR-LOOP ********/ #ifdef USE_PTHREAD // End of threadpool-based dispatch: Add a small wait-loop to ensure all threads complete #if 0//def OS_TYPE_MACOSX /*** Main execution thread executes remaining chunks in serial fashion (but in || with the pool threads): ***/ for(j = 0; j < main_work_units; ++j) { // printf("adding main task %d\n",j + pool_work_units); ASSERT(HERE, 0x0 == cy144_process_chunk( (void*)(&tdat[j + pool_work_units]) ), "Main-thread task failure!"); } #endif struct timespec ns_time; // We want a sleep interval of 0.1 mSec here... ns_time.tv_sec = 0; // (time_t)seconds - Don't use this because under OS X it's of type __darwin_time_t, which is long rather than double as under most linux distros ns_time.tv_nsec = 100000; // (long)nanoseconds - Get our desired 0.1 mSec as 10^5 nSec here while(tpool && tpool->free_tasks_queue.num_tasks != pool_work_units) { ASSERT(HERE, 0 == mlucas_nanosleep(&ns_time), "nanosleep fail!"); } /* Copy the thread-specific output carry data back to shared memory: */ for(ithread = 0; ithread < CY_THREADS; ithread++) { if(maxerr < tdat[ithread].maxerr) { maxerr = tdat[ithread].maxerr; } for(l = 0; l < RADIX; l++) { _cy[l][ithread] = tdat[ithread].cy[l]; } } #endif if(full_pass) { // printf("Iter = %d, maxerr = %20.15f\n",iter,maxerr); } else { break; } /* Wraparound carry cleanup loop is here: The cleanup carries from the end of each length-N/RADIX set of contiguous data into the begining of the next can all be neatly processed as follows: (1) Invert the forward DIF FFT of the first block of RADIX complex elements in A and unweight; (2) Propagate cleanup carries among the real and imaginary parts of the RADIX outputs of (1); (3) Reweight and perform a forward DIF FFT on the result of (2); (4) If any of the exit carries from (2) are nonzero, advance to the next RADIX elements and repeat (1-4). */ for(l = 0; l < RADIX; l++) { t[l].re = _cy[l][CY_THREADS - 1]; } for(ithread = CY_THREADS - 1; ithread > 0; ithread--) { for(l = 0; l < RADIX; l++) { _cy[l][ithread] = _cy[l][ithread-1]; } } _cy[0][0] =+t[RADIX-1].re; /* ...The wraparound carry is here: */ for(l = 1; l < RADIX; l++) { _cy[l][0] = t[l-1].re; } full_pass = 0; scale = prp_mult = 1; j_jhi = jhi_wrap; for(ithread = 0; ithread < CY_THREADS; ithread++) { for(j = ithread*pini; j <= ithread*pini + j_jhi; j++) { // Generate padded version of j, since prepadding pini is thread-count unsafe: j1 = j + ( (j >> DAT_BITS) << PAD_BITS ); for(l = 0; l < RADIX>>2; l++) { jt = j1 + poff[l]; // poff[] = p04,p08,...,p56 a[jt ] *= radix_inv; a[jt+p1] *= radix_inv; a[jt+p2] *= radix_inv; a[jt+p3] *= radix_inv; } } } } /* endfor(outer) */ dtmp = 0; for(ithread = 0; ithread < CY_THREADS; ithread++) { for(l = 0; l < RADIX; l++) { dtmp += fabs(_cy[l][ithread]); } *fracmax = maxerr; } if(dtmp != 0.0) { sprintf(cbuf,"ERROR: iter = %10d; nonzero exit carry in %s - input wordsize may be too small.\n",iter,func); mlucas_fprint(cbuf,INTERACT); err = ERR_CARRY; return(err); } return(0); } /****************/ void radix144_dif_pass1(double a[], int n) { /* !...Acronym: DIF = Decimation In Frequency ! !...Subroutine to perform an initial radix-144 complex DIF FFT pass on the data in the length-N real vector A. ! ! See the documentation in radix16_dif_pass for further details on storage and indexing. ! ! See the documentation in radix9_dif_pass for details on the radix-9 subtransform. */ const double c = 0.76604444311897803520, /* cos(2*pi/9) */ s = 0.64278760968653932631, /* sin(2*pi/9) */ c2 = 0.17364817766693034887, /* cos(2*u) */ s2 = 0.98480775301220805936, /* sin(2*u) */ c3m1= -1.50000000000000000000, /* cos(3*u)-1 */ s3 = 0.86602540378443864677, /* sin(3*u) */ c4 = -0.93969262078590838404, /* cos(4*u) */ s4 = 0.34202014332566873307; /* sin(4*u) */ int l,j,j1,j2,jt,jp; const int *iptr; // p-indexing is hexadecimal here: static int NDIVR,p1,p2,p3,p4,p5,p6,p7,p8,p9,pa,pb,pc,pd,pe,pf ,p10,p20,p30,p40,p50,p60,p70,p80, first_entry=TRUE; const uint64 dif16_oidx_lo[9] = { 0x01327654efcdab89ull, 0x10236745fedcba98ull, 0x89bafedc10236745ull, 0x5467102389bafedcull, 0xcdfe89ba54671023ull, 0x32105467cdfe89baull, 0xab89cdfe32105467ull, 0x76543210ab89cdfeull, 0xefcdab8976543210ull }; static int plo[16],phi[9]; const int dif_phi[9] = {0,8,5,2,7,4,1,6,3}, // circ-shift count of basic array needed for stage 1: dif_ncshft[16] = {0x0,0x1,0x2,0x2,0x3,0x3,0x4,0x4,0x5,0x6,0x6,0x7,0x7,0x8,0x8,0x0}, // dif_pcshft has 9 base elts followed by repeat of first 8 of those to support circ-shift perms dif_pcshft[17] = {0,8,7,6,5,4,3,2,1,0,8,7,6,5,4,3,2}; uint64 i64; int k0,k1,k2,k3,k4,k5,k6,k7,k8,k9,ka,kb,kc,kd,ke,kf; struct complex t[RADIX], *tptr; double rt,it,re; if(!first_entry && (n/RADIX) != NDIVR) /* New runlength? */ { first_entry=TRUE; } /*...initialize things upon first entry */ if(first_entry) { first_entry=FALSE; NDIVR = n/RADIX; p1 = NDIVR; p2 = p1 + p1; p3 = p2 + p1; p4 = p3 + p1; p5 = p4 + p1; p6 = p5 + p1; p7 = p6 + p1; p8 = p7 + p1; p9 = p8 + p1; pa = p9 + p1; pb = pa + p1; pc = pb + p1; pd = pc + p1; pe = pd + p1; pf = pe + p1; p10 = pf + p1; p20 = p10 + p10; p30 = p20 + p10; p40 = p30 + p10; p50 = p40 + p10; p60 = p50 + p10; p70 = p60 + p10; p80 = p70 + p10; p1 += ( (p1 >> DAT_BITS) << PAD_BITS ); p2 += ( (p2 >> DAT_BITS) << PAD_BITS ); p3 += ( (p3 >> DAT_BITS) << PAD_BITS ); p4 += ( (p4 >> DAT_BITS) << PAD_BITS ); p5 += ( (p5 >> DAT_BITS) << PAD_BITS ); p6 += ( (p6 >> DAT_BITS) << PAD_BITS ); p7 += ( (p7 >> DAT_BITS) << PAD_BITS ); p8 += ( (p8 >> DAT_BITS) << PAD_BITS ); p9 += ( (p9 >> DAT_BITS) << PAD_BITS ); pa += ( (pa >> DAT_BITS) << PAD_BITS ); pb += ( (pb >> DAT_BITS) << PAD_BITS ); pc += ( (pc >> DAT_BITS) << PAD_BITS ); pd += ( (pd >> DAT_BITS) << PAD_BITS ); pe += ( (pe >> DAT_BITS) << PAD_BITS ); pf += ( (pf >> DAT_BITS) << PAD_BITS ); p10 += ( (p10 >> DAT_BITS) << PAD_BITS ); p20 += ( (p20 >> DAT_BITS) << PAD_BITS ); p30 += ( (p30 >> DAT_BITS) << PAD_BITS ); p40 += ( (p40 >> DAT_BITS) << PAD_BITS ); p50 += ( (p50 >> DAT_BITS) << PAD_BITS ); p60 += ( (p60 >> DAT_BITS) << PAD_BITS ); p70 += ( (p70 >> DAT_BITS) << PAD_BITS ); p80 += ( (p80 >> DAT_BITS) << PAD_BITS ); plo[0x0] = 0; plo[0x1] = p1; plo[0x2] = p2; plo[0x3] = p3; plo[0x4] = p4; plo[0x5] = p5; plo[0x6] = p6; plo[0x7] = p7; plo[0x8] = p8; plo[0x9] = p9; plo[0xa] = pa; plo[0xb] = pb; plo[0xc] = pc; plo[0xd] = pd; plo[0xe] = pe; plo[0xf] = pf; l = 0; phi[l++] = 0; phi[l++] = p10; phi[l++] = p20; phi[l++] = p30; phi[l++] = p40; phi[l++] = p50; phi[l++] = p60; phi[l++] = p70; phi[l++] = p80; } /*...The radix-144 pass is here. */ for(j = 0; j < NDIVR; j += 2) { #ifdef USE_AVX512 j1 = (j & mask03) + br16[j&15]; #elif defined(USE_AVX) j1 = (j & mask02) + br8[j&7]; #elif defined(USE_SSE2) j1 = (j & mask01) + br4[j&3]; #else j1 = j; #endif j1 += ( (j1>> DAT_BITS) << PAD_BITS ); /* padded-array fetch index is here */ j2 = j1 + RE_IM_STRIDE; /* Twiddleless version arranges 16 sets of radix-9 DFT inputs as follows: 0 in upper left corner, decrement 16 horizontally and 9 vertically (mod 144). Use hex here to match p-indexing ...Can auto-generate these by running test_fft_radix with TTYPE = 0; note this input-offset pattern is shared by DIF and DIT, but DIT layers a generalized bit-reversal atop it: DIF/DIT input-scramble array = [vvv 00,...,10 vvv = basic offset array] leftward-circ-shift count of basic array 00,80,70,60,50,40,30,20,10 00,80,70,60,50,40,30,20,10 + p0 0 87,77,67,57,47,37,27,17,07 80,70,60,50,40,30,20,10,00 + p7 1 7e,6e,5e,4e,3e,2e,1e,0e,8e 70,60,50,40,30,20,10,00,80 + pe 2 75,65,55,45,35,25,15,05,85 70,60,50,40,30,20,10,00,80 + p5 2 6c,5c,4c,3c,2c,1c,0c,8c,7c 60,50,40,30,20,10,00,80,70 + pc 3 63,53,43,33,23,13,03,83,73 60,50,40,30,20,10,00,80,70 + p3 3 5a,4a,3a,2a,1a,0a,8a,7a,6a 50,40,30,20,10,00,80,70,60 + pa 4 51,41,31,21,11,01,81,71,61 = 50,40,30,20,10,00,80,70,60 + p1 4 48,38,28,18,08,88,78,68,58 40,30,20,10,00,80,70,60,50 + p8 5 3f,2f,1f,0f,8f,7f,6f,5f,4f 30,20,10,00,80,70,60,50,40 + pf 6 36,26,16,06,86,76,66,56,46 30,20,10,00,80,70,60,50,40 + p6 6 2d,1d,0d,8d,7d,6d,5d,4d,3d 20,10,00,80,70,60,50,40,30 + pd 7 24,14,04,84,74,64,54,44,34 20,10,00,80,70,60,50,40,30 + p4 7 1b,0b,8b,7b,6b,5b,4b,3b,2b 10,00,80,70,60,50,40,30,20 + pb 8 12,02,82,72,62,52,42,32,22 10,00,80,70,60,50,40,30,20 + p2 8 09,89,79,69,59,49,39,29,19 00,80,70,60,50,40,30,20,10 + p9 0 */ //...gather the needed data (144 64-bit complex) and do 16 radix-9 transforms: tptr = t; for(l = 0; l < 16; l++) { iptr = dif_pcshft + dif_ncshft[l]; // Hi-part of p-offset indices: k0 = phi[*iptr]; k1 = phi[*(iptr+0x1)]; k2 = phi[*(iptr+0x2)]; k3 = phi[*(iptr+0x3)]; k4 = phi[*(iptr+0x4)]; k5 = phi[*(iptr+0x5)]; k6 = phi[*(iptr+0x6)]; k7 = phi[*(iptr+0x7)]; k8 = phi[*(iptr+0x8)]; jp = plo[((l<<3)-l) & 0xf]; // Low-part offset = p[7*l (mod 16)] ... jt = j1 + jp; jp += j2; RADIX_09_DIF( a[jt+k0],a[jp+k0],a[jt+k1],a[jp+k1],a[jt+k2],a[jp+k2],a[jt+k3],a[jp+k3],a[jt+k4],a[jp+k4],a[jt+k5],a[jp+k5],a[jt+k6],a[jp+k6],a[jt+k7],a[jp+k7],a[jt+k8],a[jp+k8], tptr->re,tptr->im,(tptr+0x10)->re,(tptr+0x10)->im,(tptr+0x20)->re,(tptr+0x20)->im,(tptr+0x30)->re,(tptr+0x30)->im,(tptr+0x40)->re,(tptr+0x40)->im,(tptr+0x50)->re,(tptr+0x50)->im,(tptr+0x60)->re,(tptr+0x60)->im,(tptr+0x70)->re,(tptr+0x70)->im,(tptr+0x80)->re,(tptr+0x80)->im, rt,it,re ); tptr++; } /*...and now do 9 radix-16 transforms: Use the supercalafragalistic Ancient Chinese Secret index-munging formula [SACSIMPF] to properly permute the outputs of the radix-16 DFTs to the required ordering, which in terms of our p-offsets is 00,01,02,03,05,04,07,06,0b,0a,08,09,0e,0f,0d,0c 0,1,2,3,5,4,7,6,b,a,8,9,e,f,d,c 81,80,83,82,84,85,86,87,8a,8b,89,88,8f,8e,8c,8d 1,0,3,2,4,5,6,7,a,b,9,8,f,e,c,d + p80 7d,7c,7f,7e,7a,7b,79,78,71,70,73,72,74,75,76,77 d,c,f,e,a,b,9,8,1,0,3,2,4,5,6,7 + p50 67,66,64,65,61,60,63,62,6d,6c,6f,6e,6a,6b,69,68 = 7,6,4,5,1,0,3,2,d,c,f,e,a,b,9,8 + p20 58,59,5a,5b,5d,5c,5f,5e,57,56,54,55,51,50,53,52 8,9,a,b,d,c,f,e,7,6,4,5,1,0,3,2 + p70 42,43,41,40,47,46,44,45,48,49,4a,4b,4d,4c,4f,4e 2,3,1,0,7,6,4,5,8,9,a,b,d,c,f,e + p40 3e,3f,3d,3c,38,39,3a,3b,32,33,31,30,37,36,34,35 e,f,d,c,8,9,a,b,2,3,1,0,7,6,4,5 + p10 25,24,27,26,22,23,21,20,2e,2f,2d,2c,28,29,2a,2b 5,4,7,6,2,3,1,0,e,f,d,c,8,9,a,b + p60 1b,1a,18,19,1e,1f,1d,1c,15,14,17,16,12,13,11,10 b,a,8,9,e,f,d,c,5,4,7,6,2,3,1,0 + p30 Directly convert each 16-perm above into the hex digits of a little-endian uint64, store thes in dif16_oidx_lo[]. */ tptr = t; for(l = 0; l < 9; l++) { i64 = dif16_oidx_lo[l]; // p-offset indices encoded in little-endian hex-char fashion: k0 = plo[(i64 >> 60)&0xf]; k1 = plo[(i64 >> 56)&0xf]; k2 = plo[(i64 >> 52)&0xf]; k3 = plo[(i64 >> 48)&0xf]; k4 = plo[(i64 >> 44)&0xf]; k5 = plo[(i64 >> 40)&0xf]; k6 = plo[(i64 >> 36)&0xf]; k7 = plo[(i64 >> 32)&0xf]; k8 = plo[(i64 >> 28)&0xf]; k9 = plo[(i64 >> 24)&0xf]; ka = plo[(i64 >> 20)&0xf]; kb = plo[(i64 >> 16)&0xf]; kc = plo[(i64 >> 12)&0xf]; kd = plo[(i64 >> 8)&0xf]; ke = plo[(i64 >> 4)&0xf]; kf = plo[(i64 )&0xf]; jp = phi[dif_phi[l]]; // = p10*[0,8,5,2,7,4,1,6,3] jt = j1 + jp; jp += j2; RADIX_16_DIF( tptr->re,tptr->im,(tptr+0x1)->re,(tptr+0x1)->im,(tptr+0x2)->re,(tptr+0x2)->im,(tptr+0x3)->re,(tptr+0x3)->im,(tptr+0x4)->re,(tptr+0x4)->im,(tptr+0x5)->re,(tptr+0x5)->im,(tptr+0x6)->re,(tptr+0x6)->im,(tptr+0x7)->re,(tptr+0x7)->im,(tptr+0x8)->re,(tptr+0x8)->im,(tptr+0x9)->re,(tptr+0x9)->im,(tptr+0xa)->re,(tptr+0xa)->im,(tptr+0xb)->re,(tptr+0xb)->im,(tptr+0xc)->re,(tptr+0xc)->im,(tptr+0xd)->re,(tptr+0xd)->im,(tptr+0xe)->re,(tptr+0xe)->im,(tptr+0xf)->re,(tptr+0xf)->im, a[jt+k0],a[jp+k0],a[jt+k1],a[jp+k1],a[jt+k2],a[jp+k2],a[jt+k3],a[jp+k3],a[jt+k4],a[jp+k4],a[jt+k5],a[jp+k5],a[jt+k6],a[jp+k6],a[jt+k7],a[jp+k7],a[jt+k8],a[jp+k8],a[jt+k9],a[jp+k9],a[jt+ka],a[jp+ka],a[jt+kb],a[jp+kb],a[jt+kc],a[jp+kc],a[jt+kd],a[jp+kd],a[jt+ke],a[jp+ke],a[jt+kf],a[jp+kf], c16,s16); tptr += 0x10; } } } /***************/ void radix144_dit_pass1(double a[], int n) { /* !...Acronym: DIT = Decimation In Time ! !...Subroutine to perform an initial radix-144 complex DIT FFT pass on the data in the length-N real vector A. */ const double c = 0.76604444311897803520, /* cos(2*pi/9) */ s = 0.64278760968653932631, /* sin(2*pi/9) */ c2 = 0.17364817766693034887, /* cos(2*u) */ s2 = 0.98480775301220805936, /* sin(2*u) */ c3m1= -1.50000000000000000000, /* cos(3*u)-1 */ s3 = 0.86602540378443864677, /* sin(3*u) */ c4 = -0.93969262078590838404, /* cos(4*u) */ s4 = 0.34202014332566873307; /* sin(4*u) */ int l,j,j1,j2,jt,jp; const int *iptr; // p-indexing is hexadecimal here: static int NDIVR,p1,p2,p3,p4,p5,p6,p7,p8,p9,pa,pb,pc,pd,pe,pf ,p10,p20,p30,p40,p50,p60,p70,p80, first_entry=TRUE; const uint64 dit16_iidx_lo[9] = { 0x01327654fedcba98ull, 0x5467102398abefcdull, 0xab89cdfe23014576ull, 0x76543210ba98dcefull, 0x10236745efcdab89ull, 0xcdfe89ba45760132ull, 0x32105467dcef98abull, 0xefcdab8967452301ull, 0x89bafedc01327654ull }; static int plo[16],phi[9]; // circ-shift count of basic array needed for stage 2: const int dit_phi[9] = {0,2,1,6,8,7,4,3,5}, dit_ncshft[16] = {0,6,8,1,3,5,7,0,2,4,6,8,1,3,5,7}, // dit_pcshft has 9 base elts followed by repeat of first 8 of those to support circ-shift perms dit_pcshft[17] = {0,2,4,6,8,1,3,5,7,0,2,4,6,8,1,3,5}; uint64 i64; int k0,k1,k2,k3,k4,k5,k6,k7,k8,k9,ka,kb,kc,kd,ke,kf; struct complex t[RADIX], *tptr; double rt,it,re; if(!first_entry && (n/RADIX) != NDIVR) /* New runlength? */ { first_entry=TRUE; } /*...initialize things upon first entry */ if(first_entry) { first_entry=FALSE; NDIVR = n/RADIX; p1 = NDIVR; p2 = p1 + p1; p3 = p2 + p1; p4 = p3 + p1; p5 = p4 + p1; p6 = p5 + p1; p7 = p6 + p1; p8 = p7 + p1; p9 = p8 + p1; pa = p9 + p1; pb = pa + p1; pc = pb + p1; pd = pc + p1; pe = pd + p1; pf = pe + p1; p10 = pf + p1; p20 = p10 + p10; p30 = p20 + p10; p40 = p30 + p10; p50 = p40 + p10; p60 = p50 + p10; p70 = p60 + p10; p80 = p70 + p10; p1 += ( (p1 >> DAT_BITS) << PAD_BITS ); p2 += ( (p2 >> DAT_BITS) << PAD_BITS ); p3 += ( (p3 >> DAT_BITS) << PAD_BITS ); p4 += ( (p4 >> DAT_BITS) << PAD_BITS ); p5 += ( (p5 >> DAT_BITS) << PAD_BITS ); p6 += ( (p6 >> DAT_BITS) << PAD_BITS ); p7 += ( (p7 >> DAT_BITS) << PAD_BITS ); p8 += ( (p8 >> DAT_BITS) << PAD_BITS ); p9 += ( (p9 >> DAT_BITS) << PAD_BITS ); pa += ( (pa >> DAT_BITS) << PAD_BITS ); pb += ( (pb >> DAT_BITS) << PAD_BITS ); pc += ( (pc >> DAT_BITS) << PAD_BITS ); pd += ( (pd >> DAT_BITS) << PAD_BITS ); pe += ( (pe >> DAT_BITS) << PAD_BITS ); pf += ( (pf >> DAT_BITS) << PAD_BITS ); p10 += ( (p10 >> DAT_BITS) << PAD_BITS ); p20 += ( (p20 >> DAT_BITS) << PAD_BITS ); p30 += ( (p30 >> DAT_BITS) << PAD_BITS ); p40 += ( (p40 >> DAT_BITS) << PAD_BITS ); p50 += ( (p50 >> DAT_BITS) << PAD_BITS ); p60 += ( (p60 >> DAT_BITS) << PAD_BITS ); p70 += ( (p70 >> DAT_BITS) << PAD_BITS ); p80 += ( (p80 >> DAT_BITS) << PAD_BITS ); plo[0x0] = 0; plo[0x1] = p1; plo[0x2] = p2; plo[0x3] = p3; plo[0x4] = p4; plo[0x5] = p5; plo[0x6] = p6; plo[0x7] = p7; plo[0x8] = p8; plo[0x9] = p9; plo[0xa] = pa; plo[0xb] = pb; plo[0xc] = pc; plo[0xd] = pd; plo[0xe] = pe; plo[0xf] = pf; l = 0; phi[l++] = 0; phi[l++] = p10; phi[l++] = p20; phi[l++] = p30; phi[l++] = p40; phi[l++] = p50; phi[l++] = p60; phi[l++] = p70; phi[l++] = p80; } /*...The radix-144 pass is here. */ for(j = 0; j < NDIVR; j += 2) { #ifdef USE_AVX512 j1 = (j & mask03) + br16[j&15]; #elif defined(USE_AVX) j1 = (j & mask02) + br8[j&7]; #elif defined(USE_SSE2) j1 = (j & mask01) + br4[j&3]; #else j1 = j; #endif j1 += ( (j1>> DAT_BITS) << PAD_BITS ); /* padded-array fetch index is here */ j2 = j1 + RE_IM_STRIDE; //...gather the needed data (144 64-bit complex) and do 9 radix-16 transforms: /* Twiddleless version uses same linear-index-vector-form permutation as in DIF - Remember, inputs to DIT are bit-reversed, so use output of test_fft_radix() with TTYPE=0 to auto-generate needed input-index permutation: Combined DIT input-scramble array = 00,01,03,02,07,06,05,04,0f,0e,0d,0c,0b,0a,09,08 0,1,3,2,7,6,5,4,f,e,d,c,b,a,9,8 25,24,26,27,21,20,22,23,29,28,2a,2b,2e,2f,2c,2d 5,4,6,7,1,0,2,3,9,8,a,b,e,f,c,d + p20 1a,1b,18,19,1c,1d,1f,1e,12,13,10,11,14,15,17,16 a,b,8,9,c,d,f,e,2,3,0,1,4,5,7,6 + p10 67,66,65,64,63,62,61,60,6b,6a,69,68,6d,6c,6e,6f 7,6,5,4,3,2,1,0,b,a,9,8,d,c,e,f + p60 81,80,82,83,86,87,84,85,8e,8f,8c,8d,8a,8b,88,89 = 1,0,2,3,6,7,4,5,e,f,c,d,a,b,8,9 + p80 7c,7d,7f,7e,78,79,7b,7a,74,75,77,76,70,71,73,72 c,d,f,e,8,9,b,a,4,5,7,6,0,1,3,2 + p70 43,42,41,40,45,44,46,47,4d,4c,4e,4f,49,48,4a,4b 3,2,1,0,5,4,6,7,d,c,e,f,9,8,a,b + p40 3e,3f,3c,3d,3a,3b,38,39,36,37,34,35,32,33,30,31 e,f,c,d,a,b,8,9,6,7,4,5,2,3,0,1 + p30 58,59,5b,5a,5f,5e,5d,5c,50,51,53,52,57,56,55,54 8,9,b,a,f,e,d,c,0,1,3,2,7,6,5,4 + p50 In order to support a compact-object-code version of the above simply encode each 16-perm as a hex-char string: *NOTE* this means we must extract each p-offset in little-endian fashion, e.g. low 4 bits have rightmost p-offset above. */ tptr = t; for(l = 0; l < 9; l++) { i64 = dit16_iidx_lo[l]; // p-offset indices encoded in little-endian hex-char fashion: k0 = plo[(i64 >> 60)&0xf]; k1 = plo[(i64 >> 56)&0xf]; k2 = plo[(i64 >> 52)&0xf]; k3 = plo[(i64 >> 48)&0xf]; k4 = plo[(i64 >> 44)&0xf]; k5 = plo[(i64 >> 40)&0xf]; k6 = plo[(i64 >> 36)&0xf]; k7 = plo[(i64 >> 32)&0xf]; k8 = plo[(i64 >> 28)&0xf]; k9 = plo[(i64 >> 24)&0xf]; ka = plo[(i64 >> 20)&0xf]; kb = plo[(i64 >> 16)&0xf]; kc = plo[(i64 >> 12)&0xf]; kd = plo[(i64 >> 8)&0xf]; ke = plo[(i64 >> 4)&0xf]; kf = plo[(i64 )&0xf]; jp = phi[dit_phi[l]]; // offsets = p10*[0,2,1,6,8,7,4,3,5] jt = j1 + jp; jp += j2; RADIX_16_DIT( a[jt+k0],a[jp+k0],a[jt+k1],a[jp+k1],a[jt+k2],a[jp+k2],a[jt+k3],a[jp+k3],a[jt+k4],a[jp+k4],a[jt+k5],a[jp+k5],a[jt+k6],a[jp+k6],a[jt+k7],a[jp+k7],a[jt+k8],a[jp+k8],a[jt+k9],a[jp+k9],a[jt+ka],a[jp+ka],a[jt+kb],a[jp+kb],a[jt+kc],a[jp+kc],a[jt+kd],a[jp+kd],a[jt+ke],a[jp+ke],a[jt+kf],a[jp+kf], tptr->re,tptr->im,(tptr+0x1)->re,(tptr+0x1)->im,(tptr+0x2)->re,(tptr+0x2)->im,(tptr+0x3)->re,(tptr+0x3)->im,(tptr+0x4)->re,(tptr+0x4)->im,(tptr+0x5)->re,(tptr+0x5)->im,(tptr+0x6)->re,(tptr+0x6)->im,(tptr+0x7)->re,(tptr+0x7)->im,(tptr+0x8)->re,(tptr+0x8)->im,(tptr+0x9)->re,(tptr+0x9)->im,(tptr+0xa)->re,(tptr+0xa)->im,(tptr+0xb)->re,(tptr+0xb)->im,(tptr+0xc)->re,(tptr+0xc)->im,(tptr+0xd)->re,(tptr+0xd)->im,(tptr+0xe)->re,(tptr+0xe)->im,(tptr+0xf)->re,(tptr+0xf)->im, c16,s16); tptr += 0x10; } /*...and now do 16 radix-9 transforms: Since our first-look oindex ordering was +p0x[0,10,20,30,40,50,60,70,80] for each radix-9 and incr += p1 between those DFTs, arrange resulting mismatched-data-sorted index permutation into 9 vertical 16-entry columns to get needed oindex patterns: [vvvvv = basic offset array] leftward-circ-shift count of basic array 00,20,40,60,80,10,30,50,70 00,20,40,60,80,10,30,50,70 + p0 0 3f,5f,7f,0f,2f,4f,6f,8f,1f 30,50,70,00,20,40,60,80,10 + pf 6 7e,0e,2e,4e,6e,8e,1e,3e,5e 70,00,20,40,60,80,10,30,50 + pe 8 2d,4d,6d,8d,1d,3d,5d,7d,0d 20,40,60,80,10,30,50,70,00 + pd 1 6c,8c,1c,3c,5c,7c,0c,2c,4c 60,80,10,30,50,70,00,20,40 + pc 3 1b,3b,5b,7b,0b,2b,4b,6b,8b 10,30,50,70,00,20,40,60,80 + pb 5 5a,7a,0a,2a,4a,6a,8a,1a,3a 50,70,00,20,40,60,80,10,30 + pa 7 09,29,49,69,89,19,39,59,79 = 00,20,40,60,80,10,30,50,70 + p9 0 48,68,88,18,38,58,78,08,28 40,60,80,10,30,50,70,00,20 + p8 2 87,17,37,57,77,07,27,47,67 80,10,30,50,70,00,20,40,60 + p7 4 36,56,76,06,26,46,66,86,16 30,50,70,00,20,40,60,80,10 + p6 6 75,05,25,45,65,85,15,35,55 70,00,20,40,60,80,10,30,50 + p5 8 24,44,64,84,14,34,54,74,04 20,40,60,80,10,30,50,70,00 + p4 1 63,83,13,33,53,73,03,23,43 60,80,10,30,50,70,00,20,40 + p3 3 12,32,52,72,02,22,42,62,82 10,30,50,70,00,20,40,60,80 + p2 5 51,71,01,21,41,61,81,11,31 50,70,00,20,40,60,80,10,30 + p1 7 */ tptr = t; for(l = 0; l < 16; l++) { iptr = dit_pcshft + dit_ncshft[l]; // Hi-part of p-offset indices: k0 = phi[*iptr]; k1 = phi[*(iptr+0x1)]; k2 = phi[*(iptr+0x2)]; k3 = phi[*(iptr+0x3)]; k4 = phi[*(iptr+0x4)]; k5 = phi[*(iptr+0x5)]; k6 = phi[*(iptr+0x6)]; k7 = phi[*(iptr+0x7)]; k8 = phi[*(iptr+0x8)]; jp = plo[(16 - l)&0xf]; // Low-part offset = p0,f,e,...,2,1 jt = j1 + jp; jp += j2; RADIX_09_DIT( tptr->re,tptr->im,(tptr+0x10)->re,(tptr+0x10)->im,(tptr+0x20)->re,(tptr+0x20)->im,(tptr+0x30)->re,(tptr+0x30)->im,(tptr+0x40)->re,(tptr+0x40)->im,(tptr+0x50)->re,(tptr+0x50)->im,(tptr+0x60)->re,(tptr+0x60)->im,(tptr+0x70)->re,(tptr+0x70)->im,(tptr+0x80)->re,(tptr+0x80)->im, a[jt+k0],a[jp+k0],a[jt+k1],a[jp+k1],a[jt+k2],a[jp+k2],a[jt+k3],a[jp+k3],a[jt+k4],a[jp+k4],a[jt+k5],a[jp+k5],a[jt+k6],a[jp+k6],a[jt+k7],a[jp+k7],a[jt+k8],a[jp+k8], rt,it,re ); tptr++; } } } /******************** Multithreaded function body - NO STATIC VARS BELOW THIS POINT!: ***************************/ #ifdef USE_PTHREAD #ifndef COMPILER_TYPE_GCC #error pthreaded carry code requires GCC build! #endif void* cy144_process_chunk(void*targ) // Thread-arg pointer *must* be cast to void and specialized inside the function { struct cy_thread_data_t* thread_arg = targ; // Move to top because scalar-mode carry pointers taken directly from it double *addr; const int pfetch_dist = PFETCH_DIST; const int stride = (int)RE_IM_STRIDE << 1; // main-array loop stride = 2*RE_IM_STRIDE uint32 p1,p2,p3,p4,p5,p6,p7,p8,p9,pa,pb,pc,pd,pe,pf ,p10,p20,p30,p40,p50,p60,p70,p80,p90,pa0; int poff[RADIX>>2]; // Store [RADIX/4] mults of p04 offset for loop control // Shared DIF+DIT: const int *iptr; int plo[16],phi[9]; uint64 i64; int k0,k1,k2,k3,k4,k5,k6,k7,k8,k9,ka,kb,kc,kd,ke,kf,po_kperm[16],*po_ptr = &(po_kperm[0]); double rt,it,re; double wt_re,wt_im, wi_re,wi_im; // Fermat-mod/LOACC weights stuff, used in both scalar and SIMD mode // DIF: const uint64 dif16_oidx_lo[9] = { 0x01327654efcdab89ull, 0x10236745fedcba98ull, 0x89bafedc10236745ull, 0x5467102389bafedcull, 0xcdfe89ba54671023ull, 0x32105467cdfe89baull, 0xab89cdfe32105467ull, 0x76543210ab89cdfeull, 0xefcdab8976543210ull }; const int dif_phi[9] = {0,8,5,2,7,4,1,6,3}, // circ-shift count of basic array needed for stage 1: dif_ncshft[16] = {0x0,0x1,0x2,0x2,0x3,0x3,0x4,0x4,0x5,0x6,0x6,0x7,0x7,0x8,0x8,0x0}, // dif_pcshft has 9 base elts followed by repeat of first 8 of those to support circ-shift perms dif_pcshft[17] = {0,8,7,6,5,4,3,2,1,0,8,7,6,5,4,3,2}; // DIT: const uint64 dit16_iidx_lo[9] = { 0x01327654fedcba98ull, 0x5467102398abefcdull, 0xab89cdfe23014576ull, 0x76543210ba98dcefull, 0x10236745efcdab89ull, 0xcdfe89ba45760132ull, 0x32105467dcef98abull, 0xefcdab8967452301ull, 0x89bafedc01327654ull }; // circ-shift count of basic array needed for stage 2: const int dit_phi[9] = {0,2,1,6,8,7,4,3,5}, dit_ncshft[16] = {0,6,8,1,3,5,7,0,2,4,6,8,1,3,5,7}, // dit_pcshft has 9 base elts followed by repeat of first 8 of those to support circ-shift perms dit_pcshft[17] = {0,2,4,6,8,1,3,5,7,0,2,4,6,8,1,3,5}; int incr,j,j1,j2,jt,jp,k,l,ntmp; // incr = Carry-chain wts-multipliers recurrence length, which must divide // RADIX/[n-wayness of carry macro], e.g. RADIX/[16|8|4] = 9|18|36 for avx512,avx,sse, respectively: #ifdef USE_AVX512 // Oddly, FMA-based builds appear to need slightly shorter chains for radices 144,288 const int incr_long = 9,incr_med = 3,incr_short = 3; #elif defined(USE_AVX2) const int incr_long = 9,incr_med = 6,incr_short = 3; #else const int incr_long = 18,incr_med = 9,incr_short = 6; // Note: lump SSE2+AVX together for this radix, thus must divide 18! #endif // Have no specialized HIACC carry macro in USE_AVX512 and ARMv8 SIMD, so use "goes to 11" in LOACC mode via an incr_hiacc = 1: #if defined(USE_AVX512) || defined(USE_ARM_V8_SIMD) const int incr_hiacc = 1; #else const int incr_hiacc = 0; #endif // Allows cy-macro error data to be used to fiddle incr on the fly to a smaller, safer value if necessary if(USE_SHORT_CY_CHAIN == 0) incr = incr_long; else if(USE_SHORT_CY_CHAIN == 1) incr = incr_med; else if(USE_SHORT_CY_CHAIN == 2) incr = incr_short; else incr = incr_hiacc; #ifdef USE_AVX512 double t0,t1,t2,t3; #ifdef CARRY_16_WAY struct uint32x16 *n_minus_sil,*n_minus_silp1,*sinwt,*sinwtm1; #else struct uint32x8 *n_minus_sil,*n_minus_silp1,*sinwt,*sinwtm1; #endif #elif defined(USE_AVX) struct uint32x4 *n_minus_sil,*n_minus_silp1,*sinwt,*sinwtm1; #else int n_minus_sil,n_minus_silp1,sinwt,sinwtm1; double wtl,wtlp1,wtn,wtnm1; /* Mersenne-mod weights stuff */ #endif #ifdef USE_AVX2 // Due to GCC macro argc limit of 30, to enable 16-register data-doubled version of the radix-9 macros need 2 length-9 ptr arrays: vec_dbl *rad9_iptr[9], *rad9_optr[9]; #endif #ifdef USE_SSE2 const double crnd = 3.0*0x4000000*0x2000000; int *itmp,*itm2; // Pointer into the bjmodn array struct complex *ctmp; // Hybrid AVX-DFT/SSE2-carry scheme used for Mersenne-mod needs a 2-word-double pointer double *add0,*add1,*add2,*add3; /* Addresses into array sections */ int *bjmodn; // Alloc mem for this along with other SIMD stuff vec_dbl *tmp,*tm0,*tm1,*tm2, // Non-static utility ptrs *va0,*va1,*va2,*va3,*va4,*va5,*va6,*va7,*va8, *vb0,*vb1,*vb2,*vb3,*vb4,*vb5,*vb6,*vb7,*vb8, *two,*one,*sqrt2,*isrt2,*cc0,*ss0, *cc1,*ss1,*cc2,*ss2,*cc3m1,*ss3,*cc4,*ss4, *max_err, *sse2_rnd, *half_arr, *r00, // Head of RADIX*vec_cmplx-sized local store #1 *s1p00, // Head of RADIX*vec_cmplx-sized local store #2 *cy; // Need RADIX/2 slots for sse2 carries, RADIX/4 for avx double dtmp; uint64 *sign_mask, *sse_bw, *sse_sw, *sse_n; #else const double c = 0.76604444311897803520, /* cos(2*pi/9) */ s = 0.64278760968653932631, /* sin(2*pi/9) */ c2 = 0.17364817766693034887, /* cos(2*u) */ s2 = 0.98480775301220805936, /* sin(2*u) */ c3m1= -1.50000000000000000000, /* cos(3*u)-1 */ s3 = 0.86602540378443864677, /* sin(3*u) */ c4 = -0.93969262078590838404, /* cos(4*u) */ s4 = 0.34202014332566873307; /* sin(4*u) */ double *base, *baseinv; int p0123[4]; const double one_half[3] = {1.0, 0.5, 0.25}; /* Needed for small-weights-tables scheme */ int m,m2; double wt,wtinv,wtA,wtB,wtC; /* Mersenne-mod weights stuff */ int bjmodn[RADIX]; // Thread only carries a base datum here, must alloc a local array for remaining values double *cy = thread_arg->cy, temp,frac; struct complex t[RADIX], *tptr; int *itmp; // Pointer into the bjmodn array #endif // int data: int thr_id = thread_arg->tid; int iter = thread_arg->iter; int NDIVR = thread_arg->ndivr; int n = NDIVR*RADIX; int target_idx = thread_arg->target_idx; int target_set = thread_arg->target_set; double target_cy = thread_arg->target_cy; int khi = thread_arg->khi; int i = thread_arg->i; /* Pointer to the BASE and BASEINV arrays. */ int jstart = thread_arg->jstart; int jhi = thread_arg->jhi; int col = thread_arg->col; int co2 = thread_arg->co2; int co3 = thread_arg->co3; int sw = thread_arg->sw; int nwt = thread_arg->nwt; // double data: double maxerr = thread_arg->maxerr; double scale = thread_arg->scale; int full_pass = scale < 0.5; double prp_mult = thread_arg->prp_mult; // pointer data: double *a = thread_arg->arrdat; double *wt0 = thread_arg->wt0; double *wt1 = thread_arg->wt1; double *wts_mult = thread_arg->wts_mult; // Const Intra-block wts-multiplier... double *inv_mult = thread_arg->inv_mult; // ...and 2*(its multiplicative inverse). ASSERT(HERE,fabs(wts_mult[0]*inv_mult[0] - 1.0) < EPS, "wts_mults fail accuracy check!"); ASSERT(HERE,fabs(wts_mult[1]*inv_mult[1] - 1.0) < EPS, "wts_mults fail accuracy check!"); int *si = thread_arg->si; /* constant index offsets for array load/stores are here. */ p1 = NDIVR; p2 = p1 + p1; p3 = p2 + p1; p4 = p3 + p1; p5 = p4 + p1; p6 = p5 + p1; p7 = p6 + p1; p8 = p7 + p1; p9 = p8 + p1; pa = p9 + p1; pb = pa + p1; pc = pb + p1; pd = pc + p1; pe = pd + p1; pf = pe + p1; p10 = pf + p1; p20 = p10 + p10; p30 = p20 + p10; p40 = p30 + p10; p50 = p40 + p10; p60 = p50 + p10; p70 = p60 + p10; p80 = p70 + p10; p90 = p80 + p10; pa0 = p90 + p10; p1 += ( (p1 >> DAT_BITS) << PAD_BITS ); p2 += ( (p2 >> DAT_BITS) << PAD_BITS ); p3 += ( (p3 >> DAT_BITS) << PAD_BITS ); p4 += ( (p4 >> DAT_BITS) << PAD_BITS ); p5 += ( (p5 >> DAT_BITS) << PAD_BITS ); p6 += ( (p6 >> DAT_BITS) << PAD_BITS ); p7 += ( (p7 >> DAT_BITS) << PAD_BITS ); p8 += ( (p8 >> DAT_BITS) << PAD_BITS ); p9 += ( (p9 >> DAT_BITS) << PAD_BITS ); pa += ( (pa >> DAT_BITS) << PAD_BITS ); pb += ( (pb >> DAT_BITS) << PAD_BITS ); pc += ( (pc >> DAT_BITS) << PAD_BITS ); pd += ( (pd >> DAT_BITS) << PAD_BITS ); pe += ( (pe >> DAT_BITS) << PAD_BITS ); pf += ( (pf >> DAT_BITS) << PAD_BITS ); p10 += ( (p10 >> DAT_BITS) << PAD_BITS ); p20 += ( (p20 >> DAT_BITS) << PAD_BITS ); p30 += ( (p30 >> DAT_BITS) << PAD_BITS ); p40 += ( (p40 >> DAT_BITS) << PAD_BITS ); p50 += ( (p50 >> DAT_BITS) << PAD_BITS ); p60 += ( (p60 >> DAT_BITS) << PAD_BITS ); p70 += ( (p70 >> DAT_BITS) << PAD_BITS ); p80 += ( (p80 >> DAT_BITS) << PAD_BITS ); p90 += ( (p90 >> DAT_BITS) << PAD_BITS ); pa0 += ( (pa0 >> DAT_BITS) << PAD_BITS ); #ifndef USE_SSE2 p0123[0] = 0; p0123[1] = p1; p0123[2] = p2; p0123[3] = p3; #endif poff[ 0] = 0; poff[ 1] = p4; poff[ 2] = p8; poff[ 3] = pc; poff[0x04+0] = p10; poff[0x04+1] = p10+p4; poff[0x04+2] = p10+p8; poff[0x04+3] = p10+pc; poff[0x08+0] = p20; poff[0x08+1] = p20+p4; poff[0x08+2] = p20+p8; poff[0x08+3] = p20+pc; poff[0x0c+0] = p30; poff[0x0c+1] = p30+p4; poff[0x0c+2] = p30+p8; poff[0x0c+3] = p30+pc; poff[0x10+0] = p40; poff[0x10+1] = p40+p4; poff[0x10+2] = p40+p8; poff[0x10+3] = p40+pc; poff[0x14+0] = p50; poff[0x14+1] = p50+p4; poff[0x14+2] = p50+p8; poff[0x14+3] = p50+pc; poff[0x18+0] = p60; poff[0x18+1] = p60+p4; poff[0x18+2] = p60+p8; poff[0x18+3] = p60+pc; poff[0x1c+0] = p70; poff[0x1c+1] = p70+p4; poff[0x1c+2] = p70+p8; poff[0x1c+3] = p70+pc; poff[0x20+0] = p80; poff[0x20+1] = p80+p4; poff[0x20+2] = p80+p8; poff[0x20+3] = p80+pc; // Shared: plo[0x0] = 0; plo[0x1] = p1; plo[0x2] = p2; plo[0x3] = p3; plo[0x4] = p4; plo[0x5] = p5; plo[0x6] = p6; plo[0x7] = p7; plo[0x8] = p8; plo[0x9] = p9; plo[0xa] = pa; plo[0xb] = pb; plo[0xc] = pc; plo[0xd] = pd; plo[0xe] = pe; plo[0xf] = pf; l = 0; phi[l++] = 0; phi[l++] = p10; phi[l++] = p20; phi[l++] = p30; phi[l++] = p40; phi[l++] = p50; phi[l++] = p60; phi[l++] = p70; phi[l++] = p80; #ifdef USE_SSE2 tmp = r00 = thread_arg->r00; // Head of RADIX*vec_cmplx-sized local store #1 tmp += 0x120; s1p00 = tmp; // Head of RADIX*vec_cmplx-sized local store #2 tmp += 0x122; // Extra 2 slots here for one,sqrt2 below - added those late, too lazy to rejigger all the existing offsets following two = tmp - 2; // AVX+ versions of various DFT macros assume consts 2.0,1.0,sqrt2,isrt2 laid out thusly one = tmp - 1; sqrt2 = tmp + 0; // Roots for radix-16 DFTs: isrt2 = tmp + 0x1; cc0 = tmp + 0x2; ss0 = tmp + 0x3; // Roots for radix-9 DFTs: cc1 = tmp + 0x4; ss1 = tmp + 0x5; cc2 = tmp + 0x6; ss2 = tmp + 0x7; cc3m1 = tmp + 0x8; ss3 = tmp + 0x9; cc4 = tmp + 0xa; ss4 = tmp + 0xb; tmp += 0xc; // sc_ptr += 0x24e #ifdef USE_AVX512 cy = tmp; tmp += 0x12; // RADIX/8 vec_dbl slots for carry sub-array max_err = tmp + 0x00; sse2_rnd= tmp + 0x01; half_arr= tmp + 0x02; #elif defined(USE_AVX) cy = tmp; tmp += 0x24; // sc_ptr += 0x272 max_err = tmp + 0x00; sse2_rnd= tmp + 0x01; // sc_ptr += 2 = 0x274; This is where the value of half_arr_offset144 comes from half_arr= tmp + 0x02; // This table needs 68 vec_dbl in AVX mode #else cy = tmp; tmp += 0x48; // sc_ptr += 0x296 max_err = tmp + 0x00; sse2_rnd= tmp + 0x01; // sc_ptr += 2 = 0x296; This is where the value of half_arr_offset144 comes from half_arr= tmp + 0x02; // This table needs 20 x 16 bytes in SSE2 mode #endif ASSERT(HERE, (r00 == thread_arg->r00), "thread-local memcheck failed!"); ASSERT(HERE, (half_arr == thread_arg->half_arr), "thread-local memcheck failed!"); #ifndef USE_AVX512 // In AVX-512 mode, use VRNDSCALEPD for rounding and hijack this vector-data slot for the 4 base/baseinv-consts: ASSERT(HERE, (sse2_rnd->d0 == crnd && sse2_rnd->d1 == crnd), "thread-local memcheck failed!"); #endif tmp = half_arr; #ifdef USE_AVX512 /* No-Op */ #elif defined(USE_AVX) dtmp = (tmp+40)->d0 * (tmp+56)->d0; ASSERT(HERE, fabs(dtmp - 1.0) < EPS, "thread-local memcheck failed!"); dtmp = (tmp+40)->d1 * (tmp+56)->d1; ASSERT(HERE, fabs(dtmp - 1.0) < EPS, "thread-local memcheck failed!"); #else // SSE2: dtmp = (tmp+10)->d0 * (tmp+14)->d0; ASSERT(HERE, fabs(dtmp - 1.0) < EPS, "thread-local memcheck failed!"); dtmp = (tmp+10)->d1 * (tmp+14)->d1; ASSERT(HERE, fabs(dtmp - 1.0) < EPS, "thread-local memcheck failed!"); #endif VEC_DBL_INIT(max_err, 0.0); sign_mask = (uint64*)(r00 + radix144_creals_in_local_store); sse_bw = sign_mask + RE_IM_STRIDE; // ( #doubles in a SIMD complex) x 32-bits = RE_IM_STRIDE x 64-bits sse_sw = sse_bw + RE_IM_STRIDE; sse_n = sse_sw + RE_IM_STRIDE; #ifdef USE_AVX512 #ifdef CARRY_16_WAY n_minus_sil = (struct uint32x16*)sse_n + 1; n_minus_silp1 = (struct uint32x16*)sse_n + 2; sinwt = (struct uint32x16*)sse_n + 3; sinwtm1 = (struct uint32x16*)sse_n + 4; #else n_minus_sil = (struct uint32x8 *)sse_n + 1; n_minus_silp1 = (struct uint32x8 *)sse_n + 2; sinwt = (struct uint32x8 *)sse_n + 3; sinwtm1 = (struct uint32x8 *)sse_n + 4; #endif #elif defined(USE_AVX) n_minus_sil = (struct uint32x4 *)sse_n + 1; n_minus_silp1 = (struct uint32x4 *)sse_n + 2; sinwt = (struct uint32x4 *)sse_n + 3; sinwtm1 = (struct uint32x4 *)sse_n + 4; #endif #ifdef USE_AVX bjmodn = (int*)(sinwtm1 + RE_IM_STRIDE); #else bjmodn = (int*)(sse_n + RE_IM_STRIDE); #endif #else // In scalar mode use these 2 ptrs to pass the base & baseinv arrays: base = (double *)thread_arg->r00 ; baseinv = (double *)thread_arg->half_arr; #endif // USE_SSE2 ? /* Init DWT-indices: */ uint32 bjmodnini = thread_arg->bjmodnini; bjmodn[0] = thread_arg->bjmodn0; for(l = 1; l < RADIX; l++) { // must use e.g. l for loop idx here as i is used for dwt indexing MOD_ADD32(bjmodn[l-1], bjmodnini, n, bjmodn[l]); } /* init carries */ addr = thread_arg->cy; #ifdef USE_AVX512 tmp = cy; for(l = 0; l < RADIX; l += 8, ++tmp) { tmp->d0 = *(addr+l ); tmp->d1 = *(addr+l+1); tmp->d2 = *(addr+l+2); tmp->d3 = *(addr+l+3); tmp->d4 = *(addr+l+4); tmp->d5 = *(addr+l+5); tmp->d6 = *(addr+l+6); tmp->d7 = *(addr+l+7); } #elif defined(USE_AVX) tmp = cy; for(l = 0; l < RADIX; l += 4, ++tmp) { tmp->d0 = *(addr+l ); tmp->d1 = *(addr+l+1); tmp->d2 = *(addr+l+2); tmp->d3 = *(addr+l+3); } #elif defined(USE_SSE2) tmp = cy; for(l = 0; l < RADIX; l += 2, ++tmp) { tmp->d0 = *(addr+l ); tmp->d1 = *(addr+l+1); } #elif 0 // No_op in scalar case, since carry pattern matches that of thread data for(l = 0; l < RADIX; l++) { cy[l] = *(addr+l); } #endif /********************************************************************************/ /* This main loop is same for un-and-multithreaded, so stick into a header file */ /* (can't use a macro because of the #if-enclosed stuff). */ /********************************************************************************/ #include "radix144_main_carry_loop.h" /* At end of each thread-processed work chunk, dump the carryouts into their non-thread-private array slots: */ addr = thread_arg->cy; #ifdef USE_AVX512 tmp = cy; for(l = 0; l < RADIX; l += 8, ++tmp) { *(addr+l ) = tmp->d0; *(addr+l+1) = tmp->d1; *(addr+l+2) = tmp->d2; *(addr+l+3) = tmp->d3; *(addr+l+4) = tmp->d4; *(addr+l+5) = tmp->d5; *(addr+l+6) = tmp->d6; *(addr+l+7) = tmp->d7; } t0 = MAX(max_err->d0,max_err->d1); t1 = MAX(max_err->d2,max_err->d3); t2 = MAX(max_err->d4,max_err->d5); t3 = MAX(max_err->d6,max_err->d7); maxerr = MAX( MAX(t0,t1), MAX(t2,t3) ); #elif defined(USE_AVX) tmp = cy; for(l = 0; l < RADIX; l += 4, ++tmp) { *(addr+l ) = tmp->d0; *(addr+l+1) = tmp->d1; *(addr+l+2) = tmp->d2; *(addr+l+3) = tmp->d3; } maxerr = MAX( MAX(max_err->d0,max_err->d1) , MAX(max_err->d2,max_err->d3) ); #elif defined(USE_SSE2) tmp = cy; for(l = 0; l < RADIX; l += 2, ++tmp) { *(addr+l ) = tmp->d0; *(addr+l+1) = tmp->d1; } maxerr = MAX(max_err->d0,max_err->d1); #elif 0 // No_op in scalar case, since carry pattern matches that of thread data for(l = 0; l < RADIX; l++) { *(addr+l) = cy[l]; } #endif /* Since will lose separate maxerr values when threads are merged, save them after each pass. */ if(thread_arg->maxerr < maxerr) { thread_arg->maxerr = maxerr; } return 0x0; } #endif #undef RADIX #undef PFETCH_DIST