| 1 |
// linker define ntt512_7681 |
| 2 |
// linker define ntt512_10753 |
| 3 |
// linker define invntt512_7681 |
| 4 |
// linker define invntt512_10753 |
| 5 |
|
| 6 |
#include "ntt.h" |
| 7 |
|
| 8 |
// auto-generated; do not edit |
| 9 |
|
| 10 |
#include <immintrin.h> |
| 11 |
|
| 12 |
#define _mm256_permute2x128_si256_lo(f0,f1) _mm256_permute2x128_si256(f0,f1,0x20) |
| 13 |
#define _mm256_permute2x128_si256_hi(f0,f1) _mm256_permute2x128_si256(f0,f1,0x31) |
| 14 |
#define int16x16 __m256i |
| 15 |
|
| 16 |
typedef int16_t int16; |
| 17 |
typedef int32_t int32; |
| 18 |
|
| 19 |
static const int16 __attribute((aligned(32))) qdata_7681[] = { |
| 20 |
#define precomp_8_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 *(const int16x16 *)(qdata+0) |
| 21 |
-3593,-3593,-3593,-3593,-3625,-3625,-3625,-3625,-3593,-3593,-3593,-3593,-3625,-3625,-3625,-3625, |
| 22 |
#define precomp_8_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 *(const int16x16 *)(qdata+16) |
| 23 |
-3777,-3777,-3777,-3777,3182,3182,3182,3182,-3777,-3777,-3777,-3777,3182,3182,3182,3182, |
| 24 |
#define precomp_8_7_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 *(const int16x16 *)(qdata+32) |
| 25 |
-3593,-3593,-3593,-3593,-3182,-3182,-3182,-3182,-3593,-3593,-3593,-3593,-3182,-3182,-3182,-3182, |
| 26 |
#define precomp_8_7_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 *(const int16x16 *)(qdata+48) |
| 27 |
3777,3777,3777,3777,3625,3625,3625,3625,3777,3777,3777,3777,3625,3625,3625,3625, |
| 28 |
#define precomp_16_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 *(const int16x16 *)(qdata+64) |
| 29 |
-3593,-3593,-3593,-3593,2194,2194,2194,2194,-3593,-3593,-3593,-3593,2194,2194,2194,2194, |
| 30 |
#define precomp_16_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 *(const int16x16 *)(qdata+80) |
| 31 |
-3625,-3625,-3625,-3625,-1100,-1100,-1100,-1100,-3625,-3625,-3625,-3625,-1100,-1100,-1100,-1100, |
| 32 |
#define precomp_16_15_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 *(const int16x16 *)(qdata+96) |
| 33 |
-3593,-3593,-3593,-3593,3696,3696,3696,3696,-3593,-3593,-3593,-3593,3696,3696,3696,3696, |
| 34 |
#define precomp_16_15_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 *(const int16x16 *)(qdata+112) |
| 35 |
-3182,-3182,-3182,-3182,-2456,-2456,-2456,-2456,-3182,-3182,-3182,-3182,-2456,-2456,-2456,-2456, |
| 36 |
#define precomp_32_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 *(const int16x16 *)(qdata+128) |
| 37 |
-3593,1701,2194,834,-3625,2319,-1100,121,-3593,1701,2194,834,-3625,2319,-1100,121, |
| 38 |
#define precomp_32_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+144) |
| 39 |
-3777,1414,2456,2495,3182,2876,-3696,2250,-3777,1414,2456,2495,3182,2876,-3696,2250, |
| 40 |
#define precomp_32_31_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 *(const int16x16 *)(qdata+160) |
| 41 |
-3593,-2250,3696,-2876,-3182,-2495,-2456,-1414,-3593,-2250,3696,-2876,-3182,-2495,-2456,-1414, |
| 42 |
#define precomp_32_31_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+176) |
| 43 |
3777,-121,1100,-2319,3625,-834,-2194,-1701,3777,-121,1100,-2319,3625,-834,-2194,-1701, |
| 44 |
#define precomp_64_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 *(const int16x16 *)(qdata+192) |
| 45 |
-3593,3364,1701,-1599,2194,2557,834,-2816,-3593,3364,1701,-1599,2194,2557,834,-2816, |
| 46 |
#define precomp_64_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+208) |
| 47 |
-3625,617,2319,2006,-1100,-1296,121,1986,-3625,617,2319,2006,-1100,-1296,121,1986, |
| 48 |
#define precomp_64_63_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 *(const int16x16 *)(qdata+224) |
| 49 |
-3593,2237,-2250,-1483,3696,3706,-2876,1921,-3593,2237,-2250,-1483,3696,3706,-2876,1921, |
| 50 |
#define precomp_64_63_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+240) |
| 51 |
-3182,2088,-2495,-1525,-2456,1993,-1414,2830,-3182,2088,-2495,-1525,-2456,1993,-1414,2830, |
| 52 |
#define precomp_128_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+256) |
| 53 |
-3593,514,3364,438,1701,2555,-1599,-1738,2194,103,2557,1881,834,-549,-2816,638, |
| 54 |
#define precomp_128_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+272) |
| 55 |
-3625,-1399,617,-1760,2319,2535,2006,3266,-1100,-1431,-1296,3174,121,3153,1986,-810, |
| 56 |
#define precomp_128_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+288) |
| 57 |
-3777,2956,-2830,-679,1414,2440,-1993,-3689,2456,2804,1525,3555,2495,1535,-2088,-7, |
| 58 |
#define precomp_128_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+304) |
| 59 |
3182,-1321,-1921,-1305,2876,-3772,-3706,3600,-3696,-2043,1483,-396,2250,-2310,-2237,1887, |
| 60 |
#define precomp_128_127_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+320) |
| 61 |
-3593,-1887,2237,2310,-2250,396,-1483,2043,3696,-3600,3706,3772,-2876,1305,1921,1321, |
| 62 |
#define precomp_128_127_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+336) |
| 63 |
-3182,7,2088,-1535,-2495,-3555,-1525,-2804,-2456,3689,1993,-2440,-1414,679,2830,-2956, |
| 64 |
#define precomp_128_127_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+352) |
| 65 |
3777,810,-1986,-3153,-121,-3174,1296,1431,1100,-3266,-2006,-2535,-2319,1760,-617,1399, |
| 66 |
#define precomp_128_127_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+368) |
| 67 |
3625,-638,2816,549,-834,-1881,-2557,-103,-2194,1738,1599,-2555,-1701,-438,-3364,-514, |
| 68 |
#define precomp_256_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+384) |
| 69 |
-3593,-1532,514,-373,3364,-3816,438,-3456,1701,783,2555,2883,-1599,727,-1738,-2385, |
| 70 |
#define precomp_256_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+400) |
| 71 |
2194,-2160,103,-2391,2557,2762,1881,-2426,834,3310,-549,-1350,-2816,1386,638,-194, |
| 72 |
#define precomp_256_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+416) |
| 73 |
-3625,404,-1399,-3692,617,-2764,-1760,-1054,2319,1799,2535,-3588,2006,1533,3266,2113, |
| 74 |
#define precomp_256_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+432) |
| 75 |
-1100,-2579,-1431,-1756,-1296,1598,3174,-2,121,-3480,3153,-2572,1986,2743,-810,2919, |
| 76 |
#define precomp_256_255_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+448) |
| 77 |
-3593,2789,-1887,-921,2237,-1497,2310,-2133,-2250,-915,396,1390,-1483,3135,2043,-859, |
| 78 |
#define precomp_256_255_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+464) |
| 79 |
3696,2732,-3600,-1464,3706,2224,3772,-2665,-2876,1698,1305,2835,1921,730,1321,486, |
| 80 |
#define precomp_256_255_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+480) |
| 81 |
-3182,3417,7,-3428,2088,-3145,-1535,1168,-2495,-3831,-3555,-3750,-1525,660,-2804,2649, |
| 82 |
#define precomp_256_255_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+496) |
| 83 |
-2456,3405,3689,-1521,1993,1681,-2440,1056,-1414,1166,679,-2233,2830,2175,-2956,-1919, |
| 84 |
#define precomp_512_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+512) |
| 85 |
-3593,-1404,-1532,451,514,-402,-373,1278,3364,-509,-3816,-3770,438,-2345,-3456,-226, |
| 86 |
#define precomp_512_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+528) |
| 87 |
1701,-1689,783,-1509,2555,2963,2883,1242,-1599,1669,727,2719,-1738,642,-2385,-436, |
| 88 |
#define precomp_512_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+544) |
| 89 |
2194,3335,-2160,1779,103,3745,-2391,17,2557,2812,2762,-1144,1881,83,-2426,-1181, |
| 90 |
#define precomp_512_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+560) |
| 91 |
834,-1519,3310,3568,-549,-796,-1350,2072,-2816,-2460,1386,2891,638,-2083,-194,-715, |
| 92 |
#define precomp_512_5_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+576) |
| 93 |
-3593,-402,-3816,-226,2555,1669,-2385,1779,2557,83,3310,2072,638,1012,-3692,1295, |
| 94 |
#define precomp_512_5_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+592) |
| 95 |
2319,-3208,1533,-2071,-1431,-2005,-2,1586,1986,-293,1919,-929,-679,777,-1681,-3461, |
| 96 |
#define precomp_512_5_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+608) |
| 97 |
2456,3366,3750,-1203,1535,-3657,-3417,-1712,-1921,2515,2665,-1070,3600,2532,-3135,-2589, |
| 98 |
#define precomp_512_5_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+624) |
| 99 |
2250,-2258,921,-658,-514,509,3456,1509,1599,-642,2160,-17,-1881,1519,1350,-2891, |
| 100 |
#define precomp_512_507_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+640) |
| 101 |
-3593,-3434,-1497,893,396,-2422,-859,2965,3706,-2339,1698,-2937,1321,-670,-3428,-3163, |
| 102 |
#define precomp_512_507_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+656) |
| 103 |
-2495,-1072,660,1084,3689,-179,1056,-1338,2830,2786,-2919,-3677,-3153,-151,-1598,3334, |
| 104 |
#define precomp_512_507_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+672) |
| 105 |
1100,-3314,3588,2262,1760,-2230,-404,2083,2816,-3568,2426,-2812,-103,436,-727,-2963, |
| 106 |
#define precomp_512_507_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+688) |
| 107 |
-1701,3770,373,1404,1887,-1649,2133,-826,1483,434,-2732,3287,-3772,-2378,-2835,3723, |
| 108 |
#define precomp_512_511_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+704) |
| 109 |
-3593,658,2789,370,-1887,-3434,-921,-3752,2237,1649,-1497,2258,2310,3581,-2133,893, |
| 110 |
#define precomp_512_511_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+720) |
| 111 |
-2250,3794,-915,826,396,2589,1390,592,-1483,-2422,3135,3214,2043,-434,-859,-2532, |
| 112 |
#define precomp_512_511_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+736) |
| 113 |
3696,1121,2732,2965,-3600,2998,-1464,-3287,3706,1070,2224,-589,3772,-2339,-2665,2070, |
| 114 |
#define precomp_512_511_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+752) |
| 115 |
-2876,2378,1698,-2515,1305,-2815,2835,-2937,1921,-1348,730,-3723,1321,1712,486,2130, |
| 116 |
#define q_x16 *(const int16x16 *)(qdata+768) |
| 117 |
7681,7681,7681,7681,7681,7681,7681,7681,7681,7681,7681,7681,7681,7681,7681,7681, |
| 118 |
#define qinvprecomp_8_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 *(const int16x16 *)(qdata+784) |
| 119 |
-9,-9,-9,-9,-16425,-16425,-16425,-16425,-9,-9,-9,-9,-16425,-16425,-16425,-16425, |
| 120 |
#define qinvprecomp_8_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 *(const int16x16 *)(qdata+800) |
| 121 |
-28865,-28865,-28865,-28865,10350,10350,10350,10350,-28865,-28865,-28865,-28865,10350,10350,10350,10350, |
| 122 |
#define qinvprecomp_8_7_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 *(const int16x16 *)(qdata+816) |
| 123 |
-9,-9,-9,-9,-10350,-10350,-10350,-10350,-9,-9,-9,-9,-10350,-10350,-10350,-10350, |
| 124 |
#define qinvprecomp_8_7_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 *(const int16x16 *)(qdata+832) |
| 125 |
28865,28865,28865,28865,16425,16425,16425,16425,28865,28865,28865,28865,16425,16425,16425,16425, |
| 126 |
#define qinvprecomp_16_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 *(const int16x16 *)(qdata+848) |
| 127 |
-9,-9,-9,-9,-4974,-4974,-4974,-4974,-9,-9,-9,-9,-4974,-4974,-4974,-4974, |
| 128 |
#define qinvprecomp_16_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 *(const int16x16 *)(qdata+864) |
| 129 |
-16425,-16425,-16425,-16425,-7244,-7244,-7244,-7244,-16425,-16425,-16425,-16425,-7244,-7244,-7244,-7244, |
| 130 |
#define qinvprecomp_16_15_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 *(const int16x16 *)(qdata+880) |
| 131 |
-9,-9,-9,-9,-4496,-4496,-4496,-4496,-9,-9,-9,-9,-4496,-4496,-4496,-4496, |
| 132 |
#define qinvprecomp_16_15_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 *(const int16x16 *)(qdata+896) |
| 133 |
-10350,-10350,-10350,-10350,-14744,-14744,-14744,-14744,-10350,-10350,-10350,-10350,-14744,-14744,-14744,-14744, |
| 134 |
#define qinvprecomp_32_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 *(const int16x16 *)(qdata+912) |
| 135 |
-9,-20315,-4974,18242,-16425,18191,-7244,-11655,-9,-20315,-4974,18242,-16425,18191,-7244,-11655, |
| 136 |
#define qinvprecomp_32_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+928) |
| 137 |
-28865,20870,14744,-22593,10350,828,4496,23754,-28865,20870,14744,-22593,10350,828,4496,23754, |
| 138 |
#define qinvprecomp_32_31_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 *(const int16x16 *)(qdata+944) |
| 139 |
-9,-23754,-4496,-828,-10350,22593,-14744,-20870,-9,-23754,-4496,-828,-10350,22593,-14744,-20870, |
| 140 |
#define qinvprecomp_32_31_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+960) |
| 141 |
28865,11655,7244,-18191,16425,-18242,4974,20315,28865,11655,7244,-18191,16425,-18242,4974,20315, |
| 142 |
#define qinvprecomp_64_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 *(const int16x16 *)(qdata+976) |
| 143 |
-9,-10972,-20315,23489,-4974,25597,18242,-2816,-9,-10972,-20315,23489,-4974,25597,18242,-2816, |
| 144 |
#define qinvprecomp_64_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+992) |
| 145 |
-16425,-19351,18191,-3114,-7244,-9488,-11655,19394,-16425,-19351,18191,-3114,-7244,-9488,-11655,19394, |
| 146 |
#define qinvprecomp_64_63_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 *(const int16x16 *)(qdata+1008) |
| 147 |
-9,-7491,-23754,-15307,-4496,-15750,-828,-5759,-9,-7491,-23754,-15307,-4496,-15750,-828,-5759, |
| 148 |
#define qinvprecomp_64_63_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+1024) |
| 149 |
-10350,22568,22593,-20469,-14744,31177,-20870,26382,-10350,22568,22593,-20469,-14744,31177,-20870,26382, |
| 150 |
#define qinvprecomp_128_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+1040) |
| 151 |
-9,-14846,-10972,-21066,-20315,-24581,23489,-23242,-4974,-4505,25597,-26279,18242,21467,-2816,15998, |
| 152 |
#define qinvprecomp_128_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+1056) |
| 153 |
-16425,-4983,-19351,14624,18191,-2073,-3114,20674,-7244,-21399,-9488,6246,-11655,-29103,19394,-5930, |
| 154 |
#define qinvprecomp_128_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+1072) |
| 155 |
-28865,-23668,-26382,-28839,20870,6536,-31177,16279,14744,29428,20469,29667,-22593,9215,-22568,-11783, |
| 156 |
#define qinvprecomp_128_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+1088) |
| 157 |
10350,-14121,5759,-5913,828,-1724,15750,11792,4496,25093,15307,26228,23754,-21766,7491,-6817, |
| 158 |
#define qinvprecomp_128_127_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+1104) |
| 159 |
-9,6817,-7491,21766,-23754,-26228,-15307,-25093,-4496,-11792,-15750,1724,-828,5913,-5759,14121, |
| 160 |
#define qinvprecomp_128_127_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+1120) |
| 161 |
-10350,11783,22568,-9215,22593,-29667,-20469,-29428,-14744,-16279,31177,-6536,-20870,28839,26382,23668, |
| 162 |
#define qinvprecomp_128_127_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+1136) |
| 163 |
28865,5930,-19394,29103,11655,-6246,9488,21399,7244,-20674,3114,2073,-18191,-14624,19351,4983, |
| 164 |
#define qinvprecomp_128_127_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+1152) |
| 165 |
16425,-15998,2816,-21467,-18242,26279,-25597,4505,4974,23242,-23489,24581,20315,21066,10972,14846, |
| 166 |
#define qinvprecomp_256_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+1168) |
| 167 |
-9,-32252,-14846,-19317,-10972,8472,-21066,-3456,-20315,16655,-24581,12611,23489,-12073,-23242,29871, |
| 168 |
#define qinvprecomp_256_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+1184) |
| 169 |
-4974,6032,-4505,10409,25597,24266,-26279,17030,18242,10478,21467,11962,-2816,-26262,15998,-17602, |
| 170 |
#define qinvprecomp_256_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+1200) |
| 171 |
-16425,-22124,-4983,-26220,-19351,-8908,14624,32738,18191,13575,-2073,27132,-3114,24573,20674,27201, |
| 172 |
#define qinvprecomp_256_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+1216) |
| 173 |
-7244,12269,-21399,-16092,-9488,-15810,6246,15358,-11655,-15768,-29103,24052,19394,-26441,-5930,-1689, |
| 174 |
#define qinvprecomp_256_255_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+1232) |
| 175 |
-9,13541,6817,-5529,-7491,26663,21766,-4693,-23754,13933,-26228,8558,-15307,-21953,-25093,-22875, |
| 176 |
#define qinvprecomp_256_255_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+1248) |
| 177 |
-4496,-7508,-11792,-30136,-15750,26800,1724,17303,-828,2722,5913,-12013,-5759,30426,14121,3558, |
| 178 |
#define qinvprecomp_256_255_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+1264) |
| 179 |
-10350,-24743,11783,-21860,22568,-32329,-9215,9360,22593,-7415,-29667,25946,-20469,-21868,-29428,-25511, |
| 180 |
#define qinvprecomp_256_255_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+1280) |
| 181 |
-14744,1869,-16279,14351,31177,2193,-6536,17440,-20870,24718,28839,-23225,26382,9855,23668,-9599, |
| 182 |
#define qinvprecomp_512_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+1296) |
| 183 |
-9,-32124,-32252,10179,-14846,6766,-19317,16638,-10972,-23549,8472,-17082,-21066,-15145,-3456,31518, |
| 184 |
#define qinvprecomp_512_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+1312) |
| 185 |
-20315,-6297,16655,-12261,-24581,-11885,12611,30938,23489,28805,-12073,26783,-23242,-14718,29871,5708, |
| 186 |
#define qinvprecomp_512_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+1328) |
| 187 |
-4974,15111,6032,-29453,-4505,12449,10409,529,25597,-32004,24266,2952,-26279,18003,17030,24931, |
| 188 |
#define qinvprecomp_512_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+1344) |
| 189 |
18242,-1007,10478,-4624,21467,17636,11962,14360,-2816,15972,-26262,16715,15998,4573,-17602,-14539, |
| 190 |
#define qinvprecomp_512_5_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+1360) |
| 191 |
-9,6766,8472,31518,-24581,28805,29871,-29453,25597,18003,10478,14360,15998,27636,-26220,17167, |
| 192 |
#define qinvprecomp_512_5_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+1376) |
| 193 |
18191,-7304,24573,-22039,-21399,-4565,15358,10802,19394,21723,9599,-9633,-28839,-2807,-2193,-30597, |
| 194 |
#define qinvprecomp_512_5_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+1392) |
| 195 |
14744,-26330,-25946,-2739,9215,32695,24743,-26288,5759,20435,-17303,24530,11792,20964,21953,23523, |
| 196 |
#define qinvprecomp_512_5_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+1408) |
| 197 |
23754,-27858,5529,6510,14846,23549,3456,12261,-23489,14718,-6032,-529,26279,1007,-11962,-16715, |
| 198 |
#define qinvprecomp_512_507_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+1424) |
| 199 |
-9,24214,26663,23933,-26228,-13686,-22875,-27243,-15750,4317,2722,8839,14121,-32414,-21860,-25179, |
| 200 |
#define qinvprecomp_512_507_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+1440) |
| 201 |
22593,-25648,-21868,-964,-16279,-1715,17440,-14650,26382,-28958,1689,-10333,29103,-20119,15810,22790, |
| 202 |
#define qinvprecomp_512_507_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+1456) |
| 203 |
7244,20238,-27132,-2858,-14624,19274,22124,-4573,2816,4624,-17030,32004,4505,-5708,12073,11885, |
| 204 |
#define qinvprecomp_512_507_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+1472) |
| 205 |
20315,17082,19317,32124,-6817,14223,4693,-14138,15307,9650,7508,-9513,-1724,-23882,12013,-15221, |
| 206 |
#define qinvprecomp_512_511_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+1488) |
| 207 |
-9,-6510,13541,-23182,6817,24214,-5529,-24232,-7491,-14223,26663,27858,21766,26621,-4693,23933, |
| 208 |
#define qinvprecomp_512_511_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+1504) |
| 209 |
-23754,29394,13933,14138,-26228,-23523,8558,-23984,-15307,-13686,-21953,26766,-25093,-9650,-22875,-20964, |
| 210 |
#define qinvprecomp_512_511_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+1520) |
| 211 |
-4496,-22943,-7508,-27243,-11792,-18506,-30136,9513,-15750,-24530,26800,947,1724,4317,17303,29718, |
| 212 |
#define qinvprecomp_512_511_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+1536) |
| 213 |
-828,23882,2722,-20435,5913,-10495,-12013,8839,-5759,-3396,30426,15221,14121,26288,3558,27730, |
| 214 |
#define qinvscaledzeta_x16_4_1 *(const int16x16 *)(qdata+1552) |
| 215 |
-28865,-28865,-28865,-28865,-28865,-28865,-28865,-28865,-28865,-28865,-28865,-28865,-28865,-28865,-28865,-28865, |
| 216 |
#define qinvscaledzeta_x16_4_3 *(const int16x16 *)(qdata+1568) |
| 217 |
28865,28865,28865,28865,28865,28865,28865,28865,28865,28865,28865,28865,28865,28865,28865,28865, |
| 218 |
#define qinvscaledzeta_x16_8_1 *(const int16x16 *)(qdata+1584) |
| 219 |
-16425,-16425,-16425,-16425,-16425,-16425,-16425,-16425,-16425,-16425,-16425,-16425,-16425,-16425,-16425,-16425, |
| 220 |
#define qinvscaledzeta_x16_8_7 *(const int16x16 *)(qdata+1600) |
| 221 |
-10350,-10350,-10350,-10350,-10350,-10350,-10350,-10350,-10350,-10350,-10350,-10350,-10350,-10350,-10350,-10350, |
| 222 |
#define qround32_x16 *(const int16x16 *)(qdata+1616) |
| 223 |
4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4, |
| 224 |
#define scaledzeta_x16_4_1 *(const int16x16 *)(qdata+1632) |
| 225 |
-3777,-3777,-3777,-3777,-3777,-3777,-3777,-3777,-3777,-3777,-3777,-3777,-3777,-3777,-3777,-3777, |
| 226 |
#define scaledzeta_x16_4_3 *(const int16x16 *)(qdata+1648) |
| 227 |
3777,3777,3777,3777,3777,3777,3777,3777,3777,3777,3777,3777,3777,3777,3777,3777, |
| 228 |
#define scaledzeta_x16_8_1 *(const int16x16 *)(qdata+1664) |
| 229 |
-3625,-3625,-3625,-3625,-3625,-3625,-3625,-3625,-3625,-3625,-3625,-3625,-3625,-3625,-3625,-3625, |
| 230 |
#define scaledzeta_x16_8_7 *(const int16x16 *)(qdata+1680) |
| 231 |
-3182,-3182,-3182,-3182,-3182,-3182,-3182,-3182,-3182,-3182,-3182,-3182,-3182,-3182,-3182,-3182, |
| 232 |
} ; |
| 233 |
|
| 234 |
static const int16 __attribute((aligned(32))) qdata_10753[] = { |
| 235 |
// precomp_8_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 |
| 236 |
1018,1018,1018,1018,3688,3688,3688,3688,1018,1018,1018,1018,3688,3688,3688,3688, |
| 237 |
// precomp_8_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 |
| 238 |
-223,-223,-223,-223,-4188,-4188,-4188,-4188,-223,-223,-223,-223,-4188,-4188,-4188,-4188, |
| 239 |
// precomp_8_7_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 |
| 240 |
1018,1018,1018,1018,4188,4188,4188,4188,1018,1018,1018,1018,4188,4188,4188,4188, |
| 241 |
// precomp_8_7_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 |
| 242 |
223,223,223,223,-3688,-3688,-3688,-3688,223,223,223,223,-3688,-3688,-3688,-3688, |
| 243 |
// precomp_16_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 |
| 244 |
1018,1018,1018,1018,-376,-376,-376,-376,1018,1018,1018,1018,-376,-376,-376,-376, |
| 245 |
// precomp_16_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 |
| 246 |
3688,3688,3688,3688,-3686,-3686,-3686,-3686,3688,3688,3688,3688,-3686,-3686,-3686,-3686, |
| 247 |
// precomp_16_15_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 |
| 248 |
1018,1018,1018,1018,-2413,-2413,-2413,-2413,1018,1018,1018,1018,-2413,-2413,-2413,-2413, |
| 249 |
// precomp_16_15_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 |
| 250 |
4188,4188,4188,4188,-357,-357,-357,-357,4188,4188,4188,4188,-357,-357,-357,-357, |
| 251 |
// precomp_32_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 |
| 252 |
1018,-3364,-376,4855,3688,425,-3686,2695,1018,-3364,-376,4855,3688,425,-3686,2695, |
| 253 |
// precomp_32_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 |
| 254 |
-223,-3784,357,-2236,-4188,4544,2413,730,-223,-3784,357,-2236,-4188,4544,2413,730, |
| 255 |
// precomp_32_31_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 |
| 256 |
1018,-730,-2413,-4544,4188,2236,-357,3784,1018,-730,-2413,-4544,4188,2236,-357,3784, |
| 257 |
// precomp_32_31_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 |
| 258 |
223,-2695,3686,-425,-3688,-4855,376,3364,223,-2695,3686,-425,-3688,-4855,376,3364, |
| 259 |
// precomp_64_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 |
| 260 |
1018,-5175,-3364,2503,-376,1341,4855,-4875,1018,-5175,-3364,2503,-376,1341,4855,-4875, |
| 261 |
// precomp_64_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 |
| 262 |
3688,-2629,425,-4347,-3686,3823,2695,-4035,3688,-2629,425,-4347,-3686,3823,2695,-4035, |
| 263 |
// precomp_64_63_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 |
| 264 |
1018,5063,-730,341,-2413,-3012,-4544,-5213,1018,5063,-730,341,-2413,-3012,-4544,-5213, |
| 265 |
// precomp_64_63_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 |
| 266 |
4188,1520,2236,1931,-357,918,3784,4095,4188,1520,2236,1931,-357,918,3784,4095, |
| 267 |
// precomp_128_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 |
| 268 |
1018,3085,-5175,2982,-3364,-4744,2503,-4129,-376,-2576,1341,-193,4855,3062,-4875,4, |
| 269 |
// precomp_128_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 |
| 270 |
3688,2388,-2629,-4513,425,4742,-4347,2935,-3686,-544,3823,-2178,2695,847,-4035,268, |
| 271 |
// precomp_128_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 |
| 272 |
-223,-1299,-4095,-1287,-3784,-4876,-918,3091,357,-4189,-1931,4616,-2236,2984,-1520,-3550, |
| 273 |
// precomp_128_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 |
| 274 |
-4188,-1009,5213,-205,4544,-4102,3012,2790,2413,-1085,-341,-2565,730,-4379,-5063,-1284, |
| 275 |
// precomp_128_127_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 |
| 276 |
1018,1284,5063,4379,-730,2565,341,1085,-2413,-2790,-3012,4102,-4544,205,-5213,1009, |
| 277 |
// precomp_128_127_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 |
| 278 |
4188,3550,1520,-2984,2236,-4616,1931,4189,-357,-3091,918,4876,3784,1287,4095,1299, |
| 279 |
// precomp_128_127_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 |
| 280 |
223,-268,4035,-847,-2695,2178,-3823,544,3686,-2935,4347,-4742,-425,4513,2629,-2388, |
| 281 |
// precomp_128_127_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 |
| 282 |
-3688,-4,4875,-3062,-4855,193,-1341,2576,376,4129,-2503,4744,3364,-2982,5175,-3085, |
| 283 |
// precomp_256_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 |
| 284 |
1018,5116,3085,-3615,-5175,400,2982,3198,-3364,2234,-4744,-4828,2503,326,-4129,-512, |
| 285 |
// precomp_256_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 |
| 286 |
-376,1068,-2576,-4580,1341,3169,-193,-2998,4855,-635,3062,-4808,-4875,-2740,4,675, |
| 287 |
// precomp_256_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 |
| 288 |
3688,-1324,2388,5114,-2629,5294,-4513,-794,425,-864,4742,-886,-4347,336,2935,-2045, |
| 289 |
// precomp_256_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 |
| 290 |
-3686,-3715,-544,4977,3823,-2737,-2178,3441,2695,467,847,454,-4035,-779,268,2213, |
| 291 |
// precomp_256_255_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 |
| 292 |
1018,1615,1284,2206,5063,5064,4379,472,-730,-5341,2565,-4286,341,2981,1085,-1268, |
| 293 |
// precomp_256_255_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 |
| 294 |
-2413,-3057,-2790,-2884,-3012,-1356,4102,-3337,-4544,5023,205,-636,-5213,909,1009,-2973, |
| 295 |
// precomp_256_255_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 |
| 296 |
4188,2271,3550,-1572,1520,1841,-2984,970,2236,-4734,-4616,578,1931,-116,4189,1586, |
| 297 |
// precomp_256_255_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 |
| 298 |
-357,-2774,-3091,-1006,918,-5156,4876,4123,3784,-567,1287,151,4095,1458,1299,2684, |
| 299 |
// precomp_512_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 |
| 300 |
1018,-3260,5116,-1722,3085,5120,-3615,3760,-5175,73,400,4254,2982,2788,3198,-2657, |
| 301 |
// precomp_512_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 |
| 302 |
-3364,569,2234,1930,-4744,-2279,-4828,5215,2503,-4403,326,1639,-4129,5068,-512,-5015, |
| 303 |
// precomp_512_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 |
| 304 |
-376,-4859,1068,-40,-2576,4003,-4580,-4621,1341,2487,3169,-2374,-193,2625,-2998,4784, |
| 305 |
// precomp_512_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 |
| 306 |
4855,825,-635,2118,3062,-2813,-4808,-4250,-4875,-2113,-2740,-4408,4,-1893,675,458, |
| 307 |
// precomp_512_5_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 |
| 308 |
1018,5120,400,-2657,-4744,-4403,-512,-40,1341,2625,-635,-4250,4,-3360,5114,-5313, |
| 309 |
// precomp_512_5_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 |
| 310 |
425,-2151,336,-2662,-544,5334,3441,2117,-4035,2205,-2684,-3570,-1287,-4973,5156,2419, |
| 311 |
// precomp_512_5_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 |
| 312 |
357,1204,-578,1635,2984,-1111,-2271,4359,5213,-2449,3337,3453,2790,554,-2981,-1409, |
| 313 |
// precomp_512_5_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 |
| 314 |
730,-279,-2206,3524,-3085,-73,-3198,-1930,-2503,-5068,-1068,4621,193,-825,4808,4408, |
| 315 |
// precomp_512_507_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 |
| 316 |
1018,4428,5064,-4000,2565,573,-1268,3125,-3012,-4144,5023,1927,1009,-2139,-1572,3535, |
| 317 |
// precomp_512_507_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 |
| 318 |
2236,663,-116,4967,-3091,-854,4123,1160,4095,-1349,-2213,1782,-847,2062,2737,624, |
| 319 |
// precomp_512_507_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 |
| 320 |
3686,-2283,886,4889,4513,-4601,1324,1893,4875,-2118,2998,-2487,2576,5015,-326,2279, |
| 321 |
// precomp_512_507_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 |
| 322 |
3364,-4254,3615,3260,-1284,-1381,-472,-3891,-341,2087,3057,4720,-4102,3410,636,1689, |
| 323 |
// precomp_512_511_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 |
| 324 |
1018,-3524,1615,5268,1284,4428,2206,-834,5063,1381,5064,279,4379,2439,472,-4000, |
| 325 |
// precomp_512_511_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 |
| 326 |
-730,-2015,-5341,3891,2565,1409,-4286,2605,341,573,2981,5356,1085,-2087,-1268,-554, |
| 327 |
// precomp_512_511_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 |
| 328 |
-2413,3135,-3057,3125,-2790,-778,-2884,-4720,-3012,-3453,-1356,-355,4102,-4144,-3337,-152, |
| 329 |
// precomp_512_511_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 |
| 330 |
-4544,-3410,5023,2449,205,-97,-636,1927,-5213,2624,909,-1689,1009,-4359,-2973,-3419, |
| 331 |
// q_x16 |
| 332 |
10753,10753,10753,10753,10753,10753,10753,10753,10753,10753,10753,10753,10753,10753,10753,10753, |
| 333 |
// qinvprecomp_8_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 |
| 334 |
-6,-6,-6,-6,-408,-408,-408,-408,-6,-6,-6,-6,-408,-408,-408,-408, |
| 335 |
// qinvprecomp_8_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 |
| 336 |
-27359,-27359,-27359,-27359,1956,1956,1956,1956,-27359,-27359,-27359,-27359,1956,1956,1956,1956, |
| 337 |
// qinvprecomp_8_7_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 |
| 338 |
-6,-6,-6,-6,-1956,-1956,-1956,-1956,-6,-6,-6,-6,-1956,-1956,-1956,-1956, |
| 339 |
// qinvprecomp_8_7_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 |
| 340 |
27359,27359,27359,27359,408,408,408,408,27359,27359,27359,27359,408,408,408,408, |
| 341 |
// qinvprecomp_16_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 |
| 342 |
-6,-6,-6,-6,-20856,-20856,-20856,-20856,-6,-6,-6,-6,-20856,-20856,-20856,-20856, |
| 343 |
// qinvprecomp_16_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 |
| 344 |
-408,-408,-408,-408,-21094,-21094,-21094,-21094,-408,-408,-408,-408,-21094,-21094,-21094,-21094, |
| 345 |
// qinvprecomp_16_15_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 |
| 346 |
-6,-6,-6,-6,-10093,-10093,-10093,-10093,-6,-6,-6,-6,-10093,-10093,-10093,-10093, |
| 347 |
// qinvprecomp_16_15_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 |
| 348 |
-1956,-1956,-1956,-1956,-28517,-28517,-28517,-28517,-1956,-1956,-1956,-1956,-28517,-28517,-28517,-28517, |
| 349 |
// qinvprecomp_32_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 |
| 350 |
-6,-9508,-20856,-29449,-408,18345,-21094,-7033,-6,-9508,-20856,-29449,-408,18345,-21094,-7033, |
| 351 |
// qinvprecomp_32_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 |
| 352 |
-27359,-16072,28517,-12476,1956,-28224,10093,16090,-27359,-16072,28517,-12476,1956,-28224,10093,16090, |
| 353 |
// qinvprecomp_32_31_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 |
| 354 |
-6,-16090,-10093,28224,-1956,12476,-28517,16072,-6,-16090,-10093,28224,-1956,12476,-28517,16072, |
| 355 |
// qinvprecomp_32_31_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 |
| 356 |
27359,7033,21094,-18345,408,29449,20856,9508,27359,7033,21094,-18345,408,29449,20856,9508, |
| 357 |
// qinvprecomp_64_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 |
| 358 |
-6,-3639,-9508,25543,-20856,829,-29449,-17675,-6,-3639,-9508,25543,-20856,829,-29449,-17675, |
| 359 |
// qinvprecomp_64_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 |
| 360 |
-408,18363,18345,7429,-21094,-10001,-7033,-4547,-408,18363,18345,7429,-21094,-10001,-7033,-4547, |
| 361 |
// qinvprecomp_64_63_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 |
| 362 |
-6,28103,-16090,3925,-10093,7228,28224,11683,-6,28103,-16090,3925,-10093,7228,28224,11683, |
| 363 |
// qinvprecomp_64_63_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 |
| 364 |
-1956,-23056,12476,14731,-28517,26518,16072,14847,-1956,-23056,12476,14731,-28517,26518,16072,14847, |
| 365 |
// qinvprecomp_128_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 |
| 366 |
-6,-5619,-3639,-12378,-9508,15736,25543,23007,-20856,-27152,829,-22209,-29449,-20490,-17675,22532, |
| 367 |
// qinvprecomp_128_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 |
| 368 |
-408,16724,18363,22623,18345,5766,7429,-31369,-21094,15840,-10001,19326,-7033,3407,-4547,2316, |
| 369 |
// qinvprecomp_128_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 |
| 370 |
-27359,6381,-14847,8441,-16072,-6924,-26518,-4589,28517,12707,-14731,-15864,-12476,31656,23056,24098, |
| 371 |
// qinvprecomp_128_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 |
| 372 |
1956,-31217,-11683,-24269,-28224,-5126,-7228,20198,10093,-573,-3925,-14341,16090,23781,-28103,-23812, |
| 373 |
// qinvprecomp_128_127_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 |
| 374 |
-6,23812,28103,-23781,-16090,14341,3925,573,-10093,-20198,7228,5126,28224,24269,11683,31217, |
| 375 |
// qinvprecomp_128_127_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 |
| 376 |
-1956,-24098,-23056,-31656,12476,15864,14731,-12707,-28517,4589,26518,6924,16072,-8441,14847,-6381, |
| 377 |
// qinvprecomp_128_127_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 |
| 378 |
27359,-2316,4547,-3407,7033,-19326,10001,-15840,21094,31369,-7429,-5766,-18345,-22623,-18363,-16724, |
| 379 |
// qinvprecomp_128_127_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 |
| 380 |
408,-22532,17675,20490,29449,22209,-829,27152,20856,-23007,-25543,-15736,9508,12378,3639,5619, |
| 381 |
// qinvprecomp_256_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 |
| 382 |
-6,-17412,-5619,2017,-3639,24976,-12378,24702,-9508,-31558,15736,1316,25543,-31418,23007,-512, |
| 383 |
// qinvprecomp_256_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 |
| 384 |
-20856,-13268,-27152,22044,829,8801,-22209,-12214,-29449,11141,-20490,-17096,-17675,32076,22532,17571, |
| 385 |
// qinvprecomp_256_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 |
| 386 |
-408,13012,16724,4090,18363,-30546,22623,16614,18345,-17248,5766,22666,7429,-7856,-31369,31235, |
| 387 |
// qinvprecomp_256_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 |
| 388 |
-21094,28541,15840,-30351,-10001,-177,19326,-31887,-7033,25555,3407,-31290,-4547,-13579,2316,-2395, |
| 389 |
// qinvprecomp_256_255_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 |
| 390 |
-6,4175,23812,7326,28103,17352,-23781,-28200,-16090,11555,14341,6978,3925,-1627,573,780, |
| 391 |
// qinvprecomp_256_255_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 |
| 392 |
-10093,32271,-20198,7356,7228,29364,5126,27895,28224,-609,24269,21892,11683,-7795,31217,-18845, |
| 393 |
// qinvprecomp_256_255_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 |
| 394 |
-1956,29407,-24098,-7716,-23056,-719,-31656,-8246,12476,-26238,15864,11842,14731,1932,-12707,-11726, |
| 395 |
// qinvprecomp_256_255_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 |
| 396 |
-28517,4394,4589,2066,26518,-11300,6924,-24037,16072,969,-8441,14999,14847,-11854,-6381,-19844, |
| 397 |
// qinvprecomp_512_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 |
| 398 |
-6,-13500,-17412,32070,-5619,5120,2017,11952,-3639,1609,24976,9374,-12378,-23836,24702,-8289, |
| 399 |
// qinvprecomp_512_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 |
| 400 |
-9508,-22471,-31558,25482,15736,-8935,1316,32351,25543,19661,-31418,8295,23007,-25652,-512,-19863, |
| 401 |
// qinvprecomp_512_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 |
| 402 |
-20856,6917,-13268,-28712,-27152,20899,22044,4083,829,951,8801,29370,-22209,24641,-12214,12976, |
| 403 |
// qinvprecomp_512_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 |
| 404 |
-29449,-22215,11141,-29626,-20490,30467,-17096,13158,-17675,-24129,32076,7880,22532,-30053,17571,-8758, |
| 405 |
// qinvprecomp_512_5_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 |
| 406 |
-6,5120,24976,-8289,15736,19661,-512,-28712,829,24641,11141,13158,22532,13024,4090,-27329, |
| 407 |
// qinvprecomp_512_5_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 |
| 408 |
18345,-8807,-7856,-20070,15840,-1834,-31887,-18875,-4547,18077,19844,-23026,8441,-12653,11300,11123, |
| 409 |
// qinvprecomp_512_5_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 |
| 410 |
28517,31924,-11842,-14237,31656,16809,-29407,-5369,-11683,-16273,-27895,-29827,20198,7722,1627,9343, |
| 411 |
// qinvprecomp_512_5_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 |
| 412 |
16090,-15127,-7326,-6716,5619,-1609,-24702,-25482,-25543,25652,13268,-4083,22209,22215,17096,-7880, |
| 413 |
// qinvprecomp_512_507_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 |
| 414 |
-6,-26292,17352,12384,14341,61,780,23093,7228,-12336,-609,-7801,31217,-6747,-7716,6095, |
| 415 |
// qinvprecomp_512_507_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 |
| 416 |
12476,15511,1932,11623,4589,6314,-24037,-19320,14847,19643,2395,-21770,-3407,-17394,177,-23952, |
| 417 |
// qinvprecomp_512_507_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 |
| 418 |
21094,-31467,-22666,-1767,-22623,-14329,-13012,30053,17675,29626,12214,-951,27152,19863,31418,8935, |
| 419 |
// qinvprecomp_512_507_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 |
| 420 |
9508,-9374,-2017,13500,-23812,-29541,28200,20173,-3925,-24025,-32271,-19856,-5126,-26286,-21892,-4967, |
| 421 |
// qinvprecomp_512_511_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 |
| 422 |
-6,6716,4175,-13164,23812,-26292,7326,-12098,28103,29541,17352,15127,-23781,-7289,-28200,12384, |
| 423 |
// qinvprecomp_512_511_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 |
| 424 |
-16090,-29151,11555,-20173,14341,-9343,6978,-22483,3925,61,-1627,23788,573,24025,780,-7722, |
| 425 |
// qinvprecomp_512_511_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 |
| 426 |
-10093,-18881,32271,23093,-20198,-24330,7356,19856,7228,29827,29364,15517,5126,-12336,27895,-4248, |
| 427 |
// qinvprecomp_512_511_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 |
| 428 |
28224,26286,-609,16273,24269,-5729,21892,-7801,11683,-30144,-7795,4967,31217,5369,-18845,-8027, |
| 429 |
// qinvscaledzeta_x16_4_1 |
| 430 |
-27359,-27359,-27359,-27359,-27359,-27359,-27359,-27359,-27359,-27359,-27359,-27359,-27359,-27359,-27359,-27359, |
| 431 |
// qinvscaledzeta_x16_4_3 |
| 432 |
27359,27359,27359,27359,27359,27359,27359,27359,27359,27359,27359,27359,27359,27359,27359,27359, |
| 433 |
// qinvscaledzeta_x16_8_1 |
| 434 |
-408,-408,-408,-408,-408,-408,-408,-408,-408,-408,-408,-408,-408,-408,-408,-408, |
| 435 |
// qinvscaledzeta_x16_8_7 |
| 436 |
-1956,-1956,-1956,-1956,-1956,-1956,-1956,-1956,-1956,-1956,-1956,-1956,-1956,-1956,-1956,-1956, |
| 437 |
// qround32_x16 |
| 438 |
3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, |
| 439 |
// scaledzeta_x16_4_1 |
| 440 |
-223,-223,-223,-223,-223,-223,-223,-223,-223,-223,-223,-223,-223,-223,-223,-223, |
| 441 |
// scaledzeta_x16_4_3 |
| 442 |
223,223,223,223,223,223,223,223,223,223,223,223,223,223,223,223, |
| 443 |
// scaledzeta_x16_8_1 |
| 444 |
3688,3688,3688,3688,3688,3688,3688,3688,3688,3688,3688,3688,3688,3688,3688,3688, |
| 445 |
// scaledzeta_x16_8_7 |
| 446 |
4188,4188,4188,4188,4188,4188,4188,4188,4188,4188,4188,4188,4188,4188,4188,4188, |
| 447 |
} ; |
| 448 |
|
| 449 |
static inline int16x16 add_x16(int16x16 a,int16x16 b) |
| 450 |
{ |
| 451 |
return _mm256_add_epi16(a,b); |
| 452 |
} |
| 453 |
|
| 454 |
static inline int16x16 sub_x16(int16x16 a,int16x16 b) |
| 455 |
{ |
| 456 |
return _mm256_sub_epi16(a,b); |
| 457 |
} |
| 458 |
|
| 459 |
static inline int16x16 mulmod_scaled_x16(int16x16 x,int16x16 y,int16x16 yqinv,const int16 *qdata) |
| 460 |
{ |
| 461 |
int16x16 b = _mm256_mulhi_epi16(x,y); |
| 462 |
int16x16 d = _mm256_mullo_epi16(x,yqinv); |
| 463 |
int16x16 e = _mm256_mulhi_epi16(d,q_x16); |
| 464 |
return sub_x16(b,e); |
| 465 |
} |
| 466 |
|
| 467 |
static inline int16x16 reduce_x16(int16x16 x,const int16 *qdata) |
| 468 |
{ |
| 469 |
int16x16 y = _mm256_mulhrs_epi16(x,qround32_x16); |
| 470 |
y = _mm256_mullo_epi16(y,q_x16); |
| 471 |
return sub_x16(x,y); |
| 472 |
} |
| 473 |
|
| 474 |
// ----- codegen pass 1 |
| 475 |
// |
| 476 |
// startntt 512 |
| 477 |
// startbatch 512 |
| 478 |
// // ----- PRECONDITIONS |
| 479 |
// physical_map (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 480 |
// // transform size 512 |
| 481 |
// // transform indexing [0, 1, 2, 3, 4, 5, 6, 7, 8] |
| 482 |
// // transforms per batch 1 |
| 483 |
// // batch indexing [] |
| 484 |
// // total batch size 512 |
| 485 |
// |
| 486 |
// // modulus x^512-1 pos 0:512 q 7681,10753 bound 512*(5629,5800) |
| 487 |
// assertranges ... |
| 488 |
// |
| 489 |
// // ----- LAYER 1 |
| 490 |
// |
| 491 |
// // butterfly(0,256,1,256,1,0) |
| 492 |
// butterfly 0 256 1 256 1 0 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 493 |
// |
| 494 |
// // ----- POSTCONDITIONS AFTER LAYER 1 |
| 495 |
// // transform size 512 |
| 496 |
// // transform indexing [0, 1, 2, 3, 4, 5, 6, 7, 8] |
| 497 |
// // transforms per batch 1 |
| 498 |
// // batch indexing [] |
| 499 |
// // total batch size 512 |
| 500 |
// |
| 501 |
// // modulus x^256-1 pos 0:256 q 7681,10753 bound 256*(11258,11600) |
| 502 |
// assertranges ... |
| 503 |
// |
| 504 |
// // modulus x^256+1 pos 256:512 q 7681,10753 bound 256*(11258,11600) |
| 505 |
// assertranges ... |
| 506 |
// |
| 507 |
// // ----- LAYER 2 |
| 508 |
// |
| 509 |
// // reduce_ifreverse(0,64,1) |
| 510 |
// reduce_ifreverse 0 64 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 511 |
// |
| 512 |
// // reduce_ifreverse(256,320,1) |
| 513 |
// reduce_ifreverse 256 320 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 514 |
// |
| 515 |
// // butterfly(0,128,1,128,1,0) |
| 516 |
// butterfly 0 128 1 128 1 0 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 517 |
// |
| 518 |
// // butterfly(256,384,1,128,4,1) |
| 519 |
// butterfly 256 384 1 128 4 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 520 |
// |
| 521 |
// // ----- POSTCONDITIONS AFTER LAYER 2 |
| 522 |
// // transform size 512 |
| 523 |
// // transform indexing [0, 1, 2, 3, 4, 5, 6, 7, 8] |
| 524 |
// // transforms per batch 1 |
| 525 |
// // batch indexing [] |
| 526 |
// // total batch size 512 |
| 527 |
// |
| 528 |
// // modulus x^128-1 pos 0:128 q 7681,10753 bound 128*(22516,23200) |
| 529 |
// assertranges ... |
| 530 |
// |
| 531 |
// // modulus x^128+1 pos 128:256 q 7681,10753 bound 128*(22516,23200) |
| 532 |
// assertranges ... |
| 533 |
// |
| 534 |
// // modulus x^128-zeta4 pos 256:384 q 7681,10753 bound 128*(15747,17016) |
| 535 |
// assertranges ... |
| 536 |
// |
| 537 |
// // modulus x^128+zeta4 pos 384:512 q 7681,10753 bound 128*(15747,17016) |
| 538 |
// assertranges ... |
| 539 |
// |
| 540 |
// // ----- LAYER 3 |
| 541 |
// |
| 542 |
// // reduce_ifforward(64,128,1) |
| 543 |
// reduce_ifforward 64 128 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 544 |
// |
| 545 |
// // butterfly(0,64,1,64,1,0) |
| 546 |
// butterfly 0 64 1 64 1 0 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 547 |
// |
| 548 |
// // butterfly(128,192,1,64,4,1) |
| 549 |
// butterfly 128 192 1 64 4 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 550 |
// |
| 551 |
// // butterfly(256,320,1,64,8,1) |
| 552 |
// butterfly 256 320 1 64 8 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 553 |
// |
| 554 |
// // butterfly(384,448,1,64,8,-1) |
| 555 |
// butterfly 384 448 1 64 8 7 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 556 |
// |
| 557 |
// // reduce(0,64,1) |
| 558 |
// reduce 0 64 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 559 |
// |
| 560 |
// // twist(64,128,1,128,1) |
| 561 |
// twist 64 128 1 128 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 562 |
// |
| 563 |
// // twist(128,192,1,256,1) |
| 564 |
// twist 128 192 1 256 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 565 |
// |
| 566 |
// // twist(192,256,1,256,-1) |
| 567 |
// twist 192 256 1 256 255 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 568 |
// |
| 569 |
// // twist(256,320,1,512,1) |
| 570 |
// twist 256 320 1 512 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 571 |
// |
| 572 |
// // twist(320,384,1,512,5) |
| 573 |
// twist 320 384 1 512 5 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 574 |
// |
| 575 |
// // twist(384,448,1,512,-1) |
| 576 |
// twist 384 448 1 512 511 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 577 |
// |
| 578 |
// // twist(448,512,1,512,-5) |
| 579 |
// twist 448 512 1 512 507 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 580 |
// |
| 581 |
// // physical_permute(3,6) |
| 582 |
// physical_permute (3, 6) (0, 1, 2, 3, 4, 5, 6, 7, 8) () (0, 1, 2, 6, 4, 5, 3, 7, 8) () |
| 583 |
// |
| 584 |
// // fold(256) |
| 585 |
// physical_unmap (0, 1, 2, 6, 4, 5, 3, 7, 8) () |
| 586 |
// physical_map (0, 1, 2, 6, 4, 5, 3, 7) (8,) |
| 587 |
// |
| 588 |
// // fold(128) |
| 589 |
// physical_unmap (0, 1, 2, 6, 4, 5, 3, 7) (8,) |
| 590 |
// physical_map (0, 1, 2, 6, 4, 5, 3) (7, 8) |
| 591 |
// |
| 592 |
// // fold(64) |
| 593 |
// physical_unmap (0, 1, 2, 6, 4, 5, 3) (7, 8) |
| 594 |
// physical_map (0, 1, 2, 6, 4, 5) (3, 7, 8) |
| 595 |
// |
| 596 |
// // nextbatch() |
| 597 |
// stopbatch 512 |
| 598 |
// startbatch 512 |
| 599 |
// |
| 600 |
// // halfbatch() |
| 601 |
// physical_unmap (0, 1, 2, 6, 4, 5) (3, 7, 8) |
| 602 |
// stopbatch 512 |
| 603 |
// doublereps |
| 604 |
// startbatch 256 |
| 605 |
// physical_map (0, 1, 2, 6, 4, 5) (3, 7) |
| 606 |
// |
| 607 |
// // halfbatch() |
| 608 |
// physical_unmap (0, 1, 2, 6, 4, 5) (3, 7) |
| 609 |
// stopbatch 256 |
| 610 |
// doublereps |
| 611 |
// startbatch 128 |
| 612 |
// physical_map (0, 1, 2, 6, 4, 5) (3,) |
| 613 |
// |
| 614 |
// // ----- POSTCONDITIONS AFTER LAYER 3 |
| 615 |
// // transform size 64 |
| 616 |
// // transform indexing [0, 1, 2, 6, 4, 5] |
| 617 |
// // transforms per batch 2 |
| 618 |
// // batch indexing [3] |
| 619 |
// // total batch size 128 |
| 620 |
// |
| 621 |
// // modulus x^64-1 pos 0:64 q 7681,10753 bound 1*(5629,5827) 1*(5629,7613) 1*(5629,7666) 1*(5629,7264) 1*(5629,7639) 1*(5629,7591) 1*(5629,7291) 1*(5629,7204) ... |
| 622 |
// assertranges ... |
| 623 |
// |
| 624 |
// // ----- LAYER 4 |
| 625 |
// |
| 626 |
// // butterfly(0,32,1,32,1,0) |
| 627 |
// butterfly 0 32 1 32 1 0 (0, 1, 2, 6, 4, 5) (3,) |
| 628 |
// |
| 629 |
// // ----- POSTCONDITIONS AFTER LAYER 4 |
| 630 |
// // transform size 64 |
| 631 |
// // transform indexing [0, 1, 2, 6, 4, 5] |
| 632 |
// // transforms per batch 2 |
| 633 |
// // batch indexing [3] |
| 634 |
// // total batch size 128 |
| 635 |
// |
| 636 |
// // modulus x^32-1 pos 0:32 q 7681,10753 bound 1*(11258,13035) 1*(11258,14721) 1*(11258,14855) 1*(11258,14877) 1*(11258,14690) 1*(11258,15282) 1*(11258,14641) 1*(11258,14272) ... |
| 637 |
// assertranges ... |
| 638 |
// |
| 639 |
// // modulus x^32+1 pos 32:64 q 7681,10753 bound 1*(11258,13035) 1*(11258,14721) 1*(11258,14855) 1*(11258,14877) 1*(11258,14690) 1*(11258,15282) 1*(11258,14641) 1*(11258,14272) ... |
| 640 |
// assertranges ... |
| 641 |
// |
| 642 |
// // ----- LAYER 5 |
| 643 |
// |
| 644 |
// // butterfly(0,16,1,16,1,0) |
| 645 |
// butterfly 0 16 1 16 1 0 (0, 1, 2, 6, 4, 5) (3,) |
| 646 |
// |
| 647 |
// // butterfly(32,48,1,16,4,1) |
| 648 |
// butterfly 32 48 1 16 4 1 (0, 1, 2, 6, 4, 5) (3,) |
| 649 |
// |
| 650 |
// // reduce(0,16,1) |
| 651 |
// reduce 0 16 1 (0, 1, 2, 6, 4, 5) (3,) |
| 652 |
// |
| 653 |
// // twist(16,32,1,32,1) |
| 654 |
// twist 16 32 1 32 1 (0, 1, 2, 6, 4, 5) (3,) |
| 655 |
// |
| 656 |
// // twist(32,48,1,64,1) |
| 657 |
// twist 32 48 1 64 1 (0, 1, 2, 6, 4, 5) (3,) |
| 658 |
// |
| 659 |
// // twist(48,64,1,64,-1) |
| 660 |
// twist 48 64 1 64 63 (0, 1, 2, 6, 4, 5) (3,) |
| 661 |
// |
| 662 |
// // physical_permute(0,1,2,5) |
| 663 |
// physical_permute (0, 1, 2, 5) (0, 1, 2, 6, 4, 5) (3,) (1, 2, 5, 6, 4, 0) (3,) |
| 664 |
// |
| 665 |
// // fold(32) |
| 666 |
// physical_unmap (1, 2, 5, 6, 4, 0) (3,) |
| 667 |
// physical_map (1, 2, 5, 6, 4) (0, 3) |
| 668 |
// |
| 669 |
// // fold(16) |
| 670 |
// physical_unmap (1, 2, 5, 6, 4) (0, 3) |
| 671 |
// physical_map (1, 2, 5, 6) (0, 3, 4) |
| 672 |
// |
| 673 |
// // ----- POSTCONDITIONS AFTER LAYER 5 |
| 674 |
// // transform size 16 |
| 675 |
// // transform indexing [1, 2, 5, 6] |
| 676 |
// // transforms per batch 8 |
| 677 |
// // batch indexing [0, 3, 4] |
| 678 |
// // total batch size 128 |
| 679 |
// |
| 680 |
// // modulus x^16-1 pos 0:16 q 7681,10753 bound 1*(5629,5800) 1*(5629,6967) 1*(5629,6418) 1*(5629,7585) 1*(5629,7017) 1*(5629,6328) 1*(5629,7033) 1*(5629,6943) ... |
| 681 |
// assertranges ... |
| 682 |
// |
| 683 |
// // ----- LAYER 6 |
| 684 |
// |
| 685 |
// // butterfly(0,8,1,8,1,0) |
| 686 |
// butterfly 0 8 1 8 1 0 (1, 2, 5, 6) (0, 3, 4) |
| 687 |
// |
| 688 |
// // physical_permute(1,2,4) |
| 689 |
// physical_permute (1, 2, 4) (1, 2, 5, 6) (0, 3, 4) (2, 4, 5, 6) (0, 3, 1) |
| 690 |
// |
| 691 |
// // nextbatch() |
| 692 |
// stopbatch 128 |
| 693 |
// startbatch 128 |
| 694 |
// |
| 695 |
// // ----- POSTCONDITIONS AFTER LAYER 6 |
| 696 |
// // transform size 16 |
| 697 |
// // transform indexing [2, 4, 5, 6] |
| 698 |
// // transforms per batch 8 |
| 699 |
// // batch indexing [0, 3, 1] |
| 700 |
// // total batch size 128 |
| 701 |
// |
| 702 |
// // modulus x^8-1 pos 0:8 q 7681,10753 bound 1*(11258,12447) 1*(11258,14071) 1*(11258,12488) 1*(11258,14310) 1*(11258,14287) 1*(11258,13674) 1*(11258,13574) 1*(11258,13555) |
| 703 |
// assertranges ... |
| 704 |
// |
| 705 |
// // modulus x^8+1 pos 8:16 q 7681,10753 bound 1*(11258,12447) 1*(11258,14071) 1*(11258,12488) 1*(11258,14310) 1*(11258,14287) 1*(11258,13674) 1*(11258,13574) 1*(11258,13555) |
| 706 |
// assertranges ... |
| 707 |
// |
| 708 |
// // ----- LAYER 7 |
| 709 |
// |
| 710 |
// // butterfly(0,4,1,4,1,0) |
| 711 |
// butterfly 0 4 1 4 1 0 (2, 4, 5, 6) (0, 3, 1) |
| 712 |
// |
| 713 |
// // butterfly(8,12,1,4,4,1) |
| 714 |
// butterfly 8 12 1 4 4 1 (2, 4, 5, 6) (0, 3, 1) |
| 715 |
// |
| 716 |
// // reduce(0,4,1) |
| 717 |
// reduce 0 4 1 (2, 4, 5, 6) (0, 3, 1) |
| 718 |
// |
| 719 |
// // twist(4,8,1,8,1) |
| 720 |
// twist 4 8 1 8 1 (2, 4, 5, 6) (0, 3, 1) |
| 721 |
// |
| 722 |
// // twist(8,12,1,16,1) |
| 723 |
// twist 8 12 1 16 1 (2, 4, 5, 6) (0, 3, 1) |
| 724 |
// |
| 725 |
// // twist(12,16,1,16,-1) |
| 726 |
// twist 12 16 1 16 15 (2, 4, 5, 6) (0, 3, 1) |
| 727 |
// |
| 728 |
// // physical_permute(2,6) |
| 729 |
// physical_permute (2, 6) (2, 4, 5, 6) (0, 3, 1) (6, 4, 5, 2) (0, 3, 1) |
| 730 |
// |
| 731 |
// // fold(8) |
| 732 |
// physical_unmap (6, 4, 5, 2) (0, 3, 1) |
| 733 |
// physical_map (6, 4, 5) (0, 1, 2, 3) |
| 734 |
// |
| 735 |
// // fold(4) |
| 736 |
// physical_unmap (6, 4, 5) (0, 1, 2, 3) |
| 737 |
// physical_map (6, 4) (0, 1, 2, 3, 5) |
| 738 |
// |
| 739 |
// // ----- POSTCONDITIONS AFTER LAYER 7 |
| 740 |
// // transform size 4 |
| 741 |
// // transform indexing [6, 4] |
| 742 |
// // transforms per batch 32 |
| 743 |
// // batch indexing [0, 1, 2, 3, 5] |
| 744 |
// // total batch size 128 |
| 745 |
// |
| 746 |
// // modulus x^4-1 pos 0:4 q 7681,10753 bound 1*(5629,5800) 1*(5629,6938) 1*(5629,6521) 1*(5629,7157) |
| 747 |
// assertranges ... |
| 748 |
// |
| 749 |
// // ----- LAYER 8 |
| 750 |
// |
| 751 |
// // butterfly(0,2,1,2,1,0) |
| 752 |
// butterfly 0 2 1 2 1 0 (6, 4) (0, 1, 2, 3, 5) |
| 753 |
// |
| 754 |
// // ----- POSTCONDITIONS AFTER LAYER 8 |
| 755 |
// // transform size 4 |
| 756 |
// // transform indexing [6, 4] |
| 757 |
// // transforms per batch 32 |
| 758 |
// // batch indexing [0, 1, 2, 3, 5] |
| 759 |
// // total batch size 128 |
| 760 |
// |
| 761 |
// // modulus x^2-1 pos 0:2 q 7681,10753 bound 1*(11258,12321) 1*(11258,14095) |
| 762 |
// assertranges ... |
| 763 |
// |
| 764 |
// // modulus x^2+1 pos 2:4 q 7681,10753 bound 1*(11258,12321) 1*(11258,14095) |
| 765 |
// assertranges ... |
| 766 |
// |
| 767 |
// // ----- LAYER 9 |
| 768 |
// |
| 769 |
// // butterfly(0,1,1,1,1,0) |
| 770 |
// butterfly 0 1 1 1 1 0 (6, 4) (0, 1, 2, 3, 5) |
| 771 |
// |
| 772 |
// // butterfly(2,3,1,1,4,1) |
| 773 |
// butterfly 2 3 1 1 4 1 (6, 4) (0, 1, 2, 3, 5) |
| 774 |
// |
| 775 |
// // ----- POSTCONDITIONS AFTER LAYER 9 |
| 776 |
// // transform size 4 |
| 777 |
// // transform indexing [6, 4] |
| 778 |
// // transforms per batch 32 |
| 779 |
// // batch indexing [0, 1, 2, 3, 5] |
| 780 |
// // total batch size 128 |
| 781 |
// |
| 782 |
// // modulus x^1-1 pos 0:1 q 7681,10753 bound 1*(22516,26416) |
| 783 |
// assertranges ... |
| 784 |
// |
| 785 |
// // modulus x^1+1 pos 1:2 q 7681,10753 bound 1*(22516,26416) |
| 786 |
// assertranges ... |
| 787 |
// |
| 788 |
// // modulus x^1-zeta4 pos 2:3 q 7681,10753 bound 1*(15747,17745) |
| 789 |
// assertranges ... |
| 790 |
// |
| 791 |
// // modulus x^1+zeta4 pos 3:4 q 7681,10753 bound 1*(15747,17745) |
| 792 |
// assertranges ... |
| 793 |
// stopbatch 128 |
| 794 |
// physical_unmap (6, 4) (0, 1, 2, 3, 5) |
| 795 |
// stopntt 512 |
| 796 |
|
| 797 |
// ----- codegen pass 2 |
| 798 |
// |
| 799 |
// startntt 512 |
| 800 |
// startbatch 512 |
| 801 |
// vector_butterfly 0 256 1 0 |
| 802 |
// vector_butterfly 128 384 1 0 |
| 803 |
// vector_butterfly 64 320 1 0 |
| 804 |
// vector_butterfly 192 448 1 0 |
| 805 |
// vector_reduce_ifreverse 0 |
| 806 |
// vector_reduce_ifreverse 256 |
| 807 |
// vector_butterfly 0 128 1 0 |
| 808 |
// vector_butterfly 64 192 1 0 |
| 809 |
// vector_butterfly 256 384 4 1 |
| 810 |
// vector_butterfly 320 448 4 1 |
| 811 |
// vector_reduce_ifforward 64 |
| 812 |
// vector_butterfly 0 64 1 0 |
| 813 |
// vector_butterfly 128 192 4 1 |
| 814 |
// vector_butterfly 256 320 8 1 |
| 815 |
// vector_butterfly 384 448 8 7 |
| 816 |
// vector_reduce 0 |
| 817 |
// vector_twist 64 128 1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 818 |
// vector_twist 128 256 1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 819 |
// vector_twist 192 256 255 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 820 |
// vector_twist 256 512 1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 821 |
// vector_twist 320 512 5 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 822 |
// vector_twist 384 512 511 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 823 |
// vector_twist 448 512 507 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 824 |
// vector_permute 0 64 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 825 |
// vector_permute 128 192 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 826 |
// vector_permute 256 320 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 827 |
// vector_permute 384 448 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 828 |
// stopbatch 512 |
| 829 |
// startbatch 512 |
| 830 |
// vector_butterfly 16 272 1 0 |
| 831 |
// vector_butterfly 144 400 1 0 |
| 832 |
// vector_butterfly 80 336 1 0 |
| 833 |
// vector_butterfly 208 464 1 0 |
| 834 |
// vector_reduce_ifreverse 16 |
| 835 |
// vector_reduce_ifreverse 272 |
| 836 |
// vector_butterfly 16 144 1 0 |
| 837 |
// vector_butterfly 80 208 1 0 |
| 838 |
// vector_butterfly 272 400 4 1 |
| 839 |
// vector_butterfly 336 464 4 1 |
| 840 |
// vector_reduce_ifforward 80 |
| 841 |
// vector_butterfly 16 80 1 0 |
| 842 |
// vector_butterfly 144 208 4 1 |
| 843 |
// vector_butterfly 272 336 8 1 |
| 844 |
// vector_butterfly 400 464 8 7 |
| 845 |
// vector_reduce 16 |
| 846 |
// vector_twist 80 128 1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 847 |
// vector_twist 144 256 1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 848 |
// vector_twist 208 256 255 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 849 |
// vector_twist 272 512 1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 850 |
// vector_twist 336 512 5 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 851 |
// vector_twist 400 512 511 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 852 |
// vector_twist 464 512 507 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 853 |
// vector_permute 16 80 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 854 |
// vector_permute 144 208 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 855 |
// vector_permute 272 336 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 856 |
// vector_permute 400 464 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 857 |
// stopbatch 512 |
| 858 |
// startbatch 512 |
| 859 |
// vector_butterfly 32 288 1 0 |
| 860 |
// vector_butterfly 160 416 1 0 |
| 861 |
// vector_butterfly 96 352 1 0 |
| 862 |
// vector_butterfly 224 480 1 0 |
| 863 |
// vector_reduce_ifreverse 32 |
| 864 |
// vector_reduce_ifreverse 288 |
| 865 |
// vector_butterfly 32 160 1 0 |
| 866 |
// vector_butterfly 96 224 1 0 |
| 867 |
// vector_butterfly 288 416 4 1 |
| 868 |
// vector_butterfly 352 480 4 1 |
| 869 |
// vector_reduce_ifforward 96 |
| 870 |
// vector_butterfly 32 96 1 0 |
| 871 |
// vector_butterfly 160 224 4 1 |
| 872 |
// vector_butterfly 288 352 8 1 |
| 873 |
// vector_butterfly 416 480 8 7 |
| 874 |
// vector_reduce 32 |
| 875 |
// vector_twist 96 128 1 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 876 |
// vector_twist 160 256 1 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 877 |
// vector_twist 224 256 255 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 878 |
// vector_twist 288 512 1 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 879 |
// vector_twist 352 512 5 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 880 |
// vector_twist 416 512 511 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 881 |
// vector_twist 480 512 507 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 882 |
// vector_permute 32 96 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 883 |
// vector_permute 160 224 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 884 |
// vector_permute 288 352 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 885 |
// vector_permute 416 480 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 886 |
// stopbatch 512 |
| 887 |
// startbatch 512 |
| 888 |
// vector_butterfly 48 304 1 0 |
| 889 |
// vector_butterfly 176 432 1 0 |
| 890 |
// vector_butterfly 112 368 1 0 |
| 891 |
// vector_butterfly 240 496 1 0 |
| 892 |
// vector_reduce_ifreverse 48 |
| 893 |
// vector_reduce_ifreverse 304 |
| 894 |
// vector_butterfly 48 176 1 0 |
| 895 |
// vector_butterfly 112 240 1 0 |
| 896 |
// vector_butterfly 304 432 4 1 |
| 897 |
// vector_butterfly 368 496 4 1 |
| 898 |
// vector_reduce_ifforward 112 |
| 899 |
// vector_butterfly 48 112 1 0 |
| 900 |
// vector_butterfly 176 240 4 1 |
| 901 |
// vector_butterfly 304 368 8 1 |
| 902 |
// vector_butterfly 432 496 8 7 |
| 903 |
// vector_reduce 48 |
| 904 |
// vector_twist 112 128 1 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 905 |
// vector_twist 176 256 1 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 906 |
// vector_twist 240 256 255 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 907 |
// vector_twist 304 512 1 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 908 |
// vector_twist 368 512 5 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 909 |
// vector_twist 432 512 511 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 910 |
// vector_twist 496 512 507 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 911 |
// vector_permute 48 112 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 912 |
// vector_permute 176 240 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 913 |
// vector_permute 304 368 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 914 |
// vector_permute 432 496 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 915 |
// stopbatch 512 |
| 916 |
// doublereps |
| 917 |
// doublereps |
| 918 |
// startbatch 128 |
| 919 |
// vector_butterfly 0 32 1 0 |
| 920 |
// vector_butterfly 64 96 1 0 |
| 921 |
// vector_butterfly 16 48 1 0 |
| 922 |
// vector_butterfly 80 112 1 0 |
| 923 |
// vector_butterfly 0 16 1 0 |
| 924 |
// vector_butterfly 64 80 1 0 |
| 925 |
// vector_butterfly 32 48 4 1 |
| 926 |
// vector_butterfly 96 112 4 1 |
| 927 |
// vector_reduce 0 |
| 928 |
// vector_reduce 64 |
| 929 |
// vector_twist 16 32 1 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 |
| 930 |
// vector_twist 80 32 1 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15 |
| 931 |
// vector_twist 32 64 1 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 |
| 932 |
// vector_twist 96 64 1 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15 |
| 933 |
// vector_twist 48 64 63 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 |
| 934 |
// vector_twist 112 64 63 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15 |
| 935 |
// vector_permute 0 32 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16 |
| 936 |
// vector_permute 16 48 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16 |
| 937 |
// vector_permute 64 96 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16 |
| 938 |
// vector_permute 80 112 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16 |
| 939 |
// vector_butterfly 0 64 1 0 |
| 940 |
// vector_butterfly 32 96 1 0 |
| 941 |
// vector_butterfly 16 80 1 0 |
| 942 |
// vector_butterfly 48 112 1 0 |
| 943 |
// vector_permute 0 16 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32 |
| 944 |
// vector_permute 32 48 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32 |
| 945 |
// vector_permute 64 80 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32 |
| 946 |
// vector_permute 96 112 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32 |
| 947 |
// stopbatch 128 |
| 948 |
// startbatch 128 |
| 949 |
// vector_butterfly 0 32 1 0 |
| 950 |
// vector_butterfly 16 48 1 0 |
| 951 |
// vector_butterfly 64 96 4 1 |
| 952 |
// vector_butterfly 80 112 4 1 |
| 953 |
// vector_reduce 0 |
| 954 |
// vector_reduce 16 |
| 955 |
// vector_twist 32 8 1 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1 |
| 956 |
// vector_twist 48 8 1 2 2 2 2 3 3 3 3 2 2 2 2 3 3 3 3 |
| 957 |
// vector_twist 64 16 1 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1 |
| 958 |
// vector_twist 80 16 1 2 2 2 2 3 3 3 3 2 2 2 2 3 3 3 3 |
| 959 |
// vector_twist 96 16 15 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1 |
| 960 |
// vector_twist 112 16 15 2 2 2 2 3 3 3 3 2 2 2 2 3 3 3 3 |
| 961 |
// vector_permute 0 64 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64 |
| 962 |
// vector_permute 16 80 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64 |
| 963 |
// vector_permute 32 96 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64 |
| 964 |
// vector_permute 48 112 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64 |
| 965 |
// vector_butterfly 0 16 1 0 |
| 966 |
// vector_butterfly 64 80 1 0 |
| 967 |
// vector_butterfly 32 48 1 0 |
| 968 |
// vector_butterfly 96 112 1 0 |
| 969 |
// vector_butterfly 0 64 1 0 |
| 970 |
// vector_butterfly 32 96 1 0 |
| 971 |
// vector_butterfly 16 80 4 1 |
| 972 |
// vector_butterfly 48 112 4 1 |
| 973 |
// stopbatch 128 |
| 974 |
// stopntt 512 |
| 975 |
// startntt 512 |
| 976 |
|
| 977 |
static void ntt512(int16 *f,int reps,const int16 *qdata) |
| 978 |
{ |
| 979 |
// startbatch 512 |
| 980 |
for (long long r = 0;r < reps;++r) { |
| 981 |
// vector_butterfly 0 256 1 0 |
| 982 |
int16x16 a0 = _mm256_loadu_si256((int16x16 *) (f+0)); |
| 983 |
int16x16 a16 = _mm256_loadu_si256((int16x16 *) (f+256)); |
| 984 |
int16x16 b0 = add_x16(a0,a16); |
| 985 |
int16x16 b16 = sub_x16(a0,a16); |
| 986 |
// vector_butterfly 128 384 1 0 |
| 987 |
int16x16 a8 = _mm256_loadu_si256((int16x16 *) (f+128)); |
| 988 |
int16x16 a24 = _mm256_loadu_si256((int16x16 *) (f+384)); |
| 989 |
int16x16 b8 = add_x16(a8,a24); |
| 990 |
int16x16 b24 = sub_x16(a8,a24); |
| 991 |
// vector_butterfly 64 320 1 0 |
| 992 |
int16x16 a4 = _mm256_loadu_si256((int16x16 *) (f+64)); |
| 993 |
int16x16 a20 = _mm256_loadu_si256((int16x16 *) (f+320)); |
| 994 |
int16x16 b4 = add_x16(a4,a20); |
| 995 |
int16x16 b20 = sub_x16(a4,a20); |
| 996 |
// vector_butterfly 192 448 1 0 |
| 997 |
int16x16 a12 = _mm256_loadu_si256((int16x16 *) (f+192)); |
| 998 |
int16x16 a28 = _mm256_loadu_si256((int16x16 *) (f+448)); |
| 999 |
int16x16 b12 = add_x16(a12,a28); |
| 1000 |
int16x16 b28 = sub_x16(a12,a28); |
| 1001 |
// vector_reduce_ifreverse 0 |
| 1002 |
// vector_reduce_ifreverse 256 |
| 1003 |
// vector_butterfly 0 128 1 0 |
| 1004 |
int16x16 c0 = add_x16(b0,b8); |
| 1005 |
int16x16 c8 = sub_x16(b0,b8); |
| 1006 |
// vector_butterfly 64 192 1 0 |
| 1007 |
int16x16 c4 = add_x16(b4,b12); |
| 1008 |
int16x16 c12 = sub_x16(b4,b12); |
| 1009 |
// vector_butterfly 256 384 4 1 |
| 1010 |
b24 = mulmod_scaled_x16(b24,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 1011 |
int16x16 c16 = add_x16(b16,b24); |
| 1012 |
int16x16 c24 = sub_x16(b16,b24); |
| 1013 |
// vector_butterfly 320 448 4 1 |
| 1014 |
b28 = mulmod_scaled_x16(b28,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 1015 |
int16x16 c20 = add_x16(b20,b28); |
| 1016 |
int16x16 c28 = sub_x16(b20,b28); |
| 1017 |
// vector_reduce_ifforward 64 |
| 1018 |
c4 = reduce_x16(c4,qdata); |
| 1019 |
// vector_butterfly 0 64 1 0 |
| 1020 |
int16x16 d0 = add_x16(c0,c4); |
| 1021 |
int16x16 d4 = sub_x16(c0,c4); |
| 1022 |
// vector_butterfly 128 192 4 1 |
| 1023 |
c12 = mulmod_scaled_x16(c12,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 1024 |
int16x16 d8 = add_x16(c8,c12); |
| 1025 |
int16x16 d12 = sub_x16(c8,c12); |
| 1026 |
// vector_butterfly 256 320 8 1 |
| 1027 |
c20 = mulmod_scaled_x16(c20,scaledzeta_x16_8_1,qinvscaledzeta_x16_8_1,qdata); |
| 1028 |
int16x16 d16 = add_x16(c16,c20); |
| 1029 |
int16x16 d20 = sub_x16(c16,c20); |
| 1030 |
// vector_butterfly 384 448 8 7 |
| 1031 |
c28 = mulmod_scaled_x16(c28,scaledzeta_x16_8_7,qinvscaledzeta_x16_8_7,qdata); |
| 1032 |
int16x16 d24 = add_x16(c24,c28); |
| 1033 |
int16x16 d28 = sub_x16(c24,c28); |
| 1034 |
// vector_reduce 0 |
| 1035 |
d0 = reduce_x16(d0,qdata); |
| 1036 |
// vector_twist 64 128 1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 1037 |
d4 = mulmod_scaled_x16(d4,precomp_128_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_128_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 1038 |
// vector_twist 128 256 1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 1039 |
d8 = mulmod_scaled_x16(d8,precomp_256_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_256_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 1040 |
// vector_twist 192 256 255 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 1041 |
d12 = mulmod_scaled_x16(d12,precomp_256_255_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_256_255_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 1042 |
// vector_twist 256 512 1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 1043 |
d16 = mulmod_scaled_x16(d16,precomp_512_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_512_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 1044 |
// vector_twist 320 512 5 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 1045 |
d20 = mulmod_scaled_x16(d20,precomp_512_5_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_512_5_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 1046 |
// vector_twist 384 512 511 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 1047 |
d24 = mulmod_scaled_x16(d24,precomp_512_511_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_512_511_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 1048 |
// vector_twist 448 512 507 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 1049 |
d28 = mulmod_scaled_x16(d28,precomp_512_507_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_512_507_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 1050 |
// vector_permute 0 64 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1051 |
int16x16 e0 = _mm256_permute2x128_si256_lo(d0,d4); |
| 1052 |
int16x16 e4 = _mm256_permute2x128_si256_hi(d0,d4); |
| 1053 |
// vector_permute 128 192 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1054 |
int16x16 e8 = _mm256_permute2x128_si256_lo(d8,d12); |
| 1055 |
int16x16 e12 = _mm256_permute2x128_si256_hi(d8,d12); |
| 1056 |
// vector_permute 256 320 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1057 |
int16x16 e16 = _mm256_permute2x128_si256_lo(d16,d20); |
| 1058 |
int16x16 e20 = _mm256_permute2x128_si256_hi(d16,d20); |
| 1059 |
// vector_permute 384 448 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1060 |
int16x16 e24 = _mm256_permute2x128_si256_lo(d24,d28); |
| 1061 |
int16x16 e28 = _mm256_permute2x128_si256_hi(d24,d28); |
| 1062 |
// stopbatch 512 |
| 1063 |
_mm256_storeu_si256((int16x16 *) (f+0),e0); |
| 1064 |
_mm256_storeu_si256((int16x16 *) (f+64),e4); |
| 1065 |
_mm256_storeu_si256((int16x16 *) (f+128),e8); |
| 1066 |
_mm256_storeu_si256((int16x16 *) (f+192),e12); |
| 1067 |
_mm256_storeu_si256((int16x16 *) (f+256),e16); |
| 1068 |
_mm256_storeu_si256((int16x16 *) (f+320),e20); |
| 1069 |
_mm256_storeu_si256((int16x16 *) (f+384),e24); |
| 1070 |
_mm256_storeu_si256((int16x16 *) (f+448),e28); |
| 1071 |
f += 512; |
| 1072 |
} |
| 1073 |
f -= 512*reps; |
| 1074 |
// startbatch 512 |
| 1075 |
for (long long r = 0;r < reps;++r) { |
| 1076 |
// vector_butterfly 16 272 1 0 |
| 1077 |
int16x16 a1 = _mm256_loadu_si256((int16x16 *) (f+16)); |
| 1078 |
int16x16 a17 = _mm256_loadu_si256((int16x16 *) (f+272)); |
| 1079 |
int16x16 b1 = add_x16(a1,a17); |
| 1080 |
int16x16 b17 = sub_x16(a1,a17); |
| 1081 |
// vector_butterfly 144 400 1 0 |
| 1082 |
int16x16 a9 = _mm256_loadu_si256((int16x16 *) (f+144)); |
| 1083 |
int16x16 a25 = _mm256_loadu_si256((int16x16 *) (f+400)); |
| 1084 |
int16x16 b9 = add_x16(a9,a25); |
| 1085 |
int16x16 b25 = sub_x16(a9,a25); |
| 1086 |
// vector_butterfly 80 336 1 0 |
| 1087 |
int16x16 a5 = _mm256_loadu_si256((int16x16 *) (f+80)); |
| 1088 |
int16x16 a21 = _mm256_loadu_si256((int16x16 *) (f+336)); |
| 1089 |
int16x16 b5 = add_x16(a5,a21); |
| 1090 |
int16x16 b21 = sub_x16(a5,a21); |
| 1091 |
// vector_butterfly 208 464 1 0 |
| 1092 |
int16x16 a13 = _mm256_loadu_si256((int16x16 *) (f+208)); |
| 1093 |
int16x16 a29 = _mm256_loadu_si256((int16x16 *) (f+464)); |
| 1094 |
int16x16 b13 = add_x16(a13,a29); |
| 1095 |
int16x16 b29 = sub_x16(a13,a29); |
| 1096 |
// vector_reduce_ifreverse 16 |
| 1097 |
// vector_reduce_ifreverse 272 |
| 1098 |
// vector_butterfly 16 144 1 0 |
| 1099 |
int16x16 c1 = add_x16(b1,b9); |
| 1100 |
int16x16 c9 = sub_x16(b1,b9); |
| 1101 |
// vector_butterfly 80 208 1 0 |
| 1102 |
int16x16 c5 = add_x16(b5,b13); |
| 1103 |
int16x16 c13 = sub_x16(b5,b13); |
| 1104 |
// vector_butterfly 272 400 4 1 |
| 1105 |
b25 = mulmod_scaled_x16(b25,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 1106 |
int16x16 c17 = add_x16(b17,b25); |
| 1107 |
int16x16 c25 = sub_x16(b17,b25); |
| 1108 |
// vector_butterfly 336 464 4 1 |
| 1109 |
b29 = mulmod_scaled_x16(b29,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 1110 |
int16x16 c21 = add_x16(b21,b29); |
| 1111 |
int16x16 c29 = sub_x16(b21,b29); |
| 1112 |
// vector_reduce_ifforward 80 |
| 1113 |
c5 = reduce_x16(c5,qdata); |
| 1114 |
// vector_butterfly 16 80 1 0 |
| 1115 |
int16x16 d1 = add_x16(c1,c5); |
| 1116 |
int16x16 d5 = sub_x16(c1,c5); |
| 1117 |
// vector_butterfly 144 208 4 1 |
| 1118 |
c13 = mulmod_scaled_x16(c13,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 1119 |
int16x16 d9 = add_x16(c9,c13); |
| 1120 |
int16x16 d13 = sub_x16(c9,c13); |
| 1121 |
// vector_butterfly 272 336 8 1 |
| 1122 |
c21 = mulmod_scaled_x16(c21,scaledzeta_x16_8_1,qinvscaledzeta_x16_8_1,qdata); |
| 1123 |
int16x16 d17 = add_x16(c17,c21); |
| 1124 |
int16x16 d21 = sub_x16(c17,c21); |
| 1125 |
// vector_butterfly 400 464 8 7 |
| 1126 |
c29 = mulmod_scaled_x16(c29,scaledzeta_x16_8_7,qinvscaledzeta_x16_8_7,qdata); |
| 1127 |
int16x16 d25 = add_x16(c25,c29); |
| 1128 |
int16x16 d29 = sub_x16(c25,c29); |
| 1129 |
// vector_reduce 16 |
| 1130 |
d1 = reduce_x16(d1,qdata); |
| 1131 |
// vector_twist 80 128 1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 1132 |
d5 = mulmod_scaled_x16(d5,precomp_128_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_128_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 1133 |
// vector_twist 144 256 1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 1134 |
d9 = mulmod_scaled_x16(d9,precomp_256_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_256_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 1135 |
// vector_twist 208 256 255 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 1136 |
d13 = mulmod_scaled_x16(d13,precomp_256_255_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_256_255_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 1137 |
// vector_twist 272 512 1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 1138 |
d17 = mulmod_scaled_x16(d17,precomp_512_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_512_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 1139 |
// vector_twist 336 512 5 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 1140 |
d21 = mulmod_scaled_x16(d21,precomp_512_5_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_512_5_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 1141 |
// vector_twist 400 512 511 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 1142 |
d25 = mulmod_scaled_x16(d25,precomp_512_511_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_512_511_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 1143 |
// vector_twist 464 512 507 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 1144 |
d29 = mulmod_scaled_x16(d29,precomp_512_507_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_512_507_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 1145 |
// vector_permute 16 80 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1146 |
int16x16 e1 = _mm256_permute2x128_si256_lo(d1,d5); |
| 1147 |
int16x16 e5 = _mm256_permute2x128_si256_hi(d1,d5); |
| 1148 |
// vector_permute 144 208 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1149 |
int16x16 e9 = _mm256_permute2x128_si256_lo(d9,d13); |
| 1150 |
int16x16 e13 = _mm256_permute2x128_si256_hi(d9,d13); |
| 1151 |
// vector_permute 272 336 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1152 |
int16x16 e17 = _mm256_permute2x128_si256_lo(d17,d21); |
| 1153 |
int16x16 e21 = _mm256_permute2x128_si256_hi(d17,d21); |
| 1154 |
// vector_permute 400 464 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1155 |
int16x16 e25 = _mm256_permute2x128_si256_lo(d25,d29); |
| 1156 |
int16x16 e29 = _mm256_permute2x128_si256_hi(d25,d29); |
| 1157 |
// stopbatch 512 |
| 1158 |
_mm256_storeu_si256((int16x16 *) (f+16),e1); |
| 1159 |
_mm256_storeu_si256((int16x16 *) (f+80),e5); |
| 1160 |
_mm256_storeu_si256((int16x16 *) (f+144),e9); |
| 1161 |
_mm256_storeu_si256((int16x16 *) (f+208),e13); |
| 1162 |
_mm256_storeu_si256((int16x16 *) (f+272),e17); |
| 1163 |
_mm256_storeu_si256((int16x16 *) (f+336),e21); |
| 1164 |
_mm256_storeu_si256((int16x16 *) (f+400),e25); |
| 1165 |
_mm256_storeu_si256((int16x16 *) (f+464),e29); |
| 1166 |
f += 512; |
| 1167 |
} |
| 1168 |
f -= 512*reps; |
| 1169 |
// startbatch 512 |
| 1170 |
for (long long r = 0;r < reps;++r) { |
| 1171 |
// vector_butterfly 32 288 1 0 |
| 1172 |
int16x16 a2 = _mm256_loadu_si256((int16x16 *) (f+32)); |
| 1173 |
int16x16 a18 = _mm256_loadu_si256((int16x16 *) (f+288)); |
| 1174 |
int16x16 b2 = add_x16(a2,a18); |
| 1175 |
int16x16 b18 = sub_x16(a2,a18); |
| 1176 |
// vector_butterfly 160 416 1 0 |
| 1177 |
int16x16 a10 = _mm256_loadu_si256((int16x16 *) (f+160)); |
| 1178 |
int16x16 a26 = _mm256_loadu_si256((int16x16 *) (f+416)); |
| 1179 |
int16x16 b10 = add_x16(a10,a26); |
| 1180 |
int16x16 b26 = sub_x16(a10,a26); |
| 1181 |
// vector_butterfly 96 352 1 0 |
| 1182 |
int16x16 a6 = _mm256_loadu_si256((int16x16 *) (f+96)); |
| 1183 |
int16x16 a22 = _mm256_loadu_si256((int16x16 *) (f+352)); |
| 1184 |
int16x16 b6 = add_x16(a6,a22); |
| 1185 |
int16x16 b22 = sub_x16(a6,a22); |
| 1186 |
// vector_butterfly 224 480 1 0 |
| 1187 |
int16x16 a14 = _mm256_loadu_si256((int16x16 *) (f+224)); |
| 1188 |
int16x16 a30 = _mm256_loadu_si256((int16x16 *) (f+480)); |
| 1189 |
int16x16 b14 = add_x16(a14,a30); |
| 1190 |
int16x16 b30 = sub_x16(a14,a30); |
| 1191 |
// vector_reduce_ifreverse 32 |
| 1192 |
// vector_reduce_ifreverse 288 |
| 1193 |
// vector_butterfly 32 160 1 0 |
| 1194 |
int16x16 c2 = add_x16(b2,b10); |
| 1195 |
int16x16 c10 = sub_x16(b2,b10); |
| 1196 |
// vector_butterfly 96 224 1 0 |
| 1197 |
int16x16 c6 = add_x16(b6,b14); |
| 1198 |
int16x16 c14 = sub_x16(b6,b14); |
| 1199 |
// vector_butterfly 288 416 4 1 |
| 1200 |
b26 = mulmod_scaled_x16(b26,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 1201 |
int16x16 c18 = add_x16(b18,b26); |
| 1202 |
int16x16 c26 = sub_x16(b18,b26); |
| 1203 |
// vector_butterfly 352 480 4 1 |
| 1204 |
b30 = mulmod_scaled_x16(b30,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 1205 |
int16x16 c22 = add_x16(b22,b30); |
| 1206 |
int16x16 c30 = sub_x16(b22,b30); |
| 1207 |
// vector_reduce_ifforward 96 |
| 1208 |
c6 = reduce_x16(c6,qdata); |
| 1209 |
// vector_butterfly 32 96 1 0 |
| 1210 |
int16x16 d2 = add_x16(c2,c6); |
| 1211 |
int16x16 d6 = sub_x16(c2,c6); |
| 1212 |
// vector_butterfly 160 224 4 1 |
| 1213 |
c14 = mulmod_scaled_x16(c14,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 1214 |
int16x16 d10 = add_x16(c10,c14); |
| 1215 |
int16x16 d14 = sub_x16(c10,c14); |
| 1216 |
// vector_butterfly 288 352 8 1 |
| 1217 |
c22 = mulmod_scaled_x16(c22,scaledzeta_x16_8_1,qinvscaledzeta_x16_8_1,qdata); |
| 1218 |
int16x16 d18 = add_x16(c18,c22); |
| 1219 |
int16x16 d22 = sub_x16(c18,c22); |
| 1220 |
// vector_butterfly 416 480 8 7 |
| 1221 |
c30 = mulmod_scaled_x16(c30,scaledzeta_x16_8_7,qinvscaledzeta_x16_8_7,qdata); |
| 1222 |
int16x16 d26 = add_x16(c26,c30); |
| 1223 |
int16x16 d30 = sub_x16(c26,c30); |
| 1224 |
// vector_reduce 32 |
| 1225 |
d2 = reduce_x16(d2,qdata); |
| 1226 |
// vector_twist 96 128 1 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1227 |
d6 = mulmod_scaled_x16(d6,precomp_128_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_128_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1228 |
// vector_twist 160 256 1 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1229 |
d10 = mulmod_scaled_x16(d10,precomp_256_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_256_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1230 |
// vector_twist 224 256 255 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1231 |
d14 = mulmod_scaled_x16(d14,precomp_256_255_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_256_255_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1232 |
// vector_twist 288 512 1 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1233 |
d18 = mulmod_scaled_x16(d18,precomp_512_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_512_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1234 |
// vector_twist 352 512 5 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1235 |
d22 = mulmod_scaled_x16(d22,precomp_512_5_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_512_5_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1236 |
// vector_twist 416 512 511 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1237 |
d26 = mulmod_scaled_x16(d26,precomp_512_511_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_512_511_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1238 |
// vector_twist 480 512 507 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1239 |
d30 = mulmod_scaled_x16(d30,precomp_512_507_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_512_507_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1240 |
// vector_permute 32 96 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1241 |
int16x16 e2 = _mm256_permute2x128_si256_lo(d2,d6); |
| 1242 |
int16x16 e6 = _mm256_permute2x128_si256_hi(d2,d6); |
| 1243 |
// vector_permute 160 224 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1244 |
int16x16 e10 = _mm256_permute2x128_si256_lo(d10,d14); |
| 1245 |
int16x16 e14 = _mm256_permute2x128_si256_hi(d10,d14); |
| 1246 |
// vector_permute 288 352 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1247 |
int16x16 e18 = _mm256_permute2x128_si256_lo(d18,d22); |
| 1248 |
int16x16 e22 = _mm256_permute2x128_si256_hi(d18,d22); |
| 1249 |
// vector_permute 416 480 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1250 |
int16x16 e26 = _mm256_permute2x128_si256_lo(d26,d30); |
| 1251 |
int16x16 e30 = _mm256_permute2x128_si256_hi(d26,d30); |
| 1252 |
// stopbatch 512 |
| 1253 |
_mm256_storeu_si256((int16x16 *) (f+32),e2); |
| 1254 |
_mm256_storeu_si256((int16x16 *) (f+96),e6); |
| 1255 |
_mm256_storeu_si256((int16x16 *) (f+160),e10); |
| 1256 |
_mm256_storeu_si256((int16x16 *) (f+224),e14); |
| 1257 |
_mm256_storeu_si256((int16x16 *) (f+288),e18); |
| 1258 |
_mm256_storeu_si256((int16x16 *) (f+352),e22); |
| 1259 |
_mm256_storeu_si256((int16x16 *) (f+416),e26); |
| 1260 |
_mm256_storeu_si256((int16x16 *) (f+480),e30); |
| 1261 |
f += 512; |
| 1262 |
} |
| 1263 |
f -= 512*reps; |
| 1264 |
// startbatch 512 |
| 1265 |
for (long long r = 0;r < reps;++r) { |
| 1266 |
// vector_butterfly 48 304 1 0 |
| 1267 |
int16x16 a3 = _mm256_loadu_si256((int16x16 *) (f+48)); |
| 1268 |
int16x16 a19 = _mm256_loadu_si256((int16x16 *) (f+304)); |
| 1269 |
int16x16 b3 = add_x16(a3,a19); |
| 1270 |
int16x16 b19 = sub_x16(a3,a19); |
| 1271 |
// vector_butterfly 176 432 1 0 |
| 1272 |
int16x16 a11 = _mm256_loadu_si256((int16x16 *) (f+176)); |
| 1273 |
int16x16 a27 = _mm256_loadu_si256((int16x16 *) (f+432)); |
| 1274 |
int16x16 b11 = add_x16(a11,a27); |
| 1275 |
int16x16 b27 = sub_x16(a11,a27); |
| 1276 |
// vector_butterfly 112 368 1 0 |
| 1277 |
int16x16 a7 = _mm256_loadu_si256((int16x16 *) (f+112)); |
| 1278 |
int16x16 a23 = _mm256_loadu_si256((int16x16 *) (f+368)); |
| 1279 |
int16x16 b7 = add_x16(a7,a23); |
| 1280 |
int16x16 b23 = sub_x16(a7,a23); |
| 1281 |
// vector_butterfly 240 496 1 0 |
| 1282 |
int16x16 a15 = _mm256_loadu_si256((int16x16 *) (f+240)); |
| 1283 |
int16x16 a31 = _mm256_loadu_si256((int16x16 *) (f+496)); |
| 1284 |
int16x16 b15 = add_x16(a15,a31); |
| 1285 |
int16x16 b31 = sub_x16(a15,a31); |
| 1286 |
// vector_reduce_ifreverse 48 |
| 1287 |
// vector_reduce_ifreverse 304 |
| 1288 |
// vector_butterfly 48 176 1 0 |
| 1289 |
int16x16 c3 = add_x16(b3,b11); |
| 1290 |
int16x16 c11 = sub_x16(b3,b11); |
| 1291 |
// vector_butterfly 112 240 1 0 |
| 1292 |
int16x16 c7 = add_x16(b7,b15); |
| 1293 |
int16x16 c15 = sub_x16(b7,b15); |
| 1294 |
// vector_butterfly 304 432 4 1 |
| 1295 |
b27 = mulmod_scaled_x16(b27,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 1296 |
int16x16 c19 = add_x16(b19,b27); |
| 1297 |
int16x16 c27 = sub_x16(b19,b27); |
| 1298 |
// vector_butterfly 368 496 4 1 |
| 1299 |
b31 = mulmod_scaled_x16(b31,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 1300 |
int16x16 c23 = add_x16(b23,b31); |
| 1301 |
int16x16 c31 = sub_x16(b23,b31); |
| 1302 |
// vector_reduce_ifforward 112 |
| 1303 |
c7 = reduce_x16(c7,qdata); |
| 1304 |
// vector_butterfly 48 112 1 0 |
| 1305 |
int16x16 d3 = add_x16(c3,c7); |
| 1306 |
int16x16 d7 = sub_x16(c3,c7); |
| 1307 |
// vector_butterfly 176 240 4 1 |
| 1308 |
c15 = mulmod_scaled_x16(c15,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 1309 |
int16x16 d11 = add_x16(c11,c15); |
| 1310 |
int16x16 d15 = sub_x16(c11,c15); |
| 1311 |
// vector_butterfly 304 368 8 1 |
| 1312 |
c23 = mulmod_scaled_x16(c23,scaledzeta_x16_8_1,qinvscaledzeta_x16_8_1,qdata); |
| 1313 |
int16x16 d19 = add_x16(c19,c23); |
| 1314 |
int16x16 d23 = sub_x16(c19,c23); |
| 1315 |
// vector_butterfly 432 496 8 7 |
| 1316 |
c31 = mulmod_scaled_x16(c31,scaledzeta_x16_8_7,qinvscaledzeta_x16_8_7,qdata); |
| 1317 |
int16x16 d27 = add_x16(c27,c31); |
| 1318 |
int16x16 d31 = sub_x16(c27,c31); |
| 1319 |
// vector_reduce 48 |
| 1320 |
d3 = reduce_x16(d3,qdata); |
| 1321 |
// vector_twist 112 128 1 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1322 |
d7 = mulmod_scaled_x16(d7,precomp_128_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_128_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1323 |
// vector_twist 176 256 1 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1324 |
d11 = mulmod_scaled_x16(d11,precomp_256_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_256_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1325 |
// vector_twist 240 256 255 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1326 |
d15 = mulmod_scaled_x16(d15,precomp_256_255_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_256_255_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1327 |
// vector_twist 304 512 1 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1328 |
d19 = mulmod_scaled_x16(d19,precomp_512_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_512_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1329 |
// vector_twist 368 512 5 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1330 |
d23 = mulmod_scaled_x16(d23,precomp_512_5_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_512_5_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1331 |
// vector_twist 432 512 511 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1332 |
d27 = mulmod_scaled_x16(d27,precomp_512_511_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_512_511_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1333 |
// vector_twist 496 512 507 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1334 |
d31 = mulmod_scaled_x16(d31,precomp_512_507_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_512_507_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1335 |
// vector_permute 48 112 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1336 |
int16x16 e3 = _mm256_permute2x128_si256_lo(d3,d7); |
| 1337 |
int16x16 e7 = _mm256_permute2x128_si256_hi(d3,d7); |
| 1338 |
// vector_permute 176 240 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1339 |
int16x16 e11 = _mm256_permute2x128_si256_lo(d11,d15); |
| 1340 |
int16x16 e15 = _mm256_permute2x128_si256_hi(d11,d15); |
| 1341 |
// vector_permute 304 368 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1342 |
int16x16 e19 = _mm256_permute2x128_si256_lo(d19,d23); |
| 1343 |
int16x16 e23 = _mm256_permute2x128_si256_hi(d19,d23); |
| 1344 |
// vector_permute 432 496 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1345 |
int16x16 e27 = _mm256_permute2x128_si256_lo(d27,d31); |
| 1346 |
int16x16 e31 = _mm256_permute2x128_si256_hi(d27,d31); |
| 1347 |
// stopbatch 512 |
| 1348 |
_mm256_storeu_si256((int16x16 *) (f+48),e3); |
| 1349 |
_mm256_storeu_si256((int16x16 *) (f+112),e7); |
| 1350 |
_mm256_storeu_si256((int16x16 *) (f+176),e11); |
| 1351 |
_mm256_storeu_si256((int16x16 *) (f+240),e15); |
| 1352 |
_mm256_storeu_si256((int16x16 *) (f+304),e19); |
| 1353 |
_mm256_storeu_si256((int16x16 *) (f+368),e23); |
| 1354 |
_mm256_storeu_si256((int16x16 *) (f+432),e27); |
| 1355 |
_mm256_storeu_si256((int16x16 *) (f+496),e31); |
| 1356 |
f += 512; |
| 1357 |
} |
| 1358 |
f -= 512*reps; |
| 1359 |
// doublereps |
| 1360 |
reps *= 2; |
| 1361 |
// doublereps |
| 1362 |
reps *= 2; |
| 1363 |
// startbatch 128 |
| 1364 |
for (long long r = 0;r < reps;++r) { |
| 1365 |
// vector_butterfly 0 32 1 0 |
| 1366 |
int16x16 a0 = _mm256_loadu_si256((int16x16 *) (f+0)); |
| 1367 |
int16x16 a2 = _mm256_loadu_si256((int16x16 *) (f+32)); |
| 1368 |
int16x16 b0 = add_x16(a0,a2); |
| 1369 |
int16x16 b2 = sub_x16(a0,a2); |
| 1370 |
// vector_butterfly 64 96 1 0 |
| 1371 |
int16x16 a4 = _mm256_loadu_si256((int16x16 *) (f+64)); |
| 1372 |
int16x16 a6 = _mm256_loadu_si256((int16x16 *) (f+96)); |
| 1373 |
int16x16 b4 = add_x16(a4,a6); |
| 1374 |
int16x16 b6 = sub_x16(a4,a6); |
| 1375 |
// vector_butterfly 16 48 1 0 |
| 1376 |
int16x16 a1 = _mm256_loadu_si256((int16x16 *) (f+16)); |
| 1377 |
int16x16 a3 = _mm256_loadu_si256((int16x16 *) (f+48)); |
| 1378 |
int16x16 b1 = add_x16(a1,a3); |
| 1379 |
int16x16 b3 = sub_x16(a1,a3); |
| 1380 |
// vector_butterfly 80 112 1 0 |
| 1381 |
int16x16 a5 = _mm256_loadu_si256((int16x16 *) (f+80)); |
| 1382 |
int16x16 a7 = _mm256_loadu_si256((int16x16 *) (f+112)); |
| 1383 |
int16x16 b5 = add_x16(a5,a7); |
| 1384 |
int16x16 b7 = sub_x16(a5,a7); |
| 1385 |
// vector_butterfly 0 16 1 0 |
| 1386 |
int16x16 c0 = add_x16(b0,b1); |
| 1387 |
int16x16 c1 = sub_x16(b0,b1); |
| 1388 |
// vector_butterfly 64 80 1 0 |
| 1389 |
int16x16 c4 = add_x16(b4,b5); |
| 1390 |
int16x16 c5 = sub_x16(b4,b5); |
| 1391 |
// vector_butterfly 32 48 4 1 |
| 1392 |
b3 = mulmod_scaled_x16(b3,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 1393 |
int16x16 c2 = add_x16(b2,b3); |
| 1394 |
int16x16 c3 = sub_x16(b2,b3); |
| 1395 |
// vector_butterfly 96 112 4 1 |
| 1396 |
b7 = mulmod_scaled_x16(b7,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 1397 |
int16x16 c6 = add_x16(b6,b7); |
| 1398 |
int16x16 c7 = sub_x16(b6,b7); |
| 1399 |
// vector_reduce 0 |
| 1400 |
c0 = reduce_x16(c0,qdata); |
| 1401 |
// vector_reduce 64 |
| 1402 |
c4 = reduce_x16(c4,qdata); |
| 1403 |
// vector_twist 16 32 1 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 |
| 1404 |
c1 = mulmod_scaled_x16(c1,precomp_32_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qinvprecomp_32_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qdata); |
| 1405 |
// vector_twist 80 32 1 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15 |
| 1406 |
c5 = mulmod_scaled_x16(c5,precomp_32_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qinvprecomp_32_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qdata); |
| 1407 |
// vector_twist 32 64 1 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 |
| 1408 |
c2 = mulmod_scaled_x16(c2,precomp_64_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qinvprecomp_64_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qdata); |
| 1409 |
// vector_twist 96 64 1 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15 |
| 1410 |
c6 = mulmod_scaled_x16(c6,precomp_64_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qinvprecomp_64_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qdata); |
| 1411 |
// vector_twist 48 64 63 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 |
| 1412 |
c3 = mulmod_scaled_x16(c3,precomp_64_63_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qinvprecomp_64_63_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qdata); |
| 1413 |
// vector_twist 112 64 63 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15 |
| 1414 |
c7 = mulmod_scaled_x16(c7,precomp_64_63_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qinvprecomp_64_63_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qdata); |
| 1415 |
// vector_permute 0 32 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16 |
| 1416 |
int16x16 d0 = _mm256_unpacklo_epi16(c0,c2); |
| 1417 |
int16x16 d2 = _mm256_unpackhi_epi16(c0,c2); |
| 1418 |
// vector_permute 16 48 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16 |
| 1419 |
int16x16 d1 = _mm256_unpacklo_epi16(c1,c3); |
| 1420 |
int16x16 d3 = _mm256_unpackhi_epi16(c1,c3); |
| 1421 |
// vector_permute 64 96 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16 |
| 1422 |
int16x16 d4 = _mm256_unpacklo_epi16(c4,c6); |
| 1423 |
int16x16 d6 = _mm256_unpackhi_epi16(c4,c6); |
| 1424 |
// vector_permute 80 112 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16 |
| 1425 |
int16x16 d5 = _mm256_unpacklo_epi16(c5,c7); |
| 1426 |
int16x16 d7 = _mm256_unpackhi_epi16(c5,c7); |
| 1427 |
// vector_butterfly 0 64 1 0 |
| 1428 |
int16x16 e0 = add_x16(d0,d4); |
| 1429 |
int16x16 e4 = sub_x16(d0,d4); |
| 1430 |
// vector_butterfly 32 96 1 0 |
| 1431 |
int16x16 e2 = add_x16(d2,d6); |
| 1432 |
int16x16 e6 = sub_x16(d2,d6); |
| 1433 |
// vector_butterfly 16 80 1 0 |
| 1434 |
int16x16 e1 = add_x16(d1,d5); |
| 1435 |
int16x16 e5 = sub_x16(d1,d5); |
| 1436 |
// vector_butterfly 48 112 1 0 |
| 1437 |
int16x16 e3 = add_x16(d3,d7); |
| 1438 |
int16x16 e7 = sub_x16(d3,d7); |
| 1439 |
// vector_permute 0 16 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32 |
| 1440 |
int16x16 f0 = _mm256_unpacklo_epi32(e0,e1); |
| 1441 |
int16x16 f1 = _mm256_unpackhi_epi32(e0,e1); |
| 1442 |
// vector_permute 32 48 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32 |
| 1443 |
int16x16 f2 = _mm256_unpacklo_epi32(e2,e3); |
| 1444 |
int16x16 f3 = _mm256_unpackhi_epi32(e2,e3); |
| 1445 |
// vector_permute 64 80 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32 |
| 1446 |
int16x16 f4 = _mm256_unpacklo_epi32(e4,e5); |
| 1447 |
int16x16 f5 = _mm256_unpackhi_epi32(e4,e5); |
| 1448 |
// vector_permute 96 112 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32 |
| 1449 |
int16x16 f6 = _mm256_unpacklo_epi32(e6,e7); |
| 1450 |
int16x16 f7 = _mm256_unpackhi_epi32(e6,e7); |
| 1451 |
// stopbatch 128 |
| 1452 |
_mm256_storeu_si256((int16x16 *) (f+0),f0); |
| 1453 |
_mm256_storeu_si256((int16x16 *) (f+16),f1); |
| 1454 |
_mm256_storeu_si256((int16x16 *) (f+32),f2); |
| 1455 |
_mm256_storeu_si256((int16x16 *) (f+48),f3); |
| 1456 |
_mm256_storeu_si256((int16x16 *) (f+64),f4); |
| 1457 |
_mm256_storeu_si256((int16x16 *) (f+80),f5); |
| 1458 |
_mm256_storeu_si256((int16x16 *) (f+96),f6); |
| 1459 |
_mm256_storeu_si256((int16x16 *) (f+112),f7); |
| 1460 |
f += 128; |
| 1461 |
} |
| 1462 |
f -= 128*reps; |
| 1463 |
// startbatch 128 |
| 1464 |
for (long long r = 0;r < reps;++r) { |
| 1465 |
// vector_butterfly 0 32 1 0 |
| 1466 |
int16x16 a0 = _mm256_loadu_si256((int16x16 *) (f+0)); |
| 1467 |
int16x16 a2 = _mm256_loadu_si256((int16x16 *) (f+32)); |
| 1468 |
int16x16 b0 = add_x16(a0,a2); |
| 1469 |
int16x16 b2 = sub_x16(a0,a2); |
| 1470 |
// vector_butterfly 16 48 1 0 |
| 1471 |
int16x16 a1 = _mm256_loadu_si256((int16x16 *) (f+16)); |
| 1472 |
int16x16 a3 = _mm256_loadu_si256((int16x16 *) (f+48)); |
| 1473 |
int16x16 b1 = add_x16(a1,a3); |
| 1474 |
int16x16 b3 = sub_x16(a1,a3); |
| 1475 |
// vector_butterfly 64 96 4 1 |
| 1476 |
int16x16 a4 = _mm256_loadu_si256((int16x16 *) (f+64)); |
| 1477 |
int16x16 a6 = _mm256_loadu_si256((int16x16 *) (f+96)); |
| 1478 |
a6 = mulmod_scaled_x16(a6,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 1479 |
int16x16 b4 = add_x16(a4,a6); |
| 1480 |
int16x16 b6 = sub_x16(a4,a6); |
| 1481 |
// vector_butterfly 80 112 4 1 |
| 1482 |
int16x16 a5 = _mm256_loadu_si256((int16x16 *) (f+80)); |
| 1483 |
int16x16 a7 = _mm256_loadu_si256((int16x16 *) (f+112)); |
| 1484 |
a7 = mulmod_scaled_x16(a7,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 1485 |
int16x16 b5 = add_x16(a5,a7); |
| 1486 |
int16x16 b7 = sub_x16(a5,a7); |
| 1487 |
// vector_reduce 0 |
| 1488 |
b0 = reduce_x16(b0,qdata); |
| 1489 |
// vector_reduce 16 |
| 1490 |
b1 = reduce_x16(b1,qdata); |
| 1491 |
// vector_twist 32 8 1 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1 |
| 1492 |
b2 = mulmod_scaled_x16(b2,precomp_8_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qinvprecomp_8_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qdata); |
| 1493 |
// vector_twist 48 8 1 2 2 2 2 3 3 3 3 2 2 2 2 3 3 3 3 |
| 1494 |
b3 = mulmod_scaled_x16(b3,precomp_8_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qinvprecomp_8_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qdata); |
| 1495 |
// vector_twist 64 16 1 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1 |
| 1496 |
b4 = mulmod_scaled_x16(b4,precomp_16_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qinvprecomp_16_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qdata); |
| 1497 |
// vector_twist 80 16 1 2 2 2 2 3 3 3 3 2 2 2 2 3 3 3 3 |
| 1498 |
b5 = mulmod_scaled_x16(b5,precomp_16_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qinvprecomp_16_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qdata); |
| 1499 |
// vector_twist 96 16 15 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1 |
| 1500 |
b6 = mulmod_scaled_x16(b6,precomp_16_15_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qinvprecomp_16_15_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qdata); |
| 1501 |
// vector_twist 112 16 15 2 2 2 2 3 3 3 3 2 2 2 2 3 3 3 3 |
| 1502 |
b7 = mulmod_scaled_x16(b7,precomp_16_15_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qinvprecomp_16_15_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qdata); |
| 1503 |
// vector_permute 0 64 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64 |
| 1504 |
int16x16 c0 = _mm256_unpacklo_epi64(b0,b4); |
| 1505 |
int16x16 c4 = _mm256_unpackhi_epi64(b0,b4); |
| 1506 |
// vector_permute 16 80 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64 |
| 1507 |
int16x16 c1 = _mm256_unpacklo_epi64(b1,b5); |
| 1508 |
int16x16 c5 = _mm256_unpackhi_epi64(b1,b5); |
| 1509 |
// vector_permute 32 96 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64 |
| 1510 |
int16x16 c2 = _mm256_unpacklo_epi64(b2,b6); |
| 1511 |
int16x16 c6 = _mm256_unpackhi_epi64(b2,b6); |
| 1512 |
// vector_permute 48 112 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64 |
| 1513 |
int16x16 c3 = _mm256_unpacklo_epi64(b3,b7); |
| 1514 |
int16x16 c7 = _mm256_unpackhi_epi64(b3,b7); |
| 1515 |
// vector_butterfly 0 16 1 0 |
| 1516 |
int16x16 d0 = add_x16(c0,c1); |
| 1517 |
int16x16 d1 = sub_x16(c0,c1); |
| 1518 |
// vector_butterfly 64 80 1 0 |
| 1519 |
int16x16 d4 = add_x16(c4,c5); |
| 1520 |
int16x16 d5 = sub_x16(c4,c5); |
| 1521 |
// vector_butterfly 32 48 1 0 |
| 1522 |
int16x16 d2 = add_x16(c2,c3); |
| 1523 |
int16x16 d3 = sub_x16(c2,c3); |
| 1524 |
// vector_butterfly 96 112 1 0 |
| 1525 |
int16x16 d6 = add_x16(c6,c7); |
| 1526 |
int16x16 d7 = sub_x16(c6,c7); |
| 1527 |
// vector_butterfly 0 64 1 0 |
| 1528 |
int16x16 e0 = add_x16(d0,d4); |
| 1529 |
int16x16 e4 = sub_x16(d0,d4); |
| 1530 |
// vector_butterfly 32 96 1 0 |
| 1531 |
int16x16 e2 = add_x16(d2,d6); |
| 1532 |
int16x16 e6 = sub_x16(d2,d6); |
| 1533 |
// vector_butterfly 16 80 4 1 |
| 1534 |
d5 = mulmod_scaled_x16(d5,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 1535 |
int16x16 e1 = add_x16(d1,d5); |
| 1536 |
int16x16 e5 = sub_x16(d1,d5); |
| 1537 |
// vector_butterfly 48 112 4 1 |
| 1538 |
d7 = mulmod_scaled_x16(d7,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 1539 |
int16x16 e3 = add_x16(d3,d7); |
| 1540 |
int16x16 e7 = sub_x16(d3,d7); |
| 1541 |
// stopbatch 128 |
| 1542 |
_mm256_storeu_si256((int16x16 *) (f+0),e0); |
| 1543 |
_mm256_storeu_si256((int16x16 *) (f+16),e1); |
| 1544 |
_mm256_storeu_si256((int16x16 *) (f+32),e2); |
| 1545 |
_mm256_storeu_si256((int16x16 *) (f+48),e3); |
| 1546 |
_mm256_storeu_si256((int16x16 *) (f+64),e4); |
| 1547 |
_mm256_storeu_si256((int16x16 *) (f+80),e5); |
| 1548 |
_mm256_storeu_si256((int16x16 *) (f+96),e6); |
| 1549 |
_mm256_storeu_si256((int16x16 *) (f+112),e7); |
| 1550 |
f += 128; |
| 1551 |
} |
| 1552 |
f -= 128*reps; |
| 1553 |
// stopntt 512 |
| 1554 |
} |
| 1555 |
|
| 1556 |
void ntt512_7681(int16 *f,int reps) |
| 1557 |
{ |
| 1558 |
ntt512(f,reps,qdata_7681); |
| 1559 |
} |
| 1560 |
|
| 1561 |
void ntt512_10753(int16 *f,int reps) |
| 1562 |
{ |
| 1563 |
ntt512(f,reps,qdata_10753); |
| 1564 |
} |
| 1565 |
// inv stopntt 512 |
| 1566 |
|
| 1567 |
static void invntt512(int16 *f,int reps,const int16 *qdata) |
| 1568 |
{ |
| 1569 |
reps *= 4; |
| 1570 |
// inv stopbatch 128 |
| 1571 |
for (long long r = 0;r < reps;++r) { |
| 1572 |
// inv vector_butterfly 48 112 4 1 |
| 1573 |
int16x16 a3 = _mm256_loadu_si256((int16x16 *) (f+48)); |
| 1574 |
int16x16 a7 = _mm256_loadu_si256((int16x16 *) (f+112)); |
| 1575 |
int16x16 b3 = add_x16(a3,a7); |
| 1576 |
int16x16 b7 = sub_x16(a3,a7); |
| 1577 |
b7 = mulmod_scaled_x16(b7,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1578 |
// inv vector_butterfly 16 80 4 1 |
| 1579 |
int16x16 a1 = _mm256_loadu_si256((int16x16 *) (f+16)); |
| 1580 |
int16x16 a5 = _mm256_loadu_si256((int16x16 *) (f+80)); |
| 1581 |
int16x16 b1 = add_x16(a1,a5); |
| 1582 |
int16x16 b5 = sub_x16(a1,a5); |
| 1583 |
b5 = mulmod_scaled_x16(b5,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1584 |
// inv vector_butterfly 32 96 1 0 |
| 1585 |
int16x16 a2 = _mm256_loadu_si256((int16x16 *) (f+32)); |
| 1586 |
int16x16 a6 = _mm256_loadu_si256((int16x16 *) (f+96)); |
| 1587 |
int16x16 b2 = add_x16(a2,a6); |
| 1588 |
int16x16 b6 = sub_x16(a2,a6); |
| 1589 |
// inv vector_butterfly 0 64 1 0 |
| 1590 |
int16x16 a0 = _mm256_loadu_si256((int16x16 *) (f+0)); |
| 1591 |
int16x16 a4 = _mm256_loadu_si256((int16x16 *) (f+64)); |
| 1592 |
int16x16 b0 = add_x16(a0,a4); |
| 1593 |
int16x16 b4 = sub_x16(a0,a4); |
| 1594 |
// inv vector_butterfly 96 112 1 0 |
| 1595 |
int16x16 c6 = add_x16(b6,b7); |
| 1596 |
int16x16 c7 = sub_x16(b6,b7); |
| 1597 |
// inv vector_butterfly 32 48 1 0 |
| 1598 |
int16x16 c2 = add_x16(b2,b3); |
| 1599 |
int16x16 c3 = sub_x16(b2,b3); |
| 1600 |
// inv vector_butterfly 64 80 1 0 |
| 1601 |
int16x16 c4 = add_x16(b4,b5); |
| 1602 |
int16x16 c5 = sub_x16(b4,b5); |
| 1603 |
// inv vector_butterfly 0 16 1 0 |
| 1604 |
int16x16 c0 = add_x16(b0,b1); |
| 1605 |
int16x16 c1 = sub_x16(b0,b1); |
| 1606 |
// inv vector_permute 48 112 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64 |
| 1607 |
int16x16 d3 = _mm256_unpacklo_epi64(c3,c7); |
| 1608 |
int16x16 d7 = _mm256_unpackhi_epi64(c3,c7); |
| 1609 |
// inv vector_permute 32 96 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64 |
| 1610 |
int16x16 d2 = _mm256_unpacklo_epi64(c2,c6); |
| 1611 |
int16x16 d6 = _mm256_unpackhi_epi64(c2,c6); |
| 1612 |
// inv vector_permute 16 80 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64 |
| 1613 |
int16x16 d1 = _mm256_unpacklo_epi64(c1,c5); |
| 1614 |
int16x16 d5 = _mm256_unpackhi_epi64(c1,c5); |
| 1615 |
// inv vector_permute 0 64 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64 |
| 1616 |
int16x16 d0 = _mm256_unpacklo_epi64(c0,c4); |
| 1617 |
int16x16 d4 = _mm256_unpackhi_epi64(c0,c4); |
| 1618 |
// inv vector_twist 112 16 15 2 2 2 2 3 3 3 3 2 2 2 2 3 3 3 3 |
| 1619 |
d7 = mulmod_scaled_x16(d7,precomp_16_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qinvprecomp_16_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qdata); |
| 1620 |
// inv vector_twist 96 16 15 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1 |
| 1621 |
d6 = mulmod_scaled_x16(d6,precomp_16_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qinvprecomp_16_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qdata); |
| 1622 |
// inv vector_twist 80 16 1 2 2 2 2 3 3 3 3 2 2 2 2 3 3 3 3 |
| 1623 |
d5 = mulmod_scaled_x16(d5,precomp_16_15_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qinvprecomp_16_15_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qdata); |
| 1624 |
// inv vector_twist 64 16 1 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1 |
| 1625 |
d4 = mulmod_scaled_x16(d4,precomp_16_15_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qinvprecomp_16_15_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qdata); |
| 1626 |
// inv vector_twist 48 8 1 2 2 2 2 3 3 3 3 2 2 2 2 3 3 3 3 |
| 1627 |
d3 = mulmod_scaled_x16(d3,precomp_8_7_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qinvprecomp_8_7_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qdata); |
| 1628 |
// inv vector_twist 32 8 1 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1 |
| 1629 |
d2 = mulmod_scaled_x16(d2,precomp_8_7_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qinvprecomp_8_7_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qdata); |
| 1630 |
// inv vector_reduce 16 |
| 1631 |
d1 = reduce_x16(d1,qdata); |
| 1632 |
// inv vector_reduce 0 |
| 1633 |
d0 = reduce_x16(d0,qdata); |
| 1634 |
// inv vector_butterfly 80 112 4 1 |
| 1635 |
int16x16 e5 = add_x16(d5,d7); |
| 1636 |
int16x16 e7 = sub_x16(d5,d7); |
| 1637 |
e7 = mulmod_scaled_x16(e7,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1638 |
// inv vector_butterfly 64 96 4 1 |
| 1639 |
int16x16 e4 = add_x16(d4,d6); |
| 1640 |
int16x16 e6 = sub_x16(d4,d6); |
| 1641 |
e6 = mulmod_scaled_x16(e6,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1642 |
// inv vector_butterfly 16 48 1 0 |
| 1643 |
int16x16 e1 = add_x16(d1,d3); |
| 1644 |
int16x16 e3 = sub_x16(d1,d3); |
| 1645 |
// inv vector_butterfly 0 32 1 0 |
| 1646 |
int16x16 e0 = add_x16(d0,d2); |
| 1647 |
int16x16 e2 = sub_x16(d0,d2); |
| 1648 |
// inv startbatch 128 |
| 1649 |
_mm256_storeu_si256((int16x16 *) (f+0),e0); |
| 1650 |
_mm256_storeu_si256((int16x16 *) (f+16),e1); |
| 1651 |
_mm256_storeu_si256((int16x16 *) (f+32),e2); |
| 1652 |
_mm256_storeu_si256((int16x16 *) (f+48),e3); |
| 1653 |
_mm256_storeu_si256((int16x16 *) (f+64),e4); |
| 1654 |
_mm256_storeu_si256((int16x16 *) (f+80),e5); |
| 1655 |
_mm256_storeu_si256((int16x16 *) (f+96),e6); |
| 1656 |
_mm256_storeu_si256((int16x16 *) (f+112),e7); |
| 1657 |
f += 128; |
| 1658 |
} |
| 1659 |
f -= 128*reps; |
| 1660 |
// inv stopbatch 128 |
| 1661 |
for (long long r = 0;r < reps;++r) { |
| 1662 |
// inv vector_permute 96 112 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32 |
| 1663 |
int16x16 a6 = _mm256_loadu_si256((int16x16 *) (f+96)); |
| 1664 |
int16x16 a7 = _mm256_loadu_si256((int16x16 *) (f+112)); |
| 1665 |
int16x16 b6 = _mm256_unpacklo_epi32(a6,a7); |
| 1666 |
int16x16 b7 = _mm256_unpackhi_epi32(a6,a7); |
| 1667 |
int16x16 c6 = _mm256_unpacklo_epi32(b6,b7); |
| 1668 |
int16x16 c7 = _mm256_unpackhi_epi32(b6,b7); |
| 1669 |
// inv vector_permute 64 80 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32 |
| 1670 |
int16x16 a4 = _mm256_loadu_si256((int16x16 *) (f+64)); |
| 1671 |
int16x16 a5 = _mm256_loadu_si256((int16x16 *) (f+80)); |
| 1672 |
int16x16 b4 = _mm256_unpacklo_epi32(a4,a5); |
| 1673 |
int16x16 b5 = _mm256_unpackhi_epi32(a4,a5); |
| 1674 |
int16x16 c4 = _mm256_unpacklo_epi32(b4,b5); |
| 1675 |
int16x16 c5 = _mm256_unpackhi_epi32(b4,b5); |
| 1676 |
// inv vector_permute 32 48 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32 |
| 1677 |
int16x16 a2 = _mm256_loadu_si256((int16x16 *) (f+32)); |
| 1678 |
int16x16 a3 = _mm256_loadu_si256((int16x16 *) (f+48)); |
| 1679 |
int16x16 b2 = _mm256_unpacklo_epi32(a2,a3); |
| 1680 |
int16x16 b3 = _mm256_unpackhi_epi32(a2,a3); |
| 1681 |
int16x16 c2 = _mm256_unpacklo_epi32(b2,b3); |
| 1682 |
int16x16 c3 = _mm256_unpackhi_epi32(b2,b3); |
| 1683 |
// inv vector_permute 0 16 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32 |
| 1684 |
int16x16 a0 = _mm256_loadu_si256((int16x16 *) (f+0)); |
| 1685 |
int16x16 a1 = _mm256_loadu_si256((int16x16 *) (f+16)); |
| 1686 |
int16x16 b0 = _mm256_unpacklo_epi32(a0,a1); |
| 1687 |
int16x16 b1 = _mm256_unpackhi_epi32(a0,a1); |
| 1688 |
int16x16 c0 = _mm256_unpacklo_epi32(b0,b1); |
| 1689 |
int16x16 c1 = _mm256_unpackhi_epi32(b0,b1); |
| 1690 |
// inv vector_butterfly 48 112 1 0 |
| 1691 |
int16x16 d3 = add_x16(c3,c7); |
| 1692 |
int16x16 d7 = sub_x16(c3,c7); |
| 1693 |
// inv vector_butterfly 16 80 1 0 |
| 1694 |
int16x16 d1 = add_x16(c1,c5); |
| 1695 |
int16x16 d5 = sub_x16(c1,c5); |
| 1696 |
// inv vector_butterfly 32 96 1 0 |
| 1697 |
int16x16 d2 = add_x16(c2,c6); |
| 1698 |
int16x16 d6 = sub_x16(c2,c6); |
| 1699 |
// inv vector_butterfly 0 64 1 0 |
| 1700 |
int16x16 d0 = add_x16(c0,c4); |
| 1701 |
int16x16 d4 = sub_x16(c0,c4); |
| 1702 |
// inv vector_permute 80 112 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16 |
| 1703 |
int16x16 e5 = _mm256_unpacklo_epi16(d5,d7); |
| 1704 |
int16x16 e7 = _mm256_unpackhi_epi16(d5,d7); |
| 1705 |
int16x16 f5 = _mm256_unpacklo_epi16(e5,e7); |
| 1706 |
int16x16 f7 = _mm256_unpackhi_epi16(e5,e7); |
| 1707 |
int16x16 g5 = _mm256_unpacklo_epi16(f5,f7); |
| 1708 |
int16x16 g7 = _mm256_unpackhi_epi16(f5,f7); |
| 1709 |
// inv vector_permute 64 96 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16 |
| 1710 |
int16x16 e4 = _mm256_unpacklo_epi16(d4,d6); |
| 1711 |
int16x16 e6 = _mm256_unpackhi_epi16(d4,d6); |
| 1712 |
int16x16 f4 = _mm256_unpacklo_epi16(e4,e6); |
| 1713 |
int16x16 f6 = _mm256_unpackhi_epi16(e4,e6); |
| 1714 |
int16x16 g4 = _mm256_unpacklo_epi16(f4,f6); |
| 1715 |
int16x16 g6 = _mm256_unpackhi_epi16(f4,f6); |
| 1716 |
// inv vector_permute 16 48 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16 |
| 1717 |
int16x16 e1 = _mm256_unpacklo_epi16(d1,d3); |
| 1718 |
int16x16 e3 = _mm256_unpackhi_epi16(d1,d3); |
| 1719 |
int16x16 f1 = _mm256_unpacklo_epi16(e1,e3); |
| 1720 |
int16x16 f3 = _mm256_unpackhi_epi16(e1,e3); |
| 1721 |
int16x16 g1 = _mm256_unpacklo_epi16(f1,f3); |
| 1722 |
int16x16 g3 = _mm256_unpackhi_epi16(f1,f3); |
| 1723 |
// inv vector_permute 0 32 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16 |
| 1724 |
int16x16 e0 = _mm256_unpacklo_epi16(d0,d2); |
| 1725 |
int16x16 e2 = _mm256_unpackhi_epi16(d0,d2); |
| 1726 |
int16x16 f0 = _mm256_unpacklo_epi16(e0,e2); |
| 1727 |
int16x16 f2 = _mm256_unpackhi_epi16(e0,e2); |
| 1728 |
int16x16 g0 = _mm256_unpacklo_epi16(f0,f2); |
| 1729 |
int16x16 g2 = _mm256_unpackhi_epi16(f0,f2); |
| 1730 |
// inv vector_twist 112 64 63 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15 |
| 1731 |
g7 = mulmod_scaled_x16(g7,precomp_64_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qinvprecomp_64_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qdata); |
| 1732 |
// inv vector_twist 48 64 63 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 |
| 1733 |
g3 = mulmod_scaled_x16(g3,precomp_64_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qinvprecomp_64_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qdata); |
| 1734 |
// inv vector_twist 96 64 1 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15 |
| 1735 |
g6 = mulmod_scaled_x16(g6,precomp_64_63_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qinvprecomp_64_63_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qdata); |
| 1736 |
// inv vector_twist 32 64 1 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 |
| 1737 |
g2 = mulmod_scaled_x16(g2,precomp_64_63_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qinvprecomp_64_63_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qdata); |
| 1738 |
// inv vector_twist 80 32 1 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15 |
| 1739 |
g5 = mulmod_scaled_x16(g5,precomp_32_31_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qinvprecomp_32_31_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qdata); |
| 1740 |
// inv vector_twist 16 32 1 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 |
| 1741 |
g1 = mulmod_scaled_x16(g1,precomp_32_31_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qinvprecomp_32_31_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qdata); |
| 1742 |
// inv vector_reduce 64 |
| 1743 |
g4 = reduce_x16(g4,qdata); |
| 1744 |
// inv vector_reduce 0 |
| 1745 |
g0 = reduce_x16(g0,qdata); |
| 1746 |
// inv vector_butterfly 96 112 4 1 |
| 1747 |
int16x16 h6 = add_x16(g6,g7); |
| 1748 |
int16x16 h7 = sub_x16(g6,g7); |
| 1749 |
h7 = mulmod_scaled_x16(h7,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1750 |
// inv vector_butterfly 32 48 4 1 |
| 1751 |
int16x16 h2 = add_x16(g2,g3); |
| 1752 |
int16x16 h3 = sub_x16(g2,g3); |
| 1753 |
h3 = mulmod_scaled_x16(h3,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1754 |
// inv vector_butterfly 64 80 1 0 |
| 1755 |
int16x16 h4 = add_x16(g4,g5); |
| 1756 |
int16x16 h5 = sub_x16(g4,g5); |
| 1757 |
// inv vector_butterfly 0 16 1 0 |
| 1758 |
int16x16 h0 = add_x16(g0,g1); |
| 1759 |
int16x16 h1 = sub_x16(g0,g1); |
| 1760 |
// inv vector_butterfly 80 112 1 0 |
| 1761 |
int16x16 i5 = add_x16(h5,h7); |
| 1762 |
int16x16 i7 = sub_x16(h5,h7); |
| 1763 |
// inv vector_butterfly 16 48 1 0 |
| 1764 |
int16x16 i1 = add_x16(h1,h3); |
| 1765 |
int16x16 i3 = sub_x16(h1,h3); |
| 1766 |
// inv vector_butterfly 64 96 1 0 |
| 1767 |
int16x16 i4 = add_x16(h4,h6); |
| 1768 |
int16x16 i6 = sub_x16(h4,h6); |
| 1769 |
// inv vector_butterfly 0 32 1 0 |
| 1770 |
int16x16 i0 = add_x16(h0,h2); |
| 1771 |
int16x16 i2 = sub_x16(h0,h2); |
| 1772 |
// inv startbatch 128 |
| 1773 |
_mm256_storeu_si256((int16x16 *) (f+0),i0); |
| 1774 |
_mm256_storeu_si256((int16x16 *) (f+16),i1); |
| 1775 |
_mm256_storeu_si256((int16x16 *) (f+32),i2); |
| 1776 |
_mm256_storeu_si256((int16x16 *) (f+48),i3); |
| 1777 |
_mm256_storeu_si256((int16x16 *) (f+64),i4); |
| 1778 |
_mm256_storeu_si256((int16x16 *) (f+80),i5); |
| 1779 |
_mm256_storeu_si256((int16x16 *) (f+96),i6); |
| 1780 |
_mm256_storeu_si256((int16x16 *) (f+112),i7); |
| 1781 |
f += 128; |
| 1782 |
} |
| 1783 |
f -= 128*reps; |
| 1784 |
// inv doublereps |
| 1785 |
reps /= 2; |
| 1786 |
// inv doublereps |
| 1787 |
reps /= 2; |
| 1788 |
// inv stopbatch 512 |
| 1789 |
for (long long r = 0;r < reps;++r) { |
| 1790 |
// inv vector_permute 432 496 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1791 |
int16x16 a27 = _mm256_loadu_si256((int16x16 *) (f+432)); |
| 1792 |
int16x16 a31 = _mm256_loadu_si256((int16x16 *) (f+496)); |
| 1793 |
int16x16 b27 = _mm256_permute2x128_si256_lo(a27,a31); |
| 1794 |
int16x16 b31 = _mm256_permute2x128_si256_hi(a27,a31); |
| 1795 |
// inv vector_permute 304 368 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1796 |
int16x16 a19 = _mm256_loadu_si256((int16x16 *) (f+304)); |
| 1797 |
int16x16 a23 = _mm256_loadu_si256((int16x16 *) (f+368)); |
| 1798 |
int16x16 b19 = _mm256_permute2x128_si256_lo(a19,a23); |
| 1799 |
int16x16 b23 = _mm256_permute2x128_si256_hi(a19,a23); |
| 1800 |
// inv vector_permute 176 240 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1801 |
int16x16 a11 = _mm256_loadu_si256((int16x16 *) (f+176)); |
| 1802 |
int16x16 a15 = _mm256_loadu_si256((int16x16 *) (f+240)); |
| 1803 |
int16x16 b11 = _mm256_permute2x128_si256_lo(a11,a15); |
| 1804 |
int16x16 b15 = _mm256_permute2x128_si256_hi(a11,a15); |
| 1805 |
// inv vector_permute 48 112 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1806 |
int16x16 a3 = _mm256_loadu_si256((int16x16 *) (f+48)); |
| 1807 |
int16x16 a7 = _mm256_loadu_si256((int16x16 *) (f+112)); |
| 1808 |
int16x16 b3 = _mm256_permute2x128_si256_lo(a3,a7); |
| 1809 |
int16x16 b7 = _mm256_permute2x128_si256_hi(a3,a7); |
| 1810 |
// inv vector_twist 496 512 507 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1811 |
b31 = mulmod_scaled_x16(b31,precomp_512_5_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_512_5_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1812 |
// inv vector_twist 432 512 511 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1813 |
b27 = mulmod_scaled_x16(b27,precomp_512_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_512_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1814 |
// inv vector_twist 368 512 5 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1815 |
b23 = mulmod_scaled_x16(b23,precomp_512_507_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_512_507_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1816 |
// inv vector_twist 304 512 1 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1817 |
b19 = mulmod_scaled_x16(b19,precomp_512_511_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_512_511_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1818 |
// inv vector_twist 240 256 255 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1819 |
b15 = mulmod_scaled_x16(b15,precomp_256_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_256_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1820 |
// inv vector_twist 176 256 1 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1821 |
b11 = mulmod_scaled_x16(b11,precomp_256_255_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_256_255_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1822 |
// inv vector_twist 112 128 1 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1823 |
b7 = mulmod_scaled_x16(b7,precomp_128_127_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_128_127_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1824 |
// inv vector_reduce 48 |
| 1825 |
b3 = reduce_x16(b3,qdata); |
| 1826 |
// inv vector_butterfly 432 496 8 7 |
| 1827 |
int16x16 c27 = add_x16(b27,b31); |
| 1828 |
int16x16 c31 = sub_x16(b27,b31); |
| 1829 |
c31 = mulmod_scaled_x16(c31,scaledzeta_x16_8_1,qinvscaledzeta_x16_8_1,qdata); |
| 1830 |
// inv vector_butterfly 304 368 8 1 |
| 1831 |
int16x16 c19 = add_x16(b19,b23); |
| 1832 |
int16x16 c23 = sub_x16(b19,b23); |
| 1833 |
c23 = mulmod_scaled_x16(c23,scaledzeta_x16_8_7,qinvscaledzeta_x16_8_7,qdata); |
| 1834 |
// inv vector_butterfly 176 240 4 1 |
| 1835 |
int16x16 c11 = add_x16(b11,b15); |
| 1836 |
int16x16 c15 = sub_x16(b11,b15); |
| 1837 |
c15 = mulmod_scaled_x16(c15,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1838 |
// inv vector_butterfly 48 112 1 0 |
| 1839 |
int16x16 c3 = add_x16(b3,b7); |
| 1840 |
int16x16 c7 = sub_x16(b3,b7); |
| 1841 |
// inv vector_reduce_ifforward 112 |
| 1842 |
// inv vector_butterfly 368 496 4 1 |
| 1843 |
int16x16 d23 = add_x16(c23,c31); |
| 1844 |
int16x16 d31 = sub_x16(c23,c31); |
| 1845 |
d31 = mulmod_scaled_x16(d31,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1846 |
// inv vector_butterfly 304 432 4 1 |
| 1847 |
int16x16 d19 = add_x16(c19,c27); |
| 1848 |
int16x16 d27 = sub_x16(c19,c27); |
| 1849 |
d27 = mulmod_scaled_x16(d27,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1850 |
// inv vector_butterfly 112 240 1 0 |
| 1851 |
int16x16 d7 = add_x16(c7,c15); |
| 1852 |
int16x16 d15 = sub_x16(c7,c15); |
| 1853 |
// inv vector_butterfly 48 176 1 0 |
| 1854 |
int16x16 d3 = add_x16(c3,c11); |
| 1855 |
int16x16 d11 = sub_x16(c3,c11); |
| 1856 |
// inv vector_reduce_ifreverse 304 |
| 1857 |
d19 = reduce_x16(d19,qdata); |
| 1858 |
// inv vector_reduce_ifreverse 48 |
| 1859 |
d3 = reduce_x16(d3,qdata); |
| 1860 |
// inv vector_butterfly 240 496 1 0 |
| 1861 |
int16x16 e15 = add_x16(d15,d31); |
| 1862 |
int16x16 e31 = sub_x16(d15,d31); |
| 1863 |
// inv vector_butterfly 112 368 1 0 |
| 1864 |
int16x16 e7 = add_x16(d7,d23); |
| 1865 |
int16x16 e23 = sub_x16(d7,d23); |
| 1866 |
// inv vector_butterfly 176 432 1 0 |
| 1867 |
int16x16 e11 = add_x16(d11,d27); |
| 1868 |
int16x16 e27 = sub_x16(d11,d27); |
| 1869 |
// inv vector_butterfly 48 304 1 0 |
| 1870 |
int16x16 e3 = add_x16(d3,d19); |
| 1871 |
int16x16 e19 = sub_x16(d3,d19); |
| 1872 |
// inv startbatch 512 |
| 1873 |
_mm256_storeu_si256((int16x16 *) (f+48),e3); |
| 1874 |
_mm256_storeu_si256((int16x16 *) (f+112),e7); |
| 1875 |
_mm256_storeu_si256((int16x16 *) (f+176),e11); |
| 1876 |
_mm256_storeu_si256((int16x16 *) (f+240),e15); |
| 1877 |
_mm256_storeu_si256((int16x16 *) (f+304),e19); |
| 1878 |
_mm256_storeu_si256((int16x16 *) (f+368),e23); |
| 1879 |
_mm256_storeu_si256((int16x16 *) (f+432),e27); |
| 1880 |
_mm256_storeu_si256((int16x16 *) (f+496),e31); |
| 1881 |
f += 512; |
| 1882 |
} |
| 1883 |
f -= 512*reps; |
| 1884 |
// inv stopbatch 512 |
| 1885 |
for (long long r = 0;r < reps;++r) { |
| 1886 |
// inv vector_permute 416 480 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1887 |
int16x16 a26 = _mm256_loadu_si256((int16x16 *) (f+416)); |
| 1888 |
int16x16 a30 = _mm256_loadu_si256((int16x16 *) (f+480)); |
| 1889 |
int16x16 b26 = _mm256_permute2x128_si256_lo(a26,a30); |
| 1890 |
int16x16 b30 = _mm256_permute2x128_si256_hi(a26,a30); |
| 1891 |
// inv vector_permute 288 352 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1892 |
int16x16 a18 = _mm256_loadu_si256((int16x16 *) (f+288)); |
| 1893 |
int16x16 a22 = _mm256_loadu_si256((int16x16 *) (f+352)); |
| 1894 |
int16x16 b18 = _mm256_permute2x128_si256_lo(a18,a22); |
| 1895 |
int16x16 b22 = _mm256_permute2x128_si256_hi(a18,a22); |
| 1896 |
// inv vector_permute 160 224 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1897 |
int16x16 a10 = _mm256_loadu_si256((int16x16 *) (f+160)); |
| 1898 |
int16x16 a14 = _mm256_loadu_si256((int16x16 *) (f+224)); |
| 1899 |
int16x16 b10 = _mm256_permute2x128_si256_lo(a10,a14); |
| 1900 |
int16x16 b14 = _mm256_permute2x128_si256_hi(a10,a14); |
| 1901 |
// inv vector_permute 32 96 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1902 |
int16x16 a2 = _mm256_loadu_si256((int16x16 *) (f+32)); |
| 1903 |
int16x16 a6 = _mm256_loadu_si256((int16x16 *) (f+96)); |
| 1904 |
int16x16 b2 = _mm256_permute2x128_si256_lo(a2,a6); |
| 1905 |
int16x16 b6 = _mm256_permute2x128_si256_hi(a2,a6); |
| 1906 |
// inv vector_twist 480 512 507 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1907 |
b30 = mulmod_scaled_x16(b30,precomp_512_5_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_512_5_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1908 |
// inv vector_twist 416 512 511 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1909 |
b26 = mulmod_scaled_x16(b26,precomp_512_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_512_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1910 |
// inv vector_twist 352 512 5 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1911 |
b22 = mulmod_scaled_x16(b22,precomp_512_507_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_512_507_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1912 |
// inv vector_twist 288 512 1 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1913 |
b18 = mulmod_scaled_x16(b18,precomp_512_511_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_512_511_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1914 |
// inv vector_twist 224 256 255 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1915 |
b14 = mulmod_scaled_x16(b14,precomp_256_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_256_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1916 |
// inv vector_twist 160 256 1 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1917 |
b10 = mulmod_scaled_x16(b10,precomp_256_255_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_256_255_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1918 |
// inv vector_twist 96 128 1 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1919 |
b6 = mulmod_scaled_x16(b6,precomp_128_127_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_128_127_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1920 |
// inv vector_reduce 32 |
| 1921 |
b2 = reduce_x16(b2,qdata); |
| 1922 |
// inv vector_butterfly 416 480 8 7 |
| 1923 |
int16x16 c26 = add_x16(b26,b30); |
| 1924 |
int16x16 c30 = sub_x16(b26,b30); |
| 1925 |
c30 = mulmod_scaled_x16(c30,scaledzeta_x16_8_1,qinvscaledzeta_x16_8_1,qdata); |
| 1926 |
// inv vector_butterfly 288 352 8 1 |
| 1927 |
int16x16 c18 = add_x16(b18,b22); |
| 1928 |
int16x16 c22 = sub_x16(b18,b22); |
| 1929 |
c22 = mulmod_scaled_x16(c22,scaledzeta_x16_8_7,qinvscaledzeta_x16_8_7,qdata); |
| 1930 |
// inv vector_butterfly 160 224 4 1 |
| 1931 |
int16x16 c10 = add_x16(b10,b14); |
| 1932 |
int16x16 c14 = sub_x16(b10,b14); |
| 1933 |
c14 = mulmod_scaled_x16(c14,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1934 |
// inv vector_butterfly 32 96 1 0 |
| 1935 |
int16x16 c2 = add_x16(b2,b6); |
| 1936 |
int16x16 c6 = sub_x16(b2,b6); |
| 1937 |
// inv vector_reduce_ifforward 96 |
| 1938 |
// inv vector_butterfly 352 480 4 1 |
| 1939 |
int16x16 d22 = add_x16(c22,c30); |
| 1940 |
int16x16 d30 = sub_x16(c22,c30); |
| 1941 |
d30 = mulmod_scaled_x16(d30,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1942 |
// inv vector_butterfly 288 416 4 1 |
| 1943 |
int16x16 d18 = add_x16(c18,c26); |
| 1944 |
int16x16 d26 = sub_x16(c18,c26); |
| 1945 |
d26 = mulmod_scaled_x16(d26,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1946 |
// inv vector_butterfly 96 224 1 0 |
| 1947 |
int16x16 d6 = add_x16(c6,c14); |
| 1948 |
int16x16 d14 = sub_x16(c6,c14); |
| 1949 |
// inv vector_butterfly 32 160 1 0 |
| 1950 |
int16x16 d2 = add_x16(c2,c10); |
| 1951 |
int16x16 d10 = sub_x16(c2,c10); |
| 1952 |
// inv vector_reduce_ifreverse 288 |
| 1953 |
d18 = reduce_x16(d18,qdata); |
| 1954 |
// inv vector_reduce_ifreverse 32 |
| 1955 |
d2 = reduce_x16(d2,qdata); |
| 1956 |
// inv vector_butterfly 224 480 1 0 |
| 1957 |
int16x16 e14 = add_x16(d14,d30); |
| 1958 |
int16x16 e30 = sub_x16(d14,d30); |
| 1959 |
// inv vector_butterfly 96 352 1 0 |
| 1960 |
int16x16 e6 = add_x16(d6,d22); |
| 1961 |
int16x16 e22 = sub_x16(d6,d22); |
| 1962 |
// inv vector_butterfly 160 416 1 0 |
| 1963 |
int16x16 e10 = add_x16(d10,d26); |
| 1964 |
int16x16 e26 = sub_x16(d10,d26); |
| 1965 |
// inv vector_butterfly 32 288 1 0 |
| 1966 |
int16x16 e2 = add_x16(d2,d18); |
| 1967 |
int16x16 e18 = sub_x16(d2,d18); |
| 1968 |
// inv startbatch 512 |
| 1969 |
_mm256_storeu_si256((int16x16 *) (f+32),e2); |
| 1970 |
_mm256_storeu_si256((int16x16 *) (f+96),e6); |
| 1971 |
_mm256_storeu_si256((int16x16 *) (f+160),e10); |
| 1972 |
_mm256_storeu_si256((int16x16 *) (f+224),e14); |
| 1973 |
_mm256_storeu_si256((int16x16 *) (f+288),e18); |
| 1974 |
_mm256_storeu_si256((int16x16 *) (f+352),e22); |
| 1975 |
_mm256_storeu_si256((int16x16 *) (f+416),e26); |
| 1976 |
_mm256_storeu_si256((int16x16 *) (f+480),e30); |
| 1977 |
f += 512; |
| 1978 |
} |
| 1979 |
f -= 512*reps; |
| 1980 |
// inv stopbatch 512 |
| 1981 |
for (long long r = 0;r < reps;++r) { |
| 1982 |
// inv vector_permute 400 464 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1983 |
int16x16 a25 = _mm256_loadu_si256((int16x16 *) (f+400)); |
| 1984 |
int16x16 a29 = _mm256_loadu_si256((int16x16 *) (f+464)); |
| 1985 |
int16x16 b25 = _mm256_permute2x128_si256_lo(a25,a29); |
| 1986 |
int16x16 b29 = _mm256_permute2x128_si256_hi(a25,a29); |
| 1987 |
// inv vector_permute 272 336 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1988 |
int16x16 a17 = _mm256_loadu_si256((int16x16 *) (f+272)); |
| 1989 |
int16x16 a21 = _mm256_loadu_si256((int16x16 *) (f+336)); |
| 1990 |
int16x16 b17 = _mm256_permute2x128_si256_lo(a17,a21); |
| 1991 |
int16x16 b21 = _mm256_permute2x128_si256_hi(a17,a21); |
| 1992 |
// inv vector_permute 144 208 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1993 |
int16x16 a9 = _mm256_loadu_si256((int16x16 *) (f+144)); |
| 1994 |
int16x16 a13 = _mm256_loadu_si256((int16x16 *) (f+208)); |
| 1995 |
int16x16 b9 = _mm256_permute2x128_si256_lo(a9,a13); |
| 1996 |
int16x16 b13 = _mm256_permute2x128_si256_hi(a9,a13); |
| 1997 |
// inv vector_permute 16 80 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1998 |
int16x16 a1 = _mm256_loadu_si256((int16x16 *) (f+16)); |
| 1999 |
int16x16 a5 = _mm256_loadu_si256((int16x16 *) (f+80)); |
| 2000 |
int16x16 b1 = _mm256_permute2x128_si256_lo(a1,a5); |
| 2001 |
int16x16 b5 = _mm256_permute2x128_si256_hi(a1,a5); |
| 2002 |
// inv vector_twist 464 512 507 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 2003 |
b29 = mulmod_scaled_x16(b29,precomp_512_5_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_512_5_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 2004 |
// inv vector_twist 400 512 511 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 2005 |
b25 = mulmod_scaled_x16(b25,precomp_512_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_512_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 2006 |
// inv vector_twist 336 512 5 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 2007 |
b21 = mulmod_scaled_x16(b21,precomp_512_507_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_512_507_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 2008 |
// inv vector_twist 272 512 1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 2009 |
b17 = mulmod_scaled_x16(b17,precomp_512_511_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_512_511_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 2010 |
// inv vector_twist 208 256 255 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 2011 |
b13 = mulmod_scaled_x16(b13,precomp_256_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_256_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 2012 |
// inv vector_twist 144 256 1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 2013 |
b9 = mulmod_scaled_x16(b9,precomp_256_255_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_256_255_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 2014 |
// inv vector_twist 80 128 1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 2015 |
b5 = mulmod_scaled_x16(b5,precomp_128_127_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_128_127_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 2016 |
// inv vector_reduce 16 |
| 2017 |
b1 = reduce_x16(b1,qdata); |
| 2018 |
// inv vector_butterfly 400 464 8 7 |
| 2019 |
int16x16 c25 = add_x16(b25,b29); |
| 2020 |
int16x16 c29 = sub_x16(b25,b29); |
| 2021 |
c29 = mulmod_scaled_x16(c29,scaledzeta_x16_8_1,qinvscaledzeta_x16_8_1,qdata); |
| 2022 |
// inv vector_butterfly 272 336 8 1 |
| 2023 |
int16x16 c17 = add_x16(b17,b21); |
| 2024 |
int16x16 c21 = sub_x16(b17,b21); |
| 2025 |
c21 = mulmod_scaled_x16(c21,scaledzeta_x16_8_7,qinvscaledzeta_x16_8_7,qdata); |
| 2026 |
// inv vector_butterfly 144 208 4 1 |
| 2027 |
int16x16 c9 = add_x16(b9,b13); |
| 2028 |
int16x16 c13 = sub_x16(b9,b13); |
| 2029 |
c13 = mulmod_scaled_x16(c13,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 2030 |
// inv vector_butterfly 16 80 1 0 |
| 2031 |
int16x16 c1 = add_x16(b1,b5); |
| 2032 |
int16x16 c5 = sub_x16(b1,b5); |
| 2033 |
// inv vector_reduce_ifforward 80 |
| 2034 |
// inv vector_butterfly 336 464 4 1 |
| 2035 |
int16x16 d21 = add_x16(c21,c29); |
| 2036 |
int16x16 d29 = sub_x16(c21,c29); |
| 2037 |
d29 = mulmod_scaled_x16(d29,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 2038 |
// inv vector_butterfly 272 400 4 1 |
| 2039 |
int16x16 d17 = add_x16(c17,c25); |
| 2040 |
int16x16 d25 = sub_x16(c17,c25); |
| 2041 |
d25 = mulmod_scaled_x16(d25,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 2042 |
// inv vector_butterfly 80 208 1 0 |
| 2043 |
int16x16 d5 = add_x16(c5,c13); |
| 2044 |
int16x16 d13 = sub_x16(c5,c13); |
| 2045 |
// inv vector_butterfly 16 144 1 0 |
| 2046 |
int16x16 d1 = add_x16(c1,c9); |
| 2047 |
int16x16 d9 = sub_x16(c1,c9); |
| 2048 |
// inv vector_reduce_ifreverse 272 |
| 2049 |
d17 = reduce_x16(d17,qdata); |
| 2050 |
// inv vector_reduce_ifreverse 16 |
| 2051 |
d1 = reduce_x16(d1,qdata); |
| 2052 |
// inv vector_butterfly 208 464 1 0 |
| 2053 |
int16x16 e13 = add_x16(d13,d29); |
| 2054 |
int16x16 e29 = sub_x16(d13,d29); |
| 2055 |
// inv vector_butterfly 80 336 1 0 |
| 2056 |
int16x16 e5 = add_x16(d5,d21); |
| 2057 |
int16x16 e21 = sub_x16(d5,d21); |
| 2058 |
// inv vector_butterfly 144 400 1 0 |
| 2059 |
int16x16 e9 = add_x16(d9,d25); |
| 2060 |
int16x16 e25 = sub_x16(d9,d25); |
| 2061 |
// inv vector_butterfly 16 272 1 0 |
| 2062 |
int16x16 e1 = add_x16(d1,d17); |
| 2063 |
int16x16 e17 = sub_x16(d1,d17); |
| 2064 |
// inv startbatch 512 |
| 2065 |
_mm256_storeu_si256((int16x16 *) (f+16),e1); |
| 2066 |
_mm256_storeu_si256((int16x16 *) (f+80),e5); |
| 2067 |
_mm256_storeu_si256((int16x16 *) (f+144),e9); |
| 2068 |
_mm256_storeu_si256((int16x16 *) (f+208),e13); |
| 2069 |
_mm256_storeu_si256((int16x16 *) (f+272),e17); |
| 2070 |
_mm256_storeu_si256((int16x16 *) (f+336),e21); |
| 2071 |
_mm256_storeu_si256((int16x16 *) (f+400),e25); |
| 2072 |
_mm256_storeu_si256((int16x16 *) (f+464),e29); |
| 2073 |
f += 512; |
| 2074 |
} |
| 2075 |
f -= 512*reps; |
| 2076 |
// inv stopbatch 512 |
| 2077 |
for (long long r = 0;r < reps;++r) { |
| 2078 |
// inv vector_permute 384 448 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 2079 |
int16x16 a24 = _mm256_loadu_si256((int16x16 *) (f+384)); |
| 2080 |
int16x16 a28 = _mm256_loadu_si256((int16x16 *) (f+448)); |
| 2081 |
int16x16 b24 = _mm256_permute2x128_si256_lo(a24,a28); |
| 2082 |
int16x16 b28 = _mm256_permute2x128_si256_hi(a24,a28); |
| 2083 |
// inv vector_permute 256 320 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 2084 |
int16x16 a16 = _mm256_loadu_si256((int16x16 *) (f+256)); |
| 2085 |
int16x16 a20 = _mm256_loadu_si256((int16x16 *) (f+320)); |
| 2086 |
int16x16 b16 = _mm256_permute2x128_si256_lo(a16,a20); |
| 2087 |
int16x16 b20 = _mm256_permute2x128_si256_hi(a16,a20); |
| 2088 |
// inv vector_permute 128 192 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 2089 |
int16x16 a8 = _mm256_loadu_si256((int16x16 *) (f+128)); |
| 2090 |
int16x16 a12 = _mm256_loadu_si256((int16x16 *) (f+192)); |
| 2091 |
int16x16 b8 = _mm256_permute2x128_si256_lo(a8,a12); |
| 2092 |
int16x16 b12 = _mm256_permute2x128_si256_hi(a8,a12); |
| 2093 |
// inv vector_permute 0 64 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 2094 |
int16x16 a0 = _mm256_loadu_si256((int16x16 *) (f+0)); |
| 2095 |
int16x16 a4 = _mm256_loadu_si256((int16x16 *) (f+64)); |
| 2096 |
int16x16 b0 = _mm256_permute2x128_si256_lo(a0,a4); |
| 2097 |
int16x16 b4 = _mm256_permute2x128_si256_hi(a0,a4); |
| 2098 |
// inv vector_twist 448 512 507 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 2099 |
b28 = mulmod_scaled_x16(b28,precomp_512_5_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_512_5_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 2100 |
// inv vector_twist 384 512 511 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 2101 |
b24 = mulmod_scaled_x16(b24,precomp_512_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_512_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 2102 |
// inv vector_twist 320 512 5 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 2103 |
b20 = mulmod_scaled_x16(b20,precomp_512_507_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_512_507_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 2104 |
// inv vector_twist 256 512 1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 2105 |
b16 = mulmod_scaled_x16(b16,precomp_512_511_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_512_511_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 2106 |
// inv vector_twist 192 256 255 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 2107 |
b12 = mulmod_scaled_x16(b12,precomp_256_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_256_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 2108 |
// inv vector_twist 128 256 1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 2109 |
b8 = mulmod_scaled_x16(b8,precomp_256_255_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_256_255_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 2110 |
// inv vector_twist 64 128 1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 2111 |
b4 = mulmod_scaled_x16(b4,precomp_128_127_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_128_127_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 2112 |
// inv vector_reduce 0 |
| 2113 |
b0 = reduce_x16(b0,qdata); |
| 2114 |
// inv vector_butterfly 384 448 8 7 |
| 2115 |
int16x16 c24 = add_x16(b24,b28); |
| 2116 |
int16x16 c28 = sub_x16(b24,b28); |
| 2117 |
c28 = mulmod_scaled_x16(c28,scaledzeta_x16_8_1,qinvscaledzeta_x16_8_1,qdata); |
| 2118 |
// inv vector_butterfly 256 320 8 1 |
| 2119 |
int16x16 c16 = add_x16(b16,b20); |
| 2120 |
int16x16 c20 = sub_x16(b16,b20); |
| 2121 |
c20 = mulmod_scaled_x16(c20,scaledzeta_x16_8_7,qinvscaledzeta_x16_8_7,qdata); |
| 2122 |
// inv vector_butterfly 128 192 4 1 |
| 2123 |
int16x16 c8 = add_x16(b8,b12); |
| 2124 |
int16x16 c12 = sub_x16(b8,b12); |
| 2125 |
c12 = mulmod_scaled_x16(c12,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 2126 |
// inv vector_butterfly 0 64 1 0 |
| 2127 |
int16x16 c0 = add_x16(b0,b4); |
| 2128 |
int16x16 c4 = sub_x16(b0,b4); |
| 2129 |
// inv vector_reduce_ifforward 64 |
| 2130 |
// inv vector_butterfly 320 448 4 1 |
| 2131 |
int16x16 d20 = add_x16(c20,c28); |
| 2132 |
int16x16 d28 = sub_x16(c20,c28); |
| 2133 |
d28 = mulmod_scaled_x16(d28,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 2134 |
// inv vector_butterfly 256 384 4 1 |
| 2135 |
int16x16 d16 = add_x16(c16,c24); |
| 2136 |
int16x16 d24 = sub_x16(c16,c24); |
| 2137 |
d24 = mulmod_scaled_x16(d24,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 2138 |
// inv vector_butterfly 64 192 1 0 |
| 2139 |
int16x16 d4 = add_x16(c4,c12); |
| 2140 |
int16x16 d12 = sub_x16(c4,c12); |
| 2141 |
// inv vector_butterfly 0 128 1 0 |
| 2142 |
int16x16 d0 = add_x16(c0,c8); |
| 2143 |
int16x16 d8 = sub_x16(c0,c8); |
| 2144 |
// inv vector_reduce_ifreverse 256 |
| 2145 |
d16 = reduce_x16(d16,qdata); |
| 2146 |
// inv vector_reduce_ifreverse 0 |
| 2147 |
d0 = reduce_x16(d0,qdata); |
| 2148 |
// inv vector_butterfly 192 448 1 0 |
| 2149 |
int16x16 e12 = add_x16(d12,d28); |
| 2150 |
int16x16 e28 = sub_x16(d12,d28); |
| 2151 |
// inv vector_butterfly 64 320 1 0 |
| 2152 |
int16x16 e4 = add_x16(d4,d20); |
| 2153 |
int16x16 e20 = sub_x16(d4,d20); |
| 2154 |
// inv vector_butterfly 128 384 1 0 |
| 2155 |
int16x16 e8 = add_x16(d8,d24); |
| 2156 |
int16x16 e24 = sub_x16(d8,d24); |
| 2157 |
// inv vector_butterfly 0 256 1 0 |
| 2158 |
int16x16 e0 = add_x16(d0,d16); |
| 2159 |
int16x16 e16 = sub_x16(d0,d16); |
| 2160 |
// inv startbatch 512 |
| 2161 |
_mm256_storeu_si256((int16x16 *) (f+0),e0); |
| 2162 |
_mm256_storeu_si256((int16x16 *) (f+64),e4); |
| 2163 |
_mm256_storeu_si256((int16x16 *) (f+128),e8); |
| 2164 |
_mm256_storeu_si256((int16x16 *) (f+192),e12); |
| 2165 |
_mm256_storeu_si256((int16x16 *) (f+256),e16); |
| 2166 |
_mm256_storeu_si256((int16x16 *) (f+320),e20); |
| 2167 |
_mm256_storeu_si256((int16x16 *) (f+384),e24); |
| 2168 |
_mm256_storeu_si256((int16x16 *) (f+448),e28); |
| 2169 |
f += 512; |
| 2170 |
} |
| 2171 |
f -= 512*reps; |
| 2172 |
// inv startntt 512 |
| 2173 |
} |
| 2174 |
|
| 2175 |
void invntt512_7681(int16 *f,int reps) |
| 2176 |
{ |
| 2177 |
invntt512(f,reps,qdata_7681); |
| 2178 |
} |
| 2179 |
|
| 2180 |
void invntt512_10753(int16 *f,int reps) |
| 2181 |
{ |
| 2182 |
invntt512(f,reps,qdata_10753); |
| 2183 |
} |