| 1 |
// linker define ntt512_7681 |
| 2 |
// linker define invntt512_7681 |
| 3 |
|
| 4 |
#include "ntt.h" |
| 5 |
|
| 6 |
// auto-generated; do not edit |
| 7 |
|
| 8 |
#include <immintrin.h> |
| 9 |
|
| 10 |
#define _mm256_permute2x128_si256_lo(f0,f1) _mm256_permute2x128_si256(f0,f1,0x20) |
| 11 |
#define _mm256_permute2x128_si256_hi(f0,f1) _mm256_permute2x128_si256(f0,f1,0x31) |
| 12 |
#define int16x16 __m256i |
| 13 |
|
| 14 |
typedef int16_t int16; |
| 15 |
typedef int32_t int32; |
| 16 |
|
| 17 |
static const int16 __attribute((aligned(32))) qdata_7681[] = { |
| 18 |
#define precomp_8_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 *(const int16x16 *)(qdata+0) |
| 19 |
-3593,-3593,-3593,-3593,-3625,-3625,-3625,-3625,-3593,-3593,-3593,-3593,-3625,-3625,-3625,-3625, |
| 20 |
#define precomp_8_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 *(const int16x16 *)(qdata+16) |
| 21 |
-3777,-3777,-3777,-3777,3182,3182,3182,3182,-3777,-3777,-3777,-3777,3182,3182,3182,3182, |
| 22 |
#define precomp_8_7_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 *(const int16x16 *)(qdata+32) |
| 23 |
-3593,-3593,-3593,-3593,-3182,-3182,-3182,-3182,-3593,-3593,-3593,-3593,-3182,-3182,-3182,-3182, |
| 24 |
#define precomp_8_7_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 *(const int16x16 *)(qdata+48) |
| 25 |
3777,3777,3777,3777,3625,3625,3625,3625,3777,3777,3777,3777,3625,3625,3625,3625, |
| 26 |
#define precomp_16_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 *(const int16x16 *)(qdata+64) |
| 27 |
-3593,-3593,-3593,-3593,2194,2194,2194,2194,-3593,-3593,-3593,-3593,2194,2194,2194,2194, |
| 28 |
#define precomp_16_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 *(const int16x16 *)(qdata+80) |
| 29 |
-3625,-3625,-3625,-3625,-1100,-1100,-1100,-1100,-3625,-3625,-3625,-3625,-1100,-1100,-1100,-1100, |
| 30 |
#define precomp_16_15_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 *(const int16x16 *)(qdata+96) |
| 31 |
-3593,-3593,-3593,-3593,3696,3696,3696,3696,-3593,-3593,-3593,-3593,3696,3696,3696,3696, |
| 32 |
#define precomp_16_15_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 *(const int16x16 *)(qdata+112) |
| 33 |
-3182,-3182,-3182,-3182,-2456,-2456,-2456,-2456,-3182,-3182,-3182,-3182,-2456,-2456,-2456,-2456, |
| 34 |
#define precomp_32_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 *(const int16x16 *)(qdata+128) |
| 35 |
-3593,1701,2194,834,-3625,2319,-1100,121,-3593,1701,2194,834,-3625,2319,-1100,121, |
| 36 |
#define precomp_32_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+144) |
| 37 |
-3777,1414,2456,2495,3182,2876,-3696,2250,-3777,1414,2456,2495,3182,2876,-3696,2250, |
| 38 |
#define precomp_32_31_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 *(const int16x16 *)(qdata+160) |
| 39 |
-3593,-2250,3696,-2876,-3182,-2495,-2456,-1414,-3593,-2250,3696,-2876,-3182,-2495,-2456,-1414, |
| 40 |
#define precomp_32_31_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+176) |
| 41 |
3777,-121,1100,-2319,3625,-834,-2194,-1701,3777,-121,1100,-2319,3625,-834,-2194,-1701, |
| 42 |
#define precomp_64_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 *(const int16x16 *)(qdata+192) |
| 43 |
-3593,3364,1701,-1599,2194,2557,834,-2816,-3593,3364,1701,-1599,2194,2557,834,-2816, |
| 44 |
#define precomp_64_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+208) |
| 45 |
-3625,617,2319,2006,-1100,-1296,121,1986,-3625,617,2319,2006,-1100,-1296,121,1986, |
| 46 |
#define precomp_64_63_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 *(const int16x16 *)(qdata+224) |
| 47 |
-3593,2237,-2250,-1483,3696,3706,-2876,1921,-3593,2237,-2250,-1483,3696,3706,-2876,1921, |
| 48 |
#define precomp_64_63_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+240) |
| 49 |
-3182,2088,-2495,-1525,-2456,1993,-1414,2830,-3182,2088,-2495,-1525,-2456,1993,-1414,2830, |
| 50 |
#define precomp_128_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+256) |
| 51 |
-3593,514,3364,438,1701,2555,-1599,-1738,2194,103,2557,1881,834,-549,-2816,638, |
| 52 |
#define precomp_128_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+272) |
| 53 |
-3625,-1399,617,-1760,2319,2535,2006,3266,-1100,-1431,-1296,3174,121,3153,1986,-810, |
| 54 |
#define precomp_128_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+288) |
| 55 |
-3777,2956,-2830,-679,1414,2440,-1993,-3689,2456,2804,1525,3555,2495,1535,-2088,-7, |
| 56 |
#define precomp_128_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+304) |
| 57 |
3182,-1321,-1921,-1305,2876,-3772,-3706,3600,-3696,-2043,1483,-396,2250,-2310,-2237,1887, |
| 58 |
#define precomp_128_127_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+320) |
| 59 |
-3593,-1887,2237,2310,-2250,396,-1483,2043,3696,-3600,3706,3772,-2876,1305,1921,1321, |
| 60 |
#define precomp_128_127_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+336) |
| 61 |
-3182,7,2088,-1535,-2495,-3555,-1525,-2804,-2456,3689,1993,-2440,-1414,679,2830,-2956, |
| 62 |
#define precomp_128_127_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+352) |
| 63 |
3777,810,-1986,-3153,-121,-3174,1296,1431,1100,-3266,-2006,-2535,-2319,1760,-617,1399, |
| 64 |
#define precomp_128_127_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+368) |
| 65 |
3625,-638,2816,549,-834,-1881,-2557,-103,-2194,1738,1599,-2555,-1701,-438,-3364,-514, |
| 66 |
#define precomp_256_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+384) |
| 67 |
-3593,-1532,514,-373,3364,-3816,438,-3456,1701,783,2555,2883,-1599,727,-1738,-2385, |
| 68 |
#define precomp_256_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+400) |
| 69 |
2194,-2160,103,-2391,2557,2762,1881,-2426,834,3310,-549,-1350,-2816,1386,638,-194, |
| 70 |
#define precomp_256_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+416) |
| 71 |
-3625,404,-1399,-3692,617,-2764,-1760,-1054,2319,1799,2535,-3588,2006,1533,3266,2113, |
| 72 |
#define precomp_256_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+432) |
| 73 |
-1100,-2579,-1431,-1756,-1296,1598,3174,-2,121,-3480,3153,-2572,1986,2743,-810,2919, |
| 74 |
#define precomp_256_255_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+448) |
| 75 |
-3593,2789,-1887,-921,2237,-1497,2310,-2133,-2250,-915,396,1390,-1483,3135,2043,-859, |
| 76 |
#define precomp_256_255_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+464) |
| 77 |
3696,2732,-3600,-1464,3706,2224,3772,-2665,-2876,1698,1305,2835,1921,730,1321,486, |
| 78 |
#define precomp_256_255_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+480) |
| 79 |
-3182,3417,7,-3428,2088,-3145,-1535,1168,-2495,-3831,-3555,-3750,-1525,660,-2804,2649, |
| 80 |
#define precomp_256_255_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+496) |
| 81 |
-2456,3405,3689,-1521,1993,1681,-2440,1056,-1414,1166,679,-2233,2830,2175,-2956,-1919, |
| 82 |
#define precomp_512_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+512) |
| 83 |
-3593,-1404,-1532,451,514,-402,-373,1278,3364,-509,-3816,-3770,438,-2345,-3456,-226, |
| 84 |
#define precomp_512_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+528) |
| 85 |
1701,-1689,783,-1509,2555,2963,2883,1242,-1599,1669,727,2719,-1738,642,-2385,-436, |
| 86 |
#define precomp_512_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+544) |
| 87 |
2194,3335,-2160,1779,103,3745,-2391,17,2557,2812,2762,-1144,1881,83,-2426,-1181, |
| 88 |
#define precomp_512_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+560) |
| 89 |
834,-1519,3310,3568,-549,-796,-1350,2072,-2816,-2460,1386,2891,638,-2083,-194,-715, |
| 90 |
#define precomp_512_5_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+576) |
| 91 |
-3593,-402,-3816,-226,2555,1669,-2385,1779,2557,83,3310,2072,638,1012,-3692,1295, |
| 92 |
#define precomp_512_5_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+592) |
| 93 |
2319,-3208,1533,-2071,-1431,-2005,-2,1586,1986,-293,1919,-929,-679,777,-1681,-3461, |
| 94 |
#define precomp_512_5_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+608) |
| 95 |
2456,3366,3750,-1203,1535,-3657,-3417,-1712,-1921,2515,2665,-1070,3600,2532,-3135,-2589, |
| 96 |
#define precomp_512_5_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+624) |
| 97 |
2250,-2258,921,-658,-514,509,3456,1509,1599,-642,2160,-17,-1881,1519,1350,-2891, |
| 98 |
#define precomp_512_507_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+640) |
| 99 |
-3593,-3434,-1497,893,396,-2422,-859,2965,3706,-2339,1698,-2937,1321,-670,-3428,-3163, |
| 100 |
#define precomp_512_507_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+656) |
| 101 |
-2495,-1072,660,1084,3689,-179,1056,-1338,2830,2786,-2919,-3677,-3153,-151,-1598,3334, |
| 102 |
#define precomp_512_507_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+672) |
| 103 |
1100,-3314,3588,2262,1760,-2230,-404,2083,2816,-3568,2426,-2812,-103,436,-727,-2963, |
| 104 |
#define precomp_512_507_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+688) |
| 105 |
-1701,3770,373,1404,1887,-1649,2133,-826,1483,434,-2732,3287,-3772,-2378,-2835,3723, |
| 106 |
#define precomp_512_511_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+704) |
| 107 |
-3593,658,2789,370,-1887,-3434,-921,-3752,2237,1649,-1497,2258,2310,3581,-2133,893, |
| 108 |
#define precomp_512_511_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+720) |
| 109 |
-2250,3794,-915,826,396,2589,1390,592,-1483,-2422,3135,3214,2043,-434,-859,-2532, |
| 110 |
#define precomp_512_511_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+736) |
| 111 |
3696,1121,2732,2965,-3600,2998,-1464,-3287,3706,1070,2224,-589,3772,-2339,-2665,2070, |
| 112 |
#define precomp_512_511_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+752) |
| 113 |
-2876,2378,1698,-2515,1305,-2815,2835,-2937,1921,-1348,730,-3723,1321,1712,486,2130, |
| 114 |
#define q_x16 *(const int16x16 *)(qdata+768) |
| 115 |
7681,7681,7681,7681,7681,7681,7681,7681,7681,7681,7681,7681,7681,7681,7681,7681, |
| 116 |
#define qinvprecomp_8_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 *(const int16x16 *)(qdata+784) |
| 117 |
-9,-9,-9,-9,-16425,-16425,-16425,-16425,-9,-9,-9,-9,-16425,-16425,-16425,-16425, |
| 118 |
#define qinvprecomp_8_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 *(const int16x16 *)(qdata+800) |
| 119 |
-28865,-28865,-28865,-28865,10350,10350,10350,10350,-28865,-28865,-28865,-28865,10350,10350,10350,10350, |
| 120 |
#define qinvprecomp_8_7_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 *(const int16x16 *)(qdata+816) |
| 121 |
-9,-9,-9,-9,-10350,-10350,-10350,-10350,-9,-9,-9,-9,-10350,-10350,-10350,-10350, |
| 122 |
#define qinvprecomp_8_7_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 *(const int16x16 *)(qdata+832) |
| 123 |
28865,28865,28865,28865,16425,16425,16425,16425,28865,28865,28865,28865,16425,16425,16425,16425, |
| 124 |
#define qinvprecomp_16_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 *(const int16x16 *)(qdata+848) |
| 125 |
-9,-9,-9,-9,-4974,-4974,-4974,-4974,-9,-9,-9,-9,-4974,-4974,-4974,-4974, |
| 126 |
#define qinvprecomp_16_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 *(const int16x16 *)(qdata+864) |
| 127 |
-16425,-16425,-16425,-16425,-7244,-7244,-7244,-7244,-16425,-16425,-16425,-16425,-7244,-7244,-7244,-7244, |
| 128 |
#define qinvprecomp_16_15_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 *(const int16x16 *)(qdata+880) |
| 129 |
-9,-9,-9,-9,-4496,-4496,-4496,-4496,-9,-9,-9,-9,-4496,-4496,-4496,-4496, |
| 130 |
#define qinvprecomp_16_15_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 *(const int16x16 *)(qdata+896) |
| 131 |
-10350,-10350,-10350,-10350,-14744,-14744,-14744,-14744,-10350,-10350,-10350,-10350,-14744,-14744,-14744,-14744, |
| 132 |
#define qinvprecomp_32_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 *(const int16x16 *)(qdata+912) |
| 133 |
-9,-20315,-4974,18242,-16425,18191,-7244,-11655,-9,-20315,-4974,18242,-16425,18191,-7244,-11655, |
| 134 |
#define qinvprecomp_32_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+928) |
| 135 |
-28865,20870,14744,-22593,10350,828,4496,23754,-28865,20870,14744,-22593,10350,828,4496,23754, |
| 136 |
#define qinvprecomp_32_31_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 *(const int16x16 *)(qdata+944) |
| 137 |
-9,-23754,-4496,-828,-10350,22593,-14744,-20870,-9,-23754,-4496,-828,-10350,22593,-14744,-20870, |
| 138 |
#define qinvprecomp_32_31_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+960) |
| 139 |
28865,11655,7244,-18191,16425,-18242,4974,20315,28865,11655,7244,-18191,16425,-18242,4974,20315, |
| 140 |
#define qinvprecomp_64_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 *(const int16x16 *)(qdata+976) |
| 141 |
-9,-10972,-20315,23489,-4974,25597,18242,-2816,-9,-10972,-20315,23489,-4974,25597,18242,-2816, |
| 142 |
#define qinvprecomp_64_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+992) |
| 143 |
-16425,-19351,18191,-3114,-7244,-9488,-11655,19394,-16425,-19351,18191,-3114,-7244,-9488,-11655,19394, |
| 144 |
#define qinvprecomp_64_63_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 *(const int16x16 *)(qdata+1008) |
| 145 |
-9,-7491,-23754,-15307,-4496,-15750,-828,-5759,-9,-7491,-23754,-15307,-4496,-15750,-828,-5759, |
| 146 |
#define qinvprecomp_64_63_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+1024) |
| 147 |
-10350,22568,22593,-20469,-14744,31177,-20870,26382,-10350,22568,22593,-20469,-14744,31177,-20870,26382, |
| 148 |
#define qinvprecomp_128_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+1040) |
| 149 |
-9,-14846,-10972,-21066,-20315,-24581,23489,-23242,-4974,-4505,25597,-26279,18242,21467,-2816,15998, |
| 150 |
#define qinvprecomp_128_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+1056) |
| 151 |
-16425,-4983,-19351,14624,18191,-2073,-3114,20674,-7244,-21399,-9488,6246,-11655,-29103,19394,-5930, |
| 152 |
#define qinvprecomp_128_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+1072) |
| 153 |
-28865,-23668,-26382,-28839,20870,6536,-31177,16279,14744,29428,20469,29667,-22593,9215,-22568,-11783, |
| 154 |
#define qinvprecomp_128_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+1088) |
| 155 |
10350,-14121,5759,-5913,828,-1724,15750,11792,4496,25093,15307,26228,23754,-21766,7491,-6817, |
| 156 |
#define qinvprecomp_128_127_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+1104) |
| 157 |
-9,6817,-7491,21766,-23754,-26228,-15307,-25093,-4496,-11792,-15750,1724,-828,5913,-5759,14121, |
| 158 |
#define qinvprecomp_128_127_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+1120) |
| 159 |
-10350,11783,22568,-9215,22593,-29667,-20469,-29428,-14744,-16279,31177,-6536,-20870,28839,26382,23668, |
| 160 |
#define qinvprecomp_128_127_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+1136) |
| 161 |
28865,5930,-19394,29103,11655,-6246,9488,21399,7244,-20674,3114,2073,-18191,-14624,19351,4983, |
| 162 |
#define qinvprecomp_128_127_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+1152) |
| 163 |
16425,-15998,2816,-21467,-18242,26279,-25597,4505,4974,23242,-23489,24581,20315,21066,10972,14846, |
| 164 |
#define qinvprecomp_256_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+1168) |
| 165 |
-9,-32252,-14846,-19317,-10972,8472,-21066,-3456,-20315,16655,-24581,12611,23489,-12073,-23242,29871, |
| 166 |
#define qinvprecomp_256_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+1184) |
| 167 |
-4974,6032,-4505,10409,25597,24266,-26279,17030,18242,10478,21467,11962,-2816,-26262,15998,-17602, |
| 168 |
#define qinvprecomp_256_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+1200) |
| 169 |
-16425,-22124,-4983,-26220,-19351,-8908,14624,32738,18191,13575,-2073,27132,-3114,24573,20674,27201, |
| 170 |
#define qinvprecomp_256_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+1216) |
| 171 |
-7244,12269,-21399,-16092,-9488,-15810,6246,15358,-11655,-15768,-29103,24052,19394,-26441,-5930,-1689, |
| 172 |
#define qinvprecomp_256_255_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+1232) |
| 173 |
-9,13541,6817,-5529,-7491,26663,21766,-4693,-23754,13933,-26228,8558,-15307,-21953,-25093,-22875, |
| 174 |
#define qinvprecomp_256_255_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+1248) |
| 175 |
-4496,-7508,-11792,-30136,-15750,26800,1724,17303,-828,2722,5913,-12013,-5759,30426,14121,3558, |
| 176 |
#define qinvprecomp_256_255_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+1264) |
| 177 |
-10350,-24743,11783,-21860,22568,-32329,-9215,9360,22593,-7415,-29667,25946,-20469,-21868,-29428,-25511, |
| 178 |
#define qinvprecomp_256_255_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+1280) |
| 179 |
-14744,1869,-16279,14351,31177,2193,-6536,17440,-20870,24718,28839,-23225,26382,9855,23668,-9599, |
| 180 |
#define qinvprecomp_512_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+1296) |
| 181 |
-9,-32124,-32252,10179,-14846,6766,-19317,16638,-10972,-23549,8472,-17082,-21066,-15145,-3456,31518, |
| 182 |
#define qinvprecomp_512_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+1312) |
| 183 |
-20315,-6297,16655,-12261,-24581,-11885,12611,30938,23489,28805,-12073,26783,-23242,-14718,29871,5708, |
| 184 |
#define qinvprecomp_512_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+1328) |
| 185 |
-4974,15111,6032,-29453,-4505,12449,10409,529,25597,-32004,24266,2952,-26279,18003,17030,24931, |
| 186 |
#define qinvprecomp_512_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+1344) |
| 187 |
18242,-1007,10478,-4624,21467,17636,11962,14360,-2816,15972,-26262,16715,15998,4573,-17602,-14539, |
| 188 |
#define qinvprecomp_512_5_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+1360) |
| 189 |
-9,6766,8472,31518,-24581,28805,29871,-29453,25597,18003,10478,14360,15998,27636,-26220,17167, |
| 190 |
#define qinvprecomp_512_5_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+1376) |
| 191 |
18191,-7304,24573,-22039,-21399,-4565,15358,10802,19394,21723,9599,-9633,-28839,-2807,-2193,-30597, |
| 192 |
#define qinvprecomp_512_5_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+1392) |
| 193 |
14744,-26330,-25946,-2739,9215,32695,24743,-26288,5759,20435,-17303,24530,11792,20964,21953,23523, |
| 194 |
#define qinvprecomp_512_5_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+1408) |
| 195 |
23754,-27858,5529,6510,14846,23549,3456,12261,-23489,14718,-6032,-529,26279,1007,-11962,-16715, |
| 196 |
#define qinvprecomp_512_507_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+1424) |
| 197 |
-9,24214,26663,23933,-26228,-13686,-22875,-27243,-15750,4317,2722,8839,14121,-32414,-21860,-25179, |
| 198 |
#define qinvprecomp_512_507_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+1440) |
| 199 |
22593,-25648,-21868,-964,-16279,-1715,17440,-14650,26382,-28958,1689,-10333,29103,-20119,15810,22790, |
| 200 |
#define qinvprecomp_512_507_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+1456) |
| 201 |
7244,20238,-27132,-2858,-14624,19274,22124,-4573,2816,4624,-17030,32004,4505,-5708,12073,11885, |
| 202 |
#define qinvprecomp_512_507_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+1472) |
| 203 |
20315,17082,19317,32124,-6817,14223,4693,-14138,15307,9650,7508,-9513,-1724,-23882,12013,-15221, |
| 204 |
#define qinvprecomp_512_511_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+1488) |
| 205 |
-9,-6510,13541,-23182,6817,24214,-5529,-24232,-7491,-14223,26663,27858,21766,26621,-4693,23933, |
| 206 |
#define qinvprecomp_512_511_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+1504) |
| 207 |
-23754,29394,13933,14138,-26228,-23523,8558,-23984,-15307,-13686,-21953,26766,-25093,-9650,-22875,-20964, |
| 208 |
#define qinvprecomp_512_511_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+1520) |
| 209 |
-4496,-22943,-7508,-27243,-11792,-18506,-30136,9513,-15750,-24530,26800,947,1724,4317,17303,29718, |
| 210 |
#define qinvprecomp_512_511_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+1536) |
| 211 |
-828,23882,2722,-20435,5913,-10495,-12013,8839,-5759,-3396,30426,15221,14121,26288,3558,27730, |
| 212 |
#define qinvscaledzeta_x16_4_1 *(const int16x16 *)(qdata+1552) |
| 213 |
-28865,-28865,-28865,-28865,-28865,-28865,-28865,-28865,-28865,-28865,-28865,-28865,-28865,-28865,-28865,-28865, |
| 214 |
#define qinvscaledzeta_x16_4_3 *(const int16x16 *)(qdata+1568) |
| 215 |
28865,28865,28865,28865,28865,28865,28865,28865,28865,28865,28865,28865,28865,28865,28865,28865, |
| 216 |
#define qinvscaledzeta_x16_8_1 *(const int16x16 *)(qdata+1584) |
| 217 |
-16425,-16425,-16425,-16425,-16425,-16425,-16425,-16425,-16425,-16425,-16425,-16425,-16425,-16425,-16425,-16425, |
| 218 |
#define qinvscaledzeta_x16_8_7 *(const int16x16 *)(qdata+1600) |
| 219 |
-10350,-10350,-10350,-10350,-10350,-10350,-10350,-10350,-10350,-10350,-10350,-10350,-10350,-10350,-10350,-10350, |
| 220 |
#define qround32_x16 *(const int16x16 *)(qdata+1616) |
| 221 |
4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4, |
| 222 |
#define scaledzeta_x16_4_1 *(const int16x16 *)(qdata+1632) |
| 223 |
-3777,-3777,-3777,-3777,-3777,-3777,-3777,-3777,-3777,-3777,-3777,-3777,-3777,-3777,-3777,-3777, |
| 224 |
#define scaledzeta_x16_4_3 *(const int16x16 *)(qdata+1648) |
| 225 |
3777,3777,3777,3777,3777,3777,3777,3777,3777,3777,3777,3777,3777,3777,3777,3777, |
| 226 |
#define scaledzeta_x16_8_1 *(const int16x16 *)(qdata+1664) |
| 227 |
-3625,-3625,-3625,-3625,-3625,-3625,-3625,-3625,-3625,-3625,-3625,-3625,-3625,-3625,-3625,-3625, |
| 228 |
#define scaledzeta_x16_8_7 *(const int16x16 *)(qdata+1680) |
| 229 |
-3182,-3182,-3182,-3182,-3182,-3182,-3182,-3182,-3182,-3182,-3182,-3182,-3182,-3182,-3182,-3182, |
| 230 |
} ; |
| 231 |
|
| 232 |
static inline int16x16 add_x16(int16x16 a,int16x16 b) |
| 233 |
{ |
| 234 |
return _mm256_add_epi16(a,b); |
| 235 |
} |
| 236 |
|
| 237 |
static inline int16x16 sub_x16(int16x16 a,int16x16 b) |
| 238 |
{ |
| 239 |
return _mm256_sub_epi16(a,b); |
| 240 |
} |
| 241 |
|
| 242 |
static inline int16x16 mulmod_scaled_x16(int16x16 x,int16x16 y,int16x16 yqinv,const int16 *qdata) |
| 243 |
{ |
| 244 |
int16x16 b = _mm256_mulhi_epi16(x,y); |
| 245 |
int16x16 d = _mm256_mullo_epi16(x,yqinv); |
| 246 |
int16x16 e = _mm256_mulhi_epi16(d,q_x16); |
| 247 |
return sub_x16(b,e); |
| 248 |
} |
| 249 |
|
| 250 |
static inline int16x16 reduce_x16(int16x16 x,const int16 *qdata) |
| 251 |
{ |
| 252 |
int16x16 y = _mm256_mulhrs_epi16(x,qround32_x16); |
| 253 |
y = _mm256_mullo_epi16(y,q_x16); |
| 254 |
return sub_x16(x,y); |
| 255 |
} |
| 256 |
|
| 257 |
// ----- codegen pass 1 |
| 258 |
// |
| 259 |
// startntt 512 |
| 260 |
// startbatch 512 |
| 261 |
// // ----- PRECONDITIONS |
| 262 |
// physical_map (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 263 |
// // transform size 512 |
| 264 |
// // transform indexing [0, 1, 2, 3, 4, 5, 6, 7, 8] |
| 265 |
// // transforms per batch 1 |
| 266 |
// // batch indexing [] |
| 267 |
// // total batch size 512 |
| 268 |
// |
| 269 |
// // modulus x^512-1 pos 0:512 q 7681,10753 bound 512*(5629,5800) |
| 270 |
// assertranges ... |
| 271 |
// |
| 272 |
// // ----- LAYER 1 |
| 273 |
// |
| 274 |
// // butterfly(0,256,1,256,1,0) |
| 275 |
// butterfly 0 256 1 256 1 0 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 276 |
// |
| 277 |
// // ----- POSTCONDITIONS AFTER LAYER 1 |
| 278 |
// // transform size 512 |
| 279 |
// // transform indexing [0, 1, 2, 3, 4, 5, 6, 7, 8] |
| 280 |
// // transforms per batch 1 |
| 281 |
// // batch indexing [] |
| 282 |
// // total batch size 512 |
| 283 |
// |
| 284 |
// // modulus x^256-1 pos 0:256 q 7681,10753 bound 256*(11258,11600) |
| 285 |
// assertranges ... |
| 286 |
// |
| 287 |
// // modulus x^256+1 pos 256:512 q 7681,10753 bound 256*(11258,11600) |
| 288 |
// assertranges ... |
| 289 |
// |
| 290 |
// // ----- LAYER 2 |
| 291 |
// |
| 292 |
// // reduce_ifreverse(0,64,1) |
| 293 |
// reduce_ifreverse 0 64 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 294 |
// |
| 295 |
// // reduce_ifreverse(256,320,1) |
| 296 |
// reduce_ifreverse 256 320 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 297 |
// |
| 298 |
// // butterfly(0,128,1,128,1,0) |
| 299 |
// butterfly 0 128 1 128 1 0 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 300 |
// |
| 301 |
// // butterfly(256,384,1,128,4,1) |
| 302 |
// butterfly 256 384 1 128 4 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 303 |
// |
| 304 |
// // ----- POSTCONDITIONS AFTER LAYER 2 |
| 305 |
// // transform size 512 |
| 306 |
// // transform indexing [0, 1, 2, 3, 4, 5, 6, 7, 8] |
| 307 |
// // transforms per batch 1 |
| 308 |
// // batch indexing [] |
| 309 |
// // total batch size 512 |
| 310 |
// |
| 311 |
// // modulus x^128-1 pos 0:128 q 7681,10753 bound 128*(22516,23200) |
| 312 |
// assertranges ... |
| 313 |
// |
| 314 |
// // modulus x^128+1 pos 128:256 q 7681,10753 bound 128*(22516,23200) |
| 315 |
// assertranges ... |
| 316 |
// |
| 317 |
// // modulus x^128-zeta4 pos 256:384 q 7681,10753 bound 128*(15747,17016) |
| 318 |
// assertranges ... |
| 319 |
// |
| 320 |
// // modulus x^128+zeta4 pos 384:512 q 7681,10753 bound 128*(15747,17016) |
| 321 |
// assertranges ... |
| 322 |
// |
| 323 |
// // ----- LAYER 3 |
| 324 |
// |
| 325 |
// // reduce_ifforward(64,128,1) |
| 326 |
// reduce_ifforward 64 128 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 327 |
// |
| 328 |
// // butterfly(0,64,1,64,1,0) |
| 329 |
// butterfly 0 64 1 64 1 0 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 330 |
// |
| 331 |
// // butterfly(128,192,1,64,4,1) |
| 332 |
// butterfly 128 192 1 64 4 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 333 |
// |
| 334 |
// // butterfly(256,320,1,64,8,1) |
| 335 |
// butterfly 256 320 1 64 8 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 336 |
// |
| 337 |
// // butterfly(384,448,1,64,8,-1) |
| 338 |
// butterfly 384 448 1 64 8 7 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 339 |
// |
| 340 |
// // reduce(0,64,1) |
| 341 |
// reduce 0 64 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 342 |
// |
| 343 |
// // twist(64,128,1,128,1) |
| 344 |
// twist 64 128 1 128 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 345 |
// |
| 346 |
// // twist(128,192,1,256,1) |
| 347 |
// twist 128 192 1 256 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 348 |
// |
| 349 |
// // twist(192,256,1,256,-1) |
| 350 |
// twist 192 256 1 256 255 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 351 |
// |
| 352 |
// // twist(256,320,1,512,1) |
| 353 |
// twist 256 320 1 512 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 354 |
// |
| 355 |
// // twist(320,384,1,512,5) |
| 356 |
// twist 320 384 1 512 5 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 357 |
// |
| 358 |
// // twist(384,448,1,512,-1) |
| 359 |
// twist 384 448 1 512 511 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 360 |
// |
| 361 |
// // twist(448,512,1,512,-5) |
| 362 |
// twist 448 512 1 512 507 (0, 1, 2, 3, 4, 5, 6, 7, 8) () |
| 363 |
// |
| 364 |
// // physical_permute(3,6) |
| 365 |
// physical_permute (3, 6) (0, 1, 2, 3, 4, 5, 6, 7, 8) () (0, 1, 2, 6, 4, 5, 3, 7, 8) () |
| 366 |
// |
| 367 |
// // fold(256) |
| 368 |
// physical_unmap (0, 1, 2, 6, 4, 5, 3, 7, 8) () |
| 369 |
// physical_map (0, 1, 2, 6, 4, 5, 3, 7) (8,) |
| 370 |
// |
| 371 |
// // fold(128) |
| 372 |
// physical_unmap (0, 1, 2, 6, 4, 5, 3, 7) (8,) |
| 373 |
// physical_map (0, 1, 2, 6, 4, 5, 3) (7, 8) |
| 374 |
// |
| 375 |
// // fold(64) |
| 376 |
// physical_unmap (0, 1, 2, 6, 4, 5, 3) (7, 8) |
| 377 |
// physical_map (0, 1, 2, 6, 4, 5) (3, 7, 8) |
| 378 |
// |
| 379 |
// // nextbatch() |
| 380 |
// stopbatch 512 |
| 381 |
// startbatch 512 |
| 382 |
// |
| 383 |
// // halfbatch() |
| 384 |
// physical_unmap (0, 1, 2, 6, 4, 5) (3, 7, 8) |
| 385 |
// stopbatch 512 |
| 386 |
// doublereps |
| 387 |
// startbatch 256 |
| 388 |
// physical_map (0, 1, 2, 6, 4, 5) (3, 7) |
| 389 |
// |
| 390 |
// // halfbatch() |
| 391 |
// physical_unmap (0, 1, 2, 6, 4, 5) (3, 7) |
| 392 |
// stopbatch 256 |
| 393 |
// doublereps |
| 394 |
// startbatch 128 |
| 395 |
// physical_map (0, 1, 2, 6, 4, 5) (3,) |
| 396 |
// |
| 397 |
// // ----- POSTCONDITIONS AFTER LAYER 3 |
| 398 |
// // transform size 64 |
| 399 |
// // transform indexing [0, 1, 2, 6, 4, 5] |
| 400 |
// // transforms per batch 2 |
| 401 |
// // batch indexing [3] |
| 402 |
// // total batch size 128 |
| 403 |
// |
| 404 |
// // modulus x^64-1 pos 0:64 q 7681,10753 bound 1*(5629,5827) 1*(5629,7613) 1*(5629,7666) 1*(5629,7264) 1*(5629,7639) 1*(5629,7591) 1*(5629,7291) 1*(5629,7204) ... |
| 405 |
// assertranges ... |
| 406 |
// |
| 407 |
// // ----- LAYER 4 |
| 408 |
// |
| 409 |
// // butterfly(0,32,1,32,1,0) |
| 410 |
// butterfly 0 32 1 32 1 0 (0, 1, 2, 6, 4, 5) (3,) |
| 411 |
// |
| 412 |
// // ----- POSTCONDITIONS AFTER LAYER 4 |
| 413 |
// // transform size 64 |
| 414 |
// // transform indexing [0, 1, 2, 6, 4, 5] |
| 415 |
// // transforms per batch 2 |
| 416 |
// // batch indexing [3] |
| 417 |
// // total batch size 128 |
| 418 |
// |
| 419 |
// // modulus x^32-1 pos 0:32 q 7681,10753 bound 1*(11258,13035) 1*(11258,14721) 1*(11258,14855) 1*(11258,14877) 1*(11258,14690) 1*(11258,15282) 1*(11258,14641) 1*(11258,14272) ... |
| 420 |
// assertranges ... |
| 421 |
// |
| 422 |
// // modulus x^32+1 pos 32:64 q 7681,10753 bound 1*(11258,13035) 1*(11258,14721) 1*(11258,14855) 1*(11258,14877) 1*(11258,14690) 1*(11258,15282) 1*(11258,14641) 1*(11258,14272) ... |
| 423 |
// assertranges ... |
| 424 |
// |
| 425 |
// // ----- LAYER 5 |
| 426 |
// |
| 427 |
// // butterfly(0,16,1,16,1,0) |
| 428 |
// butterfly 0 16 1 16 1 0 (0, 1, 2, 6, 4, 5) (3,) |
| 429 |
// |
| 430 |
// // butterfly(32,48,1,16,4,1) |
| 431 |
// butterfly 32 48 1 16 4 1 (0, 1, 2, 6, 4, 5) (3,) |
| 432 |
// |
| 433 |
// // reduce(0,16,1) |
| 434 |
// reduce 0 16 1 (0, 1, 2, 6, 4, 5) (3,) |
| 435 |
// |
| 436 |
// // twist(16,32,1,32,1) |
| 437 |
// twist 16 32 1 32 1 (0, 1, 2, 6, 4, 5) (3,) |
| 438 |
// |
| 439 |
// // twist(32,48,1,64,1) |
| 440 |
// twist 32 48 1 64 1 (0, 1, 2, 6, 4, 5) (3,) |
| 441 |
// |
| 442 |
// // twist(48,64,1,64,-1) |
| 443 |
// twist 48 64 1 64 63 (0, 1, 2, 6, 4, 5) (3,) |
| 444 |
// |
| 445 |
// // physical_permute(0,1,2,5) |
| 446 |
// physical_permute (0, 1, 2, 5) (0, 1, 2, 6, 4, 5) (3,) (1, 2, 5, 6, 4, 0) (3,) |
| 447 |
// |
| 448 |
// // fold(32) |
| 449 |
// physical_unmap (1, 2, 5, 6, 4, 0) (3,) |
| 450 |
// physical_map (1, 2, 5, 6, 4) (0, 3) |
| 451 |
// |
| 452 |
// // fold(16) |
| 453 |
// physical_unmap (1, 2, 5, 6, 4) (0, 3) |
| 454 |
// physical_map (1, 2, 5, 6) (0, 3, 4) |
| 455 |
// |
| 456 |
// // ----- POSTCONDITIONS AFTER LAYER 5 |
| 457 |
// // transform size 16 |
| 458 |
// // transform indexing [1, 2, 5, 6] |
| 459 |
// // transforms per batch 8 |
| 460 |
// // batch indexing [0, 3, 4] |
| 461 |
// // total batch size 128 |
| 462 |
// |
| 463 |
// // modulus x^16-1 pos 0:16 q 7681,10753 bound 1*(5629,5800) 1*(5629,6967) 1*(5629,6418) 1*(5629,7585) 1*(5629,7017) 1*(5629,6328) 1*(5629,7033) 1*(5629,6943) ... |
| 464 |
// assertranges ... |
| 465 |
// |
| 466 |
// // ----- LAYER 6 |
| 467 |
// |
| 468 |
// // butterfly(0,8,1,8,1,0) |
| 469 |
// butterfly 0 8 1 8 1 0 (1, 2, 5, 6) (0, 3, 4) |
| 470 |
// |
| 471 |
// // physical_permute(1,2,4) |
| 472 |
// physical_permute (1, 2, 4) (1, 2, 5, 6) (0, 3, 4) (2, 4, 5, 6) (0, 3, 1) |
| 473 |
// |
| 474 |
// // nextbatch() |
| 475 |
// stopbatch 128 |
| 476 |
// startbatch 128 |
| 477 |
// |
| 478 |
// // ----- POSTCONDITIONS AFTER LAYER 6 |
| 479 |
// // transform size 16 |
| 480 |
// // transform indexing [2, 4, 5, 6] |
| 481 |
// // transforms per batch 8 |
| 482 |
// // batch indexing [0, 3, 1] |
| 483 |
// // total batch size 128 |
| 484 |
// |
| 485 |
// // modulus x^8-1 pos 0:8 q 7681,10753 bound 1*(11258,12447) 1*(11258,14071) 1*(11258,12488) 1*(11258,14310) 1*(11258,14287) 1*(11258,13674) 1*(11258,13574) 1*(11258,13555) |
| 486 |
// assertranges ... |
| 487 |
// |
| 488 |
// // modulus x^8+1 pos 8:16 q 7681,10753 bound 1*(11258,12447) 1*(11258,14071) 1*(11258,12488) 1*(11258,14310) 1*(11258,14287) 1*(11258,13674) 1*(11258,13574) 1*(11258,13555) |
| 489 |
// assertranges ... |
| 490 |
// |
| 491 |
// // ----- LAYER 7 |
| 492 |
// |
| 493 |
// // butterfly(0,4,1,4,1,0) |
| 494 |
// butterfly 0 4 1 4 1 0 (2, 4, 5, 6) (0, 3, 1) |
| 495 |
// |
| 496 |
// // butterfly(8,12,1,4,4,1) |
| 497 |
// butterfly 8 12 1 4 4 1 (2, 4, 5, 6) (0, 3, 1) |
| 498 |
// |
| 499 |
// // reduce(0,4,1) |
| 500 |
// reduce 0 4 1 (2, 4, 5, 6) (0, 3, 1) |
| 501 |
// |
| 502 |
// // twist(4,8,1,8,1) |
| 503 |
// twist 4 8 1 8 1 (2, 4, 5, 6) (0, 3, 1) |
| 504 |
// |
| 505 |
// // twist(8,12,1,16,1) |
| 506 |
// twist 8 12 1 16 1 (2, 4, 5, 6) (0, 3, 1) |
| 507 |
// |
| 508 |
// // twist(12,16,1,16,-1) |
| 509 |
// twist 12 16 1 16 15 (2, 4, 5, 6) (0, 3, 1) |
| 510 |
// |
| 511 |
// // physical_permute(2,6) |
| 512 |
// physical_permute (2, 6) (2, 4, 5, 6) (0, 3, 1) (6, 4, 5, 2) (0, 3, 1) |
| 513 |
// |
| 514 |
// // fold(8) |
| 515 |
// physical_unmap (6, 4, 5, 2) (0, 3, 1) |
| 516 |
// physical_map (6, 4, 5) (0, 1, 2, 3) |
| 517 |
// |
| 518 |
// // fold(4) |
| 519 |
// physical_unmap (6, 4, 5) (0, 1, 2, 3) |
| 520 |
// physical_map (6, 4) (0, 1, 2, 3, 5) |
| 521 |
// |
| 522 |
// // ----- POSTCONDITIONS AFTER LAYER 7 |
| 523 |
// // transform size 4 |
| 524 |
// // transform indexing [6, 4] |
| 525 |
// // transforms per batch 32 |
| 526 |
// // batch indexing [0, 1, 2, 3, 5] |
| 527 |
// // total batch size 128 |
| 528 |
// |
| 529 |
// // modulus x^4-1 pos 0:4 q 7681,10753 bound 1*(5629,5800) 1*(5629,6938) 1*(5629,6521) 1*(5629,7157) |
| 530 |
// assertranges ... |
| 531 |
// |
| 532 |
// // ----- LAYER 8 |
| 533 |
// |
| 534 |
// // butterfly(0,2,1,2,1,0) |
| 535 |
// butterfly 0 2 1 2 1 0 (6, 4) (0, 1, 2, 3, 5) |
| 536 |
// |
| 537 |
// // ----- POSTCONDITIONS AFTER LAYER 8 |
| 538 |
// // transform size 4 |
| 539 |
// // transform indexing [6, 4] |
| 540 |
// // transforms per batch 32 |
| 541 |
// // batch indexing [0, 1, 2, 3, 5] |
| 542 |
// // total batch size 128 |
| 543 |
// |
| 544 |
// // modulus x^2-1 pos 0:2 q 7681,10753 bound 1*(11258,12321) 1*(11258,14095) |
| 545 |
// assertranges ... |
| 546 |
// |
| 547 |
// // modulus x^2+1 pos 2:4 q 7681,10753 bound 1*(11258,12321) 1*(11258,14095) |
| 548 |
// assertranges ... |
| 549 |
// |
| 550 |
// // ----- LAYER 9 |
| 551 |
// |
| 552 |
// // butterfly(0,1,1,1,1,0) |
| 553 |
// butterfly 0 1 1 1 1 0 (6, 4) (0, 1, 2, 3, 5) |
| 554 |
// |
| 555 |
// // butterfly(2,3,1,1,4,1) |
| 556 |
// butterfly 2 3 1 1 4 1 (6, 4) (0, 1, 2, 3, 5) |
| 557 |
// |
| 558 |
// // ----- POSTCONDITIONS AFTER LAYER 9 |
| 559 |
// // transform size 4 |
| 560 |
// // transform indexing [6, 4] |
| 561 |
// // transforms per batch 32 |
| 562 |
// // batch indexing [0, 1, 2, 3, 5] |
| 563 |
// // total batch size 128 |
| 564 |
// |
| 565 |
// // modulus x^1-1 pos 0:1 q 7681,10753 bound 1*(22516,26416) |
| 566 |
// assertranges ... |
| 567 |
// |
| 568 |
// // modulus x^1+1 pos 1:2 q 7681,10753 bound 1*(22516,26416) |
| 569 |
// assertranges ... |
| 570 |
// |
| 571 |
// // modulus x^1-zeta4 pos 2:3 q 7681,10753 bound 1*(15747,17745) |
| 572 |
// assertranges ... |
| 573 |
// |
| 574 |
// // modulus x^1+zeta4 pos 3:4 q 7681,10753 bound 1*(15747,17745) |
| 575 |
// assertranges ... |
| 576 |
// stopbatch 128 |
| 577 |
// physical_unmap (6, 4) (0, 1, 2, 3, 5) |
| 578 |
// stopntt 512 |
| 579 |
|
| 580 |
// ----- codegen pass 2 |
| 581 |
// |
| 582 |
// startntt 512 |
| 583 |
// startbatch 512 |
| 584 |
// vector_butterfly 0 256 1 0 |
| 585 |
// vector_butterfly 128 384 1 0 |
| 586 |
// vector_butterfly 64 320 1 0 |
| 587 |
// vector_butterfly 192 448 1 0 |
| 588 |
// vector_reduce_ifreverse 0 |
| 589 |
// vector_reduce_ifreverse 256 |
| 590 |
// vector_butterfly 0 128 1 0 |
| 591 |
// vector_butterfly 64 192 1 0 |
| 592 |
// vector_butterfly 256 384 4 1 |
| 593 |
// vector_butterfly 320 448 4 1 |
| 594 |
// vector_reduce_ifforward 64 |
| 595 |
// vector_butterfly 0 64 1 0 |
| 596 |
// vector_butterfly 128 192 4 1 |
| 597 |
// vector_butterfly 256 320 8 1 |
| 598 |
// vector_butterfly 384 448 8 7 |
| 599 |
// vector_reduce 0 |
| 600 |
// vector_twist 64 128 1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 601 |
// vector_twist 128 256 1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 602 |
// vector_twist 192 256 255 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 603 |
// vector_twist 256 512 1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 604 |
// vector_twist 320 512 5 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 605 |
// vector_twist 384 512 511 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 606 |
// vector_twist 448 512 507 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 607 |
// vector_permute 0 64 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 608 |
// vector_permute 128 192 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 609 |
// vector_permute 256 320 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 610 |
// vector_permute 384 448 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 611 |
// stopbatch 512 |
| 612 |
// startbatch 512 |
| 613 |
// vector_butterfly 16 272 1 0 |
| 614 |
// vector_butterfly 144 400 1 0 |
| 615 |
// vector_butterfly 80 336 1 0 |
| 616 |
// vector_butterfly 208 464 1 0 |
| 617 |
// vector_reduce_ifreverse 16 |
| 618 |
// vector_reduce_ifreverse 272 |
| 619 |
// vector_butterfly 16 144 1 0 |
| 620 |
// vector_butterfly 80 208 1 0 |
| 621 |
// vector_butterfly 272 400 4 1 |
| 622 |
// vector_butterfly 336 464 4 1 |
| 623 |
// vector_reduce_ifforward 80 |
| 624 |
// vector_butterfly 16 80 1 0 |
| 625 |
// vector_butterfly 144 208 4 1 |
| 626 |
// vector_butterfly 272 336 8 1 |
| 627 |
// vector_butterfly 400 464 8 7 |
| 628 |
// vector_reduce 16 |
| 629 |
// vector_twist 80 128 1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 630 |
// vector_twist 144 256 1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 631 |
// vector_twist 208 256 255 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 632 |
// vector_twist 272 512 1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 633 |
// vector_twist 336 512 5 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 634 |
// vector_twist 400 512 511 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 635 |
// vector_twist 464 512 507 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 636 |
// vector_permute 16 80 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 637 |
// vector_permute 144 208 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 638 |
// vector_permute 272 336 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 639 |
// vector_permute 400 464 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 640 |
// stopbatch 512 |
| 641 |
// startbatch 512 |
| 642 |
// vector_butterfly 32 288 1 0 |
| 643 |
// vector_butterfly 160 416 1 0 |
| 644 |
// vector_butterfly 96 352 1 0 |
| 645 |
// vector_butterfly 224 480 1 0 |
| 646 |
// vector_reduce_ifreverse 32 |
| 647 |
// vector_reduce_ifreverse 288 |
| 648 |
// vector_butterfly 32 160 1 0 |
| 649 |
// vector_butterfly 96 224 1 0 |
| 650 |
// vector_butterfly 288 416 4 1 |
| 651 |
// vector_butterfly 352 480 4 1 |
| 652 |
// vector_reduce_ifforward 96 |
| 653 |
// vector_butterfly 32 96 1 0 |
| 654 |
// vector_butterfly 160 224 4 1 |
| 655 |
// vector_butterfly 288 352 8 1 |
| 656 |
// vector_butterfly 416 480 8 7 |
| 657 |
// vector_reduce 32 |
| 658 |
// vector_twist 96 128 1 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 659 |
// vector_twist 160 256 1 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 660 |
// vector_twist 224 256 255 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 661 |
// vector_twist 288 512 1 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 662 |
// vector_twist 352 512 5 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 663 |
// vector_twist 416 512 511 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 664 |
// vector_twist 480 512 507 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 665 |
// vector_permute 32 96 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 666 |
// vector_permute 160 224 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 667 |
// vector_permute 288 352 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 668 |
// vector_permute 416 480 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 669 |
// stopbatch 512 |
| 670 |
// startbatch 512 |
| 671 |
// vector_butterfly 48 304 1 0 |
| 672 |
// vector_butterfly 176 432 1 0 |
| 673 |
// vector_butterfly 112 368 1 0 |
| 674 |
// vector_butterfly 240 496 1 0 |
| 675 |
// vector_reduce_ifreverse 48 |
| 676 |
// vector_reduce_ifreverse 304 |
| 677 |
// vector_butterfly 48 176 1 0 |
| 678 |
// vector_butterfly 112 240 1 0 |
| 679 |
// vector_butterfly 304 432 4 1 |
| 680 |
// vector_butterfly 368 496 4 1 |
| 681 |
// vector_reduce_ifforward 112 |
| 682 |
// vector_butterfly 48 112 1 0 |
| 683 |
// vector_butterfly 176 240 4 1 |
| 684 |
// vector_butterfly 304 368 8 1 |
| 685 |
// vector_butterfly 432 496 8 7 |
| 686 |
// vector_reduce 48 |
| 687 |
// vector_twist 112 128 1 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 688 |
// vector_twist 176 256 1 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 689 |
// vector_twist 240 256 255 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 690 |
// vector_twist 304 512 1 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 691 |
// vector_twist 368 512 5 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 692 |
// vector_twist 432 512 511 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 693 |
// vector_twist 496 512 507 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 694 |
// vector_permute 48 112 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 695 |
// vector_permute 176 240 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 696 |
// vector_permute 304 368 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 697 |
// vector_permute 432 496 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 698 |
// stopbatch 512 |
| 699 |
// doublereps |
| 700 |
// doublereps |
| 701 |
// startbatch 128 |
| 702 |
// vector_butterfly 0 32 1 0 |
| 703 |
// vector_butterfly 64 96 1 0 |
| 704 |
// vector_butterfly 16 48 1 0 |
| 705 |
// vector_butterfly 80 112 1 0 |
| 706 |
// vector_butterfly 0 16 1 0 |
| 707 |
// vector_butterfly 64 80 1 0 |
| 708 |
// vector_butterfly 32 48 4 1 |
| 709 |
// vector_butterfly 96 112 4 1 |
| 710 |
// vector_reduce 0 |
| 711 |
// vector_reduce 64 |
| 712 |
// vector_twist 16 32 1 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 |
| 713 |
// vector_twist 80 32 1 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15 |
| 714 |
// vector_twist 32 64 1 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 |
| 715 |
// vector_twist 96 64 1 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15 |
| 716 |
// vector_twist 48 64 63 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 |
| 717 |
// vector_twist 112 64 63 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15 |
| 718 |
// vector_permute 0 32 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16 |
| 719 |
// vector_permute 16 48 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16 |
| 720 |
// vector_permute 64 96 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16 |
| 721 |
// vector_permute 80 112 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16 |
| 722 |
// vector_butterfly 0 64 1 0 |
| 723 |
// vector_butterfly 32 96 1 0 |
| 724 |
// vector_butterfly 16 80 1 0 |
| 725 |
// vector_butterfly 48 112 1 0 |
| 726 |
// vector_permute 0 16 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32 |
| 727 |
// vector_permute 32 48 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32 |
| 728 |
// vector_permute 64 80 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32 |
| 729 |
// vector_permute 96 112 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32 |
| 730 |
// stopbatch 128 |
| 731 |
// startbatch 128 |
| 732 |
// vector_butterfly 0 32 1 0 |
| 733 |
// vector_butterfly 16 48 1 0 |
| 734 |
// vector_butterfly 64 96 4 1 |
| 735 |
// vector_butterfly 80 112 4 1 |
| 736 |
// vector_reduce 0 |
| 737 |
// vector_reduce 16 |
| 738 |
// vector_twist 32 8 1 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1 |
| 739 |
// vector_twist 48 8 1 2 2 2 2 3 3 3 3 2 2 2 2 3 3 3 3 |
| 740 |
// vector_twist 64 16 1 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1 |
| 741 |
// vector_twist 80 16 1 2 2 2 2 3 3 3 3 2 2 2 2 3 3 3 3 |
| 742 |
// vector_twist 96 16 15 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1 |
| 743 |
// vector_twist 112 16 15 2 2 2 2 3 3 3 3 2 2 2 2 3 3 3 3 |
| 744 |
// vector_permute 0 64 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64 |
| 745 |
// vector_permute 16 80 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64 |
| 746 |
// vector_permute 32 96 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64 |
| 747 |
// vector_permute 48 112 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64 |
| 748 |
// vector_butterfly 0 16 1 0 |
| 749 |
// vector_butterfly 64 80 1 0 |
| 750 |
// vector_butterfly 32 48 1 0 |
| 751 |
// vector_butterfly 96 112 1 0 |
| 752 |
// vector_butterfly 0 64 1 0 |
| 753 |
// vector_butterfly 32 96 1 0 |
| 754 |
// vector_butterfly 16 80 4 1 |
| 755 |
// vector_butterfly 48 112 4 1 |
| 756 |
// stopbatch 128 |
| 757 |
// stopntt 512 |
| 758 |
// startntt 512 |
| 759 |
|
| 760 |
static void ntt512(int16 *f,int reps,const int16 *qdata) |
| 761 |
{ |
| 762 |
// startbatch 512 |
| 763 |
for (long long r = 0;r < reps;++r) { |
| 764 |
// vector_butterfly 0 256 1 0 |
| 765 |
int16x16 a0 = _mm256_loadu_si256((int16x16 *) (f+0)); |
| 766 |
int16x16 a16 = _mm256_loadu_si256((int16x16 *) (f+256)); |
| 767 |
int16x16 b0 = add_x16(a0,a16); |
| 768 |
int16x16 b16 = sub_x16(a0,a16); |
| 769 |
// vector_butterfly 128 384 1 0 |
| 770 |
int16x16 a8 = _mm256_loadu_si256((int16x16 *) (f+128)); |
| 771 |
int16x16 a24 = _mm256_loadu_si256((int16x16 *) (f+384)); |
| 772 |
int16x16 b8 = add_x16(a8,a24); |
| 773 |
int16x16 b24 = sub_x16(a8,a24); |
| 774 |
// vector_butterfly 64 320 1 0 |
| 775 |
int16x16 a4 = _mm256_loadu_si256((int16x16 *) (f+64)); |
| 776 |
int16x16 a20 = _mm256_loadu_si256((int16x16 *) (f+320)); |
| 777 |
int16x16 b4 = add_x16(a4,a20); |
| 778 |
int16x16 b20 = sub_x16(a4,a20); |
| 779 |
// vector_butterfly 192 448 1 0 |
| 780 |
int16x16 a12 = _mm256_loadu_si256((int16x16 *) (f+192)); |
| 781 |
int16x16 a28 = _mm256_loadu_si256((int16x16 *) (f+448)); |
| 782 |
int16x16 b12 = add_x16(a12,a28); |
| 783 |
int16x16 b28 = sub_x16(a12,a28); |
| 784 |
// vector_reduce_ifreverse 0 |
| 785 |
// vector_reduce_ifreverse 256 |
| 786 |
// vector_butterfly 0 128 1 0 |
| 787 |
int16x16 c0 = add_x16(b0,b8); |
| 788 |
int16x16 c8 = sub_x16(b0,b8); |
| 789 |
// vector_butterfly 64 192 1 0 |
| 790 |
int16x16 c4 = add_x16(b4,b12); |
| 791 |
int16x16 c12 = sub_x16(b4,b12); |
| 792 |
// vector_butterfly 256 384 4 1 |
| 793 |
b24 = mulmod_scaled_x16(b24,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 794 |
int16x16 c16 = add_x16(b16,b24); |
| 795 |
int16x16 c24 = sub_x16(b16,b24); |
| 796 |
// vector_butterfly 320 448 4 1 |
| 797 |
b28 = mulmod_scaled_x16(b28,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 798 |
int16x16 c20 = add_x16(b20,b28); |
| 799 |
int16x16 c28 = sub_x16(b20,b28); |
| 800 |
// vector_reduce_ifforward 64 |
| 801 |
c4 = reduce_x16(c4,qdata); |
| 802 |
// vector_butterfly 0 64 1 0 |
| 803 |
int16x16 d0 = add_x16(c0,c4); |
| 804 |
int16x16 d4 = sub_x16(c0,c4); |
| 805 |
// vector_butterfly 128 192 4 1 |
| 806 |
c12 = mulmod_scaled_x16(c12,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 807 |
int16x16 d8 = add_x16(c8,c12); |
| 808 |
int16x16 d12 = sub_x16(c8,c12); |
| 809 |
// vector_butterfly 256 320 8 1 |
| 810 |
c20 = mulmod_scaled_x16(c20,scaledzeta_x16_8_1,qinvscaledzeta_x16_8_1,qdata); |
| 811 |
int16x16 d16 = add_x16(c16,c20); |
| 812 |
int16x16 d20 = sub_x16(c16,c20); |
| 813 |
// vector_butterfly 384 448 8 7 |
| 814 |
c28 = mulmod_scaled_x16(c28,scaledzeta_x16_8_7,qinvscaledzeta_x16_8_7,qdata); |
| 815 |
int16x16 d24 = add_x16(c24,c28); |
| 816 |
int16x16 d28 = sub_x16(c24,c28); |
| 817 |
// vector_reduce 0 |
| 818 |
d0 = reduce_x16(d0,qdata); |
| 819 |
// vector_twist 64 128 1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 820 |
d4 = mulmod_scaled_x16(d4,precomp_128_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_128_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 821 |
// vector_twist 128 256 1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 822 |
d8 = mulmod_scaled_x16(d8,precomp_256_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_256_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 823 |
// vector_twist 192 256 255 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 824 |
d12 = mulmod_scaled_x16(d12,precomp_256_255_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_256_255_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 825 |
// vector_twist 256 512 1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 826 |
d16 = mulmod_scaled_x16(d16,precomp_512_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_512_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 827 |
// vector_twist 320 512 5 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 828 |
d20 = mulmod_scaled_x16(d20,precomp_512_5_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_512_5_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 829 |
// vector_twist 384 512 511 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 830 |
d24 = mulmod_scaled_x16(d24,precomp_512_511_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_512_511_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 831 |
// vector_twist 448 512 507 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 832 |
d28 = mulmod_scaled_x16(d28,precomp_512_507_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_512_507_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 833 |
// vector_permute 0 64 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 834 |
int16x16 e0 = _mm256_permute2x128_si256_lo(d0,d4); |
| 835 |
int16x16 e4 = _mm256_permute2x128_si256_hi(d0,d4); |
| 836 |
// vector_permute 128 192 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 837 |
int16x16 e8 = _mm256_permute2x128_si256_lo(d8,d12); |
| 838 |
int16x16 e12 = _mm256_permute2x128_si256_hi(d8,d12); |
| 839 |
// vector_permute 256 320 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 840 |
int16x16 e16 = _mm256_permute2x128_si256_lo(d16,d20); |
| 841 |
int16x16 e20 = _mm256_permute2x128_si256_hi(d16,d20); |
| 842 |
// vector_permute 384 448 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 843 |
int16x16 e24 = _mm256_permute2x128_si256_lo(d24,d28); |
| 844 |
int16x16 e28 = _mm256_permute2x128_si256_hi(d24,d28); |
| 845 |
// stopbatch 512 |
| 846 |
_mm256_storeu_si256((int16x16 *) (f+0),e0); |
| 847 |
_mm256_storeu_si256((int16x16 *) (f+64),e4); |
| 848 |
_mm256_storeu_si256((int16x16 *) (f+128),e8); |
| 849 |
_mm256_storeu_si256((int16x16 *) (f+192),e12); |
| 850 |
_mm256_storeu_si256((int16x16 *) (f+256),e16); |
| 851 |
_mm256_storeu_si256((int16x16 *) (f+320),e20); |
| 852 |
_mm256_storeu_si256((int16x16 *) (f+384),e24); |
| 853 |
_mm256_storeu_si256((int16x16 *) (f+448),e28); |
| 854 |
f += 512; |
| 855 |
} |
| 856 |
f -= 512*reps; |
| 857 |
// startbatch 512 |
| 858 |
for (long long r = 0;r < reps;++r) { |
| 859 |
// vector_butterfly 16 272 1 0 |
| 860 |
int16x16 a1 = _mm256_loadu_si256((int16x16 *) (f+16)); |
| 861 |
int16x16 a17 = _mm256_loadu_si256((int16x16 *) (f+272)); |
| 862 |
int16x16 b1 = add_x16(a1,a17); |
| 863 |
int16x16 b17 = sub_x16(a1,a17); |
| 864 |
// vector_butterfly 144 400 1 0 |
| 865 |
int16x16 a9 = _mm256_loadu_si256((int16x16 *) (f+144)); |
| 866 |
int16x16 a25 = _mm256_loadu_si256((int16x16 *) (f+400)); |
| 867 |
int16x16 b9 = add_x16(a9,a25); |
| 868 |
int16x16 b25 = sub_x16(a9,a25); |
| 869 |
// vector_butterfly 80 336 1 0 |
| 870 |
int16x16 a5 = _mm256_loadu_si256((int16x16 *) (f+80)); |
| 871 |
int16x16 a21 = _mm256_loadu_si256((int16x16 *) (f+336)); |
| 872 |
int16x16 b5 = add_x16(a5,a21); |
| 873 |
int16x16 b21 = sub_x16(a5,a21); |
| 874 |
// vector_butterfly 208 464 1 0 |
| 875 |
int16x16 a13 = _mm256_loadu_si256((int16x16 *) (f+208)); |
| 876 |
int16x16 a29 = _mm256_loadu_si256((int16x16 *) (f+464)); |
| 877 |
int16x16 b13 = add_x16(a13,a29); |
| 878 |
int16x16 b29 = sub_x16(a13,a29); |
| 879 |
// vector_reduce_ifreverse 16 |
| 880 |
// vector_reduce_ifreverse 272 |
| 881 |
// vector_butterfly 16 144 1 0 |
| 882 |
int16x16 c1 = add_x16(b1,b9); |
| 883 |
int16x16 c9 = sub_x16(b1,b9); |
| 884 |
// vector_butterfly 80 208 1 0 |
| 885 |
int16x16 c5 = add_x16(b5,b13); |
| 886 |
int16x16 c13 = sub_x16(b5,b13); |
| 887 |
// vector_butterfly 272 400 4 1 |
| 888 |
b25 = mulmod_scaled_x16(b25,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 889 |
int16x16 c17 = add_x16(b17,b25); |
| 890 |
int16x16 c25 = sub_x16(b17,b25); |
| 891 |
// vector_butterfly 336 464 4 1 |
| 892 |
b29 = mulmod_scaled_x16(b29,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 893 |
int16x16 c21 = add_x16(b21,b29); |
| 894 |
int16x16 c29 = sub_x16(b21,b29); |
| 895 |
// vector_reduce_ifforward 80 |
| 896 |
c5 = reduce_x16(c5,qdata); |
| 897 |
// vector_butterfly 16 80 1 0 |
| 898 |
int16x16 d1 = add_x16(c1,c5); |
| 899 |
int16x16 d5 = sub_x16(c1,c5); |
| 900 |
// vector_butterfly 144 208 4 1 |
| 901 |
c13 = mulmod_scaled_x16(c13,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 902 |
int16x16 d9 = add_x16(c9,c13); |
| 903 |
int16x16 d13 = sub_x16(c9,c13); |
| 904 |
// vector_butterfly 272 336 8 1 |
| 905 |
c21 = mulmod_scaled_x16(c21,scaledzeta_x16_8_1,qinvscaledzeta_x16_8_1,qdata); |
| 906 |
int16x16 d17 = add_x16(c17,c21); |
| 907 |
int16x16 d21 = sub_x16(c17,c21); |
| 908 |
// vector_butterfly 400 464 8 7 |
| 909 |
c29 = mulmod_scaled_x16(c29,scaledzeta_x16_8_7,qinvscaledzeta_x16_8_7,qdata); |
| 910 |
int16x16 d25 = add_x16(c25,c29); |
| 911 |
int16x16 d29 = sub_x16(c25,c29); |
| 912 |
// vector_reduce 16 |
| 913 |
d1 = reduce_x16(d1,qdata); |
| 914 |
// vector_twist 80 128 1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 915 |
d5 = mulmod_scaled_x16(d5,precomp_128_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_128_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 916 |
// vector_twist 144 256 1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 917 |
d9 = mulmod_scaled_x16(d9,precomp_256_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_256_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 918 |
// vector_twist 208 256 255 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 919 |
d13 = mulmod_scaled_x16(d13,precomp_256_255_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_256_255_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 920 |
// vector_twist 272 512 1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 921 |
d17 = mulmod_scaled_x16(d17,precomp_512_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_512_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 922 |
// vector_twist 336 512 5 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 923 |
d21 = mulmod_scaled_x16(d21,precomp_512_5_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_512_5_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 924 |
// vector_twist 400 512 511 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 925 |
d25 = mulmod_scaled_x16(d25,precomp_512_511_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_512_511_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 926 |
// vector_twist 464 512 507 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 927 |
d29 = mulmod_scaled_x16(d29,precomp_512_507_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_512_507_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 928 |
// vector_permute 16 80 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 929 |
int16x16 e1 = _mm256_permute2x128_si256_lo(d1,d5); |
| 930 |
int16x16 e5 = _mm256_permute2x128_si256_hi(d1,d5); |
| 931 |
// vector_permute 144 208 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 932 |
int16x16 e9 = _mm256_permute2x128_si256_lo(d9,d13); |
| 933 |
int16x16 e13 = _mm256_permute2x128_si256_hi(d9,d13); |
| 934 |
// vector_permute 272 336 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 935 |
int16x16 e17 = _mm256_permute2x128_si256_lo(d17,d21); |
| 936 |
int16x16 e21 = _mm256_permute2x128_si256_hi(d17,d21); |
| 937 |
// vector_permute 400 464 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 938 |
int16x16 e25 = _mm256_permute2x128_si256_lo(d25,d29); |
| 939 |
int16x16 e29 = _mm256_permute2x128_si256_hi(d25,d29); |
| 940 |
// stopbatch 512 |
| 941 |
_mm256_storeu_si256((int16x16 *) (f+16),e1); |
| 942 |
_mm256_storeu_si256((int16x16 *) (f+80),e5); |
| 943 |
_mm256_storeu_si256((int16x16 *) (f+144),e9); |
| 944 |
_mm256_storeu_si256((int16x16 *) (f+208),e13); |
| 945 |
_mm256_storeu_si256((int16x16 *) (f+272),e17); |
| 946 |
_mm256_storeu_si256((int16x16 *) (f+336),e21); |
| 947 |
_mm256_storeu_si256((int16x16 *) (f+400),e25); |
| 948 |
_mm256_storeu_si256((int16x16 *) (f+464),e29); |
| 949 |
f += 512; |
| 950 |
} |
| 951 |
f -= 512*reps; |
| 952 |
// startbatch 512 |
| 953 |
for (long long r = 0;r < reps;++r) { |
| 954 |
// vector_butterfly 32 288 1 0 |
| 955 |
int16x16 a2 = _mm256_loadu_si256((int16x16 *) (f+32)); |
| 956 |
int16x16 a18 = _mm256_loadu_si256((int16x16 *) (f+288)); |
| 957 |
int16x16 b2 = add_x16(a2,a18); |
| 958 |
int16x16 b18 = sub_x16(a2,a18); |
| 959 |
// vector_butterfly 160 416 1 0 |
| 960 |
int16x16 a10 = _mm256_loadu_si256((int16x16 *) (f+160)); |
| 961 |
int16x16 a26 = _mm256_loadu_si256((int16x16 *) (f+416)); |
| 962 |
int16x16 b10 = add_x16(a10,a26); |
| 963 |
int16x16 b26 = sub_x16(a10,a26); |
| 964 |
// vector_butterfly 96 352 1 0 |
| 965 |
int16x16 a6 = _mm256_loadu_si256((int16x16 *) (f+96)); |
| 966 |
int16x16 a22 = _mm256_loadu_si256((int16x16 *) (f+352)); |
| 967 |
int16x16 b6 = add_x16(a6,a22); |
| 968 |
int16x16 b22 = sub_x16(a6,a22); |
| 969 |
// vector_butterfly 224 480 1 0 |
| 970 |
int16x16 a14 = _mm256_loadu_si256((int16x16 *) (f+224)); |
| 971 |
int16x16 a30 = _mm256_loadu_si256((int16x16 *) (f+480)); |
| 972 |
int16x16 b14 = add_x16(a14,a30); |
| 973 |
int16x16 b30 = sub_x16(a14,a30); |
| 974 |
// vector_reduce_ifreverse 32 |
| 975 |
// vector_reduce_ifreverse 288 |
| 976 |
// vector_butterfly 32 160 1 0 |
| 977 |
int16x16 c2 = add_x16(b2,b10); |
| 978 |
int16x16 c10 = sub_x16(b2,b10); |
| 979 |
// vector_butterfly 96 224 1 0 |
| 980 |
int16x16 c6 = add_x16(b6,b14); |
| 981 |
int16x16 c14 = sub_x16(b6,b14); |
| 982 |
// vector_butterfly 288 416 4 1 |
| 983 |
b26 = mulmod_scaled_x16(b26,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 984 |
int16x16 c18 = add_x16(b18,b26); |
| 985 |
int16x16 c26 = sub_x16(b18,b26); |
| 986 |
// vector_butterfly 352 480 4 1 |
| 987 |
b30 = mulmod_scaled_x16(b30,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 988 |
int16x16 c22 = add_x16(b22,b30); |
| 989 |
int16x16 c30 = sub_x16(b22,b30); |
| 990 |
// vector_reduce_ifforward 96 |
| 991 |
c6 = reduce_x16(c6,qdata); |
| 992 |
// vector_butterfly 32 96 1 0 |
| 993 |
int16x16 d2 = add_x16(c2,c6); |
| 994 |
int16x16 d6 = sub_x16(c2,c6); |
| 995 |
// vector_butterfly 160 224 4 1 |
| 996 |
c14 = mulmod_scaled_x16(c14,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 997 |
int16x16 d10 = add_x16(c10,c14); |
| 998 |
int16x16 d14 = sub_x16(c10,c14); |
| 999 |
// vector_butterfly 288 352 8 1 |
| 1000 |
c22 = mulmod_scaled_x16(c22,scaledzeta_x16_8_1,qinvscaledzeta_x16_8_1,qdata); |
| 1001 |
int16x16 d18 = add_x16(c18,c22); |
| 1002 |
int16x16 d22 = sub_x16(c18,c22); |
| 1003 |
// vector_butterfly 416 480 8 7 |
| 1004 |
c30 = mulmod_scaled_x16(c30,scaledzeta_x16_8_7,qinvscaledzeta_x16_8_7,qdata); |
| 1005 |
int16x16 d26 = add_x16(c26,c30); |
| 1006 |
int16x16 d30 = sub_x16(c26,c30); |
| 1007 |
// vector_reduce 32 |
| 1008 |
d2 = reduce_x16(d2,qdata); |
| 1009 |
// vector_twist 96 128 1 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1010 |
d6 = mulmod_scaled_x16(d6,precomp_128_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_128_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1011 |
// vector_twist 160 256 1 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1012 |
d10 = mulmod_scaled_x16(d10,precomp_256_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_256_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1013 |
// vector_twist 224 256 255 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1014 |
d14 = mulmod_scaled_x16(d14,precomp_256_255_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_256_255_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1015 |
// vector_twist 288 512 1 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1016 |
d18 = mulmod_scaled_x16(d18,precomp_512_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_512_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1017 |
// vector_twist 352 512 5 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1018 |
d22 = mulmod_scaled_x16(d22,precomp_512_5_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_512_5_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1019 |
// vector_twist 416 512 511 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1020 |
d26 = mulmod_scaled_x16(d26,precomp_512_511_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_512_511_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1021 |
// vector_twist 480 512 507 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1022 |
d30 = mulmod_scaled_x16(d30,precomp_512_507_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_512_507_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1023 |
// vector_permute 32 96 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1024 |
int16x16 e2 = _mm256_permute2x128_si256_lo(d2,d6); |
| 1025 |
int16x16 e6 = _mm256_permute2x128_si256_hi(d2,d6); |
| 1026 |
// vector_permute 160 224 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1027 |
int16x16 e10 = _mm256_permute2x128_si256_lo(d10,d14); |
| 1028 |
int16x16 e14 = _mm256_permute2x128_si256_hi(d10,d14); |
| 1029 |
// vector_permute 288 352 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1030 |
int16x16 e18 = _mm256_permute2x128_si256_lo(d18,d22); |
| 1031 |
int16x16 e22 = _mm256_permute2x128_si256_hi(d18,d22); |
| 1032 |
// vector_permute 416 480 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1033 |
int16x16 e26 = _mm256_permute2x128_si256_lo(d26,d30); |
| 1034 |
int16x16 e30 = _mm256_permute2x128_si256_hi(d26,d30); |
| 1035 |
// stopbatch 512 |
| 1036 |
_mm256_storeu_si256((int16x16 *) (f+32),e2); |
| 1037 |
_mm256_storeu_si256((int16x16 *) (f+96),e6); |
| 1038 |
_mm256_storeu_si256((int16x16 *) (f+160),e10); |
| 1039 |
_mm256_storeu_si256((int16x16 *) (f+224),e14); |
| 1040 |
_mm256_storeu_si256((int16x16 *) (f+288),e18); |
| 1041 |
_mm256_storeu_si256((int16x16 *) (f+352),e22); |
| 1042 |
_mm256_storeu_si256((int16x16 *) (f+416),e26); |
| 1043 |
_mm256_storeu_si256((int16x16 *) (f+480),e30); |
| 1044 |
f += 512; |
| 1045 |
} |
| 1046 |
f -= 512*reps; |
| 1047 |
// startbatch 512 |
| 1048 |
for (long long r = 0;r < reps;++r) { |
| 1049 |
// vector_butterfly 48 304 1 0 |
| 1050 |
int16x16 a3 = _mm256_loadu_si256((int16x16 *) (f+48)); |
| 1051 |
int16x16 a19 = _mm256_loadu_si256((int16x16 *) (f+304)); |
| 1052 |
int16x16 b3 = add_x16(a3,a19); |
| 1053 |
int16x16 b19 = sub_x16(a3,a19); |
| 1054 |
// vector_butterfly 176 432 1 0 |
| 1055 |
int16x16 a11 = _mm256_loadu_si256((int16x16 *) (f+176)); |
| 1056 |
int16x16 a27 = _mm256_loadu_si256((int16x16 *) (f+432)); |
| 1057 |
int16x16 b11 = add_x16(a11,a27); |
| 1058 |
int16x16 b27 = sub_x16(a11,a27); |
| 1059 |
// vector_butterfly 112 368 1 0 |
| 1060 |
int16x16 a7 = _mm256_loadu_si256((int16x16 *) (f+112)); |
| 1061 |
int16x16 a23 = _mm256_loadu_si256((int16x16 *) (f+368)); |
| 1062 |
int16x16 b7 = add_x16(a7,a23); |
| 1063 |
int16x16 b23 = sub_x16(a7,a23); |
| 1064 |
// vector_butterfly 240 496 1 0 |
| 1065 |
int16x16 a15 = _mm256_loadu_si256((int16x16 *) (f+240)); |
| 1066 |
int16x16 a31 = _mm256_loadu_si256((int16x16 *) (f+496)); |
| 1067 |
int16x16 b15 = add_x16(a15,a31); |
| 1068 |
int16x16 b31 = sub_x16(a15,a31); |
| 1069 |
// vector_reduce_ifreverse 48 |
| 1070 |
// vector_reduce_ifreverse 304 |
| 1071 |
// vector_butterfly 48 176 1 0 |
| 1072 |
int16x16 c3 = add_x16(b3,b11); |
| 1073 |
int16x16 c11 = sub_x16(b3,b11); |
| 1074 |
// vector_butterfly 112 240 1 0 |
| 1075 |
int16x16 c7 = add_x16(b7,b15); |
| 1076 |
int16x16 c15 = sub_x16(b7,b15); |
| 1077 |
// vector_butterfly 304 432 4 1 |
| 1078 |
b27 = mulmod_scaled_x16(b27,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 1079 |
int16x16 c19 = add_x16(b19,b27); |
| 1080 |
int16x16 c27 = sub_x16(b19,b27); |
| 1081 |
// vector_butterfly 368 496 4 1 |
| 1082 |
b31 = mulmod_scaled_x16(b31,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 1083 |
int16x16 c23 = add_x16(b23,b31); |
| 1084 |
int16x16 c31 = sub_x16(b23,b31); |
| 1085 |
// vector_reduce_ifforward 112 |
| 1086 |
c7 = reduce_x16(c7,qdata); |
| 1087 |
// vector_butterfly 48 112 1 0 |
| 1088 |
int16x16 d3 = add_x16(c3,c7); |
| 1089 |
int16x16 d7 = sub_x16(c3,c7); |
| 1090 |
// vector_butterfly 176 240 4 1 |
| 1091 |
c15 = mulmod_scaled_x16(c15,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 1092 |
int16x16 d11 = add_x16(c11,c15); |
| 1093 |
int16x16 d15 = sub_x16(c11,c15); |
| 1094 |
// vector_butterfly 304 368 8 1 |
| 1095 |
c23 = mulmod_scaled_x16(c23,scaledzeta_x16_8_1,qinvscaledzeta_x16_8_1,qdata); |
| 1096 |
int16x16 d19 = add_x16(c19,c23); |
| 1097 |
int16x16 d23 = sub_x16(c19,c23); |
| 1098 |
// vector_butterfly 432 496 8 7 |
| 1099 |
c31 = mulmod_scaled_x16(c31,scaledzeta_x16_8_7,qinvscaledzeta_x16_8_7,qdata); |
| 1100 |
int16x16 d27 = add_x16(c27,c31); |
| 1101 |
int16x16 d31 = sub_x16(c27,c31); |
| 1102 |
// vector_reduce 48 |
| 1103 |
d3 = reduce_x16(d3,qdata); |
| 1104 |
// vector_twist 112 128 1 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1105 |
d7 = mulmod_scaled_x16(d7,precomp_128_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_128_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1106 |
// vector_twist 176 256 1 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1107 |
d11 = mulmod_scaled_x16(d11,precomp_256_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_256_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1108 |
// vector_twist 240 256 255 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1109 |
d15 = mulmod_scaled_x16(d15,precomp_256_255_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_256_255_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1110 |
// vector_twist 304 512 1 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1111 |
d19 = mulmod_scaled_x16(d19,precomp_512_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_512_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1112 |
// vector_twist 368 512 5 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1113 |
d23 = mulmod_scaled_x16(d23,precomp_512_5_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_512_5_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1114 |
// vector_twist 432 512 511 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1115 |
d27 = mulmod_scaled_x16(d27,precomp_512_511_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_512_511_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1116 |
// vector_twist 496 512 507 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1117 |
d31 = mulmod_scaled_x16(d31,precomp_512_507_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_512_507_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1118 |
// vector_permute 48 112 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1119 |
int16x16 e3 = _mm256_permute2x128_si256_lo(d3,d7); |
| 1120 |
int16x16 e7 = _mm256_permute2x128_si256_hi(d3,d7); |
| 1121 |
// vector_permute 176 240 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1122 |
int16x16 e11 = _mm256_permute2x128_si256_lo(d11,d15); |
| 1123 |
int16x16 e15 = _mm256_permute2x128_si256_hi(d11,d15); |
| 1124 |
// vector_permute 304 368 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1125 |
int16x16 e19 = _mm256_permute2x128_si256_lo(d19,d23); |
| 1126 |
int16x16 e23 = _mm256_permute2x128_si256_hi(d19,d23); |
| 1127 |
// vector_permute 432 496 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1128 |
int16x16 e27 = _mm256_permute2x128_si256_lo(d27,d31); |
| 1129 |
int16x16 e31 = _mm256_permute2x128_si256_hi(d27,d31); |
| 1130 |
// stopbatch 512 |
| 1131 |
_mm256_storeu_si256((int16x16 *) (f+48),e3); |
| 1132 |
_mm256_storeu_si256((int16x16 *) (f+112),e7); |
| 1133 |
_mm256_storeu_si256((int16x16 *) (f+176),e11); |
| 1134 |
_mm256_storeu_si256((int16x16 *) (f+240),e15); |
| 1135 |
_mm256_storeu_si256((int16x16 *) (f+304),e19); |
| 1136 |
_mm256_storeu_si256((int16x16 *) (f+368),e23); |
| 1137 |
_mm256_storeu_si256((int16x16 *) (f+432),e27); |
| 1138 |
_mm256_storeu_si256((int16x16 *) (f+496),e31); |
| 1139 |
f += 512; |
| 1140 |
} |
| 1141 |
f -= 512*reps; |
| 1142 |
// doublereps |
| 1143 |
reps *= 2; |
| 1144 |
// doublereps |
| 1145 |
reps *= 2; |
| 1146 |
// startbatch 128 |
| 1147 |
for (long long r = 0;r < reps;++r) { |
| 1148 |
// vector_butterfly 0 32 1 0 |
| 1149 |
int16x16 a0 = _mm256_loadu_si256((int16x16 *) (f+0)); |
| 1150 |
int16x16 a2 = _mm256_loadu_si256((int16x16 *) (f+32)); |
| 1151 |
int16x16 b0 = add_x16(a0,a2); |
| 1152 |
int16x16 b2 = sub_x16(a0,a2); |
| 1153 |
// vector_butterfly 64 96 1 0 |
| 1154 |
int16x16 a4 = _mm256_loadu_si256((int16x16 *) (f+64)); |
| 1155 |
int16x16 a6 = _mm256_loadu_si256((int16x16 *) (f+96)); |
| 1156 |
int16x16 b4 = add_x16(a4,a6); |
| 1157 |
int16x16 b6 = sub_x16(a4,a6); |
| 1158 |
// vector_butterfly 16 48 1 0 |
| 1159 |
int16x16 a1 = _mm256_loadu_si256((int16x16 *) (f+16)); |
| 1160 |
int16x16 a3 = _mm256_loadu_si256((int16x16 *) (f+48)); |
| 1161 |
int16x16 b1 = add_x16(a1,a3); |
| 1162 |
int16x16 b3 = sub_x16(a1,a3); |
| 1163 |
// vector_butterfly 80 112 1 0 |
| 1164 |
int16x16 a5 = _mm256_loadu_si256((int16x16 *) (f+80)); |
| 1165 |
int16x16 a7 = _mm256_loadu_si256((int16x16 *) (f+112)); |
| 1166 |
int16x16 b5 = add_x16(a5,a7); |
| 1167 |
int16x16 b7 = sub_x16(a5,a7); |
| 1168 |
// vector_butterfly 0 16 1 0 |
| 1169 |
int16x16 c0 = add_x16(b0,b1); |
| 1170 |
int16x16 c1 = sub_x16(b0,b1); |
| 1171 |
// vector_butterfly 64 80 1 0 |
| 1172 |
int16x16 c4 = add_x16(b4,b5); |
| 1173 |
int16x16 c5 = sub_x16(b4,b5); |
| 1174 |
// vector_butterfly 32 48 4 1 |
| 1175 |
b3 = mulmod_scaled_x16(b3,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 1176 |
int16x16 c2 = add_x16(b2,b3); |
| 1177 |
int16x16 c3 = sub_x16(b2,b3); |
| 1178 |
// vector_butterfly 96 112 4 1 |
| 1179 |
b7 = mulmod_scaled_x16(b7,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 1180 |
int16x16 c6 = add_x16(b6,b7); |
| 1181 |
int16x16 c7 = sub_x16(b6,b7); |
| 1182 |
// vector_reduce 0 |
| 1183 |
c0 = reduce_x16(c0,qdata); |
| 1184 |
// vector_reduce 64 |
| 1185 |
c4 = reduce_x16(c4,qdata); |
| 1186 |
// vector_twist 16 32 1 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 |
| 1187 |
c1 = mulmod_scaled_x16(c1,precomp_32_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qinvprecomp_32_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qdata); |
| 1188 |
// vector_twist 80 32 1 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15 |
| 1189 |
c5 = mulmod_scaled_x16(c5,precomp_32_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qinvprecomp_32_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qdata); |
| 1190 |
// vector_twist 32 64 1 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 |
| 1191 |
c2 = mulmod_scaled_x16(c2,precomp_64_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qinvprecomp_64_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qdata); |
| 1192 |
// vector_twist 96 64 1 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15 |
| 1193 |
c6 = mulmod_scaled_x16(c6,precomp_64_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qinvprecomp_64_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qdata); |
| 1194 |
// vector_twist 48 64 63 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 |
| 1195 |
c3 = mulmod_scaled_x16(c3,precomp_64_63_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qinvprecomp_64_63_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qdata); |
| 1196 |
// vector_twist 112 64 63 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15 |
| 1197 |
c7 = mulmod_scaled_x16(c7,precomp_64_63_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qinvprecomp_64_63_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qdata); |
| 1198 |
// vector_permute 0 32 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16 |
| 1199 |
int16x16 d0 = _mm256_unpacklo_epi16(c0,c2); |
| 1200 |
int16x16 d2 = _mm256_unpackhi_epi16(c0,c2); |
| 1201 |
// vector_permute 16 48 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16 |
| 1202 |
int16x16 d1 = _mm256_unpacklo_epi16(c1,c3); |
| 1203 |
int16x16 d3 = _mm256_unpackhi_epi16(c1,c3); |
| 1204 |
// vector_permute 64 96 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16 |
| 1205 |
int16x16 d4 = _mm256_unpacklo_epi16(c4,c6); |
| 1206 |
int16x16 d6 = _mm256_unpackhi_epi16(c4,c6); |
| 1207 |
// vector_permute 80 112 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16 |
| 1208 |
int16x16 d5 = _mm256_unpacklo_epi16(c5,c7); |
| 1209 |
int16x16 d7 = _mm256_unpackhi_epi16(c5,c7); |
| 1210 |
// vector_butterfly 0 64 1 0 |
| 1211 |
int16x16 e0 = add_x16(d0,d4); |
| 1212 |
int16x16 e4 = sub_x16(d0,d4); |
| 1213 |
// vector_butterfly 32 96 1 0 |
| 1214 |
int16x16 e2 = add_x16(d2,d6); |
| 1215 |
int16x16 e6 = sub_x16(d2,d6); |
| 1216 |
// vector_butterfly 16 80 1 0 |
| 1217 |
int16x16 e1 = add_x16(d1,d5); |
| 1218 |
int16x16 e5 = sub_x16(d1,d5); |
| 1219 |
// vector_butterfly 48 112 1 0 |
| 1220 |
int16x16 e3 = add_x16(d3,d7); |
| 1221 |
int16x16 e7 = sub_x16(d3,d7); |
| 1222 |
// vector_permute 0 16 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32 |
| 1223 |
int16x16 f0 = _mm256_unpacklo_epi32(e0,e1); |
| 1224 |
int16x16 f1 = _mm256_unpackhi_epi32(e0,e1); |
| 1225 |
// vector_permute 32 48 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32 |
| 1226 |
int16x16 f2 = _mm256_unpacklo_epi32(e2,e3); |
| 1227 |
int16x16 f3 = _mm256_unpackhi_epi32(e2,e3); |
| 1228 |
// vector_permute 64 80 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32 |
| 1229 |
int16x16 f4 = _mm256_unpacklo_epi32(e4,e5); |
| 1230 |
int16x16 f5 = _mm256_unpackhi_epi32(e4,e5); |
| 1231 |
// vector_permute 96 112 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32 |
| 1232 |
int16x16 f6 = _mm256_unpacklo_epi32(e6,e7); |
| 1233 |
int16x16 f7 = _mm256_unpackhi_epi32(e6,e7); |
| 1234 |
// stopbatch 128 |
| 1235 |
_mm256_storeu_si256((int16x16 *) (f+0),f0); |
| 1236 |
_mm256_storeu_si256((int16x16 *) (f+16),f1); |
| 1237 |
_mm256_storeu_si256((int16x16 *) (f+32),f2); |
| 1238 |
_mm256_storeu_si256((int16x16 *) (f+48),f3); |
| 1239 |
_mm256_storeu_si256((int16x16 *) (f+64),f4); |
| 1240 |
_mm256_storeu_si256((int16x16 *) (f+80),f5); |
| 1241 |
_mm256_storeu_si256((int16x16 *) (f+96),f6); |
| 1242 |
_mm256_storeu_si256((int16x16 *) (f+112),f7); |
| 1243 |
f += 128; |
| 1244 |
} |
| 1245 |
f -= 128*reps; |
| 1246 |
// startbatch 128 |
| 1247 |
for (long long r = 0;r < reps;++r) { |
| 1248 |
// vector_butterfly 0 32 1 0 |
| 1249 |
int16x16 a0 = _mm256_loadu_si256((int16x16 *) (f+0)); |
| 1250 |
int16x16 a2 = _mm256_loadu_si256((int16x16 *) (f+32)); |
| 1251 |
int16x16 b0 = add_x16(a0,a2); |
| 1252 |
int16x16 b2 = sub_x16(a0,a2); |
| 1253 |
// vector_butterfly 16 48 1 0 |
| 1254 |
int16x16 a1 = _mm256_loadu_si256((int16x16 *) (f+16)); |
| 1255 |
int16x16 a3 = _mm256_loadu_si256((int16x16 *) (f+48)); |
| 1256 |
int16x16 b1 = add_x16(a1,a3); |
| 1257 |
int16x16 b3 = sub_x16(a1,a3); |
| 1258 |
// vector_butterfly 64 96 4 1 |
| 1259 |
int16x16 a4 = _mm256_loadu_si256((int16x16 *) (f+64)); |
| 1260 |
int16x16 a6 = _mm256_loadu_si256((int16x16 *) (f+96)); |
| 1261 |
a6 = mulmod_scaled_x16(a6,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 1262 |
int16x16 b4 = add_x16(a4,a6); |
| 1263 |
int16x16 b6 = sub_x16(a4,a6); |
| 1264 |
// vector_butterfly 80 112 4 1 |
| 1265 |
int16x16 a5 = _mm256_loadu_si256((int16x16 *) (f+80)); |
| 1266 |
int16x16 a7 = _mm256_loadu_si256((int16x16 *) (f+112)); |
| 1267 |
a7 = mulmod_scaled_x16(a7,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 1268 |
int16x16 b5 = add_x16(a5,a7); |
| 1269 |
int16x16 b7 = sub_x16(a5,a7); |
| 1270 |
// vector_reduce 0 |
| 1271 |
b0 = reduce_x16(b0,qdata); |
| 1272 |
// vector_reduce 16 |
| 1273 |
b1 = reduce_x16(b1,qdata); |
| 1274 |
// vector_twist 32 8 1 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1 |
| 1275 |
b2 = mulmod_scaled_x16(b2,precomp_8_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qinvprecomp_8_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qdata); |
| 1276 |
// vector_twist 48 8 1 2 2 2 2 3 3 3 3 2 2 2 2 3 3 3 3 |
| 1277 |
b3 = mulmod_scaled_x16(b3,precomp_8_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qinvprecomp_8_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qdata); |
| 1278 |
// vector_twist 64 16 1 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1 |
| 1279 |
b4 = mulmod_scaled_x16(b4,precomp_16_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qinvprecomp_16_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qdata); |
| 1280 |
// vector_twist 80 16 1 2 2 2 2 3 3 3 3 2 2 2 2 3 3 3 3 |
| 1281 |
b5 = mulmod_scaled_x16(b5,precomp_16_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qinvprecomp_16_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qdata); |
| 1282 |
// vector_twist 96 16 15 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1 |
| 1283 |
b6 = mulmod_scaled_x16(b6,precomp_16_15_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qinvprecomp_16_15_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qdata); |
| 1284 |
// vector_twist 112 16 15 2 2 2 2 3 3 3 3 2 2 2 2 3 3 3 3 |
| 1285 |
b7 = mulmod_scaled_x16(b7,precomp_16_15_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qinvprecomp_16_15_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qdata); |
| 1286 |
// vector_permute 0 64 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64 |
| 1287 |
int16x16 c0 = _mm256_unpacklo_epi64(b0,b4); |
| 1288 |
int16x16 c4 = _mm256_unpackhi_epi64(b0,b4); |
| 1289 |
// vector_permute 16 80 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64 |
| 1290 |
int16x16 c1 = _mm256_unpacklo_epi64(b1,b5); |
| 1291 |
int16x16 c5 = _mm256_unpackhi_epi64(b1,b5); |
| 1292 |
// vector_permute 32 96 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64 |
| 1293 |
int16x16 c2 = _mm256_unpacklo_epi64(b2,b6); |
| 1294 |
int16x16 c6 = _mm256_unpackhi_epi64(b2,b6); |
| 1295 |
// vector_permute 48 112 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64 |
| 1296 |
int16x16 c3 = _mm256_unpacklo_epi64(b3,b7); |
| 1297 |
int16x16 c7 = _mm256_unpackhi_epi64(b3,b7); |
| 1298 |
// vector_butterfly 0 16 1 0 |
| 1299 |
int16x16 d0 = add_x16(c0,c1); |
| 1300 |
int16x16 d1 = sub_x16(c0,c1); |
| 1301 |
// vector_butterfly 64 80 1 0 |
| 1302 |
int16x16 d4 = add_x16(c4,c5); |
| 1303 |
int16x16 d5 = sub_x16(c4,c5); |
| 1304 |
// vector_butterfly 32 48 1 0 |
| 1305 |
int16x16 d2 = add_x16(c2,c3); |
| 1306 |
int16x16 d3 = sub_x16(c2,c3); |
| 1307 |
// vector_butterfly 96 112 1 0 |
| 1308 |
int16x16 d6 = add_x16(c6,c7); |
| 1309 |
int16x16 d7 = sub_x16(c6,c7); |
| 1310 |
// vector_butterfly 0 64 1 0 |
| 1311 |
int16x16 e0 = add_x16(d0,d4); |
| 1312 |
int16x16 e4 = sub_x16(d0,d4); |
| 1313 |
// vector_butterfly 32 96 1 0 |
| 1314 |
int16x16 e2 = add_x16(d2,d6); |
| 1315 |
int16x16 e6 = sub_x16(d2,d6); |
| 1316 |
// vector_butterfly 16 80 4 1 |
| 1317 |
d5 = mulmod_scaled_x16(d5,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 1318 |
int16x16 e1 = add_x16(d1,d5); |
| 1319 |
int16x16 e5 = sub_x16(d1,d5); |
| 1320 |
// vector_butterfly 48 112 4 1 |
| 1321 |
d7 = mulmod_scaled_x16(d7,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata); |
| 1322 |
int16x16 e3 = add_x16(d3,d7); |
| 1323 |
int16x16 e7 = sub_x16(d3,d7); |
| 1324 |
// stopbatch 128 |
| 1325 |
_mm256_storeu_si256((int16x16 *) (f+0),e0); |
| 1326 |
_mm256_storeu_si256((int16x16 *) (f+16),e1); |
| 1327 |
_mm256_storeu_si256((int16x16 *) (f+32),e2); |
| 1328 |
_mm256_storeu_si256((int16x16 *) (f+48),e3); |
| 1329 |
_mm256_storeu_si256((int16x16 *) (f+64),e4); |
| 1330 |
_mm256_storeu_si256((int16x16 *) (f+80),e5); |
| 1331 |
_mm256_storeu_si256((int16x16 *) (f+96),e6); |
| 1332 |
_mm256_storeu_si256((int16x16 *) (f+112),e7); |
| 1333 |
f += 128; |
| 1334 |
} |
| 1335 |
f -= 128*reps; |
| 1336 |
// stopntt 512 |
| 1337 |
} |
| 1338 |
|
| 1339 |
void ntt512_7681(int16 *f,int reps) |
| 1340 |
{ |
| 1341 |
ntt512(f,reps,qdata_7681); |
| 1342 |
} |
| 1343 |
|
| 1344 |
// inv stopntt 512 |
| 1345 |
|
| 1346 |
static void invntt512(int16 *f,int reps,const int16 *qdata) |
| 1347 |
{ |
| 1348 |
reps *= 4; |
| 1349 |
// inv stopbatch 128 |
| 1350 |
for (long long r = 0;r < reps;++r) { |
| 1351 |
// inv vector_butterfly 48 112 4 1 |
| 1352 |
int16x16 a3 = _mm256_loadu_si256((int16x16 *) (f+48)); |
| 1353 |
int16x16 a7 = _mm256_loadu_si256((int16x16 *) (f+112)); |
| 1354 |
int16x16 b3 = add_x16(a3,a7); |
| 1355 |
int16x16 b7 = sub_x16(a3,a7); |
| 1356 |
b7 = mulmod_scaled_x16(b7,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1357 |
// inv vector_butterfly 16 80 4 1 |
| 1358 |
int16x16 a1 = _mm256_loadu_si256((int16x16 *) (f+16)); |
| 1359 |
int16x16 a5 = _mm256_loadu_si256((int16x16 *) (f+80)); |
| 1360 |
int16x16 b1 = add_x16(a1,a5); |
| 1361 |
int16x16 b5 = sub_x16(a1,a5); |
| 1362 |
b5 = mulmod_scaled_x16(b5,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1363 |
// inv vector_butterfly 32 96 1 0 |
| 1364 |
int16x16 a2 = _mm256_loadu_si256((int16x16 *) (f+32)); |
| 1365 |
int16x16 a6 = _mm256_loadu_si256((int16x16 *) (f+96)); |
| 1366 |
int16x16 b2 = add_x16(a2,a6); |
| 1367 |
int16x16 b6 = sub_x16(a2,a6); |
| 1368 |
// inv vector_butterfly 0 64 1 0 |
| 1369 |
int16x16 a0 = _mm256_loadu_si256((int16x16 *) (f+0)); |
| 1370 |
int16x16 a4 = _mm256_loadu_si256((int16x16 *) (f+64)); |
| 1371 |
int16x16 b0 = add_x16(a0,a4); |
| 1372 |
int16x16 b4 = sub_x16(a0,a4); |
| 1373 |
// inv vector_butterfly 96 112 1 0 |
| 1374 |
int16x16 c6 = add_x16(b6,b7); |
| 1375 |
int16x16 c7 = sub_x16(b6,b7); |
| 1376 |
// inv vector_butterfly 32 48 1 0 |
| 1377 |
int16x16 c2 = add_x16(b2,b3); |
| 1378 |
int16x16 c3 = sub_x16(b2,b3); |
| 1379 |
// inv vector_butterfly 64 80 1 0 |
| 1380 |
int16x16 c4 = add_x16(b4,b5); |
| 1381 |
int16x16 c5 = sub_x16(b4,b5); |
| 1382 |
// inv vector_butterfly 0 16 1 0 |
| 1383 |
int16x16 c0 = add_x16(b0,b1); |
| 1384 |
int16x16 c1 = sub_x16(b0,b1); |
| 1385 |
// inv vector_permute 48 112 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64 |
| 1386 |
int16x16 d3 = _mm256_unpacklo_epi64(c3,c7); |
| 1387 |
int16x16 d7 = _mm256_unpackhi_epi64(c3,c7); |
| 1388 |
// inv vector_permute 32 96 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64 |
| 1389 |
int16x16 d2 = _mm256_unpacklo_epi64(c2,c6); |
| 1390 |
int16x16 d6 = _mm256_unpackhi_epi64(c2,c6); |
| 1391 |
// inv vector_permute 16 80 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64 |
| 1392 |
int16x16 d1 = _mm256_unpacklo_epi64(c1,c5); |
| 1393 |
int16x16 d5 = _mm256_unpackhi_epi64(c1,c5); |
| 1394 |
// inv vector_permute 0 64 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64 |
| 1395 |
int16x16 d0 = _mm256_unpacklo_epi64(c0,c4); |
| 1396 |
int16x16 d4 = _mm256_unpackhi_epi64(c0,c4); |
| 1397 |
// inv vector_twist 112 16 15 2 2 2 2 3 3 3 3 2 2 2 2 3 3 3 3 |
| 1398 |
d7 = mulmod_scaled_x16(d7,precomp_16_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qinvprecomp_16_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qdata); |
| 1399 |
// inv vector_twist 96 16 15 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1 |
| 1400 |
d6 = mulmod_scaled_x16(d6,precomp_16_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qinvprecomp_16_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qdata); |
| 1401 |
// inv vector_twist 80 16 1 2 2 2 2 3 3 3 3 2 2 2 2 3 3 3 3 |
| 1402 |
d5 = mulmod_scaled_x16(d5,precomp_16_15_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qinvprecomp_16_15_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qdata); |
| 1403 |
// inv vector_twist 64 16 1 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1 |
| 1404 |
d4 = mulmod_scaled_x16(d4,precomp_16_15_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qinvprecomp_16_15_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qdata); |
| 1405 |
// inv vector_twist 48 8 1 2 2 2 2 3 3 3 3 2 2 2 2 3 3 3 3 |
| 1406 |
d3 = mulmod_scaled_x16(d3,precomp_8_7_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qinvprecomp_8_7_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qdata); |
| 1407 |
// inv vector_twist 32 8 1 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1 |
| 1408 |
d2 = mulmod_scaled_x16(d2,precomp_8_7_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qinvprecomp_8_7_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qdata); |
| 1409 |
// inv vector_reduce 16 |
| 1410 |
d1 = reduce_x16(d1,qdata); |
| 1411 |
// inv vector_reduce 0 |
| 1412 |
d0 = reduce_x16(d0,qdata); |
| 1413 |
// inv vector_butterfly 80 112 4 1 |
| 1414 |
int16x16 e5 = add_x16(d5,d7); |
| 1415 |
int16x16 e7 = sub_x16(d5,d7); |
| 1416 |
e7 = mulmod_scaled_x16(e7,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1417 |
// inv vector_butterfly 64 96 4 1 |
| 1418 |
int16x16 e4 = add_x16(d4,d6); |
| 1419 |
int16x16 e6 = sub_x16(d4,d6); |
| 1420 |
e6 = mulmod_scaled_x16(e6,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1421 |
// inv vector_butterfly 16 48 1 0 |
| 1422 |
int16x16 e1 = add_x16(d1,d3); |
| 1423 |
int16x16 e3 = sub_x16(d1,d3); |
| 1424 |
// inv vector_butterfly 0 32 1 0 |
| 1425 |
int16x16 e0 = add_x16(d0,d2); |
| 1426 |
int16x16 e2 = sub_x16(d0,d2); |
| 1427 |
// inv startbatch 128 |
| 1428 |
_mm256_storeu_si256((int16x16 *) (f+0),e0); |
| 1429 |
_mm256_storeu_si256((int16x16 *) (f+16),e1); |
| 1430 |
_mm256_storeu_si256((int16x16 *) (f+32),e2); |
| 1431 |
_mm256_storeu_si256((int16x16 *) (f+48),e3); |
| 1432 |
_mm256_storeu_si256((int16x16 *) (f+64),e4); |
| 1433 |
_mm256_storeu_si256((int16x16 *) (f+80),e5); |
| 1434 |
_mm256_storeu_si256((int16x16 *) (f+96),e6); |
| 1435 |
_mm256_storeu_si256((int16x16 *) (f+112),e7); |
| 1436 |
f += 128; |
| 1437 |
} |
| 1438 |
f -= 128*reps; |
| 1439 |
// inv stopbatch 128 |
| 1440 |
for (long long r = 0;r < reps;++r) { |
| 1441 |
// inv vector_permute 96 112 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32 |
| 1442 |
int16x16 a6 = _mm256_loadu_si256((int16x16 *) (f+96)); |
| 1443 |
int16x16 a7 = _mm256_loadu_si256((int16x16 *) (f+112)); |
| 1444 |
int16x16 b6 = _mm256_unpacklo_epi32(a6,a7); |
| 1445 |
int16x16 b7 = _mm256_unpackhi_epi32(a6,a7); |
| 1446 |
int16x16 c6 = _mm256_unpacklo_epi32(b6,b7); |
| 1447 |
int16x16 c7 = _mm256_unpackhi_epi32(b6,b7); |
| 1448 |
// inv vector_permute 64 80 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32 |
| 1449 |
int16x16 a4 = _mm256_loadu_si256((int16x16 *) (f+64)); |
| 1450 |
int16x16 a5 = _mm256_loadu_si256((int16x16 *) (f+80)); |
| 1451 |
int16x16 b4 = _mm256_unpacklo_epi32(a4,a5); |
| 1452 |
int16x16 b5 = _mm256_unpackhi_epi32(a4,a5); |
| 1453 |
int16x16 c4 = _mm256_unpacklo_epi32(b4,b5); |
| 1454 |
int16x16 c5 = _mm256_unpackhi_epi32(b4,b5); |
| 1455 |
// inv vector_permute 32 48 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32 |
| 1456 |
int16x16 a2 = _mm256_loadu_si256((int16x16 *) (f+32)); |
| 1457 |
int16x16 a3 = _mm256_loadu_si256((int16x16 *) (f+48)); |
| 1458 |
int16x16 b2 = _mm256_unpacklo_epi32(a2,a3); |
| 1459 |
int16x16 b3 = _mm256_unpackhi_epi32(a2,a3); |
| 1460 |
int16x16 c2 = _mm256_unpacklo_epi32(b2,b3); |
| 1461 |
int16x16 c3 = _mm256_unpackhi_epi32(b2,b3); |
| 1462 |
// inv vector_permute 0 16 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32 |
| 1463 |
int16x16 a0 = _mm256_loadu_si256((int16x16 *) (f+0)); |
| 1464 |
int16x16 a1 = _mm256_loadu_si256((int16x16 *) (f+16)); |
| 1465 |
int16x16 b0 = _mm256_unpacklo_epi32(a0,a1); |
| 1466 |
int16x16 b1 = _mm256_unpackhi_epi32(a0,a1); |
| 1467 |
int16x16 c0 = _mm256_unpacklo_epi32(b0,b1); |
| 1468 |
int16x16 c1 = _mm256_unpackhi_epi32(b0,b1); |
| 1469 |
// inv vector_butterfly 48 112 1 0 |
| 1470 |
int16x16 d3 = add_x16(c3,c7); |
| 1471 |
int16x16 d7 = sub_x16(c3,c7); |
| 1472 |
// inv vector_butterfly 16 80 1 0 |
| 1473 |
int16x16 d1 = add_x16(c1,c5); |
| 1474 |
int16x16 d5 = sub_x16(c1,c5); |
| 1475 |
// inv vector_butterfly 32 96 1 0 |
| 1476 |
int16x16 d2 = add_x16(c2,c6); |
| 1477 |
int16x16 d6 = sub_x16(c2,c6); |
| 1478 |
// inv vector_butterfly 0 64 1 0 |
| 1479 |
int16x16 d0 = add_x16(c0,c4); |
| 1480 |
int16x16 d4 = sub_x16(c0,c4); |
| 1481 |
// inv vector_permute 80 112 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16 |
| 1482 |
int16x16 e5 = _mm256_unpacklo_epi16(d5,d7); |
| 1483 |
int16x16 e7 = _mm256_unpackhi_epi16(d5,d7); |
| 1484 |
int16x16 f5 = _mm256_unpacklo_epi16(e5,e7); |
| 1485 |
int16x16 f7 = _mm256_unpackhi_epi16(e5,e7); |
| 1486 |
int16x16 g5 = _mm256_unpacklo_epi16(f5,f7); |
| 1487 |
int16x16 g7 = _mm256_unpackhi_epi16(f5,f7); |
| 1488 |
// inv vector_permute 64 96 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16 |
| 1489 |
int16x16 e4 = _mm256_unpacklo_epi16(d4,d6); |
| 1490 |
int16x16 e6 = _mm256_unpackhi_epi16(d4,d6); |
| 1491 |
int16x16 f4 = _mm256_unpacklo_epi16(e4,e6); |
| 1492 |
int16x16 f6 = _mm256_unpackhi_epi16(e4,e6); |
| 1493 |
int16x16 g4 = _mm256_unpacklo_epi16(f4,f6); |
| 1494 |
int16x16 g6 = _mm256_unpackhi_epi16(f4,f6); |
| 1495 |
// inv vector_permute 16 48 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16 |
| 1496 |
int16x16 e1 = _mm256_unpacklo_epi16(d1,d3); |
| 1497 |
int16x16 e3 = _mm256_unpackhi_epi16(d1,d3); |
| 1498 |
int16x16 f1 = _mm256_unpacklo_epi16(e1,e3); |
| 1499 |
int16x16 f3 = _mm256_unpackhi_epi16(e1,e3); |
| 1500 |
int16x16 g1 = _mm256_unpacklo_epi16(f1,f3); |
| 1501 |
int16x16 g3 = _mm256_unpackhi_epi16(f1,f3); |
| 1502 |
// inv vector_permute 0 32 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16 |
| 1503 |
int16x16 e0 = _mm256_unpacklo_epi16(d0,d2); |
| 1504 |
int16x16 e2 = _mm256_unpackhi_epi16(d0,d2); |
| 1505 |
int16x16 f0 = _mm256_unpacklo_epi16(e0,e2); |
| 1506 |
int16x16 f2 = _mm256_unpackhi_epi16(e0,e2); |
| 1507 |
int16x16 g0 = _mm256_unpacklo_epi16(f0,f2); |
| 1508 |
int16x16 g2 = _mm256_unpackhi_epi16(f0,f2); |
| 1509 |
// inv vector_twist 112 64 63 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15 |
| 1510 |
g7 = mulmod_scaled_x16(g7,precomp_64_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qinvprecomp_64_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qdata); |
| 1511 |
// inv vector_twist 48 64 63 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 |
| 1512 |
g3 = mulmod_scaled_x16(g3,precomp_64_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qinvprecomp_64_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qdata); |
| 1513 |
// inv vector_twist 96 64 1 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15 |
| 1514 |
g6 = mulmod_scaled_x16(g6,precomp_64_63_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qinvprecomp_64_63_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qdata); |
| 1515 |
// inv vector_twist 32 64 1 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 |
| 1516 |
g2 = mulmod_scaled_x16(g2,precomp_64_63_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qinvprecomp_64_63_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qdata); |
| 1517 |
// inv vector_twist 80 32 1 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15 |
| 1518 |
g5 = mulmod_scaled_x16(g5,precomp_32_31_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qinvprecomp_32_31_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qdata); |
| 1519 |
// inv vector_twist 16 32 1 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 |
| 1520 |
g1 = mulmod_scaled_x16(g1,precomp_32_31_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qinvprecomp_32_31_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qdata); |
| 1521 |
// inv vector_reduce 64 |
| 1522 |
g4 = reduce_x16(g4,qdata); |
| 1523 |
// inv vector_reduce 0 |
| 1524 |
g0 = reduce_x16(g0,qdata); |
| 1525 |
// inv vector_butterfly 96 112 4 1 |
| 1526 |
int16x16 h6 = add_x16(g6,g7); |
| 1527 |
int16x16 h7 = sub_x16(g6,g7); |
| 1528 |
h7 = mulmod_scaled_x16(h7,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1529 |
// inv vector_butterfly 32 48 4 1 |
| 1530 |
int16x16 h2 = add_x16(g2,g3); |
| 1531 |
int16x16 h3 = sub_x16(g2,g3); |
| 1532 |
h3 = mulmod_scaled_x16(h3,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1533 |
// inv vector_butterfly 64 80 1 0 |
| 1534 |
int16x16 h4 = add_x16(g4,g5); |
| 1535 |
int16x16 h5 = sub_x16(g4,g5); |
| 1536 |
// inv vector_butterfly 0 16 1 0 |
| 1537 |
int16x16 h0 = add_x16(g0,g1); |
| 1538 |
int16x16 h1 = sub_x16(g0,g1); |
| 1539 |
// inv vector_butterfly 80 112 1 0 |
| 1540 |
int16x16 i5 = add_x16(h5,h7); |
| 1541 |
int16x16 i7 = sub_x16(h5,h7); |
| 1542 |
// inv vector_butterfly 16 48 1 0 |
| 1543 |
int16x16 i1 = add_x16(h1,h3); |
| 1544 |
int16x16 i3 = sub_x16(h1,h3); |
| 1545 |
// inv vector_butterfly 64 96 1 0 |
| 1546 |
int16x16 i4 = add_x16(h4,h6); |
| 1547 |
int16x16 i6 = sub_x16(h4,h6); |
| 1548 |
// inv vector_butterfly 0 32 1 0 |
| 1549 |
int16x16 i0 = add_x16(h0,h2); |
| 1550 |
int16x16 i2 = sub_x16(h0,h2); |
| 1551 |
// inv startbatch 128 |
| 1552 |
_mm256_storeu_si256((int16x16 *) (f+0),i0); |
| 1553 |
_mm256_storeu_si256((int16x16 *) (f+16),i1); |
| 1554 |
_mm256_storeu_si256((int16x16 *) (f+32),i2); |
| 1555 |
_mm256_storeu_si256((int16x16 *) (f+48),i3); |
| 1556 |
_mm256_storeu_si256((int16x16 *) (f+64),i4); |
| 1557 |
_mm256_storeu_si256((int16x16 *) (f+80),i5); |
| 1558 |
_mm256_storeu_si256((int16x16 *) (f+96),i6); |
| 1559 |
_mm256_storeu_si256((int16x16 *) (f+112),i7); |
| 1560 |
f += 128; |
| 1561 |
} |
| 1562 |
f -= 128*reps; |
| 1563 |
// inv doublereps |
| 1564 |
reps /= 2; |
| 1565 |
// inv doublereps |
| 1566 |
reps /= 2; |
| 1567 |
// inv stopbatch 512 |
| 1568 |
for (long long r = 0;r < reps;++r) { |
| 1569 |
// inv vector_permute 432 496 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1570 |
int16x16 a27 = _mm256_loadu_si256((int16x16 *) (f+432)); |
| 1571 |
int16x16 a31 = _mm256_loadu_si256((int16x16 *) (f+496)); |
| 1572 |
int16x16 b27 = _mm256_permute2x128_si256_lo(a27,a31); |
| 1573 |
int16x16 b31 = _mm256_permute2x128_si256_hi(a27,a31); |
| 1574 |
// inv vector_permute 304 368 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1575 |
int16x16 a19 = _mm256_loadu_si256((int16x16 *) (f+304)); |
| 1576 |
int16x16 a23 = _mm256_loadu_si256((int16x16 *) (f+368)); |
| 1577 |
int16x16 b19 = _mm256_permute2x128_si256_lo(a19,a23); |
| 1578 |
int16x16 b23 = _mm256_permute2x128_si256_hi(a19,a23); |
| 1579 |
// inv vector_permute 176 240 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1580 |
int16x16 a11 = _mm256_loadu_si256((int16x16 *) (f+176)); |
| 1581 |
int16x16 a15 = _mm256_loadu_si256((int16x16 *) (f+240)); |
| 1582 |
int16x16 b11 = _mm256_permute2x128_si256_lo(a11,a15); |
| 1583 |
int16x16 b15 = _mm256_permute2x128_si256_hi(a11,a15); |
| 1584 |
// inv vector_permute 48 112 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1585 |
int16x16 a3 = _mm256_loadu_si256((int16x16 *) (f+48)); |
| 1586 |
int16x16 a7 = _mm256_loadu_si256((int16x16 *) (f+112)); |
| 1587 |
int16x16 b3 = _mm256_permute2x128_si256_lo(a3,a7); |
| 1588 |
int16x16 b7 = _mm256_permute2x128_si256_hi(a3,a7); |
| 1589 |
// inv vector_twist 496 512 507 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1590 |
b31 = mulmod_scaled_x16(b31,precomp_512_5_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_512_5_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1591 |
// inv vector_twist 432 512 511 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1592 |
b27 = mulmod_scaled_x16(b27,precomp_512_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_512_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1593 |
// inv vector_twist 368 512 5 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1594 |
b23 = mulmod_scaled_x16(b23,precomp_512_507_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_512_507_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1595 |
// inv vector_twist 304 512 1 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1596 |
b19 = mulmod_scaled_x16(b19,precomp_512_511_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_512_511_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1597 |
// inv vector_twist 240 256 255 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1598 |
b15 = mulmod_scaled_x16(b15,precomp_256_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_256_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1599 |
// inv vector_twist 176 256 1 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1600 |
b11 = mulmod_scaled_x16(b11,precomp_256_255_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_256_255_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1601 |
// inv vector_twist 112 128 1 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 1602 |
b7 = mulmod_scaled_x16(b7,precomp_128_127_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_128_127_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata); |
| 1603 |
// inv vector_reduce 48 |
| 1604 |
b3 = reduce_x16(b3,qdata); |
| 1605 |
// inv vector_butterfly 432 496 8 7 |
| 1606 |
int16x16 c27 = add_x16(b27,b31); |
| 1607 |
int16x16 c31 = sub_x16(b27,b31); |
| 1608 |
c31 = mulmod_scaled_x16(c31,scaledzeta_x16_8_1,qinvscaledzeta_x16_8_1,qdata); |
| 1609 |
// inv vector_butterfly 304 368 8 1 |
| 1610 |
int16x16 c19 = add_x16(b19,b23); |
| 1611 |
int16x16 c23 = sub_x16(b19,b23); |
| 1612 |
c23 = mulmod_scaled_x16(c23,scaledzeta_x16_8_7,qinvscaledzeta_x16_8_7,qdata); |
| 1613 |
// inv vector_butterfly 176 240 4 1 |
| 1614 |
int16x16 c11 = add_x16(b11,b15); |
| 1615 |
int16x16 c15 = sub_x16(b11,b15); |
| 1616 |
c15 = mulmod_scaled_x16(c15,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1617 |
// inv vector_butterfly 48 112 1 0 |
| 1618 |
int16x16 c3 = add_x16(b3,b7); |
| 1619 |
int16x16 c7 = sub_x16(b3,b7); |
| 1620 |
// inv vector_reduce_ifforward 112 |
| 1621 |
// inv vector_butterfly 368 496 4 1 |
| 1622 |
int16x16 d23 = add_x16(c23,c31); |
| 1623 |
int16x16 d31 = sub_x16(c23,c31); |
| 1624 |
d31 = mulmod_scaled_x16(d31,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1625 |
// inv vector_butterfly 304 432 4 1 |
| 1626 |
int16x16 d19 = add_x16(c19,c27); |
| 1627 |
int16x16 d27 = sub_x16(c19,c27); |
| 1628 |
d27 = mulmod_scaled_x16(d27,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1629 |
// inv vector_butterfly 112 240 1 0 |
| 1630 |
int16x16 d7 = add_x16(c7,c15); |
| 1631 |
int16x16 d15 = sub_x16(c7,c15); |
| 1632 |
// inv vector_butterfly 48 176 1 0 |
| 1633 |
int16x16 d3 = add_x16(c3,c11); |
| 1634 |
int16x16 d11 = sub_x16(c3,c11); |
| 1635 |
// inv vector_reduce_ifreverse 304 |
| 1636 |
d19 = reduce_x16(d19,qdata); |
| 1637 |
// inv vector_reduce_ifreverse 48 |
| 1638 |
d3 = reduce_x16(d3,qdata); |
| 1639 |
// inv vector_butterfly 240 496 1 0 |
| 1640 |
int16x16 e15 = add_x16(d15,d31); |
| 1641 |
int16x16 e31 = sub_x16(d15,d31); |
| 1642 |
// inv vector_butterfly 112 368 1 0 |
| 1643 |
int16x16 e7 = add_x16(d7,d23); |
| 1644 |
int16x16 e23 = sub_x16(d7,d23); |
| 1645 |
// inv vector_butterfly 176 432 1 0 |
| 1646 |
int16x16 e11 = add_x16(d11,d27); |
| 1647 |
int16x16 e27 = sub_x16(d11,d27); |
| 1648 |
// inv vector_butterfly 48 304 1 0 |
| 1649 |
int16x16 e3 = add_x16(d3,d19); |
| 1650 |
int16x16 e19 = sub_x16(d3,d19); |
| 1651 |
// inv startbatch 512 |
| 1652 |
_mm256_storeu_si256((int16x16 *) (f+48),e3); |
| 1653 |
_mm256_storeu_si256((int16x16 *) (f+112),e7); |
| 1654 |
_mm256_storeu_si256((int16x16 *) (f+176),e11); |
| 1655 |
_mm256_storeu_si256((int16x16 *) (f+240),e15); |
| 1656 |
_mm256_storeu_si256((int16x16 *) (f+304),e19); |
| 1657 |
_mm256_storeu_si256((int16x16 *) (f+368),e23); |
| 1658 |
_mm256_storeu_si256((int16x16 *) (f+432),e27); |
| 1659 |
_mm256_storeu_si256((int16x16 *) (f+496),e31); |
| 1660 |
f += 512; |
| 1661 |
} |
| 1662 |
f -= 512*reps; |
| 1663 |
// inv stopbatch 512 |
| 1664 |
for (long long r = 0;r < reps;++r) { |
| 1665 |
// inv vector_permute 416 480 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1666 |
int16x16 a26 = _mm256_loadu_si256((int16x16 *) (f+416)); |
| 1667 |
int16x16 a30 = _mm256_loadu_si256((int16x16 *) (f+480)); |
| 1668 |
int16x16 b26 = _mm256_permute2x128_si256_lo(a26,a30); |
| 1669 |
int16x16 b30 = _mm256_permute2x128_si256_hi(a26,a30); |
| 1670 |
// inv vector_permute 288 352 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1671 |
int16x16 a18 = _mm256_loadu_si256((int16x16 *) (f+288)); |
| 1672 |
int16x16 a22 = _mm256_loadu_si256((int16x16 *) (f+352)); |
| 1673 |
int16x16 b18 = _mm256_permute2x128_si256_lo(a18,a22); |
| 1674 |
int16x16 b22 = _mm256_permute2x128_si256_hi(a18,a22); |
| 1675 |
// inv vector_permute 160 224 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1676 |
int16x16 a10 = _mm256_loadu_si256((int16x16 *) (f+160)); |
| 1677 |
int16x16 a14 = _mm256_loadu_si256((int16x16 *) (f+224)); |
| 1678 |
int16x16 b10 = _mm256_permute2x128_si256_lo(a10,a14); |
| 1679 |
int16x16 b14 = _mm256_permute2x128_si256_hi(a10,a14); |
| 1680 |
// inv vector_permute 32 96 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1681 |
int16x16 a2 = _mm256_loadu_si256((int16x16 *) (f+32)); |
| 1682 |
int16x16 a6 = _mm256_loadu_si256((int16x16 *) (f+96)); |
| 1683 |
int16x16 b2 = _mm256_permute2x128_si256_lo(a2,a6); |
| 1684 |
int16x16 b6 = _mm256_permute2x128_si256_hi(a2,a6); |
| 1685 |
// inv vector_twist 480 512 507 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1686 |
b30 = mulmod_scaled_x16(b30,precomp_512_5_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_512_5_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1687 |
// inv vector_twist 416 512 511 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1688 |
b26 = mulmod_scaled_x16(b26,precomp_512_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_512_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1689 |
// inv vector_twist 352 512 5 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1690 |
b22 = mulmod_scaled_x16(b22,precomp_512_507_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_512_507_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1691 |
// inv vector_twist 288 512 1 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1692 |
b18 = mulmod_scaled_x16(b18,precomp_512_511_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_512_511_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1693 |
// inv vector_twist 224 256 255 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1694 |
b14 = mulmod_scaled_x16(b14,precomp_256_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_256_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1695 |
// inv vector_twist 160 256 1 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1696 |
b10 = mulmod_scaled_x16(b10,precomp_256_255_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_256_255_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1697 |
// inv vector_twist 96 128 1 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 1698 |
b6 = mulmod_scaled_x16(b6,precomp_128_127_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_128_127_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata); |
| 1699 |
// inv vector_reduce 32 |
| 1700 |
b2 = reduce_x16(b2,qdata); |
| 1701 |
// inv vector_butterfly 416 480 8 7 |
| 1702 |
int16x16 c26 = add_x16(b26,b30); |
| 1703 |
int16x16 c30 = sub_x16(b26,b30); |
| 1704 |
c30 = mulmod_scaled_x16(c30,scaledzeta_x16_8_1,qinvscaledzeta_x16_8_1,qdata); |
| 1705 |
// inv vector_butterfly 288 352 8 1 |
| 1706 |
int16x16 c18 = add_x16(b18,b22); |
| 1707 |
int16x16 c22 = sub_x16(b18,b22); |
| 1708 |
c22 = mulmod_scaled_x16(c22,scaledzeta_x16_8_7,qinvscaledzeta_x16_8_7,qdata); |
| 1709 |
// inv vector_butterfly 160 224 4 1 |
| 1710 |
int16x16 c10 = add_x16(b10,b14); |
| 1711 |
int16x16 c14 = sub_x16(b10,b14); |
| 1712 |
c14 = mulmod_scaled_x16(c14,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1713 |
// inv vector_butterfly 32 96 1 0 |
| 1714 |
int16x16 c2 = add_x16(b2,b6); |
| 1715 |
int16x16 c6 = sub_x16(b2,b6); |
| 1716 |
// inv vector_reduce_ifforward 96 |
| 1717 |
// inv vector_butterfly 352 480 4 1 |
| 1718 |
int16x16 d22 = add_x16(c22,c30); |
| 1719 |
int16x16 d30 = sub_x16(c22,c30); |
| 1720 |
d30 = mulmod_scaled_x16(d30,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1721 |
// inv vector_butterfly 288 416 4 1 |
| 1722 |
int16x16 d18 = add_x16(c18,c26); |
| 1723 |
int16x16 d26 = sub_x16(c18,c26); |
| 1724 |
d26 = mulmod_scaled_x16(d26,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1725 |
// inv vector_butterfly 96 224 1 0 |
| 1726 |
int16x16 d6 = add_x16(c6,c14); |
| 1727 |
int16x16 d14 = sub_x16(c6,c14); |
| 1728 |
// inv vector_butterfly 32 160 1 0 |
| 1729 |
int16x16 d2 = add_x16(c2,c10); |
| 1730 |
int16x16 d10 = sub_x16(c2,c10); |
| 1731 |
// inv vector_reduce_ifreverse 288 |
| 1732 |
d18 = reduce_x16(d18,qdata); |
| 1733 |
// inv vector_reduce_ifreverse 32 |
| 1734 |
d2 = reduce_x16(d2,qdata); |
| 1735 |
// inv vector_butterfly 224 480 1 0 |
| 1736 |
int16x16 e14 = add_x16(d14,d30); |
| 1737 |
int16x16 e30 = sub_x16(d14,d30); |
| 1738 |
// inv vector_butterfly 96 352 1 0 |
| 1739 |
int16x16 e6 = add_x16(d6,d22); |
| 1740 |
int16x16 e22 = sub_x16(d6,d22); |
| 1741 |
// inv vector_butterfly 160 416 1 0 |
| 1742 |
int16x16 e10 = add_x16(d10,d26); |
| 1743 |
int16x16 e26 = sub_x16(d10,d26); |
| 1744 |
// inv vector_butterfly 32 288 1 0 |
| 1745 |
int16x16 e2 = add_x16(d2,d18); |
| 1746 |
int16x16 e18 = sub_x16(d2,d18); |
| 1747 |
// inv startbatch 512 |
| 1748 |
_mm256_storeu_si256((int16x16 *) (f+32),e2); |
| 1749 |
_mm256_storeu_si256((int16x16 *) (f+96),e6); |
| 1750 |
_mm256_storeu_si256((int16x16 *) (f+160),e10); |
| 1751 |
_mm256_storeu_si256((int16x16 *) (f+224),e14); |
| 1752 |
_mm256_storeu_si256((int16x16 *) (f+288),e18); |
| 1753 |
_mm256_storeu_si256((int16x16 *) (f+352),e22); |
| 1754 |
_mm256_storeu_si256((int16x16 *) (f+416),e26); |
| 1755 |
_mm256_storeu_si256((int16x16 *) (f+480),e30); |
| 1756 |
f += 512; |
| 1757 |
} |
| 1758 |
f -= 512*reps; |
| 1759 |
// inv stopbatch 512 |
| 1760 |
for (long long r = 0;r < reps;++r) { |
| 1761 |
// inv vector_permute 400 464 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1762 |
int16x16 a25 = _mm256_loadu_si256((int16x16 *) (f+400)); |
| 1763 |
int16x16 a29 = _mm256_loadu_si256((int16x16 *) (f+464)); |
| 1764 |
int16x16 b25 = _mm256_permute2x128_si256_lo(a25,a29); |
| 1765 |
int16x16 b29 = _mm256_permute2x128_si256_hi(a25,a29); |
| 1766 |
// inv vector_permute 272 336 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1767 |
int16x16 a17 = _mm256_loadu_si256((int16x16 *) (f+272)); |
| 1768 |
int16x16 a21 = _mm256_loadu_si256((int16x16 *) (f+336)); |
| 1769 |
int16x16 b17 = _mm256_permute2x128_si256_lo(a17,a21); |
| 1770 |
int16x16 b21 = _mm256_permute2x128_si256_hi(a17,a21); |
| 1771 |
// inv vector_permute 144 208 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1772 |
int16x16 a9 = _mm256_loadu_si256((int16x16 *) (f+144)); |
| 1773 |
int16x16 a13 = _mm256_loadu_si256((int16x16 *) (f+208)); |
| 1774 |
int16x16 b9 = _mm256_permute2x128_si256_lo(a9,a13); |
| 1775 |
int16x16 b13 = _mm256_permute2x128_si256_hi(a9,a13); |
| 1776 |
// inv vector_permute 16 80 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1777 |
int16x16 a1 = _mm256_loadu_si256((int16x16 *) (f+16)); |
| 1778 |
int16x16 a5 = _mm256_loadu_si256((int16x16 *) (f+80)); |
| 1779 |
int16x16 b1 = _mm256_permute2x128_si256_lo(a1,a5); |
| 1780 |
int16x16 b5 = _mm256_permute2x128_si256_hi(a1,a5); |
| 1781 |
// inv vector_twist 464 512 507 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 1782 |
b29 = mulmod_scaled_x16(b29,precomp_512_5_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_512_5_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 1783 |
// inv vector_twist 400 512 511 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 1784 |
b25 = mulmod_scaled_x16(b25,precomp_512_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_512_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 1785 |
// inv vector_twist 336 512 5 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 1786 |
b21 = mulmod_scaled_x16(b21,precomp_512_507_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_512_507_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 1787 |
// inv vector_twist 272 512 1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 1788 |
b17 = mulmod_scaled_x16(b17,precomp_512_511_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_512_511_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 1789 |
// inv vector_twist 208 256 255 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 1790 |
b13 = mulmod_scaled_x16(b13,precomp_256_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_256_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 1791 |
// inv vector_twist 144 256 1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 1792 |
b9 = mulmod_scaled_x16(b9,precomp_256_255_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_256_255_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 1793 |
// inv vector_twist 80 128 1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 1794 |
b5 = mulmod_scaled_x16(b5,precomp_128_127_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_128_127_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata); |
| 1795 |
// inv vector_reduce 16 |
| 1796 |
b1 = reduce_x16(b1,qdata); |
| 1797 |
// inv vector_butterfly 400 464 8 7 |
| 1798 |
int16x16 c25 = add_x16(b25,b29); |
| 1799 |
int16x16 c29 = sub_x16(b25,b29); |
| 1800 |
c29 = mulmod_scaled_x16(c29,scaledzeta_x16_8_1,qinvscaledzeta_x16_8_1,qdata); |
| 1801 |
// inv vector_butterfly 272 336 8 1 |
| 1802 |
int16x16 c17 = add_x16(b17,b21); |
| 1803 |
int16x16 c21 = sub_x16(b17,b21); |
| 1804 |
c21 = mulmod_scaled_x16(c21,scaledzeta_x16_8_7,qinvscaledzeta_x16_8_7,qdata); |
| 1805 |
// inv vector_butterfly 144 208 4 1 |
| 1806 |
int16x16 c9 = add_x16(b9,b13); |
| 1807 |
int16x16 c13 = sub_x16(b9,b13); |
| 1808 |
c13 = mulmod_scaled_x16(c13,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1809 |
// inv vector_butterfly 16 80 1 0 |
| 1810 |
int16x16 c1 = add_x16(b1,b5); |
| 1811 |
int16x16 c5 = sub_x16(b1,b5); |
| 1812 |
// inv vector_reduce_ifforward 80 |
| 1813 |
// inv vector_butterfly 336 464 4 1 |
| 1814 |
int16x16 d21 = add_x16(c21,c29); |
| 1815 |
int16x16 d29 = sub_x16(c21,c29); |
| 1816 |
d29 = mulmod_scaled_x16(d29,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1817 |
// inv vector_butterfly 272 400 4 1 |
| 1818 |
int16x16 d17 = add_x16(c17,c25); |
| 1819 |
int16x16 d25 = sub_x16(c17,c25); |
| 1820 |
d25 = mulmod_scaled_x16(d25,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1821 |
// inv vector_butterfly 80 208 1 0 |
| 1822 |
int16x16 d5 = add_x16(c5,c13); |
| 1823 |
int16x16 d13 = sub_x16(c5,c13); |
| 1824 |
// inv vector_butterfly 16 144 1 0 |
| 1825 |
int16x16 d1 = add_x16(c1,c9); |
| 1826 |
int16x16 d9 = sub_x16(c1,c9); |
| 1827 |
// inv vector_reduce_ifreverse 272 |
| 1828 |
d17 = reduce_x16(d17,qdata); |
| 1829 |
// inv vector_reduce_ifreverse 16 |
| 1830 |
d1 = reduce_x16(d1,qdata); |
| 1831 |
// inv vector_butterfly 208 464 1 0 |
| 1832 |
int16x16 e13 = add_x16(d13,d29); |
| 1833 |
int16x16 e29 = sub_x16(d13,d29); |
| 1834 |
// inv vector_butterfly 80 336 1 0 |
| 1835 |
int16x16 e5 = add_x16(d5,d21); |
| 1836 |
int16x16 e21 = sub_x16(d5,d21); |
| 1837 |
// inv vector_butterfly 144 400 1 0 |
| 1838 |
int16x16 e9 = add_x16(d9,d25); |
| 1839 |
int16x16 e25 = sub_x16(d9,d25); |
| 1840 |
// inv vector_butterfly 16 272 1 0 |
| 1841 |
int16x16 e1 = add_x16(d1,d17); |
| 1842 |
int16x16 e17 = sub_x16(d1,d17); |
| 1843 |
// inv startbatch 512 |
| 1844 |
_mm256_storeu_si256((int16x16 *) (f+16),e1); |
| 1845 |
_mm256_storeu_si256((int16x16 *) (f+80),e5); |
| 1846 |
_mm256_storeu_si256((int16x16 *) (f+144),e9); |
| 1847 |
_mm256_storeu_si256((int16x16 *) (f+208),e13); |
| 1848 |
_mm256_storeu_si256((int16x16 *) (f+272),e17); |
| 1849 |
_mm256_storeu_si256((int16x16 *) (f+336),e21); |
| 1850 |
_mm256_storeu_si256((int16x16 *) (f+400),e25); |
| 1851 |
_mm256_storeu_si256((int16x16 *) (f+464),e29); |
| 1852 |
f += 512; |
| 1853 |
} |
| 1854 |
f -= 512*reps; |
| 1855 |
// inv stopbatch 512 |
| 1856 |
for (long long r = 0;r < reps;++r) { |
| 1857 |
// inv vector_permute 384 448 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1858 |
int16x16 a24 = _mm256_loadu_si256((int16x16 *) (f+384)); |
| 1859 |
int16x16 a28 = _mm256_loadu_si256((int16x16 *) (f+448)); |
| 1860 |
int16x16 b24 = _mm256_permute2x128_si256_lo(a24,a28); |
| 1861 |
int16x16 b28 = _mm256_permute2x128_si256_hi(a24,a28); |
| 1862 |
// inv vector_permute 256 320 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1863 |
int16x16 a16 = _mm256_loadu_si256((int16x16 *) (f+256)); |
| 1864 |
int16x16 a20 = _mm256_loadu_si256((int16x16 *) (f+320)); |
| 1865 |
int16x16 b16 = _mm256_permute2x128_si256_lo(a16,a20); |
| 1866 |
int16x16 b20 = _mm256_permute2x128_si256_hi(a16,a20); |
| 1867 |
// inv vector_permute 128 192 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1868 |
int16x16 a8 = _mm256_loadu_si256((int16x16 *) (f+128)); |
| 1869 |
int16x16 a12 = _mm256_loadu_si256((int16x16 *) (f+192)); |
| 1870 |
int16x16 b8 = _mm256_permute2x128_si256_lo(a8,a12); |
| 1871 |
int16x16 b12 = _mm256_permute2x128_si256_hi(a8,a12); |
| 1872 |
// inv vector_permute 0 64 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi |
| 1873 |
int16x16 a0 = _mm256_loadu_si256((int16x16 *) (f+0)); |
| 1874 |
int16x16 a4 = _mm256_loadu_si256((int16x16 *) (f+64)); |
| 1875 |
int16x16 b0 = _mm256_permute2x128_si256_lo(a0,a4); |
| 1876 |
int16x16 b4 = _mm256_permute2x128_si256_hi(a0,a4); |
| 1877 |
// inv vector_twist 448 512 507 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 1878 |
b28 = mulmod_scaled_x16(b28,precomp_512_5_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_512_5_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 1879 |
// inv vector_twist 384 512 511 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 1880 |
b24 = mulmod_scaled_x16(b24,precomp_512_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_512_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 1881 |
// inv vector_twist 320 512 5 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 1882 |
b20 = mulmod_scaled_x16(b20,precomp_512_507_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_512_507_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 1883 |
// inv vector_twist 256 512 1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 1884 |
b16 = mulmod_scaled_x16(b16,precomp_512_511_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_512_511_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 1885 |
// inv vector_twist 192 256 255 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 1886 |
b12 = mulmod_scaled_x16(b12,precomp_256_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_256_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 1887 |
// inv vector_twist 128 256 1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 1888 |
b8 = mulmod_scaled_x16(b8,precomp_256_255_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_256_255_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 1889 |
// inv vector_twist 64 128 1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 1890 |
b4 = mulmod_scaled_x16(b4,precomp_128_127_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_128_127_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata); |
| 1891 |
// inv vector_reduce 0 |
| 1892 |
b0 = reduce_x16(b0,qdata); |
| 1893 |
// inv vector_butterfly 384 448 8 7 |
| 1894 |
int16x16 c24 = add_x16(b24,b28); |
| 1895 |
int16x16 c28 = sub_x16(b24,b28); |
| 1896 |
c28 = mulmod_scaled_x16(c28,scaledzeta_x16_8_1,qinvscaledzeta_x16_8_1,qdata); |
| 1897 |
// inv vector_butterfly 256 320 8 1 |
| 1898 |
int16x16 c16 = add_x16(b16,b20); |
| 1899 |
int16x16 c20 = sub_x16(b16,b20); |
| 1900 |
c20 = mulmod_scaled_x16(c20,scaledzeta_x16_8_7,qinvscaledzeta_x16_8_7,qdata); |
| 1901 |
// inv vector_butterfly 128 192 4 1 |
| 1902 |
int16x16 c8 = add_x16(b8,b12); |
| 1903 |
int16x16 c12 = sub_x16(b8,b12); |
| 1904 |
c12 = mulmod_scaled_x16(c12,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1905 |
// inv vector_butterfly 0 64 1 0 |
| 1906 |
int16x16 c0 = add_x16(b0,b4); |
| 1907 |
int16x16 c4 = sub_x16(b0,b4); |
| 1908 |
// inv vector_reduce_ifforward 64 |
| 1909 |
// inv vector_butterfly 320 448 4 1 |
| 1910 |
int16x16 d20 = add_x16(c20,c28); |
| 1911 |
int16x16 d28 = sub_x16(c20,c28); |
| 1912 |
d28 = mulmod_scaled_x16(d28,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1913 |
// inv vector_butterfly 256 384 4 1 |
| 1914 |
int16x16 d16 = add_x16(c16,c24); |
| 1915 |
int16x16 d24 = sub_x16(c16,c24); |
| 1916 |
d24 = mulmod_scaled_x16(d24,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata); |
| 1917 |
// inv vector_butterfly 64 192 1 0 |
| 1918 |
int16x16 d4 = add_x16(c4,c12); |
| 1919 |
int16x16 d12 = sub_x16(c4,c12); |
| 1920 |
// inv vector_butterfly 0 128 1 0 |
| 1921 |
int16x16 d0 = add_x16(c0,c8); |
| 1922 |
int16x16 d8 = sub_x16(c0,c8); |
| 1923 |
// inv vector_reduce_ifreverse 256 |
| 1924 |
d16 = reduce_x16(d16,qdata); |
| 1925 |
// inv vector_reduce_ifreverse 0 |
| 1926 |
d0 = reduce_x16(d0,qdata); |
| 1927 |
// inv vector_butterfly 192 448 1 0 |
| 1928 |
int16x16 e12 = add_x16(d12,d28); |
| 1929 |
int16x16 e28 = sub_x16(d12,d28); |
| 1930 |
// inv vector_butterfly 64 320 1 0 |
| 1931 |
int16x16 e4 = add_x16(d4,d20); |
| 1932 |
int16x16 e20 = sub_x16(d4,d20); |
| 1933 |
// inv vector_butterfly 128 384 1 0 |
| 1934 |
int16x16 e8 = add_x16(d8,d24); |
| 1935 |
int16x16 e24 = sub_x16(d8,d24); |
| 1936 |
// inv vector_butterfly 0 256 1 0 |
| 1937 |
int16x16 e0 = add_x16(d0,d16); |
| 1938 |
int16x16 e16 = sub_x16(d0,d16); |
| 1939 |
// inv startbatch 512 |
| 1940 |
_mm256_storeu_si256((int16x16 *) (f+0),e0); |
| 1941 |
_mm256_storeu_si256((int16x16 *) (f+64),e4); |
| 1942 |
_mm256_storeu_si256((int16x16 *) (f+128),e8); |
| 1943 |
_mm256_storeu_si256((int16x16 *) (f+192),e12); |
| 1944 |
_mm256_storeu_si256((int16x16 *) (f+256),e16); |
| 1945 |
_mm256_storeu_si256((int16x16 *) (f+320),e20); |
| 1946 |
_mm256_storeu_si256((int16x16 *) (f+384),e24); |
| 1947 |
_mm256_storeu_si256((int16x16 *) (f+448),e28); |
| 1948 |
f += 512; |
| 1949 |
} |
| 1950 |
f -= 512*reps; |
| 1951 |
// inv startntt 512 |
| 1952 |
} |
| 1953 |
|
| 1954 |
void invntt512_7681(int16 *f,int reps) |
| 1955 |
{ |
| 1956 |
invntt512(f,reps,qdata_7681); |
| 1957 |
} |