Debian Codegraph @

crypto_core/mult3sntrup1277/avx/ntt.c

raw
1 // linker define ntt512_7681
2 // linker define invntt512_7681
3
4 #include "ntt.h"
5
6 // auto-generated; do not edit
7
8 #include <immintrin.h>
9
10 #define _mm256_permute2x128_si256_lo(f0,f1) _mm256_permute2x128_si256(f0,f1,0x20)
11 #define _mm256_permute2x128_si256_hi(f0,f1) _mm256_permute2x128_si256(f0,f1,0x31)
12 #define int16x16 __m256i
13
14 typedef int16_t int16;
15 typedef int32_t int32;
16
17 static const int16 __attribute((aligned(32))) qdata_7681[] = {
18 #define precomp_8_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 *(const int16x16 *)(qdata+0)
19 -3593,-3593,-3593,-3593,-3625,-3625,-3625,-3625,-3593,-3593,-3593,-3593,-3625,-3625,-3625,-3625,
20 #define precomp_8_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 *(const int16x16 *)(qdata+16)
21 -3777,-3777,-3777,-3777,3182,3182,3182,3182,-3777,-3777,-3777,-3777,3182,3182,3182,3182,
22 #define precomp_8_7_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 *(const int16x16 *)(qdata+32)
23 -3593,-3593,-3593,-3593,-3182,-3182,-3182,-3182,-3593,-3593,-3593,-3593,-3182,-3182,-3182,-3182,
24 #define precomp_8_7_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 *(const int16x16 *)(qdata+48)
25 3777,3777,3777,3777,3625,3625,3625,3625,3777,3777,3777,3777,3625,3625,3625,3625,
26 #define precomp_16_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 *(const int16x16 *)(qdata+64)
27 -3593,-3593,-3593,-3593,2194,2194,2194,2194,-3593,-3593,-3593,-3593,2194,2194,2194,2194,
28 #define precomp_16_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 *(const int16x16 *)(qdata+80)
29 -3625,-3625,-3625,-3625,-1100,-1100,-1100,-1100,-3625,-3625,-3625,-3625,-1100,-1100,-1100,-1100,
30 #define precomp_16_15_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 *(const int16x16 *)(qdata+96)
31 -3593,-3593,-3593,-3593,3696,3696,3696,3696,-3593,-3593,-3593,-3593,3696,3696,3696,3696,
32 #define precomp_16_15_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 *(const int16x16 *)(qdata+112)
33 -3182,-3182,-3182,-3182,-2456,-2456,-2456,-2456,-3182,-3182,-3182,-3182,-2456,-2456,-2456,-2456,
34 #define precomp_32_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 *(const int16x16 *)(qdata+128)
35 -3593,1701,2194,834,-3625,2319,-1100,121,-3593,1701,2194,834,-3625,2319,-1100,121,
36 #define precomp_32_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+144)
37 -3777,1414,2456,2495,3182,2876,-3696,2250,-3777,1414,2456,2495,3182,2876,-3696,2250,
38 #define precomp_32_31_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 *(const int16x16 *)(qdata+160)
39 -3593,-2250,3696,-2876,-3182,-2495,-2456,-1414,-3593,-2250,3696,-2876,-3182,-2495,-2456,-1414,
40 #define precomp_32_31_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+176)
41 3777,-121,1100,-2319,3625,-834,-2194,-1701,3777,-121,1100,-2319,3625,-834,-2194,-1701,
42 #define precomp_64_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 *(const int16x16 *)(qdata+192)
43 -3593,3364,1701,-1599,2194,2557,834,-2816,-3593,3364,1701,-1599,2194,2557,834,-2816,
44 #define precomp_64_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+208)
45 -3625,617,2319,2006,-1100,-1296,121,1986,-3625,617,2319,2006,-1100,-1296,121,1986,
46 #define precomp_64_63_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 *(const int16x16 *)(qdata+224)
47 -3593,2237,-2250,-1483,3696,3706,-2876,1921,-3593,2237,-2250,-1483,3696,3706,-2876,1921,
48 #define precomp_64_63_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+240)
49 -3182,2088,-2495,-1525,-2456,1993,-1414,2830,-3182,2088,-2495,-1525,-2456,1993,-1414,2830,
50 #define precomp_128_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+256)
51 -3593,514,3364,438,1701,2555,-1599,-1738,2194,103,2557,1881,834,-549,-2816,638,
52 #define precomp_128_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+272)
53 -3625,-1399,617,-1760,2319,2535,2006,3266,-1100,-1431,-1296,3174,121,3153,1986,-810,
54 #define precomp_128_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+288)
55 -3777,2956,-2830,-679,1414,2440,-1993,-3689,2456,2804,1525,3555,2495,1535,-2088,-7,
56 #define precomp_128_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+304)
57 3182,-1321,-1921,-1305,2876,-3772,-3706,3600,-3696,-2043,1483,-396,2250,-2310,-2237,1887,
58 #define precomp_128_127_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+320)
59 -3593,-1887,2237,2310,-2250,396,-1483,2043,3696,-3600,3706,3772,-2876,1305,1921,1321,
60 #define precomp_128_127_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+336)
61 -3182,7,2088,-1535,-2495,-3555,-1525,-2804,-2456,3689,1993,-2440,-1414,679,2830,-2956,
62 #define precomp_128_127_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+352)
63 3777,810,-1986,-3153,-121,-3174,1296,1431,1100,-3266,-2006,-2535,-2319,1760,-617,1399,
64 #define precomp_128_127_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+368)
65 3625,-638,2816,549,-834,-1881,-2557,-103,-2194,1738,1599,-2555,-1701,-438,-3364,-514,
66 #define precomp_256_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+384)
67 -3593,-1532,514,-373,3364,-3816,438,-3456,1701,783,2555,2883,-1599,727,-1738,-2385,
68 #define precomp_256_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+400)
69 2194,-2160,103,-2391,2557,2762,1881,-2426,834,3310,-549,-1350,-2816,1386,638,-194,
70 #define precomp_256_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+416)
71 -3625,404,-1399,-3692,617,-2764,-1760,-1054,2319,1799,2535,-3588,2006,1533,3266,2113,
72 #define precomp_256_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+432)
73 -1100,-2579,-1431,-1756,-1296,1598,3174,-2,121,-3480,3153,-2572,1986,2743,-810,2919,
74 #define precomp_256_255_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+448)
75 -3593,2789,-1887,-921,2237,-1497,2310,-2133,-2250,-915,396,1390,-1483,3135,2043,-859,
76 #define precomp_256_255_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+464)
77 3696,2732,-3600,-1464,3706,2224,3772,-2665,-2876,1698,1305,2835,1921,730,1321,486,
78 #define precomp_256_255_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+480)
79 -3182,3417,7,-3428,2088,-3145,-1535,1168,-2495,-3831,-3555,-3750,-1525,660,-2804,2649,
80 #define precomp_256_255_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+496)
81 -2456,3405,3689,-1521,1993,1681,-2440,1056,-1414,1166,679,-2233,2830,2175,-2956,-1919,
82 #define precomp_512_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+512)
83 -3593,-1404,-1532,451,514,-402,-373,1278,3364,-509,-3816,-3770,438,-2345,-3456,-226,
84 #define precomp_512_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+528)
85 1701,-1689,783,-1509,2555,2963,2883,1242,-1599,1669,727,2719,-1738,642,-2385,-436,
86 #define precomp_512_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+544)
87 2194,3335,-2160,1779,103,3745,-2391,17,2557,2812,2762,-1144,1881,83,-2426,-1181,
88 #define precomp_512_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+560)
89 834,-1519,3310,3568,-549,-796,-1350,2072,-2816,-2460,1386,2891,638,-2083,-194,-715,
90 #define precomp_512_5_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+576)
91 -3593,-402,-3816,-226,2555,1669,-2385,1779,2557,83,3310,2072,638,1012,-3692,1295,
92 #define precomp_512_5_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+592)
93 2319,-3208,1533,-2071,-1431,-2005,-2,1586,1986,-293,1919,-929,-679,777,-1681,-3461,
94 #define precomp_512_5_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+608)
95 2456,3366,3750,-1203,1535,-3657,-3417,-1712,-1921,2515,2665,-1070,3600,2532,-3135,-2589,
96 #define precomp_512_5_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+624)
97 2250,-2258,921,-658,-514,509,3456,1509,1599,-642,2160,-17,-1881,1519,1350,-2891,
98 #define precomp_512_507_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+640)
99 -3593,-3434,-1497,893,396,-2422,-859,2965,3706,-2339,1698,-2937,1321,-670,-3428,-3163,
100 #define precomp_512_507_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+656)
101 -2495,-1072,660,1084,3689,-179,1056,-1338,2830,2786,-2919,-3677,-3153,-151,-1598,3334,
102 #define precomp_512_507_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+672)
103 1100,-3314,3588,2262,1760,-2230,-404,2083,2816,-3568,2426,-2812,-103,436,-727,-2963,
104 #define precomp_512_507_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+688)
105 -1701,3770,373,1404,1887,-1649,2133,-826,1483,434,-2732,3287,-3772,-2378,-2835,3723,
106 #define precomp_512_511_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+704)
107 -3593,658,2789,370,-1887,-3434,-921,-3752,2237,1649,-1497,2258,2310,3581,-2133,893,
108 #define precomp_512_511_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+720)
109 -2250,3794,-915,826,396,2589,1390,592,-1483,-2422,3135,3214,2043,-434,-859,-2532,
110 #define precomp_512_511_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+736)
111 3696,1121,2732,2965,-3600,2998,-1464,-3287,3706,1070,2224,-589,3772,-2339,-2665,2070,
112 #define precomp_512_511_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+752)
113 -2876,2378,1698,-2515,1305,-2815,2835,-2937,1921,-1348,730,-3723,1321,1712,486,2130,
114 #define q_x16 *(const int16x16 *)(qdata+768)
115 7681,7681,7681,7681,7681,7681,7681,7681,7681,7681,7681,7681,7681,7681,7681,7681,
116 #define qinvprecomp_8_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 *(const int16x16 *)(qdata+784)
117 -9,-9,-9,-9,-16425,-16425,-16425,-16425,-9,-9,-9,-9,-16425,-16425,-16425,-16425,
118 #define qinvprecomp_8_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 *(const int16x16 *)(qdata+800)
119 -28865,-28865,-28865,-28865,10350,10350,10350,10350,-28865,-28865,-28865,-28865,10350,10350,10350,10350,
120 #define qinvprecomp_8_7_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 *(const int16x16 *)(qdata+816)
121 -9,-9,-9,-9,-10350,-10350,-10350,-10350,-9,-9,-9,-9,-10350,-10350,-10350,-10350,
122 #define qinvprecomp_8_7_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 *(const int16x16 *)(qdata+832)
123 28865,28865,28865,28865,16425,16425,16425,16425,28865,28865,28865,28865,16425,16425,16425,16425,
124 #define qinvprecomp_16_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 *(const int16x16 *)(qdata+848)
125 -9,-9,-9,-9,-4974,-4974,-4974,-4974,-9,-9,-9,-9,-4974,-4974,-4974,-4974,
126 #define qinvprecomp_16_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 *(const int16x16 *)(qdata+864)
127 -16425,-16425,-16425,-16425,-7244,-7244,-7244,-7244,-16425,-16425,-16425,-16425,-7244,-7244,-7244,-7244,
128 #define qinvprecomp_16_15_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1 *(const int16x16 *)(qdata+880)
129 -9,-9,-9,-9,-4496,-4496,-4496,-4496,-9,-9,-9,-9,-4496,-4496,-4496,-4496,
130 #define qinvprecomp_16_15_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3 *(const int16x16 *)(qdata+896)
131 -10350,-10350,-10350,-10350,-14744,-14744,-14744,-14744,-10350,-10350,-10350,-10350,-14744,-14744,-14744,-14744,
132 #define qinvprecomp_32_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 *(const int16x16 *)(qdata+912)
133 -9,-20315,-4974,18242,-16425,18191,-7244,-11655,-9,-20315,-4974,18242,-16425,18191,-7244,-11655,
134 #define qinvprecomp_32_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+928)
135 -28865,20870,14744,-22593,10350,828,4496,23754,-28865,20870,14744,-22593,10350,828,4496,23754,
136 #define qinvprecomp_32_31_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 *(const int16x16 *)(qdata+944)
137 -9,-23754,-4496,-828,-10350,22593,-14744,-20870,-9,-23754,-4496,-828,-10350,22593,-14744,-20870,
138 #define qinvprecomp_32_31_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+960)
139 28865,11655,7244,-18191,16425,-18242,4974,20315,28865,11655,7244,-18191,16425,-18242,4974,20315,
140 #define qinvprecomp_64_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 *(const int16x16 *)(qdata+976)
141 -9,-10972,-20315,23489,-4974,25597,18242,-2816,-9,-10972,-20315,23489,-4974,25597,18242,-2816,
142 #define qinvprecomp_64_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+992)
143 -16425,-19351,18191,-3114,-7244,-9488,-11655,19394,-16425,-19351,18191,-3114,-7244,-9488,-11655,19394,
144 #define qinvprecomp_64_63_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7 *(const int16x16 *)(qdata+1008)
145 -9,-7491,-23754,-15307,-4496,-15750,-828,-5759,-9,-7491,-23754,-15307,-4496,-15750,-828,-5759,
146 #define qinvprecomp_64_63_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+1024)
147 -10350,22568,22593,-20469,-14744,31177,-20870,26382,-10350,22568,22593,-20469,-14744,31177,-20870,26382,
148 #define qinvprecomp_128_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+1040)
149 -9,-14846,-10972,-21066,-20315,-24581,23489,-23242,-4974,-4505,25597,-26279,18242,21467,-2816,15998,
150 #define qinvprecomp_128_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+1056)
151 -16425,-4983,-19351,14624,18191,-2073,-3114,20674,-7244,-21399,-9488,6246,-11655,-29103,19394,-5930,
152 #define qinvprecomp_128_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+1072)
153 -28865,-23668,-26382,-28839,20870,6536,-31177,16279,14744,29428,20469,29667,-22593,9215,-22568,-11783,
154 #define qinvprecomp_128_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+1088)
155 10350,-14121,5759,-5913,828,-1724,15750,11792,4496,25093,15307,26228,23754,-21766,7491,-6817,
156 #define qinvprecomp_128_127_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+1104)
157 -9,6817,-7491,21766,-23754,-26228,-15307,-25093,-4496,-11792,-15750,1724,-828,5913,-5759,14121,
158 #define qinvprecomp_128_127_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+1120)
159 -10350,11783,22568,-9215,22593,-29667,-20469,-29428,-14744,-16279,31177,-6536,-20870,28839,26382,23668,
160 #define qinvprecomp_128_127_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+1136)
161 28865,5930,-19394,29103,11655,-6246,9488,21399,7244,-20674,3114,2073,-18191,-14624,19351,4983,
162 #define qinvprecomp_128_127_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+1152)
163 16425,-15998,2816,-21467,-18242,26279,-25597,4505,4974,23242,-23489,24581,20315,21066,10972,14846,
164 #define qinvprecomp_256_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+1168)
165 -9,-32252,-14846,-19317,-10972,8472,-21066,-3456,-20315,16655,-24581,12611,23489,-12073,-23242,29871,
166 #define qinvprecomp_256_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+1184)
167 -4974,6032,-4505,10409,25597,24266,-26279,17030,18242,10478,21467,11962,-2816,-26262,15998,-17602,
168 #define qinvprecomp_256_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+1200)
169 -16425,-22124,-4983,-26220,-19351,-8908,14624,32738,18191,13575,-2073,27132,-3114,24573,20674,27201,
170 #define qinvprecomp_256_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+1216)
171 -7244,12269,-21399,-16092,-9488,-15810,6246,15358,-11655,-15768,-29103,24052,19394,-26441,-5930,-1689,
172 #define qinvprecomp_256_255_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+1232)
173 -9,13541,6817,-5529,-7491,26663,21766,-4693,-23754,13933,-26228,8558,-15307,-21953,-25093,-22875,
174 #define qinvprecomp_256_255_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+1248)
175 -4496,-7508,-11792,-30136,-15750,26800,1724,17303,-828,2722,5913,-12013,-5759,30426,14121,3558,
176 #define qinvprecomp_256_255_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+1264)
177 -10350,-24743,11783,-21860,22568,-32329,-9215,9360,22593,-7415,-29667,25946,-20469,-21868,-29428,-25511,
178 #define qinvprecomp_256_255_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+1280)
179 -14744,1869,-16279,14351,31177,2193,-6536,17440,-20870,24718,28839,-23225,26382,9855,23668,-9599,
180 #define qinvprecomp_512_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+1296)
181 -9,-32124,-32252,10179,-14846,6766,-19317,16638,-10972,-23549,8472,-17082,-21066,-15145,-3456,31518,
182 #define qinvprecomp_512_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+1312)
183 -20315,-6297,16655,-12261,-24581,-11885,12611,30938,23489,28805,-12073,26783,-23242,-14718,29871,5708,
184 #define qinvprecomp_512_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+1328)
185 -4974,15111,6032,-29453,-4505,12449,10409,529,25597,-32004,24266,2952,-26279,18003,17030,24931,
186 #define qinvprecomp_512_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+1344)
187 18242,-1007,10478,-4624,21467,17636,11962,14360,-2816,15972,-26262,16715,15998,4573,-17602,-14539,
188 #define qinvprecomp_512_5_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+1360)
189 -9,6766,8472,31518,-24581,28805,29871,-29453,25597,18003,10478,14360,15998,27636,-26220,17167,
190 #define qinvprecomp_512_5_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+1376)
191 18191,-7304,24573,-22039,-21399,-4565,15358,10802,19394,21723,9599,-9633,-28839,-2807,-2193,-30597,
192 #define qinvprecomp_512_5_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+1392)
193 14744,-26330,-25946,-2739,9215,32695,24743,-26288,5759,20435,-17303,24530,11792,20964,21953,23523,
194 #define qinvprecomp_512_5_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+1408)
195 23754,-27858,5529,6510,14846,23549,3456,12261,-23489,14718,-6032,-529,26279,1007,-11962,-16715,
196 #define qinvprecomp_512_507_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+1424)
197 -9,24214,26663,23933,-26228,-13686,-22875,-27243,-15750,4317,2722,8839,14121,-32414,-21860,-25179,
198 #define qinvprecomp_512_507_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+1440)
199 22593,-25648,-21868,-964,-16279,-1715,17440,-14650,26382,-28958,1689,-10333,29103,-20119,15810,22790,
200 #define qinvprecomp_512_507_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+1456)
201 7244,20238,-27132,-2858,-14624,19274,22124,-4573,2816,4624,-17030,32004,4505,-5708,12073,11885,
202 #define qinvprecomp_512_507_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+1472)
203 20315,17082,19317,32124,-6817,14223,4693,-14138,15307,9650,7508,-9513,-1724,-23882,12013,-15221,
204 #define qinvprecomp_512_511_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15 *(const int16x16 *)(qdata+1488)
205 -9,-6510,13541,-23182,6817,24214,-5529,-24232,-7491,-14223,26663,27858,21766,26621,-4693,23933,
206 #define qinvprecomp_512_511_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31 *(const int16x16 *)(qdata+1504)
207 -23754,29394,13933,14138,-26228,-23523,8558,-23984,-15307,-13686,-21953,26766,-25093,-9650,-22875,-20964,
208 #define qinvprecomp_512_511_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47 *(const int16x16 *)(qdata+1520)
209 -4496,-22943,-7508,-27243,-11792,-18506,-30136,9513,-15750,-24530,26800,947,1724,4317,17303,29718,
210 #define qinvprecomp_512_511_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63 *(const int16x16 *)(qdata+1536)
211 -828,23882,2722,-20435,5913,-10495,-12013,8839,-5759,-3396,30426,15221,14121,26288,3558,27730,
212 #define qinvscaledzeta_x16_4_1 *(const int16x16 *)(qdata+1552)
213 -28865,-28865,-28865,-28865,-28865,-28865,-28865,-28865,-28865,-28865,-28865,-28865,-28865,-28865,-28865,-28865,
214 #define qinvscaledzeta_x16_4_3 *(const int16x16 *)(qdata+1568)
215 28865,28865,28865,28865,28865,28865,28865,28865,28865,28865,28865,28865,28865,28865,28865,28865,
216 #define qinvscaledzeta_x16_8_1 *(const int16x16 *)(qdata+1584)
217 -16425,-16425,-16425,-16425,-16425,-16425,-16425,-16425,-16425,-16425,-16425,-16425,-16425,-16425,-16425,-16425,
218 #define qinvscaledzeta_x16_8_7 *(const int16x16 *)(qdata+1600)
219 -10350,-10350,-10350,-10350,-10350,-10350,-10350,-10350,-10350,-10350,-10350,-10350,-10350,-10350,-10350,-10350,
220 #define qround32_x16 *(const int16x16 *)(qdata+1616)
221 4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,
222 #define scaledzeta_x16_4_1 *(const int16x16 *)(qdata+1632)
223 -3777,-3777,-3777,-3777,-3777,-3777,-3777,-3777,-3777,-3777,-3777,-3777,-3777,-3777,-3777,-3777,
224 #define scaledzeta_x16_4_3 *(const int16x16 *)(qdata+1648)
225 3777,3777,3777,3777,3777,3777,3777,3777,3777,3777,3777,3777,3777,3777,3777,3777,
226 #define scaledzeta_x16_8_1 *(const int16x16 *)(qdata+1664)
227 -3625,-3625,-3625,-3625,-3625,-3625,-3625,-3625,-3625,-3625,-3625,-3625,-3625,-3625,-3625,-3625,
228 #define scaledzeta_x16_8_7 *(const int16x16 *)(qdata+1680)
229 -3182,-3182,-3182,-3182,-3182,-3182,-3182,-3182,-3182,-3182,-3182,-3182,-3182,-3182,-3182,-3182,
230 } ;
231
232 static inline int16x16 add_x16(int16x16 a,int16x16 b)
233 {
234 return _mm256_add_epi16(a,b);
235 }
236
237 static inline int16x16 sub_x16(int16x16 a,int16x16 b)
238 {
239 return _mm256_sub_epi16(a,b);
240 }
241
242 static inline int16x16 mulmod_scaled_x16(int16x16 x,int16x16 y,int16x16 yqinv,const int16 *qdata)
243 {
244 int16x16 b = _mm256_mulhi_epi16(x,y);
245 int16x16 d = _mm256_mullo_epi16(x,yqinv);
246 int16x16 e = _mm256_mulhi_epi16(d,q_x16);
247 return sub_x16(b,e);
248 }
249
250 static inline int16x16 reduce_x16(int16x16 x,const int16 *qdata)
251 {
252 int16x16 y = _mm256_mulhrs_epi16(x,qround32_x16);
253 y = _mm256_mullo_epi16(y,q_x16);
254 return sub_x16(x,y);
255 }
256
257 // ----- codegen pass 1
258 //
259 // startntt 512
260 // startbatch 512
261 // // ----- PRECONDITIONS
262 // physical_map (0, 1, 2, 3, 4, 5, 6, 7, 8) ()
263 // // transform size 512
264 // // transform indexing [0, 1, 2, 3, 4, 5, 6, 7, 8]
265 // // transforms per batch 1
266 // // batch indexing []
267 // // total batch size 512
268 //
269 // // modulus x^512-1 pos 0:512 q 7681,10753 bound 512*(5629,5800)
270 // assertranges ...
271 //
272 // // ----- LAYER 1
273 //
274 // // butterfly(0,256,1,256,1,0)
275 // butterfly 0 256 1 256 1 0 (0, 1, 2, 3, 4, 5, 6, 7, 8) ()
276 //
277 // // ----- POSTCONDITIONS AFTER LAYER 1
278 // // transform size 512
279 // // transform indexing [0, 1, 2, 3, 4, 5, 6, 7, 8]
280 // // transforms per batch 1
281 // // batch indexing []
282 // // total batch size 512
283 //
284 // // modulus x^256-1 pos 0:256 q 7681,10753 bound 256*(11258,11600)
285 // assertranges ...
286 //
287 // // modulus x^256+1 pos 256:512 q 7681,10753 bound 256*(11258,11600)
288 // assertranges ...
289 //
290 // // ----- LAYER 2
291 //
292 // // reduce_ifreverse(0,64,1)
293 // reduce_ifreverse 0 64 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) ()
294 //
295 // // reduce_ifreverse(256,320,1)
296 // reduce_ifreverse 256 320 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) ()
297 //
298 // // butterfly(0,128,1,128,1,0)
299 // butterfly 0 128 1 128 1 0 (0, 1, 2, 3, 4, 5, 6, 7, 8) ()
300 //
301 // // butterfly(256,384,1,128,4,1)
302 // butterfly 256 384 1 128 4 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) ()
303 //
304 // // ----- POSTCONDITIONS AFTER LAYER 2
305 // // transform size 512
306 // // transform indexing [0, 1, 2, 3, 4, 5, 6, 7, 8]
307 // // transforms per batch 1
308 // // batch indexing []
309 // // total batch size 512
310 //
311 // // modulus x^128-1 pos 0:128 q 7681,10753 bound 128*(22516,23200)
312 // assertranges ...
313 //
314 // // modulus x^128+1 pos 128:256 q 7681,10753 bound 128*(22516,23200)
315 // assertranges ...
316 //
317 // // modulus x^128-zeta4 pos 256:384 q 7681,10753 bound 128*(15747,17016)
318 // assertranges ...
319 //
320 // // modulus x^128+zeta4 pos 384:512 q 7681,10753 bound 128*(15747,17016)
321 // assertranges ...
322 //
323 // // ----- LAYER 3
324 //
325 // // reduce_ifforward(64,128,1)
326 // reduce_ifforward 64 128 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) ()
327 //
328 // // butterfly(0,64,1,64,1,0)
329 // butterfly 0 64 1 64 1 0 (0, 1, 2, 3, 4, 5, 6, 7, 8) ()
330 //
331 // // butterfly(128,192,1,64,4,1)
332 // butterfly 128 192 1 64 4 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) ()
333 //
334 // // butterfly(256,320,1,64,8,1)
335 // butterfly 256 320 1 64 8 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) ()
336 //
337 // // butterfly(384,448,1,64,8,-1)
338 // butterfly 384 448 1 64 8 7 (0, 1, 2, 3, 4, 5, 6, 7, 8) ()
339 //
340 // // reduce(0,64,1)
341 // reduce 0 64 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) ()
342 //
343 // // twist(64,128,1,128,1)
344 // twist 64 128 1 128 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) ()
345 //
346 // // twist(128,192,1,256,1)
347 // twist 128 192 1 256 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) ()
348 //
349 // // twist(192,256,1,256,-1)
350 // twist 192 256 1 256 255 (0, 1, 2, 3, 4, 5, 6, 7, 8) ()
351 //
352 // // twist(256,320,1,512,1)
353 // twist 256 320 1 512 1 (0, 1, 2, 3, 4, 5, 6, 7, 8) ()
354 //
355 // // twist(320,384,1,512,5)
356 // twist 320 384 1 512 5 (0, 1, 2, 3, 4, 5, 6, 7, 8) ()
357 //
358 // // twist(384,448,1,512,-1)
359 // twist 384 448 1 512 511 (0, 1, 2, 3, 4, 5, 6, 7, 8) ()
360 //
361 // // twist(448,512,1,512,-5)
362 // twist 448 512 1 512 507 (0, 1, 2, 3, 4, 5, 6, 7, 8) ()
363 //
364 // // physical_permute(3,6)
365 // physical_permute (3, 6) (0, 1, 2, 3, 4, 5, 6, 7, 8) () (0, 1, 2, 6, 4, 5, 3, 7, 8) ()
366 //
367 // // fold(256)
368 // physical_unmap (0, 1, 2, 6, 4, 5, 3, 7, 8) ()
369 // physical_map (0, 1, 2, 6, 4, 5, 3, 7) (8,)
370 //
371 // // fold(128)
372 // physical_unmap (0, 1, 2, 6, 4, 5, 3, 7) (8,)
373 // physical_map (0, 1, 2, 6, 4, 5, 3) (7, 8)
374 //
375 // // fold(64)
376 // physical_unmap (0, 1, 2, 6, 4, 5, 3) (7, 8)
377 // physical_map (0, 1, 2, 6, 4, 5) (3, 7, 8)
378 //
379 // // nextbatch()
380 // stopbatch 512
381 // startbatch 512
382 //
383 // // halfbatch()
384 // physical_unmap (0, 1, 2, 6, 4, 5) (3, 7, 8)
385 // stopbatch 512
386 // doublereps
387 // startbatch 256
388 // physical_map (0, 1, 2, 6, 4, 5) (3, 7)
389 //
390 // // halfbatch()
391 // physical_unmap (0, 1, 2, 6, 4, 5) (3, 7)
392 // stopbatch 256
393 // doublereps
394 // startbatch 128
395 // physical_map (0, 1, 2, 6, 4, 5) (3,)
396 //
397 // // ----- POSTCONDITIONS AFTER LAYER 3
398 // // transform size 64
399 // // transform indexing [0, 1, 2, 6, 4, 5]
400 // // transforms per batch 2
401 // // batch indexing [3]
402 // // total batch size 128
403 //
404 // // modulus x^64-1 pos 0:64 q 7681,10753 bound 1*(5629,5827) 1*(5629,7613) 1*(5629,7666) 1*(5629,7264) 1*(5629,7639) 1*(5629,7591) 1*(5629,7291) 1*(5629,7204) ...
405 // assertranges ...
406 //
407 // // ----- LAYER 4
408 //
409 // // butterfly(0,32,1,32,1,0)
410 // butterfly 0 32 1 32 1 0 (0, 1, 2, 6, 4, 5) (3,)
411 //
412 // // ----- POSTCONDITIONS AFTER LAYER 4
413 // // transform size 64
414 // // transform indexing [0, 1, 2, 6, 4, 5]
415 // // transforms per batch 2
416 // // batch indexing [3]
417 // // total batch size 128
418 //
419 // // modulus x^32-1 pos 0:32 q 7681,10753 bound 1*(11258,13035) 1*(11258,14721) 1*(11258,14855) 1*(11258,14877) 1*(11258,14690) 1*(11258,15282) 1*(11258,14641) 1*(11258,14272) ...
420 // assertranges ...
421 //
422 // // modulus x^32+1 pos 32:64 q 7681,10753 bound 1*(11258,13035) 1*(11258,14721) 1*(11258,14855) 1*(11258,14877) 1*(11258,14690) 1*(11258,15282) 1*(11258,14641) 1*(11258,14272) ...
423 // assertranges ...
424 //
425 // // ----- LAYER 5
426 //
427 // // butterfly(0,16,1,16,1,0)
428 // butterfly 0 16 1 16 1 0 (0, 1, 2, 6, 4, 5) (3,)
429 //
430 // // butterfly(32,48,1,16,4,1)
431 // butterfly 32 48 1 16 4 1 (0, 1, 2, 6, 4, 5) (3,)
432 //
433 // // reduce(0,16,1)
434 // reduce 0 16 1 (0, 1, 2, 6, 4, 5) (3,)
435 //
436 // // twist(16,32,1,32,1)
437 // twist 16 32 1 32 1 (0, 1, 2, 6, 4, 5) (3,)
438 //
439 // // twist(32,48,1,64,1)
440 // twist 32 48 1 64 1 (0, 1, 2, 6, 4, 5) (3,)
441 //
442 // // twist(48,64,1,64,-1)
443 // twist 48 64 1 64 63 (0, 1, 2, 6, 4, 5) (3,)
444 //
445 // // physical_permute(0,1,2,5)
446 // physical_permute (0, 1, 2, 5) (0, 1, 2, 6, 4, 5) (3,) (1, 2, 5, 6, 4, 0) (3,)
447 //
448 // // fold(32)
449 // physical_unmap (1, 2, 5, 6, 4, 0) (3,)
450 // physical_map (1, 2, 5, 6, 4) (0, 3)
451 //
452 // // fold(16)
453 // physical_unmap (1, 2, 5, 6, 4) (0, 3)
454 // physical_map (1, 2, 5, 6) (0, 3, 4)
455 //
456 // // ----- POSTCONDITIONS AFTER LAYER 5
457 // // transform size 16
458 // // transform indexing [1, 2, 5, 6]
459 // // transforms per batch 8
460 // // batch indexing [0, 3, 4]
461 // // total batch size 128
462 //
463 // // modulus x^16-1 pos 0:16 q 7681,10753 bound 1*(5629,5800) 1*(5629,6967) 1*(5629,6418) 1*(5629,7585) 1*(5629,7017) 1*(5629,6328) 1*(5629,7033) 1*(5629,6943) ...
464 // assertranges ...
465 //
466 // // ----- LAYER 6
467 //
468 // // butterfly(0,8,1,8,1,0)
469 // butterfly 0 8 1 8 1 0 (1, 2, 5, 6) (0, 3, 4)
470 //
471 // // physical_permute(1,2,4)
472 // physical_permute (1, 2, 4) (1, 2, 5, 6) (0, 3, 4) (2, 4, 5, 6) (0, 3, 1)
473 //
474 // // nextbatch()
475 // stopbatch 128
476 // startbatch 128
477 //
478 // // ----- POSTCONDITIONS AFTER LAYER 6
479 // // transform size 16
480 // // transform indexing [2, 4, 5, 6]
481 // // transforms per batch 8
482 // // batch indexing [0, 3, 1]
483 // // total batch size 128
484 //
485 // // modulus x^8-1 pos 0:8 q 7681,10753 bound 1*(11258,12447) 1*(11258,14071) 1*(11258,12488) 1*(11258,14310) 1*(11258,14287) 1*(11258,13674) 1*(11258,13574) 1*(11258,13555)
486 // assertranges ...
487 //
488 // // modulus x^8+1 pos 8:16 q 7681,10753 bound 1*(11258,12447) 1*(11258,14071) 1*(11258,12488) 1*(11258,14310) 1*(11258,14287) 1*(11258,13674) 1*(11258,13574) 1*(11258,13555)
489 // assertranges ...
490 //
491 // // ----- LAYER 7
492 //
493 // // butterfly(0,4,1,4,1,0)
494 // butterfly 0 4 1 4 1 0 (2, 4, 5, 6) (0, 3, 1)
495 //
496 // // butterfly(8,12,1,4,4,1)
497 // butterfly 8 12 1 4 4 1 (2, 4, 5, 6) (0, 3, 1)
498 //
499 // // reduce(0,4,1)
500 // reduce 0 4 1 (2, 4, 5, 6) (0, 3, 1)
501 //
502 // // twist(4,8,1,8,1)
503 // twist 4 8 1 8 1 (2, 4, 5, 6) (0, 3, 1)
504 //
505 // // twist(8,12,1,16,1)
506 // twist 8 12 1 16 1 (2, 4, 5, 6) (0, 3, 1)
507 //
508 // // twist(12,16,1,16,-1)
509 // twist 12 16 1 16 15 (2, 4, 5, 6) (0, 3, 1)
510 //
511 // // physical_permute(2,6)
512 // physical_permute (2, 6) (2, 4, 5, 6) (0, 3, 1) (6, 4, 5, 2) (0, 3, 1)
513 //
514 // // fold(8)
515 // physical_unmap (6, 4, 5, 2) (0, 3, 1)
516 // physical_map (6, 4, 5) (0, 1, 2, 3)
517 //
518 // // fold(4)
519 // physical_unmap (6, 4, 5) (0, 1, 2, 3)
520 // physical_map (6, 4) (0, 1, 2, 3, 5)
521 //
522 // // ----- POSTCONDITIONS AFTER LAYER 7
523 // // transform size 4
524 // // transform indexing [6, 4]
525 // // transforms per batch 32
526 // // batch indexing [0, 1, 2, 3, 5]
527 // // total batch size 128
528 //
529 // // modulus x^4-1 pos 0:4 q 7681,10753 bound 1*(5629,5800) 1*(5629,6938) 1*(5629,6521) 1*(5629,7157)
530 // assertranges ...
531 //
532 // // ----- LAYER 8
533 //
534 // // butterfly(0,2,1,2,1,0)
535 // butterfly 0 2 1 2 1 0 (6, 4) (0, 1, 2, 3, 5)
536 //
537 // // ----- POSTCONDITIONS AFTER LAYER 8
538 // // transform size 4
539 // // transform indexing [6, 4]
540 // // transforms per batch 32
541 // // batch indexing [0, 1, 2, 3, 5]
542 // // total batch size 128
543 //
544 // // modulus x^2-1 pos 0:2 q 7681,10753 bound 1*(11258,12321) 1*(11258,14095)
545 // assertranges ...
546 //
547 // // modulus x^2+1 pos 2:4 q 7681,10753 bound 1*(11258,12321) 1*(11258,14095)
548 // assertranges ...
549 //
550 // // ----- LAYER 9
551 //
552 // // butterfly(0,1,1,1,1,0)
553 // butterfly 0 1 1 1 1 0 (6, 4) (0, 1, 2, 3, 5)
554 //
555 // // butterfly(2,3,1,1,4,1)
556 // butterfly 2 3 1 1 4 1 (6, 4) (0, 1, 2, 3, 5)
557 //
558 // // ----- POSTCONDITIONS AFTER LAYER 9
559 // // transform size 4
560 // // transform indexing [6, 4]
561 // // transforms per batch 32
562 // // batch indexing [0, 1, 2, 3, 5]
563 // // total batch size 128
564 //
565 // // modulus x^1-1 pos 0:1 q 7681,10753 bound 1*(22516,26416)
566 // assertranges ...
567 //
568 // // modulus x^1+1 pos 1:2 q 7681,10753 bound 1*(22516,26416)
569 // assertranges ...
570 //
571 // // modulus x^1-zeta4 pos 2:3 q 7681,10753 bound 1*(15747,17745)
572 // assertranges ...
573 //
574 // // modulus x^1+zeta4 pos 3:4 q 7681,10753 bound 1*(15747,17745)
575 // assertranges ...
576 // stopbatch 128
577 // physical_unmap (6, 4) (0, 1, 2, 3, 5)
578 // stopntt 512
579
580 // ----- codegen pass 2
581 //
582 // startntt 512
583 // startbatch 512
584 // vector_butterfly 0 256 1 0
585 // vector_butterfly 128 384 1 0
586 // vector_butterfly 64 320 1 0
587 // vector_butterfly 192 448 1 0
588 // vector_reduce_ifreverse 0
589 // vector_reduce_ifreverse 256
590 // vector_butterfly 0 128 1 0
591 // vector_butterfly 64 192 1 0
592 // vector_butterfly 256 384 4 1
593 // vector_butterfly 320 448 4 1
594 // vector_reduce_ifforward 64
595 // vector_butterfly 0 64 1 0
596 // vector_butterfly 128 192 4 1
597 // vector_butterfly 256 320 8 1
598 // vector_butterfly 384 448 8 7
599 // vector_reduce 0
600 // vector_twist 64 128 1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
601 // vector_twist 128 256 1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
602 // vector_twist 192 256 255 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
603 // vector_twist 256 512 1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
604 // vector_twist 320 512 5 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
605 // vector_twist 384 512 511 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
606 // vector_twist 448 512 507 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
607 // vector_permute 0 64 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
608 // vector_permute 128 192 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
609 // vector_permute 256 320 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
610 // vector_permute 384 448 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
611 // stopbatch 512
612 // startbatch 512
613 // vector_butterfly 16 272 1 0
614 // vector_butterfly 144 400 1 0
615 // vector_butterfly 80 336 1 0
616 // vector_butterfly 208 464 1 0
617 // vector_reduce_ifreverse 16
618 // vector_reduce_ifreverse 272
619 // vector_butterfly 16 144 1 0
620 // vector_butterfly 80 208 1 0
621 // vector_butterfly 272 400 4 1
622 // vector_butterfly 336 464 4 1
623 // vector_reduce_ifforward 80
624 // vector_butterfly 16 80 1 0
625 // vector_butterfly 144 208 4 1
626 // vector_butterfly 272 336 8 1
627 // vector_butterfly 400 464 8 7
628 // vector_reduce 16
629 // vector_twist 80 128 1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
630 // vector_twist 144 256 1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
631 // vector_twist 208 256 255 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
632 // vector_twist 272 512 1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
633 // vector_twist 336 512 5 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
634 // vector_twist 400 512 511 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
635 // vector_twist 464 512 507 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
636 // vector_permute 16 80 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
637 // vector_permute 144 208 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
638 // vector_permute 272 336 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
639 // vector_permute 400 464 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
640 // stopbatch 512
641 // startbatch 512
642 // vector_butterfly 32 288 1 0
643 // vector_butterfly 160 416 1 0
644 // vector_butterfly 96 352 1 0
645 // vector_butterfly 224 480 1 0
646 // vector_reduce_ifreverse 32
647 // vector_reduce_ifreverse 288
648 // vector_butterfly 32 160 1 0
649 // vector_butterfly 96 224 1 0
650 // vector_butterfly 288 416 4 1
651 // vector_butterfly 352 480 4 1
652 // vector_reduce_ifforward 96
653 // vector_butterfly 32 96 1 0
654 // vector_butterfly 160 224 4 1
655 // vector_butterfly 288 352 8 1
656 // vector_butterfly 416 480 8 7
657 // vector_reduce 32
658 // vector_twist 96 128 1 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
659 // vector_twist 160 256 1 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
660 // vector_twist 224 256 255 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
661 // vector_twist 288 512 1 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
662 // vector_twist 352 512 5 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
663 // vector_twist 416 512 511 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
664 // vector_twist 480 512 507 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
665 // vector_permute 32 96 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
666 // vector_permute 160 224 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
667 // vector_permute 288 352 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
668 // vector_permute 416 480 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
669 // stopbatch 512
670 // startbatch 512
671 // vector_butterfly 48 304 1 0
672 // vector_butterfly 176 432 1 0
673 // vector_butterfly 112 368 1 0
674 // vector_butterfly 240 496 1 0
675 // vector_reduce_ifreverse 48
676 // vector_reduce_ifreverse 304
677 // vector_butterfly 48 176 1 0
678 // vector_butterfly 112 240 1 0
679 // vector_butterfly 304 432 4 1
680 // vector_butterfly 368 496 4 1
681 // vector_reduce_ifforward 112
682 // vector_butterfly 48 112 1 0
683 // vector_butterfly 176 240 4 1
684 // vector_butterfly 304 368 8 1
685 // vector_butterfly 432 496 8 7
686 // vector_reduce 48
687 // vector_twist 112 128 1 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
688 // vector_twist 176 256 1 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
689 // vector_twist 240 256 255 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
690 // vector_twist 304 512 1 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
691 // vector_twist 368 512 5 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
692 // vector_twist 432 512 511 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
693 // vector_twist 496 512 507 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
694 // vector_permute 48 112 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
695 // vector_permute 176 240 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
696 // vector_permute 304 368 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
697 // vector_permute 432 496 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
698 // stopbatch 512
699 // doublereps
700 // doublereps
701 // startbatch 128
702 // vector_butterfly 0 32 1 0
703 // vector_butterfly 64 96 1 0
704 // vector_butterfly 16 48 1 0
705 // vector_butterfly 80 112 1 0
706 // vector_butterfly 0 16 1 0
707 // vector_butterfly 64 80 1 0
708 // vector_butterfly 32 48 4 1
709 // vector_butterfly 96 112 4 1
710 // vector_reduce 0
711 // vector_reduce 64
712 // vector_twist 16 32 1 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7
713 // vector_twist 80 32 1 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15
714 // vector_twist 32 64 1 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7
715 // vector_twist 96 64 1 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15
716 // vector_twist 48 64 63 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7
717 // vector_twist 112 64 63 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15
718 // vector_permute 0 32 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16
719 // vector_permute 16 48 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16
720 // vector_permute 64 96 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16
721 // vector_permute 80 112 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16
722 // vector_butterfly 0 64 1 0
723 // vector_butterfly 32 96 1 0
724 // vector_butterfly 16 80 1 0
725 // vector_butterfly 48 112 1 0
726 // vector_permute 0 16 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32
727 // vector_permute 32 48 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32
728 // vector_permute 64 80 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32
729 // vector_permute 96 112 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32
730 // stopbatch 128
731 // startbatch 128
732 // vector_butterfly 0 32 1 0
733 // vector_butterfly 16 48 1 0
734 // vector_butterfly 64 96 4 1
735 // vector_butterfly 80 112 4 1
736 // vector_reduce 0
737 // vector_reduce 16
738 // vector_twist 32 8 1 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1
739 // vector_twist 48 8 1 2 2 2 2 3 3 3 3 2 2 2 2 3 3 3 3
740 // vector_twist 64 16 1 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1
741 // vector_twist 80 16 1 2 2 2 2 3 3 3 3 2 2 2 2 3 3 3 3
742 // vector_twist 96 16 15 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1
743 // vector_twist 112 16 15 2 2 2 2 3 3 3 3 2 2 2 2 3 3 3 3
744 // vector_permute 0 64 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64
745 // vector_permute 16 80 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64
746 // vector_permute 32 96 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64
747 // vector_permute 48 112 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64
748 // vector_butterfly 0 16 1 0
749 // vector_butterfly 64 80 1 0
750 // vector_butterfly 32 48 1 0
751 // vector_butterfly 96 112 1 0
752 // vector_butterfly 0 64 1 0
753 // vector_butterfly 32 96 1 0
754 // vector_butterfly 16 80 4 1
755 // vector_butterfly 48 112 4 1
756 // stopbatch 128
757 // stopntt 512
758 // startntt 512
759
760 static void ntt512(int16 *f,int reps,const int16 *qdata)
761 {
762 // startbatch 512
763 for (long long r = 0;r < reps;++r) {
764 // vector_butterfly 0 256 1 0
765 int16x16 a0 = _mm256_loadu_si256((int16x16 *) (f+0));
766 int16x16 a16 = _mm256_loadu_si256((int16x16 *) (f+256));
767 int16x16 b0 = add_x16(a0,a16);
768 int16x16 b16 = sub_x16(a0,a16);
769 // vector_butterfly 128 384 1 0
770 int16x16 a8 = _mm256_loadu_si256((int16x16 *) (f+128));
771 int16x16 a24 = _mm256_loadu_si256((int16x16 *) (f+384));
772 int16x16 b8 = add_x16(a8,a24);
773 int16x16 b24 = sub_x16(a8,a24);
774 // vector_butterfly 64 320 1 0
775 int16x16 a4 = _mm256_loadu_si256((int16x16 *) (f+64));
776 int16x16 a20 = _mm256_loadu_si256((int16x16 *) (f+320));
777 int16x16 b4 = add_x16(a4,a20);
778 int16x16 b20 = sub_x16(a4,a20);
779 // vector_butterfly 192 448 1 0
780 int16x16 a12 = _mm256_loadu_si256((int16x16 *) (f+192));
781 int16x16 a28 = _mm256_loadu_si256((int16x16 *) (f+448));
782 int16x16 b12 = add_x16(a12,a28);
783 int16x16 b28 = sub_x16(a12,a28);
784 // vector_reduce_ifreverse 0
785 // vector_reduce_ifreverse 256
786 // vector_butterfly 0 128 1 0
787 int16x16 c0 = add_x16(b0,b8);
788 int16x16 c8 = sub_x16(b0,b8);
789 // vector_butterfly 64 192 1 0
790 int16x16 c4 = add_x16(b4,b12);
791 int16x16 c12 = sub_x16(b4,b12);
792 // vector_butterfly 256 384 4 1
793 b24 = mulmod_scaled_x16(b24,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata);
794 int16x16 c16 = add_x16(b16,b24);
795 int16x16 c24 = sub_x16(b16,b24);
796 // vector_butterfly 320 448 4 1
797 b28 = mulmod_scaled_x16(b28,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata);
798 int16x16 c20 = add_x16(b20,b28);
799 int16x16 c28 = sub_x16(b20,b28);
800 // vector_reduce_ifforward 64
801 c4 = reduce_x16(c4,qdata);
802 // vector_butterfly 0 64 1 0
803 int16x16 d0 = add_x16(c0,c4);
804 int16x16 d4 = sub_x16(c0,c4);
805 // vector_butterfly 128 192 4 1
806 c12 = mulmod_scaled_x16(c12,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata);
807 int16x16 d8 = add_x16(c8,c12);
808 int16x16 d12 = sub_x16(c8,c12);
809 // vector_butterfly 256 320 8 1
810 c20 = mulmod_scaled_x16(c20,scaledzeta_x16_8_1,qinvscaledzeta_x16_8_1,qdata);
811 int16x16 d16 = add_x16(c16,c20);
812 int16x16 d20 = sub_x16(c16,c20);
813 // vector_butterfly 384 448 8 7
814 c28 = mulmod_scaled_x16(c28,scaledzeta_x16_8_7,qinvscaledzeta_x16_8_7,qdata);
815 int16x16 d24 = add_x16(c24,c28);
816 int16x16 d28 = sub_x16(c24,c28);
817 // vector_reduce 0
818 d0 = reduce_x16(d0,qdata);
819 // vector_twist 64 128 1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
820 d4 = mulmod_scaled_x16(d4,precomp_128_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_128_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata);
821 // vector_twist 128 256 1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
822 d8 = mulmod_scaled_x16(d8,precomp_256_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_256_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata);
823 // vector_twist 192 256 255 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
824 d12 = mulmod_scaled_x16(d12,precomp_256_255_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_256_255_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata);
825 // vector_twist 256 512 1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
826 d16 = mulmod_scaled_x16(d16,precomp_512_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_512_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata);
827 // vector_twist 320 512 5 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
828 d20 = mulmod_scaled_x16(d20,precomp_512_5_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_512_5_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata);
829 // vector_twist 384 512 511 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
830 d24 = mulmod_scaled_x16(d24,precomp_512_511_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_512_511_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata);
831 // vector_twist 448 512 507 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
832 d28 = mulmod_scaled_x16(d28,precomp_512_507_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_512_507_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata);
833 // vector_permute 0 64 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
834 int16x16 e0 = _mm256_permute2x128_si256_lo(d0,d4);
835 int16x16 e4 = _mm256_permute2x128_si256_hi(d0,d4);
836 // vector_permute 128 192 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
837 int16x16 e8 = _mm256_permute2x128_si256_lo(d8,d12);
838 int16x16 e12 = _mm256_permute2x128_si256_hi(d8,d12);
839 // vector_permute 256 320 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
840 int16x16 e16 = _mm256_permute2x128_si256_lo(d16,d20);
841 int16x16 e20 = _mm256_permute2x128_si256_hi(d16,d20);
842 // vector_permute 384 448 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
843 int16x16 e24 = _mm256_permute2x128_si256_lo(d24,d28);
844 int16x16 e28 = _mm256_permute2x128_si256_hi(d24,d28);
845 // stopbatch 512
846 _mm256_storeu_si256((int16x16 *) (f+0),e0);
847 _mm256_storeu_si256((int16x16 *) (f+64),e4);
848 _mm256_storeu_si256((int16x16 *) (f+128),e8);
849 _mm256_storeu_si256((int16x16 *) (f+192),e12);
850 _mm256_storeu_si256((int16x16 *) (f+256),e16);
851 _mm256_storeu_si256((int16x16 *) (f+320),e20);
852 _mm256_storeu_si256((int16x16 *) (f+384),e24);
853 _mm256_storeu_si256((int16x16 *) (f+448),e28);
854 f += 512;
855 }
856 f -= 512*reps;
857 // startbatch 512
858 for (long long r = 0;r < reps;++r) {
859 // vector_butterfly 16 272 1 0
860 int16x16 a1 = _mm256_loadu_si256((int16x16 *) (f+16));
861 int16x16 a17 = _mm256_loadu_si256((int16x16 *) (f+272));
862 int16x16 b1 = add_x16(a1,a17);
863 int16x16 b17 = sub_x16(a1,a17);
864 // vector_butterfly 144 400 1 0
865 int16x16 a9 = _mm256_loadu_si256((int16x16 *) (f+144));
866 int16x16 a25 = _mm256_loadu_si256((int16x16 *) (f+400));
867 int16x16 b9 = add_x16(a9,a25);
868 int16x16 b25 = sub_x16(a9,a25);
869 // vector_butterfly 80 336 1 0
870 int16x16 a5 = _mm256_loadu_si256((int16x16 *) (f+80));
871 int16x16 a21 = _mm256_loadu_si256((int16x16 *) (f+336));
872 int16x16 b5 = add_x16(a5,a21);
873 int16x16 b21 = sub_x16(a5,a21);
874 // vector_butterfly 208 464 1 0
875 int16x16 a13 = _mm256_loadu_si256((int16x16 *) (f+208));
876 int16x16 a29 = _mm256_loadu_si256((int16x16 *) (f+464));
877 int16x16 b13 = add_x16(a13,a29);
878 int16x16 b29 = sub_x16(a13,a29);
879 // vector_reduce_ifreverse 16
880 // vector_reduce_ifreverse 272
881 // vector_butterfly 16 144 1 0
882 int16x16 c1 = add_x16(b1,b9);
883 int16x16 c9 = sub_x16(b1,b9);
884 // vector_butterfly 80 208 1 0
885 int16x16 c5 = add_x16(b5,b13);
886 int16x16 c13 = sub_x16(b5,b13);
887 // vector_butterfly 272 400 4 1
888 b25 = mulmod_scaled_x16(b25,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata);
889 int16x16 c17 = add_x16(b17,b25);
890 int16x16 c25 = sub_x16(b17,b25);
891 // vector_butterfly 336 464 4 1
892 b29 = mulmod_scaled_x16(b29,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata);
893 int16x16 c21 = add_x16(b21,b29);
894 int16x16 c29 = sub_x16(b21,b29);
895 // vector_reduce_ifforward 80
896 c5 = reduce_x16(c5,qdata);
897 // vector_butterfly 16 80 1 0
898 int16x16 d1 = add_x16(c1,c5);
899 int16x16 d5 = sub_x16(c1,c5);
900 // vector_butterfly 144 208 4 1
901 c13 = mulmod_scaled_x16(c13,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata);
902 int16x16 d9 = add_x16(c9,c13);
903 int16x16 d13 = sub_x16(c9,c13);
904 // vector_butterfly 272 336 8 1
905 c21 = mulmod_scaled_x16(c21,scaledzeta_x16_8_1,qinvscaledzeta_x16_8_1,qdata);
906 int16x16 d17 = add_x16(c17,c21);
907 int16x16 d21 = sub_x16(c17,c21);
908 // vector_butterfly 400 464 8 7
909 c29 = mulmod_scaled_x16(c29,scaledzeta_x16_8_7,qinvscaledzeta_x16_8_7,qdata);
910 int16x16 d25 = add_x16(c25,c29);
911 int16x16 d29 = sub_x16(c25,c29);
912 // vector_reduce 16
913 d1 = reduce_x16(d1,qdata);
914 // vector_twist 80 128 1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
915 d5 = mulmod_scaled_x16(d5,precomp_128_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_128_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata);
916 // vector_twist 144 256 1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
917 d9 = mulmod_scaled_x16(d9,precomp_256_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_256_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata);
918 // vector_twist 208 256 255 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
919 d13 = mulmod_scaled_x16(d13,precomp_256_255_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_256_255_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata);
920 // vector_twist 272 512 1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
921 d17 = mulmod_scaled_x16(d17,precomp_512_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_512_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata);
922 // vector_twist 336 512 5 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
923 d21 = mulmod_scaled_x16(d21,precomp_512_5_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_512_5_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata);
924 // vector_twist 400 512 511 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
925 d25 = mulmod_scaled_x16(d25,precomp_512_511_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_512_511_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata);
926 // vector_twist 464 512 507 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
927 d29 = mulmod_scaled_x16(d29,precomp_512_507_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_512_507_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata);
928 // vector_permute 16 80 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
929 int16x16 e1 = _mm256_permute2x128_si256_lo(d1,d5);
930 int16x16 e5 = _mm256_permute2x128_si256_hi(d1,d5);
931 // vector_permute 144 208 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
932 int16x16 e9 = _mm256_permute2x128_si256_lo(d9,d13);
933 int16x16 e13 = _mm256_permute2x128_si256_hi(d9,d13);
934 // vector_permute 272 336 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
935 int16x16 e17 = _mm256_permute2x128_si256_lo(d17,d21);
936 int16x16 e21 = _mm256_permute2x128_si256_hi(d17,d21);
937 // vector_permute 400 464 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
938 int16x16 e25 = _mm256_permute2x128_si256_lo(d25,d29);
939 int16x16 e29 = _mm256_permute2x128_si256_hi(d25,d29);
940 // stopbatch 512
941 _mm256_storeu_si256((int16x16 *) (f+16),e1);
942 _mm256_storeu_si256((int16x16 *) (f+80),e5);
943 _mm256_storeu_si256((int16x16 *) (f+144),e9);
944 _mm256_storeu_si256((int16x16 *) (f+208),e13);
945 _mm256_storeu_si256((int16x16 *) (f+272),e17);
946 _mm256_storeu_si256((int16x16 *) (f+336),e21);
947 _mm256_storeu_si256((int16x16 *) (f+400),e25);
948 _mm256_storeu_si256((int16x16 *) (f+464),e29);
949 f += 512;
950 }
951 f -= 512*reps;
952 // startbatch 512
953 for (long long r = 0;r < reps;++r) {
954 // vector_butterfly 32 288 1 0
955 int16x16 a2 = _mm256_loadu_si256((int16x16 *) (f+32));
956 int16x16 a18 = _mm256_loadu_si256((int16x16 *) (f+288));
957 int16x16 b2 = add_x16(a2,a18);
958 int16x16 b18 = sub_x16(a2,a18);
959 // vector_butterfly 160 416 1 0
960 int16x16 a10 = _mm256_loadu_si256((int16x16 *) (f+160));
961 int16x16 a26 = _mm256_loadu_si256((int16x16 *) (f+416));
962 int16x16 b10 = add_x16(a10,a26);
963 int16x16 b26 = sub_x16(a10,a26);
964 // vector_butterfly 96 352 1 0
965 int16x16 a6 = _mm256_loadu_si256((int16x16 *) (f+96));
966 int16x16 a22 = _mm256_loadu_si256((int16x16 *) (f+352));
967 int16x16 b6 = add_x16(a6,a22);
968 int16x16 b22 = sub_x16(a6,a22);
969 // vector_butterfly 224 480 1 0
970 int16x16 a14 = _mm256_loadu_si256((int16x16 *) (f+224));
971 int16x16 a30 = _mm256_loadu_si256((int16x16 *) (f+480));
972 int16x16 b14 = add_x16(a14,a30);
973 int16x16 b30 = sub_x16(a14,a30);
974 // vector_reduce_ifreverse 32
975 // vector_reduce_ifreverse 288
976 // vector_butterfly 32 160 1 0
977 int16x16 c2 = add_x16(b2,b10);
978 int16x16 c10 = sub_x16(b2,b10);
979 // vector_butterfly 96 224 1 0
980 int16x16 c6 = add_x16(b6,b14);
981 int16x16 c14 = sub_x16(b6,b14);
982 // vector_butterfly 288 416 4 1
983 b26 = mulmod_scaled_x16(b26,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata);
984 int16x16 c18 = add_x16(b18,b26);
985 int16x16 c26 = sub_x16(b18,b26);
986 // vector_butterfly 352 480 4 1
987 b30 = mulmod_scaled_x16(b30,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata);
988 int16x16 c22 = add_x16(b22,b30);
989 int16x16 c30 = sub_x16(b22,b30);
990 // vector_reduce_ifforward 96
991 c6 = reduce_x16(c6,qdata);
992 // vector_butterfly 32 96 1 0
993 int16x16 d2 = add_x16(c2,c6);
994 int16x16 d6 = sub_x16(c2,c6);
995 // vector_butterfly 160 224 4 1
996 c14 = mulmod_scaled_x16(c14,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata);
997 int16x16 d10 = add_x16(c10,c14);
998 int16x16 d14 = sub_x16(c10,c14);
999 // vector_butterfly 288 352 8 1
1000 c22 = mulmod_scaled_x16(c22,scaledzeta_x16_8_1,qinvscaledzeta_x16_8_1,qdata);
1001 int16x16 d18 = add_x16(c18,c22);
1002 int16x16 d22 = sub_x16(c18,c22);
1003 // vector_butterfly 416 480 8 7
1004 c30 = mulmod_scaled_x16(c30,scaledzeta_x16_8_7,qinvscaledzeta_x16_8_7,qdata);
1005 int16x16 d26 = add_x16(c26,c30);
1006 int16x16 d30 = sub_x16(c26,c30);
1007 // vector_reduce 32
1008 d2 = reduce_x16(d2,qdata);
1009 // vector_twist 96 128 1 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
1010 d6 = mulmod_scaled_x16(d6,precomp_128_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_128_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata);
1011 // vector_twist 160 256 1 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
1012 d10 = mulmod_scaled_x16(d10,precomp_256_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_256_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata);
1013 // vector_twist 224 256 255 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
1014 d14 = mulmod_scaled_x16(d14,precomp_256_255_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_256_255_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata);
1015 // vector_twist 288 512 1 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
1016 d18 = mulmod_scaled_x16(d18,precomp_512_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_512_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata);
1017 // vector_twist 352 512 5 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
1018 d22 = mulmod_scaled_x16(d22,precomp_512_5_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_512_5_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata);
1019 // vector_twist 416 512 511 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
1020 d26 = mulmod_scaled_x16(d26,precomp_512_511_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_512_511_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata);
1021 // vector_twist 480 512 507 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
1022 d30 = mulmod_scaled_x16(d30,precomp_512_507_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_512_507_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata);
1023 // vector_permute 32 96 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
1024 int16x16 e2 = _mm256_permute2x128_si256_lo(d2,d6);
1025 int16x16 e6 = _mm256_permute2x128_si256_hi(d2,d6);
1026 // vector_permute 160 224 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
1027 int16x16 e10 = _mm256_permute2x128_si256_lo(d10,d14);
1028 int16x16 e14 = _mm256_permute2x128_si256_hi(d10,d14);
1029 // vector_permute 288 352 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
1030 int16x16 e18 = _mm256_permute2x128_si256_lo(d18,d22);
1031 int16x16 e22 = _mm256_permute2x128_si256_hi(d18,d22);
1032 // vector_permute 416 480 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
1033 int16x16 e26 = _mm256_permute2x128_si256_lo(d26,d30);
1034 int16x16 e30 = _mm256_permute2x128_si256_hi(d26,d30);
1035 // stopbatch 512
1036 _mm256_storeu_si256((int16x16 *) (f+32),e2);
1037 _mm256_storeu_si256((int16x16 *) (f+96),e6);
1038 _mm256_storeu_si256((int16x16 *) (f+160),e10);
1039 _mm256_storeu_si256((int16x16 *) (f+224),e14);
1040 _mm256_storeu_si256((int16x16 *) (f+288),e18);
1041 _mm256_storeu_si256((int16x16 *) (f+352),e22);
1042 _mm256_storeu_si256((int16x16 *) (f+416),e26);
1043 _mm256_storeu_si256((int16x16 *) (f+480),e30);
1044 f += 512;
1045 }
1046 f -= 512*reps;
1047 // startbatch 512
1048 for (long long r = 0;r < reps;++r) {
1049 // vector_butterfly 48 304 1 0
1050 int16x16 a3 = _mm256_loadu_si256((int16x16 *) (f+48));
1051 int16x16 a19 = _mm256_loadu_si256((int16x16 *) (f+304));
1052 int16x16 b3 = add_x16(a3,a19);
1053 int16x16 b19 = sub_x16(a3,a19);
1054 // vector_butterfly 176 432 1 0
1055 int16x16 a11 = _mm256_loadu_si256((int16x16 *) (f+176));
1056 int16x16 a27 = _mm256_loadu_si256((int16x16 *) (f+432));
1057 int16x16 b11 = add_x16(a11,a27);
1058 int16x16 b27 = sub_x16(a11,a27);
1059 // vector_butterfly 112 368 1 0
1060 int16x16 a7 = _mm256_loadu_si256((int16x16 *) (f+112));
1061 int16x16 a23 = _mm256_loadu_si256((int16x16 *) (f+368));
1062 int16x16 b7 = add_x16(a7,a23);
1063 int16x16 b23 = sub_x16(a7,a23);
1064 // vector_butterfly 240 496 1 0
1065 int16x16 a15 = _mm256_loadu_si256((int16x16 *) (f+240));
1066 int16x16 a31 = _mm256_loadu_si256((int16x16 *) (f+496));
1067 int16x16 b15 = add_x16(a15,a31);
1068 int16x16 b31 = sub_x16(a15,a31);
1069 // vector_reduce_ifreverse 48
1070 // vector_reduce_ifreverse 304
1071 // vector_butterfly 48 176 1 0
1072 int16x16 c3 = add_x16(b3,b11);
1073 int16x16 c11 = sub_x16(b3,b11);
1074 // vector_butterfly 112 240 1 0
1075 int16x16 c7 = add_x16(b7,b15);
1076 int16x16 c15 = sub_x16(b7,b15);
1077 // vector_butterfly 304 432 4 1
1078 b27 = mulmod_scaled_x16(b27,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata);
1079 int16x16 c19 = add_x16(b19,b27);
1080 int16x16 c27 = sub_x16(b19,b27);
1081 // vector_butterfly 368 496 4 1
1082 b31 = mulmod_scaled_x16(b31,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata);
1083 int16x16 c23 = add_x16(b23,b31);
1084 int16x16 c31 = sub_x16(b23,b31);
1085 // vector_reduce_ifforward 112
1086 c7 = reduce_x16(c7,qdata);
1087 // vector_butterfly 48 112 1 0
1088 int16x16 d3 = add_x16(c3,c7);
1089 int16x16 d7 = sub_x16(c3,c7);
1090 // vector_butterfly 176 240 4 1
1091 c15 = mulmod_scaled_x16(c15,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata);
1092 int16x16 d11 = add_x16(c11,c15);
1093 int16x16 d15 = sub_x16(c11,c15);
1094 // vector_butterfly 304 368 8 1
1095 c23 = mulmod_scaled_x16(c23,scaledzeta_x16_8_1,qinvscaledzeta_x16_8_1,qdata);
1096 int16x16 d19 = add_x16(c19,c23);
1097 int16x16 d23 = sub_x16(c19,c23);
1098 // vector_butterfly 432 496 8 7
1099 c31 = mulmod_scaled_x16(c31,scaledzeta_x16_8_7,qinvscaledzeta_x16_8_7,qdata);
1100 int16x16 d27 = add_x16(c27,c31);
1101 int16x16 d31 = sub_x16(c27,c31);
1102 // vector_reduce 48
1103 d3 = reduce_x16(d3,qdata);
1104 // vector_twist 112 128 1 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
1105 d7 = mulmod_scaled_x16(d7,precomp_128_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_128_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata);
1106 // vector_twist 176 256 1 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
1107 d11 = mulmod_scaled_x16(d11,precomp_256_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_256_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata);
1108 // vector_twist 240 256 255 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
1109 d15 = mulmod_scaled_x16(d15,precomp_256_255_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_256_255_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata);
1110 // vector_twist 304 512 1 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
1111 d19 = mulmod_scaled_x16(d19,precomp_512_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_512_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata);
1112 // vector_twist 368 512 5 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
1113 d23 = mulmod_scaled_x16(d23,precomp_512_5_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_512_5_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata);
1114 // vector_twist 432 512 511 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
1115 d27 = mulmod_scaled_x16(d27,precomp_512_511_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_512_511_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata);
1116 // vector_twist 496 512 507 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
1117 d31 = mulmod_scaled_x16(d31,precomp_512_507_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_512_507_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata);
1118 // vector_permute 48 112 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
1119 int16x16 e3 = _mm256_permute2x128_si256_lo(d3,d7);
1120 int16x16 e7 = _mm256_permute2x128_si256_hi(d3,d7);
1121 // vector_permute 176 240 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
1122 int16x16 e11 = _mm256_permute2x128_si256_lo(d11,d15);
1123 int16x16 e15 = _mm256_permute2x128_si256_hi(d11,d15);
1124 // vector_permute 304 368 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
1125 int16x16 e19 = _mm256_permute2x128_si256_lo(d19,d23);
1126 int16x16 e23 = _mm256_permute2x128_si256_hi(d19,d23);
1127 // vector_permute 432 496 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
1128 int16x16 e27 = _mm256_permute2x128_si256_lo(d27,d31);
1129 int16x16 e31 = _mm256_permute2x128_si256_hi(d27,d31);
1130 // stopbatch 512
1131 _mm256_storeu_si256((int16x16 *) (f+48),e3);
1132 _mm256_storeu_si256((int16x16 *) (f+112),e7);
1133 _mm256_storeu_si256((int16x16 *) (f+176),e11);
1134 _mm256_storeu_si256((int16x16 *) (f+240),e15);
1135 _mm256_storeu_si256((int16x16 *) (f+304),e19);
1136 _mm256_storeu_si256((int16x16 *) (f+368),e23);
1137 _mm256_storeu_si256((int16x16 *) (f+432),e27);
1138 _mm256_storeu_si256((int16x16 *) (f+496),e31);
1139 f += 512;
1140 }
1141 f -= 512*reps;
1142 // doublereps
1143 reps *= 2;
1144 // doublereps
1145 reps *= 2;
1146 // startbatch 128
1147 for (long long r = 0;r < reps;++r) {
1148 // vector_butterfly 0 32 1 0
1149 int16x16 a0 = _mm256_loadu_si256((int16x16 *) (f+0));
1150 int16x16 a2 = _mm256_loadu_si256((int16x16 *) (f+32));
1151 int16x16 b0 = add_x16(a0,a2);
1152 int16x16 b2 = sub_x16(a0,a2);
1153 // vector_butterfly 64 96 1 0
1154 int16x16 a4 = _mm256_loadu_si256((int16x16 *) (f+64));
1155 int16x16 a6 = _mm256_loadu_si256((int16x16 *) (f+96));
1156 int16x16 b4 = add_x16(a4,a6);
1157 int16x16 b6 = sub_x16(a4,a6);
1158 // vector_butterfly 16 48 1 0
1159 int16x16 a1 = _mm256_loadu_si256((int16x16 *) (f+16));
1160 int16x16 a3 = _mm256_loadu_si256((int16x16 *) (f+48));
1161 int16x16 b1 = add_x16(a1,a3);
1162 int16x16 b3 = sub_x16(a1,a3);
1163 // vector_butterfly 80 112 1 0
1164 int16x16 a5 = _mm256_loadu_si256((int16x16 *) (f+80));
1165 int16x16 a7 = _mm256_loadu_si256((int16x16 *) (f+112));
1166 int16x16 b5 = add_x16(a5,a7);
1167 int16x16 b7 = sub_x16(a5,a7);
1168 // vector_butterfly 0 16 1 0
1169 int16x16 c0 = add_x16(b0,b1);
1170 int16x16 c1 = sub_x16(b0,b1);
1171 // vector_butterfly 64 80 1 0
1172 int16x16 c4 = add_x16(b4,b5);
1173 int16x16 c5 = sub_x16(b4,b5);
1174 // vector_butterfly 32 48 4 1
1175 b3 = mulmod_scaled_x16(b3,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata);
1176 int16x16 c2 = add_x16(b2,b3);
1177 int16x16 c3 = sub_x16(b2,b3);
1178 // vector_butterfly 96 112 4 1
1179 b7 = mulmod_scaled_x16(b7,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata);
1180 int16x16 c6 = add_x16(b6,b7);
1181 int16x16 c7 = sub_x16(b6,b7);
1182 // vector_reduce 0
1183 c0 = reduce_x16(c0,qdata);
1184 // vector_reduce 64
1185 c4 = reduce_x16(c4,qdata);
1186 // vector_twist 16 32 1 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7
1187 c1 = mulmod_scaled_x16(c1,precomp_32_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qinvprecomp_32_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qdata);
1188 // vector_twist 80 32 1 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15
1189 c5 = mulmod_scaled_x16(c5,precomp_32_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qinvprecomp_32_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qdata);
1190 // vector_twist 32 64 1 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7
1191 c2 = mulmod_scaled_x16(c2,precomp_64_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qinvprecomp_64_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qdata);
1192 // vector_twist 96 64 1 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15
1193 c6 = mulmod_scaled_x16(c6,precomp_64_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qinvprecomp_64_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qdata);
1194 // vector_twist 48 64 63 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7
1195 c3 = mulmod_scaled_x16(c3,precomp_64_63_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qinvprecomp_64_63_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qdata);
1196 // vector_twist 112 64 63 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15
1197 c7 = mulmod_scaled_x16(c7,precomp_64_63_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qinvprecomp_64_63_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qdata);
1198 // vector_permute 0 32 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16
1199 int16x16 d0 = _mm256_unpacklo_epi16(c0,c2);
1200 int16x16 d2 = _mm256_unpackhi_epi16(c0,c2);
1201 // vector_permute 16 48 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16
1202 int16x16 d1 = _mm256_unpacklo_epi16(c1,c3);
1203 int16x16 d3 = _mm256_unpackhi_epi16(c1,c3);
1204 // vector_permute 64 96 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16
1205 int16x16 d4 = _mm256_unpacklo_epi16(c4,c6);
1206 int16x16 d6 = _mm256_unpackhi_epi16(c4,c6);
1207 // vector_permute 80 112 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16
1208 int16x16 d5 = _mm256_unpacklo_epi16(c5,c7);
1209 int16x16 d7 = _mm256_unpackhi_epi16(c5,c7);
1210 // vector_butterfly 0 64 1 0
1211 int16x16 e0 = add_x16(d0,d4);
1212 int16x16 e4 = sub_x16(d0,d4);
1213 // vector_butterfly 32 96 1 0
1214 int16x16 e2 = add_x16(d2,d6);
1215 int16x16 e6 = sub_x16(d2,d6);
1216 // vector_butterfly 16 80 1 0
1217 int16x16 e1 = add_x16(d1,d5);
1218 int16x16 e5 = sub_x16(d1,d5);
1219 // vector_butterfly 48 112 1 0
1220 int16x16 e3 = add_x16(d3,d7);
1221 int16x16 e7 = sub_x16(d3,d7);
1222 // vector_permute 0 16 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32
1223 int16x16 f0 = _mm256_unpacklo_epi32(e0,e1);
1224 int16x16 f1 = _mm256_unpackhi_epi32(e0,e1);
1225 // vector_permute 32 48 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32
1226 int16x16 f2 = _mm256_unpacklo_epi32(e2,e3);
1227 int16x16 f3 = _mm256_unpackhi_epi32(e2,e3);
1228 // vector_permute 64 80 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32
1229 int16x16 f4 = _mm256_unpacklo_epi32(e4,e5);
1230 int16x16 f5 = _mm256_unpackhi_epi32(e4,e5);
1231 // vector_permute 96 112 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32
1232 int16x16 f6 = _mm256_unpacklo_epi32(e6,e7);
1233 int16x16 f7 = _mm256_unpackhi_epi32(e6,e7);
1234 // stopbatch 128
1235 _mm256_storeu_si256((int16x16 *) (f+0),f0);
1236 _mm256_storeu_si256((int16x16 *) (f+16),f1);
1237 _mm256_storeu_si256((int16x16 *) (f+32),f2);
1238 _mm256_storeu_si256((int16x16 *) (f+48),f3);
1239 _mm256_storeu_si256((int16x16 *) (f+64),f4);
1240 _mm256_storeu_si256((int16x16 *) (f+80),f5);
1241 _mm256_storeu_si256((int16x16 *) (f+96),f6);
1242 _mm256_storeu_si256((int16x16 *) (f+112),f7);
1243 f += 128;
1244 }
1245 f -= 128*reps;
1246 // startbatch 128
1247 for (long long r = 0;r < reps;++r) {
1248 // vector_butterfly 0 32 1 0
1249 int16x16 a0 = _mm256_loadu_si256((int16x16 *) (f+0));
1250 int16x16 a2 = _mm256_loadu_si256((int16x16 *) (f+32));
1251 int16x16 b0 = add_x16(a0,a2);
1252 int16x16 b2 = sub_x16(a0,a2);
1253 // vector_butterfly 16 48 1 0
1254 int16x16 a1 = _mm256_loadu_si256((int16x16 *) (f+16));
1255 int16x16 a3 = _mm256_loadu_si256((int16x16 *) (f+48));
1256 int16x16 b1 = add_x16(a1,a3);
1257 int16x16 b3 = sub_x16(a1,a3);
1258 // vector_butterfly 64 96 4 1
1259 int16x16 a4 = _mm256_loadu_si256((int16x16 *) (f+64));
1260 int16x16 a6 = _mm256_loadu_si256((int16x16 *) (f+96));
1261 a6 = mulmod_scaled_x16(a6,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata);
1262 int16x16 b4 = add_x16(a4,a6);
1263 int16x16 b6 = sub_x16(a4,a6);
1264 // vector_butterfly 80 112 4 1
1265 int16x16 a5 = _mm256_loadu_si256((int16x16 *) (f+80));
1266 int16x16 a7 = _mm256_loadu_si256((int16x16 *) (f+112));
1267 a7 = mulmod_scaled_x16(a7,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata);
1268 int16x16 b5 = add_x16(a5,a7);
1269 int16x16 b7 = sub_x16(a5,a7);
1270 // vector_reduce 0
1271 b0 = reduce_x16(b0,qdata);
1272 // vector_reduce 16
1273 b1 = reduce_x16(b1,qdata);
1274 // vector_twist 32 8 1 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1
1275 b2 = mulmod_scaled_x16(b2,precomp_8_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qinvprecomp_8_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qdata);
1276 // vector_twist 48 8 1 2 2 2 2 3 3 3 3 2 2 2 2 3 3 3 3
1277 b3 = mulmod_scaled_x16(b3,precomp_8_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qinvprecomp_8_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qdata);
1278 // vector_twist 64 16 1 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1
1279 b4 = mulmod_scaled_x16(b4,precomp_16_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qinvprecomp_16_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qdata);
1280 // vector_twist 80 16 1 2 2 2 2 3 3 3 3 2 2 2 2 3 3 3 3
1281 b5 = mulmod_scaled_x16(b5,precomp_16_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qinvprecomp_16_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qdata);
1282 // vector_twist 96 16 15 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1
1283 b6 = mulmod_scaled_x16(b6,precomp_16_15_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qinvprecomp_16_15_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qdata);
1284 // vector_twist 112 16 15 2 2 2 2 3 3 3 3 2 2 2 2 3 3 3 3
1285 b7 = mulmod_scaled_x16(b7,precomp_16_15_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qinvprecomp_16_15_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qdata);
1286 // vector_permute 0 64 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64
1287 int16x16 c0 = _mm256_unpacklo_epi64(b0,b4);
1288 int16x16 c4 = _mm256_unpackhi_epi64(b0,b4);
1289 // vector_permute 16 80 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64
1290 int16x16 c1 = _mm256_unpacklo_epi64(b1,b5);
1291 int16x16 c5 = _mm256_unpackhi_epi64(b1,b5);
1292 // vector_permute 32 96 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64
1293 int16x16 c2 = _mm256_unpacklo_epi64(b2,b6);
1294 int16x16 c6 = _mm256_unpackhi_epi64(b2,b6);
1295 // vector_permute 48 112 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64
1296 int16x16 c3 = _mm256_unpacklo_epi64(b3,b7);
1297 int16x16 c7 = _mm256_unpackhi_epi64(b3,b7);
1298 // vector_butterfly 0 16 1 0
1299 int16x16 d0 = add_x16(c0,c1);
1300 int16x16 d1 = sub_x16(c0,c1);
1301 // vector_butterfly 64 80 1 0
1302 int16x16 d4 = add_x16(c4,c5);
1303 int16x16 d5 = sub_x16(c4,c5);
1304 // vector_butterfly 32 48 1 0
1305 int16x16 d2 = add_x16(c2,c3);
1306 int16x16 d3 = sub_x16(c2,c3);
1307 // vector_butterfly 96 112 1 0
1308 int16x16 d6 = add_x16(c6,c7);
1309 int16x16 d7 = sub_x16(c6,c7);
1310 // vector_butterfly 0 64 1 0
1311 int16x16 e0 = add_x16(d0,d4);
1312 int16x16 e4 = sub_x16(d0,d4);
1313 // vector_butterfly 32 96 1 0
1314 int16x16 e2 = add_x16(d2,d6);
1315 int16x16 e6 = sub_x16(d2,d6);
1316 // vector_butterfly 16 80 4 1
1317 d5 = mulmod_scaled_x16(d5,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata);
1318 int16x16 e1 = add_x16(d1,d5);
1319 int16x16 e5 = sub_x16(d1,d5);
1320 // vector_butterfly 48 112 4 1
1321 d7 = mulmod_scaled_x16(d7,scaledzeta_x16_4_1,qinvscaledzeta_x16_4_1,qdata);
1322 int16x16 e3 = add_x16(d3,d7);
1323 int16x16 e7 = sub_x16(d3,d7);
1324 // stopbatch 128
1325 _mm256_storeu_si256((int16x16 *) (f+0),e0);
1326 _mm256_storeu_si256((int16x16 *) (f+16),e1);
1327 _mm256_storeu_si256((int16x16 *) (f+32),e2);
1328 _mm256_storeu_si256((int16x16 *) (f+48),e3);
1329 _mm256_storeu_si256((int16x16 *) (f+64),e4);
1330 _mm256_storeu_si256((int16x16 *) (f+80),e5);
1331 _mm256_storeu_si256((int16x16 *) (f+96),e6);
1332 _mm256_storeu_si256((int16x16 *) (f+112),e7);
1333 f += 128;
1334 }
1335 f -= 128*reps;
1336 // stopntt 512
1337 }
1338
1339 void ntt512_7681(int16 *f,int reps)
1340 {
1341 ntt512(f,reps,qdata_7681);
1342 }
1343
1344 // inv stopntt 512
1345
1346 static void invntt512(int16 *f,int reps,const int16 *qdata)
1347 {
1348 reps *= 4;
1349 // inv stopbatch 128
1350 for (long long r = 0;r < reps;++r) {
1351 // inv vector_butterfly 48 112 4 1
1352 int16x16 a3 = _mm256_loadu_si256((int16x16 *) (f+48));
1353 int16x16 a7 = _mm256_loadu_si256((int16x16 *) (f+112));
1354 int16x16 b3 = add_x16(a3,a7);
1355 int16x16 b7 = sub_x16(a3,a7);
1356 b7 = mulmod_scaled_x16(b7,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata);
1357 // inv vector_butterfly 16 80 4 1
1358 int16x16 a1 = _mm256_loadu_si256((int16x16 *) (f+16));
1359 int16x16 a5 = _mm256_loadu_si256((int16x16 *) (f+80));
1360 int16x16 b1 = add_x16(a1,a5);
1361 int16x16 b5 = sub_x16(a1,a5);
1362 b5 = mulmod_scaled_x16(b5,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata);
1363 // inv vector_butterfly 32 96 1 0
1364 int16x16 a2 = _mm256_loadu_si256((int16x16 *) (f+32));
1365 int16x16 a6 = _mm256_loadu_si256((int16x16 *) (f+96));
1366 int16x16 b2 = add_x16(a2,a6);
1367 int16x16 b6 = sub_x16(a2,a6);
1368 // inv vector_butterfly 0 64 1 0
1369 int16x16 a0 = _mm256_loadu_si256((int16x16 *) (f+0));
1370 int16x16 a4 = _mm256_loadu_si256((int16x16 *) (f+64));
1371 int16x16 b0 = add_x16(a0,a4);
1372 int16x16 b4 = sub_x16(a0,a4);
1373 // inv vector_butterfly 96 112 1 0
1374 int16x16 c6 = add_x16(b6,b7);
1375 int16x16 c7 = sub_x16(b6,b7);
1376 // inv vector_butterfly 32 48 1 0
1377 int16x16 c2 = add_x16(b2,b3);
1378 int16x16 c3 = sub_x16(b2,b3);
1379 // inv vector_butterfly 64 80 1 0
1380 int16x16 c4 = add_x16(b4,b5);
1381 int16x16 c5 = sub_x16(b4,b5);
1382 // inv vector_butterfly 0 16 1 0
1383 int16x16 c0 = add_x16(b0,b1);
1384 int16x16 c1 = sub_x16(b0,b1);
1385 // inv vector_permute 48 112 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64
1386 int16x16 d3 = _mm256_unpacklo_epi64(c3,c7);
1387 int16x16 d7 = _mm256_unpackhi_epi64(c3,c7);
1388 // inv vector_permute 32 96 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64
1389 int16x16 d2 = _mm256_unpacklo_epi64(c2,c6);
1390 int16x16 d6 = _mm256_unpackhi_epi64(c2,c6);
1391 // inv vector_permute 16 80 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64
1392 int16x16 d1 = _mm256_unpacklo_epi64(c1,c5);
1393 int16x16 d5 = _mm256_unpackhi_epi64(c1,c5);
1394 // inv vector_permute 0 64 _mm256_unpacklo_epi64 _mm256_unpackhi_epi64
1395 int16x16 d0 = _mm256_unpacklo_epi64(c0,c4);
1396 int16x16 d4 = _mm256_unpackhi_epi64(c0,c4);
1397 // inv vector_twist 112 16 15 2 2 2 2 3 3 3 3 2 2 2 2 3 3 3 3
1398 d7 = mulmod_scaled_x16(d7,precomp_16_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qinvprecomp_16_1_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qdata);
1399 // inv vector_twist 96 16 15 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1
1400 d6 = mulmod_scaled_x16(d6,precomp_16_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qinvprecomp_16_1_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qdata);
1401 // inv vector_twist 80 16 1 2 2 2 2 3 3 3 3 2 2 2 2 3 3 3 3
1402 d5 = mulmod_scaled_x16(d5,precomp_16_15_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qinvprecomp_16_15_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qdata);
1403 // inv vector_twist 64 16 1 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1
1404 d4 = mulmod_scaled_x16(d4,precomp_16_15_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qinvprecomp_16_15_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qdata);
1405 // inv vector_twist 48 8 1 2 2 2 2 3 3 3 3 2 2 2 2 3 3 3 3
1406 d3 = mulmod_scaled_x16(d3,precomp_8_7_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qinvprecomp_8_7_2_2_2_2_3_3_3_3_2_2_2_2_3_3_3_3,qdata);
1407 // inv vector_twist 32 8 1 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1
1408 d2 = mulmod_scaled_x16(d2,precomp_8_7_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qinvprecomp_8_7_0_0_0_0_1_1_1_1_0_0_0_0_1_1_1_1,qdata);
1409 // inv vector_reduce 16
1410 d1 = reduce_x16(d1,qdata);
1411 // inv vector_reduce 0
1412 d0 = reduce_x16(d0,qdata);
1413 // inv vector_butterfly 80 112 4 1
1414 int16x16 e5 = add_x16(d5,d7);
1415 int16x16 e7 = sub_x16(d5,d7);
1416 e7 = mulmod_scaled_x16(e7,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata);
1417 // inv vector_butterfly 64 96 4 1
1418 int16x16 e4 = add_x16(d4,d6);
1419 int16x16 e6 = sub_x16(d4,d6);
1420 e6 = mulmod_scaled_x16(e6,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata);
1421 // inv vector_butterfly 16 48 1 0
1422 int16x16 e1 = add_x16(d1,d3);
1423 int16x16 e3 = sub_x16(d1,d3);
1424 // inv vector_butterfly 0 32 1 0
1425 int16x16 e0 = add_x16(d0,d2);
1426 int16x16 e2 = sub_x16(d0,d2);
1427 // inv startbatch 128
1428 _mm256_storeu_si256((int16x16 *) (f+0),e0);
1429 _mm256_storeu_si256((int16x16 *) (f+16),e1);
1430 _mm256_storeu_si256((int16x16 *) (f+32),e2);
1431 _mm256_storeu_si256((int16x16 *) (f+48),e3);
1432 _mm256_storeu_si256((int16x16 *) (f+64),e4);
1433 _mm256_storeu_si256((int16x16 *) (f+80),e5);
1434 _mm256_storeu_si256((int16x16 *) (f+96),e6);
1435 _mm256_storeu_si256((int16x16 *) (f+112),e7);
1436 f += 128;
1437 }
1438 f -= 128*reps;
1439 // inv stopbatch 128
1440 for (long long r = 0;r < reps;++r) {
1441 // inv vector_permute 96 112 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32
1442 int16x16 a6 = _mm256_loadu_si256((int16x16 *) (f+96));
1443 int16x16 a7 = _mm256_loadu_si256((int16x16 *) (f+112));
1444 int16x16 b6 = _mm256_unpacklo_epi32(a6,a7);
1445 int16x16 b7 = _mm256_unpackhi_epi32(a6,a7);
1446 int16x16 c6 = _mm256_unpacklo_epi32(b6,b7);
1447 int16x16 c7 = _mm256_unpackhi_epi32(b6,b7);
1448 // inv vector_permute 64 80 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32
1449 int16x16 a4 = _mm256_loadu_si256((int16x16 *) (f+64));
1450 int16x16 a5 = _mm256_loadu_si256((int16x16 *) (f+80));
1451 int16x16 b4 = _mm256_unpacklo_epi32(a4,a5);
1452 int16x16 b5 = _mm256_unpackhi_epi32(a4,a5);
1453 int16x16 c4 = _mm256_unpacklo_epi32(b4,b5);
1454 int16x16 c5 = _mm256_unpackhi_epi32(b4,b5);
1455 // inv vector_permute 32 48 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32
1456 int16x16 a2 = _mm256_loadu_si256((int16x16 *) (f+32));
1457 int16x16 a3 = _mm256_loadu_si256((int16x16 *) (f+48));
1458 int16x16 b2 = _mm256_unpacklo_epi32(a2,a3);
1459 int16x16 b3 = _mm256_unpackhi_epi32(a2,a3);
1460 int16x16 c2 = _mm256_unpacklo_epi32(b2,b3);
1461 int16x16 c3 = _mm256_unpackhi_epi32(b2,b3);
1462 // inv vector_permute 0 16 _mm256_unpacklo_epi32 _mm256_unpackhi_epi32
1463 int16x16 a0 = _mm256_loadu_si256((int16x16 *) (f+0));
1464 int16x16 a1 = _mm256_loadu_si256((int16x16 *) (f+16));
1465 int16x16 b0 = _mm256_unpacklo_epi32(a0,a1);
1466 int16x16 b1 = _mm256_unpackhi_epi32(a0,a1);
1467 int16x16 c0 = _mm256_unpacklo_epi32(b0,b1);
1468 int16x16 c1 = _mm256_unpackhi_epi32(b0,b1);
1469 // inv vector_butterfly 48 112 1 0
1470 int16x16 d3 = add_x16(c3,c7);
1471 int16x16 d7 = sub_x16(c3,c7);
1472 // inv vector_butterfly 16 80 1 0
1473 int16x16 d1 = add_x16(c1,c5);
1474 int16x16 d5 = sub_x16(c1,c5);
1475 // inv vector_butterfly 32 96 1 0
1476 int16x16 d2 = add_x16(c2,c6);
1477 int16x16 d6 = sub_x16(c2,c6);
1478 // inv vector_butterfly 0 64 1 0
1479 int16x16 d0 = add_x16(c0,c4);
1480 int16x16 d4 = sub_x16(c0,c4);
1481 // inv vector_permute 80 112 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16
1482 int16x16 e5 = _mm256_unpacklo_epi16(d5,d7);
1483 int16x16 e7 = _mm256_unpackhi_epi16(d5,d7);
1484 int16x16 f5 = _mm256_unpacklo_epi16(e5,e7);
1485 int16x16 f7 = _mm256_unpackhi_epi16(e5,e7);
1486 int16x16 g5 = _mm256_unpacklo_epi16(f5,f7);
1487 int16x16 g7 = _mm256_unpackhi_epi16(f5,f7);
1488 // inv vector_permute 64 96 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16
1489 int16x16 e4 = _mm256_unpacklo_epi16(d4,d6);
1490 int16x16 e6 = _mm256_unpackhi_epi16(d4,d6);
1491 int16x16 f4 = _mm256_unpacklo_epi16(e4,e6);
1492 int16x16 f6 = _mm256_unpackhi_epi16(e4,e6);
1493 int16x16 g4 = _mm256_unpacklo_epi16(f4,f6);
1494 int16x16 g6 = _mm256_unpackhi_epi16(f4,f6);
1495 // inv vector_permute 16 48 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16
1496 int16x16 e1 = _mm256_unpacklo_epi16(d1,d3);
1497 int16x16 e3 = _mm256_unpackhi_epi16(d1,d3);
1498 int16x16 f1 = _mm256_unpacklo_epi16(e1,e3);
1499 int16x16 f3 = _mm256_unpackhi_epi16(e1,e3);
1500 int16x16 g1 = _mm256_unpacklo_epi16(f1,f3);
1501 int16x16 g3 = _mm256_unpackhi_epi16(f1,f3);
1502 // inv vector_permute 0 32 _mm256_unpacklo_epi16 _mm256_unpackhi_epi16
1503 int16x16 e0 = _mm256_unpacklo_epi16(d0,d2);
1504 int16x16 e2 = _mm256_unpackhi_epi16(d0,d2);
1505 int16x16 f0 = _mm256_unpacklo_epi16(e0,e2);
1506 int16x16 f2 = _mm256_unpackhi_epi16(e0,e2);
1507 int16x16 g0 = _mm256_unpacklo_epi16(f0,f2);
1508 int16x16 g2 = _mm256_unpackhi_epi16(f0,f2);
1509 // inv vector_twist 112 64 63 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15
1510 g7 = mulmod_scaled_x16(g7,precomp_64_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qinvprecomp_64_1_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qdata);
1511 // inv vector_twist 48 64 63 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7
1512 g3 = mulmod_scaled_x16(g3,precomp_64_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qinvprecomp_64_1_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qdata);
1513 // inv vector_twist 96 64 1 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15
1514 g6 = mulmod_scaled_x16(g6,precomp_64_63_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qinvprecomp_64_63_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qdata);
1515 // inv vector_twist 32 64 1 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7
1516 g2 = mulmod_scaled_x16(g2,precomp_64_63_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qinvprecomp_64_63_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qdata);
1517 // inv vector_twist 80 32 1 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15
1518 g5 = mulmod_scaled_x16(g5,precomp_32_31_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qinvprecomp_32_31_8_9_10_11_12_13_14_15_8_9_10_11_12_13_14_15,qdata);
1519 // inv vector_twist 16 32 1 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7
1520 g1 = mulmod_scaled_x16(g1,precomp_32_31_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qinvprecomp_32_31_0_1_2_3_4_5_6_7_0_1_2_3_4_5_6_7,qdata);
1521 // inv vector_reduce 64
1522 g4 = reduce_x16(g4,qdata);
1523 // inv vector_reduce 0
1524 g0 = reduce_x16(g0,qdata);
1525 // inv vector_butterfly 96 112 4 1
1526 int16x16 h6 = add_x16(g6,g7);
1527 int16x16 h7 = sub_x16(g6,g7);
1528 h7 = mulmod_scaled_x16(h7,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata);
1529 // inv vector_butterfly 32 48 4 1
1530 int16x16 h2 = add_x16(g2,g3);
1531 int16x16 h3 = sub_x16(g2,g3);
1532 h3 = mulmod_scaled_x16(h3,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata);
1533 // inv vector_butterfly 64 80 1 0
1534 int16x16 h4 = add_x16(g4,g5);
1535 int16x16 h5 = sub_x16(g4,g5);
1536 // inv vector_butterfly 0 16 1 0
1537 int16x16 h0 = add_x16(g0,g1);
1538 int16x16 h1 = sub_x16(g0,g1);
1539 // inv vector_butterfly 80 112 1 0
1540 int16x16 i5 = add_x16(h5,h7);
1541 int16x16 i7 = sub_x16(h5,h7);
1542 // inv vector_butterfly 16 48 1 0
1543 int16x16 i1 = add_x16(h1,h3);
1544 int16x16 i3 = sub_x16(h1,h3);
1545 // inv vector_butterfly 64 96 1 0
1546 int16x16 i4 = add_x16(h4,h6);
1547 int16x16 i6 = sub_x16(h4,h6);
1548 // inv vector_butterfly 0 32 1 0
1549 int16x16 i0 = add_x16(h0,h2);
1550 int16x16 i2 = sub_x16(h0,h2);
1551 // inv startbatch 128
1552 _mm256_storeu_si256((int16x16 *) (f+0),i0);
1553 _mm256_storeu_si256((int16x16 *) (f+16),i1);
1554 _mm256_storeu_si256((int16x16 *) (f+32),i2);
1555 _mm256_storeu_si256((int16x16 *) (f+48),i3);
1556 _mm256_storeu_si256((int16x16 *) (f+64),i4);
1557 _mm256_storeu_si256((int16x16 *) (f+80),i5);
1558 _mm256_storeu_si256((int16x16 *) (f+96),i6);
1559 _mm256_storeu_si256((int16x16 *) (f+112),i7);
1560 f += 128;
1561 }
1562 f -= 128*reps;
1563 // inv doublereps
1564 reps /= 2;
1565 // inv doublereps
1566 reps /= 2;
1567 // inv stopbatch 512
1568 for (long long r = 0;r < reps;++r) {
1569 // inv vector_permute 432 496 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
1570 int16x16 a27 = _mm256_loadu_si256((int16x16 *) (f+432));
1571 int16x16 a31 = _mm256_loadu_si256((int16x16 *) (f+496));
1572 int16x16 b27 = _mm256_permute2x128_si256_lo(a27,a31);
1573 int16x16 b31 = _mm256_permute2x128_si256_hi(a27,a31);
1574 // inv vector_permute 304 368 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
1575 int16x16 a19 = _mm256_loadu_si256((int16x16 *) (f+304));
1576 int16x16 a23 = _mm256_loadu_si256((int16x16 *) (f+368));
1577 int16x16 b19 = _mm256_permute2x128_si256_lo(a19,a23);
1578 int16x16 b23 = _mm256_permute2x128_si256_hi(a19,a23);
1579 // inv vector_permute 176 240 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
1580 int16x16 a11 = _mm256_loadu_si256((int16x16 *) (f+176));
1581 int16x16 a15 = _mm256_loadu_si256((int16x16 *) (f+240));
1582 int16x16 b11 = _mm256_permute2x128_si256_lo(a11,a15);
1583 int16x16 b15 = _mm256_permute2x128_si256_hi(a11,a15);
1584 // inv vector_permute 48 112 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
1585 int16x16 a3 = _mm256_loadu_si256((int16x16 *) (f+48));
1586 int16x16 a7 = _mm256_loadu_si256((int16x16 *) (f+112));
1587 int16x16 b3 = _mm256_permute2x128_si256_lo(a3,a7);
1588 int16x16 b7 = _mm256_permute2x128_si256_hi(a3,a7);
1589 // inv vector_twist 496 512 507 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
1590 b31 = mulmod_scaled_x16(b31,precomp_512_5_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_512_5_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata);
1591 // inv vector_twist 432 512 511 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
1592 b27 = mulmod_scaled_x16(b27,precomp_512_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_512_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata);
1593 // inv vector_twist 368 512 5 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
1594 b23 = mulmod_scaled_x16(b23,precomp_512_507_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_512_507_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata);
1595 // inv vector_twist 304 512 1 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
1596 b19 = mulmod_scaled_x16(b19,precomp_512_511_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_512_511_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata);
1597 // inv vector_twist 240 256 255 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
1598 b15 = mulmod_scaled_x16(b15,precomp_256_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_256_1_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata);
1599 // inv vector_twist 176 256 1 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
1600 b11 = mulmod_scaled_x16(b11,precomp_256_255_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_256_255_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata);
1601 // inv vector_twist 112 128 1 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
1602 b7 = mulmod_scaled_x16(b7,precomp_128_127_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qinvprecomp_128_127_48_49_50_51_52_53_54_55_56_57_58_59_60_61_62_63,qdata);
1603 // inv vector_reduce 48
1604 b3 = reduce_x16(b3,qdata);
1605 // inv vector_butterfly 432 496 8 7
1606 int16x16 c27 = add_x16(b27,b31);
1607 int16x16 c31 = sub_x16(b27,b31);
1608 c31 = mulmod_scaled_x16(c31,scaledzeta_x16_8_1,qinvscaledzeta_x16_8_1,qdata);
1609 // inv vector_butterfly 304 368 8 1
1610 int16x16 c19 = add_x16(b19,b23);
1611 int16x16 c23 = sub_x16(b19,b23);
1612 c23 = mulmod_scaled_x16(c23,scaledzeta_x16_8_7,qinvscaledzeta_x16_8_7,qdata);
1613 // inv vector_butterfly 176 240 4 1
1614 int16x16 c11 = add_x16(b11,b15);
1615 int16x16 c15 = sub_x16(b11,b15);
1616 c15 = mulmod_scaled_x16(c15,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata);
1617 // inv vector_butterfly 48 112 1 0
1618 int16x16 c3 = add_x16(b3,b7);
1619 int16x16 c7 = sub_x16(b3,b7);
1620 // inv vector_reduce_ifforward 112
1621 // inv vector_butterfly 368 496 4 1
1622 int16x16 d23 = add_x16(c23,c31);
1623 int16x16 d31 = sub_x16(c23,c31);
1624 d31 = mulmod_scaled_x16(d31,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata);
1625 // inv vector_butterfly 304 432 4 1
1626 int16x16 d19 = add_x16(c19,c27);
1627 int16x16 d27 = sub_x16(c19,c27);
1628 d27 = mulmod_scaled_x16(d27,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata);
1629 // inv vector_butterfly 112 240 1 0
1630 int16x16 d7 = add_x16(c7,c15);
1631 int16x16 d15 = sub_x16(c7,c15);
1632 // inv vector_butterfly 48 176 1 0
1633 int16x16 d3 = add_x16(c3,c11);
1634 int16x16 d11 = sub_x16(c3,c11);
1635 // inv vector_reduce_ifreverse 304
1636 d19 = reduce_x16(d19,qdata);
1637 // inv vector_reduce_ifreverse 48
1638 d3 = reduce_x16(d3,qdata);
1639 // inv vector_butterfly 240 496 1 0
1640 int16x16 e15 = add_x16(d15,d31);
1641 int16x16 e31 = sub_x16(d15,d31);
1642 // inv vector_butterfly 112 368 1 0
1643 int16x16 e7 = add_x16(d7,d23);
1644 int16x16 e23 = sub_x16(d7,d23);
1645 // inv vector_butterfly 176 432 1 0
1646 int16x16 e11 = add_x16(d11,d27);
1647 int16x16 e27 = sub_x16(d11,d27);
1648 // inv vector_butterfly 48 304 1 0
1649 int16x16 e3 = add_x16(d3,d19);
1650 int16x16 e19 = sub_x16(d3,d19);
1651 // inv startbatch 512
1652 _mm256_storeu_si256((int16x16 *) (f+48),e3);
1653 _mm256_storeu_si256((int16x16 *) (f+112),e7);
1654 _mm256_storeu_si256((int16x16 *) (f+176),e11);
1655 _mm256_storeu_si256((int16x16 *) (f+240),e15);
1656 _mm256_storeu_si256((int16x16 *) (f+304),e19);
1657 _mm256_storeu_si256((int16x16 *) (f+368),e23);
1658 _mm256_storeu_si256((int16x16 *) (f+432),e27);
1659 _mm256_storeu_si256((int16x16 *) (f+496),e31);
1660 f += 512;
1661 }
1662 f -= 512*reps;
1663 // inv stopbatch 512
1664 for (long long r = 0;r < reps;++r) {
1665 // inv vector_permute 416 480 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
1666 int16x16 a26 = _mm256_loadu_si256((int16x16 *) (f+416));
1667 int16x16 a30 = _mm256_loadu_si256((int16x16 *) (f+480));
1668 int16x16 b26 = _mm256_permute2x128_si256_lo(a26,a30);
1669 int16x16 b30 = _mm256_permute2x128_si256_hi(a26,a30);
1670 // inv vector_permute 288 352 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
1671 int16x16 a18 = _mm256_loadu_si256((int16x16 *) (f+288));
1672 int16x16 a22 = _mm256_loadu_si256((int16x16 *) (f+352));
1673 int16x16 b18 = _mm256_permute2x128_si256_lo(a18,a22);
1674 int16x16 b22 = _mm256_permute2x128_si256_hi(a18,a22);
1675 // inv vector_permute 160 224 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
1676 int16x16 a10 = _mm256_loadu_si256((int16x16 *) (f+160));
1677 int16x16 a14 = _mm256_loadu_si256((int16x16 *) (f+224));
1678 int16x16 b10 = _mm256_permute2x128_si256_lo(a10,a14);
1679 int16x16 b14 = _mm256_permute2x128_si256_hi(a10,a14);
1680 // inv vector_permute 32 96 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
1681 int16x16 a2 = _mm256_loadu_si256((int16x16 *) (f+32));
1682 int16x16 a6 = _mm256_loadu_si256((int16x16 *) (f+96));
1683 int16x16 b2 = _mm256_permute2x128_si256_lo(a2,a6);
1684 int16x16 b6 = _mm256_permute2x128_si256_hi(a2,a6);
1685 // inv vector_twist 480 512 507 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
1686 b30 = mulmod_scaled_x16(b30,precomp_512_5_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_512_5_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata);
1687 // inv vector_twist 416 512 511 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
1688 b26 = mulmod_scaled_x16(b26,precomp_512_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_512_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata);
1689 // inv vector_twist 352 512 5 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
1690 b22 = mulmod_scaled_x16(b22,precomp_512_507_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_512_507_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata);
1691 // inv vector_twist 288 512 1 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
1692 b18 = mulmod_scaled_x16(b18,precomp_512_511_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_512_511_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata);
1693 // inv vector_twist 224 256 255 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
1694 b14 = mulmod_scaled_x16(b14,precomp_256_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_256_1_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata);
1695 // inv vector_twist 160 256 1 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
1696 b10 = mulmod_scaled_x16(b10,precomp_256_255_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_256_255_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata);
1697 // inv vector_twist 96 128 1 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
1698 b6 = mulmod_scaled_x16(b6,precomp_128_127_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qinvprecomp_128_127_32_33_34_35_36_37_38_39_40_41_42_43_44_45_46_47,qdata);
1699 // inv vector_reduce 32
1700 b2 = reduce_x16(b2,qdata);
1701 // inv vector_butterfly 416 480 8 7
1702 int16x16 c26 = add_x16(b26,b30);
1703 int16x16 c30 = sub_x16(b26,b30);
1704 c30 = mulmod_scaled_x16(c30,scaledzeta_x16_8_1,qinvscaledzeta_x16_8_1,qdata);
1705 // inv vector_butterfly 288 352 8 1
1706 int16x16 c18 = add_x16(b18,b22);
1707 int16x16 c22 = sub_x16(b18,b22);
1708 c22 = mulmod_scaled_x16(c22,scaledzeta_x16_8_7,qinvscaledzeta_x16_8_7,qdata);
1709 // inv vector_butterfly 160 224 4 1
1710 int16x16 c10 = add_x16(b10,b14);
1711 int16x16 c14 = sub_x16(b10,b14);
1712 c14 = mulmod_scaled_x16(c14,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata);
1713 // inv vector_butterfly 32 96 1 0
1714 int16x16 c2 = add_x16(b2,b6);
1715 int16x16 c6 = sub_x16(b2,b6);
1716 // inv vector_reduce_ifforward 96
1717 // inv vector_butterfly 352 480 4 1
1718 int16x16 d22 = add_x16(c22,c30);
1719 int16x16 d30 = sub_x16(c22,c30);
1720 d30 = mulmod_scaled_x16(d30,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata);
1721 // inv vector_butterfly 288 416 4 1
1722 int16x16 d18 = add_x16(c18,c26);
1723 int16x16 d26 = sub_x16(c18,c26);
1724 d26 = mulmod_scaled_x16(d26,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata);
1725 // inv vector_butterfly 96 224 1 0
1726 int16x16 d6 = add_x16(c6,c14);
1727 int16x16 d14 = sub_x16(c6,c14);
1728 // inv vector_butterfly 32 160 1 0
1729 int16x16 d2 = add_x16(c2,c10);
1730 int16x16 d10 = sub_x16(c2,c10);
1731 // inv vector_reduce_ifreverse 288
1732 d18 = reduce_x16(d18,qdata);
1733 // inv vector_reduce_ifreverse 32
1734 d2 = reduce_x16(d2,qdata);
1735 // inv vector_butterfly 224 480 1 0
1736 int16x16 e14 = add_x16(d14,d30);
1737 int16x16 e30 = sub_x16(d14,d30);
1738 // inv vector_butterfly 96 352 1 0
1739 int16x16 e6 = add_x16(d6,d22);
1740 int16x16 e22 = sub_x16(d6,d22);
1741 // inv vector_butterfly 160 416 1 0
1742 int16x16 e10 = add_x16(d10,d26);
1743 int16x16 e26 = sub_x16(d10,d26);
1744 // inv vector_butterfly 32 288 1 0
1745 int16x16 e2 = add_x16(d2,d18);
1746 int16x16 e18 = sub_x16(d2,d18);
1747 // inv startbatch 512
1748 _mm256_storeu_si256((int16x16 *) (f+32),e2);
1749 _mm256_storeu_si256((int16x16 *) (f+96),e6);
1750 _mm256_storeu_si256((int16x16 *) (f+160),e10);
1751 _mm256_storeu_si256((int16x16 *) (f+224),e14);
1752 _mm256_storeu_si256((int16x16 *) (f+288),e18);
1753 _mm256_storeu_si256((int16x16 *) (f+352),e22);
1754 _mm256_storeu_si256((int16x16 *) (f+416),e26);
1755 _mm256_storeu_si256((int16x16 *) (f+480),e30);
1756 f += 512;
1757 }
1758 f -= 512*reps;
1759 // inv stopbatch 512
1760 for (long long r = 0;r < reps;++r) {
1761 // inv vector_permute 400 464 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
1762 int16x16 a25 = _mm256_loadu_si256((int16x16 *) (f+400));
1763 int16x16 a29 = _mm256_loadu_si256((int16x16 *) (f+464));
1764 int16x16 b25 = _mm256_permute2x128_si256_lo(a25,a29);
1765 int16x16 b29 = _mm256_permute2x128_si256_hi(a25,a29);
1766 // inv vector_permute 272 336 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
1767 int16x16 a17 = _mm256_loadu_si256((int16x16 *) (f+272));
1768 int16x16 a21 = _mm256_loadu_si256((int16x16 *) (f+336));
1769 int16x16 b17 = _mm256_permute2x128_si256_lo(a17,a21);
1770 int16x16 b21 = _mm256_permute2x128_si256_hi(a17,a21);
1771 // inv vector_permute 144 208 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
1772 int16x16 a9 = _mm256_loadu_si256((int16x16 *) (f+144));
1773 int16x16 a13 = _mm256_loadu_si256((int16x16 *) (f+208));
1774 int16x16 b9 = _mm256_permute2x128_si256_lo(a9,a13);
1775 int16x16 b13 = _mm256_permute2x128_si256_hi(a9,a13);
1776 // inv vector_permute 16 80 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
1777 int16x16 a1 = _mm256_loadu_si256((int16x16 *) (f+16));
1778 int16x16 a5 = _mm256_loadu_si256((int16x16 *) (f+80));
1779 int16x16 b1 = _mm256_permute2x128_si256_lo(a1,a5);
1780 int16x16 b5 = _mm256_permute2x128_si256_hi(a1,a5);
1781 // inv vector_twist 464 512 507 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
1782 b29 = mulmod_scaled_x16(b29,precomp_512_5_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_512_5_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata);
1783 // inv vector_twist 400 512 511 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
1784 b25 = mulmod_scaled_x16(b25,precomp_512_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_512_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata);
1785 // inv vector_twist 336 512 5 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
1786 b21 = mulmod_scaled_x16(b21,precomp_512_507_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_512_507_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata);
1787 // inv vector_twist 272 512 1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
1788 b17 = mulmod_scaled_x16(b17,precomp_512_511_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_512_511_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata);
1789 // inv vector_twist 208 256 255 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
1790 b13 = mulmod_scaled_x16(b13,precomp_256_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_256_1_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata);
1791 // inv vector_twist 144 256 1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
1792 b9 = mulmod_scaled_x16(b9,precomp_256_255_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_256_255_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata);
1793 // inv vector_twist 80 128 1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
1794 b5 = mulmod_scaled_x16(b5,precomp_128_127_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qinvprecomp_128_127_16_17_18_19_20_21_22_23_24_25_26_27_28_29_30_31,qdata);
1795 // inv vector_reduce 16
1796 b1 = reduce_x16(b1,qdata);
1797 // inv vector_butterfly 400 464 8 7
1798 int16x16 c25 = add_x16(b25,b29);
1799 int16x16 c29 = sub_x16(b25,b29);
1800 c29 = mulmod_scaled_x16(c29,scaledzeta_x16_8_1,qinvscaledzeta_x16_8_1,qdata);
1801 // inv vector_butterfly 272 336 8 1
1802 int16x16 c17 = add_x16(b17,b21);
1803 int16x16 c21 = sub_x16(b17,b21);
1804 c21 = mulmod_scaled_x16(c21,scaledzeta_x16_8_7,qinvscaledzeta_x16_8_7,qdata);
1805 // inv vector_butterfly 144 208 4 1
1806 int16x16 c9 = add_x16(b9,b13);
1807 int16x16 c13 = sub_x16(b9,b13);
1808 c13 = mulmod_scaled_x16(c13,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata);
1809 // inv vector_butterfly 16 80 1 0
1810 int16x16 c1 = add_x16(b1,b5);
1811 int16x16 c5 = sub_x16(b1,b5);
1812 // inv vector_reduce_ifforward 80
1813 // inv vector_butterfly 336 464 4 1
1814 int16x16 d21 = add_x16(c21,c29);
1815 int16x16 d29 = sub_x16(c21,c29);
1816 d29 = mulmod_scaled_x16(d29,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata);
1817 // inv vector_butterfly 272 400 4 1
1818 int16x16 d17 = add_x16(c17,c25);
1819 int16x16 d25 = sub_x16(c17,c25);
1820 d25 = mulmod_scaled_x16(d25,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata);
1821 // inv vector_butterfly 80 208 1 0
1822 int16x16 d5 = add_x16(c5,c13);
1823 int16x16 d13 = sub_x16(c5,c13);
1824 // inv vector_butterfly 16 144 1 0
1825 int16x16 d1 = add_x16(c1,c9);
1826 int16x16 d9 = sub_x16(c1,c9);
1827 // inv vector_reduce_ifreverse 272
1828 d17 = reduce_x16(d17,qdata);
1829 // inv vector_reduce_ifreverse 16
1830 d1 = reduce_x16(d1,qdata);
1831 // inv vector_butterfly 208 464 1 0
1832 int16x16 e13 = add_x16(d13,d29);
1833 int16x16 e29 = sub_x16(d13,d29);
1834 // inv vector_butterfly 80 336 1 0
1835 int16x16 e5 = add_x16(d5,d21);
1836 int16x16 e21 = sub_x16(d5,d21);
1837 // inv vector_butterfly 144 400 1 0
1838 int16x16 e9 = add_x16(d9,d25);
1839 int16x16 e25 = sub_x16(d9,d25);
1840 // inv vector_butterfly 16 272 1 0
1841 int16x16 e1 = add_x16(d1,d17);
1842 int16x16 e17 = sub_x16(d1,d17);
1843 // inv startbatch 512
1844 _mm256_storeu_si256((int16x16 *) (f+16),e1);
1845 _mm256_storeu_si256((int16x16 *) (f+80),e5);
1846 _mm256_storeu_si256((int16x16 *) (f+144),e9);
1847 _mm256_storeu_si256((int16x16 *) (f+208),e13);
1848 _mm256_storeu_si256((int16x16 *) (f+272),e17);
1849 _mm256_storeu_si256((int16x16 *) (f+336),e21);
1850 _mm256_storeu_si256((int16x16 *) (f+400),e25);
1851 _mm256_storeu_si256((int16x16 *) (f+464),e29);
1852 f += 512;
1853 }
1854 f -= 512*reps;
1855 // inv stopbatch 512
1856 for (long long r = 0;r < reps;++r) {
1857 // inv vector_permute 384 448 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
1858 int16x16 a24 = _mm256_loadu_si256((int16x16 *) (f+384));
1859 int16x16 a28 = _mm256_loadu_si256((int16x16 *) (f+448));
1860 int16x16 b24 = _mm256_permute2x128_si256_lo(a24,a28);
1861 int16x16 b28 = _mm256_permute2x128_si256_hi(a24,a28);
1862 // inv vector_permute 256 320 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
1863 int16x16 a16 = _mm256_loadu_si256((int16x16 *) (f+256));
1864 int16x16 a20 = _mm256_loadu_si256((int16x16 *) (f+320));
1865 int16x16 b16 = _mm256_permute2x128_si256_lo(a16,a20);
1866 int16x16 b20 = _mm256_permute2x128_si256_hi(a16,a20);
1867 // inv vector_permute 128 192 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
1868 int16x16 a8 = _mm256_loadu_si256((int16x16 *) (f+128));
1869 int16x16 a12 = _mm256_loadu_si256((int16x16 *) (f+192));
1870 int16x16 b8 = _mm256_permute2x128_si256_lo(a8,a12);
1871 int16x16 b12 = _mm256_permute2x128_si256_hi(a8,a12);
1872 // inv vector_permute 0 64 _mm256_permute2x128_si256_lo _mm256_permute2x128_si256_hi
1873 int16x16 a0 = _mm256_loadu_si256((int16x16 *) (f+0));
1874 int16x16 a4 = _mm256_loadu_si256((int16x16 *) (f+64));
1875 int16x16 b0 = _mm256_permute2x128_si256_lo(a0,a4);
1876 int16x16 b4 = _mm256_permute2x128_si256_hi(a0,a4);
1877 // inv vector_twist 448 512 507 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
1878 b28 = mulmod_scaled_x16(b28,precomp_512_5_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_512_5_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata);
1879 // inv vector_twist 384 512 511 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
1880 b24 = mulmod_scaled_x16(b24,precomp_512_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_512_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata);
1881 // inv vector_twist 320 512 5 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
1882 b20 = mulmod_scaled_x16(b20,precomp_512_507_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_512_507_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata);
1883 // inv vector_twist 256 512 1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
1884 b16 = mulmod_scaled_x16(b16,precomp_512_511_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_512_511_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata);
1885 // inv vector_twist 192 256 255 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
1886 b12 = mulmod_scaled_x16(b12,precomp_256_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_256_1_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata);
1887 // inv vector_twist 128 256 1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
1888 b8 = mulmod_scaled_x16(b8,precomp_256_255_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_256_255_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata);
1889 // inv vector_twist 64 128 1 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
1890 b4 = mulmod_scaled_x16(b4,precomp_128_127_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qinvprecomp_128_127_0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15,qdata);
1891 // inv vector_reduce 0
1892 b0 = reduce_x16(b0,qdata);
1893 // inv vector_butterfly 384 448 8 7
1894 int16x16 c24 = add_x16(b24,b28);
1895 int16x16 c28 = sub_x16(b24,b28);
1896 c28 = mulmod_scaled_x16(c28,scaledzeta_x16_8_1,qinvscaledzeta_x16_8_1,qdata);
1897 // inv vector_butterfly 256 320 8 1
1898 int16x16 c16 = add_x16(b16,b20);
1899 int16x16 c20 = sub_x16(b16,b20);
1900 c20 = mulmod_scaled_x16(c20,scaledzeta_x16_8_7,qinvscaledzeta_x16_8_7,qdata);
1901 // inv vector_butterfly 128 192 4 1
1902 int16x16 c8 = add_x16(b8,b12);
1903 int16x16 c12 = sub_x16(b8,b12);
1904 c12 = mulmod_scaled_x16(c12,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata);
1905 // inv vector_butterfly 0 64 1 0
1906 int16x16 c0 = add_x16(b0,b4);
1907 int16x16 c4 = sub_x16(b0,b4);
1908 // inv vector_reduce_ifforward 64
1909 // inv vector_butterfly 320 448 4 1
1910 int16x16 d20 = add_x16(c20,c28);
1911 int16x16 d28 = sub_x16(c20,c28);
1912 d28 = mulmod_scaled_x16(d28,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata);
1913 // inv vector_butterfly 256 384 4 1
1914 int16x16 d16 = add_x16(c16,c24);
1915 int16x16 d24 = sub_x16(c16,c24);
1916 d24 = mulmod_scaled_x16(d24,scaledzeta_x16_4_3,qinvscaledzeta_x16_4_3,qdata);
1917 // inv vector_butterfly 64 192 1 0
1918 int16x16 d4 = add_x16(c4,c12);
1919 int16x16 d12 = sub_x16(c4,c12);
1920 // inv vector_butterfly 0 128 1 0
1921 int16x16 d0 = add_x16(c0,c8);
1922 int16x16 d8 = sub_x16(c0,c8);
1923 // inv vector_reduce_ifreverse 256
1924 d16 = reduce_x16(d16,qdata);
1925 // inv vector_reduce_ifreverse 0
1926 d0 = reduce_x16(d0,qdata);
1927 // inv vector_butterfly 192 448 1 0
1928 int16x16 e12 = add_x16(d12,d28);
1929 int16x16 e28 = sub_x16(d12,d28);
1930 // inv vector_butterfly 64 320 1 0
1931 int16x16 e4 = add_x16(d4,d20);
1932 int16x16 e20 = sub_x16(d4,d20);
1933 // inv vector_butterfly 128 384 1 0
1934 int16x16 e8 = add_x16(d8,d24);
1935 int16x16 e24 = sub_x16(d8,d24);
1936 // inv vector_butterfly 0 256 1 0
1937 int16x16 e0 = add_x16(d0,d16);
1938 int16x16 e16 = sub_x16(d0,d16);
1939 // inv startbatch 512
1940 _mm256_storeu_si256((int16x16 *) (f+0),e0);
1941 _mm256_storeu_si256((int16x16 *) (f+64),e4);
1942 _mm256_storeu_si256((int16x16 *) (f+128),e8);
1943 _mm256_storeu_si256((int16x16 *) (f+192),e12);
1944 _mm256_storeu_si256((int16x16 *) (f+256),e16);
1945 _mm256_storeu_si256((int16x16 *) (f+320),e20);
1946 _mm256_storeu_si256((int16x16 *) (f+384),e24);
1947 _mm256_storeu_si256((int16x16 *) (f+448),e28);
1948 f += 512;
1949 }
1950 f -= 512*reps;
1951 // inv startntt 512
1952 }
1953
1954 void invntt512_7681(int16 *f,int reps)
1955 {
1956 invntt512(f,reps,qdata_7681);
1957 }