Debian Codegraph @

crypto_core/mult3sntrup1277/avx/mult1280.c

raw
1 // 20240812 djb: more cryptoint usage
2 #include "ntt.h"
3 #include <immintrin.h>
4 #include "crypto_int8.h"
5
6 typedef int8_t int8;
7 typedef int16_t int16;
8
9 #define int16x16 __m256i
10 #define load_x16(p) _mm256_loadu_si256((int16x16 *) (p))
11 #define store_x16(p,v) _mm256_storeu_si256((int16x16 *) (p),(v))
12 #define const_x16 _mm256_set1_epi16
13 #define add_x16 _mm256_add_epi16
14 #define sub_x16 _mm256_sub_epi16
15 #define mullo_x16 _mm256_mullo_epi16
16 #define mulhi_x16 _mm256_mulhi_epi16
17 #define mulhrs_x16 _mm256_mulhrs_epi16
18 #define signmask_x16(x) _mm256_srai_epi16((x),15)
19
20 static int16x16 squeeze_3_x16(int16x16 x)
21 {
22 return sub_x16(x,mullo_x16(mulhrs_x16(x,const_x16(10923)),const_x16(3)));
23 }
24
25 static int16x16 squeeze_7681_x16(int16x16 x)
26 {
27 return sub_x16(x,mullo_x16(mulhrs_x16(x,const_x16(4)),const_x16(7681)));
28 }
29
30 static int16x16 mulmod_7681_x16(int16x16 x,int16x16 y)
31 {
32 int16x16 yqinv = mullo_x16(y,const_x16(-7679)); /* XXX: precompute */
33 int16x16 b = mulhi_x16(x,y);
34 int16x16 d = mullo_x16(x,yqinv);
35 int16x16 e = mulhi_x16(d,const_x16(7681));
36 return sub_x16(b,e);
37 }
38
39 #define mask0 _mm256_set_epi16(-1,0,0,0,0,-1,0,0,0,0,-1,0,0,0,0,-1)
40 #define mask1 _mm256_set_epi16(0,0,-1,0,0,0,0,-1,0,0,0,0,-1,0,0,0)
41 #define mask2 _mm256_set_epi16(0,0,0,0,-1,0,0,0,0,-1,0,0,0,0,-1,0)
42 #define mask3 _mm256_set_epi16(0,-1,0,0,0,0,-1,0,0,0,0,-1,0,0,0,0)
43 #define mask4 _mm256_set_epi16(0,0,0,-1,0,0,0,0,-1,0,0,0,0,-1,0,0)
44
45 static void good(int16 fpad[5][512],const int16 f[1280])
46 {
47 int j;
48 int16x16 f0,f1,f2;
49
50 j = 0;
51 for (;;) {
52 f0 = load_x16(f+j);
53 f1 = load_x16(f+512+j);
54 f2 = load_x16(f+1024+j);
55 store_x16(&fpad[0][j],(f0&mask0)|(f1&mask1)|(f2&mask2));
56 store_x16(&fpad[1][j],(f0&mask1)|(f1&mask2)|(f2&mask3));
57 store_x16(&fpad[2][j],(f0&mask2)|(f1&mask3)|(f2&mask4));
58 store_x16(&fpad[3][j],(f0&mask3)|(f1&mask4)|(f2&mask0));
59 store_x16(&fpad[4][j],(f0&mask4)|(f1&mask0)|(f2&mask1));
60 j += 16;
61 if (j == 256) break;
62
63 f0 = load_x16(f+j);
64 f1 = load_x16(f+512+j);
65 f2 = load_x16(f+1024+j);
66 store_x16(&fpad[0][j],(f0&mask3)|(f1&mask4)|(f2&mask0));
67 store_x16(&fpad[1][j],(f0&mask4)|(f1&mask0)|(f2&mask1));
68 store_x16(&fpad[2][j],(f0&mask0)|(f1&mask1)|(f2&mask2));
69 store_x16(&fpad[3][j],(f0&mask1)|(f1&mask2)|(f2&mask3));
70 store_x16(&fpad[4][j],(f0&mask2)|(f1&mask3)|(f2&mask4));
71 j += 16;
72
73 f0 = load_x16(f+j);
74 f1 = load_x16(f+512+j);
75 f2 = load_x16(f+1024+j);
76 store_x16(&fpad[0][j],(f0&mask1)|(f1&mask2)|(f2&mask3));
77 store_x16(&fpad[1][j],(f0&mask2)|(f1&mask3)|(f2&mask4));
78 store_x16(&fpad[2][j],(f0&mask3)|(f1&mask4)|(f2&mask0));
79 store_x16(&fpad[3][j],(f0&mask4)|(f1&mask0)|(f2&mask1));
80 store_x16(&fpad[4][j],(f0&mask0)|(f1&mask1)|(f2&mask2));
81 j += 16;
82
83 f0 = load_x16(f+j);
84 f1 = load_x16(f+512+j);
85 f2 = load_x16(f+1024+j);
86 store_x16(&fpad[0][j],(f0&mask4)|(f1&mask0)|(f2&mask1));
87 store_x16(&fpad[1][j],(f0&mask0)|(f1&mask1)|(f2&mask2));
88 store_x16(&fpad[2][j],(f0&mask1)|(f1&mask2)|(f2&mask3));
89 store_x16(&fpad[3][j],(f0&mask2)|(f1&mask3)|(f2&mask4));
90 store_x16(&fpad[4][j],(f0&mask3)|(f1&mask4)|(f2&mask0));
91 j += 16;
92
93 f0 = load_x16(f+j);
94 f1 = load_x16(f+512+j);
95 f2 = load_x16(f+1024+j);
96 store_x16(&fpad[0][j],(f0&mask2)|(f1&mask3)|(f2&mask4));
97 store_x16(&fpad[1][j],(f0&mask3)|(f1&mask4)|(f2&mask0));
98 store_x16(&fpad[2][j],(f0&mask4)|(f1&mask0)|(f2&mask1));
99 store_x16(&fpad[3][j],(f0&mask0)|(f1&mask1)|(f2&mask2));
100 store_x16(&fpad[4][j],(f0&mask1)|(f1&mask2)|(f2&mask3));
101 j += 16;
102 }
103 for (;;) {
104 f0 = load_x16(f+j);
105 f1 = load_x16(f+512+j);
106 store_x16(&fpad[0][j],(f0&mask3)|(f1&mask4));
107 store_x16(&fpad[1][j],(f0&mask4)|(f1&mask0));
108 store_x16(&fpad[2][j],(f0&mask0)|(f1&mask1));
109 store_x16(&fpad[3][j],(f0&mask1)|(f1&mask2));
110 store_x16(&fpad[4][j],(f0&mask2)|(f1&mask3));
111 j += 16;
112 if (j == 512) break;
113
114 f0 = load_x16(f+j);
115 f1 = load_x16(f+512+j);
116 store_x16(&fpad[0][j],(f0&mask1)|(f1&mask2));
117 store_x16(&fpad[1][j],(f0&mask2)|(f1&mask3));
118 store_x16(&fpad[2][j],(f0&mask3)|(f1&mask4));
119 store_x16(&fpad[3][j],(f0&mask4)|(f1&mask0));
120 store_x16(&fpad[4][j],(f0&mask0)|(f1&mask1));
121 j += 16;
122
123 f0 = load_x16(f+j);
124 f1 = load_x16(f+512+j);
125 store_x16(&fpad[0][j],(f0&mask4)|(f1&mask0));
126 store_x16(&fpad[1][j],(f0&mask0)|(f1&mask1));
127 store_x16(&fpad[2][j],(f0&mask1)|(f1&mask2));
128 store_x16(&fpad[3][j],(f0&mask2)|(f1&mask3));
129 store_x16(&fpad[4][j],(f0&mask3)|(f1&mask4));
130 j += 16;
131
132 f0 = load_x16(f+j);
133 f1 = load_x16(f+512+j);
134 store_x16(&fpad[0][j],(f0&mask2)|(f1&mask3));
135 store_x16(&fpad[1][j],(f0&mask3)|(f1&mask4));
136 store_x16(&fpad[2][j],(f0&mask4)|(f1&mask0));
137 store_x16(&fpad[3][j],(f0&mask0)|(f1&mask1));
138 store_x16(&fpad[4][j],(f0&mask1)|(f1&mask2));
139 j += 16;
140
141 f0 = load_x16(f+j);
142 f1 = load_x16(f+512+j);
143 store_x16(&fpad[0][j],(f0&mask0)|(f1&mask1));
144 store_x16(&fpad[1][j],(f0&mask1)|(f1&mask2));
145 store_x16(&fpad[2][j],(f0&mask2)|(f1&mask3));
146 store_x16(&fpad[3][j],(f0&mask3)|(f1&mask4));
147 store_x16(&fpad[4][j],(f0&mask4)|(f1&mask0));
148 j += 16;
149 }
150 }
151
152 static void ungood(int16 f[2560],const int16 fpad[5][512])
153 {
154 int j;
155 int16x16 f0,f1,f2,f3,f4,g0,g1,g2,g3,g4;
156
157 j = 0;
158
159 for (;;) {
160 f0 = load_x16(&fpad[0][j]);
161 f1 = load_x16(&fpad[1][j]);
162 f2 = load_x16(&fpad[2][j]);
163 f3 = load_x16(&fpad[3][j]);
164 f4 = load_x16(&fpad[4][j]);
165 g0 = (f0&mask0)|(f1&mask1)|(f2&mask2)|(f3&mask3)|(f4&mask4);
166 g1 = (f0&mask1)|(f1&mask2)|(f2&mask3)|(f3&mask4)|(f4&mask0);
167 g2 = (f0&mask2)|(f1&mask3)|(f2&mask4)|(f3&mask0)|(f4&mask1);
168 g3 = (f0&mask3)|(f1&mask4)|(f2&mask0)|(f3&mask1)|(f4&mask2);
169 g4 = f0^f1^f2^f3^f4^g0^g1^g2^g3; /* same as (f0&mask4)|(f1&mask0)|(f2&mask1)|(f3&mask2)|(f4&mask3); */
170 store_x16(f+0+j,g0);
171 store_x16(f+512+j,g1);
172 store_x16(f+1024+j,g2);
173 store_x16(f+1536+j,g3);
174 store_x16(f+2048+j,g4);
175 j += 16;
176
177 f0 = load_x16(&fpad[0][j]);
178 f1 = load_x16(&fpad[1][j]);
179 f2 = load_x16(&fpad[2][j]);
180 f3 = load_x16(&fpad[3][j]);
181 f4 = load_x16(&fpad[4][j]);
182 g0 = (f0&mask3)|(f1&mask4)|(f2&mask0)|(f3&mask1)|(f4&mask2);
183 g1 = (f0&mask4)|(f1&mask0)|(f2&mask1)|(f3&mask2)|(f4&mask3);
184 g2 = (f0&mask0)|(f1&mask1)|(f2&mask2)|(f3&mask3)|(f4&mask4);
185 g3 = (f0&mask1)|(f1&mask2)|(f2&mask3)|(f3&mask4)|(f4&mask0);
186 g4 = f0^f1^f2^f3^f4^g0^g1^g2^g3;
187 store_x16(f+0+j,g0);
188 store_x16(f+512+j,g1);
189 store_x16(f+1024+j,g2);
190 store_x16(f+1536+j,g3);
191 store_x16(f+2048+j,g4);
192 j += 16;
193 if (j == 512) break;
194
195 f0 = load_x16(&fpad[0][j]);
196 f1 = load_x16(&fpad[1][j]);
197 f2 = load_x16(&fpad[2][j]);
198 f3 = load_x16(&fpad[3][j]);
199 f4 = load_x16(&fpad[4][j]);
200 g0 = (f0&mask1)|(f1&mask2)|(f2&mask3)|(f3&mask4)|(f4&mask0);
201 g1 = (f0&mask2)|(f1&mask3)|(f2&mask4)|(f3&mask0)|(f4&mask1);
202 g2 = (f0&mask3)|(f1&mask4)|(f2&mask0)|(f3&mask1)|(f4&mask2);
203 g3 = (f0&mask4)|(f1&mask0)|(f2&mask1)|(f3&mask2)|(f4&mask3);
204 g4 = f0^f1^f2^f3^f4^g0^g1^g2^g3;
205 store_x16(f+0+j,g0);
206 store_x16(f+512+j,g1);
207 store_x16(f+1024+j,g2);
208 store_x16(f+1536+j,g3);
209 store_x16(f+2048+j,g4);
210 j += 16;
211
212 f0 = load_x16(&fpad[0][j]);
213 f1 = load_x16(&fpad[1][j]);
214 f2 = load_x16(&fpad[2][j]);
215 f3 = load_x16(&fpad[3][j]);
216 f4 = load_x16(&fpad[4][j]);
217 g0 = (f0&mask4)|(f1&mask0)|(f2&mask1)|(f3&mask2)|(f4&mask3);
218 g1 = (f0&mask0)|(f1&mask1)|(f2&mask2)|(f3&mask3)|(f4&mask4);
219 g2 = (f0&mask1)|(f1&mask2)|(f2&mask3)|(f3&mask4)|(f4&mask0);
220 g3 = (f0&mask2)|(f1&mask3)|(f2&mask4)|(f3&mask0)|(f4&mask1);
221 g4 = f0^f1^f2^f3^f4^g0^g1^g2^g3;
222 store_x16(f+0+j,g0);
223 store_x16(f+512+j,g1);
224 store_x16(f+1024+j,g2);
225 store_x16(f+1536+j,g3);
226 store_x16(f+2048+j,g4);
227 j += 16;
228
229 f0 = load_x16(&fpad[0][j]);
230 f1 = load_x16(&fpad[1][j]);
231 f2 = load_x16(&fpad[2][j]);
232 f3 = load_x16(&fpad[3][j]);
233 f4 = load_x16(&fpad[4][j]);
234 g0 = (f0&mask2)|(f1&mask3)|(f2&mask4)|(f3&mask0)|(f4&mask1);
235 g1 = (f0&mask3)|(f1&mask4)|(f2&mask0)|(f3&mask1)|(f4&mask2);
236 g2 = (f0&mask4)|(f1&mask0)|(f2&mask1)|(f3&mask2)|(f4&mask3);
237 g3 = (f0&mask0)|(f1&mask1)|(f2&mask2)|(f3&mask3)|(f4&mask4);
238 g4 = f0^f1^f2^f3^f4^g0^g1^g2^g3;
239 store_x16(f+0+j,g0);
240 store_x16(f+512+j,g1);
241 store_x16(f+1024+j,g2);
242 store_x16(f+1536+j,g3);
243 store_x16(f+2048+j,g4);
244 j += 16;
245 }
246 }
247
248 #define ALIGNED __attribute((aligned(32)))
249
250 static void mult1280(int16 h[2560],const int16 f[1280],const int16 g[1280])
251 {
252 ALIGNED int16 fgpad[10][512];
253 #define fpad fgpad
254 #define gpad (fgpad+5)
255 #define hpad fpad
256 ALIGNED int16 h_7681[2560];
257 int i;
258
259 good(fpad,f);
260 good(gpad,g);
261
262 ntt512_7681(fgpad[0],10);
263
264 for (i = 0;i < 512;i += 16) {
265 int16x16 f0 = squeeze_7681_x16(load_x16(&fpad[0][i]));
266 int16x16 f1 = squeeze_7681_x16(load_x16(&fpad[1][i]));
267 int16x16 f2 = squeeze_7681_x16(load_x16(&fpad[2][i]));
268 int16x16 f3 = squeeze_7681_x16(load_x16(&fpad[3][i]));
269 int16x16 f4 = squeeze_7681_x16(load_x16(&fpad[4][i]));
270 int16x16 g0 = squeeze_7681_x16(load_x16(&gpad[0][i]));
271 int16x16 g1 = squeeze_7681_x16(load_x16(&gpad[1][i]));
272 int16x16 g2 = squeeze_7681_x16(load_x16(&gpad[2][i]));
273 int16x16 g3 = squeeze_7681_x16(load_x16(&gpad[3][i]));
274 int16x16 g4 = squeeze_7681_x16(load_x16(&gpad[4][i]));
275 int16x16 d0 = mulmod_7681_x16(f0,g0);
276 int16x16 d1 = mulmod_7681_x16(f1,g1);
277 int16x16 d2 = mulmod_7681_x16(f2,g2);
278 int16x16 d3 = mulmod_7681_x16(f3,g3);
279 int16x16 d4 = mulmod_7681_x16(f4,g4);
280 int16x16 dsum;
281 int16x16 h0,h1,h2,h3,h4;
282
283 dsum = add_x16(add_x16(d0,d1),d2);
284 dsum = squeeze_7681_x16(dsum);
285 dsum = add_x16(add_x16(d3,d4),dsum);
286 dsum = squeeze_7681_x16(dsum);
287
288 /* could save one mult here using sum hi = (sum fi)(sum gi) */
289
290 h0 = add_x16(dsum,mulmod_7681_x16(sub_x16(f4,f1),sub_x16(g1,g4)));
291 h1 = add_x16(dsum,mulmod_7681_x16(sub_x16(f1,f0),sub_x16(g0,g1)));
292 h2 = add_x16(dsum,mulmod_7681_x16(sub_x16(f2,f0),sub_x16(g0,g2)));
293 h3 = add_x16(dsum,mulmod_7681_x16(sub_x16(f3,f0),sub_x16(g0,g3)));
294 h4 = add_x16(dsum,mulmod_7681_x16(sub_x16(f4,f0),sub_x16(g0,g4)));
295 h0 = add_x16(h0,mulmod_7681_x16(sub_x16(f3,f2),sub_x16(g2,g3)));
296 h1 = add_x16(h1,mulmod_7681_x16(sub_x16(f4,f2),sub_x16(g2,g4)));
297 h2 = add_x16(h2,mulmod_7681_x16(sub_x16(f4,f3),sub_x16(g3,g4)));
298 h3 = add_x16(h3,mulmod_7681_x16(sub_x16(f2,f1),sub_x16(g1,g2)));
299 h4 = add_x16(h4,mulmod_7681_x16(sub_x16(f3,f1),sub_x16(g1,g3)));
300
301 store_x16(&hpad[0][i],squeeze_7681_x16(h0));
302 store_x16(&hpad[1][i],squeeze_7681_x16(h1));
303 store_x16(&hpad[2][i],squeeze_7681_x16(h2));
304 store_x16(&hpad[3][i],squeeze_7681_x16(h3));
305 store_x16(&hpad[4][i],squeeze_7681_x16(h4));
306 }
307
308 invntt512_7681(hpad[0],5);
309 ungood(h_7681,hpad);
310
311 for (i = 0;i < 2560;i += 16) {
312 int16x16 u = load_x16(&h_7681[i]);
313 u = mulmod_7681_x16(u,const_x16(956));
314 store_x16(&h[i],u);
315 }
316 }
317
318 #include "crypto_core.h"
319 #include "crypto_decode_1277xint16.h"
320 #define crypto_decode_pxint16 crypto_decode_1277xint16
321 #include "crypto_encode_1277xint16.h"
322 #define crypto_encode_pxint16 crypto_encode_1277xint16
323
324 #define p 1277
325
326 static inline int16x16 freeze_3_x16(int16x16 x)
327 {
328 int16x16 mask, x3;
329 x = add_x16(x,const_x16(3)&signmask_x16(x));
330 mask = signmask_x16(sub_x16(x,const_x16(2)));
331 x3 = sub_x16(x,const_x16(3));
332 x = _mm256_blendv_epi8(x3,x,mask);
333 return x;
334 }
335
336 void crypto_core(unsigned char *outbytes,const unsigned char *inbytes,const unsigned char *kbytes,const unsigned char *cbytes)
337 {
338 ALIGNED int16 f[1280];
339 ALIGNED int16 g[1280];
340 ALIGNED int16 fg[2560];
341 #define h f
342 int i;
343 int16x16 x;
344
345 x = const_x16(0);
346 for (i = p&~15;i < 1280;i += 16) store_x16(&f[i],x);
347 for (i = p&~15;i < 1280;i += 16) store_x16(&g[i],x);
348
349 for (i = 0;i < p;++i) {
350 int8 fi = inbytes[i];
351 int8 fi0 = crypto_int8_bottombit_01(fi);
352 f[i] = fi0-(fi&(fi0<<1));
353 }
354 for (i = 0;i < p;++i) {
355 int8 gi = kbytes[i];
356 int8 gi0 = crypto_int8_bottombit_01(gi);
357 g[i] = gi0-(gi&(gi0<<1));
358 }
359
360 mult1280(fg,f,g);
361
362 fg[0] -= fg[p-1];
363 for (i = 0;i < 1280;i += 16) {
364 int16x16 fgi = load_x16(&fg[i]);
365 int16x16 fgip = load_x16(&fg[i + p]);
366 int16x16 fgip1 = load_x16(&fg[i + p - 1]);
367 x = add_x16(fgi,add_x16(fgip,fgip1));
368 x = freeze_3_x16(squeeze_3_x16(x));
369 store_x16(&h[i],x);
370 }
371
372 for (i = 0;i < p;++i) outbytes[i] = h[i];
373 }