Debian Codegraph @

crypto_core/multsntrup761/avx/mult768.c

raw
1 // 20240812 djb: more cryptoint usage
2 #include "ntt.h"
3 #include <immintrin.h>
4 #include "crypto_int8.h"
5
6 typedef int8_t int8;
7 typedef int16_t int16;
8
9 #define int16x16 __m256i
10 #define load_x16(p) _mm256_loadu_si256((int16x16 *) (p))
11 #define store_x16(p,v) _mm256_storeu_si256((int16x16 *) (p),(v))
12 #define const_x16 _mm256_set1_epi16
13 #define add_x16 _mm256_add_epi16
14 #define sub_x16 _mm256_sub_epi16
15 #define mullo_x16 _mm256_mullo_epi16
16 #define mulhi_x16 _mm256_mulhi_epi16
17 #define mulhrs_x16 _mm256_mulhrs_epi16
18 #define signmask_x16(x) _mm256_srai_epi16((x),15)
19
20 static inline int16x16 squeeze_4591_x16(int16x16 x)
21 {
22 return sub_x16(x,mullo_x16(mulhrs_x16(x,const_x16(7)),const_x16(4591)));
23 }
24
25 static inline int16x16 squeeze_7681_x16(int16x16 x)
26 {
27 return sub_x16(x,mullo_x16(mulhrs_x16(x,const_x16(4)),const_x16(7681)));
28 }
29
30 static inline int16x16 squeeze_10753_x16(int16x16 x)
31 {
32 return sub_x16(x,mullo_x16(mulhrs_x16(x,const_x16(3)),const_x16(10753)));
33 }
34
35 static inline int16x16 mulmod_4591_x16(int16x16 x,int16x16 y)
36 {
37 int16x16 yqinv = mullo_x16(y,const_x16(15631)); /* XXX: precompute */
38 int16x16 b = mulhi_x16(x,y);
39 int16x16 d = mullo_x16(x,yqinv);
40 int16x16 e = mulhi_x16(d,const_x16(4591));
41 return sub_x16(b,e);
42 }
43
44 static inline int16x16 mulmod_7681_x16(int16x16 x,int16x16 y)
45 {
46 int16x16 yqinv = mullo_x16(y,const_x16(-7679)); /* XXX: precompute */
47 int16x16 b = mulhi_x16(x,y);
48 int16x16 d = mullo_x16(x,yqinv);
49 int16x16 e = mulhi_x16(d,const_x16(7681));
50 return sub_x16(b,e);
51 }
52
53 static inline int16x16 mulmod_10753_x16(int16x16 x,int16x16 y)
54 {
55 int16x16 yqinv = mullo_x16(y,const_x16(-10751)); /* XXX: precompute */
56 int16x16 b = mulhi_x16(x,y);
57 int16x16 d = mullo_x16(x,yqinv);
58 int16x16 e = mulhi_x16(d,const_x16(10753));
59 return sub_x16(b,e);
60 }
61
62 #define mask0 _mm256_set_epi16(-1,0,0,-1,0,0,-1,0,0,-1,0,0,-1,0,0,-1)
63 #define mask1 _mm256_set_epi16(0,0,-1,0,0,-1,0,0,-1,0,0,-1,0,0,-1,0)
64 #define mask2 _mm256_set_epi16(0,-1,0,0,-1,0,0,-1,0,0,-1,0,0,-1,0,0)
65
66 static void good(int16 fpad[3][512],const int16 f[768])
67 {
68 int j;
69 int16x16 f0,f1;
70
71 j = 0;
72 for (;;) {
73 f0 = load_x16(f+j);
74 f1 = load_x16(f+512+j);
75 store_x16(&fpad[0][j],(f0&mask0)|(f1&mask1));
76 store_x16(&fpad[1][j],(f0&mask1)|(f1&mask2));
77 store_x16(&fpad[2][j],(f0&mask2)|(f1&mask0));
78 j += 16;
79 if (j == 256) break;
80
81 f0 = load_x16(f+j);
82 f1 = load_x16(f+512+j);
83 store_x16(&fpad[0][j],(f0&mask2)|(f1&mask0));
84 store_x16(&fpad[1][j],(f0&mask0)|(f1&mask1));
85 store_x16(&fpad[2][j],(f0&mask1)|(f1&mask2));
86 j += 16;
87
88 f0 = load_x16(f+j);
89 f1 = load_x16(f+512+j);
90 store_x16(&fpad[0][j],(f0&mask1)|(f1&mask2));
91 store_x16(&fpad[1][j],(f0&mask2)|(f1&mask0));
92 store_x16(&fpad[2][j],(f0&mask0)|(f1&mask1));
93 j += 16;
94 }
95 for (;;) {
96 f0 = load_x16(f+j);
97 store_x16(&fpad[0][j],f0&mask2);
98 store_x16(&fpad[1][j],f0&mask0);
99 store_x16(&fpad[2][j],f0&mask1);
100 j += 16;
101 if (j == 512) break;
102
103 f0 = load_x16(f+j);
104 store_x16(&fpad[0][j],f0&mask1);
105 store_x16(&fpad[1][j],f0&mask2);
106 store_x16(&fpad[2][j],f0&mask0);
107 j += 16;
108
109 f0 = load_x16(f+j);
110 store_x16(&fpad[0][j],f0&mask0);
111 store_x16(&fpad[1][j],f0&mask1);
112 store_x16(&fpad[2][j],f0&mask2);
113 j += 16;
114 }
115 }
116
117 static void ungood(int16 f[1536],const int16 fpad[3][512])
118 {
119 int j;
120 int16x16 f0,f1,f2,g0,g1,g2;
121
122 j = 0;
123
124 for (;;) {
125 f0 = load_x16(&fpad[0][j]);
126 f1 = load_x16(&fpad[1][j]);
127 f2 = load_x16(&fpad[2][j]);
128 g0 = (f0&mask0)|(f1&mask1)|(f2&mask2);
129 g1 = (f0&mask1)|(f1&mask2)|(f2&mask0);
130 g2 = f0^f1^f2^g0^g1; /* same as (f0&mask2)|(f1&mask0)|(f2&mask1) */
131 store_x16(f+0+j,g0);
132 store_x16(f+512+j,g1);
133 store_x16(f+1024+j,g2);
134 j += 16;
135
136 f0 = load_x16(&fpad[0][j]);
137 f1 = load_x16(&fpad[1][j]);
138 f2 = load_x16(&fpad[2][j]);
139 g0 = (f0&mask2)|(f1&mask0)|(f2&mask1);
140 g1 = (f0&mask0)|(f1&mask1)|(f2&mask2);
141 g2 = f0^f1^f2^g0^g1; /* same as (f0&mask1)|(f1&mask2)|(f2&mask0) */
142 store_x16(f+0+j,g0);
143 store_x16(f+512+j,g1);
144 store_x16(f+1024+j,g2);
145 j += 16;
146 if (j == 512) break;
147
148 f0 = load_x16(&fpad[0][j]);
149 f1 = load_x16(&fpad[1][j]);
150 f2 = load_x16(&fpad[2][j]);
151 g0 = (f0&mask1)|(f1&mask2)|(f2&mask0);
152 g1 = (f0&mask2)|(f1&mask0)|(f2&mask1);
153 g2 = f0^f1^f2^g0^g1; /* same as (f0&mask0)|(f1&mask1)|(f2&mask2) */
154 store_x16(f+0+j,g0);
155 store_x16(f+512+j,g1);
156 store_x16(f+1024+j,g2);
157 j += 16;
158 }
159 }
160
161 #define ALIGNED __attribute((aligned(512)))
162
163 static void mult768(int16 h[1536],const int16 f[768],const int16 g[768])
164 {
165 ALIGNED int16 fgpad[6][512];
166 #define fpad fgpad
167 #define gpad (fgpad+3)
168 #define hpad fpad
169 ALIGNED int16 h_7681[1536];
170 ALIGNED int16 h_10753[1536];
171 int i;
172
173 good(fpad,f);
174 good(gpad,g);
175
176 ntt512_7681(fgpad[0],6);
177
178 for (i = 0;i < 512;i += 16) {
179 int16x16 f0 = squeeze_7681_x16(load_x16(&fpad[0][i]));
180 int16x16 f1 = squeeze_7681_x16(load_x16(&fpad[1][i]));
181 int16x16 f2 = squeeze_7681_x16(load_x16(&fpad[2][i]));
182 int16x16 g0 = squeeze_7681_x16(load_x16(&gpad[0][i]));
183 int16x16 g1 = squeeze_7681_x16(load_x16(&gpad[1][i]));
184 int16x16 g2 = squeeze_7681_x16(load_x16(&gpad[2][i]));
185 int16x16 d0 = mulmod_7681_x16(f0,g0);
186 int16x16 d1 = mulmod_7681_x16(f1,g1);
187 int16x16 d2 = mulmod_7681_x16(f2,g2);
188 int16x16 dsum = add_x16(add_x16(d0,d1),d2);
189 int16x16 h0 = add_x16(dsum,mulmod_7681_x16(sub_x16(f2,f1),sub_x16(g1,g2)));
190 int16x16 h1 = add_x16(dsum,mulmod_7681_x16(sub_x16(f1,f0),sub_x16(g0,g1)));
191 int16x16 h2 = add_x16(dsum,mulmod_7681_x16(sub_x16(f0,f2),sub_x16(g2,g0)));
192 store_x16(&hpad[0][i],squeeze_7681_x16(h0));
193 store_x16(&hpad[1][i],squeeze_7681_x16(h1));
194 store_x16(&hpad[2][i],squeeze_7681_x16(h2));
195 }
196
197 invntt512_7681(hpad[0],3);
198 ungood(h_7681,hpad);
199
200 good(fpad,f);
201 good(gpad,g);
202
203 ntt512_10753(fgpad[0],6);
204
205 for (i = 0;i < 512;i += 16) {
206 int16x16 f0 = squeeze_10753_x16(load_x16(&fpad[0][i]));
207 int16x16 f1 = squeeze_10753_x16(load_x16(&fpad[1][i]));
208 int16x16 f2 = squeeze_10753_x16(load_x16(&fpad[2][i]));
209 int16x16 g0 = squeeze_10753_x16(load_x16(&gpad[0][i]));
210 int16x16 g1 = squeeze_10753_x16(load_x16(&gpad[1][i]));
211 int16x16 g2 = squeeze_10753_x16(load_x16(&gpad[2][i]));
212 int16x16 d0 = mulmod_10753_x16(f0,g0);
213 int16x16 d1 = mulmod_10753_x16(f1,g1);
214 int16x16 d2 = mulmod_10753_x16(f2,g2);
215 int16x16 dsum = add_x16(add_x16(d0,d1),d2);
216 int16x16 h0 = add_x16(dsum,mulmod_10753_x16(sub_x16(f2,f1),sub_x16(g1,g2)));
217 int16x16 h1 = add_x16(dsum,mulmod_10753_x16(sub_x16(f1,f0),sub_x16(g0,g1)));
218 int16x16 h2 = add_x16(dsum,mulmod_10753_x16(sub_x16(f0,f2),sub_x16(g2,g0)));
219 store_x16(&hpad[0][i],squeeze_10753_x16(h0));
220 store_x16(&hpad[1][i],squeeze_10753_x16(h1));
221 store_x16(&hpad[2][i],squeeze_10753_x16(h2));
222 }
223
224 invntt512_10753(hpad[0],3);
225 ungood(h_10753,hpad);
226
227 for (i = 0;i < 1536;i += 16) {
228 int16x16 u1 = load_x16(&h_10753[i]);
229 int16x16 u2 = load_x16(&h_7681[i]);
230 int16x16 t;
231 u1 = mulmod_10753_x16(u1,const_x16(1268));
232 u2 = mulmod_7681_x16(u2,const_x16(956));
233 t = mulmod_7681_x16(sub_x16(u2,u1),const_x16(-2539));
234 t = add_x16(u1,mulmod_4591_x16(t,const_x16(-710)));
235 store_x16(&h[i],t);
236 }
237 }
238
239 #include "crypto_core.h"
240 #include "crypto_decode_761xint16.h"
241 #define crypto_decode_pxint16 crypto_decode_761xint16
242 #include "crypto_encode_761xint16.h"
243 #define crypto_encode_pxint16 crypto_encode_761xint16
244
245 #define p 761
246 #define q 4591
247
248 static inline int16x16 freeze_4591_x16(int16x16 x)
249 {
250 int16x16 mask, xq;
251 x = add_x16(x,const_x16(q)&signmask_x16(x));
252 mask = signmask_x16(sub_x16(x,const_x16((q+1)/2)));
253 xq = sub_x16(x,const_x16(q));
254 x = _mm256_blendv_epi8(xq,x,mask);
255 return x;
256 }
257
258 void crypto_core(unsigned char *outbytes,const unsigned char *inbytes,const unsigned char *kbytes,const unsigned char *cbytes)
259 {
260 ALIGNED int16 f[768];
261 ALIGNED int16 g[768];
262 ALIGNED int16 fg[1536];
263 #define h f
264 int i;
265 int16x16 x;
266
267 x = const_x16(0);
268 for (i = p&~15;i < 768;i += 16) store_x16(&f[i],x);
269 for (i = p&~15;i < 768;i += 16) store_x16(&g[i],x);
270
271 crypto_decode_pxint16(f,inbytes);
272
273 for (i = 0;i < 768;i += 16) {
274 x = load_x16(&f[i]);
275 x = freeze_4591_x16(squeeze_4591_x16(x));
276 store_x16(&f[i],x);
277 }
278 for (i = 0;i < p;++i) {
279 int8 gi = kbytes[i];
280 int8 gi0 = crypto_int8_bottombit_01(gi);
281 g[i] = gi0-(gi&(gi0<<1));
282 }
283
284 mult768(fg,f,g);
285
286 fg[0] -= fg[p-1];
287 for (i = 0;i < 768;i += 16) {
288 int16x16 fgi = load_x16(&fg[i]);
289 int16x16 fgip = load_x16(&fg[i + p]);
290 int16x16 fgip1 = load_x16(&fg[i + p - 1]);
291 x = add_x16(fgi,add_x16(fgip,fgip1));
292 x = freeze_4591_x16(squeeze_4591_x16(x));
293 store_x16(&h[i],x);
294 }
295
296 crypto_encode_pxint16(outbytes,h);
297 }