Debian Codegraph @

crypto_core/mult3sntrup857/avx/mult1024.c

raw
1 // 20240812 djb: more cryptoint usage
2 #include "ntt.h"
3 #include <immintrin.h>
4 #include "crypto_int8.h"
5
6 typedef int8_t int8;
7 typedef int16_t int16;
8
9 #define int16x16 __m256i
10 #define load_x16(p) _mm256_loadu_si256((int16x16 *) (p))
11 #define store_x16(p,v) _mm256_storeu_si256((int16x16 *) (p),(v))
12 #define const_x16 _mm256_set1_epi16
13 #define add_x16 _mm256_add_epi16
14 #define sub_x16 _mm256_sub_epi16
15 #define mullo_x16 _mm256_mullo_epi16
16 #define mulhi_x16 _mm256_mulhi_epi16
17 #define mulhrs_x16 _mm256_mulhrs_epi16
18 #define signmask_x16(x) _mm256_srai_epi16((x),15)
19
20 static int16x16 squeeze_3_x16(int16x16 x)
21 {
22 return sub_x16(x,mullo_x16(mulhrs_x16(x,const_x16(10923)),const_x16(3)));
23 }
24
25 static int16x16 squeeze_7681_x16(int16x16 x)
26 {
27 return sub_x16(x,mullo_x16(mulhrs_x16(x,const_x16(4)),const_x16(7681)));
28 }
29
30 static int16x16 mulmod_7681_x16(int16x16 x,int16x16 y)
31 {
32 int16x16 yqinv = mullo_x16(y,const_x16(-7679)); /* XXX: precompute */
33 int16x16 b = mulhi_x16(x,y);
34 int16x16 d = mullo_x16(x,yqinv);
35 int16x16 e = mulhi_x16(d,const_x16(7681));
36 return sub_x16(b,e);
37 }
38
39 static void stride(int16 fpad[4][512],const int16 f[1024])
40 {
41 int16x16 f0,f1,f2,f3,g0,g1,g2,g3;
42 int i,j;
43
44 for (j = 0;j < 256;j += 16) {
45 f0 = load_x16(&f[0]);
46 f1 = load_x16(&f[16]);
47 f2 = load_x16(&f[32]);
48 f3 = load_x16(&f[48]);
49 f += 64;
50
51 g0 = _mm256_permute2x128_si256(f0,f2,0x20);
52 g1 = _mm256_permute2x128_si256(f0,f2,0x31);
53 g2 = _mm256_permute2x128_si256(f1,f3,0x20);
54 g3 = _mm256_permute2x128_si256(f1,f3,0x31);
55 f0 = _mm256_unpacklo_epi16(g0,g1);
56 f1 = _mm256_unpackhi_epi16(g0,g1);
57 f2 = _mm256_unpacklo_epi16(g2,g3);
58 f3 = _mm256_unpackhi_epi16(g2,g3);
59 g0 = _mm256_unpacklo_epi16(f0,f1);
60 g1 = _mm256_unpackhi_epi16(f0,f1);
61 g2 = _mm256_unpacklo_epi16(f2,f3);
62 g3 = _mm256_unpackhi_epi16(f2,f3);
63 f0 = _mm256_unpacklo_epi64(g0,g2);
64 f1 = _mm256_unpackhi_epi64(g0,g2);
65 f2 = _mm256_unpacklo_epi64(g1,g3);
66 f3 = _mm256_unpackhi_epi64(g1,g3);
67
68 store_x16(&fpad[0][j],f0);
69 store_x16(&fpad[1][j],f1);
70 store_x16(&fpad[2][j],f2);
71 store_x16(&fpad[3][j],f3);
72 }
73
74 for (i = 0;i < 4;++i)
75 for (j = 256;j < 512;++j)
76 fpad[i][j] = 0;
77 }
78
79 static void unstride(int16 f[2048],const int16 fpad[4][512])
80 {
81 int16x16 f0,f1,f2,f3,g0,g1,g2,g3,h0,h1,h2,h3;
82 int j;
83
84 for (j = 0;j < 512;j += 16) {
85 f0 = load_x16(&fpad[0][j]);
86 f1 = load_x16(&fpad[1][j]);
87 f2 = load_x16(&fpad[2][j]);
88 f3 = load_x16(&fpad[3][j]);
89
90 g2 = _mm256_unpacklo_epi16(f2,f3);
91 g3 = _mm256_unpackhi_epi16(f2,f3);
92 g0 = _mm256_unpacklo_epi16(f0,f1);
93 h0 = _mm256_unpacklo_epi32(g0,g2);
94 h1 = _mm256_unpackhi_epi32(g0,g2);
95 g1 = _mm256_unpackhi_epi16(f0,f1);
96 h2 = _mm256_unpacklo_epi32(g1,g3);
97 h3 = _mm256_unpackhi_epi32(g1,g3);
98 f1 = _mm256_permute2x128_si256(h2,h3,0x20);
99 f3 = _mm256_permute2x128_si256(h2,h3,0x31);
100 f0 = _mm256_permute2x128_si256(h0,h1,0x20);
101 f2 = _mm256_permute2x128_si256(h0,h1,0x31);
102
103 store_x16(&f[0],f0);
104 store_x16(&f[16],f1);
105 store_x16(&f[32],f2);
106 store_x16(&f[48],f3);
107 f += 64;
108 }
109 }
110
111 #define ALIGNED __attribute((aligned(32)))
112
113 static const ALIGNED int16 y_7681[512] = {
114 #include "precomp7681.inc"
115 } ;
116
117 static void mult1024(int16 h[2048],const int16 f[1024],const int16 g[1024])
118 {
119 ALIGNED int16 fgpad[8][512];
120 #define fpad fgpad
121 #define gpad (fgpad+4)
122 #define hpad fpad
123 ALIGNED int16 h_7681[2048];
124 int i;
125
126 stride(fpad,f);
127 stride(gpad,g);
128
129 ntt512_7681(fgpad[0],8);
130
131 /* XXX: try arbitrary-degree Karatsuba */
132
133 for (i = 0;i < 512;i += 16) {
134 int16x16 f0 = squeeze_7681_x16(load_x16(&fpad[0][i]));
135 int16x16 f1 = squeeze_7681_x16(load_x16(&fpad[1][i]));
136 int16x16 f2 = squeeze_7681_x16(load_x16(&fpad[2][i]));
137 int16x16 f3 = squeeze_7681_x16(load_x16(&fpad[3][i]));
138 int16x16 g0 = squeeze_7681_x16(load_x16(&gpad[0][i]));
139 int16x16 g1 = squeeze_7681_x16(load_x16(&gpad[1][i]));
140 int16x16 g2 = squeeze_7681_x16(load_x16(&gpad[2][i]));
141 int16x16 g3 = squeeze_7681_x16(load_x16(&gpad[3][i]));
142 int16x16 d0 = mulmod_7681_x16(f0,g0);
143 int16x16 d1 = mulmod_7681_x16(f1,g1);
144 int16x16 d2 = mulmod_7681_x16(f2,g2);
145 int16x16 d3 = mulmod_7681_x16(f3,g3);
146 int16x16 d0d1 = add_x16(d0,d1);
147 int16x16 d0d1d2 = add_x16(d0d1,d2);
148 int16x16 d0d1d2d3 = squeeze_7681_x16(add_x16(d0d1d2,d3));
149 int16x16 d2d3 = add_x16(d2,d3);
150 int16x16 d1d2d3 = add_x16(d1,d2d3);
151 int16x16 e01 = mulmod_7681_x16(sub_x16(f0,f1),sub_x16(g0,g1));
152 int16x16 e02 = mulmod_7681_x16(sub_x16(f0,f2),sub_x16(g0,g2));
153 int16x16 e03 = mulmod_7681_x16(sub_x16(f0,f3),sub_x16(g0,g3));
154 int16x16 e12 = mulmod_7681_x16(sub_x16(f1,f2),sub_x16(g1,g2));
155 int16x16 e13 = mulmod_7681_x16(sub_x16(f1,f3),sub_x16(g1,g3));
156 int16x16 e23 = mulmod_7681_x16(sub_x16(f2,f3),sub_x16(g2,g3));
157 int16x16 h0 = d0;
158 int16x16 h1 = sub_x16(d0d1,e01);
159 int16x16 h2 = sub_x16(d0d1d2,e02);
160 int16x16 h3 = sub_x16(d0d1d2d3,add_x16(e12,e03));
161 int16x16 h4 = sub_x16(d1d2d3,e13);
162 int16x16 h5 = sub_x16(d2d3,e23);
163 int16x16 h6 = d3;
164 int16x16 twist = load_x16(&y_7681[i]);
165 h4 = mulmod_7681_x16(h4,twist);
166 h5 = mulmod_7681_x16(h5,twist);
167 h6 = mulmod_7681_x16(h6,twist);
168 h0 = add_x16(h0,h4);
169 h1 = add_x16(h1,h5);
170 h2 = add_x16(h2,h6);
171 store_x16(&hpad[0][i],squeeze_7681_x16(h0));
172 store_x16(&hpad[1][i],squeeze_7681_x16(h1));
173 store_x16(&hpad[2][i],squeeze_7681_x16(h2));
174 store_x16(&hpad[3][i],squeeze_7681_x16(h3));
175 }
176
177 invntt512_7681(hpad[0],4);
178 unstride(h_7681,hpad);
179
180 for (i = 0;i < 2048;i += 16) {
181 int16x16 u = load_x16(&h_7681[i]);
182 u = mulmod_7681_x16(u,const_x16(956));
183 store_x16(&h[i],u);
184 }
185 }
186
187 #include "crypto_core.h"
188 #include "crypto_decode_857xint16.h"
189 #define crypto_decode_pxint16 crypto_decode_857xint16
190 #include "crypto_encode_857xint16.h"
191 #define crypto_encode_pxint16 crypto_encode_857xint16
192
193 #define p 857
194
195 static inline int16x16 freeze_3_x16(int16x16 x)
196 {
197 int16x16 mask, x3;
198 x = add_x16(x,const_x16(3)&signmask_x16(x));
199 mask = signmask_x16(sub_x16(x,const_x16(2)));
200 x3 = sub_x16(x,const_x16(3));
201 x = _mm256_blendv_epi8(x3,x,mask);
202 return x;
203 }
204
205 void crypto_core(unsigned char *outbytes,const unsigned char *inbytes,const unsigned char *kbytes,const unsigned char *cbytes)
206 {
207 ALIGNED int16 f[1024];
208 ALIGNED int16 g[1024];
209 ALIGNED int16 fg[2048];
210 #define h f
211 int i;
212 int16x16 x;
213
214 x = const_x16(0);
215 for (i = p&~15;i < 1024;i += 16) store_x16(&f[i],x);
216 for (i = p&~15;i < 1024;i += 16) store_x16(&g[i],x);
217
218 for (i = 0;i < p;++i) {
219 int8 fi = inbytes[i];
220 int8 fi0 = crypto_int8_bottombit_01(fi);
221 f[i] = fi0-(fi&(fi0<<1));
222 }
223 for (i = 0;i < p;++i) {
224 int8 gi = kbytes[i];
225 int8 gi0 = crypto_int8_bottombit_01(gi);
226 g[i] = gi0-(gi&(gi0<<1));
227 }
228
229 mult1024(fg,f,g);
230
231 fg[0] -= fg[p-1];
232 for (i = 0;i < 1024;i += 16) {
233 int16x16 fgi = load_x16(&fg[i]);
234 int16x16 fgip = load_x16(&fg[i + p]);
235 int16x16 fgip1 = load_x16(&fg[i + p - 1]);
236 x = add_x16(fgi,add_x16(fgip,fgip1));
237 x = freeze_3_x16(squeeze_3_x16(x));
238 store_x16(&h[i],x);
239 }
240
241 for (i = 0;i < p;++i) outbytes[i] = h[i];
242 }