Debian Codegraph @

crypto_core/mult3sntrup761/avx/mult768.c

raw
1 // 20240812 djb: more cryptoint usage
2 #include "ntt.h"
3 #include <immintrin.h>
4 #include "crypto_int8.h"
5
6 typedef int8_t int8;
7 typedef int16_t int16;
8
9 #define int16x16 __m256i
10 #define load_x16(p) _mm256_loadu_si256((int16x16 *) (p))
11 #define store_x16(p,v) _mm256_storeu_si256((int16x16 *) (p),(v))
12 #define const_x16 _mm256_set1_epi16
13 #define add_x16 _mm256_add_epi16
14 #define sub_x16 _mm256_sub_epi16
15 #define mullo_x16 _mm256_mullo_epi16
16 #define mulhi_x16 _mm256_mulhi_epi16
17 #define mulhrs_x16 _mm256_mulhrs_epi16
18 #define signmask_x16(x) _mm256_srai_epi16((x),15)
19
20 static int16x16 squeeze_3_x16(int16x16 x)
21 {
22 return sub_x16(x,mullo_x16(mulhrs_x16(x,const_x16(10923)),const_x16(3)));
23 }
24
25 static int16x16 squeeze_7681_x16(int16x16 x)
26 {
27 return sub_x16(x,mullo_x16(mulhrs_x16(x,const_x16(4)),const_x16(7681)));
28 }
29
30 static int16x16 mulmod_7681_x16(int16x16 x,int16x16 y)
31 {
32 int16x16 yqinv = mullo_x16(y,const_x16(-7679)); /* XXX: precompute */
33 int16x16 b = mulhi_x16(x,y);
34 int16x16 d = mullo_x16(x,yqinv);
35 int16x16 e = mulhi_x16(d,const_x16(7681));
36 return sub_x16(b,e);
37 }
38
39 #define mask0 _mm256_set_epi16(-1,0,0,-1,0,0,-1,0,0,-1,0,0,-1,0,0,-1)
40 #define mask1 _mm256_set_epi16(0,0,-1,0,0,-1,0,0,-1,0,0,-1,0,0,-1,0)
41 #define mask2 _mm256_set_epi16(0,-1,0,0,-1,0,0,-1,0,0,-1,0,0,-1,0,0)
42
43 static void good(int16 fpad[3][512],const int16 f[768])
44 {
45 int j;
46 int16x16 f0,f1;
47
48 j = 0;
49 for (;;) {
50 f0 = load_x16(f+j);
51 f1 = load_x16(f+512+j);
52 store_x16(&fpad[0][j],(f0&mask0)|(f1&mask1));
53 store_x16(&fpad[1][j],(f0&mask1)|(f1&mask2));
54 store_x16(&fpad[2][j],(f0&mask2)|(f1&mask0));
55 j += 16;
56 if (j == 256) break;
57
58 f0 = load_x16(f+j);
59 f1 = load_x16(f+512+j);
60 store_x16(&fpad[0][j],(f0&mask2)|(f1&mask0));
61 store_x16(&fpad[1][j],(f0&mask0)|(f1&mask1));
62 store_x16(&fpad[2][j],(f0&mask1)|(f1&mask2));
63 j += 16;
64
65 f0 = load_x16(f+j);
66 f1 = load_x16(f+512+j);
67 store_x16(&fpad[0][j],(f0&mask1)|(f1&mask2));
68 store_x16(&fpad[1][j],(f0&mask2)|(f1&mask0));
69 store_x16(&fpad[2][j],(f0&mask0)|(f1&mask1));
70 j += 16;
71 }
72 for (;;) {
73 f0 = load_x16(f+j);
74 store_x16(&fpad[0][j],f0&mask2);
75 store_x16(&fpad[1][j],f0&mask0);
76 store_x16(&fpad[2][j],f0&mask1);
77 j += 16;
78 if (j == 512) break;
79
80 f0 = load_x16(f+j);
81 store_x16(&fpad[0][j],f0&mask1);
82 store_x16(&fpad[1][j],f0&mask2);
83 store_x16(&fpad[2][j],f0&mask0);
84 j += 16;
85
86 f0 = load_x16(f+j);
87 store_x16(&fpad[0][j],f0&mask0);
88 store_x16(&fpad[1][j],f0&mask1);
89 store_x16(&fpad[2][j],f0&mask2);
90 j += 16;
91 }
92 }
93
94 static void ungood(int16 f[1536],const int16 fpad[3][512])
95 {
96 int j;
97 int16x16 f0,f1,f2,g0,g1,g2;
98
99 j = 0;
100
101 for (;;) {
102 f0 = load_x16(&fpad[0][j]);
103 f1 = load_x16(&fpad[1][j]);
104 f2 = load_x16(&fpad[2][j]);
105 g0 = (f0&mask0)|(f1&mask1)|(f2&mask2);
106 g1 = (f0&mask1)|(f1&mask2)|(f2&mask0);
107 g2 = f0^f1^f2^g0^g1; /* same as (f0&mask2)|(f1&mask0)|(f2&mask1) */
108 store_x16(f+0+j,g0);
109 store_x16(f+512+j,g1);
110 store_x16(f+1024+j,g2);
111 j += 16;
112
113 f0 = load_x16(&fpad[0][j]);
114 f1 = load_x16(&fpad[1][j]);
115 f2 = load_x16(&fpad[2][j]);
116 g0 = (f0&mask2)|(f1&mask0)|(f2&mask1);
117 g1 = (f0&mask0)|(f1&mask1)|(f2&mask2);
118 g2 = f0^f1^f2^g0^g1; /* same as (f0&mask1)|(f1&mask2)|(f2&mask0) */
119 store_x16(f+0+j,g0);
120 store_x16(f+512+j,g1);
121 store_x16(f+1024+j,g2);
122 j += 16;
123 if (j == 512) break;
124
125 f0 = load_x16(&fpad[0][j]);
126 f1 = load_x16(&fpad[1][j]);
127 f2 = load_x16(&fpad[2][j]);
128 g0 = (f0&mask1)|(f1&mask2)|(f2&mask0);
129 g1 = (f0&mask2)|(f1&mask0)|(f2&mask1);
130 g2 = f0^f1^f2^g0^g1; /* same as (f0&mask0)|(f1&mask1)|(f2&mask2) */
131 store_x16(f+0+j,g0);
132 store_x16(f+512+j,g1);
133 store_x16(f+1024+j,g2);
134 j += 16;
135 }
136 }
137
138 #define ALIGNED __attribute((aligned(32)))
139
140 static void mult768(int16 h[1536],const int16 f[768],const int16 g[768])
141 {
142 ALIGNED int16 fgpad[6][512];
143 #define fpad fgpad
144 #define gpad (fgpad+3)
145 #define hpad fpad
146 ALIGNED int16 h_7681[1536];
147 int i;
148
149 good(fpad,f);
150 good(gpad,g);
151
152 ntt512_7681(fgpad[0],6);
153
154 for (i = 0;i < 512;i += 16) {
155 int16x16 f0 = squeeze_7681_x16(load_x16(&fpad[0][i]));
156 int16x16 f1 = squeeze_7681_x16(load_x16(&fpad[1][i]));
157 int16x16 f2 = squeeze_7681_x16(load_x16(&fpad[2][i]));
158 int16x16 g0 = squeeze_7681_x16(load_x16(&gpad[0][i]));
159 int16x16 g1 = squeeze_7681_x16(load_x16(&gpad[1][i]));
160 int16x16 g2 = squeeze_7681_x16(load_x16(&gpad[2][i]));
161 int16x16 d0 = mulmod_7681_x16(f0,g0);
162 int16x16 d1 = mulmod_7681_x16(f1,g1);
163 int16x16 d2 = mulmod_7681_x16(f2,g2);
164 int16x16 dsum = add_x16(add_x16(d0,d1),d2);
165 int16x16 h0 = add_x16(dsum,mulmod_7681_x16(sub_x16(f2,f1),sub_x16(g1,g2)));
166 int16x16 h1 = add_x16(dsum,mulmod_7681_x16(sub_x16(f1,f0),sub_x16(g0,g1)));
167 int16x16 h2 = add_x16(dsum,mulmod_7681_x16(sub_x16(f0,f2),sub_x16(g2,g0)));
168 store_x16(&hpad[0][i],squeeze_7681_x16(h0));
169 store_x16(&hpad[1][i],squeeze_7681_x16(h1));
170 store_x16(&hpad[2][i],squeeze_7681_x16(h2));
171 }
172
173 invntt512_7681(hpad[0],3);
174 ungood(h_7681,hpad);
175
176 for (i = 0;i < 1536;i += 16) {
177 int16x16 u = load_x16(&h_7681[i]);
178 u = mulmod_7681_x16(u,const_x16(956));
179 store_x16(&h[i],u);
180 }
181 }
182
183 #include "crypto_core.h"
184 #include "crypto_decode_761xint16.h"
185 #define crypto_decode_pxint16 crypto_decode_761xint16
186 #include "crypto_encode_761xint16.h"
187 #define crypto_encode_pxint16 crypto_encode_761xint16
188
189 #define p 761
190
191 static inline int16x16 freeze_3_x16(int16x16 x)
192 {
193 int16x16 mask, x3;
194 x = add_x16(x,const_x16(3)&signmask_x16(x));
195 mask = signmask_x16(sub_x16(x,const_x16(2)));
196 x3 = sub_x16(x,const_x16(3));
197 x = _mm256_blendv_epi8(x3,x,mask);
198 return x;
199 }
200
201 void crypto_core(unsigned char *outbytes,const unsigned char *inbytes,const unsigned char *kbytes,const unsigned char *cbytes)
202 {
203 ALIGNED int16 f[768];
204 ALIGNED int16 g[768];
205 ALIGNED int16 fg[1536];
206 #define h f
207 int i;
208 int16x16 x;
209
210 x = const_x16(0);
211 for (i = p&~15;i < 768;i += 16) store_x16(&f[i],x);
212 for (i = p&~15;i < 768;i += 16) store_x16(&g[i],x);
213
214 for (i = 0;i < p;++i) {
215 int8 fi = inbytes[i];
216 int8 fi0 = crypto_int8_bottombit_01(fi);
217 f[i] = fi0-(fi&(fi0<<1));
218 }
219 for (i = 0;i < p;++i) {
220 int8 gi = kbytes[i];
221 int8 gi0 = crypto_int8_bottombit_01(gi);
222 g[i] = gi0-(gi&(gi0<<1));
223 }
224
225 mult768(fg,f,g);
226
227 fg[0] -= fg[p-1];
228 for (i = 0;i < 768;i += 16) {
229 int16x16 fgi = load_x16(&fg[i]);
230 int16x16 fgip = load_x16(&fg[i + p]);
231 int16x16 fgip1 = load_x16(&fg[i + p - 1]);
232 x = add_x16(fgi,add_x16(fgip,fgip1));
233 x = freeze_3_x16(squeeze_3_x16(x));
234 store_x16(&h[i],x);
235 }
236
237 for (i = 0;i < p;++i) outbytes[i] = h[i];
238 }