Debian Codegraph @

crypto_kem/348864/avx/xor_mat_vec256.c

raw
1 // 20240508 djb: split out of encrypt.c; optimize
2
3 // linker define xor_mat_vec256
4
5 #include "xor_mat_vec256.h"
6
7 // input: M, size rows*rowbytes
8 // input: v, size ceil(rowbytes/32)
9 // input: s, size ceil(rows/8)
10 // output: s, size ceil(rows/8)
11 // operation: s_i ^= dotproduct(M_i,v)
12 // where s_i is ith bit of s
13 // and M_i is ith row of M
14 // and dotproduct is dot product mod 2
15 void xor_mat_vec256(unsigned char *s, const unsigned char *M, int64_t rows, int64_t rowbytes, const vec256 *v)
16 {
17 int64_t i, j, rowwords1;
18 vec256 vj, t_0, t_1, t_2, t_3, u_0, u_1;
19 vec128 x128;
20 uint64_t x64;
21 const unsigned char *Mi0, *Mi1, *Mi2, *Mi3;
22
23 if (rows <= 0) return;
24 if (rowbytes <= 0) return;
25 rowwords1 = (rowbytes-1)/32;
26
27 i = 0;
28 Mi0 = M;
29 Mi1 = Mi0 + rowbytes;
30 Mi2 = Mi1 + rowbytes;
31 Mi3 = Mi2 + rowbytes;
32
33 if (rows <= 1) Mi1 = Mi0;
34 if (rows <= 2) Mi2 = Mi0;
35 if (rows <= 3) Mi3 = Mi0;
36
37 mainloop:
38 // handle rows i, i+1, i+2, i+3 where i is a multiple of 4
39 // pointers to the M rows are Mi0, Mi1, Mi2, Mi3
40 // if i+1 or i+2 or i+3 is >=rows: Mi1 or Mi2 or Mi3 is actually Mi0
41
42 t_0 = vec256_setzero();
43 t_1 = vec256_setzero();
44 t_2 = vec256_setzero();
45 t_3 = vec256_setzero();
46
47 for (j = 0; j < rowwords1; j++) {
48 vj = v[j];
49 t_0 ^= vec256_load(Mi0+32*j) & vj;
50 t_1 ^= vec256_load(Mi1+32*j) & vj;
51 t_2 ^= vec256_load(Mi2+32*j) & vj;
52 t_3 ^= vec256_load(Mi3+32*j) & vj;
53 }
54
55 vj = v[j];
56 t_0 ^= vec256_load(Mi0+rowbytes-32) & vj;
57 t_1 ^= vec256_load(Mi1+rowbytes-32) & vj;
58 t_2 ^= vec256_load(Mi2+rowbytes-32) & vj;
59 t_3 ^= vec256_load(Mi3+rowbytes-32) & vj;
60
61 // want: horizontal bit sums of t_0, t_1, t_2, t_3
62
63 u_0 = vec256_unpack_low(t_0,t_2) ^ vec256_unpack_high(t_0,t_2);
64 u_1 = vec256_unpack_low(t_1,t_3) ^ vec256_unpack_high(t_1,t_3);
65
66 // want: horizontal bit sums of u_0 bot, u_1 bot, u_0 top, u_1 top
67
68 u_0 ^= vec256_8x_shr(u_0,1);
69 u_1 ^= vec256_8x_shl(u_1,1);
70 u_0 &= vec256_set1_32b(0x55555555);
71 u_1 &= vec256_set1_32b(0xaaaaaaaa);
72 u_0 |= u_1;
73
74 // want: sums of u_0 bot even bits, u_0 bot odd bits, u_0 top even bits, u_0 top odd bits
75
76 u_0 ^= vec256_2x_swap64(u_0);
77 u_0 ^= vec256_4x_shr(u_0,32);
78 u_0 ^= vec256_8x_shr(u_0,16);
79 u_0 ^= vec256_8x_shr(u_0,8);
80 u_0 ^= vec256_8x_shr(u_0,4);
81 u_0 ^= vec256_8x_shr(u_0,2);
82 u_0 &= vec256_set1_32b(3);
83
84 x128 = vec256_extract2x(u_0,1);
85 x128 = vec128_4x_shl(x128,2) | vec256_extractbot(u_0);
86
87 x64 = vec128_extract(x128,0);
88
89 s[ i/8 ] ^= (x64 & 15) << (i%8);
90
91 i += 4;
92 Mi0 += 4*rowbytes;
93 Mi1 += 4*rowbytes;
94 Mi2 += 4*rowbytes;
95 Mi3 += 4*rowbytes;
96
97 if (i + 4 <= rows) // normal case
98 goto mainloop;
99
100 if (i < rows) {
101 // some trailing rows
102 // at most 3 rows out of many, so prioritize code size
103 if (rows <= i+1) Mi1 = Mi0;
104 if (rows <= i+2) Mi2 = Mi0;
105 if (rows <= i+3) Mi3 = Mi0;
106 goto mainloop;
107 }
108
109 if (i > rows) {
110 // clean up after trailing rows
111 i -= 4;
112 s[ i/8 ] ^= (x64 & 15 & (15 << (rows - i))) << (i%8);
113 }
114 }