| 1 |
// 20240508 djb: split out of encrypt.c; optimize |
| 2 |
|
| 3 |
// linker define xor_mat_vec256 |
| 4 |
|
| 5 |
#include "xor_mat_vec256.h" |
| 6 |
|
| 7 |
// input: M, size rows*rowbytes |
| 8 |
// input: v, size ceil(rowbytes/32) |
| 9 |
// input: s, size ceil(rows/8) |
| 10 |
// output: s, size ceil(rows/8) |
| 11 |
// operation: s_i ^= dotproduct(M_i,v) |
| 12 |
// where s_i is ith bit of s |
| 13 |
// and M_i is ith row of M |
| 14 |
// and dotproduct is dot product mod 2 |
| 15 |
void xor_mat_vec256(unsigned char *s, const unsigned char *M, int64_t rows, int64_t rowbytes, const vec256 *v) |
| 16 |
{ |
| 17 |
int64_t i, j, rowwords1; |
| 18 |
vec256 vj, t_0, t_1, t_2, t_3, u_0, u_1; |
| 19 |
vec128 x128; |
| 20 |
uint64_t x64; |
| 21 |
const unsigned char *Mi0, *Mi1, *Mi2, *Mi3; |
| 22 |
|
| 23 |
if (rows <= 0) return; |
| 24 |
if (rowbytes <= 0) return; |
| 25 |
rowwords1 = (rowbytes-1)/32; |
| 26 |
|
| 27 |
i = 0; |
| 28 |
Mi0 = M; |
| 29 |
Mi1 = Mi0 + rowbytes; |
| 30 |
Mi2 = Mi1 + rowbytes; |
| 31 |
Mi3 = Mi2 + rowbytes; |
| 32 |
|
| 33 |
if (rows <= 1) Mi1 = Mi0; |
| 34 |
if (rows <= 2) Mi2 = Mi0; |
| 35 |
if (rows <= 3) Mi3 = Mi0; |
| 36 |
|
| 37 |
mainloop: |
| 38 |
// handle rows i, i+1, i+2, i+3 where i is a multiple of 4 |
| 39 |
// pointers to the M rows are Mi0, Mi1, Mi2, Mi3 |
| 40 |
// if i+1 or i+2 or i+3 is >=rows: Mi1 or Mi2 or Mi3 is actually Mi0 |
| 41 |
|
| 42 |
t_0 = vec256_setzero(); |
| 43 |
t_1 = vec256_setzero(); |
| 44 |
t_2 = vec256_setzero(); |
| 45 |
t_3 = vec256_setzero(); |
| 46 |
|
| 47 |
for (j = 0; j < rowwords1; j++) { |
| 48 |
vj = v[j]; |
| 49 |
t_0 ^= vec256_load(Mi0+32*j) & vj; |
| 50 |
t_1 ^= vec256_load(Mi1+32*j) & vj; |
| 51 |
t_2 ^= vec256_load(Mi2+32*j) & vj; |
| 52 |
t_3 ^= vec256_load(Mi3+32*j) & vj; |
| 53 |
} |
| 54 |
|
| 55 |
vj = v[j]; |
| 56 |
t_0 ^= vec256_load(Mi0+rowbytes-32) & vj; |
| 57 |
t_1 ^= vec256_load(Mi1+rowbytes-32) & vj; |
| 58 |
t_2 ^= vec256_load(Mi2+rowbytes-32) & vj; |
| 59 |
t_3 ^= vec256_load(Mi3+rowbytes-32) & vj; |
| 60 |
|
| 61 |
// want: horizontal bit sums of t_0, t_1, t_2, t_3 |
| 62 |
|
| 63 |
u_0 = vec256_unpack_low(t_0,t_2) ^ vec256_unpack_high(t_0,t_2); |
| 64 |
u_1 = vec256_unpack_low(t_1,t_3) ^ vec256_unpack_high(t_1,t_3); |
| 65 |
|
| 66 |
// want: horizontal bit sums of u_0 bot, u_1 bot, u_0 top, u_1 top |
| 67 |
|
| 68 |
u_0 ^= vec256_8x_shr(u_0,1); |
| 69 |
u_1 ^= vec256_8x_shl(u_1,1); |
| 70 |
u_0 &= vec256_set1_32b(0x55555555); |
| 71 |
u_1 &= vec256_set1_32b(0xaaaaaaaa); |
| 72 |
u_0 |= u_1; |
| 73 |
|
| 74 |
// want: sums of u_0 bot even bits, u_0 bot odd bits, u_0 top even bits, u_0 top odd bits |
| 75 |
|
| 76 |
u_0 ^= vec256_2x_swap64(u_0); |
| 77 |
u_0 ^= vec256_4x_shr(u_0,32); |
| 78 |
u_0 ^= vec256_8x_shr(u_0,16); |
| 79 |
u_0 ^= vec256_8x_shr(u_0,8); |
| 80 |
u_0 ^= vec256_8x_shr(u_0,4); |
| 81 |
u_0 ^= vec256_8x_shr(u_0,2); |
| 82 |
u_0 &= vec256_set1_32b(3); |
| 83 |
|
| 84 |
x128 = vec256_extract2x(u_0,1); |
| 85 |
x128 = vec128_4x_shl(x128,2) | vec256_extractbot(u_0); |
| 86 |
|
| 87 |
x64 = vec128_extract(x128,0); |
| 88 |
|
| 89 |
s[ i/8 ] ^= (x64 & 15) << (i%8); |
| 90 |
|
| 91 |
i += 4; |
| 92 |
Mi0 += 4*rowbytes; |
| 93 |
Mi1 += 4*rowbytes; |
| 94 |
Mi2 += 4*rowbytes; |
| 95 |
Mi3 += 4*rowbytes; |
| 96 |
|
| 97 |
if (i + 4 <= rows) // normal case |
| 98 |
goto mainloop; |
| 99 |
|
| 100 |
if (i < rows) { |
| 101 |
// some trailing rows |
| 102 |
// at most 3 rows out of many, so prioritize code size |
| 103 |
if (rows <= i+1) Mi1 = Mi0; |
| 104 |
if (rows <= i+2) Mi2 = Mi0; |
| 105 |
if (rows <= i+3) Mi3 = Mi0; |
| 106 |
goto mainloop; |
| 107 |
} |
| 108 |
|
| 109 |
if (i > rows) { |
| 110 |
// clean up after trailing rows |
| 111 |
i -= 4; |
| 112 |
s[ i/8 ] ^= (x64 & 15 & (15 << (rows - i))) << (i%8); |
| 113 |
} |
| 114 |
} |