Debian Codegraph @

_gen/gf8.go

raw
1 //go:build generate
2 // +build generate
3
4 // Copyright 2022+, Klaus Post. See LICENSE for details.
5
6 package main
7
8 import (
9 "fmt"
10
11 "github.com/mmcloughlin/avo/attr"
12 . "github.com/mmcloughlin/avo/build"
13 . "github.com/mmcloughlin/avo/operand"
14 "github.com/mmcloughlin/avo/reg"
15 )
16
17 type gf8ctx struct {
18 clrMask reg.VecVirtual
19 clrMask128 reg.VecVirtual
20 }
21
22 func genGF8() {
23 var ctx gf8ctx
24 // Ported from static void IFFT_DIT2
25 // https://github.com/catid/leopard/blob/master/LeopardFF8.cpp#L599
26 if pshufb {
27 TEXT("ifftDIT28_avx2", attr.NOSPLIT, fmt.Sprintf("func(x, y []byte, table *[2*16]uint8)"))
28 Pragma("noescape")
29
30 tablePtr := Load(Param("table"), GP64())
31 var tables table256
32 tables.Lo, tables.Hi = YMM(), YMM()
33 // Load and expand tables
34 VBROADCASTI128(Mem{Base: tablePtr, Disp: 0}, tables.Lo)
35 VBROADCASTI128(Mem{Base: tablePtr, Disp: 16}, tables.Hi)
36
37 bytes := Load(Param("x").Len(), GP64())
38 x := Load(Param("x").Base(), GP64())
39 y := Load(Param("y").Base(), GP64())
40 // Generate mask
41 ctx.clrMask = YMM()
42 tmpMask := GP64()
43 MOVQ(U32(15), tmpMask)
44 MOVQ(tmpMask, ctx.clrMask.AsX())
45 VPBROADCASTB(ctx.clrMask.AsX(), ctx.clrMask)
46
47 x0, x1, y0, y1 := YMM(), YMM(), YMM(), YMM()
48 Label("loop")
49 VMOVDQU(Mem{Base: x, Disp: 0}, x0)
50 VMOVDQU(Mem{Base: x, Disp: 32}, x1)
51 VMOVDQU(Mem{Base: y, Disp: 0}, y0)
52 VMOVDQU(Mem{Base: y, Disp: 32}, y1)
53
54 // Update y and store
55 VPXOR(y0, x0, y0)
56 VPXOR(y1, x1, y1)
57 VMOVDQU(y0, Mem{Base: y, Disp: 0})
58 VMOVDQU(y1, Mem{Base: y, Disp: 32})
59
60 // Update x and store
61 leo8MulAdd256(ctx, x0, y0, tables)
62 leo8MulAdd256(ctx, x1, y1, tables)
63 VMOVDQU(x0, Mem{Base: x, Disp: 0})
64 VMOVDQU(x1, Mem{Base: x, Disp: 32})
65
66 // Move on
67 ADDQ(U8(64), x)
68 ADDQ(U8(64), y)
69 SUBQ(U8(64), bytes)
70 JA(LabelRef("loop"))
71
72 VZEROUPPER()
73 RET()
74 }
75 // https://github.com/catid/leopard/blob/master/LeopardFF8.cpp#L1323
76 if pshufb {
77 TEXT("fftDIT28_avx2", attr.NOSPLIT, fmt.Sprintf("func(x, y []byte, table *[2*16]uint8)"))
78 Pragma("noescape")
79 tablePtr := Load(Param("table"), GP64())
80 var tables table256
81 tables.Lo, tables.Hi = YMM(), YMM()
82 // Load and expand tables
83 VBROADCASTI128(Mem{Base: tablePtr, Disp: 0}, tables.Lo)
84 VBROADCASTI128(Mem{Base: tablePtr, Disp: 16}, tables.Hi)
85
86 bytes := Load(Param("x").Len(), GP64())
87 x := Load(Param("x").Base(), GP64())
88 y := Load(Param("y").Base(), GP64())
89 // Generate mask
90 ctx.clrMask = YMM()
91 tmpMask := GP64()
92 MOVQ(U32(15), tmpMask)
93 MOVQ(tmpMask, ctx.clrMask.AsX())
94 VPBROADCASTB(ctx.clrMask.AsX(), ctx.clrMask)
95
96 x0, x1, y0, y1 := YMM(), YMM(), YMM(), YMM()
97 Label("loop")
98 VMOVDQU(Mem{Base: x, Disp: 0}, x0)
99 VMOVDQU(Mem{Base: x, Disp: 32}, x1)
100 VMOVDQU(Mem{Base: y, Disp: 0}, y0)
101 VMOVDQU(Mem{Base: y, Disp: 32}, y1)
102
103 leo8MulAdd256(ctx, x0, y0, tables)
104 leo8MulAdd256(ctx, x1, y1, tables)
105 VMOVDQU(x0, Mem{Base: x, Disp: 0})
106 VMOVDQU(x1, Mem{Base: x, Disp: 32})
107
108 VPXOR(y0, x0, y0)
109 VPXOR(y1, x1, y1)
110 VMOVDQU(y0, Mem{Base: y, Disp: 0})
111 VMOVDQU(y1, Mem{Base: y, Disp: 32})
112 ADDQ(U8(64), x)
113 ADDQ(U8(64), y)
114 SUBQ(U8(64), bytes)
115 JA(LabelRef("loop"))
116
117 VZEROUPPER()
118 RET()
119 }
120
121 x := [8]int{}
122 for _, withDst := range []bool{false, true} {
123 dstString := ""
124 dstSuf := ""
125 if withDst {
126 dstString = "dst, "
127 dstSuf = "dst_"
128 }
129 for skipMask := range x[:] {
130 if !pshufb {
131 break
132 }
133 {
134 var suffix = "avx2_" + dstSuf + fmt.Sprint(skipMask)
135 TEXT("ifftDIT48_"+suffix, attr.NOSPLIT, fmt.Sprintf("func(%swork [][]byte, dist int, t01, t23, t02 *[2*16]uint8)", dstString))
136 Pragma("noescape")
137 var t01, t23, t02 table256
138 // Load and expand tables
139
140 if (skipMask & 1) == 0 {
141 tablePtr := Load(Param("t01"), GP64())
142 t01.Lo, t01.Hi = YMM(), YMM()
143 // We need one register when loading all.
144 if skipMask == 0 {
145 t01.loadLo128 = &Mem{Base: tablePtr, Disp: 0}
146 } else {
147 VBROADCASTI128(Mem{Base: tablePtr, Disp: 0}, t01.Lo)
148 }
149 VBROADCASTI128(Mem{Base: tablePtr, Disp: 16}, t01.Hi)
150 }
151 if (skipMask & 2) == 0 {
152 tablePtr := Load(Param("t23"), GP64())
153 t23.Lo, t23.Hi = YMM(), YMM()
154 VBROADCASTI128(Mem{Base: tablePtr, Disp: 0}, t23.Lo)
155 VBROADCASTI128(Mem{Base: tablePtr, Disp: 16}, t23.Hi)
156 }
157 if (skipMask & 4) == 0 {
158 tablePtr := Load(Param("t02"), GP64())
159 t02.Lo, t02.Hi = YMM(), YMM()
160 VBROADCASTI128(Mem{Base: tablePtr, Disp: 0}, t02.Lo)
161 VBROADCASTI128(Mem{Base: tablePtr, Disp: 16}, t02.Hi)
162 }
163 dist := Load(Param("dist"), GP64())
164
165 var work [4]reg.GPVirtual
166 workTable := Load(Param("work").Base(), GP64()) // &work[0]
167 bytes := GP64()
168 MOVQ(Mem{Base: workTable, Disp: 8}, bytes)
169
170 var dst [4]reg.GPVirtual
171 dstTable := GP64()
172 if withDst {
173 Load(Param("dst").Base(), dstTable) // &dst[0]
174 }
175 for i := range work {
176 work[i] = GP64()
177 // work[i] = &workTable[dist*i]
178 MOVQ(Mem{Base: workTable}, work[i])
179 if i < len(work)-1 {
180 ADDQ(dist, workTable)
181 }
182 if withDst {
183 dst[i] = GP64()
184 MOVQ(Mem{Base: dstTable}, dst[i])
185 if i < len(work)-1 {
186 ADDQ(dist, dstTable)
187 }
188 }
189 }
190
191 // Generate mask
192 ctx.clrMask = YMM()
193 tmpMask := GP64()
194 MOVQ(U32(15), tmpMask)
195 MOVQ(tmpMask, ctx.clrMask.AsX())
196 VPBROADCASTB(ctx.clrMask.AsX(), ctx.clrMask)
197
198 Label("loop")
199 var workReg [4]reg.VecVirtual
200 var workReg2 [4]reg.VecVirtual
201
202 workReg[0] = YMM()
203 workReg[1] = YMM()
204 workReg2[0] = YMM()
205 workReg2[1] = YMM()
206
207 VMOVDQU(Mem{Base: work[0], Disp: 0}, workReg[0])
208 VMOVDQU(Mem{Base: work[1], Disp: 0}, workReg[1])
209 VMOVDQU(Mem{Base: work[0], Disp: 32}, workReg2[0])
210 VMOVDQU(Mem{Base: work[1], Disp: 32}, workReg2[1])
211
212 // work1_reg = _mm256_xor_si256(work0_reg, work1_reg);
213 VPXOR(workReg[1], workReg[0], workReg[1])
214 VPXOR(workReg2[1], workReg2[0], workReg2[1])
215 if (skipMask & 1) == 0 {
216 t01.prepare()
217 leo8MulAdd256(ctx, workReg[0], workReg[1], t01)
218 leo8MulAdd256(ctx, workReg2[0], workReg2[1], t01)
219 }
220
221 workReg[2] = YMM()
222 workReg[3] = YMM()
223 workReg2[2] = YMM()
224 workReg2[3] = YMM()
225 VMOVDQU(Mem{Base: work[2], Disp: 0}, workReg[2])
226 VMOVDQU(Mem{Base: work[3], Disp: 0}, workReg[3])
227 VMOVDQU(Mem{Base: work[2], Disp: 32}, workReg2[2])
228 VMOVDQU(Mem{Base: work[3], Disp: 32}, workReg2[3])
229
230 //work3_reg = _mm256_xor_si256(work2_reg, work3_reg)
231 VPXOR(workReg[2], workReg[3], workReg[3])
232 VPXOR(workReg2[2], workReg2[3], workReg2[3])
233 if (skipMask & 2) == 0 {
234 leo8MulAdd256(ctx, workReg[2], workReg[3], t23)
235 leo8MulAdd256(ctx, workReg2[2], workReg2[3], t23)
236 }
237
238 // Second layer:
239 // work2_reg = _mm256_xor_si256(work0_reg, work2_reg);
240 // work3_reg = _mm256_xor_si256(work1_reg, work3_reg);
241 VPXOR(workReg[0], workReg[2], workReg[2])
242 VPXOR(workReg[1], workReg[3], workReg[3])
243 VPXOR(workReg2[0], workReg2[2], workReg2[2])
244 VPXOR(workReg2[1], workReg2[3], workReg2[3])
245
246 if (skipMask & 4) == 0 {
247 leo8MulAdd256(ctx, workReg[0], workReg[2], t02)
248 leo8MulAdd256(ctx, workReg[1], workReg[3], t02)
249 leo8MulAdd256(ctx, workReg2[0], workReg2[2], t02)
250 leo8MulAdd256(ctx, workReg2[1], workReg2[3], t02)
251 }
252
253 // Store + Next loop:
254 for i := range work {
255 if withDst {
256 VMOVDQU(workReg[i], Mem{Base: dst[i], Disp: 0})
257 VMOVDQU(workReg2[i], Mem{Base: dst[i], Disp: 32})
258 ADDQ(U8(64), dst[i])
259 } else {
260 VMOVDQU(workReg[i], Mem{Base: work[i], Disp: 0})
261 VMOVDQU(workReg2[i], Mem{Base: work[i], Disp: 32})
262 }
263 ADDQ(U8(64), work[i])
264 }
265
266 SUBQ(U8(64), bytes)
267 JA(LabelRef("loop"))
268
269 VZEROUPPER()
270 RET()
271 }
272 // No destination version needed at this point.
273 if !withDst {
274 var suffix = "avx2_" + dstSuf + fmt.Sprint(skipMask)
275 TEXT("fftDIT48_"+suffix, attr.NOSPLIT, fmt.Sprintf("func(%swork [][]byte, dist int, t01, t23, t02 *[2*16]uint8)", dstString))
276 Pragma("noescape")
277 var t01, t23, t02 table256
278 // Load and expand tables
279
280 if (skipMask & 2) == 0 {
281 tablePtr := Load(Param("t01"), GP64())
282 t01.Lo, t01.Hi = YMM(), YMM()
283 if skipMask == 0 {
284 t01.loadLo128 = &Mem{Base: tablePtr, Disp: 0}
285 } else {
286 // We need additional registers
287 VBROADCASTI128(Mem{Base: tablePtr, Disp: 0}, t01.Lo)
288 }
289 VBROADCASTI128(Mem{Base: tablePtr, Disp: 16}, t01.Hi)
290 }
291 if (skipMask & 4) == 0 {
292 tablePtr := Load(Param("t23"), GP64())
293 t23.Lo, t23.Hi = YMM(), YMM()
294 if skipMask == 0 {
295 t23.loadLo128 = &Mem{Base: tablePtr, Disp: 0}
296 } else {
297 VBROADCASTI128(Mem{Base: tablePtr, Disp: 0}, t23.Lo)
298 }
299 VBROADCASTI128(Mem{Base: tablePtr, Disp: 16}, t23.Hi)
300 }
301 if (skipMask & 1) == 0 {
302 tablePtr := Load(Param("t02"), GP64())
303
304 t02.Lo, t02.Hi = YMM(), YMM()
305 VBROADCASTI128(Mem{Base: tablePtr, Disp: 0}, t02.Lo)
306 VBROADCASTI128(Mem{Base: tablePtr, Disp: 16}, t02.Hi)
307 }
308 dist := Load(Param("dist"), GP64())
309
310 var work [4]reg.GPVirtual
311 workTable := Load(Param("work").Base(), GP64()) // &work[0]
312 bytes := GP64()
313 MOVQ(Mem{Base: workTable, Disp: 8}, bytes)
314
315 var dst [4]reg.GPVirtual
316 dstTable := GP64()
317 if withDst {
318 Load(Param("dst").Base(), dstTable) // &dst[0]
319 }
320 for i := range work {
321 work[i] = GP64()
322 // work[i] = &workTable[dist*i]
323 MOVQ(Mem{Base: workTable}, work[i])
324 if i < len(work)-1 {
325 ADDQ(dist, workTable)
326 }
327 if withDst {
328 dst[i] = GP64()
329 MOVQ(Mem{Base: dstTable}, dst[i])
330 if i < len(work)-1 {
331 ADDQ(dist, dstTable)
332 }
333 }
334 }
335
336 // Generate mask
337 ctx.clrMask = YMM()
338 tmpMask := GP64()
339 MOVQ(U32(15), tmpMask)
340 MOVQ(tmpMask, ctx.clrMask.AsX())
341 VPBROADCASTB(ctx.clrMask.AsX(), ctx.clrMask)
342
343 Label("loop")
344 var workReg [4]reg.VecVirtual
345 var workReg2 [4]reg.VecVirtual
346
347 for i := range workReg {
348 workReg[i] = YMM()
349 workReg2[i] = YMM()
350 }
351
352 VMOVDQU(Mem{Base: work[0], Disp: 0}, workReg[0])
353 VMOVDQU(Mem{Base: work[0], Disp: 32}, workReg2[0])
354 VMOVDQU(Mem{Base: work[2], Disp: 0}, workReg[2])
355 VMOVDQU(Mem{Base: work[2], Disp: 32}, workReg2[2])
356 VMOVDQU(Mem{Base: work[1], Disp: 0}, workReg[1])
357 VMOVDQU(Mem{Base: work[1], Disp: 32}, workReg2[1])
358 VMOVDQU(Mem{Base: work[3], Disp: 0}, workReg[3])
359 VMOVDQU(Mem{Base: work[3], Disp: 32}, workReg2[3])
360
361 // work1_reg = _mm256_xor_si256(work0_reg, work1_reg);
362 if (skipMask & 1) == 0 {
363 leo8MulAdd256(ctx, workReg[0], workReg[2], t02)
364 leo8MulAdd256(ctx, workReg2[0], workReg2[2], t02)
365
366 leo8MulAdd256(ctx, workReg[1], workReg[3], t02)
367 leo8MulAdd256(ctx, workReg2[1], workReg2[3], t02)
368 }
369 // work2_reg = _mm256_xor_si256(work0_reg, work2_reg);
370 // work3_reg = _mm256_xor_si256(work1_reg, work3_reg);
371 VPXOR(workReg[0], workReg[2], workReg[2])
372 VPXOR(workReg[1], workReg[3], workReg[3])
373 VPXOR(workReg2[0], workReg2[2], workReg2[2])
374 VPXOR(workReg2[1], workReg2[3], workReg2[3])
375
376 // Second layer:
377 if (skipMask & 2) == 0 {
378 t01.prepare()
379 leo8MulAdd256(ctx, workReg[0], workReg[1], t01)
380 leo8MulAdd256(ctx, workReg2[0], workReg2[1], t01)
381 }
382 //work1_reg = _mm256_xor_si256(work0_reg, work1_reg);
383 VPXOR(workReg[1], workReg[0], workReg[1])
384 VPXOR(workReg2[1], workReg2[0], workReg2[1])
385
386 if (skipMask & 4) == 0 {
387 t23.prepare()
388 leo8MulAdd256(ctx, workReg[2], workReg[3], t23)
389 leo8MulAdd256(ctx, workReg2[2], workReg2[3], t23)
390 }
391 // work3_reg = _mm256_xor_si256(work2_reg, work3_reg);
392 VPXOR(workReg[2], workReg[3], workReg[3])
393 VPXOR(workReg2[2], workReg2[3], workReg2[3])
394
395 // Store + Next loop:
396 for i := range work {
397 if withDst {
398 VMOVDQU(workReg[i], Mem{Base: dst[i], Disp: 0})
399 VMOVDQU(workReg2[i], Mem{Base: dst[i], Disp: 32})
400 ADDQ(U8(64), dst[i])
401 } else {
402 VMOVDQU(workReg[i], Mem{Base: work[i], Disp: 0})
403 VMOVDQU(workReg2[i], Mem{Base: work[i], Disp: 32})
404 }
405 ADDQ(U8(64), work[i])
406 }
407
408 SUBQ(U8(64), bytes)
409 JA(LabelRef("loop"))
410
411 VZEROUPPER()
412 RET()
413 }
414 }
415
416 // GFNI
417 for skipMask := range x[:] {
418 {
419 var suffix = "gfni_" + dstSuf + fmt.Sprint(skipMask)
420 TEXT("ifftDIT48_"+suffix, attr.NOSPLIT, fmt.Sprintf("func(%swork [][]byte, dist int, t01, t23, t02 uint64)", dstString))
421 Pragma("noescape")
422 var t01, t23, t02 table512 = ZMM(), ZMM(), ZMM()
423 // Load and expand tables
424
425 if (skipMask & 1) == 0 {
426 tablePtr, _ := Param("t01").Resolve()
427 VBROADCASTF32X2(tablePtr.Addr, t01)
428 }
429 if (skipMask & 2) == 0 {
430 tablePtr, _ := Param("t23").Resolve()
431 VBROADCASTF32X2(tablePtr.Addr, t23)
432 }
433 if (skipMask & 4) == 0 {
434 tablePtr, _ := Param("t02").Resolve()
435 VBROADCASTF32X2(tablePtr.Addr, t02)
436 }
437 dist := Load(Param("dist"), GP64())
438
439 var work [4]reg.GPVirtual
440 workTable := Load(Param("work").Base(), GP64()) // &work[0]
441 bytes := GP64()
442 MOVQ(Mem{Base: workTable, Disp: 8}, bytes)
443
444 var dst [4]reg.GPVirtual
445 dstTable := GP64()
446 if withDst {
447 Load(Param("dst").Base(), dstTable) // &dst[0]
448 }
449 for i := range work {
450 work[i] = GP64()
451 // work[i] = &workTable[dist*i]
452 MOVQ(Mem{Base: workTable}, work[i])
453 if i < len(work)-1 {
454 ADDQ(dist, workTable)
455 }
456 if withDst {
457 dst[i] = GP64()
458 MOVQ(Mem{Base: dstTable}, dst[i])
459 if i < len(work)-1 {
460 ADDQ(dist, dstTable)
461 }
462 }
463 }
464
465 Label("loop")
466 var workReg [4]reg.VecVirtual
467 for i := range workReg[:] {
468 workReg[i] = ZMM()
469 VMOVDQU64(Mem{Base: work[i], Disp: 0}, workReg[i])
470 }
471
472 // First layer:
473 // work1_reg = _mm256_xor_si256(work0_reg, work1_reg);
474 VXORPD(workReg[1], workReg[0], workReg[1])
475 if (skipMask & 1) == 0 {
476 leo8MulAdd512(ctx, workReg[0], workReg[1], t01, nil)
477 }
478
479 //work3_reg = _mm256_xor_si256(work2_reg, work3_reg)
480 VXORPD(workReg[2], workReg[3], workReg[3])
481 if (skipMask & 2) == 0 {
482 leo8MulAdd512(ctx, workReg[2], workReg[3], t23, workReg[0])
483 } else {
484 // Merged above when run...
485 VXORPD(workReg[0], workReg[2], workReg[2])
486 }
487
488 // Second layer:
489 // work2_reg = _mm256_xor_si256(work0_reg, work2_reg);
490 // work3_reg = _mm256_xor_si256(work1_reg, work3_reg);
491 VXORPD(workReg[1], workReg[3], workReg[3])
492
493 if (skipMask & 4) == 0 {
494 leo8MulAdd512(ctx, workReg[0], workReg[2], t02, nil)
495 leo8MulAdd512(ctx, workReg[1], workReg[3], t02, nil)
496 }
497
498 // Store + Next loop:
499 for i := range work {
500 if withDst {
501 VMOVDQU64(workReg[i], Mem{Base: dst[i], Disp: 0})
502 ADDQ(U8(64), dst[i])
503 } else {
504 VMOVDQU64(workReg[i], Mem{Base: work[i], Disp: 0})
505 }
506 ADDQ(U8(64), work[i])
507 }
508
509 SUBQ(U8(64), bytes)
510 JA(LabelRef("loop"))
511
512 VZEROUPPER()
513 RET()
514 }
515
516 // No destination version needed at this point.
517 if !withDst {
518 var suffix = "gfni_" + dstSuf + fmt.Sprint(skipMask)
519 TEXT("fftDIT48_"+suffix, attr.NOSPLIT, fmt.Sprintf("func(%swork [][]byte, dist int, t01, t23, t02 uint64)", dstString))
520 Pragma("noescape")
521 var t01, t23, t02 table512 = ZMM(), ZMM(), ZMM()
522 // Load and expand tables
523
524 if (skipMask & 2) == 0 {
525 tablePtr, _ := Param("t01").Resolve()
526 VBROADCASTF32X2(tablePtr.Addr, t01)
527 }
528 if (skipMask & 4) == 0 {
529 tablePtr, _ := Param("t23").Resolve()
530 VBROADCASTF32X2(tablePtr.Addr, t23)
531 }
532 if (skipMask & 1) == 0 {
533 tablePtr, _ := Param("t02").Resolve()
534 VBROADCASTF32X2(tablePtr.Addr, t02)
535 }
536 dist := Load(Param("dist"), GP64())
537
538 var work [4]reg.GPVirtual
539 workTable := Load(Param("work").Base(), GP64()) // &work[0]
540 bytes := GP64()
541 MOVQ(Mem{Base: workTable, Disp: 8}, bytes)
542
543 var dst [4]reg.GPVirtual
544 dstTable := GP64()
545 if withDst {
546 Load(Param("dst").Base(), dstTable) // &dst[0]
547 }
548 for i := range work {
549 work[i] = GP64()
550 // work[i] = &workTable[dist*i]
551 MOVQ(Mem{Base: workTable}, work[i])
552 if i < len(work)-1 {
553 ADDQ(dist, workTable)
554 }
555 if withDst {
556 dst[i] = GP64()
557 MOVQ(Mem{Base: dstTable}, dst[i])
558 if i < len(work)-1 {
559 ADDQ(dist, dstTable)
560 }
561 }
562 }
563
564 Label("loop")
565 var workReg [4]reg.VecVirtual
566
567 for i := range workReg {
568 workReg[i] = ZMM()
569 VMOVDQU64(Mem{Base: work[i], Disp: 0}, workReg[i])
570 }
571
572 // First layer:
573 if (skipMask & 1) == 0 {
574 leo8MulAdd512(ctx, workReg[0], workReg[2], t02, nil)
575 leo8MulAdd512(ctx, workReg[1], workReg[3], t02, nil)
576 }
577 // work2_reg = _mm256_xor_si256(work0_reg, work2_reg);
578 // work3_reg = _mm256_xor_si256(work1_reg, work3_reg);
579 VXORPD(workReg[0], workReg[2], workReg[2])
580 VXORPD(workReg[1], workReg[3], workReg[3])
581
582 // Second layer:
583 if (skipMask & 2) == 0 {
584 leo8MulAdd512(ctx, workReg[0], workReg[1], t01, nil)
585 }
586 //work1_reg = _mm256_xor_si256(work0_reg, work1_reg);
587 VXORPD(workReg[1], workReg[0], workReg[1])
588
589 if (skipMask & 4) == 0 {
590 leo8MulAdd512(ctx, workReg[2], workReg[3], t23, nil)
591 }
592 // work3_reg = _mm256_xor_si256(work2_reg, work3_reg);
593 VXORPD(workReg[2], workReg[3], workReg[3])
594
595 // Store + Next loop:
596 for i := range work {
597 if withDst {
598 VMOVDQU64(workReg[i], Mem{Base: dst[i], Disp: 0})
599 ADDQ(U8(64), dst[i])
600 } else {
601 VMOVDQU64(workReg[i], Mem{Base: work[i], Disp: 0})
602 }
603 ADDQ(U8(64), work[i])
604 }
605
606 SUBQ(U8(64), bytes)
607 JA(LabelRef("loop"))
608
609 VZEROUPPER()
610 RET()
611 }
612 } //end with_dst
613 }
614
615 }
616
617 // x updated, y preserved...
618 func leo8MulAdd256(ctx gf8ctx, x, y reg.VecVirtual, table table256) {
619 Comment("LEO_MULADD_256")
620 lo, hi := YMM(), YMM()
621
622 VPAND(y, ctx.clrMask, lo)
623 VPSRLQ(U8(4), y, hi)
624 VPSHUFB(lo, table.Lo, lo)
625
626 // Do high
627 VPAND(hi, ctx.clrMask, hi)
628 VPSHUFB(hi, table.Hi, hi)
629 VPXOR3way(lo, hi, x)
630 }
631
632 // multiply y with table and xor result into x.
633 // If z is set it is xored into x as.
634 func leo8MulAdd512(ctx gf8ctx, x, y reg.VecVirtual, table table512, z reg.VecVirtual) {
635 Comment("LEO_MULADD_512")
636 tmp := ZMM()
637 VGF2P8AFFINEQB(U8(0), table, y, tmp)
638 if z == nil {
639 VXORPD(x, tmp, x)
640 } else {
641 VPTERNLOGD(U8(0x96), tmp, z, x)
642 }
643 }