Debian Codegraph @

_gen/gf16.go

raw
1 //go:build generate
2 // +build generate
3
4 // Copyright 2022+, Klaus Post. See LICENSE for details.
5
6 package main
7
8 import (
9 "fmt"
10
11 "github.com/mmcloughlin/avo/attr"
12 . "github.com/mmcloughlin/avo/build"
13 . "github.com/mmcloughlin/avo/operand"
14 "github.com/mmcloughlin/avo/reg"
15 )
16
17 type table256 struct {
18 Lo, Hi Op
19 loadLo128, loadHi128 *Mem
20 loadLo256, loadHi256 *Mem
21 useZmmLo, useZmmHi *reg.VecPhysical
22 }
23
24 type table512 Op
25
26 func (t *table256) prepare() {
27 t.prepareLo()
28 t.prepareHi()
29 }
30
31 func (t *table256) prepareHi() {
32 if t.loadHi128 != nil {
33 t.Hi = YMM()
34 // Load and expand tables
35 VBROADCASTI128(*t.loadHi128, t.Hi)
36 }
37 if t.loadHi256 != nil {
38 t.Hi = YMM()
39 // Load and expand tables
40 VMOVDQU(*t.loadHi256, t.Hi)
41 }
42 if t.useZmmHi != nil {
43 r := *t.useZmmHi
44 t.Hi = r.AsY()
45 }
46 }
47
48 func (t *table256) prepareLo() {
49 if t.loadLo128 != nil {
50 t.Lo = YMM()
51 // Load and expand tables
52 VBROADCASTI128(*t.loadLo128, t.Lo)
53 }
54 if t.loadLo256 != nil {
55 t.Lo = YMM()
56 // Load and expand tables
57 VMOVDQU(*t.loadLo256, t.Lo)
58 }
59 if t.useZmmLo != nil {
60 r := *t.useZmmLo
61 t.Lo = r.AsY()
62 }
63 }
64
65 // table128 contains memory pointers to tables
66 type table128 struct {
67 Lo, Hi Op
68 }
69
70 type gf16ctx struct {
71 clrMask reg.VecVirtual
72 clrMask128 reg.VecVirtual
73 avx512 bool
74 }
75
76 func genGF16() {
77 var ctx gf16ctx
78 // Ported from static void IFFT_DIT2
79 // https://github.com/catid/leopard/blob/master/LeopardFF16.cpp#L629
80 if pshufb {
81 TEXT("ifftDIT2_avx2", attr.NOSPLIT, fmt.Sprintf("func(x, y []byte, table *[8*16]uint8)"))
82 Pragma("noescape")
83 tablePtr := Load(Param("table"), GP64())
84 tables := [4]table256{}
85 for i, t := range tables {
86 t.Lo, t.Hi = YMM(), YMM()
87 // Load and expand tables
88 VBROADCASTI128(Mem{Base: tablePtr, Disp: i * 16}, t.Lo)
89 VBROADCASTI128(Mem{Base: tablePtr, Disp: i*16 + 16*4}, t.Hi)
90 tables[i] = t
91 }
92 bytes := Load(Param("x").Len(), GP64())
93 x := Load(Param("x").Base(), GP64())
94 y := Load(Param("y").Base(), GP64())
95 // Generate mask
96 ctx.clrMask = YMM()
97 tmpMask := GP64()
98 MOVQ(U32(15), tmpMask)
99 MOVQ(tmpMask, ctx.clrMask.AsX())
100 VPBROADCASTB(ctx.clrMask.AsX(), ctx.clrMask)
101
102 xLo, xHi, yLo, yHi := YMM(), YMM(), YMM(), YMM()
103 Label("loop")
104 VMOVDQU(Mem{Base: x, Disp: 0}, xLo)
105 VMOVDQU(Mem{Base: x, Disp: 32}, xHi)
106 VMOVDQU(Mem{Base: y, Disp: 0}, yLo)
107 VMOVDQU(Mem{Base: y, Disp: 32}, yHi)
108 VPXOR(yLo, xLo, yLo)
109 VPXOR(yHi, xHi, yHi)
110 VMOVDQU(yLo, Mem{Base: y, Disp: 0})
111 VMOVDQU(yHi, Mem{Base: y, Disp: 32})
112 leoMulAdd256(ctx, xLo, xHi, yLo, yHi, tables)
113 VMOVDQU(xLo, Mem{Base: x, Disp: 0})
114 VMOVDQU(xHi, Mem{Base: x, Disp: 32})
115 ADDQ(U8(64), x)
116 ADDQ(U8(64), y)
117 SUBQ(U8(64), bytes)
118 JNZ(LabelRef("loop"))
119
120 VZEROUPPER()
121 RET()
122 }
123 if pshufb {
124 TEXT("fftDIT2_avx2", attr.NOSPLIT, fmt.Sprintf("func(x, y []byte, table *[8*16]uint8)"))
125 Pragma("noescape")
126 tablePtr := Load(Param("table"), GP64())
127 tables := [4]table256{}
128 for i, t := range tables {
129 t.Lo, t.Hi = YMM(), YMM()
130 // Load and expand tables
131 VBROADCASTI128(Mem{Base: tablePtr, Disp: i * 16}, t.Lo)
132 VBROADCASTI128(Mem{Base: tablePtr, Disp: i*16 + 16*4}, t.Hi)
133 tables[i] = t
134 }
135 bytes := Load(Param("x").Len(), GP64())
136 x := Load(Param("x").Base(), GP64())
137 y := Load(Param("y").Base(), GP64())
138 // Generate mask
139 ctx.clrMask = YMM()
140 tmpMask := GP64()
141 MOVQ(U32(15), tmpMask)
142 MOVQ(tmpMask, ctx.clrMask.AsX())
143 VPBROADCASTB(ctx.clrMask.AsX(), ctx.clrMask)
144
145 xLo, xHi, yLo, yHi := YMM(), YMM(), YMM(), YMM()
146 Label("loop")
147 VMOVDQU(Mem{Base: x, Disp: 0}, xLo)
148 VMOVDQU(Mem{Base: x, Disp: 32}, xHi)
149 VMOVDQU(Mem{Base: y, Disp: 0}, yLo)
150 VMOVDQU(Mem{Base: y, Disp: 32}, yHi)
151
152 leoMulAdd256(ctx, xLo, xHi, yLo, yHi, tables)
153 VMOVDQU(xLo, Mem{Base: x, Disp: 0})
154 VMOVDQU(xHi, Mem{Base: x, Disp: 32})
155
156 // Reload, or we go beyond 16 regs..
157 if true {
158 yLo, yHi = YMM(), YMM()
159 VMOVDQU(Mem{Base: y, Disp: 0}, yLo)
160 VMOVDQU(Mem{Base: y, Disp: 32}, yHi)
161 }
162
163 VPXOR(yLo, xLo, yLo)
164 VPXOR(yHi, xHi, yHi)
165 VMOVDQU(yLo, Mem{Base: y, Disp: 0})
166 VMOVDQU(yHi, Mem{Base: y, Disp: 32})
167 ADDQ(U8(64), x)
168 ADDQ(U8(64), y)
169 SUBQ(U8(64), bytes)
170 JNZ(LabelRef("loop"))
171
172 VZEROUPPER()
173 RET()
174 }
175
176 if pshufb {
177 TEXT("mulgf16_avx2", attr.NOSPLIT, fmt.Sprintf("func(x, y []byte, table *[8*16]uint8)"))
178 Pragma("noescape")
179 tablePtr := Load(Param("table"), GP64())
180 tables := [4]table256{}
181 for i, t := range tables {
182 t.Lo, t.Hi = YMM(), YMM()
183 // Load and expand tables
184 VBROADCASTI128(Mem{Base: tablePtr, Disp: i * 16}, t.Lo)
185 VBROADCASTI128(Mem{Base: tablePtr, Disp: i*16 + 16*4}, t.Hi)
186 tables[i] = t
187 }
188 bytes := Load(Param("x").Len(), GP64())
189 x := Load(Param("x").Base(), GP64())
190 y := Load(Param("y").Base(), GP64())
191 // Generate mask
192 ctx.clrMask = YMM()
193 tmpMask := GP64()
194 MOVQ(U32(15), tmpMask)
195 MOVQ(tmpMask, ctx.clrMask.AsX())
196 VPBROADCASTB(ctx.clrMask.AsX(), ctx.clrMask)
197
198 dataLo, dataHi := YMM(), YMM()
199 Label("loop")
200 VMOVDQU(Mem{Base: y, Disp: 0}, dataLo)
201 VMOVDQU(Mem{Base: y, Disp: 32}, dataHi)
202
203 prodLo, prodHi := leoMul256(ctx, dataLo, dataHi, tables)
204 VMOVDQU(prodLo, Mem{Base: x, Disp: 0})
205 VMOVDQU(prodHi, Mem{Base: x, Disp: 32})
206
207 ADDQ(U8(64), x)
208 ADDQ(U8(64), y)
209 SUBQ(U8(64), bytes)
210 JNZ(LabelRef("loop"))
211
212 VZEROUPPER()
213 RET()
214 }
215
216 if pshufb {
217 TEXT("mulgf16Xor_avx2", attr.NOSPLIT, fmt.Sprintf("func(x, y []byte, table *[8*16]uint8)"))
218 Pragma("noescape")
219 tablePtr := Load(Param("table"), GP64())
220 tables := [4]table256{}
221 for i, t := range tables {
222 t.Lo, t.Hi = YMM(), YMM()
223 VBROADCASTI128(Mem{Base: tablePtr, Disp: i * 16}, t.Lo)
224 VBROADCASTI128(Mem{Base: tablePtr, Disp: i*16 + 16*4}, t.Hi)
225 tables[i] = t
226 }
227 bytes := Load(Param("x").Len(), GP64())
228 x := Load(Param("x").Base(), GP64())
229 y := Load(Param("y").Base(), GP64())
230 ctx.clrMask = YMM()
231 tmpMask := GP64()
232 MOVQ(U32(15), tmpMask)
233 MOVQ(tmpMask, ctx.clrMask.AsX())
234 VPBROADCASTB(ctx.clrMask.AsX(), ctx.clrMask)
235
236 xLo, xHi, dataLo, dataHi := YMM(), YMM(), YMM(), YMM()
237 Label("loop")
238 VMOVDQU(Mem{Base: x, Disp: 0}, xLo)
239 VMOVDQU(Mem{Base: x, Disp: 32}, xHi)
240 VMOVDQU(Mem{Base: y, Disp: 0}, dataLo)
241 VMOVDQU(Mem{Base: y, Disp: 32}, dataHi)
242
243 leoMulAdd256(ctx, xLo, xHi, dataLo, dataHi, tables)
244 VMOVDQU(xLo, Mem{Base: x, Disp: 0})
245 VMOVDQU(xHi, Mem{Base: x, Disp: 32})
246
247 ADDQ(U8(64), x)
248 ADDQ(U8(64), y)
249 SUBQ(U8(64), bytes)
250 JNZ(LabelRef("loop"))
251
252 VZEROUPPER()
253 RET()
254 }
255
256 for _, withDst := range []bool{false, true} {
257 dstString := ""
258 dstSuf := ""
259 if withDst {
260 dstString = "dst, "
261 dstSuf = "dst_"
262 }
263 for _, avx512 := range []bool{true, false} {
264 if !pshufb {
265 continue
266 }
267 x := [8]int{}
268 for skipMask := range x[:] {
269 // AVX-512 only uses more registers for tables.
270 var suffix = "avx2_" + dstSuf + fmt.Sprint(skipMask)
271 if avx512 {
272 suffix = "avx512_" + dstSuf + fmt.Sprint(skipMask)
273 }
274 ctx.avx512 = avx512
275 extZMMs := []reg.VecPhysical{reg.Z16, reg.Z17, reg.Z18, reg.Z19, reg.Z20, reg.Z21, reg.Z22, reg.Z23, reg.Z24, reg.Z25, reg.Z26, reg.Z27, reg.Z28, reg.Z29, reg.Z30, reg.Z31}
276 {
277 TEXT("ifftDIT4_"+suffix, attr.NOSPLIT, fmt.Sprintf("func(%swork [][]byte, dist int, table01 *[8*16]uint8, table23 *[8*16]uint8, table02 *[8*16]uint8)", dstString))
278 Pragma("noescape")
279 Comment("dist must be multiplied by 24 (size of slice header)")
280
281 // Unpack tables to stack. Slower.
282 const unpackTables = false
283
284 table01Ptr := Load(Param("table01"), GP64())
285 table23Ptr := Load(Param("table23"), GP64())
286 table02Ptr := Load(Param("table02"), GP64())
287
288 // Prepare table pointers.
289 table01 := [4]table256{}
290 table23 := [4]table256{}
291 table02 := [4]table256{}
292 if avx512 {
293 usedZmm := 0
294 fill := func(t *[4]table256, ptr reg.Register) {
295 for i := range table01 {
296 t := &t[i]
297 if len(extZMMs)-usedZmm >= 2 {
298 tmpLo, tmpHi := YMM(), YMM()
299 t.useZmmLo, t.useZmmHi = &extZMMs[usedZmm], &extZMMs[usedZmm+1]
300 usedZmm += 2
301 // Load and expand tables
302 VBROADCASTI128(Mem{Base: ptr, Disp: i * 16}, tmpLo)
303 VBROADCASTI128(Mem{Base: ptr, Disp: i*16 + 16*4}, tmpHi)
304 VMOVAPS(tmpLo.AsZ(), *t.useZmmLo)
305 VMOVAPS(tmpHi.AsZ(), *t.useZmmHi)
306 } else {
307 t.loadLo128 = &Mem{Base: ptr, Disp: i * 16}
308 t.loadHi128 = &Mem{Base: ptr, Disp: i*16 + 16*4}
309 }
310 }
311 }
312 if (skipMask & 4) == 0 {
313 fill(&table02, table02Ptr)
314 }
315 if (skipMask & 1) == 0 {
316 fill(&table01, table01Ptr)
317 }
318 if (skipMask & 2) == 0 {
319 fill(&table23, table23Ptr)
320 }
321 }
322 for i := range table01 {
323 if avx512 {
324 continue
325 }
326
327 if unpackTables {
328 toStack := func(m Mem) *Mem {
329 stack := AllocLocal(32)
330 y := YMM()
331 VBROADCASTI128(m, y)
332 VMOVDQU(y, stack)
333 return &stack
334 }
335
336 table01[i].loadLo256 = toStack(Mem{Base: table01Ptr, Disp: i * 16})
337 table23[i].loadLo256 = toStack(Mem{Base: table23Ptr, Disp: i * 16})
338 table02[i].loadLo256 = toStack(Mem{Base: table02Ptr, Disp: i * 16})
339
340 table01[i].loadHi256 = toStack(Mem{Base: table01Ptr, Disp: i*16 + 16*4})
341 table23[i].loadHi256 = toStack(Mem{Base: table23Ptr, Disp: i*16 + 16*4})
342 table02[i].loadHi256 = toStack(Mem{Base: table02Ptr, Disp: i*16 + 16*4})
343 } else {
344 table01[i].loadLo128 = &Mem{Base: table01Ptr, Disp: i * 16}
345 table23[i].loadLo128 = &Mem{Base: table23Ptr, Disp: i * 16}
346 table02[i].loadLo128 = &Mem{Base: table02Ptr, Disp: i * 16}
347
348 table01[i].loadHi128 = &Mem{Base: table01Ptr, Disp: i*16 + 16*4}
349 table23[i].loadHi128 = &Mem{Base: table23Ptr, Disp: i*16 + 16*4}
350 table02[i].loadHi128 = &Mem{Base: table02Ptr, Disp: i*16 + 16*4}
351 }
352 }
353 // Generate mask
354 ctx.clrMask = YMM()
355 tmpMask := GP64()
356 MOVQ(U32(15), tmpMask)
357 MOVQ(tmpMask, ctx.clrMask.AsX())
358 VPBROADCASTB(ctx.clrMask.AsX(), ctx.clrMask)
359
360 dist := Load(Param("dist"), GP64())
361
362 // Pointers to each "work"
363 var work [4]reg.GPVirtual
364 workTable := Load(Param("work").Base(), GP64()) // &work[0]
365 bytes := GP64()
366
367 // Load length of work[0]
368 MOVQ(Mem{Base: workTable, Disp: 8}, bytes)
369
370 var dst [4]reg.GPVirtual
371 dstTable := GP64()
372 if withDst {
373 Load(Param("dst").Base(), dstTable) // &dst[0]
374 }
375 for i := range work {
376 work[i] = GP64()
377 // work[i] = &workTable[dist*i]
378 MOVQ(Mem{Base: workTable}, work[i])
379 if i < len(work)-1 {
380 ADDQ(dist, workTable)
381 }
382 if withDst {
383 dst[i] = GP64()
384 MOVQ(Mem{Base: dstTable}, dst[i])
385 if i < len(work)-1 {
386 ADDQ(dist, dstTable)
387 }
388 }
389 }
390 var workRegLo [4]reg.VecVirtual
391 var workRegHi [4]reg.VecVirtual
392
393 workRegLo[0], workRegHi[0] = YMM(), YMM()
394 workRegLo[1], workRegHi[1] = YMM(), YMM()
395
396 Label("loop_ifft4_" + suffix)
397 VMOVDQU(Mem{Base: work[0], Disp: 0}, workRegLo[0])
398 VMOVDQU(Mem{Base: work[0], Disp: 32}, workRegHi[0])
399 VMOVDQU(Mem{Base: work[1], Disp: 0}, workRegLo[1])
400 VMOVDQU(Mem{Base: work[1], Disp: 32}, workRegHi[1])
401
402 // First layer:
403 VPXOR(workRegLo[0], workRegLo[1], workRegLo[1])
404 VPXOR(workRegHi[0], workRegHi[1], workRegHi[1])
405
406 // Test bit 0
407 if (skipMask & 1) == 0 {
408 leoMulAdd256(ctx, workRegLo[0], workRegHi[0], workRegLo[1], workRegHi[1], table01)
409 }
410 workRegLo[2], workRegHi[2] = YMM(), YMM()
411 workRegLo[3], workRegHi[3] = YMM(), YMM()
412 VMOVDQU(Mem{Base: work[2], Disp: 0}, workRegLo[2])
413 VMOVDQU(Mem{Base: work[2], Disp: 32}, workRegHi[2])
414 VMOVDQU(Mem{Base: work[3], Disp: 0}, workRegLo[3])
415 VMOVDQU(Mem{Base: work[3], Disp: 32}, workRegHi[3])
416
417 VPXOR(workRegLo[2], workRegLo[3], workRegLo[3])
418 VPXOR(workRegHi[2], workRegHi[3], workRegHi[3])
419
420 // Test bit 1
421 if (skipMask & 2) == 0 {
422 leoMulAdd256(ctx, workRegLo[2], workRegHi[2], workRegLo[3], workRegHi[3], table23)
423 }
424
425 // Second layer:
426 VPXOR(workRegLo[0], workRegLo[2], workRegLo[2])
427 VPXOR(workRegHi[0], workRegHi[2], workRegHi[2])
428 VPXOR(workRegLo[1], workRegLo[3], workRegLo[3])
429 VPXOR(workRegHi[1], workRegHi[3], workRegHi[3])
430
431 // Test bit 2
432 if (skipMask & 4) == 0 {
433 leoMulAdd256(ctx, workRegLo[0], workRegHi[0], workRegLo[2], workRegHi[2], table02)
434 leoMulAdd256(ctx, workRegLo[1], workRegHi[1], workRegLo[3], workRegHi[3], table02)
435 }
436
437 // Store + Next loop:
438 for i := range work {
439 if withDst {
440 VMOVDQU(workRegLo[i], Mem{Base: dst[i], Disp: 0})
441 VMOVDQU(workRegHi[i], Mem{Base: dst[i], Disp: 32})
442 ADDQ(U8(64), dst[i])
443 } else {
444 VMOVDQU(workRegLo[i], Mem{Base: work[i], Disp: 0})
445 VMOVDQU(workRegHi[i], Mem{Base: work[i], Disp: 32})
446 }
447 ADDQ(U8(64), work[i])
448 }
449
450 SUBQ(U8(64), bytes)
451 JNZ(LabelRef("loop_ifft4_" + suffix))
452
453 VZEROUPPER()
454 RET()
455 }
456 // fftDIT4 does not need a dst variant
457 if !withDst {
458 TEXT("fftDIT4_"+suffix, attr.NOSPLIT, fmt.Sprintf("func(work [][]byte, dist int, table01 *[8*16]uint8, table23 *[8*16]uint8, table02 *[8*16]uint8)"))
459 Pragma("noescape")
460 Comment("dist must be multiplied by 24 (size of slice header)")
461
462 // Unpack tables to stack. Slower.
463 const unpackTables = false
464
465 table01Ptr := Load(Param("table01"), GP64())
466 table23Ptr := Load(Param("table23"), GP64())
467 table02Ptr := Load(Param("table02"), GP64())
468
469 // Prepare table pointers.
470 table01 := [4]table256{}
471 table23 := [4]table256{}
472 table02 := [4]table256{}
473 if avx512 {
474 usedZmm := 0
475 fill := func(t *[4]table256, ptr reg.Register) {
476 for i := range table01 {
477 t := &t[i]
478 if len(extZMMs)-usedZmm >= 2 {
479 tmpLo, tmpHi := YMM(), YMM()
480 t.useZmmLo, t.useZmmHi = &extZMMs[usedZmm], &extZMMs[usedZmm+1]
481 usedZmm += 2
482 // Load and expand tables
483 VBROADCASTI128(Mem{Base: ptr, Disp: i * 16}, tmpLo)
484 VBROADCASTI128(Mem{Base: ptr, Disp: i*16 + 16*4}, tmpHi)
485 VMOVAPS(tmpLo.AsZ(), *t.useZmmLo)
486 VMOVAPS(tmpHi.AsZ(), *t.useZmmHi)
487 } else {
488 t.loadLo128 = &Mem{Base: ptr, Disp: i * 16}
489 t.loadHi128 = &Mem{Base: ptr, Disp: i*16 + 16*4}
490 }
491 }
492 }
493 if (skipMask & 1) == 0 {
494 fill(&table02, table02Ptr)
495 }
496 if (skipMask & 2) == 0 {
497 fill(&table01, table01Ptr)
498 }
499 if (skipMask & 4) == 0 {
500 fill(&table23, table23Ptr)
501 }
502 }
503 for i := range table01 {
504 if avx512 {
505 continue
506 }
507 if unpackTables {
508 toStack := func(m Mem) *Mem {
509 stack := AllocLocal(32)
510 y := YMM()
511 VBROADCASTI128(m, y)
512 VMOVDQU(y, stack)
513 return &stack
514 }
515
516 table01[i].loadLo256 = toStack(Mem{Base: table01Ptr, Disp: i * 16})
517 table23[i].loadLo256 = toStack(Mem{Base: table23Ptr, Disp: i * 16})
518 table02[i].loadLo256 = toStack(Mem{Base: table02Ptr, Disp: i * 16})
519
520 table01[i].loadHi256 = toStack(Mem{Base: table01Ptr, Disp: i*16 + 16*4})
521 table23[i].loadHi256 = toStack(Mem{Base: table23Ptr, Disp: i*16 + 16*4})
522 table02[i].loadHi256 = toStack(Mem{Base: table02Ptr, Disp: i*16 + 16*4})
523 } else {
524 table01[i].loadLo128 = &Mem{Base: table01Ptr, Disp: i * 16}
525 table23[i].loadLo128 = &Mem{Base: table23Ptr, Disp: i * 16}
526 table02[i].loadLo128 = &Mem{Base: table02Ptr, Disp: i * 16}
527
528 table01[i].loadHi128 = &Mem{Base: table01Ptr, Disp: i*16 + 16*4}
529 table23[i].loadHi128 = &Mem{Base: table23Ptr, Disp: i*16 + 16*4}
530 table02[i].loadHi128 = &Mem{Base: table02Ptr, Disp: i*16 + 16*4}
531 }
532 }
533 // Generate mask
534 ctx.clrMask = YMM()
535 tmpMask := GP64()
536 MOVQ(U32(15), tmpMask)
537 MOVQ(tmpMask, ctx.clrMask.AsX())
538 VPBROADCASTB(ctx.clrMask.AsX(), ctx.clrMask)
539
540 dist := Load(Param("dist"), GP64())
541
542 // Pointers to each "work"
543 var work [4]reg.GPVirtual
544 workTable := Load(Param("work").Base(), GP64()) // &work[0]
545 bytes := GP64()
546
547 // Load length of work[0]
548 MOVQ(Mem{Base: workTable, Disp: 8}, bytes)
549
550 for i := range work {
551 work[i] = GP64()
552 // work[i] = &workTable[dist*i]
553 MOVQ(Mem{Base: workTable}, work[i])
554 if i < len(work)-1 {
555 ADDQ(dist, workTable)
556 }
557 }
558 var workRegLo [4]reg.VecVirtual
559 var workRegHi [4]reg.VecVirtual
560
561 workRegLo[0], workRegHi[0] = YMM(), YMM()
562 workRegLo[1], workRegHi[1] = YMM(), YMM()
563 workRegLo[2], workRegHi[2] = YMM(), YMM()
564 workRegLo[3], workRegHi[3] = YMM(), YMM()
565
566 Label("loop")
567 VMOVDQU(Mem{Base: work[0], Disp: 0}, workRegLo[0])
568 VMOVDQU(Mem{Base: work[0], Disp: 32}, workRegHi[0])
569 VMOVDQU(Mem{Base: work[2], Disp: 0}, workRegLo[2])
570 VMOVDQU(Mem{Base: work[2], Disp: 32}, workRegHi[2])
571
572 VMOVDQU(Mem{Base: work[1], Disp: 0}, workRegLo[1])
573 VMOVDQU(Mem{Base: work[1], Disp: 32}, workRegHi[1])
574 VMOVDQU(Mem{Base: work[3], Disp: 0}, workRegLo[3])
575 VMOVDQU(Mem{Base: work[3], Disp: 32}, workRegHi[3])
576
577 // First layer:
578
579 // Test bit 0
580 if (skipMask & 1) == 0 {
581 leoMulAdd256(ctx, workRegLo[0], workRegHi[0], workRegLo[2], workRegHi[2], table02)
582 leoMulAdd256(ctx, workRegLo[1], workRegHi[1], workRegLo[3], workRegHi[3], table02)
583 }
584
585 VPXOR(workRegLo[0], workRegLo[2], workRegLo[2])
586 VPXOR(workRegHi[0], workRegHi[2], workRegHi[2])
587 VPXOR(workRegLo[1], workRegLo[3], workRegLo[3])
588 VPXOR(workRegHi[1], workRegHi[3], workRegHi[3])
589
590 // Second layer:
591 // Test bit 1
592 if (skipMask & 2) == 0 {
593 leoMulAdd256(ctx, workRegLo[0], workRegHi[0], workRegLo[1], workRegHi[1], table01)
594 }
595 VPXOR(workRegLo[0], workRegLo[1], workRegLo[1])
596 VPXOR(workRegHi[0], workRegHi[1], workRegHi[1])
597
598 // Store...
599 for i := range work[:2] {
600 VMOVDQU(workRegLo[i], Mem{Base: work[i], Disp: 0})
601 VMOVDQU(workRegHi[i], Mem{Base: work[i], Disp: 32})
602 ADDQ(U8(64), work[i])
603 }
604
605 // Test bit 2
606 if (skipMask & 4) == 0 {
607 leoMulAdd256(ctx, workRegLo[2], workRegHi[2], workRegLo[3], workRegHi[3], table23)
608 }
609 VPXOR(workRegLo[2], workRegLo[3], workRegLo[3])
610 VPXOR(workRegHi[2], workRegHi[3], workRegHi[3])
611
612 // Store + Next loop:
613 for i := range work[2:] {
614 i := i + 2
615 VMOVDQU(workRegLo[i], Mem{Base: work[i], Disp: 0})
616 VMOVDQU(workRegHi[i], Mem{Base: work[i], Disp: 32})
617 ADDQ(U8(64), work[i])
618 }
619
620 SUBQ(U8(64), bytes)
621 JNZ(LabelRef("loop"))
622
623 VZEROUPPER()
624 RET()
625 }
626 }
627 }
628 }
629
630 // GFNI versions of ifftDIT4 and fftDIT4
631 // Load tables on-demand in the loop to avoid running out of YMM registers
632 for _, withDstGfni := range []bool{false, true} {
633 dstStringGfni := ""
634 dstSufGfni := ""
635 if withDstGfni {
636 dstStringGfni = "dst, "
637 dstSufGfni = "dst_"
638 }
639 for _, avx512 := range []bool{false, true} {
640 for skipMask := 0; skipMask < 8; skipMask++ {
641 suffix := "gfni_" + dstSufGfni + fmt.Sprint(skipMask)
642 if avx512 {
643 // AVX512 just uses the extra registers to avoid the load+broadcast
644 suffix = "gfni_avx512_" + dstSufGfni + fmt.Sprint(skipMask)
645 }
646 // ifftDIT4_gfni_*
647 {
648 TEXT("ifftDIT4_"+suffix, attr.NOSPLIT, fmt.Sprintf("func(%swork [][]byte, dist int, table01 *[4]uint64, table23 *[4]uint64, table02 *[4]uint64)", dstStringGfni))
649 Pragma("noescape")
650 Comment("dist must be multiplied by 24 (size of slice header)")
651
652 table01Ptr := Load(Param("table01"), GP64())
653 table23Ptr := Load(Param("table23"), GP64())
654 table02Ptr := Load(Param("table02"), GP64())
655
656 var table02 [4]reg.Register
657 if skipMask&4 == 0 {
658 for i := range table02 {
659 table02[i] = YMM().AsY()
660 VBROADCASTSD(Mem{Base: table02Ptr, Disp: i * 8}, table02[i])
661 }
662 }
663 var table01 [4]reg.Register
664 var table23 [4]reg.Register
665 if avx512 {
666 table01 = [4]reg.Register{reg.Z16.AsY(), reg.Z17.AsY(), reg.Z18.AsY(), reg.Z19.AsY()}
667 table23 = [4]reg.Register{reg.Z20.AsY(), reg.Z21.AsY(), reg.Z22.AsY(), reg.Z23.AsY()}
668 if skipMask&1 == 0 {
669 for i := range table01 {
670 VBROADCASTSD(Mem{Base: table01Ptr, Disp: i * 8}, table01[i])
671 }
672 }
673 if skipMask&2 == 0 {
674 for i := range table23 {
675 VBROADCASTSD(Mem{Base: table23Ptr, Disp: i * 8}, table23[i])
676 }
677 }
678 }
679
680 dist := Load(Param("dist"), GP64())
681
682 var work [4]reg.GPVirtual
683 workTable := Load(Param("work").Base(), GP64())
684 bytes := GP64()
685
686 MOVQ(Mem{Base: workTable, Disp: 8}, bytes)
687
688 var dst [4]reg.GPVirtual
689 dstTable := GP64()
690 if withDstGfni {
691 Load(Param("dst").Base(), dstTable) // &dst[0]
692 }
693 for i := range work {
694 work[i] = GP64()
695 // work[i] = &workTable[dist*i]
696 MOVQ(Mem{Base: workTable}, work[i])
697 if i < len(work)-1 {
698 ADDQ(dist, workTable)
699 }
700 if withDstGfni {
701 dst[i] = GP64()
702 MOVQ(Mem{Base: dstTable}, dst[i])
703 if i < len(work)-1 {
704 ADDQ(dist, dstTable)
705 }
706 }
707 }
708
709 var workRegLo [4]reg.VecVirtual
710 var workRegHi [4]reg.VecVirtual
711
712 workRegLo[0], workRegHi[0] = YMM(), YMM()
713 workRegLo[1], workRegHi[1] = YMM(), YMM()
714
715 Label("loop_ifft4_" + suffix)
716 VMOVDQU(Mem{Base: work[0], Disp: 0}, workRegLo[0])
717 VMOVDQU(Mem{Base: work[0], Disp: 32}, workRegHi[0])
718 VMOVDQU(Mem{Base: work[1], Disp: 0}, workRegLo[1])
719 VMOVDQU(Mem{Base: work[1], Disp: 32}, workRegHi[1])
720
721 // First layer: y = x XOR y
722 VPXOR(workRegLo[0], workRegLo[1], workRegLo[1])
723 VPXOR(workRegHi[0], workRegHi[1], workRegHi[1])
724
725 if (skipMask & 1) == 0 {
726 if avx512 {
727 leoMulAdd256_gfni_avx2(workRegLo[0], workRegHi[0], workRegLo[1], workRegHi[1], table01, true)
728 } else {
729 leoMulAdd256_gfni_mem(workRegLo[0], workRegHi[0], workRegLo[1], workRegHi[1], table01Ptr)
730 }
731 }
732
733 workRegLo[2], workRegHi[2] = YMM(), YMM()
734 workRegLo[3], workRegHi[3] = YMM(), YMM()
735 VMOVDQU(Mem{Base: work[2], Disp: 0}, workRegLo[2])
736 VMOVDQU(Mem{Base: work[2], Disp: 32}, workRegHi[2])
737 VMOVDQU(Mem{Base: work[3], Disp: 0}, workRegLo[3])
738 VMOVDQU(Mem{Base: work[3], Disp: 32}, workRegHi[3])
739
740 VPXOR(workRegLo[2], workRegLo[3], workRegLo[3])
741 VPXOR(workRegHi[2], workRegHi[3], workRegHi[3])
742
743 if (skipMask & 2) == 0 {
744 if avx512 {
745 leoMulAdd256_gfni_avx2(workRegLo[2], workRegHi[2], workRegLo[3], workRegHi[3], table23, true)
746 } else {
747 leoMulAdd256_gfni_mem(workRegLo[2], workRegHi[2], workRegLo[3], workRegHi[3], table23Ptr)
748 }
749 }
750
751 // Second layer
752 VPXOR(workRegLo[0], workRegLo[2], workRegLo[2])
753 VPXOR(workRegHi[0], workRegHi[2], workRegHi[2])
754 VPXOR(workRegLo[1], workRegLo[3], workRegLo[3])
755 VPXOR(workRegHi[1], workRegHi[3], workRegHi[3])
756
757 if (skipMask & 4) == 0 {
758 leoMulAdd256_gfni_avx2(workRegLo[0], workRegHi[0], workRegLo[2], workRegHi[2], table02, avx512)
759 leoMulAdd256_gfni_avx2(workRegLo[1], workRegHi[1], workRegLo[3], workRegHi[3], table02, avx512)
760 }
761
762 // Store + Next loop
763 for i := range work {
764 if withDstGfni {
765 VMOVDQU(workRegLo[i], Mem{Base: dst[i], Disp: 0})
766 VMOVDQU(workRegHi[i], Mem{Base: dst[i], Disp: 32})
767 ADDQ(U8(64), dst[i])
768 } else {
769 VMOVDQU(workRegLo[i], Mem{Base: work[i], Disp: 0})
770 VMOVDQU(workRegHi[i], Mem{Base: work[i], Disp: 32})
771 }
772 ADDQ(U8(64), work[i])
773 }
774
775 SUBQ(U8(64), bytes)
776 JNZ(LabelRef("loop_ifft4_" + suffix))
777
778 VZEROUPPER()
779 RET()
780 }
781
782 // fftDIT4_gfni_* does not need dst variant
783 if !withDstGfni {
784 // fftDIT4_gfni_*
785 TEXT("fftDIT4_"+suffix, attr.NOSPLIT, fmt.Sprintf("func(work [][]byte, dist int, table01 *[4]uint64, table23 *[4]uint64, table02 *[4]uint64)"))
786 Pragma("noescape")
787 Comment("dist must be multiplied by 24 (size of slice header)")
788
789 table01Ptr := Load(Param("table01"), GP64())
790 table23Ptr := Load(Param("table23"), GP64())
791 table02Ptr := Load(Param("table02"), GP64())
792
793 var table02 [4]reg.Register
794 if (skipMask & 1) == 0 {
795 for i := range table02 {
796 table02[i] = YMM().AsY()
797 VBROADCASTSD(Mem{Base: table02Ptr, Disp: i * 8}, table02[i])
798 }
799 }
800 var table01 [4]reg.Register
801 var table23 [4]reg.Register
802 if avx512 {
803 table01 = [4]reg.Register{reg.Z16.AsY(), reg.Z17.AsY(), reg.Z18.AsY(), reg.Z19.AsY()}
804 table23 = [4]reg.Register{reg.Z20.AsY(), reg.Z21.AsY(), reg.Z22.AsY(), reg.Z23.AsY()}
805
806 if skipMask&2 == 0 {
807 for i := range table01 {
808 VBROADCASTSD(Mem{Base: table01Ptr, Disp: i * 8}, table01[i])
809 }
810 }
811 if skipMask&4 == 0 {
812 for i := range table23 {
813 VBROADCASTSD(Mem{Base: table23Ptr, Disp: i * 8}, table23[i])
814 }
815 }
816 }
817 dist := Load(Param("dist"), GP64())
818
819 var work [4]reg.GPVirtual
820 workTable := Load(Param("work").Base(), GP64())
821 bytes := GP64()
822
823 MOVQ(Mem{Base: workTable, Disp: 8}, bytes)
824
825 for i := range work {
826 work[i] = GP64()
827 // work[i] = &workTable[dist*i]
828 MOVQ(Mem{Base: workTable}, work[i])
829 if i < len(work)-1 {
830 ADDQ(dist, workTable)
831 }
832 }
833
834 var workRegLo [4]reg.VecVirtual
835 var workRegHi [4]reg.VecVirtual
836
837 workRegLo[0], workRegHi[0] = YMM(), YMM()
838 workRegLo[1], workRegHi[1] = YMM(), YMM()
839 workRegLo[2], workRegHi[2] = YMM(), YMM()
840 workRegLo[3], workRegHi[3] = YMM(), YMM()
841
842 Label("loop_fft4_" + suffix)
843 VMOVDQU(Mem{Base: work[0], Disp: 0}, workRegLo[0])
844 VMOVDQU(Mem{Base: work[0], Disp: 32}, workRegHi[0])
845 VMOVDQU(Mem{Base: work[2], Disp: 0}, workRegLo[2])
846 VMOVDQU(Mem{Base: work[2], Disp: 32}, workRegHi[2])
847
848 VMOVDQU(Mem{Base: work[1], Disp: 0}, workRegLo[1])
849 VMOVDQU(Mem{Base: work[1], Disp: 32}, workRegHi[1])
850 VMOVDQU(Mem{Base: work[3], Disp: 0}, workRegLo[3])
851 VMOVDQU(Mem{Base: work[3], Disp: 32}, workRegHi[3])
852
853 // First layer
854 if (skipMask & 1) == 0 {
855 leoMulAdd256_gfni_avx2(workRegLo[0], workRegHi[0], workRegLo[2], workRegHi[2], table02, avx512)
856 leoMulAdd256_gfni_avx2(workRegLo[1], workRegHi[1], workRegLo[3], workRegHi[3], table02, avx512)
857 }
858
859 VPXOR(workRegLo[0], workRegLo[2], workRegLo[2])
860 VPXOR(workRegHi[0], workRegHi[2], workRegHi[2])
861 VPXOR(workRegLo[1], workRegLo[3], workRegLo[3])
862 VPXOR(workRegHi[1], workRegHi[3], workRegHi[3])
863
864 // Second layer
865 if (skipMask & 2) == 0 {
866 if avx512 {
867 leoMulAdd256_gfni_avx2(workRegLo[0], workRegHi[0], workRegLo[1], workRegHi[1], table01, true)
868 } else {
869 leoMulAdd256_gfni_mem(workRegLo[0], workRegHi[0], workRegLo[1], workRegHi[1], table01Ptr)
870 }
871 }
872 VPXOR(workRegLo[0], workRegLo[1], workRegLo[1])
873 VPXOR(workRegHi[0], workRegHi[1], workRegHi[1])
874
875 // Store work[0] and work[1]
876 for i := range work[:2] {
877 VMOVDQU(workRegLo[i], Mem{Base: work[i], Disp: 0})
878 VMOVDQU(workRegHi[i], Mem{Base: work[i], Disp: 32})
879 ADDQ(U8(64), work[i])
880 }
881
882 if (skipMask & 4) == 0 {
883 if avx512 {
884 leoMulAdd256_gfni_avx2(workRegLo[2], workRegHi[2], workRegLo[3], workRegHi[3], table23, true)
885 } else {
886 leoMulAdd256_gfni_mem(workRegLo[2], workRegHi[2], workRegLo[3], workRegHi[3], table23Ptr)
887 }
888 }
889 VPXOR(workRegLo[2], workRegLo[3], workRegLo[3])
890 VPXOR(workRegHi[2], workRegHi[3], workRegHi[3])
891
892 // Store work[2] and work[3] + Next loop
893 for i := range work[2:] {
894 i := i + 2
895 VMOVDQU(workRegLo[i], Mem{Base: work[i], Disp: 0})
896 VMOVDQU(workRegHi[i], Mem{Base: work[i], Disp: 32})
897 ADDQ(U8(64), work[i])
898 }
899
900 SUBQ(U8(64), bytes)
901 JNZ(LabelRef("loop_fft4_" + suffix))
902
903 VZEROUPPER()
904 RET()
905 }
906 }
907 }
908 }
909
910 // SSSE3:
911 ctx.avx512 = false
912 if pshufb {
913 TEXT("ifftDIT2_ssse3", attr.NOSPLIT, fmt.Sprintf("func(x, y []byte, table *[8*16]uint8)"))
914 Pragma("noescape")
915 tablePtr := Load(Param("table"), GP64())
916 tables := [4]table128{}
917 for i, t := range tables {
918 // We almost have enough space for all tables.
919 if i > 2 {
920 t.Lo, t.Hi = Mem{Base: tablePtr, Disp: i * 16}, Mem{Base: tablePtr, Disp: i*16 + 16*4}
921 } else {
922 t.Lo, t.Hi = XMM(), XMM()
923 MOVUPS(Mem{Base: tablePtr, Disp: i * 16}, t.Lo)
924 MOVUPS(Mem{Base: tablePtr, Disp: i*16 + 16*4}, t.Hi)
925 }
926 tables[i] = t
927 }
928 // Generate mask
929 zero := XMM()
930 XORPS(zero, zero) // Zero, so bytes will be copied.
931 fifteen, mask := GP64(), XMM()
932 MOVQ(U32(0xf), fifteen)
933 MOVQ(fifteen, mask)
934 PSHUFB(zero, mask)
935 ctx.clrMask128 = mask
936
937 bytes := Load(Param("x").Len(), GP64())
938 x := Load(Param("x").Base(), GP64())
939 y := Load(Param("y").Base(), GP64())
940
941 Label("loop")
942 for i := 0; i < 2; i++ {
943 xLo, xHi, yLo, yHi := XMM(), XMM(), XMM(), XMM()
944 MOVUPS(Mem{Base: x, Disp: i*16 + 0}, xLo)
945 MOVUPS(Mem{Base: x, Disp: i*16 + 32}, xHi)
946 MOVUPS(Mem{Base: y, Disp: i*16 + 0}, yLo)
947 MOVUPS(Mem{Base: y, Disp: i*16 + 32}, yHi)
948 PXOR(xLo, yLo)
949 PXOR(xHi, yHi)
950 MOVUPS(yLo, Mem{Base: y, Disp: i*16 + 0})
951 MOVUPS(yHi, Mem{Base: y, Disp: i*16 + 32})
952 leoMulAdd128(ctx, xLo, xHi, yLo, yHi, tables)
953 MOVUPS(xLo, Mem{Base: x, Disp: i*16 + 0})
954 MOVUPS(xHi, Mem{Base: x, Disp: i*16 + 32})
955 }
956 ADDQ(U8(64), x)
957 ADDQ(U8(64), y)
958 SUBQ(U8(64), bytes)
959 JNZ(LabelRef("loop"))
960
961 RET()
962 }
963 if pshufb {
964 TEXT("fftDIT2_ssse3", attr.NOSPLIT, fmt.Sprintf("func(x, y []byte, table *[8*16]uint8)"))
965 Pragma("noescape")
966 tablePtr := Load(Param("table"), GP64())
967 tables := [4]table128{}
968 for i, t := range tables {
969 // We almost have enough space for all tables.
970 if i > 2 {
971 t.Lo, t.Hi = Mem{Base: tablePtr, Disp: i * 16}, Mem{Base: tablePtr, Disp: i*16 + 16*4}
972 } else {
973 t.Lo, t.Hi = XMM(), XMM()
974 MOVUPS(Mem{Base: tablePtr, Disp: i * 16}, t.Lo)
975 MOVUPS(Mem{Base: tablePtr, Disp: i*16 + 16*4}, t.Hi)
976 }
977 tables[i] = t
978 }
979 // Generate mask
980 zero := XMM()
981 XORPS(zero, zero) // Zero, so bytes will be copied.
982 fifteen, mask := GP64(), XMM()
983 MOVQ(U32(0xf), fifteen)
984 MOVQ(fifteen, mask)
985 PSHUFB(zero, mask)
986 ctx.clrMask128 = mask
987
988 bytes := Load(Param("x").Len(), GP64())
989 x := Load(Param("x").Base(), GP64())
990 y := Load(Param("y").Base(), GP64())
991
992 Label("loop")
993 for i := 0; i < 2; i++ {
994 xLo, xHi, yLo, yHi := XMM(), XMM(), XMM(), XMM()
995 MOVUPS(Mem{Base: y, Disp: i*16 + 0}, yLo)
996 MOVUPS(Mem{Base: y, Disp: i*16 + 32}, yHi)
997
998 prodLo, prodHi := leoMul128(ctx, yLo, yHi, tables)
999
1000 MOVUPS(Mem{Base: x, Disp: i*16 + 0}, xLo)
1001 MOVUPS(Mem{Base: x, Disp: i*16 + 32}, xHi)
1002 PXOR(prodLo, xLo)
1003 PXOR(prodHi, xHi)
1004 MOVUPS(xLo, Mem{Base: x, Disp: i*16 + 0})
1005 MOVUPS(xHi, Mem{Base: x, Disp: i*16 + 32})
1006
1007 PXOR(xLo, yLo)
1008 PXOR(xHi, yHi)
1009 MOVUPS(yLo, Mem{Base: y, Disp: i*16 + 0})
1010 MOVUPS(yHi, Mem{Base: y, Disp: i*16 + 32})
1011
1012 }
1013
1014 ADDQ(U8(64), x)
1015 ADDQ(U8(64), y)
1016 SUBQ(U8(64), bytes)
1017 JNZ(LabelRef("loop"))
1018
1019 RET()
1020 }
1021 if pshufb {
1022 TEXT("mulgf16_ssse3", attr.NOSPLIT, fmt.Sprintf("func(x, y []byte, table *[8*16]uint8)"))
1023 Pragma("noescape")
1024 tablePtr := Load(Param("table"), GP64())
1025 tables := [4]table128{}
1026 for i, t := range tables {
1027 // We have enough space for all tables.
1028 if i > 3 {
1029 t.Lo, t.Hi = Mem{Base: tablePtr, Disp: i * 16}, Mem{Base: tablePtr, Disp: i*16 + 16*4}
1030 } else {
1031 t.Lo, t.Hi = XMM(), XMM()
1032 MOVUPS(Mem{Base: tablePtr, Disp: i * 16}, t.Lo)
1033 MOVUPS(Mem{Base: tablePtr, Disp: i*16 + 16*4}, t.Hi)
1034 }
1035 tables[i] = t
1036 }
1037 bytes := Load(Param("x").Len(), GP64())
1038 x := Load(Param("x").Base(), GP64())
1039 y := Load(Param("y").Base(), GP64())
1040 // Generate mask
1041 zero := XMM()
1042 XORPS(zero, zero) // Zero, so bytes will be copied.
1043 fifteen, mask := GP64(), XMM()
1044 MOVQ(U32(0xf), fifteen)
1045 MOVQ(fifteen, mask)
1046 PSHUFB(zero, mask)
1047 ctx.clrMask128 = mask
1048
1049 Label("loop")
1050 for i := 0; i < 2; i++ {
1051 dataLo, dataHi := XMM(), XMM()
1052 MOVUPS(Mem{Base: y, Disp: i*16 + 0}, dataLo)
1053 MOVUPS(Mem{Base: y, Disp: i*16 + 32}, dataHi)
1054
1055 prodLo, prodHi := leoMul128(ctx, dataLo, dataHi, tables)
1056 MOVUPS(prodLo, Mem{Base: x, Disp: i*16 + 0})
1057 MOVUPS(prodHi, Mem{Base: x, Disp: i*16 + 32})
1058 }
1059
1060 ADDQ(U8(64), x)
1061 ADDQ(U8(64), y)
1062 SUBQ(U8(64), bytes)
1063 JNZ(LabelRef("loop"))
1064
1065 RET()
1066 }
1067
1068 if pshufb {
1069 TEXT("mulgf16Xor_ssse3", attr.NOSPLIT, fmt.Sprintf("func(x, y []byte, table *[8*16]uint8)"))
1070 Pragma("noescape")
1071 tablePtr := Load(Param("table"), GP64())
1072 tables := [4]table128{}
1073 for i, t := range tables {
1074 // Keep tables 2-3 in memory to reduce register pressure
1075 // (xLo/xHi need live regs alongside leoMulAdd128 temps).
1076 if i >= 2 {
1077 t.Lo, t.Hi = Mem{Base: tablePtr, Disp: i * 16}, Mem{Base: tablePtr, Disp: i*16 + 16*4}
1078 } else {
1079 t.Lo, t.Hi = XMM(), XMM()
1080 MOVUPS(Mem{Base: tablePtr, Disp: i * 16}, t.Lo)
1081 MOVUPS(Mem{Base: tablePtr, Disp: i*16 + 16*4}, t.Hi)
1082 }
1083 tables[i] = t
1084 }
1085 bytes := Load(Param("x").Len(), GP64())
1086 x := Load(Param("x").Base(), GP64())
1087 y := Load(Param("y").Base(), GP64())
1088 zero := XMM()
1089 XORPS(zero, zero)
1090 fifteen, mask := GP64(), XMM()
1091 MOVQ(U32(0xf), fifteen)
1092 MOVQ(fifteen, mask)
1093 PSHUFB(zero, mask)
1094 ctx.clrMask128 = mask
1095
1096 Label("loop")
1097 for i := 0; i < 2; i++ {
1098 xLo, xHi := XMM(), XMM()
1099 MOVUPS(Mem{Base: x, Disp: i*16 + 0}, xLo)
1100 MOVUPS(Mem{Base: x, Disp: i*16 + 32}, xHi)
1101 dataLo, dataHi := XMM(), XMM()
1102 MOVUPS(Mem{Base: y, Disp: i*16 + 0}, dataLo)
1103 MOVUPS(Mem{Base: y, Disp: i*16 + 32}, dataHi)
1104
1105 leoMulAdd128(ctx, xLo, xHi, dataLo, dataHi, tables)
1106 MOVUPS(xLo, Mem{Base: x, Disp: i*16 + 0})
1107 MOVUPS(xHi, Mem{Base: x, Disp: i*16 + 32})
1108 }
1109
1110 ADDQ(U8(64), x)
1111 ADDQ(U8(64), y)
1112 SUBQ(U8(64), bytes)
1113 JNZ(LabelRef("loop"))
1114
1115 RET()
1116 }
1117
1118 // GFNI version of ifftDIT2
1119 // Data layout is SPLIT: x[0:32] = lo bytes, x[32:64] = hi bytes of 32 elements
1120 {
1121 TEXT("ifftDIT2_gfni", attr.NOSPLIT, fmt.Sprintf("func(x, y []byte, table *[4]uint64)"))
1122 Pragma("noescape")
1123
1124 // Load 4 GFNI matrices and broadcast to YMM registers
1125 tablePtr := Load(Param("table"), GP64())
1126 var tables [4]reg.Register
1127 for i := range tables {
1128 tables[i] = YMM().AsY()
1129 VBROADCASTSD(Mem{Base: tablePtr, Disp: i * 8}, tables[i])
1130 }
1131
1132 bytes := Load(Param("x").Len(), GP64())
1133 x := Load(Param("x").Base(), GP64())
1134 y := Load(Param("y").Base(), GP64())
1135
1136 xLo, xHi, yLo, yHi := YMM(), YMM(), YMM(), YMM()
1137
1138 Label("loop_gfni")
1139 VMOVDQU(Mem{Base: x, Disp: 0}, xLo) // lo bytes of 32 elements
1140 VMOVDQU(Mem{Base: x, Disp: 32}, xHi) // hi bytes of 32 elements
1141 VMOVDQU(Mem{Base: y, Disp: 0}, yLo) // lo bytes of 32 elements
1142 VMOVDQU(Mem{Base: y, Disp: 32}, yHi) // hi bytes of 32 elements
1143
1144 // y = x XOR y
1145 VPXOR(yLo, xLo, yLo)
1146 VPXOR(yHi, xHi, yHi)
1147 VMOVDQU(yLo, Mem{Base: y, Disp: 0})
1148 VMOVDQU(yHi, Mem{Base: y, Disp: 32})
1149
1150 // x = x + leoMul(y, table) using GFNI
1151 leoMulAdd256_gfni_avx2(xLo, xHi, yLo, yHi, tables, false)
1152
1153 VMOVDQU(xLo, Mem{Base: x, Disp: 0})
1154 VMOVDQU(xHi, Mem{Base: x, Disp: 32})
1155
1156 ADDQ(U8(64), x)
1157 ADDQ(U8(64), y)
1158 SUBQ(U8(64), bytes)
1159 JNZ(LabelRef("loop_gfni"))
1160
1161 VZEROUPPER()
1162 RET()
1163 }
1164
1165 // GFNI version of fftDIT2
1166 // Data layout is SPLIT: x[0:32] = lo bytes, x[32:64] = hi bytes of 32 elements
1167 {
1168 TEXT("fftDIT2_gfni", attr.NOSPLIT, fmt.Sprintf("func(x, y []byte, table *[4]uint64)"))
1169 Pragma("noescape")
1170
1171 // Load 4 GFNI matrices and broadcast to YMM registers
1172 tablePtr := Load(Param("table"), GP64())
1173 var tables [4]reg.Register
1174 for i := range tables {
1175 tables[i] = YMM().AsY()
1176 VBROADCASTSD(Mem{Base: tablePtr, Disp: i * 8}, tables[i])
1177 }
1178
1179 bytes := Load(Param("x").Len(), GP64())
1180 x := Load(Param("x").Base(), GP64())
1181 y := Load(Param("y").Base(), GP64())
1182
1183 xLo, xHi, yLo, yHi := YMM(), YMM(), YMM(), YMM()
1184
1185 Label("loop_fft_gfni")
1186 VMOVDQU(Mem{Base: x, Disp: 0}, xLo) // lo bytes of 32 elements
1187 VMOVDQU(Mem{Base: x, Disp: 32}, xHi) // hi bytes of 32 elements
1188 VMOVDQU(Mem{Base: y, Disp: 0}, yLo) // lo bytes of 32 elements
1189 VMOVDQU(Mem{Base: y, Disp: 32}, yHi) // hi bytes of 32 elements
1190
1191 // x = x + leoMul(y, table) using GFNI
1192 leoMulAdd256_gfni_avx2(xLo, xHi, yLo, yHi, tables, false)
1193
1194 VMOVDQU(xLo, Mem{Base: x, Disp: 0})
1195 VMOVDQU(xHi, Mem{Base: x, Disp: 32})
1196
1197 // y = x XOR y (after x is updated)
1198 VPXOR(yLo, xLo, yLo)
1199 VPXOR(yHi, xHi, yHi)
1200 VMOVDQU(yLo, Mem{Base: y, Disp: 0})
1201 VMOVDQU(yHi, Mem{Base: y, Disp: 32})
1202
1203 ADDQ(U8(64), x)
1204 ADDQ(U8(64), y)
1205 SUBQ(U8(64), bytes)
1206 JNZ(LabelRef("loop_fft_gfni"))
1207
1208 VZEROUPPER()
1209 RET()
1210 }
1211
1212 // GFNI version of mulgf16 (AVX2+GFNI)
1213 // x = y * table
1214 // Data layout is SPLIT: [0:32] = lo bytes, [32:64] = hi bytes of 32 elements
1215 {
1216 TEXT("mulgf16_gfni", attr.NOSPLIT, fmt.Sprintf("func(x, y []byte, table *[4]uint64)"))
1217 Pragma("noescape")
1218
1219 tablePtr := Load(Param("table"), GP64())
1220 var tables [4]reg.VecVirtual
1221 for i := range tables {
1222 tables[i] = YMM()
1223 VBROADCASTSD(Mem{Base: tablePtr, Disp: i * 8}, tables[i])
1224 }
1225
1226 bytes := Load(Param("x").Len(), GP64())
1227 x := Load(Param("x").Base(), GP64())
1228 y := Load(Param("y").Base(), GP64())
1229
1230 yLo, yHi := YMM(), YMM()
1231
1232 Label("loop_mulgf16_gfni")
1233 VMOVDQU(Mem{Base: y, Disp: 0}, yLo)
1234 VMOVDQU(Mem{Base: y, Disp: 32}, yHi)
1235
1236 // prodLo = A*yLo XOR B*yHi, prodHi = C*yLo XOR D*yHi
1237 tmpA, tmpB, tmpC, tmpD := YMM(), YMM(), YMM(), YMM()
1238 VGF2P8AFFINEQB(U8(0), tables[0], yLo, tmpA)
1239 VGF2P8AFFINEQB(U8(0), tables[1], yHi, tmpB)
1240 VGF2P8AFFINEQB(U8(0), tables[2], yLo, tmpC)
1241 VGF2P8AFFINEQB(U8(0), tables[3], yHi, tmpD)
1242 VPXOR(tmpA, tmpB, tmpA)
1243 VPXOR(tmpC, tmpD, tmpC)
1244
1245 VMOVDQU(tmpA, Mem{Base: x, Disp: 0})
1246 VMOVDQU(tmpC, Mem{Base: x, Disp: 32})
1247
1248 ADDQ(U8(64), x)
1249 ADDQ(U8(64), y)
1250 SUBQ(U8(64), bytes)
1251 JNZ(LabelRef("loop_mulgf16_gfni"))
1252
1253 VZEROUPPER()
1254 RET()
1255 }
1256
1257 // mulgf16Xor_gfni: x[] ^= y[] * table (GFNI mul-add)
1258 {
1259 TEXT("mulgf16Xor_gfni", attr.NOSPLIT, fmt.Sprintf("func(x, y []byte, table *[4]uint64)"))
1260 Pragma("noescape")
1261
1262 tablePtr := Load(Param("table"), GP64())
1263 var tables [4]reg.VecVirtual
1264 for i := range tables {
1265 tables[i] = YMM()
1266 VBROADCASTSD(Mem{Base: tablePtr, Disp: i * 8}, tables[i])
1267 }
1268
1269 bytes := Load(Param("x").Len(), GP64())
1270 x := Load(Param("x").Base(), GP64())
1271 y := Load(Param("y").Base(), GP64())
1272
1273 xLo, xHi, yLo, yHi := YMM(), YMM(), YMM(), YMM()
1274
1275 Label("loop_mulgf16Xor_gfni")
1276 VMOVDQU(Mem{Base: x, Disp: 0}, xLo)
1277 VMOVDQU(Mem{Base: x, Disp: 32}, xHi)
1278 VMOVDQU(Mem{Base: y, Disp: 0}, yLo)
1279 VMOVDQU(Mem{Base: y, Disp: 32}, yHi)
1280
1281 tmpA, tmpB, tmpC, tmpD := YMM(), YMM(), YMM(), YMM()
1282 VGF2P8AFFINEQB(U8(0), tables[0], yLo, tmpA)
1283 VGF2P8AFFINEQB(U8(0), tables[1], yHi, tmpB)
1284 VGF2P8AFFINEQB(U8(0), tables[2], yLo, tmpC)
1285 VGF2P8AFFINEQB(U8(0), tables[3], yHi, tmpD)
1286 VPXOR3way(tmpA, tmpB, xLo)
1287 VPXOR3way(tmpC, tmpD, xHi)
1288
1289 VMOVDQU(xLo, Mem{Base: x, Disp: 0})
1290 VMOVDQU(xHi, Mem{Base: x, Disp: 32})
1291
1292 ADDQ(U8(64), x)
1293 ADDQ(U8(64), y)
1294 SUBQ(U8(64), bytes)
1295 JNZ(LabelRef("loop_mulgf16Xor_gfni"))
1296
1297 VZEROUPPER()
1298 RET()
1299 }
1300
1301 // Fused 8-output GFNI: reads input once, scatters mul-add to 8 outputs.
1302 {
1303 TEXT("mulgf16Xor8_gfni", attr.NOSPLIT, fmt.Sprintf("func(in []byte, outs *[8][]byte, tables *[8][4]uint64)"))
1304 Pragma("noescape")
1305
1306 inPtr := Load(Param("in").Base(), GP64())
1307 bytes := Load(Param("in").Len(), GP64())
1308 outsPtr := Load(Param("outs"), GP64())
1309 tablesPtr := Load(Param("tables"), GP64())
1310
1311 // Pre-load 8 output base pointers. Slice header = 24 bytes; ptr at offset 0.
1312 var outPtrs [8]reg.GPVirtual
1313 for k := range outPtrs {
1314 outPtrs[k] = GP64()
1315 MOVQ(Mem{Base: outsPtr, Disp: k * 24}, outPtrs[k])
1316 }
1317 // outsPtr is now dead.
1318
1319 Label("loop_mulgf16Xor8_gfni")
1320 yLo, yHi := YMM(), YMM()
1321 VMOVDQU(Mem{Base: inPtr, Disp: 0}, yLo)
1322 VMOVDQU(Mem{Base: inPtr, Disp: 32}, yHi)
1323
1324 xLo, xHi := YMM(), YMM()
1325 VMOVDQU(Mem{Base: outPtrs[0], Disp: 0}, xLo)
1326 VMOVDQU(Mem{Base: outPtrs[0], Disp: 32}, xHi)
1327
1328 for k := 0; k < 8; k++ {
1329 tA, tB, tC, tD := YMM(), YMM(), YMM(), YMM()
1330 VBROADCASTSD(Mem{Base: tablesPtr, Disp: k*32 + 0}, tA)
1331 VBROADCASTSD(Mem{Base: tablesPtr, Disp: k*32 + 8}, tB)
1332 VBROADCASTSD(Mem{Base: tablesPtr, Disp: k*32 + 16}, tC)
1333 VBROADCASTSD(Mem{Base: tablesPtr, Disp: k*32 + 24}, tD)
1334
1335 VGF2P8AFFINEQB(U8(0), tA, yLo, tA)
1336 VGF2P8AFFINEQB(U8(0), tB, yHi, tB)
1337 VGF2P8AFFINEQB(U8(0), tC, yLo, tC)
1338 VGF2P8AFFINEQB(U8(0), tD, yHi, tD)
1339 VPXOR3way(xLo, tA, tB)
1340 VPXOR3way(xHi, tC, tD)
1341
1342 if k < 7 {
1343 VMOVDQU(Mem{Base: outPtrs[k+1], Disp: 0}, xLo)
1344 VMOVDQU(Mem{Base: outPtrs[k+1], Disp: 32}, xHi)
1345 }
1346
1347 VMOVDQU(tB, Mem{Base: outPtrs[k], Disp: 0})
1348 VMOVDQU(tD, Mem{Base: outPtrs[k], Disp: 32})
1349 }
1350
1351 ADDQ(U8(64), inPtr)
1352 for k := range outPtrs {
1353 ADDQ(U8(64), outPtrs[k])
1354 }
1355 SUBQ(U8(64), bytes)
1356 JNZ(LabelRef("loop_mulgf16Xor8_gfni"))
1357
1358 VZEROUPPER()
1359 RET()
1360 }
1361
1362 // Fused 8-output AVX-512 GFNI: 3 tables per scalar in Z8-Z31, VPTERNLOGD.
1363 {
1364 TEXT("mulgf16Xor8_avx512gfni", attr.NOSPLIT, fmt.Sprintf("func(in []byte, outs *[8][]byte, tables *[8][4]uint64)"))
1365 Pragma("noescape")
1366
1367 inPtr := Load(Param("in").Base(), GP64())
1368 bytes := Load(Param("in").Len(), GP64())
1369 outsPtr := Load(Param("outs"), GP64())
1370 tablesPtr := Load(Param("tables"), GP64())
1371
1372 var outPtrs [8]reg.GPVirtual
1373 for k := range outPtrs {
1374 outPtrs[k] = GP64()
1375 MOVQ(Mem{Base: outsPtr, Disp: k * 24}, outPtrs[k])
1376 }
1377
1378 // Physical registers:
1379 // Y0: yLo, Y1: yHi, Y2: xLo, Y3: xHi
1380 // Y4: tmpA, Y5: tmpB, Y6: tmpC, Y7: tD
1381 // Z8-Z31: 3 tables (A,B,C) x 8 scalars
1382 yLo, yHi := reg.Z0.AsY(), reg.Z1.AsY()
1383 xLo, xHi := reg.Z2.AsY(), reg.Z3.AsY()
1384 tmpA, tmpB := reg.Z4.AsY(), reg.Z5.AsY()
1385 tmpC, tD := reg.Z6.AsY(), reg.Z7.AsY()
1386
1387 tableZMMs := []reg.VecPhysical{
1388 reg.Z8, reg.Z9, reg.Z10, reg.Z11, reg.Z12, reg.Z13,
1389 reg.Z14, reg.Z15, reg.Z16, reg.Z17, reg.Z18, reg.Z19,
1390 reg.Z20, reg.Z21, reg.Z22, reg.Z23, reg.Z24, reg.Z25,
1391 reg.Z26, reg.Z27, reg.Z28, reg.Z29, reg.Z30, reg.Z31,
1392 }
1393
1394 // Pre-load tables A, B, C for all 8 scalars into ZMM registers.
1395 for k := 0; k < 8; k++ {
1396 for t := 0; t < 3; t++ {
1397 zr := tableZMMs[k*3+t]
1398 VBROADCASTSD(Mem{Base: tablesPtr, Disp: k*32 + t*8}, zr.AsY())
1399 }
1400 }
1401
1402 Label("loop_mulgf16Xor8_avx512gfni")
1403 VMOVDQU(Mem{Base: inPtr, Disp: 0}, yLo)
1404 VMOVDQU(Mem{Base: inPtr, Disp: 32}, yHi)
1405 VMOVDQU(Mem{Base: outPtrs[0], Disp: 0}, xLo)
1406 VMOVDQU(Mem{Base: outPtrs[0], Disp: 32}, xHi)
1407
1408 for k := 0; k < 8; k++ {
1409 tA := tableZMMs[k*3].AsY()
1410 tB := tableZMMs[k*3+1].AsY()
1411 tC := tableZMMs[k*3+2].AsY()
1412
1413 VGF2P8AFFINEQB(U8(0), tA, yLo, tmpA)
1414 VGF2P8AFFINEQB(U8(0), tB, yHi, tmpB)
1415 VPTERNLOGD(U8(0x96), xLo, tmpB, tmpA)
1416 dstLo := tmpA
1417 VGF2P8AFFINEQB(U8(0), tC, yLo, tmpC)
1418 VBROADCASTSD(Mem{Base: tablesPtr, Disp: k*32 + 24}, tD)
1419 VGF2P8AFFINEQB(U8(0), tD, yHi, tD)
1420 VPTERNLOGD(U8(0x96), tD, xHi, tmpC)
1421 dstHi := tmpC
1422
1423 if k < 7 {
1424 VMOVDQU(Mem{Base: outPtrs[k+1], Disp: 0}, xLo)
1425 VMOVDQU(Mem{Base: outPtrs[k+1], Disp: 32}, xHi)
1426 }
1427 VMOVDQU(dstLo, Mem{Base: outPtrs[k], Disp: 0})
1428 VMOVDQU(dstHi, Mem{Base: outPtrs[k], Disp: 32})
1429 }
1430
1431 ADDQ(U8(64), inPtr)
1432 for k := range outPtrs {
1433 ADDQ(U8(64), outPtrs[k])
1434 }
1435 SUBQ(U8(64), bytes)
1436 JNZ(LabelRef("loop_mulgf16Xor8_avx512gfni"))
1437
1438 VZEROUPPER()
1439 RET()
1440 }
1441
1442 // Fused 8-output AVX2 PSHUFB: reads input once, scatters mul-add to 8 outputs.
1443 if pshufb {
1444 TEXT("mulgf16Xor8_avx2", attr.NOSPLIT, fmt.Sprintf("func(in []byte, outs *[8][]byte, tables *[8][8*16]uint8)"))
1445 Pragma("noescape")
1446
1447 inPtr := Load(Param("in").Base(), GP64())
1448 bytes := Load(Param("in").Len(), GP64())
1449 outsPtr := Load(Param("outs"), GP64())
1450 tablesPtr := Load(Param("tables"), GP64())
1451
1452 var outPtrs [8]reg.GPVirtual
1453 for k := range outPtrs {
1454 outPtrs[k] = GP64()
1455 MOVQ(Mem{Base: outsPtr, Disp: k * 24}, outPtrs[k])
1456 }
1457
1458 ctx.clrMask = YMM()
1459 tmpMask := GP64()
1460 MOVQ(U32(15), tmpMask)
1461 MOVQ(tmpMask, ctx.clrMask.AsX())
1462 VPBROADCASTB(ctx.clrMask.AsX(), ctx.clrMask)
1463
1464 Label("loop_mulgf16Xor8_avx2")
1465 // Load input and extract nibbles (shared across all 8 scalars).
1466 yLo, yHi := YMM(), YMM()
1467 VMOVDQU(Mem{Base: inPtr, Disp: 0}, yLo)
1468 VMOVDQU(Mem{Base: inPtr, Disp: 32}, yHi)
1469
1470 data0Lo, data1Lo := YMM(), YMM()
1471 VPAND(ctx.clrMask, yLo, data0Lo)
1472 VPSRLQ(U8(4), yLo, data1Lo)
1473 VPAND(ctx.clrMask, data1Lo, data1Lo)
1474
1475 data0Hi, data1Hi := YMM(), YMM()
1476 VPAND(ctx.clrMask, yHi, data0Hi)
1477 VPSRLQ(U8(4), yHi, data1Hi)
1478 VPAND(ctx.clrMask, data1Hi, data1Hi)
1479 // yLo, yHi now dead.
1480
1481 for k := 0; k < 8; k++ {
1482 tableOff := k * 128
1483 xLo, xHi := YMM(), YMM()
1484 VMOVDQU(Mem{Base: outPtrs[k], Disp: 0}, xLo)
1485 VMOVDQU(Mem{Base: outPtrs[k], Disp: 32}, xHi)
1486
1487 // tables layout per scalar: [Lo0 Lo1 Lo2 Lo3 Hi0 Hi1 Hi2 Hi3] each 16 bytes.
1488 prodLo, prodHi, tLo, tHi, tmpLo, tmpHi := YMM(), YMM(), YMM(), YMM(), YMM(), YMM()
1489
1490 // table[0]: data0Lo nibbles
1491 VBROADCASTI128(Mem{Base: tablesPtr, Disp: tableOff + 0}, tLo)
1492 VBROADCASTI128(Mem{Base: tablesPtr, Disp: tableOff + 64}, tHi)
1493 VPSHUFB(data0Lo, tLo, prodLo)
1494 VPSHUFB(data0Lo, tHi, prodHi)
1495
1496 // table[1]: data1Lo nibbles
1497 VBROADCASTI128(Mem{Base: tablesPtr, Disp: tableOff + 16}, tLo)
1498 VBROADCASTI128(Mem{Base: tablesPtr, Disp: tableOff + 80}, tHi)
1499 VPSHUFB(data1Lo, tLo, tmpLo)
1500 VPSHUFB(data1Lo, tHi, tmpHi)
1501 VPXOR(prodLo, tmpLo, prodLo)
1502 VPXOR(prodHi, tmpHi, prodHi)
1503
1504 // table[2]: data0Hi nibbles
1505 VBROADCASTI128(Mem{Base: tablesPtr, Disp: tableOff + 32}, tLo)
1506 VBROADCASTI128(Mem{Base: tablesPtr, Disp: tableOff + 96}, tHi)
1507 VPSHUFB(data0Hi, tLo, tmpLo)
1508 VPSHUFB(data0Hi, tHi, tmpHi)
1509 VPXOR(prodLo, tmpLo, prodLo)
1510 VPXOR(prodHi, tmpHi, prodHi)
1511
1512 // table[3]: data1Hi nibbles, final accumulate
1513 VBROADCASTI128(Mem{Base: tablesPtr, Disp: tableOff + 48}, tLo)
1514 VBROADCASTI128(Mem{Base: tablesPtr, Disp: tableOff + 112}, tHi)
1515 VPSHUFB(data1Hi, tLo, tmpLo)
1516 VPSHUFB(data1Hi, tHi, tmpHi)
1517 VPXOR3way(prodLo, tmpLo, xLo)
1518 VPXOR3way(prodHi, tmpHi, xHi)
1519
1520 VMOVDQU(xLo, Mem{Base: outPtrs[k], Disp: 0})
1521 VMOVDQU(xHi, Mem{Base: outPtrs[k], Disp: 32})
1522 }
1523
1524 ADDQ(U8(64), inPtr)
1525 for k := range outPtrs {
1526 ADDQ(U8(64), outPtrs[k])
1527 }
1528 SUBQ(U8(64), bytes)
1529 JNZ(LabelRef("loop_mulgf16Xor8_avx2"))
1530
1531 VZEROUPPER()
1532 RET()
1533 }
1534 }
1535
1536 // xLo, xHi updated, yLo, yHi preserved...
1537 func leoMulAdd256(ctx gf16ctx, xLo, xHi, yLo, yHi reg.VecVirtual, table [4]table256) {
1538 // inlined:
1539 // prodLo, prodHi := leoMul256(ctx, yLo, yHi, table)
1540 lo := yLo
1541 hi := yHi
1542 data0, data1 := YMM(), YMM()
1543 VPSRLQ(U8(4), lo, data1) // data1 = lo >> 4
1544 VPAND(ctx.clrMask, lo, data0) // data0 = lo&0xf
1545 VPAND(ctx.clrMask, data1, data1) // data 1 = data1 &0xf
1546 prodLo, prodHi := YMM(), YMM()
1547 table[0].prepare()
1548 VPSHUFB(data0, table[0].Lo, prodLo)
1549 VPSHUFB(data0, table[0].Hi, prodHi)
1550 tmpLo, tmpHi := YMM(), YMM()
1551 table[1].prepare()
1552 VPSHUFB(data1, table[1].Lo, tmpLo)
1553 VPSHUFB(data1, table[1].Hi, tmpHi)
1554 VPXOR(prodLo, tmpLo, prodLo)
1555 VPXOR(prodHi, tmpHi, prodHi)
1556
1557 // Now process high
1558 data0, data1 = YMM(), YMM() // Realloc to break dep
1559 VPAND(hi, ctx.clrMask, data0)
1560 VPSRLQ(U8(4), hi, data1)
1561 VPAND(ctx.clrMask, data1, data1)
1562
1563 tmpLo, tmpHi = YMM(), YMM() // Realloc to break dep
1564 table[2].prepare()
1565 VPSHUFB(data0, table[2].Lo, tmpLo)
1566 VPSHUFB(data0, table[2].Hi, tmpHi)
1567 VPXOR(prodLo, tmpLo, prodLo)
1568 VPXOR(prodHi, tmpHi, prodHi)
1569 table[3].prepare()
1570 VPSHUFB(data1, table[3].Lo, tmpLo)
1571 VPSHUFB(data1, table[3].Hi, tmpHi)
1572 if ctx.avx512 {
1573 VPTERNLOGD(U8(0x96), prodLo, tmpLo, xLo)
1574 VPTERNLOGD(U8(0x96), prodHi, tmpHi, xHi)
1575 } else {
1576 VPXOR3way(prodLo, tmpLo, xLo)
1577 VPXOR3way(prodHi, tmpHi, xHi)
1578 }
1579 }
1580
1581 // leoMul256 lo, hi preserved...
1582 func leoMul256(ctx gf16ctx, lo, hi reg.VecVirtual, table [4]table256) (prodLo, prodHi reg.VecVirtual) {
1583 data0, data1 := YMM(), YMM()
1584 VPSRLQ(U8(4), lo, data1) // data1 = lo >> 4
1585 VPAND(ctx.clrMask, lo, data0) // data0 = lo&0xf
1586 VPAND(ctx.clrMask, data1, data1) // data 1 = data1 &0xf
1587 prodLo, prodHi = YMM(), YMM()
1588 table[0].prepare()
1589 VPSHUFB(data0, table[0].Lo, prodLo)
1590 VPSHUFB(data0, table[0].Hi, prodHi)
1591 tmpLo, tmpHi := YMM(), YMM()
1592 table[1].prepare()
1593 VPSHUFB(data1, table[1].Lo, tmpLo)
1594 VPSHUFB(data1, table[1].Hi, tmpHi)
1595 VPXOR(prodLo, tmpLo, prodLo)
1596 VPXOR(prodHi, tmpHi, prodHi)
1597
1598 // Now process high
1599 data0, data1 = YMM(), YMM() // Realloc to break dep
1600 VPAND(hi, ctx.clrMask, data0)
1601 VPSRLQ(U8(4), hi, data1)
1602 VPAND(ctx.clrMask, data1, data1)
1603
1604 tmpLo, tmpHi = YMM(), YMM() // Realloc to break dep
1605 table[2].prepare()
1606 VPSHUFB(data0, table[2].Lo, tmpLo)
1607 VPSHUFB(data0, table[2].Hi, tmpHi)
1608 VPXOR(prodLo, tmpLo, prodLo)
1609 VPXOR(prodHi, tmpHi, prodHi)
1610 table[3].prepare()
1611 VPSHUFB(data1, table[3].Lo, tmpLo)
1612 VPSHUFB(data1, table[3].Hi, tmpHi)
1613 VPXOR(prodLo, tmpLo, prodLo)
1614 VPXOR(prodHi, tmpHi, prodHi)
1615 return
1616 }
1617
1618 func leoMulAdd128(ctx gf16ctx, xLo, xHi, yLo, yHi reg.VecVirtual, table [4]table128) {
1619 prodLo, prodHi := leoMul128(ctx, yLo, yHi, table)
1620 PXOR(prodLo, xLo)
1621 PXOR(prodHi, xHi)
1622 }
1623
1624 // leoMul128 lo, hi preseved (but likely will take extra regs to reuse)
1625 func leoMul128(ctx gf16ctx, lo, hi reg.VecVirtual, table [4]table128) (prodLo, prodHi reg.VecVirtual) {
1626 data0, data1 := XMM(), XMM()
1627 MOVAPS(lo, data1)
1628 PSRLQ(U8(4), data1) // data1 = lo >> 4
1629 MOVAPS(lo, data0)
1630 PAND(ctx.clrMask128, data0) // data0 = lo&0xf
1631 PAND(ctx.clrMask128, data1) // data 1 = data1 &0xf
1632 prodLo, prodHi = XMM(), XMM()
1633 MOVUPS(table[0].Lo, prodLo)
1634 MOVUPS(table[0].Hi, prodHi)
1635 PSHUFB(data0, prodLo)
1636 PSHUFB(data0, prodHi)
1637 tmpLo, tmpHi := XMM(), XMM()
1638 MOVUPS(table[1].Lo, tmpLo)
1639 MOVUPS(table[1].Hi, tmpHi)
1640 PSHUFB(data1, tmpLo)
1641 PSHUFB(data1, tmpHi)
1642 PXOR(tmpLo, prodLo)
1643 PXOR(tmpHi, prodHi)
1644
1645 // Now process high
1646 data0, data1 = XMM(), XMM() // Realloc to break dep
1647 MOVAPS(hi, data0)
1648 MOVAPS(hi, data1)
1649 PAND(ctx.clrMask128, data0)
1650 PSRLQ(U8(4), data1)
1651 PAND(ctx.clrMask128, data1)
1652
1653 tmpLo, tmpHi = XMM(), XMM() // Realloc to break dep
1654 MOVUPS(table[2].Lo, tmpLo)
1655 MOVUPS(table[2].Hi, tmpHi)
1656 PSHUFB(data0, tmpLo)
1657 PSHUFB(data0, tmpHi)
1658 PXOR(tmpLo, prodLo)
1659 PXOR(tmpHi, prodHi)
1660 MOVUPS(table[3].Lo, tmpLo)
1661 MOVUPS(table[3].Hi, tmpHi)
1662 PSHUFB(data1, tmpLo)
1663 PSHUFB(data1, tmpHi)
1664 PXOR(tmpLo, prodLo)
1665 PXOR(tmpHi, prodHi)
1666 return
1667 }
1668
1669 // leoMulAdd256_gfni_mem loads tables from memory and multiplies y, XORs into x.
1670 // Loads tables on-demand to save YMM registers for DIT4 operations.
1671 func leoMulAdd256_gfni_mem(xLo, xHi, yLo, yHi reg.VecVirtual, tablePtr reg.Register) {
1672 Comment("GFNI LEO_MULADD_256 (from memory)")
1673
1674 tmpA, tmpB, tmpC, tmpD := YMM(), YMM(), YMM(), YMM()
1675 VPBROADCASTQ(Mem{Base: tablePtr, Disp: 0}, tmpA)
1676 VPBROADCASTQ(Mem{Base: tablePtr, Disp: 8}, tmpB)
1677 VPBROADCASTQ(Mem{Base: tablePtr, Disp: 16}, tmpC)
1678 VPBROADCASTQ(Mem{Base: tablePtr, Disp: 24}, tmpD)
1679
1680 VGF2P8AFFINEQB(U8(0), tmpA, yLo, tmpA)
1681 VGF2P8AFFINEQB(U8(0), tmpB, yHi, tmpB)
1682 VGF2P8AFFINEQB(U8(0), tmpC, yLo, tmpC)
1683 VGF2P8AFFINEQB(U8(0), tmpD, yHi, tmpD)
1684
1685 // XOR into x
1686 VPXOR3way(tmpA, tmpB, xLo)
1687 VPXOR3way(tmpC, tmpD, xHi)
1688 }
1689
1690 // leoMulAdd256_gfni_avx2 loads tables from registers and multiplies y, XORs into x.
1691 // Loads tables on-demand to save YMM registers for DIT4 operations.
1692 func leoMulAdd256_gfni_avx2(xLo, xHi, yLo, yHi reg.VecVirtual, tables [4]reg.Register, tern bool) {
1693 Comment("GFNI LEO_MULADD_256 (from register)")
1694
1695 // Apply GFNI transforms
1696 tmpA, tmpB, tmpC, tmpD := YMM(), YMM(), YMM(), YMM()
1697 VGF2P8AFFINEQB(U8(0), tables[0], yLo, tmpA) // A * yLo
1698 VGF2P8AFFINEQB(U8(0), tables[1], yHi, tmpB) // B * yHi
1699 VGF2P8AFFINEQB(U8(0), tables[2], yLo, tmpC) // C * yLo
1700 VGF2P8AFFINEQB(U8(0), tables[3], yHi, tmpD) // D * yHi
1701 // XOR into x
1702 if tern {
1703 VPTERNLOGD(U8(0x96), tmpA, tmpB, xLo)
1704 VPTERNLOGD(U8(0x96), tmpC, tmpD, xHi)
1705 } else {
1706 VPXOR3way(tmpA, tmpB, xLo)
1707 VPXOR3way(tmpC, tmpD, xHi)
1708 }
1709 }
1710
1711 // leoMulAddZMM_gfni multiplies packed y by GFNI tables and XORs into packed x.
1712 // x and y are in packed format: [lo_32bytes | hi_32bytes]
1713 // tables: [0] = [A|C], [1] = [B|D] - combined matrices for efficiency
1714 // Uses shuffle to duplicate lo/hi, then single GFNI per combined table.
1715 func leoMulAddZMM_gfni(x, y reg.VecVirtual, tables [2]reg.VecVirtual) {
1716 Comment("GFNI LEO_MULADD packed ZMM (combined tables)")
1717
1718 // Duplicate data halves: [lo|hi] -> [lo|lo] and [hi|hi]
1719 yLoLo, yHiHi := ZMM(), ZMM()
1720 VSHUFI64X2(U8(0x44), y, y, yLoLo) // [lo|lo]: select lanes 0,1,0,1
1721 VSHUFI64X2(U8(0xEE), y, y, yHiHi) // [hi|hi]: select lanes 2,3,2,3
1722
1723 // Apply combined tables
1724 // tableAC = [A|C], tableBD = [B|D]
1725 // GFNI([A|C], [lo|lo]) = [A*lo | C*lo]
1726 // GFNI([B|D], [hi|hi]) = [B*hi | D*hi]
1727 tmp1, tmp2 := ZMM(), ZMM()
1728 VGF2P8AFFINEQB(U8(0), tables[0], yLoLo, tmp1) // [A*lo | C*lo]
1729 VGF2P8AFFINEQB(U8(0), tables[1], yHiHi, tmp2) // [B*hi | D*hi]
1730
1731 // 3-way XOR: x ^= tmp1 ^ tmp2 using VPTERNLOG (0x96 = A XOR B XOR C)
1732 VPTERNLOGD(U8(0x96), tmp2, tmp1, x)
1733 }