Debian Codegraph @

_gen/gen.go

raw
1 //go:build generate
2
3 // Copyright 2022+, Klaus Post. See LICENSE for details.
4
5 //go:generate go run -tags=generate,nopshufb . -out ../galois_gen_nopshufb_amd64.s -stubs ../galois_gen_nopshufb_amd64.go -pkg=reedsolomon
6 //go:generate go fmt ../galois_gen_switch_nopshufb_amd64.go
7 //go:generate go fmt ../galois_gen_nopshufb_amd64.go
8 //go:generate go run cleanup.go ../galois_gen_nopshufb_amd64.s
9
10 //go:generate go run -tags=generate . -out ../galois_gen_amd64.s -stubs ../galois_gen_amd64.go -pkg=reedsolomon
11 //go:generate go fmt ../galois_gen_switch_amd64.go
12 //go:generate go fmt ../galois_gen_amd64.go
13 //go:generate go run cleanup.go ../galois_gen_amd64.s
14
15 package main
16
17 import (
18 "bufio"
19 "fmt"
20 "os"
21
22 "github.com/mmcloughlin/avo/attr"
23 . "github.com/mmcloughlin/avo/build"
24 "github.com/mmcloughlin/avo/buildtags"
25 . "github.com/mmcloughlin/avo/operand"
26 "github.com/mmcloughlin/avo/reg"
27 )
28
29 // Technically we can do slightly bigger, but we stay reasonable.
30 const inputMax = 10
31 const outputMax = 10
32
33 var switchDefs [inputMax][outputMax]string
34 var switchDefsX [inputMax][outputMax]string
35
36 var switchDefs512 [inputMax][outputMax]string
37 var switchDefsX512 [inputMax][outputMax]string
38
39 var switchDefsAvxGFNI [inputMax][outputMax]string
40 var switchDefsXAvxGFNI [inputMax][outputMax]string
41
42 // Prefetch offsets, set to 0 to disable.
43 // Disabled since they appear to be consistently slower.
44 const prefetchSrc = 0
45 const prefetchDst = 0
46
47 func main() {
48 Constraint(buildtags.Not("appengine").ToConstraint())
49 Constraint(buildtags.Not("noasm").ToConstraint())
50 Constraint(buildtags.Not("nogen").ToConstraint())
51 if pshufb {
52 Constraint(buildtags.Not("nopshufb").ToConstraint())
53 } else {
54 Constraint(buildtags.Opt("nopshufb").ToConstraint())
55 }
56 Constraint(buildtags.Term("gc").ToConstraint())
57
58 TEXT("_dummy_", 0, "func()")
59 Comment("#ifdef GOAMD64_v4")
60 Comment("#define XOR3WAY(ignore, a, b, dst)\\")
61 Comment("@\tVPTERNLOGD $0x96, a, b, dst")
62 Comment("#else")
63 Comment("#define XOR3WAY(ignore, a, b, dst)\\")
64 Comment("@\tVPXOR a, dst, dst\\")
65 Comment("@\tVPXOR b, dst, dst")
66 Comment("#endif")
67 RET()
68
69 genXor()
70
71 for i := 1; i <= inputMax; i++ {
72 for j := 1; j <= outputMax; j++ {
73 if pshufb {
74 genMulAvx2(fmt.Sprintf("mulAvxTwo_%dx%d", i, j), i, j, false)
75 genMulAvx2Sixty64(fmt.Sprintf("mulAvxTwo_%dx%d_64", i, j), i, j, false)
76 }
77 genMulAvx512GFNI(fmt.Sprintf("mulGFNI_%dx%d_64", i, j), i, j, false)
78 genMulAvxGFNI(fmt.Sprintf("mulAvxGFNI_%dx%d", i, j), i, j, false)
79 genMulAvx512GFNI(fmt.Sprintf("mulGFNI_%dx%d_64Xor", i, j), i, j, true)
80 genMulAvxGFNI(fmt.Sprintf("mulAvxGFNI_%dx%dXor", i, j), i, j, true)
81
82 if pshufb {
83 genMulAvx2(fmt.Sprintf("mulAvxTwo_%dx%dXor", i, j), i, j, true)
84 genMulAvx2Sixty64(fmt.Sprintf("mulAvxTwo_%dx%d_64Xor", i, j), i, j, true)
85 }
86 }
87 }
88
89 genSwitch()
90 genGF16()
91 genGF8()
92
93 if pshufb {
94 genArmSve()
95 genArmNeon()
96 genArmSveAllVl()
97 }
98 Generate()
99 }
100
101 func genSwitch() {
102 name := "../galois_gen_switch_amd64.go"
103 tag := "!nopshufb"
104 if !pshufb {
105 name = "../galois_gen_switch_nopshufb_amd64.go"
106 tag = "nopshufb"
107 }
108 f, err := os.Create(name)
109 if err != nil {
110 panic(err)
111 }
112 defer f.Close()
113 w := bufio.NewWriter(f)
114 defer w.Flush()
115 w.WriteString(`// Code generated by command: go generate ` + os.Getenv("GOFILE") + `. DO NOT EDIT.
116
117 //go:build !appengine && !noasm && gc && !nogen && ` + tag + `
118
119 package reedsolomon
120
121 import (
122 "fmt"
123 )
124
125 `)
126 avx2funcs := string(` fAvx2 = galMulSlicesAvx2
127 fAvx2Xor = galMulSlicesAvx2Xor
128 `)
129
130 hasCodeGenImpl := string(` return &fAvx2, &fAvx2Xor, codeGen && pshufb && r.o.useAVX2 &&
131 byteCount >= codeGenMinSize && inputs+outputs >= codeGenMinShards &&
132 inputs <= codeGenMaxInputs && outputs <= codeGenMaxOutputs`)
133
134 if !pshufb {
135 avx2funcs = ""
136 hasCodeGenImpl = ` return nil, nil, false // no code generation for generic case (only GFNI cases) `
137 }
138
139 w.WriteString(fmt.Sprintf(`const (
140 codeGen = true
141 codeGenMaxGoroutines = 8
142 codeGenMaxInputs = %d
143 codeGenMaxOutputs = %d
144 minCodeGenSize = 64
145 codeGenPadInputs = 1
146 )
147
148 var (
149 %s fGFNI = galMulSlicesGFNI
150 fGFNIXor = galMulSlicesGFNIXor
151 fAvxGFNI = galMulSlicesAvxGFNI
152 fAvxGFNIXor = galMulSlicesAvxGFNIXor
153 )
154
155 func (r *reedSolomon) hasCodeGen(byteCount int, inputs, outputs int) (_, _ *func(matrix []byte, in, out [][]byte, start, stop int) int, ok bool) {
156 %s
157 }
158
159 func (r *reedSolomon) canGFNI(byteCount int, inputs, outputs int) (_, _ *func(matrix []uint64, in, out [][]byte, start, stop int) int, ok bool) {
160 if r.o.useAvx512GFNI {
161 return &fGFNI, &fGFNIXor, codeGen &&
162 byteCount >= codeGenMinSize && inputs+outputs >= codeGenMinShards &&
163 inputs <= codeGenMaxInputs && outputs <= codeGenMaxOutputs
164 }
165 return &fAvxGFNI, &fAvxGFNIXor, codeGen && r.o.useAvxGNFI &&
166 byteCount >= codeGenMinSize && inputs+outputs >= codeGenMinShards &&
167 inputs <= codeGenMaxInputs && outputs <= codeGenMaxOutputs
168 }`, inputMax, outputMax, avx2funcs, hasCodeGenImpl))
169
170 if !pshufb {
171 w.WriteString("\n\nfunc galMulSlicesAvx2(matrix []byte, in, out [][]byte, start, stop int) int { panic(`no pshufb`)}\n")
172 w.WriteString("func galMulSlicesAvx2Xor(matrix []byte, in, out [][]byte, start, stop int) int { panic(`no pshufb`)}\n")
173 }
174
175 if pshufb {
176 w.WriteString(`
177
178 func galMulSlicesAvx2(matrix []byte, in, out [][]byte, start, stop int) (n int) {
179 n = stop - start
180 if raceEnabled {
181 defer func() {
182 raceReadSlices(in, start, n)
183 raceWriteSlices(out, start, n)
184 }()
185 }
186
187 `)
188
189 w.WriteString(`switch len(in) {
190 `)
191 for in, defs := range switchDefs[:] {
192 w.WriteString(fmt.Sprintf(" case %d:\n switch len(out) {\n", in+1))
193 for out, def := range defs[:] {
194 w.WriteString(fmt.Sprintf(" case %d:\n", out+1))
195 w.WriteString(def)
196 }
197 w.WriteString("}\n")
198 }
199 w.WriteString(`}
200 panic(fmt.Sprintf("unhandled size: %dx%d", len(in), len(out)))
201 }
202
203 func galMulSlicesAvx2Xor(matrix []byte, in, out [][]byte, start, stop int) (n int) {
204 n = stop - start
205 if raceEnabled {
206 defer func() {
207 raceReadSlices(in, start, n)
208 raceWriteSlices(out, start, n)
209 }()
210 }
211
212 `)
213
214 w.WriteString(`switch len(in) {
215 `)
216 for in, defs := range switchDefsX[:] {
217 w.WriteString(fmt.Sprintf(" case %d:\n switch len(out) {\n", in+1))
218 for out, def := range defs[:] {
219 w.WriteString(fmt.Sprintf(" case %d:\n", out+1))
220 w.WriteString(def)
221 }
222 w.WriteString("}\n")
223 }
224 w.WriteString(`}
225 panic(fmt.Sprintf("unhandled size: %dx%d", len(in), len(out)))
226 }
227 `)
228 }
229
230 w.WriteString(`
231
232 func galMulSlicesGFNI(matrix []uint64, in, out [][]byte, start, stop int) int {
233 n := (stop-start) & (maxInt - (64 - 1))
234
235 if raceEnabled {
236 raceReadSlices(in, start, n)
237 raceWriteSlices(out, start, n)
238 }
239
240 `)
241
242 w.WriteString(`switch len(in) {
243 `)
244 for in, defs := range switchDefs512[:] {
245 w.WriteString(fmt.Sprintf(" case %d:\n switch len(out) {\n", in+1))
246 for out, def := range defs[:] {
247 w.WriteString(fmt.Sprintf(" case %d:\n", out+1))
248 w.WriteString(def)
249 }
250 w.WriteString("}\n")
251 }
252 w.WriteString(`}
253 panic(fmt.Sprintf("unhandled size: %dx%d", len(in), len(out)))
254 }
255
256 func galMulSlicesGFNIXor(matrix []uint64, in, out [][]byte, start, stop int) int {
257 n := (stop-start) & (maxInt - (64 - 1))
258
259 if raceEnabled {
260 raceReadSlices(in, start, n)
261 raceWriteSlices(out, start, n)
262 }
263
264 `)
265
266 w.WriteString(`switch len(in) {
267 `)
268 for in, defs := range switchDefsX512[:] {
269 w.WriteString(fmt.Sprintf(" case %d:\n switch len(out) {\n", in+1))
270 for out, def := range defs[:] {
271 w.WriteString(fmt.Sprintf(" case %d:\n", out+1))
272 w.WriteString(def)
273 }
274 w.WriteString("}\n")
275 }
276 w.WriteString(`}
277 panic(fmt.Sprintf("unhandled size: %dx%d", len(in), len(out)))
278 }
279 `)
280
281 w.WriteString(`
282
283 func galMulSlicesAvxGFNI(matrix []uint64, in, out [][]byte, start, stop int) int {
284 n := (stop-start) & (maxInt - (32 - 1))
285
286 if raceEnabled {
287 raceReadSlices(in, start, n)
288 raceWriteSlices(out, start, n)
289 }
290
291 `)
292
293 w.WriteString(`switch len(in) {
294 `)
295 for in, defs := range switchDefsAvxGFNI[:] {
296 w.WriteString(fmt.Sprintf(" case %d:\n switch len(out) {\n", in+1))
297 for out, def := range defs[:] {
298 w.WriteString(fmt.Sprintf(" case %d:\n", out+1))
299 w.WriteString(def)
300 }
301 w.WriteString("}\n")
302 }
303 w.WriteString(`}
304 panic(fmt.Sprintf("unhandled size: %dx%d", len(in), len(out)))
305 }
306
307 func galMulSlicesAvxGFNIXor(matrix []uint64, in, out [][]byte, start, stop int) int {
308 n := (stop-start) & (maxInt - (32 - 1))
309
310 if raceEnabled {
311 raceReadSlices(in, start, n)
312 raceWriteSlices(out, start, n)
313 }
314
315 `)
316
317 w.WriteString(`switch len(in) {
318 `)
319 for in, defs := range switchDefsXAvxGFNI[:] {
320 w.WriteString(fmt.Sprintf(" case %d:\n switch len(out) {\n", in+1))
321 for out, def := range defs[:] {
322 w.WriteString(fmt.Sprintf(" case %d:\n", out+1))
323 w.WriteString(def)
324 }
325 w.WriteString("}\n")
326 }
327 w.WriteString(`}
328 panic(fmt.Sprintf("unhandled size: %dx%d", len(in), len(out)))
329 }
330 `)
331 }
332
333 // VPXOR3way will 3-way xor a and b and dst.
334 func VPXOR3way(a, b, dst reg.VecVirtual) {
335 // VPTERNLOGQ is replaced by XOR3WAY - we just use an equivalent operation
336 VPTERNLOGQ(U8(0), a, b, dst)
337 }
338
339 func genMulAvx2(name string, inputs int, outputs int, xor bool) {
340 if outputs < 4 {
341 // Covered by 64-byte version.
342 return
343 }
344 const perLoopBits = 5
345 const perLoop = 1 << perLoopBits
346
347 total := inputs * outputs
348 doc := []string{
349 fmt.Sprintf("%s takes %d inputs and produces %d outputs.", name, inputs, outputs),
350 }
351 if !xor {
352 doc = append(doc, "The output is initialized to 0.")
353 }
354
355 // Load shuffle masks on every use.
356 var loadNone bool
357 // Use registers for destination registers.
358 var regDst = true
359 var reloadLength = false
360
361 // lo, hi, 1 in, 1 out, 2 tmp, 1 mask
362 est := total*2 + outputs + 5
363 if outputs == 1 {
364 // We don't need to keep a copy of the input if only 1 output.
365 est -= 2
366 }
367
368 if est > 16 {
369 loadNone = true
370 // We run out of GP registers first, now.
371 if inputs+outputs > 13 {
372 regDst = false
373 }
374 // Save one register by reloading length.
375 if inputs+outputs > 12 && regDst {
376 reloadLength = true
377 }
378 }
379
380 x := ""
381 if xor {
382 x = "Xor"
383 }
384
385 TEXT(name, attr.NOSPLIT, fmt.Sprintf("func(matrix []byte, in [][]byte, out [][]byte, start, n int)"))
386
387 // SWITCH DEFINITION:
388 s := fmt.Sprintf(" mulAvxTwo_%dx%d%s(matrix, in, out, start, n)\n", inputs, outputs, x)
389 s += fmt.Sprintf("\t\t\t\treturn n & (maxInt - %d)\n", perLoop-1)
390 if xor {
391 switchDefsX[inputs-1][outputs-1] = s
392 } else {
393 switchDefs[inputs-1][outputs-1] = s
394 }
395
396 if loadNone {
397 Comment("Loading no tables to registers")
398 } else {
399 // loadNone == false
400 Comment("Loading all tables to registers")
401 }
402 if regDst {
403 Comment("Destination kept in GP registers")
404 } else {
405 Comment("Destination kept on stack")
406 }
407
408 Doc(doc...)
409 Pragma("noescape")
410 Commentf("Full registers estimated %d YMM used", est)
411
412 length := Load(Param("n"), GP64())
413 matrixBase := GP64()
414 addr, err := Param("matrix").Base().Resolve()
415 if err != nil {
416 panic(err)
417 }
418 MOVQ(addr.Addr, matrixBase)
419 SHRQ(U8(perLoopBits), length)
420 TESTQ(length, length)
421 JZ(LabelRef(name + "_end"))
422
423 inLo := make([]reg.VecVirtual, total)
424 inHi := make([]reg.VecVirtual, total)
425
426 for i := range inLo {
427 if loadNone {
428 break
429 }
430 tableLo := YMM()
431 tableHi := YMM()
432 VMOVDQU(Mem{Base: matrixBase, Disp: i * 64}, tableLo)
433 VMOVDQU(Mem{Base: matrixBase, Disp: i*64 + 32}, tableHi)
434 inLo[i] = tableLo
435 inHi[i] = tableHi
436 }
437
438 inPtrs := make([]reg.GPVirtual, inputs)
439 inSlicePtr := GP64()
440 addr, err = Param("in").Base().Resolve()
441 if err != nil {
442 panic(err)
443 }
444 MOVQ(addr.Addr, inSlicePtr)
445 for i := range inPtrs {
446 ptr := GP64()
447 MOVQ(Mem{Base: inSlicePtr, Disp: i * 24}, ptr)
448 inPtrs[i] = ptr
449 }
450 // Destination
451 dst := make([]reg.VecVirtual, outputs)
452 dstPtr := make([]reg.GPVirtual, outputs)
453 addr, err = Param("out").Base().Resolve()
454 if err != nil {
455 panic(err)
456 }
457 outSlicePtr := GP64()
458 MOVQ(addr.Addr, outSlicePtr)
459 for i := range dst {
460 dst[i] = YMM()
461 if !regDst {
462 continue
463 }
464 ptr := GP64()
465 MOVQ(Mem{Base: outSlicePtr, Disp: i * 24}, ptr)
466 dstPtr[i] = ptr
467 }
468
469 offset := GP64()
470 addr, err = Param("start").Resolve()
471 if err != nil {
472 panic(err)
473 }
474
475 MOVQ(addr.Addr, offset)
476 if regDst {
477 Comment("Add start offset to output")
478 for _, ptr := range dstPtr {
479 ADDQ(offset, ptr)
480 }
481 }
482
483 Comment("Add start offset to input")
484 for _, ptr := range inPtrs {
485 ADDQ(offset, ptr)
486 }
487 // Offset no longer needed unless not regDst
488
489 tmpMask := GP64()
490 MOVQ(U32(15), tmpMask)
491 lowMask := YMM()
492 MOVQ(tmpMask, lowMask.AsX())
493 VPBROADCASTB(lowMask.AsX(), lowMask)
494
495 if reloadLength {
496 length = Load(Param("n"), GP64())
497 SHRQ(U8(perLoopBits), length)
498 }
499 Label(name + "_loop")
500
501 // Load data before loop or during first iteration?
502 // No clear winner.
503 preloadInput := xor && false
504 if preloadInput {
505 Commentf("Load %d outputs", outputs)
506 for i := range dst {
507 if regDst {
508 VMOVDQU(Mem{Base: dstPtr[i]}, dst[i])
509 if prefetchDst > 0 {
510 PREFETCHT0(Mem{Base: dstPtr[i], Disp: prefetchDst})
511 }
512 continue
513 }
514 ptr := GP64()
515 MOVQ(Mem{Base: outSlicePtr, Disp: i * 24}, ptr)
516 VMOVDQU(Mem{Base: ptr, Index: offset, Scale: 1}, dst[i])
517 if prefetchDst > 0 {
518 PREFETCHT0(Mem{Base: ptr, Disp: prefetchDst, Index: offset, Scale: 1})
519 }
520 }
521 }
522
523 lookLow, lookHigh := YMM(), YMM()
524 inLow, inHigh := YMM(), YMM()
525 for i := range inPtrs {
526 Commentf("Load and process 32 bytes from input %d to %d outputs", i, outputs)
527 VMOVDQU(Mem{Base: inPtrs[i]}, inLow)
528 if prefetchSrc > 0 {
529 PREFETCHT0(Mem{Base: inPtrs[i], Disp: prefetchSrc})
530 }
531 ADDQ(U8(perLoop), inPtrs[i])
532 VPSRLQ(U8(4), inLow, inHigh)
533 VPAND(lowMask, inLow, inLow)
534 VPAND(lowMask, inHigh, inHigh)
535 for j := range dst {
536 //Commentf(" xor:%v i: %v", xor, i)
537 if !preloadInput && xor && i == 0 {
538 if regDst {
539 VMOVDQU(Mem{Base: dstPtr[j]}, dst[j])
540 if prefetchDst > 0 {
541 PREFETCHT0(Mem{Base: dstPtr[j], Disp: prefetchDst})
542 }
543 } else {
544 ptr := GP64()
545 MOVQ(Mem{Base: outSlicePtr, Disp: j * 24}, ptr)
546 VMOVDQU(Mem{Base: ptr, Index: offset, Scale: 1}, dst[j])
547 if prefetchDst > 0 {
548 PREFETCHT0(Mem{Base: ptr, Disp: prefetchDst, Index: offset, Scale: 1})
549 }
550 }
551 }
552 if loadNone {
553 VMOVDQU(Mem{Base: matrixBase, Disp: 64 * (i*outputs + j)}, lookLow)
554 VMOVDQU(Mem{Base: matrixBase, Disp: 32 + 64*(i*outputs+j)}, lookHigh)
555 VPSHUFB(inLow, lookLow, lookLow)
556 VPSHUFB(inHigh, lookHigh, lookHigh)
557 } else {
558 VPSHUFB(inLow, inLo[i*outputs+j], lookLow)
559 VPSHUFB(inHigh, inHi[i*outputs+j], lookHigh)
560 }
561 if i == 0 && !xor {
562 // We don't have any existing data, write directly.
563 VPXOR(lookLow, lookHigh, dst[j])
564 } else {
565 VPXOR3way(lookLow, lookHigh, dst[j])
566 }
567 }
568 }
569 Commentf("Store %d outputs", outputs)
570 for i := range dst {
571 if regDst {
572 VMOVDQU(dst[i], Mem{Base: dstPtr[i]})
573 if prefetchDst > 0 && !xor {
574 PREFETCHT0(Mem{Base: dstPtr[i], Disp: prefetchDst})
575 }
576 ADDQ(U8(perLoop), dstPtr[i])
577 continue
578 }
579 ptr := GP64()
580 MOVQ(Mem{Base: outSlicePtr, Disp: i * 24}, ptr)
581 VMOVDQU(dst[i], Mem{Base: ptr, Index: offset, Scale: 1})
582 if prefetchDst > 0 && !xor {
583 PREFETCHT0(Mem{Base: ptr, Disp: prefetchDst, Index: offset, Scale: 1})
584 }
585 }
586 Comment("Prepare for next loop")
587 if !regDst {
588 ADDQ(U8(perLoop), offset)
589 }
590 DECQ(length)
591 JNZ(LabelRef(name + "_loop"))
592 VZEROUPPER()
593
594 Label(name + "_end")
595 RET()
596 }
597
598 func genMulAvx2Sixty64(name string, inputs int, outputs int, xor bool) {
599 if outputs >= 4 {
600 return
601 }
602 const perLoopBits = 6
603 const perLoop = 1 << perLoopBits
604
605 total := inputs * outputs
606
607 doc := []string{
608 fmt.Sprintf("%s takes %d inputs and produces %d outputs.", name, inputs, outputs),
609 }
610 if !xor {
611 doc = append(doc, "The output is initialized to 0.")
612 }
613
614 // Load shuffle masks on every use.
615 var loadNone bool
616 // Use registers for destination registers.
617 var regDst = true
618 var reloadLength = false
619
620 // lo, hi, 1 in, 1 out, 2 tmp, 1 mask
621 est := total*4 + outputs + 7
622 if outputs == 1 {
623 // We don't need to keep a copy of the input if only 1 output.
624 est -= 2
625 }
626
627 if est > 16 {
628 loadNone = true
629 // We run out of GP registers first, now.
630 if inputs+outputs > 13 {
631 regDst = false
632 }
633 // Save one register by reloading length.
634 if inputs+outputs > 12 && regDst {
635 reloadLength = true
636 }
637 }
638
639 TEXT(name, 0, fmt.Sprintf("func(matrix []byte, in [][]byte, out [][]byte, start, n int)"))
640 x := ""
641 if xor {
642 x = "Xor"
643 }
644 // SWITCH DEFINITION:
645 //s := fmt.Sprintf("n = (n>>%d)<<%d\n", perLoopBits, perLoopBits)
646 s := fmt.Sprintf(" mulAvxTwo_%dx%d_64%s(matrix, in, out, start, n)\n", inputs, outputs, x)
647 s += fmt.Sprintf("\t\t\t\treturn n & (maxInt - %d)\n", perLoop-1)
648 if xor {
649 switchDefsX[inputs-1][outputs-1] = s
650 } else {
651 switchDefs[inputs-1][outputs-1] = s
652 }
653
654 if loadNone {
655 Comment("Loading no tables to registers")
656 } else {
657 // loadNone == false
658 Comment("Loading all tables to registers")
659 }
660 if regDst {
661 Comment("Destination kept in GP registers")
662 } else {
663 Comment("Destination kept on stack")
664 }
665
666 Doc(doc...)
667 Pragma("noescape")
668 Commentf("Full registers estimated %d YMM used", est)
669
670 length := Load(Param("n"), GP64())
671 matrixBase := GP64()
672 addr, err := Param("matrix").Base().Resolve()
673 if err != nil {
674 panic(err)
675 }
676 MOVQ(addr.Addr, matrixBase)
677 SHRQ(U8(perLoopBits), length)
678 TESTQ(length, length)
679 JZ(LabelRef(name + "_end"))
680
681 inLo := make([]reg.VecVirtual, total)
682 inHi := make([]reg.VecVirtual, total)
683
684 for i := range inLo {
685 if loadNone {
686 break
687 }
688 tableLo := YMM()
689 tableHi := YMM()
690 VMOVDQU(Mem{Base: matrixBase, Disp: i * 64}, tableLo)
691 VMOVDQU(Mem{Base: matrixBase, Disp: i*64 + 32}, tableHi)
692 inLo[i] = tableLo
693 inHi[i] = tableHi
694 }
695
696 inPtrs := make([]reg.GPVirtual, inputs)
697 inSlicePtr := GP64()
698 addr, err = Param("in").Base().Resolve()
699 if err != nil {
700 panic(err)
701 }
702 MOVQ(addr.Addr, inSlicePtr)
703 for i := range inPtrs {
704 ptr := GP64()
705 MOVQ(Mem{Base: inSlicePtr, Disp: i * 24}, ptr)
706 inPtrs[i] = ptr
707 }
708 // Destination
709 dst := make([]reg.VecVirtual, outputs)
710 dst2 := make([]reg.VecVirtual, outputs)
711 dstPtr := make([]reg.GPVirtual, outputs)
712 addr, err = Param("out").Base().Resolve()
713 if err != nil {
714 panic(err)
715 }
716 outBase := addr.Addr
717 outSlicePtr := GP64()
718 MOVQ(outBase, outSlicePtr)
719 for i := range dst {
720 dst[i] = YMM()
721 dst2[i] = YMM()
722 if !regDst {
723 continue
724 }
725 ptr := GP64()
726 MOVQ(Mem{Base: outSlicePtr, Disp: i * 24}, ptr)
727 dstPtr[i] = ptr
728 }
729
730 offset := GP64()
731 addr, err = Param("start").Resolve()
732 if err != nil {
733 panic(err)
734 }
735
736 MOVQ(addr.Addr, offset)
737 if regDst {
738 Comment("Add start offset to output")
739 for _, ptr := range dstPtr {
740 ADDQ(offset, ptr)
741 }
742 }
743
744 Comment("Add start offset to input")
745 for _, ptr := range inPtrs {
746 ADDQ(offset, ptr)
747 }
748 // Offset no longer needed unless not regdst
749
750 tmpMask := GP64()
751 MOVQ(U32(15), tmpMask)
752 lowMask := YMM()
753 MOVQ(tmpMask, lowMask.AsX())
754 VPBROADCASTB(lowMask.AsX(), lowMask)
755
756 if reloadLength {
757 Commentf("Reload length to save a register")
758 length = Load(Param("n"), GP64())
759 SHRQ(U8(perLoopBits), length)
760 }
761 Label(name + "_loop")
762
763 if xor {
764 Commentf("Load %d outputs", outputs)
765 for i := range dst {
766 if regDst {
767 VMOVDQU(Mem{Base: dstPtr[i]}, dst[i])
768 VMOVDQU(Mem{Base: dstPtr[i], Disp: 32}, dst2[i])
769 if prefetchDst > 0 {
770 PREFETCHT0(Mem{Base: dstPtr[i], Disp: prefetchDst})
771 }
772 continue
773 }
774 ptr := GP64()
775 MOVQ(Mem{Base: outSlicePtr, Disp: i * 24}, ptr)
776 VMOVDQU(Mem{Base: ptr, Index: offset, Scale: 1}, dst[i])
777 VMOVDQU(Mem{Base: ptr, Index: offset, Scale: 1, Disp: 32}, dst2[i])
778
779 if prefetchDst > 0 {
780 PREFETCHT0(Mem{Base: ptr, Disp: prefetchDst, Index: offset, Scale: 1})
781 }
782 }
783 }
784
785 lookLow, lookHigh := YMM(), YMM()
786 lookLow2, lookHigh2 := YMM(), YMM()
787 inLow, inHigh := YMM(), YMM()
788 in2Low, in2High := YMM(), YMM()
789 for i := range inPtrs {
790 Commentf("Load and process 64 bytes from input %d to %d outputs", i, outputs)
791 VMOVDQU(Mem{Base: inPtrs[i]}, inLow)
792 VMOVDQU(Mem{Base: inPtrs[i], Disp: 32}, in2Low)
793 if prefetchSrc > 0 {
794 PREFETCHT0(Mem{Base: inPtrs[i], Disp: prefetchSrc})
795 }
796 ADDQ(U8(perLoop), inPtrs[i])
797 VPSRLQ(U8(4), inLow, inHigh)
798 VPSRLQ(U8(4), in2Low, in2High)
799 VPAND(lowMask, inLow, inLow)
800 VPAND(lowMask, in2Low, in2Low)
801 VPAND(lowMask, inHigh, inHigh)
802 VPAND(lowMask, in2High, in2High)
803 for j := range dst {
804 if loadNone {
805 VMOVDQU(Mem{Base: matrixBase, Disp: 64 * (i*outputs + j)}, lookLow)
806 VMOVDQU(Mem{Base: matrixBase, Disp: 32 + 64*(i*outputs+j)}, lookHigh)
807 VPSHUFB(in2Low, lookLow, lookLow2)
808 VPSHUFB(inLow, lookLow, lookLow) // Reuse lookLow to save a reg
809 VPSHUFB(in2High, lookHigh, lookHigh2)
810 VPSHUFB(inHigh, lookHigh, lookHigh) // Reuse lookHigh to save a reg
811 } else {
812 VPSHUFB(inLow, inLo[i*outputs+j], lookLow)
813 VPSHUFB(in2Low, inLo[i*outputs+j], lookLow2)
814 VPSHUFB(inHigh, inHi[i*outputs+j], lookHigh)
815 VPSHUFB(in2High, inHi[i*outputs+j], lookHigh2)
816 }
817 if i == 0 && !xor {
818 // We don't have any existing data, write directly.
819 VPXOR(lookLow, lookHigh, dst[j])
820 VPXOR(lookLow2, lookHigh2, dst2[j])
821 } else {
822 VPXOR3way(lookLow, lookHigh, dst[j])
823 VPXOR3way(lookLow2, lookHigh2, dst2[j])
824 }
825 }
826 }
827 Commentf("Store %d outputs", outputs)
828 for i := range dst {
829 if regDst {
830 VMOVDQU(dst[i], Mem{Base: dstPtr[i]})
831 VMOVDQU(dst2[i], Mem{Base: dstPtr[i], Disp: 32})
832 if prefetchDst > 0 && !xor {
833 PREFETCHT0(Mem{Base: dstPtr[i], Disp: prefetchDst})
834 }
835 ADDQ(U8(perLoop), dstPtr[i])
836 continue
837 }
838 ptr := GP64()
839 MOVQ(Mem{Base: outSlicePtr, Disp: i * 24}, ptr)
840 VMOVDQU(dst[i], Mem{Base: ptr, Index: offset, Scale: 1})
841 VMOVDQU(dst2[i], Mem{Base: ptr, Index: offset, Scale: 1, Disp: 32})
842 if prefetchDst > 0 && !xor {
843 PREFETCHT0(Mem{Base: ptr, Disp: prefetchDst, Index: offset, Scale: 1})
844 }
845 }
846 Comment("Prepare for next loop")
847 if !regDst {
848 ADDQ(U8(perLoop), offset)
849 }
850 DECQ(length)
851 JNZ(LabelRef(name + "_loop"))
852 VZEROUPPER()
853
854 Label(name + "_end")
855 RET()
856 }
857
858 func genMulAvx512GFNI(name string, inputs int, outputs int, xor bool) {
859 const perLoopBits = 6
860 const perLoop = 1 << perLoopBits
861
862 total := inputs * outputs
863
864 doc := []string{
865 fmt.Sprintf("%s takes %d inputs and produces %d outputs.", name, inputs, outputs),
866 }
867 if !xor {
868 doc = append(doc, "The output is initialized to 0.")
869 }
870
871 // Load shuffle masks on every use.
872 var loadNone bool
873 // Use registers for destination registers.
874 var regDst = true
875 var reloadLength = false
876
877 est := total + outputs + 2
878 // When we can't hold all, keep this many in registers.
879 inReg := 0
880 if est > 32 {
881 loadNone = true
882 inReg = 32 - outputs - 2
883 // We run out of GP registers first, now.
884 if inputs+outputs > 13 {
885 regDst = false
886 }
887 // Save one register by reloading length.
888 if inputs+outputs > 12 && regDst {
889 reloadLength = true
890 }
891 }
892
893 TEXT(name, 0, fmt.Sprintf("func(matrix []uint64, in [][]byte, out [][]byte, start, n int)"))
894 x := ""
895 if xor {
896 x = "Xor"
897 }
898 // SWITCH DEFINITION:
899 //s := fmt.Sprintf("n = (n>>%d)<<%d\n", perLoopBits, perLoopBits)
900 s := fmt.Sprintf(" mulGFNI_%dx%d_64%s(matrix, in, out, start, n)\n", inputs, outputs, x)
901 s += fmt.Sprintf("\t\t\t\treturn n\n")
902 if xor {
903 switchDefsX512[inputs-1][outputs-1] = s
904 } else {
905 switchDefs512[inputs-1][outputs-1] = s
906 }
907
908 if loadNone {
909 Commentf("Loading %d of %d tables to registers", inReg, inputs*outputs)
910 } else {
911 // loadNone == false
912 Comment("Loading all tables to registers")
913 }
914 if regDst {
915 Comment("Destination kept in GP registers")
916 } else {
917 Comment("Destination kept on stack")
918 }
919
920 Doc(doc...)
921 Pragma("noescape")
922 Commentf("Full registers estimated %d YMM used", est)
923
924 length := Load(Param("n"), GP64())
925 matrixBase := GP64()
926 addr, err := Param("matrix").Base().Resolve()
927 if err != nil {
928 panic(err)
929 }
930 MOVQ(addr.Addr, matrixBase)
931 SHRQ(U8(perLoopBits), length)
932 TESTQ(length, length)
933 JZ(LabelRef(name + "_end"))
934
935 matrix := make([]reg.VecVirtual, total)
936
937 for i := range matrix {
938 if loadNone && i >= inReg {
939 break
940 }
941 table := ZMM()
942 VBROADCASTF32X2(Mem{Base: matrixBase, Disp: i * 8}, table)
943 matrix[i] = table
944 }
945
946 inPtrs := make([]reg.GPVirtual, inputs)
947 inSlicePtr := GP64()
948 addr, err = Param("in").Base().Resolve()
949 if err != nil {
950 panic(err)
951 }
952 MOVQ(addr.Addr, inSlicePtr)
953 for i := range inPtrs {
954 ptr := GP64()
955 MOVQ(Mem{Base: inSlicePtr, Disp: i * 24}, ptr)
956 inPtrs[i] = ptr
957 }
958 // Destination
959 dst := make([]reg.VecVirtual, outputs)
960 dstPtr := make([]reg.GPVirtual, outputs)
961 addr, err = Param("out").Base().Resolve()
962 if err != nil {
963 panic(err)
964 }
965 outBase := addr.Addr
966 outSlicePtr := GP64()
967 MOVQ(addr.Addr, outSlicePtr)
968 MOVQ(outBase, outSlicePtr)
969 for i := range dst {
970 dst[i] = ZMM()
971 if !regDst {
972 continue
973 }
974 ptr := GP64()
975 MOVQ(Mem{Base: outSlicePtr, Disp: i * 24}, ptr)
976 dstPtr[i] = ptr
977 }
978
979 offset := GP64()
980 addr, err = Param("start").Resolve()
981 if err != nil {
982 panic(err)
983 }
984
985 MOVQ(addr.Addr, offset)
986 if regDst {
987 Comment("Add start offset to output")
988 for _, ptr := range dstPtr {
989 ADDQ(offset, ptr)
990 }
991 }
992
993 Comment("Add start offset to input")
994 for _, ptr := range inPtrs {
995 ADDQ(offset, ptr)
996 }
997 // Offset no longer needed unless not regdst
998
999 if reloadLength {
1000 Commentf("Reload length to save a register")
1001 length = Load(Param("n"), GP64())
1002 SHRQ(U8(perLoopBits), length)
1003 }
1004 Label(name + "_loop")
1005
1006 if xor {
1007 Commentf("Load %d outputs", outputs)
1008 for i := range dst {
1009 if regDst {
1010 VMOVDQU64(Mem{Base: dstPtr[i]}, dst[i])
1011 if prefetchDst > 0 {
1012 PREFETCHT0(Mem{Base: dstPtr[i], Disp: prefetchDst})
1013 }
1014 continue
1015 }
1016 ptr := GP64()
1017 MOVQ(Mem{Base: outSlicePtr, Disp: i * 24}, ptr)
1018 VMOVDQU64(Mem{Base: ptr, Index: offset, Scale: 1}, dst[i])
1019
1020 if prefetchDst > 0 {
1021 PREFETCHT0(Mem{Base: ptr, Disp: prefetchDst, Index: offset, Scale: 1})
1022 }
1023 }
1024 }
1025
1026 in := ZMM()
1027 look := ZMM()
1028 for i := range inPtrs {
1029 Commentf("Load and process 64 bytes from input %d to %d outputs", i, outputs)
1030 VMOVDQU64(Mem{Base: inPtrs[i]}, in)
1031 if prefetchSrc > 0 {
1032 PREFETCHT0(Mem{Base: inPtrs[i], Disp: prefetchSrc})
1033 }
1034 ADDQ(U8(perLoop), inPtrs[i])
1035
1036 for j := range dst {
1037 idx := i*outputs + j
1038 if loadNone && idx >= inReg {
1039 if i == 0 && !xor {
1040 VGF2P8AFFINEQB_BCST(U8(0), Mem{Base: matrixBase, Disp: 8 * idx}, in, dst[j])
1041 } else {
1042 VGF2P8AFFINEQB_BCST(U8(0), Mem{Base: matrixBase, Disp: 8 * idx}, in, look)
1043 VXORPD(dst[j], look, dst[j])
1044 }
1045 } else {
1046 if i == 0 && !xor {
1047 VGF2P8AFFINEQB(U8(0), matrix[i*outputs+j], in, dst[j])
1048 } else {
1049 VGF2P8AFFINEQB(U8(0), matrix[i*outputs+j], in, look)
1050 VXORPD(dst[j], look, dst[j])
1051 }
1052 }
1053 }
1054 }
1055 Commentf("Store %d outputs", outputs)
1056 for i := range dst {
1057 if regDst {
1058 VMOVDQU64(dst[i], Mem{Base: dstPtr[i]})
1059 if prefetchDst > 0 && !xor {
1060 PREFETCHT0(Mem{Base: dstPtr[i], Disp: prefetchDst})
1061 }
1062 ADDQ(U8(perLoop), dstPtr[i])
1063 continue
1064 }
1065 ptr := GP64()
1066 MOVQ(Mem{Base: outSlicePtr, Disp: i * 24}, ptr)
1067 VMOVDQU64(dst[i], Mem{Base: ptr, Index: offset, Scale: 1})
1068 if prefetchDst > 0 && !xor {
1069 PREFETCHT0(Mem{Base: ptr, Disp: prefetchDst, Index: offset, Scale: 1})
1070 }
1071 }
1072 Comment("Prepare for next loop")
1073 if !regDst {
1074 ADDQ(U8(perLoop), offset)
1075 }
1076 DECQ(length)
1077 JNZ(LabelRef(name + "_loop"))
1078 VZEROUPPER()
1079
1080 Label(name + "_end")
1081 RET()
1082 }
1083
1084 func genMulAvxGFNI(name string, inputs int, outputs int, xor bool) {
1085 const perLoopBits = 5
1086 const perLoop = 1 << perLoopBits
1087
1088 total := inputs * outputs
1089
1090 doc := []string{
1091 fmt.Sprintf("%s takes %d inputs and produces %d outputs.", name, inputs, outputs),
1092 }
1093 if !xor {
1094 doc = append(doc, "The output is initialized to 0.")
1095 }
1096
1097 // Load shuffle masks on every use.
1098 var loadNone bool
1099 // Use registers for destination registers.
1100 var regDst = true
1101 var reloadLength = false
1102
1103 est := total + outputs + 2
1104 // When we can't hold all, keep this many in registers.
1105 inReg := 0
1106 if est > 16 {
1107 loadNone = true
1108 inReg = 16 - outputs - 2
1109 // We run out of GP registers first, now.
1110 if inputs+outputs > 13 {
1111 regDst = false
1112 }
1113 // Save one register by reloading length.
1114 if inputs+outputs > 12 && regDst {
1115 reloadLength = true
1116 }
1117 }
1118
1119 TEXT(name, 0, fmt.Sprintf("func(matrix []uint64, in [][]byte, out [][]byte, start, n int)"))
1120 x := ""
1121 if xor {
1122 x = "Xor"
1123 }
1124 // SWITCH DEFINITION:
1125 //s := fmt.Sprintf("n = (n>>%d)<<%d\n", perLoopBits, perLoopBits)
1126 s := fmt.Sprintf(" mulAvxGFNI_%dx%d%s(matrix, in, out, start, n)\n", inputs, outputs, x)
1127 s += fmt.Sprintf("\t\t\t\treturn n\n")
1128 if xor {
1129 switchDefsXAvxGFNI[inputs-1][outputs-1] = s
1130 } else {
1131 switchDefsAvxGFNI[inputs-1][outputs-1] = s
1132 }
1133
1134 if loadNone {
1135 Commentf("Loading %d of %d tables to registers", inReg, inputs*outputs)
1136 } else {
1137 // loadNone == false
1138 Comment("Loading all tables to registers")
1139 }
1140 if regDst {
1141 Comment("Destination kept in GP registers")
1142 } else {
1143 Comment("Destination kept on stack")
1144 }
1145
1146 Doc(doc...)
1147 Pragma("noescape")
1148 Commentf("Full registers estimated %d YMM used", est)
1149
1150 length := Load(Param("n"), GP64())
1151 matrixBase := GP64()
1152 addr, err := Param("matrix").Base().Resolve()
1153 if err != nil {
1154 panic(err)
1155 }
1156 MOVQ(addr.Addr, matrixBase)
1157 SHRQ(U8(perLoopBits), length)
1158 TESTQ(length, length)
1159 JZ(LabelRef(name + "_end"))
1160
1161 matrix := make([]reg.VecVirtual, total)
1162
1163 for i := range matrix {
1164 if loadNone && i >= inReg {
1165 break
1166 }
1167 table := YMM()
1168 VBROADCASTSD(Mem{Base: matrixBase, Disp: i * 8}, table)
1169 matrix[i] = table
1170 }
1171
1172 inPtrs := make([]reg.GPVirtual, inputs)
1173 inSlicePtr := GP64()
1174 addr, err = Param("in").Base().Resolve()
1175 if err != nil {
1176 panic(err)
1177 }
1178 MOVQ(addr.Addr, inSlicePtr)
1179 for i := range inPtrs {
1180 ptr := GP64()
1181 MOVQ(Mem{Base: inSlicePtr, Disp: i * 24}, ptr)
1182 inPtrs[i] = ptr
1183 }
1184 // Destination
1185 dst := make([]reg.VecVirtual, outputs)
1186 dstPtr := make([]reg.GPVirtual, outputs)
1187 addr, err = Param("out").Base().Resolve()
1188 if err != nil {
1189 panic(err)
1190 }
1191 outBase := addr.Addr
1192 outSlicePtr := GP64()
1193 MOVQ(addr.Addr, outSlicePtr)
1194 MOVQ(outBase, outSlicePtr)
1195 for i := range dst {
1196 dst[i] = YMM()
1197 if !regDst {
1198 continue
1199 }
1200 ptr := GP64()
1201 MOVQ(Mem{Base: outSlicePtr, Disp: i * 24}, ptr)
1202 dstPtr[i] = ptr
1203 }
1204
1205 offset := GP64()
1206 addr, err = Param("start").Resolve()
1207 if err != nil {
1208 panic(err)
1209 }
1210
1211 MOVQ(addr.Addr, offset)
1212 if regDst {
1213 Comment("Add start offset to output")
1214 for _, ptr := range dstPtr {
1215 ADDQ(offset, ptr)
1216 }
1217 }
1218
1219 Comment("Add start offset to input")
1220 for _, ptr := range inPtrs {
1221 ADDQ(offset, ptr)
1222 }
1223 // Offset no longer needed unless not regdst
1224
1225 if reloadLength {
1226 Commentf("Reload length to save a register")
1227 length = Load(Param("n"), GP64())
1228 SHRQ(U8(perLoopBits), length)
1229 }
1230 Label(name + "_loop")
1231
1232 if xor {
1233 Commentf("Load %d outputs", outputs)
1234 for i := range dst {
1235 if regDst {
1236 VMOVDQU(Mem{Base: dstPtr[i]}, dst[i])
1237 if prefetchDst > 0 {
1238 PREFETCHT0(Mem{Base: dstPtr[i], Disp: prefetchDst})
1239 }
1240 continue
1241 }
1242 ptr := GP64()
1243 MOVQ(Mem{Base: outSlicePtr, Disp: i * 24}, ptr)
1244 VMOVDQU(Mem{Base: ptr, Index: offset, Scale: 1}, dst[i])
1245
1246 if prefetchDst > 0 {
1247 PREFETCHT0(Mem{Base: ptr, Disp: prefetchDst, Index: offset, Scale: 1})
1248 }
1249 }
1250 }
1251
1252 in := YMM()
1253 look := YMM()
1254 for i := range inPtrs {
1255 Commentf("Load and process 32 bytes from input %d to %d outputs", i, outputs)
1256 VMOVDQU(Mem{Base: inPtrs[i]}, in)
1257 if prefetchSrc > 0 {
1258 PREFETCHT0(Mem{Base: inPtrs[i], Disp: prefetchSrc})
1259 }
1260 ADDQ(U8(perLoop), inPtrs[i])
1261
1262 for j := range dst {
1263 idx := i*outputs + j
1264 if loadNone && idx >= inReg {
1265 tmp := YMM()
1266 if i == 0 && !xor {
1267 VBROADCASTSD(Mem{Base: matrixBase, Disp: idx * 8}, tmp)
1268 VGF2P8AFFINEQB(U8(0), tmp, in, dst[j])
1269 } else {
1270 VBROADCASTSD(Mem{Base: matrixBase, Disp: idx * 8}, tmp)
1271 VGF2P8AFFINEQB(U8(0), tmp, in, look)
1272 VXORPD(dst[j], look, dst[j])
1273 }
1274 } else {
1275 if i == 0 && !xor {
1276 VGF2P8AFFINEQB(U8(0), matrix[i*outputs+j], in, dst[j])
1277 } else {
1278 VGF2P8AFFINEQB(U8(0), matrix[i*outputs+j], in, look)
1279 VXORPD(dst[j], look, dst[j])
1280 }
1281 }
1282 }
1283 }
1284 Commentf("Store %d outputs", outputs)
1285 for i := range dst {
1286 if regDst {
1287 VMOVDQU(dst[i], Mem{Base: dstPtr[i]})
1288 if prefetchDst > 0 && !xor {
1289 PREFETCHT0(Mem{Base: dstPtr[i], Disp: prefetchDst})
1290 }
1291 ADDQ(U8(perLoop), dstPtr[i])
1292 continue
1293 }
1294 ptr := GP64()
1295 MOVQ(Mem{Base: outSlicePtr, Disp: i * 24}, ptr)
1296 VMOVDQU(dst[i], Mem{Base: ptr, Index: offset, Scale: 1})
1297 if prefetchDst > 0 && !xor {
1298 PREFETCHT0(Mem{Base: ptr, Disp: prefetchDst, Index: offset, Scale: 1})
1299 }
1300 }
1301 Comment("Prepare for next loop")
1302 if !regDst {
1303 ADDQ(U8(perLoop), offset)
1304 }
1305 DECQ(length)
1306 JNZ(LabelRef(name + "_loop"))
1307 VZEROUPPER()
1308
1309 Label(name + "_end")
1310 RET()
1311 }
1312
1313 func genXor() {
1314 // SSE 2
1315 {
1316 Comment("sSE2XorSlice will XOR in with out and store in out.")
1317 Comment("Processes 16 bytes/loop.")
1318 TEXT("sSE2XorSlice", 0, fmt.Sprintf("func(in, out []byte)"))
1319 Pragma("noescape")
1320 src := Load(Param("in").Base(), GP64())
1321 dst := Load(Param("out").Base(), GP64())
1322 length := Load(Param("in").Len(), GP64())
1323 SHRQ(U8(4), length)
1324 srcX, dstX := XMM(), XMM()
1325 JZ(LabelRef("end"))
1326 Label("loop")
1327 MOVOU(Mem{Base: src}, srcX)
1328 MOVOU(Mem{Base: dst}, dstX)
1329 PXOR(srcX, dstX)
1330 MOVOU(dstX, Mem{Base: dst})
1331 ADDQ(U8(16), src)
1332 ADDQ(U8(16), dst)
1333 DECQ(length)
1334 JNZ(LabelRef("loop"))
1335 Label("end")
1336 RET()
1337 }
1338
1339 // SSE2 64 bytes
1340 {
1341 Comment("sSE2XorSlice_64 will XOR in with out and store in out.")
1342 Comment("Processes 64 bytes/loop.")
1343 TEXT("sSE2XorSlice_64", 0, fmt.Sprintf("func(in, out []byte)"))
1344 Pragma("noescape")
1345 src := Load(Param("in").Base(), GP64())
1346 dst := Load(Param("out").Base(), GP64())
1347 length := Load(Param("in").Len(), GP64())
1348 SHRQ(U8(6), length)
1349 var srcX, dstX [4]reg.VecVirtual
1350 for i := range srcX {
1351 srcX[i], dstX[i] = XMM(), XMM()
1352 }
1353 JZ(LabelRef("end"))
1354 Label("loop")
1355 for i := range srcX {
1356 MOVOU(Mem{Base: src, Disp: 16 * i}, srcX[i])
1357 }
1358 for i := range srcX {
1359 MOVOU(Mem{Base: dst, Disp: 16 * i}, dstX[i])
1360 }
1361 for i := range srcX {
1362 PXOR(srcX[i], dstX[i])
1363 }
1364 for i := range srcX {
1365 MOVOU(dstX[i], Mem{Base: dst, Disp: 16 * i})
1366 }
1367 ADDQ(U8(64), src)
1368 ADDQ(U8(64), dst)
1369 DECQ(length)
1370 JNZ(LabelRef("loop"))
1371 Label("end")
1372 RET()
1373 }
1374 //AVX 2
1375 {
1376 Comment("avx2XorSlice_64 will XOR in with out and store in out.")
1377 Comment("Processes 64 bytes/loop.")
1378 TEXT("avx2XorSlice_64", 0, fmt.Sprintf("func(in, out []byte)"))
1379 Pragma("noescape")
1380 src := Load(Param("in").Base(), GP64())
1381 dst := Load(Param("out").Base(), GP64())
1382 length := Load(Param("in").Len(), GP64())
1383 SHRQ(U8(6), length)
1384 srcX, dstX := YMM(), YMM()
1385 srcX2, dstX2 := YMM(), YMM()
1386 JZ(LabelRef("end"))
1387
1388 Label("loop")
1389 VMOVDQU(Mem{Base: src}, srcX)
1390 VMOVDQU(Mem{Base: src, Disp: 32}, srcX2)
1391 VMOVDQU(Mem{Base: dst}, dstX)
1392 VMOVDQU(Mem{Base: dst, Disp: 32}, dstX2)
1393 VPXOR(srcX, dstX, dstX)
1394 VPXOR(srcX2, dstX2, dstX2)
1395 VMOVDQU(dstX, Mem{Base: dst})
1396 VMOVDQU(dstX2, Mem{Base: dst, Disp: 32})
1397 ADDQ(U8(64), src)
1398 ADDQ(U8(64), dst)
1399 DECQ(length)
1400 JNZ(LabelRef("loop"))
1401
1402 Label("end")
1403 VZEROUPPER()
1404 RET()
1405 }
1406 }