Debian Codegraph @

simd/keccakf1600/internal/asm/src.go

raw
1 //go:generate go run src.go -out ../../f1600x4_amd64.s -stubs ../../f1600x4stubs_amd64.go -pkg keccakf1600
2
3 // AVX2 fourway parallelized Keccak-f[1600].
4
5 package main
6
7 import (
8 . "github.com/mmcloughlin/avo/build" // nolint:stylecheck,golint
9 . "github.com/mmcloughlin/avo/operand" // nolint:stylecheck,golint
10 )
11
12 // nolint:funlen
13 func main() {
14 ConstraintExpr("amd64,!purego")
15
16 // Must be called on 32 byte aligned memory.
17 TEXT("f1600x4AVX2", NOSPLIT, "func(state *uint64, rc *[24]uint64, turbo bool)")
18
19 Pragma("noescape")
20
21 statePtr := Load(Param("state"), GP64())
22 state := func(offset int) Op {
23 return Mem{Base: statePtr, Disp: 32 * offset}
24 }
25
26 rcPtr := Load(Param("rc"), GP64())
27
28 // We use the same approach as the normal Keccak-f[1600] implementation
29 // (in the internal/sha3 package): we group four rounds together into a
30 // super round. Thus we have six super rounds.
31 superRound := GP64()
32 MOVQ(U64(6), superRound) // count down.
33
34 turbo := Load(Param("turbo"), GP64())
35 TESTQ(turbo, turbo)
36 JZ(LabelRef("loop"))
37
38 MOVQ(U64(3), superRound) // Skip 3 * 4 = 12 rounds
39 ADDQ(Imm(8*12), rcPtr)
40
41 // XXX Because our AVX2 is significantly larger, it might better not
42 // to group four rounds together, but simply loop over the rounds
43 // themselves.
44
45 Label("loop")
46
47 for r := 0; r < 4; r++ {
48 // Compute parities: p[i] = a[i] ^ a[i + 5] ^ ... ^ a[i + 20].
49 p := []Op{YMM(), YMM(), YMM(), YMM(), YMM()}
50 for i := 0; i < 5; i++ {
51 VMOVDQA(state(i), p[i])
52 }
53 for j := 1; j < 5; j++ {
54 for i := 0; i < 5; i++ {
55 VPXOR(state(5*j+i), p[i], p[i])
56 }
57 }
58
59 // Rotate and xor parities: d[i] = rotate_left(p[i+1], 1) ^ p[i-1]
60 t := []Op{YMM(), YMM(), YMM(), YMM(), YMM()}
61 d := []Op{YMM(), YMM(), YMM(), YMM(), YMM()}
62 for i := 0; i < 5; i++ {
63 VPSLLQ(U8(1), p[(i+1)%5], t[i])
64 }
65 for i := 0; i < 5; i++ {
66 VPSRLQ(U8(63), p[(i+1)%5], d[i])
67 }
68 for i := 0; i < 5; i++ {
69 VPOR(t[i], d[i], d[i])
70 }
71 for i := 0; i < 5; i++ {
72 VPXOR(d[i], p[(i+4)%5], d[i])
73 }
74
75 // Rotation to use
76 rot := func(i, g int) int {
77 table := [][]int{
78 {0, 24, 18, 6, 12},
79 {7, 23, 2, 9, 22},
80 {1, 3, 17, 16, 20},
81 {13, 8, 4, 5, 15},
82 {19, 10, 21, 14, 11},
83 }
84 t := table[g][i]
85 return ((t + 1) * t / 2) % 64 // t'th triangle number
86 }
87
88 // Index into d to use
89 di := func(i, g int) int {
90 return (3*g + i) % 5
91 }
92
93 // Index into state to use
94 si := func(i, g, r int) int {
95 n := []int{6, 16, 11, 1}[r]
96 m := []int{10, 20, 15, 5}[r]
97 return (i*n + m*g) % 25
98 }
99
100 for g := 0; g < 5; g++ {
101 s := []Op{YMM(), YMM(), YMM(), YMM(), YMM()}
102
103 // Load the right five words from the state and XOR d into them.
104 for i := 0; i < 5; i++ {
105 VPXOR(state(si(di(i, g), g, r)), d[di(i, g)], s[i])
106 }
107
108 // Rotate each s[i] by the appropriate amount
109 for i := 0; i < 5; i++ {
110 if rot(i, g) != 0 {
111 VPSLLQ(U8(rot(i, g)), s[i], t[i])
112 }
113 }
114 for i := 0; i < 5; i++ {
115 if rot(i, g) != 0 {
116 VPSRLQ(U8(64-rot(i, g)), s[i], s[i])
117 }
118 }
119 for i := 0; i < 5; i++ {
120 if rot(i, g) != 0 {
121 VPOR(t[i], s[i], s[i])
122 }
123 }
124
125 // Compute the new words s[i] ^ (s[i+2] & ~s[i+1])
126 for i := 0; i < 5; i++ {
127 VPANDN(s[(i+2)%5], s[(i+1)%5], t[i])
128 }
129 for i := 0; i < 5; i++ {
130 VPXOR(s[i], t[i], t[i])
131 }
132
133 // Round constant
134 if g == 0 {
135 // Note that we move rcPtr by 8*4 bytes after each superround.
136 rc := YMM()
137 VPBROADCASTQ(Mem{Base: rcPtr, Disp: r * 8}, rc)
138 VPXOR(rc, t[0], t[0])
139 }
140
141 // Store back into state
142 for i := 0; i < 5; i++ {
143 VMOVDQA(t[i], state(si(i, g, r)))
144 }
145 }
146 }
147
148 ADDQ(Imm(8*4), rcPtr)
149 SUBQ(U32(1), superRound)
150 JNZ(LabelRef("loop"))
151
152 RET()
153
154 Generate()
155 }