Debian Codegraph @

avo/avx2/hashF.go

raw
1 package main
2
3 import (
4 . "github.com/mmcloughlin/avo/build"
5 . "github.com/mmcloughlin/avo/operand"
6 . "github.com/mmcloughlin/avo/reg"
7 . "github.com/zeebo/blake3/avo"
8 )
9
10 func HashF(c Ctx) {
11 TEXT("HashF", 0, `func(
12 input *[8192]byte,
13 length uint64,
14 counter uint64,
15 flags uint32,
16 key *[8]uint32,
17 out *[32]uint32,
18 chain *[8]uint32,
19 )`)
20
21 var (
22 input = Mem{Base: Load(Param("input"), GP64())}
23 length = Load(Param("length"), GP64()).(GPVirtual)
24 counter = Load(Param("counter"), GP64()).(GPVirtual)
25 flags = Load(Param("flags"), GP32()).(GPVirtual)
26 key = Mem{Base: Load(Param("key"), GP64())}
27 out = Mem{Base: Load(Param("out"), GP64())}
28 chain = Mem{Base: Load(Param("chain"), GP64())}
29 )
30
31 loop := GP64()
32 chunks := GP64()
33 blocks := GP64()
34 stash := GP64()
35
36 {
37 Comment("Allocate local space and align it")
38 local := AllocLocal(roundSize + 32)
39 LEAQ(local.Offset(31), stash)
40 // TODO: avo improvement
41 tmp := GP64()
42 MOVQ(U64(31), tmp)
43 NOTQ(tmp)
44 ANDQ(tmp, stash)
45 }
46
47 alloc := NewAlloc(Mem{Base: stash})
48 defer alloc.Free()
49
50 flags_mem := AllocLocal(8)
51 counter_mem := AllocLocal(8)
52
53 tmp := AllocLocal(32)
54 ctr_lo_mem := AllocLocal(32)
55 ctr_hi_mem := AllocLocal(32)
56 msg := AllocLocal(32 * 16)
57
58 var (
59 h_vecs []*Value
60 h_regs []int
61 vs []*Value
62 iv []*Value
63 ctr_low *Value
64 ctr_hi *Value
65 blen_vec *Value
66 flags_vec *Value
67 )
68
69 h_vecs = alloc.ValuesWith(8, key)
70 blen_vec = alloc.ValueFrom(c.BlockLen)
71 flags_vec = alloc.ValueWith(flags_mem)
72 iv = alloc.ValuesWith(4, c.IV)
73 ctr_low = alloc.ValueFrom(ctr_lo_mem)
74 ctr_hi = alloc.ValueFrom(ctr_hi_mem)
75
76 {
77 Comment("Skip if the length is zero")
78 XORQ(chunks, chunks)
79 XORQ(blocks, blocks)
80 TESTQ(length, length)
81 JZ(LabelRef("skip_compute"))
82 }
83
84 {
85 Comment("Compute complete chunks and blocks")
86
87 // chunks = (length - 1) / 1024
88 SUBQ(U8(1), length)
89 MOVQ(length, chunks)
90 SHRQ(U8(10), chunks)
91
92 // blocks = (length - 1) % 1024 / 64 * 64
93 MOVQ(length, blocks)
94 ANDQ(U32(960), blocks)
95 }
96
97 Label("skip_compute")
98
99 {
100 Comment("Load some params into the stack (avo improvment?)")
101 MOVL(flags, flags_mem)
102 MOVQ(counter, counter_mem)
103 }
104
105 {
106 Comment("Load IV into vectors")
107 h_regs = make([]int, 8)
108 for i, v := range h_vecs {
109 h_regs[i] = v.Reg()
110 _ = v.Get()
111 }
112 }
113
114 {
115 Comment("Build and store counter data on the stack")
116 loadCounter(c, alloc, counter_mem, ctr_lo_mem, ctr_hi_mem)
117 }
118
119 {
120 Comment("Set up block flags and variables for iteration")
121 XORQ(loop, loop)
122 ORL(U8(flag_chunkStart), flags_mem)
123 }
124
125 Label("loop")
126
127 {
128 Comment("Include end flags if last block")
129 CMPQ(loop, U32(15*64))
130 JNE(LabelRef("round_setup"))
131 ORL(U8(flag_chunkEnd), flags_mem)
132 }
133
134 Label("round_setup")
135
136 {
137 Comment("Load and transpose message vectors")
138 transposeMsg(c, alloc, loop, input, msg)
139 }
140
141 {
142 Comment("Load constants for the round")
143 for _, v := range h_vecs {
144 _ = v.Get()
145 }
146 _ = blen_vec.Get()
147 _ = flags_vec.Get()
148 for _, v := range iv {
149 _ = v.Get()
150 }
151 _ = ctr_low.Get()
152 _ = ctr_hi.Get()
153 }
154
155 {
156 Comment("Save state for partial chunk if necessary")
157 CMPQ(loop, blocks)
158 JNE(LabelRef("begin_rounds"))
159
160 for i, v := range h_vecs {
161 tmp32 := GP32()
162 VMOVDQU(v.Get(), tmp)
163 MOVL(tmp.Idx(chunks, 4), tmp32)
164 MOVL(tmp32, chain.Offset(4*i))
165 }
166 }
167
168 Label("begin_rounds")
169
170 {
171 Comment("Perform the rounds")
172
173 vs = []*Value{
174 h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3],
175 h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7],
176 iv[0], iv[1], iv[2], iv[3],
177 ctr_low, ctr_hi, blen_vec, flags_vec,
178 }
179
180 for r := 0; r < 7; r++ {
181 Commentf("Round %d", r+1)
182 roundF(c, alloc, vs, r, msg)
183 }
184 }
185
186 {
187 Comment("Finalize rounds")
188 finalizeRounds(alloc, vs, h_vecs, h_regs)
189 }
190
191 {
192 Comment("Fix up registers for next iteration")
193 for i := 7; i >= 0; i-- {
194 h_vecs[i].Become(h_regs[i])
195 }
196 }
197
198 {
199 Comment("If we have zero complete chunks, we're done")
200 CMPQ(chunks, U8(0))
201 JNE(LabelRef("loop_trailer"))
202 CMPQ(blocks, loop)
203 JEQ(LabelRef("finalize"))
204 }
205
206 Label("loop_trailer")
207
208 {
209 Comment("Increment, reset flags, and loop")
210 CMPQ(loop, U32(15*64))
211 JEQ(LabelRef("finalize"))
212
213 ADDQ(Imm(64), loop)
214 MOVL(flags, flags_mem)
215 JMP(LabelRef("loop"))
216 }
217
218 Label("finalize")
219
220 {
221 Comment("Store result into out")
222 for i, v := range h_vecs {
223 VMOVDQU(v.Consume(), out.Offset(32*i))
224 }
225 }
226
227 VZEROUPPER()
228 RET()
229 }
230
231 func roundF(c Ctx, alloc *Alloc, vs []*Value, r int, mp Mem) {
232 round(c, alloc, vs, r, func(n int) Mem {
233 return mp.Offset(n * 32)
234 })
235 }