Debian Codegraph @

avo/avx2/common.go

raw
1 package main
2
3 import (
4 . "github.com/mmcloughlin/avo/build"
5 . "github.com/mmcloughlin/avo/operand"
6 . "github.com/mmcloughlin/avo/reg"
7 . "github.com/zeebo/blake3/avo"
8 )
9
10 var msgSched = [7][16]int{
11 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
12 {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8},
13 {3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1},
14 {10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6},
15 {12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4},
16 {9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7},
17 {11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13},
18 }
19
20 const roundSize = 32
21
22 const (
23 flag_chunkStart = 1 << 0
24 flag_chunkEnd = 1 << 1
25 flag_parent = 1 << 2
26 )
27
28 func transpose(c Ctx, alloc *Alloc, vs []*Value) {
29 L01, H01, L23, H23 := alloc.Value(), alloc.Value(), alloc.Value(), alloc.Value()
30 L45, H45, L67, H67 := alloc.Value(), alloc.Value(), alloc.Value(), alloc.Value()
31
32 VPUNPCKLDQ(vs[1].GetOp(), vs[0].Get(), L01.Get())
33 VPUNPCKHDQ(vs[1].ConsumeOp(), vs[0].Consume(), H01.Get())
34 VPUNPCKLDQ(vs[3].GetOp(), vs[2].Get(), L23.Get())
35 VPUNPCKHDQ(vs[3].ConsumeOp(), vs[2].Consume(), H23.Get())
36 VPUNPCKLDQ(vs[5].GetOp(), vs[4].Get(), L45.Get())
37 VPUNPCKHDQ(vs[5].ConsumeOp(), vs[4].Consume(), H45.Get())
38 VPUNPCKLDQ(vs[7].GetOp(), vs[6].Get(), L67.Get())
39 VPUNPCKHDQ(vs[7].ConsumeOp(), vs[6].Consume(), H67.Get())
40
41 LL0123, HL0123, LH0123, HH0123 := alloc.Value(), alloc.Value(), alloc.Value(), alloc.Value()
42 LL4567, HL4567, LH4567, HH4567 := alloc.Value(), alloc.Value(), alloc.Value(), alloc.Value()
43
44 VPUNPCKLQDQ(L23.GetOp(), L01.Get(), LL0123.Get())
45 VPUNPCKHQDQ(L23.ConsumeOp(), L01.Consume(), HL0123.Get())
46 VPUNPCKLQDQ(H23.GetOp(), H01.Get(), LH0123.Get())
47 VPUNPCKHQDQ(H23.ConsumeOp(), H01.Consume(), HH0123.Get())
48 VPUNPCKLQDQ(L67.GetOp(), L45.Get(), LL4567.Get())
49 VPUNPCKHQDQ(L67.ConsumeOp(), L45.Consume(), HL4567.Get())
50 VPUNPCKLQDQ(H67.GetOp(), H45.Get(), LH4567.Get())
51 VPUNPCKHQDQ(H67.ConsumeOp(), H45.Consume(), HH4567.Get())
52
53 vs[0], vs[1], vs[2], vs[3] = alloc.Value(), alloc.Value(), alloc.Value(), alloc.Value()
54 vs[4], vs[5], vs[6], vs[7] = alloc.Value(), alloc.Value(), alloc.Value(), alloc.Value()
55
56 VINSERTI128(Imm(1), LL4567.Get().(VecPhysical).AsX(), LL0123.Get(), vs[0].Get())
57 VPERM2I128(Imm(49), LL4567.Consume(), LL0123.Consume(), vs[4].Get())
58 VINSERTI128(Imm(1), HL4567.Get().(VecPhysical).AsX(), HL0123.Get(), vs[1].Get())
59 VPERM2I128(Imm(49), HL4567.Consume(), HL0123.Consume(), vs[5].Get())
60 VINSERTI128(Imm(1), LH4567.Get().(VecPhysical).AsX(), LH0123.Get(), vs[2].Get())
61 VPERM2I128(Imm(49), LH4567.Consume(), LH0123.Consume(), vs[6].Get())
62 VINSERTI128(Imm(1), HH4567.Get().(VecPhysical).AsX(), HH0123.Get(), vs[3].Get())
63 VPERM2I128(Imm(49), HH4567.Consume(), HH0123.Consume(), vs[7].Get())
64 }
65
66 func transposeMsg(c Ctx, alloc *Alloc, block GPVirtual, input, msg Mem) {
67 for j := 0; j < 2; j++ {
68 vs := alloc.Values(8)
69 for i, v := range vs {
70 VMOVDQU(input.Offset(1024*i+32*j).Idx(block, 1), v.Get())
71 }
72 transpose(c, alloc, vs)
73 for i, v := range vs {
74 VMOVDQU(v.Consume(), msg.Offset(32*i+256*j))
75 }
76 }
77 }
78
79 func transposeMsgN(c Ctx, alloc *Alloc, block GPVirtual, input, msg Mem, j int) {
80 vs := alloc.Values(8)
81 for i, v := range vs {
82 VMOVDQU(input.Offset(1024*i+32*j).Idx(block, 1), v.Get())
83 }
84 transpose(c, alloc, vs)
85 for i, v := range vs {
86 VMOVDQU(v.Consume(), msg.Offset(32*i+256*j))
87 }
88 }
89
90 func loadCounter(c Ctx, alloc *Alloc, mem, lo_mem, hi_mem Mem) {
91 ctr0, ctr1 := alloc.Value(), alloc.Value()
92 VPBROADCASTQ(mem, ctr0.Get())
93 VPADDQ(c.Counter, ctr0.Get(), ctr0.Get())
94 VPBROADCASTQ(mem, ctr1.Get())
95 VPADDQ(c.Counter.Offset(32), ctr1.Get(), ctr1.Get())
96
97 L, H := alloc.Value(), alloc.Value()
98 VPUNPCKLDQ(ctr1.GetOp(), ctr0.Get(), L.Get())
99 VPUNPCKHDQ(ctr1.ConsumeOp(), ctr0.Consume(), H.Get())
100
101 LLH, HLH := alloc.Value(), alloc.Value()
102 VPUNPCKLDQ(H.GetOp(), L.Get(), LLH.Get())
103 VPUNPCKHDQ(H.ConsumeOp(), L.Consume(), HLH.Get())
104
105 ctrl, ctrh := alloc.Value(), alloc.Value()
106 VPERMQ(U8(0b11_01_10_00), LLH.ConsumeOp(), ctrl.Get())
107 VPERMQ(U8(0b11_01_10_00), HLH.ConsumeOp(), ctrh.Get())
108
109 VMOVDQU(ctrl.Consume(), lo_mem)
110 VMOVDQU(ctrh.Consume(), hi_mem)
111 }
112
113 func finalizeRounds(alloc *Alloc, vs, h_vecs []*Value, h_regs []int) {
114 finalized := [8]bool{}
115
116 finalize:
117 for j := 0; j < 8; j++ {
118 free := alloc.FreeReg()
119 for i, reg := range h_regs {
120 if reg == free && !finalized[i] {
121 h_vecs[i] = xorb(alloc, vs[i], vs[8+i])
122 finalized[i] = true
123 continue finalize
124 }
125 }
126
127 for i, f := range finalized[:] {
128 if !f {
129 h_vecs[i] = xorb(alloc, vs[i], vs[8+i])
130 finalized[i] = true
131 continue finalize
132 }
133 }
134 }
135 }
136
137 func round(c Ctx, alloc *Alloc, vs []*Value, r int, m func(n int) Mem) {
138 ms := func(ns ...int) (o []Mem) {
139 for _, n := range ns {
140 o = append(o, m(msgSched[r][n]))
141 }
142 return o
143 }
144
145 partials := []struct {
146 ms []Mem
147 tab Mem
148 rot int
149 }{
150 {ms(0, 2, 4, 6), c.Rot16, 12},
151 {ms(1, 3, 5, 7), c.Rot8, 7},
152 {ms(8, 10, 12, 14), c.Rot16, 12},
153 {ms(9, 11, 13, 15), c.Rot8, 7},
154 }
155
156 for i, p := range partials {
157 addms(alloc, p.ms, vs[0:4])
158
159 tab := alloc.ValueFrom(p.tab)
160 for j := 0; j < 4; j++ {
161 vs[0+j] = add(alloc, vs[4+j], vs[0+j])
162 vs[12+j] = xor(alloc, vs[0+j], vs[12+j])
163 vs[12+j] = rotTv(alloc, tab, vs[12+j])
164 }
165 tab.Free()
166
167 for j := 0; j < 4; j++ {
168 vs[8+j] = add(alloc, vs[12+j], vs[8+j])
169 vs[4+j] = xor(alloc, vs[8+j], vs[4+j])
170 }
171
172 rotNs(alloc, p.rot, vs[4:8])
173
174 // roll the blocks
175 if i == 1 {
176 vs[4], vs[5], vs[6], vs[7] = vs[5], vs[6], vs[7], vs[4]
177 vs[8], vs[9], vs[10], vs[11] = vs[10], vs[11], vs[8], vs[9]
178 vs[12], vs[13], vs[14], vs[15] = vs[15], vs[12], vs[13], vs[14]
179 } else if i == 3 {
180 vs[4], vs[5], vs[6], vs[7] = vs[7], vs[4], vs[5], vs[6]
181 vs[8], vs[9], vs[10], vs[11] = vs[10], vs[11], vs[8], vs[9]
182 vs[12], vs[13], vs[14], vs[15] = vs[13], vs[14], vs[15], vs[12]
183 }
184 }
185 }
186
187 func addm(alloc *Alloc, mp Mem, a *Value) *Value {
188 o := alloc.Value()
189 VPADDD(mp, a.Consume(), o.Get())
190 return o
191 }
192
193 func addms(alloc *Alloc, mps []Mem, as []*Value) {
194 for i, a := range as {
195 as[i] = addm(alloc, mps[i], a)
196 }
197 }
198
199 func add(alloc *Alloc, a, b *Value) *Value {
200 o := alloc.Value()
201 VPADDD(a.Get(), b.Consume(), o.Get())
202 return o
203 }
204
205 func xor(alloc *Alloc, a, b *Value) *Value {
206 o := alloc.Value()
207 VPXOR(a.Get(), b.Consume(), o.Get())
208 return o
209 }
210
211 func xorb(alloc *Alloc, a, b *Value) *Value {
212 o := alloc.Value()
213 switch {
214 case a.HasReg():
215 VPXOR(b.ConsumeOp(), a.Consume(), o.Get())
216 case b.HasReg():
217 VPXOR(a.ConsumeOp(), b.Consume(), o.Get())
218 default:
219 VPXOR(a.ConsumeOp(), b.Consume(), o.Get())
220 }
221 return o
222 }
223
224 func rotN(alloc *Alloc, n int, a *Value) *Value {
225 tmp, o := alloc.Value(), alloc.Value()
226 VPSRLD(U8(n), a.Get(), tmp.Get())
227 VPSLLD(U8(32-n), a.Get(), a.Get())
228 VPOR(tmp.ConsumeOp(), a.Consume(), o.Get())
229 return o
230 }
231
232 func rotNs(alloc *Alloc, n int, as []*Value) {
233 for i, a := range as {
234 as[i] = rotN(alloc, n, a)
235 }
236 }
237
238 func rotTv(alloc *Alloc, tab, a *Value) *Value {
239 o := alloc.Value()
240 VPSHUFB(tab.GetOp(), a.Consume(), o.Get())
241 return o
242 }