| 1 |
package main |
| 2 |
|
| 3 |
import ( |
| 4 |
. "github.com/mmcloughlin/avo/build" |
| 5 |
. "github.com/mmcloughlin/avo/operand" |
| 6 |
. "github.com/mmcloughlin/avo/reg" |
| 7 |
. "github.com/zeebo/blake3/avo" |
| 8 |
) |
| 9 |
|
| 10 |
var msgSched = [7][16]int{ |
| 11 |
{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, |
| 12 |
{2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8}, |
| 13 |
{3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1}, |
| 14 |
{10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6}, |
| 15 |
{12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4}, |
| 16 |
{9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7}, |
| 17 |
{11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13}, |
| 18 |
} |
| 19 |
|
| 20 |
const roundSize = 32 |
| 21 |
|
| 22 |
const ( |
| 23 |
flag_chunkStart = 1 << 0 |
| 24 |
flag_chunkEnd = 1 << 1 |
| 25 |
flag_parent = 1 << 2 |
| 26 |
) |
| 27 |
|
| 28 |
func transpose(c Ctx, alloc *Alloc, vs []*Value) { |
| 29 |
L01, H01, L23, H23 := alloc.Value(), alloc.Value(), alloc.Value(), alloc.Value() |
| 30 |
L45, H45, L67, H67 := alloc.Value(), alloc.Value(), alloc.Value(), alloc.Value() |
| 31 |
|
| 32 |
VPUNPCKLDQ(vs[1].GetOp(), vs[0].Get(), L01.Get()) |
| 33 |
VPUNPCKHDQ(vs[1].ConsumeOp(), vs[0].Consume(), H01.Get()) |
| 34 |
VPUNPCKLDQ(vs[3].GetOp(), vs[2].Get(), L23.Get()) |
| 35 |
VPUNPCKHDQ(vs[3].ConsumeOp(), vs[2].Consume(), H23.Get()) |
| 36 |
VPUNPCKLDQ(vs[5].GetOp(), vs[4].Get(), L45.Get()) |
| 37 |
VPUNPCKHDQ(vs[5].ConsumeOp(), vs[4].Consume(), H45.Get()) |
| 38 |
VPUNPCKLDQ(vs[7].GetOp(), vs[6].Get(), L67.Get()) |
| 39 |
VPUNPCKHDQ(vs[7].ConsumeOp(), vs[6].Consume(), H67.Get()) |
| 40 |
|
| 41 |
LL0123, HL0123, LH0123, HH0123 := alloc.Value(), alloc.Value(), alloc.Value(), alloc.Value() |
| 42 |
LL4567, HL4567, LH4567, HH4567 := alloc.Value(), alloc.Value(), alloc.Value(), alloc.Value() |
| 43 |
|
| 44 |
VPUNPCKLQDQ(L23.GetOp(), L01.Get(), LL0123.Get()) |
| 45 |
VPUNPCKHQDQ(L23.ConsumeOp(), L01.Consume(), HL0123.Get()) |
| 46 |
VPUNPCKLQDQ(H23.GetOp(), H01.Get(), LH0123.Get()) |
| 47 |
VPUNPCKHQDQ(H23.ConsumeOp(), H01.Consume(), HH0123.Get()) |
| 48 |
VPUNPCKLQDQ(L67.GetOp(), L45.Get(), LL4567.Get()) |
| 49 |
VPUNPCKHQDQ(L67.ConsumeOp(), L45.Consume(), HL4567.Get()) |
| 50 |
VPUNPCKLQDQ(H67.GetOp(), H45.Get(), LH4567.Get()) |
| 51 |
VPUNPCKHQDQ(H67.ConsumeOp(), H45.Consume(), HH4567.Get()) |
| 52 |
|
| 53 |
vs[0], vs[1], vs[2], vs[3] = alloc.Value(), alloc.Value(), alloc.Value(), alloc.Value() |
| 54 |
vs[4], vs[5], vs[6], vs[7] = alloc.Value(), alloc.Value(), alloc.Value(), alloc.Value() |
| 55 |
|
| 56 |
VINSERTI128(Imm(1), LL4567.Get().(VecPhysical).AsX(), LL0123.Get(), vs[0].Get()) |
| 57 |
VPERM2I128(Imm(49), LL4567.Consume(), LL0123.Consume(), vs[4].Get()) |
| 58 |
VINSERTI128(Imm(1), HL4567.Get().(VecPhysical).AsX(), HL0123.Get(), vs[1].Get()) |
| 59 |
VPERM2I128(Imm(49), HL4567.Consume(), HL0123.Consume(), vs[5].Get()) |
| 60 |
VINSERTI128(Imm(1), LH4567.Get().(VecPhysical).AsX(), LH0123.Get(), vs[2].Get()) |
| 61 |
VPERM2I128(Imm(49), LH4567.Consume(), LH0123.Consume(), vs[6].Get()) |
| 62 |
VINSERTI128(Imm(1), HH4567.Get().(VecPhysical).AsX(), HH0123.Get(), vs[3].Get()) |
| 63 |
VPERM2I128(Imm(49), HH4567.Consume(), HH0123.Consume(), vs[7].Get()) |
| 64 |
} |
| 65 |
|
| 66 |
func transposeMsg(c Ctx, alloc *Alloc, block GPVirtual, input, msg Mem) { |
| 67 |
for j := 0; j < 2; j++ { |
| 68 |
vs := alloc.Values(8) |
| 69 |
for i, v := range vs { |
| 70 |
VMOVDQU(input.Offset(1024*i+32*j).Idx(block, 1), v.Get()) |
| 71 |
} |
| 72 |
transpose(c, alloc, vs) |
| 73 |
for i, v := range vs { |
| 74 |
VMOVDQU(v.Consume(), msg.Offset(32*i+256*j)) |
| 75 |
} |
| 76 |
} |
| 77 |
} |
| 78 |
|
| 79 |
func transposeMsgN(c Ctx, alloc *Alloc, block GPVirtual, input, msg Mem, j int) { |
| 80 |
vs := alloc.Values(8) |
| 81 |
for i, v := range vs { |
| 82 |
VMOVDQU(input.Offset(1024*i+32*j).Idx(block, 1), v.Get()) |
| 83 |
} |
| 84 |
transpose(c, alloc, vs) |
| 85 |
for i, v := range vs { |
| 86 |
VMOVDQU(v.Consume(), msg.Offset(32*i+256*j)) |
| 87 |
} |
| 88 |
} |
| 89 |
|
| 90 |
func loadCounter(c Ctx, alloc *Alloc, mem, lo_mem, hi_mem Mem) { |
| 91 |
ctr0, ctr1 := alloc.Value(), alloc.Value() |
| 92 |
VPBROADCASTQ(mem, ctr0.Get()) |
| 93 |
VPADDQ(c.Counter, ctr0.Get(), ctr0.Get()) |
| 94 |
VPBROADCASTQ(mem, ctr1.Get()) |
| 95 |
VPADDQ(c.Counter.Offset(32), ctr1.Get(), ctr1.Get()) |
| 96 |
|
| 97 |
L, H := alloc.Value(), alloc.Value() |
| 98 |
VPUNPCKLDQ(ctr1.GetOp(), ctr0.Get(), L.Get()) |
| 99 |
VPUNPCKHDQ(ctr1.ConsumeOp(), ctr0.Consume(), H.Get()) |
| 100 |
|
| 101 |
LLH, HLH := alloc.Value(), alloc.Value() |
| 102 |
VPUNPCKLDQ(H.GetOp(), L.Get(), LLH.Get()) |
| 103 |
VPUNPCKHDQ(H.ConsumeOp(), L.Consume(), HLH.Get()) |
| 104 |
|
| 105 |
ctrl, ctrh := alloc.Value(), alloc.Value() |
| 106 |
VPERMQ(U8(0b11_01_10_00), LLH.ConsumeOp(), ctrl.Get()) |
| 107 |
VPERMQ(U8(0b11_01_10_00), HLH.ConsumeOp(), ctrh.Get()) |
| 108 |
|
| 109 |
VMOVDQU(ctrl.Consume(), lo_mem) |
| 110 |
VMOVDQU(ctrh.Consume(), hi_mem) |
| 111 |
} |
| 112 |
|
| 113 |
func finalizeRounds(alloc *Alloc, vs, h_vecs []*Value, h_regs []int) { |
| 114 |
finalized := [8]bool{} |
| 115 |
|
| 116 |
finalize: |
| 117 |
for j := 0; j < 8; j++ { |
| 118 |
free := alloc.FreeReg() |
| 119 |
for i, reg := range h_regs { |
| 120 |
if reg == free && !finalized[i] { |
| 121 |
h_vecs[i] = xorb(alloc, vs[i], vs[8+i]) |
| 122 |
finalized[i] = true |
| 123 |
continue finalize |
| 124 |
} |
| 125 |
} |
| 126 |
|
| 127 |
for i, f := range finalized[:] { |
| 128 |
if !f { |
| 129 |
h_vecs[i] = xorb(alloc, vs[i], vs[8+i]) |
| 130 |
finalized[i] = true |
| 131 |
continue finalize |
| 132 |
} |
| 133 |
} |
| 134 |
} |
| 135 |
} |
| 136 |
|
| 137 |
func round(c Ctx, alloc *Alloc, vs []*Value, r int, m func(n int) Mem) { |
| 138 |
ms := func(ns ...int) (o []Mem) { |
| 139 |
for _, n := range ns { |
| 140 |
o = append(o, m(msgSched[r][n])) |
| 141 |
} |
| 142 |
return o |
| 143 |
} |
| 144 |
|
| 145 |
partials := []struct { |
| 146 |
ms []Mem |
| 147 |
tab Mem |
| 148 |
rot int |
| 149 |
}{ |
| 150 |
{ms(0, 2, 4, 6), c.Rot16, 12}, |
| 151 |
{ms(1, 3, 5, 7), c.Rot8, 7}, |
| 152 |
{ms(8, 10, 12, 14), c.Rot16, 12}, |
| 153 |
{ms(9, 11, 13, 15), c.Rot8, 7}, |
| 154 |
} |
| 155 |
|
| 156 |
for i, p := range partials { |
| 157 |
addms(alloc, p.ms, vs[0:4]) |
| 158 |
|
| 159 |
tab := alloc.ValueFrom(p.tab) |
| 160 |
for j := 0; j < 4; j++ { |
| 161 |
vs[0+j] = add(alloc, vs[4+j], vs[0+j]) |
| 162 |
vs[12+j] = xor(alloc, vs[0+j], vs[12+j]) |
| 163 |
vs[12+j] = rotTv(alloc, tab, vs[12+j]) |
| 164 |
} |
| 165 |
tab.Free() |
| 166 |
|
| 167 |
for j := 0; j < 4; j++ { |
| 168 |
vs[8+j] = add(alloc, vs[12+j], vs[8+j]) |
| 169 |
vs[4+j] = xor(alloc, vs[8+j], vs[4+j]) |
| 170 |
} |
| 171 |
|
| 172 |
rotNs(alloc, p.rot, vs[4:8]) |
| 173 |
|
| 174 |
// roll the blocks |
| 175 |
if i == 1 { |
| 176 |
vs[4], vs[5], vs[6], vs[7] = vs[5], vs[6], vs[7], vs[4] |
| 177 |
vs[8], vs[9], vs[10], vs[11] = vs[10], vs[11], vs[8], vs[9] |
| 178 |
vs[12], vs[13], vs[14], vs[15] = vs[15], vs[12], vs[13], vs[14] |
| 179 |
} else if i == 3 { |
| 180 |
vs[4], vs[5], vs[6], vs[7] = vs[7], vs[4], vs[5], vs[6] |
| 181 |
vs[8], vs[9], vs[10], vs[11] = vs[10], vs[11], vs[8], vs[9] |
| 182 |
vs[12], vs[13], vs[14], vs[15] = vs[13], vs[14], vs[15], vs[12] |
| 183 |
} |
| 184 |
} |
| 185 |
} |
| 186 |
|
| 187 |
func addm(alloc *Alloc, mp Mem, a *Value) *Value { |
| 188 |
o := alloc.Value() |
| 189 |
VPADDD(mp, a.Consume(), o.Get()) |
| 190 |
return o |
| 191 |
} |
| 192 |
|
| 193 |
func addms(alloc *Alloc, mps []Mem, as []*Value) { |
| 194 |
for i, a := range as { |
| 195 |
as[i] = addm(alloc, mps[i], a) |
| 196 |
} |
| 197 |
} |
| 198 |
|
| 199 |
func add(alloc *Alloc, a, b *Value) *Value { |
| 200 |
o := alloc.Value() |
| 201 |
VPADDD(a.Get(), b.Consume(), o.Get()) |
| 202 |
return o |
| 203 |
} |
| 204 |
|
| 205 |
func xor(alloc *Alloc, a, b *Value) *Value { |
| 206 |
o := alloc.Value() |
| 207 |
VPXOR(a.Get(), b.Consume(), o.Get()) |
| 208 |
return o |
| 209 |
} |
| 210 |
|
| 211 |
func xorb(alloc *Alloc, a, b *Value) *Value { |
| 212 |
o := alloc.Value() |
| 213 |
switch { |
| 214 |
case a.HasReg(): |
| 215 |
VPXOR(b.ConsumeOp(), a.Consume(), o.Get()) |
| 216 |
case b.HasReg(): |
| 217 |
VPXOR(a.ConsumeOp(), b.Consume(), o.Get()) |
| 218 |
default: |
| 219 |
VPXOR(a.ConsumeOp(), b.Consume(), o.Get()) |
| 220 |
} |
| 221 |
return o |
| 222 |
} |
| 223 |
|
| 224 |
func rotN(alloc *Alloc, n int, a *Value) *Value { |
| 225 |
tmp, o := alloc.Value(), alloc.Value() |
| 226 |
VPSRLD(U8(n), a.Get(), tmp.Get()) |
| 227 |
VPSLLD(U8(32-n), a.Get(), a.Get()) |
| 228 |
VPOR(tmp.ConsumeOp(), a.Consume(), o.Get()) |
| 229 |
return o |
| 230 |
} |
| 231 |
|
| 232 |
func rotNs(alloc *Alloc, n int, as []*Value) { |
| 233 |
for i, a := range as { |
| 234 |
as[i] = rotN(alloc, n, a) |
| 235 |
} |
| 236 |
} |
| 237 |
|
| 238 |
func rotTv(alloc *Alloc, tab, a *Value) *Value { |
| 239 |
o := alloc.Value() |
| 240 |
VPSHUFB(tab.GetOp(), a.Consume(), o.Get()) |
| 241 |
return o |
| 242 |
} |