| 1 |
package main |
| 2 |
|
| 3 |
import ( |
| 4 |
. "github.com/mmcloughlin/avo/build" |
| 5 |
. "github.com/mmcloughlin/avo/operand" |
| 6 |
. "github.com/mmcloughlin/avo/reg" |
| 7 |
. "github.com/zeebo/blake3/avo" |
| 8 |
) |
| 9 |
|
| 10 |
func HashF(c Ctx) { |
| 11 |
TEXT("HashF", 0, `func( |
| 12 |
input *[8192]byte, |
| 13 |
length uint64, |
| 14 |
counter uint64, |
| 15 |
flags uint32, |
| 16 |
key *[8]uint32, |
| 17 |
out *[32]uint32, |
| 18 |
chain *[8]uint32, |
| 19 |
)`) |
| 20 |
|
| 21 |
var ( |
| 22 |
input = Mem{Base: Load(Param("input"), GP64())} |
| 23 |
length = Load(Param("length"), GP64()).(GPVirtual) |
| 24 |
counter = Load(Param("counter"), GP64()).(GPVirtual) |
| 25 |
flags = Load(Param("flags"), GP32()).(GPVirtual) |
| 26 |
key = Mem{Base: Load(Param("key"), GP64())} |
| 27 |
out = Mem{Base: Load(Param("out"), GP64())} |
| 28 |
chain = Mem{Base: Load(Param("chain"), GP64())} |
| 29 |
) |
| 30 |
|
| 31 |
loop := GP64() |
| 32 |
chunks := GP64() |
| 33 |
blocks := GP64() |
| 34 |
stash := GP64() |
| 35 |
|
| 36 |
{ |
| 37 |
Comment("Allocate local space and align it") |
| 38 |
local := AllocLocal(roundSize + 32) |
| 39 |
LEAQ(local.Offset(31), stash) |
| 40 |
// TODO: avo improvement |
| 41 |
tmp := GP64() |
| 42 |
MOVQ(U64(31), tmp) |
| 43 |
NOTQ(tmp) |
| 44 |
ANDQ(tmp, stash) |
| 45 |
} |
| 46 |
|
| 47 |
alloc := NewAlloc(Mem{Base: stash}) |
| 48 |
defer alloc.Free() |
| 49 |
|
| 50 |
flags_mem := AllocLocal(8) |
| 51 |
counter_mem := AllocLocal(8) |
| 52 |
|
| 53 |
tmp := AllocLocal(32) |
| 54 |
ctr_lo_mem := AllocLocal(32) |
| 55 |
ctr_hi_mem := AllocLocal(32) |
| 56 |
msg := AllocLocal(32 * 16) |
| 57 |
|
| 58 |
var ( |
| 59 |
h_vecs []*Value |
| 60 |
h_regs []int |
| 61 |
vs []*Value |
| 62 |
iv []*Value |
| 63 |
ctr_low *Value |
| 64 |
ctr_hi *Value |
| 65 |
blen_vec *Value |
| 66 |
flags_vec *Value |
| 67 |
) |
| 68 |
|
| 69 |
h_vecs = alloc.ValuesWith(8, key) |
| 70 |
blen_vec = alloc.ValueFrom(c.BlockLen) |
| 71 |
flags_vec = alloc.ValueWith(flags_mem) |
| 72 |
iv = alloc.ValuesWith(4, c.IV) |
| 73 |
ctr_low = alloc.ValueFrom(ctr_lo_mem) |
| 74 |
ctr_hi = alloc.ValueFrom(ctr_hi_mem) |
| 75 |
|
| 76 |
{ |
| 77 |
Comment("Skip if the length is zero") |
| 78 |
XORQ(chunks, chunks) |
| 79 |
XORQ(blocks, blocks) |
| 80 |
TESTQ(length, length) |
| 81 |
JZ(LabelRef("skip_compute")) |
| 82 |
} |
| 83 |
|
| 84 |
{ |
| 85 |
Comment("Compute complete chunks and blocks") |
| 86 |
|
| 87 |
// chunks = (length - 1) / 1024 |
| 88 |
SUBQ(U8(1), length) |
| 89 |
MOVQ(length, chunks) |
| 90 |
SHRQ(U8(10), chunks) |
| 91 |
|
| 92 |
// blocks = (length - 1) % 1024 / 64 * 64 |
| 93 |
MOVQ(length, blocks) |
| 94 |
ANDQ(U32(960), blocks) |
| 95 |
} |
| 96 |
|
| 97 |
Label("skip_compute") |
| 98 |
|
| 99 |
{ |
| 100 |
Comment("Load some params into the stack (avo improvment?)") |
| 101 |
MOVL(flags, flags_mem) |
| 102 |
MOVQ(counter, counter_mem) |
| 103 |
} |
| 104 |
|
| 105 |
{ |
| 106 |
Comment("Load IV into vectors") |
| 107 |
h_regs = make([]int, 8) |
| 108 |
for i, v := range h_vecs { |
| 109 |
h_regs[i] = v.Reg() |
| 110 |
_ = v.Get() |
| 111 |
} |
| 112 |
} |
| 113 |
|
| 114 |
{ |
| 115 |
Comment("Build and store counter data on the stack") |
| 116 |
loadCounter(c, alloc, counter_mem, ctr_lo_mem, ctr_hi_mem) |
| 117 |
} |
| 118 |
|
| 119 |
{ |
| 120 |
Comment("Set up block flags and variables for iteration") |
| 121 |
XORQ(loop, loop) |
| 122 |
ORL(U8(flag_chunkStart), flags_mem) |
| 123 |
} |
| 124 |
|
| 125 |
Label("loop") |
| 126 |
|
| 127 |
{ |
| 128 |
Comment("Include end flags if last block") |
| 129 |
CMPQ(loop, U32(15*64)) |
| 130 |
JNE(LabelRef("round_setup")) |
| 131 |
ORL(U8(flag_chunkEnd), flags_mem) |
| 132 |
} |
| 133 |
|
| 134 |
Label("round_setup") |
| 135 |
|
| 136 |
{ |
| 137 |
Comment("Load and transpose message vectors") |
| 138 |
transposeMsg(c, alloc, loop, input, msg) |
| 139 |
} |
| 140 |
|
| 141 |
{ |
| 142 |
Comment("Load constants for the round") |
| 143 |
for _, v := range h_vecs { |
| 144 |
_ = v.Get() |
| 145 |
} |
| 146 |
_ = blen_vec.Get() |
| 147 |
_ = flags_vec.Get() |
| 148 |
for _, v := range iv { |
| 149 |
_ = v.Get() |
| 150 |
} |
| 151 |
_ = ctr_low.Get() |
| 152 |
_ = ctr_hi.Get() |
| 153 |
} |
| 154 |
|
| 155 |
{ |
| 156 |
Comment("Save state for partial chunk if necessary") |
| 157 |
CMPQ(loop, blocks) |
| 158 |
JNE(LabelRef("begin_rounds")) |
| 159 |
|
| 160 |
for i, v := range h_vecs { |
| 161 |
tmp32 := GP32() |
| 162 |
VMOVDQU(v.Get(), tmp) |
| 163 |
MOVL(tmp.Idx(chunks, 4), tmp32) |
| 164 |
MOVL(tmp32, chain.Offset(4*i)) |
| 165 |
} |
| 166 |
} |
| 167 |
|
| 168 |
Label("begin_rounds") |
| 169 |
|
| 170 |
{ |
| 171 |
Comment("Perform the rounds") |
| 172 |
|
| 173 |
vs = []*Value{ |
| 174 |
h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], |
| 175 |
h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], |
| 176 |
iv[0], iv[1], iv[2], iv[3], |
| 177 |
ctr_low, ctr_hi, blen_vec, flags_vec, |
| 178 |
} |
| 179 |
|
| 180 |
for r := 0; r < 7; r++ { |
| 181 |
Commentf("Round %d", r+1) |
| 182 |
roundF(c, alloc, vs, r, msg) |
| 183 |
} |
| 184 |
} |
| 185 |
|
| 186 |
{ |
| 187 |
Comment("Finalize rounds") |
| 188 |
finalizeRounds(alloc, vs, h_vecs, h_regs) |
| 189 |
} |
| 190 |
|
| 191 |
{ |
| 192 |
Comment("Fix up registers for next iteration") |
| 193 |
for i := 7; i >= 0; i-- { |
| 194 |
h_vecs[i].Become(h_regs[i]) |
| 195 |
} |
| 196 |
} |
| 197 |
|
| 198 |
{ |
| 199 |
Comment("If we have zero complete chunks, we're done") |
| 200 |
CMPQ(chunks, U8(0)) |
| 201 |
JNE(LabelRef("loop_trailer")) |
| 202 |
CMPQ(blocks, loop) |
| 203 |
JEQ(LabelRef("finalize")) |
| 204 |
} |
| 205 |
|
| 206 |
Label("loop_trailer") |
| 207 |
|
| 208 |
{ |
| 209 |
Comment("Increment, reset flags, and loop") |
| 210 |
CMPQ(loop, U32(15*64)) |
| 211 |
JEQ(LabelRef("finalize")) |
| 212 |
|
| 213 |
ADDQ(Imm(64), loop) |
| 214 |
MOVL(flags, flags_mem) |
| 215 |
JMP(LabelRef("loop")) |
| 216 |
} |
| 217 |
|
| 218 |
Label("finalize") |
| 219 |
|
| 220 |
{ |
| 221 |
Comment("Store result into out") |
| 222 |
for i, v := range h_vecs { |
| 223 |
VMOVDQU(v.Consume(), out.Offset(32*i)) |
| 224 |
} |
| 225 |
} |
| 226 |
|
| 227 |
VZEROUPPER() |
| 228 |
RET() |
| 229 |
} |
| 230 |
|
| 231 |
func roundF(c Ctx, alloc *Alloc, vs []*Value, r int, mp Mem) { |
| 232 |
round(c, alloc, vs, r, func(n int) Mem { |
| 233 |
return mp.Offset(n * 32) |
| 234 |
}) |
| 235 |
} |