Debian Codegraph @

blake2b/_asm/standard/blake2b_amd64_asm.go

raw
1 // Copyright 2024 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 package main
6
7 import (
8 _ "github.com/go-crypt/x/blake2b"
9 . "github.com/mmcloughlin/avo/build"
10 . "github.com/mmcloughlin/avo/operand"
11 . "github.com/mmcloughlin/avo/reg"
12 )
13
14 //go:generate go run . -out ../../blake2b_amd64.s -pkg blake2b
15
16 const ThatPeskyUnicodeDot = "\u00b7"
17
18 var iv0_DATA_ptr, iv1_DATA_ptr, iv2_DATA_ptr, iv3_DATA_ptr, c40_DATA_ptr, c48_DATA_ptr *Mem
19
20 func main() {
21 Package("github.com/go-crypt/x/blake2b")
22 ConstraintExpr("amd64,gc,!purego")
23 hashBlocksSSE4()
24 Generate()
25 }
26
27 func SHUFFLE(v2, v3, v4, v5, v6, v7, t1, t2 VecPhysical) {
28 MOVO(v4, t1)
29 MOVO(v5, v4)
30 MOVO(t1, v5)
31 MOVO(v6, t1)
32 PUNPCKLQDQ(v6, t2)
33 PUNPCKHQDQ(v7, v6)
34 PUNPCKHQDQ(t2, v6)
35 PUNPCKLQDQ(v7, t2)
36 MOVO(t1, v7)
37 MOVO(v2, t1)
38 PUNPCKHQDQ(t2, v7)
39 PUNPCKLQDQ(v3, t2)
40 PUNPCKHQDQ(t2, v2)
41 PUNPCKLQDQ(t1, t2)
42 PUNPCKHQDQ(t2, v3)
43 }
44
45 func SHUFFLE_INV(v2, v3, v4, v5, v6, v7, t1, t2 VecPhysical) {
46 MOVO(v4, t1)
47 MOVO(v5, v4)
48 MOVO(t1, v5)
49 MOVO(v2, t1)
50 PUNPCKLQDQ(v2, t2)
51 PUNPCKHQDQ(v3, v2)
52 PUNPCKHQDQ(t2, v2)
53 PUNPCKLQDQ(v3, t2)
54 MOVO(t1, v3)
55 MOVO(v6, t1)
56 PUNPCKHQDQ(t2, v3)
57 PUNPCKLQDQ(v7, t2)
58 PUNPCKHQDQ(t2, v6)
59 PUNPCKLQDQ(t1, t2)
60 PUNPCKHQDQ(t2, v7)
61 }
62
63 func HALF_ROUND(v0, v1, v2, v3, v4, v5, v6, v7 VecPhysical, m0, m1, m2, m3 Op, t0, c40, c48 VecPhysical) {
64 PADDQ(m0, v0)
65 PADDQ(m1, v1)
66 PADDQ(v2, v0)
67 PADDQ(v3, v1)
68 PXOR(v0, v6)
69 PXOR(v1, v7)
70 PSHUFD(Imm(0xB1), v6, v6)
71 PSHUFD(Imm(0xB1), v7, v7)
72 PADDQ(v6, v4)
73 PADDQ(v7, v5)
74 PXOR(v4, v2)
75 PXOR(v5, v3)
76 PSHUFB(c40, v2)
77 PSHUFB(c40, v3)
78 PADDQ(m2, v0)
79 PADDQ(m3, v1)
80 PADDQ(v2, v0)
81 PADDQ(v3, v1)
82 PXOR(v0, v6)
83 PXOR(v1, v7)
84 PSHUFB(c48, v6)
85 PSHUFB(c48, v7)
86 PADDQ(v6, v4)
87 PADDQ(v7, v5)
88 PXOR(v4, v2)
89 PXOR(v5, v3)
90 MOVOU(v2, t0)
91 PADDQ(v2, t0)
92 PSRLQ(Imm(63), v2)
93 PXOR(t0, v2)
94 MOVOU(v3, t0)
95 PADDQ(v3, t0)
96 PSRLQ(Imm(63), v3)
97 PXOR(t0, v3)
98 }
99
100 func LOAD_MSG(m0, m1, m2, m3 VecPhysical, src GPPhysical, i0, i1, i2, i3, i4, i5, i6, i7 int) {
101 MOVQ(Mem{Base: src}.Offset(i0*8), m0)
102 PINSRQ(Imm(1), Mem{Base: src}.Offset(i1*8), m0)
103 MOVQ(Mem{Base: src}.Offset(i2*8), m1)
104 PINSRQ(Imm(1), Mem{Base: src}.Offset(i3*8), m1)
105 MOVQ(Mem{Base: src}.Offset(i4*8), m2)
106 PINSRQ(Imm(1), Mem{Base: src}.Offset(i5*8), m2)
107 MOVQ(Mem{Base: src}.Offset(i6*8), m3)
108 PINSRQ(Imm(1), Mem{Base: src}.Offset(i7*8), m3)
109 }
110
111 func hashBlocksSSE4() {
112 Implement("hashBlocksSSE4")
113 Attributes(4)
114 AllocLocal(288) // frame size = 272 + 16 byte alignment
115
116 Load(Param("h"), RAX)
117 Load(Param("c"), RBX)
118 Load(Param("flag"), RCX)
119 Load(Param("blocks").Base(), RSI)
120 Load(Param("blocks").Len(), RDI)
121
122 MOVQ(RSP, R10)
123 ADDQ(Imm(15), R10)
124 ANDQ(I32(-16), R10)
125
126 iv3 := iv3_DATA()
127 MOVOU(iv3, X0)
128 MOVO(X0, Mem{Base: R10}.Offset(0))
129 XORQ(RCX, Mem{Base: R10}.Offset(0)) // 0(R10) = ·iv3 ^ (CX || 0)
130
131 c40 := c40_DATA()
132 c48 := c48_DATA()
133 MOVOU(c40, X13)
134 MOVOU(c48, X14)
135
136 MOVOU(Mem{Base: AX}.Offset(0), X12)
137 MOVOU(Mem{Base: AX}.Offset(16), X15)
138
139 MOVQ(Mem{Base: BX}.Offset(0), R8)
140 MOVQ(Mem{Base: BX}.Offset(8), R9)
141
142 Label("loop")
143 ADDQ(Imm(128), R8)
144 CMPQ(R8, Imm(128))
145 JGE(LabelRef("noinc"))
146 INCQ(R9)
147
148 Label("noinc")
149 MOVQ(R8, X8)
150 PINSRQ(Imm(1), R9, X8)
151
152 iv0 := iv0_DATA()
153 iv1 := iv1_DATA()
154 iv2 := iv2_DATA()
155
156 MOVO(X12, X0)
157 MOVO(X15, X1)
158 MOVOU(Mem{Base: AX}.Offset(32), X2)
159 MOVOU(Mem{Base: AX}.Offset(48), X3)
160 MOVOU(iv0, X4)
161 MOVOU(iv1, X5)
162 MOVOU(iv2, X6)
163
164 PXOR(X8, X6)
165 MOVO(Mem{Base: R10}.Offset(0), X7)
166
167 LOAD_MSG(X8, X9, X10, X11, SI, 0, 2, 4, 6, 1, 3, 5, 7)
168 MOVO(X8, Mem{Base: R10}.Offset(16))
169 MOVO(X9, Mem{Base: R10}.Offset(32))
170 MOVO(X10, Mem{Base: R10}.Offset(48))
171 MOVO(X11, Mem{Base: R10}.Offset(64))
172 HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
173 SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
174 LOAD_MSG(X8, X9, X10, X11, SI, 8, 10, 12, 14, 9, 11, 13, 15)
175 MOVO(X8, Mem{Base: R10}.Offset(80))
176 MOVO(X9, Mem{Base: R10}.Offset(96))
177 MOVO(X10, Mem{Base: R10}.Offset(112))
178 MOVO(X11, Mem{Base: R10}.Offset(128))
179 HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
180 SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
181
182 LOAD_MSG(X8, X9, X10, X11, SI, 14, 4, 9, 13, 10, 8, 15, 6)
183 MOVO(X8, Mem{Base: R10}.Offset(144))
184 MOVO(X9, Mem{Base: R10}.Offset(160))
185 MOVO(X10, Mem{Base: R10}.Offset(176))
186 MOVO(X11, Mem{Base: R10}.Offset(192))
187 HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
188 SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
189 LOAD_MSG(X8, X9, X10, X11, SI, 1, 0, 11, 5, 12, 2, 7, 3)
190 MOVO(X8, Mem{Base: R10}.Offset(208))
191 MOVO(X9, Mem{Base: R10}.Offset(224))
192 MOVO(X10, Mem{Base: R10}.Offset(240))
193 MOVO(X11, Mem{Base: R10}.Offset(256))
194 HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
195 SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
196
197 LOAD_MSG(X8, X9, X10, X11, SI, 11, 12, 5, 15, 8, 0, 2, 13)
198 HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
199 SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
200 LOAD_MSG(X8, X9, X10, X11, SI, 10, 3, 7, 9, 14, 6, 1, 4)
201 HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
202 SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
203
204 LOAD_MSG(X8, X9, X10, X11, SI, 7, 3, 13, 11, 9, 1, 12, 14)
205 HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
206 SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
207 LOAD_MSG(X8, X9, X10, X11, SI, 2, 5, 4, 15, 6, 10, 0, 8)
208 HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
209 SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
210
211 LOAD_MSG(X8, X9, X10, X11, SI, 9, 5, 2, 10, 0, 7, 4, 15)
212 HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
213 SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
214 LOAD_MSG(X8, X9, X10, X11, SI, 14, 11, 6, 3, 1, 12, 8, 13)
215 HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
216 SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
217
218 LOAD_MSG(X8, X9, X10, X11, SI, 2, 6, 0, 8, 12, 10, 11, 3)
219 HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
220 SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
221 LOAD_MSG(X8, X9, X10, X11, SI, 4, 7, 15, 1, 13, 5, 14, 9)
222 HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
223 SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
224
225 LOAD_MSG(X8, X9, X10, X11, SI, 12, 1, 14, 4, 5, 15, 13, 10)
226 HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
227 SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
228 LOAD_MSG(X8, X9, X10, X11, SI, 0, 6, 9, 8, 7, 3, 2, 11)
229 HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
230 SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
231
232 LOAD_MSG(X8, X9, X10, X11, SI, 13, 7, 12, 3, 11, 14, 1, 9)
233 HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
234 SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
235 LOAD_MSG(X8, X9, X10, X11, SI, 5, 15, 8, 2, 0, 4, 6, 10)
236 HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
237 SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
238
239 LOAD_MSG(X8, X9, X10, X11, SI, 6, 14, 11, 0, 15, 9, 3, 8)
240 HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
241 SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
242 LOAD_MSG(X8, X9, X10, X11, SI, 12, 13, 1, 10, 2, 7, 4, 5)
243 HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
244 SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
245
246 LOAD_MSG(X8, X9, X10, X11, SI, 10, 8, 7, 1, 2, 4, 6, 5)
247 HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
248 SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
249 LOAD_MSG(X8, X9, X10, X11, SI, 15, 9, 3, 13, 11, 14, 12, 0)
250 HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
251 SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
252
253 HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, Mem{Base: R10}.Offset(16), Mem{Base: R10}.Offset(32), Mem{Base: R10}.Offset(48), Mem{Base: R10}.Offset(64), X11, X13, X14)
254 SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
255 HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, Mem{Base: R10}.Offset(80), Mem{Base: R10}.Offset(96), Mem{Base: R10}.Offset(112), Mem{Base: R10}.Offset(128), X11, X13, X14)
256 SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
257
258 HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, Mem{Base: R10}.Offset(144), Mem{Base: R10}.Offset(160), Mem{Base: R10}.Offset(176), Mem{Base: R10}.Offset(192), X11, X13, X14)
259 SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
260 HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, Mem{Base: R10}.Offset(208), Mem{Base: R10}.Offset(224), Mem{Base: R10}.Offset(240), Mem{Base: R10}.Offset(256), X11, X13, X14)
261 SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
262
263 MOVOU(Mem{Base: AX}.Offset(32), X10)
264 MOVOU(Mem{Base: AX}.Offset(48), X11)
265 PXOR(X0, X12)
266 PXOR(X1, X15)
267 PXOR(X2, X10)
268 PXOR(X3, X11)
269 PXOR(X4, X12)
270 PXOR(X5, X15)
271 PXOR(X6, X10)
272 PXOR(X7, X11)
273 MOVOU(X10, Mem{Base: AX}.Offset(32))
274 MOVOU(X11, Mem{Base: AX}.Offset(48))
275
276 LEAQ(Mem{Base: SI}.Offset(128), RSI)
277 SUBQ(Imm(128), RDI)
278 JNE(LabelRef("loop"))
279
280 MOVOU(X12, Mem{Base: AX}.Offset(0))
281 MOVOU(X15, Mem{Base: AX}.Offset(16))
282
283 MOVQ(R8, Mem{Base: BX}.Offset(0))
284 MOVQ(R9, Mem{Base: BX}.Offset(8))
285
286 RET()
287 }
288
289 // #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~DATA SECTION~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~##
290
291 func iv0_DATA() Mem {
292 if iv0_DATA_ptr != nil {
293 return *iv0_DATA_ptr
294 }
295
296 iv0 := GLOBL(ThatPeskyUnicodeDot+"iv0", NOPTR|RODATA)
297 iv0_DATA_ptr = &iv0
298 DATA(0x00, U64(0x6a09e667f3bcc908))
299 DATA(0x08, U64(0xbb67ae8584caa73b))
300 return iv0
301 }
302
303 func iv1_DATA() Mem {
304 if iv1_DATA_ptr != nil {
305 return *iv1_DATA_ptr
306 }
307
308 iv1 := GLOBL(ThatPeskyUnicodeDot+"iv1", NOPTR|RODATA)
309 iv1_DATA_ptr = &iv1
310 DATA(0x00, U64(0x3c6ef372fe94f82b))
311 DATA(0x08, U64(0xa54ff53a5f1d36f1))
312 return iv1
313 }
314
315 func iv2_DATA() Mem {
316 if iv2_DATA_ptr != nil {
317 return *iv2_DATA_ptr
318 }
319
320 iv2 := GLOBL(ThatPeskyUnicodeDot+"iv2", NOPTR|RODATA)
321 iv2_DATA_ptr = &iv2
322 DATA(0x00, U64(0x510e527fade682d1))
323 DATA(0x08, U64(0x9b05688c2b3e6c1f))
324 return iv2
325 }
326
327 func iv3_DATA() Mem {
328 if iv3_DATA_ptr != nil {
329 return *iv3_DATA_ptr
330 }
331
332 iv3 := GLOBL(ThatPeskyUnicodeDot+"iv3", NOPTR|RODATA)
333 iv3_DATA_ptr = &iv3
334 DATA(0x00, U64(0x1f83d9abfb41bd6b))
335 DATA(0x08, U64(0x5be0cd19137e2179))
336 return iv3
337 }
338
339 func c40_DATA() Mem {
340 if c40_DATA_ptr != nil {
341 return *c40_DATA_ptr
342 }
343
344 c40 := GLOBL(ThatPeskyUnicodeDot+"c40", NOPTR|RODATA)
345 c40_DATA_ptr = &c40
346 DATA(0x00, U64(0x0201000706050403))
347 DATA(0x08, U64(0x0a09080f0e0d0c0b))
348 return c40
349 }
350
351 func c48_DATA() Mem {
352 if c48_DATA_ptr != nil {
353 return *c48_DATA_ptr
354 }
355
356 c48 := GLOBL(ThatPeskyUnicodeDot+"c48", NOPTR|RODATA)
357 c48_DATA_ptr = &c48
358 DATA(0x00, U64(0x0100070605040302))
359 DATA(0x08, U64(0x09080f0e0d0c0b0a))
360 return c48
361 }