Debian Codegraph @

_gen/gen-arm-neon.go

raw
1 // Copyright 2024, Klaus Post/Minio Inc. See LICENSE for details.
2
3 package main
4
5 import (
6 "bytes"
7 "fmt"
8 "log"
9 "os"
10 "regexp"
11 "strconv"
12 "strings"
13 )
14
15 func convert2Neon(asmBuf *bytes.Buffer, lines []string) {
16
17 asmF := func(format string, args ...interface{}) {
18 (*asmBuf).WriteString(fmt.Sprintf(format, args...))
19 }
20
21 reAddrMode := regexp.MustCompile(`\[(.*?)\]`) // regexp to match content between square brackets
22
23 getZregister := func(reg string) int {
24 if reg[0] == 'z' {
25 reg = strings.NewReplacer(",", "", ".d", "", ".b", "", "[0]", "").Replace(reg[1:])
26 num, err := strconv.Atoi(reg)
27 if err != nil {
28 panic(err)
29 }
30 return num
31 }
32 return -1
33 }
34
35 getXregister := func(reg string) int {
36 if reg[0] == 'x' {
37 reg = strings.ReplaceAll(reg, ",", "")
38 num, err := strconv.Atoi(reg[1:])
39 if err != nil {
40 panic(err)
41 }
42 return num
43 }
44 return -1
45 }
46
47 getHashImm := func(imm string) int {
48 if imm[0] == '#' {
49 num, err := strconv.Atoi(imm[1:])
50 if err != nil {
51 panic(err)
52 }
53 return num
54 } else {
55 panic("bad immediate")
56 }
57 }
58
59 parseAddrModeMulVl := func(addrMode string) string {
60 addrMode = strings.NewReplacer("[", "", "]", "").Replace(addrMode)
61 f := strings.Fields(addrMode)
62 xn, offset := getXregister(f[0]), ""
63 if len(f) > 1 {
64 if len(f) == 4 && f[2] == "MUL" && f[3] == "VL" {
65 num, err := strconv.Atoi(strings.NewReplacer("#", "", ",", "").Replace(f[1]))
66 if err != nil {
67 panic(err)
68 }
69 offset = fmt.Sprintf("%d", num*32)
70 } else {
71 panic("bad addressing mode")
72 }
73 }
74
75 return fmt.Sprintf("%s(R%d)", offset, xn)
76 }
77
78 parseAddrModeIndexed := func(addrMode string) (string, string) {
79 addrMode = strings.NewReplacer("[", "", "]", "").Replace(addrMode)
80 f := strings.Fields(addrMode)
81 regbase := getXregister(f[0])
82 regshifted := getXregister(f[1])
83 shift := getHashImm(f[3])
84 return fmt.Sprintf("(R%d)", regbase), fmt.Sprintf("R%d<<%d", regshifted, shift)
85 }
86
87 for _, line := range lines {
88 if strings.Contains(line, " // ldr z") {
89 if matches := reAddrMode.FindAllStringSubmatch(line, -1); len(matches) == 1 {
90 line = strings.ReplaceAll(line, matches[0][0], "ADDRMODE")
91 f := strings.Fields(line)
92 zd := getZregister(f[len(f)-2])
93 var am string
94 { // HACK: ignore offset since we're fixating it at 32
95 baseReg := strings.Split(matches[0][0], ",")[0]
96 am = parseAddrModeMulVl(baseReg)
97 }
98 asmF(" VLD1.P 32%s, [V%d.B16, V%d.B16]\n", am, zd*2, zd*2+1)
99 } else {
100 panic("bad 'ldr' instrunction")
101 }
102 } else if strings.Contains(line, " // str z") {
103 if matches := reAddrMode.FindAllStringSubmatch(line, -1); len(matches) == 1 {
104 line = strings.ReplaceAll(line, matches[0][0], "ADDRMODE")
105 f := strings.Fields(line)
106 zd := getZregister(f[len(f)-2])
107 var am string
108 { // HACK: ignore offset since we're fixating it at 32
109 baseReg := strings.Split(matches[0][0], ",")[0]
110 am = parseAddrModeMulVl(baseReg)
111 }
112 asmF(" VST1.P [V%d.D2, V%d.D2], 32%s\n", zd*2, zd*2+1, am)
113 } else {
114 panic("bad 'str' instrunction")
115 }
116 } else if strings.Contains(line, " // ld1d { z") {
117 if matches := reAddrMode.FindAllStringSubmatch(line, -1); len(matches) == 1 {
118 line = strings.ReplaceAll(line, matches[0][0], "ADDRMODE")
119 f := strings.Fields(line)
120 zd := getZregister(f[5])
121 base, shifted := parseAddrModeIndexed(matches[0][0])
122 asmF(" ADD %s, %s\n", shifted, strings.NewReplacer("(", "", ")", "").Replace(base))
123 asmF(" VLD1 %s, [V%d.B16, V%d.B16]\n", base, zd*2, zd*2+1)
124 } else {
125 panic("bad 'ld1d' instrunction")
126 }
127 } else if strings.Contains(line, " // st1d { z") {
128 if matches := reAddrMode.FindAllStringSubmatch(line, -1); len(matches) == 1 {
129 line = strings.ReplaceAll(line, matches[0][0], "ADDRMODE")
130 f := strings.Fields(line)
131 zd := getZregister(f[5])
132 base, shifted := parseAddrModeIndexed(matches[0][0])
133 asmF(" ADD %s, %s\n", shifted, strings.NewReplacer("(", "", ")", "").Replace(base))
134 asmF(" VST1 [V%d.D2, V%d.D2], %s\n", zd*2, zd*2+1, base)
135 } else {
136 panic("bad 'st1d' instrunction")
137 }
138 } else if strings.Contains(line, " // lsr z") {
139 f := strings.Fields(line)
140 zd := getZregister(f[len(f)-3])
141 zn := getZregister(f[len(f)-2])
142 imm := getHashImm(f[len(f)-1])
143 asmF(" VUSHR $%d, V%d.B16, V%d.B16\n", imm, zn*2, zd*2)
144 asmF(" VUSHR $%d, V%d.B16, V%d.B16\n", imm, zn*2+1, zd*2+1)
145 } else if strings.Contains(line, " // and z") {
146 f := strings.Fields(line)
147 zd := getZregister(f[len(f)-3])
148 zn := getZregister(f[len(f)-2])
149 zn2 := getZregister(f[len(f)-1])
150 asmF(" VAND V%d.B16, V%d.B16, V%d.B16\n", zn2*2, zn*2, zd*2)
151 asmF(" VAND V%d.B16, V%d.B16, V%d.B16\n", zn2*2 /*+1*/, zn*2+1, zd*2+1)
152 } else if strings.Contains(line, " // tbl z") {
153 f := strings.Fields(line)
154 zd := getZregister(f[len(f)-3])
155 zn := getZregister(f[len(f)-2])
156 zn2 := getZregister(f[len(f)-1])
157 asmF(" VTBL V%d.B16, [V%d.B16], V%d.B16\n", zn2*2, zn*2, zd*2)
158 asmF(" VTBL V%d.B16, [V%d.B16], V%d.B16\n", zn2*2+1, zn*2+1, zd*2+1)
159 } else if strings.Contains(line, " // eor z") {
160 f := strings.Fields(line)
161 zd := getZregister(f[len(f)-3])
162 zn := getZregister(f[len(f)-2])
163 zn2 := getZregister(f[len(f)-1])
164 asmF(" VEOR V%d.B16, V%d.B16, V%d.B16\n", zn2*2, zn*2, zd*2)
165 asmF(" VEOR V%d.B16, V%d.B16, V%d.B16\n", zn2*2+1, zn*2+1, zd*2+1)
166 } else if strings.Contains(line, " // mov z") {
167 f := strings.Fields(line)
168 zd := getZregister(f[len(f)-2])
169 xn := getXregister(f[len(f)-1])
170 asmF(" VMOV R%d, V%d.B[0]\n", xn, zd*2)
171 } else if strings.Contains(line, " // dup z") {
172 f := strings.Fields(line)
173 zd := getZregister(f[len(f)-2])
174 zn := getZregister(f[len(f)-1])
175 asmF(" VDUP V%d.B[0], V%d.B16\n", zn*2, zd*2)
176 } else if strings.Contains(line, " // add x") {
177 f := strings.Fields(line)
178 xd := getXregister(f[len(f)-3])
179 if xd != getXregister(f[len(f)-2]) {
180 panic("registers don't match")
181 }
182 if f[len(f)-1][0] == '#' {
183 imm := getHashImm(f[len(f)-1])
184 asmF(" ADD $%d, R%d\n", imm, xd)
185 } else {
186 xn := getXregister(f[len(f)-1])
187 asmF(" ADD R%d, R%d\n", xn, xd)
188 }
189 } else if strings.Contains(line, " // subs x") {
190 f := strings.Fields(line)
191 xd := getXregister(f[len(f)-3])
192 if xd != getXregister(f[len(f)-2]) {
193 panic("registers don't match")
194 }
195 imm := getHashImm(f[len(f)-1])
196 asmF(" SUBS $%d, R%d\n", imm, xd)
197 } else if strings.Contains(line, " // lsr x") {
198 f := strings.Fields(line)
199 xd := getXregister(f[len(f)-3])
200 if xd != getXregister(f[len(f)-2]) {
201 panic("registers don't match")
202 }
203 imm := getHashImm(f[len(f)-1])
204 asmF(" LSR $%d, R%d\n", imm, xd)
205 } else if strings.Contains(line, " // tst x") {
206 f := strings.Fields(line)
207 xd := getXregister(f[len(f)-2])
208 xn := getXregister(f[len(f)-1])
209 asmF(" TST R%d, R%d\n", xn, xd)
210 } else if strings.Contains(line, " // mov x") {
211 f := strings.Fields(line)
212 xd := getXregister(f[len(f)-2])
213 imm := getHashImm(f[len(f)-1])
214 asmF(" MOVD $%d, R%d\n", imm, xd)
215 } else if strings.HasSuffix(line, ":") ||
216 strings.HasPrefix(line, " BEQ") ||
217 strings.HasPrefix(line, " BNE") ||
218 strings.HasPrefix(line, "TEXT ·mulSve") ||
219 strings.HasPrefix(line, "// func mulSve") {
220 line = strings.ReplaceAll(line, "Sve", "Neon")
221 asmF("%s\n", line)
222 } else if strings.Contains(line, "Requires: SVE") {
223 line = strings.ReplaceAll(line, "SVE", "NEON")
224 asmF("%s\n", line)
225 } else if strings.Contains(line, " // ptrue p") {
226 // intentionally drop line
227 } else if strings.HasPrefix(line, " // ") ||
228 strings.HasPrefix(line, " MOVD ") ||
229 strings.HasPrefix(line, " CMP ") ||
230 strings.HasPrefix(line, " RET") ||
231 len(line) == 0 {
232 asmF("%s\n", line)
233 } else {
234 panic(fmt.Sprintf("convert2Neon unsupported: `%s`", line))
235 }
236 }
237 }
238
239 func fixPostIncrementNeon(asmBuf *bytes.Buffer, lines []string) {
240
241 asmF := func(format string, args ...interface{}) {
242 (*asmBuf).WriteString(fmt.Sprintf(format, args...))
243 }
244
245 const MATRIX_BASE = "matrix_base"
246
247 skipResetMatrixBase := false
248 {
249 routine := strings.Join(lines, "\n")
250 reFramePtr := regexp.MustCompile(`MOVD\s*` + MATRIX_BASE + `\+\d*(\(FP\),\s*R\d*)`)
251
252 if matches := reFramePtr.FindAllStringSubmatch(routine, -1); len(matches) == 1 {
253 framePtrToDest := matches[0][1]
254
255 // check if we're loading into register
256 // more than once from the stack frame
257 // (meaning we overwrite the 'matrix_base' value)
258 escaped := strings.NewReplacer("(", `\(`, ")", `\)`).Replace(framePtrToDest)
259 reSameDest := regexp.MustCompile(`MOVD\s*\w*\+\d*` + escaped)
260 if m := reSameDest.FindAllStringSubmatch(routine, -1); len(m) == 2 {
261 skipResetMatrixBase = true
262 }
263 }
264 }
265
266 isXor := false
267 {
268 routine := strings.Join(lines, "\n")
269 isXor = strings.Count(routine, "Xor(SB)") > 0
270 }
271
272 resetMatrixBaseAtStartOfLoop := ""
273 for i := 0; i < len(lines); i++ {
274
275 if !skipResetMatrixBase {
276 //
277 // Since we are loading with post-increment,
278 // reset register holding matrix array at
279 // start of each loop
280 //
281 if strings.Contains(lines[i], MATRIX_BASE) {
282 resetMatrixBaseAtStartOfLoop = lines[i]
283 continue
284 } else if strings.HasSuffix(lines[i], "_loop:") {
285 asmF("%s\n", lines[i])
286 asmF("%s\n", resetMatrixBaseAtStartOfLoop)
287 resetMatrixBaseAtStartOfLoop = ""
288 continue
289 }
290 }
291
292 //
293 // Remove the explicit ADDition of the
294 // pointer to the shard (since we are already
295 // using post-increments for the loads/stores)
296 //
297 if i < len(lines)-1 &&
298 strings.Contains(lines[i], "32(R") &&
299 strings.Contains(lines[i+1], "ADD") && strings.Contains(lines[i+1], "$32, R") {
300
301 storing := strings.Contains(lines[i], "VST1.P")
302 if storing && isXor {
303 // move post-increment into a "pre-decrement" to offset
304 // post-increment for loading of existing content in case of Xor-case
305 asmF("%s\n", strings.ReplaceAll(lines[i+1], "ADD", "SUB"))
306 asmF("%s\n", lines[i])
307 } else {
308 asmF("%s\n", lines[i])
309 // intentionally skip line with ADD
310 }
311 i += 1
312 continue
313 }
314 if i < len(lines)-2 &&
315 strings.Contains(lines[i], "32(R") &&
316 strings.Contains(lines[i+1], "32(R") &&
317 strings.Contains(lines[i+2], "ADD") && strings.Contains(lines[i+2], "$64, R") {
318
319 storing := strings.Contains(lines[i], "VST1.P") && strings.Contains(lines[i+1], "VST1.P")
320 if storing && isXor {
321 // move post-increment into a "pre-decrement" to offset
322 // post-increment for loading of existing content in case of Xor-case
323 asmF("%s\n", strings.ReplaceAll(lines[i+2], "ADD", "SUB"))
324 asmF("%s\n", lines[i])
325 asmF("%s\n", lines[i+1])
326 } else {
327 asmF("%s\n", lines[i])
328 asmF("%s\n", lines[i+1])
329 // intentionally skip line with ADD
330 }
331 i += 2
332 continue
333 }
334
335 asmF("%s\n", lines[i])
336 }
337 }
338
339 func genArmNeon() {
340 const SVE_CODE = "../galois_gen_arm64.s"
341
342 asmOut, goOut := &bytes.Buffer{}, &bytes.Buffer{}
343
344 if asmSve, err := os.ReadFile(SVE_CODE); err != nil {
345 log.Fatalf("Failed to read %s: %v", SVE_CODE, err)
346 } else {
347 // start with SVE code
348 asmOut.WriteString(string(asmSve))
349 }
350 if goSve, err := os.ReadFile(strings.ReplaceAll(SVE_CODE, ".s", ".go")); err != nil {
351 log.Fatalf("Failed to read %s: %v", SVE_CODE, err)
352 } else {
353 goOut.WriteString(string(goSve))
354 }
355
356 const input = 10
357
358 // Processing 64 bytes variants
359 for output := 1; output <= 3; output++ {
360 for op := ""; len(op) <= 3; op += "Xor" {
361 templName := fmt.Sprintf("mulSve_%dx%d_64%s", input, output, op)
362 funcDef := fmt.Sprintf("func %s(matrix []byte, in [][]byte, out [][]byte, start int, n int)", strings.ReplaceAll(templName, "Sve", "Neon"))
363
364 lines, err := extractRoutine(SVE_CODE, fmt.Sprintf("TEXT ·%s(SB)", templName))
365 if err != nil {
366 log.Fatal(err)
367 }
368
369 // prepend output with commented out function definition and comment
370 asmOut.WriteString(fmt.Sprintf("// %s\n", funcDef))
371 asmOut.WriteString("// Requires: NEON\n")
372
373 {
374 asmTemp := &bytes.Buffer{}
375 convert2Neon(asmTemp, lines)
376 fixPostIncrementNeon(asmOut, strings.Split(string(asmTemp.Bytes()), "\n"))
377 }
378
379 // golang declaration
380 goOut.WriteString(fmt.Sprintf("//go:noescape\n%s\n\n", funcDef))
381 }
382 }
383
384 // Processing 32 bytes variants
385 for output := 4; output <= 10; output++ {
386 for op := ""; len(op) <= 3; op += "Xor" {
387 templName := fmt.Sprintf("mulSve_%dx%d%s", input, output, op)
388 funcDef := fmt.Sprintf("func %s(matrix []byte, in [][]byte, out [][]byte, start int, n int)", strings.ReplaceAll(templName, "Sve", "Neon"))
389
390 lines, err := extractRoutine(SVE_CODE, fmt.Sprintf("TEXT ·%s(SB)", templName))
391 if err != nil {
392 log.Fatal(err)
393 }
394
395 // prepend output with commented out function definition and comment
396 asmOut.WriteString(fmt.Sprintf("// %s\n", funcDef))
397 asmOut.WriteString("// Requires: NEON\n")
398
399 {
400 asmTemp := &bytes.Buffer{}
401 convert2Neon(asmTemp, lines)
402 fixPostIncrementNeon(asmOut, strings.Split(string(asmTemp.Bytes()), "\n"))
403 }
404
405 // golang declaration
406 goOut.WriteString(fmt.Sprintf("//go:noescape\n%s\n", funcDef))
407
408 if !(output == 10 && op == "Xor") {
409 goOut.WriteString("\n")
410 }
411 }
412 }
413 if err := os.WriteFile("../galois_gen_arm64.s", asmOut.Bytes(), 0644); err != nil {
414 log.Fatal(err)
415 }
416 if err := os.WriteFile("../galois_gen_arm64.go", goOut.Bytes(), 0644); err != nil {
417 log.Fatal(err)
418 }
419 }