| 1 |
// Copyright 2024, Klaus Post/Minio Inc. See LICENSE for details. |
| 2 |
|
| 3 |
package main |
| 4 |
|
| 5 |
import ( |
| 6 |
"bytes" |
| 7 |
"fmt" |
| 8 |
"log" |
| 9 |
"os" |
| 10 |
"regexp" |
| 11 |
"strconv" |
| 12 |
"strings" |
| 13 |
) |
| 14 |
|
| 15 |
func convert2Neon(asmBuf *bytes.Buffer, lines []string) { |
| 16 |
|
| 17 |
asmF := func(format string, args ...interface{}) { |
| 18 |
(*asmBuf).WriteString(fmt.Sprintf(format, args...)) |
| 19 |
} |
| 20 |
|
| 21 |
reAddrMode := regexp.MustCompile(`\[(.*?)\]`) // regexp to match content between square brackets |
| 22 |
|
| 23 |
getZregister := func(reg string) int { |
| 24 |
if reg[0] == 'z' { |
| 25 |
reg = strings.NewReplacer(",", "", ".d", "", ".b", "", "[0]", "").Replace(reg[1:]) |
| 26 |
num, err := strconv.Atoi(reg) |
| 27 |
if err != nil { |
| 28 |
panic(err) |
| 29 |
} |
| 30 |
return num |
| 31 |
} |
| 32 |
return -1 |
| 33 |
} |
| 34 |
|
| 35 |
getXregister := func(reg string) int { |
| 36 |
if reg[0] == 'x' { |
| 37 |
reg = strings.ReplaceAll(reg, ",", "") |
| 38 |
num, err := strconv.Atoi(reg[1:]) |
| 39 |
if err != nil { |
| 40 |
panic(err) |
| 41 |
} |
| 42 |
return num |
| 43 |
} |
| 44 |
return -1 |
| 45 |
} |
| 46 |
|
| 47 |
getHashImm := func(imm string) int { |
| 48 |
if imm[0] == '#' { |
| 49 |
num, err := strconv.Atoi(imm[1:]) |
| 50 |
if err != nil { |
| 51 |
panic(err) |
| 52 |
} |
| 53 |
return num |
| 54 |
} else { |
| 55 |
panic("bad immediate") |
| 56 |
} |
| 57 |
} |
| 58 |
|
| 59 |
parseAddrModeMulVl := func(addrMode string) string { |
| 60 |
addrMode = strings.NewReplacer("[", "", "]", "").Replace(addrMode) |
| 61 |
f := strings.Fields(addrMode) |
| 62 |
xn, offset := getXregister(f[0]), "" |
| 63 |
if len(f) > 1 { |
| 64 |
if len(f) == 4 && f[2] == "MUL" && f[3] == "VL" { |
| 65 |
num, err := strconv.Atoi(strings.NewReplacer("#", "", ",", "").Replace(f[1])) |
| 66 |
if err != nil { |
| 67 |
panic(err) |
| 68 |
} |
| 69 |
offset = fmt.Sprintf("%d", num*32) |
| 70 |
} else { |
| 71 |
panic("bad addressing mode") |
| 72 |
} |
| 73 |
} |
| 74 |
|
| 75 |
return fmt.Sprintf("%s(R%d)", offset, xn) |
| 76 |
} |
| 77 |
|
| 78 |
parseAddrModeIndexed := func(addrMode string) (string, string) { |
| 79 |
addrMode = strings.NewReplacer("[", "", "]", "").Replace(addrMode) |
| 80 |
f := strings.Fields(addrMode) |
| 81 |
regbase := getXregister(f[0]) |
| 82 |
regshifted := getXregister(f[1]) |
| 83 |
shift := getHashImm(f[3]) |
| 84 |
return fmt.Sprintf("(R%d)", regbase), fmt.Sprintf("R%d<<%d", regshifted, shift) |
| 85 |
} |
| 86 |
|
| 87 |
for _, line := range lines { |
| 88 |
if strings.Contains(line, " // ldr z") { |
| 89 |
if matches := reAddrMode.FindAllStringSubmatch(line, -1); len(matches) == 1 { |
| 90 |
line = strings.ReplaceAll(line, matches[0][0], "ADDRMODE") |
| 91 |
f := strings.Fields(line) |
| 92 |
zd := getZregister(f[len(f)-2]) |
| 93 |
var am string |
| 94 |
{ // HACK: ignore offset since we're fixating it at 32 |
| 95 |
baseReg := strings.Split(matches[0][0], ",")[0] |
| 96 |
am = parseAddrModeMulVl(baseReg) |
| 97 |
} |
| 98 |
asmF(" VLD1.P 32%s, [V%d.B16, V%d.B16]\n", am, zd*2, zd*2+1) |
| 99 |
} else { |
| 100 |
panic("bad 'ldr' instrunction") |
| 101 |
} |
| 102 |
} else if strings.Contains(line, " // str z") { |
| 103 |
if matches := reAddrMode.FindAllStringSubmatch(line, -1); len(matches) == 1 { |
| 104 |
line = strings.ReplaceAll(line, matches[0][0], "ADDRMODE") |
| 105 |
f := strings.Fields(line) |
| 106 |
zd := getZregister(f[len(f)-2]) |
| 107 |
var am string |
| 108 |
{ // HACK: ignore offset since we're fixating it at 32 |
| 109 |
baseReg := strings.Split(matches[0][0], ",")[0] |
| 110 |
am = parseAddrModeMulVl(baseReg) |
| 111 |
} |
| 112 |
asmF(" VST1.P [V%d.D2, V%d.D2], 32%s\n", zd*2, zd*2+1, am) |
| 113 |
} else { |
| 114 |
panic("bad 'str' instrunction") |
| 115 |
} |
| 116 |
} else if strings.Contains(line, " // ld1d { z") { |
| 117 |
if matches := reAddrMode.FindAllStringSubmatch(line, -1); len(matches) == 1 { |
| 118 |
line = strings.ReplaceAll(line, matches[0][0], "ADDRMODE") |
| 119 |
f := strings.Fields(line) |
| 120 |
zd := getZregister(f[5]) |
| 121 |
base, shifted := parseAddrModeIndexed(matches[0][0]) |
| 122 |
asmF(" ADD %s, %s\n", shifted, strings.NewReplacer("(", "", ")", "").Replace(base)) |
| 123 |
asmF(" VLD1 %s, [V%d.B16, V%d.B16]\n", base, zd*2, zd*2+1) |
| 124 |
} else { |
| 125 |
panic("bad 'ld1d' instrunction") |
| 126 |
} |
| 127 |
} else if strings.Contains(line, " // st1d { z") { |
| 128 |
if matches := reAddrMode.FindAllStringSubmatch(line, -1); len(matches) == 1 { |
| 129 |
line = strings.ReplaceAll(line, matches[0][0], "ADDRMODE") |
| 130 |
f := strings.Fields(line) |
| 131 |
zd := getZregister(f[5]) |
| 132 |
base, shifted := parseAddrModeIndexed(matches[0][0]) |
| 133 |
asmF(" ADD %s, %s\n", shifted, strings.NewReplacer("(", "", ")", "").Replace(base)) |
| 134 |
asmF(" VST1 [V%d.D2, V%d.D2], %s\n", zd*2, zd*2+1, base) |
| 135 |
} else { |
| 136 |
panic("bad 'st1d' instrunction") |
| 137 |
} |
| 138 |
} else if strings.Contains(line, " // lsr z") { |
| 139 |
f := strings.Fields(line) |
| 140 |
zd := getZregister(f[len(f)-3]) |
| 141 |
zn := getZregister(f[len(f)-2]) |
| 142 |
imm := getHashImm(f[len(f)-1]) |
| 143 |
asmF(" VUSHR $%d, V%d.B16, V%d.B16\n", imm, zn*2, zd*2) |
| 144 |
asmF(" VUSHR $%d, V%d.B16, V%d.B16\n", imm, zn*2+1, zd*2+1) |
| 145 |
} else if strings.Contains(line, " // and z") { |
| 146 |
f := strings.Fields(line) |
| 147 |
zd := getZregister(f[len(f)-3]) |
| 148 |
zn := getZregister(f[len(f)-2]) |
| 149 |
zn2 := getZregister(f[len(f)-1]) |
| 150 |
asmF(" VAND V%d.B16, V%d.B16, V%d.B16\n", zn2*2, zn*2, zd*2) |
| 151 |
asmF(" VAND V%d.B16, V%d.B16, V%d.B16\n", zn2*2 /*+1*/, zn*2+1, zd*2+1) |
| 152 |
} else if strings.Contains(line, " // tbl z") { |
| 153 |
f := strings.Fields(line) |
| 154 |
zd := getZregister(f[len(f)-3]) |
| 155 |
zn := getZregister(f[len(f)-2]) |
| 156 |
zn2 := getZregister(f[len(f)-1]) |
| 157 |
asmF(" VTBL V%d.B16, [V%d.B16], V%d.B16\n", zn2*2, zn*2, zd*2) |
| 158 |
asmF(" VTBL V%d.B16, [V%d.B16], V%d.B16\n", zn2*2+1, zn*2+1, zd*2+1) |
| 159 |
} else if strings.Contains(line, " // eor z") { |
| 160 |
f := strings.Fields(line) |
| 161 |
zd := getZregister(f[len(f)-3]) |
| 162 |
zn := getZregister(f[len(f)-2]) |
| 163 |
zn2 := getZregister(f[len(f)-1]) |
| 164 |
asmF(" VEOR V%d.B16, V%d.B16, V%d.B16\n", zn2*2, zn*2, zd*2) |
| 165 |
asmF(" VEOR V%d.B16, V%d.B16, V%d.B16\n", zn2*2+1, zn*2+1, zd*2+1) |
| 166 |
} else if strings.Contains(line, " // mov z") { |
| 167 |
f := strings.Fields(line) |
| 168 |
zd := getZregister(f[len(f)-2]) |
| 169 |
xn := getXregister(f[len(f)-1]) |
| 170 |
asmF(" VMOV R%d, V%d.B[0]\n", xn, zd*2) |
| 171 |
} else if strings.Contains(line, " // dup z") { |
| 172 |
f := strings.Fields(line) |
| 173 |
zd := getZregister(f[len(f)-2]) |
| 174 |
zn := getZregister(f[len(f)-1]) |
| 175 |
asmF(" VDUP V%d.B[0], V%d.B16\n", zn*2, zd*2) |
| 176 |
} else if strings.Contains(line, " // add x") { |
| 177 |
f := strings.Fields(line) |
| 178 |
xd := getXregister(f[len(f)-3]) |
| 179 |
if xd != getXregister(f[len(f)-2]) { |
| 180 |
panic("registers don't match") |
| 181 |
} |
| 182 |
if f[len(f)-1][0] == '#' { |
| 183 |
imm := getHashImm(f[len(f)-1]) |
| 184 |
asmF(" ADD $%d, R%d\n", imm, xd) |
| 185 |
} else { |
| 186 |
xn := getXregister(f[len(f)-1]) |
| 187 |
asmF(" ADD R%d, R%d\n", xn, xd) |
| 188 |
} |
| 189 |
} else if strings.Contains(line, " // subs x") { |
| 190 |
f := strings.Fields(line) |
| 191 |
xd := getXregister(f[len(f)-3]) |
| 192 |
if xd != getXregister(f[len(f)-2]) { |
| 193 |
panic("registers don't match") |
| 194 |
} |
| 195 |
imm := getHashImm(f[len(f)-1]) |
| 196 |
asmF(" SUBS $%d, R%d\n", imm, xd) |
| 197 |
} else if strings.Contains(line, " // lsr x") { |
| 198 |
f := strings.Fields(line) |
| 199 |
xd := getXregister(f[len(f)-3]) |
| 200 |
if xd != getXregister(f[len(f)-2]) { |
| 201 |
panic("registers don't match") |
| 202 |
} |
| 203 |
imm := getHashImm(f[len(f)-1]) |
| 204 |
asmF(" LSR $%d, R%d\n", imm, xd) |
| 205 |
} else if strings.Contains(line, " // tst x") { |
| 206 |
f := strings.Fields(line) |
| 207 |
xd := getXregister(f[len(f)-2]) |
| 208 |
xn := getXregister(f[len(f)-1]) |
| 209 |
asmF(" TST R%d, R%d\n", xn, xd) |
| 210 |
} else if strings.Contains(line, " // mov x") { |
| 211 |
f := strings.Fields(line) |
| 212 |
xd := getXregister(f[len(f)-2]) |
| 213 |
imm := getHashImm(f[len(f)-1]) |
| 214 |
asmF(" MOVD $%d, R%d\n", imm, xd) |
| 215 |
} else if strings.HasSuffix(line, ":") || |
| 216 |
strings.HasPrefix(line, " BEQ") || |
| 217 |
strings.HasPrefix(line, " BNE") || |
| 218 |
strings.HasPrefix(line, "TEXT ·mulSve") || |
| 219 |
strings.HasPrefix(line, "// func mulSve") { |
| 220 |
line = strings.ReplaceAll(line, "Sve", "Neon") |
| 221 |
asmF("%s\n", line) |
| 222 |
} else if strings.Contains(line, "Requires: SVE") { |
| 223 |
line = strings.ReplaceAll(line, "SVE", "NEON") |
| 224 |
asmF("%s\n", line) |
| 225 |
} else if strings.Contains(line, " // ptrue p") { |
| 226 |
// intentionally drop line |
| 227 |
} else if strings.HasPrefix(line, " // ") || |
| 228 |
strings.HasPrefix(line, " MOVD ") || |
| 229 |
strings.HasPrefix(line, " CMP ") || |
| 230 |
strings.HasPrefix(line, " RET") || |
| 231 |
len(line) == 0 { |
| 232 |
asmF("%s\n", line) |
| 233 |
} else { |
| 234 |
panic(fmt.Sprintf("convert2Neon unsupported: `%s`", line)) |
| 235 |
} |
| 236 |
} |
| 237 |
} |
| 238 |
|
| 239 |
func fixPostIncrementNeon(asmBuf *bytes.Buffer, lines []string) { |
| 240 |
|
| 241 |
asmF := func(format string, args ...interface{}) { |
| 242 |
(*asmBuf).WriteString(fmt.Sprintf(format, args...)) |
| 243 |
} |
| 244 |
|
| 245 |
const MATRIX_BASE = "matrix_base" |
| 246 |
|
| 247 |
skipResetMatrixBase := false |
| 248 |
{ |
| 249 |
routine := strings.Join(lines, "\n") |
| 250 |
reFramePtr := regexp.MustCompile(`MOVD\s*` + MATRIX_BASE + `\+\d*(\(FP\),\s*R\d*)`) |
| 251 |
|
| 252 |
if matches := reFramePtr.FindAllStringSubmatch(routine, -1); len(matches) == 1 { |
| 253 |
framePtrToDest := matches[0][1] |
| 254 |
|
| 255 |
// check if we're loading into register |
| 256 |
// more than once from the stack frame |
| 257 |
// (meaning we overwrite the 'matrix_base' value) |
| 258 |
escaped := strings.NewReplacer("(", `\(`, ")", `\)`).Replace(framePtrToDest) |
| 259 |
reSameDest := regexp.MustCompile(`MOVD\s*\w*\+\d*` + escaped) |
| 260 |
if m := reSameDest.FindAllStringSubmatch(routine, -1); len(m) == 2 { |
| 261 |
skipResetMatrixBase = true |
| 262 |
} |
| 263 |
} |
| 264 |
} |
| 265 |
|
| 266 |
isXor := false |
| 267 |
{ |
| 268 |
routine := strings.Join(lines, "\n") |
| 269 |
isXor = strings.Count(routine, "Xor(SB)") > 0 |
| 270 |
} |
| 271 |
|
| 272 |
resetMatrixBaseAtStartOfLoop := "" |
| 273 |
for i := 0; i < len(lines); i++ { |
| 274 |
|
| 275 |
if !skipResetMatrixBase { |
| 276 |
// |
| 277 |
// Since we are loading with post-increment, |
| 278 |
// reset register holding matrix array at |
| 279 |
// start of each loop |
| 280 |
// |
| 281 |
if strings.Contains(lines[i], MATRIX_BASE) { |
| 282 |
resetMatrixBaseAtStartOfLoop = lines[i] |
| 283 |
continue |
| 284 |
} else if strings.HasSuffix(lines[i], "_loop:") { |
| 285 |
asmF("%s\n", lines[i]) |
| 286 |
asmF("%s\n", resetMatrixBaseAtStartOfLoop) |
| 287 |
resetMatrixBaseAtStartOfLoop = "" |
| 288 |
continue |
| 289 |
} |
| 290 |
} |
| 291 |
|
| 292 |
// |
| 293 |
// Remove the explicit ADDition of the |
| 294 |
// pointer to the shard (since we are already |
| 295 |
// using post-increments for the loads/stores) |
| 296 |
// |
| 297 |
if i < len(lines)-1 && |
| 298 |
strings.Contains(lines[i], "32(R") && |
| 299 |
strings.Contains(lines[i+1], "ADD") && strings.Contains(lines[i+1], "$32, R") { |
| 300 |
|
| 301 |
storing := strings.Contains(lines[i], "VST1.P") |
| 302 |
if storing && isXor { |
| 303 |
// move post-increment into a "pre-decrement" to offset |
| 304 |
// post-increment for loading of existing content in case of Xor-case |
| 305 |
asmF("%s\n", strings.ReplaceAll(lines[i+1], "ADD", "SUB")) |
| 306 |
asmF("%s\n", lines[i]) |
| 307 |
} else { |
| 308 |
asmF("%s\n", lines[i]) |
| 309 |
// intentionally skip line with ADD |
| 310 |
} |
| 311 |
i += 1 |
| 312 |
continue |
| 313 |
} |
| 314 |
if i < len(lines)-2 && |
| 315 |
strings.Contains(lines[i], "32(R") && |
| 316 |
strings.Contains(lines[i+1], "32(R") && |
| 317 |
strings.Contains(lines[i+2], "ADD") && strings.Contains(lines[i+2], "$64, R") { |
| 318 |
|
| 319 |
storing := strings.Contains(lines[i], "VST1.P") && strings.Contains(lines[i+1], "VST1.P") |
| 320 |
if storing && isXor { |
| 321 |
// move post-increment into a "pre-decrement" to offset |
| 322 |
// post-increment for loading of existing content in case of Xor-case |
| 323 |
asmF("%s\n", strings.ReplaceAll(lines[i+2], "ADD", "SUB")) |
| 324 |
asmF("%s\n", lines[i]) |
| 325 |
asmF("%s\n", lines[i+1]) |
| 326 |
} else { |
| 327 |
asmF("%s\n", lines[i]) |
| 328 |
asmF("%s\n", lines[i+1]) |
| 329 |
// intentionally skip line with ADD |
| 330 |
} |
| 331 |
i += 2 |
| 332 |
continue |
| 333 |
} |
| 334 |
|
| 335 |
asmF("%s\n", lines[i]) |
| 336 |
} |
| 337 |
} |
| 338 |
|
| 339 |
func genArmNeon() { |
| 340 |
const SVE_CODE = "../galois_gen_arm64.s" |
| 341 |
|
| 342 |
asmOut, goOut := &bytes.Buffer{}, &bytes.Buffer{} |
| 343 |
|
| 344 |
if asmSve, err := os.ReadFile(SVE_CODE); err != nil { |
| 345 |
log.Fatalf("Failed to read %s: %v", SVE_CODE, err) |
| 346 |
} else { |
| 347 |
// start with SVE code |
| 348 |
asmOut.WriteString(string(asmSve)) |
| 349 |
} |
| 350 |
if goSve, err := os.ReadFile(strings.ReplaceAll(SVE_CODE, ".s", ".go")); err != nil { |
| 351 |
log.Fatalf("Failed to read %s: %v", SVE_CODE, err) |
| 352 |
} else { |
| 353 |
goOut.WriteString(string(goSve)) |
| 354 |
} |
| 355 |
|
| 356 |
const input = 10 |
| 357 |
|
| 358 |
// Processing 64 bytes variants |
| 359 |
for output := 1; output <= 3; output++ { |
| 360 |
for op := ""; len(op) <= 3; op += "Xor" { |
| 361 |
templName := fmt.Sprintf("mulSve_%dx%d_64%s", input, output, op) |
| 362 |
funcDef := fmt.Sprintf("func %s(matrix []byte, in [][]byte, out [][]byte, start int, n int)", strings.ReplaceAll(templName, "Sve", "Neon")) |
| 363 |
|
| 364 |
lines, err := extractRoutine(SVE_CODE, fmt.Sprintf("TEXT ·%s(SB)", templName)) |
| 365 |
if err != nil { |
| 366 |
log.Fatal(err) |
| 367 |
} |
| 368 |
|
| 369 |
// prepend output with commented out function definition and comment |
| 370 |
asmOut.WriteString(fmt.Sprintf("// %s\n", funcDef)) |
| 371 |
asmOut.WriteString("// Requires: NEON\n") |
| 372 |
|
| 373 |
{ |
| 374 |
asmTemp := &bytes.Buffer{} |
| 375 |
convert2Neon(asmTemp, lines) |
| 376 |
fixPostIncrementNeon(asmOut, strings.Split(string(asmTemp.Bytes()), "\n")) |
| 377 |
} |
| 378 |
|
| 379 |
// golang declaration |
| 380 |
goOut.WriteString(fmt.Sprintf("//go:noescape\n%s\n\n", funcDef)) |
| 381 |
} |
| 382 |
} |
| 383 |
|
| 384 |
// Processing 32 bytes variants |
| 385 |
for output := 4; output <= 10; output++ { |
| 386 |
for op := ""; len(op) <= 3; op += "Xor" { |
| 387 |
templName := fmt.Sprintf("mulSve_%dx%d%s", input, output, op) |
| 388 |
funcDef := fmt.Sprintf("func %s(matrix []byte, in [][]byte, out [][]byte, start int, n int)", strings.ReplaceAll(templName, "Sve", "Neon")) |
| 389 |
|
| 390 |
lines, err := extractRoutine(SVE_CODE, fmt.Sprintf("TEXT ·%s(SB)", templName)) |
| 391 |
if err != nil { |
| 392 |
log.Fatal(err) |
| 393 |
} |
| 394 |
|
| 395 |
// prepend output with commented out function definition and comment |
| 396 |
asmOut.WriteString(fmt.Sprintf("// %s\n", funcDef)) |
| 397 |
asmOut.WriteString("// Requires: NEON\n") |
| 398 |
|
| 399 |
{ |
| 400 |
asmTemp := &bytes.Buffer{} |
| 401 |
convert2Neon(asmTemp, lines) |
| 402 |
fixPostIncrementNeon(asmOut, strings.Split(string(asmTemp.Bytes()), "\n")) |
| 403 |
} |
| 404 |
|
| 405 |
// golang declaration |
| 406 |
goOut.WriteString(fmt.Sprintf("//go:noescape\n%s\n", funcDef)) |
| 407 |
|
| 408 |
if !(output == 10 && op == "Xor") { |
| 409 |
goOut.WriteString("\n") |
| 410 |
} |
| 411 |
} |
| 412 |
} |
| 413 |
if err := os.WriteFile("../galois_gen_arm64.s", asmOut.Bytes(), 0644); err != nil { |
| 414 |
log.Fatal(err) |
| 415 |
} |
| 416 |
if err := os.WriteFile("../galois_gen_arm64.go", goOut.Bytes(), 0644); err != nil { |
| 417 |
log.Fatal(err) |
| 418 |
} |
| 419 |
} |