| 1 |
package arm64 |
| 2 |
|
| 3 |
import ( |
| 4 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" |
| 5 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" |
| 6 |
) |
| 7 |
|
| 8 |
// lowerConstant allocates a new VReg and inserts the instruction to load the constant value. |
| 9 |
func (m *machine) lowerConstant(instr *ssa.Instruction) (vr regalloc.VReg) { |
| 10 |
val := instr.Return() |
| 11 |
valType := val.Type() |
| 12 |
|
| 13 |
vr = m.compiler.AllocateVReg(valType) |
| 14 |
v := instr.ConstantVal() |
| 15 |
m.insertLoadConstant(v, valType, vr) |
| 16 |
return |
| 17 |
} |
| 18 |
|
| 19 |
// InsertLoadConstantBlockArg implements backend.Machine. |
| 20 |
func (m *machine) InsertLoadConstantBlockArg(instr *ssa.Instruction, vr regalloc.VReg) { |
| 21 |
val := instr.Return() |
| 22 |
valType := val.Type() |
| 23 |
v := instr.ConstantVal() |
| 24 |
load := m.allocateInstr() |
| 25 |
load.asLoadConstBlockArg(v, valType, vr) |
| 26 |
m.insert(load) |
| 27 |
} |
| 28 |
|
| 29 |
func (m *machine) lowerLoadConstantBlockArgAfterRegAlloc(i *instruction) { |
| 30 |
v, typ, dst := i.loadConstBlockArgData() |
| 31 |
m.insertLoadConstant(v, typ, dst) |
| 32 |
} |
| 33 |
|
| 34 |
func (m *machine) insertLoadConstant(v uint64, valType ssa.Type, vr regalloc.VReg) { |
| 35 |
if valType.Bits() < 64 { // Clear the redundant bits just in case it's unexpectedly sign-extended, etc. |
| 36 |
v = v & ((1 << valType.Bits()) - 1) |
| 37 |
} |
| 38 |
|
| 39 |
switch valType { |
| 40 |
case ssa.TypeF32: |
| 41 |
loadF := m.allocateInstr() |
| 42 |
loadF.asLoadFpuConst32(vr, v) |
| 43 |
m.insert(loadF) |
| 44 |
case ssa.TypeF64: |
| 45 |
loadF := m.allocateInstr() |
| 46 |
loadF.asLoadFpuConst64(vr, v) |
| 47 |
m.insert(loadF) |
| 48 |
case ssa.TypeI32: |
| 49 |
if v == 0 { |
| 50 |
m.InsertMove(vr, xzrVReg, ssa.TypeI32) |
| 51 |
} else { |
| 52 |
m.lowerConstantI32(vr, int32(v)) |
| 53 |
} |
| 54 |
case ssa.TypeI64: |
| 55 |
if v == 0 { |
| 56 |
m.InsertMove(vr, xzrVReg, ssa.TypeI64) |
| 57 |
} else { |
| 58 |
m.lowerConstantI64(vr, int64(v)) |
| 59 |
} |
| 60 |
default: |
| 61 |
panic("TODO") |
| 62 |
} |
| 63 |
} |
| 64 |
|
| 65 |
// The following logics are based on the old asm/arm64 package. |
| 66 |
// https://github.com/tetratelabs/wazero/blob/39f2ff23a6d609e10c82b9cc0b981f6de5b87a9c/internal/asm/arm64/impl.go |
| 67 |
|
| 68 |
func (m *machine) lowerConstantI32(dst regalloc.VReg, c int32) { |
| 69 |
// Following the logic here: |
| 70 |
// https://github.com/golang/go/blob/release-branch.go1.15/src/cmd/internal/obj/arm64/asm7.go#L1637 |
| 71 |
ic := int64(uint32(c)) |
| 72 |
if ic >= 0 && (ic <= 0xfff || (ic&0xfff) == 0 && (uint64(ic>>12) <= 0xfff)) { |
| 73 |
if isBitMaskImmediate(uint64(c), false) { |
| 74 |
m.lowerConstViaBitMaskImmediate(uint64(uint32(c)), dst, false) |
| 75 |
return |
| 76 |
} |
| 77 |
} |
| 78 |
|
| 79 |
if t := const16bitAligned(int64(uint32(c))); t >= 0 { |
| 80 |
// If the const can fit within 16-bit alignment, for example, 0xffff, 0xffff_0000 or 0xffff_0000_0000_0000 |
| 81 |
// We could load it into temporary with movk. |
| 82 |
m.insertMOVZ(dst, uint64(uint32(c)>>(16*t)), t, false) |
| 83 |
} else if t := const16bitAligned(int64(^c)); t >= 0 { |
| 84 |
// Also, if the inverse of the const can fit within 16-bit range, do the same ^^. |
| 85 |
m.insertMOVN(dst, uint64(^c>>(16*t)), t, false) |
| 86 |
} else if isBitMaskImmediate(uint64(uint32(c)), false) { |
| 87 |
m.lowerConstViaBitMaskImmediate(uint64(c), dst, false) |
| 88 |
} else { |
| 89 |
// Otherwise, we use MOVZ and MOVK to load it. |
| 90 |
c16 := uint16(c) |
| 91 |
m.insertMOVZ(dst, uint64(c16), 0, false) |
| 92 |
c16 = uint16(uint32(c) >> 16) |
| 93 |
m.insertMOVK(dst, uint64(c16), 1, false) |
| 94 |
} |
| 95 |
} |
| 96 |
|
| 97 |
func (m *machine) lowerConstantI64(dst regalloc.VReg, c int64) { |
| 98 |
// Following the logic here: |
| 99 |
// https://github.com/golang/go/blob/go1.24.0/src/cmd/internal/obj/arm64/asm7.go#L2161-L2215 |
| 100 |
if c >= 0 && (c <= 0xfff || (c&0xfff) == 0 && (uint64(c>>12) <= 0xfff)) { |
| 101 |
if isBitMaskImmediate(uint64(c), true) { |
| 102 |
m.lowerConstViaBitMaskImmediate(uint64(c), dst, true) |
| 103 |
return |
| 104 |
} |
| 105 |
} |
| 106 |
|
| 107 |
if t := const16bitAligned(c); t >= 0 { |
| 108 |
// If the const can fit within 16-bit alignment, for example, 0xffff, 0xffff_0000 or 0xffff_0000_0000_0000 |
| 109 |
// We could load it into temporary with movk. |
| 110 |
m.insertMOVZ(dst, uint64(c)>>(16*t), t, true) |
| 111 |
} else if t := const16bitAligned(^c); t >= 0 { |
| 112 |
// Also, if the reverse of the const can fit within 16-bit range, do the same ^^. |
| 113 |
m.insertMOVN(dst, uint64(^c)>>(16*t), t, true) |
| 114 |
} else if isBitMaskImmediate(uint64(c), true) { |
| 115 |
m.lowerConstViaBitMaskImmediate(uint64(c), dst, true) |
| 116 |
} else { |
| 117 |
m.load64bitConst(c, dst) |
| 118 |
} |
| 119 |
} |
| 120 |
|
| 121 |
func (m *machine) lowerConstViaBitMaskImmediate(c uint64, dst regalloc.VReg, b64 bool) { |
| 122 |
instr := m.allocateInstr() |
| 123 |
instr.asALUBitmaskImm(aluOpOrr, dst, xzrVReg, c, b64) |
| 124 |
m.insert(instr) |
| 125 |
} |
| 126 |
|
| 127 |
// isBitMaskImmediate determines if the value can be encoded as "bitmask immediate". |
| 128 |
// |
| 129 |
// Such an immediate is a 32-bit or 64-bit pattern viewed as a vector of identical elements of size e = 2, 4, 8, 16, 32, or 64 bits. |
| 130 |
// Each element contains the same sub-pattern: a single run of 1 to e-1 non-zero bits, rotated by 0 to e-1 bits. |
| 131 |
// |
| 132 |
// See https://developer.arm.com/documentation/dui0802/b/A64-General-Instructions/MOV--bitmask-immediate- |
| 133 |
func isBitMaskImmediate(x uint64, _64 bool) bool { |
| 134 |
// All zeros and ones are not "bitmask immediate" by definition. |
| 135 |
if x == 0 || (_64 && x == 0xffff_ffff_ffff_ffff) || (!_64 && x == 0xffff_ffff) { |
| 136 |
return false |
| 137 |
} |
| 138 |
|
| 139 |
switch { |
| 140 |
case x != x>>32|x<<32: |
| 141 |
// e = 64 |
| 142 |
case x != x>>16|x<<48: |
| 143 |
// e = 32 (x == x>>32|x<<32). |
| 144 |
// e.g. 0x00ff_ff00_00ff_ff00 |
| 145 |
x = uint64(int32(x)) |
| 146 |
case x != x>>8|x<<56: |
| 147 |
// e = 16 (x == x>>16|x<<48). |
| 148 |
// e.g. 0x00ff_00ff_00ff_00ff |
| 149 |
x = uint64(int16(x)) |
| 150 |
case x != x>>4|x<<60: |
| 151 |
// e = 8 (x == x>>8|x<<56). |
| 152 |
// e.g. 0x0f0f_0f0f_0f0f_0f0f |
| 153 |
x = uint64(int8(x)) |
| 154 |
default: |
| 155 |
// e = 4 or 2. |
| 156 |
return true |
| 157 |
} |
| 158 |
return sequenceOfSetbits(x) || sequenceOfSetbits(^x) |
| 159 |
} |
| 160 |
|
| 161 |
// sequenceOfSetbits returns true if the number's binary representation is the sequence set bit (1). |
| 162 |
// For example: 0b1110 -> true, 0b1010 -> false |
| 163 |
func sequenceOfSetbits(x uint64) bool { |
| 164 |
y := getLowestBit(x) |
| 165 |
// If x is a sequence of set bit, this should results in the number |
| 166 |
// with only one set bit (i.e. power of two). |
| 167 |
y += x |
| 168 |
return (y-1)&y == 0 |
| 169 |
} |
| 170 |
|
| 171 |
func getLowestBit(x uint64) uint64 { |
| 172 |
return x & (^x + 1) |
| 173 |
} |
| 174 |
|
| 175 |
// const16bitAligned check if the value is on the 16-bit alignment. |
| 176 |
// If so, returns the shift num divided by 16, and otherwise -1. |
| 177 |
func const16bitAligned(v int64) (ret int) { |
| 178 |
ret = -1 |
| 179 |
for s := 0; s < 64; s += 16 { |
| 180 |
if (uint64(v) &^ (uint64(0xffff) << uint(s))) == 0 { |
| 181 |
ret = s / 16 |
| 182 |
break |
| 183 |
} |
| 184 |
} |
| 185 |
return |
| 186 |
} |
| 187 |
|
| 188 |
// load64bitConst loads a 64-bit constant into the register, following the same logic to decide how to load large 64-bit |
| 189 |
// consts as in the Go assembler. |
| 190 |
// |
| 191 |
// See https://github.com/golang/go/blob/go1.24.0/src/cmd/internal/obj/arm64/asm7.go#L7555-L7682 |
| 192 |
func (m *machine) load64bitConst(c int64, dst regalloc.VReg) { |
| 193 |
var bits [4]uint64 |
| 194 |
var zeros, negs int |
| 195 |
for i := 0; i < 4; i++ { |
| 196 |
bits[i] = uint64(c) >> uint(i*16) & 0xffff |
| 197 |
if v := bits[i]; v == 0 { |
| 198 |
zeros++ |
| 199 |
} else if v == 0xffff { |
| 200 |
negs++ |
| 201 |
} |
| 202 |
} |
| 203 |
|
| 204 |
if zeros == 3 { |
| 205 |
// one MOVZ instruction. |
| 206 |
for i, v := range bits { |
| 207 |
if v != 0 { |
| 208 |
m.insertMOVZ(dst, v, i, true) |
| 209 |
} |
| 210 |
} |
| 211 |
} else if negs == 3 { |
| 212 |
// one MOVN instruction. |
| 213 |
for i, v := range bits { |
| 214 |
if v != 0xffff { |
| 215 |
v = ^v |
| 216 |
m.insertMOVN(dst, v, i, true) |
| 217 |
} |
| 218 |
} |
| 219 |
} else if zeros == 2 { |
| 220 |
// one MOVZ then one OVK. |
| 221 |
var movz bool |
| 222 |
for i, v := range bits { |
| 223 |
if !movz && v != 0 { // MOVZ. |
| 224 |
m.insertMOVZ(dst, v, i, true) |
| 225 |
movz = true |
| 226 |
} else if v != 0 { |
| 227 |
m.insertMOVK(dst, v, i, true) |
| 228 |
} |
| 229 |
} |
| 230 |
|
| 231 |
} else if negs == 2 { |
| 232 |
// one MOVN then one or two MOVK. |
| 233 |
var movn bool |
| 234 |
for i, v := range bits { // Emit MOVN. |
| 235 |
if !movn && v != 0xffff { |
| 236 |
v = ^v |
| 237 |
// https://developer.arm.com/documentation/dui0802/a/A64-General-Instructions/MOVN |
| 238 |
m.insertMOVN(dst, v, i, true) |
| 239 |
movn = true |
| 240 |
} else if v != 0xffff { |
| 241 |
m.insertMOVK(dst, v, i, true) |
| 242 |
} |
| 243 |
} |
| 244 |
|
| 245 |
} else if zeros == 1 { |
| 246 |
// one MOVZ then two MOVK. |
| 247 |
var movz bool |
| 248 |
for i, v := range bits { |
| 249 |
if !movz && v != 0 { // MOVZ. |
| 250 |
m.insertMOVZ(dst, v, i, true) |
| 251 |
movz = true |
| 252 |
} else if v != 0 { |
| 253 |
m.insertMOVK(dst, v, i, true) |
| 254 |
} |
| 255 |
} |
| 256 |
|
| 257 |
} else if negs == 1 { |
| 258 |
// one MOVN then two MOVK. |
| 259 |
var movn bool |
| 260 |
for i, v := range bits { // Emit MOVN. |
| 261 |
if !movn && v != 0xffff { |
| 262 |
v = ^v |
| 263 |
// https://developer.arm.com/documentation/dui0802/a/A64-General-Instructions/MOVN |
| 264 |
m.insertMOVN(dst, v, i, true) |
| 265 |
movn = true |
| 266 |
} else if v != 0xffff { |
| 267 |
m.insertMOVK(dst, v, i, true) |
| 268 |
} |
| 269 |
} |
| 270 |
|
| 271 |
} else { |
| 272 |
// one MOVZ then up to three MOVK. |
| 273 |
var movz bool |
| 274 |
for i, v := range bits { |
| 275 |
if !movz && v != 0 { // MOVZ. |
| 276 |
m.insertMOVZ(dst, v, i, true) |
| 277 |
movz = true |
| 278 |
} else if v != 0 { |
| 279 |
m.insertMOVK(dst, v, i, true) |
| 280 |
} |
| 281 |
} |
| 282 |
} |
| 283 |
} |
| 284 |
|
| 285 |
func (m *machine) insertMOVZ(dst regalloc.VReg, v uint64, shift int, dst64 bool) { |
| 286 |
instr := m.allocateInstr() |
| 287 |
instr.asMOVZ(dst, v, uint32(shift), dst64) |
| 288 |
m.insert(instr) |
| 289 |
} |
| 290 |
|
| 291 |
func (m *machine) insertMOVK(dst regalloc.VReg, v uint64, shift int, dst64 bool) { |
| 292 |
instr := m.allocateInstr() |
| 293 |
instr.asMOVK(dst, v, uint32(shift), dst64) |
| 294 |
m.insert(instr) |
| 295 |
} |
| 296 |
|
| 297 |
func (m *machine) insertMOVN(dst regalloc.VReg, v uint64, shift int, dst64 bool) { |
| 298 |
instr := m.allocateInstr() |
| 299 |
instr.asMOVN(dst, v, uint32(shift), dst64) |
| 300 |
m.insert(instr) |
| 301 |
} |