| 1 |
package arm64 |
| 2 |
|
| 3 |
import ( |
| 4 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/backend" |
| 5 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" |
| 6 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" |
| 7 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" |
| 8 |
) |
| 9 |
|
| 10 |
var calleeSavedRegistersSorted = []regalloc.VReg{ |
| 11 |
x19VReg, x20VReg, x21VReg, x22VReg, x23VReg, x24VReg, x25VReg, x26VReg, x28VReg, |
| 12 |
v18VReg, v19VReg, v20VReg, v21VReg, v22VReg, v23VReg, v24VReg, v25VReg, v26VReg, v27VReg, v28VReg, v29VReg, v30VReg, v31VReg, |
| 13 |
} |
| 14 |
|
| 15 |
// CompileGoFunctionTrampoline implements backend.Machine. |
| 16 |
func (m *machine) CompileGoFunctionTrampoline(exitCode wazevoapi.ExitCode, sig *ssa.Signature, needModuleContextPtr bool) []byte { |
| 17 |
argBegin := 1 // Skips exec context by default. |
| 18 |
if needModuleContextPtr { |
| 19 |
argBegin++ |
| 20 |
} |
| 21 |
|
| 22 |
abi := &backend.FunctionABI{} |
| 23 |
abi.Init(sig, intParamResultRegs, floatParamResultRegs) |
| 24 |
m.currentABI = abi |
| 25 |
|
| 26 |
cur := m.allocateInstr() |
| 27 |
cur.asNop0() |
| 28 |
m.rootInstr = cur |
| 29 |
|
| 30 |
// Execution context is always the first argument. |
| 31 |
execCtrPtr := x0VReg |
| 32 |
|
| 33 |
// In the following, we create the following stack layout: |
| 34 |
// |
| 35 |
// (high address) |
| 36 |
// SP ------> +-----------------+ <----+ |
| 37 |
// | ....... | | |
| 38 |
// | ret Y | | |
| 39 |
// | ....... | | |
| 40 |
// | ret 0 | | |
| 41 |
// | arg X | | size_of_arg_ret |
| 42 |
// | ....... | | |
| 43 |
// | arg 1 | | |
| 44 |
// | arg 0 | <----+ <-------- originalArg0Reg |
| 45 |
// | size_of_arg_ret | |
| 46 |
// | ReturnAddress | |
| 47 |
// +-----------------+ <----+ |
| 48 |
// | xxxx | | ;; might be padded to make it 16-byte aligned. |
| 49 |
// +--->| arg[N]/ret[M] | | |
| 50 |
// sliceSize| | ............ | | goCallStackSize |
| 51 |
// | | arg[1]/ret[1] | | |
| 52 |
// +--->| arg[0]/ret[0] | <----+ <-------- arg0ret0AddrReg |
| 53 |
// | sliceSize | |
| 54 |
// | frame_size | |
| 55 |
// +-----------------+ |
| 56 |
// (low address) |
| 57 |
// |
| 58 |
// where the region of "arg[0]/ret[0] ... arg[N]/ret[M]" is the stack used by the Go functions, |
| 59 |
// therefore will be accessed as the usual []uint64. So that's where we need to pass/receive |
| 60 |
// the arguments/return values. |
| 61 |
|
| 62 |
// First of all, to update the SP, and create "ReturnAddress + size_of_arg_ret". |
| 63 |
cur = m.createReturnAddrAndSizeOfArgRetSlot(cur) |
| 64 |
|
| 65 |
const frameInfoSize = 16 // == frame_size + sliceSize. |
| 66 |
|
| 67 |
// Next, we should allocate the stack for the Go function call if necessary. |
| 68 |
goCallStackSize, sliceSizeInBytes := backend.GoFunctionCallRequiredStackSize(sig, argBegin) |
| 69 |
cur = m.insertStackBoundsCheck(goCallStackSize+frameInfoSize, cur) |
| 70 |
|
| 71 |
originalArg0Reg := x17VReg // Caller save, so we can use it for whatever we want. |
| 72 |
if m.currentABI.AlignedArgResultStackSlotSize() > 0 { |
| 73 |
// At this point, SP points to `ReturnAddress`, so add 16 to get the original arg 0 slot. |
| 74 |
cur = m.addsAddOrSubStackPointer(cur, originalArg0Reg, frameInfoSize, true) |
| 75 |
} |
| 76 |
|
| 77 |
// Save the callee saved registers. |
| 78 |
cur = m.saveRegistersInExecutionContext(cur, calleeSavedRegistersSorted) |
| 79 |
|
| 80 |
if needModuleContextPtr { |
| 81 |
offset := wazevoapi.ExecutionContextOffsetGoFunctionCallCalleeModuleContextOpaque.I64() |
| 82 |
if !offsetFitsInAddressModeKindRegUnsignedImm12(64, offset) { |
| 83 |
panic("BUG: too large or un-aligned offset for goFunctionCallCalleeModuleContextOpaque in execution context") |
| 84 |
} |
| 85 |
|
| 86 |
// Module context is always the second argument. |
| 87 |
moduleCtrPtr := x1VReg |
| 88 |
store := m.allocateInstr() |
| 89 |
amode := m.amodePool.Allocate() |
| 90 |
*amode = addressMode{kind: addressModeKindRegUnsignedImm12, rn: execCtrPtr, imm: offset} |
| 91 |
store.asStore(operandNR(moduleCtrPtr), amode, 64) |
| 92 |
cur = linkInstr(cur, store) |
| 93 |
} |
| 94 |
|
| 95 |
// Advances the stack pointer. |
| 96 |
cur = m.addsAddOrSubStackPointer(cur, spVReg, goCallStackSize, false) |
| 97 |
|
| 98 |
// Copy the pointer to x15VReg. |
| 99 |
arg0ret0AddrReg := x15VReg // Caller save, so we can use it for whatever we want. |
| 100 |
copySp := m.allocateInstr() |
| 101 |
copySp.asMove64(arg0ret0AddrReg, spVReg) |
| 102 |
cur = linkInstr(cur, copySp) |
| 103 |
|
| 104 |
// Next, we need to store all the arguments to the stack in the typical Wasm stack style. |
| 105 |
for i := range abi.Args[argBegin:] { |
| 106 |
arg := &abi.Args[argBegin+i] |
| 107 |
store := m.allocateInstr() |
| 108 |
var v regalloc.VReg |
| 109 |
if arg.Kind == backend.ABIArgKindReg { |
| 110 |
v = arg.Reg |
| 111 |
} else { |
| 112 |
cur, v = m.goFunctionCallLoadStackArg(cur, originalArg0Reg, arg, |
| 113 |
// Caller save, so we can use it for whatever we want. |
| 114 |
x11VReg, v11VReg) |
| 115 |
} |
| 116 |
|
| 117 |
var sizeInBits byte |
| 118 |
if arg.Type == ssa.TypeV128 { |
| 119 |
sizeInBits = 128 |
| 120 |
} else { |
| 121 |
sizeInBits = 64 |
| 122 |
} |
| 123 |
amode := m.amodePool.Allocate() |
| 124 |
*amode = addressMode{kind: addressModeKindPostIndex, rn: arg0ret0AddrReg, imm: int64(sizeInBits / 8)} |
| 125 |
store.asStore(operandNR(v), amode, sizeInBits) |
| 126 |
cur = linkInstr(cur, store) |
| 127 |
} |
| 128 |
|
| 129 |
// Finally, now that we've advanced SP to arg[0]/ret[0], we allocate `frame_size + sliceSize`. |
| 130 |
var frameSizeReg, sliceSizeReg regalloc.VReg |
| 131 |
if goCallStackSize > 0 { |
| 132 |
cur = m.lowerConstantI64AndInsert(cur, tmpRegVReg, goCallStackSize) |
| 133 |
frameSizeReg = tmpRegVReg |
| 134 |
cur = m.lowerConstantI64AndInsert(cur, x16VReg, sliceSizeInBytes/8) |
| 135 |
sliceSizeReg = x16VReg |
| 136 |
} else { |
| 137 |
frameSizeReg = xzrVReg |
| 138 |
sliceSizeReg = xzrVReg |
| 139 |
} |
| 140 |
_amode := addressModePreOrPostIndex(m, spVReg, -16, true) |
| 141 |
storeP := m.allocateInstr() |
| 142 |
storeP.asStorePair64(frameSizeReg, sliceSizeReg, _amode) |
| 143 |
cur = linkInstr(cur, storeP) |
| 144 |
|
| 145 |
// Set the exit status on the execution context. |
| 146 |
cur = m.setExitCode(cur, x0VReg, exitCode) |
| 147 |
|
| 148 |
// Save the current stack pointer. |
| 149 |
cur = m.saveCurrentStackPointer(cur, x0VReg) |
| 150 |
|
| 151 |
// Exit the execution. |
| 152 |
cur = m.storeReturnAddressAndExit(cur) |
| 153 |
|
| 154 |
// After the call, we need to restore the callee saved registers. |
| 155 |
cur = m.restoreRegistersInExecutionContext(cur, calleeSavedRegistersSorted) |
| 156 |
|
| 157 |
// Get the pointer to the arg[0]/ret[0]: We need to skip `frame_size + sliceSize`. |
| 158 |
if len(abi.Rets) > 0 { |
| 159 |
cur = m.addsAddOrSubStackPointer(cur, arg0ret0AddrReg, frameInfoSize, true) |
| 160 |
} |
| 161 |
|
| 162 |
// Advances the SP so that it points to `ReturnAddress`. |
| 163 |
cur = m.addsAddOrSubStackPointer(cur, spVReg, frameInfoSize+goCallStackSize, true) |
| 164 |
ldr := m.allocateInstr() |
| 165 |
// And load the return address. |
| 166 |
amode := addressModePreOrPostIndex(m, spVReg, 16 /* stack pointer must be 16-byte aligned. */, false /* increment after loads */) |
| 167 |
ldr.asULoad(lrVReg, amode, 64) |
| 168 |
cur = linkInstr(cur, ldr) |
| 169 |
|
| 170 |
originalRet0Reg := x17VReg // Caller save, so we can use it for whatever we want. |
| 171 |
if m.currentABI.RetStackSize > 0 { |
| 172 |
cur = m.addsAddOrSubStackPointer(cur, originalRet0Reg, m.currentABI.ArgStackSize, true) |
| 173 |
} |
| 174 |
|
| 175 |
// Make the SP point to the original address (above the result slot). |
| 176 |
if s := int64(m.currentABI.AlignedArgResultStackSlotSize()); s > 0 { |
| 177 |
cur = m.addsAddOrSubStackPointer(cur, spVReg, s, true) |
| 178 |
} |
| 179 |
|
| 180 |
for i := range abi.Rets { |
| 181 |
r := &abi.Rets[i] |
| 182 |
if r.Kind == backend.ABIArgKindReg { |
| 183 |
loadIntoReg := m.allocateInstr() |
| 184 |
mode := m.amodePool.Allocate() |
| 185 |
*mode = addressMode{kind: addressModeKindPostIndex, rn: arg0ret0AddrReg} |
| 186 |
switch r.Type { |
| 187 |
case ssa.TypeI32: |
| 188 |
mode.imm = 8 // We use uint64 for all basic types, except SIMD v128. |
| 189 |
loadIntoReg.asULoad(r.Reg, mode, 32) |
| 190 |
case ssa.TypeI64: |
| 191 |
mode.imm = 8 // We use uint64 for all basic types, except SIMD v128. |
| 192 |
loadIntoReg.asULoad(r.Reg, mode, 64) |
| 193 |
case ssa.TypeF32: |
| 194 |
mode.imm = 8 // We use uint64 for all basic types, except SIMD v128. |
| 195 |
loadIntoReg.asFpuLoad(r.Reg, mode, 32) |
| 196 |
case ssa.TypeF64: |
| 197 |
mode.imm = 8 // We use uint64 for all basic types, except SIMD v128. |
| 198 |
loadIntoReg.asFpuLoad(r.Reg, mode, 64) |
| 199 |
case ssa.TypeV128: |
| 200 |
mode.imm = 16 |
| 201 |
loadIntoReg.asFpuLoad(r.Reg, mode, 128) |
| 202 |
default: |
| 203 |
panic("TODO") |
| 204 |
} |
| 205 |
cur = linkInstr(cur, loadIntoReg) |
| 206 |
} else { |
| 207 |
// First we need to load the value to a temporary just like ^^. |
| 208 |
intTmp, floatTmp := x11VReg, v11VReg |
| 209 |
loadIntoTmpReg := m.allocateInstr() |
| 210 |
mode := m.amodePool.Allocate() |
| 211 |
*mode = addressMode{kind: addressModeKindPostIndex, rn: arg0ret0AddrReg} |
| 212 |
var resultReg regalloc.VReg |
| 213 |
switch r.Type { |
| 214 |
case ssa.TypeI32: |
| 215 |
mode.imm = 8 // We use uint64 for all basic types, except SIMD v128. |
| 216 |
loadIntoTmpReg.asULoad(intTmp, mode, 32) |
| 217 |
resultReg = intTmp |
| 218 |
case ssa.TypeI64: |
| 219 |
mode.imm = 8 // We use uint64 for all basic types, except SIMD v128. |
| 220 |
loadIntoTmpReg.asULoad(intTmp, mode, 64) |
| 221 |
resultReg = intTmp |
| 222 |
case ssa.TypeF32: |
| 223 |
mode.imm = 8 // We use uint64 for all basic types, except SIMD v128. |
| 224 |
loadIntoTmpReg.asFpuLoad(floatTmp, mode, 32) |
| 225 |
resultReg = floatTmp |
| 226 |
case ssa.TypeF64: |
| 227 |
mode.imm = 8 // We use uint64 for all basic types, except SIMD v128. |
| 228 |
loadIntoTmpReg.asFpuLoad(floatTmp, mode, 64) |
| 229 |
resultReg = floatTmp |
| 230 |
case ssa.TypeV128: |
| 231 |
mode.imm = 16 |
| 232 |
loadIntoTmpReg.asFpuLoad(floatTmp, mode, 128) |
| 233 |
resultReg = floatTmp |
| 234 |
default: |
| 235 |
panic("TODO") |
| 236 |
} |
| 237 |
cur = linkInstr(cur, loadIntoTmpReg) |
| 238 |
cur = m.goFunctionCallStoreStackResult(cur, originalRet0Reg, r, resultReg) |
| 239 |
} |
| 240 |
} |
| 241 |
|
| 242 |
ret := m.allocateInstr() |
| 243 |
ret.asRet() |
| 244 |
linkInstr(cur, ret) |
| 245 |
|
| 246 |
m.encode(m.rootInstr) |
| 247 |
return m.compiler.Buf() |
| 248 |
} |
| 249 |
|
| 250 |
func (m *machine) saveRegistersInExecutionContext(cur *instruction, regs []regalloc.VReg) *instruction { |
| 251 |
offset := wazevoapi.ExecutionContextOffsetSavedRegistersBegin.I64() |
| 252 |
for _, v := range regs { |
| 253 |
store := m.allocateInstr() |
| 254 |
var sizeInBits byte |
| 255 |
switch v.RegType() { |
| 256 |
case regalloc.RegTypeInt: |
| 257 |
sizeInBits = 64 |
| 258 |
case regalloc.RegTypeFloat: |
| 259 |
sizeInBits = 128 |
| 260 |
} |
| 261 |
mode := m.amodePool.Allocate() |
| 262 |
*mode = addressMode{ |
| 263 |
kind: addressModeKindRegUnsignedImm12, |
| 264 |
// Execution context is always the first argument. |
| 265 |
rn: x0VReg, imm: offset, |
| 266 |
} |
| 267 |
store.asStore(operandNR(v), mode, sizeInBits) |
| 268 |
store.prev = cur |
| 269 |
cur.next = store |
| 270 |
cur = store |
| 271 |
offset += 16 // Imm12 must be aligned 16 for vector regs, so we unconditionally store regs at the offset of multiple of 16. |
| 272 |
} |
| 273 |
return cur |
| 274 |
} |
| 275 |
|
| 276 |
func (m *machine) restoreRegistersInExecutionContext(cur *instruction, regs []regalloc.VReg) *instruction { |
| 277 |
offset := wazevoapi.ExecutionContextOffsetSavedRegistersBegin.I64() |
| 278 |
for _, v := range regs { |
| 279 |
load := m.allocateInstr() |
| 280 |
var as func(dst regalloc.VReg, amode *addressMode, sizeInBits byte) |
| 281 |
var sizeInBits byte |
| 282 |
switch v.RegType() { |
| 283 |
case regalloc.RegTypeInt: |
| 284 |
as = load.asULoad |
| 285 |
sizeInBits = 64 |
| 286 |
case regalloc.RegTypeFloat: |
| 287 |
as = load.asFpuLoad |
| 288 |
sizeInBits = 128 |
| 289 |
} |
| 290 |
mode := m.amodePool.Allocate() |
| 291 |
*mode = addressMode{ |
| 292 |
kind: addressModeKindRegUnsignedImm12, |
| 293 |
// Execution context is always the first argument. |
| 294 |
rn: x0VReg, imm: offset, |
| 295 |
} |
| 296 |
as(v, mode, sizeInBits) |
| 297 |
cur = linkInstr(cur, load) |
| 298 |
offset += 16 // Imm12 must be aligned 16 for vector regs, so we unconditionally load regs at the offset of multiple of 16. |
| 299 |
} |
| 300 |
return cur |
| 301 |
} |
| 302 |
|
| 303 |
func (m *machine) lowerConstantI64AndInsert(cur *instruction, dst regalloc.VReg, v int64) *instruction { |
| 304 |
m.pendingInstructions = m.pendingInstructions[:0] |
| 305 |
m.lowerConstantI64(dst, v) |
| 306 |
for _, instr := range m.pendingInstructions { |
| 307 |
cur = linkInstr(cur, instr) |
| 308 |
} |
| 309 |
return cur |
| 310 |
} |
| 311 |
|
| 312 |
func (m *machine) lowerConstantI32AndInsert(cur *instruction, dst regalloc.VReg, v int32) *instruction { |
| 313 |
m.pendingInstructions = m.pendingInstructions[:0] |
| 314 |
m.lowerConstantI32(dst, v) |
| 315 |
for _, instr := range m.pendingInstructions { |
| 316 |
cur = linkInstr(cur, instr) |
| 317 |
} |
| 318 |
return cur |
| 319 |
} |
| 320 |
|
| 321 |
func (m *machine) setExitCode(cur *instruction, execCtr regalloc.VReg, exitCode wazevoapi.ExitCode) *instruction { |
| 322 |
constReg := x17VReg // caller-saved, so we can use it. |
| 323 |
cur = m.lowerConstantI32AndInsert(cur, constReg, int32(exitCode)) |
| 324 |
|
| 325 |
// Set the exit status on the execution context. |
| 326 |
setExistStatus := m.allocateInstr() |
| 327 |
mode := m.amodePool.Allocate() |
| 328 |
*mode = addressMode{kind: addressModeKindRegUnsignedImm12, rn: execCtr, imm: wazevoapi.ExecutionContextOffsetExitCodeOffset.I64()} |
| 329 |
setExistStatus.asStore(operandNR(constReg), mode, 32) |
| 330 |
cur = linkInstr(cur, setExistStatus) |
| 331 |
return cur |
| 332 |
} |
| 333 |
|
| 334 |
func (m *machine) storeReturnAddressAndExit(cur *instruction) *instruction { |
| 335 |
// Read the return address into tmp, and store it in the execution context. |
| 336 |
adr := m.allocateInstr() |
| 337 |
adr.asAdr(tmpRegVReg, exitSequenceSize+8) |
| 338 |
cur = linkInstr(cur, adr) |
| 339 |
|
| 340 |
storeReturnAddr := m.allocateInstr() |
| 341 |
mode := m.amodePool.Allocate() |
| 342 |
*mode = addressMode{ |
| 343 |
kind: addressModeKindRegUnsignedImm12, |
| 344 |
// Execution context is always the first argument. |
| 345 |
rn: x0VReg, imm: wazevoapi.ExecutionContextOffsetGoCallReturnAddress.I64(), |
| 346 |
} |
| 347 |
storeReturnAddr.asStore(operandNR(tmpRegVReg), mode, 64) |
| 348 |
cur = linkInstr(cur, storeReturnAddr) |
| 349 |
|
| 350 |
// Exit the execution. |
| 351 |
trapSeq := m.allocateInstr() |
| 352 |
trapSeq.asExitSequence(x0VReg) |
| 353 |
cur = linkInstr(cur, trapSeq) |
| 354 |
return cur |
| 355 |
} |
| 356 |
|
| 357 |
func (m *machine) saveCurrentStackPointer(cur *instruction, execCtr regalloc.VReg) *instruction { |
| 358 |
// Save the current stack pointer: |
| 359 |
// mov tmp, sp, |
| 360 |
// str tmp, [exec_ctx, #stackPointerBeforeGoCall] |
| 361 |
movSp := m.allocateInstr() |
| 362 |
movSp.asMove64(tmpRegVReg, spVReg) |
| 363 |
cur = linkInstr(cur, movSp) |
| 364 |
|
| 365 |
strSp := m.allocateInstr() |
| 366 |
mode := m.amodePool.Allocate() |
| 367 |
*mode = addressMode{ |
| 368 |
kind: addressModeKindRegUnsignedImm12, |
| 369 |
rn: execCtr, imm: wazevoapi.ExecutionContextOffsetStackPointerBeforeGoCall.I64(), |
| 370 |
} |
| 371 |
strSp.asStore(operandNR(tmpRegVReg), mode, 64) |
| 372 |
cur = linkInstr(cur, strSp) |
| 373 |
return cur |
| 374 |
} |
| 375 |
|
| 376 |
func (m *machine) goFunctionCallLoadStackArg(cur *instruction, originalArg0Reg regalloc.VReg, arg *backend.ABIArg, intVReg, floatVReg regalloc.VReg) (*instruction, regalloc.VReg) { |
| 377 |
load := m.allocateInstr() |
| 378 |
var result regalloc.VReg |
| 379 |
mode := m.amodePool.Allocate() |
| 380 |
*mode = addressMode{kind: addressModeKindPostIndex, rn: originalArg0Reg} |
| 381 |
switch arg.Type { |
| 382 |
case ssa.TypeI32: |
| 383 |
mode.imm = 8 // We use uint64 for all basic types, except SIMD v128. |
| 384 |
load.asULoad(intVReg, mode, 32) |
| 385 |
result = intVReg |
| 386 |
case ssa.TypeI64: |
| 387 |
mode.imm = 8 // We use uint64 for all basic types, except SIMD v128. |
| 388 |
load.asULoad(intVReg, mode, 64) |
| 389 |
result = intVReg |
| 390 |
case ssa.TypeF32: |
| 391 |
mode.imm = 8 // We use uint64 for all basic types, except SIMD v128. |
| 392 |
load.asFpuLoad(floatVReg, mode, 32) |
| 393 |
result = floatVReg |
| 394 |
case ssa.TypeF64: |
| 395 |
mode.imm = 8 // We use uint64 for all basic types, except SIMD v128. |
| 396 |
load.asFpuLoad(floatVReg, mode, 64) |
| 397 |
result = floatVReg |
| 398 |
case ssa.TypeV128: |
| 399 |
mode.imm = 16 |
| 400 |
load.asFpuLoad(floatVReg, mode, 128) |
| 401 |
result = floatVReg |
| 402 |
default: |
| 403 |
panic("TODO") |
| 404 |
} |
| 405 |
|
| 406 |
cur = linkInstr(cur, load) |
| 407 |
return cur, result |
| 408 |
} |
| 409 |
|
| 410 |
func (m *machine) goFunctionCallStoreStackResult(cur *instruction, originalRet0Reg regalloc.VReg, result *backend.ABIArg, resultVReg regalloc.VReg) *instruction { |
| 411 |
store := m.allocateInstr() |
| 412 |
mode := m.amodePool.Allocate() |
| 413 |
*mode = addressMode{kind: addressModeKindPostIndex, rn: originalRet0Reg} |
| 414 |
var sizeInBits byte |
| 415 |
switch result.Type { |
| 416 |
case ssa.TypeI32, ssa.TypeF32: |
| 417 |
mode.imm = 8 |
| 418 |
sizeInBits = 32 |
| 419 |
case ssa.TypeI64, ssa.TypeF64: |
| 420 |
mode.imm = 8 |
| 421 |
sizeInBits = 64 |
| 422 |
case ssa.TypeV128: |
| 423 |
mode.imm = 16 |
| 424 |
sizeInBits = 128 |
| 425 |
default: |
| 426 |
panic("TODO") |
| 427 |
} |
| 428 |
store.asStore(operandNR(resultVReg), mode, sizeInBits) |
| 429 |
return linkInstr(cur, store) |
| 430 |
} |