| 1 |
/* SPDX-License-Identifier: GPL-2.0-only */ |
| 2 |
/* |
| 3 |
* Just-In-Time compiler for eBPF bytecode on 32-bit and 64-bit MIPS. |
| 4 |
* |
| 5 |
* Copyright (c) 2021 Anyfi Networks AB. |
| 6 |
* Author: Johan Almbladh <[email protected]> |
| 7 |
* |
| 8 |
* Based on code and ideas from |
| 9 |
* Copyright (c) 2017 Cavium, Inc. |
| 10 |
* Copyright (c) 2017 Shubham Bansal <[email protected]> |
| 11 |
* Copyright (c) 2011 Mircea Gherzan <[email protected]> |
| 12 |
*/ |
| 13 |
|
| 14 |
#ifndef _BPF_JIT_COMP_H |
| 15 |
#define _BPF_JIT_COMP_H |
| 16 |
|
| 17 |
/* MIPS registers */ |
| 18 |
#define MIPS_R_ZERO 0 /* Const zero */ |
| 19 |
#define MIPS_R_AT 1 /* Asm temp */ |
| 20 |
#define MIPS_R_V0 2 /* Result */ |
| 21 |
#define MIPS_R_V1 3 /* Result */ |
| 22 |
#define MIPS_R_A0 4 /* Argument */ |
| 23 |
#define MIPS_R_A1 5 /* Argument */ |
| 24 |
#define MIPS_R_A2 6 /* Argument */ |
| 25 |
#define MIPS_R_A3 7 /* Argument */ |
| 26 |
#define MIPS_R_A4 8 /* Arg (n64) */ |
| 27 |
#define MIPS_R_A5 9 /* Arg (n64) */ |
| 28 |
#define MIPS_R_A6 10 /* Arg (n64) */ |
| 29 |
#define MIPS_R_A7 11 /* Arg (n64) */ |
| 30 |
#define MIPS_R_T0 8 /* Temp (o32) */ |
| 31 |
#define MIPS_R_T1 9 /* Temp (o32) */ |
| 32 |
#define MIPS_R_T2 10 /* Temp (o32) */ |
| 33 |
#define MIPS_R_T3 11 /* Temp (o32) */ |
| 34 |
#define MIPS_R_T4 12 /* Temporary */ |
| 35 |
#define MIPS_R_T5 13 /* Temporary */ |
| 36 |
#define MIPS_R_T6 14 /* Temporary */ |
| 37 |
#define MIPS_R_T7 15 /* Temporary */ |
| 38 |
#define MIPS_R_S0 16 /* Saved */ |
| 39 |
#define MIPS_R_S1 17 /* Saved */ |
| 40 |
#define MIPS_R_S2 18 /* Saved */ |
| 41 |
#define MIPS_R_S3 19 /* Saved */ |
| 42 |
#define MIPS_R_S4 20 /* Saved */ |
| 43 |
#define MIPS_R_S5 21 /* Saved */ |
| 44 |
#define MIPS_R_S6 22 /* Saved */ |
| 45 |
#define MIPS_R_S7 23 /* Saved */ |
| 46 |
#define MIPS_R_T8 24 /* Temporary */ |
| 47 |
#define MIPS_R_T9 25 /* Temporary */ |
| 48 |
/* MIPS_R_K0 26 Reserved */ |
| 49 |
/* MIPS_R_K1 27 Reserved */ |
| 50 |
#define MIPS_R_GP 28 /* Global ptr */ |
| 51 |
#define MIPS_R_SP 29 /* Stack ptr */ |
| 52 |
#define MIPS_R_FP 30 /* Frame ptr */ |
| 53 |
#define MIPS_R_RA 31 /* Return */ |
| 54 |
|
| 55 |
/* |
| 56 |
* Jump address mask for immediate jumps. The four most significant bits |
| 57 |
* must be equal to PC. |
| 58 |
*/ |
| 59 |
#define MIPS_JMP_MASK 0x0fffffffUL |
| 60 |
|
| 61 |
/* Maximum number of iterations in offset table computation */ |
| 62 |
#define JIT_MAX_ITERATIONS 8 |
| 63 |
|
| 64 |
/* |
| 65 |
* Jump pseudo-instructions used internally |
| 66 |
* for branch conversion and branch optimization. |
| 67 |
*/ |
| 68 |
#define JIT_JNSET 0xe0 |
| 69 |
#define JIT_JNOP 0xf0 |
| 70 |
|
| 71 |
/* Descriptor flag for PC-relative branch conversion */ |
| 72 |
#define JIT_DESC_CONVERT BIT(31) |
| 73 |
|
| 74 |
/* JIT context for an eBPF program */ |
| 75 |
struct jit_context { |
| 76 |
struct bpf_prog *program; /* The eBPF program being JITed */ |
| 77 |
u32 *descriptors; /* eBPF to JITed CPU insn descriptors */ |
| 78 |
u32 *target; /* JITed code buffer */ |
| 79 |
u32 bpf_index; /* Index of current BPF program insn */ |
| 80 |
u32 jit_index; /* Index of current JIT target insn */ |
| 81 |
u32 changes; /* Number of PC-relative branch conv */ |
| 82 |
u32 accessed; /* Bit mask of read eBPF registers */ |
| 83 |
u32 clobbered; /* Bit mask of modified CPU registers */ |
| 84 |
u32 stack_size; /* Total allocated stack size in bytes */ |
| 85 |
u32 saved_size; /* Size of callee-saved registers */ |
| 86 |
u32 stack_used; /* Stack size used for function calls */ |
| 87 |
}; |
| 88 |
|
| 89 |
/* Emit the instruction if the JIT memory space has been allocated */ |
| 90 |
#define __emit(ctx, func, ...) \ |
| 91 |
do { \ |
| 92 |
if ((ctx)->target != NULL) { \ |
| 93 |
u32 *p = &(ctx)->target[ctx->jit_index]; \ |
| 94 |
uasm_i_##func(&p, ##__VA_ARGS__); \ |
| 95 |
} \ |
| 96 |
(ctx)->jit_index++; \ |
| 97 |
} while (0) |
| 98 |
#define emit(...) __emit(__VA_ARGS__) |
| 99 |
|
| 100 |
/* Workaround for R10000 ll/sc errata */ |
| 101 |
#ifdef CONFIG_WAR_R10000_LLSC |
| 102 |
#define LLSC_beqz beqzl |
| 103 |
#else |
| 104 |
#define LLSC_beqz beqz |
| 105 |
#endif |
| 106 |
|
| 107 |
/* Workaround for Loongson-3 ll/sc errata */ |
| 108 |
#ifdef CONFIG_CPU_LOONGSON3_WORKAROUNDS |
| 109 |
#define LLSC_sync(ctx) emit(ctx, sync, 0) |
| 110 |
#define LLSC_offset 4 |
| 111 |
#else |
| 112 |
#define LLSC_sync(ctx) |
| 113 |
#define LLSC_offset 0 |
| 114 |
#endif |
| 115 |
|
| 116 |
/* Workaround for Loongson-2F jump errata */ |
| 117 |
#ifdef CONFIG_CPU_JUMP_WORKAROUNDS |
| 118 |
#define JALR_MASK 0xffffffffcfffffffULL |
| 119 |
#else |
| 120 |
#define JALR_MASK (~0ULL) |
| 121 |
#endif |
| 122 |
|
| 123 |
/* |
| 124 |
* Mark a BPF register as accessed, it needs to be |
| 125 |
* initialized by the program if expected, e.g. FP. |
| 126 |
*/ |
| 127 |
static inline void access_reg(struct jit_context *ctx, u8 reg) |
| 128 |
{ |
| 129 |
ctx->accessed |= BIT(reg); |
| 130 |
} |
| 131 |
|
| 132 |
/* |
| 133 |
* Mark a CPU register as clobbered, it needs to be |
| 134 |
* saved/restored by the program if callee-saved. |
| 135 |
*/ |
| 136 |
static inline void clobber_reg(struct jit_context *ctx, u8 reg) |
| 137 |
{ |
| 138 |
ctx->clobbered |= BIT(reg); |
| 139 |
} |
| 140 |
|
| 141 |
/* |
| 142 |
* Push registers on the stack, starting at a given depth from the stack |
| 143 |
* pointer and increasing. The next depth to be written is returned. |
| 144 |
*/ |
| 145 |
int push_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth); |
| 146 |
|
| 147 |
/* |
| 148 |
* Pop registers from the stack, starting at a given depth from the stack |
| 149 |
* pointer and increasing. The next depth to be read is returned. |
| 150 |
*/ |
| 151 |
int pop_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth); |
| 152 |
|
| 153 |
/* Compute the 28-bit jump target address from a BPF program location */ |
| 154 |
int get_target(struct jit_context *ctx, u32 loc); |
| 155 |
|
| 156 |
/* Compute the PC-relative offset to relative BPF program offset */ |
| 157 |
int get_offset(const struct jit_context *ctx, int off); |
| 158 |
|
| 159 |
/* dst = imm (32-bit) */ |
| 160 |
void emit_mov_i(struct jit_context *ctx, u8 dst, s32 imm); |
| 161 |
|
| 162 |
/* dst = src (32-bit) */ |
| 163 |
void emit_mov_r(struct jit_context *ctx, u8 dst, u8 src); |
| 164 |
|
| 165 |
/* Validate ALU/ALU64 immediate range */ |
| 166 |
bool valid_alu_i(u8 op, s32 imm); |
| 167 |
|
| 168 |
/* Rewrite ALU/ALU64 immediate operation */ |
| 169 |
bool rewrite_alu_i(u8 op, s32 imm, u8 *alu, s32 *val); |
| 170 |
|
| 171 |
/* ALU immediate operation (32-bit) */ |
| 172 |
void emit_alu_i(struct jit_context *ctx, u8 dst, s32 imm, u8 op); |
| 173 |
|
| 174 |
/* ALU register operation (32-bit) */ |
| 175 |
void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op); |
| 176 |
|
| 177 |
/* Atomic read-modify-write (32-bit) */ |
| 178 |
void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code); |
| 179 |
|
| 180 |
/* Atomic compare-and-exchange (32-bit) */ |
| 181 |
void emit_cmpxchg_r(struct jit_context *ctx, u8 dst, u8 src, u8 res, s16 off); |
| 182 |
|
| 183 |
/* Swap bytes and truncate a register word or half word */ |
| 184 |
void emit_bswap_r(struct jit_context *ctx, u8 dst, u32 width); |
| 185 |
|
| 186 |
/* Validate JMP/JMP32 immediate range */ |
| 187 |
bool valid_jmp_i(u8 op, s32 imm); |
| 188 |
|
| 189 |
/* Prepare a PC-relative jump operation with immediate conditional */ |
| 190 |
void setup_jmp_i(struct jit_context *ctx, s32 imm, u8 width, |
| 191 |
u8 bpf_op, s16 bpf_off, u8 *jit_op, s32 *jit_off); |
| 192 |
|
| 193 |
/* Prepare a PC-relative jump operation with register conditional */ |
| 194 |
void setup_jmp_r(struct jit_context *ctx, bool same_reg, |
| 195 |
u8 bpf_op, s16 bpf_off, u8 *jit_op, s32 *jit_off); |
| 196 |
|
| 197 |
/* Finish a PC-relative jump operation */ |
| 198 |
int finish_jmp(struct jit_context *ctx, u8 jit_op, s16 bpf_off); |
| 199 |
|
| 200 |
/* Conditional JMP/JMP32 immediate */ |
| 201 |
void emit_jmp_i(struct jit_context *ctx, u8 dst, s32 imm, s32 off, u8 op); |
| 202 |
|
| 203 |
/* Conditional JMP/JMP32 register */ |
| 204 |
void emit_jmp_r(struct jit_context *ctx, u8 dst, u8 src, s32 off, u8 op); |
| 205 |
|
| 206 |
/* Jump always */ |
| 207 |
int emit_ja(struct jit_context *ctx, s16 off); |
| 208 |
|
| 209 |
/* Jump to epilogue */ |
| 210 |
int emit_exit(struct jit_context *ctx); |
| 211 |
|
| 212 |
/* |
| 213 |
* Build program prologue to set up the stack and registers. |
| 214 |
* This function is implemented separately for 32-bit and 64-bit JITs. |
| 215 |
*/ |
| 216 |
void build_prologue(struct jit_context *ctx); |
| 217 |
|
| 218 |
/* |
| 219 |
* Build the program epilogue to restore the stack and registers. |
| 220 |
* This function is implemented separately for 32-bit and 64-bit JITs. |
| 221 |
*/ |
| 222 |
void build_epilogue(struct jit_context *ctx, int dest_reg); |
| 223 |
|
| 224 |
/* |
| 225 |
* Convert an eBPF instruction to native instruction, i.e |
| 226 |
* JITs an eBPF instruction. |
| 227 |
* Returns : |
| 228 |
* 0 - Successfully JITed an 8-byte eBPF instruction |
| 229 |
* >0 - Successfully JITed a 16-byte eBPF instruction |
| 230 |
* <0 - Failed to JIT. |
| 231 |
* This function is implemented separately for 32-bit and 64-bit JITs. |
| 232 |
*/ |
| 233 |
int build_insn(const struct bpf_insn *insn, struct jit_context *ctx); |
| 234 |
|
| 235 |
#endif /* _BPF_JIT_COMP_H */ |