| 1 |
/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ |
| 2 |
#ifndef __BPF_TRACING_H__ |
| 3 |
#define __BPF_TRACING_H__ |
| 4 |
|
| 5 |
#include "bpf_helpers.h" |
| 6 |
|
| 7 |
/* Scan the ARCH passed in from ARCH env variable (see Makefile) */ |
| 8 |
#if defined(__TARGET_ARCH_x86) |
| 9 |
#define bpf_target_x86 |
| 10 |
#define bpf_target_defined |
| 11 |
#elif defined(__TARGET_ARCH_s390) |
| 12 |
#define bpf_target_s390 |
| 13 |
#define bpf_target_defined |
| 14 |
#elif defined(__TARGET_ARCH_arm) |
| 15 |
#define bpf_target_arm |
| 16 |
#define bpf_target_defined |
| 17 |
#elif defined(__TARGET_ARCH_arm64) |
| 18 |
#define bpf_target_arm64 |
| 19 |
#define bpf_target_defined |
| 20 |
#elif defined(__TARGET_ARCH_mips) |
| 21 |
#define bpf_target_mips |
| 22 |
#define bpf_target_defined |
| 23 |
#elif defined(__TARGET_ARCH_powerpc) |
| 24 |
#define bpf_target_powerpc |
| 25 |
#define bpf_target_defined |
| 26 |
#elif defined(__TARGET_ARCH_sparc) |
| 27 |
#define bpf_target_sparc |
| 28 |
#define bpf_target_defined |
| 29 |
#elif defined(__TARGET_ARCH_riscv) |
| 30 |
#define bpf_target_riscv |
| 31 |
#define bpf_target_defined |
| 32 |
#elif defined(__TARGET_ARCH_arc) |
| 33 |
#define bpf_target_arc |
| 34 |
#define bpf_target_defined |
| 35 |
#elif defined(__TARGET_ARCH_loongarch) |
| 36 |
#define bpf_target_loongarch |
| 37 |
#define bpf_target_defined |
| 38 |
#else |
| 39 |
|
| 40 |
/* Fall back to what the compiler says */ |
| 41 |
#if defined(__x86_64__) |
| 42 |
#define bpf_target_x86 |
| 43 |
#define bpf_target_defined |
| 44 |
#elif defined(__s390__) |
| 45 |
#define bpf_target_s390 |
| 46 |
#define bpf_target_defined |
| 47 |
#elif defined(__arm__) |
| 48 |
#define bpf_target_arm |
| 49 |
#define bpf_target_defined |
| 50 |
#elif defined(__aarch64__) |
| 51 |
#define bpf_target_arm64 |
| 52 |
#define bpf_target_defined |
| 53 |
#elif defined(__mips__) |
| 54 |
#define bpf_target_mips |
| 55 |
#define bpf_target_defined |
| 56 |
#elif defined(__powerpc__) |
| 57 |
#define bpf_target_powerpc |
| 58 |
#define bpf_target_defined |
| 59 |
#elif defined(__sparc__) |
| 60 |
#define bpf_target_sparc |
| 61 |
#define bpf_target_defined |
| 62 |
#elif defined(__riscv) && __riscv_xlen == 64 |
| 63 |
#define bpf_target_riscv |
| 64 |
#define bpf_target_defined |
| 65 |
#elif defined(__arc__) |
| 66 |
#define bpf_target_arc |
| 67 |
#define bpf_target_defined |
| 68 |
#elif defined(__loongarch__) |
| 69 |
#define bpf_target_loongarch |
| 70 |
#define bpf_target_defined |
| 71 |
#endif /* no compiler target */ |
| 72 |
|
| 73 |
#endif |
| 74 |
|
| 75 |
#ifndef __BPF_TARGET_MISSING |
| 76 |
#define __BPF_TARGET_MISSING "GCC error \"Must specify a BPF target arch via __TARGET_ARCH_xxx\"" |
| 77 |
#endif |
| 78 |
|
| 79 |
#if defined(bpf_target_x86) |
| 80 |
|
| 81 |
/* |
| 82 |
* https://en.wikipedia.org/wiki/X86_calling_conventions#System_V_AMD64_ABI |
| 83 |
*/ |
| 84 |
|
| 85 |
#if defined(__KERNEL__) || defined(__VMLINUX_H__) |
| 86 |
|
| 87 |
#define __PT_PARM1_REG di |
| 88 |
#define __PT_PARM2_REG si |
| 89 |
#define __PT_PARM3_REG dx |
| 90 |
#define __PT_PARM4_REG cx |
| 91 |
#define __PT_PARM5_REG r8 |
| 92 |
#define __PT_PARM6_REG r9 |
| 93 |
/* |
| 94 |
* Syscall uses r10 for PARM4. See arch/x86/entry/entry_64.S:entry_SYSCALL_64 |
| 95 |
* comments in Linux sources. And refer to syscall(2) manpage. |
| 96 |
*/ |
| 97 |
#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG |
| 98 |
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG |
| 99 |
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG |
| 100 |
#define __PT_PARM4_SYSCALL_REG r10 |
| 101 |
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG |
| 102 |
#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG |
| 103 |
|
| 104 |
#define __PT_RET_REG sp |
| 105 |
#define __PT_FP_REG bp |
| 106 |
#define __PT_RC_REG ax |
| 107 |
#define __PT_SP_REG sp |
| 108 |
#define __PT_IP_REG ip |
| 109 |
|
| 110 |
#else |
| 111 |
|
| 112 |
#ifdef __i386__ |
| 113 |
|
| 114 |
/* i386 kernel is built with -mregparm=3 */ |
| 115 |
#define __PT_PARM1_REG eax |
| 116 |
#define __PT_PARM2_REG edx |
| 117 |
#define __PT_PARM3_REG ecx |
| 118 |
/* i386 syscall ABI is very different, refer to syscall(2) manpage */ |
| 119 |
#define __PT_PARM1_SYSCALL_REG ebx |
| 120 |
#define __PT_PARM2_SYSCALL_REG ecx |
| 121 |
#define __PT_PARM3_SYSCALL_REG edx |
| 122 |
#define __PT_PARM4_SYSCALL_REG esi |
| 123 |
#define __PT_PARM5_SYSCALL_REG edi |
| 124 |
#define __PT_PARM6_SYSCALL_REG ebp |
| 125 |
|
| 126 |
#define __PT_RET_REG esp |
| 127 |
#define __PT_FP_REG ebp |
| 128 |
#define __PT_RC_REG eax |
| 129 |
#define __PT_SP_REG esp |
| 130 |
#define __PT_IP_REG eip |
| 131 |
|
| 132 |
#else /* __i386__ */ |
| 133 |
|
| 134 |
#define __PT_PARM1_REG rdi |
| 135 |
#define __PT_PARM2_REG rsi |
| 136 |
#define __PT_PARM3_REG rdx |
| 137 |
#define __PT_PARM4_REG rcx |
| 138 |
#define __PT_PARM5_REG r8 |
| 139 |
#define __PT_PARM6_REG r9 |
| 140 |
|
| 141 |
#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG |
| 142 |
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG |
| 143 |
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG |
| 144 |
#define __PT_PARM4_SYSCALL_REG r10 |
| 145 |
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG |
| 146 |
#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG |
| 147 |
|
| 148 |
#define __PT_RET_REG rsp |
| 149 |
#define __PT_FP_REG rbp |
| 150 |
#define __PT_RC_REG rax |
| 151 |
#define __PT_SP_REG rsp |
| 152 |
#define __PT_IP_REG rip |
| 153 |
|
| 154 |
#endif /* __i386__ */ |
| 155 |
|
| 156 |
#endif /* __KERNEL__ || __VMLINUX_H__ */ |
| 157 |
|
| 158 |
#elif defined(bpf_target_s390) |
| 159 |
|
| 160 |
/* |
| 161 |
* https://github.com/IBM/s390x-abi/releases/download/v1.6/lzsabi_s390x.pdf |
| 162 |
*/ |
| 163 |
|
| 164 |
struct pt_regs___s390 { |
| 165 |
unsigned long orig_gpr2; |
| 166 |
} __attribute__((preserve_access_index)); |
| 167 |
|
| 168 |
/* s390 provides user_pt_regs instead of struct pt_regs to userspace */ |
| 169 |
#define __PT_REGS_CAST(x) ((const user_pt_regs *)(x)) |
| 170 |
#define __PT_PARM1_REG gprs[2] |
| 171 |
#define __PT_PARM2_REG gprs[3] |
| 172 |
#define __PT_PARM3_REG gprs[4] |
| 173 |
#define __PT_PARM4_REG gprs[5] |
| 174 |
#define __PT_PARM5_REG gprs[6] |
| 175 |
|
| 176 |
#define __PT_PARM1_SYSCALL_REG orig_gpr2 |
| 177 |
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG |
| 178 |
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG |
| 179 |
#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG |
| 180 |
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG |
| 181 |
#define __PT_PARM6_SYSCALL_REG gprs[7] |
| 182 |
#define PT_REGS_PARM1_SYSCALL(x) (((const struct pt_regs___s390 *)(x))->__PT_PARM1_SYSCALL_REG) |
| 183 |
#define PT_REGS_PARM1_CORE_SYSCALL(x) \ |
| 184 |
BPF_CORE_READ((const struct pt_regs___s390 *)(x), __PT_PARM1_SYSCALL_REG) |
| 185 |
|
| 186 |
#define __PT_RET_REG gprs[14] |
| 187 |
#define __PT_FP_REG gprs[11] /* Works only with CONFIG_FRAME_POINTER */ |
| 188 |
#define __PT_RC_REG gprs[2] |
| 189 |
#define __PT_SP_REG gprs[15] |
| 190 |
#define __PT_IP_REG psw.addr |
| 191 |
|
| 192 |
#elif defined(bpf_target_arm) |
| 193 |
|
| 194 |
/* |
| 195 |
* https://github.com/ARM-software/abi-aa/blob/main/aapcs32/aapcs32.rst#machine-registers |
| 196 |
*/ |
| 197 |
|
| 198 |
#define __PT_PARM1_REG uregs[0] |
| 199 |
#define __PT_PARM2_REG uregs[1] |
| 200 |
#define __PT_PARM3_REG uregs[2] |
| 201 |
#define __PT_PARM4_REG uregs[3] |
| 202 |
|
| 203 |
#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG |
| 204 |
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG |
| 205 |
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG |
| 206 |
#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG |
| 207 |
#define __PT_PARM5_SYSCALL_REG uregs[4] |
| 208 |
#define __PT_PARM6_SYSCALL_REG uregs[5] |
| 209 |
#define __PT_PARM7_SYSCALL_REG uregs[6] |
| 210 |
|
| 211 |
#define __PT_RET_REG uregs[14] |
| 212 |
#define __PT_FP_REG uregs[11] /* Works only with CONFIG_FRAME_POINTER */ |
| 213 |
#define __PT_RC_REG uregs[0] |
| 214 |
#define __PT_SP_REG uregs[13] |
| 215 |
#define __PT_IP_REG uregs[12] |
| 216 |
|
| 217 |
#elif defined(bpf_target_arm64) |
| 218 |
|
| 219 |
/* |
| 220 |
* https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst#machine-registers |
| 221 |
*/ |
| 222 |
|
| 223 |
struct pt_regs___arm64 { |
| 224 |
unsigned long orig_x0; |
| 225 |
} __attribute__((preserve_access_index)); |
| 226 |
|
| 227 |
/* arm64 provides struct user_pt_regs instead of struct pt_regs to userspace */ |
| 228 |
#define __PT_REGS_CAST(x) ((const struct user_pt_regs *)(x)) |
| 229 |
#define __PT_PARM1_REG regs[0] |
| 230 |
#define __PT_PARM2_REG regs[1] |
| 231 |
#define __PT_PARM3_REG regs[2] |
| 232 |
#define __PT_PARM4_REG regs[3] |
| 233 |
#define __PT_PARM5_REG regs[4] |
| 234 |
#define __PT_PARM6_REG regs[5] |
| 235 |
#define __PT_PARM7_REG regs[6] |
| 236 |
#define __PT_PARM8_REG regs[7] |
| 237 |
|
| 238 |
#define __PT_PARM1_SYSCALL_REG orig_x0 |
| 239 |
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG |
| 240 |
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG |
| 241 |
#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG |
| 242 |
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG |
| 243 |
#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG |
| 244 |
#define PT_REGS_PARM1_SYSCALL(x) (((const struct pt_regs___arm64 *)(x))->__PT_PARM1_SYSCALL_REG) |
| 245 |
#define PT_REGS_PARM1_CORE_SYSCALL(x) \ |
| 246 |
BPF_CORE_READ((const struct pt_regs___arm64 *)(x), __PT_PARM1_SYSCALL_REG) |
| 247 |
|
| 248 |
#define __PT_RET_REG regs[30] |
| 249 |
#define __PT_FP_REG regs[29] /* Works only with CONFIG_FRAME_POINTER */ |
| 250 |
#define __PT_RC_REG regs[0] |
| 251 |
#define __PT_SP_REG sp |
| 252 |
#define __PT_IP_REG pc |
| 253 |
|
| 254 |
#elif defined(bpf_target_mips) |
| 255 |
|
| 256 |
/* |
| 257 |
* N64 ABI is assumed right now. |
| 258 |
* https://en.wikipedia.org/wiki/MIPS_architecture#Calling_conventions |
| 259 |
*/ |
| 260 |
|
| 261 |
#define __PT_PARM1_REG regs[4] |
| 262 |
#define __PT_PARM2_REG regs[5] |
| 263 |
#define __PT_PARM3_REG regs[6] |
| 264 |
#define __PT_PARM4_REG regs[7] |
| 265 |
#define __PT_PARM5_REG regs[8] |
| 266 |
#define __PT_PARM6_REG regs[9] |
| 267 |
#define __PT_PARM7_REG regs[10] |
| 268 |
#define __PT_PARM8_REG regs[11] |
| 269 |
|
| 270 |
#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG |
| 271 |
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG |
| 272 |
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG |
| 273 |
#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG |
| 274 |
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG /* only N32/N64 */ |
| 275 |
#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG /* only N32/N64 */ |
| 276 |
|
| 277 |
#define __PT_RET_REG regs[31] |
| 278 |
#define __PT_FP_REG regs[30] /* Works only with CONFIG_FRAME_POINTER */ |
| 279 |
#define __PT_RC_REG regs[2] |
| 280 |
#define __PT_SP_REG regs[29] |
| 281 |
#define __PT_IP_REG cp0_epc |
| 282 |
|
| 283 |
#elif defined(bpf_target_powerpc) |
| 284 |
|
| 285 |
/* |
| 286 |
* http://refspecs.linux-foundation.org/elf/elfspec_ppc.pdf (page 3-14, |
| 287 |
* section "Function Calling Sequence") |
| 288 |
*/ |
| 289 |
|
| 290 |
#define __PT_PARM1_REG gpr[3] |
| 291 |
#define __PT_PARM2_REG gpr[4] |
| 292 |
#define __PT_PARM3_REG gpr[5] |
| 293 |
#define __PT_PARM4_REG gpr[6] |
| 294 |
#define __PT_PARM5_REG gpr[7] |
| 295 |
#define __PT_PARM6_REG gpr[8] |
| 296 |
#define __PT_PARM7_REG gpr[9] |
| 297 |
#define __PT_PARM8_REG gpr[10] |
| 298 |
|
| 299 |
/* powerpc does not select ARCH_HAS_SYSCALL_WRAPPER. */ |
| 300 |
#define PT_REGS_SYSCALL_REGS(ctx) ctx |
| 301 |
#define __PT_PARM1_SYSCALL_REG orig_gpr3 |
| 302 |
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG |
| 303 |
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG |
| 304 |
#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG |
| 305 |
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG |
| 306 |
#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG |
| 307 |
#if !defined(__arch64__) |
| 308 |
#define __PT_PARM7_SYSCALL_REG __PT_PARM7_REG /* only powerpc (not powerpc64) */ |
| 309 |
#endif |
| 310 |
|
| 311 |
#define __PT_RET_REG regs[31] |
| 312 |
#define __PT_FP_REG __unsupported__ |
| 313 |
#define __PT_RC_REG gpr[3] |
| 314 |
#define __PT_SP_REG gpr[1] |
| 315 |
#define __PT_IP_REG nip |
| 316 |
|
| 317 |
#elif defined(bpf_target_sparc) |
| 318 |
|
| 319 |
/* |
| 320 |
* https://en.wikipedia.org/wiki/Calling_convention#SPARC |
| 321 |
*/ |
| 322 |
|
| 323 |
#define __PT_PARM1_REG u_regs[UREG_I0] |
| 324 |
#define __PT_PARM2_REG u_regs[UREG_I1] |
| 325 |
#define __PT_PARM3_REG u_regs[UREG_I2] |
| 326 |
#define __PT_PARM4_REG u_regs[UREG_I3] |
| 327 |
#define __PT_PARM5_REG u_regs[UREG_I4] |
| 328 |
#define __PT_PARM6_REG u_regs[UREG_I5] |
| 329 |
|
| 330 |
#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG |
| 331 |
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG |
| 332 |
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG |
| 333 |
#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG |
| 334 |
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG |
| 335 |
#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG |
| 336 |
|
| 337 |
#define __PT_RET_REG u_regs[UREG_I7] |
| 338 |
#define __PT_FP_REG __unsupported__ |
| 339 |
#define __PT_RC_REG u_regs[UREG_I0] |
| 340 |
#define __PT_SP_REG u_regs[UREG_FP] |
| 341 |
/* Should this also be a bpf_target check for the sparc case? */ |
| 342 |
#if defined(__arch64__) |
| 343 |
#define __PT_IP_REG tpc |
| 344 |
#else |
| 345 |
#define __PT_IP_REG pc |
| 346 |
#endif |
| 347 |
|
| 348 |
#elif defined(bpf_target_riscv) |
| 349 |
|
| 350 |
/* |
| 351 |
* https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-cc.adoc#risc-v-calling-conventions |
| 352 |
*/ |
| 353 |
|
| 354 |
struct pt_regs___riscv { |
| 355 |
unsigned long orig_a0; |
| 356 |
} __attribute__((preserve_access_index)); |
| 357 |
|
| 358 |
/* riscv provides struct user_regs_struct instead of struct pt_regs to userspace */ |
| 359 |
#define __PT_REGS_CAST(x) ((const struct user_regs_struct *)(x)) |
| 360 |
#define __PT_PARM1_REG a0 |
| 361 |
#define __PT_PARM2_REG a1 |
| 362 |
#define __PT_PARM3_REG a2 |
| 363 |
#define __PT_PARM4_REG a3 |
| 364 |
#define __PT_PARM5_REG a4 |
| 365 |
#define __PT_PARM6_REG a5 |
| 366 |
#define __PT_PARM7_REG a6 |
| 367 |
#define __PT_PARM8_REG a7 |
| 368 |
|
| 369 |
#define __PT_PARM1_SYSCALL_REG orig_a0 |
| 370 |
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG |
| 371 |
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG |
| 372 |
#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG |
| 373 |
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG |
| 374 |
#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG |
| 375 |
#define PT_REGS_PARM1_SYSCALL(x) (((const struct pt_regs___riscv *)(x))->__PT_PARM1_SYSCALL_REG) |
| 376 |
#define PT_REGS_PARM1_CORE_SYSCALL(x) \ |
| 377 |
BPF_CORE_READ((const struct pt_regs___riscv *)(x), __PT_PARM1_SYSCALL_REG) |
| 378 |
|
| 379 |
#define __PT_RET_REG ra |
| 380 |
#define __PT_FP_REG s0 |
| 381 |
#define __PT_RC_REG a0 |
| 382 |
#define __PT_SP_REG sp |
| 383 |
#define __PT_IP_REG pc |
| 384 |
|
| 385 |
#elif defined(bpf_target_arc) |
| 386 |
|
| 387 |
/* |
| 388 |
* Section "Function Calling Sequence" (page 24): |
| 389 |
* https://raw.githubusercontent.com/wiki/foss-for-synopsys-dwc-arc-processors/toolchain/files/ARCv2_ABI.pdf |
| 390 |
*/ |
| 391 |
|
| 392 |
/* arc provides struct user_regs_struct instead of struct pt_regs to userspace */ |
| 393 |
#define __PT_REGS_CAST(x) ((const struct user_regs_struct *)(x)) |
| 394 |
#define __PT_PARM1_REG scratch.r0 |
| 395 |
#define __PT_PARM2_REG scratch.r1 |
| 396 |
#define __PT_PARM3_REG scratch.r2 |
| 397 |
#define __PT_PARM4_REG scratch.r3 |
| 398 |
#define __PT_PARM5_REG scratch.r4 |
| 399 |
#define __PT_PARM6_REG scratch.r5 |
| 400 |
#define __PT_PARM7_REG scratch.r6 |
| 401 |
#define __PT_PARM8_REG scratch.r7 |
| 402 |
|
| 403 |
/* arc does not select ARCH_HAS_SYSCALL_WRAPPER. */ |
| 404 |
#define PT_REGS_SYSCALL_REGS(ctx) ctx |
| 405 |
#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG |
| 406 |
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG |
| 407 |
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG |
| 408 |
#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG |
| 409 |
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG |
| 410 |
#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG |
| 411 |
|
| 412 |
#define __PT_RET_REG scratch.blink |
| 413 |
#define __PT_FP_REG scratch.fp |
| 414 |
#define __PT_RC_REG scratch.r0 |
| 415 |
#define __PT_SP_REG scratch.sp |
| 416 |
#define __PT_IP_REG scratch.ret |
| 417 |
|
| 418 |
#elif defined(bpf_target_loongarch) |
| 419 |
|
| 420 |
/* |
| 421 |
* https://docs.kernel.org/loongarch/introduction.html |
| 422 |
* https://loongson.github.io/LoongArch-Documentation/LoongArch-ELF-ABI-EN.html |
| 423 |
*/ |
| 424 |
|
| 425 |
/* loongarch provides struct user_pt_regs instead of struct pt_regs to userspace */ |
| 426 |
#define __PT_REGS_CAST(x) ((const struct user_pt_regs *)(x)) |
| 427 |
#define __PT_PARM1_REG regs[4] |
| 428 |
#define __PT_PARM2_REG regs[5] |
| 429 |
#define __PT_PARM3_REG regs[6] |
| 430 |
#define __PT_PARM4_REG regs[7] |
| 431 |
#define __PT_PARM5_REG regs[8] |
| 432 |
#define __PT_PARM6_REG regs[9] |
| 433 |
#define __PT_PARM7_REG regs[10] |
| 434 |
#define __PT_PARM8_REG regs[11] |
| 435 |
|
| 436 |
/* loongarch does not select ARCH_HAS_SYSCALL_WRAPPER. */ |
| 437 |
#define PT_REGS_SYSCALL_REGS(ctx) ctx |
| 438 |
#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG |
| 439 |
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG |
| 440 |
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG |
| 441 |
#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG |
| 442 |
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG |
| 443 |
#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG |
| 444 |
|
| 445 |
#define __PT_RET_REG regs[1] |
| 446 |
#define __PT_FP_REG regs[22] |
| 447 |
#define __PT_RC_REG regs[4] |
| 448 |
#define __PT_SP_REG regs[3] |
| 449 |
#define __PT_IP_REG csr_era |
| 450 |
|
| 451 |
#endif |
| 452 |
|
| 453 |
#if defined(bpf_target_defined) |
| 454 |
|
| 455 |
struct pt_regs; |
| 456 |
|
| 457 |
/* allow some architectures to override `struct pt_regs` */ |
| 458 |
#ifndef __PT_REGS_CAST |
| 459 |
#define __PT_REGS_CAST(x) (x) |
| 460 |
#endif |
| 461 |
|
| 462 |
/* |
| 463 |
* Different architectures support different number of arguments passed |
| 464 |
* through registers. i386 supports just 3, some arches support up to 8. |
| 465 |
*/ |
| 466 |
#ifndef __PT_PARM4_REG |
| 467 |
#define __PT_PARM4_REG __unsupported__ |
| 468 |
#endif |
| 469 |
#ifndef __PT_PARM5_REG |
| 470 |
#define __PT_PARM5_REG __unsupported__ |
| 471 |
#endif |
| 472 |
#ifndef __PT_PARM6_REG |
| 473 |
#define __PT_PARM6_REG __unsupported__ |
| 474 |
#endif |
| 475 |
#ifndef __PT_PARM7_REG |
| 476 |
#define __PT_PARM7_REG __unsupported__ |
| 477 |
#endif |
| 478 |
#ifndef __PT_PARM8_REG |
| 479 |
#define __PT_PARM8_REG __unsupported__ |
| 480 |
#endif |
| 481 |
/* |
| 482 |
* Similarly, syscall-specific conventions might differ between function call |
| 483 |
* conventions within each architecture. All supported architectures pass |
| 484 |
* either 6 or 7 syscall arguments in registers. |
| 485 |
* |
| 486 |
* See syscall(2) manpage for succinct table with information on each arch. |
| 487 |
*/ |
| 488 |
#ifndef __PT_PARM7_SYSCALL_REG |
| 489 |
#define __PT_PARM7_SYSCALL_REG __unsupported__ |
| 490 |
#endif |
| 491 |
|
| 492 |
#define PT_REGS_PARM1(x) (__PT_REGS_CAST(x)->__PT_PARM1_REG) |
| 493 |
#define PT_REGS_PARM2(x) (__PT_REGS_CAST(x)->__PT_PARM2_REG) |
| 494 |
#define PT_REGS_PARM3(x) (__PT_REGS_CAST(x)->__PT_PARM3_REG) |
| 495 |
#define PT_REGS_PARM4(x) (__PT_REGS_CAST(x)->__PT_PARM4_REG) |
| 496 |
#define PT_REGS_PARM5(x) (__PT_REGS_CAST(x)->__PT_PARM5_REG) |
| 497 |
#define PT_REGS_PARM6(x) (__PT_REGS_CAST(x)->__PT_PARM6_REG) |
| 498 |
#define PT_REGS_PARM7(x) (__PT_REGS_CAST(x)->__PT_PARM7_REG) |
| 499 |
#define PT_REGS_PARM8(x) (__PT_REGS_CAST(x)->__PT_PARM8_REG) |
| 500 |
#define PT_REGS_RET(x) (__PT_REGS_CAST(x)->__PT_RET_REG) |
| 501 |
#define PT_REGS_FP(x) (__PT_REGS_CAST(x)->__PT_FP_REG) |
| 502 |
#define PT_REGS_RC(x) (__PT_REGS_CAST(x)->__PT_RC_REG) |
| 503 |
#define PT_REGS_SP(x) (__PT_REGS_CAST(x)->__PT_SP_REG) |
| 504 |
#define PT_REGS_IP(x) (__PT_REGS_CAST(x)->__PT_IP_REG) |
| 505 |
|
| 506 |
#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM1_REG) |
| 507 |
#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM2_REG) |
| 508 |
#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM3_REG) |
| 509 |
#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM4_REG) |
| 510 |
#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM5_REG) |
| 511 |
#define PT_REGS_PARM6_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM6_REG) |
| 512 |
#define PT_REGS_PARM7_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM7_REG) |
| 513 |
#define PT_REGS_PARM8_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM8_REG) |
| 514 |
#define PT_REGS_RET_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_RET_REG) |
| 515 |
#define PT_REGS_FP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_FP_REG) |
| 516 |
#define PT_REGS_RC_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_RC_REG) |
| 517 |
#define PT_REGS_SP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_SP_REG) |
| 518 |
#define PT_REGS_IP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_IP_REG) |
| 519 |
|
| 520 |
#if defined(bpf_target_powerpc) |
| 521 |
|
| 522 |
#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = (ctx)->link; }) |
| 523 |
#define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP |
| 524 |
|
| 525 |
#elif defined(bpf_target_sparc) || defined(bpf_target_arm64) |
| 526 |
|
| 527 |
#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = PT_REGS_RET(ctx); }) |
| 528 |
#define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP |
| 529 |
|
| 530 |
#else |
| 531 |
|
| 532 |
#define BPF_KPROBE_READ_RET_IP(ip, ctx) \ |
| 533 |
({ bpf_probe_read_kernel(&(ip), sizeof(ip), (void *)PT_REGS_RET(ctx)); }) |
| 534 |
#define BPF_KRETPROBE_READ_RET_IP(ip, ctx) \ |
| 535 |
({ bpf_probe_read_kernel(&(ip), sizeof(ip), (void *)(PT_REGS_FP(ctx) + sizeof(ip))); }) |
| 536 |
|
| 537 |
#endif |
| 538 |
|
| 539 |
#ifndef PT_REGS_PARM1_SYSCALL |
| 540 |
#define PT_REGS_PARM1_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM1_SYSCALL_REG) |
| 541 |
#define PT_REGS_PARM1_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM1_SYSCALL_REG) |
| 542 |
#endif |
| 543 |
#ifndef PT_REGS_PARM2_SYSCALL |
| 544 |
#define PT_REGS_PARM2_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM2_SYSCALL_REG) |
| 545 |
#define PT_REGS_PARM2_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM2_SYSCALL_REG) |
| 546 |
#endif |
| 547 |
#ifndef PT_REGS_PARM3_SYSCALL |
| 548 |
#define PT_REGS_PARM3_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM3_SYSCALL_REG) |
| 549 |
#define PT_REGS_PARM3_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM3_SYSCALL_REG) |
| 550 |
#endif |
| 551 |
#ifndef PT_REGS_PARM4_SYSCALL |
| 552 |
#define PT_REGS_PARM4_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM4_SYSCALL_REG) |
| 553 |
#define PT_REGS_PARM4_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM4_SYSCALL_REG) |
| 554 |
#endif |
| 555 |
#ifndef PT_REGS_PARM5_SYSCALL |
| 556 |
#define PT_REGS_PARM5_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM5_SYSCALL_REG) |
| 557 |
#define PT_REGS_PARM5_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM5_SYSCALL_REG) |
| 558 |
#endif |
| 559 |
#ifndef PT_REGS_PARM6_SYSCALL |
| 560 |
#define PT_REGS_PARM6_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM6_SYSCALL_REG) |
| 561 |
#define PT_REGS_PARM6_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM6_SYSCALL_REG) |
| 562 |
#endif |
| 563 |
#ifndef PT_REGS_PARM7_SYSCALL |
| 564 |
#define PT_REGS_PARM7_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM7_SYSCALL_REG) |
| 565 |
#define PT_REGS_PARM7_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM7_SYSCALL_REG) |
| 566 |
#endif |
| 567 |
|
| 568 |
#else /* defined(bpf_target_defined) */ |
| 569 |
|
| 570 |
#define PT_REGS_PARM1(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 571 |
#define PT_REGS_PARM2(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 572 |
#define PT_REGS_PARM3(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 573 |
#define PT_REGS_PARM4(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 574 |
#define PT_REGS_PARM5(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 575 |
#define PT_REGS_PARM6(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 576 |
#define PT_REGS_PARM7(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 577 |
#define PT_REGS_PARM8(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 578 |
#define PT_REGS_RET(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 579 |
#define PT_REGS_FP(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 580 |
#define PT_REGS_RC(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 581 |
#define PT_REGS_SP(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 582 |
#define PT_REGS_IP(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 583 |
|
| 584 |
#define PT_REGS_PARM1_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 585 |
#define PT_REGS_PARM2_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 586 |
#define PT_REGS_PARM3_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 587 |
#define PT_REGS_PARM4_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 588 |
#define PT_REGS_PARM5_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 589 |
#define PT_REGS_PARM6_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 590 |
#define PT_REGS_PARM7_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 591 |
#define PT_REGS_PARM8_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 592 |
#define PT_REGS_RET_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 593 |
#define PT_REGS_FP_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 594 |
#define PT_REGS_RC_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 595 |
#define PT_REGS_SP_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 596 |
#define PT_REGS_IP_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 597 |
|
| 598 |
#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 599 |
#define BPF_KRETPROBE_READ_RET_IP(ip, ctx) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 600 |
|
| 601 |
#define PT_REGS_PARM1_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 602 |
#define PT_REGS_PARM2_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 603 |
#define PT_REGS_PARM3_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 604 |
#define PT_REGS_PARM4_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 605 |
#define PT_REGS_PARM5_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 606 |
#define PT_REGS_PARM6_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 607 |
#define PT_REGS_PARM7_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 608 |
|
| 609 |
#define PT_REGS_PARM1_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 610 |
#define PT_REGS_PARM2_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 611 |
#define PT_REGS_PARM3_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 612 |
#define PT_REGS_PARM4_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 613 |
#define PT_REGS_PARM5_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 614 |
#define PT_REGS_PARM6_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 615 |
#define PT_REGS_PARM7_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) |
| 616 |
|
| 617 |
#endif /* defined(bpf_target_defined) */ |
| 618 |
|
| 619 |
/* |
| 620 |
* When invoked from a syscall handler kprobe, returns a pointer to a |
| 621 |
* struct pt_regs containing syscall arguments and suitable for passing to |
| 622 |
* PT_REGS_PARMn_SYSCALL() and PT_REGS_PARMn_CORE_SYSCALL(). |
| 623 |
*/ |
| 624 |
#ifndef PT_REGS_SYSCALL_REGS |
| 625 |
/* By default, assume that the arch selects ARCH_HAS_SYSCALL_WRAPPER. */ |
| 626 |
#define PT_REGS_SYSCALL_REGS(ctx) ((struct pt_regs *)PT_REGS_PARM1(ctx)) |
| 627 |
#endif |
| 628 |
|
| 629 |
#ifndef ___bpf_concat |
| 630 |
#define ___bpf_concat(a, b) a ## b |
| 631 |
#endif |
| 632 |
#ifndef ___bpf_apply |
| 633 |
#define ___bpf_apply(fn, n) ___bpf_concat(fn, n) |
| 634 |
#endif |
| 635 |
#ifndef ___bpf_nth |
| 636 |
#define ___bpf_nth(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _a, _b, _c, N, ...) N |
| 637 |
#endif |
| 638 |
#ifndef ___bpf_narg |
| 639 |
#define ___bpf_narg(...) ___bpf_nth(_, ##__VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) |
| 640 |
#endif |
| 641 |
|
| 642 |
#define ___bpf_ctx_cast0() ctx |
| 643 |
#define ___bpf_ctx_cast1(x) ___bpf_ctx_cast0(), ctx[0] |
| 644 |
#define ___bpf_ctx_cast2(x, args...) ___bpf_ctx_cast1(args), ctx[1] |
| 645 |
#define ___bpf_ctx_cast3(x, args...) ___bpf_ctx_cast2(args), ctx[2] |
| 646 |
#define ___bpf_ctx_cast4(x, args...) ___bpf_ctx_cast3(args), ctx[3] |
| 647 |
#define ___bpf_ctx_cast5(x, args...) ___bpf_ctx_cast4(args), ctx[4] |
| 648 |
#define ___bpf_ctx_cast6(x, args...) ___bpf_ctx_cast5(args), ctx[5] |
| 649 |
#define ___bpf_ctx_cast7(x, args...) ___bpf_ctx_cast6(args), ctx[6] |
| 650 |
#define ___bpf_ctx_cast8(x, args...) ___bpf_ctx_cast7(args), ctx[7] |
| 651 |
#define ___bpf_ctx_cast9(x, args...) ___bpf_ctx_cast8(args), ctx[8] |
| 652 |
#define ___bpf_ctx_cast10(x, args...) ___bpf_ctx_cast9(args), ctx[9] |
| 653 |
#define ___bpf_ctx_cast11(x, args...) ___bpf_ctx_cast10(args), ctx[10] |
| 654 |
#define ___bpf_ctx_cast12(x, args...) ___bpf_ctx_cast11(args), ctx[11] |
| 655 |
#define ___bpf_ctx_cast(args...) ___bpf_apply(___bpf_ctx_cast, ___bpf_narg(args))(args) |
| 656 |
|
| 657 |
/* |
| 658 |
* BPF_PROG is a convenience wrapper for generic tp_btf/fentry/fexit and |
| 659 |
* similar kinds of BPF programs, that accept input arguments as a single |
| 660 |
* pointer to untyped u64 array, where each u64 can actually be a typed |
| 661 |
* pointer or integer of different size. Instead of requiring user to write |
| 662 |
* manual casts and work with array elements by index, BPF_PROG macro |
| 663 |
* allows user to declare a list of named and typed input arguments in the |
| 664 |
* same syntax as for normal C function. All the casting is hidden and |
| 665 |
* performed transparently, while user code can just assume working with |
| 666 |
* function arguments of specified type and name. |
| 667 |
* |
| 668 |
* Original raw context argument is preserved as well as 'ctx' argument. |
| 669 |
* This is useful when using BPF helpers that expect original context |
| 670 |
* as one of the parameters (e.g., for bpf_perf_event_output()). |
| 671 |
*/ |
| 672 |
#define BPF_PROG(name, args...) \ |
| 673 |
name(unsigned long long *ctx); \ |
| 674 |
static __always_inline typeof(name(0)) \ |
| 675 |
____##name(unsigned long long *ctx, ##args); \ |
| 676 |
typeof(name(0)) name(unsigned long long *ctx) \ |
| 677 |
{ \ |
| 678 |
_Pragma("GCC diagnostic push") \ |
| 679 |
_Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ |
| 680 |
return ____##name(___bpf_ctx_cast(args)); \ |
| 681 |
_Pragma("GCC diagnostic pop") \ |
| 682 |
} \ |
| 683 |
static __always_inline typeof(name(0)) \ |
| 684 |
____##name(unsigned long long *ctx, ##args) |
| 685 |
|
| 686 |
#ifndef ___bpf_nth2 |
| 687 |
#define ___bpf_nth2(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, \ |
| 688 |
_14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, N, ...) N |
| 689 |
#endif |
| 690 |
#ifndef ___bpf_narg2 |
| 691 |
#define ___bpf_narg2(...) \ |
| 692 |
___bpf_nth2(_, ##__VA_ARGS__, 12, 12, 11, 11, 10, 10, 9, 9, 8, 8, 7, 7, \ |
| 693 |
6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0) |
| 694 |
#endif |
| 695 |
|
| 696 |
#define ___bpf_treg_cnt(t) \ |
| 697 |
__builtin_choose_expr(sizeof(t) == 1, 1, \ |
| 698 |
__builtin_choose_expr(sizeof(t) == 2, 1, \ |
| 699 |
__builtin_choose_expr(sizeof(t) == 4, 1, \ |
| 700 |
__builtin_choose_expr(sizeof(t) == 8, 1, \ |
| 701 |
__builtin_choose_expr(sizeof(t) == 16, 2, \ |
| 702 |
(void)0))))) |
| 703 |
|
| 704 |
#define ___bpf_reg_cnt0() (0) |
| 705 |
#define ___bpf_reg_cnt1(t, x) (___bpf_reg_cnt0() + ___bpf_treg_cnt(t)) |
| 706 |
#define ___bpf_reg_cnt2(t, x, args...) (___bpf_reg_cnt1(args) + ___bpf_treg_cnt(t)) |
| 707 |
#define ___bpf_reg_cnt3(t, x, args...) (___bpf_reg_cnt2(args) + ___bpf_treg_cnt(t)) |
| 708 |
#define ___bpf_reg_cnt4(t, x, args...) (___bpf_reg_cnt3(args) + ___bpf_treg_cnt(t)) |
| 709 |
#define ___bpf_reg_cnt5(t, x, args...) (___bpf_reg_cnt4(args) + ___bpf_treg_cnt(t)) |
| 710 |
#define ___bpf_reg_cnt6(t, x, args...) (___bpf_reg_cnt5(args) + ___bpf_treg_cnt(t)) |
| 711 |
#define ___bpf_reg_cnt7(t, x, args...) (___bpf_reg_cnt6(args) + ___bpf_treg_cnt(t)) |
| 712 |
#define ___bpf_reg_cnt8(t, x, args...) (___bpf_reg_cnt7(args) + ___bpf_treg_cnt(t)) |
| 713 |
#define ___bpf_reg_cnt9(t, x, args...) (___bpf_reg_cnt8(args) + ___bpf_treg_cnt(t)) |
| 714 |
#define ___bpf_reg_cnt10(t, x, args...) (___bpf_reg_cnt9(args) + ___bpf_treg_cnt(t)) |
| 715 |
#define ___bpf_reg_cnt11(t, x, args...) (___bpf_reg_cnt10(args) + ___bpf_treg_cnt(t)) |
| 716 |
#define ___bpf_reg_cnt12(t, x, args...) (___bpf_reg_cnt11(args) + ___bpf_treg_cnt(t)) |
| 717 |
#define ___bpf_reg_cnt(args...) ___bpf_apply(___bpf_reg_cnt, ___bpf_narg2(args))(args) |
| 718 |
|
| 719 |
#define ___bpf_union_arg(t, x, n) \ |
| 720 |
__builtin_choose_expr(sizeof(t) == 1, ({ union { __u8 z[1]; t x; } ___t = { .z = {ctx[n]}}; ___t.x; }), \ |
| 721 |
__builtin_choose_expr(sizeof(t) == 2, ({ union { __u16 z[1]; t x; } ___t = { .z = {ctx[n]} }; ___t.x; }), \ |
| 722 |
__builtin_choose_expr(sizeof(t) == 4, ({ union { __u32 z[1]; t x; } ___t = { .z = {ctx[n]} }; ___t.x; }), \ |
| 723 |
__builtin_choose_expr(sizeof(t) == 8, ({ union { __u64 z[1]; t x; } ___t = {.z = {ctx[n]} }; ___t.x; }), \ |
| 724 |
__builtin_choose_expr(sizeof(t) == 16, ({ union { __u64 z[2]; t x; } ___t = {.z = {ctx[n], ctx[n + 1]} }; ___t.x; }), \ |
| 725 |
(void)0))))) |
| 726 |
|
| 727 |
#define ___bpf_ctx_arg0(n, args...) |
| 728 |
#define ___bpf_ctx_arg1(n, t, x) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt1(t, x)) |
| 729 |
#define ___bpf_ctx_arg2(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt2(t, x, args)) ___bpf_ctx_arg1(n, args) |
| 730 |
#define ___bpf_ctx_arg3(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt3(t, x, args)) ___bpf_ctx_arg2(n, args) |
| 731 |
#define ___bpf_ctx_arg4(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt4(t, x, args)) ___bpf_ctx_arg3(n, args) |
| 732 |
#define ___bpf_ctx_arg5(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt5(t, x, args)) ___bpf_ctx_arg4(n, args) |
| 733 |
#define ___bpf_ctx_arg6(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt6(t, x, args)) ___bpf_ctx_arg5(n, args) |
| 734 |
#define ___bpf_ctx_arg7(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt7(t, x, args)) ___bpf_ctx_arg6(n, args) |
| 735 |
#define ___bpf_ctx_arg8(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt8(t, x, args)) ___bpf_ctx_arg7(n, args) |
| 736 |
#define ___bpf_ctx_arg9(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt9(t, x, args)) ___bpf_ctx_arg8(n, args) |
| 737 |
#define ___bpf_ctx_arg10(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt10(t, x, args)) ___bpf_ctx_arg9(n, args) |
| 738 |
#define ___bpf_ctx_arg11(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt11(t, x, args)) ___bpf_ctx_arg10(n, args) |
| 739 |
#define ___bpf_ctx_arg12(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt12(t, x, args)) ___bpf_ctx_arg11(n, args) |
| 740 |
#define ___bpf_ctx_arg(args...) ___bpf_apply(___bpf_ctx_arg, ___bpf_narg2(args))(___bpf_reg_cnt(args), args) |
| 741 |
|
| 742 |
#define ___bpf_ctx_decl0() |
| 743 |
#define ___bpf_ctx_decl1(t, x) , t x |
| 744 |
#define ___bpf_ctx_decl2(t, x, args...) , t x ___bpf_ctx_decl1(args) |
| 745 |
#define ___bpf_ctx_decl3(t, x, args...) , t x ___bpf_ctx_decl2(args) |
| 746 |
#define ___bpf_ctx_decl4(t, x, args...) , t x ___bpf_ctx_decl3(args) |
| 747 |
#define ___bpf_ctx_decl5(t, x, args...) , t x ___bpf_ctx_decl4(args) |
| 748 |
#define ___bpf_ctx_decl6(t, x, args...) , t x ___bpf_ctx_decl5(args) |
| 749 |
#define ___bpf_ctx_decl7(t, x, args...) , t x ___bpf_ctx_decl6(args) |
| 750 |
#define ___bpf_ctx_decl8(t, x, args...) , t x ___bpf_ctx_decl7(args) |
| 751 |
#define ___bpf_ctx_decl9(t, x, args...) , t x ___bpf_ctx_decl8(args) |
| 752 |
#define ___bpf_ctx_decl10(t, x, args...) , t x ___bpf_ctx_decl9(args) |
| 753 |
#define ___bpf_ctx_decl11(t, x, args...) , t x ___bpf_ctx_decl10(args) |
| 754 |
#define ___bpf_ctx_decl12(t, x, args...) , t x ___bpf_ctx_decl11(args) |
| 755 |
#define ___bpf_ctx_decl(args...) ___bpf_apply(___bpf_ctx_decl, ___bpf_narg2(args))(args) |
| 756 |
|
| 757 |
/* |
| 758 |
* BPF_PROG2 is an enhanced version of BPF_PROG in order to handle struct |
| 759 |
* arguments. Since each struct argument might take one or two u64 values |
| 760 |
* in the trampoline stack, argument type size is needed to place proper number |
| 761 |
* of u64 values for each argument. Therefore, BPF_PROG2 has different |
| 762 |
* syntax from BPF_PROG. For example, for the following BPF_PROG syntax: |
| 763 |
* |
| 764 |
* int BPF_PROG(test2, int a, int b) { ... } |
| 765 |
* |
| 766 |
* the corresponding BPF_PROG2 syntax is: |
| 767 |
* |
| 768 |
* int BPF_PROG2(test2, int, a, int, b) { ... } |
| 769 |
* |
| 770 |
* where type and the corresponding argument name are separated by comma. |
| 771 |
* |
| 772 |
* Use BPF_PROG2 macro if one of the arguments might be a struct/union larger |
| 773 |
* than 8 bytes: |
| 774 |
* |
| 775 |
* int BPF_PROG2(test_struct_arg, struct bpf_testmod_struct_arg_1, a, int, b, |
| 776 |
* int, c, int, d, struct bpf_testmod_struct_arg_2, e, int, ret) |
| 777 |
* { |
| 778 |
* // access a, b, c, d, e, and ret directly |
| 779 |
* ... |
| 780 |
* } |
| 781 |
*/ |
| 782 |
#define BPF_PROG2(name, args...) \ |
| 783 |
name(unsigned long long *ctx); \ |
| 784 |
static __always_inline typeof(name(0)) \ |
| 785 |
____##name(unsigned long long *ctx ___bpf_ctx_decl(args)); \ |
| 786 |
typeof(name(0)) name(unsigned long long *ctx) \ |
| 787 |
{ \ |
| 788 |
return ____##name(ctx ___bpf_ctx_arg(args)); \ |
| 789 |
} \ |
| 790 |
static __always_inline typeof(name(0)) \ |
| 791 |
____##name(unsigned long long *ctx ___bpf_ctx_decl(args)) |
| 792 |
|
| 793 |
struct pt_regs; |
| 794 |
|
| 795 |
#define ___bpf_kprobe_args0() ctx |
| 796 |
#define ___bpf_kprobe_args1(x) ___bpf_kprobe_args0(), (unsigned long long)PT_REGS_PARM1(ctx) |
| 797 |
#define ___bpf_kprobe_args2(x, args...) ___bpf_kprobe_args1(args), (unsigned long long)PT_REGS_PARM2(ctx) |
| 798 |
#define ___bpf_kprobe_args3(x, args...) ___bpf_kprobe_args2(args), (unsigned long long)PT_REGS_PARM3(ctx) |
| 799 |
#define ___bpf_kprobe_args4(x, args...) ___bpf_kprobe_args3(args), (unsigned long long)PT_REGS_PARM4(ctx) |
| 800 |
#define ___bpf_kprobe_args5(x, args...) ___bpf_kprobe_args4(args), (unsigned long long)PT_REGS_PARM5(ctx) |
| 801 |
#define ___bpf_kprobe_args6(x, args...) ___bpf_kprobe_args5(args), (unsigned long long)PT_REGS_PARM6(ctx) |
| 802 |
#define ___bpf_kprobe_args7(x, args...) ___bpf_kprobe_args6(args), (unsigned long long)PT_REGS_PARM7(ctx) |
| 803 |
#define ___bpf_kprobe_args8(x, args...) ___bpf_kprobe_args7(args), (unsigned long long)PT_REGS_PARM8(ctx) |
| 804 |
#define ___bpf_kprobe_args(args...) ___bpf_apply(___bpf_kprobe_args, ___bpf_narg(args))(args) |
| 805 |
|
| 806 |
/* |
| 807 |
* BPF_KPROBE serves the same purpose for kprobes as BPF_PROG for |
| 808 |
* tp_btf/fentry/fexit BPF programs. It hides the underlying platform-specific |
| 809 |
* low-level way of getting kprobe input arguments from struct pt_regs, and |
| 810 |
* provides a familiar typed and named function arguments syntax and |
| 811 |
* semantics of accessing kprobe input parameters. |
| 812 |
* |
| 813 |
* Original struct pt_regs* context is preserved as 'ctx' argument. This might |
| 814 |
* be necessary when using BPF helpers like bpf_perf_event_output(). |
| 815 |
*/ |
| 816 |
#define BPF_KPROBE(name, args...) \ |
| 817 |
name(struct pt_regs *ctx); \ |
| 818 |
static __always_inline typeof(name(0)) \ |
| 819 |
____##name(struct pt_regs *ctx, ##args); \ |
| 820 |
typeof(name(0)) name(struct pt_regs *ctx) \ |
| 821 |
{ \ |
| 822 |
_Pragma("GCC diagnostic push") \ |
| 823 |
_Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ |
| 824 |
return ____##name(___bpf_kprobe_args(args)); \ |
| 825 |
_Pragma("GCC diagnostic pop") \ |
| 826 |
} \ |
| 827 |
static __always_inline typeof(name(0)) \ |
| 828 |
____##name(struct pt_regs *ctx, ##args) |
| 829 |
|
| 830 |
#define ___bpf_kretprobe_args0() ctx |
| 831 |
#define ___bpf_kretprobe_args1(x) ___bpf_kretprobe_args0(), (unsigned long long)PT_REGS_RC(ctx) |
| 832 |
#define ___bpf_kretprobe_args(args...) ___bpf_apply(___bpf_kretprobe_args, ___bpf_narg(args))(args) |
| 833 |
|
| 834 |
/* |
| 835 |
* BPF_KRETPROBE is similar to BPF_KPROBE, except, it only provides optional |
| 836 |
* return value (in addition to `struct pt_regs *ctx`), but no input |
| 837 |
* arguments, because they will be clobbered by the time probed function |
| 838 |
* returns. |
| 839 |
*/ |
| 840 |
#define BPF_KRETPROBE(name, args...) \ |
| 841 |
name(struct pt_regs *ctx); \ |
| 842 |
static __always_inline typeof(name(0)) \ |
| 843 |
____##name(struct pt_regs *ctx, ##args); \ |
| 844 |
typeof(name(0)) name(struct pt_regs *ctx) \ |
| 845 |
{ \ |
| 846 |
_Pragma("GCC diagnostic push") \ |
| 847 |
_Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ |
| 848 |
return ____##name(___bpf_kretprobe_args(args)); \ |
| 849 |
_Pragma("GCC diagnostic pop") \ |
| 850 |
} \ |
| 851 |
static __always_inline typeof(name(0)) ____##name(struct pt_regs *ctx, ##args) |
| 852 |
|
| 853 |
/* If kernel has CONFIG_ARCH_HAS_SYSCALL_WRAPPER, read pt_regs directly */ |
| 854 |
#define ___bpf_syscall_args0() ctx |
| 855 |
#define ___bpf_syscall_args1(x) ___bpf_syscall_args0(), (unsigned long long)PT_REGS_PARM1_SYSCALL(regs) |
| 856 |
#define ___bpf_syscall_args2(x, args...) ___bpf_syscall_args1(args), (unsigned long long)PT_REGS_PARM2_SYSCALL(regs) |
| 857 |
#define ___bpf_syscall_args3(x, args...) ___bpf_syscall_args2(args), (unsigned long long)PT_REGS_PARM3_SYSCALL(regs) |
| 858 |
#define ___bpf_syscall_args4(x, args...) ___bpf_syscall_args3(args), (unsigned long long)PT_REGS_PARM4_SYSCALL(regs) |
| 859 |
#define ___bpf_syscall_args5(x, args...) ___bpf_syscall_args4(args), (unsigned long long)PT_REGS_PARM5_SYSCALL(regs) |
| 860 |
#define ___bpf_syscall_args6(x, args...) ___bpf_syscall_args5(args), (unsigned long long)PT_REGS_PARM6_SYSCALL(regs) |
| 861 |
#define ___bpf_syscall_args7(x, args...) ___bpf_syscall_args6(args), (unsigned long long)PT_REGS_PARM7_SYSCALL(regs) |
| 862 |
#define ___bpf_syscall_args(args...) ___bpf_apply(___bpf_syscall_args, ___bpf_narg(args))(args) |
| 863 |
|
| 864 |
/* If kernel doesn't have CONFIG_ARCH_HAS_SYSCALL_WRAPPER, we have to BPF_CORE_READ from pt_regs */ |
| 865 |
#define ___bpf_syswrap_args0() ctx |
| 866 |
#define ___bpf_syswrap_args1(x) ___bpf_syswrap_args0(), (unsigned long long)PT_REGS_PARM1_CORE_SYSCALL(regs) |
| 867 |
#define ___bpf_syswrap_args2(x, args...) ___bpf_syswrap_args1(args), (unsigned long long)PT_REGS_PARM2_CORE_SYSCALL(regs) |
| 868 |
#define ___bpf_syswrap_args3(x, args...) ___bpf_syswrap_args2(args), (unsigned long long)PT_REGS_PARM3_CORE_SYSCALL(regs) |
| 869 |
#define ___bpf_syswrap_args4(x, args...) ___bpf_syswrap_args3(args), (unsigned long long)PT_REGS_PARM4_CORE_SYSCALL(regs) |
| 870 |
#define ___bpf_syswrap_args5(x, args...) ___bpf_syswrap_args4(args), (unsigned long long)PT_REGS_PARM5_CORE_SYSCALL(regs) |
| 871 |
#define ___bpf_syswrap_args6(x, args...) ___bpf_syswrap_args5(args), (unsigned long long)PT_REGS_PARM6_CORE_SYSCALL(regs) |
| 872 |
#define ___bpf_syswrap_args7(x, args...) ___bpf_syswrap_args6(args), (unsigned long long)PT_REGS_PARM7_CORE_SYSCALL(regs) |
| 873 |
#define ___bpf_syswrap_args(args...) ___bpf_apply(___bpf_syswrap_args, ___bpf_narg(args))(args) |
| 874 |
|
| 875 |
/* |
| 876 |
* BPF_KSYSCALL is a variant of BPF_KPROBE, which is intended for |
| 877 |
* tracing syscall functions, like __x64_sys_close. It hides the underlying |
| 878 |
* platform-specific low-level way of getting syscall input arguments from |
| 879 |
* struct pt_regs, and provides a familiar typed and named function arguments |
| 880 |
* syntax and semantics of accessing syscall input parameters. |
| 881 |
* |
| 882 |
* Original struct pt_regs * context is preserved as 'ctx' argument. This might |
| 883 |
* be necessary when using BPF helpers like bpf_perf_event_output(). |
| 884 |
* |
| 885 |
* At the moment BPF_KSYSCALL does not transparently handle all the calling |
| 886 |
* convention quirks for the following syscalls: |
| 887 |
* |
| 888 |
* - mmap(): __ARCH_WANT_SYS_OLD_MMAP. |
| 889 |
* - clone(): CONFIG_CLONE_BACKWARDS, CONFIG_CLONE_BACKWARDS2 and |
| 890 |
* CONFIG_CLONE_BACKWARDS3. |
| 891 |
* - socket-related syscalls: __ARCH_WANT_SYS_SOCKETCALL. |
| 892 |
* - compat syscalls. |
| 893 |
* |
| 894 |
* This may or may not change in the future. User needs to take extra measures |
| 895 |
* to handle such quirks explicitly, if necessary. |
| 896 |
* |
| 897 |
* This macro relies on BPF CO-RE support and virtual __kconfig externs. |
| 898 |
*/ |
| 899 |
#define BPF_KSYSCALL(name, args...) \ |
| 900 |
name(struct pt_regs *ctx); \ |
| 901 |
extern _Bool LINUX_HAS_SYSCALL_WRAPPER __kconfig; \ |
| 902 |
static __always_inline typeof(name(0)) \ |
| 903 |
____##name(struct pt_regs *ctx, ##args); \ |
| 904 |
typeof(name(0)) name(struct pt_regs *ctx) \ |
| 905 |
{ \ |
| 906 |
struct pt_regs *regs = LINUX_HAS_SYSCALL_WRAPPER \ |
| 907 |
? (struct pt_regs *)PT_REGS_PARM1(ctx) \ |
| 908 |
: ctx; \ |
| 909 |
_Pragma("GCC diagnostic push") \ |
| 910 |
_Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ |
| 911 |
if (LINUX_HAS_SYSCALL_WRAPPER) \ |
| 912 |
return ____##name(___bpf_syswrap_args(args)); \ |
| 913 |
else \ |
| 914 |
return ____##name(___bpf_syscall_args(args)); \ |
| 915 |
_Pragma("GCC diagnostic pop") \ |
| 916 |
} \ |
| 917 |
static __always_inline typeof(name(0)) \ |
| 918 |
____##name(struct pt_regs *ctx, ##args) |
| 919 |
|
| 920 |
#define BPF_KPROBE_SYSCALL BPF_KSYSCALL |
| 921 |
|
| 922 |
/* BPF_UPROBE and BPF_URETPROBE are identical to BPF_KPROBE and BPF_KRETPROBE, |
| 923 |
* but are named way less confusingly for SEC("uprobe") and SEC("uretprobe") |
| 924 |
* use cases. |
| 925 |
*/ |
| 926 |
#define BPF_UPROBE(name, args...) BPF_KPROBE(name, ##args) |
| 927 |
#define BPF_URETPROBE(name, args...) BPF_KRETPROBE(name, ##args) |
| 928 |
|
| 929 |
#endif |