| 1 |
// SPDX-License-Identifier: GPL-2.0-only |
| 2 |
/* |
| 3 |
* Branch Record Buffer Extension Driver. |
| 4 |
* |
| 5 |
* Copyright (C) 2022-2025 ARM Limited |
| 6 |
* |
| 7 |
* Author: Anshuman Khandual <[email protected]> |
| 8 |
*/ |
| 9 |
#include <linux/types.h> |
| 10 |
#include <linux/bitmap.h> |
| 11 |
#include <linux/perf/arm_pmu.h> |
| 12 |
#include "arm_brbe.h" |
| 13 |
|
| 14 |
#define BRBFCR_EL1_BRANCH_FILTERS (BRBFCR_EL1_DIRECT | \ |
| 15 |
BRBFCR_EL1_INDIRECT | \ |
| 16 |
BRBFCR_EL1_RTN | \ |
| 17 |
BRBFCR_EL1_INDCALL | \ |
| 18 |
BRBFCR_EL1_DIRCALL | \ |
| 19 |
BRBFCR_EL1_CONDDIR) |
| 20 |
|
| 21 |
/* |
| 22 |
* BRBTS_EL1 is currently not used for branch stack implementation |
| 23 |
* purpose but BRBCR_ELx.TS needs to have a valid value from all |
| 24 |
* available options. BRBCR_ELx_TS_VIRTUAL is selected for this. |
| 25 |
*/ |
| 26 |
#define BRBCR_ELx_DEFAULT_TS FIELD_PREP(BRBCR_ELx_TS_MASK, BRBCR_ELx_TS_VIRTUAL) |
| 27 |
|
| 28 |
/* |
| 29 |
* BRBE Buffer Organization |
| 30 |
* |
| 31 |
* BRBE buffer is arranged as multiple banks of 32 branch record |
| 32 |
* entries each. An individual branch record in a given bank could |
| 33 |
* be accessed, after selecting the bank in BRBFCR_EL1.BANK and |
| 34 |
* accessing the registers i.e [BRBSRC, BRBTGT, BRBINF] set with |
| 35 |
* indices [0..31]. |
| 36 |
* |
| 37 |
* Bank 0 |
| 38 |
* |
| 39 |
* --------------------------------- ------ |
| 40 |
* | 00 | BRBSRC | BRBTGT | BRBINF | | 00 | |
| 41 |
* --------------------------------- ------ |
| 42 |
* | 01 | BRBSRC | BRBTGT | BRBINF | | 01 | |
| 43 |
* --------------------------------- ------ |
| 44 |
* | .. | BRBSRC | BRBTGT | BRBINF | | .. | |
| 45 |
* --------------------------------- ------ |
| 46 |
* | 31 | BRBSRC | BRBTGT | BRBINF | | 31 | |
| 47 |
* --------------------------------- ------ |
| 48 |
* |
| 49 |
* Bank 1 |
| 50 |
* |
| 51 |
* --------------------------------- ------ |
| 52 |
* | 32 | BRBSRC | BRBTGT | BRBINF | | 00 | |
| 53 |
* --------------------------------- ------ |
| 54 |
* | 33 | BRBSRC | BRBTGT | BRBINF | | 01 | |
| 55 |
* --------------------------------- ------ |
| 56 |
* | .. | BRBSRC | BRBTGT | BRBINF | | .. | |
| 57 |
* --------------------------------- ------ |
| 58 |
* | 63 | BRBSRC | BRBTGT | BRBINF | | 31 | |
| 59 |
* --------------------------------- ------ |
| 60 |
*/ |
| 61 |
#define BRBE_BANK_MAX_ENTRIES 32 |
| 62 |
|
| 63 |
struct brbe_regset { |
| 64 |
u64 brbsrc; |
| 65 |
u64 brbtgt; |
| 66 |
u64 brbinf; |
| 67 |
}; |
| 68 |
|
| 69 |
#define PERF_BR_ARM64_MAX (PERF_BR_MAX + PERF_BR_NEW_MAX) |
| 70 |
|
| 71 |
struct brbe_hw_attr { |
| 72 |
int brbe_version; |
| 73 |
int brbe_cc; |
| 74 |
int brbe_nr; |
| 75 |
int brbe_format; |
| 76 |
}; |
| 77 |
|
| 78 |
#define BRBE_REGN_CASE(n, case_macro) \ |
| 79 |
case n: case_macro(n); break |
| 80 |
|
| 81 |
#define BRBE_REGN_SWITCH(x, case_macro) \ |
| 82 |
do { \ |
| 83 |
switch (x) { \ |
| 84 |
BRBE_REGN_CASE(0, case_macro); \ |
| 85 |
BRBE_REGN_CASE(1, case_macro); \ |
| 86 |
BRBE_REGN_CASE(2, case_macro); \ |
| 87 |
BRBE_REGN_CASE(3, case_macro); \ |
| 88 |
BRBE_REGN_CASE(4, case_macro); \ |
| 89 |
BRBE_REGN_CASE(5, case_macro); \ |
| 90 |
BRBE_REGN_CASE(6, case_macro); \ |
| 91 |
BRBE_REGN_CASE(7, case_macro); \ |
| 92 |
BRBE_REGN_CASE(8, case_macro); \ |
| 93 |
BRBE_REGN_CASE(9, case_macro); \ |
| 94 |
BRBE_REGN_CASE(10, case_macro); \ |
| 95 |
BRBE_REGN_CASE(11, case_macro); \ |
| 96 |
BRBE_REGN_CASE(12, case_macro); \ |
| 97 |
BRBE_REGN_CASE(13, case_macro); \ |
| 98 |
BRBE_REGN_CASE(14, case_macro); \ |
| 99 |
BRBE_REGN_CASE(15, case_macro); \ |
| 100 |
BRBE_REGN_CASE(16, case_macro); \ |
| 101 |
BRBE_REGN_CASE(17, case_macro); \ |
| 102 |
BRBE_REGN_CASE(18, case_macro); \ |
| 103 |
BRBE_REGN_CASE(19, case_macro); \ |
| 104 |
BRBE_REGN_CASE(20, case_macro); \ |
| 105 |
BRBE_REGN_CASE(21, case_macro); \ |
| 106 |
BRBE_REGN_CASE(22, case_macro); \ |
| 107 |
BRBE_REGN_CASE(23, case_macro); \ |
| 108 |
BRBE_REGN_CASE(24, case_macro); \ |
| 109 |
BRBE_REGN_CASE(25, case_macro); \ |
| 110 |
BRBE_REGN_CASE(26, case_macro); \ |
| 111 |
BRBE_REGN_CASE(27, case_macro); \ |
| 112 |
BRBE_REGN_CASE(28, case_macro); \ |
| 113 |
BRBE_REGN_CASE(29, case_macro); \ |
| 114 |
BRBE_REGN_CASE(30, case_macro); \ |
| 115 |
BRBE_REGN_CASE(31, case_macro); \ |
| 116 |
default: WARN(1, "Invalid BRB* index %d\n", x); \ |
| 117 |
} \ |
| 118 |
} while (0) |
| 119 |
|
| 120 |
#define RETURN_READ_BRBSRCN(n) \ |
| 121 |
return read_sysreg_s(SYS_BRBSRC_EL1(n)) |
| 122 |
static inline u64 get_brbsrc_reg(int idx) |
| 123 |
{ |
| 124 |
BRBE_REGN_SWITCH(idx, RETURN_READ_BRBSRCN); |
| 125 |
return 0; |
| 126 |
} |
| 127 |
|
| 128 |
#define RETURN_READ_BRBTGTN(n) \ |
| 129 |
return read_sysreg_s(SYS_BRBTGT_EL1(n)) |
| 130 |
static u64 get_brbtgt_reg(int idx) |
| 131 |
{ |
| 132 |
BRBE_REGN_SWITCH(idx, RETURN_READ_BRBTGTN); |
| 133 |
return 0; |
| 134 |
} |
| 135 |
|
| 136 |
#define RETURN_READ_BRBINFN(n) \ |
| 137 |
return read_sysreg_s(SYS_BRBINF_EL1(n)) |
| 138 |
static u64 get_brbinf_reg(int idx) |
| 139 |
{ |
| 140 |
BRBE_REGN_SWITCH(idx, RETURN_READ_BRBINFN); |
| 141 |
return 0; |
| 142 |
} |
| 143 |
|
| 144 |
static u64 brbe_record_valid(u64 brbinf) |
| 145 |
{ |
| 146 |
return FIELD_GET(BRBINFx_EL1_VALID_MASK, brbinf); |
| 147 |
} |
| 148 |
|
| 149 |
static bool brbe_invalid(u64 brbinf) |
| 150 |
{ |
| 151 |
return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_NONE; |
| 152 |
} |
| 153 |
|
| 154 |
static bool brbe_record_is_complete(u64 brbinf) |
| 155 |
{ |
| 156 |
return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_FULL; |
| 157 |
} |
| 158 |
|
| 159 |
static bool brbe_record_is_source_only(u64 brbinf) |
| 160 |
{ |
| 161 |
return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_SOURCE; |
| 162 |
} |
| 163 |
|
| 164 |
static bool brbe_record_is_target_only(u64 brbinf) |
| 165 |
{ |
| 166 |
return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_TARGET; |
| 167 |
} |
| 168 |
|
| 169 |
static int brbinf_get_in_tx(u64 brbinf) |
| 170 |
{ |
| 171 |
return FIELD_GET(BRBINFx_EL1_T_MASK, brbinf); |
| 172 |
} |
| 173 |
|
| 174 |
static int brbinf_get_mispredict(u64 brbinf) |
| 175 |
{ |
| 176 |
return FIELD_GET(BRBINFx_EL1_MPRED_MASK, brbinf); |
| 177 |
} |
| 178 |
|
| 179 |
static int brbinf_get_lastfailed(u64 brbinf) |
| 180 |
{ |
| 181 |
return FIELD_GET(BRBINFx_EL1_LASTFAILED_MASK, brbinf); |
| 182 |
} |
| 183 |
|
| 184 |
static u16 brbinf_get_cycles(u64 brbinf) |
| 185 |
{ |
| 186 |
u32 exp, mant, cycles; |
| 187 |
/* |
| 188 |
* Captured cycle count is unknown and hence |
| 189 |
* should not be passed on to userspace. |
| 190 |
*/ |
| 191 |
if (brbinf & BRBINFx_EL1_CCU) |
| 192 |
return 0; |
| 193 |
|
| 194 |
exp = FIELD_GET(BRBINFx_EL1_CC_EXP_MASK, brbinf); |
| 195 |
mant = FIELD_GET(BRBINFx_EL1_CC_MANT_MASK, brbinf); |
| 196 |
|
| 197 |
if (!exp) |
| 198 |
return mant; |
| 199 |
|
| 200 |
cycles = (mant | 0x100) << (exp - 1); |
| 201 |
|
| 202 |
return min(cycles, U16_MAX); |
| 203 |
} |
| 204 |
|
| 205 |
static int brbinf_get_type(u64 brbinf) |
| 206 |
{ |
| 207 |
return FIELD_GET(BRBINFx_EL1_TYPE_MASK, brbinf); |
| 208 |
} |
| 209 |
|
| 210 |
static int brbinf_get_el(u64 brbinf) |
| 211 |
{ |
| 212 |
return FIELD_GET(BRBINFx_EL1_EL_MASK, brbinf); |
| 213 |
} |
| 214 |
|
| 215 |
void brbe_invalidate(void) |
| 216 |
{ |
| 217 |
/* Ensure all branches before this point are recorded */ |
| 218 |
isb(); |
| 219 |
asm volatile(BRB_IALL_INSN); |
| 220 |
/* Ensure all branch records are invalidated after this point */ |
| 221 |
isb(); |
| 222 |
} |
| 223 |
|
| 224 |
static bool valid_brbe_nr(int brbe_nr) |
| 225 |
{ |
| 226 |
return brbe_nr == BRBIDR0_EL1_NUMREC_8 || |
| 227 |
brbe_nr == BRBIDR0_EL1_NUMREC_16 || |
| 228 |
brbe_nr == BRBIDR0_EL1_NUMREC_32 || |
| 229 |
brbe_nr == BRBIDR0_EL1_NUMREC_64; |
| 230 |
} |
| 231 |
|
| 232 |
static bool valid_brbe_cc(int brbe_cc) |
| 233 |
{ |
| 234 |
return brbe_cc == BRBIDR0_EL1_CC_20_BIT; |
| 235 |
} |
| 236 |
|
| 237 |
static bool valid_brbe_format(int brbe_format) |
| 238 |
{ |
| 239 |
return brbe_format == BRBIDR0_EL1_FORMAT_FORMAT_0; |
| 240 |
} |
| 241 |
|
| 242 |
static bool valid_brbidr(u64 brbidr) |
| 243 |
{ |
| 244 |
int brbe_format, brbe_cc, brbe_nr; |
| 245 |
|
| 246 |
brbe_format = FIELD_GET(BRBIDR0_EL1_FORMAT_MASK, brbidr); |
| 247 |
brbe_cc = FIELD_GET(BRBIDR0_EL1_CC_MASK, brbidr); |
| 248 |
brbe_nr = FIELD_GET(BRBIDR0_EL1_NUMREC_MASK, brbidr); |
| 249 |
|
| 250 |
return valid_brbe_format(brbe_format) && valid_brbe_cc(brbe_cc) && valid_brbe_nr(brbe_nr); |
| 251 |
} |
| 252 |
|
| 253 |
static bool valid_brbe_version(int brbe_version) |
| 254 |
{ |
| 255 |
return brbe_version == ID_AA64DFR0_EL1_BRBE_IMP || |
| 256 |
brbe_version == ID_AA64DFR0_EL1_BRBE_BRBE_V1P1; |
| 257 |
} |
| 258 |
|
| 259 |
static void select_brbe_bank(int bank) |
| 260 |
{ |
| 261 |
u64 brbfcr; |
| 262 |
|
| 263 |
brbfcr = read_sysreg_s(SYS_BRBFCR_EL1); |
| 264 |
brbfcr &= ~BRBFCR_EL1_BANK_MASK; |
| 265 |
brbfcr |= SYS_FIELD_PREP(BRBFCR_EL1, BANK, bank); |
| 266 |
write_sysreg_s(brbfcr, SYS_BRBFCR_EL1); |
| 267 |
/* |
| 268 |
* Arm ARM (DDI 0487K.a) D.18.4 rule PPBZP requires explicit sync |
| 269 |
* between setting BANK and accessing branch records. |
| 270 |
*/ |
| 271 |
isb(); |
| 272 |
} |
| 273 |
|
| 274 |
static bool __read_brbe_regset(struct brbe_regset *entry, int idx) |
| 275 |
{ |
| 276 |
entry->brbinf = get_brbinf_reg(idx); |
| 277 |
|
| 278 |
if (brbe_invalid(entry->brbinf)) |
| 279 |
return false; |
| 280 |
|
| 281 |
entry->brbsrc = get_brbsrc_reg(idx); |
| 282 |
entry->brbtgt = get_brbtgt_reg(idx); |
| 283 |
return true; |
| 284 |
} |
| 285 |
|
| 286 |
/* |
| 287 |
* Generic perf branch filters supported on BRBE |
| 288 |
* |
| 289 |
* New branch filters need to be evaluated whether they could be supported on |
| 290 |
* BRBE. This ensures that such branch filters would not just be accepted, to |
| 291 |
* fail silently. PERF_SAMPLE_BRANCH_HV is a special case that is selectively |
| 292 |
* supported only on platforms where kernel is in hyp mode. |
| 293 |
*/ |
| 294 |
#define BRBE_EXCLUDE_BRANCH_FILTERS (PERF_SAMPLE_BRANCH_ABORT_TX | \ |
| 295 |
PERF_SAMPLE_BRANCH_IN_TX | \ |
| 296 |
PERF_SAMPLE_BRANCH_NO_TX | \ |
| 297 |
PERF_SAMPLE_BRANCH_CALL_STACK | \ |
| 298 |
PERF_SAMPLE_BRANCH_COUNTERS) |
| 299 |
|
| 300 |
#define BRBE_ALLOWED_BRANCH_TYPES (PERF_SAMPLE_BRANCH_ANY | \ |
| 301 |
PERF_SAMPLE_BRANCH_ANY_CALL | \ |
| 302 |
PERF_SAMPLE_BRANCH_ANY_RETURN | \ |
| 303 |
PERF_SAMPLE_BRANCH_IND_CALL | \ |
| 304 |
PERF_SAMPLE_BRANCH_COND | \ |
| 305 |
PERF_SAMPLE_BRANCH_IND_JUMP | \ |
| 306 |
PERF_SAMPLE_BRANCH_CALL) |
| 307 |
|
| 308 |
|
| 309 |
#define BRBE_ALLOWED_BRANCH_FILTERS (PERF_SAMPLE_BRANCH_USER | \ |
| 310 |
PERF_SAMPLE_BRANCH_KERNEL | \ |
| 311 |
PERF_SAMPLE_BRANCH_HV | \ |
| 312 |
BRBE_ALLOWED_BRANCH_TYPES | \ |
| 313 |
PERF_SAMPLE_BRANCH_NO_FLAGS | \ |
| 314 |
PERF_SAMPLE_BRANCH_NO_CYCLES | \ |
| 315 |
PERF_SAMPLE_BRANCH_TYPE_SAVE | \ |
| 316 |
PERF_SAMPLE_BRANCH_HW_INDEX | \ |
| 317 |
PERF_SAMPLE_BRANCH_PRIV_SAVE) |
| 318 |
|
| 319 |
#define BRBE_PERF_BRANCH_FILTERS (BRBE_ALLOWED_BRANCH_FILTERS | \ |
| 320 |
BRBE_EXCLUDE_BRANCH_FILTERS) |
| 321 |
|
| 322 |
/* |
| 323 |
* BRBE supports the following functional branch type filters while |
| 324 |
* generating branch records. These branch filters can be enabled, |
| 325 |
* either individually or as a group i.e ORing multiple filters |
| 326 |
* with each other. |
| 327 |
* |
| 328 |
* BRBFCR_EL1_CONDDIR - Conditional direct branch |
| 329 |
* BRBFCR_EL1_DIRCALL - Direct call |
| 330 |
* BRBFCR_EL1_INDCALL - Indirect call |
| 331 |
* BRBFCR_EL1_INDIRECT - Indirect branch |
| 332 |
* BRBFCR_EL1_DIRECT - Direct branch |
| 333 |
* BRBFCR_EL1_RTN - Subroutine return |
| 334 |
*/ |
| 335 |
static u64 branch_type_to_brbfcr(int branch_type) |
| 336 |
{ |
| 337 |
u64 brbfcr = 0; |
| 338 |
|
| 339 |
if (branch_type & PERF_SAMPLE_BRANCH_ANY) { |
| 340 |
brbfcr |= BRBFCR_EL1_BRANCH_FILTERS; |
| 341 |
return brbfcr; |
| 342 |
} |
| 343 |
|
| 344 |
if (branch_type & PERF_SAMPLE_BRANCH_ANY_CALL) { |
| 345 |
brbfcr |= BRBFCR_EL1_INDCALL; |
| 346 |
brbfcr |= BRBFCR_EL1_DIRCALL; |
| 347 |
} |
| 348 |
|
| 349 |
if (branch_type & PERF_SAMPLE_BRANCH_ANY_RETURN) |
| 350 |
brbfcr |= BRBFCR_EL1_RTN; |
| 351 |
|
| 352 |
if (branch_type & PERF_SAMPLE_BRANCH_IND_CALL) |
| 353 |
brbfcr |= BRBFCR_EL1_INDCALL; |
| 354 |
|
| 355 |
if (branch_type & PERF_SAMPLE_BRANCH_COND) |
| 356 |
brbfcr |= BRBFCR_EL1_CONDDIR; |
| 357 |
|
| 358 |
if (branch_type & PERF_SAMPLE_BRANCH_IND_JUMP) |
| 359 |
brbfcr |= BRBFCR_EL1_INDIRECT; |
| 360 |
|
| 361 |
if (branch_type & PERF_SAMPLE_BRANCH_CALL) |
| 362 |
brbfcr |= BRBFCR_EL1_DIRCALL; |
| 363 |
|
| 364 |
return brbfcr; |
| 365 |
} |
| 366 |
|
| 367 |
/* |
| 368 |
* BRBE supports the following privilege mode filters while generating |
| 369 |
* branch records. |
| 370 |
* |
| 371 |
* BRBCR_ELx_E0BRE - EL0 branch records |
| 372 |
* BRBCR_ELx_ExBRE - EL1/EL2 branch records |
| 373 |
* |
| 374 |
* BRBE also supports the following additional functional branch type |
| 375 |
* filters while generating branch records. |
| 376 |
* |
| 377 |
* BRBCR_ELx_EXCEPTION - Exception |
| 378 |
* BRBCR_ELx_ERTN - Exception return |
| 379 |
*/ |
| 380 |
static u64 branch_type_to_brbcr(int branch_type) |
| 381 |
{ |
| 382 |
u64 brbcr = BRBCR_ELx_FZP | BRBCR_ELx_DEFAULT_TS; |
| 383 |
|
| 384 |
if (branch_type & PERF_SAMPLE_BRANCH_USER) |
| 385 |
brbcr |= BRBCR_ELx_E0BRE; |
| 386 |
|
| 387 |
/* |
| 388 |
* When running in the hyp mode, writing into BRBCR_EL1 |
| 389 |
* actually writes into BRBCR_EL2 instead. Field E2BRE |
| 390 |
* is also at the same position as E1BRE. |
| 391 |
*/ |
| 392 |
if (branch_type & PERF_SAMPLE_BRANCH_KERNEL) |
| 393 |
brbcr |= BRBCR_ELx_ExBRE; |
| 394 |
|
| 395 |
if (branch_type & PERF_SAMPLE_BRANCH_HV) { |
| 396 |
if (is_kernel_in_hyp_mode()) |
| 397 |
brbcr |= BRBCR_ELx_ExBRE; |
| 398 |
} |
| 399 |
|
| 400 |
if (!(branch_type & PERF_SAMPLE_BRANCH_NO_CYCLES)) |
| 401 |
brbcr |= BRBCR_ELx_CC; |
| 402 |
|
| 403 |
if (!(branch_type & PERF_SAMPLE_BRANCH_NO_FLAGS)) |
| 404 |
brbcr |= BRBCR_ELx_MPRED; |
| 405 |
|
| 406 |
/* |
| 407 |
* The exception and exception return branches could be |
| 408 |
* captured, irrespective of the perf event's privilege. |
| 409 |
* If the perf event does not have enough privilege for |
| 410 |
* a given exception level, then addresses which falls |
| 411 |
* under that exception level will be reported as zero |
| 412 |
* for the captured branch record, creating source only |
| 413 |
* or target only records. |
| 414 |
*/ |
| 415 |
if (branch_type & PERF_SAMPLE_BRANCH_KERNEL) { |
| 416 |
if (branch_type & PERF_SAMPLE_BRANCH_ANY) { |
| 417 |
brbcr |= BRBCR_ELx_EXCEPTION; |
| 418 |
brbcr |= BRBCR_ELx_ERTN; |
| 419 |
} |
| 420 |
|
| 421 |
if (branch_type & PERF_SAMPLE_BRANCH_ANY_CALL) |
| 422 |
brbcr |= BRBCR_ELx_EXCEPTION; |
| 423 |
|
| 424 |
if (branch_type & PERF_SAMPLE_BRANCH_ANY_RETURN) |
| 425 |
brbcr |= BRBCR_ELx_ERTN; |
| 426 |
} |
| 427 |
return brbcr; |
| 428 |
} |
| 429 |
|
| 430 |
bool brbe_branch_attr_valid(struct perf_event *event) |
| 431 |
{ |
| 432 |
u64 branch_type = event->attr.branch_sample_type; |
| 433 |
|
| 434 |
/* |
| 435 |
* Ensure both perf branch filter allowed and exclude |
| 436 |
* masks are always in sync with the generic perf ABI. |
| 437 |
*/ |
| 438 |
BUILD_BUG_ON(BRBE_PERF_BRANCH_FILTERS != (PERF_SAMPLE_BRANCH_MAX - 1)); |
| 439 |
|
| 440 |
if (branch_type & BRBE_EXCLUDE_BRANCH_FILTERS) { |
| 441 |
pr_debug("requested branch filter not supported 0x%llx\n", branch_type); |
| 442 |
return false; |
| 443 |
} |
| 444 |
|
| 445 |
/* Ensure at least 1 branch type is enabled */ |
| 446 |
if (!(branch_type & BRBE_ALLOWED_BRANCH_TYPES)) { |
| 447 |
pr_debug("no branch type enabled 0x%llx\n", branch_type); |
| 448 |
return false; |
| 449 |
} |
| 450 |
|
| 451 |
/* |
| 452 |
* No branches are recorded in guests nor nVHE hypervisors, so |
| 453 |
* excluding the host or both kernel and user is invalid. |
| 454 |
* |
| 455 |
* Ideally we'd just require exclude_guest and exclude_hv, but setting |
| 456 |
* event filters with perf for kernel or user don't set exclude_guest. |
| 457 |
* So effectively, exclude_guest and exclude_hv are ignored. |
| 458 |
*/ |
| 459 |
if (event->attr.exclude_host || (event->attr.exclude_user && event->attr.exclude_kernel)) { |
| 460 |
pr_debug("branch filter in hypervisor or guest only not supported 0x%llx\n", branch_type); |
| 461 |
return false; |
| 462 |
} |
| 463 |
|
| 464 |
event->hw.branch_reg.config = branch_type_to_brbfcr(event->attr.branch_sample_type); |
| 465 |
event->hw.extra_reg.config = branch_type_to_brbcr(event->attr.branch_sample_type); |
| 466 |
|
| 467 |
return true; |
| 468 |
} |
| 469 |
|
| 470 |
unsigned int brbe_num_branch_records(const struct arm_pmu *armpmu) |
| 471 |
{ |
| 472 |
return FIELD_GET(BRBIDR0_EL1_NUMREC_MASK, armpmu->reg_brbidr); |
| 473 |
} |
| 474 |
|
| 475 |
void brbe_probe(struct arm_pmu *armpmu) |
| 476 |
{ |
| 477 |
u64 brbidr, aa64dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1); |
| 478 |
u32 brbe; |
| 479 |
|
| 480 |
brbe = cpuid_feature_extract_unsigned_field(aa64dfr0, ID_AA64DFR0_EL1_BRBE_SHIFT); |
| 481 |
if (!valid_brbe_version(brbe)) |
| 482 |
return; |
| 483 |
|
| 484 |
brbidr = read_sysreg_s(SYS_BRBIDR0_EL1); |
| 485 |
if (!valid_brbidr(brbidr)) |
| 486 |
return; |
| 487 |
|
| 488 |
armpmu->reg_brbidr = brbidr; |
| 489 |
} |
| 490 |
|
| 491 |
/* |
| 492 |
* BRBE is assumed to be disabled/paused on entry |
| 493 |
*/ |
| 494 |
void brbe_enable(const struct arm_pmu *arm_pmu) |
| 495 |
{ |
| 496 |
struct pmu_hw_events *cpuc = this_cpu_ptr(arm_pmu->hw_events); |
| 497 |
u64 brbfcr = 0, brbcr = 0; |
| 498 |
|
| 499 |
/* |
| 500 |
* Discard existing records to avoid a discontinuity, e.g. records |
| 501 |
* missed during handling an overflow. |
| 502 |
*/ |
| 503 |
brbe_invalidate(); |
| 504 |
|
| 505 |
/* |
| 506 |
* Merge the permitted branch filters of all events. |
| 507 |
*/ |
| 508 |
for (int i = 0; i < ARMPMU_MAX_HWEVENTS; i++) { |
| 509 |
struct perf_event *event = cpuc->events[i]; |
| 510 |
|
| 511 |
if (event && has_branch_stack(event)) { |
| 512 |
brbfcr |= event->hw.branch_reg.config; |
| 513 |
brbcr |= event->hw.extra_reg.config; |
| 514 |
} |
| 515 |
} |
| 516 |
|
| 517 |
/* |
| 518 |
* In VHE mode with MDCR_EL2.HPMN equal to PMCR_EL0.N, BRBCR_EL1.FZP |
| 519 |
* controls freezing the branch records on counter overflow rather than |
| 520 |
* BRBCR_EL2.FZP (which writes to BRBCR_EL1 are redirected to). |
| 521 |
* The exception levels are enabled/disabled in BRBCR_EL2, so keep EL1 |
| 522 |
* and EL0 recording disabled for guests. |
| 523 |
* |
| 524 |
* As BRBCR_EL1 CC and MPRED bits also need to match, use the same |
| 525 |
* value for both registers just masking the exception levels. |
| 526 |
*/ |
| 527 |
if (is_kernel_in_hyp_mode()) |
| 528 |
write_sysreg_s(brbcr & ~(BRBCR_ELx_ExBRE | BRBCR_ELx_E0BRE), SYS_BRBCR_EL12); |
| 529 |
write_sysreg_s(brbcr, SYS_BRBCR_EL1); |
| 530 |
/* Ensure BRBCR_ELx settings take effect before unpausing */ |
| 531 |
isb(); |
| 532 |
|
| 533 |
/* Finally write SYS_BRBFCR_EL to unpause BRBE */ |
| 534 |
write_sysreg_s(brbfcr, SYS_BRBFCR_EL1); |
| 535 |
/* Synchronization in PMCR write ensures ordering WRT PMU enabling */ |
| 536 |
} |
| 537 |
|
| 538 |
void brbe_disable(void) |
| 539 |
{ |
| 540 |
/* |
| 541 |
* No need for synchronization here as synchronization in PMCR write |
| 542 |
* ensures ordering and in the interrupt handler this is a NOP as |
| 543 |
* we're already paused. |
| 544 |
*/ |
| 545 |
write_sysreg_s(BRBFCR_EL1_PAUSED, SYS_BRBFCR_EL1); |
| 546 |
write_sysreg_s(0, SYS_BRBCR_EL1); |
| 547 |
} |
| 548 |
|
| 549 |
static const int brbe_type_to_perf_type_map[BRBINFx_EL1_TYPE_DEBUG_EXIT + 1][2] = { |
| 550 |
[BRBINFx_EL1_TYPE_DIRECT_UNCOND] = { PERF_BR_UNCOND, 0 }, |
| 551 |
[BRBINFx_EL1_TYPE_INDIRECT] = { PERF_BR_IND, 0 }, |
| 552 |
[BRBINFx_EL1_TYPE_DIRECT_LINK] = { PERF_BR_CALL, 0 }, |
| 553 |
[BRBINFx_EL1_TYPE_INDIRECT_LINK] = { PERF_BR_IND_CALL, 0 }, |
| 554 |
[BRBINFx_EL1_TYPE_RET] = { PERF_BR_RET, 0 }, |
| 555 |
[BRBINFx_EL1_TYPE_DIRECT_COND] = { PERF_BR_COND, 0 }, |
| 556 |
[BRBINFx_EL1_TYPE_CALL] = { PERF_BR_SYSCALL, 0 }, |
| 557 |
[BRBINFx_EL1_TYPE_ERET] = { PERF_BR_ERET, 0 }, |
| 558 |
[BRBINFx_EL1_TYPE_IRQ] = { PERF_BR_IRQ, 0 }, |
| 559 |
[BRBINFx_EL1_TYPE_TRAP] = { PERF_BR_IRQ, 0 }, |
| 560 |
[BRBINFx_EL1_TYPE_SERROR] = { PERF_BR_SERROR, 0 }, |
| 561 |
[BRBINFx_EL1_TYPE_ALIGN_FAULT] = { PERF_BR_EXTEND_ABI, PERF_BR_NEW_FAULT_ALGN }, |
| 562 |
[BRBINFx_EL1_TYPE_INSN_FAULT] = { PERF_BR_EXTEND_ABI, PERF_BR_NEW_FAULT_INST }, |
| 563 |
[BRBINFx_EL1_TYPE_DATA_FAULT] = { PERF_BR_EXTEND_ABI, PERF_BR_NEW_FAULT_DATA }, |
| 564 |
}; |
| 565 |
|
| 566 |
static void brbe_set_perf_entry_type(struct perf_branch_entry *entry, u64 brbinf) |
| 567 |
{ |
| 568 |
int brbe_type = brbinf_get_type(brbinf); |
| 569 |
|
| 570 |
if (brbe_type <= BRBINFx_EL1_TYPE_DEBUG_EXIT) { |
| 571 |
const int *br_type = brbe_type_to_perf_type_map[brbe_type]; |
| 572 |
|
| 573 |
entry->type = br_type[0]; |
| 574 |
entry->new_type = br_type[1]; |
| 575 |
} |
| 576 |
} |
| 577 |
|
| 578 |
static int brbinf_get_perf_priv(u64 brbinf) |
| 579 |
{ |
| 580 |
int brbe_el = brbinf_get_el(brbinf); |
| 581 |
|
| 582 |
switch (brbe_el) { |
| 583 |
case BRBINFx_EL1_EL_EL0: |
| 584 |
return PERF_BR_PRIV_USER; |
| 585 |
case BRBINFx_EL1_EL_EL1: |
| 586 |
return PERF_BR_PRIV_KERNEL; |
| 587 |
case BRBINFx_EL1_EL_EL2: |
| 588 |
if (is_kernel_in_hyp_mode()) |
| 589 |
return PERF_BR_PRIV_KERNEL; |
| 590 |
return PERF_BR_PRIV_HV; |
| 591 |
default: |
| 592 |
pr_warn_once("%d - unknown branch privilege captured\n", brbe_el); |
| 593 |
return PERF_BR_PRIV_UNKNOWN; |
| 594 |
} |
| 595 |
} |
| 596 |
|
| 597 |
static bool perf_entry_from_brbe_regset(int index, struct perf_branch_entry *entry, |
| 598 |
const struct perf_event *event) |
| 599 |
{ |
| 600 |
struct brbe_regset bregs; |
| 601 |
u64 brbinf; |
| 602 |
|
| 603 |
if (!__read_brbe_regset(&bregs, index)) |
| 604 |
return false; |
| 605 |
|
| 606 |
brbinf = bregs.brbinf; |
| 607 |
perf_clear_branch_entry_bitfields(entry); |
| 608 |
if (brbe_record_is_complete(brbinf)) { |
| 609 |
entry->from = bregs.brbsrc; |
| 610 |
entry->to = bregs.brbtgt; |
| 611 |
} else if (brbe_record_is_source_only(brbinf)) { |
| 612 |
entry->from = bregs.brbsrc; |
| 613 |
entry->to = 0; |
| 614 |
} else if (brbe_record_is_target_only(brbinf)) { |
| 615 |
entry->from = 0; |
| 616 |
entry->to = bregs.brbtgt; |
| 617 |
} |
| 618 |
|
| 619 |
brbe_set_perf_entry_type(entry, brbinf); |
| 620 |
|
| 621 |
if (!branch_sample_no_cycles(event)) |
| 622 |
entry->cycles = brbinf_get_cycles(brbinf); |
| 623 |
|
| 624 |
if (!branch_sample_no_flags(event)) { |
| 625 |
/* Mispredict info is available for source only and complete branch records. */ |
| 626 |
if (!brbe_record_is_target_only(brbinf)) { |
| 627 |
entry->mispred = brbinf_get_mispredict(brbinf); |
| 628 |
entry->predicted = !entry->mispred; |
| 629 |
} |
| 630 |
|
| 631 |
/* |
| 632 |
* Currently TME feature is neither implemented in any hardware |
| 633 |
* nor it is being supported in the kernel. Just warn here once |
| 634 |
* if TME related information shows up rather unexpectedly. |
| 635 |
*/ |
| 636 |
if (brbinf_get_lastfailed(brbinf) || brbinf_get_in_tx(brbinf)) |
| 637 |
pr_warn_once("Unknown transaction states\n"); |
| 638 |
} |
| 639 |
|
| 640 |
/* |
| 641 |
* Branch privilege level is available for target only and complete |
| 642 |
* branch records. |
| 643 |
*/ |
| 644 |
if (!brbe_record_is_source_only(brbinf)) |
| 645 |
entry->priv = brbinf_get_perf_priv(brbinf); |
| 646 |
|
| 647 |
return true; |
| 648 |
} |
| 649 |
|
| 650 |
#define PERF_BR_ARM64_ALL ( \ |
| 651 |
BIT(PERF_BR_COND) | \ |
| 652 |
BIT(PERF_BR_UNCOND) | \ |
| 653 |
BIT(PERF_BR_IND) | \ |
| 654 |
BIT(PERF_BR_CALL) | \ |
| 655 |
BIT(PERF_BR_IND_CALL) | \ |
| 656 |
BIT(PERF_BR_RET)) |
| 657 |
|
| 658 |
#define PERF_BR_ARM64_ALL_KERNEL ( \ |
| 659 |
BIT(PERF_BR_SYSCALL) | \ |
| 660 |
BIT(PERF_BR_IRQ) | \ |
| 661 |
BIT(PERF_BR_SERROR) | \ |
| 662 |
BIT(PERF_BR_MAX + PERF_BR_NEW_FAULT_ALGN) | \ |
| 663 |
BIT(PERF_BR_MAX + PERF_BR_NEW_FAULT_DATA) | \ |
| 664 |
BIT(PERF_BR_MAX + PERF_BR_NEW_FAULT_INST)) |
| 665 |
|
| 666 |
static void prepare_event_branch_type_mask(u64 branch_sample, |
| 667 |
unsigned long *event_type_mask) |
| 668 |
{ |
| 669 |
if (branch_sample & PERF_SAMPLE_BRANCH_ANY) { |
| 670 |
if (branch_sample & PERF_SAMPLE_BRANCH_KERNEL) |
| 671 |
bitmap_from_u64(event_type_mask, |
| 672 |
BIT(PERF_BR_ERET) | PERF_BR_ARM64_ALL | |
| 673 |
PERF_BR_ARM64_ALL_KERNEL); |
| 674 |
else |
| 675 |
bitmap_from_u64(event_type_mask, PERF_BR_ARM64_ALL); |
| 676 |
return; |
| 677 |
} |
| 678 |
|
| 679 |
bitmap_zero(event_type_mask, PERF_BR_ARM64_MAX); |
| 680 |
|
| 681 |
if (branch_sample & PERF_SAMPLE_BRANCH_ANY_CALL) { |
| 682 |
if (branch_sample & PERF_SAMPLE_BRANCH_KERNEL) |
| 683 |
bitmap_from_u64(event_type_mask, PERF_BR_ARM64_ALL_KERNEL); |
| 684 |
|
| 685 |
set_bit(PERF_BR_CALL, event_type_mask); |
| 686 |
set_bit(PERF_BR_IND_CALL, event_type_mask); |
| 687 |
} |
| 688 |
|
| 689 |
if (branch_sample & PERF_SAMPLE_BRANCH_IND_JUMP) |
| 690 |
set_bit(PERF_BR_IND, event_type_mask); |
| 691 |
|
| 692 |
if (branch_sample & PERF_SAMPLE_BRANCH_COND) |
| 693 |
set_bit(PERF_BR_COND, event_type_mask); |
| 694 |
|
| 695 |
if (branch_sample & PERF_SAMPLE_BRANCH_CALL) |
| 696 |
set_bit(PERF_BR_CALL, event_type_mask); |
| 697 |
|
| 698 |
if (branch_sample & PERF_SAMPLE_BRANCH_IND_CALL) |
| 699 |
set_bit(PERF_BR_IND_CALL, event_type_mask); |
| 700 |
|
| 701 |
if (branch_sample & PERF_SAMPLE_BRANCH_ANY_RETURN) { |
| 702 |
set_bit(PERF_BR_RET, event_type_mask); |
| 703 |
|
| 704 |
if (branch_sample & PERF_SAMPLE_BRANCH_KERNEL) |
| 705 |
set_bit(PERF_BR_ERET, event_type_mask); |
| 706 |
} |
| 707 |
} |
| 708 |
|
| 709 |
/* |
| 710 |
* BRBE is configured with an OR of permissions from all events, so there may |
| 711 |
* be events which have to be dropped or events where just the source or target |
| 712 |
* address has to be zeroed. |
| 713 |
*/ |
| 714 |
static bool filter_branch_privilege(struct perf_branch_entry *entry, u64 branch_sample_type) |
| 715 |
{ |
| 716 |
bool from_user = access_ok((void __user *)(unsigned long)entry->from, 4); |
| 717 |
bool to_user = access_ok((void __user *)(unsigned long)entry->to, 4); |
| 718 |
bool exclude_kernel = !((branch_sample_type & PERF_SAMPLE_BRANCH_KERNEL) || |
| 719 |
(is_kernel_in_hyp_mode() && (branch_sample_type & PERF_SAMPLE_BRANCH_HV))); |
| 720 |
|
| 721 |
/* We can only have a half record if permissions have not been expanded */ |
| 722 |
if (!entry->from || !entry->to) |
| 723 |
return true; |
| 724 |
|
| 725 |
/* |
| 726 |
* If record is within a single exception level, just need to either |
| 727 |
* drop or keep the entire record. |
| 728 |
*/ |
| 729 |
if (from_user == to_user) |
| 730 |
return ((entry->priv == PERF_BR_PRIV_KERNEL) && !exclude_kernel) || |
| 731 |
((entry->priv == PERF_BR_PRIV_USER) && |
| 732 |
(branch_sample_type & PERF_SAMPLE_BRANCH_USER)); |
| 733 |
|
| 734 |
/* |
| 735 |
* Record is across exception levels, mask addresses for the exception |
| 736 |
* level we're not capturing. |
| 737 |
*/ |
| 738 |
if (!(branch_sample_type & PERF_SAMPLE_BRANCH_USER)) { |
| 739 |
if (from_user) |
| 740 |
entry->from = 0; |
| 741 |
if (to_user) |
| 742 |
entry->to = 0; |
| 743 |
} |
| 744 |
|
| 745 |
if (exclude_kernel) { |
| 746 |
if (!from_user) |
| 747 |
entry->from = 0; |
| 748 |
if (!to_user) |
| 749 |
entry->to = 0; |
| 750 |
} |
| 751 |
|
| 752 |
return true; |
| 753 |
} |
| 754 |
|
| 755 |
static bool filter_branch_type(struct perf_branch_entry *entry, |
| 756 |
const unsigned long *event_type_mask) |
| 757 |
{ |
| 758 |
if (entry->type == PERF_BR_EXTEND_ABI) |
| 759 |
return test_bit(PERF_BR_MAX + entry->new_type, event_type_mask); |
| 760 |
else |
| 761 |
return test_bit(entry->type, event_type_mask); |
| 762 |
} |
| 763 |
|
| 764 |
static bool filter_branch_record(struct perf_branch_entry *entry, |
| 765 |
u64 branch_sample, |
| 766 |
const unsigned long *event_type_mask) |
| 767 |
{ |
| 768 |
return filter_branch_type(entry, event_type_mask) && |
| 769 |
filter_branch_privilege(entry, branch_sample); |
| 770 |
} |
| 771 |
|
| 772 |
void brbe_read_filtered_entries(struct perf_branch_stack *branch_stack, |
| 773 |
const struct perf_event *event) |
| 774 |
{ |
| 775 |
struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); |
| 776 |
int nr_hw = brbe_num_branch_records(cpu_pmu); |
| 777 |
int nr_banks = DIV_ROUND_UP(nr_hw, BRBE_BANK_MAX_ENTRIES); |
| 778 |
int nr_filtered = 0; |
| 779 |
u64 branch_sample_type = event->attr.branch_sample_type; |
| 780 |
DECLARE_BITMAP(event_type_mask, PERF_BR_ARM64_MAX); |
| 781 |
|
| 782 |
prepare_event_branch_type_mask(branch_sample_type, event_type_mask); |
| 783 |
|
| 784 |
for (int bank = 0; bank < nr_banks; bank++) { |
| 785 |
int nr_remaining = nr_hw - (bank * BRBE_BANK_MAX_ENTRIES); |
| 786 |
int nr_this_bank = min(nr_remaining, BRBE_BANK_MAX_ENTRIES); |
| 787 |
|
| 788 |
select_brbe_bank(bank); |
| 789 |
|
| 790 |
for (int i = 0; i < nr_this_bank; i++) { |
| 791 |
struct perf_branch_entry *pbe = &branch_stack->entries[nr_filtered]; |
| 792 |
|
| 793 |
if (!perf_entry_from_brbe_regset(i, pbe, event)) |
| 794 |
goto done; |
| 795 |
|
| 796 |
if (!filter_branch_record(pbe, branch_sample_type, event_type_mask)) |
| 797 |
continue; |
| 798 |
|
| 799 |
nr_filtered++; |
| 800 |
} |
| 801 |
} |
| 802 |
|
| 803 |
done: |
| 804 |
branch_stack->nr = nr_filtered; |
| 805 |
} |