| 1 |
// SPDX-License-Identifier: GPL-2.0 |
| 2 |
/* |
| 3 |
* CPU PMU driver for the Apple M1 and derivatives |
| 4 |
* |
| 5 |
* Copyright (C) 2021 Google LLC |
| 6 |
* |
| 7 |
* Author: Marc Zyngier <[email protected]> |
| 8 |
* |
| 9 |
* Most of the information used in this driver was provided by the |
| 10 |
* Asahi Linux project. The rest was experimentally discovered. |
| 11 |
*/ |
| 12 |
|
| 13 |
#include <linux/of.h> |
| 14 |
#include <linux/perf/arm_pmu.h> |
| 15 |
#include <linux/perf/arm_pmuv3.h> |
| 16 |
#include <linux/platform_device.h> |
| 17 |
|
| 18 |
#include <asm/apple_m1_pmu.h> |
| 19 |
#include <asm/irq_regs.h> |
| 20 |
#include <asm/perf_event.h> |
| 21 |
|
| 22 |
#define M1_PMU_NR_COUNTERS 10 |
| 23 |
|
| 24 |
#define M1_PMU_CFG_EVENT GENMASK(7, 0) |
| 25 |
|
| 26 |
#define ANY_BUT_0_1 GENMASK(9, 2) |
| 27 |
#define ONLY_2_TO_7 GENMASK(7, 2) |
| 28 |
#define ONLY_2_4_6 (BIT(2) | BIT(4) | BIT(6)) |
| 29 |
#define ONLY_5_6_7 (BIT(5) | BIT(6) | BIT(7)) |
| 30 |
|
| 31 |
/* |
| 32 |
* Description of the events we actually know about, as well as those with |
| 33 |
* a specific counter affinity. Yes, this is a grand total of two known |
| 34 |
* counters, and the rest is anybody's guess. |
| 35 |
* |
| 36 |
* Not all counters can count all events. Counters #0 and #1 are wired to |
| 37 |
* count cycles and instructions respectively, and some events have |
| 38 |
* bizarre mappings (every other counter, or even *one* counter). These |
| 39 |
* restrictions equally apply to both P and E cores. |
| 40 |
* |
| 41 |
* It is worth noting that the PMUs attached to P and E cores are likely |
| 42 |
* to be different because the underlying uarches are different. At the |
| 43 |
* moment, we don't really need to distinguish between the two because we |
| 44 |
* know next to nothing about the events themselves, and we already have |
| 45 |
* per cpu-type PMU abstractions. |
| 46 |
* |
| 47 |
* If we eventually find out that the events are different across |
| 48 |
* implementations, we'll have to introduce per cpu-type tables. |
| 49 |
*/ |
| 50 |
enum m1_pmu_events { |
| 51 |
M1_PMU_PERFCTR_RETIRE_UOP = 0x1, |
| 52 |
M1_PMU_PERFCTR_CORE_ACTIVE_CYCLE = 0x2, |
| 53 |
M1_PMU_PERFCTR_L1I_TLB_FILL = 0x4, |
| 54 |
M1_PMU_PERFCTR_L1D_TLB_FILL = 0x5, |
| 55 |
M1_PMU_PERFCTR_MMU_TABLE_WALK_INSTRUCTION = 0x7, |
| 56 |
M1_PMU_PERFCTR_MMU_TABLE_WALK_DATA = 0x8, |
| 57 |
M1_PMU_PERFCTR_L2_TLB_MISS_INSTRUCTION = 0xa, |
| 58 |
M1_PMU_PERFCTR_L2_TLB_MISS_DATA = 0xb, |
| 59 |
M1_PMU_PERFCTR_MMU_VIRTUAL_MEMORY_FAULT_NONSPEC = 0xd, |
| 60 |
M1_PMU_PERFCTR_SCHEDULE_UOP = 0x52, |
| 61 |
M1_PMU_PERFCTR_INTERRUPT_PENDING = 0x6c, |
| 62 |
M1_PMU_PERFCTR_MAP_STALL_DISPATCH = 0x70, |
| 63 |
M1_PMU_PERFCTR_MAP_REWIND = 0x75, |
| 64 |
M1_PMU_PERFCTR_MAP_STALL = 0x76, |
| 65 |
M1_PMU_PERFCTR_MAP_INT_UOP = 0x7c, |
| 66 |
M1_PMU_PERFCTR_MAP_LDST_UOP = 0x7d, |
| 67 |
M1_PMU_PERFCTR_MAP_SIMD_UOP = 0x7e, |
| 68 |
M1_PMU_PERFCTR_FLUSH_RESTART_OTHER_NONSPEC = 0x84, |
| 69 |
M1_PMU_PERFCTR_INST_ALL = 0x8c, |
| 70 |
M1_PMU_PERFCTR_INST_BRANCH = 0x8d, |
| 71 |
M1_PMU_PERFCTR_INST_BRANCH_CALL = 0x8e, |
| 72 |
M1_PMU_PERFCTR_INST_BRANCH_RET = 0x8f, |
| 73 |
M1_PMU_PERFCTR_INST_BRANCH_TAKEN = 0x90, |
| 74 |
M1_PMU_PERFCTR_INST_BRANCH_INDIR = 0x93, |
| 75 |
M1_PMU_PERFCTR_INST_BRANCH_COND = 0x94, |
| 76 |
M1_PMU_PERFCTR_INST_INT_LD = 0x95, |
| 77 |
M1_PMU_PERFCTR_INST_INT_ST = 0x96, |
| 78 |
M1_PMU_PERFCTR_INST_INT_ALU = 0x97, |
| 79 |
M1_PMU_PERFCTR_INST_SIMD_LD = 0x98, |
| 80 |
M1_PMU_PERFCTR_INST_SIMD_ST = 0x99, |
| 81 |
M1_PMU_PERFCTR_INST_SIMD_ALU = 0x9a, |
| 82 |
M1_PMU_PERFCTR_INST_LDST = 0x9b, |
| 83 |
M1_PMU_PERFCTR_INST_BARRIER = 0x9c, |
| 84 |
M1_PMU_PERFCTR_UNKNOWN_9f = 0x9f, |
| 85 |
M1_PMU_PERFCTR_L1D_TLB_ACCESS = 0xa0, |
| 86 |
M1_PMU_PERFCTR_L1D_TLB_MISS = 0xa1, |
| 87 |
M1_PMU_PERFCTR_L1D_CACHE_MISS_ST = 0xa2, |
| 88 |
M1_PMU_PERFCTR_L1D_CACHE_MISS_LD = 0xa3, |
| 89 |
M1_PMU_PERFCTR_LD_UNIT_UOP = 0xa6, |
| 90 |
M1_PMU_PERFCTR_ST_UNIT_UOP = 0xa7, |
| 91 |
M1_PMU_PERFCTR_L1D_CACHE_WRITEBACK = 0xa8, |
| 92 |
M1_PMU_PERFCTR_LDST_X64_UOP = 0xb1, |
| 93 |
M1_PMU_PERFCTR_LDST_XPG_UOP = 0xb2, |
| 94 |
M1_PMU_PERFCTR_ATOMIC_OR_EXCLUSIVE_SUCC = 0xb3, |
| 95 |
M1_PMU_PERFCTR_ATOMIC_OR_EXCLUSIVE_FAIL = 0xb4, |
| 96 |
M1_PMU_PERFCTR_L1D_CACHE_MISS_LD_NONSPEC = 0xbf, |
| 97 |
M1_PMU_PERFCTR_L1D_CACHE_MISS_ST_NONSPEC = 0xc0, |
| 98 |
M1_PMU_PERFCTR_L1D_TLB_MISS_NONSPEC = 0xc1, |
| 99 |
M1_PMU_PERFCTR_ST_MEMORY_ORDER_VIOLATION_NONSPEC = 0xc4, |
| 100 |
M1_PMU_PERFCTR_BRANCH_COND_MISPRED_NONSPEC = 0xc5, |
| 101 |
M1_PMU_PERFCTR_BRANCH_INDIR_MISPRED_NONSPEC = 0xc6, |
| 102 |
M1_PMU_PERFCTR_BRANCH_RET_INDIR_MISPRED_NONSPEC = 0xc8, |
| 103 |
M1_PMU_PERFCTR_BRANCH_CALL_INDIR_MISPRED_NONSPEC = 0xca, |
| 104 |
M1_PMU_PERFCTR_BRANCH_MISPRED_NONSPEC = 0xcb, |
| 105 |
M1_PMU_PERFCTR_L1I_TLB_MISS_DEMAND = 0xd4, |
| 106 |
M1_PMU_PERFCTR_MAP_DISPATCH_BUBBLE = 0xd6, |
| 107 |
M1_PMU_PERFCTR_L1I_CACHE_MISS_DEMAND = 0xdb, |
| 108 |
M1_PMU_PERFCTR_FETCH_RESTART = 0xde, |
| 109 |
M1_PMU_PERFCTR_ST_NT_UOP = 0xe5, |
| 110 |
M1_PMU_PERFCTR_LD_NT_UOP = 0xe6, |
| 111 |
M1_PMU_PERFCTR_UNKNOWN_f5 = 0xf5, |
| 112 |
M1_PMU_PERFCTR_UNKNOWN_f6 = 0xf6, |
| 113 |
M1_PMU_PERFCTR_UNKNOWN_f7 = 0xf7, |
| 114 |
M1_PMU_PERFCTR_UNKNOWN_f8 = 0xf8, |
| 115 |
M1_PMU_PERFCTR_UNKNOWN_fd = 0xfd, |
| 116 |
M1_PMU_PERFCTR_LAST = M1_PMU_CFG_EVENT, |
| 117 |
|
| 118 |
/* |
| 119 |
* From this point onwards, these are not actual HW events, |
| 120 |
* but attributes that get stored in hw->config_base. |
| 121 |
*/ |
| 122 |
M1_PMU_CFG_COUNT_USER = BIT(8), |
| 123 |
M1_PMU_CFG_COUNT_KERNEL = BIT(9), |
| 124 |
M1_PMU_CFG_COUNT_HOST = BIT(10), |
| 125 |
M1_PMU_CFG_COUNT_GUEST = BIT(11), |
| 126 |
}; |
| 127 |
|
| 128 |
/* |
| 129 |
* Per-event affinity table. Most events can be installed on counter |
| 130 |
* 2-9, but there are a number of exceptions. Note that this table |
| 131 |
* has been created experimentally, and I wouldn't be surprised if more |
| 132 |
* counters had strange affinities. |
| 133 |
*/ |
| 134 |
static const u16 m1_pmu_event_affinity[M1_PMU_PERFCTR_LAST + 1] = { |
| 135 |
[0 ... M1_PMU_PERFCTR_LAST] = ANY_BUT_0_1, |
| 136 |
[M1_PMU_PERFCTR_RETIRE_UOP] = BIT(7), |
| 137 |
[M1_PMU_PERFCTR_CORE_ACTIVE_CYCLE] = ANY_BUT_0_1 | BIT(0), |
| 138 |
[M1_PMU_PERFCTR_INST_ALL] = BIT(7) | BIT(1), |
| 139 |
[M1_PMU_PERFCTR_INST_BRANCH] = ONLY_5_6_7, |
| 140 |
[M1_PMU_PERFCTR_INST_BRANCH_CALL] = ONLY_5_6_7, |
| 141 |
[M1_PMU_PERFCTR_INST_BRANCH_RET] = ONLY_5_6_7, |
| 142 |
[M1_PMU_PERFCTR_INST_BRANCH_TAKEN] = ONLY_5_6_7, |
| 143 |
[M1_PMU_PERFCTR_INST_BRANCH_INDIR] = ONLY_5_6_7, |
| 144 |
[M1_PMU_PERFCTR_INST_BRANCH_COND] = ONLY_5_6_7, |
| 145 |
[M1_PMU_PERFCTR_INST_INT_LD] = ONLY_5_6_7, |
| 146 |
[M1_PMU_PERFCTR_INST_INT_ST] = BIT(7), |
| 147 |
[M1_PMU_PERFCTR_INST_INT_ALU] = BIT(7), |
| 148 |
[M1_PMU_PERFCTR_INST_SIMD_LD] = ONLY_5_6_7, |
| 149 |
[M1_PMU_PERFCTR_INST_SIMD_ST] = ONLY_5_6_7, |
| 150 |
[M1_PMU_PERFCTR_INST_SIMD_ALU] = BIT(7), |
| 151 |
[M1_PMU_PERFCTR_INST_LDST] = BIT(7), |
| 152 |
[M1_PMU_PERFCTR_INST_BARRIER] = ONLY_5_6_7, |
| 153 |
[M1_PMU_PERFCTR_UNKNOWN_9f] = BIT(7), |
| 154 |
[M1_PMU_PERFCTR_L1D_CACHE_MISS_LD_NONSPEC] = ONLY_5_6_7, |
| 155 |
[M1_PMU_PERFCTR_L1D_CACHE_MISS_ST_NONSPEC] = ONLY_5_6_7, |
| 156 |
[M1_PMU_PERFCTR_L1D_TLB_MISS_NONSPEC] = ONLY_5_6_7, |
| 157 |
[M1_PMU_PERFCTR_ST_MEMORY_ORDER_VIOLATION_NONSPEC] = ONLY_5_6_7, |
| 158 |
[M1_PMU_PERFCTR_BRANCH_COND_MISPRED_NONSPEC] = ONLY_5_6_7, |
| 159 |
[M1_PMU_PERFCTR_BRANCH_INDIR_MISPRED_NONSPEC] = ONLY_5_6_7, |
| 160 |
[M1_PMU_PERFCTR_BRANCH_RET_INDIR_MISPRED_NONSPEC] = ONLY_5_6_7, |
| 161 |
[M1_PMU_PERFCTR_BRANCH_CALL_INDIR_MISPRED_NONSPEC] = ONLY_5_6_7, |
| 162 |
[M1_PMU_PERFCTR_BRANCH_MISPRED_NONSPEC] = ONLY_5_6_7, |
| 163 |
[M1_PMU_PERFCTR_UNKNOWN_f5] = ONLY_2_4_6, |
| 164 |
[M1_PMU_PERFCTR_UNKNOWN_f6] = ONLY_2_4_6, |
| 165 |
[M1_PMU_PERFCTR_UNKNOWN_f7] = ONLY_2_4_6, |
| 166 |
[M1_PMU_PERFCTR_UNKNOWN_f8] = ONLY_2_TO_7, |
| 167 |
[M1_PMU_PERFCTR_UNKNOWN_fd] = ONLY_2_4_6, |
| 168 |
}; |
| 169 |
|
| 170 |
static const unsigned m1_pmu_perf_map[PERF_COUNT_HW_MAX] = { |
| 171 |
PERF_MAP_ALL_UNSUPPORTED, |
| 172 |
[PERF_COUNT_HW_CPU_CYCLES] = M1_PMU_PERFCTR_CORE_ACTIVE_CYCLE, |
| 173 |
[PERF_COUNT_HW_INSTRUCTIONS] = M1_PMU_PERFCTR_INST_ALL, |
| 174 |
[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = M1_PMU_PERFCTR_INST_BRANCH, |
| 175 |
[PERF_COUNT_HW_BRANCH_MISSES] = M1_PMU_PERFCTR_BRANCH_MISPRED_NONSPEC, |
| 176 |
}; |
| 177 |
|
| 178 |
#define M1_PMUV3_EVENT_MAP(pmuv3_event, m1_event) \ |
| 179 |
[ARMV8_PMUV3_PERFCTR_##pmuv3_event] = M1_PMU_PERFCTR_##m1_event |
| 180 |
|
| 181 |
static const u16 m1_pmu_pmceid_map[ARMV8_PMUV3_MAX_COMMON_EVENTS] = { |
| 182 |
[0 ... ARMV8_PMUV3_MAX_COMMON_EVENTS - 1] = HW_OP_UNSUPPORTED, |
| 183 |
M1_PMUV3_EVENT_MAP(INST_RETIRED, INST_ALL), |
| 184 |
M1_PMUV3_EVENT_MAP(CPU_CYCLES, CORE_ACTIVE_CYCLE), |
| 185 |
M1_PMUV3_EVENT_MAP(BR_RETIRED, INST_BRANCH), |
| 186 |
M1_PMUV3_EVENT_MAP(BR_MIS_PRED_RETIRED, BRANCH_MISPRED_NONSPEC), |
| 187 |
}; |
| 188 |
|
| 189 |
/* sysfs definitions */ |
| 190 |
static ssize_t m1_pmu_events_sysfs_show(struct device *dev, |
| 191 |
struct device_attribute *attr, |
| 192 |
char *page) |
| 193 |
{ |
| 194 |
struct perf_pmu_events_attr *pmu_attr; |
| 195 |
|
| 196 |
pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr); |
| 197 |
|
| 198 |
return sprintf(page, "event=0x%04llx\n", pmu_attr->id); |
| 199 |
} |
| 200 |
|
| 201 |
#define M1_PMU_EVENT_ATTR(name, config) \ |
| 202 |
PMU_EVENT_ATTR_ID(name, m1_pmu_events_sysfs_show, config) |
| 203 |
|
| 204 |
static struct attribute *m1_pmu_event_attrs[] = { |
| 205 |
M1_PMU_EVENT_ATTR(cycles, M1_PMU_PERFCTR_CORE_ACTIVE_CYCLE), |
| 206 |
M1_PMU_EVENT_ATTR(instructions, M1_PMU_PERFCTR_INST_ALL), |
| 207 |
NULL, |
| 208 |
}; |
| 209 |
|
| 210 |
static const struct attribute_group m1_pmu_events_attr_group = { |
| 211 |
.name = "events", |
| 212 |
.attrs = m1_pmu_event_attrs, |
| 213 |
}; |
| 214 |
|
| 215 |
PMU_FORMAT_ATTR(event, "config:0-7"); |
| 216 |
|
| 217 |
static struct attribute *m1_pmu_format_attrs[] = { |
| 218 |
&format_attr_event.attr, |
| 219 |
NULL, |
| 220 |
}; |
| 221 |
|
| 222 |
static const struct attribute_group m1_pmu_format_attr_group = { |
| 223 |
.name = "format", |
| 224 |
.attrs = m1_pmu_format_attrs, |
| 225 |
}; |
| 226 |
|
| 227 |
/* Low level accessors. No synchronisation. */ |
| 228 |
#define PMU_READ_COUNTER(_idx) \ |
| 229 |
case _idx: return read_sysreg_s(SYS_IMP_APL_PMC## _idx ##_EL1) |
| 230 |
|
| 231 |
#define PMU_WRITE_COUNTER(_val, _idx) \ |
| 232 |
case _idx: \ |
| 233 |
write_sysreg_s(_val, SYS_IMP_APL_PMC## _idx ##_EL1); \ |
| 234 |
return |
| 235 |
|
| 236 |
static u64 m1_pmu_read_hw_counter(unsigned int index) |
| 237 |
{ |
| 238 |
switch (index) { |
| 239 |
PMU_READ_COUNTER(0); |
| 240 |
PMU_READ_COUNTER(1); |
| 241 |
PMU_READ_COUNTER(2); |
| 242 |
PMU_READ_COUNTER(3); |
| 243 |
PMU_READ_COUNTER(4); |
| 244 |
PMU_READ_COUNTER(5); |
| 245 |
PMU_READ_COUNTER(6); |
| 246 |
PMU_READ_COUNTER(7); |
| 247 |
PMU_READ_COUNTER(8); |
| 248 |
PMU_READ_COUNTER(9); |
| 249 |
} |
| 250 |
|
| 251 |
BUG(); |
| 252 |
} |
| 253 |
|
| 254 |
static void m1_pmu_write_hw_counter(u64 val, unsigned int index) |
| 255 |
{ |
| 256 |
switch (index) { |
| 257 |
PMU_WRITE_COUNTER(val, 0); |
| 258 |
PMU_WRITE_COUNTER(val, 1); |
| 259 |
PMU_WRITE_COUNTER(val, 2); |
| 260 |
PMU_WRITE_COUNTER(val, 3); |
| 261 |
PMU_WRITE_COUNTER(val, 4); |
| 262 |
PMU_WRITE_COUNTER(val, 5); |
| 263 |
PMU_WRITE_COUNTER(val, 6); |
| 264 |
PMU_WRITE_COUNTER(val, 7); |
| 265 |
PMU_WRITE_COUNTER(val, 8); |
| 266 |
PMU_WRITE_COUNTER(val, 9); |
| 267 |
} |
| 268 |
|
| 269 |
BUG(); |
| 270 |
} |
| 271 |
|
| 272 |
#define get_bit_offset(index, mask) (__ffs(mask) + (index)) |
| 273 |
|
| 274 |
static void __m1_pmu_enable_counter(unsigned int index, bool en) |
| 275 |
{ |
| 276 |
u64 val, bit; |
| 277 |
|
| 278 |
switch (index) { |
| 279 |
case 0 ... 7: |
| 280 |
bit = BIT(get_bit_offset(index, PMCR0_CNT_ENABLE_0_7)); |
| 281 |
break; |
| 282 |
case 8 ... 9: |
| 283 |
bit = BIT(get_bit_offset(index - 8, PMCR0_CNT_ENABLE_8_9)); |
| 284 |
break; |
| 285 |
default: |
| 286 |
BUG(); |
| 287 |
} |
| 288 |
|
| 289 |
val = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1); |
| 290 |
|
| 291 |
if (en) |
| 292 |
val |= bit; |
| 293 |
else |
| 294 |
val &= ~bit; |
| 295 |
|
| 296 |
write_sysreg_s(val, SYS_IMP_APL_PMCR0_EL1); |
| 297 |
} |
| 298 |
|
| 299 |
static void m1_pmu_enable_counter(unsigned int index) |
| 300 |
{ |
| 301 |
__m1_pmu_enable_counter(index, true); |
| 302 |
} |
| 303 |
|
| 304 |
static void m1_pmu_disable_counter(unsigned int index) |
| 305 |
{ |
| 306 |
__m1_pmu_enable_counter(index, false); |
| 307 |
} |
| 308 |
|
| 309 |
static void __m1_pmu_enable_counter_interrupt(unsigned int index, bool en) |
| 310 |
{ |
| 311 |
u64 val, bit; |
| 312 |
|
| 313 |
switch (index) { |
| 314 |
case 0 ... 7: |
| 315 |
bit = BIT(get_bit_offset(index, PMCR0_PMI_ENABLE_0_7)); |
| 316 |
break; |
| 317 |
case 8 ... 9: |
| 318 |
bit = BIT(get_bit_offset(index - 8, PMCR0_PMI_ENABLE_8_9)); |
| 319 |
break; |
| 320 |
default: |
| 321 |
BUG(); |
| 322 |
} |
| 323 |
|
| 324 |
val = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1); |
| 325 |
|
| 326 |
if (en) |
| 327 |
val |= bit; |
| 328 |
else |
| 329 |
val &= ~bit; |
| 330 |
|
| 331 |
write_sysreg_s(val, SYS_IMP_APL_PMCR0_EL1); |
| 332 |
} |
| 333 |
|
| 334 |
static void m1_pmu_enable_counter_interrupt(unsigned int index) |
| 335 |
{ |
| 336 |
__m1_pmu_enable_counter_interrupt(index, true); |
| 337 |
} |
| 338 |
|
| 339 |
static void m1_pmu_disable_counter_interrupt(unsigned int index) |
| 340 |
{ |
| 341 |
__m1_pmu_enable_counter_interrupt(index, false); |
| 342 |
} |
| 343 |
|
| 344 |
static void __m1_pmu_configure_event_filter(unsigned int index, bool user, |
| 345 |
bool kernel, bool host) |
| 346 |
{ |
| 347 |
u64 clear, set, user_bit, kernel_bit; |
| 348 |
|
| 349 |
switch (index) { |
| 350 |
case 0 ... 7: |
| 351 |
user_bit = BIT(get_bit_offset(index, PMCR1_COUNT_A64_EL0_0_7)); |
| 352 |
kernel_bit = BIT(get_bit_offset(index, PMCR1_COUNT_A64_EL1_0_7)); |
| 353 |
break; |
| 354 |
case 8 ... 9: |
| 355 |
user_bit = BIT(get_bit_offset(index - 8, PMCR1_COUNT_A64_EL0_8_9)); |
| 356 |
kernel_bit = BIT(get_bit_offset(index - 8, PMCR1_COUNT_A64_EL1_8_9)); |
| 357 |
break; |
| 358 |
default: |
| 359 |
BUG(); |
| 360 |
} |
| 361 |
|
| 362 |
clear = set = 0; |
| 363 |
if (user) |
| 364 |
set |= user_bit; |
| 365 |
else |
| 366 |
clear |= user_bit; |
| 367 |
|
| 368 |
if (kernel) |
| 369 |
set |= kernel_bit; |
| 370 |
else |
| 371 |
clear |= kernel_bit; |
| 372 |
|
| 373 |
if (host) |
| 374 |
sysreg_clear_set_s(SYS_IMP_APL_PMCR1_EL1, clear, set); |
| 375 |
else if (is_kernel_in_hyp_mode()) |
| 376 |
sysreg_clear_set_s(SYS_IMP_APL_PMCR1_EL12, clear, set); |
| 377 |
} |
| 378 |
|
| 379 |
static void __m1_pmu_configure_eventsel(unsigned int index, u8 event) |
| 380 |
{ |
| 381 |
u64 clear = 0, set = 0; |
| 382 |
int shift; |
| 383 |
|
| 384 |
/* |
| 385 |
* Counters 0 and 1 have fixed events. For anything else, |
| 386 |
* place the event at the expected location in the relevant |
| 387 |
* register (PMESR0 holds the event configuration for counters |
| 388 |
* 2-5, resp. PMESR1 for counters 6-9). |
| 389 |
*/ |
| 390 |
switch (index) { |
| 391 |
case 0 ... 1: |
| 392 |
break; |
| 393 |
case 2 ... 5: |
| 394 |
shift = (index - 2) * 8; |
| 395 |
clear |= (u64)0xff << shift; |
| 396 |
set |= (u64)event << shift; |
| 397 |
sysreg_clear_set_s(SYS_IMP_APL_PMESR0_EL1, clear, set); |
| 398 |
break; |
| 399 |
case 6 ... 9: |
| 400 |
shift = (index - 6) * 8; |
| 401 |
clear |= (u64)0xff << shift; |
| 402 |
set |= (u64)event << shift; |
| 403 |
sysreg_clear_set_s(SYS_IMP_APL_PMESR1_EL1, clear, set); |
| 404 |
break; |
| 405 |
} |
| 406 |
} |
| 407 |
|
| 408 |
static void m1_pmu_configure_counter(unsigned int index, unsigned long config_base) |
| 409 |
{ |
| 410 |
bool kernel = config_base & M1_PMU_CFG_COUNT_KERNEL; |
| 411 |
bool guest = config_base & M1_PMU_CFG_COUNT_GUEST; |
| 412 |
bool host = config_base & M1_PMU_CFG_COUNT_HOST; |
| 413 |
bool user = config_base & M1_PMU_CFG_COUNT_USER; |
| 414 |
u8 evt = config_base & M1_PMU_CFG_EVENT; |
| 415 |
|
| 416 |
__m1_pmu_configure_event_filter(index, user && host, kernel && host, true); |
| 417 |
__m1_pmu_configure_event_filter(index, user && guest, kernel && guest, false); |
| 418 |
__m1_pmu_configure_eventsel(index, evt); |
| 419 |
} |
| 420 |
|
| 421 |
/* arm_pmu backend */ |
| 422 |
static void m1_pmu_enable_event(struct perf_event *event) |
| 423 |
{ |
| 424 |
bool user, kernel; |
| 425 |
u8 evt; |
| 426 |
|
| 427 |
evt = event->hw.config_base & M1_PMU_CFG_EVENT; |
| 428 |
user = event->hw.config_base & M1_PMU_CFG_COUNT_USER; |
| 429 |
kernel = event->hw.config_base & M1_PMU_CFG_COUNT_KERNEL; |
| 430 |
|
| 431 |
m1_pmu_configure_counter(event->hw.idx, event->hw.config_base); |
| 432 |
m1_pmu_enable_counter(event->hw.idx); |
| 433 |
m1_pmu_enable_counter_interrupt(event->hw.idx); |
| 434 |
isb(); |
| 435 |
} |
| 436 |
|
| 437 |
static void m1_pmu_disable_event(struct perf_event *event) |
| 438 |
{ |
| 439 |
m1_pmu_disable_counter_interrupt(event->hw.idx); |
| 440 |
m1_pmu_disable_counter(event->hw.idx); |
| 441 |
isb(); |
| 442 |
} |
| 443 |
|
| 444 |
static irqreturn_t m1_pmu_handle_irq(struct arm_pmu *cpu_pmu) |
| 445 |
{ |
| 446 |
struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events); |
| 447 |
struct pt_regs *regs; |
| 448 |
u64 overflow, state; |
| 449 |
int idx; |
| 450 |
|
| 451 |
overflow = read_sysreg_s(SYS_IMP_APL_PMSR_EL1); |
| 452 |
if (!overflow) { |
| 453 |
/* Spurious interrupt? */ |
| 454 |
state = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1); |
| 455 |
state &= ~PMCR0_IACT; |
| 456 |
write_sysreg_s(state, SYS_IMP_APL_PMCR0_EL1); |
| 457 |
isb(); |
| 458 |
return IRQ_NONE; |
| 459 |
} |
| 460 |
|
| 461 |
cpu_pmu->stop(cpu_pmu); |
| 462 |
|
| 463 |
regs = get_irq_regs(); |
| 464 |
|
| 465 |
for_each_set_bit(idx, cpu_pmu->cntr_mask, M1_PMU_NR_COUNTERS) { |
| 466 |
struct perf_event *event = cpuc->events[idx]; |
| 467 |
struct perf_sample_data data; |
| 468 |
|
| 469 |
if (!event) |
| 470 |
continue; |
| 471 |
|
| 472 |
armpmu_event_update(event); |
| 473 |
perf_sample_data_init(&data, 0, event->hw.last_period); |
| 474 |
if (!armpmu_event_set_period(event)) |
| 475 |
continue; |
| 476 |
|
| 477 |
perf_event_overflow(event, &data, regs); |
| 478 |
} |
| 479 |
|
| 480 |
cpu_pmu->start(cpu_pmu); |
| 481 |
|
| 482 |
return IRQ_HANDLED; |
| 483 |
} |
| 484 |
|
| 485 |
static u64 m1_pmu_read_counter(struct perf_event *event) |
| 486 |
{ |
| 487 |
return m1_pmu_read_hw_counter(event->hw.idx); |
| 488 |
} |
| 489 |
|
| 490 |
static void m1_pmu_write_counter(struct perf_event *event, u64 value) |
| 491 |
{ |
| 492 |
m1_pmu_write_hw_counter(value, event->hw.idx); |
| 493 |
isb(); |
| 494 |
} |
| 495 |
|
| 496 |
static int m1_pmu_get_event_idx(struct pmu_hw_events *cpuc, |
| 497 |
struct perf_event *event) |
| 498 |
{ |
| 499 |
unsigned long evtype = event->hw.config_base & M1_PMU_CFG_EVENT; |
| 500 |
unsigned long affinity = m1_pmu_event_affinity[evtype]; |
| 501 |
int idx; |
| 502 |
|
| 503 |
/* |
| 504 |
* Place the event on the first free counter that can count |
| 505 |
* this event. |
| 506 |
* |
| 507 |
* We could do a better job if we had a view of all the events |
| 508 |
* counting on the PMU at any given time, and by placing the |
| 509 |
* most constraining events first. |
| 510 |
*/ |
| 511 |
for_each_set_bit(idx, &affinity, M1_PMU_NR_COUNTERS) { |
| 512 |
if (!test_and_set_bit(idx, cpuc->used_mask)) |
| 513 |
return idx; |
| 514 |
} |
| 515 |
|
| 516 |
return -EAGAIN; |
| 517 |
} |
| 518 |
|
| 519 |
static void m1_pmu_clear_event_idx(struct pmu_hw_events *cpuc, |
| 520 |
struct perf_event *event) |
| 521 |
{ |
| 522 |
clear_bit(event->hw.idx, cpuc->used_mask); |
| 523 |
} |
| 524 |
|
| 525 |
static void __m1_pmu_set_mode(u8 mode) |
| 526 |
{ |
| 527 |
u64 val; |
| 528 |
|
| 529 |
val = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1); |
| 530 |
val &= ~(PMCR0_IMODE | PMCR0_IACT); |
| 531 |
val |= FIELD_PREP(PMCR0_IMODE, mode); |
| 532 |
write_sysreg_s(val, SYS_IMP_APL_PMCR0_EL1); |
| 533 |
isb(); |
| 534 |
} |
| 535 |
|
| 536 |
static void m1_pmu_start(struct arm_pmu *cpu_pmu) |
| 537 |
{ |
| 538 |
__m1_pmu_set_mode(PMCR0_IMODE_FIQ); |
| 539 |
} |
| 540 |
|
| 541 |
static void m1_pmu_stop(struct arm_pmu *cpu_pmu) |
| 542 |
{ |
| 543 |
__m1_pmu_set_mode(PMCR0_IMODE_OFF); |
| 544 |
} |
| 545 |
|
| 546 |
static int m1_pmu_map_event(struct perf_event *event) |
| 547 |
{ |
| 548 |
/* |
| 549 |
* Although the counters are 48bit wide, bit 47 is what |
| 550 |
* triggers the overflow interrupt. Advertise the counters |
| 551 |
* being 47bit wide to mimick the behaviour of the ARM PMU. |
| 552 |
*/ |
| 553 |
event->hw.flags |= ARMPMU_EVT_47BIT; |
| 554 |
return armpmu_map_event(event, &m1_pmu_perf_map, NULL, M1_PMU_CFG_EVENT); |
| 555 |
} |
| 556 |
|
| 557 |
static int m2_pmu_map_event(struct perf_event *event) |
| 558 |
{ |
| 559 |
/* |
| 560 |
* Same deal as the above, except that M2 has 64bit counters. |
| 561 |
* Which, as far as we're concerned, actually means 63 bits. |
| 562 |
* Yes, this is getting awkward. |
| 563 |
*/ |
| 564 |
event->hw.flags |= ARMPMU_EVT_63BIT; |
| 565 |
return armpmu_map_event(event, &m1_pmu_perf_map, NULL, M1_PMU_CFG_EVENT); |
| 566 |
} |
| 567 |
|
| 568 |
static int m1_pmu_map_pmuv3_event(unsigned int eventsel) |
| 569 |
{ |
| 570 |
u16 m1_event = HW_OP_UNSUPPORTED; |
| 571 |
|
| 572 |
if (eventsel < ARMV8_PMUV3_MAX_COMMON_EVENTS) |
| 573 |
m1_event = m1_pmu_pmceid_map[eventsel]; |
| 574 |
|
| 575 |
return m1_event == HW_OP_UNSUPPORTED ? -EOPNOTSUPP : m1_event; |
| 576 |
} |
| 577 |
|
| 578 |
static void m1_pmu_init_pmceid(struct arm_pmu *pmu) |
| 579 |
{ |
| 580 |
unsigned int event; |
| 581 |
|
| 582 |
for (event = 0; event < ARMV8_PMUV3_MAX_COMMON_EVENTS; event++) { |
| 583 |
if (m1_pmu_map_pmuv3_event(event) >= 0) |
| 584 |
set_bit(event, pmu->pmceid_bitmap); |
| 585 |
} |
| 586 |
} |
| 587 |
|
| 588 |
static void m1_pmu_reset(void *info) |
| 589 |
{ |
| 590 |
int i; |
| 591 |
|
| 592 |
__m1_pmu_set_mode(PMCR0_IMODE_OFF); |
| 593 |
|
| 594 |
for (i = 0; i < M1_PMU_NR_COUNTERS; i++) { |
| 595 |
m1_pmu_disable_counter(i); |
| 596 |
m1_pmu_disable_counter_interrupt(i); |
| 597 |
m1_pmu_write_hw_counter(0, i); |
| 598 |
} |
| 599 |
|
| 600 |
isb(); |
| 601 |
} |
| 602 |
|
| 603 |
static int m1_pmu_set_event_filter(struct hw_perf_event *event, |
| 604 |
struct perf_event_attr *attr) |
| 605 |
{ |
| 606 |
unsigned long config_base = 0; |
| 607 |
|
| 608 |
if (!attr->exclude_guest && !is_kernel_in_hyp_mode()) { |
| 609 |
pr_debug("ARM performance counters do not support mode exclusion\n"); |
| 610 |
return -EOPNOTSUPP; |
| 611 |
} |
| 612 |
if (!attr->exclude_kernel) |
| 613 |
config_base |= M1_PMU_CFG_COUNT_KERNEL; |
| 614 |
if (!attr->exclude_user) |
| 615 |
config_base |= M1_PMU_CFG_COUNT_USER; |
| 616 |
if (!attr->exclude_host) |
| 617 |
config_base |= M1_PMU_CFG_COUNT_HOST; |
| 618 |
if (!attr->exclude_guest) |
| 619 |
config_base |= M1_PMU_CFG_COUNT_GUEST; |
| 620 |
|
| 621 |
event->config_base = config_base; |
| 622 |
|
| 623 |
return 0; |
| 624 |
} |
| 625 |
|
| 626 |
static int m1_pmu_init(struct arm_pmu *cpu_pmu, u32 flags) |
| 627 |
{ |
| 628 |
cpu_pmu->handle_irq = m1_pmu_handle_irq; |
| 629 |
cpu_pmu->enable = m1_pmu_enable_event; |
| 630 |
cpu_pmu->disable = m1_pmu_disable_event; |
| 631 |
cpu_pmu->read_counter = m1_pmu_read_counter; |
| 632 |
cpu_pmu->write_counter = m1_pmu_write_counter; |
| 633 |
cpu_pmu->get_event_idx = m1_pmu_get_event_idx; |
| 634 |
cpu_pmu->clear_event_idx = m1_pmu_clear_event_idx; |
| 635 |
cpu_pmu->start = m1_pmu_start; |
| 636 |
cpu_pmu->stop = m1_pmu_stop; |
| 637 |
|
| 638 |
if (flags & ARMPMU_EVT_47BIT) |
| 639 |
cpu_pmu->map_event = m1_pmu_map_event; |
| 640 |
else if (flags & ARMPMU_EVT_63BIT) |
| 641 |
cpu_pmu->map_event = m2_pmu_map_event; |
| 642 |
else |
| 643 |
return WARN_ON(-EINVAL); |
| 644 |
|
| 645 |
cpu_pmu->reset = m1_pmu_reset; |
| 646 |
cpu_pmu->set_event_filter = m1_pmu_set_event_filter; |
| 647 |
|
| 648 |
cpu_pmu->map_pmuv3_event = m1_pmu_map_pmuv3_event; |
| 649 |
m1_pmu_init_pmceid(cpu_pmu); |
| 650 |
|
| 651 |
bitmap_set(cpu_pmu->cntr_mask, 0, M1_PMU_NR_COUNTERS); |
| 652 |
cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = &m1_pmu_events_attr_group; |
| 653 |
cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = &m1_pmu_format_attr_group; |
| 654 |
return 0; |
| 655 |
} |
| 656 |
|
| 657 |
/* Device driver gunk */ |
| 658 |
static int m1_pmu_ice_init(struct arm_pmu *cpu_pmu) |
| 659 |
{ |
| 660 |
cpu_pmu->name = "apple_icestorm_pmu"; |
| 661 |
return m1_pmu_init(cpu_pmu, ARMPMU_EVT_47BIT); |
| 662 |
} |
| 663 |
|
| 664 |
static int m1_pmu_fire_init(struct arm_pmu *cpu_pmu) |
| 665 |
{ |
| 666 |
cpu_pmu->name = "apple_firestorm_pmu"; |
| 667 |
return m1_pmu_init(cpu_pmu, ARMPMU_EVT_47BIT); |
| 668 |
} |
| 669 |
|
| 670 |
static int m2_pmu_avalanche_init(struct arm_pmu *cpu_pmu) |
| 671 |
{ |
| 672 |
cpu_pmu->name = "apple_avalanche_pmu"; |
| 673 |
return m1_pmu_init(cpu_pmu, ARMPMU_EVT_63BIT); |
| 674 |
} |
| 675 |
|
| 676 |
static int m2_pmu_blizzard_init(struct arm_pmu *cpu_pmu) |
| 677 |
{ |
| 678 |
cpu_pmu->name = "apple_blizzard_pmu"; |
| 679 |
return m1_pmu_init(cpu_pmu, ARMPMU_EVT_63BIT); |
| 680 |
} |
| 681 |
|
| 682 |
static const struct of_device_id m1_pmu_of_device_ids[] = { |
| 683 |
{ .compatible = "apple,avalanche-pmu", .data = m2_pmu_avalanche_init, }, |
| 684 |
{ .compatible = "apple,blizzard-pmu", .data = m2_pmu_blizzard_init, }, |
| 685 |
{ .compatible = "apple,icestorm-pmu", .data = m1_pmu_ice_init, }, |
| 686 |
{ .compatible = "apple,firestorm-pmu", .data = m1_pmu_fire_init, }, |
| 687 |
{ }, |
| 688 |
}; |
| 689 |
MODULE_DEVICE_TABLE(of, m1_pmu_of_device_ids); |
| 690 |
|
| 691 |
static int m1_pmu_device_probe(struct platform_device *pdev) |
| 692 |
{ |
| 693 |
return arm_pmu_device_probe(pdev, m1_pmu_of_device_ids, NULL); |
| 694 |
} |
| 695 |
|
| 696 |
static struct platform_driver m1_pmu_driver = { |
| 697 |
.driver = { |
| 698 |
.name = "apple-m1-cpu-pmu", |
| 699 |
.of_match_table = m1_pmu_of_device_ids, |
| 700 |
.suppress_bind_attrs = true, |
| 701 |
}, |
| 702 |
.probe = m1_pmu_device_probe, |
| 703 |
}; |
| 704 |
|
| 705 |
module_platform_driver(m1_pmu_driver); |