| 1 |
// SPDX-License-Identifier: GPL-2.0-only |
| 2 |
#undef DEBUG |
| 3 |
|
| 4 |
/* |
| 5 |
* ARM performance counter support. |
| 6 |
* |
| 7 |
* Copyright (C) 2009 picoChip Designs, Ltd., Jamie Iles |
| 8 |
* Copyright (C) 2010 ARM Ltd., Will Deacon <[email protected]> |
| 9 |
* |
| 10 |
* This code is based on the sparc64 perf event code, which is in turn based |
| 11 |
* on the x86 code. |
| 12 |
*/ |
| 13 |
#define pr_fmt(fmt) "hw perfevents: " fmt |
| 14 |
|
| 15 |
#include <linux/bitmap.h> |
| 16 |
#include <linux/cpumask.h> |
| 17 |
#include <linux/cpu_pm.h> |
| 18 |
#include <linux/export.h> |
| 19 |
#include <linux/kernel.h> |
| 20 |
#include <linux/perf/arm_pmu.h> |
| 21 |
#include <linux/slab.h> |
| 22 |
#include <linux/sched/clock.h> |
| 23 |
#include <linux/spinlock.h> |
| 24 |
#include <linux/irq.h> |
| 25 |
#include <linux/irqdesc.h> |
| 26 |
|
| 27 |
#include <asm/irq_regs.h> |
| 28 |
|
| 29 |
static int armpmu_count_irq_users(const struct cpumask *affinity, |
| 30 |
const int irq); |
| 31 |
|
| 32 |
struct pmu_irq_ops { |
| 33 |
void (*enable_pmuirq)(unsigned int irq); |
| 34 |
void (*disable_pmuirq)(unsigned int irq); |
| 35 |
void (*free_pmuirq)(unsigned int irq, int cpu, void __percpu *devid); |
| 36 |
}; |
| 37 |
|
| 38 |
static void armpmu_free_pmuirq(unsigned int irq, int cpu, void __percpu *devid) |
| 39 |
{ |
| 40 |
free_irq(irq, per_cpu_ptr(devid, cpu)); |
| 41 |
} |
| 42 |
|
| 43 |
static const struct pmu_irq_ops pmuirq_ops = { |
| 44 |
.enable_pmuirq = enable_irq, |
| 45 |
.disable_pmuirq = disable_irq_nosync, |
| 46 |
.free_pmuirq = armpmu_free_pmuirq |
| 47 |
}; |
| 48 |
|
| 49 |
static void armpmu_free_pmunmi(unsigned int irq, int cpu, void __percpu *devid) |
| 50 |
{ |
| 51 |
free_nmi(irq, per_cpu_ptr(devid, cpu)); |
| 52 |
} |
| 53 |
|
| 54 |
static const struct pmu_irq_ops pmunmi_ops = { |
| 55 |
.enable_pmuirq = enable_nmi, |
| 56 |
.disable_pmuirq = disable_nmi_nosync, |
| 57 |
.free_pmuirq = armpmu_free_pmunmi |
| 58 |
}; |
| 59 |
|
| 60 |
static void armpmu_enable_percpu_pmuirq(unsigned int irq) |
| 61 |
{ |
| 62 |
enable_percpu_irq(irq, IRQ_TYPE_NONE); |
| 63 |
} |
| 64 |
|
| 65 |
static void armpmu_free_percpu_pmuirq(unsigned int irq, int cpu, |
| 66 |
void __percpu *devid) |
| 67 |
{ |
| 68 |
struct arm_pmu *armpmu = *per_cpu_ptr((void * __percpu *)devid, cpu); |
| 69 |
|
| 70 |
if (armpmu_count_irq_users(&armpmu->supported_cpus, irq) == 1) |
| 71 |
free_percpu_irq(irq, devid); |
| 72 |
} |
| 73 |
|
| 74 |
static const struct pmu_irq_ops percpu_pmuirq_ops = { |
| 75 |
.enable_pmuirq = armpmu_enable_percpu_pmuirq, |
| 76 |
.disable_pmuirq = disable_percpu_irq, |
| 77 |
.free_pmuirq = armpmu_free_percpu_pmuirq |
| 78 |
}; |
| 79 |
|
| 80 |
static void armpmu_enable_percpu_pmunmi(unsigned int irq) |
| 81 |
{ |
| 82 |
if (!prepare_percpu_nmi(irq)) |
| 83 |
enable_percpu_nmi(irq, IRQ_TYPE_NONE); |
| 84 |
} |
| 85 |
|
| 86 |
static void armpmu_disable_percpu_pmunmi(unsigned int irq) |
| 87 |
{ |
| 88 |
disable_percpu_nmi(irq); |
| 89 |
teardown_percpu_nmi(irq); |
| 90 |
} |
| 91 |
|
| 92 |
static void armpmu_free_percpu_pmunmi(unsigned int irq, int cpu, |
| 93 |
void __percpu *devid) |
| 94 |
{ |
| 95 |
struct arm_pmu *armpmu = *per_cpu_ptr((void * __percpu *)devid, cpu); |
| 96 |
|
| 97 |
if (armpmu_count_irq_users(&armpmu->supported_cpus, irq) == 1) |
| 98 |
free_percpu_nmi(irq, devid); |
| 99 |
} |
| 100 |
|
| 101 |
static const struct pmu_irq_ops percpu_pmunmi_ops = { |
| 102 |
.enable_pmuirq = armpmu_enable_percpu_pmunmi, |
| 103 |
.disable_pmuirq = armpmu_disable_percpu_pmunmi, |
| 104 |
.free_pmuirq = armpmu_free_percpu_pmunmi |
| 105 |
}; |
| 106 |
|
| 107 |
static DEFINE_PER_CPU(int, cpu_irq); |
| 108 |
static DEFINE_PER_CPU(const struct pmu_irq_ops *, cpu_irq_ops); |
| 109 |
|
| 110 |
static bool has_nmi; |
| 111 |
|
| 112 |
static inline u64 arm_pmu_event_max_period(struct perf_event *event) |
| 113 |
{ |
| 114 |
if (event->hw.flags & ARMPMU_EVT_64BIT) |
| 115 |
return GENMASK_ULL(63, 0); |
| 116 |
else if (event->hw.flags & ARMPMU_EVT_63BIT) |
| 117 |
return GENMASK_ULL(62, 0); |
| 118 |
else if (event->hw.flags & ARMPMU_EVT_47BIT) |
| 119 |
return GENMASK_ULL(46, 0); |
| 120 |
else |
| 121 |
return GENMASK_ULL(31, 0); |
| 122 |
} |
| 123 |
|
| 124 |
static int |
| 125 |
armpmu_map_cache_event(const unsigned (*cache_map) |
| 126 |
[PERF_COUNT_HW_CACHE_MAX] |
| 127 |
[PERF_COUNT_HW_CACHE_OP_MAX] |
| 128 |
[PERF_COUNT_HW_CACHE_RESULT_MAX], |
| 129 |
u64 config) |
| 130 |
{ |
| 131 |
unsigned int cache_type, cache_op, cache_result, ret; |
| 132 |
|
| 133 |
cache_type = (config >> 0) & 0xff; |
| 134 |
if (cache_type >= PERF_COUNT_HW_CACHE_MAX) |
| 135 |
return -EINVAL; |
| 136 |
|
| 137 |
cache_op = (config >> 8) & 0xff; |
| 138 |
if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) |
| 139 |
return -EINVAL; |
| 140 |
|
| 141 |
cache_result = (config >> 16) & 0xff; |
| 142 |
if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) |
| 143 |
return -EINVAL; |
| 144 |
|
| 145 |
if (!cache_map) |
| 146 |
return -ENOENT; |
| 147 |
|
| 148 |
ret = (int)(*cache_map)[cache_type][cache_op][cache_result]; |
| 149 |
|
| 150 |
if (ret == CACHE_OP_UNSUPPORTED) |
| 151 |
return -ENOENT; |
| 152 |
|
| 153 |
return ret; |
| 154 |
} |
| 155 |
|
| 156 |
static int |
| 157 |
armpmu_map_hw_event(const unsigned (*event_map)[PERF_COUNT_HW_MAX], u64 config) |
| 158 |
{ |
| 159 |
int mapping; |
| 160 |
|
| 161 |
if (config >= PERF_COUNT_HW_MAX) |
| 162 |
return -EINVAL; |
| 163 |
|
| 164 |
if (!event_map) |
| 165 |
return -ENOENT; |
| 166 |
|
| 167 |
mapping = (*event_map)[config]; |
| 168 |
return mapping == HW_OP_UNSUPPORTED ? -ENOENT : mapping; |
| 169 |
} |
| 170 |
|
| 171 |
static int |
| 172 |
armpmu_map_raw_event(u32 raw_event_mask, u64 config) |
| 173 |
{ |
| 174 |
return (int)(config & raw_event_mask); |
| 175 |
} |
| 176 |
|
| 177 |
int |
| 178 |
armpmu_map_event(struct perf_event *event, |
| 179 |
const unsigned (*event_map)[PERF_COUNT_HW_MAX], |
| 180 |
const unsigned (*cache_map) |
| 181 |
[PERF_COUNT_HW_CACHE_MAX] |
| 182 |
[PERF_COUNT_HW_CACHE_OP_MAX] |
| 183 |
[PERF_COUNT_HW_CACHE_RESULT_MAX], |
| 184 |
u32 raw_event_mask) |
| 185 |
{ |
| 186 |
u64 config = event->attr.config; |
| 187 |
int type = event->attr.type; |
| 188 |
|
| 189 |
if (type == event->pmu->type) |
| 190 |
return armpmu_map_raw_event(raw_event_mask, config); |
| 191 |
|
| 192 |
switch (type) { |
| 193 |
case PERF_TYPE_HARDWARE: |
| 194 |
return armpmu_map_hw_event(event_map, config); |
| 195 |
case PERF_TYPE_HW_CACHE: |
| 196 |
return armpmu_map_cache_event(cache_map, config); |
| 197 |
case PERF_TYPE_RAW: |
| 198 |
return armpmu_map_raw_event(raw_event_mask, config); |
| 199 |
} |
| 200 |
|
| 201 |
return -ENOENT; |
| 202 |
} |
| 203 |
|
| 204 |
int armpmu_event_set_period(struct perf_event *event) |
| 205 |
{ |
| 206 |
struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
| 207 |
struct hw_perf_event *hwc = &event->hw; |
| 208 |
s64 left = local64_read(&hwc->period_left); |
| 209 |
s64 period = hwc->sample_period; |
| 210 |
u64 max_period; |
| 211 |
int ret = 0; |
| 212 |
|
| 213 |
max_period = arm_pmu_event_max_period(event); |
| 214 |
if (unlikely(left <= -period)) { |
| 215 |
left = period; |
| 216 |
local64_set(&hwc->period_left, left); |
| 217 |
hwc->last_period = period; |
| 218 |
ret = 1; |
| 219 |
} |
| 220 |
|
| 221 |
if (unlikely(left <= 0)) { |
| 222 |
left += period; |
| 223 |
local64_set(&hwc->period_left, left); |
| 224 |
hwc->last_period = period; |
| 225 |
ret = 1; |
| 226 |
} |
| 227 |
|
| 228 |
/* |
| 229 |
* Limit the maximum period to prevent the counter value |
| 230 |
* from overtaking the one we are about to program. In |
| 231 |
* effect we are reducing max_period to account for |
| 232 |
* interrupt latency (and we are being very conservative). |
| 233 |
*/ |
| 234 |
if (left > (max_period >> 1)) |
| 235 |
left = (max_period >> 1); |
| 236 |
|
| 237 |
local64_set(&hwc->prev_count, (u64)-left); |
| 238 |
|
| 239 |
armpmu->write_counter(event, (u64)(-left) & max_period); |
| 240 |
|
| 241 |
perf_event_update_userpage(event); |
| 242 |
|
| 243 |
return ret; |
| 244 |
} |
| 245 |
|
| 246 |
u64 armpmu_event_update(struct perf_event *event) |
| 247 |
{ |
| 248 |
struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
| 249 |
struct hw_perf_event *hwc = &event->hw; |
| 250 |
u64 delta, prev_raw_count, new_raw_count; |
| 251 |
u64 max_period = arm_pmu_event_max_period(event); |
| 252 |
|
| 253 |
again: |
| 254 |
prev_raw_count = local64_read(&hwc->prev_count); |
| 255 |
new_raw_count = armpmu->read_counter(event); |
| 256 |
|
| 257 |
if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, |
| 258 |
new_raw_count) != prev_raw_count) |
| 259 |
goto again; |
| 260 |
|
| 261 |
delta = (new_raw_count - prev_raw_count) & max_period; |
| 262 |
|
| 263 |
local64_add(delta, &event->count); |
| 264 |
local64_sub(delta, &hwc->period_left); |
| 265 |
|
| 266 |
return new_raw_count; |
| 267 |
} |
| 268 |
|
| 269 |
static void |
| 270 |
armpmu_read(struct perf_event *event) |
| 271 |
{ |
| 272 |
armpmu_event_update(event); |
| 273 |
} |
| 274 |
|
| 275 |
static void |
| 276 |
armpmu_stop(struct perf_event *event, int flags) |
| 277 |
{ |
| 278 |
struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
| 279 |
struct hw_perf_event *hwc = &event->hw; |
| 280 |
|
| 281 |
/* |
| 282 |
* ARM pmu always has to update the counter, so ignore |
| 283 |
* PERF_EF_UPDATE, see comments in armpmu_start(). |
| 284 |
*/ |
| 285 |
if (!(hwc->state & PERF_HES_STOPPED)) { |
| 286 |
armpmu->disable(event); |
| 287 |
armpmu_event_update(event); |
| 288 |
hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; |
| 289 |
} |
| 290 |
} |
| 291 |
|
| 292 |
static void armpmu_start(struct perf_event *event, int flags) |
| 293 |
{ |
| 294 |
struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
| 295 |
struct hw_perf_event *hwc = &event->hw; |
| 296 |
|
| 297 |
/* |
| 298 |
* ARM pmu always has to reprogram the period, so ignore |
| 299 |
* PERF_EF_RELOAD, see the comment below. |
| 300 |
*/ |
| 301 |
if (flags & PERF_EF_RELOAD) |
| 302 |
WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); |
| 303 |
|
| 304 |
hwc->state = 0; |
| 305 |
/* |
| 306 |
* Set the period again. Some counters can't be stopped, so when we |
| 307 |
* were stopped we simply disabled the IRQ source and the counter |
| 308 |
* may have been left counting. If we don't do this step then we may |
| 309 |
* get an interrupt too soon or *way* too late if the overflow has |
| 310 |
* happened since disabling. |
| 311 |
*/ |
| 312 |
armpmu_event_set_period(event); |
| 313 |
armpmu->enable(event); |
| 314 |
} |
| 315 |
|
| 316 |
static void |
| 317 |
armpmu_del(struct perf_event *event, int flags) |
| 318 |
{ |
| 319 |
struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
| 320 |
struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events); |
| 321 |
struct hw_perf_event *hwc = &event->hw; |
| 322 |
int idx = hwc->idx; |
| 323 |
|
| 324 |
armpmu_stop(event, PERF_EF_UPDATE); |
| 325 |
|
| 326 |
if (has_branch_stack(event)) { |
| 327 |
hw_events->branch_users--; |
| 328 |
perf_sched_cb_dec(event->pmu); |
| 329 |
} |
| 330 |
|
| 331 |
hw_events->events[idx] = NULL; |
| 332 |
armpmu->clear_event_idx(hw_events, event); |
| 333 |
perf_event_update_userpage(event); |
| 334 |
/* Clear the allocated counter */ |
| 335 |
hwc->idx = -1; |
| 336 |
} |
| 337 |
|
| 338 |
static int |
| 339 |
armpmu_add(struct perf_event *event, int flags) |
| 340 |
{ |
| 341 |
struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
| 342 |
struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events); |
| 343 |
struct hw_perf_event *hwc = &event->hw; |
| 344 |
int idx; |
| 345 |
|
| 346 |
/* An event following a process won't be stopped earlier */ |
| 347 |
if (!cpumask_test_cpu(smp_processor_id(), &armpmu->supported_cpus)) |
| 348 |
return -ENOENT; |
| 349 |
|
| 350 |
/* If we don't have a space for the counter then finish early. */ |
| 351 |
idx = armpmu->get_event_idx(hw_events, event); |
| 352 |
if (idx < 0) |
| 353 |
return idx; |
| 354 |
|
| 355 |
/* The newly-allocated counter should be empty */ |
| 356 |
WARN_ON_ONCE(hw_events->events[idx]); |
| 357 |
|
| 358 |
if (has_branch_stack(event)) { |
| 359 |
hw_events->branch_users++; |
| 360 |
perf_sched_cb_inc(event->pmu); |
| 361 |
} |
| 362 |
|
| 363 |
event->hw.idx = idx; |
| 364 |
hw_events->events[idx] = event; |
| 365 |
|
| 366 |
hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; |
| 367 |
if (flags & PERF_EF_START) |
| 368 |
armpmu_start(event, PERF_EF_RELOAD); |
| 369 |
|
| 370 |
/* Propagate our changes to the userspace mapping. */ |
| 371 |
perf_event_update_userpage(event); |
| 372 |
|
| 373 |
return 0; |
| 374 |
} |
| 375 |
|
| 376 |
static int |
| 377 |
validate_event(struct pmu *pmu, struct pmu_hw_events *hw_events, |
| 378 |
struct perf_event *event) |
| 379 |
{ |
| 380 |
struct arm_pmu *armpmu; |
| 381 |
|
| 382 |
if (is_software_event(event)) |
| 383 |
return 1; |
| 384 |
|
| 385 |
/* |
| 386 |
* Reject groups spanning multiple HW PMUs (e.g. CPU + CCI). The |
| 387 |
* core perf code won't check that the pmu->ctx == leader->ctx |
| 388 |
* until after pmu->event_init(event). |
| 389 |
*/ |
| 390 |
if (event->pmu != pmu) |
| 391 |
return 0; |
| 392 |
|
| 393 |
if (event->state < PERF_EVENT_STATE_OFF) |
| 394 |
return 1; |
| 395 |
|
| 396 |
if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec) |
| 397 |
return 1; |
| 398 |
|
| 399 |
armpmu = to_arm_pmu(event->pmu); |
| 400 |
return armpmu->get_event_idx(hw_events, event) >= 0; |
| 401 |
} |
| 402 |
|
| 403 |
static int |
| 404 |
validate_group(struct perf_event *event) |
| 405 |
{ |
| 406 |
struct perf_event *sibling, *leader = event->group_leader; |
| 407 |
struct pmu_hw_events fake_pmu; |
| 408 |
|
| 409 |
/* |
| 410 |
* Initialise the fake PMU. We only need to populate the |
| 411 |
* used_mask for the purposes of validation. |
| 412 |
*/ |
| 413 |
memset(&fake_pmu.used_mask, 0, sizeof(fake_pmu.used_mask)); |
| 414 |
|
| 415 |
if (!validate_event(event->pmu, &fake_pmu, leader)) |
| 416 |
return -EINVAL; |
| 417 |
|
| 418 |
if (event == leader) |
| 419 |
return 0; |
| 420 |
|
| 421 |
for_each_sibling_event(sibling, leader) { |
| 422 |
if (!validate_event(event->pmu, &fake_pmu, sibling)) |
| 423 |
return -EINVAL; |
| 424 |
} |
| 425 |
|
| 426 |
if (!validate_event(event->pmu, &fake_pmu, event)) |
| 427 |
return -EINVAL; |
| 428 |
|
| 429 |
return 0; |
| 430 |
} |
| 431 |
|
| 432 |
static irqreturn_t armpmu_dispatch_irq(int irq, void *dev) |
| 433 |
{ |
| 434 |
struct arm_pmu *armpmu; |
| 435 |
int ret; |
| 436 |
u64 start_clock, finish_clock; |
| 437 |
|
| 438 |
/* |
| 439 |
* we request the IRQ with a (possibly percpu) struct arm_pmu**, but |
| 440 |
* the handlers expect a struct arm_pmu*. The percpu_irq framework will |
| 441 |
* do any necessary shifting, we just need to perform the first |
| 442 |
* dereference. |
| 443 |
*/ |
| 444 |
armpmu = *(void **)dev; |
| 445 |
if (WARN_ON_ONCE(!armpmu)) |
| 446 |
return IRQ_NONE; |
| 447 |
|
| 448 |
start_clock = sched_clock(); |
| 449 |
ret = armpmu->handle_irq(armpmu); |
| 450 |
finish_clock = sched_clock(); |
| 451 |
|
| 452 |
perf_sample_event_took(finish_clock - start_clock); |
| 453 |
return ret; |
| 454 |
} |
| 455 |
|
| 456 |
static int |
| 457 |
__hw_perf_event_init(struct perf_event *event) |
| 458 |
{ |
| 459 |
struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
| 460 |
struct hw_perf_event *hwc = &event->hw; |
| 461 |
int mapping, ret; |
| 462 |
|
| 463 |
hwc->flags = 0; |
| 464 |
mapping = armpmu->map_event(event); |
| 465 |
|
| 466 |
if (mapping < 0) { |
| 467 |
pr_debug("event %x:%llx not supported\n", event->attr.type, |
| 468 |
event->attr.config); |
| 469 |
return mapping; |
| 470 |
} |
| 471 |
|
| 472 |
/* |
| 473 |
* We don't assign an index until we actually place the event onto |
| 474 |
* hardware. Use -1 to signify that we haven't decided where to put it |
| 475 |
* yet. For SMP systems, each core has it's own PMU so we can't do any |
| 476 |
* clever allocation or constraints checking at this point. |
| 477 |
*/ |
| 478 |
hwc->idx = -1; |
| 479 |
hwc->config_base = 0; |
| 480 |
hwc->config = 0; |
| 481 |
hwc->event_base = 0; |
| 482 |
|
| 483 |
/* |
| 484 |
* Check whether we need to exclude the counter from certain modes. |
| 485 |
*/ |
| 486 |
if (armpmu->set_event_filter) { |
| 487 |
ret = armpmu->set_event_filter(hwc, &event->attr); |
| 488 |
if (ret) |
| 489 |
return ret; |
| 490 |
} |
| 491 |
|
| 492 |
/* |
| 493 |
* Store the event encoding into the config_base field. |
| 494 |
*/ |
| 495 |
hwc->config_base |= (unsigned long)mapping; |
| 496 |
|
| 497 |
if (!is_sampling_event(event)) { |
| 498 |
/* |
| 499 |
* For non-sampling runs, limit the sample_period to half |
| 500 |
* of the counter width. That way, the new counter value |
| 501 |
* is far less likely to overtake the previous one unless |
| 502 |
* you have some serious IRQ latency issues. |
| 503 |
*/ |
| 504 |
hwc->sample_period = arm_pmu_event_max_period(event) >> 1; |
| 505 |
hwc->last_period = hwc->sample_period; |
| 506 |
local64_set(&hwc->period_left, hwc->sample_period); |
| 507 |
} |
| 508 |
|
| 509 |
return validate_group(event); |
| 510 |
} |
| 511 |
|
| 512 |
static int armpmu_event_init(struct perf_event *event) |
| 513 |
{ |
| 514 |
struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
| 515 |
|
| 516 |
/* |
| 517 |
* Reject CPU-affine events for CPUs that are of a different class to |
| 518 |
* that which this PMU handles. Process-following events (where |
| 519 |
* event->cpu == -1) can be migrated between CPUs, and thus we have to |
| 520 |
* reject them later (in armpmu_add) if they're scheduled on a |
| 521 |
* different class of CPU. |
| 522 |
*/ |
| 523 |
if (event->cpu != -1 && |
| 524 |
!cpumask_test_cpu(event->cpu, &armpmu->supported_cpus)) |
| 525 |
return -ENOENT; |
| 526 |
|
| 527 |
if (has_branch_stack(event) && !armpmu->reg_brbidr) |
| 528 |
return -EOPNOTSUPP; |
| 529 |
|
| 530 |
return __hw_perf_event_init(event); |
| 531 |
} |
| 532 |
|
| 533 |
static void armpmu_enable(struct pmu *pmu) |
| 534 |
{ |
| 535 |
struct arm_pmu *armpmu = to_arm_pmu(pmu); |
| 536 |
struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events); |
| 537 |
bool enabled = !bitmap_empty(hw_events->used_mask, ARMPMU_MAX_HWEVENTS); |
| 538 |
|
| 539 |
/* For task-bound events we may be called on other CPUs */ |
| 540 |
if (!cpumask_test_cpu(smp_processor_id(), &armpmu->supported_cpus)) |
| 541 |
return; |
| 542 |
|
| 543 |
if (enabled) |
| 544 |
armpmu->start(armpmu); |
| 545 |
} |
| 546 |
|
| 547 |
static void armpmu_disable(struct pmu *pmu) |
| 548 |
{ |
| 549 |
struct arm_pmu *armpmu = to_arm_pmu(pmu); |
| 550 |
|
| 551 |
/* For task-bound events we may be called on other CPUs */ |
| 552 |
if (!cpumask_test_cpu(smp_processor_id(), &armpmu->supported_cpus)) |
| 553 |
return; |
| 554 |
|
| 555 |
armpmu->stop(armpmu); |
| 556 |
} |
| 557 |
|
| 558 |
/* |
| 559 |
* In heterogeneous systems, events are specific to a particular |
| 560 |
* microarchitecture, and aren't suitable for another. Thus, only match CPUs of |
| 561 |
* the same microarchitecture. |
| 562 |
*/ |
| 563 |
static bool armpmu_filter(struct pmu *pmu, int cpu) |
| 564 |
{ |
| 565 |
struct arm_pmu *armpmu = to_arm_pmu(pmu); |
| 566 |
return !cpumask_test_cpu(cpu, &armpmu->supported_cpus); |
| 567 |
} |
| 568 |
|
| 569 |
static ssize_t cpus_show(struct device *dev, |
| 570 |
struct device_attribute *attr, char *buf) |
| 571 |
{ |
| 572 |
struct arm_pmu *armpmu = to_arm_pmu(dev_get_drvdata(dev)); |
| 573 |
return cpumap_print_to_pagebuf(true, buf, &armpmu->supported_cpus); |
| 574 |
} |
| 575 |
|
| 576 |
static DEVICE_ATTR_RO(cpus); |
| 577 |
|
| 578 |
static struct attribute *armpmu_common_attrs[] = { |
| 579 |
&dev_attr_cpus.attr, |
| 580 |
NULL, |
| 581 |
}; |
| 582 |
|
| 583 |
static const struct attribute_group armpmu_common_attr_group = { |
| 584 |
.attrs = armpmu_common_attrs, |
| 585 |
}; |
| 586 |
|
| 587 |
static int armpmu_count_irq_users(const struct cpumask *affinity, const int irq) |
| 588 |
{ |
| 589 |
int cpu, count = 0; |
| 590 |
|
| 591 |
for_each_cpu(cpu, affinity) { |
| 592 |
if (per_cpu(cpu_irq, cpu) == irq) |
| 593 |
count++; |
| 594 |
} |
| 595 |
|
| 596 |
return count; |
| 597 |
} |
| 598 |
|
| 599 |
static const struct pmu_irq_ops * |
| 600 |
armpmu_find_irq_ops(const struct cpumask *affinity, int irq) |
| 601 |
{ |
| 602 |
const struct pmu_irq_ops *ops = NULL; |
| 603 |
int cpu; |
| 604 |
|
| 605 |
for_each_cpu(cpu, affinity) { |
| 606 |
if (per_cpu(cpu_irq, cpu) != irq) |
| 607 |
continue; |
| 608 |
|
| 609 |
ops = per_cpu(cpu_irq_ops, cpu); |
| 610 |
if (ops) |
| 611 |
break; |
| 612 |
} |
| 613 |
|
| 614 |
return ops; |
| 615 |
} |
| 616 |
|
| 617 |
void armpmu_free_irq(struct arm_pmu * __percpu *armpmu, int irq, int cpu) |
| 618 |
{ |
| 619 |
if (per_cpu(cpu_irq, cpu) == 0) |
| 620 |
return; |
| 621 |
if (WARN_ON(irq != per_cpu(cpu_irq, cpu))) |
| 622 |
return; |
| 623 |
|
| 624 |
per_cpu(cpu_irq_ops, cpu)->free_pmuirq(irq, cpu, armpmu); |
| 625 |
|
| 626 |
per_cpu(cpu_irq, cpu) = 0; |
| 627 |
per_cpu(cpu_irq_ops, cpu) = NULL; |
| 628 |
} |
| 629 |
|
| 630 |
int armpmu_request_irq(struct arm_pmu * __percpu *pcpu_armpmu, int irq, int cpu) |
| 631 |
{ |
| 632 |
int err = 0; |
| 633 |
struct arm_pmu **armpmu = per_cpu_ptr(pcpu_armpmu, cpu); |
| 634 |
const struct cpumask *affinity = *armpmu ? &(*armpmu)->supported_cpus : |
| 635 |
cpu_possible_mask; /* ACPI */ |
| 636 |
const irq_handler_t handler = armpmu_dispatch_irq; |
| 637 |
const struct pmu_irq_ops *irq_ops; |
| 638 |
|
| 639 |
if (!irq) |
| 640 |
return 0; |
| 641 |
|
| 642 |
if (!irq_is_percpu_devid(irq)) { |
| 643 |
unsigned long irq_flags; |
| 644 |
|
| 645 |
err = irq_force_affinity(irq, cpumask_of(cpu)); |
| 646 |
|
| 647 |
if (err && num_possible_cpus() > 1) { |
| 648 |
pr_warn("unable to set irq affinity (irq=%d, cpu=%u)\n", |
| 649 |
irq, cpu); |
| 650 |
goto err_out; |
| 651 |
} |
| 652 |
|
| 653 |
irq_flags = IRQF_PERCPU | |
| 654 |
IRQF_NOBALANCING | IRQF_NO_AUTOEN | |
| 655 |
IRQF_NO_THREAD; |
| 656 |
|
| 657 |
err = request_nmi(irq, handler, irq_flags, "arm-pmu", armpmu); |
| 658 |
|
| 659 |
/* If cannot get an NMI, get a normal interrupt */ |
| 660 |
if (err) { |
| 661 |
err = request_irq(irq, handler, irq_flags, "arm-pmu", |
| 662 |
armpmu); |
| 663 |
irq_ops = &pmuirq_ops; |
| 664 |
} else { |
| 665 |
has_nmi = true; |
| 666 |
irq_ops = &pmunmi_ops; |
| 667 |
} |
| 668 |
} else if (armpmu_count_irq_users(affinity, irq) == 0) { |
| 669 |
err = request_percpu_nmi(irq, handler, "arm-pmu", affinity, pcpu_armpmu); |
| 670 |
|
| 671 |
/* If cannot get an NMI, get a normal interrupt */ |
| 672 |
if (err) { |
| 673 |
err = request_percpu_irq_affinity(irq, handler, "arm-pmu", |
| 674 |
affinity, pcpu_armpmu); |
| 675 |
irq_ops = &percpu_pmuirq_ops; |
| 676 |
} else { |
| 677 |
has_nmi = true; |
| 678 |
irq_ops = &percpu_pmunmi_ops; |
| 679 |
} |
| 680 |
} else { |
| 681 |
/* Per cpudevid irq was already requested by another CPU */ |
| 682 |
irq_ops = armpmu_find_irq_ops(affinity, irq); |
| 683 |
|
| 684 |
if (WARN_ON(!irq_ops)) |
| 685 |
err = -EINVAL; |
| 686 |
} |
| 687 |
|
| 688 |
if (err) |
| 689 |
goto err_out; |
| 690 |
|
| 691 |
per_cpu(cpu_irq, cpu) = irq; |
| 692 |
per_cpu(cpu_irq_ops, cpu) = irq_ops; |
| 693 |
return 0; |
| 694 |
|
| 695 |
err_out: |
| 696 |
pr_err("unable to request IRQ%d for ARM PMU counters\n", irq); |
| 697 |
return err; |
| 698 |
} |
| 699 |
|
| 700 |
static int armpmu_get_cpu_irq(struct arm_pmu *pmu, int cpu) |
| 701 |
{ |
| 702 |
struct pmu_hw_events __percpu *hw_events = pmu->hw_events; |
| 703 |
return per_cpu(hw_events->irq, cpu); |
| 704 |
} |
| 705 |
|
| 706 |
bool arm_pmu_irq_is_nmi(void) |
| 707 |
{ |
| 708 |
return has_nmi; |
| 709 |
} |
| 710 |
|
| 711 |
/* |
| 712 |
* PMU hardware loses all context when a CPU goes offline. |
| 713 |
* When a CPU is hotplugged back in, since some hardware registers are |
| 714 |
* UNKNOWN at reset, the PMU must be explicitly reset to avoid reading |
| 715 |
* junk values out of them. |
| 716 |
*/ |
| 717 |
static int arm_perf_starting_cpu(unsigned int cpu, struct hlist_node *node) |
| 718 |
{ |
| 719 |
struct arm_pmu *pmu = hlist_entry_safe(node, struct arm_pmu, node); |
| 720 |
int irq; |
| 721 |
|
| 722 |
if (!cpumask_test_cpu(cpu, &pmu->supported_cpus)) |
| 723 |
return 0; |
| 724 |
if (pmu->reset) |
| 725 |
pmu->reset(pmu); |
| 726 |
|
| 727 |
irq = armpmu_get_cpu_irq(pmu, cpu); |
| 728 |
if (irq) |
| 729 |
per_cpu(cpu_irq_ops, cpu)->enable_pmuirq(irq); |
| 730 |
|
| 731 |
return 0; |
| 732 |
} |
| 733 |
|
| 734 |
static int arm_perf_teardown_cpu(unsigned int cpu, struct hlist_node *node) |
| 735 |
{ |
| 736 |
struct arm_pmu *pmu = hlist_entry_safe(node, struct arm_pmu, node); |
| 737 |
int irq; |
| 738 |
|
| 739 |
if (!cpumask_test_cpu(cpu, &pmu->supported_cpus)) |
| 740 |
return 0; |
| 741 |
|
| 742 |
irq = armpmu_get_cpu_irq(pmu, cpu); |
| 743 |
if (irq) |
| 744 |
per_cpu(cpu_irq_ops, cpu)->disable_pmuirq(irq); |
| 745 |
|
| 746 |
return 0; |
| 747 |
} |
| 748 |
|
| 749 |
#ifdef CONFIG_CPU_PM |
| 750 |
static void cpu_pm_pmu_setup(struct arm_pmu *armpmu, unsigned long cmd) |
| 751 |
{ |
| 752 |
struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events); |
| 753 |
struct perf_event *event; |
| 754 |
int idx; |
| 755 |
|
| 756 |
for_each_set_bit(idx, armpmu->cntr_mask, ARMPMU_MAX_HWEVENTS) { |
| 757 |
event = hw_events->events[idx]; |
| 758 |
if (!event) |
| 759 |
continue; |
| 760 |
|
| 761 |
switch (cmd) { |
| 762 |
case CPU_PM_ENTER: |
| 763 |
/* |
| 764 |
* Stop and update the counter |
| 765 |
*/ |
| 766 |
armpmu_stop(event, PERF_EF_UPDATE); |
| 767 |
break; |
| 768 |
case CPU_PM_EXIT: |
| 769 |
case CPU_PM_ENTER_FAILED: |
| 770 |
/* |
| 771 |
* Restore and enable the counter. |
| 772 |
*/ |
| 773 |
armpmu_start(event, PERF_EF_RELOAD); |
| 774 |
break; |
| 775 |
default: |
| 776 |
break; |
| 777 |
} |
| 778 |
} |
| 779 |
} |
| 780 |
|
| 781 |
static int cpu_pm_pmu_notify(struct notifier_block *b, unsigned long cmd, |
| 782 |
void *v) |
| 783 |
{ |
| 784 |
struct arm_pmu *armpmu = container_of(b, struct arm_pmu, cpu_pm_nb); |
| 785 |
struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events); |
| 786 |
bool enabled = !bitmap_empty(hw_events->used_mask, ARMPMU_MAX_HWEVENTS); |
| 787 |
|
| 788 |
if (!cpumask_test_cpu(smp_processor_id(), &armpmu->supported_cpus)) |
| 789 |
return NOTIFY_DONE; |
| 790 |
|
| 791 |
/* |
| 792 |
* Always reset the PMU registers on power-up even if |
| 793 |
* there are no events running. |
| 794 |
*/ |
| 795 |
if (cmd == CPU_PM_EXIT && armpmu->reset) |
| 796 |
armpmu->reset(armpmu); |
| 797 |
|
| 798 |
if (!enabled) |
| 799 |
return NOTIFY_OK; |
| 800 |
|
| 801 |
switch (cmd) { |
| 802 |
case CPU_PM_ENTER: |
| 803 |
armpmu->stop(armpmu); |
| 804 |
cpu_pm_pmu_setup(armpmu, cmd); |
| 805 |
break; |
| 806 |
case CPU_PM_EXIT: |
| 807 |
case CPU_PM_ENTER_FAILED: |
| 808 |
cpu_pm_pmu_setup(armpmu, cmd); |
| 809 |
armpmu->start(armpmu); |
| 810 |
break; |
| 811 |
default: |
| 812 |
return NOTIFY_DONE; |
| 813 |
} |
| 814 |
|
| 815 |
return NOTIFY_OK; |
| 816 |
} |
| 817 |
|
| 818 |
static int cpu_pm_pmu_register(struct arm_pmu *cpu_pmu) |
| 819 |
{ |
| 820 |
cpu_pmu->cpu_pm_nb.notifier_call = cpu_pm_pmu_notify; |
| 821 |
return cpu_pm_register_notifier(&cpu_pmu->cpu_pm_nb); |
| 822 |
} |
| 823 |
|
| 824 |
static void cpu_pm_pmu_unregister(struct arm_pmu *cpu_pmu) |
| 825 |
{ |
| 826 |
cpu_pm_unregister_notifier(&cpu_pmu->cpu_pm_nb); |
| 827 |
} |
| 828 |
#else |
| 829 |
static inline int cpu_pm_pmu_register(struct arm_pmu *cpu_pmu) { return 0; } |
| 830 |
static inline void cpu_pm_pmu_unregister(struct arm_pmu *cpu_pmu) { } |
| 831 |
#endif |
| 832 |
|
| 833 |
static int cpu_pmu_init(struct arm_pmu *cpu_pmu) |
| 834 |
{ |
| 835 |
int err; |
| 836 |
|
| 837 |
err = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_STARTING, |
| 838 |
&cpu_pmu->node); |
| 839 |
if (err) |
| 840 |
goto out; |
| 841 |
|
| 842 |
err = cpu_pm_pmu_register(cpu_pmu); |
| 843 |
if (err) |
| 844 |
goto out_unregister; |
| 845 |
|
| 846 |
return 0; |
| 847 |
|
| 848 |
out_unregister: |
| 849 |
cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_STARTING, |
| 850 |
&cpu_pmu->node); |
| 851 |
out: |
| 852 |
return err; |
| 853 |
} |
| 854 |
|
| 855 |
static void cpu_pmu_destroy(struct arm_pmu *cpu_pmu) |
| 856 |
{ |
| 857 |
cpu_pm_pmu_unregister(cpu_pmu); |
| 858 |
cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_STARTING, |
| 859 |
&cpu_pmu->node); |
| 860 |
} |
| 861 |
|
| 862 |
struct arm_pmu *armpmu_alloc(void) |
| 863 |
{ |
| 864 |
struct arm_pmu *pmu; |
| 865 |
int cpu; |
| 866 |
|
| 867 |
pmu = kzalloc_obj(*pmu); |
| 868 |
if (!pmu) |
| 869 |
goto out; |
| 870 |
|
| 871 |
pmu->hw_events = alloc_percpu_gfp(struct pmu_hw_events, GFP_KERNEL); |
| 872 |
if (!pmu->hw_events) { |
| 873 |
pr_info("failed to allocate per-cpu PMU data.\n"); |
| 874 |
goto out_free_pmu; |
| 875 |
} |
| 876 |
|
| 877 |
pmu->pmu = (struct pmu) { |
| 878 |
.pmu_enable = armpmu_enable, |
| 879 |
.pmu_disable = armpmu_disable, |
| 880 |
.event_init = armpmu_event_init, |
| 881 |
.add = armpmu_add, |
| 882 |
.del = armpmu_del, |
| 883 |
.start = armpmu_start, |
| 884 |
.stop = armpmu_stop, |
| 885 |
.read = armpmu_read, |
| 886 |
.filter = armpmu_filter, |
| 887 |
.attr_groups = pmu->attr_groups, |
| 888 |
/* |
| 889 |
* This is a CPU PMU potentially in a heterogeneous |
| 890 |
* configuration (e.g. big.LITTLE) so |
| 891 |
* PERF_PMU_CAP_EXTENDED_HW_TYPE is required to open |
| 892 |
* PERF_TYPE_HARDWARE and PERF_TYPE_HW_CACHE events on a |
| 893 |
* specific PMU. |
| 894 |
*/ |
| 895 |
.capabilities = PERF_PMU_CAP_EXTENDED_REGS | |
| 896 |
PERF_PMU_CAP_EXTENDED_HW_TYPE, |
| 897 |
}; |
| 898 |
|
| 899 |
pmu->attr_groups[ARMPMU_ATTR_GROUP_COMMON] = |
| 900 |
&armpmu_common_attr_group; |
| 901 |
|
| 902 |
for_each_possible_cpu(cpu) { |
| 903 |
struct pmu_hw_events *events; |
| 904 |
|
| 905 |
events = per_cpu_ptr(pmu->hw_events, cpu); |
| 906 |
events->percpu_pmu = pmu; |
| 907 |
} |
| 908 |
|
| 909 |
return pmu; |
| 910 |
|
| 911 |
out_free_pmu: |
| 912 |
kfree(pmu); |
| 913 |
out: |
| 914 |
return NULL; |
| 915 |
} |
| 916 |
|
| 917 |
void armpmu_free(struct arm_pmu *pmu) |
| 918 |
{ |
| 919 |
free_percpu(pmu->hw_events); |
| 920 |
kfree(pmu); |
| 921 |
} |
| 922 |
|
| 923 |
int armpmu_register(struct arm_pmu *pmu) |
| 924 |
{ |
| 925 |
int ret; |
| 926 |
|
| 927 |
ret = cpu_pmu_init(pmu); |
| 928 |
if (ret) |
| 929 |
return ret; |
| 930 |
|
| 931 |
/* |
| 932 |
* By this stage we know our supported CPUs on either DT/ACPI platforms, |
| 933 |
* detect the SMT implementation. |
| 934 |
*/ |
| 935 |
pmu->has_smt = topology_core_has_smt(cpumask_first(&pmu->supported_cpus)); |
| 936 |
|
| 937 |
if (!pmu->set_event_filter) |
| 938 |
pmu->pmu.capabilities |= PERF_PMU_CAP_NO_EXCLUDE; |
| 939 |
|
| 940 |
ret = perf_pmu_register(&pmu->pmu, pmu->name, -1); |
| 941 |
if (ret) |
| 942 |
goto out_destroy; |
| 943 |
|
| 944 |
pr_info("enabled with %s PMU driver, %d (%*pb) counters available%s\n", |
| 945 |
pmu->name, bitmap_weight(pmu->cntr_mask, ARMPMU_MAX_HWEVENTS), |
| 946 |
ARMPMU_MAX_HWEVENTS, &pmu->cntr_mask, |
| 947 |
has_nmi ? ", using NMIs" : ""); |
| 948 |
|
| 949 |
kvm_host_pmu_init(pmu); |
| 950 |
|
| 951 |
return 0; |
| 952 |
|
| 953 |
out_destroy: |
| 954 |
cpu_pmu_destroy(pmu); |
| 955 |
return ret; |
| 956 |
} |
| 957 |
|
| 958 |
static int arm_pmu_hp_init(void) |
| 959 |
{ |
| 960 |
int ret; |
| 961 |
|
| 962 |
ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_STARTING, |
| 963 |
"perf/arm/pmu:starting", |
| 964 |
arm_perf_starting_cpu, |
| 965 |
arm_perf_teardown_cpu); |
| 966 |
if (ret) |
| 967 |
pr_err("CPU hotplug notifier for ARM PMU could not be registered: %d\n", |
| 968 |
ret); |
| 969 |
return ret; |
| 970 |
} |
| 971 |
subsys_initcall(arm_pmu_hp_init); |