| 1 |
// SPDX-License-Identifier: GPL-2.0-only |
| 2 |
/* |
| 3 |
* ARM DynamIQ Shared Unit (DSU) PMU driver |
| 4 |
* |
| 5 |
* Copyright (C) ARM Limited, 2017. |
| 6 |
* |
| 7 |
* Based on ARM CCI-PMU, ARMv8 PMU-v3 drivers. |
| 8 |
*/ |
| 9 |
|
| 10 |
#define PMUNAME "arm_dsu" |
| 11 |
#define DRVNAME PMUNAME "_pmu" |
| 12 |
#define pr_fmt(fmt) DRVNAME ": " fmt |
| 13 |
|
| 14 |
#include <linux/acpi.h> |
| 15 |
#include <linux/bitmap.h> |
| 16 |
#include <linux/bitops.h> |
| 17 |
#include <linux/bug.h> |
| 18 |
#include <linux/cpumask.h> |
| 19 |
#include <linux/device.h> |
| 20 |
#include <linux/interrupt.h> |
| 21 |
#include <linux/kernel.h> |
| 22 |
#include <linux/module.h> |
| 23 |
#include <linux/of.h> |
| 24 |
#include <linux/perf_event.h> |
| 25 |
#include <linux/platform_device.h> |
| 26 |
#include <linux/spinlock.h> |
| 27 |
#include <linux/smp.h> |
| 28 |
#include <linux/sysfs.h> |
| 29 |
#include <linux/types.h> |
| 30 |
|
| 31 |
#include <asm/arm_dsu_pmu.h> |
| 32 |
#include <asm/local64.h> |
| 33 |
|
| 34 |
/* PMU event codes */ |
| 35 |
#define DSU_PMU_EVT_CYCLES 0x11 |
| 36 |
#define DSU_PMU_EVT_CHAIN 0x1e |
| 37 |
|
| 38 |
#define DSU_PMU_MAX_COMMON_EVENTS 0x40 |
| 39 |
|
| 40 |
#define DSU_PMU_MAX_HW_CNTRS 32 |
| 41 |
#define DSU_PMU_HW_COUNTER_MASK (DSU_PMU_MAX_HW_CNTRS - 1) |
| 42 |
|
| 43 |
#define CLUSTERPMCR_E BIT(0) |
| 44 |
#define CLUSTERPMCR_P BIT(1) |
| 45 |
#define CLUSTERPMCR_C BIT(2) |
| 46 |
#define CLUSTERPMCR_N_SHIFT 11 |
| 47 |
#define CLUSTERPMCR_N_MASK 0x1f |
| 48 |
#define CLUSTERPMCR_IDCODE_SHIFT 16 |
| 49 |
#define CLUSTERPMCR_IDCODE_MASK 0xff |
| 50 |
#define CLUSTERPMCR_IMP_SHIFT 24 |
| 51 |
#define CLUSTERPMCR_IMP_MASK 0xff |
| 52 |
#define CLUSTERPMCR_RES_MASK 0x7e8 |
| 53 |
#define CLUSTERPMCR_RES_VAL 0x40 |
| 54 |
|
| 55 |
#define DSU_ACTIVE_CPU_MASK 0x0 |
| 56 |
#define DSU_ASSOCIATED_CPU_MASK 0x1 |
| 57 |
|
| 58 |
/* |
| 59 |
* We use the index of the counters as they appear in the counter |
| 60 |
* bit maps in the PMU registers (e.g CLUSTERPMSELR). |
| 61 |
* i.e, |
| 62 |
* counter 0 - Bit 0 |
| 63 |
* counter 1 - Bit 1 |
| 64 |
* ... |
| 65 |
* Cycle counter - Bit 31 |
| 66 |
*/ |
| 67 |
#define DSU_PMU_IDX_CYCLE_COUNTER 31 |
| 68 |
|
| 69 |
#define DSU_EXT_ATTR(_name, _func, _config) \ |
| 70 |
(&((struct dev_ext_attribute[]) { \ |
| 71 |
{ \ |
| 72 |
.attr = __ATTR(_name, 0444, _func, NULL), \ |
| 73 |
.var = (void *)_config \ |
| 74 |
} \ |
| 75 |
})[0].attr.attr) |
| 76 |
|
| 77 |
#define DSU_EVENT_ATTR(_name, _config) \ |
| 78 |
DSU_EXT_ATTR(_name, dsu_pmu_sysfs_event_show, (unsigned long)_config) |
| 79 |
|
| 80 |
#define DSU_FORMAT_ATTR(_name, _config) \ |
| 81 |
DSU_EXT_ATTR(_name, device_show_string, _config) |
| 82 |
|
| 83 |
#define DSU_CPUMASK_ATTR(_name, _config) \ |
| 84 |
DSU_EXT_ATTR(_name, dsu_pmu_cpumask_show, (unsigned long)_config) |
| 85 |
|
| 86 |
struct dsu_hw_events { |
| 87 |
DECLARE_BITMAP(used_mask, DSU_PMU_MAX_HW_CNTRS); |
| 88 |
struct perf_event *events[DSU_PMU_MAX_HW_CNTRS]; |
| 89 |
}; |
| 90 |
|
| 91 |
/* |
| 92 |
* struct dsu_pmu - DSU PMU descriptor |
| 93 |
* |
| 94 |
* @pmu_lock : Protects accesses to DSU PMU register from normal vs |
| 95 |
* interrupt handler contexts. |
| 96 |
* @hw_events : Holds the event counter state. |
| 97 |
* @associated_cpus : CPUs attached to the DSU. |
| 98 |
* @active_cpu : CPU to which the PMU is bound for accesses. |
| 99 |
* @cpuhp_node : Node for CPU hotplug notifier link. |
| 100 |
* @num_counters : Number of event counters implemented by the PMU, |
| 101 |
* excluding the cycle counter. |
| 102 |
* @irq : Interrupt line for counter overflow. |
| 103 |
* @has_32b_pmevcntr : Are the non-cycle counters only 32-bit? |
| 104 |
* @has_pmccntr : Do we even have a dedicated cycle counter? |
| 105 |
* @cpmceid_bitmap : Bitmap for the availability of architected common |
| 106 |
* events (event_code < 0x40). |
| 107 |
*/ |
| 108 |
struct dsu_pmu { |
| 109 |
struct pmu pmu; |
| 110 |
struct device *dev; |
| 111 |
raw_spinlock_t pmu_lock; |
| 112 |
struct dsu_hw_events hw_events; |
| 113 |
cpumask_t associated_cpus; |
| 114 |
cpumask_t active_cpu; |
| 115 |
struct hlist_node cpuhp_node; |
| 116 |
s8 num_counters; |
| 117 |
int irq; |
| 118 |
bool has_32b_pmevcntr; |
| 119 |
bool has_pmccntr; |
| 120 |
DECLARE_BITMAP(cpmceid_bitmap, DSU_PMU_MAX_COMMON_EVENTS); |
| 121 |
}; |
| 122 |
|
| 123 |
static unsigned long dsu_pmu_cpuhp_state; |
| 124 |
|
| 125 |
static inline struct dsu_pmu *to_dsu_pmu(struct pmu *pmu) |
| 126 |
{ |
| 127 |
return container_of(pmu, struct dsu_pmu, pmu); |
| 128 |
} |
| 129 |
|
| 130 |
static ssize_t dsu_pmu_sysfs_event_show(struct device *dev, |
| 131 |
struct device_attribute *attr, |
| 132 |
char *buf) |
| 133 |
{ |
| 134 |
struct dev_ext_attribute *eattr = container_of(attr, |
| 135 |
struct dev_ext_attribute, attr); |
| 136 |
return sysfs_emit(buf, "event=0x%lx\n", (unsigned long)eattr->var); |
| 137 |
} |
| 138 |
|
| 139 |
static ssize_t dsu_pmu_cpumask_show(struct device *dev, |
| 140 |
struct device_attribute *attr, |
| 141 |
char *buf) |
| 142 |
{ |
| 143 |
struct pmu *pmu = dev_get_drvdata(dev); |
| 144 |
struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu); |
| 145 |
struct dev_ext_attribute *eattr = container_of(attr, |
| 146 |
struct dev_ext_attribute, attr); |
| 147 |
unsigned long mask_id = (unsigned long)eattr->var; |
| 148 |
const cpumask_t *cpumask; |
| 149 |
|
| 150 |
switch (mask_id) { |
| 151 |
case DSU_ACTIVE_CPU_MASK: |
| 152 |
cpumask = &dsu_pmu->active_cpu; |
| 153 |
break; |
| 154 |
case DSU_ASSOCIATED_CPU_MASK: |
| 155 |
cpumask = &dsu_pmu->associated_cpus; |
| 156 |
break; |
| 157 |
default: |
| 158 |
return 0; |
| 159 |
} |
| 160 |
return cpumap_print_to_pagebuf(true, buf, cpumask); |
| 161 |
} |
| 162 |
|
| 163 |
static struct attribute *dsu_pmu_format_attrs[] = { |
| 164 |
DSU_FORMAT_ATTR(event, "config:0-31"), |
| 165 |
NULL, |
| 166 |
}; |
| 167 |
|
| 168 |
static const struct attribute_group dsu_pmu_format_attr_group = { |
| 169 |
.name = "format", |
| 170 |
.attrs = dsu_pmu_format_attrs, |
| 171 |
}; |
| 172 |
|
| 173 |
static struct attribute *dsu_pmu_event_attrs[] = { |
| 174 |
DSU_EVENT_ATTR(cycles, 0x11), |
| 175 |
DSU_EVENT_ATTR(bus_access, 0x19), |
| 176 |
DSU_EVENT_ATTR(memory_error, 0x1a), |
| 177 |
DSU_EVENT_ATTR(bus_cycles, 0x1d), |
| 178 |
DSU_EVENT_ATTR(l3d_cache_allocate, 0x29), |
| 179 |
DSU_EVENT_ATTR(l3d_cache_refill, 0x2a), |
| 180 |
DSU_EVENT_ATTR(l3d_cache, 0x2b), |
| 181 |
DSU_EVENT_ATTR(l3d_cache_wb, 0x2c), |
| 182 |
NULL, |
| 183 |
}; |
| 184 |
|
| 185 |
static umode_t |
| 186 |
dsu_pmu_event_attr_is_visible(struct kobject *kobj, struct attribute *attr, |
| 187 |
int unused) |
| 188 |
{ |
| 189 |
struct pmu *pmu = dev_get_drvdata(kobj_to_dev(kobj)); |
| 190 |
struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu); |
| 191 |
struct dev_ext_attribute *eattr = container_of(attr, |
| 192 |
struct dev_ext_attribute, attr.attr); |
| 193 |
unsigned long evt = (unsigned long)eattr->var; |
| 194 |
|
| 195 |
return test_bit(evt, dsu_pmu->cpmceid_bitmap) ? attr->mode : 0; |
| 196 |
} |
| 197 |
|
| 198 |
static const struct attribute_group dsu_pmu_events_attr_group = { |
| 199 |
.name = "events", |
| 200 |
.attrs = dsu_pmu_event_attrs, |
| 201 |
.is_visible = dsu_pmu_event_attr_is_visible, |
| 202 |
}; |
| 203 |
|
| 204 |
static struct attribute *dsu_pmu_cpumask_attrs[] = { |
| 205 |
DSU_CPUMASK_ATTR(cpumask, DSU_ACTIVE_CPU_MASK), |
| 206 |
DSU_CPUMASK_ATTR(associated_cpus, DSU_ASSOCIATED_CPU_MASK), |
| 207 |
NULL, |
| 208 |
}; |
| 209 |
|
| 210 |
static const struct attribute_group dsu_pmu_cpumask_attr_group = { |
| 211 |
.attrs = dsu_pmu_cpumask_attrs, |
| 212 |
}; |
| 213 |
|
| 214 |
static const struct attribute_group *dsu_pmu_attr_groups[] = { |
| 215 |
&dsu_pmu_cpumask_attr_group, |
| 216 |
&dsu_pmu_events_attr_group, |
| 217 |
&dsu_pmu_format_attr_group, |
| 218 |
NULL, |
| 219 |
}; |
| 220 |
|
| 221 |
static inline bool dsu_pmu_counter_valid(struct dsu_pmu *dsu_pmu, u32 idx) |
| 222 |
{ |
| 223 |
return (idx < dsu_pmu->num_counters) || |
| 224 |
(idx == DSU_PMU_IDX_CYCLE_COUNTER); |
| 225 |
} |
| 226 |
|
| 227 |
static inline u64 dsu_pmu_read_counter(struct perf_event *event) |
| 228 |
{ |
| 229 |
u64 val; |
| 230 |
unsigned long flags; |
| 231 |
struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu); |
| 232 |
int idx = event->hw.idx; |
| 233 |
|
| 234 |
if (WARN_ON(!cpumask_test_cpu(smp_processor_id(), |
| 235 |
&dsu_pmu->associated_cpus))) |
| 236 |
return 0; |
| 237 |
|
| 238 |
if (!dsu_pmu_counter_valid(dsu_pmu, idx)) { |
| 239 |
dev_err(event->pmu->dev, |
| 240 |
"Trying reading invalid counter %d\n", idx); |
| 241 |
return 0; |
| 242 |
} |
| 243 |
|
| 244 |
raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags); |
| 245 |
if (idx == DSU_PMU_IDX_CYCLE_COUNTER) |
| 246 |
val = __dsu_pmu_read_pmccntr(); |
| 247 |
else |
| 248 |
val = __dsu_pmu_read_counter(idx); |
| 249 |
raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags); |
| 250 |
|
| 251 |
return val; |
| 252 |
} |
| 253 |
|
| 254 |
static void dsu_pmu_write_counter(struct perf_event *event, u64 val) |
| 255 |
{ |
| 256 |
unsigned long flags; |
| 257 |
struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu); |
| 258 |
int idx = event->hw.idx; |
| 259 |
|
| 260 |
if (WARN_ON(!cpumask_test_cpu(smp_processor_id(), |
| 261 |
&dsu_pmu->associated_cpus))) |
| 262 |
return; |
| 263 |
|
| 264 |
if (!dsu_pmu_counter_valid(dsu_pmu, idx)) { |
| 265 |
dev_err(event->pmu->dev, |
| 266 |
"writing to invalid counter %d\n", idx); |
| 267 |
return; |
| 268 |
} |
| 269 |
|
| 270 |
raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags); |
| 271 |
if (idx == DSU_PMU_IDX_CYCLE_COUNTER) |
| 272 |
__dsu_pmu_write_pmccntr(val); |
| 273 |
else |
| 274 |
__dsu_pmu_write_counter(idx, val); |
| 275 |
raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags); |
| 276 |
} |
| 277 |
|
| 278 |
static int dsu_pmu_get_event_idx(struct dsu_hw_events *hw_events, |
| 279 |
struct perf_event *event) |
| 280 |
{ |
| 281 |
int idx; |
| 282 |
unsigned long evtype = event->attr.config; |
| 283 |
struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu); |
| 284 |
unsigned long *used_mask = hw_events->used_mask; |
| 285 |
|
| 286 |
if (evtype == DSU_PMU_EVT_CYCLES && dsu_pmu->has_pmccntr) { |
| 287 |
if (!test_and_set_bit(DSU_PMU_IDX_CYCLE_COUNTER, used_mask)) |
| 288 |
return DSU_PMU_IDX_CYCLE_COUNTER; |
| 289 |
} |
| 290 |
|
| 291 |
idx = find_first_zero_bit(used_mask, dsu_pmu->num_counters); |
| 292 |
if (idx >= dsu_pmu->num_counters) |
| 293 |
return -EAGAIN; |
| 294 |
set_bit(idx, hw_events->used_mask); |
| 295 |
return idx; |
| 296 |
} |
| 297 |
|
| 298 |
static void dsu_pmu_enable_counter(struct dsu_pmu *dsu_pmu, int idx) |
| 299 |
{ |
| 300 |
__dsu_pmu_counter_interrupt_enable(idx); |
| 301 |
__dsu_pmu_enable_counter(idx); |
| 302 |
} |
| 303 |
|
| 304 |
static void dsu_pmu_disable_counter(struct dsu_pmu *dsu_pmu, int idx) |
| 305 |
{ |
| 306 |
__dsu_pmu_disable_counter(idx); |
| 307 |
__dsu_pmu_counter_interrupt_disable(idx); |
| 308 |
} |
| 309 |
|
| 310 |
static inline void dsu_pmu_set_event(struct dsu_pmu *dsu_pmu, |
| 311 |
struct perf_event *event) |
| 312 |
{ |
| 313 |
int idx = event->hw.idx; |
| 314 |
unsigned long flags; |
| 315 |
|
| 316 |
if (!dsu_pmu_counter_valid(dsu_pmu, idx)) { |
| 317 |
dev_err(event->pmu->dev, |
| 318 |
"Trying to set invalid counter %d\n", idx); |
| 319 |
return; |
| 320 |
} |
| 321 |
|
| 322 |
raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags); |
| 323 |
__dsu_pmu_set_event(idx, event->hw.config_base); |
| 324 |
raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags); |
| 325 |
} |
| 326 |
|
| 327 |
static u64 dsu_pmu_counter_mask(struct hw_perf_event *hw) |
| 328 |
{ |
| 329 |
return (hw->flags && hw->idx != DSU_PMU_IDX_CYCLE_COUNTER) ? U32_MAX : U64_MAX; |
| 330 |
} |
| 331 |
|
| 332 |
static void dsu_pmu_event_update(struct perf_event *event) |
| 333 |
{ |
| 334 |
struct hw_perf_event *hwc = &event->hw; |
| 335 |
u64 delta, prev_count, new_count; |
| 336 |
|
| 337 |
do { |
| 338 |
/* We may also be called from the irq handler */ |
| 339 |
prev_count = local64_read(&hwc->prev_count); |
| 340 |
new_count = dsu_pmu_read_counter(event); |
| 341 |
} while (local64_cmpxchg(&hwc->prev_count, prev_count, new_count) != |
| 342 |
prev_count); |
| 343 |
delta = (new_count - prev_count) & dsu_pmu_counter_mask(hwc); |
| 344 |
local64_add(delta, &event->count); |
| 345 |
} |
| 346 |
|
| 347 |
static void dsu_pmu_read(struct perf_event *event) |
| 348 |
{ |
| 349 |
dsu_pmu_event_update(event); |
| 350 |
} |
| 351 |
|
| 352 |
static inline u32 dsu_pmu_get_reset_overflow(void) |
| 353 |
{ |
| 354 |
return __dsu_pmu_get_reset_overflow(); |
| 355 |
} |
| 356 |
|
| 357 |
/* |
| 358 |
* dsu_pmu_set_event_period: Set the period for the counter. |
| 359 |
* |
| 360 |
* All DSU PMU event counters, except the cycle counter are 32bit |
| 361 |
* counters. To handle cases of extreme interrupt latency, we program |
| 362 |
* the counter with half of the max count for the counters. |
| 363 |
*/ |
| 364 |
static void dsu_pmu_set_event_period(struct perf_event *event) |
| 365 |
{ |
| 366 |
u64 val = dsu_pmu_counter_mask(&event->hw) >> 1; |
| 367 |
|
| 368 |
local64_set(&event->hw.prev_count, val); |
| 369 |
dsu_pmu_write_counter(event, val); |
| 370 |
} |
| 371 |
|
| 372 |
static irqreturn_t dsu_pmu_handle_irq(int irq_num, void *dev) |
| 373 |
{ |
| 374 |
int i; |
| 375 |
bool handled = false; |
| 376 |
struct dsu_pmu *dsu_pmu = dev; |
| 377 |
struct dsu_hw_events *hw_events = &dsu_pmu->hw_events; |
| 378 |
unsigned long overflow; |
| 379 |
|
| 380 |
overflow = dsu_pmu_get_reset_overflow(); |
| 381 |
if (!overflow) |
| 382 |
return IRQ_NONE; |
| 383 |
|
| 384 |
for_each_set_bit(i, &overflow, DSU_PMU_MAX_HW_CNTRS) { |
| 385 |
struct perf_event *event = hw_events->events[i]; |
| 386 |
|
| 387 |
if (!event) |
| 388 |
continue; |
| 389 |
dsu_pmu_event_update(event); |
| 390 |
dsu_pmu_set_event_period(event); |
| 391 |
handled = true; |
| 392 |
} |
| 393 |
|
| 394 |
return IRQ_RETVAL(handled); |
| 395 |
} |
| 396 |
|
| 397 |
static void dsu_pmu_start(struct perf_event *event, int pmu_flags) |
| 398 |
{ |
| 399 |
struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu); |
| 400 |
|
| 401 |
/* We always reprogram the counter */ |
| 402 |
if (pmu_flags & PERF_EF_RELOAD) |
| 403 |
WARN_ON(!(event->hw.state & PERF_HES_UPTODATE)); |
| 404 |
dsu_pmu_set_event_period(event); |
| 405 |
if (event->hw.idx != DSU_PMU_IDX_CYCLE_COUNTER) |
| 406 |
dsu_pmu_set_event(dsu_pmu, event); |
| 407 |
event->hw.state = 0; |
| 408 |
dsu_pmu_enable_counter(dsu_pmu, event->hw.idx); |
| 409 |
} |
| 410 |
|
| 411 |
static void dsu_pmu_stop(struct perf_event *event, int pmu_flags) |
| 412 |
{ |
| 413 |
struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu); |
| 414 |
|
| 415 |
if (event->hw.state & PERF_HES_STOPPED) |
| 416 |
return; |
| 417 |
dsu_pmu_disable_counter(dsu_pmu, event->hw.idx); |
| 418 |
dsu_pmu_event_update(event); |
| 419 |
event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; |
| 420 |
} |
| 421 |
|
| 422 |
static int dsu_pmu_add(struct perf_event *event, int flags) |
| 423 |
{ |
| 424 |
struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu); |
| 425 |
struct dsu_hw_events *hw_events = &dsu_pmu->hw_events; |
| 426 |
struct hw_perf_event *hwc = &event->hw; |
| 427 |
int idx; |
| 428 |
|
| 429 |
if (WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), |
| 430 |
&dsu_pmu->associated_cpus))) |
| 431 |
return -ENOENT; |
| 432 |
|
| 433 |
idx = dsu_pmu_get_event_idx(hw_events, event); |
| 434 |
if (idx < 0) |
| 435 |
return idx; |
| 436 |
|
| 437 |
hwc->idx = idx; |
| 438 |
hw_events->events[idx] = event; |
| 439 |
hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; |
| 440 |
|
| 441 |
if (flags & PERF_EF_START) |
| 442 |
dsu_pmu_start(event, PERF_EF_RELOAD); |
| 443 |
|
| 444 |
perf_event_update_userpage(event); |
| 445 |
return 0; |
| 446 |
} |
| 447 |
|
| 448 |
static void dsu_pmu_del(struct perf_event *event, int flags) |
| 449 |
{ |
| 450 |
struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu); |
| 451 |
struct dsu_hw_events *hw_events = &dsu_pmu->hw_events; |
| 452 |
struct hw_perf_event *hwc = &event->hw; |
| 453 |
int idx = hwc->idx; |
| 454 |
|
| 455 |
dsu_pmu_stop(event, PERF_EF_UPDATE); |
| 456 |
hw_events->events[idx] = NULL; |
| 457 |
clear_bit(idx, hw_events->used_mask); |
| 458 |
perf_event_update_userpage(event); |
| 459 |
} |
| 460 |
|
| 461 |
static void dsu_pmu_enable(struct pmu *pmu) |
| 462 |
{ |
| 463 |
u32 pmcr; |
| 464 |
unsigned long flags; |
| 465 |
struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu); |
| 466 |
|
| 467 |
/* If no counters are added, skip enabling the PMU */ |
| 468 |
if (bitmap_empty(dsu_pmu->hw_events.used_mask, DSU_PMU_MAX_HW_CNTRS)) |
| 469 |
return; |
| 470 |
|
| 471 |
raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags); |
| 472 |
pmcr = __dsu_pmu_read_pmcr(); |
| 473 |
pmcr |= CLUSTERPMCR_E; |
| 474 |
__dsu_pmu_write_pmcr(pmcr); |
| 475 |
raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags); |
| 476 |
} |
| 477 |
|
| 478 |
static void dsu_pmu_disable(struct pmu *pmu) |
| 479 |
{ |
| 480 |
u32 pmcr; |
| 481 |
unsigned long flags; |
| 482 |
struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu); |
| 483 |
|
| 484 |
raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags); |
| 485 |
pmcr = __dsu_pmu_read_pmcr(); |
| 486 |
pmcr &= ~CLUSTERPMCR_E; |
| 487 |
__dsu_pmu_write_pmcr(pmcr); |
| 488 |
raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags); |
| 489 |
} |
| 490 |
|
| 491 |
static bool dsu_pmu_validate_event(struct pmu *pmu, |
| 492 |
struct dsu_hw_events *hw_events, |
| 493 |
struct perf_event *event) |
| 494 |
{ |
| 495 |
if (is_software_event(event)) |
| 496 |
return true; |
| 497 |
/* Reject groups spanning multiple HW PMUs. */ |
| 498 |
if (event->pmu != pmu) |
| 499 |
return false; |
| 500 |
return dsu_pmu_get_event_idx(hw_events, event) >= 0; |
| 501 |
} |
| 502 |
|
| 503 |
/* |
| 504 |
* Make sure the group of events can be scheduled at once |
| 505 |
* on the PMU. |
| 506 |
*/ |
| 507 |
static bool dsu_pmu_validate_group(struct perf_event *event) |
| 508 |
{ |
| 509 |
struct perf_event *sibling, *leader = event->group_leader; |
| 510 |
struct dsu_hw_events fake_hw; |
| 511 |
|
| 512 |
if (event->group_leader == event) |
| 513 |
return true; |
| 514 |
|
| 515 |
memset(fake_hw.used_mask, 0, sizeof(fake_hw.used_mask)); |
| 516 |
if (!dsu_pmu_validate_event(event->pmu, &fake_hw, leader)) |
| 517 |
return false; |
| 518 |
for_each_sibling_event(sibling, leader) { |
| 519 |
if (!dsu_pmu_validate_event(event->pmu, &fake_hw, sibling)) |
| 520 |
return false; |
| 521 |
} |
| 522 |
return dsu_pmu_validate_event(event->pmu, &fake_hw, event); |
| 523 |
} |
| 524 |
|
| 525 |
static int dsu_pmu_event_init(struct perf_event *event) |
| 526 |
{ |
| 527 |
struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu); |
| 528 |
|
| 529 |
if (event->attr.type != event->pmu->type) |
| 530 |
return -ENOENT; |
| 531 |
|
| 532 |
/* We don't support sampling */ |
| 533 |
if (is_sampling_event(event)) { |
| 534 |
dev_dbg(dsu_pmu->pmu.dev, "Can't support sampling events\n"); |
| 535 |
return -EOPNOTSUPP; |
| 536 |
} |
| 537 |
|
| 538 |
/* We cannot support task bound events */ |
| 539 |
if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK) { |
| 540 |
dev_dbg(dsu_pmu->pmu.dev, "Can't support per-task counters\n"); |
| 541 |
return -EINVAL; |
| 542 |
} |
| 543 |
|
| 544 |
if (has_branch_stack(event)) { |
| 545 |
dev_dbg(dsu_pmu->pmu.dev, "Can't support filtering\n"); |
| 546 |
return -EINVAL; |
| 547 |
} |
| 548 |
|
| 549 |
if (!cpumask_test_cpu(event->cpu, &dsu_pmu->associated_cpus)) { |
| 550 |
dev_dbg(dsu_pmu->pmu.dev, |
| 551 |
"Requested cpu is not associated with the DSU\n"); |
| 552 |
return -EINVAL; |
| 553 |
} |
| 554 |
/* |
| 555 |
* Choose the current active CPU to read the events. We don't want |
| 556 |
* to migrate the event contexts, irq handling etc to the requested |
| 557 |
* CPU. As long as the requested CPU is within the same DSU, we |
| 558 |
* are fine. |
| 559 |
*/ |
| 560 |
event->cpu = cpumask_first(&dsu_pmu->active_cpu); |
| 561 |
if (event->cpu >= nr_cpu_ids) |
| 562 |
return -EINVAL; |
| 563 |
if (!dsu_pmu_validate_group(event)) |
| 564 |
return -EINVAL; |
| 565 |
|
| 566 |
event->hw.config_base = event->attr.config; |
| 567 |
event->hw.flags = dsu_pmu->has_32b_pmevcntr; |
| 568 |
return 0; |
| 569 |
} |
| 570 |
|
| 571 |
static struct dsu_pmu *dsu_pmu_alloc(struct platform_device *pdev) |
| 572 |
{ |
| 573 |
struct dsu_pmu *dsu_pmu; |
| 574 |
|
| 575 |
dsu_pmu = devm_kzalloc(&pdev->dev, sizeof(*dsu_pmu), GFP_KERNEL); |
| 576 |
if (!dsu_pmu) |
| 577 |
return ERR_PTR(-ENOMEM); |
| 578 |
|
| 579 |
raw_spin_lock_init(&dsu_pmu->pmu_lock); |
| 580 |
/* |
| 581 |
* Initialise the number of counters to -1, until we probe |
| 582 |
* the real number on a connected CPU. |
| 583 |
*/ |
| 584 |
dsu_pmu->num_counters = -1; |
| 585 |
return dsu_pmu; |
| 586 |
} |
| 587 |
|
| 588 |
/* |
| 589 |
* dsu_pmu_dt_get_cpus: Get the list of CPUs in the cluster |
| 590 |
* from device tree. |
| 591 |
*/ |
| 592 |
static int dsu_pmu_dt_get_cpus(struct device *dev, cpumask_t *mask) |
| 593 |
{ |
| 594 |
int i = 0, n, cpu; |
| 595 |
struct device_node *cpu_node; |
| 596 |
|
| 597 |
n = of_count_phandle_with_args(dev->of_node, "cpus", NULL); |
| 598 |
if (n <= 0) |
| 599 |
return -ENODEV; |
| 600 |
for (; i < n; i++) { |
| 601 |
cpu_node = of_parse_phandle(dev->of_node, "cpus", i); |
| 602 |
if (!cpu_node) |
| 603 |
break; |
| 604 |
cpu = of_cpu_node_to_id(cpu_node); |
| 605 |
of_node_put(cpu_node); |
| 606 |
/* |
| 607 |
* We have to ignore the failures here and continue scanning |
| 608 |
* the list to handle cases where the nr_cpus could be capped |
| 609 |
* in the running kernel. |
| 610 |
*/ |
| 611 |
if (cpu < 0) |
| 612 |
continue; |
| 613 |
cpumask_set_cpu(cpu, mask); |
| 614 |
} |
| 615 |
return 0; |
| 616 |
} |
| 617 |
|
| 618 |
/* |
| 619 |
* dsu_pmu_acpi_get_cpus: Get the list of CPUs in the cluster |
| 620 |
* from ACPI. |
| 621 |
*/ |
| 622 |
static int dsu_pmu_acpi_get_cpus(struct device *dev, cpumask_t *mask) |
| 623 |
{ |
| 624 |
#ifdef CONFIG_ACPI |
| 625 |
struct acpi_device *parent_adev = acpi_dev_parent(ACPI_COMPANION(dev)); |
| 626 |
int cpu; |
| 627 |
|
| 628 |
/* |
| 629 |
* A dsu pmu node is inside a cluster parent node along with cpu nodes. |
| 630 |
* We need to find out all cpus that have the same parent with this pmu. |
| 631 |
*/ |
| 632 |
for_each_possible_cpu(cpu) { |
| 633 |
struct acpi_device *acpi_dev; |
| 634 |
struct device *cpu_dev = get_cpu_device(cpu); |
| 635 |
|
| 636 |
if (!cpu_dev) |
| 637 |
continue; |
| 638 |
|
| 639 |
acpi_dev = ACPI_COMPANION(cpu_dev); |
| 640 |
if (acpi_dev && acpi_dev_parent(acpi_dev) == parent_adev) |
| 641 |
cpumask_set_cpu(cpu, mask); |
| 642 |
} |
| 643 |
#endif |
| 644 |
|
| 645 |
return 0; |
| 646 |
} |
| 647 |
|
| 648 |
/* |
| 649 |
* dsu_pmu_probe_pmu: Probe the PMU details on a CPU in the cluster. |
| 650 |
*/ |
| 651 |
static void dsu_pmu_probe_pmu(struct dsu_pmu *dsu_pmu) |
| 652 |
{ |
| 653 |
u64 num_counters; |
| 654 |
u32 cpmceid[2]; |
| 655 |
|
| 656 |
num_counters = (__dsu_pmu_read_pmcr() >> CLUSTERPMCR_N_SHIFT) & |
| 657 |
CLUSTERPMCR_N_MASK; |
| 658 |
/* We can only support up to 31 independent counters */ |
| 659 |
if (WARN_ON(num_counters > 31)) |
| 660 |
num_counters = 31; |
| 661 |
dsu_pmu->num_counters = num_counters; |
| 662 |
if (!dsu_pmu->num_counters) |
| 663 |
return; |
| 664 |
cpmceid[0] = __dsu_pmu_read_pmceid(0); |
| 665 |
cpmceid[1] = __dsu_pmu_read_pmceid(1); |
| 666 |
bitmap_from_arr32(dsu_pmu->cpmceid_bitmap, cpmceid, |
| 667 |
DSU_PMU_MAX_COMMON_EVENTS); |
| 668 |
/* Newer DSUs have 64-bit counters */ |
| 669 |
__dsu_pmu_write_counter(0, U64_MAX); |
| 670 |
if (__dsu_pmu_read_counter(0) != U64_MAX) |
| 671 |
dsu_pmu->has_32b_pmevcntr = true; |
| 672 |
/* On even newer DSUs, PMCCNTR is RAZ/WI */ |
| 673 |
__dsu_pmu_write_pmccntr(U64_MAX); |
| 674 |
if (__dsu_pmu_read_pmccntr() == U64_MAX) |
| 675 |
dsu_pmu->has_pmccntr = true; |
| 676 |
} |
| 677 |
|
| 678 |
static void dsu_pmu_set_active_cpu(int cpu, struct dsu_pmu *dsu_pmu) |
| 679 |
{ |
| 680 |
cpumask_set_cpu(cpu, &dsu_pmu->active_cpu); |
| 681 |
if (irq_set_affinity(dsu_pmu->irq, &dsu_pmu->active_cpu)) |
| 682 |
pr_warn("Failed to set irq affinity to %d\n", cpu); |
| 683 |
} |
| 684 |
|
| 685 |
/* |
| 686 |
* dsu_pmu_init_pmu: Initialise the DSU PMU configurations if |
| 687 |
* we haven't done it already. |
| 688 |
*/ |
| 689 |
static void dsu_pmu_init_pmu(struct dsu_pmu *dsu_pmu) |
| 690 |
{ |
| 691 |
if (dsu_pmu->num_counters == -1) |
| 692 |
dsu_pmu_probe_pmu(dsu_pmu); |
| 693 |
/* Reset the interrupt overflow mask */ |
| 694 |
dsu_pmu_get_reset_overflow(); |
| 695 |
} |
| 696 |
|
| 697 |
static int dsu_pmu_device_probe(struct platform_device *pdev) |
| 698 |
{ |
| 699 |
int irq, rc; |
| 700 |
struct dsu_pmu *dsu_pmu; |
| 701 |
struct fwnode_handle *fwnode = dev_fwnode(&pdev->dev); |
| 702 |
char *name; |
| 703 |
static atomic_t pmu_idx = ATOMIC_INIT(-1); |
| 704 |
|
| 705 |
dsu_pmu = dsu_pmu_alloc(pdev); |
| 706 |
if (IS_ERR(dsu_pmu)) |
| 707 |
return PTR_ERR(dsu_pmu); |
| 708 |
|
| 709 |
if (is_of_node(fwnode)) |
| 710 |
rc = dsu_pmu_dt_get_cpus(&pdev->dev, &dsu_pmu->associated_cpus); |
| 711 |
else if (is_acpi_device_node(fwnode)) |
| 712 |
rc = dsu_pmu_acpi_get_cpus(&pdev->dev, &dsu_pmu->associated_cpus); |
| 713 |
else |
| 714 |
return -ENOENT; |
| 715 |
|
| 716 |
if (rc) { |
| 717 |
dev_warn(&pdev->dev, "Failed to parse the CPUs\n"); |
| 718 |
return rc; |
| 719 |
} |
| 720 |
|
| 721 |
irq = platform_get_irq(pdev, 0); |
| 722 |
if (irq < 0) |
| 723 |
return -EINVAL; |
| 724 |
|
| 725 |
name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_%d", |
| 726 |
PMUNAME, atomic_inc_return(&pmu_idx)); |
| 727 |
if (!name) |
| 728 |
return -ENOMEM; |
| 729 |
rc = devm_request_irq(&pdev->dev, irq, dsu_pmu_handle_irq, |
| 730 |
IRQF_NOBALANCING, name, dsu_pmu); |
| 731 |
if (rc) { |
| 732 |
dev_warn(&pdev->dev, "Failed to request IRQ %d\n", irq); |
| 733 |
return rc; |
| 734 |
} |
| 735 |
|
| 736 |
dsu_pmu->irq = irq; |
| 737 |
platform_set_drvdata(pdev, dsu_pmu); |
| 738 |
rc = cpuhp_state_add_instance(dsu_pmu_cpuhp_state, |
| 739 |
&dsu_pmu->cpuhp_node); |
| 740 |
if (rc) |
| 741 |
return rc; |
| 742 |
|
| 743 |
dsu_pmu->pmu = (struct pmu) { |
| 744 |
.task_ctx_nr = perf_invalid_context, |
| 745 |
.parent = &pdev->dev, |
| 746 |
.module = THIS_MODULE, |
| 747 |
.pmu_enable = dsu_pmu_enable, |
| 748 |
.pmu_disable = dsu_pmu_disable, |
| 749 |
.event_init = dsu_pmu_event_init, |
| 750 |
.add = dsu_pmu_add, |
| 751 |
.del = dsu_pmu_del, |
| 752 |
.start = dsu_pmu_start, |
| 753 |
.stop = dsu_pmu_stop, |
| 754 |
.read = dsu_pmu_read, |
| 755 |
|
| 756 |
.attr_groups = dsu_pmu_attr_groups, |
| 757 |
.capabilities = PERF_PMU_CAP_NO_EXCLUDE, |
| 758 |
}; |
| 759 |
|
| 760 |
rc = perf_pmu_register(&dsu_pmu->pmu, name, -1); |
| 761 |
if (rc) { |
| 762 |
cpuhp_state_remove_instance(dsu_pmu_cpuhp_state, |
| 763 |
&dsu_pmu->cpuhp_node); |
| 764 |
} |
| 765 |
|
| 766 |
return rc; |
| 767 |
} |
| 768 |
|
| 769 |
static void dsu_pmu_device_remove(struct platform_device *pdev) |
| 770 |
{ |
| 771 |
struct dsu_pmu *dsu_pmu = platform_get_drvdata(pdev); |
| 772 |
|
| 773 |
perf_pmu_unregister(&dsu_pmu->pmu); |
| 774 |
cpuhp_state_remove_instance(dsu_pmu_cpuhp_state, &dsu_pmu->cpuhp_node); |
| 775 |
} |
| 776 |
|
| 777 |
static const struct of_device_id dsu_pmu_of_match[] = { |
| 778 |
{ .compatible = "arm,dsu-pmu", }, |
| 779 |
{}, |
| 780 |
}; |
| 781 |
MODULE_DEVICE_TABLE(of, dsu_pmu_of_match); |
| 782 |
|
| 783 |
#ifdef CONFIG_ACPI |
| 784 |
static const struct acpi_device_id dsu_pmu_acpi_match[] = { |
| 785 |
{ "ARMHD500", 0}, |
| 786 |
{}, |
| 787 |
}; |
| 788 |
MODULE_DEVICE_TABLE(acpi, dsu_pmu_acpi_match); |
| 789 |
#endif |
| 790 |
|
| 791 |
static struct platform_driver dsu_pmu_driver = { |
| 792 |
.driver = { |
| 793 |
.name = DRVNAME, |
| 794 |
.of_match_table = of_match_ptr(dsu_pmu_of_match), |
| 795 |
.acpi_match_table = ACPI_PTR(dsu_pmu_acpi_match), |
| 796 |
.suppress_bind_attrs = true, |
| 797 |
}, |
| 798 |
.probe = dsu_pmu_device_probe, |
| 799 |
.remove = dsu_pmu_device_remove, |
| 800 |
}; |
| 801 |
|
| 802 |
static int dsu_pmu_cpu_online(unsigned int cpu, struct hlist_node *node) |
| 803 |
{ |
| 804 |
struct dsu_pmu *dsu_pmu = hlist_entry_safe(node, struct dsu_pmu, |
| 805 |
cpuhp_node); |
| 806 |
|
| 807 |
if (!cpumask_test_cpu(cpu, &dsu_pmu->associated_cpus)) |
| 808 |
return 0; |
| 809 |
|
| 810 |
/* If the PMU is already managed, there is nothing to do */ |
| 811 |
if (!cpumask_empty(&dsu_pmu->active_cpu)) |
| 812 |
return 0; |
| 813 |
|
| 814 |
dsu_pmu_init_pmu(dsu_pmu); |
| 815 |
dsu_pmu_set_active_cpu(cpu, dsu_pmu); |
| 816 |
|
| 817 |
return 0; |
| 818 |
} |
| 819 |
|
| 820 |
static int dsu_pmu_cpu_teardown(unsigned int cpu, struct hlist_node *node) |
| 821 |
{ |
| 822 |
struct dsu_pmu *dsu_pmu; |
| 823 |
unsigned int dst; |
| 824 |
|
| 825 |
dsu_pmu = hlist_entry_safe(node, struct dsu_pmu, cpuhp_node); |
| 826 |
|
| 827 |
if (!cpumask_test_and_clear_cpu(cpu, &dsu_pmu->active_cpu)) |
| 828 |
return 0; |
| 829 |
|
| 830 |
dst = cpumask_any_and_but(&dsu_pmu->associated_cpus, |
| 831 |
cpu_online_mask, cpu); |
| 832 |
/* If there are no active CPUs in the DSU, leave IRQ disabled */ |
| 833 |
if (dst >= nr_cpu_ids) |
| 834 |
return 0; |
| 835 |
|
| 836 |
perf_pmu_migrate_context(&dsu_pmu->pmu, cpu, dst); |
| 837 |
dsu_pmu_set_active_cpu(dst, dsu_pmu); |
| 838 |
|
| 839 |
return 0; |
| 840 |
} |
| 841 |
|
| 842 |
static int __init dsu_pmu_init(void) |
| 843 |
{ |
| 844 |
int ret; |
| 845 |
|
| 846 |
ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, |
| 847 |
DRVNAME, |
| 848 |
dsu_pmu_cpu_online, |
| 849 |
dsu_pmu_cpu_teardown); |
| 850 |
if (ret < 0) |
| 851 |
return ret; |
| 852 |
dsu_pmu_cpuhp_state = ret; |
| 853 |
ret = platform_driver_register(&dsu_pmu_driver); |
| 854 |
if (ret) |
| 855 |
cpuhp_remove_multi_state(dsu_pmu_cpuhp_state); |
| 856 |
|
| 857 |
return ret; |
| 858 |
} |
| 859 |
|
| 860 |
static void __exit dsu_pmu_exit(void) |
| 861 |
{ |
| 862 |
platform_driver_unregister(&dsu_pmu_driver); |
| 863 |
cpuhp_remove_multi_state(dsu_pmu_cpuhp_state); |
| 864 |
} |
| 865 |
|
| 866 |
module_init(dsu_pmu_init); |
| 867 |
module_exit(dsu_pmu_exit); |
| 868 |
|
| 869 |
MODULE_DESCRIPTION("Perf driver for ARM DynamIQ Shared Unit"); |
| 870 |
MODULE_AUTHOR("Suzuki K Poulose <[email protected]>"); |
| 871 |
MODULE_LICENSE("GPL v2"); |