| 1 |
// SPDX-License-Identifier: GPL-2.0-only |
| 2 |
/* |
| 3 |
* Perf support for the Statistical Profiling Extension, introduced as |
| 4 |
* part of ARMv8.2. |
| 5 |
* |
| 6 |
* Copyright (C) 2016 ARM Limited |
| 7 |
* |
| 8 |
* Author: Will Deacon <[email protected]> |
| 9 |
*/ |
| 10 |
|
| 11 |
#define PMUNAME "arm_spe" |
| 12 |
#define DRVNAME PMUNAME "_pmu" |
| 13 |
#define pr_fmt(fmt) DRVNAME ": " fmt |
| 14 |
|
| 15 |
#include <linux/bitfield.h> |
| 16 |
#include <linux/bitops.h> |
| 17 |
#include <linux/bug.h> |
| 18 |
#include <linux/capability.h> |
| 19 |
#include <linux/cpuhotplug.h> |
| 20 |
#include <linux/cpumask.h> |
| 21 |
#include <linux/device.h> |
| 22 |
#include <linux/errno.h> |
| 23 |
#include <linux/interrupt.h> |
| 24 |
#include <linux/irq.h> |
| 25 |
#include <linux/kernel.h> |
| 26 |
#include <linux/list.h> |
| 27 |
#include <linux/module.h> |
| 28 |
#include <linux/of.h> |
| 29 |
#include <linux/perf_event.h> |
| 30 |
#include <linux/perf/arm_pmu.h> |
| 31 |
#include <linux/platform_device.h> |
| 32 |
#include <linux/printk.h> |
| 33 |
#include <linux/slab.h> |
| 34 |
#include <linux/smp.h> |
| 35 |
#include <linux/vmalloc.h> |
| 36 |
|
| 37 |
#include <asm/barrier.h> |
| 38 |
#include <asm/cpufeature.h> |
| 39 |
#include <asm/mmu.h> |
| 40 |
#include <asm/sysreg.h> |
| 41 |
|
| 42 |
/* |
| 43 |
* Cache if the event is allowed to trace Context information. |
| 44 |
* This allows us to perform the check, i.e, perf_allow_kernel(), |
| 45 |
* in the context of the event owner, once, during the event_init(). |
| 46 |
*/ |
| 47 |
#define SPE_PMU_HW_FLAGS_CX 0x00001 |
| 48 |
|
| 49 |
static_assert((PERF_EVENT_FLAG_ARCH & SPE_PMU_HW_FLAGS_CX) == SPE_PMU_HW_FLAGS_CX); |
| 50 |
|
| 51 |
static void set_spe_event_has_cx(struct perf_event *event) |
| 52 |
{ |
| 53 |
if (IS_ENABLED(CONFIG_PID_IN_CONTEXTIDR) && !perf_allow_kernel()) |
| 54 |
event->hw.flags |= SPE_PMU_HW_FLAGS_CX; |
| 55 |
} |
| 56 |
|
| 57 |
static bool get_spe_event_has_cx(struct perf_event *event) |
| 58 |
{ |
| 59 |
return !!(event->hw.flags & SPE_PMU_HW_FLAGS_CX); |
| 60 |
} |
| 61 |
|
| 62 |
#define ARM_SPE_BUF_PAD_BYTE 0 |
| 63 |
|
| 64 |
struct arm_spe_pmu_buf { |
| 65 |
int nr_pages; |
| 66 |
bool snapshot; |
| 67 |
void *base; |
| 68 |
}; |
| 69 |
|
| 70 |
struct arm_spe_pmu { |
| 71 |
struct pmu pmu; |
| 72 |
struct platform_device *pdev; |
| 73 |
cpumask_t supported_cpus; |
| 74 |
struct hlist_node hotplug_node; |
| 75 |
|
| 76 |
int irq; /* PPI */ |
| 77 |
u16 pmsver; |
| 78 |
u16 min_period; |
| 79 |
u16 counter_sz; |
| 80 |
|
| 81 |
#define SPE_PMU_FEAT_FILT_EVT (1UL << 0) |
| 82 |
#define SPE_PMU_FEAT_FILT_TYP (1UL << 1) |
| 83 |
#define SPE_PMU_FEAT_FILT_LAT (1UL << 2) |
| 84 |
#define SPE_PMU_FEAT_ARCH_INST (1UL << 3) |
| 85 |
#define SPE_PMU_FEAT_LDS (1UL << 4) |
| 86 |
#define SPE_PMU_FEAT_ERND (1UL << 5) |
| 87 |
#define SPE_PMU_FEAT_INV_FILT_EVT (1UL << 6) |
| 88 |
#define SPE_PMU_FEAT_DISCARD (1UL << 7) |
| 89 |
#define SPE_PMU_FEAT_EFT (1UL << 8) |
| 90 |
#define SPE_PMU_FEAT_FDS (1UL << 9) |
| 91 |
#define SPE_PMU_FEAT_DEV_PROBED (1UL << 63) |
| 92 |
u64 features; |
| 93 |
|
| 94 |
u64 pmsevfr_res0; |
| 95 |
u16 max_record_sz; |
| 96 |
u16 align; |
| 97 |
struct perf_output_handle __percpu *handle; |
| 98 |
}; |
| 99 |
|
| 100 |
#define to_spe_pmu(p) (container_of(p, struct arm_spe_pmu, pmu)) |
| 101 |
|
| 102 |
/* Convert a free-running index from perf into an SPE buffer offset */ |
| 103 |
#define PERF_IDX2OFF(idx, buf) \ |
| 104 |
((idx) % ((unsigned long)(buf)->nr_pages << PAGE_SHIFT)) |
| 105 |
|
| 106 |
/* Keep track of our dynamic hotplug state */ |
| 107 |
static enum cpuhp_state arm_spe_pmu_online; |
| 108 |
|
| 109 |
static void arm_spe_pmu_stop(struct perf_event *event, int flags); |
| 110 |
|
| 111 |
enum arm_spe_pmu_buf_fault_action { |
| 112 |
SPE_PMU_BUF_FAULT_ACT_SPURIOUS, |
| 113 |
SPE_PMU_BUF_FAULT_ACT_FATAL, |
| 114 |
SPE_PMU_BUF_FAULT_ACT_OK, |
| 115 |
}; |
| 116 |
|
| 117 |
/* This sysfs gunk was really good fun to write. */ |
| 118 |
enum arm_spe_pmu_capabilities { |
| 119 |
SPE_PMU_CAP_ARCH_INST = 0, |
| 120 |
SPE_PMU_CAP_ERND, |
| 121 |
SPE_PMU_CAP_FEAT_MAX, |
| 122 |
SPE_PMU_CAP_CNT_SZ = SPE_PMU_CAP_FEAT_MAX, |
| 123 |
SPE_PMU_CAP_MIN_IVAL, |
| 124 |
SPE_PMU_CAP_EVENT_FILTER, |
| 125 |
}; |
| 126 |
|
| 127 |
static int arm_spe_pmu_feat_caps[SPE_PMU_CAP_FEAT_MAX] = { |
| 128 |
[SPE_PMU_CAP_ARCH_INST] = SPE_PMU_FEAT_ARCH_INST, |
| 129 |
[SPE_PMU_CAP_ERND] = SPE_PMU_FEAT_ERND, |
| 130 |
}; |
| 131 |
|
| 132 |
static u64 arm_spe_pmu_cap_get(struct arm_spe_pmu *spe_pmu, int cap) |
| 133 |
{ |
| 134 |
if (cap < SPE_PMU_CAP_FEAT_MAX) |
| 135 |
return !!(spe_pmu->features & arm_spe_pmu_feat_caps[cap]); |
| 136 |
|
| 137 |
switch (cap) { |
| 138 |
case SPE_PMU_CAP_CNT_SZ: |
| 139 |
return spe_pmu->counter_sz; |
| 140 |
case SPE_PMU_CAP_MIN_IVAL: |
| 141 |
return spe_pmu->min_period; |
| 142 |
case SPE_PMU_CAP_EVENT_FILTER: |
| 143 |
return ~spe_pmu->pmsevfr_res0; |
| 144 |
default: |
| 145 |
WARN(1, "unknown cap %d\n", cap); |
| 146 |
} |
| 147 |
|
| 148 |
return 0; |
| 149 |
} |
| 150 |
|
| 151 |
static ssize_t arm_spe_pmu_cap_show(struct device *dev, |
| 152 |
struct device_attribute *attr, |
| 153 |
char *buf) |
| 154 |
{ |
| 155 |
struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev); |
| 156 |
struct dev_ext_attribute *ea = |
| 157 |
container_of(attr, struct dev_ext_attribute, attr); |
| 158 |
int cap = (long)ea->var; |
| 159 |
|
| 160 |
return sysfs_emit(buf, "%llu\n", arm_spe_pmu_cap_get(spe_pmu, cap)); |
| 161 |
} |
| 162 |
|
| 163 |
static ssize_t arm_spe_pmu_cap_show_hex(struct device *dev, |
| 164 |
struct device_attribute *attr, |
| 165 |
char *buf) |
| 166 |
{ |
| 167 |
struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev); |
| 168 |
struct dev_ext_attribute *ea = |
| 169 |
container_of(attr, struct dev_ext_attribute, attr); |
| 170 |
int cap = (long)ea->var; |
| 171 |
|
| 172 |
return sysfs_emit(buf, "0x%llx\n", arm_spe_pmu_cap_get(spe_pmu, cap)); |
| 173 |
} |
| 174 |
|
| 175 |
#define SPE_EXT_ATTR_ENTRY(_name, _func, _var) \ |
| 176 |
&((struct dev_ext_attribute[]) { \ |
| 177 |
{ __ATTR(_name, S_IRUGO, _func, NULL), (void *)_var } \ |
| 178 |
})[0].attr.attr |
| 179 |
|
| 180 |
#define SPE_CAP_EXT_ATTR_ENTRY(_name, _var) \ |
| 181 |
SPE_EXT_ATTR_ENTRY(_name, arm_spe_pmu_cap_show, _var) |
| 182 |
#define SPE_CAP_EXT_ATTR_ENTRY_HEX(_name, _var) \ |
| 183 |
SPE_EXT_ATTR_ENTRY(_name, arm_spe_pmu_cap_show_hex, _var) |
| 184 |
|
| 185 |
static struct attribute *arm_spe_pmu_cap_attr[] = { |
| 186 |
SPE_CAP_EXT_ATTR_ENTRY(arch_inst, SPE_PMU_CAP_ARCH_INST), |
| 187 |
SPE_CAP_EXT_ATTR_ENTRY(ernd, SPE_PMU_CAP_ERND), |
| 188 |
SPE_CAP_EXT_ATTR_ENTRY(count_size, SPE_PMU_CAP_CNT_SZ), |
| 189 |
SPE_CAP_EXT_ATTR_ENTRY(min_interval, SPE_PMU_CAP_MIN_IVAL), |
| 190 |
SPE_CAP_EXT_ATTR_ENTRY_HEX(event_filter, SPE_PMU_CAP_EVENT_FILTER), |
| 191 |
NULL, |
| 192 |
}; |
| 193 |
|
| 194 |
static const struct attribute_group arm_spe_pmu_cap_group = { |
| 195 |
.name = "caps", |
| 196 |
.attrs = arm_spe_pmu_cap_attr, |
| 197 |
}; |
| 198 |
|
| 199 |
/* User ABI */ |
| 200 |
#define ATTR_CFG_FLD_ts_enable_CFG config /* PMSCR_EL1.TS */ |
| 201 |
#define ATTR_CFG_FLD_ts_enable_LO 0 |
| 202 |
#define ATTR_CFG_FLD_ts_enable_HI 0 |
| 203 |
#define ATTR_CFG_FLD_pa_enable_CFG config /* PMSCR_EL1.PA */ |
| 204 |
#define ATTR_CFG_FLD_pa_enable_LO 1 |
| 205 |
#define ATTR_CFG_FLD_pa_enable_HI 1 |
| 206 |
#define ATTR_CFG_FLD_pct_enable_CFG config /* PMSCR_EL1.PCT */ |
| 207 |
#define ATTR_CFG_FLD_pct_enable_LO 2 |
| 208 |
#define ATTR_CFG_FLD_pct_enable_HI 2 |
| 209 |
#define ATTR_CFG_FLD_jitter_CFG config /* PMSIRR_EL1.RND */ |
| 210 |
#define ATTR_CFG_FLD_jitter_LO 16 |
| 211 |
#define ATTR_CFG_FLD_jitter_HI 16 |
| 212 |
#define ATTR_CFG_FLD_branch_filter_CFG config /* PMSFCR_EL1.B */ |
| 213 |
#define ATTR_CFG_FLD_branch_filter_LO 32 |
| 214 |
#define ATTR_CFG_FLD_branch_filter_HI 32 |
| 215 |
#define ATTR_CFG_FLD_load_filter_CFG config /* PMSFCR_EL1.LD */ |
| 216 |
#define ATTR_CFG_FLD_load_filter_LO 33 |
| 217 |
#define ATTR_CFG_FLD_load_filter_HI 33 |
| 218 |
#define ATTR_CFG_FLD_store_filter_CFG config /* PMSFCR_EL1.ST */ |
| 219 |
#define ATTR_CFG_FLD_store_filter_LO 34 |
| 220 |
#define ATTR_CFG_FLD_store_filter_HI 34 |
| 221 |
#define ATTR_CFG_FLD_discard_CFG config /* PMBLIMITR_EL1.FM = DISCARD */ |
| 222 |
#define ATTR_CFG_FLD_discard_LO 35 |
| 223 |
#define ATTR_CFG_FLD_discard_HI 35 |
| 224 |
#define ATTR_CFG_FLD_branch_filter_mask_CFG config /* PMSFCR_EL1.Bm */ |
| 225 |
#define ATTR_CFG_FLD_branch_filter_mask_LO 36 |
| 226 |
#define ATTR_CFG_FLD_branch_filter_mask_HI 36 |
| 227 |
#define ATTR_CFG_FLD_load_filter_mask_CFG config /* PMSFCR_EL1.LDm */ |
| 228 |
#define ATTR_CFG_FLD_load_filter_mask_LO 37 |
| 229 |
#define ATTR_CFG_FLD_load_filter_mask_HI 37 |
| 230 |
#define ATTR_CFG_FLD_store_filter_mask_CFG config /* PMSFCR_EL1.STm */ |
| 231 |
#define ATTR_CFG_FLD_store_filter_mask_LO 38 |
| 232 |
#define ATTR_CFG_FLD_store_filter_mask_HI 38 |
| 233 |
#define ATTR_CFG_FLD_simd_filter_CFG config /* PMSFCR_EL1.SIMD */ |
| 234 |
#define ATTR_CFG_FLD_simd_filter_LO 39 |
| 235 |
#define ATTR_CFG_FLD_simd_filter_HI 39 |
| 236 |
#define ATTR_CFG_FLD_simd_filter_mask_CFG config /* PMSFCR_EL1.SIMDm */ |
| 237 |
#define ATTR_CFG_FLD_simd_filter_mask_LO 40 |
| 238 |
#define ATTR_CFG_FLD_simd_filter_mask_HI 40 |
| 239 |
#define ATTR_CFG_FLD_float_filter_CFG config /* PMSFCR_EL1.FP */ |
| 240 |
#define ATTR_CFG_FLD_float_filter_LO 41 |
| 241 |
#define ATTR_CFG_FLD_float_filter_HI 41 |
| 242 |
#define ATTR_CFG_FLD_float_filter_mask_CFG config /* PMSFCR_EL1.FPm */ |
| 243 |
#define ATTR_CFG_FLD_float_filter_mask_LO 42 |
| 244 |
#define ATTR_CFG_FLD_float_filter_mask_HI 42 |
| 245 |
|
| 246 |
#define ATTR_CFG_FLD_event_filter_CFG config1 /* PMSEVFR_EL1 */ |
| 247 |
#define ATTR_CFG_FLD_event_filter_LO 0 |
| 248 |
#define ATTR_CFG_FLD_event_filter_HI 63 |
| 249 |
|
| 250 |
#define ATTR_CFG_FLD_min_latency_CFG config2 /* PMSLATFR_EL1.MINLAT */ |
| 251 |
#define ATTR_CFG_FLD_min_latency_LO 0 |
| 252 |
#define ATTR_CFG_FLD_min_latency_HI 11 |
| 253 |
|
| 254 |
#define ATTR_CFG_FLD_inv_event_filter_CFG config3 /* PMSNEVFR_EL1 */ |
| 255 |
#define ATTR_CFG_FLD_inv_event_filter_LO 0 |
| 256 |
#define ATTR_CFG_FLD_inv_event_filter_HI 63 |
| 257 |
|
| 258 |
#define ATTR_CFG_FLD_inv_data_src_filter_CFG config4 /* inverse of PMSDSFR_EL1 */ |
| 259 |
#define ATTR_CFG_FLD_inv_data_src_filter_LO 0 |
| 260 |
#define ATTR_CFG_FLD_inv_data_src_filter_HI 63 |
| 261 |
|
| 262 |
GEN_PMU_FORMAT_ATTR(ts_enable); |
| 263 |
GEN_PMU_FORMAT_ATTR(pa_enable); |
| 264 |
GEN_PMU_FORMAT_ATTR(pct_enable); |
| 265 |
GEN_PMU_FORMAT_ATTR(jitter); |
| 266 |
GEN_PMU_FORMAT_ATTR(branch_filter); |
| 267 |
GEN_PMU_FORMAT_ATTR(branch_filter_mask); |
| 268 |
GEN_PMU_FORMAT_ATTR(load_filter); |
| 269 |
GEN_PMU_FORMAT_ATTR(load_filter_mask); |
| 270 |
GEN_PMU_FORMAT_ATTR(store_filter); |
| 271 |
GEN_PMU_FORMAT_ATTR(store_filter_mask); |
| 272 |
GEN_PMU_FORMAT_ATTR(simd_filter); |
| 273 |
GEN_PMU_FORMAT_ATTR(simd_filter_mask); |
| 274 |
GEN_PMU_FORMAT_ATTR(float_filter); |
| 275 |
GEN_PMU_FORMAT_ATTR(float_filter_mask); |
| 276 |
GEN_PMU_FORMAT_ATTR(event_filter); |
| 277 |
GEN_PMU_FORMAT_ATTR(inv_event_filter); |
| 278 |
GEN_PMU_FORMAT_ATTR(inv_data_src_filter); |
| 279 |
GEN_PMU_FORMAT_ATTR(min_latency); |
| 280 |
GEN_PMU_FORMAT_ATTR(discard); |
| 281 |
|
| 282 |
static struct attribute *arm_spe_pmu_formats_attr[] = { |
| 283 |
&format_attr_ts_enable.attr, |
| 284 |
&format_attr_pa_enable.attr, |
| 285 |
&format_attr_pct_enable.attr, |
| 286 |
&format_attr_jitter.attr, |
| 287 |
&format_attr_branch_filter.attr, |
| 288 |
&format_attr_branch_filter_mask.attr, |
| 289 |
&format_attr_load_filter.attr, |
| 290 |
&format_attr_load_filter_mask.attr, |
| 291 |
&format_attr_store_filter.attr, |
| 292 |
&format_attr_store_filter_mask.attr, |
| 293 |
&format_attr_simd_filter.attr, |
| 294 |
&format_attr_simd_filter_mask.attr, |
| 295 |
&format_attr_float_filter.attr, |
| 296 |
&format_attr_float_filter_mask.attr, |
| 297 |
&format_attr_event_filter.attr, |
| 298 |
&format_attr_inv_event_filter.attr, |
| 299 |
&format_attr_inv_data_src_filter.attr, |
| 300 |
&format_attr_min_latency.attr, |
| 301 |
&format_attr_discard.attr, |
| 302 |
NULL, |
| 303 |
}; |
| 304 |
|
| 305 |
static umode_t arm_spe_pmu_format_attr_is_visible(struct kobject *kobj, |
| 306 |
struct attribute *attr, |
| 307 |
int unused) |
| 308 |
{ |
| 309 |
struct device *dev = kobj_to_dev(kobj); |
| 310 |
struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev); |
| 311 |
|
| 312 |
if (attr == &format_attr_discard.attr && !(spe_pmu->features & SPE_PMU_FEAT_DISCARD)) |
| 313 |
return 0; |
| 314 |
|
| 315 |
if (attr == &format_attr_inv_event_filter.attr && !(spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT)) |
| 316 |
return 0; |
| 317 |
|
| 318 |
if (attr == &format_attr_inv_data_src_filter.attr && |
| 319 |
!(spe_pmu->features & SPE_PMU_FEAT_FDS)) |
| 320 |
return 0; |
| 321 |
|
| 322 |
if ((attr == &format_attr_branch_filter_mask.attr || |
| 323 |
attr == &format_attr_load_filter_mask.attr || |
| 324 |
attr == &format_attr_store_filter_mask.attr || |
| 325 |
attr == &format_attr_simd_filter.attr || |
| 326 |
attr == &format_attr_simd_filter_mask.attr || |
| 327 |
attr == &format_attr_float_filter.attr || |
| 328 |
attr == &format_attr_float_filter_mask.attr) && |
| 329 |
!(spe_pmu->features & SPE_PMU_FEAT_EFT)) |
| 330 |
return 0; |
| 331 |
|
| 332 |
return attr->mode; |
| 333 |
} |
| 334 |
|
| 335 |
static const struct attribute_group arm_spe_pmu_format_group = { |
| 336 |
.name = "format", |
| 337 |
.is_visible = arm_spe_pmu_format_attr_is_visible, |
| 338 |
.attrs = arm_spe_pmu_formats_attr, |
| 339 |
}; |
| 340 |
|
| 341 |
static ssize_t cpumask_show(struct device *dev, |
| 342 |
struct device_attribute *attr, char *buf) |
| 343 |
{ |
| 344 |
struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev); |
| 345 |
|
| 346 |
return cpumap_print_to_pagebuf(true, buf, &spe_pmu->supported_cpus); |
| 347 |
} |
| 348 |
static DEVICE_ATTR_RO(cpumask); |
| 349 |
|
| 350 |
static struct attribute *arm_spe_pmu_attrs[] = { |
| 351 |
&dev_attr_cpumask.attr, |
| 352 |
NULL, |
| 353 |
}; |
| 354 |
|
| 355 |
static const struct attribute_group arm_spe_pmu_group = { |
| 356 |
.attrs = arm_spe_pmu_attrs, |
| 357 |
}; |
| 358 |
|
| 359 |
static const struct attribute_group *arm_spe_pmu_attr_groups[] = { |
| 360 |
&arm_spe_pmu_group, |
| 361 |
&arm_spe_pmu_cap_group, |
| 362 |
&arm_spe_pmu_format_group, |
| 363 |
NULL, |
| 364 |
}; |
| 365 |
|
| 366 |
/* Convert between user ABI and register values */ |
| 367 |
static u64 arm_spe_event_to_pmscr(struct perf_event *event) |
| 368 |
{ |
| 369 |
struct perf_event_attr *attr = &event->attr; |
| 370 |
u64 reg = 0; |
| 371 |
|
| 372 |
reg |= FIELD_PREP(PMSCR_EL1_TS, ATTR_CFG_GET_FLD(attr, ts_enable)); |
| 373 |
reg |= FIELD_PREP(PMSCR_EL1_PA, ATTR_CFG_GET_FLD(attr, pa_enable)); |
| 374 |
reg |= FIELD_PREP(PMSCR_EL1_PCT, ATTR_CFG_GET_FLD(attr, pct_enable)); |
| 375 |
|
| 376 |
if (!attr->exclude_user) |
| 377 |
reg |= PMSCR_EL1_E0SPE; |
| 378 |
|
| 379 |
if (!attr->exclude_kernel) |
| 380 |
reg |= PMSCR_EL1_E1SPE; |
| 381 |
|
| 382 |
if (get_spe_event_has_cx(event)) |
| 383 |
reg |= PMSCR_EL1_CX; |
| 384 |
|
| 385 |
return reg; |
| 386 |
} |
| 387 |
|
| 388 |
static void arm_spe_event_sanitise_period(struct perf_event *event) |
| 389 |
{ |
| 390 |
u64 period = event->hw.sample_period; |
| 391 |
u64 max_period = PMSIRR_EL1_INTERVAL_MASK; |
| 392 |
|
| 393 |
/* |
| 394 |
* The PMSIDR_EL1.Interval field (stored in spe_pmu->min_period) is a |
| 395 |
* recommendation for the minimum interval, not a hardware limitation. |
| 396 |
* |
| 397 |
* According to the Arm ARM (DDI 0487 L.a), section D24.7.12 PMSIRR_EL1, |
| 398 |
* Sampling Interval Reload Register, the INTERVAL field (bits [31:8]) |
| 399 |
* states: "Software must set this to a nonzero value". Use 1 as the |
| 400 |
* minimum value. |
| 401 |
*/ |
| 402 |
u64 min_period = FIELD_PREP(PMSIRR_EL1_INTERVAL_MASK, 1); |
| 403 |
|
| 404 |
period = clamp_t(u64, period, min_period, max_period) & max_period; |
| 405 |
event->hw.sample_period = period; |
| 406 |
} |
| 407 |
|
| 408 |
static u64 arm_spe_event_to_pmsirr(struct perf_event *event) |
| 409 |
{ |
| 410 |
struct perf_event_attr *attr = &event->attr; |
| 411 |
u64 reg = 0; |
| 412 |
|
| 413 |
arm_spe_event_sanitise_period(event); |
| 414 |
|
| 415 |
reg |= FIELD_PREP(PMSIRR_EL1_RND, ATTR_CFG_GET_FLD(attr, jitter)); |
| 416 |
reg |= event->hw.sample_period; |
| 417 |
|
| 418 |
return reg; |
| 419 |
} |
| 420 |
|
| 421 |
static u64 arm_spe_event_to_pmsfcr(struct perf_event *event) |
| 422 |
{ |
| 423 |
struct perf_event_attr *attr = &event->attr; |
| 424 |
u64 reg = 0; |
| 425 |
|
| 426 |
reg |= FIELD_PREP(PMSFCR_EL1_LD, ATTR_CFG_GET_FLD(attr, load_filter)); |
| 427 |
reg |= FIELD_PREP(PMSFCR_EL1_LDm, ATTR_CFG_GET_FLD(attr, load_filter_mask)); |
| 428 |
reg |= FIELD_PREP(PMSFCR_EL1_ST, ATTR_CFG_GET_FLD(attr, store_filter)); |
| 429 |
reg |= FIELD_PREP(PMSFCR_EL1_STm, ATTR_CFG_GET_FLD(attr, store_filter_mask)); |
| 430 |
reg |= FIELD_PREP(PMSFCR_EL1_B, ATTR_CFG_GET_FLD(attr, branch_filter)); |
| 431 |
reg |= FIELD_PREP(PMSFCR_EL1_Bm, ATTR_CFG_GET_FLD(attr, branch_filter_mask)); |
| 432 |
reg |= FIELD_PREP(PMSFCR_EL1_SIMD, ATTR_CFG_GET_FLD(attr, simd_filter)); |
| 433 |
reg |= FIELD_PREP(PMSFCR_EL1_SIMDm, ATTR_CFG_GET_FLD(attr, simd_filter_mask)); |
| 434 |
reg |= FIELD_PREP(PMSFCR_EL1_FP, ATTR_CFG_GET_FLD(attr, float_filter)); |
| 435 |
reg |= FIELD_PREP(PMSFCR_EL1_FPm, ATTR_CFG_GET_FLD(attr, float_filter_mask)); |
| 436 |
|
| 437 |
if (reg) |
| 438 |
reg |= PMSFCR_EL1_FT; |
| 439 |
|
| 440 |
if (ATTR_CFG_GET_FLD(attr, event_filter)) |
| 441 |
reg |= PMSFCR_EL1_FE; |
| 442 |
|
| 443 |
if (ATTR_CFG_GET_FLD(attr, inv_event_filter)) |
| 444 |
reg |= PMSFCR_EL1_FnE; |
| 445 |
|
| 446 |
if (ATTR_CFG_GET_FLD(attr, inv_data_src_filter)) |
| 447 |
reg |= PMSFCR_EL1_FDS; |
| 448 |
|
| 449 |
if (ATTR_CFG_GET_FLD(attr, min_latency)) |
| 450 |
reg |= PMSFCR_EL1_FL; |
| 451 |
|
| 452 |
return reg; |
| 453 |
} |
| 454 |
|
| 455 |
static u64 arm_spe_event_to_pmsevfr(struct perf_event *event) |
| 456 |
{ |
| 457 |
struct perf_event_attr *attr = &event->attr; |
| 458 |
return ATTR_CFG_GET_FLD(attr, event_filter); |
| 459 |
} |
| 460 |
|
| 461 |
static u64 arm_spe_event_to_pmsnevfr(struct perf_event *event) |
| 462 |
{ |
| 463 |
struct perf_event_attr *attr = &event->attr; |
| 464 |
return ATTR_CFG_GET_FLD(attr, inv_event_filter); |
| 465 |
} |
| 466 |
|
| 467 |
static u64 arm_spe_event_to_pmslatfr(struct perf_event *event) |
| 468 |
{ |
| 469 |
struct perf_event_attr *attr = &event->attr; |
| 470 |
return FIELD_PREP(PMSLATFR_EL1_MINLAT, ATTR_CFG_GET_FLD(attr, min_latency)); |
| 471 |
} |
| 472 |
|
| 473 |
static u64 arm_spe_event_to_pmsdsfr(struct perf_event *event) |
| 474 |
{ |
| 475 |
struct perf_event_attr *attr = &event->attr; |
| 476 |
|
| 477 |
/* |
| 478 |
* Data src filter is inverted so that the default value of 0 is |
| 479 |
* equivalent to no filtering. |
| 480 |
*/ |
| 481 |
return ~ATTR_CFG_GET_FLD(attr, inv_data_src_filter); |
| 482 |
} |
| 483 |
|
| 484 |
static void arm_spe_pmu_pad_buf(struct perf_output_handle *handle, int len) |
| 485 |
{ |
| 486 |
struct arm_spe_pmu_buf *buf = perf_get_aux(handle); |
| 487 |
u64 head = PERF_IDX2OFF(handle->head, buf); |
| 488 |
|
| 489 |
memset(buf->base + head, ARM_SPE_BUF_PAD_BYTE, len); |
| 490 |
if (!buf->snapshot) |
| 491 |
perf_aux_output_skip(handle, len); |
| 492 |
} |
| 493 |
|
| 494 |
static u64 arm_spe_pmu_next_snapshot_off(struct perf_output_handle *handle) |
| 495 |
{ |
| 496 |
struct arm_spe_pmu_buf *buf = perf_get_aux(handle); |
| 497 |
struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu); |
| 498 |
u64 head = PERF_IDX2OFF(handle->head, buf); |
| 499 |
u64 limit = buf->nr_pages * PAGE_SIZE; |
| 500 |
|
| 501 |
/* |
| 502 |
* The trace format isn't parseable in reverse, so clamp |
| 503 |
* the limit to half of the buffer size in snapshot mode |
| 504 |
* so that the worst case is half a buffer of records, as |
| 505 |
* opposed to a single record. |
| 506 |
*/ |
| 507 |
if (head < limit >> 1) |
| 508 |
limit >>= 1; |
| 509 |
|
| 510 |
/* |
| 511 |
* If we're within max_record_sz of the limit, we must |
| 512 |
* pad, move the head index and recompute the limit. |
| 513 |
*/ |
| 514 |
if (limit - head < spe_pmu->max_record_sz) { |
| 515 |
arm_spe_pmu_pad_buf(handle, limit - head); |
| 516 |
handle->head = PERF_IDX2OFF(limit, buf); |
| 517 |
limit = ((buf->nr_pages * PAGE_SIZE) >> 1) + handle->head; |
| 518 |
} |
| 519 |
|
| 520 |
return limit; |
| 521 |
} |
| 522 |
|
| 523 |
static u64 __arm_spe_pmu_next_off(struct perf_output_handle *handle) |
| 524 |
{ |
| 525 |
struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu); |
| 526 |
struct arm_spe_pmu_buf *buf = perf_get_aux(handle); |
| 527 |
const u64 bufsize = buf->nr_pages * PAGE_SIZE; |
| 528 |
u64 limit = bufsize; |
| 529 |
u64 head, tail, wakeup; |
| 530 |
|
| 531 |
/* |
| 532 |
* The head can be misaligned for two reasons: |
| 533 |
* |
| 534 |
* 1. The hardware left PMBPTR pointing to the first byte after |
| 535 |
* a record when generating a buffer management event. |
| 536 |
* |
| 537 |
* 2. We used perf_aux_output_skip to consume handle->size bytes |
| 538 |
* and CIRC_SPACE was used to compute the size, which always |
| 539 |
* leaves one entry free. |
| 540 |
* |
| 541 |
* Deal with this by padding to the next alignment boundary and |
| 542 |
* moving the head index. If we run out of buffer space, we'll |
| 543 |
* reduce handle->size to zero and end up reporting truncation. |
| 544 |
*/ |
| 545 |
head = PERF_IDX2OFF(handle->head, buf); |
| 546 |
if (!IS_ALIGNED(head, spe_pmu->align)) { |
| 547 |
unsigned long delta = roundup(head, spe_pmu->align) - head; |
| 548 |
|
| 549 |
delta = min(delta, handle->size); |
| 550 |
arm_spe_pmu_pad_buf(handle, delta); |
| 551 |
head = PERF_IDX2OFF(handle->head, buf); |
| 552 |
} |
| 553 |
|
| 554 |
/* If we've run out of free space, then nothing more to do */ |
| 555 |
if (!handle->size) |
| 556 |
goto no_space; |
| 557 |
|
| 558 |
/* Compute the tail and wakeup indices now that we've aligned head */ |
| 559 |
tail = PERF_IDX2OFF(handle->head + handle->size, buf); |
| 560 |
wakeup = PERF_IDX2OFF(handle->wakeup, buf); |
| 561 |
|
| 562 |
/* |
| 563 |
* Avoid clobbering unconsumed data. We know we have space, so |
| 564 |
* if we see head == tail we know that the buffer is empty. If |
| 565 |
* head > tail, then there's nothing to clobber prior to |
| 566 |
* wrapping. |
| 567 |
*/ |
| 568 |
if (head < tail) |
| 569 |
limit = round_down(tail, PAGE_SIZE); |
| 570 |
|
| 571 |
/* |
| 572 |
* Wakeup may be arbitrarily far into the future. If it's not in |
| 573 |
* the current generation, either we'll wrap before hitting it, |
| 574 |
* or it's in the past and has been handled already. |
| 575 |
* |
| 576 |
* If there's a wakeup before we wrap, arrange to be woken up by |
| 577 |
* the page boundary following it. Keep the tail boundary if |
| 578 |
* that's lower. |
| 579 |
*/ |
| 580 |
if (handle->wakeup < (handle->head + handle->size) && head <= wakeup) |
| 581 |
limit = min(limit, round_up(wakeup, PAGE_SIZE)); |
| 582 |
|
| 583 |
if (limit > head) |
| 584 |
return limit; |
| 585 |
|
| 586 |
arm_spe_pmu_pad_buf(handle, handle->size); |
| 587 |
no_space: |
| 588 |
perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); |
| 589 |
perf_aux_output_end(handle, 0); |
| 590 |
return 0; |
| 591 |
} |
| 592 |
|
| 593 |
static u64 arm_spe_pmu_next_off(struct perf_output_handle *handle) |
| 594 |
{ |
| 595 |
struct arm_spe_pmu_buf *buf = perf_get_aux(handle); |
| 596 |
struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu); |
| 597 |
u64 limit = __arm_spe_pmu_next_off(handle); |
| 598 |
u64 head = PERF_IDX2OFF(handle->head, buf); |
| 599 |
|
| 600 |
/* |
| 601 |
* If the head has come too close to the end of the buffer, |
| 602 |
* then pad to the end and recompute the limit. |
| 603 |
*/ |
| 604 |
if (limit && (limit - head < spe_pmu->max_record_sz)) { |
| 605 |
arm_spe_pmu_pad_buf(handle, limit - head); |
| 606 |
limit = __arm_spe_pmu_next_off(handle); |
| 607 |
} |
| 608 |
|
| 609 |
return limit; |
| 610 |
} |
| 611 |
|
| 612 |
static int arm_spe_perf_aux_output_begin(struct perf_output_handle *handle, |
| 613 |
struct perf_event *event) |
| 614 |
{ |
| 615 |
u64 base, limit; |
| 616 |
struct arm_spe_pmu_buf *buf; |
| 617 |
|
| 618 |
if (ATTR_CFG_GET_FLD(&event->attr, discard)) { |
| 619 |
limit = FIELD_PREP(PMBLIMITR_EL1_FM, PMBLIMITR_EL1_FM_DISCARD); |
| 620 |
limit |= PMBLIMITR_EL1_E; |
| 621 |
goto out_write_limit; |
| 622 |
} |
| 623 |
|
| 624 |
/* Start a new aux session */ |
| 625 |
buf = perf_aux_output_begin(handle, event); |
| 626 |
if (!buf) { |
| 627 |
/* |
| 628 |
* We still need to clear the limit pointer, since the |
| 629 |
* profiler might only be disabled by virtue of a fault. |
| 630 |
*/ |
| 631 |
limit = 0; |
| 632 |
goto out_write_limit; |
| 633 |
} |
| 634 |
|
| 635 |
limit = buf->snapshot ? arm_spe_pmu_next_snapshot_off(handle) |
| 636 |
: arm_spe_pmu_next_off(handle); |
| 637 |
if (limit) |
| 638 |
limit |= PMBLIMITR_EL1_E; |
| 639 |
|
| 640 |
limit += (u64)buf->base; |
| 641 |
base = (u64)buf->base + PERF_IDX2OFF(handle->head, buf); |
| 642 |
write_sysreg_s(base, SYS_PMBPTR_EL1); |
| 643 |
|
| 644 |
out_write_limit: |
| 645 |
write_sysreg_s(limit, SYS_PMBLIMITR_EL1); |
| 646 |
return (limit & PMBLIMITR_EL1_E) ? 0 : -EIO; |
| 647 |
} |
| 648 |
|
| 649 |
static void arm_spe_perf_aux_output_end(struct perf_output_handle *handle) |
| 650 |
{ |
| 651 |
struct arm_spe_pmu_buf *buf = perf_get_aux(handle); |
| 652 |
u64 offset, size; |
| 653 |
|
| 654 |
offset = read_sysreg_s(SYS_PMBPTR_EL1) - (u64)buf->base; |
| 655 |
size = offset - PERF_IDX2OFF(handle->head, buf); |
| 656 |
|
| 657 |
if (buf->snapshot) |
| 658 |
handle->head = offset; |
| 659 |
|
| 660 |
perf_aux_output_end(handle, size); |
| 661 |
} |
| 662 |
|
| 663 |
static void arm_spe_pmu_disable_and_drain_local(void) |
| 664 |
{ |
| 665 |
/* Disable profiling at EL0 and EL1 */ |
| 666 |
write_sysreg_s(0, SYS_PMSCR_EL1); |
| 667 |
isb(); |
| 668 |
|
| 669 |
/* Drain any buffered data */ |
| 670 |
psb_csync(); |
| 671 |
dsb(nsh); |
| 672 |
|
| 673 |
/* Disable the profiling buffer */ |
| 674 |
write_sysreg_s(0, SYS_PMBLIMITR_EL1); |
| 675 |
isb(); |
| 676 |
} |
| 677 |
|
| 678 |
/* IRQ handling */ |
| 679 |
static enum arm_spe_pmu_buf_fault_action |
| 680 |
arm_spe_pmu_buf_get_fault_act(struct perf_output_handle *handle) |
| 681 |
{ |
| 682 |
const char *err_str; |
| 683 |
u64 pmbsr; |
| 684 |
enum arm_spe_pmu_buf_fault_action ret; |
| 685 |
|
| 686 |
/* |
| 687 |
* Ensure new profiling data is visible to the CPU and any external |
| 688 |
* aborts have been resolved. |
| 689 |
*/ |
| 690 |
psb_csync(); |
| 691 |
dsb(nsh); |
| 692 |
|
| 693 |
/* Ensure hardware updates to PMBPTR_EL1 are visible */ |
| 694 |
isb(); |
| 695 |
|
| 696 |
/* Service required? */ |
| 697 |
pmbsr = read_sysreg_s(SYS_PMBSR_EL1); |
| 698 |
if (!FIELD_GET(PMBSR_EL1_S, pmbsr)) |
| 699 |
return SPE_PMU_BUF_FAULT_ACT_SPURIOUS; |
| 700 |
|
| 701 |
/* |
| 702 |
* If we've lost data, disable profiling and also set the PARTIAL |
| 703 |
* flag to indicate that the last record is corrupted. |
| 704 |
*/ |
| 705 |
if (FIELD_GET(PMBSR_EL1_DL, pmbsr)) |
| 706 |
perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED | |
| 707 |
PERF_AUX_FLAG_PARTIAL); |
| 708 |
|
| 709 |
/* Report collisions to userspace so that it can up the period */ |
| 710 |
if (FIELD_GET(PMBSR_EL1_COLL, pmbsr)) |
| 711 |
perf_aux_output_flag(handle, PERF_AUX_FLAG_COLLISION); |
| 712 |
|
| 713 |
/* We only expect buffer management events */ |
| 714 |
switch (FIELD_GET(PMBSR_EL1_EC, pmbsr)) { |
| 715 |
case PMBSR_EL1_EC_BUF: |
| 716 |
/* Handled below */ |
| 717 |
break; |
| 718 |
case PMBSR_EL1_EC_FAULT_S1: |
| 719 |
case PMBSR_EL1_EC_FAULT_S2: |
| 720 |
err_str = "Unexpected buffer fault"; |
| 721 |
goto out_err; |
| 722 |
default: |
| 723 |
err_str = "Unknown error code"; |
| 724 |
goto out_err; |
| 725 |
} |
| 726 |
|
| 727 |
/* Buffer management event */ |
| 728 |
switch (FIELD_GET(PMBSR_EL1_BUF_BSC_MASK, pmbsr)) { |
| 729 |
case PMBSR_EL1_BUF_BSC_FULL: |
| 730 |
ret = SPE_PMU_BUF_FAULT_ACT_OK; |
| 731 |
goto out_stop; |
| 732 |
default: |
| 733 |
err_str = "Unknown buffer status code"; |
| 734 |
} |
| 735 |
|
| 736 |
out_err: |
| 737 |
pr_err_ratelimited("%s on CPU %d [PMBSR=0x%016llx, PMBPTR=0x%016llx, PMBLIMITR=0x%016llx]\n", |
| 738 |
err_str, smp_processor_id(), pmbsr, |
| 739 |
read_sysreg_s(SYS_PMBPTR_EL1), |
| 740 |
read_sysreg_s(SYS_PMBLIMITR_EL1)); |
| 741 |
ret = SPE_PMU_BUF_FAULT_ACT_FATAL; |
| 742 |
|
| 743 |
out_stop: |
| 744 |
arm_spe_perf_aux_output_end(handle); |
| 745 |
return ret; |
| 746 |
} |
| 747 |
|
| 748 |
static irqreturn_t arm_spe_pmu_irq_handler(int irq, void *dev) |
| 749 |
{ |
| 750 |
struct perf_output_handle *handle = dev; |
| 751 |
struct perf_event *event = handle->event; |
| 752 |
enum arm_spe_pmu_buf_fault_action act; |
| 753 |
|
| 754 |
if (!perf_get_aux(handle)) |
| 755 |
return IRQ_NONE; |
| 756 |
|
| 757 |
act = arm_spe_pmu_buf_get_fault_act(handle); |
| 758 |
if (act == SPE_PMU_BUF_FAULT_ACT_SPURIOUS) |
| 759 |
return IRQ_NONE; |
| 760 |
|
| 761 |
/* |
| 762 |
* Ensure perf callbacks have completed, which may disable the |
| 763 |
* profiling buffer in response to a TRUNCATION flag. |
| 764 |
*/ |
| 765 |
irq_work_run(); |
| 766 |
|
| 767 |
switch (act) { |
| 768 |
case SPE_PMU_BUF_FAULT_ACT_FATAL: |
| 769 |
/* |
| 770 |
* If a fatal exception occurred then leaving the profiling |
| 771 |
* buffer enabled is a recipe waiting to happen. Since |
| 772 |
* fatal faults don't always imply truncation, make sure |
| 773 |
* that the profiling buffer is disabled explicitly before |
| 774 |
* clearing the syndrome register. |
| 775 |
*/ |
| 776 |
arm_spe_pmu_disable_and_drain_local(); |
| 777 |
break; |
| 778 |
case SPE_PMU_BUF_FAULT_ACT_OK: |
| 779 |
/* |
| 780 |
* We handled the fault (the buffer was full), so resume |
| 781 |
* profiling as long as we didn't detect truncation. |
| 782 |
* PMBPTR might be misaligned, but we'll burn that bridge |
| 783 |
* when we get to it. |
| 784 |
*/ |
| 785 |
if (!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED)) { |
| 786 |
if (arm_spe_perf_aux_output_begin(handle, event)) { |
| 787 |
arm_spe_pmu_stop(event, PERF_EF_UPDATE); |
| 788 |
break; |
| 789 |
} |
| 790 |
isb(); |
| 791 |
} |
| 792 |
break; |
| 793 |
case SPE_PMU_BUF_FAULT_ACT_SPURIOUS: |
| 794 |
/* We've seen you before, but GCC has the memory of a sieve. */ |
| 795 |
break; |
| 796 |
} |
| 797 |
|
| 798 |
/* The buffer pointers are now sane, so resume profiling. */ |
| 799 |
write_sysreg_s(0, SYS_PMBSR_EL1); |
| 800 |
return IRQ_HANDLED; |
| 801 |
} |
| 802 |
|
| 803 |
/* Perf callbacks */ |
| 804 |
static int arm_spe_pmu_event_init(struct perf_event *event) |
| 805 |
{ |
| 806 |
u64 reg; |
| 807 |
struct perf_event_attr *attr = &event->attr; |
| 808 |
struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu); |
| 809 |
|
| 810 |
/* This is, of course, deeply driver-specific */ |
| 811 |
if (attr->type != event->pmu->type) |
| 812 |
return -ENOENT; |
| 813 |
|
| 814 |
if (event->cpu >= 0 && |
| 815 |
!cpumask_test_cpu(event->cpu, &spe_pmu->supported_cpus)) |
| 816 |
return -ENOENT; |
| 817 |
|
| 818 |
if (arm_spe_event_to_pmsevfr(event) & spe_pmu->pmsevfr_res0) |
| 819 |
return -EOPNOTSUPP; |
| 820 |
|
| 821 |
if (arm_spe_event_to_pmsnevfr(event) & spe_pmu->pmsevfr_res0) |
| 822 |
return -EOPNOTSUPP; |
| 823 |
|
| 824 |
if (arm_spe_event_to_pmsdsfr(event) != U64_MAX && |
| 825 |
!(spe_pmu->features & SPE_PMU_FEAT_FDS)) |
| 826 |
return -EOPNOTSUPP; |
| 827 |
|
| 828 |
if (attr->exclude_idle) |
| 829 |
return -EOPNOTSUPP; |
| 830 |
|
| 831 |
/* |
| 832 |
* Feedback-directed frequency throttling doesn't work when we |
| 833 |
* have a buffer of samples. We'd need to manually count the |
| 834 |
* samples in the buffer when it fills up and adjust the event |
| 835 |
* count to reflect that. Instead, just force the user to specify |
| 836 |
* a sample period. |
| 837 |
*/ |
| 838 |
if (attr->freq) |
| 839 |
return -EINVAL; |
| 840 |
|
| 841 |
reg = arm_spe_event_to_pmsfcr(event); |
| 842 |
if ((FIELD_GET(PMSFCR_EL1_FE, reg)) && |
| 843 |
!(spe_pmu->features & SPE_PMU_FEAT_FILT_EVT)) |
| 844 |
return -EOPNOTSUPP; |
| 845 |
|
| 846 |
if ((FIELD_GET(PMSFCR_EL1_FnE, reg)) && |
| 847 |
!(spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT)) |
| 848 |
return -EOPNOTSUPP; |
| 849 |
|
| 850 |
if ((FIELD_GET(PMSFCR_EL1_FT, reg)) && |
| 851 |
!(spe_pmu->features & SPE_PMU_FEAT_FILT_TYP)) |
| 852 |
return -EOPNOTSUPP; |
| 853 |
|
| 854 |
if ((FIELD_GET(PMSFCR_EL1_FL, reg)) && |
| 855 |
!(spe_pmu->features & SPE_PMU_FEAT_FILT_LAT)) |
| 856 |
return -EOPNOTSUPP; |
| 857 |
|
| 858 |
if ((FIELD_GET(PMSFCR_EL1_LDm, reg) || |
| 859 |
FIELD_GET(PMSFCR_EL1_STm, reg) || |
| 860 |
FIELD_GET(PMSFCR_EL1_Bm, reg) || |
| 861 |
FIELD_GET(PMSFCR_EL1_SIMD, reg) || |
| 862 |
FIELD_GET(PMSFCR_EL1_SIMDm, reg) || |
| 863 |
FIELD_GET(PMSFCR_EL1_FP, reg) || |
| 864 |
FIELD_GET(PMSFCR_EL1_FPm, reg)) && |
| 865 |
!(spe_pmu->features & SPE_PMU_FEAT_EFT)) |
| 866 |
return -EOPNOTSUPP; |
| 867 |
|
| 868 |
if (ATTR_CFG_GET_FLD(&event->attr, discard) && |
| 869 |
!(spe_pmu->features & SPE_PMU_FEAT_DISCARD)) |
| 870 |
return -EOPNOTSUPP; |
| 871 |
|
| 872 |
set_spe_event_has_cx(event); |
| 873 |
reg = arm_spe_event_to_pmscr(event); |
| 874 |
if (reg & (PMSCR_EL1_PA | PMSCR_EL1_PCT)) |
| 875 |
return perf_allow_kernel(); |
| 876 |
|
| 877 |
return 0; |
| 878 |
} |
| 879 |
|
| 880 |
static void arm_spe_pmu_start(struct perf_event *event, int flags) |
| 881 |
{ |
| 882 |
u64 reg; |
| 883 |
struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu); |
| 884 |
struct hw_perf_event *hwc = &event->hw; |
| 885 |
struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle); |
| 886 |
|
| 887 |
hwc->state = 0; |
| 888 |
if (arm_spe_perf_aux_output_begin(handle, event)) { |
| 889 |
arm_spe_pmu_stop(event, 0); |
| 890 |
return; |
| 891 |
} |
| 892 |
|
| 893 |
reg = arm_spe_event_to_pmsfcr(event); |
| 894 |
write_sysreg_s(reg, SYS_PMSFCR_EL1); |
| 895 |
|
| 896 |
reg = arm_spe_event_to_pmsevfr(event); |
| 897 |
write_sysreg_s(reg, SYS_PMSEVFR_EL1); |
| 898 |
|
| 899 |
if (spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT) { |
| 900 |
reg = arm_spe_event_to_pmsnevfr(event); |
| 901 |
write_sysreg_s(reg, SYS_PMSNEVFR_EL1); |
| 902 |
} |
| 903 |
|
| 904 |
if (spe_pmu->features & SPE_PMU_FEAT_FDS) { |
| 905 |
reg = arm_spe_event_to_pmsdsfr(event); |
| 906 |
write_sysreg_s(reg, SYS_PMSDSFR_EL1); |
| 907 |
} |
| 908 |
|
| 909 |
reg = arm_spe_event_to_pmslatfr(event); |
| 910 |
write_sysreg_s(reg, SYS_PMSLATFR_EL1); |
| 911 |
|
| 912 |
if (flags & PERF_EF_RELOAD) { |
| 913 |
reg = arm_spe_event_to_pmsirr(event); |
| 914 |
write_sysreg_s(reg, SYS_PMSIRR_EL1); |
| 915 |
isb(); |
| 916 |
reg = local64_read(&hwc->period_left); |
| 917 |
write_sysreg_s(reg, SYS_PMSICR_EL1); |
| 918 |
} |
| 919 |
|
| 920 |
reg = arm_spe_event_to_pmscr(event); |
| 921 |
isb(); |
| 922 |
write_sysreg_s(reg, SYS_PMSCR_EL1); |
| 923 |
} |
| 924 |
|
| 925 |
static void arm_spe_pmu_stop(struct perf_event *event, int flags) |
| 926 |
{ |
| 927 |
struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu); |
| 928 |
struct hw_perf_event *hwc = &event->hw; |
| 929 |
struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle); |
| 930 |
|
| 931 |
/* If we're already stopped, then nothing to do */ |
| 932 |
if (hwc->state & PERF_HES_STOPPED) |
| 933 |
return; |
| 934 |
|
| 935 |
/* Stop all trace generation */ |
| 936 |
arm_spe_pmu_disable_and_drain_local(); |
| 937 |
|
| 938 |
if (flags & PERF_EF_UPDATE) { |
| 939 |
/* |
| 940 |
* If there's a fault pending then ensure we contain it |
| 941 |
* to this buffer, since we might be on the context-switch |
| 942 |
* path. |
| 943 |
*/ |
| 944 |
if (perf_get_aux(handle)) { |
| 945 |
enum arm_spe_pmu_buf_fault_action act; |
| 946 |
|
| 947 |
act = arm_spe_pmu_buf_get_fault_act(handle); |
| 948 |
if (act == SPE_PMU_BUF_FAULT_ACT_SPURIOUS) |
| 949 |
arm_spe_perf_aux_output_end(handle); |
| 950 |
else |
| 951 |
write_sysreg_s(0, SYS_PMBSR_EL1); |
| 952 |
} |
| 953 |
|
| 954 |
/* |
| 955 |
* This may also contain ECOUNT, but nobody else should |
| 956 |
* be looking at period_left, since we forbid frequency |
| 957 |
* based sampling. |
| 958 |
*/ |
| 959 |
local64_set(&hwc->period_left, read_sysreg_s(SYS_PMSICR_EL1)); |
| 960 |
hwc->state |= PERF_HES_UPTODATE; |
| 961 |
} |
| 962 |
|
| 963 |
hwc->state |= PERF_HES_STOPPED; |
| 964 |
} |
| 965 |
|
| 966 |
static int arm_spe_pmu_add(struct perf_event *event, int flags) |
| 967 |
{ |
| 968 |
int ret = 0; |
| 969 |
struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu); |
| 970 |
struct hw_perf_event *hwc = &event->hw; |
| 971 |
int cpu = event->cpu == -1 ? smp_processor_id() : event->cpu; |
| 972 |
|
| 973 |
if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus)) |
| 974 |
return -ENOENT; |
| 975 |
|
| 976 |
hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; |
| 977 |
|
| 978 |
if (flags & PERF_EF_START) { |
| 979 |
arm_spe_pmu_start(event, PERF_EF_RELOAD); |
| 980 |
if (hwc->state & PERF_HES_STOPPED) |
| 981 |
ret = -EINVAL; |
| 982 |
} |
| 983 |
|
| 984 |
return ret; |
| 985 |
} |
| 986 |
|
| 987 |
static void arm_spe_pmu_del(struct perf_event *event, int flags) |
| 988 |
{ |
| 989 |
arm_spe_pmu_stop(event, PERF_EF_UPDATE); |
| 990 |
} |
| 991 |
|
| 992 |
static void arm_spe_pmu_read(struct perf_event *event) |
| 993 |
{ |
| 994 |
} |
| 995 |
|
| 996 |
static void *arm_spe_pmu_setup_aux(struct perf_event *event, void **pages, |
| 997 |
int nr_pages, bool snapshot) |
| 998 |
{ |
| 999 |
int i, cpu = event->cpu; |
| 1000 |
struct page **pglist; |
| 1001 |
struct arm_spe_pmu_buf *buf; |
| 1002 |
|
| 1003 |
/* We need at least two pages for this to work. */ |
| 1004 |
if (nr_pages < 2) |
| 1005 |
return NULL; |
| 1006 |
|
| 1007 |
/* |
| 1008 |
* We require an even number of pages for snapshot mode, so that |
| 1009 |
* we can effectively treat the buffer as consisting of two equal |
| 1010 |
* parts and give userspace a fighting chance of getting some |
| 1011 |
* useful data out of it. |
| 1012 |
*/ |
| 1013 |
if (snapshot && (nr_pages & 1)) |
| 1014 |
return NULL; |
| 1015 |
|
| 1016 |
if (cpu == -1) |
| 1017 |
cpu = raw_smp_processor_id(); |
| 1018 |
|
| 1019 |
buf = kzalloc_node(sizeof(*buf), GFP_KERNEL, cpu_to_node(cpu)); |
| 1020 |
if (!buf) |
| 1021 |
return NULL; |
| 1022 |
|
| 1023 |
pglist = kzalloc_objs(*pglist, nr_pages); |
| 1024 |
if (!pglist) |
| 1025 |
goto out_free_buf; |
| 1026 |
|
| 1027 |
for (i = 0; i < nr_pages; ++i) |
| 1028 |
pglist[i] = virt_to_page(pages[i]); |
| 1029 |
|
| 1030 |
buf->base = vmap(pglist, nr_pages, VM_MAP, PAGE_KERNEL); |
| 1031 |
if (!buf->base) |
| 1032 |
goto out_free_pglist; |
| 1033 |
|
| 1034 |
buf->nr_pages = nr_pages; |
| 1035 |
buf->snapshot = snapshot; |
| 1036 |
|
| 1037 |
kfree(pglist); |
| 1038 |
return buf; |
| 1039 |
|
| 1040 |
out_free_pglist: |
| 1041 |
kfree(pglist); |
| 1042 |
out_free_buf: |
| 1043 |
kfree(buf); |
| 1044 |
return NULL; |
| 1045 |
} |
| 1046 |
|
| 1047 |
static void arm_spe_pmu_free_aux(void *aux) |
| 1048 |
{ |
| 1049 |
struct arm_spe_pmu_buf *buf = aux; |
| 1050 |
|
| 1051 |
vunmap(buf->base); |
| 1052 |
kfree(buf); |
| 1053 |
} |
| 1054 |
|
| 1055 |
/* Initialisation and teardown functions */ |
| 1056 |
static int arm_spe_pmu_perf_init(struct arm_spe_pmu *spe_pmu) |
| 1057 |
{ |
| 1058 |
static atomic_t pmu_idx = ATOMIC_INIT(-1); |
| 1059 |
|
| 1060 |
int idx; |
| 1061 |
char *name; |
| 1062 |
struct device *dev = &spe_pmu->pdev->dev; |
| 1063 |
|
| 1064 |
spe_pmu->pmu = (struct pmu) { |
| 1065 |
.module = THIS_MODULE, |
| 1066 |
.parent = &spe_pmu->pdev->dev, |
| 1067 |
.capabilities = PERF_PMU_CAP_EXCLUSIVE | PERF_PMU_CAP_ITRACE, |
| 1068 |
.attr_groups = arm_spe_pmu_attr_groups, |
| 1069 |
/* |
| 1070 |
* We hitch a ride on the software context here, so that |
| 1071 |
* we can support per-task profiling (which is not possible |
| 1072 |
* with the invalid context as it doesn't get sched callbacks). |
| 1073 |
* This requires that userspace either uses a dummy event for |
| 1074 |
* perf_event_open, since the aux buffer is not setup until |
| 1075 |
* a subsequent mmap, or creates the profiling event in a |
| 1076 |
* disabled state and explicitly PERF_EVENT_IOC_ENABLEs it |
| 1077 |
* once the buffer has been created. |
| 1078 |
*/ |
| 1079 |
.task_ctx_nr = perf_sw_context, |
| 1080 |
.event_init = arm_spe_pmu_event_init, |
| 1081 |
.add = arm_spe_pmu_add, |
| 1082 |
.del = arm_spe_pmu_del, |
| 1083 |
.start = arm_spe_pmu_start, |
| 1084 |
.stop = arm_spe_pmu_stop, |
| 1085 |
.read = arm_spe_pmu_read, |
| 1086 |
.setup_aux = arm_spe_pmu_setup_aux, |
| 1087 |
.free_aux = arm_spe_pmu_free_aux, |
| 1088 |
}; |
| 1089 |
|
| 1090 |
idx = atomic_inc_return(&pmu_idx); |
| 1091 |
name = devm_kasprintf(dev, GFP_KERNEL, "%s_%d", PMUNAME, idx); |
| 1092 |
if (!name) { |
| 1093 |
dev_err(dev, "failed to allocate name for pmu %d\n", idx); |
| 1094 |
return -ENOMEM; |
| 1095 |
} |
| 1096 |
|
| 1097 |
return perf_pmu_register(&spe_pmu->pmu, name, -1); |
| 1098 |
} |
| 1099 |
|
| 1100 |
static void arm_spe_pmu_perf_destroy(struct arm_spe_pmu *spe_pmu) |
| 1101 |
{ |
| 1102 |
perf_pmu_unregister(&spe_pmu->pmu); |
| 1103 |
} |
| 1104 |
|
| 1105 |
static void __arm_spe_pmu_dev_probe(void *info) |
| 1106 |
{ |
| 1107 |
int fld; |
| 1108 |
u64 reg; |
| 1109 |
struct arm_spe_pmu *spe_pmu = info; |
| 1110 |
struct device *dev = &spe_pmu->pdev->dev; |
| 1111 |
|
| 1112 |
fld = cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64DFR0_EL1), |
| 1113 |
ID_AA64DFR0_EL1_PMSVer_SHIFT); |
| 1114 |
if (!fld) { |
| 1115 |
dev_err(dev, |
| 1116 |
"unsupported ID_AA64DFR0_EL1.PMSVer [%d] on CPU %d\n", |
| 1117 |
fld, smp_processor_id()); |
| 1118 |
return; |
| 1119 |
} |
| 1120 |
spe_pmu->pmsver = (u16)fld; |
| 1121 |
|
| 1122 |
/* Read PMBIDR first to determine whether or not we have access */ |
| 1123 |
reg = read_sysreg_s(SYS_PMBIDR_EL1); |
| 1124 |
if (FIELD_GET(PMBIDR_EL1_P, reg)) { |
| 1125 |
dev_err(dev, |
| 1126 |
"profiling buffer owned by higher exception level\n"); |
| 1127 |
return; |
| 1128 |
} |
| 1129 |
|
| 1130 |
/* Minimum alignment. If it's out-of-range, then fail the probe */ |
| 1131 |
fld = FIELD_GET(PMBIDR_EL1_ALIGN, reg); |
| 1132 |
spe_pmu->align = 1 << fld; |
| 1133 |
if (spe_pmu->align > SZ_2K) { |
| 1134 |
dev_err(dev, "unsupported PMBIDR.Align [%d] on CPU %d\n", |
| 1135 |
fld, smp_processor_id()); |
| 1136 |
return; |
| 1137 |
} |
| 1138 |
|
| 1139 |
/* It's now safe to read PMSIDR and figure out what we've got */ |
| 1140 |
reg = read_sysreg_s(SYS_PMSIDR_EL1); |
| 1141 |
if (FIELD_GET(PMSIDR_EL1_FE, reg)) |
| 1142 |
spe_pmu->features |= SPE_PMU_FEAT_FILT_EVT; |
| 1143 |
|
| 1144 |
if (FIELD_GET(PMSIDR_EL1_FnE, reg)) |
| 1145 |
spe_pmu->features |= SPE_PMU_FEAT_INV_FILT_EVT; |
| 1146 |
|
| 1147 |
if (FIELD_GET(PMSIDR_EL1_FT, reg)) |
| 1148 |
spe_pmu->features |= SPE_PMU_FEAT_FILT_TYP; |
| 1149 |
|
| 1150 |
if (FIELD_GET(PMSIDR_EL1_FL, reg)) |
| 1151 |
spe_pmu->features |= SPE_PMU_FEAT_FILT_LAT; |
| 1152 |
|
| 1153 |
if (FIELD_GET(PMSIDR_EL1_ARCHINST, reg)) |
| 1154 |
spe_pmu->features |= SPE_PMU_FEAT_ARCH_INST; |
| 1155 |
|
| 1156 |
if (FIELD_GET(PMSIDR_EL1_LDS, reg)) |
| 1157 |
spe_pmu->features |= SPE_PMU_FEAT_LDS; |
| 1158 |
|
| 1159 |
if (FIELD_GET(PMSIDR_EL1_ERND, reg)) |
| 1160 |
spe_pmu->features |= SPE_PMU_FEAT_ERND; |
| 1161 |
|
| 1162 |
if (spe_pmu->pmsver >= ID_AA64DFR0_EL1_PMSVer_V1P2) |
| 1163 |
spe_pmu->features |= SPE_PMU_FEAT_DISCARD; |
| 1164 |
|
| 1165 |
if (FIELD_GET(PMSIDR_EL1_EFT, reg)) |
| 1166 |
spe_pmu->features |= SPE_PMU_FEAT_EFT; |
| 1167 |
|
| 1168 |
if (FIELD_GET(PMSIDR_EL1_FDS, reg)) |
| 1169 |
spe_pmu->features |= SPE_PMU_FEAT_FDS; |
| 1170 |
|
| 1171 |
/* This field has a spaced out encoding, so just use a look-up */ |
| 1172 |
fld = FIELD_GET(PMSIDR_EL1_INTERVAL, reg); |
| 1173 |
switch (fld) { |
| 1174 |
case PMSIDR_EL1_INTERVAL_256: |
| 1175 |
spe_pmu->min_period = 256; |
| 1176 |
break; |
| 1177 |
case PMSIDR_EL1_INTERVAL_512: |
| 1178 |
spe_pmu->min_period = 512; |
| 1179 |
break; |
| 1180 |
case PMSIDR_EL1_INTERVAL_768: |
| 1181 |
spe_pmu->min_period = 768; |
| 1182 |
break; |
| 1183 |
case PMSIDR_EL1_INTERVAL_1024: |
| 1184 |
spe_pmu->min_period = 1024; |
| 1185 |
break; |
| 1186 |
case PMSIDR_EL1_INTERVAL_1536: |
| 1187 |
spe_pmu->min_period = 1536; |
| 1188 |
break; |
| 1189 |
case PMSIDR_EL1_INTERVAL_2048: |
| 1190 |
spe_pmu->min_period = 2048; |
| 1191 |
break; |
| 1192 |
case PMSIDR_EL1_INTERVAL_3072: |
| 1193 |
spe_pmu->min_period = 3072; |
| 1194 |
break; |
| 1195 |
default: |
| 1196 |
dev_warn(dev, "unknown PMSIDR_EL1.Interval [%d]; assuming 8\n", |
| 1197 |
fld); |
| 1198 |
fallthrough; |
| 1199 |
case PMSIDR_EL1_INTERVAL_4096: |
| 1200 |
spe_pmu->min_period = 4096; |
| 1201 |
} |
| 1202 |
|
| 1203 |
/* Maximum record size. If it's out-of-range, then fail the probe */ |
| 1204 |
fld = FIELD_GET(PMSIDR_EL1_MAXSIZE, reg); |
| 1205 |
spe_pmu->max_record_sz = 1 << fld; |
| 1206 |
if (spe_pmu->max_record_sz > SZ_2K || spe_pmu->max_record_sz < 16) { |
| 1207 |
dev_err(dev, "unsupported PMSIDR_EL1.MaxSize [%d] on CPU %d\n", |
| 1208 |
fld, smp_processor_id()); |
| 1209 |
return; |
| 1210 |
} |
| 1211 |
|
| 1212 |
fld = FIELD_GET(PMSIDR_EL1_COUNTSIZE, reg); |
| 1213 |
switch (fld) { |
| 1214 |
default: |
| 1215 |
dev_warn(dev, "unknown PMSIDR_EL1.CountSize [%d]; assuming 2\n", |
| 1216 |
fld); |
| 1217 |
fallthrough; |
| 1218 |
case PMSIDR_EL1_COUNTSIZE_12_BIT_SAT: |
| 1219 |
spe_pmu->counter_sz = 12; |
| 1220 |
break; |
| 1221 |
case PMSIDR_EL1_COUNTSIZE_16_BIT_SAT: |
| 1222 |
spe_pmu->counter_sz = 16; |
| 1223 |
} |
| 1224 |
|
| 1225 |
/* Write all 1s and then read back. Unsupported filter bits are RAZ/WI. */ |
| 1226 |
write_sysreg_s(U64_MAX, SYS_PMSEVFR_EL1); |
| 1227 |
spe_pmu->pmsevfr_res0 = ~read_sysreg_s(SYS_PMSEVFR_EL1); |
| 1228 |
|
| 1229 |
dev_info(dev, |
| 1230 |
"probed SPEv1.%d for CPUs %*pbl [max_record_sz %u, align %u, features 0x%llx]\n", |
| 1231 |
spe_pmu->pmsver - 1, cpumask_pr_args(&spe_pmu->supported_cpus), |
| 1232 |
spe_pmu->max_record_sz, spe_pmu->align, spe_pmu->features); |
| 1233 |
|
| 1234 |
spe_pmu->features |= SPE_PMU_FEAT_DEV_PROBED; |
| 1235 |
} |
| 1236 |
|
| 1237 |
static void __arm_spe_pmu_reset_local(void) |
| 1238 |
{ |
| 1239 |
/* |
| 1240 |
* This is probably overkill, as we have no idea where we're |
| 1241 |
* draining any buffered data to... |
| 1242 |
*/ |
| 1243 |
arm_spe_pmu_disable_and_drain_local(); |
| 1244 |
|
| 1245 |
/* Reset the buffer base pointer */ |
| 1246 |
write_sysreg_s(0, SYS_PMBPTR_EL1); |
| 1247 |
isb(); |
| 1248 |
|
| 1249 |
/* Clear any pending management interrupts */ |
| 1250 |
write_sysreg_s(0, SYS_PMBSR_EL1); |
| 1251 |
isb(); |
| 1252 |
} |
| 1253 |
|
| 1254 |
static void __arm_spe_pmu_setup_one(void *info) |
| 1255 |
{ |
| 1256 |
struct arm_spe_pmu *spe_pmu = info; |
| 1257 |
|
| 1258 |
__arm_spe_pmu_reset_local(); |
| 1259 |
enable_percpu_irq(spe_pmu->irq, IRQ_TYPE_NONE); |
| 1260 |
} |
| 1261 |
|
| 1262 |
static void __arm_spe_pmu_stop_one(void *info) |
| 1263 |
{ |
| 1264 |
struct arm_spe_pmu *spe_pmu = info; |
| 1265 |
|
| 1266 |
disable_percpu_irq(spe_pmu->irq); |
| 1267 |
__arm_spe_pmu_reset_local(); |
| 1268 |
} |
| 1269 |
|
| 1270 |
static int arm_spe_pmu_cpu_startup(unsigned int cpu, struct hlist_node *node) |
| 1271 |
{ |
| 1272 |
struct arm_spe_pmu *spe_pmu; |
| 1273 |
|
| 1274 |
spe_pmu = hlist_entry_safe(node, struct arm_spe_pmu, hotplug_node); |
| 1275 |
if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus)) |
| 1276 |
return 0; |
| 1277 |
|
| 1278 |
__arm_spe_pmu_setup_one(spe_pmu); |
| 1279 |
return 0; |
| 1280 |
} |
| 1281 |
|
| 1282 |
static int arm_spe_pmu_cpu_teardown(unsigned int cpu, struct hlist_node *node) |
| 1283 |
{ |
| 1284 |
struct arm_spe_pmu *spe_pmu; |
| 1285 |
|
| 1286 |
spe_pmu = hlist_entry_safe(node, struct arm_spe_pmu, hotplug_node); |
| 1287 |
if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus)) |
| 1288 |
return 0; |
| 1289 |
|
| 1290 |
__arm_spe_pmu_stop_one(spe_pmu); |
| 1291 |
return 0; |
| 1292 |
} |
| 1293 |
|
| 1294 |
static int arm_spe_pmu_dev_init(struct arm_spe_pmu *spe_pmu) |
| 1295 |
{ |
| 1296 |
int ret; |
| 1297 |
cpumask_t *mask = &spe_pmu->supported_cpus; |
| 1298 |
|
| 1299 |
/* Make sure we probe the hardware on a relevant CPU */ |
| 1300 |
ret = smp_call_function_any(mask, __arm_spe_pmu_dev_probe, spe_pmu, 1); |
| 1301 |
if (ret || !(spe_pmu->features & SPE_PMU_FEAT_DEV_PROBED)) |
| 1302 |
return -ENXIO; |
| 1303 |
|
| 1304 |
/* Request our PPIs (note that the IRQ is still disabled) */ |
| 1305 |
ret = request_percpu_irq_affinity(spe_pmu->irq, arm_spe_pmu_irq_handler, |
| 1306 |
DRVNAME, mask, spe_pmu->handle); |
| 1307 |
if (ret) |
| 1308 |
return ret; |
| 1309 |
|
| 1310 |
/* |
| 1311 |
* Register our hotplug notifier now so we don't miss any events. |
| 1312 |
* This will enable the IRQ for any supported CPUs that are already |
| 1313 |
* up. |
| 1314 |
*/ |
| 1315 |
ret = cpuhp_state_add_instance(arm_spe_pmu_online, |
| 1316 |
&spe_pmu->hotplug_node); |
| 1317 |
if (ret) |
| 1318 |
free_percpu_irq(spe_pmu->irq, spe_pmu->handle); |
| 1319 |
|
| 1320 |
return ret; |
| 1321 |
} |
| 1322 |
|
| 1323 |
static void arm_spe_pmu_dev_teardown(struct arm_spe_pmu *spe_pmu) |
| 1324 |
{ |
| 1325 |
cpuhp_state_remove_instance(arm_spe_pmu_online, &spe_pmu->hotplug_node); |
| 1326 |
free_percpu_irq(spe_pmu->irq, spe_pmu->handle); |
| 1327 |
} |
| 1328 |
|
| 1329 |
/* Driver and device probing */ |
| 1330 |
static int arm_spe_pmu_irq_probe(struct arm_spe_pmu *spe_pmu) |
| 1331 |
{ |
| 1332 |
struct platform_device *pdev = spe_pmu->pdev; |
| 1333 |
const struct cpumask *affinity; |
| 1334 |
int irq; |
| 1335 |
|
| 1336 |
irq = platform_get_irq_affinity(pdev, 0, &affinity); |
| 1337 |
if (irq < 0) |
| 1338 |
return -ENXIO; |
| 1339 |
|
| 1340 |
if (!irq_is_percpu(irq)) { |
| 1341 |
dev_err(&pdev->dev, "expected PPI but got SPI (%d)\n", irq); |
| 1342 |
return -EINVAL; |
| 1343 |
} |
| 1344 |
|
| 1345 |
cpumask_copy(&spe_pmu->supported_cpus, affinity); |
| 1346 |
|
| 1347 |
spe_pmu->irq = irq; |
| 1348 |
return 0; |
| 1349 |
} |
| 1350 |
|
| 1351 |
static const struct of_device_id arm_spe_pmu_of_match[] = { |
| 1352 |
{ .compatible = "arm,statistical-profiling-extension-v1", .data = (void *)1 }, |
| 1353 |
{ /* Sentinel */ }, |
| 1354 |
}; |
| 1355 |
MODULE_DEVICE_TABLE(of, arm_spe_pmu_of_match); |
| 1356 |
|
| 1357 |
static const struct platform_device_id arm_spe_match[] = { |
| 1358 |
{ ARMV8_SPE_PDEV_NAME, 0}, |
| 1359 |
{ } |
| 1360 |
}; |
| 1361 |
MODULE_DEVICE_TABLE(platform, arm_spe_match); |
| 1362 |
|
| 1363 |
static int arm_spe_pmu_device_probe(struct platform_device *pdev) |
| 1364 |
{ |
| 1365 |
int ret; |
| 1366 |
struct arm_spe_pmu *spe_pmu; |
| 1367 |
struct device *dev = &pdev->dev; |
| 1368 |
|
| 1369 |
/* |
| 1370 |
* If kernelspace is unmapped when running at EL0, then the SPE |
| 1371 |
* buffer will fault and prematurely terminate the AUX session. |
| 1372 |
*/ |
| 1373 |
if (arm64_kernel_unmapped_at_el0()) { |
| 1374 |
dev_warn_once(dev, "profiling buffer inaccessible. Try passing \"kpti=off\" on the kernel command line\n"); |
| 1375 |
return -EPERM; |
| 1376 |
} |
| 1377 |
|
| 1378 |
spe_pmu = devm_kzalloc(dev, sizeof(*spe_pmu), GFP_KERNEL); |
| 1379 |
if (!spe_pmu) |
| 1380 |
return -ENOMEM; |
| 1381 |
|
| 1382 |
spe_pmu->handle = alloc_percpu(typeof(*spe_pmu->handle)); |
| 1383 |
if (!spe_pmu->handle) |
| 1384 |
return -ENOMEM; |
| 1385 |
|
| 1386 |
spe_pmu->pdev = pdev; |
| 1387 |
platform_set_drvdata(pdev, spe_pmu); |
| 1388 |
|
| 1389 |
ret = arm_spe_pmu_irq_probe(spe_pmu); |
| 1390 |
if (ret) |
| 1391 |
goto out_free_handle; |
| 1392 |
|
| 1393 |
ret = arm_spe_pmu_dev_init(spe_pmu); |
| 1394 |
if (ret) |
| 1395 |
goto out_free_handle; |
| 1396 |
|
| 1397 |
ret = arm_spe_pmu_perf_init(spe_pmu); |
| 1398 |
if (ret) |
| 1399 |
goto out_teardown_dev; |
| 1400 |
|
| 1401 |
return 0; |
| 1402 |
|
| 1403 |
out_teardown_dev: |
| 1404 |
arm_spe_pmu_dev_teardown(spe_pmu); |
| 1405 |
out_free_handle: |
| 1406 |
free_percpu(spe_pmu->handle); |
| 1407 |
return ret; |
| 1408 |
} |
| 1409 |
|
| 1410 |
static void arm_spe_pmu_device_remove(struct platform_device *pdev) |
| 1411 |
{ |
| 1412 |
struct arm_spe_pmu *spe_pmu = platform_get_drvdata(pdev); |
| 1413 |
|
| 1414 |
arm_spe_pmu_perf_destroy(spe_pmu); |
| 1415 |
arm_spe_pmu_dev_teardown(spe_pmu); |
| 1416 |
free_percpu(spe_pmu->handle); |
| 1417 |
} |
| 1418 |
|
| 1419 |
static struct platform_driver arm_spe_pmu_driver = { |
| 1420 |
.id_table = arm_spe_match, |
| 1421 |
.driver = { |
| 1422 |
.name = DRVNAME, |
| 1423 |
.of_match_table = of_match_ptr(arm_spe_pmu_of_match), |
| 1424 |
.suppress_bind_attrs = true, |
| 1425 |
}, |
| 1426 |
.probe = arm_spe_pmu_device_probe, |
| 1427 |
.remove = arm_spe_pmu_device_remove, |
| 1428 |
}; |
| 1429 |
|
| 1430 |
static int __init arm_spe_pmu_init(void) |
| 1431 |
{ |
| 1432 |
int ret; |
| 1433 |
|
| 1434 |
ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRVNAME, |
| 1435 |
arm_spe_pmu_cpu_startup, |
| 1436 |
arm_spe_pmu_cpu_teardown); |
| 1437 |
if (ret < 0) |
| 1438 |
return ret; |
| 1439 |
arm_spe_pmu_online = ret; |
| 1440 |
|
| 1441 |
ret = platform_driver_register(&arm_spe_pmu_driver); |
| 1442 |
if (ret) |
| 1443 |
cpuhp_remove_multi_state(arm_spe_pmu_online); |
| 1444 |
|
| 1445 |
return ret; |
| 1446 |
} |
| 1447 |
|
| 1448 |
static void __exit arm_spe_pmu_exit(void) |
| 1449 |
{ |
| 1450 |
platform_driver_unregister(&arm_spe_pmu_driver); |
| 1451 |
cpuhp_remove_multi_state(arm_spe_pmu_online); |
| 1452 |
} |
| 1453 |
|
| 1454 |
module_init(arm_spe_pmu_init); |
| 1455 |
module_exit(arm_spe_pmu_exit); |
| 1456 |
|
| 1457 |
MODULE_DESCRIPTION("Perf driver for the ARMv8.2 Statistical Profiling Extension"); |
| 1458 |
MODULE_AUTHOR("Will Deacon <[email protected]>"); |
| 1459 |
MODULE_LICENSE("GPL v2"); |