| 1 |
// SPDX-License-Identifier: GPL-2.0 |
| 2 |
|
| 3 |
/* |
| 4 |
* This driver adds support for perf events to use the Performance |
| 5 |
* Monitor Counter Groups (PMCG) associated with an SMMUv3 node |
| 6 |
* to monitor that node. |
| 7 |
* |
| 8 |
* SMMUv3 PMCG devices are named as smmuv3_pmcg_<phys_addr_page> where |
| 9 |
* <phys_addr_page> is the physical page address of the SMMU PMCG wrapped |
| 10 |
* to 4K boundary. For example, the PMCG at 0xff88840000 is named |
| 11 |
* smmuv3_pmcg_ff88840 |
| 12 |
* |
| 13 |
* Filtering by stream id is done by specifying filtering parameters |
| 14 |
* with the event. options are: |
| 15 |
* filter_enable - 0 = no filtering, 1 = filtering enabled |
| 16 |
* filter_span - 0 = exact match, 1 = pattern match |
| 17 |
* filter_stream_id - pattern to filter against |
| 18 |
* |
| 19 |
* To match a partial StreamID where the X most-significant bits must match |
| 20 |
* but the Y least-significant bits might differ, STREAMID is programmed |
| 21 |
* with a value that contains: |
| 22 |
* STREAMID[Y - 1] == 0. |
| 23 |
* STREAMID[Y - 2:0] == 1 (where Y > 1). |
| 24 |
* The remainder of implemented bits of STREAMID (X bits, from bit Y upwards) |
| 25 |
* contain a value to match from the corresponding bits of event StreamID. |
| 26 |
* |
| 27 |
* Example: perf stat -e smmuv3_pmcg_ff88840/transaction,filter_enable=1, |
| 28 |
* filter_span=1,filter_stream_id=0x42/ -a netperf |
| 29 |
* Applies filter pattern 0x42 to transaction events, which means events |
| 30 |
* matching stream ids 0x42 and 0x43 are counted. Further filtering |
| 31 |
* information is available in the SMMU documentation. |
| 32 |
* |
| 33 |
* SMMU events are not attributable to a CPU, so task mode and sampling |
| 34 |
* are not supported. |
| 35 |
*/ |
| 36 |
|
| 37 |
#include <linux/acpi.h> |
| 38 |
#include <linux/acpi_iort.h> |
| 39 |
#include <linux/bitfield.h> |
| 40 |
#include <linux/bitops.h> |
| 41 |
#include <linux/cpuhotplug.h> |
| 42 |
#include <linux/cpumask.h> |
| 43 |
#include <linux/device.h> |
| 44 |
#include <linux/errno.h> |
| 45 |
#include <linux/interrupt.h> |
| 46 |
#include <linux/irq.h> |
| 47 |
#include <linux/kernel.h> |
| 48 |
#include <linux/list.h> |
| 49 |
#include <linux/msi.h> |
| 50 |
#include <linux/of.h> |
| 51 |
#include <linux/perf_event.h> |
| 52 |
#include <linux/platform_device.h> |
| 53 |
#include <linux/smp.h> |
| 54 |
#include <linux/sysfs.h> |
| 55 |
#include <linux/types.h> |
| 56 |
|
| 57 |
#define SMMU_PMCG_EVCNTR0 0x0 |
| 58 |
#define SMMU_PMCG_EVCNTR(n, stride) (SMMU_PMCG_EVCNTR0 + (n) * (stride)) |
| 59 |
#define SMMU_PMCG_EVTYPER0 0x400 |
| 60 |
#define SMMU_PMCG_EVTYPER(n) (SMMU_PMCG_EVTYPER0 + (n) * 4) |
| 61 |
#define SMMU_PMCG_SID_SPAN_SHIFT 29 |
| 62 |
#define SMMU_PMCG_SMR0 0xA00 |
| 63 |
#define SMMU_PMCG_SMR(n) (SMMU_PMCG_SMR0 + (n) * 4) |
| 64 |
#define SMMU_PMCG_CNTENSET0 0xC00 |
| 65 |
#define SMMU_PMCG_CNTENCLR0 0xC20 |
| 66 |
#define SMMU_PMCG_INTENSET0 0xC40 |
| 67 |
#define SMMU_PMCG_INTENCLR0 0xC60 |
| 68 |
#define SMMU_PMCG_OVSCLR0 0xC80 |
| 69 |
#define SMMU_PMCG_OVSSET0 0xCC0 |
| 70 |
#define SMMU_PMCG_CFGR 0xE00 |
| 71 |
#define SMMU_PMCG_CFGR_SID_FILTER_TYPE BIT(23) |
| 72 |
#define SMMU_PMCG_CFGR_MSI BIT(21) |
| 73 |
#define SMMU_PMCG_CFGR_RELOC_CTRS BIT(20) |
| 74 |
#define SMMU_PMCG_CFGR_SIZE GENMASK(13, 8) |
| 75 |
#define SMMU_PMCG_CFGR_NCTR GENMASK(5, 0) |
| 76 |
#define SMMU_PMCG_CR 0xE04 |
| 77 |
#define SMMU_PMCG_CR_ENABLE BIT(0) |
| 78 |
#define SMMU_PMCG_IIDR 0xE08 |
| 79 |
#define SMMU_PMCG_IIDR_PRODUCTID GENMASK(31, 20) |
| 80 |
#define SMMU_PMCG_IIDR_VARIANT GENMASK(19, 16) |
| 81 |
#define SMMU_PMCG_IIDR_REVISION GENMASK(15, 12) |
| 82 |
#define SMMU_PMCG_IIDR_IMPLEMENTER GENMASK(11, 0) |
| 83 |
#define SMMU_PMCG_CEID0 0xE20 |
| 84 |
#define SMMU_PMCG_CEID1 0xE28 |
| 85 |
#define SMMU_PMCG_IRQ_CTRL 0xE50 |
| 86 |
#define SMMU_PMCG_IRQ_CTRL_IRQEN BIT(0) |
| 87 |
#define SMMU_PMCG_IRQ_CFG0 0xE58 |
| 88 |
#define SMMU_PMCG_IRQ_CFG1 0xE60 |
| 89 |
#define SMMU_PMCG_IRQ_CFG2 0xE64 |
| 90 |
|
| 91 |
/* IMP-DEF ID registers */ |
| 92 |
#define SMMU_PMCG_PIDR0 0xFE0 |
| 93 |
#define SMMU_PMCG_PIDR0_PART_0 GENMASK(7, 0) |
| 94 |
#define SMMU_PMCG_PIDR1 0xFE4 |
| 95 |
#define SMMU_PMCG_PIDR1_DES_0 GENMASK(7, 4) |
| 96 |
#define SMMU_PMCG_PIDR1_PART_1 GENMASK(3, 0) |
| 97 |
#define SMMU_PMCG_PIDR2 0xFE8 |
| 98 |
#define SMMU_PMCG_PIDR2_REVISION GENMASK(7, 4) |
| 99 |
#define SMMU_PMCG_PIDR2_DES_1 GENMASK(2, 0) |
| 100 |
#define SMMU_PMCG_PIDR3 0xFEC |
| 101 |
#define SMMU_PMCG_PIDR3_REVAND GENMASK(7, 4) |
| 102 |
#define SMMU_PMCG_PIDR4 0xFD0 |
| 103 |
#define SMMU_PMCG_PIDR4_DES_2 GENMASK(3, 0) |
| 104 |
|
| 105 |
/* MSI config fields */ |
| 106 |
#define MSI_CFG0_ADDR_MASK GENMASK_ULL(51, 2) |
| 107 |
#define MSI_CFG2_MEMATTR_DEVICE_nGnRE 0x1 |
| 108 |
|
| 109 |
#define SMMU_PMCG_DEFAULT_FILTER_SPAN 1 |
| 110 |
#define SMMU_PMCG_DEFAULT_FILTER_SID GENMASK(31, 0) |
| 111 |
|
| 112 |
#define SMMU_PMCG_MAX_COUNTERS 64 |
| 113 |
#define SMMU_PMCG_ARCH_MAX_EVENTS 128 |
| 114 |
|
| 115 |
#define SMMU_PMCG_PA_SHIFT 12 |
| 116 |
|
| 117 |
#define SMMU_PMCG_EVCNTR_RDONLY BIT(0) |
| 118 |
#define SMMU_PMCG_HARDEN_DISABLE BIT(1) |
| 119 |
|
| 120 |
static int cpuhp_state_num; |
| 121 |
|
| 122 |
struct smmu_pmu { |
| 123 |
struct hlist_node node; |
| 124 |
struct perf_event *events[SMMU_PMCG_MAX_COUNTERS]; |
| 125 |
DECLARE_BITMAP(used_counters, SMMU_PMCG_MAX_COUNTERS); |
| 126 |
DECLARE_BITMAP(supported_events, SMMU_PMCG_ARCH_MAX_EVENTS); |
| 127 |
unsigned int irq; |
| 128 |
unsigned int on_cpu; |
| 129 |
struct pmu pmu; |
| 130 |
unsigned int num_counters; |
| 131 |
struct device *dev; |
| 132 |
void __iomem *reg_base; |
| 133 |
void __iomem *reloc_base; |
| 134 |
u64 counter_mask; |
| 135 |
u32 options; |
| 136 |
u32 iidr; |
| 137 |
bool global_filter; |
| 138 |
}; |
| 139 |
|
| 140 |
#define to_smmu_pmu(p) (container_of(p, struct smmu_pmu, pmu)) |
| 141 |
|
| 142 |
#define SMMU_PMU_EVENT_ATTR_EXTRACTOR(_name, _config, _start, _end) \ |
| 143 |
static inline u32 get_##_name(struct perf_event *event) \ |
| 144 |
{ \ |
| 145 |
return FIELD_GET(GENMASK_ULL(_end, _start), \ |
| 146 |
event->attr._config); \ |
| 147 |
} \ |
| 148 |
|
| 149 |
SMMU_PMU_EVENT_ATTR_EXTRACTOR(event, config, 0, 15); |
| 150 |
SMMU_PMU_EVENT_ATTR_EXTRACTOR(filter_stream_id, config1, 0, 31); |
| 151 |
SMMU_PMU_EVENT_ATTR_EXTRACTOR(filter_span, config1, 32, 32); |
| 152 |
SMMU_PMU_EVENT_ATTR_EXTRACTOR(filter_enable, config1, 33, 33); |
| 153 |
|
| 154 |
static inline void smmu_pmu_enable(struct pmu *pmu) |
| 155 |
{ |
| 156 |
struct smmu_pmu *smmu_pmu = to_smmu_pmu(pmu); |
| 157 |
|
| 158 |
writel(SMMU_PMCG_IRQ_CTRL_IRQEN, |
| 159 |
smmu_pmu->reg_base + SMMU_PMCG_IRQ_CTRL); |
| 160 |
writel(SMMU_PMCG_CR_ENABLE, smmu_pmu->reg_base + SMMU_PMCG_CR); |
| 161 |
} |
| 162 |
|
| 163 |
static int smmu_pmu_apply_event_filter(struct smmu_pmu *smmu_pmu, |
| 164 |
struct perf_event *event, int idx); |
| 165 |
|
| 166 |
static inline void smmu_pmu_enable_quirk_hip08_09(struct pmu *pmu) |
| 167 |
{ |
| 168 |
struct smmu_pmu *smmu_pmu = to_smmu_pmu(pmu); |
| 169 |
unsigned int idx; |
| 170 |
|
| 171 |
for_each_set_bit(idx, smmu_pmu->used_counters, smmu_pmu->num_counters) |
| 172 |
smmu_pmu_apply_event_filter(smmu_pmu, smmu_pmu->events[idx], idx); |
| 173 |
|
| 174 |
smmu_pmu_enable(pmu); |
| 175 |
} |
| 176 |
|
| 177 |
static inline void smmu_pmu_disable(struct pmu *pmu) |
| 178 |
{ |
| 179 |
struct smmu_pmu *smmu_pmu = to_smmu_pmu(pmu); |
| 180 |
|
| 181 |
writel(0, smmu_pmu->reg_base + SMMU_PMCG_CR); |
| 182 |
writel(0, smmu_pmu->reg_base + SMMU_PMCG_IRQ_CTRL); |
| 183 |
} |
| 184 |
|
| 185 |
static inline void smmu_pmu_disable_quirk_hip08_09(struct pmu *pmu) |
| 186 |
{ |
| 187 |
struct smmu_pmu *smmu_pmu = to_smmu_pmu(pmu); |
| 188 |
unsigned int idx; |
| 189 |
|
| 190 |
/* |
| 191 |
* The global disable of PMU sometimes fail to stop the counting. |
| 192 |
* Harden this by writing an invalid event type to each used counter |
| 193 |
* to forcibly stop counting. |
| 194 |
*/ |
| 195 |
for_each_set_bit(idx, smmu_pmu->used_counters, smmu_pmu->num_counters) |
| 196 |
writel(0xffff, smmu_pmu->reg_base + SMMU_PMCG_EVTYPER(idx)); |
| 197 |
|
| 198 |
smmu_pmu_disable(pmu); |
| 199 |
} |
| 200 |
|
| 201 |
static inline void smmu_pmu_counter_set_value(struct smmu_pmu *smmu_pmu, |
| 202 |
u32 idx, u64 value) |
| 203 |
{ |
| 204 |
if (smmu_pmu->counter_mask & BIT(32)) |
| 205 |
writeq(value, smmu_pmu->reloc_base + SMMU_PMCG_EVCNTR(idx, 8)); |
| 206 |
else |
| 207 |
writel(value, smmu_pmu->reloc_base + SMMU_PMCG_EVCNTR(idx, 4)); |
| 208 |
} |
| 209 |
|
| 210 |
static inline u64 smmu_pmu_counter_get_value(struct smmu_pmu *smmu_pmu, u32 idx) |
| 211 |
{ |
| 212 |
u64 value; |
| 213 |
|
| 214 |
if (smmu_pmu->counter_mask & BIT(32)) |
| 215 |
value = readq(smmu_pmu->reloc_base + SMMU_PMCG_EVCNTR(idx, 8)); |
| 216 |
else |
| 217 |
value = readl(smmu_pmu->reloc_base + SMMU_PMCG_EVCNTR(idx, 4)); |
| 218 |
|
| 219 |
return value; |
| 220 |
} |
| 221 |
|
| 222 |
static inline void smmu_pmu_counter_enable(struct smmu_pmu *smmu_pmu, u32 idx) |
| 223 |
{ |
| 224 |
writeq(BIT(idx), smmu_pmu->reg_base + SMMU_PMCG_CNTENSET0); |
| 225 |
} |
| 226 |
|
| 227 |
static inline void smmu_pmu_counter_disable(struct smmu_pmu *smmu_pmu, u32 idx) |
| 228 |
{ |
| 229 |
writeq(BIT(idx), smmu_pmu->reg_base + SMMU_PMCG_CNTENCLR0); |
| 230 |
} |
| 231 |
|
| 232 |
static inline void smmu_pmu_interrupt_enable(struct smmu_pmu *smmu_pmu, u32 idx) |
| 233 |
{ |
| 234 |
writeq(BIT(idx), smmu_pmu->reg_base + SMMU_PMCG_INTENSET0); |
| 235 |
} |
| 236 |
|
| 237 |
static inline void smmu_pmu_interrupt_disable(struct smmu_pmu *smmu_pmu, |
| 238 |
u32 idx) |
| 239 |
{ |
| 240 |
writeq(BIT(idx), smmu_pmu->reg_base + SMMU_PMCG_INTENCLR0); |
| 241 |
} |
| 242 |
|
| 243 |
static inline void smmu_pmu_set_evtyper(struct smmu_pmu *smmu_pmu, u32 idx, |
| 244 |
u32 val) |
| 245 |
{ |
| 246 |
writel(val, smmu_pmu->reg_base + SMMU_PMCG_EVTYPER(idx)); |
| 247 |
} |
| 248 |
|
| 249 |
static inline void smmu_pmu_set_smr(struct smmu_pmu *smmu_pmu, u32 idx, u32 val) |
| 250 |
{ |
| 251 |
writel(val, smmu_pmu->reg_base + SMMU_PMCG_SMR(idx)); |
| 252 |
} |
| 253 |
|
| 254 |
static void smmu_pmu_event_update(struct perf_event *event) |
| 255 |
{ |
| 256 |
struct hw_perf_event *hwc = &event->hw; |
| 257 |
struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu); |
| 258 |
u64 delta, prev, now; |
| 259 |
u32 idx = hwc->idx; |
| 260 |
|
| 261 |
do { |
| 262 |
prev = local64_read(&hwc->prev_count); |
| 263 |
now = smmu_pmu_counter_get_value(smmu_pmu, idx); |
| 264 |
} while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev); |
| 265 |
|
| 266 |
/* handle overflow. */ |
| 267 |
delta = now - prev; |
| 268 |
delta &= smmu_pmu->counter_mask; |
| 269 |
|
| 270 |
local64_add(delta, &event->count); |
| 271 |
} |
| 272 |
|
| 273 |
static void smmu_pmu_set_period(struct smmu_pmu *smmu_pmu, |
| 274 |
struct hw_perf_event *hwc) |
| 275 |
{ |
| 276 |
u32 idx = hwc->idx; |
| 277 |
u64 new; |
| 278 |
|
| 279 |
if (smmu_pmu->options & SMMU_PMCG_EVCNTR_RDONLY) { |
| 280 |
/* |
| 281 |
* On platforms that require this quirk, if the counter starts |
| 282 |
* at < half_counter value and wraps, the current logic of |
| 283 |
* handling the overflow may not work. It is expected that, |
| 284 |
* those platforms will have full 64 counter bits implemented |
| 285 |
* so that such a possibility is remote(eg: HiSilicon HIP08). |
| 286 |
*/ |
| 287 |
new = smmu_pmu_counter_get_value(smmu_pmu, idx); |
| 288 |
} else { |
| 289 |
/* |
| 290 |
* We limit the max period to half the max counter value |
| 291 |
* of the counter size, so that even in the case of extreme |
| 292 |
* interrupt latency the counter will (hopefully) not wrap |
| 293 |
* past its initial value. |
| 294 |
*/ |
| 295 |
new = smmu_pmu->counter_mask >> 1; |
| 296 |
smmu_pmu_counter_set_value(smmu_pmu, idx, new); |
| 297 |
} |
| 298 |
|
| 299 |
local64_set(&hwc->prev_count, new); |
| 300 |
} |
| 301 |
|
| 302 |
static void smmu_pmu_set_event_filter(struct perf_event *event, |
| 303 |
int idx, u32 span, u32 sid) |
| 304 |
{ |
| 305 |
struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu); |
| 306 |
u32 evtyper; |
| 307 |
|
| 308 |
evtyper = get_event(event) | span << SMMU_PMCG_SID_SPAN_SHIFT; |
| 309 |
smmu_pmu_set_evtyper(smmu_pmu, idx, evtyper); |
| 310 |
smmu_pmu_set_smr(smmu_pmu, idx, sid); |
| 311 |
} |
| 312 |
|
| 313 |
static bool smmu_pmu_check_global_filter(struct perf_event *curr, |
| 314 |
struct perf_event *new) |
| 315 |
{ |
| 316 |
if (get_filter_enable(new) != get_filter_enable(curr)) |
| 317 |
return false; |
| 318 |
|
| 319 |
if (!get_filter_enable(new)) |
| 320 |
return true; |
| 321 |
|
| 322 |
return get_filter_span(new) == get_filter_span(curr) && |
| 323 |
get_filter_stream_id(new) == get_filter_stream_id(curr); |
| 324 |
} |
| 325 |
|
| 326 |
static int smmu_pmu_apply_event_filter(struct smmu_pmu *smmu_pmu, |
| 327 |
struct perf_event *event, int idx) |
| 328 |
{ |
| 329 |
u32 span, sid; |
| 330 |
unsigned int cur_idx, num_ctrs = smmu_pmu->num_counters; |
| 331 |
bool filter_en = !!get_filter_enable(event); |
| 332 |
|
| 333 |
span = filter_en ? get_filter_span(event) : |
| 334 |
SMMU_PMCG_DEFAULT_FILTER_SPAN; |
| 335 |
sid = filter_en ? get_filter_stream_id(event) : |
| 336 |
SMMU_PMCG_DEFAULT_FILTER_SID; |
| 337 |
|
| 338 |
cur_idx = find_first_bit(smmu_pmu->used_counters, num_ctrs); |
| 339 |
/* |
| 340 |
* Per-counter filtering, or scheduling the first globally-filtered |
| 341 |
* event into an empty PMU so idx == 0 and it works out equivalent. |
| 342 |
*/ |
| 343 |
if (!smmu_pmu->global_filter || cur_idx == num_ctrs) { |
| 344 |
smmu_pmu_set_event_filter(event, idx, span, sid); |
| 345 |
return 0; |
| 346 |
} |
| 347 |
|
| 348 |
/* Otherwise, must match whatever's currently scheduled */ |
| 349 |
if (smmu_pmu_check_global_filter(smmu_pmu->events[cur_idx], event)) { |
| 350 |
smmu_pmu_set_evtyper(smmu_pmu, idx, get_event(event)); |
| 351 |
return 0; |
| 352 |
} |
| 353 |
|
| 354 |
return -EAGAIN; |
| 355 |
} |
| 356 |
|
| 357 |
static int smmu_pmu_get_event_idx(struct smmu_pmu *smmu_pmu, |
| 358 |
struct perf_event *event) |
| 359 |
{ |
| 360 |
int idx, err; |
| 361 |
unsigned int num_ctrs = smmu_pmu->num_counters; |
| 362 |
|
| 363 |
idx = find_first_zero_bit(smmu_pmu->used_counters, num_ctrs); |
| 364 |
if (idx == num_ctrs) |
| 365 |
/* The counters are all in use. */ |
| 366 |
return -EAGAIN; |
| 367 |
|
| 368 |
err = smmu_pmu_apply_event_filter(smmu_pmu, event, idx); |
| 369 |
if (err) |
| 370 |
return err; |
| 371 |
|
| 372 |
set_bit(idx, smmu_pmu->used_counters); |
| 373 |
|
| 374 |
return idx; |
| 375 |
} |
| 376 |
|
| 377 |
static bool smmu_pmu_events_compatible(struct perf_event *curr, |
| 378 |
struct perf_event *new) |
| 379 |
{ |
| 380 |
if (new->pmu != curr->pmu) |
| 381 |
return false; |
| 382 |
|
| 383 |
if (to_smmu_pmu(new->pmu)->global_filter && |
| 384 |
!smmu_pmu_check_global_filter(curr, new)) |
| 385 |
return false; |
| 386 |
|
| 387 |
return true; |
| 388 |
} |
| 389 |
|
| 390 |
/* |
| 391 |
* Implementation of abstract pmu functionality required by |
| 392 |
* the core perf events code. |
| 393 |
*/ |
| 394 |
|
| 395 |
static int smmu_pmu_event_init(struct perf_event *event) |
| 396 |
{ |
| 397 |
struct hw_perf_event *hwc = &event->hw; |
| 398 |
struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu); |
| 399 |
struct device *dev = smmu_pmu->dev; |
| 400 |
struct perf_event *sibling; |
| 401 |
int group_num_events = 1; |
| 402 |
u16 event_id; |
| 403 |
|
| 404 |
if (event->attr.type != event->pmu->type) |
| 405 |
return -ENOENT; |
| 406 |
|
| 407 |
if (hwc->sample_period) { |
| 408 |
dev_dbg(dev, "Sampling not supported\n"); |
| 409 |
return -EOPNOTSUPP; |
| 410 |
} |
| 411 |
|
| 412 |
if (event->cpu < 0) { |
| 413 |
dev_dbg(dev, "Per-task mode not supported\n"); |
| 414 |
return -EOPNOTSUPP; |
| 415 |
} |
| 416 |
|
| 417 |
/* Verify specified event is supported on this PMU */ |
| 418 |
event_id = get_event(event); |
| 419 |
if (event_id < SMMU_PMCG_ARCH_MAX_EVENTS && |
| 420 |
(!test_bit(event_id, smmu_pmu->supported_events))) { |
| 421 |
dev_dbg(dev, "Invalid event %d for this PMU\n", event_id); |
| 422 |
return -EINVAL; |
| 423 |
} |
| 424 |
|
| 425 |
/* Don't allow groups with mixed PMUs, except for s/w events */ |
| 426 |
if (!is_software_event(event->group_leader)) { |
| 427 |
if (!smmu_pmu_events_compatible(event->group_leader, event)) |
| 428 |
return -EINVAL; |
| 429 |
|
| 430 |
if (++group_num_events > smmu_pmu->num_counters) |
| 431 |
return -EINVAL; |
| 432 |
} |
| 433 |
|
| 434 |
/* |
| 435 |
* Ensure all events are on the same cpu so all events are in the |
| 436 |
* same cpu context, to avoid races on pmu_enable etc. |
| 437 |
*/ |
| 438 |
event->cpu = smmu_pmu->on_cpu; |
| 439 |
|
| 440 |
hwc->idx = -1; |
| 441 |
|
| 442 |
if (event->group_leader == event) |
| 443 |
return 0; |
| 444 |
|
| 445 |
for_each_sibling_event(sibling, event->group_leader) { |
| 446 |
if (is_software_event(sibling)) |
| 447 |
continue; |
| 448 |
|
| 449 |
if (!smmu_pmu_events_compatible(sibling, event)) |
| 450 |
return -EINVAL; |
| 451 |
|
| 452 |
if (++group_num_events > smmu_pmu->num_counters) |
| 453 |
return -EINVAL; |
| 454 |
} |
| 455 |
|
| 456 |
return 0; |
| 457 |
} |
| 458 |
|
| 459 |
static void smmu_pmu_event_start(struct perf_event *event, int flags) |
| 460 |
{ |
| 461 |
struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu); |
| 462 |
struct hw_perf_event *hwc = &event->hw; |
| 463 |
int idx = hwc->idx; |
| 464 |
|
| 465 |
hwc->state = 0; |
| 466 |
|
| 467 |
smmu_pmu_set_period(smmu_pmu, hwc); |
| 468 |
|
| 469 |
smmu_pmu_counter_enable(smmu_pmu, idx); |
| 470 |
} |
| 471 |
|
| 472 |
static void smmu_pmu_event_stop(struct perf_event *event, int flags) |
| 473 |
{ |
| 474 |
struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu); |
| 475 |
struct hw_perf_event *hwc = &event->hw; |
| 476 |
int idx = hwc->idx; |
| 477 |
|
| 478 |
if (hwc->state & PERF_HES_STOPPED) |
| 479 |
return; |
| 480 |
|
| 481 |
smmu_pmu_counter_disable(smmu_pmu, idx); |
| 482 |
/* As the counter gets updated on _start, ignore PERF_EF_UPDATE */ |
| 483 |
smmu_pmu_event_update(event); |
| 484 |
hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; |
| 485 |
} |
| 486 |
|
| 487 |
static int smmu_pmu_event_add(struct perf_event *event, int flags) |
| 488 |
{ |
| 489 |
struct hw_perf_event *hwc = &event->hw; |
| 490 |
int idx; |
| 491 |
struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu); |
| 492 |
|
| 493 |
idx = smmu_pmu_get_event_idx(smmu_pmu, event); |
| 494 |
if (idx < 0) |
| 495 |
return idx; |
| 496 |
|
| 497 |
hwc->idx = idx; |
| 498 |
hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; |
| 499 |
smmu_pmu->events[idx] = event; |
| 500 |
local64_set(&hwc->prev_count, 0); |
| 501 |
|
| 502 |
smmu_pmu_interrupt_enable(smmu_pmu, idx); |
| 503 |
|
| 504 |
if (flags & PERF_EF_START) |
| 505 |
smmu_pmu_event_start(event, flags); |
| 506 |
|
| 507 |
/* Propagate changes to the userspace mapping. */ |
| 508 |
perf_event_update_userpage(event); |
| 509 |
|
| 510 |
return 0; |
| 511 |
} |
| 512 |
|
| 513 |
static void smmu_pmu_event_del(struct perf_event *event, int flags) |
| 514 |
{ |
| 515 |
struct hw_perf_event *hwc = &event->hw; |
| 516 |
struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu); |
| 517 |
int idx = hwc->idx; |
| 518 |
|
| 519 |
smmu_pmu_event_stop(event, flags | PERF_EF_UPDATE); |
| 520 |
smmu_pmu_interrupt_disable(smmu_pmu, idx); |
| 521 |
smmu_pmu->events[idx] = NULL; |
| 522 |
clear_bit(idx, smmu_pmu->used_counters); |
| 523 |
|
| 524 |
perf_event_update_userpage(event); |
| 525 |
} |
| 526 |
|
| 527 |
static void smmu_pmu_event_read(struct perf_event *event) |
| 528 |
{ |
| 529 |
smmu_pmu_event_update(event); |
| 530 |
} |
| 531 |
|
| 532 |
/* cpumask */ |
| 533 |
|
| 534 |
static ssize_t smmu_pmu_cpumask_show(struct device *dev, |
| 535 |
struct device_attribute *attr, |
| 536 |
char *buf) |
| 537 |
{ |
| 538 |
struct smmu_pmu *smmu_pmu = to_smmu_pmu(dev_get_drvdata(dev)); |
| 539 |
|
| 540 |
return cpumap_print_to_pagebuf(true, buf, cpumask_of(smmu_pmu->on_cpu)); |
| 541 |
} |
| 542 |
|
| 543 |
static struct device_attribute smmu_pmu_cpumask_attr = |
| 544 |
__ATTR(cpumask, 0444, smmu_pmu_cpumask_show, NULL); |
| 545 |
|
| 546 |
static struct attribute *smmu_pmu_cpumask_attrs[] = { |
| 547 |
&smmu_pmu_cpumask_attr.attr, |
| 548 |
NULL |
| 549 |
}; |
| 550 |
|
| 551 |
static const struct attribute_group smmu_pmu_cpumask_group = { |
| 552 |
.attrs = smmu_pmu_cpumask_attrs, |
| 553 |
}; |
| 554 |
|
| 555 |
/* Events */ |
| 556 |
|
| 557 |
static ssize_t smmu_pmu_event_show(struct device *dev, |
| 558 |
struct device_attribute *attr, char *page) |
| 559 |
{ |
| 560 |
struct perf_pmu_events_attr *pmu_attr; |
| 561 |
|
| 562 |
pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr); |
| 563 |
|
| 564 |
return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id); |
| 565 |
} |
| 566 |
|
| 567 |
#define SMMU_EVENT_ATTR(name, config) \ |
| 568 |
PMU_EVENT_ATTR_ID(name, smmu_pmu_event_show, config) |
| 569 |
|
| 570 |
static struct attribute *smmu_pmu_events[] = { |
| 571 |
SMMU_EVENT_ATTR(cycles, 0), |
| 572 |
SMMU_EVENT_ATTR(transaction, 1), |
| 573 |
SMMU_EVENT_ATTR(tlb_miss, 2), |
| 574 |
SMMU_EVENT_ATTR(config_cache_miss, 3), |
| 575 |
SMMU_EVENT_ATTR(trans_table_walk_access, 4), |
| 576 |
SMMU_EVENT_ATTR(config_struct_access, 5), |
| 577 |
SMMU_EVENT_ATTR(pcie_ats_trans_rq, 6), |
| 578 |
SMMU_EVENT_ATTR(pcie_ats_trans_passed, 7), |
| 579 |
NULL |
| 580 |
}; |
| 581 |
|
| 582 |
static umode_t smmu_pmu_event_is_visible(struct kobject *kobj, |
| 583 |
struct attribute *attr, int unused) |
| 584 |
{ |
| 585 |
struct device *dev = kobj_to_dev(kobj); |
| 586 |
struct smmu_pmu *smmu_pmu = to_smmu_pmu(dev_get_drvdata(dev)); |
| 587 |
struct perf_pmu_events_attr *pmu_attr; |
| 588 |
|
| 589 |
pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr.attr); |
| 590 |
|
| 591 |
if (test_bit(pmu_attr->id, smmu_pmu->supported_events)) |
| 592 |
return attr->mode; |
| 593 |
|
| 594 |
return 0; |
| 595 |
} |
| 596 |
|
| 597 |
static const struct attribute_group smmu_pmu_events_group = { |
| 598 |
.name = "events", |
| 599 |
.attrs = smmu_pmu_events, |
| 600 |
.is_visible = smmu_pmu_event_is_visible, |
| 601 |
}; |
| 602 |
|
| 603 |
static ssize_t smmu_pmu_identifier_attr_show(struct device *dev, |
| 604 |
struct device_attribute *attr, |
| 605 |
char *page) |
| 606 |
{ |
| 607 |
struct smmu_pmu *smmu_pmu = to_smmu_pmu(dev_get_drvdata(dev)); |
| 608 |
|
| 609 |
return sysfs_emit(page, "0x%08x\n", smmu_pmu->iidr); |
| 610 |
} |
| 611 |
|
| 612 |
static umode_t smmu_pmu_identifier_attr_visible(struct kobject *kobj, |
| 613 |
struct attribute *attr, |
| 614 |
int n) |
| 615 |
{ |
| 616 |
struct device *dev = kobj_to_dev(kobj); |
| 617 |
struct smmu_pmu *smmu_pmu = to_smmu_pmu(dev_get_drvdata(dev)); |
| 618 |
|
| 619 |
if (!smmu_pmu->iidr) |
| 620 |
return 0; |
| 621 |
return attr->mode; |
| 622 |
} |
| 623 |
|
| 624 |
static struct device_attribute smmu_pmu_identifier_attr = |
| 625 |
__ATTR(identifier, 0444, smmu_pmu_identifier_attr_show, NULL); |
| 626 |
|
| 627 |
static struct attribute *smmu_pmu_identifier_attrs[] = { |
| 628 |
&smmu_pmu_identifier_attr.attr, |
| 629 |
NULL |
| 630 |
}; |
| 631 |
|
| 632 |
static const struct attribute_group smmu_pmu_identifier_group = { |
| 633 |
.attrs = smmu_pmu_identifier_attrs, |
| 634 |
.is_visible = smmu_pmu_identifier_attr_visible, |
| 635 |
}; |
| 636 |
|
| 637 |
/* Formats */ |
| 638 |
PMU_FORMAT_ATTR(event, "config:0-15"); |
| 639 |
PMU_FORMAT_ATTR(filter_stream_id, "config1:0-31"); |
| 640 |
PMU_FORMAT_ATTR(filter_span, "config1:32"); |
| 641 |
PMU_FORMAT_ATTR(filter_enable, "config1:33"); |
| 642 |
|
| 643 |
static struct attribute *smmu_pmu_formats[] = { |
| 644 |
&format_attr_event.attr, |
| 645 |
&format_attr_filter_stream_id.attr, |
| 646 |
&format_attr_filter_span.attr, |
| 647 |
&format_attr_filter_enable.attr, |
| 648 |
NULL |
| 649 |
}; |
| 650 |
|
| 651 |
static const struct attribute_group smmu_pmu_format_group = { |
| 652 |
.name = "format", |
| 653 |
.attrs = smmu_pmu_formats, |
| 654 |
}; |
| 655 |
|
| 656 |
static const struct attribute_group *smmu_pmu_attr_grps[] = { |
| 657 |
&smmu_pmu_cpumask_group, |
| 658 |
&smmu_pmu_events_group, |
| 659 |
&smmu_pmu_format_group, |
| 660 |
&smmu_pmu_identifier_group, |
| 661 |
NULL |
| 662 |
}; |
| 663 |
|
| 664 |
/* |
| 665 |
* Generic device handlers |
| 666 |
*/ |
| 667 |
|
| 668 |
static int smmu_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) |
| 669 |
{ |
| 670 |
struct smmu_pmu *smmu_pmu; |
| 671 |
unsigned int target; |
| 672 |
|
| 673 |
smmu_pmu = hlist_entry_safe(node, struct smmu_pmu, node); |
| 674 |
if (cpu != smmu_pmu->on_cpu) |
| 675 |
return 0; |
| 676 |
|
| 677 |
target = cpumask_any_but(cpu_online_mask, cpu); |
| 678 |
if (target >= nr_cpu_ids) |
| 679 |
return 0; |
| 680 |
|
| 681 |
perf_pmu_migrate_context(&smmu_pmu->pmu, cpu, target); |
| 682 |
smmu_pmu->on_cpu = target; |
| 683 |
WARN_ON(irq_set_affinity(smmu_pmu->irq, cpumask_of(target))); |
| 684 |
|
| 685 |
return 0; |
| 686 |
} |
| 687 |
|
| 688 |
static irqreturn_t smmu_pmu_handle_irq(int irq_num, void *data) |
| 689 |
{ |
| 690 |
struct smmu_pmu *smmu_pmu = data; |
| 691 |
DECLARE_BITMAP(ovs, BITS_PER_TYPE(u64)); |
| 692 |
u64 ovsr; |
| 693 |
unsigned int idx; |
| 694 |
|
| 695 |
ovsr = readq(smmu_pmu->reloc_base + SMMU_PMCG_OVSSET0); |
| 696 |
if (!ovsr) |
| 697 |
return IRQ_NONE; |
| 698 |
|
| 699 |
writeq(ovsr, smmu_pmu->reloc_base + SMMU_PMCG_OVSCLR0); |
| 700 |
|
| 701 |
bitmap_from_u64(ovs, ovsr); |
| 702 |
for_each_set_bit(idx, ovs, smmu_pmu->num_counters) { |
| 703 |
struct perf_event *event = smmu_pmu->events[idx]; |
| 704 |
struct hw_perf_event *hwc; |
| 705 |
|
| 706 |
if (WARN_ON_ONCE(!event)) |
| 707 |
continue; |
| 708 |
|
| 709 |
smmu_pmu_event_update(event); |
| 710 |
hwc = &event->hw; |
| 711 |
|
| 712 |
smmu_pmu_set_period(smmu_pmu, hwc); |
| 713 |
} |
| 714 |
|
| 715 |
return IRQ_HANDLED; |
| 716 |
} |
| 717 |
|
| 718 |
static void smmu_pmu_free_msis(void *data) |
| 719 |
{ |
| 720 |
struct device *dev = data; |
| 721 |
|
| 722 |
platform_device_msi_free_irqs_all(dev); |
| 723 |
} |
| 724 |
|
| 725 |
static void smmu_pmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg) |
| 726 |
{ |
| 727 |
phys_addr_t doorbell; |
| 728 |
struct device *dev = msi_desc_to_dev(desc); |
| 729 |
struct smmu_pmu *pmu = dev_get_drvdata(dev); |
| 730 |
|
| 731 |
doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo; |
| 732 |
doorbell &= MSI_CFG0_ADDR_MASK; |
| 733 |
|
| 734 |
writeq_relaxed(doorbell, pmu->reg_base + SMMU_PMCG_IRQ_CFG0); |
| 735 |
writel_relaxed(msg->data, pmu->reg_base + SMMU_PMCG_IRQ_CFG1); |
| 736 |
writel_relaxed(MSI_CFG2_MEMATTR_DEVICE_nGnRE, |
| 737 |
pmu->reg_base + SMMU_PMCG_IRQ_CFG2); |
| 738 |
} |
| 739 |
|
| 740 |
static void smmu_pmu_setup_msi(struct smmu_pmu *pmu) |
| 741 |
{ |
| 742 |
struct device *dev = pmu->dev; |
| 743 |
int ret; |
| 744 |
|
| 745 |
/* Clear MSI address reg */ |
| 746 |
writeq_relaxed(0, pmu->reg_base + SMMU_PMCG_IRQ_CFG0); |
| 747 |
|
| 748 |
/* MSI supported or not */ |
| 749 |
if (!(readl(pmu->reg_base + SMMU_PMCG_CFGR) & SMMU_PMCG_CFGR_MSI)) |
| 750 |
return; |
| 751 |
|
| 752 |
ret = platform_device_msi_init_and_alloc_irqs(dev, 1, smmu_pmu_write_msi_msg); |
| 753 |
if (ret) { |
| 754 |
dev_warn(dev, "failed to allocate MSIs\n"); |
| 755 |
return; |
| 756 |
} |
| 757 |
|
| 758 |
pmu->irq = msi_get_virq(dev, 0); |
| 759 |
|
| 760 |
/* Add callback to free MSIs on teardown */ |
| 761 |
devm_add_action(dev, smmu_pmu_free_msis, dev); |
| 762 |
} |
| 763 |
|
| 764 |
static int smmu_pmu_setup_irq(struct smmu_pmu *pmu) |
| 765 |
{ |
| 766 |
unsigned long flags = IRQF_NOBALANCING | IRQF_SHARED | IRQF_NO_THREAD; |
| 767 |
int irq, ret = -ENXIO; |
| 768 |
|
| 769 |
smmu_pmu_setup_msi(pmu); |
| 770 |
|
| 771 |
irq = pmu->irq; |
| 772 |
if (irq) |
| 773 |
ret = devm_request_irq(pmu->dev, irq, smmu_pmu_handle_irq, |
| 774 |
flags, "smmuv3-pmu", pmu); |
| 775 |
return ret; |
| 776 |
} |
| 777 |
|
| 778 |
static void smmu_pmu_reset(struct smmu_pmu *smmu_pmu) |
| 779 |
{ |
| 780 |
u64 counter_present_mask = GENMASK_ULL(smmu_pmu->num_counters - 1, 0); |
| 781 |
|
| 782 |
smmu_pmu_disable(&smmu_pmu->pmu); |
| 783 |
|
| 784 |
/* Disable counter and interrupt */ |
| 785 |
writeq_relaxed(counter_present_mask, |
| 786 |
smmu_pmu->reg_base + SMMU_PMCG_CNTENCLR0); |
| 787 |
writeq_relaxed(counter_present_mask, |
| 788 |
smmu_pmu->reg_base + SMMU_PMCG_INTENCLR0); |
| 789 |
writeq_relaxed(counter_present_mask, |
| 790 |
smmu_pmu->reloc_base + SMMU_PMCG_OVSCLR0); |
| 791 |
} |
| 792 |
|
| 793 |
static void smmu_pmu_get_acpi_options(struct smmu_pmu *smmu_pmu) |
| 794 |
{ |
| 795 |
u32 model; |
| 796 |
|
| 797 |
model = *(u32 *)dev_get_platdata(smmu_pmu->dev); |
| 798 |
|
| 799 |
switch (model) { |
| 800 |
case IORT_SMMU_V3_PMCG_HISI_HIP08: |
| 801 |
/* HiSilicon Erratum 162001800 */ |
| 802 |
smmu_pmu->options |= SMMU_PMCG_EVCNTR_RDONLY | SMMU_PMCG_HARDEN_DISABLE; |
| 803 |
break; |
| 804 |
case IORT_SMMU_V3_PMCG_HISI_HIP09: |
| 805 |
smmu_pmu->options |= SMMU_PMCG_HARDEN_DISABLE; |
| 806 |
break; |
| 807 |
} |
| 808 |
|
| 809 |
dev_notice(smmu_pmu->dev, "option mask 0x%x\n", smmu_pmu->options); |
| 810 |
} |
| 811 |
|
| 812 |
static bool smmu_pmu_coresight_id_regs(struct smmu_pmu *smmu_pmu) |
| 813 |
{ |
| 814 |
return of_device_is_compatible(smmu_pmu->dev->of_node, |
| 815 |
"arm,mmu-600-pmcg"); |
| 816 |
} |
| 817 |
|
| 818 |
static void smmu_pmu_get_iidr(struct smmu_pmu *smmu_pmu) |
| 819 |
{ |
| 820 |
u32 iidr = readl_relaxed(smmu_pmu->reg_base + SMMU_PMCG_IIDR); |
| 821 |
|
| 822 |
if (!iidr && smmu_pmu_coresight_id_regs(smmu_pmu)) { |
| 823 |
u32 pidr0 = readl(smmu_pmu->reg_base + SMMU_PMCG_PIDR0); |
| 824 |
u32 pidr1 = readl(smmu_pmu->reg_base + SMMU_PMCG_PIDR1); |
| 825 |
u32 pidr2 = readl(smmu_pmu->reg_base + SMMU_PMCG_PIDR2); |
| 826 |
u32 pidr3 = readl(smmu_pmu->reg_base + SMMU_PMCG_PIDR3); |
| 827 |
u32 pidr4 = readl(smmu_pmu->reg_base + SMMU_PMCG_PIDR4); |
| 828 |
|
| 829 |
u32 productid = FIELD_GET(SMMU_PMCG_PIDR0_PART_0, pidr0) | |
| 830 |
(FIELD_GET(SMMU_PMCG_PIDR1_PART_1, pidr1) << 8); |
| 831 |
u32 variant = FIELD_GET(SMMU_PMCG_PIDR2_REVISION, pidr2); |
| 832 |
u32 revision = FIELD_GET(SMMU_PMCG_PIDR3_REVAND, pidr3); |
| 833 |
u32 implementer = |
| 834 |
FIELD_GET(SMMU_PMCG_PIDR1_DES_0, pidr1) | |
| 835 |
(FIELD_GET(SMMU_PMCG_PIDR2_DES_1, pidr2) << 4) | |
| 836 |
(FIELD_GET(SMMU_PMCG_PIDR4_DES_2, pidr4) << 8); |
| 837 |
|
| 838 |
iidr = FIELD_PREP(SMMU_PMCG_IIDR_PRODUCTID, productid) | |
| 839 |
FIELD_PREP(SMMU_PMCG_IIDR_VARIANT, variant) | |
| 840 |
FIELD_PREP(SMMU_PMCG_IIDR_REVISION, revision) | |
| 841 |
FIELD_PREP(SMMU_PMCG_IIDR_IMPLEMENTER, implementer); |
| 842 |
} |
| 843 |
|
| 844 |
smmu_pmu->iidr = iidr; |
| 845 |
} |
| 846 |
|
| 847 |
static int smmu_pmu_probe(struct platform_device *pdev) |
| 848 |
{ |
| 849 |
struct smmu_pmu *smmu_pmu; |
| 850 |
struct resource *res_0; |
| 851 |
u32 cfgr, reg_size; |
| 852 |
u64 ceid_64[2]; |
| 853 |
int irq, err; |
| 854 |
char *name; |
| 855 |
struct device *dev = &pdev->dev; |
| 856 |
|
| 857 |
smmu_pmu = devm_kzalloc(dev, sizeof(*smmu_pmu), GFP_KERNEL); |
| 858 |
if (!smmu_pmu) |
| 859 |
return -ENOMEM; |
| 860 |
|
| 861 |
smmu_pmu->dev = dev; |
| 862 |
platform_set_drvdata(pdev, smmu_pmu); |
| 863 |
|
| 864 |
smmu_pmu->pmu = (struct pmu) { |
| 865 |
.module = THIS_MODULE, |
| 866 |
.parent = &pdev->dev, |
| 867 |
.task_ctx_nr = perf_invalid_context, |
| 868 |
.pmu_enable = smmu_pmu_enable, |
| 869 |
.pmu_disable = smmu_pmu_disable, |
| 870 |
.event_init = smmu_pmu_event_init, |
| 871 |
.add = smmu_pmu_event_add, |
| 872 |
.del = smmu_pmu_event_del, |
| 873 |
.start = smmu_pmu_event_start, |
| 874 |
.stop = smmu_pmu_event_stop, |
| 875 |
.read = smmu_pmu_event_read, |
| 876 |
.attr_groups = smmu_pmu_attr_grps, |
| 877 |
.capabilities = PERF_PMU_CAP_NO_EXCLUDE, |
| 878 |
}; |
| 879 |
|
| 880 |
smmu_pmu->reg_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res_0); |
| 881 |
if (IS_ERR(smmu_pmu->reg_base)) |
| 882 |
return PTR_ERR(smmu_pmu->reg_base); |
| 883 |
|
| 884 |
cfgr = readl_relaxed(smmu_pmu->reg_base + SMMU_PMCG_CFGR); |
| 885 |
|
| 886 |
/* Determine if page 1 is present */ |
| 887 |
if (cfgr & SMMU_PMCG_CFGR_RELOC_CTRS) { |
| 888 |
smmu_pmu->reloc_base = devm_platform_ioremap_resource(pdev, 1); |
| 889 |
if (IS_ERR(smmu_pmu->reloc_base)) |
| 890 |
return PTR_ERR(smmu_pmu->reloc_base); |
| 891 |
} else { |
| 892 |
smmu_pmu->reloc_base = smmu_pmu->reg_base; |
| 893 |
} |
| 894 |
|
| 895 |
irq = platform_get_irq_optional(pdev, 0); |
| 896 |
if (irq > 0) |
| 897 |
smmu_pmu->irq = irq; |
| 898 |
|
| 899 |
ceid_64[0] = readq_relaxed(smmu_pmu->reg_base + SMMU_PMCG_CEID0); |
| 900 |
ceid_64[1] = readq_relaxed(smmu_pmu->reg_base + SMMU_PMCG_CEID1); |
| 901 |
bitmap_from_arr32(smmu_pmu->supported_events, (u32 *)ceid_64, |
| 902 |
SMMU_PMCG_ARCH_MAX_EVENTS); |
| 903 |
|
| 904 |
smmu_pmu->num_counters = FIELD_GET(SMMU_PMCG_CFGR_NCTR, cfgr) + 1; |
| 905 |
|
| 906 |
smmu_pmu->global_filter = !!(cfgr & SMMU_PMCG_CFGR_SID_FILTER_TYPE); |
| 907 |
|
| 908 |
reg_size = FIELD_GET(SMMU_PMCG_CFGR_SIZE, cfgr); |
| 909 |
smmu_pmu->counter_mask = GENMASK_ULL(reg_size, 0); |
| 910 |
|
| 911 |
smmu_pmu_reset(smmu_pmu); |
| 912 |
|
| 913 |
err = smmu_pmu_setup_irq(smmu_pmu); |
| 914 |
if (err) { |
| 915 |
dev_err(dev, "Setup irq failed, PMU @%pa\n", &res_0->start); |
| 916 |
return err; |
| 917 |
} |
| 918 |
|
| 919 |
smmu_pmu_get_iidr(smmu_pmu); |
| 920 |
|
| 921 |
name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "smmuv3_pmcg_%llx", |
| 922 |
(res_0->start) >> SMMU_PMCG_PA_SHIFT); |
| 923 |
if (!name) { |
| 924 |
dev_err(dev, "Create name failed, PMU @%pa\n", &res_0->start); |
| 925 |
return -EINVAL; |
| 926 |
} |
| 927 |
|
| 928 |
if (!dev->of_node) |
| 929 |
smmu_pmu_get_acpi_options(smmu_pmu); |
| 930 |
|
| 931 |
/* |
| 932 |
* For platforms suffer this quirk, the PMU disable sometimes fails to |
| 933 |
* stop the counters. This will leads to inaccurate or error counting. |
| 934 |
* Forcibly disable the counters with these quirk handler. |
| 935 |
*/ |
| 936 |
if (smmu_pmu->options & SMMU_PMCG_HARDEN_DISABLE) { |
| 937 |
smmu_pmu->pmu.pmu_enable = smmu_pmu_enable_quirk_hip08_09; |
| 938 |
smmu_pmu->pmu.pmu_disable = smmu_pmu_disable_quirk_hip08_09; |
| 939 |
} |
| 940 |
|
| 941 |
/* Pick one CPU to be the preferred one to use */ |
| 942 |
smmu_pmu->on_cpu = raw_smp_processor_id(); |
| 943 |
WARN_ON(irq_set_affinity(smmu_pmu->irq, cpumask_of(smmu_pmu->on_cpu))); |
| 944 |
|
| 945 |
err = cpuhp_state_add_instance_nocalls(cpuhp_state_num, |
| 946 |
&smmu_pmu->node); |
| 947 |
if (err) { |
| 948 |
dev_err(dev, "Error %d registering hotplug, PMU @%pa\n", |
| 949 |
err, &res_0->start); |
| 950 |
return err; |
| 951 |
} |
| 952 |
|
| 953 |
err = perf_pmu_register(&smmu_pmu->pmu, name, -1); |
| 954 |
if (err) { |
| 955 |
dev_err(dev, "Error %d registering PMU @%pa\n", |
| 956 |
err, &res_0->start); |
| 957 |
goto out_unregister; |
| 958 |
} |
| 959 |
|
| 960 |
dev_info(dev, "Registered PMU @ %pa using %d counters with %s filter settings\n", |
| 961 |
&res_0->start, smmu_pmu->num_counters, |
| 962 |
smmu_pmu->global_filter ? "Global(Counter0)" : |
| 963 |
"Individual"); |
| 964 |
|
| 965 |
return 0; |
| 966 |
|
| 967 |
out_unregister: |
| 968 |
cpuhp_state_remove_instance_nocalls(cpuhp_state_num, &smmu_pmu->node); |
| 969 |
return err; |
| 970 |
} |
| 971 |
|
| 972 |
static void smmu_pmu_remove(struct platform_device *pdev) |
| 973 |
{ |
| 974 |
struct smmu_pmu *smmu_pmu = platform_get_drvdata(pdev); |
| 975 |
|
| 976 |
perf_pmu_unregister(&smmu_pmu->pmu); |
| 977 |
cpuhp_state_remove_instance_nocalls(cpuhp_state_num, &smmu_pmu->node); |
| 978 |
} |
| 979 |
|
| 980 |
static void smmu_pmu_shutdown(struct platform_device *pdev) |
| 981 |
{ |
| 982 |
struct smmu_pmu *smmu_pmu = platform_get_drvdata(pdev); |
| 983 |
|
| 984 |
smmu_pmu_disable(&smmu_pmu->pmu); |
| 985 |
} |
| 986 |
|
| 987 |
#ifdef CONFIG_OF |
| 988 |
static const struct of_device_id smmu_pmu_of_match[] = { |
| 989 |
{ .compatible = "arm,smmu-v3-pmcg" }, |
| 990 |
{} |
| 991 |
}; |
| 992 |
MODULE_DEVICE_TABLE(of, smmu_pmu_of_match); |
| 993 |
#endif |
| 994 |
|
| 995 |
static struct platform_driver smmu_pmu_driver = { |
| 996 |
.driver = { |
| 997 |
.name = "arm-smmu-v3-pmcg", |
| 998 |
.of_match_table = of_match_ptr(smmu_pmu_of_match), |
| 999 |
.suppress_bind_attrs = true, |
| 1000 |
}, |
| 1001 |
.probe = smmu_pmu_probe, |
| 1002 |
.remove = smmu_pmu_remove, |
| 1003 |
.shutdown = smmu_pmu_shutdown, |
| 1004 |
}; |
| 1005 |
|
| 1006 |
static int __init arm_smmu_pmu_init(void) |
| 1007 |
{ |
| 1008 |
int ret; |
| 1009 |
|
| 1010 |
cpuhp_state_num = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, |
| 1011 |
"perf/arm/pmcg:online", |
| 1012 |
NULL, |
| 1013 |
smmu_pmu_offline_cpu); |
| 1014 |
if (cpuhp_state_num < 0) |
| 1015 |
return cpuhp_state_num; |
| 1016 |
|
| 1017 |
ret = platform_driver_register(&smmu_pmu_driver); |
| 1018 |
if (ret) |
| 1019 |
cpuhp_remove_multi_state(cpuhp_state_num); |
| 1020 |
|
| 1021 |
return ret; |
| 1022 |
} |
| 1023 |
module_init(arm_smmu_pmu_init); |
| 1024 |
|
| 1025 |
static void __exit arm_smmu_pmu_exit(void) |
| 1026 |
{ |
| 1027 |
platform_driver_unregister(&smmu_pmu_driver); |
| 1028 |
cpuhp_remove_multi_state(cpuhp_state_num); |
| 1029 |
} |
| 1030 |
|
| 1031 |
module_exit(arm_smmu_pmu_exit); |
| 1032 |
|
| 1033 |
MODULE_ALIAS("platform:arm-smmu-v3-pmcg"); |
| 1034 |
MODULE_DESCRIPTION("PMU driver for ARM SMMUv3 Performance Monitors Extension"); |
| 1035 |
MODULE_AUTHOR("Neil Leeder <[email protected]>"); |
| 1036 |
MODULE_AUTHOR("Shameer Kolothum <[email protected]>"); |
| 1037 |
MODULE_LICENSE("GPL v2"); |