| 1 |
// SPDX-License-Identifier: GPL-2.0 |
| 2 |
// Copyright (C) 2022-2024 Arm Limited |
| 3 |
// NI-700 Network-on-Chip PMU driver |
| 4 |
|
| 5 |
#include <linux/acpi.h> |
| 6 |
#include <linux/bitfield.h> |
| 7 |
#include <linux/interrupt.h> |
| 8 |
#include <linux/io.h> |
| 9 |
#include <linux/io-64-nonatomic-lo-hi.h> |
| 10 |
#include <linux/kernel.h> |
| 11 |
#include <linux/module.h> |
| 12 |
#include <linux/of.h> |
| 13 |
#include <linux/perf_event.h> |
| 14 |
#include <linux/platform_device.h> |
| 15 |
#include <linux/slab.h> |
| 16 |
|
| 17 |
/* Common registers */ |
| 18 |
#define NI_NODE_TYPE 0x000 |
| 19 |
#define NI_NODE_TYPE_NODE_ID GENMASK(31, 16) |
| 20 |
#define NI_NODE_TYPE_NODE_TYPE GENMASK(15, 0) |
| 21 |
|
| 22 |
#define NI_CHILD_NODE_INFO 0x004 |
| 23 |
#define NI_CHILD_PTR(n) (0x008 + (n) * 4) |
| 24 |
#define NI_NUM_SUB_FEATURES 0x100 |
| 25 |
#define NI_SUB_FEATURE_TYPE(n) (0x108 + (n) * 8) |
| 26 |
#define NI_SUB_FEATURE_PTR(n) (0x10c + (n) * 8) |
| 27 |
|
| 28 |
#define NI_SUB_FEATURE_TYPE_FCU 0x2 |
| 29 |
|
| 30 |
#define NI700_PMUSELA 0x00c |
| 31 |
|
| 32 |
/* Config node */ |
| 33 |
#define NI_PERIPHERAL_ID0 0xfe0 |
| 34 |
#define NI_PIDR0_PART_7_0 GENMASK(7, 0) |
| 35 |
#define NI_PERIPHERAL_ID1 0xfe4 |
| 36 |
#define NI_PIDR1_PART_11_8 GENMASK(3, 0) |
| 37 |
#define NI_PERIPHERAL_ID2 0xfe8 |
| 38 |
#define NI_PIDR2_VERSION GENMASK(7, 4) |
| 39 |
|
| 40 |
/* PMU node */ |
| 41 |
#define NI700_PMEVCNTR(n) (0x008 + (n) * 8) |
| 42 |
#define NI700_PMCCNTR_L 0x0f8 |
| 43 |
#define NI_PMEVCNTR(n) (0x200 + (n) * 8) |
| 44 |
#define NI_PMCCNTR_L 0x2f8 |
| 45 |
#define NI_PMEVTYPER(n) (0x400 + (n) * 4) |
| 46 |
#define NI_PMEVTYPER_NODE_TYPE GENMASK(12, 9) |
| 47 |
#define NI_PMEVTYPER_NODE_ID GENMASK(8, 0) |
| 48 |
#define NI_PMCNTENSET 0xc00 |
| 49 |
#define NI_PMCNTENCLR 0xc20 |
| 50 |
#define NI_PMINTENSET 0xc40 |
| 51 |
#define NI_PMINTENCLR 0xc60 |
| 52 |
#define NI_PMOVSCLR 0xc80 |
| 53 |
#define NI_PMOVSSET 0xcc0 |
| 54 |
#define NI_PMCFGR 0xe00 |
| 55 |
#define NI_PMCR 0xe04 |
| 56 |
#define NI_PMCR_RESET_CCNT BIT(2) |
| 57 |
#define NI_PMCR_RESET_EVCNT BIT(1) |
| 58 |
#define NI_PMCR_ENABLE BIT(0) |
| 59 |
|
| 60 |
#define NI_NUM_COUNTERS 8 |
| 61 |
#define NI_CCNT_IDX 31 |
| 62 |
|
| 63 |
/* Event attributes */ |
| 64 |
#define NI_CONFIG_TYPE GENMASK_ULL(15, 0) |
| 65 |
#define NI_CONFIG_NODEID GENMASK_ULL(31, 16) |
| 66 |
#define NI_CONFIG_EVENTID GENMASK_ULL(47, 32) |
| 67 |
|
| 68 |
#define NI_EVENT_TYPE(event) FIELD_GET(NI_CONFIG_TYPE, (event)->attr.config) |
| 69 |
#define NI_EVENT_NODEID(event) FIELD_GET(NI_CONFIG_NODEID, (event)->attr.config) |
| 70 |
#define NI_EVENT_EVENTID(event) FIELD_GET(NI_CONFIG_EVENTID, (event)->attr.config) |
| 71 |
|
| 72 |
enum ni_part { |
| 73 |
PART_NI_700 = 0x43b, |
| 74 |
PART_NI_710AE = 0x43d, |
| 75 |
PART_NOC_S3 = 0x43f, |
| 76 |
PART_SI_L1 = 0x455, |
| 77 |
}; |
| 78 |
|
| 79 |
enum ni_node_type { |
| 80 |
NI_GLOBAL, |
| 81 |
NI_VOLTAGE, |
| 82 |
NI_POWER, |
| 83 |
NI_CLOCK, |
| 84 |
NI_ASNI, |
| 85 |
NI_AMNI, |
| 86 |
NI_PMU, |
| 87 |
NI_HSNI, |
| 88 |
NI_HMNI, |
| 89 |
NI_PMNI, |
| 90 |
NI_TSNI, |
| 91 |
NI_TMNI, |
| 92 |
NI_CMNI = 0x0e, |
| 93 |
NI_MCN = 0x63, |
| 94 |
}; |
| 95 |
|
| 96 |
struct arm_ni_node { |
| 97 |
void __iomem *base; |
| 98 |
enum ni_node_type type; |
| 99 |
u16 id; |
| 100 |
u32 num_components; |
| 101 |
}; |
| 102 |
|
| 103 |
struct arm_ni_unit { |
| 104 |
void __iomem *pmusela; |
| 105 |
enum ni_node_type type; |
| 106 |
u16 id; |
| 107 |
bool ns; |
| 108 |
union { |
| 109 |
__le64 pmusel; |
| 110 |
u8 event[8]; |
| 111 |
}; |
| 112 |
}; |
| 113 |
|
| 114 |
struct arm_ni_cd { |
| 115 |
void __iomem *pmu_base; |
| 116 |
u16 id; |
| 117 |
s8 irq_friend; |
| 118 |
int num_units; |
| 119 |
int irq; |
| 120 |
struct pmu pmu; |
| 121 |
struct arm_ni_unit *units; |
| 122 |
struct perf_event *evcnt[NI_NUM_COUNTERS]; |
| 123 |
struct perf_event *ccnt; |
| 124 |
}; |
| 125 |
|
| 126 |
struct arm_ni { |
| 127 |
struct device *dev; |
| 128 |
void __iomem *base; |
| 129 |
enum ni_part part; |
| 130 |
int id; |
| 131 |
int cpu; |
| 132 |
int num_cds; |
| 133 |
struct hlist_node cpuhp_node; |
| 134 |
struct arm_ni_cd cds[] __counted_by(num_cds); |
| 135 |
}; |
| 136 |
|
| 137 |
#define cd_to_ni(cd) container_of((cd), struct arm_ni, cds[(cd)->id]) |
| 138 |
#define pmu_to_cd(p) container_of((p), struct arm_ni_cd, pmu) |
| 139 |
|
| 140 |
#define ni_for_each_cd(n, c) \ |
| 141 |
for (struct arm_ni_cd *c = n->cds; c < n->cds + n->num_cds; c++) if (c->pmu_base) |
| 142 |
|
| 143 |
#define cd_for_each_unit(cd, u) \ |
| 144 |
for (struct arm_ni_unit *u = cd->units; u < cd->units + cd->num_units; u++) |
| 145 |
|
| 146 |
static int arm_ni_hp_state; |
| 147 |
|
| 148 |
struct arm_ni_event_attr { |
| 149 |
struct device_attribute attr; |
| 150 |
enum ni_node_type type; |
| 151 |
}; |
| 152 |
|
| 153 |
#define NI_EVENT_ATTR(_name, _type) \ |
| 154 |
(&((struct arm_ni_event_attr[]) {{ \ |
| 155 |
.attr = __ATTR(_name, 0444, arm_ni_event_show, NULL), \ |
| 156 |
.type = _type, \ |
| 157 |
}})[0].attr.attr) |
| 158 |
|
| 159 |
static ssize_t arm_ni_event_show(struct device *dev, |
| 160 |
struct device_attribute *attr, char *buf) |
| 161 |
{ |
| 162 |
struct arm_ni_event_attr *eattr = container_of(attr, typeof(*eattr), attr); |
| 163 |
|
| 164 |
if (eattr->type == NI_PMU) |
| 165 |
return sysfs_emit(buf, "type=0x%x\n", eattr->type); |
| 166 |
|
| 167 |
return sysfs_emit(buf, "type=0x%x,eventid=?,nodeid=?\n", eattr->type); |
| 168 |
} |
| 169 |
|
| 170 |
static umode_t arm_ni_event_attr_is_visible(struct kobject *kobj, |
| 171 |
struct attribute *attr, int unused) |
| 172 |
{ |
| 173 |
struct device *dev = kobj_to_dev(kobj); |
| 174 |
struct arm_ni_cd *cd = pmu_to_cd(dev_get_drvdata(dev)); |
| 175 |
struct arm_ni_event_attr *eattr; |
| 176 |
|
| 177 |
eattr = container_of(attr, typeof(*eattr), attr.attr); |
| 178 |
|
| 179 |
cd_for_each_unit(cd, unit) { |
| 180 |
if (unit->type == eattr->type && unit->ns) |
| 181 |
return attr->mode; |
| 182 |
} |
| 183 |
|
| 184 |
return 0; |
| 185 |
} |
| 186 |
|
| 187 |
static struct attribute *arm_ni_event_attrs[] = { |
| 188 |
NI_EVENT_ATTR(asni, NI_ASNI), |
| 189 |
NI_EVENT_ATTR(amni, NI_AMNI), |
| 190 |
NI_EVENT_ATTR(cycles, NI_PMU), |
| 191 |
NI_EVENT_ATTR(hsni, NI_HSNI), |
| 192 |
NI_EVENT_ATTR(hmni, NI_HMNI), |
| 193 |
NI_EVENT_ATTR(pmni, NI_PMNI), |
| 194 |
NI_EVENT_ATTR(tsni, NI_TSNI), |
| 195 |
NI_EVENT_ATTR(tmni, NI_TMNI), |
| 196 |
NI_EVENT_ATTR(cmni, NI_CMNI), |
| 197 |
NULL |
| 198 |
}; |
| 199 |
|
| 200 |
static const struct attribute_group arm_ni_event_attrs_group = { |
| 201 |
.name = "events", |
| 202 |
.attrs = arm_ni_event_attrs, |
| 203 |
.is_visible = arm_ni_event_attr_is_visible, |
| 204 |
}; |
| 205 |
|
| 206 |
struct arm_ni_format_attr { |
| 207 |
struct device_attribute attr; |
| 208 |
u64 field; |
| 209 |
}; |
| 210 |
|
| 211 |
#define NI_FORMAT_ATTR(_name, _fld) \ |
| 212 |
(&((struct arm_ni_format_attr[]) {{ \ |
| 213 |
.attr = __ATTR(_name, 0444, arm_ni_format_show, NULL), \ |
| 214 |
.field = _fld, \ |
| 215 |
}})[0].attr.attr) |
| 216 |
|
| 217 |
static ssize_t arm_ni_format_show(struct device *dev, |
| 218 |
struct device_attribute *attr, char *buf) |
| 219 |
{ |
| 220 |
struct arm_ni_format_attr *fmt = container_of(attr, typeof(*fmt), attr); |
| 221 |
|
| 222 |
return sysfs_emit(buf, "config:%*pbl\n", 64, &fmt->field); |
| 223 |
} |
| 224 |
|
| 225 |
static struct attribute *arm_ni_format_attrs[] = { |
| 226 |
NI_FORMAT_ATTR(type, NI_CONFIG_TYPE), |
| 227 |
NI_FORMAT_ATTR(nodeid, NI_CONFIG_NODEID), |
| 228 |
NI_FORMAT_ATTR(eventid, NI_CONFIG_EVENTID), |
| 229 |
NULL |
| 230 |
}; |
| 231 |
|
| 232 |
static const struct attribute_group arm_ni_format_attrs_group = { |
| 233 |
.name = "format", |
| 234 |
.attrs = arm_ni_format_attrs, |
| 235 |
}; |
| 236 |
|
| 237 |
static ssize_t arm_ni_cpumask_show(struct device *dev, |
| 238 |
struct device_attribute *attr, char *buf) |
| 239 |
{ |
| 240 |
struct arm_ni *ni = cd_to_ni(pmu_to_cd(dev_get_drvdata(dev))); |
| 241 |
|
| 242 |
return cpumap_print_to_pagebuf(true, buf, cpumask_of(ni->cpu)); |
| 243 |
} |
| 244 |
|
| 245 |
static struct device_attribute arm_ni_cpumask_attr = |
| 246 |
__ATTR(cpumask, 0444, arm_ni_cpumask_show, NULL); |
| 247 |
|
| 248 |
static ssize_t arm_ni_identifier_show(struct device *dev, |
| 249 |
struct device_attribute *attr, char *buf) |
| 250 |
{ |
| 251 |
struct arm_ni *ni = cd_to_ni(pmu_to_cd(dev_get_drvdata(dev))); |
| 252 |
u32 reg = readl_relaxed(ni->base + NI_PERIPHERAL_ID2); |
| 253 |
int version = FIELD_GET(NI_PIDR2_VERSION, reg); |
| 254 |
|
| 255 |
return sysfs_emit(buf, "%03x%02x\n", ni->part, version); |
| 256 |
} |
| 257 |
|
| 258 |
static struct device_attribute arm_ni_identifier_attr = |
| 259 |
__ATTR(identifier, 0444, arm_ni_identifier_show, NULL); |
| 260 |
|
| 261 |
static struct attribute *arm_ni_other_attrs[] = { |
| 262 |
&arm_ni_cpumask_attr.attr, |
| 263 |
&arm_ni_identifier_attr.attr, |
| 264 |
NULL |
| 265 |
}; |
| 266 |
|
| 267 |
static const struct attribute_group arm_ni_other_attr_group = { |
| 268 |
.attrs = arm_ni_other_attrs, |
| 269 |
}; |
| 270 |
|
| 271 |
static const struct attribute_group *arm_ni_attr_groups[] = { |
| 272 |
&arm_ni_event_attrs_group, |
| 273 |
&arm_ni_format_attrs_group, |
| 274 |
&arm_ni_other_attr_group, |
| 275 |
NULL |
| 276 |
}; |
| 277 |
|
| 278 |
static void arm_ni_pmu_enable(struct pmu *pmu) |
| 279 |
{ |
| 280 |
writel_relaxed(NI_PMCR_ENABLE, pmu_to_cd(pmu)->pmu_base + NI_PMCR); |
| 281 |
} |
| 282 |
|
| 283 |
static void arm_ni_pmu_disable(struct pmu *pmu) |
| 284 |
{ |
| 285 |
writel_relaxed(0, pmu_to_cd(pmu)->pmu_base + NI_PMCR); |
| 286 |
} |
| 287 |
|
| 288 |
struct arm_ni_val { |
| 289 |
unsigned int evcnt; |
| 290 |
unsigned int ccnt; |
| 291 |
}; |
| 292 |
|
| 293 |
static bool arm_ni_val_count_event(struct perf_event *evt, struct arm_ni_val *val) |
| 294 |
{ |
| 295 |
if (is_software_event(evt)) |
| 296 |
return true; |
| 297 |
|
| 298 |
if (NI_EVENT_TYPE(evt) == NI_PMU) { |
| 299 |
val->ccnt++; |
| 300 |
return val->ccnt <= 1; |
| 301 |
} |
| 302 |
|
| 303 |
val->evcnt++; |
| 304 |
return val->evcnt <= NI_NUM_COUNTERS; |
| 305 |
} |
| 306 |
|
| 307 |
static int arm_ni_validate_group(struct perf_event *event) |
| 308 |
{ |
| 309 |
struct perf_event *sibling, *leader = event->group_leader; |
| 310 |
struct arm_ni_val val = { 0 }; |
| 311 |
|
| 312 |
if (leader == event) |
| 313 |
return 0; |
| 314 |
|
| 315 |
arm_ni_val_count_event(event, &val); |
| 316 |
if (!arm_ni_val_count_event(leader, &val)) |
| 317 |
return -EINVAL; |
| 318 |
|
| 319 |
for_each_sibling_event(sibling, leader) { |
| 320 |
if (!arm_ni_val_count_event(sibling, &val)) |
| 321 |
return -EINVAL; |
| 322 |
} |
| 323 |
return 0; |
| 324 |
} |
| 325 |
|
| 326 |
static bool arm_ni_is_7xx(const struct arm_ni *ni) |
| 327 |
{ |
| 328 |
return ni->part == PART_NI_700 || ni->part == PART_NI_710AE; |
| 329 |
} |
| 330 |
|
| 331 |
static int arm_ni_event_init(struct perf_event *event) |
| 332 |
{ |
| 333 |
struct arm_ni_cd *cd = pmu_to_cd(event->pmu); |
| 334 |
struct arm_ni *ni; |
| 335 |
|
| 336 |
if (event->attr.type != event->pmu->type) |
| 337 |
return -ENOENT; |
| 338 |
|
| 339 |
if (is_sampling_event(event)) |
| 340 |
return -EINVAL; |
| 341 |
|
| 342 |
ni = cd_to_ni(cd); |
| 343 |
event->cpu = ni->cpu; |
| 344 |
event->hw.flags = arm_ni_is_7xx(ni); |
| 345 |
|
| 346 |
if (NI_EVENT_TYPE(event) == NI_PMU) |
| 347 |
return arm_ni_validate_group(event); |
| 348 |
|
| 349 |
cd_for_each_unit(cd, unit) { |
| 350 |
if (unit->type == NI_EVENT_TYPE(event) && |
| 351 |
unit->id == NI_EVENT_NODEID(event) && unit->ns) { |
| 352 |
event->hw.config_base = (unsigned long)unit; |
| 353 |
return arm_ni_validate_group(event); |
| 354 |
} |
| 355 |
} |
| 356 |
return -EINVAL; |
| 357 |
} |
| 358 |
|
| 359 |
static u64 arm_ni_read_ccnt(void __iomem *pmccntr) |
| 360 |
{ |
| 361 |
u64 l, u_old, u_new; |
| 362 |
int retries = 3; /* 1st time unlucky, 2nd improbable, 3rd just broken */ |
| 363 |
|
| 364 |
u_new = readl_relaxed(pmccntr + 4); |
| 365 |
do { |
| 366 |
u_old = u_new; |
| 367 |
l = readl_relaxed(pmccntr); |
| 368 |
u_new = readl_relaxed(pmccntr + 4); |
| 369 |
} while (u_new != u_old && --retries); |
| 370 |
WARN_ON(!retries); |
| 371 |
|
| 372 |
return (u_new << 32) | l; |
| 373 |
} |
| 374 |
|
| 375 |
static void arm_ni_event_read(struct perf_event *event) |
| 376 |
{ |
| 377 |
struct hw_perf_event *hw = &event->hw; |
| 378 |
u64 count, prev; |
| 379 |
bool ccnt = hw->idx == NI_CCNT_IDX; |
| 380 |
|
| 381 |
do { |
| 382 |
prev = local64_read(&hw->prev_count); |
| 383 |
if (ccnt) |
| 384 |
count = arm_ni_read_ccnt((void __iomem *)event->hw.event_base); |
| 385 |
else |
| 386 |
count = readl_relaxed((void __iomem *)event->hw.event_base); |
| 387 |
} while (local64_cmpxchg(&hw->prev_count, prev, count) != prev); |
| 388 |
|
| 389 |
count -= prev; |
| 390 |
if (!ccnt) |
| 391 |
count = (u32)count; |
| 392 |
local64_add(count, &event->count); |
| 393 |
} |
| 394 |
|
| 395 |
static void arm_ni_event_start(struct perf_event *event, int flags) |
| 396 |
{ |
| 397 |
struct arm_ni_cd *cd = pmu_to_cd(event->pmu); |
| 398 |
|
| 399 |
writel_relaxed(1U << event->hw.idx, cd->pmu_base + NI_PMCNTENSET); |
| 400 |
} |
| 401 |
|
| 402 |
static void arm_ni_event_stop(struct perf_event *event, int flags) |
| 403 |
{ |
| 404 |
struct arm_ni_cd *cd = pmu_to_cd(event->pmu); |
| 405 |
|
| 406 |
writel_relaxed(1U << event->hw.idx, cd->pmu_base + NI_PMCNTENCLR); |
| 407 |
if (flags & PERF_EF_UPDATE) |
| 408 |
arm_ni_event_read(event); |
| 409 |
} |
| 410 |
|
| 411 |
static void arm_ni_init_ccnt(struct hw_perf_event *hw) |
| 412 |
{ |
| 413 |
local64_set(&hw->prev_count, S64_MIN); |
| 414 |
lo_hi_writeq_relaxed(S64_MIN, (void __iomem *)hw->event_base); |
| 415 |
} |
| 416 |
|
| 417 |
static void arm_ni_init_evcnt(struct hw_perf_event *hw) |
| 418 |
{ |
| 419 |
local64_set(&hw->prev_count, S32_MIN); |
| 420 |
writel_relaxed(S32_MIN, (void __iomem *)hw->event_base); |
| 421 |
} |
| 422 |
|
| 423 |
static int arm_ni_event_add(struct perf_event *event, int flags) |
| 424 |
{ |
| 425 |
struct arm_ni_cd *cd = pmu_to_cd(event->pmu); |
| 426 |
struct hw_perf_event *hw = &event->hw; |
| 427 |
struct arm_ni_unit *unit; |
| 428 |
enum ni_node_type type = NI_EVENT_TYPE(event); |
| 429 |
u32 reg; |
| 430 |
|
| 431 |
if (type == NI_PMU) { |
| 432 |
if (cd->ccnt) |
| 433 |
return -ENOSPC; |
| 434 |
hw->idx = NI_CCNT_IDX; |
| 435 |
hw->event_base = (unsigned long)cd->pmu_base + |
| 436 |
(hw->flags ? NI700_PMCCNTR_L : NI_PMCCNTR_L); |
| 437 |
cd->ccnt = event; |
| 438 |
arm_ni_init_ccnt(hw); |
| 439 |
} else { |
| 440 |
hw->idx = 0; |
| 441 |
while (cd->evcnt[hw->idx]) { |
| 442 |
if (++hw->idx == NI_NUM_COUNTERS) |
| 443 |
return -ENOSPC; |
| 444 |
} |
| 445 |
cd->evcnt[hw->idx] = event; |
| 446 |
unit = (void *)hw->config_base; |
| 447 |
unit->event[hw->idx] = NI_EVENT_EVENTID(event); |
| 448 |
hw->event_base = (unsigned long)cd->pmu_base + |
| 449 |
(hw->flags ? NI700_PMEVCNTR(hw->idx) : NI_PMEVCNTR(hw->idx)); |
| 450 |
arm_ni_init_evcnt(hw); |
| 451 |
lo_hi_writeq_relaxed(le64_to_cpu(unit->pmusel), unit->pmusela); |
| 452 |
|
| 453 |
reg = FIELD_PREP(NI_PMEVTYPER_NODE_TYPE, type) | |
| 454 |
FIELD_PREP(NI_PMEVTYPER_NODE_ID, NI_EVENT_NODEID(event)); |
| 455 |
writel_relaxed(reg, cd->pmu_base + NI_PMEVTYPER(hw->idx)); |
| 456 |
} |
| 457 |
if (flags & PERF_EF_START) |
| 458 |
arm_ni_event_start(event, 0); |
| 459 |
return 0; |
| 460 |
} |
| 461 |
|
| 462 |
static void arm_ni_event_del(struct perf_event *event, int flags) |
| 463 |
{ |
| 464 |
struct arm_ni_cd *cd = pmu_to_cd(event->pmu); |
| 465 |
struct hw_perf_event *hw = &event->hw; |
| 466 |
|
| 467 |
arm_ni_event_stop(event, PERF_EF_UPDATE); |
| 468 |
|
| 469 |
if (hw->idx == NI_CCNT_IDX) |
| 470 |
cd->ccnt = NULL; |
| 471 |
else |
| 472 |
cd->evcnt[hw->idx] = NULL; |
| 473 |
} |
| 474 |
|
| 475 |
static irqreturn_t arm_ni_handle_irq(int irq, void *dev_id) |
| 476 |
{ |
| 477 |
struct arm_ni_cd *cd = dev_id; |
| 478 |
irqreturn_t ret = IRQ_NONE; |
| 479 |
|
| 480 |
for (;;) { |
| 481 |
u32 reg = readl_relaxed(cd->pmu_base + NI_PMOVSCLR); |
| 482 |
|
| 483 |
if (reg & (1U << NI_CCNT_IDX)) { |
| 484 |
ret = IRQ_HANDLED; |
| 485 |
if (!(WARN_ON(!cd->ccnt))) { |
| 486 |
arm_ni_event_read(cd->ccnt); |
| 487 |
arm_ni_init_ccnt(&cd->ccnt->hw); |
| 488 |
} |
| 489 |
} |
| 490 |
for (int i = 0; i < NI_NUM_COUNTERS; i++) { |
| 491 |
if (!(reg & (1U << i))) |
| 492 |
continue; |
| 493 |
ret = IRQ_HANDLED; |
| 494 |
if (!(WARN_ON(!cd->evcnt[i]))) { |
| 495 |
arm_ni_event_read(cd->evcnt[i]); |
| 496 |
arm_ni_init_evcnt(&cd->evcnt[i]->hw); |
| 497 |
} |
| 498 |
} |
| 499 |
writel_relaxed(reg, cd->pmu_base + NI_PMOVSCLR); |
| 500 |
if (!cd->irq_friend) |
| 501 |
return ret; |
| 502 |
cd += cd->irq_friend; |
| 503 |
} |
| 504 |
} |
| 505 |
|
| 506 |
static void __iomem *arm_ni_get_pmusel(struct arm_ni *ni, void __iomem *unit_base) |
| 507 |
{ |
| 508 |
u32 type, ptr, num; |
| 509 |
|
| 510 |
if (arm_ni_is_7xx(ni)) |
| 511 |
return unit_base + NI700_PMUSELA; |
| 512 |
|
| 513 |
num = readl_relaxed(unit_base + NI_NUM_SUB_FEATURES); |
| 514 |
for (int i = 0; i < num; i++) { |
| 515 |
type = readl_relaxed(unit_base + NI_SUB_FEATURE_TYPE(i)); |
| 516 |
if (type != NI_SUB_FEATURE_TYPE_FCU) |
| 517 |
continue; |
| 518 |
ptr = readl_relaxed(unit_base + NI_SUB_FEATURE_PTR(i)); |
| 519 |
return ni->base + ptr; |
| 520 |
} |
| 521 |
/* Should be impossible */ |
| 522 |
return NULL; |
| 523 |
} |
| 524 |
|
| 525 |
static int arm_ni_init_cd(struct arm_ni *ni, struct arm_ni_node *node, u64 res_start) |
| 526 |
{ |
| 527 |
struct arm_ni_cd *cd = ni->cds + node->id; |
| 528 |
const char *name; |
| 529 |
|
| 530 |
cd->id = node->id; |
| 531 |
cd->num_units = node->num_components; |
| 532 |
cd->units = devm_kcalloc(ni->dev, cd->num_units, sizeof(*(cd->units)), GFP_KERNEL); |
| 533 |
if (!cd->units) |
| 534 |
return -ENOMEM; |
| 535 |
|
| 536 |
for (int i = 0; i < cd->num_units; i++) { |
| 537 |
u32 reg = readl_relaxed(node->base + NI_CHILD_PTR(i)); |
| 538 |
void __iomem *unit_base = ni->base + reg; |
| 539 |
struct arm_ni_unit *unit = cd->units + i; |
| 540 |
|
| 541 |
reg = readl_relaxed(unit_base + NI_NODE_TYPE); |
| 542 |
unit->type = FIELD_GET(NI_NODE_TYPE_NODE_TYPE, reg); |
| 543 |
unit->id = FIELD_GET(NI_NODE_TYPE_NODE_ID, reg); |
| 544 |
|
| 545 |
switch (unit->type) { |
| 546 |
case NI_PMU: |
| 547 |
reg = readl_relaxed(unit_base + NI_PMCFGR); |
| 548 |
if (!reg) { |
| 549 |
dev_info(ni->dev, "No access to PMU %d\n", cd->id); |
| 550 |
devm_kfree(ni->dev, cd->units); |
| 551 |
return 0; |
| 552 |
} |
| 553 |
unit->ns = true; |
| 554 |
cd->pmu_base = unit_base; |
| 555 |
break; |
| 556 |
case NI_ASNI: |
| 557 |
case NI_AMNI: |
| 558 |
case NI_HSNI: |
| 559 |
case NI_HMNI: |
| 560 |
case NI_PMNI: |
| 561 |
case NI_TSNI: |
| 562 |
case NI_TMNI: |
| 563 |
case NI_CMNI: |
| 564 |
unit->pmusela = arm_ni_get_pmusel(ni, unit_base); |
| 565 |
writel_relaxed(1, unit->pmusela); |
| 566 |
if (readl_relaxed(unit->pmusela) != 1) |
| 567 |
dev_info(ni->dev, "No access to node 0x%04x%04x\n", unit->id, unit->type); |
| 568 |
else |
| 569 |
unit->ns = true; |
| 570 |
break; |
| 571 |
case NI_MCN: |
| 572 |
break; |
| 573 |
default: |
| 574 |
/* |
| 575 |
* e.g. FMU - thankfully bits 3:2 of FMU_ERR_FR0 are RES0 so |
| 576 |
* can't alias any of the leaf node types we're looking for. |
| 577 |
*/ |
| 578 |
dev_dbg(ni->dev, "Mystery node 0x%04x%04x\n", unit->id, unit->type); |
| 579 |
break; |
| 580 |
} |
| 581 |
} |
| 582 |
|
| 583 |
res_start += cd->pmu_base - ni->base; |
| 584 |
if (!devm_request_mem_region(ni->dev, res_start, SZ_4K, dev_name(ni->dev))) { |
| 585 |
dev_err(ni->dev, "Failed to request PMU region 0x%llx\n", res_start); |
| 586 |
return -EBUSY; |
| 587 |
} |
| 588 |
|
| 589 |
writel_relaxed(NI_PMCR_RESET_CCNT | NI_PMCR_RESET_EVCNT, |
| 590 |
cd->pmu_base + NI_PMCR); |
| 591 |
writel_relaxed(U32_MAX, cd->pmu_base + NI_PMCNTENCLR); |
| 592 |
writel_relaxed(U32_MAX, cd->pmu_base + NI_PMOVSCLR); |
| 593 |
|
| 594 |
cd->irq = platform_get_irq(to_platform_device(ni->dev), cd->id); |
| 595 |
if (cd->irq < 0) |
| 596 |
return cd->irq; |
| 597 |
|
| 598 |
cd->pmu = (struct pmu) { |
| 599 |
.module = THIS_MODULE, |
| 600 |
.parent = ni->dev, |
| 601 |
.attr_groups = arm_ni_attr_groups, |
| 602 |
.capabilities = PERF_PMU_CAP_NO_EXCLUDE, |
| 603 |
.task_ctx_nr = perf_invalid_context, |
| 604 |
.pmu_enable = arm_ni_pmu_enable, |
| 605 |
.pmu_disable = arm_ni_pmu_disable, |
| 606 |
.event_init = arm_ni_event_init, |
| 607 |
.add = arm_ni_event_add, |
| 608 |
.del = arm_ni_event_del, |
| 609 |
.start = arm_ni_event_start, |
| 610 |
.stop = arm_ni_event_stop, |
| 611 |
.read = arm_ni_event_read, |
| 612 |
}; |
| 613 |
|
| 614 |
name = devm_kasprintf(ni->dev, GFP_KERNEL, "arm_ni_%d_cd_%d", ni->id, cd->id); |
| 615 |
if (!name) |
| 616 |
return -ENOMEM; |
| 617 |
|
| 618 |
return perf_pmu_register(&cd->pmu, name, -1); |
| 619 |
} |
| 620 |
|
| 621 |
static void arm_ni_remove(struct platform_device *pdev) |
| 622 |
{ |
| 623 |
struct arm_ni *ni = platform_get_drvdata(pdev); |
| 624 |
|
| 625 |
ni_for_each_cd(ni, cd) { |
| 626 |
writel_relaxed(0, cd->pmu_base + NI_PMCR); |
| 627 |
writel_relaxed(U32_MAX, cd->pmu_base + NI_PMINTENCLR); |
| 628 |
perf_pmu_unregister(&cd->pmu); |
| 629 |
} |
| 630 |
cpuhp_state_remove_instance_nocalls(arm_ni_hp_state, &ni->cpuhp_node); |
| 631 |
} |
| 632 |
|
| 633 |
static void arm_ni_probe_domain(void __iomem *base, struct arm_ni_node *node) |
| 634 |
{ |
| 635 |
u32 reg = readl_relaxed(base + NI_NODE_TYPE); |
| 636 |
|
| 637 |
node->base = base; |
| 638 |
node->type = FIELD_GET(NI_NODE_TYPE_NODE_TYPE, reg); |
| 639 |
node->id = FIELD_GET(NI_NODE_TYPE_NODE_ID, reg); |
| 640 |
node->num_components = readl_relaxed(base + NI_CHILD_NODE_INFO); |
| 641 |
} |
| 642 |
|
| 643 |
static int arm_ni_init_irqs(struct arm_ni *ni) |
| 644 |
{ |
| 645 |
int err; |
| 646 |
|
| 647 |
ni_for_each_cd(ni, cd) { |
| 648 |
for (struct arm_ni_cd *prev = cd; prev-- > ni->cds; ) { |
| 649 |
if (prev->irq == cd->irq) { |
| 650 |
prev->irq_friend = cd - prev; |
| 651 |
goto set_inten; |
| 652 |
} |
| 653 |
} |
| 654 |
err = devm_request_irq(ni->dev, cd->irq, arm_ni_handle_irq, |
| 655 |
IRQF_NOBALANCING | IRQF_NO_THREAD | IRQF_NO_AUTOEN, |
| 656 |
dev_name(ni->dev), cd); |
| 657 |
if (err) |
| 658 |
return err; |
| 659 |
|
| 660 |
irq_set_affinity(cd->irq, cpumask_of(ni->cpu)); |
| 661 |
set_inten: |
| 662 |
writel_relaxed(U32_MAX, cd->pmu_base + NI_PMINTENSET); |
| 663 |
} |
| 664 |
|
| 665 |
ni_for_each_cd(ni, cd) |
| 666 |
if (!cd->irq_friend) |
| 667 |
enable_irq(cd->irq); |
| 668 |
return 0; |
| 669 |
} |
| 670 |
|
| 671 |
static int arm_ni_probe(struct platform_device *pdev) |
| 672 |
{ |
| 673 |
struct arm_ni_node cfg, vd, pd, cd; |
| 674 |
struct arm_ni *ni; |
| 675 |
struct resource *res; |
| 676 |
void __iomem *base; |
| 677 |
static atomic_t id; |
| 678 |
int ret, num_cds; |
| 679 |
u32 reg, part; |
| 680 |
|
| 681 |
/* |
| 682 |
* We want to map the whole configuration space for ease of discovery, |
| 683 |
* but the PMU pages are the only ones for which we can honestly claim |
| 684 |
* exclusive ownership, so we'll request them explicitly once found. |
| 685 |
*/ |
| 686 |
res = platform_get_resource(pdev, IORESOURCE_MEM, 0); |
| 687 |
base = devm_ioremap(&pdev->dev, res->start, resource_size(res)); |
| 688 |
if (!base) |
| 689 |
return -ENOMEM; |
| 690 |
|
| 691 |
arm_ni_probe_domain(base, &cfg); |
| 692 |
if (cfg.type != NI_GLOBAL) |
| 693 |
return -ENODEV; |
| 694 |
|
| 695 |
reg = readl_relaxed(cfg.base + NI_PERIPHERAL_ID0); |
| 696 |
part = FIELD_GET(NI_PIDR0_PART_7_0, reg); |
| 697 |
reg = readl_relaxed(cfg.base + NI_PERIPHERAL_ID1); |
| 698 |
part |= FIELD_GET(NI_PIDR1_PART_11_8, reg) << 8; |
| 699 |
|
| 700 |
switch (part) { |
| 701 |
case PART_NI_700: |
| 702 |
case PART_NI_710AE: |
| 703 |
case PART_NOC_S3: |
| 704 |
case PART_SI_L1: |
| 705 |
break; |
| 706 |
default: |
| 707 |
dev_WARN(&pdev->dev, "Unknown part number: 0x%03x, this may go badly\n", part); |
| 708 |
break; |
| 709 |
} |
| 710 |
|
| 711 |
num_cds = 0; |
| 712 |
for (int v = 0; v < cfg.num_components; v++) { |
| 713 |
reg = readl_relaxed(cfg.base + NI_CHILD_PTR(v)); |
| 714 |
arm_ni_probe_domain(base + reg, &vd); |
| 715 |
for (int p = 0; p < vd.num_components; p++) { |
| 716 |
reg = readl_relaxed(vd.base + NI_CHILD_PTR(p)); |
| 717 |
arm_ni_probe_domain(base + reg, &pd); |
| 718 |
num_cds += pd.num_components; |
| 719 |
} |
| 720 |
} |
| 721 |
|
| 722 |
ni = devm_kzalloc(&pdev->dev, struct_size(ni, cds, num_cds), GFP_KERNEL); |
| 723 |
if (!ni) |
| 724 |
return -ENOMEM; |
| 725 |
|
| 726 |
ni->dev = &pdev->dev; |
| 727 |
ni->base = base; |
| 728 |
ni->num_cds = num_cds; |
| 729 |
ni->part = part; |
| 730 |
ni->id = atomic_fetch_inc(&id); |
| 731 |
ni->cpu = cpumask_local_spread(0, dev_to_node(ni->dev)); |
| 732 |
platform_set_drvdata(pdev, ni); |
| 733 |
|
| 734 |
ret = cpuhp_state_add_instance_nocalls(arm_ni_hp_state, &ni->cpuhp_node); |
| 735 |
if (ret) |
| 736 |
return ret; |
| 737 |
|
| 738 |
for (int v = 0; v < cfg.num_components; v++) { |
| 739 |
reg = readl_relaxed(cfg.base + NI_CHILD_PTR(v)); |
| 740 |
arm_ni_probe_domain(base + reg, &vd); |
| 741 |
for (int p = 0; p < vd.num_components; p++) { |
| 742 |
reg = readl_relaxed(vd.base + NI_CHILD_PTR(p)); |
| 743 |
arm_ni_probe_domain(base + reg, &pd); |
| 744 |
for (int c = 0; c < pd.num_components; c++) { |
| 745 |
reg = readl_relaxed(pd.base + NI_CHILD_PTR(c)); |
| 746 |
arm_ni_probe_domain(base + reg, &cd); |
| 747 |
ret = arm_ni_init_cd(ni, &cd, res->start); |
| 748 |
if (ret) { |
| 749 |
ni->cds[cd.id].pmu_base = NULL; |
| 750 |
arm_ni_remove(pdev); |
| 751 |
return ret; |
| 752 |
} |
| 753 |
} |
| 754 |
} |
| 755 |
} |
| 756 |
|
| 757 |
ret = arm_ni_init_irqs(ni); |
| 758 |
if (ret) |
| 759 |
arm_ni_remove(pdev); |
| 760 |
|
| 761 |
return ret; |
| 762 |
} |
| 763 |
|
| 764 |
#ifdef CONFIG_OF |
| 765 |
static const struct of_device_id arm_ni_of_match[] = { |
| 766 |
{ .compatible = "arm,ni-700" }, |
| 767 |
{} |
| 768 |
}; |
| 769 |
MODULE_DEVICE_TABLE(of, arm_ni_of_match); |
| 770 |
#endif |
| 771 |
|
| 772 |
#ifdef CONFIG_ACPI |
| 773 |
static const struct acpi_device_id arm_ni_acpi_match[] = { |
| 774 |
{ "ARMHCB70" }, |
| 775 |
{} |
| 776 |
}; |
| 777 |
MODULE_DEVICE_TABLE(acpi, arm_ni_acpi_match); |
| 778 |
#endif |
| 779 |
|
| 780 |
static struct platform_driver arm_ni_driver = { |
| 781 |
.driver = { |
| 782 |
.name = "arm-ni", |
| 783 |
.of_match_table = of_match_ptr(arm_ni_of_match), |
| 784 |
.acpi_match_table = ACPI_PTR(arm_ni_acpi_match), |
| 785 |
.suppress_bind_attrs = true, |
| 786 |
}, |
| 787 |
.probe = arm_ni_probe, |
| 788 |
.remove = arm_ni_remove, |
| 789 |
}; |
| 790 |
|
| 791 |
static void arm_ni_pmu_migrate(struct arm_ni *ni, unsigned int cpu) |
| 792 |
{ |
| 793 |
ni_for_each_cd(ni, cd) { |
| 794 |
perf_pmu_migrate_context(&cd->pmu, ni->cpu, cpu); |
| 795 |
irq_set_affinity(cd->irq, cpumask_of(cpu)); |
| 796 |
} |
| 797 |
ni->cpu = cpu; |
| 798 |
} |
| 799 |
|
| 800 |
static int arm_ni_pmu_online_cpu(unsigned int cpu, struct hlist_node *cpuhp_node) |
| 801 |
{ |
| 802 |
struct arm_ni *ni; |
| 803 |
int node; |
| 804 |
|
| 805 |
ni = hlist_entry_safe(cpuhp_node, struct arm_ni, cpuhp_node); |
| 806 |
node = dev_to_node(ni->dev); |
| 807 |
if (cpu_to_node(ni->cpu) != node && cpu_to_node(cpu) == node) |
| 808 |
arm_ni_pmu_migrate(ni, cpu); |
| 809 |
return 0; |
| 810 |
} |
| 811 |
|
| 812 |
static int arm_ni_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_node) |
| 813 |
{ |
| 814 |
struct arm_ni *ni; |
| 815 |
unsigned int target; |
| 816 |
int node; |
| 817 |
|
| 818 |
ni = hlist_entry_safe(cpuhp_node, struct arm_ni, cpuhp_node); |
| 819 |
if (cpu != ni->cpu) |
| 820 |
return 0; |
| 821 |
|
| 822 |
node = dev_to_node(ni->dev); |
| 823 |
target = cpumask_any_and_but(cpumask_of_node(node), cpu_online_mask, cpu); |
| 824 |
if (target >= nr_cpu_ids) |
| 825 |
target = cpumask_any_but(cpu_online_mask, cpu); |
| 826 |
|
| 827 |
if (target < nr_cpu_ids) |
| 828 |
arm_ni_pmu_migrate(ni, target); |
| 829 |
return 0; |
| 830 |
} |
| 831 |
|
| 832 |
static int __init arm_ni_init(void) |
| 833 |
{ |
| 834 |
int ret; |
| 835 |
|
| 836 |
ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, |
| 837 |
"perf/arm/ni:online", |
| 838 |
arm_ni_pmu_online_cpu, |
| 839 |
arm_ni_pmu_offline_cpu); |
| 840 |
if (ret < 0) |
| 841 |
return ret; |
| 842 |
|
| 843 |
arm_ni_hp_state = ret; |
| 844 |
|
| 845 |
ret = platform_driver_register(&arm_ni_driver); |
| 846 |
if (ret) |
| 847 |
cpuhp_remove_multi_state(arm_ni_hp_state); |
| 848 |
return ret; |
| 849 |
} |
| 850 |
|
| 851 |
static void __exit arm_ni_exit(void) |
| 852 |
{ |
| 853 |
platform_driver_unregister(&arm_ni_driver); |
| 854 |
cpuhp_remove_multi_state(arm_ni_hp_state); |
| 855 |
} |
| 856 |
|
| 857 |
module_init(arm_ni_init); |
| 858 |
module_exit(arm_ni_exit); |
| 859 |
|
| 860 |
MODULE_AUTHOR("Robin Murphy <[email protected]>"); |
| 861 |
MODULE_DESCRIPTION("Arm NI-700 PMU driver"); |
| 862 |
MODULE_LICENSE("GPL v2"); |