| 1 |
// SPDX-License-Identifier: GPL-2.0 |
| 2 |
/* |
| 3 |
* Performance event support framework for SuperH hardware counters. |
| 4 |
* |
| 5 |
* Copyright (C) 2009 Paul Mundt |
| 6 |
* |
| 7 |
* Heavily based on the x86 and PowerPC implementations. |
| 8 |
* |
| 9 |
* x86: |
| 10 |
* Copyright (C) 2008 Linutronix GmbH, Thomas Gleixner <[email protected]> |
| 11 |
* Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar |
| 12 |
* Copyright (C) 2009 Jaswinder Singh Rajput |
| 13 |
* Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter |
| 14 |
* Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra |
| 15 |
* Copyright (C) 2009 Intel Corporation, <[email protected]> |
| 16 |
* |
| 17 |
* ppc: |
| 18 |
* Copyright 2008-2009 Paul Mackerras, IBM Corporation. |
| 19 |
*/ |
| 20 |
#include <linux/kernel.h> |
| 21 |
#include <linux/init.h> |
| 22 |
#include <linux/io.h> |
| 23 |
#include <linux/irq.h> |
| 24 |
#include <linux/perf_event.h> |
| 25 |
#include <linux/export.h> |
| 26 |
#include <asm/processor.h> |
| 27 |
|
| 28 |
struct cpu_hw_events { |
| 29 |
struct perf_event *events[MAX_HWEVENTS]; |
| 30 |
unsigned long used_mask[BITS_TO_LONGS(MAX_HWEVENTS)]; |
| 31 |
unsigned long active_mask[BITS_TO_LONGS(MAX_HWEVENTS)]; |
| 32 |
}; |
| 33 |
|
| 34 |
DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events); |
| 35 |
|
| 36 |
static struct sh_pmu *sh_pmu __read_mostly; |
| 37 |
|
| 38 |
/* Number of perf_events counting hardware events */ |
| 39 |
static atomic_t num_events; |
| 40 |
/* Used to avoid races in calling reserve/release_pmc_hardware */ |
| 41 |
static DEFINE_MUTEX(pmc_reserve_mutex); |
| 42 |
|
| 43 |
/* |
| 44 |
* Stub these out for now, do something more profound later. |
| 45 |
*/ |
| 46 |
int reserve_pmc_hardware(void) |
| 47 |
{ |
| 48 |
return 0; |
| 49 |
} |
| 50 |
|
| 51 |
void release_pmc_hardware(void) |
| 52 |
{ |
| 53 |
} |
| 54 |
|
| 55 |
static inline int sh_pmu_initialized(void) |
| 56 |
{ |
| 57 |
return !!sh_pmu; |
| 58 |
} |
| 59 |
|
| 60 |
/* |
| 61 |
* Release the PMU if this is the last perf_event. |
| 62 |
*/ |
| 63 |
static void hw_perf_event_destroy(struct perf_event *event) |
| 64 |
{ |
| 65 |
if (!atomic_add_unless(&num_events, -1, 1)) { |
| 66 |
mutex_lock(&pmc_reserve_mutex); |
| 67 |
if (atomic_dec_return(&num_events) == 0) |
| 68 |
release_pmc_hardware(); |
| 69 |
mutex_unlock(&pmc_reserve_mutex); |
| 70 |
} |
| 71 |
} |
| 72 |
|
| 73 |
static int hw_perf_cache_event(int config, int *evp) |
| 74 |
{ |
| 75 |
unsigned long type, op, result; |
| 76 |
int ev; |
| 77 |
|
| 78 |
if (!sh_pmu->cache_events) |
| 79 |
return -EINVAL; |
| 80 |
|
| 81 |
/* unpack config */ |
| 82 |
type = config & 0xff; |
| 83 |
op = (config >> 8) & 0xff; |
| 84 |
result = (config >> 16) & 0xff; |
| 85 |
|
| 86 |
if (type >= PERF_COUNT_HW_CACHE_MAX || |
| 87 |
op >= PERF_COUNT_HW_CACHE_OP_MAX || |
| 88 |
result >= PERF_COUNT_HW_CACHE_RESULT_MAX) |
| 89 |
return -EINVAL; |
| 90 |
|
| 91 |
ev = (*sh_pmu->cache_events)[type][op][result]; |
| 92 |
if (ev == 0) |
| 93 |
return -EOPNOTSUPP; |
| 94 |
if (ev == -1) |
| 95 |
return -EINVAL; |
| 96 |
*evp = ev; |
| 97 |
return 0; |
| 98 |
} |
| 99 |
|
| 100 |
static int __hw_perf_event_init(struct perf_event *event) |
| 101 |
{ |
| 102 |
struct perf_event_attr *attr = &event->attr; |
| 103 |
struct hw_perf_event *hwc = &event->hw; |
| 104 |
int config = -1; |
| 105 |
int err; |
| 106 |
|
| 107 |
if (!sh_pmu_initialized()) |
| 108 |
return -ENODEV; |
| 109 |
|
| 110 |
/* |
| 111 |
* See if we need to reserve the counter. |
| 112 |
* |
| 113 |
* If no events are currently in use, then we have to take a |
| 114 |
* mutex to ensure that we don't race with another task doing |
| 115 |
* reserve_pmc_hardware or release_pmc_hardware. |
| 116 |
*/ |
| 117 |
err = 0; |
| 118 |
if (!atomic_inc_not_zero(&num_events)) { |
| 119 |
mutex_lock(&pmc_reserve_mutex); |
| 120 |
if (atomic_read(&num_events) == 0 && |
| 121 |
reserve_pmc_hardware()) |
| 122 |
err = -EBUSY; |
| 123 |
else |
| 124 |
atomic_inc(&num_events); |
| 125 |
mutex_unlock(&pmc_reserve_mutex); |
| 126 |
} |
| 127 |
|
| 128 |
if (err) |
| 129 |
return err; |
| 130 |
|
| 131 |
event->destroy = hw_perf_event_destroy; |
| 132 |
|
| 133 |
switch (attr->type) { |
| 134 |
case PERF_TYPE_RAW: |
| 135 |
config = attr->config & sh_pmu->raw_event_mask; |
| 136 |
break; |
| 137 |
case PERF_TYPE_HW_CACHE: |
| 138 |
err = hw_perf_cache_event(attr->config, &config); |
| 139 |
if (err) |
| 140 |
return err; |
| 141 |
break; |
| 142 |
case PERF_TYPE_HARDWARE: |
| 143 |
if (attr->config >= sh_pmu->max_events) |
| 144 |
return -EINVAL; |
| 145 |
|
| 146 |
config = sh_pmu->event_map(attr->config); |
| 147 |
break; |
| 148 |
} |
| 149 |
|
| 150 |
if (config == -1) |
| 151 |
return -EINVAL; |
| 152 |
|
| 153 |
hwc->config |= config; |
| 154 |
|
| 155 |
return 0; |
| 156 |
} |
| 157 |
|
| 158 |
static void sh_perf_event_update(struct perf_event *event, |
| 159 |
struct hw_perf_event *hwc, int idx) |
| 160 |
{ |
| 161 |
u64 prev_raw_count, new_raw_count; |
| 162 |
s64 delta; |
| 163 |
int shift = 0; |
| 164 |
|
| 165 |
/* |
| 166 |
* Depending on the counter configuration, they may or may not |
| 167 |
* be chained, in which case the previous counter value can be |
| 168 |
* updated underneath us if the lower-half overflows. |
| 169 |
* |
| 170 |
* Our tactic to handle this is to first atomically read and |
| 171 |
* exchange a new raw count - then add that new-prev delta |
| 172 |
* count to the generic counter atomically. |
| 173 |
* |
| 174 |
* As there is no interrupt associated with the overflow events, |
| 175 |
* this is the simplest approach for maintaining consistency. |
| 176 |
*/ |
| 177 |
again: |
| 178 |
prev_raw_count = local64_read(&hwc->prev_count); |
| 179 |
new_raw_count = sh_pmu->read(idx); |
| 180 |
|
| 181 |
if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, |
| 182 |
new_raw_count) != prev_raw_count) |
| 183 |
goto again; |
| 184 |
|
| 185 |
/* |
| 186 |
* Now we have the new raw value and have updated the prev |
| 187 |
* timestamp already. We can now calculate the elapsed delta |
| 188 |
* (counter-)time and add that to the generic counter. |
| 189 |
* |
| 190 |
* Careful, not all hw sign-extends above the physical width |
| 191 |
* of the count. |
| 192 |
*/ |
| 193 |
delta = (new_raw_count << shift) - (prev_raw_count << shift); |
| 194 |
delta >>= shift; |
| 195 |
|
| 196 |
local64_add(delta, &event->count); |
| 197 |
} |
| 198 |
|
| 199 |
static void sh_pmu_stop(struct perf_event *event, int flags) |
| 200 |
{ |
| 201 |
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
| 202 |
struct hw_perf_event *hwc = &event->hw; |
| 203 |
int idx = hwc->idx; |
| 204 |
|
| 205 |
if (!(event->hw.state & PERF_HES_STOPPED)) { |
| 206 |
sh_pmu->disable(hwc, idx); |
| 207 |
cpuc->events[idx] = NULL; |
| 208 |
event->hw.state |= PERF_HES_STOPPED; |
| 209 |
} |
| 210 |
|
| 211 |
if ((flags & PERF_EF_UPDATE) && !(event->hw.state & PERF_HES_UPTODATE)) { |
| 212 |
sh_perf_event_update(event, &event->hw, idx); |
| 213 |
event->hw.state |= PERF_HES_UPTODATE; |
| 214 |
} |
| 215 |
} |
| 216 |
|
| 217 |
static void sh_pmu_start(struct perf_event *event, int flags) |
| 218 |
{ |
| 219 |
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
| 220 |
struct hw_perf_event *hwc = &event->hw; |
| 221 |
int idx = hwc->idx; |
| 222 |
|
| 223 |
if (WARN_ON_ONCE(idx == -1)) |
| 224 |
return; |
| 225 |
|
| 226 |
if (flags & PERF_EF_RELOAD) |
| 227 |
WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); |
| 228 |
|
| 229 |
cpuc->events[idx] = event; |
| 230 |
event->hw.state = 0; |
| 231 |
sh_pmu->enable(hwc, idx); |
| 232 |
} |
| 233 |
|
| 234 |
static void sh_pmu_del(struct perf_event *event, int flags) |
| 235 |
{ |
| 236 |
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
| 237 |
|
| 238 |
sh_pmu_stop(event, PERF_EF_UPDATE); |
| 239 |
__clear_bit(event->hw.idx, cpuc->used_mask); |
| 240 |
|
| 241 |
perf_event_update_userpage(event); |
| 242 |
} |
| 243 |
|
| 244 |
static int sh_pmu_add(struct perf_event *event, int flags) |
| 245 |
{ |
| 246 |
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
| 247 |
struct hw_perf_event *hwc = &event->hw; |
| 248 |
int idx = hwc->idx; |
| 249 |
int ret = -EAGAIN; |
| 250 |
|
| 251 |
perf_pmu_disable(event->pmu); |
| 252 |
|
| 253 |
if (__test_and_set_bit(idx, cpuc->used_mask)) { |
| 254 |
idx = find_first_zero_bit(cpuc->used_mask, sh_pmu->num_events); |
| 255 |
if (idx == sh_pmu->num_events) |
| 256 |
goto out; |
| 257 |
|
| 258 |
__set_bit(idx, cpuc->used_mask); |
| 259 |
hwc->idx = idx; |
| 260 |
} |
| 261 |
|
| 262 |
sh_pmu->disable(hwc, idx); |
| 263 |
|
| 264 |
event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED; |
| 265 |
if (flags & PERF_EF_START) |
| 266 |
sh_pmu_start(event, PERF_EF_RELOAD); |
| 267 |
|
| 268 |
perf_event_update_userpage(event); |
| 269 |
ret = 0; |
| 270 |
out: |
| 271 |
perf_pmu_enable(event->pmu); |
| 272 |
return ret; |
| 273 |
} |
| 274 |
|
| 275 |
static void sh_pmu_read(struct perf_event *event) |
| 276 |
{ |
| 277 |
sh_perf_event_update(event, &event->hw, event->hw.idx); |
| 278 |
} |
| 279 |
|
| 280 |
static int sh_pmu_event_init(struct perf_event *event) |
| 281 |
{ |
| 282 |
int err; |
| 283 |
|
| 284 |
/* does not support taken branch sampling */ |
| 285 |
if (has_branch_stack(event)) |
| 286 |
return -EOPNOTSUPP; |
| 287 |
|
| 288 |
switch (event->attr.type) { |
| 289 |
case PERF_TYPE_RAW: |
| 290 |
case PERF_TYPE_HW_CACHE: |
| 291 |
case PERF_TYPE_HARDWARE: |
| 292 |
err = __hw_perf_event_init(event); |
| 293 |
break; |
| 294 |
|
| 295 |
default: |
| 296 |
return -ENOENT; |
| 297 |
} |
| 298 |
|
| 299 |
if (unlikely(err)) { |
| 300 |
if (event->destroy) |
| 301 |
event->destroy(event); |
| 302 |
} |
| 303 |
|
| 304 |
return err; |
| 305 |
} |
| 306 |
|
| 307 |
static void sh_pmu_enable(struct pmu *pmu) |
| 308 |
{ |
| 309 |
if (!sh_pmu_initialized()) |
| 310 |
return; |
| 311 |
|
| 312 |
sh_pmu->enable_all(); |
| 313 |
} |
| 314 |
|
| 315 |
static void sh_pmu_disable(struct pmu *pmu) |
| 316 |
{ |
| 317 |
if (!sh_pmu_initialized()) |
| 318 |
return; |
| 319 |
|
| 320 |
sh_pmu->disable_all(); |
| 321 |
} |
| 322 |
|
| 323 |
static struct pmu pmu = { |
| 324 |
.pmu_enable = sh_pmu_enable, |
| 325 |
.pmu_disable = sh_pmu_disable, |
| 326 |
.event_init = sh_pmu_event_init, |
| 327 |
.add = sh_pmu_add, |
| 328 |
.del = sh_pmu_del, |
| 329 |
.start = sh_pmu_start, |
| 330 |
.stop = sh_pmu_stop, |
| 331 |
.read = sh_pmu_read, |
| 332 |
}; |
| 333 |
|
| 334 |
static int sh_pmu_prepare_cpu(unsigned int cpu) |
| 335 |
{ |
| 336 |
struct cpu_hw_events *cpuhw = &per_cpu(cpu_hw_events, cpu); |
| 337 |
|
| 338 |
memset(cpuhw, 0, sizeof(struct cpu_hw_events)); |
| 339 |
return 0; |
| 340 |
} |
| 341 |
|
| 342 |
int register_sh_pmu(struct sh_pmu *_pmu) |
| 343 |
{ |
| 344 |
if (sh_pmu) |
| 345 |
return -EBUSY; |
| 346 |
sh_pmu = _pmu; |
| 347 |
|
| 348 |
pr_info("Performance Events: %s support registered\n", _pmu->name); |
| 349 |
|
| 350 |
/* |
| 351 |
* All of the on-chip counters are "limited", in that they have |
| 352 |
* no interrupts, and are therefore unable to do sampling without |
| 353 |
* further work and timer assistance. |
| 354 |
*/ |
| 355 |
pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT; |
| 356 |
|
| 357 |
WARN_ON(_pmu->num_events > MAX_HWEVENTS); |
| 358 |
|
| 359 |
perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW); |
| 360 |
cpuhp_setup_state(CPUHP_PERF_SUPERH, "PERF_SUPERH", sh_pmu_prepare_cpu, |
| 361 |
NULL); |
| 362 |
return 0; |
| 363 |
} |