| 1 |
// SPDX-License-Identifier: GPL-2.0 |
| 2 |
/* |
| 3 |
* Ampere SoC PMU (Performance Monitor Unit) |
| 4 |
* |
| 5 |
* Copyright (c) 2023, Ampere Computing LLC |
| 6 |
*/ |
| 7 |
#include <linux/io.h> |
| 8 |
#include <linux/module.h> |
| 9 |
#include <linux/topology.h> |
| 10 |
|
| 11 |
#include "arm_cspmu.h" |
| 12 |
|
| 13 |
#define PMAUXR0 PMIMPDEF |
| 14 |
#define PMAUXR1 (PMIMPDEF + 0x4) |
| 15 |
#define PMAUXR2 (PMIMPDEF + 0x8) |
| 16 |
#define PMAUXR3 (PMIMPDEF + 0xC) |
| 17 |
|
| 18 |
#define to_ampere_cspmu_ctx(cspmu) ((struct ampere_cspmu_ctx *)(cspmu->impl.ctx)) |
| 19 |
|
| 20 |
struct ampere_cspmu_ctx { |
| 21 |
const char *name; |
| 22 |
struct attribute **event_attr; |
| 23 |
struct attribute **format_attr; |
| 24 |
}; |
| 25 |
|
| 26 |
static DEFINE_IDA(mcu_pmu_ida); |
| 27 |
|
| 28 |
#define SOC_PMU_EVENT_ATTR_EXTRACTOR(_name, _config, _start, _end) \ |
| 29 |
static inline u32 get_##_name(const struct perf_event *event) \ |
| 30 |
{ \ |
| 31 |
return FIELD_GET(GENMASK_ULL(_end, _start), \ |
| 32 |
event->attr._config); \ |
| 33 |
} \ |
| 34 |
|
| 35 |
SOC_PMU_EVENT_ATTR_EXTRACTOR(event, config, 0, 8); |
| 36 |
SOC_PMU_EVENT_ATTR_EXTRACTOR(threshold, config1, 0, 7); |
| 37 |
SOC_PMU_EVENT_ATTR_EXTRACTOR(rank, config1, 8, 23); |
| 38 |
SOC_PMU_EVENT_ATTR_EXTRACTOR(bank, config1, 24, 55); |
| 39 |
|
| 40 |
static struct attribute *ampereone_mcu_pmu_event_attrs[] = { |
| 41 |
ARM_CSPMU_EVENT_ATTR(cycle_count, 0x00), |
| 42 |
ARM_CSPMU_EVENT_ATTR(act_sent, 0x01), |
| 43 |
ARM_CSPMU_EVENT_ATTR(pre_sent, 0x02), |
| 44 |
ARM_CSPMU_EVENT_ATTR(rd_sent, 0x03), |
| 45 |
ARM_CSPMU_EVENT_ATTR(rda_sent, 0x04), |
| 46 |
ARM_CSPMU_EVENT_ATTR(wr_sent, 0x05), |
| 47 |
ARM_CSPMU_EVENT_ATTR(wra_sent, 0x06), |
| 48 |
ARM_CSPMU_EVENT_ATTR(pd_entry_vld, 0x07), |
| 49 |
ARM_CSPMU_EVENT_ATTR(sref_entry_vld, 0x08), |
| 50 |
ARM_CSPMU_EVENT_ATTR(prea_sent, 0x09), |
| 51 |
ARM_CSPMU_EVENT_ATTR(pre_sb_sent, 0x0a), |
| 52 |
ARM_CSPMU_EVENT_ATTR(ref_sent, 0x0b), |
| 53 |
ARM_CSPMU_EVENT_ATTR(rfm_sent, 0x0c), |
| 54 |
ARM_CSPMU_EVENT_ATTR(ref_sb_sent, 0x0d), |
| 55 |
ARM_CSPMU_EVENT_ATTR(rfm_sb_sent, 0x0e), |
| 56 |
ARM_CSPMU_EVENT_ATTR(rd_rda_sent, 0x0f), |
| 57 |
ARM_CSPMU_EVENT_ATTR(wr_wra_sent, 0x10), |
| 58 |
ARM_CSPMU_EVENT_ATTR(raw_hazard, 0x11), |
| 59 |
ARM_CSPMU_EVENT_ATTR(war_hazard, 0x12), |
| 60 |
ARM_CSPMU_EVENT_ATTR(waw_hazard, 0x13), |
| 61 |
ARM_CSPMU_EVENT_ATTR(rar_hazard, 0x14), |
| 62 |
ARM_CSPMU_EVENT_ATTR(raw_war_waw_hazard, 0x15), |
| 63 |
ARM_CSPMU_EVENT_ATTR(hprd_lprd_wr_req_vld, 0x16), |
| 64 |
ARM_CSPMU_EVENT_ATTR(lprd_req_vld, 0x17), |
| 65 |
ARM_CSPMU_EVENT_ATTR(hprd_req_vld, 0x18), |
| 66 |
ARM_CSPMU_EVENT_ATTR(hprd_lprd_req_vld, 0x19), |
| 67 |
ARM_CSPMU_EVENT_ATTR(prefetch_tgt, 0x1a), |
| 68 |
ARM_CSPMU_EVENT_ATTR(wr_req_vld, 0x1b), |
| 69 |
ARM_CSPMU_EVENT_ATTR(partial_wr_req_vld, 0x1c), |
| 70 |
ARM_CSPMU_EVENT_ATTR(rd_retry, 0x1d), |
| 71 |
ARM_CSPMU_EVENT_ATTR(wr_retry, 0x1e), |
| 72 |
ARM_CSPMU_EVENT_ATTR(retry_gnt, 0x1f), |
| 73 |
ARM_CSPMU_EVENT_ATTR(rank_change, 0x20), |
| 74 |
ARM_CSPMU_EVENT_ATTR(dir_change, 0x21), |
| 75 |
ARM_CSPMU_EVENT_ATTR(rank_dir_change, 0x22), |
| 76 |
ARM_CSPMU_EVENT_ATTR(rank_active, 0x23), |
| 77 |
ARM_CSPMU_EVENT_ATTR(rank_idle, 0x24), |
| 78 |
ARM_CSPMU_EVENT_ATTR(rank_pd, 0x25), |
| 79 |
ARM_CSPMU_EVENT_ATTR(rank_sref, 0x26), |
| 80 |
ARM_CSPMU_EVENT_ATTR(queue_fill_gt_thresh, 0x27), |
| 81 |
ARM_CSPMU_EVENT_ATTR(queue_rds_gt_thresh, 0x28), |
| 82 |
ARM_CSPMU_EVENT_ATTR(queue_wrs_gt_thresh, 0x29), |
| 83 |
ARM_CSPMU_EVENT_ATTR(phy_updt_complt, 0x2a), |
| 84 |
ARM_CSPMU_EVENT_ATTR(tz_fail, 0x2b), |
| 85 |
ARM_CSPMU_EVENT_ATTR(dram_errc, 0x2c), |
| 86 |
ARM_CSPMU_EVENT_ATTR(dram_errd, 0x2d), |
| 87 |
ARM_CSPMU_EVENT_ATTR(read_data_return, 0x32), |
| 88 |
ARM_CSPMU_EVENT_ATTR(chi_wr_data_delta, 0x33), |
| 89 |
ARM_CSPMU_EVENT_ATTR(zq_start, 0x34), |
| 90 |
ARM_CSPMU_EVENT_ATTR(zq_latch, 0x35), |
| 91 |
ARM_CSPMU_EVENT_ATTR(wr_fifo_full, 0x36), |
| 92 |
ARM_CSPMU_EVENT_ATTR(info_fifo_full, 0x37), |
| 93 |
ARM_CSPMU_EVENT_ATTR(cmd_fifo_full, 0x38), |
| 94 |
ARM_CSPMU_EVENT_ATTR(dfi_nop, 0x39), |
| 95 |
ARM_CSPMU_EVENT_ATTR(dfi_cmd, 0x3a), |
| 96 |
ARM_CSPMU_EVENT_ATTR(rd_run_len, 0x3b), |
| 97 |
ARM_CSPMU_EVENT_ATTR(wr_run_len, 0x3c), |
| 98 |
|
| 99 |
ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT), |
| 100 |
NULL, |
| 101 |
}; |
| 102 |
|
| 103 |
static struct attribute *ampereone_mcu_format_attrs[] = { |
| 104 |
ARM_CSPMU_FORMAT_EVENT_ATTR, |
| 105 |
ARM_CSPMU_FORMAT_ATTR(threshold, "config1:0-7"), |
| 106 |
ARM_CSPMU_FORMAT_ATTR(rank, "config1:8-23"), |
| 107 |
ARM_CSPMU_FORMAT_ATTR(bank, "config1:24-55"), |
| 108 |
NULL, |
| 109 |
}; |
| 110 |
|
| 111 |
static struct attribute ** |
| 112 |
ampere_cspmu_get_event_attrs(const struct arm_cspmu *cspmu) |
| 113 |
{ |
| 114 |
const struct ampere_cspmu_ctx *ctx = to_ampere_cspmu_ctx(cspmu); |
| 115 |
|
| 116 |
return ctx->event_attr; |
| 117 |
} |
| 118 |
|
| 119 |
static struct attribute ** |
| 120 |
ampere_cspmu_get_format_attrs(const struct arm_cspmu *cspmu) |
| 121 |
{ |
| 122 |
const struct ampere_cspmu_ctx *ctx = to_ampere_cspmu_ctx(cspmu); |
| 123 |
|
| 124 |
return ctx->format_attr; |
| 125 |
} |
| 126 |
|
| 127 |
static const char * |
| 128 |
ampere_cspmu_get_name(const struct arm_cspmu *cspmu) |
| 129 |
{ |
| 130 |
const struct ampere_cspmu_ctx *ctx = to_ampere_cspmu_ctx(cspmu); |
| 131 |
|
| 132 |
return ctx->name; |
| 133 |
} |
| 134 |
|
| 135 |
static void ampere_cspmu_set_cc_filter(struct arm_cspmu *cspmu, |
| 136 |
const struct perf_event *event) |
| 137 |
{ |
| 138 |
/* |
| 139 |
* PMCCFILTR is RES0, so this is just a dummy callback to override |
| 140 |
* the default implementation and avoid writing to it. |
| 141 |
*/ |
| 142 |
} |
| 143 |
|
| 144 |
static void ampere_cspmu_set_ev_filter(struct arm_cspmu *cspmu, |
| 145 |
const struct perf_event *event) |
| 146 |
{ |
| 147 |
u32 threshold, rank, bank; |
| 148 |
|
| 149 |
threshold = get_threshold(event); |
| 150 |
rank = get_rank(event); |
| 151 |
bank = get_bank(event); |
| 152 |
|
| 153 |
writel(threshold, cspmu->base0 + PMAUXR0); |
| 154 |
writel(rank, cspmu->base0 + PMAUXR1); |
| 155 |
writel(bank, cspmu->base0 + PMAUXR2); |
| 156 |
} |
| 157 |
|
| 158 |
static int ampere_cspmu_validate_configs(struct perf_event *event, |
| 159 |
struct perf_event *event2) |
| 160 |
{ |
| 161 |
if (get_threshold(event) != get_threshold(event2) || |
| 162 |
get_rank(event) != get_rank(event2) || |
| 163 |
get_bank(event) != get_bank(event2)) |
| 164 |
return -EINVAL; |
| 165 |
|
| 166 |
return 0; |
| 167 |
} |
| 168 |
|
| 169 |
static int ampere_cspmu_validate_event(struct arm_cspmu *cspmu, |
| 170 |
struct perf_event *new) |
| 171 |
{ |
| 172 |
struct perf_event *curr, *leader = new->group_leader; |
| 173 |
unsigned int idx; |
| 174 |
int ret; |
| 175 |
|
| 176 |
ret = ampere_cspmu_validate_configs(new, leader); |
| 177 |
if (ret) |
| 178 |
return ret; |
| 179 |
|
| 180 |
/* We compare the global filter settings to the existing events */ |
| 181 |
idx = find_first_bit(cspmu->hw_events.used_ctrs, |
| 182 |
cspmu->cycle_counter_logical_idx); |
| 183 |
|
| 184 |
/* This is the first event, thus any configuration is fine */ |
| 185 |
if (idx == cspmu->cycle_counter_logical_idx) |
| 186 |
return 0; |
| 187 |
|
| 188 |
curr = cspmu->hw_events.events[idx]; |
| 189 |
|
| 190 |
return ampere_cspmu_validate_configs(curr, new); |
| 191 |
} |
| 192 |
|
| 193 |
static char *ampere_cspmu_format_name(const struct arm_cspmu *cspmu, |
| 194 |
const char *name_pattern) |
| 195 |
{ |
| 196 |
struct device *dev = cspmu->dev; |
| 197 |
int id; |
| 198 |
|
| 199 |
id = ida_alloc(&mcu_pmu_ida, GFP_KERNEL); |
| 200 |
if (id < 0) |
| 201 |
return ERR_PTR(id); |
| 202 |
|
| 203 |
return devm_kasprintf(dev, GFP_KERNEL, name_pattern, id); |
| 204 |
} |
| 205 |
|
| 206 |
static int ampere_cspmu_init_ops(struct arm_cspmu *cspmu) |
| 207 |
{ |
| 208 |
struct device *dev = cspmu->dev; |
| 209 |
struct ampere_cspmu_ctx *ctx; |
| 210 |
struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops; |
| 211 |
|
| 212 |
ctx = devm_kzalloc(dev, sizeof(struct ampere_cspmu_ctx), GFP_KERNEL); |
| 213 |
if (!ctx) |
| 214 |
return -ENOMEM; |
| 215 |
|
| 216 |
ctx->event_attr = ampereone_mcu_pmu_event_attrs; |
| 217 |
ctx->format_attr = ampereone_mcu_format_attrs; |
| 218 |
ctx->name = ampere_cspmu_format_name(cspmu, "ampere_mcu_pmu_%d"); |
| 219 |
if (IS_ERR_OR_NULL(ctx->name)) |
| 220 |
return ctx->name ? PTR_ERR(ctx->name) : -ENOMEM; |
| 221 |
|
| 222 |
cspmu->impl.ctx = ctx; |
| 223 |
|
| 224 |
impl_ops->set_cc_filter = ampere_cspmu_set_cc_filter; |
| 225 |
impl_ops->set_ev_filter = ampere_cspmu_set_ev_filter; |
| 226 |
impl_ops->validate_event = ampere_cspmu_validate_event; |
| 227 |
impl_ops->get_name = ampere_cspmu_get_name; |
| 228 |
impl_ops->get_event_attrs = ampere_cspmu_get_event_attrs; |
| 229 |
impl_ops->get_format_attrs = ampere_cspmu_get_format_attrs; |
| 230 |
|
| 231 |
return 0; |
| 232 |
} |
| 233 |
|
| 234 |
/* Match all Ampere Coresight PMU devices */ |
| 235 |
static const struct arm_cspmu_impl_match ampere_cspmu_param = { |
| 236 |
.pmiidr_val = ARM_CSPMU_IMPL_ID_AMPERE, |
| 237 |
.module = THIS_MODULE, |
| 238 |
.impl_init_ops = ampere_cspmu_init_ops |
| 239 |
}; |
| 240 |
|
| 241 |
static int __init ampere_cspmu_init(void) |
| 242 |
{ |
| 243 |
int ret; |
| 244 |
|
| 245 |
ret = arm_cspmu_impl_register(&ere_cspmu_param); |
| 246 |
if (ret) |
| 247 |
pr_err("ampere_cspmu backend registration error: %d\n", ret); |
| 248 |
|
| 249 |
return ret; |
| 250 |
} |
| 251 |
|
| 252 |
static void __exit ampere_cspmu_exit(void) |
| 253 |
{ |
| 254 |
arm_cspmu_impl_unregister(&ere_cspmu_param); |
| 255 |
} |
| 256 |
|
| 257 |
module_init(ampere_cspmu_init); |
| 258 |
module_exit(ampere_cspmu_exit); |
| 259 |
|
| 260 |
MODULE_DESCRIPTION("Ampere SoC Performance Monitor Driver"); |
| 261 |
MODULE_LICENSE("GPL"); |