| 1 |
// SPDX-License-Identifier: GPL-2.0 |
| 2 |
/* |
| 3 |
* Synopsys DesignWare PCIe PMU driver |
| 4 |
* |
| 5 |
* Copyright (C) 2021-2023 Alibaba Inc. |
| 6 |
*/ |
| 7 |
|
| 8 |
#include <linux/bitfield.h> |
| 9 |
#include <linux/bitops.h> |
| 10 |
#include <linux/cpuhotplug.h> |
| 11 |
#include <linux/cpumask.h> |
| 12 |
#include <linux/device.h> |
| 13 |
#include <linux/errno.h> |
| 14 |
#include <linux/kernel.h> |
| 15 |
#include <linux/list.h> |
| 16 |
#include <linux/pcie-dwc.h> |
| 17 |
#include <linux/perf_event.h> |
| 18 |
#include <linux/pci.h> |
| 19 |
#include <linux/platform_device.h> |
| 20 |
#include <linux/smp.h> |
| 21 |
#include <linux/sysfs.h> |
| 22 |
#include <linux/types.h> |
| 23 |
|
| 24 |
#define DWC_PCIE_EVENT_CNT_CTL 0x8 |
| 25 |
|
| 26 |
/* |
| 27 |
* Event Counter Data Select includes two parts: |
| 28 |
* - 27-24: Group number(4-bit: 0..0x7) |
| 29 |
* - 23-16: Event number(8-bit: 0..0x13) within the Group |
| 30 |
* |
| 31 |
* Put them together as in TRM. |
| 32 |
*/ |
| 33 |
#define DWC_PCIE_CNT_EVENT_SEL GENMASK(27, 16) |
| 34 |
#define DWC_PCIE_CNT_LANE_SEL GENMASK(11, 8) |
| 35 |
#define DWC_PCIE_CNT_STATUS BIT(7) |
| 36 |
#define DWC_PCIE_CNT_ENABLE GENMASK(4, 2) |
| 37 |
#define DWC_PCIE_PER_EVENT_OFF 0x1 |
| 38 |
#define DWC_PCIE_PER_EVENT_ON 0x3 |
| 39 |
#define DWC_PCIE_EVENT_CLEAR GENMASK(1, 0) |
| 40 |
#define DWC_PCIE_EVENT_PER_CLEAR 0x1 |
| 41 |
|
| 42 |
/* Event Selection Field has two subfields */ |
| 43 |
#define DWC_PCIE_CNT_EVENT_SEL_GROUP GENMASK(11, 8) |
| 44 |
#define DWC_PCIE_CNT_EVENT_SEL_EVID GENMASK(7, 0) |
| 45 |
|
| 46 |
#define DWC_PCIE_EVENT_CNT_DATA 0xC |
| 47 |
|
| 48 |
#define DWC_PCIE_TIME_BASED_ANAL_CTL 0x10 |
| 49 |
#define DWC_PCIE_TIME_BASED_REPORT_SEL GENMASK(31, 24) |
| 50 |
#define DWC_PCIE_TIME_BASED_DURATION_SEL GENMASK(15, 8) |
| 51 |
#define DWC_PCIE_DURATION_MANUAL_CTL 0x0 |
| 52 |
#define DWC_PCIE_DURATION_1MS 0x1 |
| 53 |
#define DWC_PCIE_DURATION_10MS 0x2 |
| 54 |
#define DWC_PCIE_DURATION_100MS 0x3 |
| 55 |
#define DWC_PCIE_DURATION_1S 0x4 |
| 56 |
#define DWC_PCIE_DURATION_2S 0x5 |
| 57 |
#define DWC_PCIE_DURATION_4S 0x6 |
| 58 |
#define DWC_PCIE_DURATION_4US 0xFF |
| 59 |
#define DWC_PCIE_TIME_BASED_TIMER_START BIT(0) |
| 60 |
#define DWC_PCIE_TIME_BASED_CNT_ENABLE 0x1 |
| 61 |
|
| 62 |
#define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW 0x14 |
| 63 |
#define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH 0x18 |
| 64 |
|
| 65 |
/* Event attributes */ |
| 66 |
#define DWC_PCIE_CONFIG_EVENTID GENMASK(15, 0) |
| 67 |
#define DWC_PCIE_CONFIG_TYPE GENMASK(19, 16) |
| 68 |
#define DWC_PCIE_CONFIG_LANE GENMASK(27, 20) |
| 69 |
|
| 70 |
#define DWC_PCIE_EVENT_ID(event) FIELD_GET(DWC_PCIE_CONFIG_EVENTID, (event)->attr.config) |
| 71 |
#define DWC_PCIE_EVENT_TYPE(event) FIELD_GET(DWC_PCIE_CONFIG_TYPE, (event)->attr.config) |
| 72 |
#define DWC_PCIE_EVENT_LANE(event) FIELD_GET(DWC_PCIE_CONFIG_LANE, (event)->attr.config) |
| 73 |
|
| 74 |
enum dwc_pcie_event_type { |
| 75 |
DWC_PCIE_TIME_BASE_EVENT, |
| 76 |
DWC_PCIE_LANE_EVENT, |
| 77 |
DWC_PCIE_EVENT_TYPE_MAX, |
| 78 |
}; |
| 79 |
|
| 80 |
#define DWC_PCIE_LANE_GROUP_6 6 |
| 81 |
#define DWC_PCIE_LANE_GROUP_7 7 |
| 82 |
#define DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP 256 |
| 83 |
|
| 84 |
#define DWC_PCIE_LANE_EVENT_MAX_PERIOD GENMASK_ULL(31, 0) |
| 85 |
#define DWC_PCIE_MAX_PERIOD GENMASK_ULL(63, 0) |
| 86 |
|
| 87 |
struct dwc_pcie_pmu { |
| 88 |
struct pmu pmu; |
| 89 |
struct pci_dev *pdev; /* Root Port device */ |
| 90 |
u16 ras_des_offset; |
| 91 |
u32 nr_lanes; |
| 92 |
|
| 93 |
/* Groups #6 and #7 */ |
| 94 |
DECLARE_BITMAP(lane_events, 2 * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP); |
| 95 |
struct perf_event *time_based_event; |
| 96 |
|
| 97 |
struct hlist_node cpuhp_node; |
| 98 |
int on_cpu; |
| 99 |
}; |
| 100 |
|
| 101 |
#define to_dwc_pcie_pmu(p) (container_of(p, struct dwc_pcie_pmu, pmu)) |
| 102 |
|
| 103 |
static int dwc_pcie_pmu_hp_state; |
| 104 |
static struct list_head dwc_pcie_dev_info_head = |
| 105 |
LIST_HEAD_INIT(dwc_pcie_dev_info_head); |
| 106 |
static bool notify; |
| 107 |
|
| 108 |
struct dwc_pcie_dev_info { |
| 109 |
struct platform_device *plat_dev; |
| 110 |
struct pci_dev *pdev; |
| 111 |
struct list_head dev_node; |
| 112 |
}; |
| 113 |
|
| 114 |
static ssize_t cpumask_show(struct device *dev, |
| 115 |
struct device_attribute *attr, |
| 116 |
char *buf) |
| 117 |
{ |
| 118 |
struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(dev_get_drvdata(dev)); |
| 119 |
|
| 120 |
return cpumap_print_to_pagebuf(true, buf, cpumask_of(pcie_pmu->on_cpu)); |
| 121 |
} |
| 122 |
static DEVICE_ATTR_RO(cpumask); |
| 123 |
|
| 124 |
static struct attribute *dwc_pcie_pmu_cpumask_attrs[] = { |
| 125 |
&dev_attr_cpumask.attr, |
| 126 |
NULL |
| 127 |
}; |
| 128 |
|
| 129 |
static struct attribute_group dwc_pcie_cpumask_attr_group = { |
| 130 |
.attrs = dwc_pcie_pmu_cpumask_attrs, |
| 131 |
}; |
| 132 |
|
| 133 |
struct dwc_pcie_format_attr { |
| 134 |
struct device_attribute attr; |
| 135 |
u64 field; |
| 136 |
int config; |
| 137 |
}; |
| 138 |
|
| 139 |
PMU_FORMAT_ATTR(eventid, "config:0-15"); |
| 140 |
PMU_FORMAT_ATTR(type, "config:16-19"); |
| 141 |
PMU_FORMAT_ATTR(lane, "config:20-27"); |
| 142 |
|
| 143 |
static struct attribute *dwc_pcie_format_attrs[] = { |
| 144 |
&format_attr_type.attr, |
| 145 |
&format_attr_eventid.attr, |
| 146 |
&format_attr_lane.attr, |
| 147 |
NULL, |
| 148 |
}; |
| 149 |
|
| 150 |
static struct attribute_group dwc_pcie_format_attrs_group = { |
| 151 |
.name = "format", |
| 152 |
.attrs = dwc_pcie_format_attrs, |
| 153 |
}; |
| 154 |
|
| 155 |
struct dwc_pcie_event_attr { |
| 156 |
struct device_attribute attr; |
| 157 |
enum dwc_pcie_event_type type; |
| 158 |
u16 eventid; |
| 159 |
u8 lane; |
| 160 |
}; |
| 161 |
|
| 162 |
static ssize_t dwc_pcie_event_show(struct device *dev, |
| 163 |
struct device_attribute *attr, char *buf) |
| 164 |
{ |
| 165 |
struct dwc_pcie_event_attr *eattr; |
| 166 |
|
| 167 |
eattr = container_of(attr, typeof(*eattr), attr); |
| 168 |
|
| 169 |
if (eattr->type == DWC_PCIE_LANE_EVENT) |
| 170 |
return sysfs_emit(buf, "eventid=0x%x,type=0x%x,lane=?\n", |
| 171 |
eattr->eventid, eattr->type); |
| 172 |
else if (eattr->type == DWC_PCIE_TIME_BASE_EVENT) |
| 173 |
return sysfs_emit(buf, "eventid=0x%x,type=0x%x\n", |
| 174 |
eattr->eventid, eattr->type); |
| 175 |
|
| 176 |
return 0; |
| 177 |
} |
| 178 |
|
| 179 |
#define DWC_PCIE_EVENT_ATTR(_name, _type, _eventid, _lane) \ |
| 180 |
(&((struct dwc_pcie_event_attr[]) {{ \ |
| 181 |
.attr = __ATTR(_name, 0444, dwc_pcie_event_show, NULL), \ |
| 182 |
.type = _type, \ |
| 183 |
.eventid = _eventid, \ |
| 184 |
.lane = _lane, \ |
| 185 |
}})[0].attr.attr) |
| 186 |
|
| 187 |
#define DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(_name, _eventid) \ |
| 188 |
DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_TIME_BASE_EVENT, _eventid, 0) |
| 189 |
#define DWC_PCIE_PMU_LANE_EVENT_ATTR(_name, _eventid) \ |
| 190 |
DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_LANE_EVENT, _eventid, 0) |
| 191 |
|
| 192 |
static struct attribute *dwc_pcie_pmu_time_event_attrs[] = { |
| 193 |
/* Group #0 */ |
| 194 |
DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(one_cycle, 0x00), |
| 195 |
DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_L0S, 0x01), |
| 196 |
DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(RX_L0S, 0x02), |
| 197 |
DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L0, 0x03), |
| 198 |
DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1, 0x04), |
| 199 |
DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_1, 0x05), |
| 200 |
DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_2, 0x06), |
| 201 |
DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(CFG_RCVRY, 0x07), |
| 202 |
DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_AUX, 0x08), |
| 203 |
DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_RX_L0S, 0x09), |
| 204 |
|
| 205 |
/* Group #1 */ |
| 206 |
DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(tx_pcie_tlp_data_payload, 0x20), |
| 207 |
DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(rx_pcie_tlp_data_payload, 0x21), |
| 208 |
DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(tx_ccix_tlp_data_payload, 0x22), |
| 209 |
DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(rx_ccix_tlp_data_payload, 0x23), |
| 210 |
|
| 211 |
/* |
| 212 |
* Leave it to the user to specify the lane ID to avoid generating |
| 213 |
* a list of hundreds of events. |
| 214 |
*/ |
| 215 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ack_dllp, 0x600), |
| 216 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_update_fc_dllp, 0x601), |
| 217 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ack_dllp, 0x602), |
| 218 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_update_fc_dllp, 0x603), |
| 219 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_nullified_tlp, 0x604), |
| 220 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_nullified_tlp, 0x605), |
| 221 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_duplicate_tlp, 0x606), |
| 222 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_write, 0x700), |
| 223 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_read, 0x701), |
| 224 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_write, 0x702), |
| 225 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_read, 0x703), |
| 226 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_write, 0x704), |
| 227 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_read, 0x705), |
| 228 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_without_data, 0x706), |
| 229 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_with_data, 0x707), |
| 230 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_message_tlp, 0x708), |
| 231 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_atomic, 0x709), |
| 232 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_tlp_with_prefix, 0x70A), |
| 233 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_write, 0x70B), |
| 234 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_read, 0x70C), |
| 235 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_write, 0x70F), |
| 236 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_read, 0x710), |
| 237 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_without_data, 0x711), |
| 238 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_with_data, 0x712), |
| 239 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_message_tlp, 0x713), |
| 240 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_atomic, 0x714), |
| 241 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_tlp_with_prefix, 0x715), |
| 242 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ccix_tlp, 0x716), |
| 243 |
DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ccix_tlp, 0x717), |
| 244 |
NULL |
| 245 |
}; |
| 246 |
|
| 247 |
static const struct attribute_group dwc_pcie_event_attrs_group = { |
| 248 |
.name = "events", |
| 249 |
.attrs = dwc_pcie_pmu_time_event_attrs, |
| 250 |
}; |
| 251 |
|
| 252 |
static const struct attribute_group *dwc_pcie_attr_groups[] = { |
| 253 |
&dwc_pcie_event_attrs_group, |
| 254 |
&dwc_pcie_format_attrs_group, |
| 255 |
&dwc_pcie_cpumask_attr_group, |
| 256 |
NULL |
| 257 |
}; |
| 258 |
|
| 259 |
static void dwc_pcie_pmu_lane_event_enable(struct dwc_pcie_pmu *pcie_pmu, |
| 260 |
struct perf_event *event, |
| 261 |
bool enable) |
| 262 |
{ |
| 263 |
struct pci_dev *pdev = pcie_pmu->pdev; |
| 264 |
u16 ras_des_offset = pcie_pmu->ras_des_offset; |
| 265 |
int event_id = DWC_PCIE_EVENT_ID(event); |
| 266 |
int lane = DWC_PCIE_EVENT_LANE(event); |
| 267 |
u32 ctrl; |
| 268 |
|
| 269 |
ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) | |
| 270 |
FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) | |
| 271 |
FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR); |
| 272 |
|
| 273 |
if (enable) |
| 274 |
ctrl |= FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_ON); |
| 275 |
else |
| 276 |
ctrl |= FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF); |
| 277 |
|
| 278 |
pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL, |
| 279 |
ctrl); |
| 280 |
} |
| 281 |
|
| 282 |
static void dwc_pcie_pmu_time_based_event_enable(struct dwc_pcie_pmu *pcie_pmu, |
| 283 |
bool enable) |
| 284 |
{ |
| 285 |
struct pci_dev *pdev = pcie_pmu->pdev; |
| 286 |
u16 ras_des_offset = pcie_pmu->ras_des_offset; |
| 287 |
|
| 288 |
pci_clear_and_set_config_dword(pdev, |
| 289 |
ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL, |
| 290 |
DWC_PCIE_TIME_BASED_TIMER_START, enable); |
| 291 |
} |
| 292 |
|
| 293 |
static u64 dwc_pcie_pmu_read_lane_event_counter(struct perf_event *event) |
| 294 |
{ |
| 295 |
struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); |
| 296 |
struct pci_dev *pdev = pcie_pmu->pdev; |
| 297 |
int event_id = DWC_PCIE_EVENT_ID(event); |
| 298 |
int lane = DWC_PCIE_EVENT_LANE(event); |
| 299 |
u16 ras_des_offset = pcie_pmu->ras_des_offset; |
| 300 |
u32 val, ctrl; |
| 301 |
|
| 302 |
ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) | |
| 303 |
FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) | |
| 304 |
FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_ON); |
| 305 |
pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL, |
| 306 |
ctrl); |
| 307 |
pci_read_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_DATA, &val); |
| 308 |
|
| 309 |
ctrl |= FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR); |
| 310 |
pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL, |
| 311 |
ctrl); |
| 312 |
|
| 313 |
return val; |
| 314 |
} |
| 315 |
|
| 316 |
static u64 dwc_pcie_pmu_read_time_based_counter(struct perf_event *event) |
| 317 |
{ |
| 318 |
struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); |
| 319 |
struct pci_dev *pdev = pcie_pmu->pdev; |
| 320 |
int event_id = DWC_PCIE_EVENT_ID(event); |
| 321 |
u16 ras_des_offset = pcie_pmu->ras_des_offset; |
| 322 |
u32 lo, hi, ss; |
| 323 |
u64 val; |
| 324 |
|
| 325 |
/* |
| 326 |
* The 64-bit value of the data counter is spread across two |
| 327 |
* registers that are not synchronized. In order to read them |
| 328 |
* atomically, ensure that the high 32 bits match before and after |
| 329 |
* reading the low 32 bits. |
| 330 |
*/ |
| 331 |
pci_read_config_dword(pdev, |
| 332 |
ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH, &hi); |
| 333 |
do { |
| 334 |
/* snapshot the high 32 bits */ |
| 335 |
ss = hi; |
| 336 |
|
| 337 |
pci_read_config_dword( |
| 338 |
pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW, |
| 339 |
&lo); |
| 340 |
pci_read_config_dword( |
| 341 |
pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH, |
| 342 |
&hi); |
| 343 |
} while (hi != ss); |
| 344 |
|
| 345 |
val = ((u64)hi << 32) | lo; |
| 346 |
/* |
| 347 |
* The Group#1 event measures the amount of data processed in 16-byte |
| 348 |
* units. Simplify the end-user interface by multiplying the counter |
| 349 |
* at the point of read. |
| 350 |
*/ |
| 351 |
if (event_id >= 0x20 && event_id <= 0x23) |
| 352 |
val *= 16; |
| 353 |
|
| 354 |
return val; |
| 355 |
} |
| 356 |
|
| 357 |
static void dwc_pcie_pmu_event_update(struct perf_event *event) |
| 358 |
{ |
| 359 |
struct hw_perf_event *hwc = &event->hw; |
| 360 |
enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); |
| 361 |
u64 delta, prev, now; |
| 362 |
|
| 363 |
if (type == DWC_PCIE_LANE_EVENT) { |
| 364 |
now = dwc_pcie_pmu_read_lane_event_counter(event) & |
| 365 |
DWC_PCIE_LANE_EVENT_MAX_PERIOD; |
| 366 |
local64_add(now, &event->count); |
| 367 |
return; |
| 368 |
} |
| 369 |
|
| 370 |
do { |
| 371 |
prev = local64_read(&hwc->prev_count); |
| 372 |
now = dwc_pcie_pmu_read_time_based_counter(event); |
| 373 |
|
| 374 |
} while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev); |
| 375 |
|
| 376 |
delta = (now - prev) & DWC_PCIE_MAX_PERIOD; |
| 377 |
local64_add(delta, &event->count); |
| 378 |
} |
| 379 |
|
| 380 |
static int dwc_pcie_pmu_validate_add_lane_event(struct perf_event *event, |
| 381 |
unsigned long val_lane_events[]) |
| 382 |
{ |
| 383 |
int event_id, event_nr, group; |
| 384 |
|
| 385 |
event_id = DWC_PCIE_EVENT_ID(event); |
| 386 |
event_nr = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_EVID, event_id); |
| 387 |
group = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_GROUP, event_id); |
| 388 |
|
| 389 |
if (group != DWC_PCIE_LANE_GROUP_6 && group != DWC_PCIE_LANE_GROUP_7) |
| 390 |
return -EINVAL; |
| 391 |
|
| 392 |
group -= DWC_PCIE_LANE_GROUP_6; |
| 393 |
|
| 394 |
if (test_and_set_bit(group * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP + event_nr, |
| 395 |
val_lane_events)) |
| 396 |
return -EINVAL; |
| 397 |
|
| 398 |
return 0; |
| 399 |
} |
| 400 |
|
| 401 |
static int dwc_pcie_pmu_validate_group(struct perf_event *event) |
| 402 |
{ |
| 403 |
struct perf_event *sibling, *leader = event->group_leader; |
| 404 |
DECLARE_BITMAP(val_lane_events, 2 * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP); |
| 405 |
bool time_event = false; |
| 406 |
int type; |
| 407 |
|
| 408 |
type = DWC_PCIE_EVENT_TYPE(leader); |
| 409 |
if (type == DWC_PCIE_TIME_BASE_EVENT) |
| 410 |
time_event = true; |
| 411 |
else |
| 412 |
if (dwc_pcie_pmu_validate_add_lane_event(leader, val_lane_events)) |
| 413 |
return -ENOSPC; |
| 414 |
|
| 415 |
for_each_sibling_event(sibling, leader) { |
| 416 |
type = DWC_PCIE_EVENT_TYPE(sibling); |
| 417 |
if (type == DWC_PCIE_TIME_BASE_EVENT) { |
| 418 |
if (time_event) |
| 419 |
return -ENOSPC; |
| 420 |
|
| 421 |
time_event = true; |
| 422 |
continue; |
| 423 |
} |
| 424 |
|
| 425 |
if (dwc_pcie_pmu_validate_add_lane_event(sibling, val_lane_events)) |
| 426 |
return -ENOSPC; |
| 427 |
} |
| 428 |
|
| 429 |
return 0; |
| 430 |
} |
| 431 |
|
| 432 |
static int dwc_pcie_pmu_event_init(struct perf_event *event) |
| 433 |
{ |
| 434 |
struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); |
| 435 |
enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); |
| 436 |
struct perf_event *sibling; |
| 437 |
u32 lane; |
| 438 |
|
| 439 |
if (event->attr.type != event->pmu->type) |
| 440 |
return -ENOENT; |
| 441 |
|
| 442 |
/* We don't support sampling */ |
| 443 |
if (is_sampling_event(event)) |
| 444 |
return -EINVAL; |
| 445 |
|
| 446 |
/* We cannot support task bound events */ |
| 447 |
if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK) |
| 448 |
return -EINVAL; |
| 449 |
|
| 450 |
for_each_sibling_event(sibling, event->group_leader) { |
| 451 |
if (sibling->pmu != event->pmu && !is_software_event(sibling)) |
| 452 |
return -EINVAL; |
| 453 |
} |
| 454 |
|
| 455 |
if (type < 0 || type >= DWC_PCIE_EVENT_TYPE_MAX) |
| 456 |
return -EINVAL; |
| 457 |
|
| 458 |
if (type == DWC_PCIE_LANE_EVENT) { |
| 459 |
lane = DWC_PCIE_EVENT_LANE(event); |
| 460 |
if (lane < 0 || lane >= pcie_pmu->nr_lanes) |
| 461 |
return -EINVAL; |
| 462 |
} |
| 463 |
|
| 464 |
if (dwc_pcie_pmu_validate_group(event)) |
| 465 |
return -ENOSPC; |
| 466 |
|
| 467 |
event->cpu = pcie_pmu->on_cpu; |
| 468 |
|
| 469 |
return 0; |
| 470 |
} |
| 471 |
|
| 472 |
static void dwc_pcie_pmu_event_start(struct perf_event *event, int flags) |
| 473 |
{ |
| 474 |
struct hw_perf_event *hwc = &event->hw; |
| 475 |
struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); |
| 476 |
enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); |
| 477 |
|
| 478 |
hwc->state = 0; |
| 479 |
local64_set(&hwc->prev_count, 0); |
| 480 |
|
| 481 |
if (type == DWC_PCIE_LANE_EVENT) |
| 482 |
dwc_pcie_pmu_lane_event_enable(pcie_pmu, event, true); |
| 483 |
else if (type == DWC_PCIE_TIME_BASE_EVENT) |
| 484 |
dwc_pcie_pmu_time_based_event_enable(pcie_pmu, true); |
| 485 |
} |
| 486 |
|
| 487 |
static void dwc_pcie_pmu_event_stop(struct perf_event *event, int flags) |
| 488 |
{ |
| 489 |
struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); |
| 490 |
enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); |
| 491 |
struct hw_perf_event *hwc = &event->hw; |
| 492 |
|
| 493 |
if (event->hw.state & PERF_HES_STOPPED) |
| 494 |
return; |
| 495 |
|
| 496 |
dwc_pcie_pmu_event_update(event); |
| 497 |
|
| 498 |
if (type == DWC_PCIE_LANE_EVENT) |
| 499 |
dwc_pcie_pmu_lane_event_enable(pcie_pmu, event, false); |
| 500 |
else if (type == DWC_PCIE_TIME_BASE_EVENT) |
| 501 |
dwc_pcie_pmu_time_based_event_enable(pcie_pmu, false); |
| 502 |
|
| 503 |
hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; |
| 504 |
} |
| 505 |
|
| 506 |
static int dwc_pcie_pmu_event_add(struct perf_event *event, int flags) |
| 507 |
{ |
| 508 |
struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); |
| 509 |
struct pci_dev *pdev = pcie_pmu->pdev; |
| 510 |
struct hw_perf_event *hwc = &event->hw; |
| 511 |
enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); |
| 512 |
int event_id = DWC_PCIE_EVENT_ID(event); |
| 513 |
int lane = DWC_PCIE_EVENT_LANE(event); |
| 514 |
u16 ras_des_offset = pcie_pmu->ras_des_offset; |
| 515 |
u32 ctrl; |
| 516 |
|
| 517 |
hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; |
| 518 |
|
| 519 |
if (type == DWC_PCIE_LANE_EVENT) { |
| 520 |
int event_nr = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_EVID, event_id); |
| 521 |
int group = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_GROUP, event_id) - |
| 522 |
DWC_PCIE_LANE_GROUP_6; |
| 523 |
|
| 524 |
if (test_and_set_bit(group * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP + event_nr, |
| 525 |
pcie_pmu->lane_events)) |
| 526 |
return -ENOSPC; |
| 527 |
|
| 528 |
/* EVENT_COUNTER_DATA_REG needs clear manually */ |
| 529 |
ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) | |
| 530 |
FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) | |
| 531 |
FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF) | |
| 532 |
FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR); |
| 533 |
pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL, |
| 534 |
ctrl); |
| 535 |
} else if (type == DWC_PCIE_TIME_BASE_EVENT) { |
| 536 |
if (pcie_pmu->time_based_event) |
| 537 |
return -ENOSPC; |
| 538 |
|
| 539 |
pcie_pmu->time_based_event = event; |
| 540 |
|
| 541 |
/* |
| 542 |
* TIME_BASED_ANAL_DATA_REG is a 64 bit register, we can safely |
| 543 |
* use it with any manually controlled duration. And it is |
| 544 |
* cleared when next measurement starts. |
| 545 |
*/ |
| 546 |
ctrl = FIELD_PREP(DWC_PCIE_TIME_BASED_REPORT_SEL, event_id) | |
| 547 |
FIELD_PREP(DWC_PCIE_TIME_BASED_DURATION_SEL, |
| 548 |
DWC_PCIE_DURATION_MANUAL_CTL) | |
| 549 |
DWC_PCIE_TIME_BASED_CNT_ENABLE; |
| 550 |
pci_write_config_dword( |
| 551 |
pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL, ctrl); |
| 552 |
} |
| 553 |
|
| 554 |
if (flags & PERF_EF_START) |
| 555 |
dwc_pcie_pmu_event_start(event, PERF_EF_RELOAD); |
| 556 |
|
| 557 |
perf_event_update_userpage(event); |
| 558 |
|
| 559 |
return 0; |
| 560 |
} |
| 561 |
|
| 562 |
static void dwc_pcie_pmu_event_del(struct perf_event *event, int flags) |
| 563 |
{ |
| 564 |
struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); |
| 565 |
enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); |
| 566 |
|
| 567 |
dwc_pcie_pmu_event_stop(event, flags | PERF_EF_UPDATE); |
| 568 |
perf_event_update_userpage(event); |
| 569 |
|
| 570 |
if (type == DWC_PCIE_TIME_BASE_EVENT) { |
| 571 |
pcie_pmu->time_based_event = NULL; |
| 572 |
} else { |
| 573 |
int event_id = DWC_PCIE_EVENT_ID(event); |
| 574 |
int event_nr = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_EVID, event_id); |
| 575 |
int group = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_GROUP, event_id) - |
| 576 |
DWC_PCIE_LANE_GROUP_6; |
| 577 |
|
| 578 |
clear_bit(group * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP + event_nr, |
| 579 |
pcie_pmu->lane_events); |
| 580 |
} |
| 581 |
} |
| 582 |
|
| 583 |
static void dwc_pcie_pmu_remove_cpuhp_instance(void *hotplug_node) |
| 584 |
{ |
| 585 |
cpuhp_state_remove_instance_nocalls(dwc_pcie_pmu_hp_state, hotplug_node); |
| 586 |
} |
| 587 |
|
| 588 |
/* |
| 589 |
* Find the binded DES capability device info of a PCI device. |
| 590 |
* @pdev: The PCI device. |
| 591 |
*/ |
| 592 |
static struct dwc_pcie_dev_info *dwc_pcie_find_dev_info(struct pci_dev *pdev) |
| 593 |
{ |
| 594 |
struct dwc_pcie_dev_info *dev_info; |
| 595 |
|
| 596 |
list_for_each_entry(dev_info, &dwc_pcie_dev_info_head, dev_node) |
| 597 |
if (dev_info->pdev == pdev) |
| 598 |
return dev_info; |
| 599 |
|
| 600 |
return NULL; |
| 601 |
} |
| 602 |
|
| 603 |
static void dwc_pcie_unregister_pmu(void *data) |
| 604 |
{ |
| 605 |
struct dwc_pcie_pmu *pcie_pmu = data; |
| 606 |
|
| 607 |
perf_pmu_unregister(&pcie_pmu->pmu); |
| 608 |
} |
| 609 |
|
| 610 |
static u16 dwc_pcie_des_cap(struct pci_dev *pdev) |
| 611 |
{ |
| 612 |
const struct dwc_pcie_vsec_id *vid; |
| 613 |
u16 vsec; |
| 614 |
u32 val; |
| 615 |
|
| 616 |
if (!pci_is_pcie(pdev) || !(pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT)) |
| 617 |
return 0; |
| 618 |
|
| 619 |
for (vid = dwc_pcie_rasdes_vsec_ids; vid->vendor_id; vid++) { |
| 620 |
vsec = pci_find_vsec_capability(pdev, vid->vendor_id, |
| 621 |
vid->vsec_id); |
| 622 |
if (vsec) { |
| 623 |
pci_read_config_dword(pdev, vsec + PCI_VNDR_HEADER, |
| 624 |
&val); |
| 625 |
if (PCI_VNDR_HEADER_REV(val) == vid->vsec_rev) { |
| 626 |
pci_dbg(pdev, "Detected PCIe Vendor-Specific Extended Capability RAS DES\n"); |
| 627 |
return vsec; |
| 628 |
} |
| 629 |
} |
| 630 |
} |
| 631 |
return 0; |
| 632 |
} |
| 633 |
|
| 634 |
static void dwc_pcie_unregister_dev(struct dwc_pcie_dev_info *dev_info) |
| 635 |
{ |
| 636 |
platform_device_unregister(dev_info->plat_dev); |
| 637 |
list_del(&dev_info->dev_node); |
| 638 |
kfree(dev_info); |
| 639 |
} |
| 640 |
|
| 641 |
static int dwc_pcie_register_dev(struct pci_dev *pdev) |
| 642 |
{ |
| 643 |
struct platform_device *plat_dev; |
| 644 |
struct dwc_pcie_dev_info *dev_info; |
| 645 |
u32 sbdf; |
| 646 |
|
| 647 |
sbdf = (pci_domain_nr(pdev->bus) << 16) | PCI_DEVID(pdev->bus->number, pdev->devfn); |
| 648 |
plat_dev = platform_device_register_simple("dwc_pcie_pmu", sbdf, NULL, 0); |
| 649 |
if (IS_ERR(plat_dev)) |
| 650 |
return PTR_ERR(plat_dev); |
| 651 |
|
| 652 |
dev_info = kzalloc_obj(*dev_info); |
| 653 |
if (!dev_info) { |
| 654 |
platform_device_unregister(plat_dev); |
| 655 |
return -ENOMEM; |
| 656 |
} |
| 657 |
|
| 658 |
/* Cache platform device to handle pci device hotplug */ |
| 659 |
dev_info->plat_dev = plat_dev; |
| 660 |
dev_info->pdev = pdev; |
| 661 |
list_add(&dev_info->dev_node, &dwc_pcie_dev_info_head); |
| 662 |
|
| 663 |
return 0; |
| 664 |
} |
| 665 |
|
| 666 |
static int dwc_pcie_pmu_notifier(struct notifier_block *nb, |
| 667 |
unsigned long action, void *data) |
| 668 |
{ |
| 669 |
struct device *dev = data; |
| 670 |
struct pci_dev *pdev = to_pci_dev(dev); |
| 671 |
struct dwc_pcie_dev_info *dev_info; |
| 672 |
|
| 673 |
switch (action) { |
| 674 |
case BUS_NOTIFY_ADD_DEVICE: |
| 675 |
if (!dwc_pcie_des_cap(pdev)) |
| 676 |
return NOTIFY_DONE; |
| 677 |
if (dwc_pcie_register_dev(pdev)) |
| 678 |
return NOTIFY_BAD; |
| 679 |
break; |
| 680 |
case BUS_NOTIFY_DEL_DEVICE: |
| 681 |
dev_info = dwc_pcie_find_dev_info(pdev); |
| 682 |
if (!dev_info) |
| 683 |
return NOTIFY_DONE; |
| 684 |
dwc_pcie_unregister_dev(dev_info); |
| 685 |
break; |
| 686 |
} |
| 687 |
|
| 688 |
return NOTIFY_OK; |
| 689 |
} |
| 690 |
|
| 691 |
static struct notifier_block dwc_pcie_pmu_nb = { |
| 692 |
.notifier_call = dwc_pcie_pmu_notifier, |
| 693 |
}; |
| 694 |
|
| 695 |
static int dwc_pcie_pmu_probe(struct platform_device *plat_dev) |
| 696 |
{ |
| 697 |
struct pci_dev *pdev; |
| 698 |
struct dwc_pcie_pmu *pcie_pmu; |
| 699 |
char *name; |
| 700 |
u32 sbdf; |
| 701 |
u16 vsec; |
| 702 |
int ret; |
| 703 |
|
| 704 |
sbdf = plat_dev->id; |
| 705 |
pdev = pci_get_domain_bus_and_slot(sbdf >> 16, PCI_BUS_NUM(sbdf & 0xffff), |
| 706 |
sbdf & 0xff); |
| 707 |
if (!pdev) { |
| 708 |
pr_err("No pdev found for the sbdf 0x%x\n", sbdf); |
| 709 |
return -ENODEV; |
| 710 |
} |
| 711 |
|
| 712 |
vsec = dwc_pcie_des_cap(pdev); |
| 713 |
if (!vsec) |
| 714 |
return -ENODEV; |
| 715 |
|
| 716 |
pci_dev_put(pdev); |
| 717 |
name = devm_kasprintf(&plat_dev->dev, GFP_KERNEL, "dwc_rootport_%x", sbdf); |
| 718 |
if (!name) |
| 719 |
return -ENOMEM; |
| 720 |
|
| 721 |
pcie_pmu = devm_kzalloc(&plat_dev->dev, sizeof(*pcie_pmu), GFP_KERNEL); |
| 722 |
if (!pcie_pmu) |
| 723 |
return -ENOMEM; |
| 724 |
|
| 725 |
pcie_pmu->pdev = pdev; |
| 726 |
pcie_pmu->ras_des_offset = vsec; |
| 727 |
pcie_pmu->nr_lanes = pcie_get_width_cap(pdev); |
| 728 |
pcie_pmu->on_cpu = -1; |
| 729 |
pcie_pmu->pmu = (struct pmu){ |
| 730 |
.name = name, |
| 731 |
.parent = &plat_dev->dev, |
| 732 |
.module = THIS_MODULE, |
| 733 |
.attr_groups = dwc_pcie_attr_groups, |
| 734 |
.capabilities = PERF_PMU_CAP_NO_EXCLUDE, |
| 735 |
.task_ctx_nr = perf_invalid_context, |
| 736 |
.event_init = dwc_pcie_pmu_event_init, |
| 737 |
.add = dwc_pcie_pmu_event_add, |
| 738 |
.del = dwc_pcie_pmu_event_del, |
| 739 |
.start = dwc_pcie_pmu_event_start, |
| 740 |
.stop = dwc_pcie_pmu_event_stop, |
| 741 |
.read = dwc_pcie_pmu_event_update, |
| 742 |
}; |
| 743 |
|
| 744 |
/* Add this instance to the list used by the offline callback */ |
| 745 |
ret = cpuhp_state_add_instance(dwc_pcie_pmu_hp_state, |
| 746 |
&pcie_pmu->cpuhp_node); |
| 747 |
if (ret) { |
| 748 |
pci_err(pdev, "Error %d registering hotplug @%x\n", ret, sbdf); |
| 749 |
return ret; |
| 750 |
} |
| 751 |
|
| 752 |
/* Unwind when platform driver removes */ |
| 753 |
ret = devm_add_action_or_reset(&plat_dev->dev, |
| 754 |
dwc_pcie_pmu_remove_cpuhp_instance, |
| 755 |
&pcie_pmu->cpuhp_node); |
| 756 |
if (ret) |
| 757 |
return ret; |
| 758 |
|
| 759 |
ret = perf_pmu_register(&pcie_pmu->pmu, name, -1); |
| 760 |
if (ret) { |
| 761 |
pci_err(pdev, "Error %d registering PMU @%x\n", ret, sbdf); |
| 762 |
return ret; |
| 763 |
} |
| 764 |
ret = devm_add_action_or_reset(&plat_dev->dev, dwc_pcie_unregister_pmu, |
| 765 |
pcie_pmu); |
| 766 |
if (ret) |
| 767 |
return ret; |
| 768 |
|
| 769 |
return 0; |
| 770 |
} |
| 771 |
|
| 772 |
static int dwc_pcie_pmu_online_cpu(unsigned int cpu, struct hlist_node *cpuhp_node) |
| 773 |
{ |
| 774 |
struct dwc_pcie_pmu *pcie_pmu; |
| 775 |
|
| 776 |
pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node); |
| 777 |
if (pcie_pmu->on_cpu == -1) |
| 778 |
pcie_pmu->on_cpu = cpumask_local_spread( |
| 779 |
0, dev_to_node(&pcie_pmu->pdev->dev)); |
| 780 |
|
| 781 |
return 0; |
| 782 |
} |
| 783 |
|
| 784 |
static int dwc_pcie_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_node) |
| 785 |
{ |
| 786 |
struct dwc_pcie_pmu *pcie_pmu; |
| 787 |
struct pci_dev *pdev; |
| 788 |
unsigned int target; |
| 789 |
int node; |
| 790 |
|
| 791 |
pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node); |
| 792 |
/* Nothing to do if this CPU doesn't own the PMU */ |
| 793 |
if (cpu != pcie_pmu->on_cpu) |
| 794 |
return 0; |
| 795 |
|
| 796 |
pcie_pmu->on_cpu = -1; |
| 797 |
pdev = pcie_pmu->pdev; |
| 798 |
node = dev_to_node(&pdev->dev); |
| 799 |
|
| 800 |
target = cpumask_any_and_but(cpumask_of_node(node), cpu_online_mask, cpu); |
| 801 |
if (target >= nr_cpu_ids) |
| 802 |
target = cpumask_any_but(cpu_online_mask, cpu); |
| 803 |
|
| 804 |
if (target >= nr_cpu_ids) { |
| 805 |
pci_err(pdev, "There is no CPU to set\n"); |
| 806 |
return 0; |
| 807 |
} |
| 808 |
|
| 809 |
/* This PMU does NOT support interrupt, just migrate context. */ |
| 810 |
perf_pmu_migrate_context(&pcie_pmu->pmu, cpu, target); |
| 811 |
pcie_pmu->on_cpu = target; |
| 812 |
|
| 813 |
return 0; |
| 814 |
} |
| 815 |
|
| 816 |
static struct platform_driver dwc_pcie_pmu_driver = { |
| 817 |
.probe = dwc_pcie_pmu_probe, |
| 818 |
.driver = {.name = "dwc_pcie_pmu",}, |
| 819 |
}; |
| 820 |
|
| 821 |
static void dwc_pcie_cleanup_devices(void) |
| 822 |
{ |
| 823 |
struct dwc_pcie_dev_info *dev_info, *tmp; |
| 824 |
|
| 825 |
list_for_each_entry_safe(dev_info, tmp, &dwc_pcie_dev_info_head, dev_node) { |
| 826 |
dwc_pcie_unregister_dev(dev_info); |
| 827 |
} |
| 828 |
} |
| 829 |
|
| 830 |
static int __init dwc_pcie_pmu_init(void) |
| 831 |
{ |
| 832 |
struct pci_dev *pdev = NULL; |
| 833 |
int ret; |
| 834 |
|
| 835 |
for_each_pci_dev(pdev) { |
| 836 |
if (!dwc_pcie_des_cap(pdev)) |
| 837 |
continue; |
| 838 |
|
| 839 |
ret = dwc_pcie_register_dev(pdev); |
| 840 |
if (ret) { |
| 841 |
pci_dev_put(pdev); |
| 842 |
goto err_cleanup; |
| 843 |
} |
| 844 |
} |
| 845 |
|
| 846 |
ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, |
| 847 |
"perf/dwc_pcie_pmu:online", |
| 848 |
dwc_pcie_pmu_online_cpu, |
| 849 |
dwc_pcie_pmu_offline_cpu); |
| 850 |
if (ret < 0) |
| 851 |
goto err_cleanup; |
| 852 |
|
| 853 |
dwc_pcie_pmu_hp_state = ret; |
| 854 |
|
| 855 |
ret = platform_driver_register(&dwc_pcie_pmu_driver); |
| 856 |
if (ret) |
| 857 |
goto err_remove_cpuhp; |
| 858 |
|
| 859 |
ret = bus_register_notifier(&pci_bus_type, &dwc_pcie_pmu_nb); |
| 860 |
if (ret) |
| 861 |
goto err_unregister_driver; |
| 862 |
notify = true; |
| 863 |
|
| 864 |
return 0; |
| 865 |
|
| 866 |
err_unregister_driver: |
| 867 |
platform_driver_unregister(&dwc_pcie_pmu_driver); |
| 868 |
err_remove_cpuhp: |
| 869 |
cpuhp_remove_multi_state(dwc_pcie_pmu_hp_state); |
| 870 |
err_cleanup: |
| 871 |
dwc_pcie_cleanup_devices(); |
| 872 |
return ret; |
| 873 |
} |
| 874 |
|
| 875 |
static void __exit dwc_pcie_pmu_exit(void) |
| 876 |
{ |
| 877 |
if (notify) |
| 878 |
bus_unregister_notifier(&pci_bus_type, &dwc_pcie_pmu_nb); |
| 879 |
dwc_pcie_cleanup_devices(); |
| 880 |
platform_driver_unregister(&dwc_pcie_pmu_driver); |
| 881 |
cpuhp_remove_multi_state(dwc_pcie_pmu_hp_state); |
| 882 |
} |
| 883 |
|
| 884 |
module_init(dwc_pcie_pmu_init); |
| 885 |
module_exit(dwc_pcie_pmu_exit); |
| 886 |
|
| 887 |
MODULE_DESCRIPTION("PMU driver for DesignWare Cores PCI Express Controller"); |
| 888 |
MODULE_AUTHOR("Shuai Xue <[email protected]>"); |
| 889 |
MODULE_LICENSE("GPL v2"); |