Commit | Line | Data |
---|---|---|
cf7b6107 SX |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * Alibaba DDR Sub-System Driveway PMU driver | |
4 | * | |
5 | * Copyright (C) 2022 Alibaba Inc | |
6 | */ | |
7 | ||
8 | #define ALI_DRW_PMUNAME "ali_drw" | |
9 | #define ALI_DRW_DRVNAME ALI_DRW_PMUNAME "_pmu" | |
10 | #define pr_fmt(fmt) ALI_DRW_DRVNAME ": " fmt | |
11 | ||
12 | #include <linux/acpi.h> | |
13 | #include <linux/bitfield.h> | |
14 | #include <linux/bitmap.h> | |
15 | #include <linux/bitops.h> | |
16 | #include <linux/cpuhotplug.h> | |
17 | #include <linux/cpumask.h> | |
18 | #include <linux/device.h> | |
19 | #include <linux/errno.h> | |
20 | #include <linux/interrupt.h> | |
21 | #include <linux/irq.h> | |
22 | #include <linux/kernel.h> | |
23 | #include <linux/list.h> | |
24 | #include <linux/module.h> | |
25 | #include <linux/mutex.h> | |
26 | #include <linux/perf_event.h> | |
27 | #include <linux/platform_device.h> | |
28 | #include <linux/printk.h> | |
29 | #include <linux/rculist.h> | |
30 | #include <linux/refcount.h> | |
31 | ||
32 | ||
33 | #define ALI_DRW_PMU_COMMON_MAX_COUNTERS 16 | |
34 | #define ALI_DRW_PMU_TEST_SEL_COMMON_COUNTER_BASE 19 | |
35 | ||
36 | #define ALI_DRW_PMU_PA_SHIFT 12 | |
37 | #define ALI_DRW_PMU_CNT_INIT 0x00000000 | |
38 | #define ALI_DRW_CNT_MAX_PERIOD 0xffffffff | |
39 | #define ALI_DRW_PMU_CYCLE_EVT_ID 0x80 | |
40 | ||
41 | #define ALI_DRW_PMU_CNT_CTRL 0xC00 | |
42 | #define ALI_DRW_PMU_CNT_RST BIT(2) | |
43 | #define ALI_DRW_PMU_CNT_STOP BIT(1) | |
44 | #define ALI_DRW_PMU_CNT_START BIT(0) | |
45 | ||
46 | #define ALI_DRW_PMU_CNT_STATE 0xC04 | |
47 | #define ALI_DRW_PMU_TEST_CTRL 0xC08 | |
48 | #define ALI_DRW_PMU_CNT_PRELOAD 0xC0C | |
49 | ||
50 | #define ALI_DRW_PMU_CYCLE_CNT_HIGH_MASK GENMASK(23, 0) | |
51 | #define ALI_DRW_PMU_CYCLE_CNT_LOW_MASK GENMASK(31, 0) | |
52 | #define ALI_DRW_PMU_CYCLE_CNT_HIGH 0xC10 | |
53 | #define ALI_DRW_PMU_CYCLE_CNT_LOW 0xC14 | |
54 | ||
55 | /* PMU EVENT SEL 0-3 are paired in 32-bit registers on a 4-byte stride */ | |
56 | #define ALI_DRW_PMU_EVENT_SEL0 0xC68 | |
57 | /* counter 0-3 use sel0, counter 4-7 use sel1...*/ | |
58 | #define ALI_DRW_PMU_EVENT_SELn(n) \ | |
59 | (ALI_DRW_PMU_EVENT_SEL0 + (n / 4) * 0x4) | |
60 | #define ALI_DRW_PMCOM_CNT_EN BIT(7) | |
61 | #define ALI_DRW_PMCOM_CNT_EVENT_MASK GENMASK(5, 0) | |
62 | #define ALI_DRW_PMCOM_CNT_EVENT_OFFSET(n) \ | |
63 | (8 * (n % 4)) | |
64 | ||
65 | /* PMU COMMON COUNTER 0-15, are paired in 32-bit registers on a 4-byte stride */ | |
66 | #define ALI_DRW_PMU_COMMON_COUNTER0 0xC78 | |
67 | #define ALI_DRW_PMU_COMMON_COUNTERn(n) \ | |
68 | (ALI_DRW_PMU_COMMON_COUNTER0 + 0x4 * (n)) | |
69 | ||
70 | #define ALI_DRW_PMU_OV_INTR_ENABLE_CTL 0xCB8 | |
71 | #define ALI_DRW_PMU_OV_INTR_DISABLE_CTL 0xCBC | |
72 | #define ALI_DRW_PMU_OV_INTR_ENABLE_STATUS 0xCC0 | |
73 | #define ALI_DRW_PMU_OV_INTR_CLR 0xCC4 | |
74 | #define ALI_DRW_PMU_OV_INTR_STATUS 0xCC8 | |
75 | #define ALI_DRW_PMCOM_CNT_OV_INTR_MASK GENMASK(23, 8) | |
76 | #define ALI_DRW_PMBW_CNT_OV_INTR_MASK GENMASK(7, 0) | |
77 | #define ALI_DRW_PMU_OV_INTR_MASK GENMASK_ULL(63, 0) | |
78 | ||
79 | static int ali_drw_cpuhp_state_num; | |
80 | ||
81 | static LIST_HEAD(ali_drw_pmu_irqs); | |
82 | static DEFINE_MUTEX(ali_drw_pmu_irqs_lock); | |
83 | ||
84 | struct ali_drw_pmu_irq { | |
85 | struct hlist_node node; | |
86 | struct list_head irqs_node; | |
87 | struct list_head pmus_node; | |
88 | int irq_num; | |
89 | int cpu; | |
90 | refcount_t refcount; | |
91 | }; | |
92 | ||
93 | struct ali_drw_pmu { | |
94 | void __iomem *cfg_base; | |
95 | struct device *dev; | |
96 | ||
97 | struct list_head pmus_node; | |
98 | struct ali_drw_pmu_irq *irq; | |
99 | int irq_num; | |
100 | int cpu; | |
101 | DECLARE_BITMAP(used_mask, ALI_DRW_PMU_COMMON_MAX_COUNTERS); | |
102 | struct perf_event *events[ALI_DRW_PMU_COMMON_MAX_COUNTERS]; | |
103 | int evtids[ALI_DRW_PMU_COMMON_MAX_COUNTERS]; | |
104 | ||
105 | struct pmu pmu; | |
106 | }; | |
107 | ||
108 | #define to_ali_drw_pmu(p) (container_of(p, struct ali_drw_pmu, pmu)) | |
109 | ||
110 | #define DRW_CONFIG_EVENTID GENMASK(7, 0) | |
111 | #define GET_DRW_EVENTID(event) FIELD_GET(DRW_CONFIG_EVENTID, (event)->attr.config) | |
112 | ||
113 | static ssize_t ali_drw_pmu_format_show(struct device *dev, | |
114 | struct device_attribute *attr, char *buf) | |
115 | { | |
116 | struct dev_ext_attribute *eattr; | |
117 | ||
118 | eattr = container_of(attr, struct dev_ext_attribute, attr); | |
119 | ||
120 | return sprintf(buf, "%s\n", (char *)eattr->var); | |
121 | } | |
122 | ||
123 | /* | |
124 | * PMU event attributes | |
125 | */ | |
126 | static ssize_t ali_drw_pmu_event_show(struct device *dev, | |
127 | struct device_attribute *attr, char *page) | |
128 | { | |
129 | struct dev_ext_attribute *eattr; | |
130 | ||
131 | eattr = container_of(attr, struct dev_ext_attribute, attr); | |
132 | ||
133 | return sprintf(page, "config=0x%lx\n", (unsigned long)eattr->var); | |
134 | } | |
135 | ||
136 | #define ALI_DRW_PMU_ATTR(_name, _func, _config) \ | |
137 | (&((struct dev_ext_attribute[]) { \ | |
138 | { __ATTR(_name, 0444, _func, NULL), (void *)_config } \ | |
139 | })[0].attr.attr) | |
140 | ||
141 | #define ALI_DRW_PMU_FORMAT_ATTR(_name, _config) \ | |
142 | ALI_DRW_PMU_ATTR(_name, ali_drw_pmu_format_show, (void *)_config) | |
143 | #define ALI_DRW_PMU_EVENT_ATTR(_name, _config) \ | |
144 | ALI_DRW_PMU_ATTR(_name, ali_drw_pmu_event_show, (unsigned long)_config) | |
145 | ||
146 | static struct attribute *ali_drw_pmu_events_attrs[] = { | |
147 | ALI_DRW_PMU_EVENT_ATTR(hif_rd_or_wr, 0x0), | |
148 | ALI_DRW_PMU_EVENT_ATTR(hif_wr, 0x1), | |
149 | ALI_DRW_PMU_EVENT_ATTR(hif_rd, 0x2), | |
150 | ALI_DRW_PMU_EVENT_ATTR(hif_rmw, 0x3), | |
151 | ALI_DRW_PMU_EVENT_ATTR(hif_hi_pri_rd, 0x4), | |
152 | ALI_DRW_PMU_EVENT_ATTR(dfi_wr_data_cycles, 0x7), | |
153 | ALI_DRW_PMU_EVENT_ATTR(dfi_rd_data_cycles, 0x8), | |
154 | ALI_DRW_PMU_EVENT_ATTR(hpr_xact_when_critical, 0x9), | |
155 | ALI_DRW_PMU_EVENT_ATTR(lpr_xact_when_critical, 0xA), | |
156 | ALI_DRW_PMU_EVENT_ATTR(wr_xact_when_critical, 0xB), | |
157 | ALI_DRW_PMU_EVENT_ATTR(op_is_activate, 0xC), | |
158 | ALI_DRW_PMU_EVENT_ATTR(op_is_rd_or_wr, 0xD), | |
159 | ALI_DRW_PMU_EVENT_ATTR(op_is_rd_activate, 0xE), | |
160 | ALI_DRW_PMU_EVENT_ATTR(op_is_rd, 0xF), | |
161 | ALI_DRW_PMU_EVENT_ATTR(op_is_wr, 0x10), | |
162 | ALI_DRW_PMU_EVENT_ATTR(op_is_mwr, 0x11), | |
163 | ALI_DRW_PMU_EVENT_ATTR(op_is_precharge, 0x12), | |
164 | ALI_DRW_PMU_EVENT_ATTR(precharge_for_rdwr, 0x13), | |
165 | ALI_DRW_PMU_EVENT_ATTR(precharge_for_other, 0x14), | |
166 | ALI_DRW_PMU_EVENT_ATTR(rdwr_transitions, 0x15), | |
167 | ALI_DRW_PMU_EVENT_ATTR(write_combine, 0x16), | |
168 | ALI_DRW_PMU_EVENT_ATTR(war_hazard, 0x17), | |
169 | ALI_DRW_PMU_EVENT_ATTR(raw_hazard, 0x18), | |
170 | ALI_DRW_PMU_EVENT_ATTR(waw_hazard, 0x19), | |
171 | ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk0, 0x1A), | |
172 | ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk1, 0x1B), | |
173 | ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk2, 0x1C), | |
174 | ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk3, 0x1D), | |
175 | ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk0, 0x1E), | |
176 | ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk1, 0x1F), | |
177 | ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk2, 0x20), | |
178 | ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk3, 0x21), | |
179 | ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk0, 0x26), | |
180 | ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk1, 0x27), | |
181 | ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk2, 0x28), | |
182 | ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk3, 0x29), | |
183 | ALI_DRW_PMU_EVENT_ATTR(op_is_refresh, 0x2A), | |
184 | ALI_DRW_PMU_EVENT_ATTR(op_is_crit_ref, 0x2B), | |
185 | ALI_DRW_PMU_EVENT_ATTR(op_is_load_mode, 0x2D), | |
186 | ALI_DRW_PMU_EVENT_ATTR(op_is_zqcl, 0x2E), | |
187 | ALI_DRW_PMU_EVENT_ATTR(visible_window_limit_reached_rd, 0x30), | |
188 | ALI_DRW_PMU_EVENT_ATTR(visible_window_limit_reached_wr, 0x31), | |
189 | ALI_DRW_PMU_EVENT_ATTR(op_is_dqsosc_mpc, 0x34), | |
190 | ALI_DRW_PMU_EVENT_ATTR(op_is_dqsosc_mrr, 0x35), | |
191 | ALI_DRW_PMU_EVENT_ATTR(op_is_tcr_mrr, 0x36), | |
192 | ALI_DRW_PMU_EVENT_ATTR(op_is_zqstart, 0x37), | |
193 | ALI_DRW_PMU_EVENT_ATTR(op_is_zqlatch, 0x38), | |
194 | ALI_DRW_PMU_EVENT_ATTR(chi_txreq, 0x39), | |
195 | ALI_DRW_PMU_EVENT_ATTR(chi_txdat, 0x3A), | |
196 | ALI_DRW_PMU_EVENT_ATTR(chi_rxdat, 0x3B), | |
197 | ALI_DRW_PMU_EVENT_ATTR(chi_rxrsp, 0x3C), | |
198 | ALI_DRW_PMU_EVENT_ATTR(tsz_vio, 0x3D), | |
199 | ALI_DRW_PMU_EVENT_ATTR(cycle, 0x80), | |
200 | NULL, | |
201 | }; | |
202 | ||
203 | static struct attribute_group ali_drw_pmu_events_attr_group = { | |
204 | .name = "events", | |
205 | .attrs = ali_drw_pmu_events_attrs, | |
206 | }; | |
207 | ||
208 | static struct attribute *ali_drw_pmu_format_attr[] = { | |
209 | ALI_DRW_PMU_FORMAT_ATTR(event, "config:0-7"), | |
210 | NULL, | |
211 | }; | |
212 | ||
213 | static const struct attribute_group ali_drw_pmu_format_group = { | |
214 | .name = "format", | |
215 | .attrs = ali_drw_pmu_format_attr, | |
216 | }; | |
217 | ||
218 | static ssize_t ali_drw_pmu_cpumask_show(struct device *dev, | |
219 | struct device_attribute *attr, | |
220 | char *buf) | |
221 | { | |
222 | struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(dev_get_drvdata(dev)); | |
223 | ||
224 | return cpumap_print_to_pagebuf(true, buf, cpumask_of(drw_pmu->cpu)); | |
225 | } | |
226 | ||
227 | static struct device_attribute ali_drw_pmu_cpumask_attr = | |
228 | __ATTR(cpumask, 0444, ali_drw_pmu_cpumask_show, NULL); | |
229 | ||
230 | static struct attribute *ali_drw_pmu_cpumask_attrs[] = { | |
231 | &ali_drw_pmu_cpumask_attr.attr, | |
232 | NULL, | |
233 | }; | |
234 | ||
235 | static const struct attribute_group ali_drw_pmu_cpumask_attr_group = { | |
236 | .attrs = ali_drw_pmu_cpumask_attrs, | |
237 | }; | |
238 | ||
239 | static const struct attribute_group *ali_drw_pmu_attr_groups[] = { | |
240 | &ali_drw_pmu_events_attr_group, | |
241 | &ali_drw_pmu_cpumask_attr_group, | |
242 | &ali_drw_pmu_format_group, | |
243 | NULL, | |
244 | }; | |
245 | ||
246 | /* find a counter for event, then in add func, hw.idx will equal to counter */ | |
247 | static int ali_drw_get_counter_idx(struct perf_event *event) | |
248 | { | |
249 | struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); | |
250 | int idx; | |
251 | ||
252 | for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; ++idx) { | |
253 | if (!test_and_set_bit(idx, drw_pmu->used_mask)) | |
254 | return idx; | |
255 | } | |
256 | ||
257 | /* The counters are all in use. */ | |
258 | return -EBUSY; | |
259 | } | |
260 | ||
261 | static u64 ali_drw_pmu_read_counter(struct perf_event *event) | |
262 | { | |
263 | struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); | |
264 | u64 cycle_high, cycle_low; | |
265 | ||
266 | if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) { | |
267 | cycle_high = readl(drw_pmu->cfg_base + ALI_DRW_PMU_CYCLE_CNT_HIGH); | |
268 | cycle_high &= ALI_DRW_PMU_CYCLE_CNT_HIGH_MASK; | |
269 | cycle_low = readl(drw_pmu->cfg_base + ALI_DRW_PMU_CYCLE_CNT_LOW); | |
270 | cycle_low &= ALI_DRW_PMU_CYCLE_CNT_LOW_MASK; | |
271 | return (cycle_high << 32 | cycle_low); | |
272 | } | |
273 | ||
274 | return readl(drw_pmu->cfg_base + | |
275 | ALI_DRW_PMU_COMMON_COUNTERn(event->hw.idx)); | |
276 | } | |
277 | ||
278 | static void ali_drw_pmu_event_update(struct perf_event *event) | |
279 | { | |
280 | struct hw_perf_event *hwc = &event->hw; | |
281 | u64 delta, prev, now; | |
282 | ||
283 | do { | |
284 | prev = local64_read(&hwc->prev_count); | |
285 | now = ali_drw_pmu_read_counter(event); | |
286 | } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev); | |
287 | ||
288 | /* handle overflow. */ | |
289 | delta = now - prev; | |
290 | if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) | |
291 | delta &= ALI_DRW_PMU_OV_INTR_MASK; | |
292 | else | |
293 | delta &= ALI_DRW_CNT_MAX_PERIOD; | |
294 | local64_add(delta, &event->count); | |
295 | } | |
296 | ||
297 | static void ali_drw_pmu_event_set_period(struct perf_event *event) | |
298 | { | |
299 | u64 pre_val; | |
300 | struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); | |
301 | ||
302 | /* set a preload counter for test purpose */ | |
303 | writel(ALI_DRW_PMU_TEST_SEL_COMMON_COUNTER_BASE + event->hw.idx, | |
304 | drw_pmu->cfg_base + ALI_DRW_PMU_TEST_CTRL); | |
305 | ||
306 | /* set conunter initial value */ | |
307 | pre_val = ALI_DRW_PMU_CNT_INIT; | |
308 | writel(pre_val, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_PRELOAD); | |
309 | local64_set(&event->hw.prev_count, pre_val); | |
310 | ||
311 | /* set sel mode to zero to start test */ | |
312 | writel(0x0, drw_pmu->cfg_base + ALI_DRW_PMU_TEST_CTRL); | |
313 | } | |
314 | ||
315 | static void ali_drw_pmu_enable_counter(struct perf_event *event) | |
316 | { | |
317 | u32 val, subval, reg, shift; | |
318 | int counter = event->hw.idx; | |
319 | struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); | |
320 | ||
321 | reg = ALI_DRW_PMU_EVENT_SELn(counter); | |
322 | val = readl(drw_pmu->cfg_base + reg); | |
323 | subval = FIELD_PREP(ALI_DRW_PMCOM_CNT_EN, 1) | | |
324 | FIELD_PREP(ALI_DRW_PMCOM_CNT_EVENT_MASK, drw_pmu->evtids[counter]); | |
325 | ||
326 | shift = ALI_DRW_PMCOM_CNT_EVENT_OFFSET(counter); | |
327 | val &= ~(GENMASK(7, 0) << shift); | |
328 | val |= subval << shift; | |
329 | ||
330 | writel(val, drw_pmu->cfg_base + reg); | |
331 | } | |
332 | ||
333 | static void ali_drw_pmu_disable_counter(struct perf_event *event) | |
334 | { | |
335 | u32 val, reg, subval, shift; | |
336 | struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); | |
337 | int counter = event->hw.idx; | |
338 | ||
339 | reg = ALI_DRW_PMU_EVENT_SELn(counter); | |
340 | val = readl(drw_pmu->cfg_base + reg); | |
341 | subval = FIELD_PREP(ALI_DRW_PMCOM_CNT_EN, 0) | | |
342 | FIELD_PREP(ALI_DRW_PMCOM_CNT_EVENT_MASK, 0); | |
343 | ||
344 | shift = ALI_DRW_PMCOM_CNT_EVENT_OFFSET(counter); | |
345 | val &= ~(GENMASK(7, 0) << shift); | |
346 | val |= subval << shift; | |
347 | ||
348 | writel(val, drw_pmu->cfg_base + reg); | |
349 | } | |
350 | ||
351 | static irqreturn_t ali_drw_pmu_isr(int irq_num, void *data) | |
352 | { | |
353 | struct ali_drw_pmu_irq *irq = data; | |
354 | struct ali_drw_pmu *drw_pmu; | |
355 | irqreturn_t ret = IRQ_NONE; | |
356 | ||
357 | rcu_read_lock(); | |
358 | list_for_each_entry_rcu(drw_pmu, &irq->pmus_node, pmus_node) { | |
359 | unsigned long status, clr_status; | |
360 | struct perf_event *event; | |
361 | unsigned int idx; | |
362 | ||
363 | for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; idx++) { | |
364 | event = drw_pmu->events[idx]; | |
365 | if (!event) | |
366 | continue; | |
367 | ali_drw_pmu_disable_counter(event); | |
368 | } | |
369 | ||
370 | /* common counter intr status */ | |
371 | status = readl(drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_STATUS); | |
372 | status = FIELD_GET(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, status); | |
373 | if (status) { | |
374 | for_each_set_bit(idx, &status, | |
375 | ALI_DRW_PMU_COMMON_MAX_COUNTERS) { | |
376 | event = drw_pmu->events[idx]; | |
377 | if (WARN_ON_ONCE(!event)) | |
378 | continue; | |
379 | ali_drw_pmu_event_update(event); | |
380 | ali_drw_pmu_event_set_period(event); | |
381 | } | |
382 | ||
383 | /* clear common counter intr status */ | |
384 | clr_status = FIELD_PREP(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, 1); | |
385 | writel(clr_status, | |
386 | drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_CLR); | |
387 | } | |
388 | ||
389 | for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; idx++) { | |
390 | event = drw_pmu->events[idx]; | |
391 | if (!event) | |
392 | continue; | |
393 | if (!(event->hw.state & PERF_HES_STOPPED)) | |
394 | ali_drw_pmu_enable_counter(event); | |
395 | } | |
396 | if (status) | |
397 | ret = IRQ_HANDLED; | |
398 | } | |
399 | rcu_read_unlock(); | |
400 | return ret; | |
401 | } | |
402 | ||
403 | static struct ali_drw_pmu_irq *__ali_drw_pmu_init_irq(struct platform_device | |
404 | *pdev, int irq_num) | |
405 | { | |
406 | int ret; | |
407 | struct ali_drw_pmu_irq *irq; | |
408 | ||
409 | list_for_each_entry(irq, &ali_drw_pmu_irqs, irqs_node) { | |
410 | if (irq->irq_num == irq_num | |
411 | && refcount_inc_not_zero(&irq->refcount)) | |
412 | return irq; | |
413 | } | |
414 | ||
415 | irq = kzalloc(sizeof(*irq), GFP_KERNEL); | |
416 | if (!irq) | |
417 | return ERR_PTR(-ENOMEM); | |
418 | ||
419 | INIT_LIST_HEAD(&irq->pmus_node); | |
420 | ||
421 | /* Pick one CPU to be the preferred one to use */ | |
422 | irq->cpu = smp_processor_id(); | |
423 | refcount_set(&irq->refcount, 1); | |
424 | ||
425 | /* | |
426 | * FIXME: one of DDRSS Driveway PMU overflow interrupt shares the same | |
427 | * irq number with MPAM ERR_IRQ. To register DDRSS PMU and MPAM drivers | |
428 | * successfully, add IRQF_SHARED flag. Howerer, PMU interrupt should not | |
429 | * share with other component. | |
430 | */ | |
431 | ret = devm_request_irq(&pdev->dev, irq_num, ali_drw_pmu_isr, | |
432 | IRQF_SHARED, dev_name(&pdev->dev), irq); | |
433 | if (ret < 0) { | |
434 | dev_err(&pdev->dev, | |
435 | "Fail to request IRQ:%d ret:%d\n", irq_num, ret); | |
436 | goto out_free; | |
437 | } | |
438 | ||
439 | ret = irq_set_affinity_hint(irq_num, cpumask_of(irq->cpu)); | |
440 | if (ret) | |
441 | goto out_free; | |
442 | ||
443 | ret = cpuhp_state_add_instance_nocalls(ali_drw_cpuhp_state_num, | |
444 | &irq->node); | |
445 | if (ret) | |
446 | goto out_free; | |
447 | ||
448 | irq->irq_num = irq_num; | |
449 | list_add(&irq->irqs_node, &ali_drw_pmu_irqs); | |
450 | ||
451 | return irq; | |
452 | ||
453 | out_free: | |
454 | kfree(irq); | |
455 | return ERR_PTR(ret); | |
456 | } | |
457 | ||
458 | static int ali_drw_pmu_init_irq(struct ali_drw_pmu *drw_pmu, | |
459 | struct platform_device *pdev) | |
460 | { | |
461 | int irq_num; | |
462 | struct ali_drw_pmu_irq *irq; | |
463 | ||
464 | /* Read and init IRQ */ | |
465 | irq_num = platform_get_irq(pdev, 0); | |
466 | if (irq_num < 0) | |
467 | return irq_num; | |
468 | ||
469 | mutex_lock(&ali_drw_pmu_irqs_lock); | |
470 | irq = __ali_drw_pmu_init_irq(pdev, irq_num); | |
471 | mutex_unlock(&ali_drw_pmu_irqs_lock); | |
472 | ||
473 | if (IS_ERR(irq)) | |
474 | return PTR_ERR(irq); | |
475 | ||
476 | drw_pmu->irq = irq; | |
477 | ||
478 | mutex_lock(&ali_drw_pmu_irqs_lock); | |
479 | list_add_rcu(&drw_pmu->pmus_node, &irq->pmus_node); | |
480 | mutex_unlock(&ali_drw_pmu_irqs_lock); | |
481 | ||
482 | return 0; | |
483 | } | |
484 | ||
485 | static void ali_drw_pmu_uninit_irq(struct ali_drw_pmu *drw_pmu) | |
486 | { | |
487 | struct ali_drw_pmu_irq *irq = drw_pmu->irq; | |
488 | ||
489 | mutex_lock(&ali_drw_pmu_irqs_lock); | |
490 | list_del_rcu(&drw_pmu->pmus_node); | |
491 | ||
492 | if (!refcount_dec_and_test(&irq->refcount)) { | |
493 | mutex_unlock(&ali_drw_pmu_irqs_lock); | |
494 | return; | |
495 | } | |
496 | ||
497 | list_del(&irq->irqs_node); | |
498 | mutex_unlock(&ali_drw_pmu_irqs_lock); | |
499 | ||
500 | WARN_ON(irq_set_affinity_hint(irq->irq_num, NULL)); | |
501 | cpuhp_state_remove_instance_nocalls(ali_drw_cpuhp_state_num, | |
502 | &irq->node); | |
503 | kfree(irq); | |
504 | } | |
505 | ||
506 | static int ali_drw_pmu_event_init(struct perf_event *event) | |
507 | { | |
508 | struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); | |
509 | struct hw_perf_event *hwc = &event->hw; | |
510 | struct perf_event *sibling; | |
511 | struct device *dev = drw_pmu->pmu.dev; | |
512 | ||
513 | if (event->attr.type != event->pmu->type) | |
514 | return -ENOENT; | |
515 | ||
516 | if (is_sampling_event(event)) { | |
517 | dev_err(dev, "Sampling not supported!\n"); | |
518 | return -EOPNOTSUPP; | |
519 | } | |
520 | ||
521 | if (event->attach_state & PERF_ATTACH_TASK) { | |
522 | dev_err(dev, "Per-task counter cannot allocate!\n"); | |
523 | return -EOPNOTSUPP; | |
524 | } | |
525 | ||
526 | event->cpu = drw_pmu->cpu; | |
527 | if (event->cpu < 0) { | |
528 | dev_err(dev, "Per-task mode not supported!\n"); | |
529 | return -EOPNOTSUPP; | |
530 | } | |
531 | ||
532 | if (event->group_leader != event && | |
533 | !is_software_event(event->group_leader)) { | |
534 | dev_err(dev, "driveway only allow one event!\n"); | |
535 | return -EINVAL; | |
536 | } | |
537 | ||
538 | for_each_sibling_event(sibling, event->group_leader) { | |
539 | if (sibling != event && !is_software_event(sibling)) { | |
540 | dev_err(dev, "driveway event not allowed!\n"); | |
541 | return -EINVAL; | |
542 | } | |
543 | } | |
544 | ||
545 | /* reset all the pmu counters */ | |
546 | writel(ALI_DRW_PMU_CNT_RST, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL); | |
547 | ||
548 | hwc->idx = -1; | |
549 | ||
550 | return 0; | |
551 | } | |
552 | ||
553 | static void ali_drw_pmu_start(struct perf_event *event, int flags) | |
554 | { | |
555 | struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); | |
556 | ||
557 | event->hw.state = 0; | |
558 | ||
559 | if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) { | |
560 | writel(ALI_DRW_PMU_CNT_START, | |
561 | drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL); | |
562 | return; | |
563 | } | |
564 | ||
565 | ali_drw_pmu_event_set_period(event); | |
566 | if (flags & PERF_EF_RELOAD) { | |
567 | unsigned long prev_raw_count = | |
568 | local64_read(&event->hw.prev_count); | |
569 | writel(prev_raw_count, | |
570 | drw_pmu->cfg_base + ALI_DRW_PMU_CNT_PRELOAD); | |
571 | } | |
572 | ||
573 | ali_drw_pmu_enable_counter(event); | |
574 | ||
575 | writel(ALI_DRW_PMU_CNT_START, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL); | |
576 | } | |
577 | ||
578 | static void ali_drw_pmu_stop(struct perf_event *event, int flags) | |
579 | { | |
580 | struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); | |
581 | ||
582 | if (event->hw.state & PERF_HES_STOPPED) | |
583 | return; | |
584 | ||
585 | if (GET_DRW_EVENTID(event) != ALI_DRW_PMU_CYCLE_EVT_ID) | |
586 | ali_drw_pmu_disable_counter(event); | |
587 | ||
588 | writel(ALI_DRW_PMU_CNT_STOP, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL); | |
589 | ||
590 | ali_drw_pmu_event_update(event); | |
591 | event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; | |
592 | } | |
593 | ||
594 | static int ali_drw_pmu_add(struct perf_event *event, int flags) | |
595 | { | |
596 | struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); | |
597 | struct hw_perf_event *hwc = &event->hw; | |
598 | int idx = -1; | |
599 | int evtid; | |
600 | ||
601 | evtid = GET_DRW_EVENTID(event); | |
602 | ||
603 | if (evtid != ALI_DRW_PMU_CYCLE_EVT_ID) { | |
604 | idx = ali_drw_get_counter_idx(event); | |
605 | if (idx < 0) | |
606 | return idx; | |
607 | drw_pmu->events[idx] = event; | |
608 | drw_pmu->evtids[idx] = evtid; | |
609 | } | |
610 | hwc->idx = idx; | |
611 | ||
612 | hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; | |
613 | ||
614 | if (flags & PERF_EF_START) | |
615 | ali_drw_pmu_start(event, PERF_EF_RELOAD); | |
616 | ||
617 | /* Propagate our changes to the userspace mapping. */ | |
618 | perf_event_update_userpage(event); | |
619 | ||
620 | return 0; | |
621 | } | |
622 | ||
623 | static void ali_drw_pmu_del(struct perf_event *event, int flags) | |
624 | { | |
625 | struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); | |
626 | struct hw_perf_event *hwc = &event->hw; | |
627 | int idx = hwc->idx; | |
628 | ||
629 | ali_drw_pmu_stop(event, PERF_EF_UPDATE); | |
630 | ||
631 | if (idx >= 0 && idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS) { | |
632 | drw_pmu->events[idx] = NULL; | |
633 | drw_pmu->evtids[idx] = 0; | |
634 | clear_bit(idx, drw_pmu->used_mask); | |
635 | } | |
636 | ||
637 | perf_event_update_userpage(event); | |
638 | } | |
639 | ||
640 | static void ali_drw_pmu_read(struct perf_event *event) | |
641 | { | |
642 | ali_drw_pmu_event_update(event); | |
643 | } | |
644 | ||
645 | static int ali_drw_pmu_probe(struct platform_device *pdev) | |
646 | { | |
647 | struct ali_drw_pmu *drw_pmu; | |
648 | struct resource *res; | |
649 | char *name; | |
650 | int ret; | |
651 | ||
652 | drw_pmu = devm_kzalloc(&pdev->dev, sizeof(*drw_pmu), GFP_KERNEL); | |
653 | if (!drw_pmu) | |
654 | return -ENOMEM; | |
655 | ||
656 | drw_pmu->dev = &pdev->dev; | |
657 | platform_set_drvdata(pdev, drw_pmu); | |
658 | ||
bc12f344 | 659 | drw_pmu->cfg_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); |
ad0112f2 SK |
660 | if (IS_ERR(drw_pmu->cfg_base)) |
661 | return PTR_ERR(drw_pmu->cfg_base); | |
cf7b6107 SX |
662 | |
663 | name = devm_kasprintf(drw_pmu->dev, GFP_KERNEL, "ali_drw_%llx", | |
664 | (u64) (res->start >> ALI_DRW_PMU_PA_SHIFT)); | |
665 | if (!name) | |
666 | return -ENOMEM; | |
667 | ||
668 | writel(ALI_DRW_PMU_CNT_RST, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL); | |
669 | ||
670 | /* enable the generation of interrupt by all common counters */ | |
671 | writel(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, | |
672 | drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_ENABLE_CTL); | |
673 | ||
674 | /* clearing interrupt status */ | |
675 | writel(0xffffff, drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_CLR); | |
676 | ||
677 | drw_pmu->cpu = smp_processor_id(); | |
678 | ||
679 | ret = ali_drw_pmu_init_irq(drw_pmu, pdev); | |
680 | if (ret) | |
681 | return ret; | |
682 | ||
683 | drw_pmu->pmu = (struct pmu) { | |
684 | .module = THIS_MODULE, | |
685 | .task_ctx_nr = perf_invalid_context, | |
686 | .event_init = ali_drw_pmu_event_init, | |
687 | .add = ali_drw_pmu_add, | |
688 | .del = ali_drw_pmu_del, | |
689 | .start = ali_drw_pmu_start, | |
690 | .stop = ali_drw_pmu_stop, | |
691 | .read = ali_drw_pmu_read, | |
692 | .attr_groups = ali_drw_pmu_attr_groups, | |
693 | .capabilities = PERF_PMU_CAP_NO_EXCLUDE, | |
694 | }; | |
695 | ||
696 | ret = perf_pmu_register(&drw_pmu->pmu, name, -1); | |
697 | if (ret) { | |
698 | dev_err(drw_pmu->dev, "DRW Driveway PMU PMU register failed!\n"); | |
699 | ali_drw_pmu_uninit_irq(drw_pmu); | |
700 | } | |
701 | ||
702 | return ret; | |
703 | } | |
704 | ||
705 | static int ali_drw_pmu_remove(struct platform_device *pdev) | |
706 | { | |
707 | struct ali_drw_pmu *drw_pmu = platform_get_drvdata(pdev); | |
708 | ||
709 | /* disable the generation of interrupt by all common counters */ | |
710 | writel(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, | |
711 | drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_DISABLE_CTL); | |
712 | ||
713 | ali_drw_pmu_uninit_irq(drw_pmu); | |
714 | perf_pmu_unregister(&drw_pmu->pmu); | |
715 | ||
716 | return 0; | |
717 | } | |
718 | ||
719 | static int ali_drw_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) | |
720 | { | |
721 | struct ali_drw_pmu_irq *irq; | |
722 | struct ali_drw_pmu *drw_pmu; | |
723 | unsigned int target; | |
724 | int ret; | |
725 | cpumask_t node_online_cpus; | |
726 | ||
727 | irq = hlist_entry_safe(node, struct ali_drw_pmu_irq, node); | |
728 | if (cpu != irq->cpu) | |
729 | return 0; | |
730 | ||
731 | ret = cpumask_and(&node_online_cpus, | |
732 | cpumask_of_node(cpu_to_node(cpu)), cpu_online_mask); | |
733 | if (ret) | |
734 | target = cpumask_any_but(&node_online_cpus, cpu); | |
735 | else | |
736 | target = cpumask_any_but(cpu_online_mask, cpu); | |
737 | ||
738 | if (target >= nr_cpu_ids) | |
739 | return 0; | |
740 | ||
741 | /* We're only reading, but this isn't the place to be involving RCU */ | |
742 | mutex_lock(&ali_drw_pmu_irqs_lock); | |
743 | list_for_each_entry(drw_pmu, &irq->pmus_node, pmus_node) | |
744 | perf_pmu_migrate_context(&drw_pmu->pmu, irq->cpu, target); | |
745 | mutex_unlock(&ali_drw_pmu_irqs_lock); | |
746 | ||
747 | WARN_ON(irq_set_affinity_hint(irq->irq_num, cpumask_of(target))); | |
748 | irq->cpu = target; | |
749 | ||
750 | return 0; | |
751 | } | |
752 | ||
753 | /* | |
754 | * Due to historical reasons, the HID used in the production environment is | |
755 | * ARMHD700, so we leave ARMHD700 as Compatible ID. | |
756 | */ | |
757 | static const struct acpi_device_id ali_drw_acpi_match[] = { | |
758 | {"BABA5000", 0}, | |
759 | {"ARMHD700", 0}, | |
760 | {} | |
761 | }; | |
762 | ||
763 | MODULE_DEVICE_TABLE(acpi, ali_drw_acpi_match); | |
764 | ||
765 | static struct platform_driver ali_drw_pmu_driver = { | |
766 | .driver = { | |
767 | .name = "ali_drw_pmu", | |
768 | .acpi_match_table = ali_drw_acpi_match, | |
769 | }, | |
770 | .probe = ali_drw_pmu_probe, | |
771 | .remove = ali_drw_pmu_remove, | |
772 | }; | |
773 | ||
774 | static int __init ali_drw_pmu_init(void) | |
775 | { | |
776 | int ret; | |
777 | ||
778 | ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, | |
779 | "ali_drw_pmu:online", | |
780 | NULL, ali_drw_pmu_offline_cpu); | |
781 | ||
782 | if (ret < 0) { | |
783 | pr_err("DRW Driveway PMU: setup hotplug failed, ret = %d\n", | |
784 | ret); | |
785 | return ret; | |
786 | } | |
787 | ali_drw_cpuhp_state_num = ret; | |
788 | ||
789 | ret = platform_driver_register(&ali_drw_pmu_driver); | |
790 | if (ret) | |
791 | cpuhp_remove_multi_state(ali_drw_cpuhp_state_num); | |
792 | ||
793 | return ret; | |
794 | } | |
795 | ||
796 | static void __exit ali_drw_pmu_exit(void) | |
797 | { | |
798 | platform_driver_unregister(&ali_drw_pmu_driver); | |
799 | cpuhp_remove_multi_state(ali_drw_cpuhp_state_num); | |
800 | } | |
801 | ||
802 | module_init(ali_drw_pmu_init); | |
803 | module_exit(ali_drw_pmu_exit); | |
804 | ||
805 | MODULE_AUTHOR("Hongbo Yao <yaohongbo@linux.alibaba.com>"); | |
806 | MODULE_AUTHOR("Neng Chen <nengchen@linux.alibaba.com>"); | |
807 | MODULE_AUTHOR("Shuai Xue <xueshuai@linux.alibaba.com>"); | |
808 | MODULE_DESCRIPTION("Alibaba DDR Sub-System Driveway PMU driver"); | |
809 | MODULE_LICENSE("GPL v2"); |