Commit | Line | Data |
---|---|---|
a6a5006d KL |
1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* | |
3 | * Support Intel IOMMU PerfMon | |
4 | * Copyright(c) 2023 Intel Corporation. | |
5 | */ | |
6 | #define pr_fmt(fmt) "DMAR: " fmt | |
7 | #define dev_fmt(fmt) pr_fmt(fmt) | |
8 | ||
9 | #include <linux/dmar.h> | |
10 | #include "iommu.h" | |
11 | #include "perfmon.h" | |
12 | ||
7232ab8b KL |
13 | PMU_FORMAT_ATTR(event, "config:0-27"); /* ES: Events Select */ |
14 | PMU_FORMAT_ATTR(event_group, "config:28-31"); /* EGI: Event Group Index */ | |
15 | ||
16 | static struct attribute *iommu_pmu_format_attrs[] = { | |
17 | &format_attr_event_group.attr, | |
18 | &format_attr_event.attr, | |
19 | NULL | |
20 | }; | |
21 | ||
22 | static struct attribute_group iommu_pmu_format_attr_group = { | |
23 | .name = "format", | |
24 | .attrs = iommu_pmu_format_attrs, | |
25 | }; | |
26 | ||
27 | /* The available events are added in attr_update later */ | |
28 | static struct attribute *attrs_empty[] = { | |
29 | NULL | |
30 | }; | |
31 | ||
32 | static struct attribute_group iommu_pmu_events_attr_group = { | |
33 | .name = "events", | |
34 | .attrs = attrs_empty, | |
35 | }; | |
36 | ||
46284c6c KL |
37 | static cpumask_t iommu_pmu_cpu_mask; |
38 | ||
39 | static ssize_t | |
40 | cpumask_show(struct device *dev, struct device_attribute *attr, char *buf) | |
41 | { | |
42 | return cpumap_print_to_pagebuf(true, buf, &iommu_pmu_cpu_mask); | |
43 | } | |
44 | static DEVICE_ATTR_RO(cpumask); | |
45 | ||
46 | static struct attribute *iommu_pmu_cpumask_attrs[] = { | |
47 | &dev_attr_cpumask.attr, | |
48 | NULL | |
49 | }; | |
50 | ||
51 | static struct attribute_group iommu_pmu_cpumask_attr_group = { | |
52 | .attrs = iommu_pmu_cpumask_attrs, | |
53 | }; | |
54 | ||
7232ab8b KL |
55 | static const struct attribute_group *iommu_pmu_attr_groups[] = { |
56 | &iommu_pmu_format_attr_group, | |
57 | &iommu_pmu_events_attr_group, | |
46284c6c | 58 | &iommu_pmu_cpumask_attr_group, |
7232ab8b KL |
59 | NULL |
60 | }; | |
61 | ||
62 | static inline struct iommu_pmu *dev_to_iommu_pmu(struct device *dev) | |
63 | { | |
64 | /* | |
65 | * The perf_event creates its own dev for each PMU. | |
66 | * See pmu_dev_alloc() | |
67 | */ | |
68 | return container_of(dev_get_drvdata(dev), struct iommu_pmu, pmu); | |
69 | } | |
70 | ||
71 | #define IOMMU_PMU_ATTR(_name, _format, _filter) \ | |
72 | PMU_FORMAT_ATTR(_name, _format); \ | |
73 | \ | |
74 | static struct attribute *_name##_attr[] = { \ | |
75 | &format_attr_##_name.attr, \ | |
76 | NULL \ | |
77 | }; \ | |
78 | \ | |
79 | static umode_t \ | |
80 | _name##_is_visible(struct kobject *kobj, struct attribute *attr, int i) \ | |
81 | { \ | |
82 | struct device *dev = kobj_to_dev(kobj); \ | |
83 | struct iommu_pmu *iommu_pmu = dev_to_iommu_pmu(dev); \ | |
84 | \ | |
85 | if (!iommu_pmu) \ | |
86 | return 0; \ | |
87 | return (iommu_pmu->filter & _filter) ? attr->mode : 0; \ | |
88 | } \ | |
89 | \ | |
90 | static struct attribute_group _name = { \ | |
91 | .name = "format", \ | |
92 | .attrs = _name##_attr, \ | |
93 | .is_visible = _name##_is_visible, \ | |
94 | }; | |
95 | ||
96 | IOMMU_PMU_ATTR(filter_requester_id_en, "config1:0", IOMMU_PMU_FILTER_REQUESTER_ID); | |
97 | IOMMU_PMU_ATTR(filter_domain_en, "config1:1", IOMMU_PMU_FILTER_DOMAIN); | |
98 | IOMMU_PMU_ATTR(filter_pasid_en, "config1:2", IOMMU_PMU_FILTER_PASID); | |
99 | IOMMU_PMU_ATTR(filter_ats_en, "config1:3", IOMMU_PMU_FILTER_ATS); | |
100 | IOMMU_PMU_ATTR(filter_page_table_en, "config1:4", IOMMU_PMU_FILTER_PAGE_TABLE); | |
101 | IOMMU_PMU_ATTR(filter_requester_id, "config1:16-31", IOMMU_PMU_FILTER_REQUESTER_ID); | |
102 | IOMMU_PMU_ATTR(filter_domain, "config1:32-47", IOMMU_PMU_FILTER_DOMAIN); | |
103 | IOMMU_PMU_ATTR(filter_pasid, "config2:0-21", IOMMU_PMU_FILTER_PASID); | |
104 | IOMMU_PMU_ATTR(filter_ats, "config2:24-28", IOMMU_PMU_FILTER_ATS); | |
105 | IOMMU_PMU_ATTR(filter_page_table, "config2:32-36", IOMMU_PMU_FILTER_PAGE_TABLE); | |
106 | ||
107 | #define iommu_pmu_en_requester_id(e) ((e) & 0x1) | |
108 | #define iommu_pmu_en_domain(e) (((e) >> 1) & 0x1) | |
109 | #define iommu_pmu_en_pasid(e) (((e) >> 2) & 0x1) | |
110 | #define iommu_pmu_en_ats(e) (((e) >> 3) & 0x1) | |
111 | #define iommu_pmu_en_page_table(e) (((e) >> 4) & 0x1) | |
112 | #define iommu_pmu_get_requester_id(filter) (((filter) >> 16) & 0xffff) | |
113 | #define iommu_pmu_get_domain(filter) (((filter) >> 32) & 0xffff) | |
114 | #define iommu_pmu_get_pasid(filter) ((filter) & 0x3fffff) | |
115 | #define iommu_pmu_get_ats(filter) (((filter) >> 24) & 0x1f) | |
116 | #define iommu_pmu_get_page_table(filter) (((filter) >> 32) & 0x1f) | |
117 | ||
118 | #define iommu_pmu_set_filter(_name, _config, _filter, _idx, _econfig) \ | |
119 | { \ | |
120 | if ((iommu_pmu->filter & _filter) && iommu_pmu_en_##_name(_econfig)) { \ | |
121 | dmar_writel(iommu_pmu->cfg_reg + _idx * IOMMU_PMU_CFG_OFFSET + \ | |
122 | IOMMU_PMU_CFG_SIZE + \ | |
123 | (ffs(_filter) - 1) * IOMMU_PMU_CFG_FILTERS_OFFSET, \ | |
124 | iommu_pmu_get_##_name(_config) | IOMMU_PMU_FILTER_EN);\ | |
125 | } \ | |
126 | } | |
127 | ||
128 | #define iommu_pmu_clear_filter(_filter, _idx) \ | |
129 | { \ | |
130 | if (iommu_pmu->filter & _filter) { \ | |
131 | dmar_writel(iommu_pmu->cfg_reg + _idx * IOMMU_PMU_CFG_OFFSET + \ | |
132 | IOMMU_PMU_CFG_SIZE + \ | |
133 | (ffs(_filter) - 1) * IOMMU_PMU_CFG_FILTERS_OFFSET, \ | |
134 | 0); \ | |
135 | } \ | |
136 | } | |
137 | ||
138 | /* | |
139 | * Define the event attr related functions | |
140 | * Input: _name: event attr name | |
141 | * _string: string of the event in sysfs | |
142 | * _g_idx: event group encoding | |
143 | * _event: event encoding | |
144 | */ | |
145 | #define IOMMU_PMU_EVENT_ATTR(_name, _string, _g_idx, _event) \ | |
146 | PMU_EVENT_ATTR_STRING(_name, event_attr_##_name, _string) \ | |
147 | \ | |
148 | static struct attribute *_name##_attr[] = { \ | |
149 | &event_attr_##_name.attr.attr, \ | |
150 | NULL \ | |
151 | }; \ | |
152 | \ | |
153 | static umode_t \ | |
154 | _name##_is_visible(struct kobject *kobj, struct attribute *attr, int i) \ | |
155 | { \ | |
156 | struct device *dev = kobj_to_dev(kobj); \ | |
157 | struct iommu_pmu *iommu_pmu = dev_to_iommu_pmu(dev); \ | |
158 | \ | |
159 | if (!iommu_pmu) \ | |
160 | return 0; \ | |
161 | return (iommu_pmu->evcap[_g_idx] & _event) ? attr->mode : 0; \ | |
162 | } \ | |
163 | \ | |
164 | static struct attribute_group _name = { \ | |
165 | .name = "events", \ | |
166 | .attrs = _name##_attr, \ | |
167 | .is_visible = _name##_is_visible, \ | |
168 | }; | |
169 | ||
170 | IOMMU_PMU_EVENT_ATTR(iommu_clocks, "event_group=0x0,event=0x001", 0x0, 0x001) | |
171 | IOMMU_PMU_EVENT_ATTR(iommu_requests, "event_group=0x0,event=0x002", 0x0, 0x002) | |
172 | IOMMU_PMU_EVENT_ATTR(pw_occupancy, "event_group=0x0,event=0x004", 0x0, 0x004) | |
173 | IOMMU_PMU_EVENT_ATTR(ats_blocked, "event_group=0x0,event=0x008", 0x0, 0x008) | |
174 | IOMMU_PMU_EVENT_ATTR(iommu_mrds, "event_group=0x1,event=0x001", 0x1, 0x001) | |
175 | IOMMU_PMU_EVENT_ATTR(iommu_mem_blocked, "event_group=0x1,event=0x020", 0x1, 0x020) | |
176 | IOMMU_PMU_EVENT_ATTR(pg_req_posted, "event_group=0x1,event=0x040", 0x1, 0x040) | |
177 | IOMMU_PMU_EVENT_ATTR(ctxt_cache_lookup, "event_group=0x2,event=0x001", 0x2, 0x001) | |
178 | IOMMU_PMU_EVENT_ATTR(ctxt_cache_hit, "event_group=0x2,event=0x002", 0x2, 0x002) | |
179 | IOMMU_PMU_EVENT_ATTR(pasid_cache_lookup, "event_group=0x2,event=0x004", 0x2, 0x004) | |
180 | IOMMU_PMU_EVENT_ATTR(pasid_cache_hit, "event_group=0x2,event=0x008", 0x2, 0x008) | |
181 | IOMMU_PMU_EVENT_ATTR(ss_nonleaf_lookup, "event_group=0x2,event=0x010", 0x2, 0x010) | |
182 | IOMMU_PMU_EVENT_ATTR(ss_nonleaf_hit, "event_group=0x2,event=0x020", 0x2, 0x020) | |
183 | IOMMU_PMU_EVENT_ATTR(fs_nonleaf_lookup, "event_group=0x2,event=0x040", 0x2, 0x040) | |
184 | IOMMU_PMU_EVENT_ATTR(fs_nonleaf_hit, "event_group=0x2,event=0x080", 0x2, 0x080) | |
185 | IOMMU_PMU_EVENT_ATTR(hpt_nonleaf_lookup, "event_group=0x2,event=0x100", 0x2, 0x100) | |
186 | IOMMU_PMU_EVENT_ATTR(hpt_nonleaf_hit, "event_group=0x2,event=0x200", 0x2, 0x200) | |
187 | IOMMU_PMU_EVENT_ATTR(iotlb_lookup, "event_group=0x3,event=0x001", 0x3, 0x001) | |
188 | IOMMU_PMU_EVENT_ATTR(iotlb_hit, "event_group=0x3,event=0x002", 0x3, 0x002) | |
189 | IOMMU_PMU_EVENT_ATTR(hpt_leaf_lookup, "event_group=0x3,event=0x004", 0x3, 0x004) | |
190 | IOMMU_PMU_EVENT_ATTR(hpt_leaf_hit, "event_group=0x3,event=0x008", 0x3, 0x008) | |
191 | IOMMU_PMU_EVENT_ATTR(int_cache_lookup, "event_group=0x4,event=0x001", 0x4, 0x001) | |
192 | IOMMU_PMU_EVENT_ATTR(int_cache_hit_nonposted, "event_group=0x4,event=0x002", 0x4, 0x002) | |
193 | IOMMU_PMU_EVENT_ATTR(int_cache_hit_posted, "event_group=0x4,event=0x004", 0x4, 0x004) | |
194 | ||
195 | static const struct attribute_group *iommu_pmu_attr_update[] = { | |
196 | &filter_requester_id_en, | |
197 | &filter_domain_en, | |
198 | &filter_pasid_en, | |
199 | &filter_ats_en, | |
200 | &filter_page_table_en, | |
201 | &filter_requester_id, | |
202 | &filter_domain, | |
203 | &filter_pasid, | |
204 | &filter_ats, | |
205 | &filter_page_table, | |
206 | &iommu_clocks, | |
207 | &iommu_requests, | |
208 | &pw_occupancy, | |
209 | &ats_blocked, | |
210 | &iommu_mrds, | |
211 | &iommu_mem_blocked, | |
212 | &pg_req_posted, | |
213 | &ctxt_cache_lookup, | |
214 | &ctxt_cache_hit, | |
215 | &pasid_cache_lookup, | |
216 | &pasid_cache_hit, | |
217 | &ss_nonleaf_lookup, | |
218 | &ss_nonleaf_hit, | |
219 | &fs_nonleaf_lookup, | |
220 | &fs_nonleaf_hit, | |
221 | &hpt_nonleaf_lookup, | |
222 | &hpt_nonleaf_hit, | |
223 | &iotlb_lookup, | |
224 | &iotlb_hit, | |
225 | &hpt_leaf_lookup, | |
226 | &hpt_leaf_hit, | |
227 | &int_cache_lookup, | |
228 | &int_cache_hit_nonposted, | |
229 | &int_cache_hit_posted, | |
230 | NULL | |
231 | }; | |
232 | ||
233 | static inline void __iomem * | |
234 | iommu_event_base(struct iommu_pmu *iommu_pmu, int idx) | |
235 | { | |
236 | return iommu_pmu->cntr_reg + idx * iommu_pmu->cntr_stride; | |
237 | } | |
238 | ||
239 | static inline void __iomem * | |
240 | iommu_config_base(struct iommu_pmu *iommu_pmu, int idx) | |
241 | { | |
242 | return iommu_pmu->cfg_reg + idx * IOMMU_PMU_CFG_OFFSET; | |
243 | } | |
244 | ||
245 | static inline struct iommu_pmu *iommu_event_to_pmu(struct perf_event *event) | |
246 | { | |
247 | return container_of(event->pmu, struct iommu_pmu, pmu); | |
248 | } | |
249 | ||
250 | static inline u64 iommu_event_config(struct perf_event *event) | |
251 | { | |
252 | u64 config = event->attr.config; | |
253 | ||
254 | return (iommu_event_select(config) << IOMMU_EVENT_CFG_ES_SHIFT) | | |
255 | (iommu_event_group(config) << IOMMU_EVENT_CFG_EGI_SHIFT) | | |
256 | IOMMU_EVENT_CFG_INT; | |
257 | } | |
258 | ||
259 | static inline bool is_iommu_pmu_event(struct iommu_pmu *iommu_pmu, | |
260 | struct perf_event *event) | |
261 | { | |
262 | return event->pmu == &iommu_pmu->pmu; | |
263 | } | |
264 | ||
265 | static int iommu_pmu_validate_event(struct perf_event *event) | |
266 | { | |
267 | struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event); | |
268 | u32 event_group = iommu_event_group(event->attr.config); | |
269 | ||
270 | if (event_group >= iommu_pmu->num_eg) | |
271 | return -EINVAL; | |
272 | ||
273 | return 0; | |
274 | } | |
275 | ||
276 | static int iommu_pmu_validate_group(struct perf_event *event) | |
277 | { | |
278 | struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event); | |
279 | struct perf_event *sibling; | |
280 | int nr = 0; | |
281 | ||
282 | /* | |
283 | * All events in a group must be scheduled simultaneously. | |
284 | * Check whether there is enough counters for all the events. | |
285 | */ | |
286 | for_each_sibling_event(sibling, event->group_leader) { | |
287 | if (!is_iommu_pmu_event(iommu_pmu, sibling) || | |
288 | sibling->state <= PERF_EVENT_STATE_OFF) | |
289 | continue; | |
290 | ||
291 | if (++nr > iommu_pmu->num_cntr) | |
292 | return -EINVAL; | |
293 | } | |
294 | ||
295 | return 0; | |
296 | } | |
297 | ||
298 | static int iommu_pmu_event_init(struct perf_event *event) | |
299 | { | |
300 | struct hw_perf_event *hwc = &event->hw; | |
301 | ||
302 | if (event->attr.type != event->pmu->type) | |
303 | return -ENOENT; | |
304 | ||
305 | /* sampling not supported */ | |
306 | if (event->attr.sample_period) | |
307 | return -EINVAL; | |
308 | ||
309 | if (event->cpu < 0) | |
310 | return -EINVAL; | |
311 | ||
312 | if (iommu_pmu_validate_event(event)) | |
313 | return -EINVAL; | |
314 | ||
315 | hwc->config = iommu_event_config(event); | |
316 | ||
317 | return iommu_pmu_validate_group(event); | |
318 | } | |
319 | ||
320 | static void iommu_pmu_event_update(struct perf_event *event) | |
321 | { | |
322 | struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event); | |
323 | struct hw_perf_event *hwc = &event->hw; | |
324 | u64 prev_count, new_count, delta; | |
325 | int shift = 64 - iommu_pmu->cntr_width; | |
326 | ||
327 | again: | |
328 | prev_count = local64_read(&hwc->prev_count); | |
329 | new_count = dmar_readq(iommu_event_base(iommu_pmu, hwc->idx)); | |
330 | if (local64_xchg(&hwc->prev_count, new_count) != prev_count) | |
331 | goto again; | |
332 | ||
333 | /* | |
334 | * The counter width is enumerated. Always shift the counter | |
335 | * before using it. | |
336 | */ | |
337 | delta = (new_count << shift) - (prev_count << shift); | |
338 | delta >>= shift; | |
339 | ||
340 | local64_add(delta, &event->count); | |
341 | } | |
342 | ||
343 | static void iommu_pmu_start(struct perf_event *event, int flags) | |
344 | { | |
345 | struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event); | |
346 | struct intel_iommu *iommu = iommu_pmu->iommu; | |
347 | struct hw_perf_event *hwc = &event->hw; | |
348 | u64 count; | |
349 | ||
350 | if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) | |
351 | return; | |
352 | ||
353 | if (WARN_ON_ONCE(hwc->idx < 0 || hwc->idx >= IOMMU_PMU_IDX_MAX)) | |
354 | return; | |
355 | ||
356 | if (flags & PERF_EF_RELOAD) | |
357 | WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); | |
358 | ||
359 | hwc->state = 0; | |
360 | ||
361 | /* Always reprogram the period */ | |
362 | count = dmar_readq(iommu_event_base(iommu_pmu, hwc->idx)); | |
363 | local64_set((&hwc->prev_count), count); | |
364 | ||
365 | /* | |
366 | * The error of ecmd will be ignored. | |
367 | * - The existing perf_event subsystem doesn't handle the error. | |
368 | * Only IOMMU PMU returns runtime HW error. We don't want to | |
369 | * change the existing generic interfaces for the specific case. | |
370 | * - It's a corner case caused by HW, which is very unlikely to | |
371 | * happen. There is nothing SW can do. | |
372 | * - The worst case is that the user will get <not count> with | |
373 | * perf command, which can give the user some hints. | |
374 | */ | |
375 | ecmd_submit_sync(iommu, DMA_ECMD_ENABLE, hwc->idx, 0); | |
376 | ||
377 | perf_event_update_userpage(event); | |
378 | } | |
379 | ||
380 | static void iommu_pmu_stop(struct perf_event *event, int flags) | |
381 | { | |
382 | struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event); | |
383 | struct intel_iommu *iommu = iommu_pmu->iommu; | |
384 | struct hw_perf_event *hwc = &event->hw; | |
385 | ||
386 | if (!(hwc->state & PERF_HES_STOPPED)) { | |
387 | ecmd_submit_sync(iommu, DMA_ECMD_DISABLE, hwc->idx, 0); | |
388 | ||
389 | iommu_pmu_event_update(event); | |
390 | ||
391 | hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; | |
392 | } | |
393 | } | |
394 | ||
395 | static inline int | |
396 | iommu_pmu_validate_per_cntr_event(struct iommu_pmu *iommu_pmu, | |
397 | int idx, struct perf_event *event) | |
398 | { | |
399 | u32 event_group = iommu_event_group(event->attr.config); | |
400 | u32 select = iommu_event_select(event->attr.config); | |
401 | ||
402 | if (!(iommu_pmu->cntr_evcap[idx][event_group] & select)) | |
403 | return -EINVAL; | |
404 | ||
405 | return 0; | |
406 | } | |
407 | ||
408 | static int iommu_pmu_assign_event(struct iommu_pmu *iommu_pmu, | |
409 | struct perf_event *event) | |
410 | { | |
411 | struct hw_perf_event *hwc = &event->hw; | |
412 | int idx; | |
413 | ||
414 | /* | |
415 | * The counters which support limited events are usually at the end. | |
416 | * Schedule them first to accommodate more events. | |
417 | */ | |
418 | for (idx = iommu_pmu->num_cntr - 1; idx >= 0; idx--) { | |
419 | if (test_and_set_bit(idx, iommu_pmu->used_mask)) | |
420 | continue; | |
421 | /* Check per-counter event capabilities */ | |
422 | if (!iommu_pmu_validate_per_cntr_event(iommu_pmu, idx, event)) | |
423 | break; | |
424 | clear_bit(idx, iommu_pmu->used_mask); | |
425 | } | |
426 | if (idx < 0) | |
427 | return -EINVAL; | |
428 | ||
429 | iommu_pmu->event_list[idx] = event; | |
430 | hwc->idx = idx; | |
431 | ||
432 | /* config events */ | |
433 | dmar_writeq(iommu_config_base(iommu_pmu, idx), hwc->config); | |
434 | ||
435 | iommu_pmu_set_filter(requester_id, event->attr.config1, | |
436 | IOMMU_PMU_FILTER_REQUESTER_ID, idx, | |
437 | event->attr.config1); | |
438 | iommu_pmu_set_filter(domain, event->attr.config1, | |
439 | IOMMU_PMU_FILTER_DOMAIN, idx, | |
440 | event->attr.config1); | |
441 | iommu_pmu_set_filter(pasid, event->attr.config1, | |
442 | IOMMU_PMU_FILTER_PASID, idx, | |
443 | event->attr.config1); | |
444 | iommu_pmu_set_filter(ats, event->attr.config2, | |
445 | IOMMU_PMU_FILTER_ATS, idx, | |
446 | event->attr.config1); | |
447 | iommu_pmu_set_filter(page_table, event->attr.config2, | |
448 | IOMMU_PMU_FILTER_PAGE_TABLE, idx, | |
449 | event->attr.config1); | |
450 | ||
451 | return 0; | |
452 | } | |
453 | ||
454 | static int iommu_pmu_add(struct perf_event *event, int flags) | |
455 | { | |
456 | struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event); | |
457 | struct hw_perf_event *hwc = &event->hw; | |
458 | int ret; | |
459 | ||
460 | ret = iommu_pmu_assign_event(iommu_pmu, event); | |
461 | if (ret < 0) | |
462 | return ret; | |
463 | ||
464 | hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; | |
465 | ||
466 | if (flags & PERF_EF_START) | |
467 | iommu_pmu_start(event, 0); | |
468 | ||
469 | return 0; | |
470 | } | |
471 | ||
472 | static void iommu_pmu_del(struct perf_event *event, int flags) | |
473 | { | |
474 | struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event); | |
475 | int idx = event->hw.idx; | |
476 | ||
477 | iommu_pmu_stop(event, PERF_EF_UPDATE); | |
478 | ||
479 | iommu_pmu_clear_filter(IOMMU_PMU_FILTER_REQUESTER_ID, idx); | |
480 | iommu_pmu_clear_filter(IOMMU_PMU_FILTER_DOMAIN, idx); | |
481 | iommu_pmu_clear_filter(IOMMU_PMU_FILTER_PASID, idx); | |
482 | iommu_pmu_clear_filter(IOMMU_PMU_FILTER_ATS, idx); | |
483 | iommu_pmu_clear_filter(IOMMU_PMU_FILTER_PAGE_TABLE, idx); | |
484 | ||
485 | iommu_pmu->event_list[idx] = NULL; | |
486 | event->hw.idx = -1; | |
487 | clear_bit(idx, iommu_pmu->used_mask); | |
488 | ||
489 | perf_event_update_userpage(event); | |
490 | } | |
491 | ||
492 | static void iommu_pmu_enable(struct pmu *pmu) | |
493 | { | |
494 | struct iommu_pmu *iommu_pmu = container_of(pmu, struct iommu_pmu, pmu); | |
495 | struct intel_iommu *iommu = iommu_pmu->iommu; | |
496 | ||
497 | ecmd_submit_sync(iommu, DMA_ECMD_UNFREEZE, 0, 0); | |
498 | } | |
499 | ||
500 | static void iommu_pmu_disable(struct pmu *pmu) | |
501 | { | |
502 | struct iommu_pmu *iommu_pmu = container_of(pmu, struct iommu_pmu, pmu); | |
503 | struct intel_iommu *iommu = iommu_pmu->iommu; | |
504 | ||
505 | ecmd_submit_sync(iommu, DMA_ECMD_FREEZE, 0, 0); | |
506 | } | |
507 | ||
4a0d4265 KL |
508 | static void iommu_pmu_counter_overflow(struct iommu_pmu *iommu_pmu) |
509 | { | |
510 | struct perf_event *event; | |
511 | u64 status; | |
512 | int i; | |
513 | ||
514 | /* | |
515 | * Two counters may be overflowed very close. Always check | |
516 | * whether there are more to handle. | |
517 | */ | |
518 | while ((status = dmar_readq(iommu_pmu->overflow))) { | |
519 | for_each_set_bit(i, (unsigned long *)&status, iommu_pmu->num_cntr) { | |
520 | /* | |
521 | * Find the assigned event of the counter. | |
522 | * Accumulate the value into the event->count. | |
523 | */ | |
524 | event = iommu_pmu->event_list[i]; | |
525 | if (!event) { | |
526 | pr_warn_once("Cannot find the assigned event for counter %d\n", i); | |
527 | continue; | |
528 | } | |
529 | iommu_pmu_event_update(event); | |
530 | } | |
531 | ||
532 | dmar_writeq(iommu_pmu->overflow, status); | |
533 | } | |
534 | } | |
535 | ||
536 | static irqreturn_t iommu_pmu_irq_handler(int irq, void *dev_id) | |
537 | { | |
538 | struct intel_iommu *iommu = dev_id; | |
539 | ||
540 | if (!dmar_readl(iommu->reg + DMAR_PERFINTRSTS_REG)) | |
541 | return IRQ_NONE; | |
542 | ||
543 | iommu_pmu_counter_overflow(iommu->pmu); | |
544 | ||
545 | /* Clear the status bit */ | |
546 | dmar_writel(iommu->reg + DMAR_PERFINTRSTS_REG, DMA_PERFINTRSTS_PIS); | |
547 | ||
548 | return IRQ_HANDLED; | |
549 | } | |
550 | ||
7232ab8b KL |
551 | static int __iommu_pmu_register(struct intel_iommu *iommu) |
552 | { | |
553 | struct iommu_pmu *iommu_pmu = iommu->pmu; | |
554 | ||
555 | iommu_pmu->pmu.name = iommu->name; | |
556 | iommu_pmu->pmu.task_ctx_nr = perf_invalid_context; | |
557 | iommu_pmu->pmu.event_init = iommu_pmu_event_init; | |
558 | iommu_pmu->pmu.pmu_enable = iommu_pmu_enable; | |
559 | iommu_pmu->pmu.pmu_disable = iommu_pmu_disable; | |
560 | iommu_pmu->pmu.add = iommu_pmu_add; | |
561 | iommu_pmu->pmu.del = iommu_pmu_del; | |
562 | iommu_pmu->pmu.start = iommu_pmu_start; | |
563 | iommu_pmu->pmu.stop = iommu_pmu_stop; | |
564 | iommu_pmu->pmu.read = iommu_pmu_event_update; | |
565 | iommu_pmu->pmu.attr_groups = iommu_pmu_attr_groups; | |
566 | iommu_pmu->pmu.attr_update = iommu_pmu_attr_update; | |
567 | iommu_pmu->pmu.capabilities = PERF_PMU_CAP_NO_EXCLUDE; | |
568 | iommu_pmu->pmu.module = THIS_MODULE; | |
569 | ||
570 | return perf_pmu_register(&iommu_pmu->pmu, iommu_pmu->pmu.name, -1); | |
571 | } | |
572 | ||
a6a5006d KL |
573 | static inline void __iomem * |
574 | get_perf_reg_address(struct intel_iommu *iommu, u32 offset) | |
575 | { | |
576 | u32 off = dmar_readl(iommu->reg + offset); | |
577 | ||
578 | return iommu->reg + off; | |
579 | } | |
580 | ||
581 | int alloc_iommu_pmu(struct intel_iommu *iommu) | |
582 | { | |
583 | struct iommu_pmu *iommu_pmu; | |
584 | int i, j, ret; | |
585 | u64 perfcap; | |
586 | u32 cap; | |
587 | ||
588 | if (!ecap_pms(iommu->ecap)) | |
589 | return 0; | |
590 | ||
591 | /* The IOMMU PMU requires the ECMD support as well */ | |
592 | if (!cap_ecmds(iommu->cap)) | |
593 | return -ENODEV; | |
594 | ||
595 | perfcap = dmar_readq(iommu->reg + DMAR_PERFCAP_REG); | |
596 | /* The performance monitoring is not supported. */ | |
597 | if (!perfcap) | |
598 | return -ENODEV; | |
599 | ||
600 | /* Sanity check for the number of the counters and event groups */ | |
601 | if (!pcap_num_cntr(perfcap) || !pcap_num_event_group(perfcap)) | |
602 | return -ENODEV; | |
603 | ||
604 | /* The interrupt on overflow is required */ | |
605 | if (!pcap_interrupt(perfcap)) | |
606 | return -ENODEV; | |
607 | ||
7232ab8b KL |
608 | /* Check required Enhanced Command Capability */ |
609 | if (!ecmd_has_pmu_essential(iommu)) | |
610 | return -ENODEV; | |
611 | ||
a6a5006d KL |
612 | iommu_pmu = kzalloc(sizeof(*iommu_pmu), GFP_KERNEL); |
613 | if (!iommu_pmu) | |
614 | return -ENOMEM; | |
615 | ||
616 | iommu_pmu->num_cntr = pcap_num_cntr(perfcap); | |
7232ab8b KL |
617 | if (iommu_pmu->num_cntr > IOMMU_PMU_IDX_MAX) { |
618 | pr_warn_once("The number of IOMMU counters %d > max(%d), clipping!", | |
619 | iommu_pmu->num_cntr, IOMMU_PMU_IDX_MAX); | |
620 | iommu_pmu->num_cntr = IOMMU_PMU_IDX_MAX; | |
621 | } | |
622 | ||
a6a5006d KL |
623 | iommu_pmu->cntr_width = pcap_cntr_width(perfcap); |
624 | iommu_pmu->filter = pcap_filters_mask(perfcap); | |
625 | iommu_pmu->cntr_stride = pcap_cntr_stride(perfcap); | |
626 | iommu_pmu->num_eg = pcap_num_event_group(perfcap); | |
627 | ||
628 | iommu_pmu->evcap = kcalloc(iommu_pmu->num_eg, sizeof(u64), GFP_KERNEL); | |
629 | if (!iommu_pmu->evcap) { | |
630 | ret = -ENOMEM; | |
631 | goto free_pmu; | |
632 | } | |
633 | ||
634 | /* Parse event group capabilities */ | |
635 | for (i = 0; i < iommu_pmu->num_eg; i++) { | |
636 | u64 pcap; | |
637 | ||
638 | pcap = dmar_readq(iommu->reg + DMAR_PERFEVNTCAP_REG + | |
639 | i * IOMMU_PMU_CAP_REGS_STEP); | |
640 | iommu_pmu->evcap[i] = pecap_es(pcap); | |
641 | } | |
642 | ||
643 | iommu_pmu->cntr_evcap = kcalloc(iommu_pmu->num_cntr, sizeof(u32 *), GFP_KERNEL); | |
644 | if (!iommu_pmu->cntr_evcap) { | |
645 | ret = -ENOMEM; | |
646 | goto free_pmu_evcap; | |
647 | } | |
648 | for (i = 0; i < iommu_pmu->num_cntr; i++) { | |
649 | iommu_pmu->cntr_evcap[i] = kcalloc(iommu_pmu->num_eg, sizeof(u32), GFP_KERNEL); | |
650 | if (!iommu_pmu->cntr_evcap[i]) { | |
651 | ret = -ENOMEM; | |
652 | goto free_pmu_cntr_evcap; | |
653 | } | |
654 | /* | |
655 | * Set to the global capabilities, will adjust according | |
656 | * to per-counter capabilities later. | |
657 | */ | |
658 | for (j = 0; j < iommu_pmu->num_eg; j++) | |
659 | iommu_pmu->cntr_evcap[i][j] = (u32)iommu_pmu->evcap[j]; | |
660 | } | |
661 | ||
662 | iommu_pmu->cfg_reg = get_perf_reg_address(iommu, DMAR_PERFCFGOFF_REG); | |
663 | iommu_pmu->cntr_reg = get_perf_reg_address(iommu, DMAR_PERFCNTROFF_REG); | |
664 | iommu_pmu->overflow = get_perf_reg_address(iommu, DMAR_PERFOVFOFF_REG); | |
665 | ||
666 | /* | |
667 | * Check per-counter capabilities. All counters should have the | |
668 | * same capabilities on Interrupt on Overflow Support and Counter | |
669 | * Width. | |
670 | */ | |
671 | for (i = 0; i < iommu_pmu->num_cntr; i++) { | |
672 | cap = dmar_readl(iommu_pmu->cfg_reg + | |
673 | i * IOMMU_PMU_CFG_OFFSET + | |
674 | IOMMU_PMU_CFG_CNTRCAP_OFFSET); | |
675 | if (!iommu_cntrcap_pcc(cap)) | |
676 | continue; | |
677 | ||
678 | /* | |
679 | * It's possible that some counters have a different | |
680 | * capability because of e.g., HW bug. Check the corner | |
681 | * case here and simply drop those counters. | |
682 | */ | |
683 | if ((iommu_cntrcap_cw(cap) != iommu_pmu->cntr_width) || | |
684 | !iommu_cntrcap_ios(cap)) { | |
685 | iommu_pmu->num_cntr = i; | |
686 | pr_warn("PMU counter capability inconsistent, counter number reduced to %d\n", | |
687 | iommu_pmu->num_cntr); | |
688 | } | |
689 | ||
690 | /* Clear the pre-defined events group */ | |
691 | for (j = 0; j < iommu_pmu->num_eg; j++) | |
692 | iommu_pmu->cntr_evcap[i][j] = 0; | |
693 | ||
694 | /* Override with per-counter event capabilities */ | |
695 | for (j = 0; j < iommu_cntrcap_egcnt(cap); j++) { | |
696 | cap = dmar_readl(iommu_pmu->cfg_reg + i * IOMMU_PMU_CFG_OFFSET + | |
697 | IOMMU_PMU_CFG_CNTREVCAP_OFFSET + | |
698 | (j * IOMMU_PMU_OFF_REGS_STEP)); | |
699 | iommu_pmu->cntr_evcap[i][iommu_event_group(cap)] = iommu_event_select(cap); | |
700 | /* | |
701 | * Some events may only be supported by a specific counter. | |
702 | * Track them in the evcap as well. | |
703 | */ | |
704 | iommu_pmu->evcap[iommu_event_group(cap)] |= iommu_event_select(cap); | |
705 | } | |
706 | } | |
707 | ||
708 | iommu_pmu->iommu = iommu; | |
709 | iommu->pmu = iommu_pmu; | |
710 | ||
711 | return 0; | |
712 | ||
713 | free_pmu_cntr_evcap: | |
714 | for (i = 0; i < iommu_pmu->num_cntr; i++) | |
715 | kfree(iommu_pmu->cntr_evcap[i]); | |
716 | kfree(iommu_pmu->cntr_evcap); | |
717 | free_pmu_evcap: | |
718 | kfree(iommu_pmu->evcap); | |
719 | free_pmu: | |
720 | kfree(iommu_pmu); | |
721 | ||
722 | return ret; | |
723 | } | |
724 | ||
725 | void free_iommu_pmu(struct intel_iommu *iommu) | |
726 | { | |
727 | struct iommu_pmu *iommu_pmu = iommu->pmu; | |
728 | ||
729 | if (!iommu_pmu) | |
730 | return; | |
731 | ||
732 | if (iommu_pmu->evcap) { | |
733 | int i; | |
734 | ||
735 | for (i = 0; i < iommu_pmu->num_cntr; i++) | |
736 | kfree(iommu_pmu->cntr_evcap[i]); | |
737 | kfree(iommu_pmu->cntr_evcap); | |
738 | } | |
739 | kfree(iommu_pmu->evcap); | |
740 | kfree(iommu_pmu); | |
741 | iommu->pmu = NULL; | |
742 | } | |
7232ab8b | 743 | |
4a0d4265 KL |
744 | static int iommu_pmu_set_interrupt(struct intel_iommu *iommu) |
745 | { | |
746 | struct iommu_pmu *iommu_pmu = iommu->pmu; | |
747 | int irq, ret; | |
748 | ||
749 | irq = dmar_alloc_hwirq(IOMMU_IRQ_ID_OFFSET_PERF + iommu->seq_id, iommu->node, iommu); | |
750 | if (irq <= 0) | |
751 | return -EINVAL; | |
752 | ||
753 | snprintf(iommu_pmu->irq_name, sizeof(iommu_pmu->irq_name), "dmar%d-perf", iommu->seq_id); | |
754 | ||
755 | iommu->perf_irq = irq; | |
756 | ret = request_threaded_irq(irq, NULL, iommu_pmu_irq_handler, | |
757 | IRQF_ONESHOT, iommu_pmu->irq_name, iommu); | |
758 | if (ret) { | |
759 | dmar_free_hwirq(irq); | |
760 | iommu->perf_irq = 0; | |
761 | return ret; | |
762 | } | |
763 | return 0; | |
764 | } | |
765 | ||
766 | static void iommu_pmu_unset_interrupt(struct intel_iommu *iommu) | |
767 | { | |
768 | if (!iommu->perf_irq) | |
769 | return; | |
770 | ||
771 | free_irq(iommu->perf_irq, iommu); | |
772 | dmar_free_hwirq(iommu->perf_irq); | |
773 | iommu->perf_irq = 0; | |
774 | } | |
775 | ||
16812c96 | 776 | static int iommu_pmu_cpu_online(unsigned int cpu, struct hlist_node *node) |
46284c6c | 777 | { |
16812c96 KL |
778 | struct iommu_pmu *iommu_pmu = hlist_entry_safe(node, typeof(*iommu_pmu), cpuhp_node); |
779 | ||
46284c6c KL |
780 | if (cpumask_empty(&iommu_pmu_cpu_mask)) |
781 | cpumask_set_cpu(cpu, &iommu_pmu_cpu_mask); | |
782 | ||
16812c96 KL |
783 | if (cpumask_test_cpu(cpu, &iommu_pmu_cpu_mask)) |
784 | iommu_pmu->cpu = cpu; | |
785 | ||
46284c6c KL |
786 | return 0; |
787 | } | |
788 | ||
16812c96 | 789 | static int iommu_pmu_cpu_offline(unsigned int cpu, struct hlist_node *node) |
46284c6c | 790 | { |
16812c96 KL |
791 | struct iommu_pmu *iommu_pmu = hlist_entry_safe(node, typeof(*iommu_pmu), cpuhp_node); |
792 | int target = cpumask_first(&iommu_pmu_cpu_mask); | |
793 | ||
794 | /* | |
795 | * The iommu_pmu_cpu_mask has been updated when offline the CPU | |
796 | * for the first iommu_pmu. Migrate the other iommu_pmu to the | |
797 | * new target. | |
798 | */ | |
799 | if (target < nr_cpu_ids && target != iommu_pmu->cpu) { | |
800 | perf_pmu_migrate_context(&iommu_pmu->pmu, cpu, target); | |
801 | iommu_pmu->cpu = target; | |
802 | return 0; | |
803 | } | |
46284c6c KL |
804 | |
805 | if (!cpumask_test_and_clear_cpu(cpu, &iommu_pmu_cpu_mask)) | |
806 | return 0; | |
807 | ||
808 | target = cpumask_any_but(cpu_online_mask, cpu); | |
809 | ||
810 | if (target < nr_cpu_ids) | |
811 | cpumask_set_cpu(target, &iommu_pmu_cpu_mask); | |
812 | else | |
16812c96 | 813 | return 0; |
46284c6c | 814 | |
16812c96 KL |
815 | perf_pmu_migrate_context(&iommu_pmu->pmu, cpu, target); |
816 | iommu_pmu->cpu = target; | |
46284c6c KL |
817 | |
818 | return 0; | |
819 | } | |
820 | ||
821 | static int nr_iommu_pmu; | |
16812c96 | 822 | static enum cpuhp_state iommu_cpuhp_slot; |
46284c6c KL |
823 | |
824 | static int iommu_pmu_cpuhp_setup(struct iommu_pmu *iommu_pmu) | |
825 | { | |
826 | int ret; | |
827 | ||
16812c96 KL |
828 | if (!nr_iommu_pmu) { |
829 | ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, | |
830 | "driver/iommu/intel/perfmon:online", | |
831 | iommu_pmu_cpu_online, | |
832 | iommu_pmu_cpu_offline); | |
833 | if (ret < 0) | |
834 | return ret; | |
835 | iommu_cpuhp_slot = ret; | |
836 | } | |
46284c6c | 837 | |
16812c96 KL |
838 | ret = cpuhp_state_add_instance(iommu_cpuhp_slot, &iommu_pmu->cpuhp_node); |
839 | if (ret) { | |
840 | if (!nr_iommu_pmu) | |
841 | cpuhp_remove_multi_state(iommu_cpuhp_slot); | |
842 | return ret; | |
843 | } | |
844 | nr_iommu_pmu++; | |
46284c6c | 845 | |
16812c96 | 846 | return 0; |
46284c6c KL |
847 | } |
848 | ||
849 | static void iommu_pmu_cpuhp_free(struct iommu_pmu *iommu_pmu) | |
850 | { | |
16812c96 KL |
851 | cpuhp_state_remove_instance(iommu_cpuhp_slot, &iommu_pmu->cpuhp_node); |
852 | ||
46284c6c KL |
853 | if (--nr_iommu_pmu) |
854 | return; | |
855 | ||
16812c96 | 856 | cpuhp_remove_multi_state(iommu_cpuhp_slot); |
46284c6c KL |
857 | } |
858 | ||
7232ab8b KL |
859 | void iommu_pmu_register(struct intel_iommu *iommu) |
860 | { | |
46284c6c KL |
861 | struct iommu_pmu *iommu_pmu = iommu->pmu; |
862 | ||
863 | if (!iommu_pmu) | |
7232ab8b KL |
864 | return; |
865 | ||
46284c6c KL |
866 | if (__iommu_pmu_register(iommu)) |
867 | goto err; | |
868 | ||
869 | if (iommu_pmu_cpuhp_setup(iommu_pmu)) | |
870 | goto unregister; | |
871 | ||
4a0d4265 KL |
872 | /* Set interrupt for overflow */ |
873 | if (iommu_pmu_set_interrupt(iommu)) | |
874 | goto cpuhp_free; | |
875 | ||
46284c6c KL |
876 | return; |
877 | ||
4a0d4265 KL |
878 | cpuhp_free: |
879 | iommu_pmu_cpuhp_free(iommu_pmu); | |
46284c6c KL |
880 | unregister: |
881 | perf_pmu_unregister(&iommu_pmu->pmu); | |
882 | err: | |
883 | pr_err("Failed to register PMU for iommu (seq_id = %d)\n", iommu->seq_id); | |
884 | free_iommu_pmu(iommu); | |
7232ab8b KL |
885 | } |
886 | ||
887 | void iommu_pmu_unregister(struct intel_iommu *iommu) | |
888 | { | |
46284c6c KL |
889 | struct iommu_pmu *iommu_pmu = iommu->pmu; |
890 | ||
891 | if (!iommu_pmu) | |
892 | return; | |
893 | ||
4a0d4265 | 894 | iommu_pmu_unset_interrupt(iommu); |
46284c6c KL |
895 | iommu_pmu_cpuhp_free(iommu_pmu); |
896 | perf_pmu_unregister(&iommu_pmu->pmu); | |
7232ab8b | 897 | } |