Commit | Line | Data |
---|---|---|
a6a5006d KL |
1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* | |
3 | * Support Intel IOMMU PerfMon | |
4 | * Copyright(c) 2023 Intel Corporation. | |
5 | */ | |
6 | #define pr_fmt(fmt) "DMAR: " fmt | |
7 | #define dev_fmt(fmt) pr_fmt(fmt) | |
8 | ||
9 | #include <linux/dmar.h> | |
10 | #include "iommu.h" | |
11 | #include "perfmon.h" | |
12 | ||
7232ab8b KL |
13 | PMU_FORMAT_ATTR(event, "config:0-27"); /* ES: Events Select */ |
14 | PMU_FORMAT_ATTR(event_group, "config:28-31"); /* EGI: Event Group Index */ | |
15 | ||
16 | static struct attribute *iommu_pmu_format_attrs[] = { | |
17 | &format_attr_event_group.attr, | |
18 | &format_attr_event.attr, | |
19 | NULL | |
20 | }; | |
21 | ||
22 | static struct attribute_group iommu_pmu_format_attr_group = { | |
23 | .name = "format", | |
24 | .attrs = iommu_pmu_format_attrs, | |
25 | }; | |
26 | ||
27 | /* The available events are added in attr_update later */ | |
28 | static struct attribute *attrs_empty[] = { | |
29 | NULL | |
30 | }; | |
31 | ||
32 | static struct attribute_group iommu_pmu_events_attr_group = { | |
33 | .name = "events", | |
34 | .attrs = attrs_empty, | |
35 | }; | |
36 | ||
37 | static const struct attribute_group *iommu_pmu_attr_groups[] = { | |
38 | &iommu_pmu_format_attr_group, | |
39 | &iommu_pmu_events_attr_group, | |
40 | NULL | |
41 | }; | |
42 | ||
43 | static inline struct iommu_pmu *dev_to_iommu_pmu(struct device *dev) | |
44 | { | |
45 | /* | |
46 | * The perf_event creates its own dev for each PMU. | |
47 | * See pmu_dev_alloc() | |
48 | */ | |
49 | return container_of(dev_get_drvdata(dev), struct iommu_pmu, pmu); | |
50 | } | |
51 | ||
52 | #define IOMMU_PMU_ATTR(_name, _format, _filter) \ | |
53 | PMU_FORMAT_ATTR(_name, _format); \ | |
54 | \ | |
55 | static struct attribute *_name##_attr[] = { \ | |
56 | &format_attr_##_name.attr, \ | |
57 | NULL \ | |
58 | }; \ | |
59 | \ | |
60 | static umode_t \ | |
61 | _name##_is_visible(struct kobject *kobj, struct attribute *attr, int i) \ | |
62 | { \ | |
63 | struct device *dev = kobj_to_dev(kobj); \ | |
64 | struct iommu_pmu *iommu_pmu = dev_to_iommu_pmu(dev); \ | |
65 | \ | |
66 | if (!iommu_pmu) \ | |
67 | return 0; \ | |
68 | return (iommu_pmu->filter & _filter) ? attr->mode : 0; \ | |
69 | } \ | |
70 | \ | |
71 | static struct attribute_group _name = { \ | |
72 | .name = "format", \ | |
73 | .attrs = _name##_attr, \ | |
74 | .is_visible = _name##_is_visible, \ | |
75 | }; | |
76 | ||
77 | IOMMU_PMU_ATTR(filter_requester_id_en, "config1:0", IOMMU_PMU_FILTER_REQUESTER_ID); | |
78 | IOMMU_PMU_ATTR(filter_domain_en, "config1:1", IOMMU_PMU_FILTER_DOMAIN); | |
79 | IOMMU_PMU_ATTR(filter_pasid_en, "config1:2", IOMMU_PMU_FILTER_PASID); | |
80 | IOMMU_PMU_ATTR(filter_ats_en, "config1:3", IOMMU_PMU_FILTER_ATS); | |
81 | IOMMU_PMU_ATTR(filter_page_table_en, "config1:4", IOMMU_PMU_FILTER_PAGE_TABLE); | |
82 | IOMMU_PMU_ATTR(filter_requester_id, "config1:16-31", IOMMU_PMU_FILTER_REQUESTER_ID); | |
83 | IOMMU_PMU_ATTR(filter_domain, "config1:32-47", IOMMU_PMU_FILTER_DOMAIN); | |
84 | IOMMU_PMU_ATTR(filter_pasid, "config2:0-21", IOMMU_PMU_FILTER_PASID); | |
85 | IOMMU_PMU_ATTR(filter_ats, "config2:24-28", IOMMU_PMU_FILTER_ATS); | |
86 | IOMMU_PMU_ATTR(filter_page_table, "config2:32-36", IOMMU_PMU_FILTER_PAGE_TABLE); | |
87 | ||
88 | #define iommu_pmu_en_requester_id(e) ((e) & 0x1) | |
89 | #define iommu_pmu_en_domain(e) (((e) >> 1) & 0x1) | |
90 | #define iommu_pmu_en_pasid(e) (((e) >> 2) & 0x1) | |
91 | #define iommu_pmu_en_ats(e) (((e) >> 3) & 0x1) | |
92 | #define iommu_pmu_en_page_table(e) (((e) >> 4) & 0x1) | |
93 | #define iommu_pmu_get_requester_id(filter) (((filter) >> 16) & 0xffff) | |
94 | #define iommu_pmu_get_domain(filter) (((filter) >> 32) & 0xffff) | |
95 | #define iommu_pmu_get_pasid(filter) ((filter) & 0x3fffff) | |
96 | #define iommu_pmu_get_ats(filter) (((filter) >> 24) & 0x1f) | |
97 | #define iommu_pmu_get_page_table(filter) (((filter) >> 32) & 0x1f) | |
98 | ||
99 | #define iommu_pmu_set_filter(_name, _config, _filter, _idx, _econfig) \ | |
100 | { \ | |
101 | if ((iommu_pmu->filter & _filter) && iommu_pmu_en_##_name(_econfig)) { \ | |
102 | dmar_writel(iommu_pmu->cfg_reg + _idx * IOMMU_PMU_CFG_OFFSET + \ | |
103 | IOMMU_PMU_CFG_SIZE + \ | |
104 | (ffs(_filter) - 1) * IOMMU_PMU_CFG_FILTERS_OFFSET, \ | |
105 | iommu_pmu_get_##_name(_config) | IOMMU_PMU_FILTER_EN);\ | |
106 | } \ | |
107 | } | |
108 | ||
109 | #define iommu_pmu_clear_filter(_filter, _idx) \ | |
110 | { \ | |
111 | if (iommu_pmu->filter & _filter) { \ | |
112 | dmar_writel(iommu_pmu->cfg_reg + _idx * IOMMU_PMU_CFG_OFFSET + \ | |
113 | IOMMU_PMU_CFG_SIZE + \ | |
114 | (ffs(_filter) - 1) * IOMMU_PMU_CFG_FILTERS_OFFSET, \ | |
115 | 0); \ | |
116 | } \ | |
117 | } | |
118 | ||
119 | /* | |
120 | * Define the event attr related functions | |
121 | * Input: _name: event attr name | |
122 | * _string: string of the event in sysfs | |
123 | * _g_idx: event group encoding | |
124 | * _event: event encoding | |
125 | */ | |
126 | #define IOMMU_PMU_EVENT_ATTR(_name, _string, _g_idx, _event) \ | |
127 | PMU_EVENT_ATTR_STRING(_name, event_attr_##_name, _string) \ | |
128 | \ | |
129 | static struct attribute *_name##_attr[] = { \ | |
130 | &event_attr_##_name.attr.attr, \ | |
131 | NULL \ | |
132 | }; \ | |
133 | \ | |
134 | static umode_t \ | |
135 | _name##_is_visible(struct kobject *kobj, struct attribute *attr, int i) \ | |
136 | { \ | |
137 | struct device *dev = kobj_to_dev(kobj); \ | |
138 | struct iommu_pmu *iommu_pmu = dev_to_iommu_pmu(dev); \ | |
139 | \ | |
140 | if (!iommu_pmu) \ | |
141 | return 0; \ | |
142 | return (iommu_pmu->evcap[_g_idx] & _event) ? attr->mode : 0; \ | |
143 | } \ | |
144 | \ | |
145 | static struct attribute_group _name = { \ | |
146 | .name = "events", \ | |
147 | .attrs = _name##_attr, \ | |
148 | .is_visible = _name##_is_visible, \ | |
149 | }; | |
150 | ||
151 | IOMMU_PMU_EVENT_ATTR(iommu_clocks, "event_group=0x0,event=0x001", 0x0, 0x001) | |
152 | IOMMU_PMU_EVENT_ATTR(iommu_requests, "event_group=0x0,event=0x002", 0x0, 0x002) | |
153 | IOMMU_PMU_EVENT_ATTR(pw_occupancy, "event_group=0x0,event=0x004", 0x0, 0x004) | |
154 | IOMMU_PMU_EVENT_ATTR(ats_blocked, "event_group=0x0,event=0x008", 0x0, 0x008) | |
155 | IOMMU_PMU_EVENT_ATTR(iommu_mrds, "event_group=0x1,event=0x001", 0x1, 0x001) | |
156 | IOMMU_PMU_EVENT_ATTR(iommu_mem_blocked, "event_group=0x1,event=0x020", 0x1, 0x020) | |
157 | IOMMU_PMU_EVENT_ATTR(pg_req_posted, "event_group=0x1,event=0x040", 0x1, 0x040) | |
158 | IOMMU_PMU_EVENT_ATTR(ctxt_cache_lookup, "event_group=0x2,event=0x001", 0x2, 0x001) | |
159 | IOMMU_PMU_EVENT_ATTR(ctxt_cache_hit, "event_group=0x2,event=0x002", 0x2, 0x002) | |
160 | IOMMU_PMU_EVENT_ATTR(pasid_cache_lookup, "event_group=0x2,event=0x004", 0x2, 0x004) | |
161 | IOMMU_PMU_EVENT_ATTR(pasid_cache_hit, "event_group=0x2,event=0x008", 0x2, 0x008) | |
162 | IOMMU_PMU_EVENT_ATTR(ss_nonleaf_lookup, "event_group=0x2,event=0x010", 0x2, 0x010) | |
163 | IOMMU_PMU_EVENT_ATTR(ss_nonleaf_hit, "event_group=0x2,event=0x020", 0x2, 0x020) | |
164 | IOMMU_PMU_EVENT_ATTR(fs_nonleaf_lookup, "event_group=0x2,event=0x040", 0x2, 0x040) | |
165 | IOMMU_PMU_EVENT_ATTR(fs_nonleaf_hit, "event_group=0x2,event=0x080", 0x2, 0x080) | |
166 | IOMMU_PMU_EVENT_ATTR(hpt_nonleaf_lookup, "event_group=0x2,event=0x100", 0x2, 0x100) | |
167 | IOMMU_PMU_EVENT_ATTR(hpt_nonleaf_hit, "event_group=0x2,event=0x200", 0x2, 0x200) | |
168 | IOMMU_PMU_EVENT_ATTR(iotlb_lookup, "event_group=0x3,event=0x001", 0x3, 0x001) | |
169 | IOMMU_PMU_EVENT_ATTR(iotlb_hit, "event_group=0x3,event=0x002", 0x3, 0x002) | |
170 | IOMMU_PMU_EVENT_ATTR(hpt_leaf_lookup, "event_group=0x3,event=0x004", 0x3, 0x004) | |
171 | IOMMU_PMU_EVENT_ATTR(hpt_leaf_hit, "event_group=0x3,event=0x008", 0x3, 0x008) | |
172 | IOMMU_PMU_EVENT_ATTR(int_cache_lookup, "event_group=0x4,event=0x001", 0x4, 0x001) | |
173 | IOMMU_PMU_EVENT_ATTR(int_cache_hit_nonposted, "event_group=0x4,event=0x002", 0x4, 0x002) | |
174 | IOMMU_PMU_EVENT_ATTR(int_cache_hit_posted, "event_group=0x4,event=0x004", 0x4, 0x004) | |
175 | ||
176 | static const struct attribute_group *iommu_pmu_attr_update[] = { | |
177 | &filter_requester_id_en, | |
178 | &filter_domain_en, | |
179 | &filter_pasid_en, | |
180 | &filter_ats_en, | |
181 | &filter_page_table_en, | |
182 | &filter_requester_id, | |
183 | &filter_domain, | |
184 | &filter_pasid, | |
185 | &filter_ats, | |
186 | &filter_page_table, | |
187 | &iommu_clocks, | |
188 | &iommu_requests, | |
189 | &pw_occupancy, | |
190 | &ats_blocked, | |
191 | &iommu_mrds, | |
192 | &iommu_mem_blocked, | |
193 | &pg_req_posted, | |
194 | &ctxt_cache_lookup, | |
195 | &ctxt_cache_hit, | |
196 | &pasid_cache_lookup, | |
197 | &pasid_cache_hit, | |
198 | &ss_nonleaf_lookup, | |
199 | &ss_nonleaf_hit, | |
200 | &fs_nonleaf_lookup, | |
201 | &fs_nonleaf_hit, | |
202 | &hpt_nonleaf_lookup, | |
203 | &hpt_nonleaf_hit, | |
204 | &iotlb_lookup, | |
205 | &iotlb_hit, | |
206 | &hpt_leaf_lookup, | |
207 | &hpt_leaf_hit, | |
208 | &int_cache_lookup, | |
209 | &int_cache_hit_nonposted, | |
210 | &int_cache_hit_posted, | |
211 | NULL | |
212 | }; | |
213 | ||
214 | static inline void __iomem * | |
215 | iommu_event_base(struct iommu_pmu *iommu_pmu, int idx) | |
216 | { | |
217 | return iommu_pmu->cntr_reg + idx * iommu_pmu->cntr_stride; | |
218 | } | |
219 | ||
220 | static inline void __iomem * | |
221 | iommu_config_base(struct iommu_pmu *iommu_pmu, int idx) | |
222 | { | |
223 | return iommu_pmu->cfg_reg + idx * IOMMU_PMU_CFG_OFFSET; | |
224 | } | |
225 | ||
226 | static inline struct iommu_pmu *iommu_event_to_pmu(struct perf_event *event) | |
227 | { | |
228 | return container_of(event->pmu, struct iommu_pmu, pmu); | |
229 | } | |
230 | ||
231 | static inline u64 iommu_event_config(struct perf_event *event) | |
232 | { | |
233 | u64 config = event->attr.config; | |
234 | ||
235 | return (iommu_event_select(config) << IOMMU_EVENT_CFG_ES_SHIFT) | | |
236 | (iommu_event_group(config) << IOMMU_EVENT_CFG_EGI_SHIFT) | | |
237 | IOMMU_EVENT_CFG_INT; | |
238 | } | |
239 | ||
240 | static inline bool is_iommu_pmu_event(struct iommu_pmu *iommu_pmu, | |
241 | struct perf_event *event) | |
242 | { | |
243 | return event->pmu == &iommu_pmu->pmu; | |
244 | } | |
245 | ||
246 | static int iommu_pmu_validate_event(struct perf_event *event) | |
247 | { | |
248 | struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event); | |
249 | u32 event_group = iommu_event_group(event->attr.config); | |
250 | ||
251 | if (event_group >= iommu_pmu->num_eg) | |
252 | return -EINVAL; | |
253 | ||
254 | return 0; | |
255 | } | |
256 | ||
257 | static int iommu_pmu_validate_group(struct perf_event *event) | |
258 | { | |
259 | struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event); | |
260 | struct perf_event *sibling; | |
261 | int nr = 0; | |
262 | ||
263 | /* | |
264 | * All events in a group must be scheduled simultaneously. | |
265 | * Check whether there is enough counters for all the events. | |
266 | */ | |
267 | for_each_sibling_event(sibling, event->group_leader) { | |
268 | if (!is_iommu_pmu_event(iommu_pmu, sibling) || | |
269 | sibling->state <= PERF_EVENT_STATE_OFF) | |
270 | continue; | |
271 | ||
272 | if (++nr > iommu_pmu->num_cntr) | |
273 | return -EINVAL; | |
274 | } | |
275 | ||
276 | return 0; | |
277 | } | |
278 | ||
279 | static int iommu_pmu_event_init(struct perf_event *event) | |
280 | { | |
281 | struct hw_perf_event *hwc = &event->hw; | |
282 | ||
283 | if (event->attr.type != event->pmu->type) | |
284 | return -ENOENT; | |
285 | ||
286 | /* sampling not supported */ | |
287 | if (event->attr.sample_period) | |
288 | return -EINVAL; | |
289 | ||
290 | if (event->cpu < 0) | |
291 | return -EINVAL; | |
292 | ||
293 | if (iommu_pmu_validate_event(event)) | |
294 | return -EINVAL; | |
295 | ||
296 | hwc->config = iommu_event_config(event); | |
297 | ||
298 | return iommu_pmu_validate_group(event); | |
299 | } | |
300 | ||
301 | static void iommu_pmu_event_update(struct perf_event *event) | |
302 | { | |
303 | struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event); | |
304 | struct hw_perf_event *hwc = &event->hw; | |
305 | u64 prev_count, new_count, delta; | |
306 | int shift = 64 - iommu_pmu->cntr_width; | |
307 | ||
308 | again: | |
309 | prev_count = local64_read(&hwc->prev_count); | |
310 | new_count = dmar_readq(iommu_event_base(iommu_pmu, hwc->idx)); | |
311 | if (local64_xchg(&hwc->prev_count, new_count) != prev_count) | |
312 | goto again; | |
313 | ||
314 | /* | |
315 | * The counter width is enumerated. Always shift the counter | |
316 | * before using it. | |
317 | */ | |
318 | delta = (new_count << shift) - (prev_count << shift); | |
319 | delta >>= shift; | |
320 | ||
321 | local64_add(delta, &event->count); | |
322 | } | |
323 | ||
324 | static void iommu_pmu_start(struct perf_event *event, int flags) | |
325 | { | |
326 | struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event); | |
327 | struct intel_iommu *iommu = iommu_pmu->iommu; | |
328 | struct hw_perf_event *hwc = &event->hw; | |
329 | u64 count; | |
330 | ||
331 | if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) | |
332 | return; | |
333 | ||
334 | if (WARN_ON_ONCE(hwc->idx < 0 || hwc->idx >= IOMMU_PMU_IDX_MAX)) | |
335 | return; | |
336 | ||
337 | if (flags & PERF_EF_RELOAD) | |
338 | WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); | |
339 | ||
340 | hwc->state = 0; | |
341 | ||
342 | /* Always reprogram the period */ | |
343 | count = dmar_readq(iommu_event_base(iommu_pmu, hwc->idx)); | |
344 | local64_set((&hwc->prev_count), count); | |
345 | ||
346 | /* | |
347 | * The error of ecmd will be ignored. | |
348 | * - The existing perf_event subsystem doesn't handle the error. | |
349 | * Only IOMMU PMU returns runtime HW error. We don't want to | |
350 | * change the existing generic interfaces for the specific case. | |
351 | * - It's a corner case caused by HW, which is very unlikely to | |
352 | * happen. There is nothing SW can do. | |
353 | * - The worst case is that the user will get <not count> with | |
354 | * perf command, which can give the user some hints. | |
355 | */ | |
356 | ecmd_submit_sync(iommu, DMA_ECMD_ENABLE, hwc->idx, 0); | |
357 | ||
358 | perf_event_update_userpage(event); | |
359 | } | |
360 | ||
361 | static void iommu_pmu_stop(struct perf_event *event, int flags) | |
362 | { | |
363 | struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event); | |
364 | struct intel_iommu *iommu = iommu_pmu->iommu; | |
365 | struct hw_perf_event *hwc = &event->hw; | |
366 | ||
367 | if (!(hwc->state & PERF_HES_STOPPED)) { | |
368 | ecmd_submit_sync(iommu, DMA_ECMD_DISABLE, hwc->idx, 0); | |
369 | ||
370 | iommu_pmu_event_update(event); | |
371 | ||
372 | hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; | |
373 | } | |
374 | } | |
375 | ||
376 | static inline int | |
377 | iommu_pmu_validate_per_cntr_event(struct iommu_pmu *iommu_pmu, | |
378 | int idx, struct perf_event *event) | |
379 | { | |
380 | u32 event_group = iommu_event_group(event->attr.config); | |
381 | u32 select = iommu_event_select(event->attr.config); | |
382 | ||
383 | if (!(iommu_pmu->cntr_evcap[idx][event_group] & select)) | |
384 | return -EINVAL; | |
385 | ||
386 | return 0; | |
387 | } | |
388 | ||
389 | static int iommu_pmu_assign_event(struct iommu_pmu *iommu_pmu, | |
390 | struct perf_event *event) | |
391 | { | |
392 | struct hw_perf_event *hwc = &event->hw; | |
393 | int idx; | |
394 | ||
395 | /* | |
396 | * The counters which support limited events are usually at the end. | |
397 | * Schedule them first to accommodate more events. | |
398 | */ | |
399 | for (idx = iommu_pmu->num_cntr - 1; idx >= 0; idx--) { | |
400 | if (test_and_set_bit(idx, iommu_pmu->used_mask)) | |
401 | continue; | |
402 | /* Check per-counter event capabilities */ | |
403 | if (!iommu_pmu_validate_per_cntr_event(iommu_pmu, idx, event)) | |
404 | break; | |
405 | clear_bit(idx, iommu_pmu->used_mask); | |
406 | } | |
407 | if (idx < 0) | |
408 | return -EINVAL; | |
409 | ||
410 | iommu_pmu->event_list[idx] = event; | |
411 | hwc->idx = idx; | |
412 | ||
413 | /* config events */ | |
414 | dmar_writeq(iommu_config_base(iommu_pmu, idx), hwc->config); | |
415 | ||
416 | iommu_pmu_set_filter(requester_id, event->attr.config1, | |
417 | IOMMU_PMU_FILTER_REQUESTER_ID, idx, | |
418 | event->attr.config1); | |
419 | iommu_pmu_set_filter(domain, event->attr.config1, | |
420 | IOMMU_PMU_FILTER_DOMAIN, idx, | |
421 | event->attr.config1); | |
5b3625a4 | 422 | iommu_pmu_set_filter(pasid, event->attr.config2, |
7232ab8b KL |
423 | IOMMU_PMU_FILTER_PASID, idx, |
424 | event->attr.config1); | |
425 | iommu_pmu_set_filter(ats, event->attr.config2, | |
426 | IOMMU_PMU_FILTER_ATS, idx, | |
427 | event->attr.config1); | |
428 | iommu_pmu_set_filter(page_table, event->attr.config2, | |
429 | IOMMU_PMU_FILTER_PAGE_TABLE, idx, | |
430 | event->attr.config1); | |
431 | ||
432 | return 0; | |
433 | } | |
434 | ||
435 | static int iommu_pmu_add(struct perf_event *event, int flags) | |
436 | { | |
437 | struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event); | |
438 | struct hw_perf_event *hwc = &event->hw; | |
439 | int ret; | |
440 | ||
441 | ret = iommu_pmu_assign_event(iommu_pmu, event); | |
442 | if (ret < 0) | |
443 | return ret; | |
444 | ||
445 | hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; | |
446 | ||
447 | if (flags & PERF_EF_START) | |
448 | iommu_pmu_start(event, 0); | |
449 | ||
450 | return 0; | |
451 | } | |
452 | ||
453 | static void iommu_pmu_del(struct perf_event *event, int flags) | |
454 | { | |
455 | struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event); | |
456 | int idx = event->hw.idx; | |
457 | ||
458 | iommu_pmu_stop(event, PERF_EF_UPDATE); | |
459 | ||
460 | iommu_pmu_clear_filter(IOMMU_PMU_FILTER_REQUESTER_ID, idx); | |
461 | iommu_pmu_clear_filter(IOMMU_PMU_FILTER_DOMAIN, idx); | |
462 | iommu_pmu_clear_filter(IOMMU_PMU_FILTER_PASID, idx); | |
463 | iommu_pmu_clear_filter(IOMMU_PMU_FILTER_ATS, idx); | |
464 | iommu_pmu_clear_filter(IOMMU_PMU_FILTER_PAGE_TABLE, idx); | |
465 | ||
466 | iommu_pmu->event_list[idx] = NULL; | |
467 | event->hw.idx = -1; | |
468 | clear_bit(idx, iommu_pmu->used_mask); | |
469 | ||
470 | perf_event_update_userpage(event); | |
471 | } | |
472 | ||
473 | static void iommu_pmu_enable(struct pmu *pmu) | |
474 | { | |
475 | struct iommu_pmu *iommu_pmu = container_of(pmu, struct iommu_pmu, pmu); | |
476 | struct intel_iommu *iommu = iommu_pmu->iommu; | |
477 | ||
478 | ecmd_submit_sync(iommu, DMA_ECMD_UNFREEZE, 0, 0); | |
479 | } | |
480 | ||
481 | static void iommu_pmu_disable(struct pmu *pmu) | |
482 | { | |
483 | struct iommu_pmu *iommu_pmu = container_of(pmu, struct iommu_pmu, pmu); | |
484 | struct intel_iommu *iommu = iommu_pmu->iommu; | |
485 | ||
486 | ecmd_submit_sync(iommu, DMA_ECMD_FREEZE, 0, 0); | |
487 | } | |
488 | ||
4a0d4265 KL |
489 | static void iommu_pmu_counter_overflow(struct iommu_pmu *iommu_pmu) |
490 | { | |
491 | struct perf_event *event; | |
492 | u64 status; | |
493 | int i; | |
494 | ||
495 | /* | |
496 | * Two counters may be overflowed very close. Always check | |
497 | * whether there are more to handle. | |
498 | */ | |
499 | while ((status = dmar_readq(iommu_pmu->overflow))) { | |
500 | for_each_set_bit(i, (unsigned long *)&status, iommu_pmu->num_cntr) { | |
501 | /* | |
502 | * Find the assigned event of the counter. | |
503 | * Accumulate the value into the event->count. | |
504 | */ | |
505 | event = iommu_pmu->event_list[i]; | |
506 | if (!event) { | |
507 | pr_warn_once("Cannot find the assigned event for counter %d\n", i); | |
508 | continue; | |
509 | } | |
510 | iommu_pmu_event_update(event); | |
511 | } | |
512 | ||
513 | dmar_writeq(iommu_pmu->overflow, status); | |
514 | } | |
515 | } | |
516 | ||
517 | static irqreturn_t iommu_pmu_irq_handler(int irq, void *dev_id) | |
518 | { | |
519 | struct intel_iommu *iommu = dev_id; | |
520 | ||
521 | if (!dmar_readl(iommu->reg + DMAR_PERFINTRSTS_REG)) | |
522 | return IRQ_NONE; | |
523 | ||
524 | iommu_pmu_counter_overflow(iommu->pmu); | |
525 | ||
526 | /* Clear the status bit */ | |
527 | dmar_writel(iommu->reg + DMAR_PERFINTRSTS_REG, DMA_PERFINTRSTS_PIS); | |
528 | ||
529 | return IRQ_HANDLED; | |
530 | } | |
531 | ||
7232ab8b KL |
532 | static int __iommu_pmu_register(struct intel_iommu *iommu) |
533 | { | |
534 | struct iommu_pmu *iommu_pmu = iommu->pmu; | |
535 | ||
536 | iommu_pmu->pmu.name = iommu->name; | |
537 | iommu_pmu->pmu.task_ctx_nr = perf_invalid_context; | |
538 | iommu_pmu->pmu.event_init = iommu_pmu_event_init; | |
539 | iommu_pmu->pmu.pmu_enable = iommu_pmu_enable; | |
540 | iommu_pmu->pmu.pmu_disable = iommu_pmu_disable; | |
541 | iommu_pmu->pmu.add = iommu_pmu_add; | |
542 | iommu_pmu->pmu.del = iommu_pmu_del; | |
543 | iommu_pmu->pmu.start = iommu_pmu_start; | |
544 | iommu_pmu->pmu.stop = iommu_pmu_stop; | |
545 | iommu_pmu->pmu.read = iommu_pmu_event_update; | |
546 | iommu_pmu->pmu.attr_groups = iommu_pmu_attr_groups; | |
547 | iommu_pmu->pmu.attr_update = iommu_pmu_attr_update; | |
548 | iommu_pmu->pmu.capabilities = PERF_PMU_CAP_NO_EXCLUDE; | |
a8c73b82 | 549 | iommu_pmu->pmu.scope = PERF_PMU_SCOPE_SYS_WIDE; |
7232ab8b KL |
550 | iommu_pmu->pmu.module = THIS_MODULE; |
551 | ||
552 | return perf_pmu_register(&iommu_pmu->pmu, iommu_pmu->pmu.name, -1); | |
553 | } | |
554 | ||
a6a5006d KL |
555 | static inline void __iomem * |
556 | get_perf_reg_address(struct intel_iommu *iommu, u32 offset) | |
557 | { | |
558 | u32 off = dmar_readl(iommu->reg + offset); | |
559 | ||
560 | return iommu->reg + off; | |
561 | } | |
562 | ||
563 | int alloc_iommu_pmu(struct intel_iommu *iommu) | |
564 | { | |
565 | struct iommu_pmu *iommu_pmu; | |
566 | int i, j, ret; | |
567 | u64 perfcap; | |
568 | u32 cap; | |
569 | ||
570 | if (!ecap_pms(iommu->ecap)) | |
571 | return 0; | |
572 | ||
573 | /* The IOMMU PMU requires the ECMD support as well */ | |
574 | if (!cap_ecmds(iommu->cap)) | |
575 | return -ENODEV; | |
576 | ||
577 | perfcap = dmar_readq(iommu->reg + DMAR_PERFCAP_REG); | |
578 | /* The performance monitoring is not supported. */ | |
579 | if (!perfcap) | |
580 | return -ENODEV; | |
581 | ||
582 | /* Sanity check for the number of the counters and event groups */ | |
583 | if (!pcap_num_cntr(perfcap) || !pcap_num_event_group(perfcap)) | |
584 | return -ENODEV; | |
585 | ||
586 | /* The interrupt on overflow is required */ | |
587 | if (!pcap_interrupt(perfcap)) | |
588 | return -ENODEV; | |
589 | ||
7232ab8b KL |
590 | /* Check required Enhanced Command Capability */ |
591 | if (!ecmd_has_pmu_essential(iommu)) | |
592 | return -ENODEV; | |
593 | ||
a6a5006d KL |
594 | iommu_pmu = kzalloc(sizeof(*iommu_pmu), GFP_KERNEL); |
595 | if (!iommu_pmu) | |
596 | return -ENOMEM; | |
597 | ||
598 | iommu_pmu->num_cntr = pcap_num_cntr(perfcap); | |
7232ab8b KL |
599 | if (iommu_pmu->num_cntr > IOMMU_PMU_IDX_MAX) { |
600 | pr_warn_once("The number of IOMMU counters %d > max(%d), clipping!", | |
601 | iommu_pmu->num_cntr, IOMMU_PMU_IDX_MAX); | |
602 | iommu_pmu->num_cntr = IOMMU_PMU_IDX_MAX; | |
603 | } | |
604 | ||
a6a5006d KL |
605 | iommu_pmu->cntr_width = pcap_cntr_width(perfcap); |
606 | iommu_pmu->filter = pcap_filters_mask(perfcap); | |
607 | iommu_pmu->cntr_stride = pcap_cntr_stride(perfcap); | |
608 | iommu_pmu->num_eg = pcap_num_event_group(perfcap); | |
609 | ||
610 | iommu_pmu->evcap = kcalloc(iommu_pmu->num_eg, sizeof(u64), GFP_KERNEL); | |
611 | if (!iommu_pmu->evcap) { | |
612 | ret = -ENOMEM; | |
613 | goto free_pmu; | |
614 | } | |
615 | ||
616 | /* Parse event group capabilities */ | |
617 | for (i = 0; i < iommu_pmu->num_eg; i++) { | |
618 | u64 pcap; | |
619 | ||
620 | pcap = dmar_readq(iommu->reg + DMAR_PERFEVNTCAP_REG + | |
621 | i * IOMMU_PMU_CAP_REGS_STEP); | |
622 | iommu_pmu->evcap[i] = pecap_es(pcap); | |
623 | } | |
624 | ||
625 | iommu_pmu->cntr_evcap = kcalloc(iommu_pmu->num_cntr, sizeof(u32 *), GFP_KERNEL); | |
626 | if (!iommu_pmu->cntr_evcap) { | |
627 | ret = -ENOMEM; | |
628 | goto free_pmu_evcap; | |
629 | } | |
630 | for (i = 0; i < iommu_pmu->num_cntr; i++) { | |
631 | iommu_pmu->cntr_evcap[i] = kcalloc(iommu_pmu->num_eg, sizeof(u32), GFP_KERNEL); | |
632 | if (!iommu_pmu->cntr_evcap[i]) { | |
633 | ret = -ENOMEM; | |
634 | goto free_pmu_cntr_evcap; | |
635 | } | |
636 | /* | |
637 | * Set to the global capabilities, will adjust according | |
638 | * to per-counter capabilities later. | |
639 | */ | |
640 | for (j = 0; j < iommu_pmu->num_eg; j++) | |
641 | iommu_pmu->cntr_evcap[i][j] = (u32)iommu_pmu->evcap[j]; | |
642 | } | |
643 | ||
644 | iommu_pmu->cfg_reg = get_perf_reg_address(iommu, DMAR_PERFCFGOFF_REG); | |
645 | iommu_pmu->cntr_reg = get_perf_reg_address(iommu, DMAR_PERFCNTROFF_REG); | |
646 | iommu_pmu->overflow = get_perf_reg_address(iommu, DMAR_PERFOVFOFF_REG); | |
647 | ||
648 | /* | |
649 | * Check per-counter capabilities. All counters should have the | |
650 | * same capabilities on Interrupt on Overflow Support and Counter | |
651 | * Width. | |
652 | */ | |
653 | for (i = 0; i < iommu_pmu->num_cntr; i++) { | |
654 | cap = dmar_readl(iommu_pmu->cfg_reg + | |
655 | i * IOMMU_PMU_CFG_OFFSET + | |
656 | IOMMU_PMU_CFG_CNTRCAP_OFFSET); | |
657 | if (!iommu_cntrcap_pcc(cap)) | |
658 | continue; | |
659 | ||
660 | /* | |
661 | * It's possible that some counters have a different | |
662 | * capability because of e.g., HW bug. Check the corner | |
663 | * case here and simply drop those counters. | |
664 | */ | |
665 | if ((iommu_cntrcap_cw(cap) != iommu_pmu->cntr_width) || | |
666 | !iommu_cntrcap_ios(cap)) { | |
667 | iommu_pmu->num_cntr = i; | |
668 | pr_warn("PMU counter capability inconsistent, counter number reduced to %d\n", | |
669 | iommu_pmu->num_cntr); | |
670 | } | |
671 | ||
672 | /* Clear the pre-defined events group */ | |
673 | for (j = 0; j < iommu_pmu->num_eg; j++) | |
674 | iommu_pmu->cntr_evcap[i][j] = 0; | |
675 | ||
676 | /* Override with per-counter event capabilities */ | |
677 | for (j = 0; j < iommu_cntrcap_egcnt(cap); j++) { | |
678 | cap = dmar_readl(iommu_pmu->cfg_reg + i * IOMMU_PMU_CFG_OFFSET + | |
679 | IOMMU_PMU_CFG_CNTREVCAP_OFFSET + | |
680 | (j * IOMMU_PMU_OFF_REGS_STEP)); | |
681 | iommu_pmu->cntr_evcap[i][iommu_event_group(cap)] = iommu_event_select(cap); | |
682 | /* | |
683 | * Some events may only be supported by a specific counter. | |
684 | * Track them in the evcap as well. | |
685 | */ | |
686 | iommu_pmu->evcap[iommu_event_group(cap)] |= iommu_event_select(cap); | |
687 | } | |
688 | } | |
689 | ||
690 | iommu_pmu->iommu = iommu; | |
691 | iommu->pmu = iommu_pmu; | |
692 | ||
693 | return 0; | |
694 | ||
695 | free_pmu_cntr_evcap: | |
696 | for (i = 0; i < iommu_pmu->num_cntr; i++) | |
697 | kfree(iommu_pmu->cntr_evcap[i]); | |
698 | kfree(iommu_pmu->cntr_evcap); | |
699 | free_pmu_evcap: | |
700 | kfree(iommu_pmu->evcap); | |
701 | free_pmu: | |
702 | kfree(iommu_pmu); | |
703 | ||
704 | return ret; | |
705 | } | |
706 | ||
707 | void free_iommu_pmu(struct intel_iommu *iommu) | |
708 | { | |
709 | struct iommu_pmu *iommu_pmu = iommu->pmu; | |
710 | ||
711 | if (!iommu_pmu) | |
712 | return; | |
713 | ||
714 | if (iommu_pmu->evcap) { | |
715 | int i; | |
716 | ||
717 | for (i = 0; i < iommu_pmu->num_cntr; i++) | |
718 | kfree(iommu_pmu->cntr_evcap[i]); | |
719 | kfree(iommu_pmu->cntr_evcap); | |
720 | } | |
721 | kfree(iommu_pmu->evcap); | |
722 | kfree(iommu_pmu); | |
723 | iommu->pmu = NULL; | |
724 | } | |
7232ab8b | 725 | |
4a0d4265 KL |
726 | static int iommu_pmu_set_interrupt(struct intel_iommu *iommu) |
727 | { | |
728 | struct iommu_pmu *iommu_pmu = iommu->pmu; | |
729 | int irq, ret; | |
730 | ||
731 | irq = dmar_alloc_hwirq(IOMMU_IRQ_ID_OFFSET_PERF + iommu->seq_id, iommu->node, iommu); | |
732 | if (irq <= 0) | |
733 | return -EINVAL; | |
734 | ||
735 | snprintf(iommu_pmu->irq_name, sizeof(iommu_pmu->irq_name), "dmar%d-perf", iommu->seq_id); | |
736 | ||
737 | iommu->perf_irq = irq; | |
738 | ret = request_threaded_irq(irq, NULL, iommu_pmu_irq_handler, | |
739 | IRQF_ONESHOT, iommu_pmu->irq_name, iommu); | |
740 | if (ret) { | |
741 | dmar_free_hwirq(irq); | |
742 | iommu->perf_irq = 0; | |
743 | return ret; | |
744 | } | |
745 | return 0; | |
746 | } | |
747 | ||
748 | static void iommu_pmu_unset_interrupt(struct intel_iommu *iommu) | |
749 | { | |
750 | if (!iommu->perf_irq) | |
751 | return; | |
752 | ||
753 | free_irq(iommu->perf_irq, iommu); | |
754 | dmar_free_hwirq(iommu->perf_irq); | |
755 | iommu->perf_irq = 0; | |
756 | } | |
757 | ||
7232ab8b KL |
758 | void iommu_pmu_register(struct intel_iommu *iommu) |
759 | { | |
46284c6c KL |
760 | struct iommu_pmu *iommu_pmu = iommu->pmu; |
761 | ||
762 | if (!iommu_pmu) | |
7232ab8b KL |
763 | return; |
764 | ||
46284c6c KL |
765 | if (__iommu_pmu_register(iommu)) |
766 | goto err; | |
767 | ||
4a0d4265 KL |
768 | /* Set interrupt for overflow */ |
769 | if (iommu_pmu_set_interrupt(iommu)) | |
a8c73b82 | 770 | goto unregister; |
4a0d4265 | 771 | |
46284c6c KL |
772 | return; |
773 | ||
774 | unregister: | |
775 | perf_pmu_unregister(&iommu_pmu->pmu); | |
776 | err: | |
777 | pr_err("Failed to register PMU for iommu (seq_id = %d)\n", iommu->seq_id); | |
778 | free_iommu_pmu(iommu); | |
7232ab8b KL |
779 | } |
780 | ||
781 | void iommu_pmu_unregister(struct intel_iommu *iommu) | |
782 | { | |
46284c6c KL |
783 | struct iommu_pmu *iommu_pmu = iommu->pmu; |
784 | ||
785 | if (!iommu_pmu) | |
786 | return; | |
787 | ||
4a0d4265 | 788 | iommu_pmu_unset_interrupt(iommu); |
46284c6c | 789 | perf_pmu_unregister(&iommu_pmu->pmu); |
7232ab8b | 790 | } |