Commit | Line | Data |
---|---|---|
eec688e1 RB |
1 | /* |
2 | * Copyright © 2015-2016 Intel Corporation | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice (including the next | |
12 | * paragraph) shall be included in all copies or substantial portions of the | |
13 | * Software. | |
14 | * | |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS | |
21 | * IN THE SOFTWARE. | |
22 | * | |
23 | * Authors: | |
24 | * Robert Bragg <robert@sixbynine.org> | |
25 | */ | |
26 | ||
7abbd8d6 RB |
27 | |
28 | /** | |
16d98b31 | 29 | * DOC: i915 Perf Overview |
7abbd8d6 RB |
30 | * |
31 | * Gen graphics supports a large number of performance counters that can help | |
32 | * driver and application developers understand and optimize their use of the | |
33 | * GPU. | |
34 | * | |
35 | * This i915 perf interface enables userspace to configure and open a file | |
36 | * descriptor representing a stream of GPU metrics which can then be read() as | |
37 | * a stream of sample records. | |
38 | * | |
39 | * The interface is particularly suited to exposing buffered metrics that are | |
40 | * captured by DMA from the GPU, unsynchronized with and unrelated to the CPU. | |
41 | * | |
42 | * Streams representing a single context are accessible to applications with a | |
43 | * corresponding drm file descriptor, such that OpenGL can use the interface | |
44 | * without special privileges. Access to system-wide metrics requires root | |
45 | * privileges by default, unless changed via the dev.i915.perf_event_paranoid | |
46 | * sysctl option. | |
47 | * | |
16d98b31 RB |
48 | */ |
49 | ||
50 | /** | |
51 | * DOC: i915 Perf History and Comparison with Core Perf | |
7abbd8d6 RB |
52 | * |
53 | * The interface was initially inspired by the core Perf infrastructure but | |
54 | * some notable differences are: | |
55 | * | |
56 | * i915 perf file descriptors represent a "stream" instead of an "event"; where | |
57 | * a perf event primarily corresponds to a single 64bit value, while a stream | |
58 | * might sample sets of tightly-coupled counters, depending on the | |
59 | * configuration. For example the Gen OA unit isn't designed to support | |
60 | * orthogonal configurations of individual counters; it's configured for a set | |
61 | * of related counters. Samples for an i915 perf stream capturing OA metrics | |
62 | * will include a set of counter values packed in a compact HW specific format. | |
63 | * The OA unit supports a number of different packing formats which can be | |
64 | * selected by the user opening the stream. Perf has support for grouping | |
65 | * events, but each event in the group is configured, validated and | |
66 | * authenticated individually with separate system calls. | |
67 | * | |
68 | * i915 perf stream configurations are provided as an array of u64 (key,value) | |
69 | * pairs, instead of a fixed struct with multiple miscellaneous config members, | |
70 | * interleaved with event-type specific members. | |
71 | * | |
72 | * i915 perf doesn't support exposing metrics via an mmap'd circular buffer. | |
73 | * The supported metrics are being written to memory by the GPU unsynchronized | |
74 | * with the CPU, using HW specific packing formats for counter sets. Sometimes | |
75 | * the constraints on HW configuration require reports to be filtered before it | |
76 | * would be acceptable to expose them to unprivileged applications - to hide | |
77 | * the metrics of other processes/contexts. For these use cases a read() based | |
78 | * interface is a good fit, and provides an opportunity to filter data as it | |
79 | * gets copied from the GPU mapped buffers to userspace buffers. | |
80 | * | |
81 | * | |
16d98b31 RB |
82 | * Issues hit with first prototype based on Core Perf |
83 | * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ | |
7abbd8d6 RB |
84 | * |
85 | * The first prototype of this driver was based on the core perf | |
86 | * infrastructure, and while we did make that mostly work, with some changes to | |
87 | * perf, we found we were breaking or working around too many assumptions baked | |
88 | * into perf's currently cpu centric design. | |
89 | * | |
90 | * In the end we didn't see a clear benefit to making perf's implementation and | |
91 | * interface more complex by changing design assumptions while we knew we still | |
92 | * wouldn't be able to use any existing perf based userspace tools. | |
93 | * | |
94 | * Also considering the Gen specific nature of the Observability hardware and | |
95 | * how userspace will sometimes need to combine i915 perf OA metrics with | |
96 | * side-band OA data captured via MI_REPORT_PERF_COUNT commands; we're | |
97 | * expecting the interface to be used by a platform specific userspace such as | |
98 | * OpenGL or tools. This is to say; we aren't inherently missing out on having | |
99 | * a standard vendor/architecture agnostic interface by not using perf. | |
100 | * | |
101 | * | |
102 | * For posterity, in case we might re-visit trying to adapt core perf to be | |
103 | * better suited to exposing i915 metrics these were the main pain points we | |
104 | * hit: | |
105 | * | |
106 | * - The perf based OA PMU driver broke some significant design assumptions: | |
107 | * | |
108 | * Existing perf pmus are used for profiling work on a cpu and we were | |
109 | * introducing the idea of _IS_DEVICE pmus with different security | |
110 | * implications, the need to fake cpu-related data (such as user/kernel | |
111 | * registers) to fit with perf's current design, and adding _DEVICE records | |
112 | * as a way to forward device-specific status records. | |
113 | * | |
114 | * The OA unit writes reports of counters into a circular buffer, without | |
115 | * involvement from the CPU, making our PMU driver the first of a kind. | |
116 | * | |
117 | * Given the way we were periodically forward data from the GPU-mapped, OA | |
118 | * buffer to perf's buffer, those bursts of sample writes looked to perf like | |
119 | * we were sampling too fast and so we had to subvert its throttling checks. | |
120 | * | |
121 | * Perf supports groups of counters and allows those to be read via | |
122 | * transactions internally but transactions currently seem designed to be | |
123 | * explicitly initiated from the cpu (say in response to a userspace read()) | |
124 | * and while we could pull a report out of the OA buffer we can't | |
125 | * trigger a report from the cpu on demand. | |
126 | * | |
127 | * Related to being report based; the OA counters are configured in HW as a | |
128 | * set while perf generally expects counter configurations to be orthogonal. | |
129 | * Although counters can be associated with a group leader as they are | |
130 | * opened, there's no clear precedent for being able to provide group-wide | |
131 | * configuration attributes (for example we want to let userspace choose the | |
132 | * OA unit report format used to capture all counters in a set, or specify a | |
133 | * GPU context to filter metrics on). We avoided using perf's grouping | |
134 | * feature and forwarded OA reports to userspace via perf's 'raw' sample | |
135 | * field. This suited our userspace well considering how coupled the counters | |
136 | * are when dealing with normalizing. It would be inconvenient to split | |
137 | * counters up into separate events, only to require userspace to recombine | |
138 | * them. For Mesa it's also convenient to be forwarded raw, periodic reports | |
139 | * for combining with the side-band raw reports it captures using | |
140 | * MI_REPORT_PERF_COUNT commands. | |
141 | * | |
16d98b31 | 142 | * - As a side note on perf's grouping feature; there was also some concern |
7abbd8d6 RB |
143 | * that using PERF_FORMAT_GROUP as a way to pack together counter values |
144 | * would quite drastically inflate our sample sizes, which would likely | |
145 | * lower the effective sampling resolutions we could use when the available | |
146 | * memory bandwidth is limited. | |
147 | * | |
148 | * With the OA unit's report formats, counters are packed together as 32 | |
149 | * or 40bit values, with the largest report size being 256 bytes. | |
150 | * | |
151 | * PERF_FORMAT_GROUP values are 64bit, but there doesn't appear to be a | |
152 | * documented ordering to the values, implying PERF_FORMAT_ID must also be | |
153 | * used to add a 64bit ID before each value; giving 16 bytes per counter. | |
154 | * | |
155 | * Related to counter orthogonality; we can't time share the OA unit, while | |
156 | * event scheduling is a central design idea within perf for allowing | |
157 | * userspace to open + enable more events than can be configured in HW at any | |
158 | * one time. The OA unit is not designed to allow re-configuration while in | |
159 | * use. We can't reconfigure the OA unit without losing internal OA unit | |
160 | * state which we can't access explicitly to save and restore. Reconfiguring | |
161 | * the OA unit is also relatively slow, involving ~100 register writes. From | |
162 | * userspace Mesa also depends on a stable OA configuration when emitting | |
163 | * MI_REPORT_PERF_COUNT commands and importantly the OA unit can't be | |
164 | * disabled while there are outstanding MI_RPC commands lest we hang the | |
165 | * command streamer. | |
166 | * | |
167 | * The contents of sample records aren't extensible by device drivers (i.e. | |
168 | * the sample_type bits). As an example; Sourab Gupta had been looking to | |
169 | * attach GPU timestamps to our OA samples. We were shoehorning OA reports | |
170 | * into sample records by using the 'raw' field, but it's tricky to pack more | |
171 | * than one thing into this field because events/core.c currently only lets a | |
172 | * pmu give a single raw data pointer plus len which will be copied into the | |
173 | * ring buffer. To include more than the OA report we'd have to copy the | |
174 | * report into an intermediate larger buffer. I'd been considering allowing a | |
175 | * vector of data+len values to be specified for copying the raw data, but | |
176 | * it felt like a kludge to being using the raw field for this purpose. | |
177 | * | |
178 | * - It felt like our perf based PMU was making some technical compromises | |
179 | * just for the sake of using perf: | |
180 | * | |
181 | * perf_event_open() requires events to either relate to a pid or a specific | |
182 | * cpu core, while our device pmu related to neither. Events opened with a | |
183 | * pid will be automatically enabled/disabled according to the scheduling of | |
184 | * that process - so not appropriate for us. When an event is related to a | |
185 | * cpu id, perf ensures pmu methods will be invoked via an inter process | |
186 | * interrupt on that core. To avoid invasive changes our userspace opened OA | |
187 | * perf events for a specific cpu. This was workable but it meant the | |
188 | * majority of the OA driver ran in atomic context, including all OA report | |
189 | * forwarding, which wasn't really necessary in our case and seems to make | |
190 | * our locking requirements somewhat complex as we handled the interaction | |
191 | * with the rest of the i915 driver. | |
192 | */ | |
193 | ||
eec688e1 | 194 | #include <linux/anon_inodes.h> |
d7965152 | 195 | #include <linux/sizes.h> |
eec688e1 RB |
196 | |
197 | #include "i915_drv.h" | |
d7965152 RB |
198 | #include "i915_oa_hsw.h" |
199 | ||
200 | /* HW requires this to be a power of two, between 128k and 16M, though driver | |
201 | * is currently generally designed assuming the largest 16M size is used such | |
202 | * that the overflow cases are unlikely in normal operation. | |
203 | */ | |
204 | #define OA_BUFFER_SIZE SZ_16M | |
205 | ||
206 | #define OA_TAKEN(tail, head) ((tail - head) & (OA_BUFFER_SIZE - 1)) | |
207 | ||
208 | /* There's a HW race condition between OA unit tail pointer register updates and | |
209 | * writes to memory whereby the tail pointer can sometimes get ahead of what's | |
210 | * been written out to the OA buffer so far. | |
211 | * | |
212 | * Although this can be observed explicitly by checking for a zeroed report-id | |
213 | * field in tail reports, it seems preferable to account for this earlier e.g. | |
214 | * as part of the _oa_buffer_is_empty checks to minimize -EAGAIN polling cycles | |
215 | * in this situation. | |
216 | * | |
217 | * To give time for the most recent reports to land before they may be copied to | |
218 | * userspace, the driver operates as if the tail pointer effectively lags behind | |
219 | * the HW tail pointer by 'tail_margin' bytes. The margin in bytes is calculated | |
220 | * based on this constant in nanoseconds, the current OA sampling exponent | |
221 | * and current report size. | |
222 | * | |
223 | * There is also a fallback check while reading to simply skip over reports with | |
224 | * a zeroed report-id. | |
225 | */ | |
226 | #define OA_TAIL_MARGIN_NSEC 100000ULL | |
227 | ||
228 | /* frequency for checking whether the OA unit has written new reports to the | |
229 | * circular OA buffer... | |
230 | */ | |
231 | #define POLL_FREQUENCY 200 | |
232 | #define POLL_PERIOD (NSEC_PER_SEC / POLL_FREQUENCY) | |
233 | ||
ccdf6341 RB |
234 | /* for sysctl proc_dointvec_minmax of dev.i915.perf_stream_paranoid */ |
235 | static int zero; | |
236 | static int one = 1; | |
237 | static u32 i915_perf_stream_paranoid = true; | |
238 | ||
d7965152 RB |
239 | /* The maximum exponent the hardware accepts is 63 (essentially it selects one |
240 | * of the 64bit timestamp bits to trigger reports from) but there's currently | |
241 | * no known use case for sampling as infrequently as once per 47 thousand years. | |
242 | * | |
243 | * Since the timestamps included in OA reports are only 32bits it seems | |
244 | * reasonable to limit the OA exponent where it's still possible to account for | |
245 | * overflow in OA report timestamps. | |
246 | */ | |
247 | #define OA_EXPONENT_MAX 31 | |
248 | ||
249 | #define INVALID_CTX_ID 0xffffffff | |
250 | ||
251 | ||
00319ba0 RB |
252 | /* For sysctl proc_dointvec_minmax of i915_oa_max_sample_rate |
253 | * | |
254 | * 160ns is the smallest sampling period we can theoretically program the OA | |
255 | * unit with on Haswell, corresponding to 6.25MHz. | |
256 | */ | |
257 | static int oa_sample_rate_hard_limit = 6250000; | |
258 | ||
259 | /* Theoretically we can program the OA unit to sample every 160ns but don't | |
260 | * allow that by default unless root... | |
261 | * | |
262 | * The default threshold of 100000Hz is based on perf's similar | |
263 | * kernel.perf_event_max_sample_rate sysctl parameter. | |
264 | */ | |
265 | static u32 i915_oa_max_sample_rate = 100000; | |
266 | ||
d7965152 RB |
267 | /* XXX: beware if future OA HW adds new report formats that the current |
268 | * code assumes all reports have a power-of-two size and ~(size - 1) can | |
269 | * be used as a mask to align the OA tail pointer. | |
270 | */ | |
271 | static struct i915_oa_format hsw_oa_formats[I915_OA_FORMAT_MAX] = { | |
272 | [I915_OA_FORMAT_A13] = { 0, 64 }, | |
273 | [I915_OA_FORMAT_A29] = { 1, 128 }, | |
274 | [I915_OA_FORMAT_A13_B8_C8] = { 2, 128 }, | |
275 | /* A29_B8_C8 Disallowed as 192 bytes doesn't factor into buffer size */ | |
276 | [I915_OA_FORMAT_B4_C8] = { 4, 64 }, | |
277 | [I915_OA_FORMAT_A45_B8_C8] = { 5, 256 }, | |
278 | [I915_OA_FORMAT_B4_C8_A16] = { 6, 128 }, | |
279 | [I915_OA_FORMAT_C4_B8] = { 7, 64 }, | |
280 | }; | |
281 | ||
282 | #define SAMPLE_OA_REPORT (1<<0) | |
eec688e1 | 283 | |
16d98b31 RB |
284 | /** |
285 | * struct perf_open_properties - for validated properties given to open a stream | |
286 | * @sample_flags: `DRM_I915_PERF_PROP_SAMPLE_*` properties are tracked as flags | |
287 | * @single_context: Whether a single or all gpu contexts should be monitored | |
288 | * @ctx_handle: A gem ctx handle for use with @single_context | |
289 | * @metrics_set: An ID for an OA unit metric set advertised via sysfs | |
290 | * @oa_format: An OA unit HW report format | |
291 | * @oa_periodic: Whether to enable periodic OA unit sampling | |
292 | * @oa_period_exponent: The OA unit sampling period is derived from this | |
293 | * | |
294 | * As read_properties_unlocked() enumerates and validates the properties given | |
295 | * to open a stream of metrics the configuration is built up in the structure | |
296 | * which starts out zero initialized. | |
297 | */ | |
eec688e1 RB |
298 | struct perf_open_properties { |
299 | u32 sample_flags; | |
300 | ||
301 | u64 single_context:1; | |
302 | u64 ctx_handle; | |
d7965152 RB |
303 | |
304 | /* OA sampling state */ | |
305 | int metrics_set; | |
306 | int oa_format; | |
307 | bool oa_periodic; | |
308 | int oa_period_exponent; | |
309 | }; | |
310 | ||
311 | /* NB: This is either called via fops or the poll check hrtimer (atomic ctx) | |
312 | * | |
313 | * It's safe to read OA config state here unlocked, assuming that this is only | |
314 | * called while the stream is enabled, while the global OA configuration can't | |
315 | * be modified. | |
316 | * | |
317 | * Note: we don't lock around the head/tail reads even though there's the slim | |
318 | * possibility of read() fop errors forcing a re-init of the OA buffer | |
319 | * pointers. A race here could result in a false positive !empty status which | |
320 | * is acceptable. | |
321 | */ | |
322 | static bool gen7_oa_buffer_is_empty_fop_unlocked(struct drm_i915_private *dev_priv) | |
323 | { | |
324 | int report_size = dev_priv->perf.oa.oa_buffer.format_size; | |
325 | u32 oastatus2 = I915_READ(GEN7_OASTATUS2); | |
326 | u32 oastatus1 = I915_READ(GEN7_OASTATUS1); | |
327 | u32 head = oastatus2 & GEN7_OASTATUS2_HEAD_MASK; | |
328 | u32 tail = oastatus1 & GEN7_OASTATUS1_TAIL_MASK; | |
329 | ||
330 | return OA_TAKEN(tail, head) < | |
331 | dev_priv->perf.oa.tail_margin + report_size; | |
332 | } | |
333 | ||
334 | /** | |
16d98b31 RB |
335 | * append_oa_status - Appends a status record to a userspace read() buffer. |
336 | * @stream: An i915-perf stream opened for OA metrics | |
337 | * @buf: destination buffer given by userspace | |
338 | * @count: the number of bytes userspace wants to read | |
339 | * @offset: (inout): the current position for writing into @buf | |
340 | * @type: The kind of status to report to userspace | |
341 | * | |
342 | * Writes a status record (such as `DRM_I915_PERF_RECORD_OA_REPORT_LOST`) | |
343 | * into the userspace read() buffer. | |
344 | * | |
345 | * The @buf @offset will only be updated on success. | |
346 | * | |
347 | * Returns: 0 on success, negative error code on failure. | |
d7965152 RB |
348 | */ |
349 | static int append_oa_status(struct i915_perf_stream *stream, | |
350 | char __user *buf, | |
351 | size_t count, | |
352 | size_t *offset, | |
353 | enum drm_i915_perf_record_type type) | |
354 | { | |
355 | struct drm_i915_perf_record_header header = { type, 0, sizeof(header) }; | |
356 | ||
357 | if ((count - *offset) < header.size) | |
358 | return -ENOSPC; | |
359 | ||
360 | if (copy_to_user(buf + *offset, &header, sizeof(header))) | |
361 | return -EFAULT; | |
362 | ||
363 | (*offset) += header.size; | |
364 | ||
365 | return 0; | |
366 | } | |
367 | ||
368 | /** | |
16d98b31 RB |
369 | * append_oa_sample - Copies single OA report into userspace read() buffer. |
370 | * @stream: An i915-perf stream opened for OA metrics | |
371 | * @buf: destination buffer given by userspace | |
372 | * @count: the number of bytes userspace wants to read | |
373 | * @offset: (inout): the current position for writing into @buf | |
374 | * @report: A single OA report to (optionally) include as part of the sample | |
375 | * | |
376 | * The contents of a sample are configured through `DRM_I915_PERF_PROP_SAMPLE_*` | |
377 | * properties when opening a stream, tracked as `stream->sample_flags`. This | |
378 | * function copies the requested components of a single sample to the given | |
379 | * read() @buf. | |
380 | * | |
381 | * The @buf @offset will only be updated on success. | |
382 | * | |
383 | * Returns: 0 on success, negative error code on failure. | |
d7965152 RB |
384 | */ |
385 | static int append_oa_sample(struct i915_perf_stream *stream, | |
386 | char __user *buf, | |
387 | size_t count, | |
388 | size_t *offset, | |
389 | const u8 *report) | |
390 | { | |
391 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
392 | int report_size = dev_priv->perf.oa.oa_buffer.format_size; | |
393 | struct drm_i915_perf_record_header header; | |
394 | u32 sample_flags = stream->sample_flags; | |
395 | ||
396 | header.type = DRM_I915_PERF_RECORD_SAMPLE; | |
397 | header.pad = 0; | |
398 | header.size = stream->sample_size; | |
399 | ||
400 | if ((count - *offset) < header.size) | |
401 | return -ENOSPC; | |
402 | ||
403 | buf += *offset; | |
404 | if (copy_to_user(buf, &header, sizeof(header))) | |
405 | return -EFAULT; | |
406 | buf += sizeof(header); | |
407 | ||
408 | if (sample_flags & SAMPLE_OA_REPORT) { | |
409 | if (copy_to_user(buf, report, report_size)) | |
410 | return -EFAULT; | |
411 | } | |
412 | ||
413 | (*offset) += header.size; | |
414 | ||
415 | return 0; | |
416 | } | |
417 | ||
418 | /** | |
419 | * Copies all buffered OA reports into userspace read() buffer. | |
420 | * @stream: An i915-perf stream opened for OA metrics | |
421 | * @buf: destination buffer given by userspace | |
422 | * @count: the number of bytes userspace wants to read | |
423 | * @offset: (inout): the current position for writing into @buf | |
424 | * @head_ptr: (inout): the current oa buffer cpu read position | |
425 | * @tail: the current oa buffer gpu write position | |
426 | * | |
16d98b31 RB |
427 | * Notably any error condition resulting in a short read (-%ENOSPC or |
428 | * -%EFAULT) will be returned even though one or more records may | |
d7965152 RB |
429 | * have been successfully copied. In this case it's up to the caller |
430 | * to decide if the error should be squashed before returning to | |
431 | * userspace. | |
432 | * | |
433 | * Note: reports are consumed from the head, and appended to the | |
434 | * tail, so the head chases the tail?... If you think that's mad | |
435 | * and back-to-front you're not alone, but this follows the | |
436 | * Gen PRM naming convention. | |
16d98b31 RB |
437 | * |
438 | * Returns: 0 on success, negative error code on failure. | |
d7965152 RB |
439 | */ |
440 | static int gen7_append_oa_reports(struct i915_perf_stream *stream, | |
441 | char __user *buf, | |
442 | size_t count, | |
443 | size_t *offset, | |
444 | u32 *head_ptr, | |
445 | u32 tail) | |
446 | { | |
447 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
448 | int report_size = dev_priv->perf.oa.oa_buffer.format_size; | |
449 | u8 *oa_buf_base = dev_priv->perf.oa.oa_buffer.vaddr; | |
450 | int tail_margin = dev_priv->perf.oa.tail_margin; | |
451 | u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma); | |
452 | u32 mask = (OA_BUFFER_SIZE - 1); | |
453 | u32 head; | |
454 | u32 taken; | |
455 | int ret = 0; | |
456 | ||
457 | if (WARN_ON(!stream->enabled)) | |
458 | return -EIO; | |
459 | ||
460 | head = *head_ptr - gtt_offset; | |
461 | tail -= gtt_offset; | |
462 | ||
463 | /* The OA unit is expected to wrap the tail pointer according to the OA | |
464 | * buffer size and since we should never write a misaligned head | |
465 | * pointer we don't expect to read one back either... | |
466 | */ | |
467 | if (tail > OA_BUFFER_SIZE || head > OA_BUFFER_SIZE || | |
468 | head % report_size) { | |
469 | DRM_ERROR("Inconsistent OA buffer pointer (head = %u, tail = %u): force restart\n", | |
470 | head, tail); | |
471 | dev_priv->perf.oa.ops.oa_disable(dev_priv); | |
472 | dev_priv->perf.oa.ops.oa_enable(dev_priv); | |
473 | *head_ptr = I915_READ(GEN7_OASTATUS2) & | |
474 | GEN7_OASTATUS2_HEAD_MASK; | |
475 | return -EIO; | |
476 | } | |
477 | ||
478 | ||
479 | /* The tail pointer increases in 64 byte increments, not in report_size | |
480 | * steps... | |
481 | */ | |
482 | tail &= ~(report_size - 1); | |
483 | ||
484 | /* Move the tail pointer back by the current tail_margin to account for | |
485 | * the possibility that the latest reports may not have really landed | |
486 | * in memory yet... | |
487 | */ | |
488 | ||
489 | if (OA_TAKEN(tail, head) < report_size + tail_margin) | |
490 | return -EAGAIN; | |
491 | ||
492 | tail -= tail_margin; | |
493 | tail &= mask; | |
494 | ||
495 | for (/* none */; | |
496 | (taken = OA_TAKEN(tail, head)); | |
497 | head = (head + report_size) & mask) { | |
498 | u8 *report = oa_buf_base + head; | |
499 | u32 *report32 = (void *)report; | |
500 | ||
501 | /* All the report sizes factor neatly into the buffer | |
502 | * size so we never expect to see a report split | |
503 | * between the beginning and end of the buffer. | |
504 | * | |
505 | * Given the initial alignment check a misalignment | |
506 | * here would imply a driver bug that would result | |
507 | * in an overrun. | |
508 | */ | |
509 | if (WARN_ON((OA_BUFFER_SIZE - head) < report_size)) { | |
510 | DRM_ERROR("Spurious OA head ptr: non-integral report offset\n"); | |
511 | break; | |
512 | } | |
513 | ||
514 | /* The report-ID field for periodic samples includes | |
515 | * some undocumented flags related to what triggered | |
516 | * the report and is never expected to be zero so we | |
517 | * can check that the report isn't invalid before | |
518 | * copying it to userspace... | |
519 | */ | |
520 | if (report32[0] == 0) { | |
7708550c | 521 | DRM_NOTE("Skipping spurious, invalid OA report\n"); |
d7965152 RB |
522 | continue; |
523 | } | |
524 | ||
525 | ret = append_oa_sample(stream, buf, count, offset, report); | |
526 | if (ret) | |
527 | break; | |
528 | ||
529 | /* The above report-id field sanity check is based on | |
530 | * the assumption that the OA buffer is initially | |
531 | * zeroed and we reset the field after copying so the | |
532 | * check is still meaningful once old reports start | |
533 | * being overwritten. | |
534 | */ | |
535 | report32[0] = 0; | |
536 | } | |
537 | ||
538 | *head_ptr = gtt_offset + head; | |
539 | ||
540 | return ret; | |
541 | } | |
542 | ||
16d98b31 RB |
543 | /** |
544 | * gen7_oa_read - copy status records then buffered OA reports | |
545 | * @stream: An i915-perf stream opened for OA metrics | |
546 | * @buf: destination buffer given by userspace | |
547 | * @count: the number of bytes userspace wants to read | |
548 | * @offset: (inout): the current position for writing into @buf | |
549 | * | |
550 | * Checks Gen 7 specific OA unit status registers and if necessary appends | |
551 | * corresponding status records for userspace (such as for a buffer full | |
552 | * condition) and then initiate appending any buffered OA reports. | |
553 | * | |
554 | * Updates @offset according to the number of bytes successfully copied into | |
555 | * the userspace buffer. | |
556 | * | |
557 | * Returns: zero on success or a negative error code | |
558 | */ | |
d7965152 RB |
559 | static int gen7_oa_read(struct i915_perf_stream *stream, |
560 | char __user *buf, | |
561 | size_t count, | |
562 | size_t *offset) | |
563 | { | |
564 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
565 | int report_size = dev_priv->perf.oa.oa_buffer.format_size; | |
566 | u32 oastatus2; | |
567 | u32 oastatus1; | |
568 | u32 head; | |
569 | u32 tail; | |
570 | int ret; | |
571 | ||
572 | if (WARN_ON(!dev_priv->perf.oa.oa_buffer.vaddr)) | |
573 | return -EIO; | |
574 | ||
575 | oastatus2 = I915_READ(GEN7_OASTATUS2); | |
576 | oastatus1 = I915_READ(GEN7_OASTATUS1); | |
577 | ||
578 | head = oastatus2 & GEN7_OASTATUS2_HEAD_MASK; | |
579 | tail = oastatus1 & GEN7_OASTATUS1_TAIL_MASK; | |
580 | ||
581 | /* XXX: On Haswell we don't have a safe way to clear oastatus1 | |
582 | * bits while the OA unit is enabled (while the tail pointer | |
583 | * may be updated asynchronously) so we ignore status bits | |
584 | * that have already been reported to userspace. | |
585 | */ | |
586 | oastatus1 &= ~dev_priv->perf.oa.gen7_latched_oastatus1; | |
587 | ||
588 | /* We treat OABUFFER_OVERFLOW as a significant error: | |
589 | * | |
590 | * - The status can be interpreted to mean that the buffer is | |
591 | * currently full (with a higher precedence than OA_TAKEN() | |
592 | * which will start to report a near-empty buffer after an | |
593 | * overflow) but it's awkward that we can't clear the status | |
594 | * on Haswell, so without a reset we won't be able to catch | |
595 | * the state again. | |
596 | * | |
597 | * - Since it also implies the HW has started overwriting old | |
598 | * reports it may also affect our sanity checks for invalid | |
599 | * reports when copying to userspace that assume new reports | |
600 | * are being written to cleared memory. | |
601 | * | |
602 | * - In the future we may want to introduce a flight recorder | |
603 | * mode where the driver will automatically maintain a safe | |
604 | * guard band between head/tail, avoiding this overflow | |
605 | * condition, but we avoid the added driver complexity for | |
606 | * now. | |
607 | */ | |
608 | if (unlikely(oastatus1 & GEN7_OASTATUS1_OABUFFER_OVERFLOW)) { | |
609 | ret = append_oa_status(stream, buf, count, offset, | |
610 | DRM_I915_PERF_RECORD_OA_BUFFER_LOST); | |
611 | if (ret) | |
612 | return ret; | |
613 | ||
7708550c | 614 | DRM_DEBUG("OA buffer overflow: force restart\n"); |
d7965152 RB |
615 | |
616 | dev_priv->perf.oa.ops.oa_disable(dev_priv); | |
617 | dev_priv->perf.oa.ops.oa_enable(dev_priv); | |
618 | ||
619 | oastatus2 = I915_READ(GEN7_OASTATUS2); | |
620 | oastatus1 = I915_READ(GEN7_OASTATUS1); | |
621 | ||
622 | head = oastatus2 & GEN7_OASTATUS2_HEAD_MASK; | |
623 | tail = oastatus1 & GEN7_OASTATUS1_TAIL_MASK; | |
624 | } | |
625 | ||
626 | if (unlikely(oastatus1 & GEN7_OASTATUS1_REPORT_LOST)) { | |
627 | ret = append_oa_status(stream, buf, count, offset, | |
628 | DRM_I915_PERF_RECORD_OA_REPORT_LOST); | |
629 | if (ret) | |
630 | return ret; | |
631 | dev_priv->perf.oa.gen7_latched_oastatus1 |= | |
632 | GEN7_OASTATUS1_REPORT_LOST; | |
633 | } | |
634 | ||
635 | ret = gen7_append_oa_reports(stream, buf, count, offset, | |
636 | &head, tail); | |
637 | ||
638 | /* All the report sizes are a power of two and the | |
639 | * head should always be incremented by some multiple | |
640 | * of the report size. | |
641 | * | |
642 | * A warning here, but notably if we later read back a | |
643 | * misaligned pointer we will treat that as a bug since | |
644 | * it could lead to a buffer overrun. | |
645 | */ | |
646 | WARN_ONCE(head & (report_size - 1), | |
647 | "i915: Writing misaligned OA head pointer"); | |
648 | ||
649 | /* Note: we update the head pointer here even if an error | |
650 | * was returned since the error may represent a short read | |
651 | * where some some reports were successfully copied. | |
652 | */ | |
653 | I915_WRITE(GEN7_OASTATUS2, | |
654 | ((head & GEN7_OASTATUS2_HEAD_MASK) | | |
655 | OA_MEM_SELECT_GGTT)); | |
656 | ||
657 | return ret; | |
658 | } | |
659 | ||
16d98b31 RB |
660 | /** |
661 | * i915_oa_wait_unlocked - handles blocking IO until OA data available | |
662 | * @stream: An i915-perf stream opened for OA metrics | |
663 | * | |
664 | * Called when userspace tries to read() from a blocking stream FD opened | |
665 | * for OA metrics. It waits until the hrtimer callback finds a non-empty | |
666 | * OA buffer and wakes us. | |
667 | * | |
668 | * Note: it's acceptable to have this return with some false positives | |
669 | * since any subsequent read handling will return -EAGAIN if there isn't | |
670 | * really data ready for userspace yet. | |
671 | * | |
672 | * Returns: zero on success or a negative error code | |
673 | */ | |
d7965152 RB |
674 | static int i915_oa_wait_unlocked(struct i915_perf_stream *stream) |
675 | { | |
676 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
677 | ||
678 | /* We would wait indefinitely if periodic sampling is not enabled */ | |
679 | if (!dev_priv->perf.oa.periodic) | |
680 | return -EIO; | |
681 | ||
682 | /* Note: the oa_buffer_is_empty() condition is ok to run unlocked as it | |
683 | * just performs mmio reads of the OA buffer head + tail pointers and | |
684 | * it's assumed we're handling some operation that implies the stream | |
685 | * can't be destroyed until completion (such as a read()) that ensures | |
686 | * the device + OA buffer can't disappear | |
687 | */ | |
688 | return wait_event_interruptible(dev_priv->perf.oa.poll_wq, | |
689 | !dev_priv->perf.oa.ops.oa_buffer_is_empty(dev_priv)); | |
690 | } | |
691 | ||
16d98b31 RB |
692 | /** |
693 | * i915_oa_poll_wait - call poll_wait() for an OA stream poll() | |
694 | * @stream: An i915-perf stream opened for OA metrics | |
695 | * @file: An i915 perf stream file | |
696 | * @wait: poll() state table | |
697 | * | |
698 | * For handling userspace polling on an i915 perf stream opened for OA metrics, | |
699 | * this starts a poll_wait with the wait queue that our hrtimer callback wakes | |
700 | * when it sees data ready to read in the circular OA buffer. | |
701 | */ | |
d7965152 RB |
702 | static void i915_oa_poll_wait(struct i915_perf_stream *stream, |
703 | struct file *file, | |
704 | poll_table *wait) | |
705 | { | |
706 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
707 | ||
708 | poll_wait(file, &dev_priv->perf.oa.poll_wq, wait); | |
709 | } | |
710 | ||
16d98b31 RB |
711 | /** |
712 | * i915_oa_read - just calls through to &i915_oa_ops->read | |
713 | * @stream: An i915-perf stream opened for OA metrics | |
714 | * @buf: destination buffer given by userspace | |
715 | * @count: the number of bytes userspace wants to read | |
716 | * @offset: (inout): the current position for writing into @buf | |
717 | * | |
718 | * Updates @offset according to the number of bytes successfully copied into | |
719 | * the userspace buffer. | |
720 | * | |
721 | * Returns: zero on success or a negative error code | |
722 | */ | |
d7965152 RB |
723 | static int i915_oa_read(struct i915_perf_stream *stream, |
724 | char __user *buf, | |
725 | size_t count, | |
726 | size_t *offset) | |
727 | { | |
728 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
729 | ||
730 | return dev_priv->perf.oa.ops.read(stream, buf, count, offset); | |
731 | } | |
732 | ||
16d98b31 RB |
733 | /** |
734 | * oa_get_render_ctx_id - determine and hold ctx hw id | |
735 | * @stream: An i915-perf stream opened for OA metrics | |
736 | * | |
737 | * Determine the render context hw id, and ensure it remains fixed for the | |
d7965152 RB |
738 | * lifetime of the stream. This ensures that we don't have to worry about |
739 | * updating the context ID in OACONTROL on the fly. | |
16d98b31 RB |
740 | * |
741 | * Returns: zero on success or a negative error code | |
d7965152 RB |
742 | */ |
743 | static int oa_get_render_ctx_id(struct i915_perf_stream *stream) | |
744 | { | |
745 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
e8a9c58f | 746 | struct intel_engine_cs *engine = dev_priv->engine[RCS]; |
266a240b | 747 | struct intel_ring *ring; |
d7965152 RB |
748 | int ret; |
749 | ||
750 | ret = i915_mutex_lock_interruptible(&dev_priv->drm); | |
751 | if (ret) | |
752 | return ret; | |
753 | ||
754 | /* As the ID is the gtt offset of the context's vma we pin | |
755 | * the vma to ensure the ID remains fixed. | |
756 | * | |
757 | * NB: implied RCS engine... | |
758 | */ | |
266a240b CW |
759 | ring = engine->context_pin(engine, stream->ctx); |
760 | mutex_unlock(&dev_priv->drm.struct_mutex); | |
761 | if (IS_ERR(ring)) | |
762 | return PTR_ERR(ring); | |
d7965152 RB |
763 | |
764 | /* Explicitly track the ID (instead of calling i915_ggtt_offset() | |
765 | * on the fly) considering the difference with gen8+ and | |
766 | * execlists | |
767 | */ | |
e8a9c58f CW |
768 | dev_priv->perf.oa.specific_ctx_id = |
769 | i915_ggtt_offset(stream->ctx->engine[engine->id].state); | |
d7965152 | 770 | |
266a240b | 771 | return 0; |
d7965152 RB |
772 | } |
773 | ||
16d98b31 RB |
774 | /** |
775 | * oa_put_render_ctx_id - counterpart to oa_get_render_ctx_id releases hold | |
776 | * @stream: An i915-perf stream opened for OA metrics | |
777 | * | |
778 | * In case anything needed doing to ensure the context HW ID would remain valid | |
779 | * for the lifetime of the stream, then that can be undone here. | |
780 | */ | |
d7965152 RB |
781 | static void oa_put_render_ctx_id(struct i915_perf_stream *stream) |
782 | { | |
783 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
e8a9c58f | 784 | struct intel_engine_cs *engine = dev_priv->engine[RCS]; |
d7965152 RB |
785 | |
786 | mutex_lock(&dev_priv->drm.struct_mutex); | |
787 | ||
d7965152 | 788 | dev_priv->perf.oa.specific_ctx_id = INVALID_CTX_ID; |
e8a9c58f | 789 | engine->context_unpin(engine, stream->ctx); |
d7965152 RB |
790 | |
791 | mutex_unlock(&dev_priv->drm.struct_mutex); | |
792 | } | |
793 | ||
794 | static void | |
795 | free_oa_buffer(struct drm_i915_private *i915) | |
796 | { | |
797 | mutex_lock(&i915->drm.struct_mutex); | |
798 | ||
799 | i915_gem_object_unpin_map(i915->perf.oa.oa_buffer.vma->obj); | |
800 | i915_vma_unpin(i915->perf.oa.oa_buffer.vma); | |
801 | i915_gem_object_put(i915->perf.oa.oa_buffer.vma->obj); | |
802 | ||
803 | i915->perf.oa.oa_buffer.vma = NULL; | |
804 | i915->perf.oa.oa_buffer.vaddr = NULL; | |
805 | ||
806 | mutex_unlock(&i915->drm.struct_mutex); | |
807 | } | |
808 | ||
809 | static void i915_oa_stream_destroy(struct i915_perf_stream *stream) | |
810 | { | |
811 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
812 | ||
813 | BUG_ON(stream != dev_priv->perf.oa.exclusive_stream); | |
814 | ||
815 | dev_priv->perf.oa.ops.disable_metric_set(dev_priv); | |
816 | ||
817 | free_oa_buffer(dev_priv); | |
818 | ||
819 | intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); | |
820 | intel_runtime_pm_put(dev_priv); | |
821 | ||
822 | if (stream->ctx) | |
823 | oa_put_render_ctx_id(stream); | |
824 | ||
825 | dev_priv->perf.oa.exclusive_stream = NULL; | |
826 | } | |
827 | ||
828 | static void gen7_init_oa_buffer(struct drm_i915_private *dev_priv) | |
829 | { | |
830 | u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma); | |
831 | ||
832 | /* Pre-DevBDW: OABUFFER must be set with counters off, | |
833 | * before OASTATUS1, but after OASTATUS2 | |
834 | */ | |
835 | I915_WRITE(GEN7_OASTATUS2, gtt_offset | OA_MEM_SELECT_GGTT); /* head */ | |
836 | I915_WRITE(GEN7_OABUFFER, gtt_offset); | |
837 | I915_WRITE(GEN7_OASTATUS1, gtt_offset | OABUFFER_SIZE_16M); /* tail */ | |
838 | ||
839 | /* On Haswell we have to track which OASTATUS1 flags we've | |
840 | * already seen since they can't be cleared while periodic | |
841 | * sampling is enabled. | |
842 | */ | |
843 | dev_priv->perf.oa.gen7_latched_oastatus1 = 0; | |
844 | ||
845 | /* NB: although the OA buffer will initially be allocated | |
846 | * zeroed via shmfs (and so this memset is redundant when | |
847 | * first allocating), we may re-init the OA buffer, either | |
848 | * when re-enabling a stream or in error/reset paths. | |
849 | * | |
850 | * The reason we clear the buffer for each re-init is for the | |
851 | * sanity check in gen7_append_oa_reports() that looks at the | |
852 | * report-id field to make sure it's non-zero which relies on | |
853 | * the assumption that new reports are being written to zeroed | |
854 | * memory... | |
855 | */ | |
856 | memset(dev_priv->perf.oa.oa_buffer.vaddr, 0, OA_BUFFER_SIZE); | |
857 | ||
858 | /* Maybe make ->pollin per-stream state if we support multiple | |
859 | * concurrent streams in the future. | |
860 | */ | |
861 | dev_priv->perf.oa.pollin = false; | |
862 | } | |
863 | ||
864 | static int alloc_oa_buffer(struct drm_i915_private *dev_priv) | |
865 | { | |
866 | struct drm_i915_gem_object *bo; | |
867 | struct i915_vma *vma; | |
868 | int ret; | |
869 | ||
870 | if (WARN_ON(dev_priv->perf.oa.oa_buffer.vma)) | |
871 | return -ENODEV; | |
872 | ||
873 | ret = i915_mutex_lock_interruptible(&dev_priv->drm); | |
874 | if (ret) | |
875 | return ret; | |
876 | ||
877 | BUILD_BUG_ON_NOT_POWER_OF_2(OA_BUFFER_SIZE); | |
878 | BUILD_BUG_ON(OA_BUFFER_SIZE < SZ_128K || OA_BUFFER_SIZE > SZ_16M); | |
879 | ||
12d79d78 | 880 | bo = i915_gem_object_create(dev_priv, OA_BUFFER_SIZE); |
d7965152 RB |
881 | if (IS_ERR(bo)) { |
882 | DRM_ERROR("Failed to allocate OA buffer\n"); | |
883 | ret = PTR_ERR(bo); | |
884 | goto unlock; | |
885 | } | |
886 | ||
887 | ret = i915_gem_object_set_cache_level(bo, I915_CACHE_LLC); | |
888 | if (ret) | |
889 | goto err_unref; | |
890 | ||
891 | /* PreHSW required 512K alignment, HSW requires 16M */ | |
892 | vma = i915_gem_object_ggtt_pin(bo, NULL, 0, SZ_16M, 0); | |
893 | if (IS_ERR(vma)) { | |
894 | ret = PTR_ERR(vma); | |
895 | goto err_unref; | |
896 | } | |
897 | dev_priv->perf.oa.oa_buffer.vma = vma; | |
898 | ||
899 | dev_priv->perf.oa.oa_buffer.vaddr = | |
900 | i915_gem_object_pin_map(bo, I915_MAP_WB); | |
901 | if (IS_ERR(dev_priv->perf.oa.oa_buffer.vaddr)) { | |
902 | ret = PTR_ERR(dev_priv->perf.oa.oa_buffer.vaddr); | |
903 | goto err_unpin; | |
904 | } | |
905 | ||
906 | dev_priv->perf.oa.ops.init_oa_buffer(dev_priv); | |
907 | ||
908 | DRM_DEBUG_DRIVER("OA Buffer initialized, gtt offset = 0x%x, vaddr = %p\n", | |
909 | i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma), | |
910 | dev_priv->perf.oa.oa_buffer.vaddr); | |
911 | ||
912 | goto unlock; | |
913 | ||
914 | err_unpin: | |
915 | __i915_vma_unpin(vma); | |
916 | ||
917 | err_unref: | |
918 | i915_gem_object_put(bo); | |
919 | ||
920 | dev_priv->perf.oa.oa_buffer.vaddr = NULL; | |
921 | dev_priv->perf.oa.oa_buffer.vma = NULL; | |
922 | ||
923 | unlock: | |
924 | mutex_unlock(&dev_priv->drm.struct_mutex); | |
925 | return ret; | |
926 | } | |
927 | ||
928 | static void config_oa_regs(struct drm_i915_private *dev_priv, | |
929 | const struct i915_oa_reg *regs, | |
930 | int n_regs) | |
931 | { | |
932 | int i; | |
933 | ||
934 | for (i = 0; i < n_regs; i++) { | |
935 | const struct i915_oa_reg *reg = regs + i; | |
936 | ||
937 | I915_WRITE(reg->addr, reg->value); | |
938 | } | |
939 | } | |
940 | ||
941 | static int hsw_enable_metric_set(struct drm_i915_private *dev_priv) | |
942 | { | |
943 | int ret = i915_oa_select_metric_set_hsw(dev_priv); | |
944 | ||
945 | if (ret) | |
946 | return ret; | |
947 | ||
948 | I915_WRITE(GDT_CHICKEN_BITS, (I915_READ(GDT_CHICKEN_BITS) | | |
949 | GT_NOA_ENABLE)); | |
950 | ||
951 | /* PRM: | |
952 | * | |
953 | * OA unit is using “crclk” for its functionality. When trunk | |
954 | * level clock gating takes place, OA clock would be gated, | |
955 | * unable to count the events from non-render clock domain. | |
956 | * Render clock gating must be disabled when OA is enabled to | |
957 | * count the events from non-render domain. Unit level clock | |
958 | * gating for RCS should also be disabled. | |
959 | */ | |
960 | I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) & | |
961 | ~GEN7_DOP_CLOCK_GATE_ENABLE)); | |
962 | I915_WRITE(GEN6_UCGCTL1, (I915_READ(GEN6_UCGCTL1) | | |
963 | GEN6_CSUNIT_CLOCK_GATE_DISABLE)); | |
964 | ||
965 | config_oa_regs(dev_priv, dev_priv->perf.oa.mux_regs, | |
966 | dev_priv->perf.oa.mux_regs_len); | |
967 | ||
968 | /* It apparently takes a fairly long time for a new MUX | |
969 | * configuration to be be applied after these register writes. | |
970 | * This delay duration was derived empirically based on the | |
971 | * render_basic config but hopefully it covers the maximum | |
972 | * configuration latency. | |
973 | * | |
974 | * As a fallback, the checks in _append_oa_reports() to skip | |
975 | * invalid OA reports do also seem to work to discard reports | |
976 | * generated before this config has completed - albeit not | |
977 | * silently. | |
978 | * | |
979 | * Unfortunately this is essentially a magic number, since we | |
980 | * don't currently know of a reliable mechanism for predicting | |
981 | * how long the MUX config will take to apply and besides | |
982 | * seeing invalid reports we don't know of a reliable way to | |
983 | * explicitly check that the MUX config has landed. | |
984 | * | |
985 | * It's even possible we've miss characterized the underlying | |
986 | * problem - it just seems like the simplest explanation why | |
987 | * a delay at this location would mitigate any invalid reports. | |
988 | */ | |
989 | usleep_range(15000, 20000); | |
990 | ||
991 | config_oa_regs(dev_priv, dev_priv->perf.oa.b_counter_regs, | |
992 | dev_priv->perf.oa.b_counter_regs_len); | |
993 | ||
994 | return 0; | |
995 | } | |
996 | ||
997 | static void hsw_disable_metric_set(struct drm_i915_private *dev_priv) | |
998 | { | |
999 | I915_WRITE(GEN6_UCGCTL1, (I915_READ(GEN6_UCGCTL1) & | |
1000 | ~GEN6_CSUNIT_CLOCK_GATE_DISABLE)); | |
1001 | I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) | | |
1002 | GEN7_DOP_CLOCK_GATE_ENABLE)); | |
1003 | ||
1004 | I915_WRITE(GDT_CHICKEN_BITS, (I915_READ(GDT_CHICKEN_BITS) & | |
1005 | ~GT_NOA_ENABLE)); | |
1006 | } | |
1007 | ||
1008 | static void gen7_update_oacontrol_locked(struct drm_i915_private *dev_priv) | |
1009 | { | |
67520415 | 1010 | lockdep_assert_held(&dev_priv->perf.hook_lock); |
d7965152 RB |
1011 | |
1012 | if (dev_priv->perf.oa.exclusive_stream->enabled) { | |
1013 | struct i915_gem_context *ctx = | |
1014 | dev_priv->perf.oa.exclusive_stream->ctx; | |
1015 | u32 ctx_id = dev_priv->perf.oa.specific_ctx_id; | |
1016 | ||
1017 | bool periodic = dev_priv->perf.oa.periodic; | |
1018 | u32 period_exponent = dev_priv->perf.oa.period_exponent; | |
1019 | u32 report_format = dev_priv->perf.oa.oa_buffer.format; | |
1020 | ||
1021 | I915_WRITE(GEN7_OACONTROL, | |
1022 | (ctx_id & GEN7_OACONTROL_CTX_MASK) | | |
1023 | (period_exponent << | |
1024 | GEN7_OACONTROL_TIMER_PERIOD_SHIFT) | | |
1025 | (periodic ? GEN7_OACONTROL_TIMER_ENABLE : 0) | | |
1026 | (report_format << GEN7_OACONTROL_FORMAT_SHIFT) | | |
1027 | (ctx ? GEN7_OACONTROL_PER_CTX_ENABLE : 0) | | |
1028 | GEN7_OACONTROL_ENABLE); | |
1029 | } else | |
1030 | I915_WRITE(GEN7_OACONTROL, 0); | |
1031 | } | |
1032 | ||
1033 | static void gen7_oa_enable(struct drm_i915_private *dev_priv) | |
1034 | { | |
1035 | unsigned long flags; | |
1036 | ||
1037 | /* Reset buf pointers so we don't forward reports from before now. | |
1038 | * | |
1039 | * Think carefully if considering trying to avoid this, since it | |
1040 | * also ensures status flags and the buffer itself are cleared | |
1041 | * in error paths, and we have checks for invalid reports based | |
1042 | * on the assumption that certain fields are written to zeroed | |
1043 | * memory which this helps maintains. | |
1044 | */ | |
1045 | gen7_init_oa_buffer(dev_priv); | |
1046 | ||
1047 | spin_lock_irqsave(&dev_priv->perf.hook_lock, flags); | |
1048 | gen7_update_oacontrol_locked(dev_priv); | |
1049 | spin_unlock_irqrestore(&dev_priv->perf.hook_lock, flags); | |
1050 | } | |
1051 | ||
16d98b31 RB |
1052 | /** |
1053 | * i915_oa_stream_enable - handle `I915_PERF_IOCTL_ENABLE` for OA stream | |
1054 | * @stream: An i915 perf stream opened for OA metrics | |
1055 | * | |
1056 | * [Re]enables hardware periodic sampling according to the period configured | |
1057 | * when opening the stream. This also starts a hrtimer that will periodically | |
1058 | * check for data in the circular OA buffer for notifying userspace (e.g. | |
1059 | * during a read() or poll()). | |
1060 | */ | |
d7965152 RB |
1061 | static void i915_oa_stream_enable(struct i915_perf_stream *stream) |
1062 | { | |
1063 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
1064 | ||
1065 | dev_priv->perf.oa.ops.oa_enable(dev_priv); | |
1066 | ||
1067 | if (dev_priv->perf.oa.periodic) | |
1068 | hrtimer_start(&dev_priv->perf.oa.poll_check_timer, | |
1069 | ns_to_ktime(POLL_PERIOD), | |
1070 | HRTIMER_MODE_REL_PINNED); | |
1071 | } | |
1072 | ||
1073 | static void gen7_oa_disable(struct drm_i915_private *dev_priv) | |
1074 | { | |
1075 | I915_WRITE(GEN7_OACONTROL, 0); | |
1076 | } | |
1077 | ||
16d98b31 RB |
1078 | /** |
1079 | * i915_oa_stream_disable - handle `I915_PERF_IOCTL_DISABLE` for OA stream | |
1080 | * @stream: An i915 perf stream opened for OA metrics | |
1081 | * | |
1082 | * Stops the OA unit from periodically writing counter reports into the | |
1083 | * circular OA buffer. This also stops the hrtimer that periodically checks for | |
1084 | * data in the circular OA buffer, for notifying userspace. | |
1085 | */ | |
d7965152 RB |
1086 | static void i915_oa_stream_disable(struct i915_perf_stream *stream) |
1087 | { | |
1088 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
1089 | ||
1090 | dev_priv->perf.oa.ops.oa_disable(dev_priv); | |
1091 | ||
1092 | if (dev_priv->perf.oa.periodic) | |
1093 | hrtimer_cancel(&dev_priv->perf.oa.poll_check_timer); | |
1094 | } | |
1095 | ||
1096 | static u64 oa_exponent_to_ns(struct drm_i915_private *dev_priv, int exponent) | |
1097 | { | |
24603935 CW |
1098 | return div_u64(1000000000ULL * (2ULL << exponent), |
1099 | dev_priv->perf.oa.timestamp_frequency); | |
d7965152 RB |
1100 | } |
1101 | ||
1102 | static const struct i915_perf_stream_ops i915_oa_stream_ops = { | |
1103 | .destroy = i915_oa_stream_destroy, | |
1104 | .enable = i915_oa_stream_enable, | |
1105 | .disable = i915_oa_stream_disable, | |
1106 | .wait_unlocked = i915_oa_wait_unlocked, | |
1107 | .poll_wait = i915_oa_poll_wait, | |
1108 | .read = i915_oa_read, | |
eec688e1 RB |
1109 | }; |
1110 | ||
16d98b31 RB |
1111 | /** |
1112 | * i915_oa_stream_init - validate combined props for OA stream and init | |
1113 | * @stream: An i915 perf stream | |
1114 | * @param: The open parameters passed to `DRM_I915_PERF_OPEN` | |
1115 | * @props: The property state that configures stream (individually validated) | |
1116 | * | |
1117 | * While read_properties_unlocked() validates properties in isolation it | |
1118 | * doesn't ensure that the combination necessarily makes sense. | |
1119 | * | |
1120 | * At this point it has been determined that userspace wants a stream of | |
1121 | * OA metrics, but still we need to further validate the combined | |
1122 | * properties are OK. | |
1123 | * | |
1124 | * If the configuration makes sense then we can allocate memory for | |
1125 | * a circular OA buffer and apply the requested metric set configuration. | |
1126 | * | |
1127 | * Returns: zero on success or a negative error code. | |
1128 | */ | |
d7965152 RB |
1129 | static int i915_oa_stream_init(struct i915_perf_stream *stream, |
1130 | struct drm_i915_perf_open_param *param, | |
1131 | struct perf_open_properties *props) | |
1132 | { | |
1133 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
1134 | int format_size; | |
1135 | int ret; | |
1136 | ||
442b8c06 RB |
1137 | /* If the sysfs metrics/ directory wasn't registered for some |
1138 | * reason then don't let userspace try their luck with config | |
1139 | * IDs | |
1140 | */ | |
1141 | if (!dev_priv->perf.metrics_kobj) { | |
7708550c | 1142 | DRM_DEBUG("OA metrics weren't advertised via sysfs\n"); |
442b8c06 RB |
1143 | return -EINVAL; |
1144 | } | |
1145 | ||
d7965152 | 1146 | if (!(props->sample_flags & SAMPLE_OA_REPORT)) { |
7708550c | 1147 | DRM_DEBUG("Only OA report sampling supported\n"); |
d7965152 RB |
1148 | return -EINVAL; |
1149 | } | |
1150 | ||
1151 | if (!dev_priv->perf.oa.ops.init_oa_buffer) { | |
7708550c | 1152 | DRM_DEBUG("OA unit not supported\n"); |
d7965152 RB |
1153 | return -ENODEV; |
1154 | } | |
1155 | ||
1156 | /* To avoid the complexity of having to accurately filter | |
1157 | * counter reports and marshal to the appropriate client | |
1158 | * we currently only allow exclusive access | |
1159 | */ | |
1160 | if (dev_priv->perf.oa.exclusive_stream) { | |
7708550c | 1161 | DRM_DEBUG("OA unit already in use\n"); |
d7965152 RB |
1162 | return -EBUSY; |
1163 | } | |
1164 | ||
1165 | if (!props->metrics_set) { | |
7708550c | 1166 | DRM_DEBUG("OA metric set not specified\n"); |
d7965152 RB |
1167 | return -EINVAL; |
1168 | } | |
1169 | ||
1170 | if (!props->oa_format) { | |
7708550c | 1171 | DRM_DEBUG("OA report format not specified\n"); |
d7965152 RB |
1172 | return -EINVAL; |
1173 | } | |
1174 | ||
1175 | stream->sample_size = sizeof(struct drm_i915_perf_record_header); | |
1176 | ||
1177 | format_size = dev_priv->perf.oa.oa_formats[props->oa_format].size; | |
1178 | ||
1179 | stream->sample_flags |= SAMPLE_OA_REPORT; | |
1180 | stream->sample_size += format_size; | |
1181 | ||
1182 | dev_priv->perf.oa.oa_buffer.format_size = format_size; | |
1183 | if (WARN_ON(dev_priv->perf.oa.oa_buffer.format_size == 0)) | |
1184 | return -EINVAL; | |
1185 | ||
1186 | dev_priv->perf.oa.oa_buffer.format = | |
1187 | dev_priv->perf.oa.oa_formats[props->oa_format].format; | |
1188 | ||
1189 | dev_priv->perf.oa.metrics_set = props->metrics_set; | |
1190 | ||
1191 | dev_priv->perf.oa.periodic = props->oa_periodic; | |
1192 | if (dev_priv->perf.oa.periodic) { | |
24603935 | 1193 | u32 tail; |
d7965152 RB |
1194 | |
1195 | dev_priv->perf.oa.period_exponent = props->oa_period_exponent; | |
1196 | ||
1197 | /* See comment for OA_TAIL_MARGIN_NSEC for details | |
1198 | * about this tail_margin... | |
1199 | */ | |
24603935 CW |
1200 | tail = div64_u64(OA_TAIL_MARGIN_NSEC, |
1201 | oa_exponent_to_ns(dev_priv, | |
1202 | props->oa_period_exponent)); | |
1203 | dev_priv->perf.oa.tail_margin = (tail + 1) * format_size; | |
d7965152 RB |
1204 | } |
1205 | ||
1206 | if (stream->ctx) { | |
1207 | ret = oa_get_render_ctx_id(stream); | |
1208 | if (ret) | |
1209 | return ret; | |
1210 | } | |
1211 | ||
1212 | ret = alloc_oa_buffer(dev_priv); | |
1213 | if (ret) | |
1214 | goto err_oa_buf_alloc; | |
1215 | ||
1216 | /* PRM - observability performance counters: | |
1217 | * | |
1218 | * OACONTROL, performance counter enable, note: | |
1219 | * | |
1220 | * "When this bit is set, in order to have coherent counts, | |
1221 | * RC6 power state and trunk clock gating must be disabled. | |
1222 | * This can be achieved by programming MMIO registers as | |
1223 | * 0xA094=0 and 0xA090[31]=1" | |
1224 | * | |
1225 | * In our case we are expecting that taking pm + FORCEWAKE | |
1226 | * references will effectively disable RC6. | |
1227 | */ | |
1228 | intel_runtime_pm_get(dev_priv); | |
1229 | intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); | |
1230 | ||
1231 | ret = dev_priv->perf.oa.ops.enable_metric_set(dev_priv); | |
1232 | if (ret) | |
1233 | goto err_enable; | |
1234 | ||
1235 | stream->ops = &i915_oa_stream_ops; | |
1236 | ||
1237 | dev_priv->perf.oa.exclusive_stream = stream; | |
1238 | ||
1239 | return 0; | |
1240 | ||
1241 | err_enable: | |
1242 | intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); | |
1243 | intel_runtime_pm_put(dev_priv); | |
1244 | free_oa_buffer(dev_priv); | |
1245 | ||
1246 | err_oa_buf_alloc: | |
1247 | if (stream->ctx) | |
1248 | oa_put_render_ctx_id(stream); | |
1249 | ||
1250 | return ret; | |
1251 | } | |
1252 | ||
16d98b31 RB |
1253 | /** |
1254 | * i915_perf_read_locked - &i915_perf_stream_ops->read with error normalisation | |
1255 | * @stream: An i915 perf stream | |
1256 | * @file: An i915 perf stream file | |
1257 | * @buf: destination buffer given by userspace | |
1258 | * @count: the number of bytes userspace wants to read | |
1259 | * @ppos: (inout) file seek position (unused) | |
1260 | * | |
1261 | * Besides wrapping &i915_perf_stream_ops->read this provides a common place to | |
1262 | * ensure that if we've successfully copied any data then reporting that takes | |
1263 | * precedence over any internal error status, so the data isn't lost. | |
1264 | * | |
1265 | * For example ret will be -ENOSPC whenever there is more buffered data than | |
1266 | * can be copied to userspace, but that's only interesting if we weren't able | |
1267 | * to copy some data because it implies the userspace buffer is too small to | |
1268 | * receive a single record (and we never split records). | |
1269 | * | |
1270 | * Another case with ret == -EFAULT is more of a grey area since it would seem | |
1271 | * like bad form for userspace to ask us to overrun its buffer, but the user | |
1272 | * knows best: | |
1273 | * | |
1274 | * http://yarchive.net/comp/linux/partial_reads_writes.html | |
1275 | * | |
1276 | * Returns: The number of bytes copied or a negative error code on failure. | |
1277 | */ | |
eec688e1 RB |
1278 | static ssize_t i915_perf_read_locked(struct i915_perf_stream *stream, |
1279 | struct file *file, | |
1280 | char __user *buf, | |
1281 | size_t count, | |
1282 | loff_t *ppos) | |
1283 | { | |
1284 | /* Note we keep the offset (aka bytes read) separate from any | |
1285 | * error status so that the final check for whether we return | |
1286 | * the bytes read with a higher precedence than any error (see | |
1287 | * comment below) doesn't need to be handled/duplicated in | |
1288 | * stream->ops->read() implementations. | |
1289 | */ | |
1290 | size_t offset = 0; | |
1291 | int ret = stream->ops->read(stream, buf, count, &offset); | |
1292 | ||
eec688e1 RB |
1293 | return offset ?: (ret ?: -EAGAIN); |
1294 | } | |
1295 | ||
16d98b31 RB |
1296 | /** |
1297 | * i915_perf_read - handles read() FOP for i915 perf stream FDs | |
1298 | * @file: An i915 perf stream file | |
1299 | * @buf: destination buffer given by userspace | |
1300 | * @count: the number of bytes userspace wants to read | |
1301 | * @ppos: (inout) file seek position (unused) | |
1302 | * | |
1303 | * The entry point for handling a read() on a stream file descriptor from | |
1304 | * userspace. Most of the work is left to the i915_perf_read_locked() and | |
1305 | * &i915_perf_stream_ops->read but to save having stream implementations (of | |
1306 | * which we might have multiple later) we handle blocking read here. | |
1307 | * | |
1308 | * We can also consistently treat trying to read from a disabled stream | |
1309 | * as an IO error so implementations can assume the stream is enabled | |
1310 | * while reading. | |
1311 | * | |
1312 | * Returns: The number of bytes copied or a negative error code on failure. | |
1313 | */ | |
eec688e1 RB |
1314 | static ssize_t i915_perf_read(struct file *file, |
1315 | char __user *buf, | |
1316 | size_t count, | |
1317 | loff_t *ppos) | |
1318 | { | |
1319 | struct i915_perf_stream *stream = file->private_data; | |
1320 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
1321 | ssize_t ret; | |
1322 | ||
d7965152 RB |
1323 | /* To ensure it's handled consistently we simply treat all reads of a |
1324 | * disabled stream as an error. In particular it might otherwise lead | |
1325 | * to a deadlock for blocking file descriptors... | |
1326 | */ | |
1327 | if (!stream->enabled) | |
1328 | return -EIO; | |
1329 | ||
eec688e1 | 1330 | if (!(file->f_flags & O_NONBLOCK)) { |
d7965152 RB |
1331 | /* There's the small chance of false positives from |
1332 | * stream->ops->wait_unlocked. | |
1333 | * | |
1334 | * E.g. with single context filtering since we only wait until | |
1335 | * oabuffer has >= 1 report we don't immediately know whether | |
1336 | * any reports really belong to the current context | |
eec688e1 RB |
1337 | */ |
1338 | do { | |
1339 | ret = stream->ops->wait_unlocked(stream); | |
1340 | if (ret) | |
1341 | return ret; | |
1342 | ||
1343 | mutex_lock(&dev_priv->perf.lock); | |
1344 | ret = i915_perf_read_locked(stream, file, | |
1345 | buf, count, ppos); | |
1346 | mutex_unlock(&dev_priv->perf.lock); | |
1347 | } while (ret == -EAGAIN); | |
1348 | } else { | |
1349 | mutex_lock(&dev_priv->perf.lock); | |
1350 | ret = i915_perf_read_locked(stream, file, buf, count, ppos); | |
1351 | mutex_unlock(&dev_priv->perf.lock); | |
1352 | } | |
1353 | ||
d7965152 RB |
1354 | if (ret >= 0) { |
1355 | /* Maybe make ->pollin per-stream state if we support multiple | |
1356 | * concurrent streams in the future. | |
1357 | */ | |
1358 | dev_priv->perf.oa.pollin = false; | |
1359 | } | |
1360 | ||
eec688e1 RB |
1361 | return ret; |
1362 | } | |
1363 | ||
d7965152 RB |
1364 | static enum hrtimer_restart oa_poll_check_timer_cb(struct hrtimer *hrtimer) |
1365 | { | |
1366 | struct drm_i915_private *dev_priv = | |
1367 | container_of(hrtimer, typeof(*dev_priv), | |
1368 | perf.oa.poll_check_timer); | |
1369 | ||
1370 | if (!dev_priv->perf.oa.ops.oa_buffer_is_empty(dev_priv)) { | |
1371 | dev_priv->perf.oa.pollin = true; | |
1372 | wake_up(&dev_priv->perf.oa.poll_wq); | |
1373 | } | |
1374 | ||
1375 | hrtimer_forward_now(hrtimer, ns_to_ktime(POLL_PERIOD)); | |
1376 | ||
1377 | return HRTIMER_RESTART; | |
1378 | } | |
1379 | ||
16d98b31 RB |
1380 | /** |
1381 | * i915_perf_poll_locked - poll_wait() with a suitable wait queue for stream | |
1382 | * @dev_priv: i915 device instance | |
1383 | * @stream: An i915 perf stream | |
1384 | * @file: An i915 perf stream file | |
1385 | * @wait: poll() state table | |
1386 | * | |
1387 | * For handling userspace polling on an i915 perf stream, this calls through to | |
1388 | * &i915_perf_stream_ops->poll_wait to call poll_wait() with a wait queue that | |
1389 | * will be woken for new stream data. | |
1390 | * | |
1391 | * Note: The &drm_i915_private->perf.lock mutex has been taken to serialize | |
1392 | * with any non-file-operation driver hooks. | |
1393 | * | |
1394 | * Returns: any poll events that are ready without sleeping | |
1395 | */ | |
d7965152 RB |
1396 | static unsigned int i915_perf_poll_locked(struct drm_i915_private *dev_priv, |
1397 | struct i915_perf_stream *stream, | |
eec688e1 RB |
1398 | struct file *file, |
1399 | poll_table *wait) | |
1400 | { | |
d7965152 | 1401 | unsigned int events = 0; |
eec688e1 RB |
1402 | |
1403 | stream->ops->poll_wait(stream, file, wait); | |
1404 | ||
d7965152 RB |
1405 | /* Note: we don't explicitly check whether there's something to read |
1406 | * here since this path may be very hot depending on what else | |
1407 | * userspace is polling, or on the timeout in use. We rely solely on | |
1408 | * the hrtimer/oa_poll_check_timer_cb to notify us when there are | |
1409 | * samples to read. | |
1410 | */ | |
1411 | if (dev_priv->perf.oa.pollin) | |
1412 | events |= POLLIN; | |
eec688e1 | 1413 | |
d7965152 | 1414 | return events; |
eec688e1 RB |
1415 | } |
1416 | ||
16d98b31 RB |
1417 | /** |
1418 | * i915_perf_poll - call poll_wait() with a suitable wait queue for stream | |
1419 | * @file: An i915 perf stream file | |
1420 | * @wait: poll() state table | |
1421 | * | |
1422 | * For handling userspace polling on an i915 perf stream, this ensures | |
1423 | * poll_wait() gets called with a wait queue that will be woken for new stream | |
1424 | * data. | |
1425 | * | |
1426 | * Note: Implementation deferred to i915_perf_poll_locked() | |
1427 | * | |
1428 | * Returns: any poll events that are ready without sleeping | |
1429 | */ | |
eec688e1 RB |
1430 | static unsigned int i915_perf_poll(struct file *file, poll_table *wait) |
1431 | { | |
1432 | struct i915_perf_stream *stream = file->private_data; | |
1433 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
1434 | int ret; | |
1435 | ||
1436 | mutex_lock(&dev_priv->perf.lock); | |
d7965152 | 1437 | ret = i915_perf_poll_locked(dev_priv, stream, file, wait); |
eec688e1 RB |
1438 | mutex_unlock(&dev_priv->perf.lock); |
1439 | ||
1440 | return ret; | |
1441 | } | |
1442 | ||
16d98b31 RB |
1443 | /** |
1444 | * i915_perf_enable_locked - handle `I915_PERF_IOCTL_ENABLE` ioctl | |
1445 | * @stream: A disabled i915 perf stream | |
1446 | * | |
1447 | * [Re]enables the associated capture of data for this stream. | |
1448 | * | |
1449 | * If a stream was previously enabled then there's currently no intention | |
1450 | * to provide userspace any guarantee about the preservation of previously | |
1451 | * buffered data. | |
1452 | */ | |
eec688e1 RB |
1453 | static void i915_perf_enable_locked(struct i915_perf_stream *stream) |
1454 | { | |
1455 | if (stream->enabled) | |
1456 | return; | |
1457 | ||
1458 | /* Allow stream->ops->enable() to refer to this */ | |
1459 | stream->enabled = true; | |
1460 | ||
1461 | if (stream->ops->enable) | |
1462 | stream->ops->enable(stream); | |
1463 | } | |
1464 | ||
16d98b31 RB |
1465 | /** |
1466 | * i915_perf_disable_locked - handle `I915_PERF_IOCTL_DISABLE` ioctl | |
1467 | * @stream: An enabled i915 perf stream | |
1468 | * | |
1469 | * Disables the associated capture of data for this stream. | |
1470 | * | |
1471 | * The intention is that disabling an re-enabling a stream will ideally be | |
1472 | * cheaper than destroying and re-opening a stream with the same configuration, | |
1473 | * though there are no formal guarantees about what state or buffered data | |
1474 | * must be retained between disabling and re-enabling a stream. | |
1475 | * | |
1476 | * Note: while a stream is disabled it's considered an error for userspace | |
1477 | * to attempt to read from the stream (-EIO). | |
1478 | */ | |
eec688e1 RB |
1479 | static void i915_perf_disable_locked(struct i915_perf_stream *stream) |
1480 | { | |
1481 | if (!stream->enabled) | |
1482 | return; | |
1483 | ||
1484 | /* Allow stream->ops->disable() to refer to this */ | |
1485 | stream->enabled = false; | |
1486 | ||
1487 | if (stream->ops->disable) | |
1488 | stream->ops->disable(stream); | |
1489 | } | |
1490 | ||
16d98b31 RB |
1491 | /** |
1492 | * i915_perf_ioctl - support ioctl() usage with i915 perf stream FDs | |
1493 | * @stream: An i915 perf stream | |
1494 | * @cmd: the ioctl request | |
1495 | * @arg: the ioctl data | |
1496 | * | |
1497 | * Note: The &drm_i915_private->perf.lock mutex has been taken to serialize | |
1498 | * with any non-file-operation driver hooks. | |
1499 | * | |
1500 | * Returns: zero on success or a negative error code. Returns -EINVAL for | |
1501 | * an unknown ioctl request. | |
1502 | */ | |
eec688e1 RB |
1503 | static long i915_perf_ioctl_locked(struct i915_perf_stream *stream, |
1504 | unsigned int cmd, | |
1505 | unsigned long arg) | |
1506 | { | |
1507 | switch (cmd) { | |
1508 | case I915_PERF_IOCTL_ENABLE: | |
1509 | i915_perf_enable_locked(stream); | |
1510 | return 0; | |
1511 | case I915_PERF_IOCTL_DISABLE: | |
1512 | i915_perf_disable_locked(stream); | |
1513 | return 0; | |
1514 | } | |
1515 | ||
1516 | return -EINVAL; | |
1517 | } | |
1518 | ||
16d98b31 RB |
1519 | /** |
1520 | * i915_perf_ioctl - support ioctl() usage with i915 perf stream FDs | |
1521 | * @file: An i915 perf stream file | |
1522 | * @cmd: the ioctl request | |
1523 | * @arg: the ioctl data | |
1524 | * | |
1525 | * Implementation deferred to i915_perf_ioctl_locked(). | |
1526 | * | |
1527 | * Returns: zero on success or a negative error code. Returns -EINVAL for | |
1528 | * an unknown ioctl request. | |
1529 | */ | |
eec688e1 RB |
1530 | static long i915_perf_ioctl(struct file *file, |
1531 | unsigned int cmd, | |
1532 | unsigned long arg) | |
1533 | { | |
1534 | struct i915_perf_stream *stream = file->private_data; | |
1535 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
1536 | long ret; | |
1537 | ||
1538 | mutex_lock(&dev_priv->perf.lock); | |
1539 | ret = i915_perf_ioctl_locked(stream, cmd, arg); | |
1540 | mutex_unlock(&dev_priv->perf.lock); | |
1541 | ||
1542 | return ret; | |
1543 | } | |
1544 | ||
16d98b31 RB |
1545 | /** |
1546 | * i915_perf_destroy_locked - destroy an i915 perf stream | |
1547 | * @stream: An i915 perf stream | |
1548 | * | |
1549 | * Frees all resources associated with the given i915 perf @stream, disabling | |
1550 | * any associated data capture in the process. | |
1551 | * | |
1552 | * Note: The &drm_i915_private->perf.lock mutex has been taken to serialize | |
1553 | * with any non-file-operation driver hooks. | |
1554 | */ | |
eec688e1 RB |
1555 | static void i915_perf_destroy_locked(struct i915_perf_stream *stream) |
1556 | { | |
eec688e1 RB |
1557 | if (stream->enabled) |
1558 | i915_perf_disable_locked(stream); | |
1559 | ||
1560 | if (stream->ops->destroy) | |
1561 | stream->ops->destroy(stream); | |
1562 | ||
1563 | list_del(&stream->link); | |
1564 | ||
69df05e1 CW |
1565 | if (stream->ctx) |
1566 | i915_gem_context_put_unlocked(stream->ctx); | |
eec688e1 RB |
1567 | |
1568 | kfree(stream); | |
1569 | } | |
1570 | ||
16d98b31 RB |
1571 | /** |
1572 | * i915_perf_release - handles userspace close() of a stream file | |
1573 | * @inode: anonymous inode associated with file | |
1574 | * @file: An i915 perf stream file | |
1575 | * | |
1576 | * Cleans up any resources associated with an open i915 perf stream file. | |
1577 | * | |
1578 | * NB: close() can't really fail from the userspace point of view. | |
1579 | * | |
1580 | * Returns: zero on success or a negative error code. | |
1581 | */ | |
eec688e1 RB |
1582 | static int i915_perf_release(struct inode *inode, struct file *file) |
1583 | { | |
1584 | struct i915_perf_stream *stream = file->private_data; | |
1585 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
1586 | ||
1587 | mutex_lock(&dev_priv->perf.lock); | |
1588 | i915_perf_destroy_locked(stream); | |
1589 | mutex_unlock(&dev_priv->perf.lock); | |
1590 | ||
1591 | return 0; | |
1592 | } | |
1593 | ||
1594 | ||
1595 | static const struct file_operations fops = { | |
1596 | .owner = THIS_MODULE, | |
1597 | .llseek = no_llseek, | |
1598 | .release = i915_perf_release, | |
1599 | .poll = i915_perf_poll, | |
1600 | .read = i915_perf_read, | |
1601 | .unlocked_ioctl = i915_perf_ioctl, | |
1602 | }; | |
1603 | ||
1604 | ||
1605 | static struct i915_gem_context * | |
1606 | lookup_context(struct drm_i915_private *dev_priv, | |
1607 | struct drm_i915_file_private *file_priv, | |
1608 | u32 ctx_user_handle) | |
1609 | { | |
1610 | struct i915_gem_context *ctx; | |
1611 | int ret; | |
1612 | ||
1613 | ret = i915_mutex_lock_interruptible(&dev_priv->drm); | |
1614 | if (ret) | |
1615 | return ERR_PTR(ret); | |
1616 | ||
1617 | ctx = i915_gem_context_lookup(file_priv, ctx_user_handle); | |
1618 | if (!IS_ERR(ctx)) | |
1619 | i915_gem_context_get(ctx); | |
1620 | ||
1621 | mutex_unlock(&dev_priv->drm.struct_mutex); | |
1622 | ||
1623 | return ctx; | |
1624 | } | |
1625 | ||
16d98b31 RB |
1626 | /** |
1627 | * i915_perf_open_ioctl_locked - DRM ioctl() for userspace to open a stream FD | |
1628 | * @dev_priv: i915 device instance | |
1629 | * @param: The open parameters passed to 'DRM_I915_PERF_OPEN` | |
1630 | * @props: individually validated u64 property value pairs | |
1631 | * @file: drm file | |
1632 | * | |
1633 | * See i915_perf_ioctl_open() for interface details. | |
1634 | * | |
1635 | * Implements further stream config validation and stream initialization on | |
1636 | * behalf of i915_perf_open_ioctl() with the &drm_i915_private->perf.lock mutex | |
1637 | * taken to serialize with any non-file-operation driver hooks. | |
1638 | * | |
1639 | * Note: at this point the @props have only been validated in isolation and | |
1640 | * it's still necessary to validate that the combination of properties makes | |
1641 | * sense. | |
1642 | * | |
1643 | * In the case where userspace is interested in OA unit metrics then further | |
1644 | * config validation and stream initialization details will be handled by | |
1645 | * i915_oa_stream_init(). The code here should only validate config state that | |
1646 | * will be relevant to all stream types / backends. | |
1647 | * | |
1648 | * Returns: zero on success or a negative error code. | |
1649 | */ | |
eec688e1 RB |
1650 | static int |
1651 | i915_perf_open_ioctl_locked(struct drm_i915_private *dev_priv, | |
1652 | struct drm_i915_perf_open_param *param, | |
1653 | struct perf_open_properties *props, | |
1654 | struct drm_file *file) | |
1655 | { | |
1656 | struct i915_gem_context *specific_ctx = NULL; | |
1657 | struct i915_perf_stream *stream = NULL; | |
1658 | unsigned long f_flags = 0; | |
1659 | int stream_fd; | |
1660 | int ret; | |
1661 | ||
1662 | if (props->single_context) { | |
1663 | u32 ctx_handle = props->ctx_handle; | |
1664 | struct drm_i915_file_private *file_priv = file->driver_priv; | |
1665 | ||
1666 | specific_ctx = lookup_context(dev_priv, file_priv, ctx_handle); | |
1667 | if (IS_ERR(specific_ctx)) { | |
1668 | ret = PTR_ERR(specific_ctx); | |
1669 | if (ret != -EINTR) | |
7708550c | 1670 | DRM_DEBUG("Failed to look up context with ID %u for opening perf stream\n", |
eec688e1 RB |
1671 | ctx_handle); |
1672 | goto err; | |
1673 | } | |
1674 | } | |
1675 | ||
ccdf6341 RB |
1676 | /* Similar to perf's kernel.perf_paranoid_cpu sysctl option |
1677 | * we check a dev.i915.perf_stream_paranoid sysctl option | |
1678 | * to determine if it's ok to access system wide OA counters | |
1679 | * without CAP_SYS_ADMIN privileges. | |
1680 | */ | |
1681 | if (!specific_ctx && | |
1682 | i915_perf_stream_paranoid && !capable(CAP_SYS_ADMIN)) { | |
7708550c | 1683 | DRM_DEBUG("Insufficient privileges to open system-wide i915 perf stream\n"); |
eec688e1 RB |
1684 | ret = -EACCES; |
1685 | goto err_ctx; | |
1686 | } | |
1687 | ||
1688 | stream = kzalloc(sizeof(*stream), GFP_KERNEL); | |
1689 | if (!stream) { | |
1690 | ret = -ENOMEM; | |
1691 | goto err_ctx; | |
1692 | } | |
1693 | ||
eec688e1 RB |
1694 | stream->dev_priv = dev_priv; |
1695 | stream->ctx = specific_ctx; | |
1696 | ||
d7965152 RB |
1697 | ret = i915_oa_stream_init(stream, param, props); |
1698 | if (ret) | |
1699 | goto err_alloc; | |
1700 | ||
1701 | /* we avoid simply assigning stream->sample_flags = props->sample_flags | |
1702 | * to have _stream_init check the combination of sample flags more | |
1703 | * thoroughly, but still this is the expected result at this point. | |
eec688e1 | 1704 | */ |
d7965152 RB |
1705 | if (WARN_ON(stream->sample_flags != props->sample_flags)) { |
1706 | ret = -ENODEV; | |
22f880ca | 1707 | goto err_flags; |
d7965152 | 1708 | } |
eec688e1 RB |
1709 | |
1710 | list_add(&stream->link, &dev_priv->perf.streams); | |
1711 | ||
1712 | if (param->flags & I915_PERF_FLAG_FD_CLOEXEC) | |
1713 | f_flags |= O_CLOEXEC; | |
1714 | if (param->flags & I915_PERF_FLAG_FD_NONBLOCK) | |
1715 | f_flags |= O_NONBLOCK; | |
1716 | ||
1717 | stream_fd = anon_inode_getfd("[i915_perf]", &fops, stream, f_flags); | |
1718 | if (stream_fd < 0) { | |
1719 | ret = stream_fd; | |
1720 | goto err_open; | |
1721 | } | |
1722 | ||
1723 | if (!(param->flags & I915_PERF_FLAG_DISABLED)) | |
1724 | i915_perf_enable_locked(stream); | |
1725 | ||
1726 | return stream_fd; | |
1727 | ||
1728 | err_open: | |
1729 | list_del(&stream->link); | |
22f880ca | 1730 | err_flags: |
eec688e1 RB |
1731 | if (stream->ops->destroy) |
1732 | stream->ops->destroy(stream); | |
1733 | err_alloc: | |
1734 | kfree(stream); | |
1735 | err_ctx: | |
69df05e1 CW |
1736 | if (specific_ctx) |
1737 | i915_gem_context_put_unlocked(specific_ctx); | |
eec688e1 RB |
1738 | err: |
1739 | return ret; | |
1740 | } | |
1741 | ||
16d98b31 RB |
1742 | /** |
1743 | * read_properties_unlocked - validate + copy userspace stream open properties | |
1744 | * @dev_priv: i915 device instance | |
1745 | * @uprops: The array of u64 key value pairs given by userspace | |
1746 | * @n_props: The number of key value pairs expected in @uprops | |
1747 | * @props: The stream configuration built up while validating properties | |
eec688e1 RB |
1748 | * |
1749 | * Note this function only validates properties in isolation it doesn't | |
1750 | * validate that the combination of properties makes sense or that all | |
1751 | * properties necessary for a particular kind of stream have been set. | |
16d98b31 RB |
1752 | * |
1753 | * Note that there currently aren't any ordering requirements for properties so | |
1754 | * we shouldn't validate or assume anything about ordering here. This doesn't | |
1755 | * rule out defining new properties with ordering requirements in the future. | |
eec688e1 RB |
1756 | */ |
1757 | static int read_properties_unlocked(struct drm_i915_private *dev_priv, | |
1758 | u64 __user *uprops, | |
1759 | u32 n_props, | |
1760 | struct perf_open_properties *props) | |
1761 | { | |
1762 | u64 __user *uprop = uprops; | |
1763 | int i; | |
1764 | ||
1765 | memset(props, 0, sizeof(struct perf_open_properties)); | |
1766 | ||
1767 | if (!n_props) { | |
7708550c | 1768 | DRM_DEBUG("No i915 perf properties given\n"); |
eec688e1 RB |
1769 | return -EINVAL; |
1770 | } | |
1771 | ||
1772 | /* Considering that ID = 0 is reserved and assuming that we don't | |
1773 | * (currently) expect any configurations to ever specify duplicate | |
1774 | * values for a particular property ID then the last _PROP_MAX value is | |
1775 | * one greater than the maximum number of properties we expect to get | |
1776 | * from userspace. | |
1777 | */ | |
1778 | if (n_props >= DRM_I915_PERF_PROP_MAX) { | |
7708550c | 1779 | DRM_DEBUG("More i915 perf properties specified than exist\n"); |
eec688e1 RB |
1780 | return -EINVAL; |
1781 | } | |
1782 | ||
1783 | for (i = 0; i < n_props; i++) { | |
00319ba0 | 1784 | u64 oa_period, oa_freq_hz; |
eec688e1 RB |
1785 | u64 id, value; |
1786 | int ret; | |
1787 | ||
1788 | ret = get_user(id, uprop); | |
1789 | if (ret) | |
1790 | return ret; | |
1791 | ||
1792 | ret = get_user(value, uprop + 1); | |
1793 | if (ret) | |
1794 | return ret; | |
1795 | ||
0a309f9e MA |
1796 | if (id == 0 || id >= DRM_I915_PERF_PROP_MAX) { |
1797 | DRM_DEBUG("Unknown i915 perf property ID\n"); | |
1798 | return -EINVAL; | |
1799 | } | |
1800 | ||
eec688e1 RB |
1801 | switch ((enum drm_i915_perf_property_id)id) { |
1802 | case DRM_I915_PERF_PROP_CTX_HANDLE: | |
1803 | props->single_context = 1; | |
1804 | props->ctx_handle = value; | |
1805 | break; | |
d7965152 RB |
1806 | case DRM_I915_PERF_PROP_SAMPLE_OA: |
1807 | props->sample_flags |= SAMPLE_OA_REPORT; | |
1808 | break; | |
1809 | case DRM_I915_PERF_PROP_OA_METRICS_SET: | |
1810 | if (value == 0 || | |
1811 | value > dev_priv->perf.oa.n_builtin_sets) { | |
7708550c | 1812 | DRM_DEBUG("Unknown OA metric set ID\n"); |
d7965152 RB |
1813 | return -EINVAL; |
1814 | } | |
1815 | props->metrics_set = value; | |
1816 | break; | |
1817 | case DRM_I915_PERF_PROP_OA_FORMAT: | |
1818 | if (value == 0 || value >= I915_OA_FORMAT_MAX) { | |
7708550c | 1819 | DRM_DEBUG("Invalid OA report format\n"); |
d7965152 RB |
1820 | return -EINVAL; |
1821 | } | |
1822 | if (!dev_priv->perf.oa.oa_formats[value].size) { | |
7708550c | 1823 | DRM_DEBUG("Invalid OA report format\n"); |
d7965152 RB |
1824 | return -EINVAL; |
1825 | } | |
1826 | props->oa_format = value; | |
1827 | break; | |
1828 | case DRM_I915_PERF_PROP_OA_EXPONENT: | |
1829 | if (value > OA_EXPONENT_MAX) { | |
7708550c RB |
1830 | DRM_DEBUG("OA timer exponent too high (> %u)\n", |
1831 | OA_EXPONENT_MAX); | |
d7965152 RB |
1832 | return -EINVAL; |
1833 | } | |
1834 | ||
00319ba0 | 1835 | /* Theoretically we can program the OA unit to sample |
d7965152 RB |
1836 | * every 160ns but don't allow that by default unless |
1837 | * root. | |
1838 | * | |
00319ba0 RB |
1839 | * On Haswell the period is derived from the exponent |
1840 | * as: | |
1841 | * | |
1842 | * period = 80ns * 2^(exponent + 1) | |
1843 | */ | |
1844 | BUILD_BUG_ON(sizeof(oa_period) != 8); | |
1845 | oa_period = 80ull * (2ull << value); | |
1846 | ||
1847 | /* This check is primarily to ensure that oa_period <= | |
1848 | * UINT32_MAX (before passing to do_div which only | |
1849 | * accepts a u32 denominator), but we can also skip | |
1850 | * checking anything < 1Hz which implicitly can't be | |
1851 | * limited via an integer oa_max_sample_rate. | |
d7965152 | 1852 | */ |
00319ba0 RB |
1853 | if (oa_period <= NSEC_PER_SEC) { |
1854 | u64 tmp = NSEC_PER_SEC; | |
1855 | do_div(tmp, oa_period); | |
1856 | oa_freq_hz = tmp; | |
1857 | } else | |
1858 | oa_freq_hz = 0; | |
1859 | ||
1860 | if (oa_freq_hz > i915_oa_max_sample_rate && | |
1861 | !capable(CAP_SYS_ADMIN)) { | |
7708550c | 1862 | DRM_DEBUG("OA exponent would exceed the max sampling frequency (sysctl dev.i915.oa_max_sample_rate) %uHz without root privileges\n", |
00319ba0 | 1863 | i915_oa_max_sample_rate); |
d7965152 RB |
1864 | return -EACCES; |
1865 | } | |
1866 | ||
1867 | props->oa_periodic = true; | |
1868 | props->oa_period_exponent = value; | |
1869 | break; | |
0a309f9e | 1870 | case DRM_I915_PERF_PROP_MAX: |
eec688e1 | 1871 | MISSING_CASE(id); |
eec688e1 RB |
1872 | return -EINVAL; |
1873 | } | |
1874 | ||
1875 | uprop += 2; | |
1876 | } | |
1877 | ||
1878 | return 0; | |
1879 | } | |
1880 | ||
16d98b31 RB |
1881 | /** |
1882 | * i915_perf_open_ioctl - DRM ioctl() for userspace to open a stream FD | |
1883 | * @dev: drm device | |
1884 | * @data: ioctl data copied from userspace (unvalidated) | |
1885 | * @file: drm file | |
1886 | * | |
1887 | * Validates the stream open parameters given by userspace including flags | |
1888 | * and an array of u64 key, value pair properties. | |
1889 | * | |
1890 | * Very little is assumed up front about the nature of the stream being | |
1891 | * opened (for instance we don't assume it's for periodic OA unit metrics). An | |
1892 | * i915-perf stream is expected to be a suitable interface for other forms of | |
1893 | * buffered data written by the GPU besides periodic OA metrics. | |
1894 | * | |
1895 | * Note we copy the properties from userspace outside of the i915 perf | |
1896 | * mutex to avoid an awkward lockdep with mmap_sem. | |
1897 | * | |
1898 | * Most of the implementation details are handled by | |
1899 | * i915_perf_open_ioctl_locked() after taking the &drm_i915_private->perf.lock | |
1900 | * mutex for serializing with any non-file-operation driver hooks. | |
1901 | * | |
1902 | * Return: A newly opened i915 Perf stream file descriptor or negative | |
1903 | * error code on failure. | |
1904 | */ | |
eec688e1 RB |
1905 | int i915_perf_open_ioctl(struct drm_device *dev, void *data, |
1906 | struct drm_file *file) | |
1907 | { | |
1908 | struct drm_i915_private *dev_priv = dev->dev_private; | |
1909 | struct drm_i915_perf_open_param *param = data; | |
1910 | struct perf_open_properties props; | |
1911 | u32 known_open_flags; | |
1912 | int ret; | |
1913 | ||
1914 | if (!dev_priv->perf.initialized) { | |
7708550c | 1915 | DRM_DEBUG("i915 perf interface not available for this system\n"); |
eec688e1 RB |
1916 | return -ENOTSUPP; |
1917 | } | |
1918 | ||
1919 | known_open_flags = I915_PERF_FLAG_FD_CLOEXEC | | |
1920 | I915_PERF_FLAG_FD_NONBLOCK | | |
1921 | I915_PERF_FLAG_DISABLED; | |
1922 | if (param->flags & ~known_open_flags) { | |
7708550c | 1923 | DRM_DEBUG("Unknown drm_i915_perf_open_param flag\n"); |
eec688e1 RB |
1924 | return -EINVAL; |
1925 | } | |
1926 | ||
1927 | ret = read_properties_unlocked(dev_priv, | |
1928 | u64_to_user_ptr(param->properties_ptr), | |
1929 | param->num_properties, | |
1930 | &props); | |
1931 | if (ret) | |
1932 | return ret; | |
1933 | ||
1934 | mutex_lock(&dev_priv->perf.lock); | |
1935 | ret = i915_perf_open_ioctl_locked(dev_priv, param, &props, file); | |
1936 | mutex_unlock(&dev_priv->perf.lock); | |
1937 | ||
1938 | return ret; | |
1939 | } | |
1940 | ||
16d98b31 RB |
1941 | /** |
1942 | * i915_perf_register - exposes i915-perf to userspace | |
1943 | * @dev_priv: i915 device instance | |
1944 | * | |
1945 | * In particular OA metric sets are advertised under a sysfs metrics/ | |
1946 | * directory allowing userspace to enumerate valid IDs that can be | |
1947 | * used to open an i915-perf stream. | |
1948 | */ | |
442b8c06 RB |
1949 | void i915_perf_register(struct drm_i915_private *dev_priv) |
1950 | { | |
1951 | if (!IS_HASWELL(dev_priv)) | |
1952 | return; | |
1953 | ||
1954 | if (!dev_priv->perf.initialized) | |
1955 | return; | |
1956 | ||
1957 | /* To be sure we're synchronized with an attempted | |
1958 | * i915_perf_open_ioctl(); considering that we register after | |
1959 | * being exposed to userspace. | |
1960 | */ | |
1961 | mutex_lock(&dev_priv->perf.lock); | |
1962 | ||
1963 | dev_priv->perf.metrics_kobj = | |
1964 | kobject_create_and_add("metrics", | |
1965 | &dev_priv->drm.primary->kdev->kobj); | |
1966 | if (!dev_priv->perf.metrics_kobj) | |
1967 | goto exit; | |
1968 | ||
1969 | if (i915_perf_register_sysfs_hsw(dev_priv)) { | |
1970 | kobject_put(dev_priv->perf.metrics_kobj); | |
1971 | dev_priv->perf.metrics_kobj = NULL; | |
1972 | } | |
1973 | ||
1974 | exit: | |
1975 | mutex_unlock(&dev_priv->perf.lock); | |
1976 | } | |
1977 | ||
16d98b31 RB |
1978 | /** |
1979 | * i915_perf_unregister - hide i915-perf from userspace | |
1980 | * @dev_priv: i915 device instance | |
1981 | * | |
1982 | * i915-perf state cleanup is split up into an 'unregister' and | |
1983 | * 'deinit' phase where the interface is first hidden from | |
1984 | * userspace by i915_perf_unregister() before cleaning up | |
1985 | * remaining state in i915_perf_fini(). | |
1986 | */ | |
442b8c06 RB |
1987 | void i915_perf_unregister(struct drm_i915_private *dev_priv) |
1988 | { | |
1989 | if (!IS_HASWELL(dev_priv)) | |
1990 | return; | |
1991 | ||
1992 | if (!dev_priv->perf.metrics_kobj) | |
1993 | return; | |
1994 | ||
1995 | i915_perf_unregister_sysfs_hsw(dev_priv); | |
1996 | ||
1997 | kobject_put(dev_priv->perf.metrics_kobj); | |
1998 | dev_priv->perf.metrics_kobj = NULL; | |
1999 | } | |
2000 | ||
ccdf6341 RB |
2001 | static struct ctl_table oa_table[] = { |
2002 | { | |
2003 | .procname = "perf_stream_paranoid", | |
2004 | .data = &i915_perf_stream_paranoid, | |
2005 | .maxlen = sizeof(i915_perf_stream_paranoid), | |
2006 | .mode = 0644, | |
2007 | .proc_handler = proc_dointvec_minmax, | |
2008 | .extra1 = &zero, | |
2009 | .extra2 = &one, | |
2010 | }, | |
00319ba0 RB |
2011 | { |
2012 | .procname = "oa_max_sample_rate", | |
2013 | .data = &i915_oa_max_sample_rate, | |
2014 | .maxlen = sizeof(i915_oa_max_sample_rate), | |
2015 | .mode = 0644, | |
2016 | .proc_handler = proc_dointvec_minmax, | |
2017 | .extra1 = &zero, | |
2018 | .extra2 = &oa_sample_rate_hard_limit, | |
2019 | }, | |
ccdf6341 RB |
2020 | {} |
2021 | }; | |
2022 | ||
2023 | static struct ctl_table i915_root[] = { | |
2024 | { | |
2025 | .procname = "i915", | |
2026 | .maxlen = 0, | |
2027 | .mode = 0555, | |
2028 | .child = oa_table, | |
2029 | }, | |
2030 | {} | |
2031 | }; | |
2032 | ||
2033 | static struct ctl_table dev_root[] = { | |
2034 | { | |
2035 | .procname = "dev", | |
2036 | .maxlen = 0, | |
2037 | .mode = 0555, | |
2038 | .child = i915_root, | |
2039 | }, | |
2040 | {} | |
2041 | }; | |
2042 | ||
16d98b31 RB |
2043 | /** |
2044 | * i915_perf_init - initialize i915-perf state on module load | |
2045 | * @dev_priv: i915 device instance | |
2046 | * | |
2047 | * Initializes i915-perf state without exposing anything to userspace. | |
2048 | * | |
2049 | * Note: i915-perf initialization is split into an 'init' and 'register' | |
2050 | * phase with the i915_perf_register() exposing state to userspace. | |
2051 | */ | |
eec688e1 RB |
2052 | void i915_perf_init(struct drm_i915_private *dev_priv) |
2053 | { | |
d7965152 RB |
2054 | if (!IS_HASWELL(dev_priv)) |
2055 | return; | |
2056 | ||
2057 | hrtimer_init(&dev_priv->perf.oa.poll_check_timer, | |
2058 | CLOCK_MONOTONIC, HRTIMER_MODE_REL); | |
2059 | dev_priv->perf.oa.poll_check_timer.function = oa_poll_check_timer_cb; | |
2060 | init_waitqueue_head(&dev_priv->perf.oa.poll_wq); | |
2061 | ||
eec688e1 RB |
2062 | INIT_LIST_HEAD(&dev_priv->perf.streams); |
2063 | mutex_init(&dev_priv->perf.lock); | |
d7965152 RB |
2064 | spin_lock_init(&dev_priv->perf.hook_lock); |
2065 | ||
2066 | dev_priv->perf.oa.ops.init_oa_buffer = gen7_init_oa_buffer; | |
2067 | dev_priv->perf.oa.ops.enable_metric_set = hsw_enable_metric_set; | |
2068 | dev_priv->perf.oa.ops.disable_metric_set = hsw_disable_metric_set; | |
2069 | dev_priv->perf.oa.ops.oa_enable = gen7_oa_enable; | |
2070 | dev_priv->perf.oa.ops.oa_disable = gen7_oa_disable; | |
2071 | dev_priv->perf.oa.ops.read = gen7_oa_read; | |
2072 | dev_priv->perf.oa.ops.oa_buffer_is_empty = | |
2073 | gen7_oa_buffer_is_empty_fop_unlocked; | |
2074 | ||
2075 | dev_priv->perf.oa.timestamp_frequency = 12500000; | |
2076 | ||
2077 | dev_priv->perf.oa.oa_formats = hsw_oa_formats; | |
2078 | ||
2079 | dev_priv->perf.oa.n_builtin_sets = | |
2080 | i915_oa_n_builtin_metric_sets_hsw; | |
eec688e1 | 2081 | |
ccdf6341 RB |
2082 | dev_priv->perf.sysctl_header = register_sysctl_table(dev_root); |
2083 | ||
eec688e1 RB |
2084 | dev_priv->perf.initialized = true; |
2085 | } | |
2086 | ||
16d98b31 RB |
2087 | /** |
2088 | * i915_perf_fini - Counter part to i915_perf_init() | |
2089 | * @dev_priv: i915 device instance | |
2090 | */ | |
eec688e1 RB |
2091 | void i915_perf_fini(struct drm_i915_private *dev_priv) |
2092 | { | |
2093 | if (!dev_priv->perf.initialized) | |
2094 | return; | |
2095 | ||
ccdf6341 RB |
2096 | unregister_sysctl_table(dev_priv->perf.sysctl_header); |
2097 | ||
d7965152 | 2098 | memset(&dev_priv->perf.oa.ops, 0, sizeof(dev_priv->perf.oa.ops)); |
eec688e1 RB |
2099 | dev_priv->perf.initialized = false; |
2100 | } |