Commit | Line | Data |
---|---|---|
a0d3fdb6 CW |
1 | // SPDX-License-Identifier: MIT |
2 | /* | |
3 | * Copyright © 2018 Intel Corporation | |
4 | */ | |
5 | ||
6 | #include <linux/prime_numbers.h> | |
7 | ||
8 | #include "i915_selftest.h" | |
9 | #include "intel_engine_heartbeat.h" | |
10 | #include "intel_engine_pm.h" | |
11 | #include "intel_reset.h" | |
12 | #include "intel_ring.h" | |
13 | #include "selftest_engine_heartbeat.h" | |
14 | #include "selftests/i915_random.h" | |
15 | #include "selftests/igt_flush_test.h" | |
16 | #include "selftests/igt_live_test.h" | |
17 | #include "selftests/igt_spinner.h" | |
18 | #include "selftests/lib_sw_fence.h" | |
19 | #include "shmem_utils.h" | |
20 | ||
21 | #include "gem/selftests/igt_gem_utils.h" | |
22 | #include "gem/selftests/mock_context.h" | |
23 | ||
24 | #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4) | |
25 | #define NUM_GPR 16 | |
26 | #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */ | |
27 | ||
28 | static struct i915_vma *create_scratch(struct intel_gt *gt) | |
29 | { | |
a4d86249 | 30 | return __vm_create_scratch_for_read(>->ggtt->vm, PAGE_SIZE); |
a0d3fdb6 CW |
31 | } |
32 | ||
33 | static bool is_active(struct i915_request *rq) | |
34 | { | |
35 | if (i915_request_is_active(rq)) | |
36 | return true; | |
37 | ||
38 | if (i915_request_on_hold(rq)) | |
39 | return true; | |
40 | ||
41 | if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq)) | |
42 | return true; | |
43 | ||
44 | return false; | |
45 | } | |
46 | ||
47 | static int wait_for_submit(struct intel_engine_cs *engine, | |
48 | struct i915_request *rq, | |
49 | unsigned long timeout) | |
50 | { | |
d484bd0d CW |
51 | /* Ignore our own attempts to suppress excess tasklets */ |
52 | tasklet_hi_schedule(&engine->execlists.tasklet); | |
53 | ||
a0d3fdb6 CW |
54 | timeout += jiffies; |
55 | do { | |
56 | bool done = time_after(jiffies, timeout); | |
57 | ||
58 | if (i915_request_completed(rq)) /* that was quick! */ | |
59 | return 0; | |
60 | ||
61 | /* Wait until the HW has acknowleged the submission (or err) */ | |
62 | intel_engine_flush_submission(engine); | |
63 | if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq)) | |
64 | return 0; | |
65 | ||
66 | if (done) | |
67 | return -ETIME; | |
68 | ||
69 | cond_resched(); | |
70 | } while (1); | |
71 | } | |
72 | ||
73 | static int emit_semaphore_signal(struct intel_context *ce, void *slot) | |
74 | { | |
75 | const u32 offset = | |
76 | i915_ggtt_offset(ce->engine->status_page.vma) + | |
77 | offset_in_page(slot); | |
78 | struct i915_request *rq; | |
79 | u32 *cs; | |
80 | ||
81 | rq = intel_context_create_request(ce); | |
82 | if (IS_ERR(rq)) | |
83 | return PTR_ERR(rq); | |
84 | ||
85 | cs = intel_ring_begin(rq, 4); | |
86 | if (IS_ERR(cs)) { | |
87 | i915_request_add(rq); | |
88 | return PTR_ERR(cs); | |
89 | } | |
90 | ||
91 | *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; | |
92 | *cs++ = offset; | |
93 | *cs++ = 0; | |
94 | *cs++ = 1; | |
95 | ||
96 | intel_ring_advance(rq, cs); | |
97 | ||
98 | rq->sched.attr.priority = I915_PRIORITY_BARRIER; | |
99 | i915_request_add(rq); | |
100 | return 0; | |
101 | } | |
102 | ||
103 | static int context_flush(struct intel_context *ce, long timeout) | |
104 | { | |
105 | struct i915_request *rq; | |
106 | struct dma_fence *fence; | |
107 | int err = 0; | |
108 | ||
109 | rq = intel_engine_create_kernel_request(ce->engine); | |
110 | if (IS_ERR(rq)) | |
111 | return PTR_ERR(rq); | |
112 | ||
113 | fence = i915_active_fence_get(&ce->timeline->last_request); | |
114 | if (fence) { | |
115 | i915_request_await_dma_fence(rq, fence); | |
116 | dma_fence_put(fence); | |
117 | } | |
118 | ||
119 | rq = i915_request_get(rq); | |
120 | i915_request_add(rq); | |
121 | if (i915_request_wait(rq, 0, timeout) < 0) | |
122 | err = -ETIME; | |
123 | i915_request_put(rq); | |
124 | ||
125 | rmb(); /* We know the request is written, make sure all state is too! */ | |
126 | return err; | |
127 | } | |
128 | ||
129 | static int live_lrc_layout(void *arg) | |
130 | { | |
131 | struct intel_gt *gt = arg; | |
132 | struct intel_engine_cs *engine; | |
133 | enum intel_engine_id id; | |
134 | u32 *lrc; | |
135 | int err; | |
136 | ||
137 | /* | |
138 | * Check the registers offsets we use to create the initial reg state | |
139 | * match the layout saved by HW. | |
140 | */ | |
141 | ||
8d03344b | 142 | lrc = (u32 *)__get_free_page(GFP_KERNEL); /* requires page alignment */ |
a0d3fdb6 CW |
143 | if (!lrc) |
144 | return -ENOMEM; | |
8d03344b | 145 | GEM_BUG_ON(offset_in_page(lrc)); |
a0d3fdb6 CW |
146 | |
147 | err = 0; | |
148 | for_each_engine(engine, gt, id) { | |
149 | u32 *hw; | |
150 | int dw; | |
151 | ||
152 | if (!engine->default_state) | |
153 | continue; | |
154 | ||
155 | hw = shmem_pin_map(engine->default_state); | |
156 | if (IS_ERR(hw)) { | |
157 | err = PTR_ERR(hw); | |
158 | break; | |
159 | } | |
160 | hw += LRC_STATE_OFFSET / sizeof(*hw); | |
161 | ||
162 | __lrc_init_regs(memset(lrc, POISON_INUSE, PAGE_SIZE), | |
163 | engine->kernel_context, engine, true); | |
164 | ||
165 | dw = 0; | |
166 | do { | |
989536a4 | 167 | u32 lri = READ_ONCE(hw[dw]); |
a0d3fdb6 CW |
168 | |
169 | if (lri == 0) { | |
170 | dw++; | |
171 | continue; | |
172 | } | |
173 | ||
174 | if (lrc[dw] == 0) { | |
175 | pr_debug("%s: skipped instruction %x at dword %d\n", | |
176 | engine->name, lri, dw); | |
177 | dw++; | |
178 | continue; | |
179 | } | |
180 | ||
181 | if ((lri & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { | |
182 | pr_err("%s: Expected LRI command at dword %d, found %08x\n", | |
183 | engine->name, dw, lri); | |
184 | err = -EINVAL; | |
185 | break; | |
186 | } | |
187 | ||
188 | if (lrc[dw] != lri) { | |
189 | pr_err("%s: LRI command mismatch at dword %d, expected %08x found %08x\n", | |
190 | engine->name, dw, lri, lrc[dw]); | |
191 | err = -EINVAL; | |
192 | break; | |
193 | } | |
194 | ||
195 | lri &= 0x7f; | |
196 | lri++; | |
197 | dw++; | |
198 | ||
199 | while (lri) { | |
989536a4 CW |
200 | u32 offset = READ_ONCE(hw[dw]); |
201 | ||
202 | if (offset != lrc[dw]) { | |
a0d3fdb6 | 203 | pr_err("%s: Different registers found at dword %d, expected %x, found %x\n", |
989536a4 | 204 | engine->name, dw, offset, lrc[dw]); |
a0d3fdb6 CW |
205 | err = -EINVAL; |
206 | break; | |
207 | } | |
208 | ||
209 | /* | |
210 | * Skip over the actual register value as we | |
211 | * expect that to differ. | |
212 | */ | |
213 | dw += 2; | |
214 | lri -= 2; | |
215 | } | |
989536a4 | 216 | } while (!err && (lrc[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); |
a0d3fdb6 CW |
217 | |
218 | if (err) { | |
219 | pr_info("%s: HW register image:\n", engine->name); | |
220 | igt_hexdump(hw, PAGE_SIZE); | |
221 | ||
222 | pr_info("%s: SW register image:\n", engine->name); | |
223 | igt_hexdump(lrc, PAGE_SIZE); | |
224 | } | |
225 | ||
226 | shmem_unpin_map(engine->default_state, hw); | |
227 | if (err) | |
228 | break; | |
229 | } | |
230 | ||
8d03344b | 231 | free_page((unsigned long)lrc); |
a0d3fdb6 CW |
232 | return err; |
233 | } | |
234 | ||
235 | static int find_offset(const u32 *lri, u32 offset) | |
236 | { | |
237 | int i; | |
238 | ||
239 | for (i = 0; i < PAGE_SIZE / sizeof(u32); i++) | |
240 | if (lri[i] == offset) | |
241 | return i; | |
242 | ||
243 | return -1; | |
244 | } | |
245 | ||
246 | static int live_lrc_fixed(void *arg) | |
247 | { | |
248 | struct intel_gt *gt = arg; | |
249 | struct intel_engine_cs *engine; | |
250 | enum intel_engine_id id; | |
251 | int err = 0; | |
252 | ||
253 | /* | |
254 | * Check the assumed register offsets match the actual locations in | |
255 | * the context image. | |
256 | */ | |
257 | ||
258 | for_each_engine(engine, gt, id) { | |
259 | const struct { | |
260 | u32 reg; | |
261 | u32 offset; | |
262 | const char *name; | |
263 | } tbl[] = { | |
264 | { | |
265 | i915_mmio_reg_offset(RING_START(engine->mmio_base)), | |
266 | CTX_RING_START - 1, | |
267 | "RING_START" | |
268 | }, | |
269 | { | |
270 | i915_mmio_reg_offset(RING_CTL(engine->mmio_base)), | |
271 | CTX_RING_CTL - 1, | |
272 | "RING_CTL" | |
273 | }, | |
274 | { | |
275 | i915_mmio_reg_offset(RING_HEAD(engine->mmio_base)), | |
276 | CTX_RING_HEAD - 1, | |
277 | "RING_HEAD" | |
278 | }, | |
279 | { | |
280 | i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)), | |
281 | CTX_RING_TAIL - 1, | |
282 | "RING_TAIL" | |
283 | }, | |
284 | { | |
285 | i915_mmio_reg_offset(RING_MI_MODE(engine->mmio_base)), | |
286 | lrc_ring_mi_mode(engine), | |
287 | "RING_MI_MODE" | |
288 | }, | |
289 | { | |
290 | i915_mmio_reg_offset(RING_BBSTATE(engine->mmio_base)), | |
291 | CTX_BB_STATE - 1, | |
292 | "BB_STATE" | |
293 | }, | |
294 | { | |
295 | i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(engine->mmio_base)), | |
296 | lrc_ring_wa_bb_per_ctx(engine), | |
297 | "RING_BB_PER_CTX_PTR" | |
298 | }, | |
299 | { | |
300 | i915_mmio_reg_offset(RING_INDIRECT_CTX(engine->mmio_base)), | |
301 | lrc_ring_indirect_ptr(engine), | |
302 | "RING_INDIRECT_CTX_PTR" | |
303 | }, | |
304 | { | |
305 | i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(engine->mmio_base)), | |
306 | lrc_ring_indirect_offset(engine), | |
307 | "RING_INDIRECT_CTX_OFFSET" | |
308 | }, | |
309 | { | |
310 | i915_mmio_reg_offset(RING_CTX_TIMESTAMP(engine->mmio_base)), | |
311 | CTX_TIMESTAMP - 1, | |
312 | "RING_CTX_TIMESTAMP" | |
313 | }, | |
314 | { | |
315 | i915_mmio_reg_offset(GEN8_RING_CS_GPR(engine->mmio_base, 0)), | |
316 | lrc_ring_gpr0(engine), | |
317 | "RING_CS_GPR0" | |
318 | }, | |
319 | { | |
320 | i915_mmio_reg_offset(RING_CMD_BUF_CCTL(engine->mmio_base)), | |
321 | lrc_ring_cmd_buf_cctl(engine), | |
322 | "RING_CMD_BUF_CCTL" | |
323 | }, | |
324 | { }, | |
325 | }, *t; | |
326 | u32 *hw; | |
327 | ||
328 | if (!engine->default_state) | |
329 | continue; | |
330 | ||
331 | hw = shmem_pin_map(engine->default_state); | |
332 | if (IS_ERR(hw)) { | |
333 | err = PTR_ERR(hw); | |
334 | break; | |
335 | } | |
336 | hw += LRC_STATE_OFFSET / sizeof(*hw); | |
337 | ||
338 | for (t = tbl; t->name; t++) { | |
339 | int dw = find_offset(hw, t->reg); | |
340 | ||
341 | if (dw != t->offset) { | |
342 | pr_err("%s: Offset for %s [0x%x] mismatch, found %x, expected %x\n", | |
343 | engine->name, | |
344 | t->name, | |
345 | t->reg, | |
346 | dw, | |
347 | t->offset); | |
348 | err = -EINVAL; | |
349 | } | |
350 | } | |
351 | ||
352 | shmem_unpin_map(engine->default_state, hw); | |
353 | } | |
354 | ||
355 | return err; | |
356 | } | |
357 | ||
358 | static int __live_lrc_state(struct intel_engine_cs *engine, | |
359 | struct i915_vma *scratch) | |
360 | { | |
361 | struct intel_context *ce; | |
362 | struct i915_request *rq; | |
363 | struct i915_gem_ww_ctx ww; | |
364 | enum { | |
365 | RING_START_IDX = 0, | |
366 | RING_TAIL_IDX, | |
367 | MAX_IDX | |
368 | }; | |
369 | u32 expected[MAX_IDX]; | |
370 | u32 *cs; | |
371 | int err; | |
372 | int n; | |
373 | ||
374 | ce = intel_context_create(engine); | |
375 | if (IS_ERR(ce)) | |
376 | return PTR_ERR(ce); | |
377 | ||
378 | i915_gem_ww_ctx_init(&ww, false); | |
379 | retry: | |
380 | err = i915_gem_object_lock(scratch->obj, &ww); | |
381 | if (!err) | |
382 | err = intel_context_pin_ww(ce, &ww); | |
383 | if (err) | |
384 | goto err_put; | |
385 | ||
386 | rq = i915_request_create(ce); | |
387 | if (IS_ERR(rq)) { | |
388 | err = PTR_ERR(rq); | |
389 | goto err_unpin; | |
390 | } | |
391 | ||
392 | cs = intel_ring_begin(rq, 4 * MAX_IDX); | |
393 | if (IS_ERR(cs)) { | |
394 | err = PTR_ERR(cs); | |
395 | i915_request_add(rq); | |
396 | goto err_unpin; | |
397 | } | |
398 | ||
399 | *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; | |
400 | *cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base)); | |
401 | *cs++ = i915_ggtt_offset(scratch) + RING_START_IDX * sizeof(u32); | |
402 | *cs++ = 0; | |
403 | ||
404 | expected[RING_START_IDX] = i915_ggtt_offset(ce->ring->vma); | |
405 | ||
406 | *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; | |
407 | *cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)); | |
408 | *cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32); | |
409 | *cs++ = 0; | |
410 | ||
411 | err = i915_request_await_object(rq, scratch->obj, true); | |
412 | if (!err) | |
413 | err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE); | |
414 | ||
415 | i915_request_get(rq); | |
416 | i915_request_add(rq); | |
417 | if (err) | |
418 | goto err_rq; | |
419 | ||
420 | intel_engine_flush_submission(engine); | |
421 | expected[RING_TAIL_IDX] = ce->ring->tail; | |
422 | ||
423 | if (i915_request_wait(rq, 0, HZ / 5) < 0) { | |
424 | err = -ETIME; | |
425 | goto err_rq; | |
426 | } | |
427 | ||
428 | cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB); | |
429 | if (IS_ERR(cs)) { | |
430 | err = PTR_ERR(cs); | |
431 | goto err_rq; | |
432 | } | |
433 | ||
434 | for (n = 0; n < MAX_IDX; n++) { | |
435 | if (cs[n] != expected[n]) { | |
436 | pr_err("%s: Stored register[%d] value[0x%x] did not match expected[0x%x]\n", | |
437 | engine->name, n, cs[n], expected[n]); | |
438 | err = -EINVAL; | |
439 | break; | |
440 | } | |
441 | } | |
442 | ||
443 | i915_gem_object_unpin_map(scratch->obj); | |
444 | ||
445 | err_rq: | |
446 | i915_request_put(rq); | |
447 | err_unpin: | |
448 | intel_context_unpin(ce); | |
449 | err_put: | |
450 | if (err == -EDEADLK) { | |
451 | err = i915_gem_ww_ctx_backoff(&ww); | |
452 | if (!err) | |
453 | goto retry; | |
454 | } | |
455 | i915_gem_ww_ctx_fini(&ww); | |
456 | intel_context_put(ce); | |
457 | return err; | |
458 | } | |
459 | ||
460 | static int live_lrc_state(void *arg) | |
461 | { | |
462 | struct intel_gt *gt = arg; | |
463 | struct intel_engine_cs *engine; | |
464 | struct i915_vma *scratch; | |
465 | enum intel_engine_id id; | |
466 | int err = 0; | |
467 | ||
468 | /* | |
469 | * Check the live register state matches what we expect for this | |
470 | * intel_context. | |
471 | */ | |
472 | ||
473 | scratch = create_scratch(gt); | |
474 | if (IS_ERR(scratch)) | |
475 | return PTR_ERR(scratch); | |
476 | ||
477 | for_each_engine(engine, gt, id) { | |
478 | err = __live_lrc_state(engine, scratch); | |
479 | if (err) | |
480 | break; | |
481 | } | |
482 | ||
483 | if (igt_flush_test(gt->i915)) | |
484 | err = -EIO; | |
485 | ||
486 | i915_vma_unpin_and_release(&scratch, 0); | |
487 | return err; | |
488 | } | |
489 | ||
490 | static int gpr_make_dirty(struct intel_context *ce) | |
491 | { | |
492 | struct i915_request *rq; | |
493 | u32 *cs; | |
494 | int n; | |
495 | ||
496 | rq = intel_context_create_request(ce); | |
497 | if (IS_ERR(rq)) | |
498 | return PTR_ERR(rq); | |
499 | ||
500 | cs = intel_ring_begin(rq, 2 * NUM_GPR_DW + 2); | |
501 | if (IS_ERR(cs)) { | |
502 | i915_request_add(rq); | |
503 | return PTR_ERR(cs); | |
504 | } | |
505 | ||
506 | *cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW); | |
507 | for (n = 0; n < NUM_GPR_DW; n++) { | |
508 | *cs++ = CS_GPR(ce->engine, n); | |
509 | *cs++ = STACK_MAGIC; | |
510 | } | |
511 | *cs++ = MI_NOOP; | |
512 | ||
513 | intel_ring_advance(rq, cs); | |
514 | ||
515 | rq->sched.attr.priority = I915_PRIORITY_BARRIER; | |
516 | i915_request_add(rq); | |
517 | ||
518 | return 0; | |
519 | } | |
520 | ||
521 | static struct i915_request * | |
522 | __gpr_read(struct intel_context *ce, struct i915_vma *scratch, u32 *slot) | |
523 | { | |
524 | const u32 offset = | |
525 | i915_ggtt_offset(ce->engine->status_page.vma) + | |
526 | offset_in_page(slot); | |
527 | struct i915_request *rq; | |
528 | u32 *cs; | |
529 | int err; | |
530 | int n; | |
531 | ||
532 | rq = intel_context_create_request(ce); | |
533 | if (IS_ERR(rq)) | |
534 | return rq; | |
535 | ||
536 | cs = intel_ring_begin(rq, 6 + 4 * NUM_GPR_DW); | |
537 | if (IS_ERR(cs)) { | |
538 | i915_request_add(rq); | |
539 | return ERR_CAST(cs); | |
540 | } | |
541 | ||
542 | *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; | |
543 | *cs++ = MI_NOOP; | |
544 | ||
545 | *cs++ = MI_SEMAPHORE_WAIT | | |
546 | MI_SEMAPHORE_GLOBAL_GTT | | |
547 | MI_SEMAPHORE_POLL | | |
548 | MI_SEMAPHORE_SAD_NEQ_SDD; | |
549 | *cs++ = 0; | |
550 | *cs++ = offset; | |
551 | *cs++ = 0; | |
552 | ||
553 | for (n = 0; n < NUM_GPR_DW; n++) { | |
554 | *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; | |
555 | *cs++ = CS_GPR(ce->engine, n); | |
556 | *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32); | |
557 | *cs++ = 0; | |
558 | } | |
559 | ||
560 | i915_vma_lock(scratch); | |
561 | err = i915_request_await_object(rq, scratch->obj, true); | |
562 | if (!err) | |
563 | err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE); | |
564 | i915_vma_unlock(scratch); | |
565 | ||
566 | i915_request_get(rq); | |
567 | i915_request_add(rq); | |
568 | if (err) { | |
569 | i915_request_put(rq); | |
570 | rq = ERR_PTR(err); | |
571 | } | |
572 | ||
573 | return rq; | |
574 | } | |
575 | ||
576 | static int __live_lrc_gpr(struct intel_engine_cs *engine, | |
577 | struct i915_vma *scratch, | |
578 | bool preempt) | |
579 | { | |
580 | u32 *slot = memset32(engine->status_page.addr + 1000, 0, 4); | |
581 | struct intel_context *ce; | |
582 | struct i915_request *rq; | |
583 | u32 *cs; | |
584 | int err; | |
585 | int n; | |
586 | ||
587 | if (INTEL_GEN(engine->i915) < 9 && engine->class != RENDER_CLASS) | |
588 | return 0; /* GPR only on rcs0 for gen8 */ | |
589 | ||
590 | err = gpr_make_dirty(engine->kernel_context); | |
591 | if (err) | |
592 | return err; | |
593 | ||
594 | ce = intel_context_create(engine); | |
595 | if (IS_ERR(ce)) | |
596 | return PTR_ERR(ce); | |
597 | ||
598 | rq = __gpr_read(ce, scratch, slot); | |
599 | if (IS_ERR(rq)) { | |
600 | err = PTR_ERR(rq); | |
601 | goto err_put; | |
602 | } | |
603 | ||
604 | err = wait_for_submit(engine, rq, HZ / 2); | |
605 | if (err) | |
606 | goto err_rq; | |
607 | ||
608 | if (preempt) { | |
609 | err = gpr_make_dirty(engine->kernel_context); | |
610 | if (err) | |
611 | goto err_rq; | |
612 | ||
613 | err = emit_semaphore_signal(engine->kernel_context, slot); | |
614 | if (err) | |
615 | goto err_rq; | |
9559511b CW |
616 | |
617 | err = wait_for_submit(engine, rq, HZ / 2); | |
618 | if (err) | |
619 | goto err_rq; | |
a0d3fdb6 CW |
620 | } else { |
621 | slot[0] = 1; | |
622 | wmb(); | |
623 | } | |
624 | ||
625 | if (i915_request_wait(rq, 0, HZ / 5) < 0) { | |
626 | err = -ETIME; | |
627 | goto err_rq; | |
628 | } | |
629 | ||
630 | cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB); | |
631 | if (IS_ERR(cs)) { | |
632 | err = PTR_ERR(cs); | |
633 | goto err_rq; | |
634 | } | |
635 | ||
636 | for (n = 0; n < NUM_GPR_DW; n++) { | |
637 | if (cs[n]) { | |
638 | pr_err("%s: GPR[%d].%s was not zero, found 0x%08x!\n", | |
639 | engine->name, | |
640 | n / 2, n & 1 ? "udw" : "ldw", | |
641 | cs[n]); | |
642 | err = -EINVAL; | |
643 | break; | |
644 | } | |
645 | } | |
646 | ||
647 | i915_gem_object_unpin_map(scratch->obj); | |
648 | ||
649 | err_rq: | |
650 | memset32(&slot[0], -1, 4); | |
651 | wmb(); | |
652 | i915_request_put(rq); | |
653 | err_put: | |
654 | intel_context_put(ce); | |
655 | return err; | |
656 | } | |
657 | ||
658 | static int live_lrc_gpr(void *arg) | |
659 | { | |
660 | struct intel_gt *gt = arg; | |
661 | struct intel_engine_cs *engine; | |
662 | struct i915_vma *scratch; | |
663 | enum intel_engine_id id; | |
664 | int err = 0; | |
665 | ||
666 | /* | |
667 | * Check that GPR registers are cleared in new contexts as we need | |
668 | * to avoid leaking any information from previous contexts. | |
669 | */ | |
670 | ||
671 | scratch = create_scratch(gt); | |
672 | if (IS_ERR(scratch)) | |
673 | return PTR_ERR(scratch); | |
674 | ||
675 | for_each_engine(engine, gt, id) { | |
676 | st_engine_heartbeat_disable(engine); | |
677 | ||
678 | err = __live_lrc_gpr(engine, scratch, false); | |
679 | if (err) | |
680 | goto err; | |
681 | ||
682 | err = __live_lrc_gpr(engine, scratch, true); | |
683 | if (err) | |
684 | goto err; | |
685 | ||
686 | err: | |
687 | st_engine_heartbeat_enable(engine); | |
688 | if (igt_flush_test(gt->i915)) | |
689 | err = -EIO; | |
690 | if (err) | |
691 | break; | |
692 | } | |
693 | ||
694 | i915_vma_unpin_and_release(&scratch, 0); | |
695 | return err; | |
696 | } | |
697 | ||
698 | static struct i915_request * | |
699 | create_timestamp(struct intel_context *ce, void *slot, int idx) | |
700 | { | |
701 | const u32 offset = | |
702 | i915_ggtt_offset(ce->engine->status_page.vma) + | |
703 | offset_in_page(slot); | |
704 | struct i915_request *rq; | |
705 | u32 *cs; | |
706 | int err; | |
707 | ||
708 | rq = intel_context_create_request(ce); | |
709 | if (IS_ERR(rq)) | |
710 | return rq; | |
711 | ||
712 | cs = intel_ring_begin(rq, 10); | |
713 | if (IS_ERR(cs)) { | |
714 | err = PTR_ERR(cs); | |
715 | goto err; | |
716 | } | |
717 | ||
718 | *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; | |
719 | *cs++ = MI_NOOP; | |
720 | ||
721 | *cs++ = MI_SEMAPHORE_WAIT | | |
722 | MI_SEMAPHORE_GLOBAL_GTT | | |
723 | MI_SEMAPHORE_POLL | | |
724 | MI_SEMAPHORE_SAD_NEQ_SDD; | |
725 | *cs++ = 0; | |
726 | *cs++ = offset; | |
727 | *cs++ = 0; | |
728 | ||
729 | *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; | |
730 | *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(rq->engine->mmio_base)); | |
731 | *cs++ = offset + idx * sizeof(u32); | |
732 | *cs++ = 0; | |
733 | ||
734 | intel_ring_advance(rq, cs); | |
735 | ||
736 | rq->sched.attr.priority = I915_PRIORITY_MASK; | |
737 | err = 0; | |
738 | err: | |
739 | i915_request_get(rq); | |
740 | i915_request_add(rq); | |
741 | if (err) { | |
742 | i915_request_put(rq); | |
743 | return ERR_PTR(err); | |
744 | } | |
745 | ||
746 | return rq; | |
747 | } | |
748 | ||
749 | struct lrc_timestamp { | |
750 | struct intel_engine_cs *engine; | |
751 | struct intel_context *ce[2]; | |
752 | u32 poison; | |
753 | }; | |
754 | ||
755 | static bool timestamp_advanced(u32 start, u32 end) | |
756 | { | |
757 | return (s32)(end - start) > 0; | |
758 | } | |
759 | ||
760 | static int __lrc_timestamp(const struct lrc_timestamp *arg, bool preempt) | |
761 | { | |
762 | u32 *slot = memset32(arg->engine->status_page.addr + 1000, 0, 4); | |
763 | struct i915_request *rq; | |
764 | u32 timestamp; | |
765 | int err = 0; | |
766 | ||
767 | arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP] = arg->poison; | |
768 | rq = create_timestamp(arg->ce[0], slot, 1); | |
769 | if (IS_ERR(rq)) | |
770 | return PTR_ERR(rq); | |
771 | ||
772 | err = wait_for_submit(rq->engine, rq, HZ / 2); | |
773 | if (err) | |
774 | goto err; | |
775 | ||
776 | if (preempt) { | |
777 | arg->ce[1]->lrc_reg_state[CTX_TIMESTAMP] = 0xdeadbeef; | |
778 | err = emit_semaphore_signal(arg->ce[1], slot); | |
779 | if (err) | |
780 | goto err; | |
781 | } else { | |
782 | slot[0] = 1; | |
783 | wmb(); | |
784 | } | |
785 | ||
786 | /* And wait for switch to kernel (to save our context to memory) */ | |
787 | err = context_flush(arg->ce[0], HZ / 2); | |
788 | if (err) | |
789 | goto err; | |
790 | ||
791 | if (!timestamp_advanced(arg->poison, slot[1])) { | |
792 | pr_err("%s(%s): invalid timestamp on restore, context:%x, request:%x\n", | |
793 | arg->engine->name, preempt ? "preempt" : "simple", | |
794 | arg->poison, slot[1]); | |
795 | err = -EINVAL; | |
796 | } | |
797 | ||
798 | timestamp = READ_ONCE(arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP]); | |
799 | if (!timestamp_advanced(slot[1], timestamp)) { | |
800 | pr_err("%s(%s): invalid timestamp on save, request:%x, context:%x\n", | |
801 | arg->engine->name, preempt ? "preempt" : "simple", | |
802 | slot[1], timestamp); | |
803 | err = -EINVAL; | |
804 | } | |
805 | ||
806 | err: | |
807 | memset32(slot, -1, 4); | |
808 | i915_request_put(rq); | |
809 | return err; | |
810 | } | |
811 | ||
812 | static int live_lrc_timestamp(void *arg) | |
813 | { | |
814 | struct lrc_timestamp data = {}; | |
815 | struct intel_gt *gt = arg; | |
816 | enum intel_engine_id id; | |
817 | const u32 poison[] = { | |
818 | 0, | |
819 | S32_MAX, | |
820 | (u32)S32_MAX + 1, | |
821 | U32_MAX, | |
822 | }; | |
823 | ||
824 | /* | |
825 | * We want to verify that the timestamp is saved and restore across | |
826 | * context switches and is monotonic. | |
827 | * | |
828 | * So we do this with a little bit of LRC poisoning to check various | |
829 | * boundary conditions, and see what happens if we preempt the context | |
830 | * with a second request (carrying more poison into the timestamp). | |
831 | */ | |
832 | ||
833 | for_each_engine(data.engine, gt, id) { | |
834 | int i, err = 0; | |
835 | ||
836 | st_engine_heartbeat_disable(data.engine); | |
837 | ||
838 | for (i = 0; i < ARRAY_SIZE(data.ce); i++) { | |
839 | struct intel_context *tmp; | |
840 | ||
841 | tmp = intel_context_create(data.engine); | |
842 | if (IS_ERR(tmp)) { | |
843 | err = PTR_ERR(tmp); | |
844 | goto err; | |
845 | } | |
846 | ||
847 | err = intel_context_pin(tmp); | |
848 | if (err) { | |
849 | intel_context_put(tmp); | |
850 | goto err; | |
851 | } | |
852 | ||
853 | data.ce[i] = tmp; | |
854 | } | |
855 | ||
856 | for (i = 0; i < ARRAY_SIZE(poison); i++) { | |
857 | data.poison = poison[i]; | |
858 | ||
859 | err = __lrc_timestamp(&data, false); | |
860 | if (err) | |
861 | break; | |
862 | ||
863 | err = __lrc_timestamp(&data, true); | |
864 | if (err) | |
865 | break; | |
866 | } | |
867 | ||
868 | err: | |
869 | st_engine_heartbeat_enable(data.engine); | |
870 | for (i = 0; i < ARRAY_SIZE(data.ce); i++) { | |
871 | if (!data.ce[i]) | |
872 | break; | |
873 | ||
874 | intel_context_unpin(data.ce[i]); | |
875 | intel_context_put(data.ce[i]); | |
876 | } | |
877 | ||
878 | if (igt_flush_test(gt->i915)) | |
879 | err = -EIO; | |
880 | if (err) | |
881 | return err; | |
882 | } | |
883 | ||
884 | return 0; | |
885 | } | |
886 | ||
887 | static struct i915_vma * | |
888 | create_user_vma(struct i915_address_space *vm, unsigned long size) | |
889 | { | |
890 | struct drm_i915_gem_object *obj; | |
891 | struct i915_vma *vma; | |
892 | int err; | |
893 | ||
894 | obj = i915_gem_object_create_internal(vm->i915, size); | |
895 | if (IS_ERR(obj)) | |
896 | return ERR_CAST(obj); | |
897 | ||
898 | vma = i915_vma_instance(obj, vm, NULL); | |
899 | if (IS_ERR(vma)) { | |
900 | i915_gem_object_put(obj); | |
901 | return vma; | |
902 | } | |
903 | ||
904 | err = i915_vma_pin(vma, 0, 0, PIN_USER); | |
905 | if (err) { | |
906 | i915_gem_object_put(obj); | |
907 | return ERR_PTR(err); | |
908 | } | |
909 | ||
910 | return vma; | |
911 | } | |
912 | ||
913 | static struct i915_vma * | |
914 | store_context(struct intel_context *ce, struct i915_vma *scratch) | |
915 | { | |
916 | struct i915_vma *batch; | |
917 | u32 dw, x, *cs, *hw; | |
918 | u32 *defaults; | |
919 | ||
920 | batch = create_user_vma(ce->vm, SZ_64K); | |
921 | if (IS_ERR(batch)) | |
922 | return batch; | |
923 | ||
924 | cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC); | |
925 | if (IS_ERR(cs)) { | |
926 | i915_vma_put(batch); | |
927 | return ERR_CAST(cs); | |
928 | } | |
929 | ||
930 | defaults = shmem_pin_map(ce->engine->default_state); | |
931 | if (!defaults) { | |
932 | i915_gem_object_unpin_map(batch->obj); | |
933 | i915_vma_put(batch); | |
934 | return ERR_PTR(-ENOMEM); | |
935 | } | |
936 | ||
937 | x = 0; | |
938 | dw = 0; | |
939 | hw = defaults; | |
940 | hw += LRC_STATE_OFFSET / sizeof(*hw); | |
941 | do { | |
942 | u32 len = hw[dw] & 0x7f; | |
943 | ||
944 | if (hw[dw] == 0) { | |
945 | dw++; | |
946 | continue; | |
947 | } | |
948 | ||
949 | if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { | |
950 | dw += len + 2; | |
951 | continue; | |
952 | } | |
953 | ||
954 | dw++; | |
955 | len = (len + 1) / 2; | |
956 | while (len--) { | |
957 | *cs++ = MI_STORE_REGISTER_MEM_GEN8; | |
958 | *cs++ = hw[dw]; | |
959 | *cs++ = lower_32_bits(scratch->node.start + x); | |
960 | *cs++ = upper_32_bits(scratch->node.start + x); | |
961 | ||
962 | dw += 2; | |
963 | x += 4; | |
964 | } | |
965 | } while (dw < PAGE_SIZE / sizeof(u32) && | |
966 | (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); | |
967 | ||
968 | *cs++ = MI_BATCH_BUFFER_END; | |
969 | ||
970 | shmem_unpin_map(ce->engine->default_state, defaults); | |
971 | ||
972 | i915_gem_object_flush_map(batch->obj); | |
973 | i915_gem_object_unpin_map(batch->obj); | |
974 | ||
975 | return batch; | |
976 | } | |
977 | ||
978 | static int move_to_active(struct i915_request *rq, | |
979 | struct i915_vma *vma, | |
980 | unsigned int flags) | |
981 | { | |
982 | int err; | |
983 | ||
984 | i915_vma_lock(vma); | |
985 | err = i915_request_await_object(rq, vma->obj, flags); | |
986 | if (!err) | |
987 | err = i915_vma_move_to_active(vma, rq, flags); | |
988 | i915_vma_unlock(vma); | |
989 | ||
990 | return err; | |
991 | } | |
992 | ||
993 | static struct i915_request * | |
994 | record_registers(struct intel_context *ce, | |
995 | struct i915_vma *before, | |
996 | struct i915_vma *after, | |
997 | u32 *sema) | |
998 | { | |
999 | struct i915_vma *b_before, *b_after; | |
1000 | struct i915_request *rq; | |
1001 | u32 *cs; | |
1002 | int err; | |
1003 | ||
1004 | b_before = store_context(ce, before); | |
1005 | if (IS_ERR(b_before)) | |
1006 | return ERR_CAST(b_before); | |
1007 | ||
1008 | b_after = store_context(ce, after); | |
1009 | if (IS_ERR(b_after)) { | |
1010 | rq = ERR_CAST(b_after); | |
1011 | goto err_before; | |
1012 | } | |
1013 | ||
1014 | rq = intel_context_create_request(ce); | |
1015 | if (IS_ERR(rq)) | |
1016 | goto err_after; | |
1017 | ||
1018 | err = move_to_active(rq, before, EXEC_OBJECT_WRITE); | |
1019 | if (err) | |
1020 | goto err_rq; | |
1021 | ||
1022 | err = move_to_active(rq, b_before, 0); | |
1023 | if (err) | |
1024 | goto err_rq; | |
1025 | ||
1026 | err = move_to_active(rq, after, EXEC_OBJECT_WRITE); | |
1027 | if (err) | |
1028 | goto err_rq; | |
1029 | ||
1030 | err = move_to_active(rq, b_after, 0); | |
1031 | if (err) | |
1032 | goto err_rq; | |
1033 | ||
1034 | cs = intel_ring_begin(rq, 14); | |
1035 | if (IS_ERR(cs)) { | |
1036 | err = PTR_ERR(cs); | |
1037 | goto err_rq; | |
1038 | } | |
1039 | ||
1040 | *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; | |
1041 | *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8); | |
1042 | *cs++ = lower_32_bits(b_before->node.start); | |
1043 | *cs++ = upper_32_bits(b_before->node.start); | |
1044 | ||
1045 | *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; | |
1046 | *cs++ = MI_SEMAPHORE_WAIT | | |
1047 | MI_SEMAPHORE_GLOBAL_GTT | | |
1048 | MI_SEMAPHORE_POLL | | |
1049 | MI_SEMAPHORE_SAD_NEQ_SDD; | |
1050 | *cs++ = 0; | |
1051 | *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) + | |
1052 | offset_in_page(sema); | |
1053 | *cs++ = 0; | |
1054 | *cs++ = MI_NOOP; | |
1055 | ||
1056 | *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; | |
1057 | *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8); | |
1058 | *cs++ = lower_32_bits(b_after->node.start); | |
1059 | *cs++ = upper_32_bits(b_after->node.start); | |
1060 | ||
1061 | intel_ring_advance(rq, cs); | |
1062 | ||
1063 | WRITE_ONCE(*sema, 0); | |
1064 | i915_request_get(rq); | |
1065 | i915_request_add(rq); | |
1066 | err_after: | |
1067 | i915_vma_put(b_after); | |
1068 | err_before: | |
1069 | i915_vma_put(b_before); | |
1070 | return rq; | |
1071 | ||
1072 | err_rq: | |
1073 | i915_request_add(rq); | |
1074 | rq = ERR_PTR(err); | |
1075 | goto err_after; | |
1076 | } | |
1077 | ||
1078 | static struct i915_vma *load_context(struct intel_context *ce, u32 poison) | |
1079 | { | |
1080 | struct i915_vma *batch; | |
1081 | u32 dw, *cs, *hw; | |
1082 | u32 *defaults; | |
1083 | ||
1084 | batch = create_user_vma(ce->vm, SZ_64K); | |
1085 | if (IS_ERR(batch)) | |
1086 | return batch; | |
1087 | ||
1088 | cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC); | |
1089 | if (IS_ERR(cs)) { | |
1090 | i915_vma_put(batch); | |
1091 | return ERR_CAST(cs); | |
1092 | } | |
1093 | ||
1094 | defaults = shmem_pin_map(ce->engine->default_state); | |
1095 | if (!defaults) { | |
1096 | i915_gem_object_unpin_map(batch->obj); | |
1097 | i915_vma_put(batch); | |
1098 | return ERR_PTR(-ENOMEM); | |
1099 | } | |
1100 | ||
1101 | dw = 0; | |
1102 | hw = defaults; | |
1103 | hw += LRC_STATE_OFFSET / sizeof(*hw); | |
1104 | do { | |
1105 | u32 len = hw[dw] & 0x7f; | |
1106 | ||
1107 | if (hw[dw] == 0) { | |
1108 | dw++; | |
1109 | continue; | |
1110 | } | |
1111 | ||
1112 | if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { | |
1113 | dw += len + 2; | |
1114 | continue; | |
1115 | } | |
1116 | ||
1117 | dw++; | |
1118 | len = (len + 1) / 2; | |
1119 | *cs++ = MI_LOAD_REGISTER_IMM(len); | |
1120 | while (len--) { | |
1121 | *cs++ = hw[dw]; | |
1122 | *cs++ = poison; | |
1123 | dw += 2; | |
1124 | } | |
1125 | } while (dw < PAGE_SIZE / sizeof(u32) && | |
1126 | (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); | |
1127 | ||
1128 | *cs++ = MI_BATCH_BUFFER_END; | |
1129 | ||
1130 | shmem_unpin_map(ce->engine->default_state, defaults); | |
1131 | ||
1132 | i915_gem_object_flush_map(batch->obj); | |
1133 | i915_gem_object_unpin_map(batch->obj); | |
1134 | ||
1135 | return batch; | |
1136 | } | |
1137 | ||
1138 | static int poison_registers(struct intel_context *ce, u32 poison, u32 *sema) | |
1139 | { | |
1140 | struct i915_request *rq; | |
1141 | struct i915_vma *batch; | |
1142 | u32 *cs; | |
1143 | int err; | |
1144 | ||
1145 | batch = load_context(ce, poison); | |
1146 | if (IS_ERR(batch)) | |
1147 | return PTR_ERR(batch); | |
1148 | ||
1149 | rq = intel_context_create_request(ce); | |
1150 | if (IS_ERR(rq)) { | |
1151 | err = PTR_ERR(rq); | |
1152 | goto err_batch; | |
1153 | } | |
1154 | ||
1155 | err = move_to_active(rq, batch, 0); | |
1156 | if (err) | |
1157 | goto err_rq; | |
1158 | ||
1159 | cs = intel_ring_begin(rq, 8); | |
1160 | if (IS_ERR(cs)) { | |
1161 | err = PTR_ERR(cs); | |
1162 | goto err_rq; | |
1163 | } | |
1164 | ||
1165 | *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; | |
1166 | *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8); | |
1167 | *cs++ = lower_32_bits(batch->node.start); | |
1168 | *cs++ = upper_32_bits(batch->node.start); | |
1169 | ||
1170 | *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; | |
1171 | *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) + | |
1172 | offset_in_page(sema); | |
1173 | *cs++ = 0; | |
1174 | *cs++ = 1; | |
1175 | ||
1176 | intel_ring_advance(rq, cs); | |
1177 | ||
1178 | rq->sched.attr.priority = I915_PRIORITY_BARRIER; | |
1179 | err_rq: | |
1180 | i915_request_add(rq); | |
1181 | err_batch: | |
1182 | i915_vma_put(batch); | |
1183 | return err; | |
1184 | } | |
1185 | ||
1186 | static bool is_moving(u32 a, u32 b) | |
1187 | { | |
1188 | return a != b; | |
1189 | } | |
1190 | ||
1191 | static int compare_isolation(struct intel_engine_cs *engine, | |
1192 | struct i915_vma *ref[2], | |
1193 | struct i915_vma *result[2], | |
1194 | struct intel_context *ce, | |
1195 | u32 poison) | |
1196 | { | |
1197 | u32 x, dw, *hw, *lrc; | |
1198 | u32 *A[2], *B[2]; | |
1199 | u32 *defaults; | |
1200 | int err = 0; | |
1201 | ||
1202 | A[0] = i915_gem_object_pin_map(ref[0]->obj, I915_MAP_WC); | |
1203 | if (IS_ERR(A[0])) | |
1204 | return PTR_ERR(A[0]); | |
1205 | ||
1206 | A[1] = i915_gem_object_pin_map(ref[1]->obj, I915_MAP_WC); | |
1207 | if (IS_ERR(A[1])) { | |
1208 | err = PTR_ERR(A[1]); | |
1209 | goto err_A0; | |
1210 | } | |
1211 | ||
1212 | B[0] = i915_gem_object_pin_map(result[0]->obj, I915_MAP_WC); | |
1213 | if (IS_ERR(B[0])) { | |
1214 | err = PTR_ERR(B[0]); | |
1215 | goto err_A1; | |
1216 | } | |
1217 | ||
1218 | B[1] = i915_gem_object_pin_map(result[1]->obj, I915_MAP_WC); | |
1219 | if (IS_ERR(B[1])) { | |
1220 | err = PTR_ERR(B[1]); | |
1221 | goto err_B0; | |
1222 | } | |
1223 | ||
1224 | lrc = i915_gem_object_pin_map(ce->state->obj, | |
1225 | i915_coherent_map_type(engine->i915)); | |
1226 | if (IS_ERR(lrc)) { | |
1227 | err = PTR_ERR(lrc); | |
1228 | goto err_B1; | |
1229 | } | |
1230 | lrc += LRC_STATE_OFFSET / sizeof(*hw); | |
1231 | ||
1232 | defaults = shmem_pin_map(ce->engine->default_state); | |
1233 | if (!defaults) { | |
1234 | err = -ENOMEM; | |
1235 | goto err_lrc; | |
1236 | } | |
1237 | ||
1238 | x = 0; | |
1239 | dw = 0; | |
1240 | hw = defaults; | |
1241 | hw += LRC_STATE_OFFSET / sizeof(*hw); | |
1242 | do { | |
1243 | u32 len = hw[dw] & 0x7f; | |
1244 | ||
1245 | if (hw[dw] == 0) { | |
1246 | dw++; | |
1247 | continue; | |
1248 | } | |
1249 | ||
1250 | if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { | |
1251 | dw += len + 2; | |
1252 | continue; | |
1253 | } | |
1254 | ||
1255 | dw++; | |
1256 | len = (len + 1) / 2; | |
1257 | while (len--) { | |
1258 | if (!is_moving(A[0][x], A[1][x]) && | |
1259 | (A[0][x] != B[0][x] || A[1][x] != B[1][x])) { | |
1260 | switch (hw[dw] & 4095) { | |
1261 | case 0x30: /* RING_HEAD */ | |
1262 | case 0x34: /* RING_TAIL */ | |
1263 | break; | |
1264 | ||
1265 | default: | |
1266 | pr_err("%s[%d]: Mismatch for register %4x, default %08x, reference %08x, result (%08x, %08x), poison %08x, context %08x\n", | |
1267 | engine->name, dw, | |
1268 | hw[dw], hw[dw + 1], | |
1269 | A[0][x], B[0][x], B[1][x], | |
1270 | poison, lrc[dw + 1]); | |
1271 | err = -EINVAL; | |
1272 | } | |
1273 | } | |
1274 | dw += 2; | |
1275 | x++; | |
1276 | } | |
1277 | } while (dw < PAGE_SIZE / sizeof(u32) && | |
1278 | (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); | |
1279 | ||
1280 | shmem_unpin_map(ce->engine->default_state, defaults); | |
1281 | err_lrc: | |
1282 | i915_gem_object_unpin_map(ce->state->obj); | |
1283 | err_B1: | |
1284 | i915_gem_object_unpin_map(result[1]->obj); | |
1285 | err_B0: | |
1286 | i915_gem_object_unpin_map(result[0]->obj); | |
1287 | err_A1: | |
1288 | i915_gem_object_unpin_map(ref[1]->obj); | |
1289 | err_A0: | |
1290 | i915_gem_object_unpin_map(ref[0]->obj); | |
1291 | return err; | |
1292 | } | |
1293 | ||
1294 | static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison) | |
1295 | { | |
1296 | u32 *sema = memset32(engine->status_page.addr + 1000, 0, 1); | |
1297 | struct i915_vma *ref[2], *result[2]; | |
1298 | struct intel_context *A, *B; | |
1299 | struct i915_request *rq; | |
1300 | int err; | |
1301 | ||
1302 | A = intel_context_create(engine); | |
1303 | if (IS_ERR(A)) | |
1304 | return PTR_ERR(A); | |
1305 | ||
1306 | B = intel_context_create(engine); | |
1307 | if (IS_ERR(B)) { | |
1308 | err = PTR_ERR(B); | |
1309 | goto err_A; | |
1310 | } | |
1311 | ||
1312 | ref[0] = create_user_vma(A->vm, SZ_64K); | |
1313 | if (IS_ERR(ref[0])) { | |
1314 | err = PTR_ERR(ref[0]); | |
1315 | goto err_B; | |
1316 | } | |
1317 | ||
1318 | ref[1] = create_user_vma(A->vm, SZ_64K); | |
1319 | if (IS_ERR(ref[1])) { | |
1320 | err = PTR_ERR(ref[1]); | |
1321 | goto err_ref0; | |
1322 | } | |
1323 | ||
1324 | rq = record_registers(A, ref[0], ref[1], sema); | |
1325 | if (IS_ERR(rq)) { | |
1326 | err = PTR_ERR(rq); | |
1327 | goto err_ref1; | |
1328 | } | |
1329 | ||
1330 | WRITE_ONCE(*sema, 1); | |
1331 | wmb(); | |
1332 | ||
1333 | if (i915_request_wait(rq, 0, HZ / 2) < 0) { | |
1334 | i915_request_put(rq); | |
1335 | err = -ETIME; | |
1336 | goto err_ref1; | |
1337 | } | |
1338 | i915_request_put(rq); | |
1339 | ||
1340 | result[0] = create_user_vma(A->vm, SZ_64K); | |
1341 | if (IS_ERR(result[0])) { | |
1342 | err = PTR_ERR(result[0]); | |
1343 | goto err_ref1; | |
1344 | } | |
1345 | ||
1346 | result[1] = create_user_vma(A->vm, SZ_64K); | |
1347 | if (IS_ERR(result[1])) { | |
1348 | err = PTR_ERR(result[1]); | |
1349 | goto err_result0; | |
1350 | } | |
1351 | ||
1352 | rq = record_registers(A, result[0], result[1], sema); | |
1353 | if (IS_ERR(rq)) { | |
1354 | err = PTR_ERR(rq); | |
1355 | goto err_result1; | |
1356 | } | |
1357 | ||
1358 | err = poison_registers(B, poison, sema); | |
1359 | if (err) { | |
1360 | WRITE_ONCE(*sema, -1); | |
1361 | i915_request_put(rq); | |
1362 | goto err_result1; | |
1363 | } | |
1364 | ||
1365 | if (i915_request_wait(rq, 0, HZ / 2) < 0) { | |
1366 | i915_request_put(rq); | |
1367 | err = -ETIME; | |
1368 | goto err_result1; | |
1369 | } | |
1370 | i915_request_put(rq); | |
1371 | ||
1372 | err = compare_isolation(engine, ref, result, A, poison); | |
1373 | ||
1374 | err_result1: | |
1375 | i915_vma_put(result[1]); | |
1376 | err_result0: | |
1377 | i915_vma_put(result[0]); | |
1378 | err_ref1: | |
1379 | i915_vma_put(ref[1]); | |
1380 | err_ref0: | |
1381 | i915_vma_put(ref[0]); | |
1382 | err_B: | |
1383 | intel_context_put(B); | |
1384 | err_A: | |
1385 | intel_context_put(A); | |
1386 | return err; | |
1387 | } | |
1388 | ||
1389 | static bool skip_isolation(const struct intel_engine_cs *engine) | |
1390 | { | |
1391 | if (engine->class == COPY_ENGINE_CLASS && INTEL_GEN(engine->i915) == 9) | |
1392 | return true; | |
1393 | ||
1394 | if (engine->class == RENDER_CLASS && INTEL_GEN(engine->i915) == 11) | |
1395 | return true; | |
1396 | ||
1397 | return false; | |
1398 | } | |
1399 | ||
1400 | static int live_lrc_isolation(void *arg) | |
1401 | { | |
1402 | struct intel_gt *gt = arg; | |
1403 | struct intel_engine_cs *engine; | |
1404 | enum intel_engine_id id; | |
1405 | const u32 poison[] = { | |
1406 | STACK_MAGIC, | |
1407 | 0x3a3a3a3a, | |
1408 | 0x5c5c5c5c, | |
1409 | 0xffffffff, | |
1410 | 0xffff0000, | |
1411 | }; | |
1412 | int err = 0; | |
1413 | ||
1414 | /* | |
1415 | * Our goal is try and verify that per-context state cannot be | |
1416 | * tampered with by another non-privileged client. | |
1417 | * | |
1418 | * We take the list of context registers from the LRI in the default | |
1419 | * context image and attempt to modify that list from a remote context. | |
1420 | */ | |
1421 | ||
1422 | for_each_engine(engine, gt, id) { | |
1423 | int i; | |
1424 | ||
1425 | /* Just don't even ask */ | |
1426 | if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN) && | |
1427 | skip_isolation(engine)) | |
1428 | continue; | |
1429 | ||
1430 | intel_engine_pm_get(engine); | |
1431 | for (i = 0; i < ARRAY_SIZE(poison); i++) { | |
1432 | int result; | |
1433 | ||
1434 | result = __lrc_isolation(engine, poison[i]); | |
1435 | if (result && !err) | |
1436 | err = result; | |
1437 | ||
1438 | result = __lrc_isolation(engine, ~poison[i]); | |
1439 | if (result && !err) | |
1440 | err = result; | |
1441 | } | |
1442 | intel_engine_pm_put(engine); | |
1443 | if (igt_flush_test(gt->i915)) { | |
1444 | err = -EIO; | |
1445 | break; | |
1446 | } | |
1447 | } | |
1448 | ||
1449 | return err; | |
1450 | } | |
1451 | ||
1452 | static int indirect_ctx_submit_req(struct intel_context *ce) | |
1453 | { | |
1454 | struct i915_request *rq; | |
1455 | int err = 0; | |
1456 | ||
1457 | rq = intel_context_create_request(ce); | |
1458 | if (IS_ERR(rq)) | |
1459 | return PTR_ERR(rq); | |
1460 | ||
1461 | i915_request_get(rq); | |
1462 | i915_request_add(rq); | |
1463 | ||
1464 | if (i915_request_wait(rq, 0, HZ / 5) < 0) | |
1465 | err = -ETIME; | |
1466 | ||
1467 | i915_request_put(rq); | |
1468 | ||
1469 | return err; | |
1470 | } | |
1471 | ||
1472 | #define CTX_BB_CANARY_OFFSET (3 * 1024) | |
1473 | #define CTX_BB_CANARY_INDEX (CTX_BB_CANARY_OFFSET / sizeof(u32)) | |
1474 | ||
1475 | static u32 * | |
1476 | emit_indirect_ctx_bb_canary(const struct intel_context *ce, u32 *cs) | |
1477 | { | |
1478 | *cs++ = MI_STORE_REGISTER_MEM_GEN8 | | |
1479 | MI_SRM_LRM_GLOBAL_GTT | | |
1480 | MI_LRI_LRM_CS_MMIO; | |
1481 | *cs++ = i915_mmio_reg_offset(RING_START(0)); | |
1482 | *cs++ = i915_ggtt_offset(ce->state) + | |
1483 | context_wa_bb_offset(ce) + | |
1484 | CTX_BB_CANARY_OFFSET; | |
1485 | *cs++ = 0; | |
1486 | ||
1487 | return cs; | |
1488 | } | |
1489 | ||
1490 | static void | |
1491 | indirect_ctx_bb_setup(struct intel_context *ce) | |
1492 | { | |
1493 | u32 *cs = context_indirect_bb(ce); | |
1494 | ||
1495 | cs[CTX_BB_CANARY_INDEX] = 0xdeadf00d; | |
1496 | ||
1497 | setup_indirect_ctx_bb(ce, ce->engine, emit_indirect_ctx_bb_canary); | |
1498 | } | |
1499 | ||
1500 | static bool check_ring_start(struct intel_context *ce) | |
1501 | { | |
1502 | const u32 * const ctx_bb = (void *)(ce->lrc_reg_state) - | |
1503 | LRC_STATE_OFFSET + context_wa_bb_offset(ce); | |
1504 | ||
1505 | if (ctx_bb[CTX_BB_CANARY_INDEX] == ce->lrc_reg_state[CTX_RING_START]) | |
1506 | return true; | |
1507 | ||
1508 | pr_err("ring start mismatch: canary 0x%08x vs state 0x%08x\n", | |
1509 | ctx_bb[CTX_BB_CANARY_INDEX], | |
1510 | ce->lrc_reg_state[CTX_RING_START]); | |
1511 | ||
1512 | return false; | |
1513 | } | |
1514 | ||
1515 | static int indirect_ctx_bb_check(struct intel_context *ce) | |
1516 | { | |
1517 | int err; | |
1518 | ||
1519 | err = indirect_ctx_submit_req(ce); | |
1520 | if (err) | |
1521 | return err; | |
1522 | ||
1523 | if (!check_ring_start(ce)) | |
1524 | return -EINVAL; | |
1525 | ||
1526 | return 0; | |
1527 | } | |
1528 | ||
1529 | static int __live_lrc_indirect_ctx_bb(struct intel_engine_cs *engine) | |
1530 | { | |
1531 | struct intel_context *a, *b; | |
1532 | int err; | |
1533 | ||
1534 | a = intel_context_create(engine); | |
1535 | if (IS_ERR(a)) | |
1536 | return PTR_ERR(a); | |
1537 | err = intel_context_pin(a); | |
1538 | if (err) | |
1539 | goto put_a; | |
1540 | ||
1541 | b = intel_context_create(engine); | |
1542 | if (IS_ERR(b)) { | |
1543 | err = PTR_ERR(b); | |
1544 | goto unpin_a; | |
1545 | } | |
1546 | err = intel_context_pin(b); | |
1547 | if (err) | |
1548 | goto put_b; | |
1549 | ||
1550 | /* We use the already reserved extra page in context state */ | |
1551 | if (!a->wa_bb_page) { | |
1552 | GEM_BUG_ON(b->wa_bb_page); | |
1553 | GEM_BUG_ON(INTEL_GEN(engine->i915) == 12); | |
1554 | goto unpin_b; | |
1555 | } | |
1556 | ||
1557 | /* | |
1558 | * In order to test that our per context bb is truly per context, | |
1559 | * and executes at the intended spot on context restoring process, | |
1560 | * make the batch store the ring start value to memory. | |
1561 | * As ring start is restored apriori of starting the indirect ctx bb and | |
1562 | * as it will be different for each context, it fits to this purpose. | |
1563 | */ | |
1564 | indirect_ctx_bb_setup(a); | |
1565 | indirect_ctx_bb_setup(b); | |
1566 | ||
1567 | err = indirect_ctx_bb_check(a); | |
1568 | if (err) | |
1569 | goto unpin_b; | |
1570 | ||
1571 | err = indirect_ctx_bb_check(b); | |
1572 | ||
1573 | unpin_b: | |
1574 | intel_context_unpin(b); | |
1575 | put_b: | |
1576 | intel_context_put(b); | |
1577 | unpin_a: | |
1578 | intel_context_unpin(a); | |
1579 | put_a: | |
1580 | intel_context_put(a); | |
1581 | ||
1582 | return err; | |
1583 | } | |
1584 | ||
1585 | static int live_lrc_indirect_ctx_bb(void *arg) | |
1586 | { | |
1587 | struct intel_gt *gt = arg; | |
1588 | struct intel_engine_cs *engine; | |
1589 | enum intel_engine_id id; | |
1590 | int err = 0; | |
1591 | ||
1592 | for_each_engine(engine, gt, id) { | |
1593 | intel_engine_pm_get(engine); | |
1594 | err = __live_lrc_indirect_ctx_bb(engine); | |
1595 | intel_engine_pm_put(engine); | |
1596 | ||
1597 | if (igt_flush_test(gt->i915)) | |
1598 | err = -EIO; | |
1599 | ||
1600 | if (err) | |
1601 | break; | |
1602 | } | |
1603 | ||
1604 | return err; | |
1605 | } | |
1606 | ||
1607 | static void garbage_reset(struct intel_engine_cs *engine, | |
1608 | struct i915_request *rq) | |
1609 | { | |
1610 | const unsigned int bit = I915_RESET_ENGINE + engine->id; | |
1611 | unsigned long *lock = &engine->gt->reset.flags; | |
1612 | ||
16f2941a CW |
1613 | local_bh_disable(); |
1614 | if (!test_and_set_bit(bit, lock)) { | |
1615 | tasklet_disable(&engine->execlists.tasklet); | |
a0d3fdb6 | 1616 | |
16f2941a CW |
1617 | if (!rq->fence.error) |
1618 | __intel_engine_reset_bh(engine, NULL); | |
a0d3fdb6 | 1619 | |
16f2941a CW |
1620 | tasklet_enable(&engine->execlists.tasklet); |
1621 | clear_and_wake_up_bit(bit, lock); | |
1622 | } | |
1623 | local_bh_enable(); | |
a0d3fdb6 CW |
1624 | } |
1625 | ||
1626 | static struct i915_request *garbage(struct intel_context *ce, | |
1627 | struct rnd_state *prng) | |
1628 | { | |
1629 | struct i915_request *rq; | |
1630 | int err; | |
1631 | ||
1632 | err = intel_context_pin(ce); | |
1633 | if (err) | |
1634 | return ERR_PTR(err); | |
1635 | ||
1636 | prandom_bytes_state(prng, | |
1637 | ce->lrc_reg_state, | |
1638 | ce->engine->context_size - | |
1639 | LRC_STATE_OFFSET); | |
1640 | ||
1641 | rq = intel_context_create_request(ce); | |
1642 | if (IS_ERR(rq)) { | |
1643 | err = PTR_ERR(rq); | |
1644 | goto err_unpin; | |
1645 | } | |
1646 | ||
1647 | i915_request_get(rq); | |
1648 | i915_request_add(rq); | |
1649 | return rq; | |
1650 | ||
1651 | err_unpin: | |
1652 | intel_context_unpin(ce); | |
1653 | return ERR_PTR(err); | |
1654 | } | |
1655 | ||
1656 | static int __lrc_garbage(struct intel_engine_cs *engine, struct rnd_state *prng) | |
1657 | { | |
1658 | struct intel_context *ce; | |
1659 | struct i915_request *hang; | |
1660 | int err = 0; | |
1661 | ||
1662 | ce = intel_context_create(engine); | |
1663 | if (IS_ERR(ce)) | |
1664 | return PTR_ERR(ce); | |
1665 | ||
1666 | hang = garbage(ce, prng); | |
1667 | if (IS_ERR(hang)) { | |
1668 | err = PTR_ERR(hang); | |
1669 | goto err_ce; | |
1670 | } | |
1671 | ||
1672 | if (wait_for_submit(engine, hang, HZ / 2)) { | |
1673 | i915_request_put(hang); | |
1674 | err = -ETIME; | |
1675 | goto err_ce; | |
1676 | } | |
1677 | ||
1678 | intel_context_set_banned(ce); | |
1679 | garbage_reset(engine, hang); | |
1680 | ||
1681 | intel_engine_flush_submission(engine); | |
1682 | if (!hang->fence.error) { | |
1683 | i915_request_put(hang); | |
1684 | pr_err("%s: corrupted context was not reset\n", | |
1685 | engine->name); | |
1686 | err = -EINVAL; | |
1687 | goto err_ce; | |
1688 | } | |
1689 | ||
1690 | if (i915_request_wait(hang, 0, HZ / 2) < 0) { | |
1691 | pr_err("%s: corrupted context did not recover\n", | |
1692 | engine->name); | |
1693 | i915_request_put(hang); | |
1694 | err = -EIO; | |
1695 | goto err_ce; | |
1696 | } | |
1697 | i915_request_put(hang); | |
1698 | ||
1699 | err_ce: | |
1700 | intel_context_put(ce); | |
1701 | return err; | |
1702 | } | |
1703 | ||
1704 | static int live_lrc_garbage(void *arg) | |
1705 | { | |
1706 | struct intel_gt *gt = arg; | |
1707 | struct intel_engine_cs *engine; | |
1708 | enum intel_engine_id id; | |
1709 | ||
1710 | /* | |
1711 | * Verify that we can recover if one context state is completely | |
1712 | * corrupted. | |
1713 | */ | |
1714 | ||
1715 | if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN)) | |
1716 | return 0; | |
1717 | ||
1718 | for_each_engine(engine, gt, id) { | |
1719 | I915_RND_STATE(prng); | |
1720 | int err = 0, i; | |
1721 | ||
1722 | if (!intel_has_reset_engine(engine->gt)) | |
1723 | continue; | |
1724 | ||
1725 | intel_engine_pm_get(engine); | |
1726 | for (i = 0; i < 3; i++) { | |
1727 | err = __lrc_garbage(engine, &prng); | |
1728 | if (err) | |
1729 | break; | |
1730 | } | |
1731 | intel_engine_pm_put(engine); | |
1732 | ||
1733 | if (igt_flush_test(gt->i915)) | |
1734 | err = -EIO; | |
1735 | if (err) | |
1736 | return err; | |
1737 | } | |
1738 | ||
1739 | return 0; | |
1740 | } | |
1741 | ||
1742 | static int __live_pphwsp_runtime(struct intel_engine_cs *engine) | |
1743 | { | |
1744 | struct intel_context *ce; | |
1745 | struct i915_request *rq; | |
1746 | IGT_TIMEOUT(end_time); | |
1747 | int err; | |
1748 | ||
1749 | ce = intel_context_create(engine); | |
1750 | if (IS_ERR(ce)) | |
1751 | return PTR_ERR(ce); | |
1752 | ||
1753 | ce->runtime.num_underflow = 0; | |
1754 | ce->runtime.max_underflow = 0; | |
1755 | ||
1756 | do { | |
1757 | unsigned int loop = 1024; | |
1758 | ||
1759 | while (loop) { | |
1760 | rq = intel_context_create_request(ce); | |
1761 | if (IS_ERR(rq)) { | |
1762 | err = PTR_ERR(rq); | |
1763 | goto err_rq; | |
1764 | } | |
1765 | ||
1766 | if (--loop == 0) | |
1767 | i915_request_get(rq); | |
1768 | ||
1769 | i915_request_add(rq); | |
1770 | } | |
1771 | ||
1772 | if (__igt_timeout(end_time, NULL)) | |
1773 | break; | |
1774 | ||
1775 | i915_request_put(rq); | |
1776 | } while (1); | |
1777 | ||
1778 | err = i915_request_wait(rq, 0, HZ / 5); | |
1779 | if (err < 0) { | |
1780 | pr_err("%s: request not completed!\n", engine->name); | |
1781 | goto err_wait; | |
1782 | } | |
1783 | ||
1784 | igt_flush_test(engine->i915); | |
1785 | ||
1786 | pr_info("%s: pphwsp runtime %lluns, average %lluns\n", | |
1787 | engine->name, | |
1788 | intel_context_get_total_runtime_ns(ce), | |
1789 | intel_context_get_avg_runtime_ns(ce)); | |
1790 | ||
1791 | err = 0; | |
1792 | if (ce->runtime.num_underflow) { | |
1793 | pr_err("%s: pphwsp underflow %u time(s), max %u cycles!\n", | |
1794 | engine->name, | |
1795 | ce->runtime.num_underflow, | |
1796 | ce->runtime.max_underflow); | |
1797 | GEM_TRACE_DUMP(); | |
1798 | err = -EOVERFLOW; | |
1799 | } | |
1800 | ||
1801 | err_wait: | |
1802 | i915_request_put(rq); | |
1803 | err_rq: | |
1804 | intel_context_put(ce); | |
1805 | return err; | |
1806 | } | |
1807 | ||
1808 | static int live_pphwsp_runtime(void *arg) | |
1809 | { | |
1810 | struct intel_gt *gt = arg; | |
1811 | struct intel_engine_cs *engine; | |
1812 | enum intel_engine_id id; | |
1813 | int err = 0; | |
1814 | ||
1815 | /* | |
1816 | * Check that cumulative context runtime as stored in the pphwsp[16] | |
1817 | * is monotonic. | |
1818 | */ | |
1819 | ||
1820 | for_each_engine(engine, gt, id) { | |
1821 | err = __live_pphwsp_runtime(engine); | |
1822 | if (err) | |
1823 | break; | |
1824 | } | |
1825 | ||
1826 | if (igt_flush_test(gt->i915)) | |
1827 | err = -EIO; | |
1828 | ||
1829 | return err; | |
1830 | } | |
1831 | ||
1832 | int intel_lrc_live_selftests(struct drm_i915_private *i915) | |
1833 | { | |
1834 | static const struct i915_subtest tests[] = { | |
1835 | SUBTEST(live_lrc_layout), | |
1836 | SUBTEST(live_lrc_fixed), | |
1837 | SUBTEST(live_lrc_state), | |
1838 | SUBTEST(live_lrc_gpr), | |
1839 | SUBTEST(live_lrc_isolation), | |
1840 | SUBTEST(live_lrc_timestamp), | |
1841 | SUBTEST(live_lrc_garbage), | |
1842 | SUBTEST(live_pphwsp_runtime), | |
1843 | SUBTEST(live_lrc_indirect_ctx_bb), | |
1844 | }; | |
1845 | ||
1846 | if (!HAS_LOGICAL_RING_CONTEXTS(i915)) | |
1847 | return 0; | |
1848 | ||
1849 | return intel_gt_live_subtests(tests, &i915->gt); | |
1850 | } |