Commit | Line | Data |
---|---|---|
24f90d66 | 1 | // SPDX-License-Identifier: MIT |
f97fbf96 CW |
2 | /* |
3 | * Copyright © 2016 Intel Corporation | |
f97fbf96 CW |
4 | */ |
5 | ||
10be98a7 | 6 | #include "gem/i915_gem_context.h" |
2871ea85 | 7 | #include "gt/intel_ring.h" |
10be98a7 | 8 | |
112ed2d3 CW |
9 | #include "i915_drv.h" |
10 | #include "intel_context.h" | |
79ffac85 | 11 | #include "intel_engine_pm.h" |
112ed2d3 | 12 | |
f97fbf96 | 13 | #include "mock_engine.h" |
112ed2d3 | 14 | #include "selftests/mock_request.h" |
f97fbf96 | 15 | |
2c8ab333 | 16 | static int mock_timeline_pin(struct intel_timeline *tl) |
5013eb8c | 17 | { |
2c8ab333 ML |
18 | int err; |
19 | ||
d8be1357 | 20 | if (WARN_ON(!i915_gem_object_trylock(tl->hwsp_ggtt->obj, NULL))) |
2c8ab333 ML |
21 | return -EBUSY; |
22 | ||
23 | err = intel_timeline_pin_map(tl); | |
24 | i915_gem_object_unlock(tl->hwsp_ggtt->obj); | |
25 | if (err) | |
26 | return err; | |
27 | ||
ccb23d2d | 28 | atomic_inc(&tl->pin_count); |
2c8ab333 | 29 | return 0; |
5013eb8c CW |
30 | } |
31 | ||
f0c02c1b | 32 | static void mock_timeline_unpin(struct intel_timeline *tl) |
5013eb8c | 33 | { |
ccb23d2d CW |
34 | GEM_BUG_ON(!atomic_read(&tl->pin_count)); |
35 | atomic_dec(&tl->pin_count); | |
5013eb8c CW |
36 | } |
37 | ||
d03a29e0 ML |
38 | static struct i915_vma *create_ring_vma(struct i915_ggtt *ggtt, int size) |
39 | { | |
40 | struct i915_address_space *vm = &ggtt->vm; | |
41 | struct drm_i915_private *i915 = vm->i915; | |
42 | struct drm_i915_gem_object *obj; | |
43 | struct i915_vma *vma; | |
44 | ||
45 | obj = i915_gem_object_create_internal(i915, size); | |
46 | if (IS_ERR(obj)) | |
47 | return ERR_CAST(obj); | |
48 | ||
49 | vma = i915_vma_instance(obj, vm, NULL); | |
50 | if (IS_ERR(vma)) | |
51 | goto err; | |
52 | ||
53 | return vma; | |
54 | ||
55 | err: | |
56 | i915_gem_object_put(obj); | |
57 | return vma; | |
58 | } | |
59 | ||
209760b7 CW |
60 | static struct intel_ring *mock_ring(struct intel_engine_cs *engine) |
61 | { | |
d03a29e0 | 62 | const unsigned long sz = PAGE_SIZE; |
75d0a7f3 | 63 | struct intel_ring *ring; |
209760b7 CW |
64 | |
65 | ring = kzalloc(sizeof(*ring) + sz, GFP_KERNEL); | |
66 | if (!ring) | |
67 | return NULL; | |
68 | ||
75d0a7f3 CW |
69 | kref_init(&ring->ref); |
70 | ring->size = sz; | |
71 | ring->effective_size = sz; | |
72 | ring->vaddr = (void *)(ring + 1); | |
73 | atomic_set(&ring->pin_count, 1); | |
209760b7 | 74 | |
d03a29e0 ML |
75 | ring->vma = create_ring_vma(engine->gt->ggtt, PAGE_SIZE); |
76 | if (IS_ERR(ring->vma)) { | |
b63b4fea CW |
77 | kfree(ring); |
78 | return NULL; | |
79 | } | |
b63b4fea | 80 | |
75d0a7f3 | 81 | intel_ring_update_space(ring); |
209760b7 | 82 | |
75d0a7f3 | 83 | return ring; |
209760b7 CW |
84 | } |
85 | ||
b63b4fea CW |
86 | static void mock_ring_free(struct intel_ring *ring) |
87 | { | |
d03a29e0 | 88 | i915_vma_put(ring->vma); |
b63b4fea CW |
89 | |
90 | kfree(ring); | |
91 | } | |
92 | ||
32eb6bcf | 93 | static struct i915_request *first_request(struct mock_engine *engine) |
f97fbf96 | 94 | { |
0daf0113 | 95 | return list_first_entry_or_null(&engine->hw_queue, |
32eb6bcf CW |
96 | struct i915_request, |
97 | mock.link); | |
0daf0113 CW |
98 | } |
99 | ||
32eb6bcf | 100 | static void advance(struct i915_request *request) |
b1f9107e | 101 | { |
32eb6bcf CW |
102 | list_del_init(&request->mock.link); |
103 | i915_request_mark_complete(request); | |
104 | GEM_BUG_ON(!i915_request_completed(request)); | |
52c0fdb2 | 105 | |
54400257 | 106 | intel_engine_signal_breadcrumbs(request->engine); |
b1f9107e CW |
107 | } |
108 | ||
39cbf2aa | 109 | static void hw_delay_complete(struct timer_list *t) |
0daf0113 | 110 | { |
39cbf2aa | 111 | struct mock_engine *engine = from_timer(engine, t, hw_delay); |
32eb6bcf | 112 | struct i915_request *request; |
52c0fdb2 | 113 | unsigned long flags; |
0daf0113 | 114 | |
52c0fdb2 | 115 | spin_lock_irqsave(&engine->hw_lock, flags); |
0daf0113 | 116 | |
b1f9107e | 117 | /* Timer fired, first request is complete */ |
0daf0113 CW |
118 | request = first_request(engine); |
119 | if (request) | |
1579ab2d | 120 | advance(request); |
b1f9107e CW |
121 | |
122 | /* | |
123 | * Also immediately signal any subsequent 0-delay requests, but | |
124 | * requeue the timer for the next delayed request. | |
125 | */ | |
126 | while ((request = first_request(engine))) { | |
32eb6bcf CW |
127 | if (request->mock.delay) { |
128 | mod_timer(&engine->hw_delay, | |
129 | jiffies + request->mock.delay); | |
b1f9107e CW |
130 | break; |
131 | } | |
132 | ||
1579ab2d | 133 | advance(request); |
b1f9107e | 134 | } |
0daf0113 | 135 | |
52c0fdb2 | 136 | spin_unlock_irqrestore(&engine->hw_lock, flags); |
0daf0113 CW |
137 | } |
138 | ||
1fc44d9b | 139 | static void mock_context_unpin(struct intel_context *ce) |
0daf0113 | 140 | { |
1fc44d9b | 141 | } |
ab82a063 | 142 | |
3999a708 ML |
143 | static void mock_context_post_unpin(struct intel_context *ce) |
144 | { | |
d03a29e0 | 145 | i915_vma_unpin(ce->ring->vma); |
3999a708 ML |
146 | } |
147 | ||
4c5896dc | 148 | static void mock_context_destroy(struct kref *ref) |
1fc44d9b | 149 | { |
4c5896dc CW |
150 | struct intel_context *ce = container_of(ref, typeof(*ce), ref); |
151 | ||
08819549 | 152 | GEM_BUG_ON(intel_context_is_pinned(ce)); |
209760b7 | 153 | |
75d0a7f3 | 154 | if (test_bit(CONTEXT_ALLOC_BIT, &ce->flags)) { |
b63b4fea | 155 | mock_ring_free(ce->ring); |
75d0a7f3 CW |
156 | mock_timeline_unpin(ce->timeline); |
157 | } | |
4c5896dc | 158 | |
df8cf31e | 159 | intel_context_fini(ce); |
4c5896dc | 160 | intel_context_free(ce); |
0daf0113 CW |
161 | } |
162 | ||
4c60b1aa CW |
163 | static int mock_context_alloc(struct intel_context *ce) |
164 | { | |
2c8ab333 ML |
165 | int err; |
166 | ||
4c60b1aa CW |
167 | ce->ring = mock_ring(ce->engine); |
168 | if (!ce->ring) | |
169 | return -ENOMEM; | |
170 | ||
d1bf5dd8 | 171 | ce->timeline = intel_timeline_create(ce->engine->gt); |
75d0a7f3 CW |
172 | if (IS_ERR(ce->timeline)) { |
173 | kfree(ce->engine); | |
174 | return PTR_ERR(ce->timeline); | |
175 | } | |
176 | ||
2c8ab333 ML |
177 | err = mock_timeline_pin(ce->timeline); |
178 | if (err) { | |
179 | intel_timeline_put(ce->timeline); | |
180 | ce->timeline = NULL; | |
181 | return err; | |
182 | } | |
75d0a7f3 | 183 | |
4c60b1aa CW |
184 | return 0; |
185 | } | |
186 | ||
47b08693 ML |
187 | static int mock_context_pre_pin(struct intel_context *ce, |
188 | struct i915_gem_ww_ctx *ww, void **unused) | |
3999a708 | 189 | { |
d03a29e0 | 190 | return i915_vma_pin_ww(ce->ring->vma, ww, 0, 0, PIN_GLOBAL | PIN_HIGH); |
3999a708 ML |
191 | } |
192 | ||
193 | static int mock_context_pin(struct intel_context *ce, void *unused) | |
0daf0113 | 194 | { |
b11b28ea | 195 | return 0; |
0daf0113 CW |
196 | } |
197 | ||
fb218f20 CW |
198 | static void mock_context_reset(struct intel_context *ce) |
199 | { | |
200 | } | |
201 | ||
4dc84b77 | 202 | static const struct intel_context_ops mock_context_ops = { |
4c60b1aa CW |
203 | .alloc = mock_context_alloc, |
204 | ||
3999a708 | 205 | .pre_pin = mock_context_pre_pin, |
95f697eb | 206 | .pin = mock_context_pin, |
4dc84b77 | 207 | .unpin = mock_context_unpin, |
3999a708 | 208 | .post_unpin = mock_context_post_unpin, |
4c5896dc | 209 | |
6eee33e8 CW |
210 | .enter = intel_context_enter_engine, |
211 | .exit = intel_context_exit_engine, | |
212 | ||
fb218f20 | 213 | .reset = mock_context_reset, |
4dc84b77 CW |
214 | .destroy = mock_context_destroy, |
215 | }; | |
216 | ||
e61e0f51 | 217 | static int mock_request_alloc(struct i915_request *request) |
0daf0113 | 218 | { |
32eb6bcf CW |
219 | INIT_LIST_HEAD(&request->mock.link); |
220 | request->mock.delay = 0; | |
0daf0113 | 221 | |
0daf0113 CW |
222 | return 0; |
223 | } | |
224 | ||
e61e0f51 | 225 | static int mock_emit_flush(struct i915_request *request, |
0daf0113 CW |
226 | unsigned int flags) |
227 | { | |
228 | return 0; | |
229 | } | |
230 | ||
e1a73a54 | 231 | static u32 *mock_emit_breadcrumb(struct i915_request *request, u32 *cs) |
0daf0113 | 232 | { |
e1a73a54 | 233 | return cs; |
0daf0113 CW |
234 | } |
235 | ||
e61e0f51 | 236 | static void mock_submit_request(struct i915_request *request) |
0daf0113 | 237 | { |
0daf0113 CW |
238 | struct mock_engine *engine = |
239 | container_of(request->engine, typeof(*engine), base); | |
52c0fdb2 | 240 | unsigned long flags; |
0daf0113 | 241 | |
e61e0f51 | 242 | i915_request_submit(request); |
0daf0113 | 243 | |
52c0fdb2 | 244 | spin_lock_irqsave(&engine->hw_lock, flags); |
32eb6bcf CW |
245 | list_add_tail(&request->mock.link, &engine->hw_queue); |
246 | if (list_is_first(&request->mock.link, &engine->hw_queue)) { | |
247 | if (request->mock.delay) | |
248 | mod_timer(&engine->hw_delay, | |
249 | jiffies + request->mock.delay); | |
b1f9107e | 250 | else |
32eb6bcf | 251 | advance(request); |
b1f9107e | 252 | } |
52c0fdb2 | 253 | spin_unlock_irqrestore(&engine->hw_lock, flags); |
0daf0113 CW |
254 | } |
255 | ||
d1cee2d3 MB |
256 | static void mock_add_to_engine(struct i915_request *rq) |
257 | { | |
258 | lockdep_assert_held(&rq->engine->sched_engine->lock); | |
259 | list_move_tail(&rq->sched.link, &rq->engine->sched_engine->requests); | |
260 | } | |
261 | ||
262 | static void mock_remove_from_engine(struct i915_request *rq) | |
263 | { | |
264 | struct intel_engine_cs *engine, *locked; | |
265 | ||
266 | /* | |
267 | * Virtual engines complicate acquiring the engine timeline lock, | |
268 | * as their rq->engine pointer is not stable until under that | |
269 | * engine lock. The simple ploy we use is to take the lock then | |
270 | * check that the rq still belongs to the newly locked engine. | |
271 | */ | |
272 | ||
273 | locked = READ_ONCE(rq->engine); | |
274 | spin_lock_irq(&locked->sched_engine->lock); | |
275 | while (unlikely(locked != (engine = READ_ONCE(rq->engine)))) { | |
276 | spin_unlock(&locked->sched_engine->lock); | |
277 | spin_lock(&engine->sched_engine->lock); | |
278 | locked = engine; | |
279 | } | |
280 | list_del_init(&rq->sched.link); | |
281 | spin_unlock_irq(&locked->sched_engine->lock); | |
282 | } | |
283 | ||
d315d4fa CW |
284 | static void mock_reset_prepare(struct intel_engine_cs *engine) |
285 | { | |
286 | } | |
287 | ||
e26b6d43 | 288 | static void mock_reset_rewind(struct intel_engine_cs *engine, bool stalled) |
d315d4fa CW |
289 | { |
290 | GEM_BUG_ON(stalled); | |
291 | } | |
292 | ||
e26b6d43 | 293 | static void mock_reset_cancel(struct intel_engine_cs *engine) |
d315d4fa | 294 | { |
eb3afbe1 CW |
295 | struct mock_engine *mock = |
296 | container_of(engine, typeof(*mock), base); | |
297 | struct i915_request *rq; | |
d315d4fa CW |
298 | unsigned long flags; |
299 | ||
eb3afbe1 CW |
300 | del_timer_sync(&mock->hw_delay); |
301 | ||
349a2bc5 | 302 | spin_lock_irqsave(&engine->sched_engine->lock, flags); |
d315d4fa CW |
303 | |
304 | /* Mark all submitted requests as skipped. */ | |
349a2bc5 | 305 | list_for_each_entry(rq, &engine->sched_engine->requests, sched.link) |
c10e4a79 | 306 | i915_request_put(i915_request_mark_eio(rq)); |
eb3afbe1 CW |
307 | intel_engine_signal_breadcrumbs(engine); |
308 | ||
309 | /* Cancel and submit all pending requests. */ | |
310 | list_for_each_entry(rq, &mock->hw_queue, mock.link) { | |
c10e4a79 CW |
311 | if (i915_request_mark_eio(rq)) { |
312 | __i915_request_submit(rq); | |
313 | i915_request_put(rq); | |
314 | } | |
d315d4fa | 315 | } |
eb3afbe1 | 316 | INIT_LIST_HEAD(&mock->hw_queue); |
d315d4fa | 317 | |
349a2bc5 | 318 | spin_unlock_irqrestore(&engine->sched_engine->lock, flags); |
d315d4fa CW |
319 | } |
320 | ||
e26b6d43 CW |
321 | static void mock_reset_finish(struct intel_engine_cs *engine) |
322 | { | |
323 | } | |
324 | ||
325 | static void mock_engine_release(struct intel_engine_cs *engine) | |
326 | { | |
327 | struct mock_engine *mock = | |
328 | container_of(engine, typeof(*mock), base); | |
329 | ||
330 | GEM_BUG_ON(timer_pending(&mock->hw_delay)); | |
331 | ||
3e28d371 | 332 | i915_sched_engine_put(engine->sched_engine); |
a95d1160 | 333 | intel_breadcrumbs_put(engine->breadcrumbs); |
b3786b29 | 334 | |
e26b6d43 CW |
335 | intel_context_unpin(engine->kernel_context); |
336 | intel_context_put(engine->kernel_context); | |
337 | ||
338 | intel_engine_fini_retire(engine); | |
e26b6d43 CW |
339 | } |
340 | ||
0daf0113 | 341 | struct intel_engine_cs *mock_engine(struct drm_i915_private *i915, |
3ec0af7f CW |
342 | const char *name, |
343 | int id) | |
0daf0113 CW |
344 | { |
345 | struct mock_engine *engine; | |
3ec0af7f CW |
346 | |
347 | GEM_BUG_ON(id >= I915_NUM_ENGINES); | |
c14adcbd | 348 | GEM_BUG_ON(!to_gt(i915)->uncore); |
f97fbf96 CW |
349 | |
350 | engine = kzalloc(sizeof(*engine) + PAGE_SIZE, GFP_KERNEL); | |
351 | if (!engine) | |
352 | return NULL; | |
353 | ||
0daf0113 CW |
354 | /* minimal engine setup for requests */ |
355 | engine->base.i915 = i915; | |
c14adcbd MW |
356 | engine->base.gt = to_gt(i915); |
357 | engine->base.uncore = to_gt(i915)->uncore; | |
6e516148 | 358 | snprintf(engine->base.name, sizeof(engine->base.name), "%s", name); |
3ec0af7f | 359 | engine->base.id = id; |
8a68d464 | 360 | engine->base.mask = BIT(id); |
bcce7d90 | 361 | engine->base.legacy_idx = INVALID_ENGINE; |
cbb153c5 | 362 | engine->base.instance = id; |
0ca88ba0 | 363 | engine->base.status_page.addr = (void *)(engine + 1); |
f97fbf96 | 364 | |
4dc84b77 | 365 | engine->base.cops = &mock_context_ops; |
0daf0113 CW |
366 | engine->base.request_alloc = mock_request_alloc; |
367 | engine->base.emit_flush = mock_emit_flush; | |
85474441 | 368 | engine->base.emit_fini_breadcrumb = mock_emit_breadcrumb; |
0daf0113 | 369 | engine->base.submit_request = mock_submit_request; |
d1cee2d3 MB |
370 | engine->base.add_active_request = mock_add_to_engine; |
371 | engine->base.remove_active_request = mock_remove_from_engine; | |
0daf0113 | 372 | |
d315d4fa | 373 | engine->base.reset.prepare = mock_reset_prepare; |
e26b6d43 CW |
374 | engine->base.reset.rewind = mock_reset_rewind; |
375 | engine->base.reset.cancel = mock_reset_cancel; | |
d315d4fa | 376 | engine->base.reset.finish = mock_reset_finish; |
e26b6d43 CW |
377 | |
378 | engine->base.release = mock_engine_release; | |
d315d4fa | 379 | |
c14adcbd MW |
380 | to_gt(i915)->engine[id] = &engine->base; |
381 | to_gt(i915)->engine_class[0][id] = &engine->base; | |
bcce7d90 | 382 | |
0daf0113 CW |
383 | /* fake hw queue */ |
384 | spin_lock_init(&engine->hw_lock); | |
39cbf2aa | 385 | timer_setup(&engine->hw_delay, hw_delay_complete, 0); |
0daf0113 CW |
386 | INIT_LIST_HEAD(&engine->hw_queue); |
387 | ||
cbb153c5 CW |
388 | intel_engine_add_user(&engine->base); |
389 | ||
11334c6a CW |
390 | return &engine->base; |
391 | } | |
392 | ||
393 | int mock_engine_init(struct intel_engine_cs *engine) | |
394 | { | |
38775829 | 395 | struct intel_context *ce; |
11334c6a | 396 | |
3e42cc61 TH |
397 | INIT_LIST_HEAD(&engine->pinned_contexts_list); |
398 | ||
3e28d371 MB |
399 | engine->sched_engine = i915_sched_engine_create(ENGINE_MOCK); |
400 | if (!engine->sched_engine) | |
401 | return -ENOMEM; | |
22916bad | 402 | engine->sched_engine->private_data = engine; |
3e28d371 | 403 | |
11334c6a CW |
404 | intel_engine_init_execlists(engine); |
405 | intel_engine_init__pm(engine); | |
ed7dd73c | 406 | intel_engine_init_retire(engine); |
11334c6a | 407 | |
b3786b29 CW |
408 | engine->breadcrumbs = intel_breadcrumbs_create(NULL); |
409 | if (!engine->breadcrumbs) | |
3e28d371 | 410 | goto err_schedule; |
b3786b29 | 411 | |
38775829 CW |
412 | ce = create_kernel_context(engine); |
413 | if (IS_ERR(ce)) | |
422d7df4 | 414 | goto err_breadcrumbs; |
fa9f6681 | 415 | |
e3d0e213 CW |
416 | /* We insist the kernel context is using the status_page */ |
417 | engine->status_page.vma = ce->timeline->hwsp_ggtt; | |
418 | ||
38775829 | 419 | engine->kernel_context = ce; |
11334c6a | 420 | return 0; |
b887d615 CW |
421 | |
422 | err_breadcrumbs: | |
a95d1160 | 423 | intel_breadcrumbs_put(engine->breadcrumbs); |
3e28d371 MB |
424 | err_schedule: |
425 | i915_sched_engine_put(engine->sched_engine); | |
11334c6a | 426 | return -ENOMEM; |
f97fbf96 CW |
427 | } |
428 | ||
429 | void mock_engine_flush(struct intel_engine_cs *engine) | |
430 | { | |
0daf0113 CW |
431 | struct mock_engine *mock = |
432 | container_of(engine, typeof(*mock), base); | |
32eb6bcf | 433 | struct i915_request *request, *rn; |
0daf0113 CW |
434 | |
435 | del_timer_sync(&mock->hw_delay); | |
436 | ||
437 | spin_lock_irq(&mock->hw_lock); | |
32eb6bcf | 438 | list_for_each_entry_safe(request, rn, &mock->hw_queue, mock.link) |
1579ab2d | 439 | advance(request); |
0daf0113 | 440 | spin_unlock_irq(&mock->hw_lock); |
f97fbf96 CW |
441 | } |
442 | ||
443 | void mock_engine_reset(struct intel_engine_cs *engine) | |
444 | { | |
f97fbf96 | 445 | } |