Commit | Line | Data |
---|---|---|
c4d52feb CW |
1 | /* |
2 | * SPDX-License-Identifier: MIT | |
3 | * | |
4 | * Copyright © 2019 Intel Corporation | |
5 | */ | |
6 | ||
10be98a7 CW |
7 | #include "gem/i915_gem_context.h" |
8 | #include "gem/i915_gem_pm.h" | |
9 | ||
c4d52feb | 10 | #include "i915_drv.h" |
c4d52feb | 11 | #include "i915_globals.h" |
112ed2d3 | 12 | |
c4d52feb | 13 | #include "intel_context.h" |
112ed2d3 | 14 | #include "intel_engine.h" |
79ffac85 | 15 | #include "intel_engine_pm.h" |
2871ea85 | 16 | #include "intel_ring.h" |
c4d52feb CW |
17 | |
18 | static struct i915_global_context { | |
19 | struct i915_global base; | |
20 | struct kmem_cache *slab_ce; | |
21 | } global; | |
22 | ||
5e2a0419 | 23 | static struct intel_context *intel_context_alloc(void) |
c4d52feb CW |
24 | { |
25 | return kmem_cache_zalloc(global.slab_ce, GFP_KERNEL); | |
26 | } | |
27 | ||
28 | void intel_context_free(struct intel_context *ce) | |
29 | { | |
30 | kmem_cache_free(global.slab_ce, ce); | |
31 | } | |
32 | ||
33 | struct intel_context * | |
e6ba7648 | 34 | intel_context_create(struct intel_engine_cs *engine) |
c4d52feb | 35 | { |
5e2a0419 | 36 | struct intel_context *ce; |
c4d52feb CW |
37 | |
38 | ce = intel_context_alloc(); | |
39 | if (!ce) | |
40 | return ERR_PTR(-ENOMEM); | |
41 | ||
e6ba7648 | 42 | intel_context_init(ce, engine); |
5e2a0419 | 43 | return ce; |
c4d52feb CW |
44 | } |
45 | ||
89f98d63 CW |
46 | int intel_context_alloc_state(struct intel_context *ce) |
47 | { | |
48 | int err = 0; | |
49 | ||
50 | if (mutex_lock_interruptible(&ce->pin_mutex)) | |
51 | return -EINTR; | |
52 | ||
53 | if (!test_bit(CONTEXT_ALLOC_BIT, &ce->flags)) { | |
373f27f2 CW |
54 | if (intel_context_is_banned(ce)) { |
55 | err = -EIO; | |
56 | goto unlock; | |
57 | } | |
58 | ||
89f98d63 CW |
59 | err = ce->ops->alloc(ce); |
60 | if (unlikely(err)) | |
61 | goto unlock; | |
62 | ||
63 | set_bit(CONTEXT_ALLOC_BIT, &ce->flags); | |
64 | } | |
65 | ||
66 | unlock: | |
67 | mutex_unlock(&ce->pin_mutex); | |
68 | return err; | |
69 | } | |
70 | ||
b11b28ea CW |
71 | static int intel_context_active_acquire(struct intel_context *ce) |
72 | { | |
73 | int err; | |
74 | ||
e5429340 CW |
75 | __i915_active_acquire(&ce->active); |
76 | ||
77 | if (intel_context_is_barrier(ce)) | |
78 | return 0; | |
b11b28ea CW |
79 | |
80 | /* Preallocate tracking nodes */ | |
e5429340 CW |
81 | err = i915_active_acquire_preallocate_barrier(&ce->active, |
82 | ce->engine); | |
83 | if (err) | |
84 | i915_active_release(&ce->active); | |
b11b28ea | 85 | |
e5429340 | 86 | return err; |
b11b28ea CW |
87 | } |
88 | ||
89 | static void intel_context_active_release(struct intel_context *ce) | |
90 | { | |
91 | /* Nodes preallocated in intel_context_active() */ | |
92 | i915_active_acquire_barrier(&ce->active); | |
93 | i915_active_release(&ce->active); | |
94 | } | |
95 | ||
47b08693 | 96 | static int __context_pin_state(struct i915_vma *vma, struct i915_gem_ww_ctx *ww) |
c4d52feb | 97 | { |
ccd20945 | 98 | unsigned int bias = i915_ggtt_pin_bias(vma) | PIN_OFFSET_BIAS; |
ce476c80 | 99 | int err; |
c4d52feb | 100 | |
47b08693 | 101 | err = i915_ggtt_pin(vma, ww, 0, bias | PIN_HIGH); |
ce476c80 CW |
102 | if (err) |
103 | return err; | |
104 | ||
1b8bfc57 CW |
105 | err = i915_active_acquire(&vma->active); |
106 | if (err) | |
107 | goto err_unpin; | |
108 | ||
ce476c80 CW |
109 | /* |
110 | * And mark it as a globally pinned object to let the shrinker know | |
111 | * it cannot reclaim the object until we release it. | |
112 | */ | |
1aff1903 | 113 | i915_vma_make_unshrinkable(vma); |
ce476c80 CW |
114 | vma->obj->mm.dirty = true; |
115 | ||
116 | return 0; | |
1b8bfc57 CW |
117 | |
118 | err_unpin: | |
119 | i915_vma_unpin(vma); | |
120 | return err; | |
ce476c80 CW |
121 | } |
122 | ||
123 | static void __context_unpin_state(struct i915_vma *vma) | |
124 | { | |
1aff1903 | 125 | i915_vma_make_shrinkable(vma); |
1b8bfc57 | 126 | i915_active_release(&vma->active); |
99013b10 | 127 | __i915_vma_unpin(vma); |
ce476c80 CW |
128 | } |
129 | ||
47b08693 ML |
130 | static int __ring_active(struct intel_ring *ring, |
131 | struct i915_gem_ww_ctx *ww) | |
8ccfc20a CW |
132 | { |
133 | int err; | |
134 | ||
47b08693 | 135 | err = intel_ring_pin(ring, ww); |
8ccfc20a CW |
136 | if (err) |
137 | return err; | |
138 | ||
5a383d44 | 139 | err = i915_active_acquire(&ring->vma->active); |
8ccfc20a | 140 | if (err) |
5a383d44 | 141 | goto err_pin; |
8ccfc20a CW |
142 | |
143 | return 0; | |
144 | ||
5a383d44 CW |
145 | err_pin: |
146 | intel_ring_unpin(ring); | |
8ccfc20a CW |
147 | return err; |
148 | } | |
149 | ||
150 | static void __ring_retire(struct intel_ring *ring) | |
151 | { | |
8ccfc20a | 152 | i915_active_release(&ring->vma->active); |
5a383d44 | 153 | intel_ring_unpin(ring); |
8ccfc20a CW |
154 | } |
155 | ||
47b08693 ML |
156 | static int intel_context_pre_pin(struct intel_context *ce, |
157 | struct i915_gem_ww_ctx *ww) | |
3999a708 ML |
158 | { |
159 | int err; | |
160 | ||
161 | CE_TRACE(ce, "active\n"); | |
162 | ||
47b08693 | 163 | err = __ring_active(ce->ring, ww); |
3999a708 ML |
164 | if (err) |
165 | return err; | |
166 | ||
47b08693 | 167 | err = intel_timeline_pin(ce->timeline, ww); |
3999a708 ML |
168 | if (err) |
169 | goto err_ring; | |
170 | ||
171 | if (!ce->state) | |
172 | return 0; | |
173 | ||
47b08693 | 174 | err = __context_pin_state(ce->state, ww); |
3999a708 ML |
175 | if (err) |
176 | goto err_timeline; | |
177 | ||
178 | ||
179 | return 0; | |
180 | ||
181 | err_timeline: | |
182 | intel_timeline_unpin(ce->timeline); | |
183 | err_ring: | |
184 | __ring_retire(ce->ring); | |
185 | return err; | |
186 | } | |
187 | ||
188 | static void intel_context_post_unpin(struct intel_context *ce) | |
189 | { | |
190 | if (ce->state) | |
191 | __context_unpin_state(ce->state); | |
192 | ||
193 | intel_timeline_unpin(ce->timeline); | |
194 | __ring_retire(ce->ring); | |
195 | } | |
196 | ||
47b08693 ML |
197 | int __intel_context_do_pin_ww(struct intel_context *ce, |
198 | struct i915_gem_ww_ctx *ww) | |
3999a708 ML |
199 | { |
200 | bool handoff = false; | |
201 | void *vaddr; | |
202 | int err = 0; | |
203 | ||
204 | if (unlikely(!test_bit(CONTEXT_ALLOC_BIT, &ce->flags))) { | |
205 | err = intel_context_alloc_state(ce); | |
206 | if (err) | |
207 | return err; | |
208 | } | |
209 | ||
210 | /* | |
211 | * We always pin the context/ring/timeline here, to ensure a pin | |
212 | * refcount for __intel_context_active(), which prevent a lock | |
213 | * inversion of ce->pin_mutex vs dma_resv_lock(). | |
214 | */ | |
47b08693 ML |
215 | |
216 | err = i915_gem_object_lock(ce->timeline->hwsp_ggtt->obj, ww); | |
217 | if (!err && ce->ring->vma->obj) | |
218 | err = i915_gem_object_lock(ce->ring->vma->obj, ww); | |
219 | if (!err && ce->state) | |
220 | err = i915_gem_object_lock(ce->state->obj, ww); | |
221 | if (!err) | |
222 | err = intel_context_pre_pin(ce, ww); | |
3999a708 ML |
223 | if (err) |
224 | return err; | |
225 | ||
226 | err = i915_active_acquire(&ce->active); | |
227 | if (err) | |
228 | goto err_ctx_unpin; | |
229 | ||
47b08693 | 230 | err = ce->ops->pre_pin(ce, ww, &vaddr); |
3999a708 ML |
231 | if (err) |
232 | goto err_release; | |
233 | ||
234 | err = mutex_lock_interruptible(&ce->pin_mutex); | |
235 | if (err) | |
236 | goto err_post_unpin; | |
237 | ||
238 | if (unlikely(intel_context_is_closed(ce))) { | |
239 | err = -ENOENT; | |
240 | goto err_unlock; | |
241 | } | |
242 | ||
243 | if (likely(!atomic_add_unless(&ce->pin_count, 1, 0))) { | |
244 | err = intel_context_active_acquire(ce); | |
245 | if (unlikely(err)) | |
246 | goto err_unlock; | |
247 | ||
248 | err = ce->ops->pin(ce, vaddr); | |
249 | if (err) { | |
250 | intel_context_active_release(ce); | |
251 | goto err_unlock; | |
252 | } | |
253 | ||
254 | CE_TRACE(ce, "pin ring:{start:%08x, head:%04x, tail:%04x}\n", | |
255 | i915_ggtt_offset(ce->ring->vma), | |
256 | ce->ring->head, ce->ring->tail); | |
257 | ||
258 | handoff = true; | |
259 | smp_mb__before_atomic(); /* flush pin before it is visible */ | |
260 | atomic_inc(&ce->pin_count); | |
261 | } | |
262 | ||
263 | GEM_BUG_ON(!intel_context_is_pinned(ce)); /* no overflow! */ | |
264 | ||
265 | err_unlock: | |
266 | mutex_unlock(&ce->pin_mutex); | |
267 | err_post_unpin: | |
268 | if (!handoff) | |
269 | ce->ops->post_unpin(ce); | |
270 | err_release: | |
271 | i915_active_release(&ce->active); | |
272 | err_ctx_unpin: | |
273 | intel_context_post_unpin(ce); | |
e0ee152f TH |
274 | |
275 | /* | |
276 | * Unlock the hwsp_ggtt object since it's shared. | |
277 | * In principle we can unlock all the global state locked above | |
278 | * since it's pinned and doesn't need fencing, and will | |
279 | * thus remain resident until it is explicitly unpinned. | |
280 | */ | |
281 | i915_gem_ww_unlock_single(ce->timeline->hwsp_ggtt->obj); | |
282 | ||
3999a708 ML |
283 | return err; |
284 | } | |
285 | ||
47b08693 ML |
286 | int __intel_context_do_pin(struct intel_context *ce) |
287 | { | |
288 | struct i915_gem_ww_ctx ww; | |
289 | int err; | |
290 | ||
291 | i915_gem_ww_ctx_init(&ww, true); | |
292 | retry: | |
293 | err = __intel_context_do_pin_ww(ce, &ww); | |
294 | if (err == -EDEADLK) { | |
295 | err = i915_gem_ww_ctx_backoff(&ww); | |
296 | if (!err) | |
297 | goto retry; | |
298 | } | |
299 | i915_gem_ww_ctx_fini(&ww); | |
300 | return err; | |
301 | } | |
302 | ||
3999a708 ML |
303 | void intel_context_unpin(struct intel_context *ce) |
304 | { | |
305 | if (!atomic_dec_and_test(&ce->pin_count)) | |
306 | return; | |
307 | ||
308 | CE_TRACE(ce, "unpin\n"); | |
309 | ce->ops->unpin(ce); | |
310 | ce->ops->post_unpin(ce); | |
311 | ||
312 | /* | |
313 | * Once released, we may asynchronously drop the active reference. | |
314 | * As that may be the only reference keeping the context alive, | |
315 | * take an extra now so that it is not freed before we finish | |
316 | * dereferencing it. | |
317 | */ | |
318 | intel_context_get(ce); | |
319 | intel_context_active_release(ce); | |
320 | intel_context_put(ce); | |
321 | } | |
322 | ||
274cbf20 | 323 | __i915_active_call |
12c255b5 | 324 | static void __intel_context_retire(struct i915_active *active) |
ce476c80 CW |
325 | { |
326 | struct intel_context *ce = container_of(active, typeof(*ce), active); | |
327 | ||
1883a0a4 TU |
328 | CE_TRACE(ce, "retire runtime: { total:%lluns, avg:%lluns }\n", |
329 | intel_context_get_total_runtime_ns(ce), | |
330 | intel_context_get_avg_runtime_ns(ce)); | |
cba17e5d | 331 | |
f70de8d2 | 332 | set_bit(CONTEXT_VALID_BIT, &ce->flags); |
3999a708 | 333 | intel_context_post_unpin(ce); |
ce476c80 | 334 | intel_context_put(ce); |
c4d52feb CW |
335 | } |
336 | ||
12c255b5 | 337 | static int __intel_context_active(struct i915_active *active) |
ce476c80 | 338 | { |
12c255b5 | 339 | struct intel_context *ce = container_of(active, typeof(*ce), active); |
ce476c80 | 340 | |
ce476c80 CW |
341 | intel_context_get(ce); |
342 | ||
3999a708 | 343 | /* everything should already be activated by intel_context_pre_pin() */ |
47b08693 ML |
344 | GEM_WARN_ON(!i915_active_acquire_if_busy(&ce->ring->vma->active)); |
345 | __intel_ring_pin(ce->ring); | |
09c5ab38 | 346 | |
47b08693 | 347 | __intel_timeline_pin(ce->timeline); |
75d0a7f3 | 348 | |
3999a708 ML |
349 | if (ce->state) { |
350 | GEM_WARN_ON(!i915_active_acquire_if_busy(&ce->state->active)); | |
351 | __i915_vma_pin(ce->state); | |
352 | i915_vma_make_unshrinkable(ce->state); | |
353 | } | |
ce476c80 | 354 | |
d8af05ff | 355 | return 0; |
d8af05ff CW |
356 | } |
357 | ||
12c255b5 CW |
358 | void |
359 | intel_context_init(struct intel_context *ce, | |
12c255b5 | 360 | struct intel_engine_cs *engine) |
ce476c80 | 361 | { |
12c255b5 | 362 | GEM_BUG_ON(!engine->cops); |
e6ba7648 | 363 | GEM_BUG_ON(!engine->gt->vm); |
12c255b5 CW |
364 | |
365 | kref_init(&ce->ref); | |
366 | ||
12c255b5 CW |
367 | ce->engine = engine; |
368 | ce->ops = engine->cops; | |
369 | ce->sseu = engine->sseu; | |
e6ba7648 CW |
370 | ce->ring = __intel_context_ring_size(SZ_4K); |
371 | ||
1883a0a4 TU |
372 | ewma_runtime_init(&ce->runtime.avg); |
373 | ||
e6ba7648 | 374 | ce->vm = i915_vm_get(engine->gt->vm); |
12c255b5 CW |
375 | |
376 | INIT_LIST_HEAD(&ce->signal_link); | |
377 | INIT_LIST_HEAD(&ce->signals); | |
378 | ||
379 | mutex_init(&ce->pin_mutex); | |
380 | ||
b1e3177b | 381 | i915_active_init(&ce->active, |
12c255b5 | 382 | __intel_context_active, __intel_context_retire); |
c4d52feb CW |
383 | } |
384 | ||
df8cf31e CW |
385 | void intel_context_fini(struct intel_context *ce) |
386 | { | |
75d0a7f3 CW |
387 | if (ce->timeline) |
388 | intel_timeline_put(ce->timeline); | |
f5d974f9 CW |
389 | i915_vm_put(ce->vm); |
390 | ||
df8cf31e CW |
391 | mutex_destroy(&ce->pin_mutex); |
392 | i915_active_fini(&ce->active); | |
393 | } | |
394 | ||
c4d52feb CW |
395 | static void i915_global_context_shrink(void) |
396 | { | |
397 | kmem_cache_shrink(global.slab_ce); | |
398 | } | |
399 | ||
400 | static void i915_global_context_exit(void) | |
401 | { | |
402 | kmem_cache_destroy(global.slab_ce); | |
403 | } | |
404 | ||
405 | static struct i915_global_context global = { { | |
406 | .shrink = i915_global_context_shrink, | |
407 | .exit = i915_global_context_exit, | |
408 | } }; | |
409 | ||
410 | int __init i915_global_context_init(void) | |
411 | { | |
412 | global.slab_ce = KMEM_CACHE(intel_context, SLAB_HWCACHE_ALIGN); | |
413 | if (!global.slab_ce) | |
414 | return -ENOMEM; | |
415 | ||
416 | i915_global_register(&global.base); | |
417 | return 0; | |
418 | } | |
6eee33e8 CW |
419 | |
420 | void intel_context_enter_engine(struct intel_context *ce) | |
421 | { | |
79ffac85 | 422 | intel_engine_pm_get(ce->engine); |
531958f6 | 423 | intel_timeline_enter(ce->timeline); |
6eee33e8 CW |
424 | } |
425 | ||
426 | void intel_context_exit_engine(struct intel_context *ce) | |
427 | { | |
531958f6 | 428 | intel_timeline_exit(ce->timeline); |
af7a272e | 429 | intel_engine_pm_put(ce->engine); |
6eee33e8 | 430 | } |
5e2a0419 | 431 | |
a9877da2 CW |
432 | int intel_context_prepare_remote_request(struct intel_context *ce, |
433 | struct i915_request *rq) | |
434 | { | |
75d0a7f3 | 435 | struct intel_timeline *tl = ce->timeline; |
a9877da2 CW |
436 | int err; |
437 | ||
438 | /* Only suitable for use in remotely modifying this context */ | |
9f3ccd40 | 439 | GEM_BUG_ON(rq->context == ce); |
a9877da2 | 440 | |
d19d71fc | 441 | if (rcu_access_pointer(rq->timeline) != tl) { /* timeline sharing! */ |
340c4c8d | 442 | /* Queue this switch after current activity by this context. */ |
b1e3177b | 443 | err = i915_active_fence_set(&tl->last_request, rq); |
340c4c8d | 444 | if (err) |
25ffd4b1 | 445 | return err; |
340c4c8d | 446 | } |
a9877da2 CW |
447 | |
448 | /* | |
449 | * Guarantee context image and the timeline remains pinned until the | |
450 | * modifying request is retired by setting the ce activity tracker. | |
451 | * | |
452 | * But we only need to take one pin on the account of it. Or in other | |
453 | * words transfer the pinned ce object to tracked active request. | |
454 | */ | |
455 | GEM_BUG_ON(i915_active_is_idle(&ce->active)); | |
d19d71fc | 456 | return i915_active_add_request(&ce->active, rq); |
a9877da2 CW |
457 | } |
458 | ||
5e2a0419 CW |
459 | struct i915_request *intel_context_create_request(struct intel_context *ce) |
460 | { | |
8a929c9e | 461 | struct i915_gem_ww_ctx ww; |
5e2a0419 CW |
462 | struct i915_request *rq; |
463 | int err; | |
464 | ||
8a929c9e ML |
465 | i915_gem_ww_ctx_init(&ww, true); |
466 | retry: | |
467 | err = intel_context_pin_ww(ce, &ww); | |
468 | if (!err) { | |
469 | rq = i915_request_create(ce); | |
470 | intel_context_unpin(ce); | |
471 | } else if (err == -EDEADLK) { | |
472 | err = i915_gem_ww_ctx_backoff(&ww); | |
473 | if (!err) | |
474 | goto retry; | |
159ace7f | 475 | rq = ERR_PTR(err); |
8a929c9e ML |
476 | } else { |
477 | rq = ERR_PTR(err); | |
478 | } | |
5e2a0419 | 479 | |
8a929c9e | 480 | i915_gem_ww_ctx_fini(&ww); |
5e2a0419 | 481 | |
dd878c0c ML |
482 | if (IS_ERR(rq)) |
483 | return rq; | |
484 | ||
485 | /* | |
486 | * timeline->mutex should be the inner lock, but is used as outer lock. | |
487 | * Hack around this to shut up lockdep in selftests.. | |
488 | */ | |
489 | lockdep_unpin_lock(&ce->timeline->mutex, rq->cookie); | |
490 | mutex_release(&ce->timeline->mutex.dep_map, _RET_IP_); | |
491 | mutex_acquire(&ce->timeline->mutex.dep_map, SINGLE_DEPTH_NESTING, 0, _RET_IP_); | |
492 | rq->cookie = lockdep_pin_lock(&ce->timeline->mutex); | |
493 | ||
5e2a0419 CW |
494 | return rq; |
495 | } | |
d8af05ff CW |
496 | |
497 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) | |
498 | #include "selftest_context.c" | |
499 | #endif |