Commit | Line | Data |
---|---|---|
dd08ebf6 MB |
1 | // SPDX-License-Identifier: MIT |
2 | /* | |
3 | * Copyright © 2022 Intel Corporation | |
4 | */ | |
5 | ||
ea9f879d LDM |
6 | #include "xe_exec.h" |
7 | ||
dd08ebf6 MB |
8 | #include <drm/drm_device.h> |
9 | #include <drm/drm_file.h> | |
10 | #include <drm/xe_drm.h> | |
34f89ac8 | 11 | #include <linux/delay.h> |
dd08ebf6 MB |
12 | |
13 | #include "xe_bo.h" | |
14 | #include "xe_device.h" | |
c22a4ed0 | 15 | #include "xe_exec_queue.h" |
dd08ebf6 | 16 | #include "xe_macros.h" |
8ae8a2e8 | 17 | #include "xe_ring_ops_types.h" |
dd08ebf6 MB |
18 | #include "xe_sched_job.h" |
19 | #include "xe_sync.h" | |
20 | #include "xe_vm.h" | |
21 | ||
22 | /** | |
23 | * DOC: Execbuf (User GPU command submission) | |
24 | * | |
25 | * Execs have historically been rather complicated in DRM drivers (at least in | |
26 | * the i915) because a few things: | |
27 | * | |
28 | * - Passing in a list BO which are read / written to creating implicit syncs | |
29 | * - Binding at exec time | |
30 | * - Flow controlling the ring at exec time | |
31 | * | |
32 | * In XE we avoid all of this complication by not allowing a BO list to be | |
33 | * passed into an exec, using the dma-buf implicit sync uAPI, have binds as | |
34 | * seperate operations, and using the DRM scheduler to flow control the ring. | |
35 | * Let's deep dive on each of these. | |
36 | * | |
37 | * We can get away from a BO list by forcing the user to use in / out fences on | |
38 | * every exec rather than the kernel tracking dependencies of BO (e.g. if the | |
39 | * user knows an exec writes to a BO and reads from the BO in the next exec, it | |
40 | * is the user's responsibility to pass in / out fence between the two execs). | |
41 | * | |
42 | * Implicit dependencies for external BOs are handled by using the dma-buf | |
43 | * implicit dependency uAPI (TODO: add link). To make this works each exec must | |
44 | * install the job's fence into the DMA_RESV_USAGE_WRITE slot of every external | |
45 | * BO mapped in the VM. | |
46 | * | |
47 | * We do not allow a user to trigger a bind at exec time rather we have a VM | |
48 | * bind IOCTL which uses the same in / out fence interface as exec. In that | |
49 | * sense, a VM bind is basically the same operation as an exec from the user | |
50 | * perspective. e.g. If an exec depends on a VM bind use the in / out fence | |
51 | * interface (struct drm_xe_sync) to synchronize like syncing between two | |
52 | * dependent execs. | |
53 | * | |
54 | * Although a user cannot trigger a bind, we still have to rebind userptrs in | |
55 | * the VM that have been invalidated since the last exec, likewise we also have | |
56 | * to rebind BOs that have been evicted by the kernel. We schedule these rebinds | |
57 | * behind any pending kernel operations on any external BOs in VM or any BOs | |
58 | * private to the VM. This is accomplished by the rebinds waiting on BOs | |
59 | * DMA_RESV_USAGE_KERNEL slot (kernel ops) and kernel ops waiting on all BOs | |
60 | * slots (inflight execs are in the DMA_RESV_USAGE_BOOKING for private BOs and | |
61 | * in DMA_RESV_USAGE_WRITE for external BOs). | |
62 | * | |
63 | * Rebinds / dma-resv usage applies to non-compute mode VMs only as for compute | |
64 | * mode VMs we use preempt fences and a rebind worker (TODO: add link). | |
65 | * | |
66 | * There is no need to flow control the ring in the exec as we write the ring at | |
67 | * submission time and set the DRM scheduler max job limit SIZE_OF_RING / | |
68 | * MAX_JOB_SIZE. The DRM scheduler will then hold all jobs until space in the | |
69 | * ring is available. | |
70 | * | |
71 | * All of this results in a rather simple exec implementation. | |
72 | * | |
73 | * Flow | |
74 | * ~~~~ | |
75 | * | |
76 | * .. code-block:: | |
77 | * | |
78 | * Parse input arguments | |
79 | * Wait for any async VM bind passed as in-fences to start | |
80 | * <----------------------------------------------------------------------| | |
81 | * Lock global VM lock in read mode | | |
82 | * Pin userptrs (also finds userptr invalidated since last exec) | | |
83 | * Lock exec (VM dma-resv lock, external BOs dma-resv locks) | | |
84 | * Validate BOs that have been evicted | | |
85 | * Create job | | |
86 | * Rebind invalidated userptrs + evicted BOs (non-compute-mode) | | |
87 | * Add rebind fence dependency to job | | |
88 | * Add job VM dma-resv bookkeeping slot (non-compute mode) | | |
89 | * Add job to external BOs dma-resv write slots (non-compute mode) | | |
90 | * Check if any userptrs invalidated since pin ------ Drop locks ---------| | |
91 | * Install in / out fences for job | |
92 | * Submit job | |
93 | * Unlock all | |
94 | */ | |
95 | ||
34f89ac8 NV |
96 | #define XE_EXEC_BIND_RETRY_TIMEOUT_MS 1000 |
97 | ||
9b9529ce | 98 | static int xe_exec_begin(struct xe_exec_queue *q, struct ww_acquire_ctx *ww, |
dd08ebf6 MB |
99 | struct ttm_validate_buffer tv_onstack[], |
100 | struct ttm_validate_buffer **tv, | |
101 | struct list_head *objs) | |
102 | { | |
9b9529ce | 103 | struct xe_vm *vm = q->vm; |
dd08ebf6 MB |
104 | struct xe_vma *vma; |
105 | LIST_HEAD(dups); | |
34f89ac8 NV |
106 | ktime_t end = 0; |
107 | int err = 0; | |
dd08ebf6 MB |
108 | |
109 | *tv = NULL; | |
9b9529ce | 110 | if (xe_vm_no_dma_fences(q->vm)) |
dd08ebf6 MB |
111 | return 0; |
112 | ||
34f89ac8 | 113 | retry: |
dd08ebf6 MB |
114 | err = xe_vm_lock_dma_resv(vm, ww, tv_onstack, tv, objs, true, 1); |
115 | if (err) | |
116 | return err; | |
117 | ||
118 | /* | |
119 | * Validate BOs that have been evicted (i.e. make sure the | |
120 | * BOs have valid placements possibly moving an evicted BO back | |
121 | * to a location where the GPU can access it). | |
122 | */ | |
1655c893 | 123 | list_for_each_entry(vma, &vm->rebind_list, combined_links.rebind) { |
37430402 MB |
124 | XE_WARN_ON(xe_vma_is_null(vma)); |
125 | ||
dd08ebf6 MB |
126 | if (xe_vma_is_userptr(vma)) |
127 | continue; | |
128 | ||
21ed3327 | 129 | err = xe_bo_validate(xe_vma_bo(vma), vm, false); |
dd08ebf6 MB |
130 | if (err) { |
131 | xe_vm_unlock_dma_resv(vm, tv_onstack, *tv, ww, objs); | |
132 | *tv = NULL; | |
34f89ac8 NV |
133 | break; |
134 | } | |
135 | } | |
136 | ||
137 | /* | |
138 | * With multiple active VMs, under memory pressure, it is possible that | |
139 | * ttm_bo_validate() run into -EDEADLK and in such case returns -ENOMEM. | |
140 | * Until ttm properly handles locking in such scenarios, best thing the | |
141 | * driver can do is retry with a timeout. | |
142 | */ | |
143 | if (err == -ENOMEM) { | |
144 | ktime_t cur = ktime_get(); | |
145 | ||
146 | end = end ? : ktime_add_ms(cur, XE_EXEC_BIND_RETRY_TIMEOUT_MS); | |
147 | if (ktime_before(cur, end)) { | |
148 | msleep(20); | |
149 | goto retry; | |
dd08ebf6 MB |
150 | } |
151 | } | |
152 | ||
34f89ac8 | 153 | return err; |
dd08ebf6 MB |
154 | } |
155 | ||
9b9529ce | 156 | static void xe_exec_end(struct xe_exec_queue *q, |
dd08ebf6 MB |
157 | struct ttm_validate_buffer *tv_onstack, |
158 | struct ttm_validate_buffer *tv, | |
159 | struct ww_acquire_ctx *ww, | |
160 | struct list_head *objs) | |
161 | { | |
9b9529ce FD |
162 | if (!xe_vm_no_dma_fences(q->vm)) |
163 | xe_vm_unlock_dma_resv(q->vm, tv_onstack, tv, ww, objs); | |
dd08ebf6 MB |
164 | } |
165 | ||
166 | int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file) | |
167 | { | |
168 | struct xe_device *xe = to_xe_device(dev); | |
169 | struct xe_file *xef = to_xe_file(file); | |
170 | struct drm_xe_exec *args = data; | |
171 | struct drm_xe_sync __user *syncs_user = u64_to_user_ptr(args->syncs); | |
172 | u64 __user *addresses_user = u64_to_user_ptr(args->address); | |
9b9529ce | 173 | struct xe_exec_queue *q; |
dd08ebf6 MB |
174 | struct xe_sync_entry *syncs = NULL; |
175 | u64 addresses[XE_HW_ENGINE_MAX_INSTANCE]; | |
176 | struct ttm_validate_buffer tv_onstack[XE_ONSTACK_TV]; | |
177 | struct ttm_validate_buffer *tv = NULL; | |
178 | u32 i, num_syncs = 0; | |
179 | struct xe_sched_job *job; | |
180 | struct dma_fence *rebind_fence; | |
181 | struct xe_vm *vm; | |
182 | struct ww_acquire_ctx ww; | |
183 | struct list_head objs; | |
184 | bool write_locked; | |
185 | int err = 0; | |
186 | ||
b8c1ba83 FD |
187 | if (XE_IOCTL_DBG(xe, args->extensions) || |
188 | XE_IOCTL_DBG(xe, args->pad[0] || args->pad[1] || args->pad[2]) || | |
189 | XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) | |
dd08ebf6 MB |
190 | return -EINVAL; |
191 | ||
9b9529ce FD |
192 | q = xe_exec_queue_lookup(xef, args->exec_queue_id); |
193 | if (XE_IOCTL_DBG(xe, !q)) | |
dd08ebf6 MB |
194 | return -ENOENT; |
195 | ||
9b9529ce | 196 | if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_VM)) |
dd08ebf6 MB |
197 | return -EINVAL; |
198 | ||
9b9529ce | 199 | if (XE_IOCTL_DBG(xe, q->width != args->num_batch_buffer)) |
dd08ebf6 MB |
200 | return -EINVAL; |
201 | ||
9b9529ce | 202 | if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_BANNED)) { |
dd08ebf6 | 203 | err = -ECANCELED; |
9b9529ce | 204 | goto err_exec_queue; |
dd08ebf6 MB |
205 | } |
206 | ||
207 | if (args->num_syncs) { | |
208 | syncs = kcalloc(args->num_syncs, sizeof(*syncs), GFP_KERNEL); | |
209 | if (!syncs) { | |
210 | err = -ENOMEM; | |
9b9529ce | 211 | goto err_exec_queue; |
dd08ebf6 MB |
212 | } |
213 | } | |
214 | ||
9b9529ce | 215 | vm = q->vm; |
dd08ebf6 MB |
216 | |
217 | for (i = 0; i < args->num_syncs; i++) { | |
218 | err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs++], | |
219 | &syncs_user[i], true, | |
220 | xe_vm_no_dma_fences(vm)); | |
221 | if (err) | |
222 | goto err_syncs; | |
223 | } | |
224 | ||
9b9529ce | 225 | if (xe_exec_queue_is_parallel(q)) { |
dd08ebf6 | 226 | err = __copy_from_user(addresses, addresses_user, sizeof(u64) * |
9b9529ce | 227 | q->width); |
dd08ebf6 MB |
228 | if (err) { |
229 | err = -EFAULT; | |
230 | goto err_syncs; | |
231 | } | |
232 | } | |
233 | ||
234 | /* | |
235 | * We can't install a job into the VM dma-resv shared slot before an | |
236 | * async VM bind passed in as a fence without the risk of deadlocking as | |
237 | * the bind can trigger an eviction which in turn depends on anything in | |
238 | * the VM dma-resv shared slots. Not an ideal solution, but we wait for | |
239 | * all dependent async VM binds to start (install correct fences into | |
240 | * dma-resv slots) before moving forward. | |
241 | */ | |
242 | if (!xe_vm_no_dma_fences(vm) && | |
243 | vm->flags & XE_VM_FLAG_ASYNC_BIND_OPS) { | |
244 | for (i = 0; i < args->num_syncs; i++) { | |
245 | struct dma_fence *fence = syncs[i].fence; | |
3e8e7ee6 | 246 | |
dd08ebf6 MB |
247 | if (fence) { |
248 | err = xe_vm_async_fence_wait_start(fence); | |
249 | if (err) | |
250 | goto err_syncs; | |
251 | } | |
252 | } | |
253 | } | |
254 | ||
255 | retry: | |
256 | if (!xe_vm_no_dma_fences(vm) && xe_vm_userptr_check_repin(vm)) { | |
257 | err = down_write_killable(&vm->lock); | |
258 | write_locked = true; | |
259 | } else { | |
260 | /* We don't allow execs while the VM is in error state */ | |
261 | err = down_read_interruptible(&vm->lock); | |
262 | write_locked = false; | |
263 | } | |
264 | if (err) | |
265 | goto err_syncs; | |
266 | ||
267 | /* We don't allow execs while the VM is in error state */ | |
268 | if (vm->async_ops.error) { | |
269 | err = vm->async_ops.error; | |
270 | goto err_unlock_list; | |
271 | } | |
272 | ||
273 | /* | |
274 | * Extreme corner where we exit a VM error state with a munmap style VM | |
275 | * unbind inflight which requires a rebind. In this case the rebind | |
276 | * needs to install some fences into the dma-resv slots. The worker to | |
277 | * do this queued, let that worker make progress by dropping vm->lock, | |
278 | * flushing the worker and retrying the exec. | |
279 | */ | |
280 | if (vm->async_ops.munmap_rebind_inflight) { | |
281 | if (write_locked) | |
282 | up_write(&vm->lock); | |
283 | else | |
284 | up_read(&vm->lock); | |
285 | flush_work(&vm->async_ops.work); | |
286 | goto retry; | |
287 | } | |
288 | ||
289 | if (write_locked) { | |
290 | err = xe_vm_userptr_pin(vm); | |
291 | downgrade_write(&vm->lock); | |
292 | write_locked = false; | |
293 | if (err) | |
294 | goto err_unlock_list; | |
295 | } | |
296 | ||
9b9529ce | 297 | err = xe_exec_begin(q, &ww, tv_onstack, &tv, &objs); |
dd08ebf6 MB |
298 | if (err) |
299 | goto err_unlock_list; | |
300 | ||
9b9529ce | 301 | if (xe_vm_is_closed_or_banned(q->vm)) { |
9d858b69 MB |
302 | drm_warn(&xe->drm, "Trying to schedule after vm is closed or banned\n"); |
303 | err = -ECANCELED; | |
9b9529ce | 304 | goto err_exec_queue_end; |
dd08ebf6 MB |
305 | } |
306 | ||
9b9529ce | 307 | if (xe_exec_queue_is_lr(q) && xe_exec_queue_ring_full(q)) { |
8ae8a2e8 | 308 | err = -EWOULDBLOCK; |
9b9529ce | 309 | goto err_exec_queue_end; |
8ae8a2e8 MB |
310 | } |
311 | ||
9b9529ce | 312 | job = xe_sched_job_create(q, xe_exec_queue_is_parallel(q) ? |
dd08ebf6 MB |
313 | addresses : &args->address); |
314 | if (IS_ERR(job)) { | |
315 | err = PTR_ERR(job); | |
9b9529ce | 316 | goto err_exec_queue_end; |
dd08ebf6 MB |
317 | } |
318 | ||
319 | /* | |
320 | * Rebind any invalidated userptr or evicted BOs in the VM, non-compute | |
321 | * VM mode only. | |
322 | */ | |
323 | rebind_fence = xe_vm_rebind(vm, false); | |
324 | if (IS_ERR(rebind_fence)) { | |
325 | err = PTR_ERR(rebind_fence); | |
326 | goto err_put_job; | |
327 | } | |
328 | ||
329 | /* | |
330 | * We store the rebind_fence in the VM so subsequent execs don't get | |
331 | * scheduled before the rebinds of userptrs / evicted BOs is complete. | |
332 | */ | |
333 | if (rebind_fence) { | |
334 | dma_fence_put(vm->rebind_fence); | |
335 | vm->rebind_fence = rebind_fence; | |
336 | } | |
337 | if (vm->rebind_fence) { | |
338 | if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, | |
339 | &vm->rebind_fence->flags)) { | |
340 | dma_fence_put(vm->rebind_fence); | |
341 | vm->rebind_fence = NULL; | |
342 | } else { | |
343 | dma_fence_get(vm->rebind_fence); | |
344 | err = drm_sched_job_add_dependency(&job->drm, | |
345 | vm->rebind_fence); | |
346 | if (err) | |
347 | goto err_put_job; | |
348 | } | |
349 | } | |
350 | ||
351 | /* Wait behind munmap style rebinds */ | |
352 | if (!xe_vm_no_dma_fences(vm)) { | |
353 | err = drm_sched_job_add_resv_dependencies(&job->drm, | |
b06d47be | 354 | xe_vm_resv(vm), |
dd08ebf6 MB |
355 | DMA_RESV_USAGE_KERNEL); |
356 | if (err) | |
357 | goto err_put_job; | |
358 | } | |
359 | ||
360 | for (i = 0; i < num_syncs && !err; i++) | |
361 | err = xe_sync_entry_add_deps(&syncs[i], job); | |
362 | if (err) | |
363 | goto err_put_job; | |
364 | ||
365 | if (!xe_vm_no_dma_fences(vm)) { | |
366 | err = down_read_interruptible(&vm->userptr.notifier_lock); | |
367 | if (err) | |
368 | goto err_put_job; | |
369 | ||
370 | err = __xe_vm_userptr_needs_repin(vm); | |
371 | if (err) | |
372 | goto err_repin; | |
373 | } | |
374 | ||
375 | /* | |
376 | * Point of no return, if we error after this point just set an error on | |
377 | * the job and let the DRM scheduler / backend clean up the job. | |
378 | */ | |
379 | xe_sched_job_arm(job); | |
380 | if (!xe_vm_no_dma_fences(vm)) { | |
381 | /* Block userptr invalidations / BO eviction */ | |
b06d47be | 382 | dma_resv_add_fence(xe_vm_resv(vm), |
dd08ebf6 MB |
383 | &job->drm.s_fence->finished, |
384 | DMA_RESV_USAGE_BOOKKEEP); | |
385 | ||
386 | /* | |
387 | * Make implicit sync work across drivers, assuming all external | |
388 | * BOs are written as we don't pass in a read / write list. | |
389 | */ | |
390 | xe_vm_fence_all_extobjs(vm, &job->drm.s_fence->finished, | |
391 | DMA_RESV_USAGE_WRITE); | |
392 | } | |
393 | ||
394 | for (i = 0; i < num_syncs; i++) | |
395 | xe_sync_entry_signal(&syncs[i], job, | |
396 | &job->drm.s_fence->finished); | |
397 | ||
9b9529ce FD |
398 | if (xe_exec_queue_is_lr(q)) |
399 | q->ring_ops->emit_job(job); | |
dd08ebf6 | 400 | xe_sched_job_push(job); |
8e41443e | 401 | xe_vm_reactivate_rebind(vm); |
dd08ebf6 | 402 | |
7ba4c5f0 MB |
403 | if (!err && !xe_vm_no_dma_fences(vm)) { |
404 | spin_lock(&xe->ttm.lru_lock); | |
405 | ttm_lru_bulk_move_tail(&vm->lru_bulk_move); | |
406 | spin_unlock(&xe->ttm.lru_lock); | |
407 | } | |
408 | ||
dd08ebf6 MB |
409 | err_repin: |
410 | if (!xe_vm_no_dma_fences(vm)) | |
411 | up_read(&vm->userptr.notifier_lock); | |
412 | err_put_job: | |
413 | if (err) | |
414 | xe_sched_job_put(job); | |
9b9529ce FD |
415 | err_exec_queue_end: |
416 | xe_exec_end(q, tv_onstack, tv, &ww, &objs); | |
dd08ebf6 MB |
417 | err_unlock_list: |
418 | if (write_locked) | |
419 | up_write(&vm->lock); | |
420 | else | |
421 | up_read(&vm->lock); | |
422 | if (err == -EAGAIN) | |
423 | goto retry; | |
424 | err_syncs: | |
425 | for (i = 0; i < num_syncs; i++) | |
426 | xe_sync_entry_cleanup(&syncs[i]); | |
427 | kfree(syncs); | |
9b9529ce FD |
428 | err_exec_queue: |
429 | xe_exec_queue_put(q); | |
dd08ebf6 MB |
430 | |
431 | return err; | |
432 | } |