Commit | Line | Data |
---|---|---|
a8c21a54 T |
1 | /* |
2 | * Copyright (C) 2015 Etnaviv Project | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify it | |
5 | * under the terms of the GNU General Public License version 2 as published by | |
6 | * the Free Software Foundation. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, but WITHOUT | |
9 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
10 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
11 | * more details. | |
12 | * | |
13 | * You should have received a copy of the GNU General Public License along with | |
14 | * this program. If not, see <http://www.gnu.org/licenses/>. | |
15 | */ | |
16 | ||
9ad59fea | 17 | #include <linux/dma-fence-array.h> |
a8c21a54 | 18 | #include <linux/reservation.h> |
9ad59fea | 19 | #include <linux/sync_file.h> |
ea1f5729 | 20 | #include "etnaviv_cmdbuf.h" |
a8c21a54 T |
21 | #include "etnaviv_drv.h" |
22 | #include "etnaviv_gpu.h" | |
23 | #include "etnaviv_gem.h" | |
24 | ||
25 | /* | |
26 | * Cmdstream submission: | |
27 | */ | |
28 | ||
29 | #define BO_INVALID_FLAGS ~(ETNA_SUBMIT_BO_READ | ETNA_SUBMIT_BO_WRITE) | |
30 | /* make sure these don't conflict w/ ETNAVIV_SUBMIT_BO_x */ | |
31 | #define BO_LOCKED 0x4000 | |
32 | #define BO_PINNED 0x2000 | |
33 | ||
a8c21a54 T |
34 | static struct etnaviv_gem_submit *submit_create(struct drm_device *dev, |
35 | struct etnaviv_gpu *gpu, size_t nr) | |
36 | { | |
37 | struct etnaviv_gem_submit *submit; | |
38 | size_t sz = size_vstruct(nr, sizeof(submit->bos[0]), sizeof(*submit)); | |
39 | ||
0ee931c4 | 40 | submit = kmalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY); |
a8c21a54 T |
41 | if (submit) { |
42 | submit->dev = dev; | |
43 | submit->gpu = gpu; | |
44 | ||
45 | /* initially, until copy_from_user() and bo lookup succeeds: */ | |
46 | submit->nr_bos = 0; | |
657314b7 | 47 | submit->fence = NULL; |
a8c21a54 T |
48 | |
49 | ww_acquire_init(&submit->ticket, &reservation_ww_class); | |
50 | } | |
51 | ||
52 | return submit; | |
53 | } | |
54 | ||
55 | static int submit_lookup_objects(struct etnaviv_gem_submit *submit, | |
56 | struct drm_file *file, struct drm_etnaviv_gem_submit_bo *submit_bos, | |
57 | unsigned nr_bos) | |
58 | { | |
59 | struct drm_etnaviv_gem_submit_bo *bo; | |
60 | unsigned i; | |
61 | int ret = 0; | |
62 | ||
63 | spin_lock(&file->table_lock); | |
64 | ||
65 | for (i = 0, bo = submit_bos; i < nr_bos; i++, bo++) { | |
66 | struct drm_gem_object *obj; | |
67 | ||
68 | if (bo->flags & BO_INVALID_FLAGS) { | |
69 | DRM_ERROR("invalid flags: %x\n", bo->flags); | |
70 | ret = -EINVAL; | |
71 | goto out_unlock; | |
72 | } | |
73 | ||
74 | submit->bos[i].flags = bo->flags; | |
75 | ||
76 | /* normally use drm_gem_object_lookup(), but for bulk lookup | |
77 | * all under single table_lock just hit object_idr directly: | |
78 | */ | |
79 | obj = idr_find(&file->object_idr, bo->handle); | |
80 | if (!obj) { | |
81 | DRM_ERROR("invalid handle %u at index %u\n", | |
82 | bo->handle, i); | |
83 | ret = -EINVAL; | |
84 | goto out_unlock; | |
85 | } | |
86 | ||
87 | /* | |
88 | * Take a refcount on the object. The file table lock | |
89 | * prevents the object_idr's refcount on this being dropped. | |
90 | */ | |
23d1dd03 | 91 | drm_gem_object_get(obj); |
a8c21a54 T |
92 | |
93 | submit->bos[i].obj = to_etnaviv_bo(obj); | |
94 | } | |
95 | ||
96 | out_unlock: | |
97 | submit->nr_bos = i; | |
98 | spin_unlock(&file->table_lock); | |
99 | ||
100 | return ret; | |
101 | } | |
102 | ||
103 | static void submit_unlock_object(struct etnaviv_gem_submit *submit, int i) | |
104 | { | |
105 | if (submit->bos[i].flags & BO_LOCKED) { | |
106 | struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj; | |
107 | ||
108 | ww_mutex_unlock(&etnaviv_obj->resv->lock); | |
109 | submit->bos[i].flags &= ~BO_LOCKED; | |
110 | } | |
111 | } | |
112 | ||
113 | static int submit_lock_objects(struct etnaviv_gem_submit *submit) | |
114 | { | |
115 | int contended, slow_locked = -1, i, ret = 0; | |
116 | ||
117 | retry: | |
118 | for (i = 0; i < submit->nr_bos; i++) { | |
119 | struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj; | |
120 | ||
121 | if (slow_locked == i) | |
122 | slow_locked = -1; | |
123 | ||
124 | contended = i; | |
125 | ||
126 | if (!(submit->bos[i].flags & BO_LOCKED)) { | |
127 | ret = ww_mutex_lock_interruptible(&etnaviv_obj->resv->lock, | |
128 | &submit->ticket); | |
129 | if (ret == -EALREADY) | |
130 | DRM_ERROR("BO at index %u already on submit list\n", | |
131 | i); | |
132 | if (ret) | |
133 | goto fail; | |
134 | submit->bos[i].flags |= BO_LOCKED; | |
135 | } | |
136 | } | |
137 | ||
138 | ww_acquire_done(&submit->ticket); | |
139 | ||
140 | return 0; | |
141 | ||
142 | fail: | |
143 | for (; i >= 0; i--) | |
144 | submit_unlock_object(submit, i); | |
145 | ||
146 | if (slow_locked > 0) | |
147 | submit_unlock_object(submit, slow_locked); | |
148 | ||
149 | if (ret == -EDEADLK) { | |
150 | struct etnaviv_gem_object *etnaviv_obj; | |
151 | ||
152 | etnaviv_obj = submit->bos[contended].obj; | |
153 | ||
154 | /* we lost out in a seqno race, lock and retry.. */ | |
155 | ret = ww_mutex_lock_slow_interruptible(&etnaviv_obj->resv->lock, | |
156 | &submit->ticket); | |
157 | if (!ret) { | |
158 | submit->bos[contended].flags |= BO_LOCKED; | |
159 | slow_locked = contended; | |
160 | goto retry; | |
161 | } | |
162 | } | |
163 | ||
164 | return ret; | |
165 | } | |
166 | ||
167 | static int submit_fence_sync(const struct etnaviv_gem_submit *submit) | |
168 | { | |
169 | unsigned int context = submit->gpu->fence_context; | |
170 | int i, ret = 0; | |
171 | ||
172 | for (i = 0; i < submit->nr_bos; i++) { | |
173 | struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj; | |
174 | bool write = submit->bos[i].flags & ETNA_SUBMIT_BO_WRITE; | |
426ef1bb | 175 | bool explicit = !!(submit->flags & ETNA_SUBMIT_NO_IMPLICIT); |
a8c21a54 | 176 | |
9ad59fea PZ |
177 | ret = etnaviv_gpu_fence_sync_obj(etnaviv_obj, context, write, |
178 | explicit); | |
a8c21a54 T |
179 | if (ret) |
180 | break; | |
181 | } | |
182 | ||
183 | return ret; | |
184 | } | |
185 | ||
186 | static void submit_unpin_objects(struct etnaviv_gem_submit *submit) | |
187 | { | |
188 | int i; | |
189 | ||
190 | for (i = 0; i < submit->nr_bos; i++) { | |
a8c21a54 | 191 | if (submit->bos[i].flags & BO_PINNED) |
b6325f40 | 192 | etnaviv_gem_mapping_unreference(submit->bos[i].mapping); |
a8c21a54 | 193 | |
b6325f40 | 194 | submit->bos[i].mapping = NULL; |
a8c21a54 T |
195 | submit->bos[i].flags &= ~BO_PINNED; |
196 | } | |
197 | } | |
198 | ||
199 | static int submit_pin_objects(struct etnaviv_gem_submit *submit) | |
200 | { | |
201 | int i, ret = 0; | |
202 | ||
203 | for (i = 0; i < submit->nr_bos; i++) { | |
204 | struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj; | |
b6325f40 | 205 | struct etnaviv_vram_mapping *mapping; |
a8c21a54 | 206 | |
b6325f40 RK |
207 | mapping = etnaviv_gem_mapping_get(&etnaviv_obj->base, |
208 | submit->gpu); | |
209 | if (IS_ERR(mapping)) { | |
210 | ret = PTR_ERR(mapping); | |
a8c21a54 | 211 | break; |
b6325f40 | 212 | } |
a8c21a54 T |
213 | |
214 | submit->bos[i].flags |= BO_PINNED; | |
b6325f40 | 215 | submit->bos[i].mapping = mapping; |
a8c21a54 T |
216 | } |
217 | ||
218 | return ret; | |
219 | } | |
220 | ||
221 | static int submit_bo(struct etnaviv_gem_submit *submit, u32 idx, | |
8779aa8f | 222 | struct etnaviv_gem_submit_bo **bo) |
a8c21a54 T |
223 | { |
224 | if (idx >= submit->nr_bos) { | |
225 | DRM_ERROR("invalid buffer index: %u (out of %u)\n", | |
226 | idx, submit->nr_bos); | |
227 | return -EINVAL; | |
228 | } | |
229 | ||
8779aa8f | 230 | *bo = &submit->bos[idx]; |
a8c21a54 T |
231 | |
232 | return 0; | |
233 | } | |
234 | ||
235 | /* process the reloc's and patch up the cmdstream as needed: */ | |
236 | static int submit_reloc(struct etnaviv_gem_submit *submit, void *stream, | |
237 | u32 size, const struct drm_etnaviv_gem_submit_reloc *relocs, | |
238 | u32 nr_relocs) | |
239 | { | |
240 | u32 i, last_offset = 0; | |
241 | u32 *ptr = stream; | |
242 | int ret; | |
243 | ||
244 | for (i = 0; i < nr_relocs; i++) { | |
245 | const struct drm_etnaviv_gem_submit_reloc *r = relocs + i; | |
8779aa8f RK |
246 | struct etnaviv_gem_submit_bo *bo; |
247 | u32 off; | |
a8c21a54 T |
248 | |
249 | if (unlikely(r->flags)) { | |
250 | DRM_ERROR("invalid reloc flags\n"); | |
251 | return -EINVAL; | |
252 | } | |
253 | ||
254 | if (r->submit_offset % 4) { | |
255 | DRM_ERROR("non-aligned reloc offset: %u\n", | |
256 | r->submit_offset); | |
257 | return -EINVAL; | |
258 | } | |
259 | ||
260 | /* offset in dwords: */ | |
261 | off = r->submit_offset / 4; | |
262 | ||
263 | if ((off >= size ) || | |
264 | (off < last_offset)) { | |
265 | DRM_ERROR("invalid offset %u at reloc %u\n", off, i); | |
266 | return -EINVAL; | |
267 | } | |
268 | ||
8779aa8f | 269 | ret = submit_bo(submit, r->reloc_idx, &bo); |
a8c21a54 T |
270 | if (ret) |
271 | return ret; | |
272 | ||
d6f756e0 WL |
273 | if (r->reloc_offset > bo->obj->base.size - sizeof(*ptr)) { |
274 | DRM_ERROR("relocation %u outside object\n", i); | |
a8c21a54 T |
275 | return -EINVAL; |
276 | } | |
277 | ||
8779aa8f | 278 | ptr[off] = bo->mapping->iova + r->reloc_offset; |
a8c21a54 T |
279 | |
280 | last_offset = off; | |
281 | } | |
282 | ||
283 | return 0; | |
284 | } | |
285 | ||
286 | static void submit_cleanup(struct etnaviv_gem_submit *submit) | |
287 | { | |
288 | unsigned i; | |
289 | ||
290 | for (i = 0; i < submit->nr_bos; i++) { | |
291 | struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj; | |
292 | ||
293 | submit_unlock_object(submit, i); | |
23d1dd03 | 294 | drm_gem_object_put_unlocked(&etnaviv_obj->base); |
a8c21a54 T |
295 | } |
296 | ||
297 | ww_acquire_fini(&submit->ticket); | |
657314b7 LS |
298 | if (submit->fence) |
299 | dma_fence_put(submit->fence); | |
a8c21a54 T |
300 | kfree(submit); |
301 | } | |
302 | ||
303 | int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data, | |
304 | struct drm_file *file) | |
305 | { | |
306 | struct etnaviv_drm_private *priv = dev->dev_private; | |
307 | struct drm_etnaviv_gem_submit *args = data; | |
308 | struct drm_etnaviv_gem_submit_reloc *relocs; | |
309 | struct drm_etnaviv_gem_submit_bo *bos; | |
310 | struct etnaviv_gem_submit *submit; | |
311 | struct etnaviv_cmdbuf *cmdbuf; | |
312 | struct etnaviv_gpu *gpu; | |
9ad59fea | 313 | struct dma_fence *in_fence = NULL; |
78ec187f PZ |
314 | struct sync_file *sync_file = NULL; |
315 | int out_fence_fd = -1; | |
a8c21a54 T |
316 | void *stream; |
317 | int ret; | |
318 | ||
319 | if (args->pipe >= ETNA_MAX_PIPES) | |
320 | return -EINVAL; | |
321 | ||
322 | gpu = priv->gpu[args->pipe]; | |
323 | if (!gpu) | |
324 | return -ENXIO; | |
325 | ||
326 | if (args->stream_size % 4) { | |
327 | DRM_ERROR("non-aligned cmdstream buffer size: %u\n", | |
328 | args->stream_size); | |
329 | return -EINVAL; | |
330 | } | |
331 | ||
332 | if (args->exec_state != ETNA_PIPE_3D && | |
333 | args->exec_state != ETNA_PIPE_2D && | |
334 | args->exec_state != ETNA_PIPE_VG) { | |
335 | DRM_ERROR("invalid exec_state: 0x%x\n", args->exec_state); | |
336 | return -EINVAL; | |
337 | } | |
338 | ||
9ad59fea PZ |
339 | if (args->flags & ~ETNA_SUBMIT_FLAGS) { |
340 | DRM_ERROR("invalid flags: 0x%x\n", args->flags); | |
341 | return -EINVAL; | |
342 | } | |
343 | ||
a8c21a54 T |
344 | /* |
345 | * Copy the command submission and bo array to kernel space in | |
346 | * one go, and do this outside of any locks. | |
347 | */ | |
2098105e MH |
348 | bos = kvmalloc_array(args->nr_bos, sizeof(*bos), GFP_KERNEL); |
349 | relocs = kvmalloc_array(args->nr_relocs, sizeof(*relocs), GFP_KERNEL); | |
350 | stream = kvmalloc_array(1, args->stream_size, GFP_KERNEL); | |
e66774dd LS |
351 | cmdbuf = etnaviv_cmdbuf_new(gpu->cmdbuf_suballoc, |
352 | ALIGN(args->stream_size, 8) + 8, | |
353 | args->nr_bos); | |
a8c21a54 T |
354 | if (!bos || !relocs || !stream || !cmdbuf) { |
355 | ret = -ENOMEM; | |
356 | goto err_submit_cmds; | |
357 | } | |
358 | ||
359 | cmdbuf->exec_state = args->exec_state; | |
360 | cmdbuf->ctx = file->driver_priv; | |
361 | ||
3ed605bc | 362 | ret = copy_from_user(bos, u64_to_user_ptr(args->bos), |
a8c21a54 T |
363 | args->nr_bos * sizeof(*bos)); |
364 | if (ret) { | |
365 | ret = -EFAULT; | |
366 | goto err_submit_cmds; | |
367 | } | |
368 | ||
3ed605bc | 369 | ret = copy_from_user(relocs, u64_to_user_ptr(args->relocs), |
a8c21a54 T |
370 | args->nr_relocs * sizeof(*relocs)); |
371 | if (ret) { | |
372 | ret = -EFAULT; | |
373 | goto err_submit_cmds; | |
374 | } | |
375 | ||
3ed605bc | 376 | ret = copy_from_user(stream, u64_to_user_ptr(args->stream), |
a8c21a54 T |
377 | args->stream_size); |
378 | if (ret) { | |
379 | ret = -EFAULT; | |
380 | goto err_submit_cmds; | |
381 | } | |
382 | ||
78ec187f PZ |
383 | if (args->flags & ETNA_SUBMIT_FENCE_FD_OUT) { |
384 | out_fence_fd = get_unused_fd_flags(O_CLOEXEC); | |
385 | if (out_fence_fd < 0) { | |
386 | ret = out_fence_fd; | |
387 | goto err_submit_cmds; | |
388 | } | |
389 | } | |
390 | ||
a8c21a54 T |
391 | submit = submit_create(dev, gpu, args->nr_bos); |
392 | if (!submit) { | |
393 | ret = -ENOMEM; | |
394 | goto err_submit_cmds; | |
395 | } | |
396 | ||
9ad59fea PZ |
397 | submit->flags = args->flags; |
398 | ||
a8c21a54 T |
399 | ret = submit_lookup_objects(submit, file, bos, args->nr_bos); |
400 | if (ret) | |
401 | goto err_submit_objects; | |
402 | ||
403 | ret = submit_lock_objects(submit); | |
404 | if (ret) | |
405 | goto err_submit_objects; | |
406 | ||
407 | if (!etnaviv_cmd_validate_one(gpu, stream, args->stream_size / 4, | |
408 | relocs, args->nr_relocs)) { | |
409 | ret = -EINVAL; | |
410 | goto err_submit_objects; | |
411 | } | |
412 | ||
9ad59fea PZ |
413 | if (args->flags & ETNA_SUBMIT_FENCE_FD_IN) { |
414 | in_fence = sync_file_get_fence(args->fence_fd); | |
415 | if (!in_fence) { | |
416 | ret = -EINVAL; | |
417 | goto err_submit_objects; | |
418 | } | |
419 | ||
420 | /* | |
421 | * Wait if the fence is from a foreign context, or if the fence | |
422 | * array contains any fence from a foreign context. | |
423 | */ | |
424 | if (!dma_fence_match_context(in_fence, gpu->fence_context)) { | |
425 | ret = dma_fence_wait(in_fence, true); | |
426 | if (ret) | |
427 | goto err_submit_objects; | |
428 | } | |
429 | } | |
430 | ||
a8c21a54 T |
431 | ret = submit_fence_sync(submit); |
432 | if (ret) | |
433 | goto err_submit_objects; | |
434 | ||
435 | ret = submit_pin_objects(submit); | |
436 | if (ret) | |
437 | goto out; | |
438 | ||
439 | ret = submit_reloc(submit, stream, args->stream_size / 4, | |
440 | relocs, args->nr_relocs); | |
441 | if (ret) | |
442 | goto out; | |
443 | ||
444 | memcpy(cmdbuf->vaddr, stream, args->stream_size); | |
445 | cmdbuf->user_size = ALIGN(args->stream_size, 8); | |
446 | ||
447 | ret = etnaviv_gpu_submit(gpu, submit, cmdbuf); | |
5a642e6b LS |
448 | if (ret) |
449 | goto out; | |
450 | ||
451 | cmdbuf = NULL; | |
a8c21a54 | 452 | |
78ec187f PZ |
453 | if (args->flags & ETNA_SUBMIT_FENCE_FD_OUT) { |
454 | /* | |
455 | * This can be improved: ideally we want to allocate the sync | |
456 | * file before kicking off the GPU job and just attach the | |
457 | * fence to the sync file here, eliminating the ENOMEM | |
458 | * possibility at this stage. | |
459 | */ | |
460 | sync_file = sync_file_create(submit->fence); | |
461 | if (!sync_file) { | |
462 | ret = -ENOMEM; | |
463 | goto out; | |
464 | } | |
465 | fd_install(out_fence_fd, sync_file->file); | |
466 | } | |
467 | ||
468 | args->fence_fd = out_fence_fd; | |
6e2b98cf | 469 | args->fence = submit->fence->seqno; |
a8c21a54 T |
470 | |
471 | out: | |
472 | submit_unpin_objects(submit); | |
473 | ||
474 | /* | |
475 | * If we're returning -EAGAIN, it may be due to the userptr code | |
476 | * wanting to run its workqueue outside of any locks. Flush our | |
477 | * workqueue to ensure that it is run in a timely manner. | |
478 | */ | |
479 | if (ret == -EAGAIN) | |
480 | flush_workqueue(priv->wq); | |
481 | ||
482 | err_submit_objects: | |
9ad59fea PZ |
483 | if (in_fence) |
484 | dma_fence_put(in_fence); | |
a8c21a54 T |
485 | submit_cleanup(submit); |
486 | ||
487 | err_submit_cmds: | |
78ec187f PZ |
488 | if (ret && (out_fence_fd >= 0)) |
489 | put_unused_fd(out_fence_fd); | |
a8c21a54 T |
490 | /* if we still own the cmdbuf */ |
491 | if (cmdbuf) | |
ea1f5729 | 492 | etnaviv_cmdbuf_free(cmdbuf); |
a8c21a54 | 493 | if (stream) |
2098105e | 494 | kvfree(stream); |
a8c21a54 | 495 | if (bos) |
2098105e | 496 | kvfree(bos); |
a8c21a54 | 497 | if (relocs) |
2098105e | 498 | kvfree(relocs); |
a8c21a54 T |
499 | |
500 | return ret; | |
501 | } |