Commit | Line | Data |
---|---|---|
a8c21a54 T |
1 | /* |
2 | * Copyright (C) 2015 Etnaviv Project | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify it | |
5 | * under the terms of the GNU General Public License version 2 as published by | |
6 | * the Free Software Foundation. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, but WITHOUT | |
9 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
10 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
11 | * more details. | |
12 | * | |
13 | * You should have received a copy of the GNU General Public License along with | |
14 | * this program. If not, see <http://www.gnu.org/licenses/>. | |
15 | */ | |
16 | ||
17 | #include <linux/reservation.h> | |
18 | #include "etnaviv_drv.h" | |
19 | #include "etnaviv_gpu.h" | |
20 | #include "etnaviv_gem.h" | |
21 | ||
22 | /* | |
23 | * Cmdstream submission: | |
24 | */ | |
25 | ||
26 | #define BO_INVALID_FLAGS ~(ETNA_SUBMIT_BO_READ | ETNA_SUBMIT_BO_WRITE) | |
27 | /* make sure these don't conflict w/ ETNAVIV_SUBMIT_BO_x */ | |
28 | #define BO_LOCKED 0x4000 | |
29 | #define BO_PINNED 0x2000 | |
30 | ||
31 | static inline void __user *to_user_ptr(u64 address) | |
32 | { | |
33 | return (void __user *)(uintptr_t)address; | |
34 | } | |
35 | ||
36 | static struct etnaviv_gem_submit *submit_create(struct drm_device *dev, | |
37 | struct etnaviv_gpu *gpu, size_t nr) | |
38 | { | |
39 | struct etnaviv_gem_submit *submit; | |
40 | size_t sz = size_vstruct(nr, sizeof(submit->bos[0]), sizeof(*submit)); | |
41 | ||
42 | submit = kmalloc(sz, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY); | |
43 | if (submit) { | |
44 | submit->dev = dev; | |
45 | submit->gpu = gpu; | |
46 | ||
47 | /* initially, until copy_from_user() and bo lookup succeeds: */ | |
48 | submit->nr_bos = 0; | |
49 | ||
50 | ww_acquire_init(&submit->ticket, &reservation_ww_class); | |
51 | } | |
52 | ||
53 | return submit; | |
54 | } | |
55 | ||
56 | static int submit_lookup_objects(struct etnaviv_gem_submit *submit, | |
57 | struct drm_file *file, struct drm_etnaviv_gem_submit_bo *submit_bos, | |
58 | unsigned nr_bos) | |
59 | { | |
60 | struct drm_etnaviv_gem_submit_bo *bo; | |
61 | unsigned i; | |
62 | int ret = 0; | |
63 | ||
64 | spin_lock(&file->table_lock); | |
65 | ||
66 | for (i = 0, bo = submit_bos; i < nr_bos; i++, bo++) { | |
67 | struct drm_gem_object *obj; | |
68 | ||
69 | if (bo->flags & BO_INVALID_FLAGS) { | |
70 | DRM_ERROR("invalid flags: %x\n", bo->flags); | |
71 | ret = -EINVAL; | |
72 | goto out_unlock; | |
73 | } | |
74 | ||
75 | submit->bos[i].flags = bo->flags; | |
76 | ||
77 | /* normally use drm_gem_object_lookup(), but for bulk lookup | |
78 | * all under single table_lock just hit object_idr directly: | |
79 | */ | |
80 | obj = idr_find(&file->object_idr, bo->handle); | |
81 | if (!obj) { | |
82 | DRM_ERROR("invalid handle %u at index %u\n", | |
83 | bo->handle, i); | |
84 | ret = -EINVAL; | |
85 | goto out_unlock; | |
86 | } | |
87 | ||
88 | /* | |
89 | * Take a refcount on the object. The file table lock | |
90 | * prevents the object_idr's refcount on this being dropped. | |
91 | */ | |
92 | drm_gem_object_reference(obj); | |
93 | ||
94 | submit->bos[i].obj = to_etnaviv_bo(obj); | |
95 | } | |
96 | ||
97 | out_unlock: | |
98 | submit->nr_bos = i; | |
99 | spin_unlock(&file->table_lock); | |
100 | ||
101 | return ret; | |
102 | } | |
103 | ||
104 | static void submit_unlock_object(struct etnaviv_gem_submit *submit, int i) | |
105 | { | |
106 | if (submit->bos[i].flags & BO_LOCKED) { | |
107 | struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj; | |
108 | ||
109 | ww_mutex_unlock(&etnaviv_obj->resv->lock); | |
110 | submit->bos[i].flags &= ~BO_LOCKED; | |
111 | } | |
112 | } | |
113 | ||
114 | static int submit_lock_objects(struct etnaviv_gem_submit *submit) | |
115 | { | |
116 | int contended, slow_locked = -1, i, ret = 0; | |
117 | ||
118 | retry: | |
119 | for (i = 0; i < submit->nr_bos; i++) { | |
120 | struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj; | |
121 | ||
122 | if (slow_locked == i) | |
123 | slow_locked = -1; | |
124 | ||
125 | contended = i; | |
126 | ||
127 | if (!(submit->bos[i].flags & BO_LOCKED)) { | |
128 | ret = ww_mutex_lock_interruptible(&etnaviv_obj->resv->lock, | |
129 | &submit->ticket); | |
130 | if (ret == -EALREADY) | |
131 | DRM_ERROR("BO at index %u already on submit list\n", | |
132 | i); | |
133 | if (ret) | |
134 | goto fail; | |
135 | submit->bos[i].flags |= BO_LOCKED; | |
136 | } | |
137 | } | |
138 | ||
139 | ww_acquire_done(&submit->ticket); | |
140 | ||
141 | return 0; | |
142 | ||
143 | fail: | |
144 | for (; i >= 0; i--) | |
145 | submit_unlock_object(submit, i); | |
146 | ||
147 | if (slow_locked > 0) | |
148 | submit_unlock_object(submit, slow_locked); | |
149 | ||
150 | if (ret == -EDEADLK) { | |
151 | struct etnaviv_gem_object *etnaviv_obj; | |
152 | ||
153 | etnaviv_obj = submit->bos[contended].obj; | |
154 | ||
155 | /* we lost out in a seqno race, lock and retry.. */ | |
156 | ret = ww_mutex_lock_slow_interruptible(&etnaviv_obj->resv->lock, | |
157 | &submit->ticket); | |
158 | if (!ret) { | |
159 | submit->bos[contended].flags |= BO_LOCKED; | |
160 | slow_locked = contended; | |
161 | goto retry; | |
162 | } | |
163 | } | |
164 | ||
165 | return ret; | |
166 | } | |
167 | ||
168 | static int submit_fence_sync(const struct etnaviv_gem_submit *submit) | |
169 | { | |
170 | unsigned int context = submit->gpu->fence_context; | |
171 | int i, ret = 0; | |
172 | ||
173 | for (i = 0; i < submit->nr_bos; i++) { | |
174 | struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj; | |
175 | bool write = submit->bos[i].flags & ETNA_SUBMIT_BO_WRITE; | |
176 | ||
177 | ret = etnaviv_gpu_fence_sync_obj(etnaviv_obj, context, write); | |
178 | if (ret) | |
179 | break; | |
180 | } | |
181 | ||
182 | return ret; | |
183 | } | |
184 | ||
185 | static void submit_unpin_objects(struct etnaviv_gem_submit *submit) | |
186 | { | |
187 | int i; | |
188 | ||
189 | for (i = 0; i < submit->nr_bos; i++) { | |
190 | struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj; | |
191 | ||
192 | if (submit->bos[i].flags & BO_PINNED) | |
193 | etnaviv_gem_put_iova(submit->gpu, &etnaviv_obj->base); | |
194 | ||
195 | submit->bos[i].iova = 0; | |
196 | submit->bos[i].flags &= ~BO_PINNED; | |
197 | } | |
198 | } | |
199 | ||
200 | static int submit_pin_objects(struct etnaviv_gem_submit *submit) | |
201 | { | |
202 | int i, ret = 0; | |
203 | ||
204 | for (i = 0; i < submit->nr_bos; i++) { | |
205 | struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj; | |
206 | u32 iova; | |
207 | ||
208 | ret = etnaviv_gem_get_iova(submit->gpu, &etnaviv_obj->base, | |
209 | &iova); | |
210 | if (ret) | |
211 | break; | |
212 | ||
213 | submit->bos[i].flags |= BO_PINNED; | |
214 | submit->bos[i].iova = iova; | |
215 | } | |
216 | ||
217 | return ret; | |
218 | } | |
219 | ||
220 | static int submit_bo(struct etnaviv_gem_submit *submit, u32 idx, | |
221 | struct etnaviv_gem_object **obj, u32 *iova) | |
222 | { | |
223 | if (idx >= submit->nr_bos) { | |
224 | DRM_ERROR("invalid buffer index: %u (out of %u)\n", | |
225 | idx, submit->nr_bos); | |
226 | return -EINVAL; | |
227 | } | |
228 | ||
229 | if (obj) | |
230 | *obj = submit->bos[idx].obj; | |
231 | if (iova) | |
232 | *iova = submit->bos[idx].iova; | |
233 | ||
234 | return 0; | |
235 | } | |
236 | ||
237 | /* process the reloc's and patch up the cmdstream as needed: */ | |
238 | static int submit_reloc(struct etnaviv_gem_submit *submit, void *stream, | |
239 | u32 size, const struct drm_etnaviv_gem_submit_reloc *relocs, | |
240 | u32 nr_relocs) | |
241 | { | |
242 | u32 i, last_offset = 0; | |
243 | u32 *ptr = stream; | |
244 | int ret; | |
245 | ||
246 | for (i = 0; i < nr_relocs; i++) { | |
247 | const struct drm_etnaviv_gem_submit_reloc *r = relocs + i; | |
248 | struct etnaviv_gem_object *bobj; | |
249 | u32 iova, off; | |
250 | ||
251 | if (unlikely(r->flags)) { | |
252 | DRM_ERROR("invalid reloc flags\n"); | |
253 | return -EINVAL; | |
254 | } | |
255 | ||
256 | if (r->submit_offset % 4) { | |
257 | DRM_ERROR("non-aligned reloc offset: %u\n", | |
258 | r->submit_offset); | |
259 | return -EINVAL; | |
260 | } | |
261 | ||
262 | /* offset in dwords: */ | |
263 | off = r->submit_offset / 4; | |
264 | ||
265 | if ((off >= size ) || | |
266 | (off < last_offset)) { | |
267 | DRM_ERROR("invalid offset %u at reloc %u\n", off, i); | |
268 | return -EINVAL; | |
269 | } | |
270 | ||
271 | ret = submit_bo(submit, r->reloc_idx, &bobj, &iova); | |
272 | if (ret) | |
273 | return ret; | |
274 | ||
275 | if (r->reloc_offset >= | |
276 | bobj->base.size - sizeof(*ptr)) { | |
277 | DRM_ERROR("relocation %u outside object", i); | |
278 | return -EINVAL; | |
279 | } | |
280 | ||
281 | ptr[off] = iova + r->reloc_offset; | |
282 | ||
283 | last_offset = off; | |
284 | } | |
285 | ||
286 | return 0; | |
287 | } | |
288 | ||
289 | static void submit_cleanup(struct etnaviv_gem_submit *submit) | |
290 | { | |
291 | unsigned i; | |
292 | ||
293 | for (i = 0; i < submit->nr_bos; i++) { | |
294 | struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj; | |
295 | ||
296 | submit_unlock_object(submit, i); | |
297 | drm_gem_object_unreference_unlocked(&etnaviv_obj->base); | |
298 | } | |
299 | ||
300 | ww_acquire_fini(&submit->ticket); | |
301 | kfree(submit); | |
302 | } | |
303 | ||
304 | int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data, | |
305 | struct drm_file *file) | |
306 | { | |
307 | struct etnaviv_drm_private *priv = dev->dev_private; | |
308 | struct drm_etnaviv_gem_submit *args = data; | |
309 | struct drm_etnaviv_gem_submit_reloc *relocs; | |
310 | struct drm_etnaviv_gem_submit_bo *bos; | |
311 | struct etnaviv_gem_submit *submit; | |
312 | struct etnaviv_cmdbuf *cmdbuf; | |
313 | struct etnaviv_gpu *gpu; | |
314 | void *stream; | |
315 | int ret; | |
316 | ||
317 | if (args->pipe >= ETNA_MAX_PIPES) | |
318 | return -EINVAL; | |
319 | ||
320 | gpu = priv->gpu[args->pipe]; | |
321 | if (!gpu) | |
322 | return -ENXIO; | |
323 | ||
324 | if (args->stream_size % 4) { | |
325 | DRM_ERROR("non-aligned cmdstream buffer size: %u\n", | |
326 | args->stream_size); | |
327 | return -EINVAL; | |
328 | } | |
329 | ||
330 | if (args->exec_state != ETNA_PIPE_3D && | |
331 | args->exec_state != ETNA_PIPE_2D && | |
332 | args->exec_state != ETNA_PIPE_VG) { | |
333 | DRM_ERROR("invalid exec_state: 0x%x\n", args->exec_state); | |
334 | return -EINVAL; | |
335 | } | |
336 | ||
337 | /* | |
338 | * Copy the command submission and bo array to kernel space in | |
339 | * one go, and do this outside of any locks. | |
340 | */ | |
341 | bos = drm_malloc_ab(args->nr_bos, sizeof(*bos)); | |
342 | relocs = drm_malloc_ab(args->nr_relocs, sizeof(*relocs)); | |
343 | stream = drm_malloc_ab(1, args->stream_size); | |
344 | cmdbuf = etnaviv_gpu_cmdbuf_new(gpu, ALIGN(args->stream_size, 8) + 8, | |
345 | args->nr_bos); | |
346 | if (!bos || !relocs || !stream || !cmdbuf) { | |
347 | ret = -ENOMEM; | |
348 | goto err_submit_cmds; | |
349 | } | |
350 | ||
351 | cmdbuf->exec_state = args->exec_state; | |
352 | cmdbuf->ctx = file->driver_priv; | |
353 | ||
354 | ret = copy_from_user(bos, to_user_ptr(args->bos), | |
355 | args->nr_bos * sizeof(*bos)); | |
356 | if (ret) { | |
357 | ret = -EFAULT; | |
358 | goto err_submit_cmds; | |
359 | } | |
360 | ||
361 | ret = copy_from_user(relocs, to_user_ptr(args->relocs), | |
362 | args->nr_relocs * sizeof(*relocs)); | |
363 | if (ret) { | |
364 | ret = -EFAULT; | |
365 | goto err_submit_cmds; | |
366 | } | |
367 | ||
368 | ret = copy_from_user(stream, to_user_ptr(args->stream), | |
369 | args->stream_size); | |
370 | if (ret) { | |
371 | ret = -EFAULT; | |
372 | goto err_submit_cmds; | |
373 | } | |
374 | ||
375 | submit = submit_create(dev, gpu, args->nr_bos); | |
376 | if (!submit) { | |
377 | ret = -ENOMEM; | |
378 | goto err_submit_cmds; | |
379 | } | |
380 | ||
381 | ret = submit_lookup_objects(submit, file, bos, args->nr_bos); | |
382 | if (ret) | |
383 | goto err_submit_objects; | |
384 | ||
385 | ret = submit_lock_objects(submit); | |
386 | if (ret) | |
387 | goto err_submit_objects; | |
388 | ||
389 | if (!etnaviv_cmd_validate_one(gpu, stream, args->stream_size / 4, | |
390 | relocs, args->nr_relocs)) { | |
391 | ret = -EINVAL; | |
392 | goto err_submit_objects; | |
393 | } | |
394 | ||
395 | ret = submit_fence_sync(submit); | |
396 | if (ret) | |
397 | goto err_submit_objects; | |
398 | ||
399 | ret = submit_pin_objects(submit); | |
400 | if (ret) | |
401 | goto out; | |
402 | ||
403 | ret = submit_reloc(submit, stream, args->stream_size / 4, | |
404 | relocs, args->nr_relocs); | |
405 | if (ret) | |
406 | goto out; | |
407 | ||
408 | memcpy(cmdbuf->vaddr, stream, args->stream_size); | |
409 | cmdbuf->user_size = ALIGN(args->stream_size, 8); | |
410 | ||
411 | ret = etnaviv_gpu_submit(gpu, submit, cmdbuf); | |
412 | if (ret == 0) | |
413 | cmdbuf = NULL; | |
414 | ||
415 | args->fence = submit->fence; | |
416 | ||
417 | out: | |
418 | submit_unpin_objects(submit); | |
419 | ||
420 | /* | |
421 | * If we're returning -EAGAIN, it may be due to the userptr code | |
422 | * wanting to run its workqueue outside of any locks. Flush our | |
423 | * workqueue to ensure that it is run in a timely manner. | |
424 | */ | |
425 | if (ret == -EAGAIN) | |
426 | flush_workqueue(priv->wq); | |
427 | ||
428 | err_submit_objects: | |
429 | submit_cleanup(submit); | |
430 | ||
431 | err_submit_cmds: | |
432 | /* if we still own the cmdbuf */ | |
433 | if (cmdbuf) | |
434 | etnaviv_gpu_cmdbuf_free(cmdbuf); | |
435 | if (stream) | |
436 | drm_free_large(stream); | |
437 | if (bos) | |
438 | drm_free_large(bos); | |
439 | if (relocs) | |
440 | drm_free_large(relocs); | |
441 | ||
442 | return ret; | |
443 | } |