Commit | Line | Data |
---|---|---|
932d6562 OA |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | ||
3 | /* | |
4 | * Xen dma-buf functionality for gntdev. | |
5 | * | |
a240d6e4 OA |
6 | * DMA buffer implementation is based on drivers/gpu/drm/drm_prime.c. |
7 | * | |
932d6562 OA |
8 | * Copyright (c) 2018 Oleksandr Andrushchenko, EPAM Systems Inc. |
9 | */ | |
10 | ||
11 | #include <linux/kernel.h> | |
12 | #include <linux/errno.h> | |
a240d6e4 | 13 | #include <linux/dma-buf.h> |
932d6562 OA |
14 | #include <linux/slab.h> |
15 | #include <linux/types.h> | |
16 | #include <linux/uaccess.h> | |
17 | ||
18 | #include <xen/xen.h> | |
19 | #include <xen/grant_table.h> | |
20 | ||
21 | #include "gntdev-common.h" | |
22 | #include "gntdev-dmabuf.h" | |
23 | ||
bf8dc55b OA |
24 | #ifndef GRANT_INVALID_REF |
25 | /* | |
26 | * Note on usage of grant reference 0 as invalid grant reference: | |
27 | * grant reference 0 is valid, but never exposed to a driver, | |
28 | * because of the fact it is already in use/reserved by the PV console. | |
29 | */ | |
30 | #define GRANT_INVALID_REF 0 | |
31 | #endif | |
32 | ||
a240d6e4 OA |
33 | struct gntdev_dmabuf { |
34 | struct gntdev_dmabuf_priv *priv; | |
35 | struct dma_buf *dmabuf; | |
36 | struct list_head next; | |
37 | int fd; | |
38 | ||
39 | union { | |
40 | struct { | |
41 | /* Exported buffers are reference counted. */ | |
42 | struct kref refcount; | |
43 | ||
44 | struct gntdev_priv *priv; | |
45 | struct gntdev_grant_map *map; | |
46 | } exp; | |
bf8dc55b OA |
47 | struct { |
48 | /* Granted references of the imported buffer. */ | |
49 | grant_ref_t *refs; | |
50 | /* Scatter-gather table of the imported buffer. */ | |
51 | struct sg_table *sgt; | |
52 | /* dma-buf attachment of the imported buffer. */ | |
53 | struct dma_buf_attachment *attach; | |
54 | } imp; | |
a240d6e4 OA |
55 | } u; |
56 | ||
57 | /* Number of pages this buffer has. */ | |
58 | int nr_pages; | |
59 | /* Pages of this buffer. */ | |
60 | struct page **pages; | |
61 | }; | |
62 | ||
63 | struct gntdev_dmabuf_wait_obj { | |
64 | struct list_head next; | |
65 | struct gntdev_dmabuf *gntdev_dmabuf; | |
66 | struct completion completion; | |
67 | }; | |
68 | ||
69 | struct gntdev_dmabuf_attachment { | |
70 | struct sg_table *sgt; | |
71 | enum dma_data_direction dir; | |
72 | }; | |
73 | ||
932d6562 OA |
74 | struct gntdev_dmabuf_priv { |
75 | /* List of exported DMA buffers. */ | |
76 | struct list_head exp_list; | |
77 | /* List of wait objects. */ | |
78 | struct list_head exp_wait_list; | |
bf8dc55b OA |
79 | /* List of imported DMA buffers. */ |
80 | struct list_head imp_list; | |
932d6562 OA |
81 | /* This is the lock which protects dma_buf_xxx lists. */ |
82 | struct mutex lock; | |
fa13e665 OA |
83 | /* |
84 | * We reference this file while exporting dma-bufs, so | |
85 | * the grant device context is not destroyed while there are | |
86 | * external users alive. | |
87 | */ | |
88 | struct file *filp; | |
932d6562 OA |
89 | }; |
90 | ||
91 | /* DMA buffer export support. */ | |
92 | ||
93 | /* Implementation of wait for exported DMA buffer to be released. */ | |
94 | ||
a240d6e4 OA |
95 | static void dmabuf_exp_release(struct kref *kref); |
96 | ||
97 | static struct gntdev_dmabuf_wait_obj * | |
98 | dmabuf_exp_wait_obj_new(struct gntdev_dmabuf_priv *priv, | |
99 | struct gntdev_dmabuf *gntdev_dmabuf) | |
100 | { | |
101 | struct gntdev_dmabuf_wait_obj *obj; | |
102 | ||
103 | obj = kzalloc(sizeof(*obj), GFP_KERNEL); | |
104 | if (!obj) | |
105 | return ERR_PTR(-ENOMEM); | |
106 | ||
107 | init_completion(&obj->completion); | |
108 | obj->gntdev_dmabuf = gntdev_dmabuf; | |
109 | ||
110 | mutex_lock(&priv->lock); | |
111 | list_add(&obj->next, &priv->exp_wait_list); | |
112 | /* Put our reference and wait for gntdev_dmabuf's release to fire. */ | |
113 | kref_put(&gntdev_dmabuf->u.exp.refcount, dmabuf_exp_release); | |
114 | mutex_unlock(&priv->lock); | |
115 | return obj; | |
116 | } | |
117 | ||
118 | static void dmabuf_exp_wait_obj_free(struct gntdev_dmabuf_priv *priv, | |
119 | struct gntdev_dmabuf_wait_obj *obj) | |
120 | { | |
121 | mutex_lock(&priv->lock); | |
122 | list_del(&obj->next); | |
123 | mutex_unlock(&priv->lock); | |
124 | kfree(obj); | |
125 | } | |
126 | ||
127 | static int dmabuf_exp_wait_obj_wait(struct gntdev_dmabuf_wait_obj *obj, | |
128 | u32 wait_to_ms) | |
129 | { | |
130 | if (wait_for_completion_timeout(&obj->completion, | |
131 | msecs_to_jiffies(wait_to_ms)) <= 0) | |
132 | return -ETIMEDOUT; | |
133 | ||
134 | return 0; | |
135 | } | |
136 | ||
137 | static void dmabuf_exp_wait_obj_signal(struct gntdev_dmabuf_priv *priv, | |
138 | struct gntdev_dmabuf *gntdev_dmabuf) | |
139 | { | |
140 | struct gntdev_dmabuf_wait_obj *obj; | |
141 | ||
142 | list_for_each_entry(obj, &priv->exp_wait_list, next) | |
143 | if (obj->gntdev_dmabuf == gntdev_dmabuf) { | |
144 | pr_debug("Found gntdev_dmabuf in the wait list, wake\n"); | |
145 | complete_all(&obj->completion); | |
146 | break; | |
147 | } | |
148 | } | |
149 | ||
150 | static struct gntdev_dmabuf * | |
151 | dmabuf_exp_wait_obj_get_dmabuf(struct gntdev_dmabuf_priv *priv, int fd) | |
152 | { | |
153 | struct gntdev_dmabuf *gntdev_dmabuf, *ret = ERR_PTR(-ENOENT); | |
154 | ||
155 | mutex_lock(&priv->lock); | |
156 | list_for_each_entry(gntdev_dmabuf, &priv->exp_list, next) | |
157 | if (gntdev_dmabuf->fd == fd) { | |
158 | pr_debug("Found gntdev_dmabuf in the wait list\n"); | |
159 | kref_get(&gntdev_dmabuf->u.exp.refcount); | |
160 | ret = gntdev_dmabuf; | |
161 | break; | |
162 | } | |
163 | mutex_unlock(&priv->lock); | |
164 | return ret; | |
165 | } | |
166 | ||
932d6562 OA |
167 | static int dmabuf_exp_wait_released(struct gntdev_dmabuf_priv *priv, int fd, |
168 | int wait_to_ms) | |
169 | { | |
a240d6e4 OA |
170 | struct gntdev_dmabuf *gntdev_dmabuf; |
171 | struct gntdev_dmabuf_wait_obj *obj; | |
172 | int ret; | |
173 | ||
174 | pr_debug("Will wait for dma-buf with fd %d\n", fd); | |
175 | /* | |
176 | * Try to find the DMA buffer: if not found means that | |
177 | * either the buffer has already been released or file descriptor | |
178 | * provided is wrong. | |
179 | */ | |
180 | gntdev_dmabuf = dmabuf_exp_wait_obj_get_dmabuf(priv, fd); | |
181 | if (IS_ERR(gntdev_dmabuf)) | |
182 | return PTR_ERR(gntdev_dmabuf); | |
183 | ||
184 | /* | |
185 | * gntdev_dmabuf still exists and is reference count locked by us now, | |
186 | * so prepare to wait: allocate wait object and add it to the wait list, | |
187 | * so we can find it on release. | |
188 | */ | |
189 | obj = dmabuf_exp_wait_obj_new(priv, gntdev_dmabuf); | |
190 | if (IS_ERR(obj)) | |
191 | return PTR_ERR(obj); | |
192 | ||
193 | ret = dmabuf_exp_wait_obj_wait(obj, wait_to_ms); | |
194 | dmabuf_exp_wait_obj_free(priv, obj); | |
195 | return ret; | |
196 | } | |
197 | ||
198 | /* DMA buffer export support. */ | |
199 | ||
200 | static struct sg_table * | |
201 | dmabuf_pages_to_sgt(struct page **pages, unsigned int nr_pages) | |
202 | { | |
203 | struct sg_table *sgt; | |
204 | int ret; | |
205 | ||
206 | sgt = kmalloc(sizeof(*sgt), GFP_KERNEL); | |
207 | if (!sgt) { | |
208 | ret = -ENOMEM; | |
209 | goto out; | |
210 | } | |
211 | ||
212 | ret = sg_alloc_table_from_pages(sgt, pages, nr_pages, 0, | |
213 | nr_pages << PAGE_SHIFT, | |
214 | GFP_KERNEL); | |
215 | if (ret) | |
216 | goto out; | |
217 | ||
218 | return sgt; | |
219 | ||
220 | out: | |
221 | kfree(sgt); | |
222 | return ERR_PTR(ret); | |
223 | } | |
224 | ||
225 | static int dmabuf_exp_ops_attach(struct dma_buf *dma_buf, | |
a240d6e4 OA |
226 | struct dma_buf_attachment *attach) |
227 | { | |
228 | struct gntdev_dmabuf_attachment *gntdev_dmabuf_attach; | |
229 | ||
230 | gntdev_dmabuf_attach = kzalloc(sizeof(*gntdev_dmabuf_attach), | |
231 | GFP_KERNEL); | |
232 | if (!gntdev_dmabuf_attach) | |
233 | return -ENOMEM; | |
234 | ||
235 | gntdev_dmabuf_attach->dir = DMA_NONE; | |
236 | attach->priv = gntdev_dmabuf_attach; | |
237 | return 0; | |
238 | } | |
239 | ||
240 | static void dmabuf_exp_ops_detach(struct dma_buf *dma_buf, | |
241 | struct dma_buf_attachment *attach) | |
242 | { | |
243 | struct gntdev_dmabuf_attachment *gntdev_dmabuf_attach = attach->priv; | |
244 | ||
245 | if (gntdev_dmabuf_attach) { | |
246 | struct sg_table *sgt = gntdev_dmabuf_attach->sgt; | |
247 | ||
248 | if (sgt) { | |
249 | if (gntdev_dmabuf_attach->dir != DMA_NONE) | |
250 | dma_unmap_sg_attrs(attach->dev, sgt->sgl, | |
251 | sgt->nents, | |
252 | gntdev_dmabuf_attach->dir, | |
253 | DMA_ATTR_SKIP_CPU_SYNC); | |
254 | sg_free_table(sgt); | |
255 | } | |
256 | ||
257 | kfree(sgt); | |
258 | kfree(gntdev_dmabuf_attach); | |
259 | attach->priv = NULL; | |
260 | } | |
261 | } | |
262 | ||
263 | static struct sg_table * | |
264 | dmabuf_exp_ops_map_dma_buf(struct dma_buf_attachment *attach, | |
265 | enum dma_data_direction dir) | |
266 | { | |
267 | struct gntdev_dmabuf_attachment *gntdev_dmabuf_attach = attach->priv; | |
268 | struct gntdev_dmabuf *gntdev_dmabuf = attach->dmabuf->priv; | |
269 | struct sg_table *sgt; | |
270 | ||
271 | pr_debug("Mapping %d pages for dev %p\n", gntdev_dmabuf->nr_pages, | |
272 | attach->dev); | |
273 | ||
274 | if (dir == DMA_NONE || !gntdev_dmabuf_attach) | |
275 | return ERR_PTR(-EINVAL); | |
276 | ||
277 | /* Return the cached mapping when possible. */ | |
278 | if (gntdev_dmabuf_attach->dir == dir) | |
279 | return gntdev_dmabuf_attach->sgt; | |
280 | ||
281 | /* | |
282 | * Two mappings with different directions for the same attachment are | |
283 | * not allowed. | |
284 | */ | |
285 | if (gntdev_dmabuf_attach->dir != DMA_NONE) | |
286 | return ERR_PTR(-EBUSY); | |
287 | ||
288 | sgt = dmabuf_pages_to_sgt(gntdev_dmabuf->pages, | |
289 | gntdev_dmabuf->nr_pages); | |
290 | if (!IS_ERR(sgt)) { | |
291 | if (!dma_map_sg_attrs(attach->dev, sgt->sgl, sgt->nents, dir, | |
292 | DMA_ATTR_SKIP_CPU_SYNC)) { | |
293 | sg_free_table(sgt); | |
294 | kfree(sgt); | |
295 | sgt = ERR_PTR(-ENOMEM); | |
296 | } else { | |
297 | gntdev_dmabuf_attach->sgt = sgt; | |
298 | gntdev_dmabuf_attach->dir = dir; | |
299 | } | |
300 | } | |
301 | if (IS_ERR(sgt)) | |
302 | pr_debug("Failed to map sg table for dev %p\n", attach->dev); | |
303 | return sgt; | |
304 | } | |
305 | ||
306 | static void dmabuf_exp_ops_unmap_dma_buf(struct dma_buf_attachment *attach, | |
307 | struct sg_table *sgt, | |
308 | enum dma_data_direction dir) | |
309 | { | |
310 | /* Not implemented. The unmap is done at dmabuf_exp_ops_detach(). */ | |
311 | } | |
312 | ||
313 | static void dmabuf_exp_release(struct kref *kref) | |
314 | { | |
315 | struct gntdev_dmabuf *gntdev_dmabuf = | |
316 | container_of(kref, struct gntdev_dmabuf, u.exp.refcount); | |
317 | ||
318 | dmabuf_exp_wait_obj_signal(gntdev_dmabuf->priv, gntdev_dmabuf); | |
319 | list_del(&gntdev_dmabuf->next); | |
fa13e665 | 320 | fput(gntdev_dmabuf->priv->filp); |
a240d6e4 OA |
321 | kfree(gntdev_dmabuf); |
322 | } | |
323 | ||
324 | static void dmabuf_exp_remove_map(struct gntdev_priv *priv, | |
325 | struct gntdev_grant_map *map) | |
326 | { | |
327 | mutex_lock(&priv->lock); | |
328 | list_del(&map->next); | |
329 | gntdev_put_map(NULL /* already removed */, map); | |
330 | mutex_unlock(&priv->lock); | |
331 | } | |
332 | ||
333 | static void dmabuf_exp_ops_release(struct dma_buf *dma_buf) | |
334 | { | |
335 | struct gntdev_dmabuf *gntdev_dmabuf = dma_buf->priv; | |
336 | struct gntdev_dmabuf_priv *priv = gntdev_dmabuf->priv; | |
337 | ||
338 | dmabuf_exp_remove_map(gntdev_dmabuf->u.exp.priv, | |
339 | gntdev_dmabuf->u.exp.map); | |
340 | mutex_lock(&priv->lock); | |
341 | kref_put(&gntdev_dmabuf->u.exp.refcount, dmabuf_exp_release); | |
342 | mutex_unlock(&priv->lock); | |
343 | } | |
344 | ||
345 | static void *dmabuf_exp_ops_kmap(struct dma_buf *dma_buf, | |
346 | unsigned long page_num) | |
347 | { | |
348 | /* Not implemented. */ | |
349 | return NULL; | |
350 | } | |
351 | ||
352 | static void dmabuf_exp_ops_kunmap(struct dma_buf *dma_buf, | |
353 | unsigned long page_num, void *addr) | |
354 | { | |
355 | /* Not implemented. */ | |
356 | } | |
357 | ||
358 | static int dmabuf_exp_ops_mmap(struct dma_buf *dma_buf, | |
359 | struct vm_area_struct *vma) | |
360 | { | |
361 | /* Not implemented. */ | |
362 | return 0; | |
363 | } | |
364 | ||
365 | static const struct dma_buf_ops dmabuf_exp_ops = { | |
366 | .attach = dmabuf_exp_ops_attach, | |
367 | .detach = dmabuf_exp_ops_detach, | |
368 | .map_dma_buf = dmabuf_exp_ops_map_dma_buf, | |
369 | .unmap_dma_buf = dmabuf_exp_ops_unmap_dma_buf, | |
370 | .release = dmabuf_exp_ops_release, | |
371 | .map = dmabuf_exp_ops_kmap, | |
372 | .unmap = dmabuf_exp_ops_kunmap, | |
373 | .mmap = dmabuf_exp_ops_mmap, | |
374 | }; | |
375 | ||
376 | struct gntdev_dmabuf_export_args { | |
377 | struct gntdev_priv *priv; | |
378 | struct gntdev_grant_map *map; | |
379 | struct gntdev_dmabuf_priv *dmabuf_priv; | |
380 | struct device *dev; | |
381 | int count; | |
382 | struct page **pages; | |
383 | u32 fd; | |
384 | }; | |
385 | ||
386 | static int dmabuf_exp_from_pages(struct gntdev_dmabuf_export_args *args) | |
387 | { | |
388 | DEFINE_DMA_BUF_EXPORT_INFO(exp_info); | |
389 | struct gntdev_dmabuf *gntdev_dmabuf; | |
390 | int ret; | |
391 | ||
392 | gntdev_dmabuf = kzalloc(sizeof(*gntdev_dmabuf), GFP_KERNEL); | |
393 | if (!gntdev_dmabuf) | |
394 | return -ENOMEM; | |
395 | ||
396 | kref_init(&gntdev_dmabuf->u.exp.refcount); | |
397 | ||
398 | gntdev_dmabuf->priv = args->dmabuf_priv; | |
399 | gntdev_dmabuf->nr_pages = args->count; | |
400 | gntdev_dmabuf->pages = args->pages; | |
401 | gntdev_dmabuf->u.exp.priv = args->priv; | |
402 | gntdev_dmabuf->u.exp.map = args->map; | |
403 | ||
404 | exp_info.exp_name = KBUILD_MODNAME; | |
405 | if (args->dev->driver && args->dev->driver->owner) | |
406 | exp_info.owner = args->dev->driver->owner; | |
407 | else | |
408 | exp_info.owner = THIS_MODULE; | |
409 | exp_info.ops = &dmabuf_exp_ops; | |
410 | exp_info.size = args->count << PAGE_SHIFT; | |
411 | exp_info.flags = O_RDWR; | |
412 | exp_info.priv = gntdev_dmabuf; | |
413 | ||
414 | gntdev_dmabuf->dmabuf = dma_buf_export(&exp_info); | |
415 | if (IS_ERR(gntdev_dmabuf->dmabuf)) { | |
416 | ret = PTR_ERR(gntdev_dmabuf->dmabuf); | |
417 | gntdev_dmabuf->dmabuf = NULL; | |
418 | goto fail; | |
419 | } | |
420 | ||
421 | ret = dma_buf_fd(gntdev_dmabuf->dmabuf, O_CLOEXEC); | |
422 | if (ret < 0) | |
423 | goto fail; | |
424 | ||
425 | gntdev_dmabuf->fd = ret; | |
426 | args->fd = ret; | |
427 | ||
428 | pr_debug("Exporting DMA buffer with fd %d\n", ret); | |
429 | ||
430 | mutex_lock(&args->dmabuf_priv->lock); | |
431 | list_add(&gntdev_dmabuf->next, &args->dmabuf_priv->exp_list); | |
432 | mutex_unlock(&args->dmabuf_priv->lock); | |
fa13e665 | 433 | get_file(gntdev_dmabuf->priv->filp); |
a240d6e4 OA |
434 | return 0; |
435 | ||
436 | fail: | |
437 | if (gntdev_dmabuf->dmabuf) | |
438 | dma_buf_put(gntdev_dmabuf->dmabuf); | |
439 | kfree(gntdev_dmabuf); | |
440 | return ret; | |
441 | } | |
442 | ||
443 | static struct gntdev_grant_map * | |
444 | dmabuf_exp_alloc_backing_storage(struct gntdev_priv *priv, int dmabuf_flags, | |
445 | int count) | |
446 | { | |
447 | struct gntdev_grant_map *map; | |
448 | ||
449 | if (unlikely(count <= 0)) | |
450 | return ERR_PTR(-EINVAL); | |
451 | ||
452 | if ((dmabuf_flags & GNTDEV_DMA_FLAG_WC) && | |
453 | (dmabuf_flags & GNTDEV_DMA_FLAG_COHERENT)) { | |
454 | pr_debug("Wrong dma-buf flags: 0x%x\n", dmabuf_flags); | |
455 | return ERR_PTR(-EINVAL); | |
456 | } | |
457 | ||
458 | map = gntdev_alloc_map(priv, count, dmabuf_flags); | |
459 | if (!map) | |
460 | return ERR_PTR(-ENOMEM); | |
461 | ||
462 | if (unlikely(gntdev_account_mapped_pages(count))) { | |
463 | pr_debug("can't map %d pages: over limit\n", count); | |
464 | gntdev_put_map(NULL, map); | |
465 | return ERR_PTR(-ENOMEM); | |
466 | } | |
467 | return map; | |
932d6562 OA |
468 | } |
469 | ||
470 | static int dmabuf_exp_from_refs(struct gntdev_priv *priv, int flags, | |
471 | int count, u32 domid, u32 *refs, u32 *fd) | |
472 | { | |
a240d6e4 OA |
473 | struct gntdev_grant_map *map; |
474 | struct gntdev_dmabuf_export_args args; | |
475 | int i, ret; | |
476 | ||
477 | map = dmabuf_exp_alloc_backing_storage(priv, flags, count); | |
478 | if (IS_ERR(map)) | |
479 | return PTR_ERR(map); | |
480 | ||
481 | for (i = 0; i < count; i++) { | |
482 | map->grants[i].domid = domid; | |
483 | map->grants[i].ref = refs[i]; | |
484 | } | |
485 | ||
486 | mutex_lock(&priv->lock); | |
487 | gntdev_add_map(priv, map); | |
488 | mutex_unlock(&priv->lock); | |
489 | ||
490 | map->flags |= GNTMAP_host_map; | |
491 | #if defined(CONFIG_X86) | |
492 | map->flags |= GNTMAP_device_map; | |
493 | #endif | |
494 | ||
495 | ret = gntdev_map_grant_pages(map); | |
496 | if (ret < 0) | |
497 | goto out; | |
498 | ||
499 | args.priv = priv; | |
500 | args.map = map; | |
501 | args.dev = priv->dma_dev; | |
502 | args.dmabuf_priv = priv->dmabuf_priv; | |
503 | args.count = map->count; | |
504 | args.pages = map->pages; | |
505 | args.fd = -1; /* Shut up unnecessary gcc warning for i386 */ | |
506 | ||
507 | ret = dmabuf_exp_from_pages(&args); | |
508 | if (ret < 0) | |
509 | goto out; | |
510 | ||
511 | *fd = args.fd; | |
512 | return 0; | |
513 | ||
514 | out: | |
515 | dmabuf_exp_remove_map(priv, map); | |
516 | return ret; | |
932d6562 OA |
517 | } |
518 | ||
519 | /* DMA buffer import support. */ | |
520 | ||
bf8dc55b OA |
521 | static int |
522 | dmabuf_imp_grant_foreign_access(struct page **pages, u32 *refs, | |
523 | int count, int domid) | |
524 | { | |
525 | grant_ref_t priv_gref_head; | |
526 | int i, ret; | |
527 | ||
528 | ret = gnttab_alloc_grant_references(count, &priv_gref_head); | |
529 | if (ret < 0) { | |
530 | pr_debug("Cannot allocate grant references, ret %d\n", ret); | |
531 | return ret; | |
532 | } | |
533 | ||
534 | for (i = 0; i < count; i++) { | |
535 | int cur_ref; | |
536 | ||
537 | cur_ref = gnttab_claim_grant_reference(&priv_gref_head); | |
538 | if (cur_ref < 0) { | |
539 | ret = cur_ref; | |
540 | pr_debug("Cannot claim grant reference, ret %d\n", ret); | |
541 | goto out; | |
542 | } | |
543 | ||
544 | gnttab_grant_foreign_access_ref(cur_ref, domid, | |
545 | xen_page_to_gfn(pages[i]), 0); | |
546 | refs[i] = cur_ref; | |
547 | } | |
548 | ||
549 | return 0; | |
550 | ||
551 | out: | |
552 | gnttab_free_grant_references(priv_gref_head); | |
553 | return ret; | |
554 | } | |
555 | ||
556 | static void dmabuf_imp_end_foreign_access(u32 *refs, int count) | |
557 | { | |
558 | int i; | |
559 | ||
560 | for (i = 0; i < count; i++) | |
561 | if (refs[i] != GRANT_INVALID_REF) | |
562 | gnttab_end_foreign_access(refs[i], 0, 0UL); | |
563 | } | |
564 | ||
565 | static void dmabuf_imp_free_storage(struct gntdev_dmabuf *gntdev_dmabuf) | |
566 | { | |
567 | kfree(gntdev_dmabuf->pages); | |
568 | kfree(gntdev_dmabuf->u.imp.refs); | |
569 | kfree(gntdev_dmabuf); | |
570 | } | |
571 | ||
572 | static struct gntdev_dmabuf *dmabuf_imp_alloc_storage(int count) | |
573 | { | |
574 | struct gntdev_dmabuf *gntdev_dmabuf; | |
575 | int i; | |
576 | ||
577 | gntdev_dmabuf = kzalloc(sizeof(*gntdev_dmabuf), GFP_KERNEL); | |
578 | if (!gntdev_dmabuf) | |
2789e83c | 579 | goto fail_no_free; |
bf8dc55b OA |
580 | |
581 | gntdev_dmabuf->u.imp.refs = kcalloc(count, | |
582 | sizeof(gntdev_dmabuf->u.imp.refs[0]), | |
583 | GFP_KERNEL); | |
584 | if (!gntdev_dmabuf->u.imp.refs) | |
585 | goto fail; | |
586 | ||
587 | gntdev_dmabuf->pages = kcalloc(count, | |
588 | sizeof(gntdev_dmabuf->pages[0]), | |
589 | GFP_KERNEL); | |
590 | if (!gntdev_dmabuf->pages) | |
591 | goto fail; | |
592 | ||
593 | gntdev_dmabuf->nr_pages = count; | |
594 | ||
595 | for (i = 0; i < count; i++) | |
596 | gntdev_dmabuf->u.imp.refs[i] = GRANT_INVALID_REF; | |
597 | ||
598 | return gntdev_dmabuf; | |
599 | ||
600 | fail: | |
601 | dmabuf_imp_free_storage(gntdev_dmabuf); | |
2789e83c | 602 | fail_no_free: |
bf8dc55b OA |
603 | return ERR_PTR(-ENOMEM); |
604 | } | |
605 | ||
932d6562 OA |
606 | static struct gntdev_dmabuf * |
607 | dmabuf_imp_to_refs(struct gntdev_dmabuf_priv *priv, struct device *dev, | |
608 | int fd, int count, int domid) | |
609 | { | |
bf8dc55b OA |
610 | struct gntdev_dmabuf *gntdev_dmabuf, *ret; |
611 | struct dma_buf *dma_buf; | |
612 | struct dma_buf_attachment *attach; | |
613 | struct sg_table *sgt; | |
614 | struct sg_page_iter sg_iter; | |
615 | int i; | |
616 | ||
617 | dma_buf = dma_buf_get(fd); | |
618 | if (IS_ERR(dma_buf)) | |
619 | return ERR_CAST(dma_buf); | |
620 | ||
621 | gntdev_dmabuf = dmabuf_imp_alloc_storage(count); | |
622 | if (IS_ERR(gntdev_dmabuf)) { | |
623 | ret = gntdev_dmabuf; | |
624 | goto fail_put; | |
625 | } | |
626 | ||
627 | gntdev_dmabuf->priv = priv; | |
628 | gntdev_dmabuf->fd = fd; | |
629 | ||
630 | attach = dma_buf_attach(dma_buf, dev); | |
631 | if (IS_ERR(attach)) { | |
632 | ret = ERR_CAST(attach); | |
633 | goto fail_free_obj; | |
634 | } | |
635 | ||
636 | gntdev_dmabuf->u.imp.attach = attach; | |
637 | ||
638 | sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL); | |
639 | if (IS_ERR(sgt)) { | |
640 | ret = ERR_CAST(sgt); | |
641 | goto fail_detach; | |
642 | } | |
643 | ||
644 | /* Check number of pages that imported buffer has. */ | |
645 | if (attach->dmabuf->size != gntdev_dmabuf->nr_pages << PAGE_SHIFT) { | |
646 | ret = ERR_PTR(-EINVAL); | |
647 | pr_debug("DMA buffer has %zu pages, user-space expects %d\n", | |
648 | attach->dmabuf->size, gntdev_dmabuf->nr_pages); | |
649 | goto fail_unmap; | |
650 | } | |
651 | ||
652 | gntdev_dmabuf->u.imp.sgt = sgt; | |
653 | ||
654 | /* Now convert sgt to array of pages and check for page validity. */ | |
655 | i = 0; | |
656 | for_each_sg_page(sgt->sgl, &sg_iter, sgt->nents, 0) { | |
657 | struct page *page = sg_page_iter_page(&sg_iter); | |
658 | /* | |
659 | * Check if page is valid: this can happen if we are given | |
660 | * a page from VRAM or other resources which are not backed | |
661 | * by a struct page. | |
662 | */ | |
663 | if (!pfn_valid(page_to_pfn(page))) { | |
664 | ret = ERR_PTR(-EINVAL); | |
665 | goto fail_unmap; | |
666 | } | |
667 | ||
668 | gntdev_dmabuf->pages[i++] = page; | |
669 | } | |
670 | ||
671 | ret = ERR_PTR(dmabuf_imp_grant_foreign_access(gntdev_dmabuf->pages, | |
672 | gntdev_dmabuf->u.imp.refs, | |
673 | count, domid)); | |
674 | if (IS_ERR(ret)) | |
675 | goto fail_end_access; | |
676 | ||
677 | pr_debug("Imported DMA buffer with fd %d\n", fd); | |
678 | ||
679 | mutex_lock(&priv->lock); | |
680 | list_add(&gntdev_dmabuf->next, &priv->imp_list); | |
681 | mutex_unlock(&priv->lock); | |
682 | ||
683 | return gntdev_dmabuf; | |
684 | ||
685 | fail_end_access: | |
686 | dmabuf_imp_end_foreign_access(gntdev_dmabuf->u.imp.refs, count); | |
687 | fail_unmap: | |
688 | dma_buf_unmap_attachment(attach, sgt, DMA_BIDIRECTIONAL); | |
689 | fail_detach: | |
690 | dma_buf_detach(dma_buf, attach); | |
691 | fail_free_obj: | |
692 | dmabuf_imp_free_storage(gntdev_dmabuf); | |
693 | fail_put: | |
694 | dma_buf_put(dma_buf); | |
695 | return ret; | |
932d6562 OA |
696 | } |
697 | ||
bf8dc55b OA |
698 | /* |
699 | * Find the hyper dma-buf by its file descriptor and remove | |
700 | * it from the buffer's list. | |
701 | */ | |
702 | static struct gntdev_dmabuf * | |
703 | dmabuf_imp_find_unlink(struct gntdev_dmabuf_priv *priv, int fd) | |
932d6562 | 704 | { |
bf8dc55b OA |
705 | struct gntdev_dmabuf *q, *gntdev_dmabuf, *ret = ERR_PTR(-ENOENT); |
706 | ||
707 | mutex_lock(&priv->lock); | |
708 | list_for_each_entry_safe(gntdev_dmabuf, q, &priv->imp_list, next) { | |
709 | if (gntdev_dmabuf->fd == fd) { | |
710 | pr_debug("Found gntdev_dmabuf in the import list\n"); | |
711 | ret = gntdev_dmabuf; | |
712 | list_del(&gntdev_dmabuf->next); | |
713 | break; | |
714 | } | |
715 | } | |
716 | mutex_unlock(&priv->lock); | |
717 | return ret; | |
932d6562 OA |
718 | } |
719 | ||
720 | static int dmabuf_imp_release(struct gntdev_dmabuf_priv *priv, u32 fd) | |
721 | { | |
bf8dc55b OA |
722 | struct gntdev_dmabuf *gntdev_dmabuf; |
723 | struct dma_buf_attachment *attach; | |
724 | struct dma_buf *dma_buf; | |
725 | ||
726 | gntdev_dmabuf = dmabuf_imp_find_unlink(priv, fd); | |
727 | if (IS_ERR(gntdev_dmabuf)) | |
728 | return PTR_ERR(gntdev_dmabuf); | |
729 | ||
730 | pr_debug("Releasing DMA buffer with fd %d\n", fd); | |
731 | ||
732 | dmabuf_imp_end_foreign_access(gntdev_dmabuf->u.imp.refs, | |
733 | gntdev_dmabuf->nr_pages); | |
734 | ||
735 | attach = gntdev_dmabuf->u.imp.attach; | |
736 | ||
737 | if (gntdev_dmabuf->u.imp.sgt) | |
738 | dma_buf_unmap_attachment(attach, gntdev_dmabuf->u.imp.sgt, | |
739 | DMA_BIDIRECTIONAL); | |
740 | dma_buf = attach->dmabuf; | |
741 | dma_buf_detach(attach->dmabuf, attach); | |
742 | dma_buf_put(dma_buf); | |
743 | ||
744 | dmabuf_imp_free_storage(gntdev_dmabuf); | |
745 | return 0; | |
932d6562 OA |
746 | } |
747 | ||
068e79f4 OA |
748 | static void dmabuf_imp_release_all(struct gntdev_dmabuf_priv *priv) |
749 | { | |
750 | struct gntdev_dmabuf *q, *gntdev_dmabuf; | |
751 | ||
752 | list_for_each_entry_safe(gntdev_dmabuf, q, &priv->imp_list, next) | |
753 | dmabuf_imp_release(priv, gntdev_dmabuf->fd); | |
754 | } | |
755 | ||
932d6562 OA |
756 | /* DMA buffer IOCTL support. */ |
757 | ||
758 | long gntdev_ioctl_dmabuf_exp_from_refs(struct gntdev_priv *priv, int use_ptemod, | |
759 | struct ioctl_gntdev_dmabuf_exp_from_refs __user *u) | |
760 | { | |
761 | struct ioctl_gntdev_dmabuf_exp_from_refs op; | |
762 | u32 *refs; | |
763 | long ret; | |
764 | ||
765 | if (use_ptemod) { | |
766 | pr_debug("Cannot provide dma-buf: use_ptemode %d\n", | |
767 | use_ptemod); | |
768 | return -EINVAL; | |
769 | } | |
770 | ||
771 | if (copy_from_user(&op, u, sizeof(op)) != 0) | |
772 | return -EFAULT; | |
773 | ||
774 | if (unlikely(op.count <= 0)) | |
775 | return -EINVAL; | |
776 | ||
777 | refs = kcalloc(op.count, sizeof(*refs), GFP_KERNEL); | |
778 | if (!refs) | |
779 | return -ENOMEM; | |
780 | ||
781 | if (copy_from_user(refs, u->refs, sizeof(*refs) * op.count) != 0) { | |
782 | ret = -EFAULT; | |
783 | goto out; | |
784 | } | |
785 | ||
786 | ret = dmabuf_exp_from_refs(priv, op.flags, op.count, | |
787 | op.domid, refs, &op.fd); | |
788 | if (ret) | |
789 | goto out; | |
790 | ||
791 | if (copy_to_user(u, &op, sizeof(op)) != 0) | |
792 | ret = -EFAULT; | |
793 | ||
794 | out: | |
795 | kfree(refs); | |
796 | return ret; | |
797 | } | |
798 | ||
799 | long gntdev_ioctl_dmabuf_exp_wait_released(struct gntdev_priv *priv, | |
800 | struct ioctl_gntdev_dmabuf_exp_wait_released __user *u) | |
801 | { | |
802 | struct ioctl_gntdev_dmabuf_exp_wait_released op; | |
803 | ||
804 | if (copy_from_user(&op, u, sizeof(op)) != 0) | |
805 | return -EFAULT; | |
806 | ||
807 | return dmabuf_exp_wait_released(priv->dmabuf_priv, op.fd, | |
808 | op.wait_to_ms); | |
809 | } | |
810 | ||
811 | long gntdev_ioctl_dmabuf_imp_to_refs(struct gntdev_priv *priv, | |
812 | struct ioctl_gntdev_dmabuf_imp_to_refs __user *u) | |
813 | { | |
814 | struct ioctl_gntdev_dmabuf_imp_to_refs op; | |
815 | struct gntdev_dmabuf *gntdev_dmabuf; | |
816 | long ret; | |
817 | ||
818 | if (copy_from_user(&op, u, sizeof(op)) != 0) | |
819 | return -EFAULT; | |
820 | ||
821 | if (unlikely(op.count <= 0)) | |
822 | return -EINVAL; | |
823 | ||
824 | gntdev_dmabuf = dmabuf_imp_to_refs(priv->dmabuf_priv, | |
825 | priv->dma_dev, op.fd, | |
826 | op.count, op.domid); | |
827 | if (IS_ERR(gntdev_dmabuf)) | |
828 | return PTR_ERR(gntdev_dmabuf); | |
829 | ||
bf8dc55b | 830 | if (copy_to_user(u->refs, gntdev_dmabuf->u.imp.refs, |
932d6562 OA |
831 | sizeof(*u->refs) * op.count) != 0) { |
832 | ret = -EFAULT; | |
833 | goto out_release; | |
834 | } | |
835 | return 0; | |
836 | ||
837 | out_release: | |
838 | dmabuf_imp_release(priv->dmabuf_priv, op.fd); | |
839 | return ret; | |
840 | } | |
841 | ||
842 | long gntdev_ioctl_dmabuf_imp_release(struct gntdev_priv *priv, | |
843 | struct ioctl_gntdev_dmabuf_imp_release __user *u) | |
844 | { | |
845 | struct ioctl_gntdev_dmabuf_imp_release op; | |
846 | ||
847 | if (copy_from_user(&op, u, sizeof(op)) != 0) | |
848 | return -EFAULT; | |
849 | ||
850 | return dmabuf_imp_release(priv->dmabuf_priv, op.fd); | |
851 | } | |
852 | ||
fa13e665 | 853 | struct gntdev_dmabuf_priv *gntdev_dmabuf_init(struct file *filp) |
932d6562 OA |
854 | { |
855 | struct gntdev_dmabuf_priv *priv; | |
856 | ||
857 | priv = kzalloc(sizeof(*priv), GFP_KERNEL); | |
858 | if (!priv) | |
859 | return ERR_PTR(-ENOMEM); | |
860 | ||
a240d6e4 OA |
861 | mutex_init(&priv->lock); |
862 | INIT_LIST_HEAD(&priv->exp_list); | |
863 | INIT_LIST_HEAD(&priv->exp_wait_list); | |
bf8dc55b | 864 | INIT_LIST_HEAD(&priv->imp_list); |
a240d6e4 | 865 | |
fa13e665 OA |
866 | priv->filp = filp; |
867 | ||
932d6562 OA |
868 | return priv; |
869 | } | |
870 | ||
871 | void gntdev_dmabuf_fini(struct gntdev_dmabuf_priv *priv) | |
872 | { | |
068e79f4 | 873 | dmabuf_imp_release_all(priv); |
932d6562 OA |
874 | kfree(priv); |
875 | } |