Commit | Line | Data |
---|---|---|
fbb0de79 | 1 | // SPDX-License-Identifier: GPL-2.0 |
913965c4 | 2 | #include <linux/cred.h> |
fbb0de79 | 3 | #include <linux/device.h> |
fbb0de79 GH |
4 | #include <linux/dma-buf.h> |
5 | #include <linux/highmem.h> | |
913965c4 GH |
6 | #include <linux/init.h> |
7 | #include <linux/kernel.h> | |
fbb0de79 | 8 | #include <linux/memfd.h> |
913965c4 GH |
9 | #include <linux/miscdevice.h> |
10 | #include <linux/module.h> | |
11 | #include <linux/shmem_fs.h> | |
12 | #include <linux/slab.h> | |
13 | #include <linux/udmabuf.h> | |
fbb0de79 | 14 | |
dc4716d7 GH |
15 | static const u32 list_limit = 1024; /* udmabuf_create_list->count limit */ |
16 | static const size_t size_limit_mb = 64; /* total dmabuf size, in megabytes */ | |
17 | ||
fbb0de79 | 18 | struct udmabuf { |
b35f57c4 | 19 | pgoff_t pagecount; |
fbb0de79 | 20 | struct page **pages; |
284562e1 | 21 | struct sg_table *sg; |
c1bbed66 | 22 | struct miscdevice *device; |
fbb0de79 GH |
23 | }; |
24 | ||
300133d3 | 25 | static vm_fault_t udmabuf_vm_fault(struct vm_fault *vmf) |
fbb0de79 GH |
26 | { |
27 | struct vm_area_struct *vma = vmf->vma; | |
28 | struct udmabuf *ubuf = vma->vm_private_data; | |
29 | ||
fbb0de79 GH |
30 | vmf->page = ubuf->pages[vmf->pgoff]; |
31 | get_page(vmf->page); | |
32 | return 0; | |
33 | } | |
34 | ||
35 | static const struct vm_operations_struct udmabuf_vm_ops = { | |
36 | .fault = udmabuf_vm_fault, | |
37 | }; | |
38 | ||
39 | static int mmap_udmabuf(struct dma_buf *buf, struct vm_area_struct *vma) | |
40 | { | |
41 | struct udmabuf *ubuf = buf->priv; | |
42 | ||
43 | if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) == 0) | |
44 | return -EINVAL; | |
45 | ||
46 | vma->vm_ops = &udmabuf_vm_ops; | |
47 | vma->vm_private_data = ubuf; | |
48 | return 0; | |
49 | } | |
50 | ||
17a7ce20 GS |
51 | static struct sg_table *get_sg_table(struct device *dev, struct dma_buf *buf, |
52 | enum dma_data_direction direction) | |
fbb0de79 | 53 | { |
17a7ce20 | 54 | struct udmabuf *ubuf = buf->priv; |
fbb0de79 | 55 | struct sg_table *sg; |
a3e722da | 56 | int ret; |
fbb0de79 GH |
57 | |
58 | sg = kzalloc(sizeof(*sg), GFP_KERNEL); | |
59 | if (!sg) | |
a3e722da GH |
60 | return ERR_PTR(-ENOMEM); |
61 | ret = sg_alloc_table_from_pages(sg, ubuf->pages, ubuf->pagecount, | |
62 | 0, ubuf->pagecount << PAGE_SHIFT, | |
63 | GFP_KERNEL); | |
64 | if (ret < 0) | |
65 | goto err; | |
17a7ce20 | 66 | if (!dma_map_sg(dev, sg->sgl, sg->nents, direction)) { |
6f19eb21 | 67 | ret = -EINVAL; |
a3e722da | 68 | goto err; |
6f19eb21 | 69 | } |
fbb0de79 GH |
70 | return sg; |
71 | ||
a3e722da | 72 | err: |
fbb0de79 | 73 | sg_free_table(sg); |
fbb0de79 | 74 | kfree(sg); |
a3e722da | 75 | return ERR_PTR(ret); |
fbb0de79 GH |
76 | } |
77 | ||
17a7ce20 GS |
78 | static void put_sg_table(struct device *dev, struct sg_table *sg, |
79 | enum dma_data_direction direction) | |
80 | { | |
81 | dma_unmap_sg(dev, sg->sgl, sg->nents, direction); | |
82 | sg_free_table(sg); | |
83 | kfree(sg); | |
84 | } | |
85 | ||
86 | static struct sg_table *map_udmabuf(struct dma_buf_attachment *at, | |
87 | enum dma_data_direction direction) | |
88 | { | |
89 | return get_sg_table(at->dev, at->dmabuf, direction); | |
90 | } | |
91 | ||
fbb0de79 GH |
92 | static void unmap_udmabuf(struct dma_buf_attachment *at, |
93 | struct sg_table *sg, | |
94 | enum dma_data_direction direction) | |
95 | { | |
17a7ce20 | 96 | return put_sg_table(at->dev, sg, direction); |
fbb0de79 GH |
97 | } |
98 | ||
99 | static void release_udmabuf(struct dma_buf *buf) | |
100 | { | |
101 | struct udmabuf *ubuf = buf->priv; | |
284562e1 | 102 | struct device *dev = ubuf->device->this_device; |
fbb0de79 GH |
103 | pgoff_t pg; |
104 | ||
284562e1 GS |
105 | if (ubuf->sg) |
106 | put_sg_table(dev, ubuf->sg, DMA_BIDIRECTIONAL); | |
107 | ||
fbb0de79 GH |
108 | for (pg = 0; pg < ubuf->pagecount; pg++) |
109 | put_page(ubuf->pages[pg]); | |
110 | kfree(ubuf->pages); | |
111 | kfree(ubuf); | |
112 | } | |
113 | ||
284562e1 GS |
114 | static int begin_cpu_udmabuf(struct dma_buf *buf, |
115 | enum dma_data_direction direction) | |
116 | { | |
117 | struct udmabuf *ubuf = buf->priv; | |
118 | struct device *dev = ubuf->device->this_device; | |
119 | ||
120 | if (!ubuf->sg) { | |
121 | ubuf->sg = get_sg_table(dev, buf, direction); | |
122 | if (IS_ERR(ubuf->sg)) | |
123 | return PTR_ERR(ubuf->sg); | |
124 | } else { | |
1ffe0959 GS |
125 | dma_sync_sg_for_cpu(dev, ubuf->sg->sgl, ubuf->sg->nents, |
126 | direction); | |
284562e1 GS |
127 | } |
128 | ||
129 | return 0; | |
130 | } | |
131 | ||
132 | static int end_cpu_udmabuf(struct dma_buf *buf, | |
133 | enum dma_data_direction direction) | |
134 | { | |
135 | struct udmabuf *ubuf = buf->priv; | |
136 | struct device *dev = ubuf->device->this_device; | |
137 | ||
138 | if (!ubuf->sg) | |
139 | return -EINVAL; | |
140 | ||
1ffe0959 | 141 | dma_sync_sg_for_device(dev, ubuf->sg->sgl, ubuf->sg->nents, direction); |
284562e1 GS |
142 | return 0; |
143 | } | |
144 | ||
a3485289 | 145 | static const struct dma_buf_ops udmabuf_ops = { |
bc7a71da GS |
146 | .cache_sgt_mapping = true, |
147 | .map_dma_buf = map_udmabuf, | |
148 | .unmap_dma_buf = unmap_udmabuf, | |
149 | .release = release_udmabuf, | |
150 | .mmap = mmap_udmabuf, | |
284562e1 GS |
151 | .begin_cpu_access = begin_cpu_udmabuf, |
152 | .end_cpu_access = end_cpu_udmabuf, | |
fbb0de79 GH |
153 | }; |
154 | ||
155 | #define SEALS_WANTED (F_SEAL_SHRINK) | |
156 | #define SEALS_DENIED (F_SEAL_WRITE) | |
157 | ||
c1bbed66 GS |
158 | static long udmabuf_create(struct miscdevice *device, |
159 | struct udmabuf_create_list *head, | |
160 | struct udmabuf_create_item *list) | |
fbb0de79 GH |
161 | { |
162 | DEFINE_DMA_BUF_EXPORT_INFO(exp_info); | |
163 | struct file *memfd = NULL; | |
164 | struct udmabuf *ubuf; | |
165 | struct dma_buf *buf; | |
0d17455c | 166 | pgoff_t pgoff, pgcnt, pgidx, pgbuf = 0, pglimit; |
fbb0de79 GH |
167 | struct page *page; |
168 | int seals, ret = -EINVAL; | |
169 | u32 i, flags; | |
170 | ||
33f35429 | 171 | ubuf = kzalloc(sizeof(*ubuf), GFP_KERNEL); |
fbb0de79 GH |
172 | if (!ubuf) |
173 | return -ENOMEM; | |
174 | ||
dc4716d7 | 175 | pglimit = (size_limit_mb * 1024 * 1024) >> PAGE_SHIFT; |
fbb0de79 GH |
176 | for (i = 0; i < head->count; i++) { |
177 | if (!IS_ALIGNED(list[i].offset, PAGE_SIZE)) | |
0d17455c | 178 | goto err; |
fbb0de79 | 179 | if (!IS_ALIGNED(list[i].size, PAGE_SIZE)) |
0d17455c | 180 | goto err; |
fbb0de79 | 181 | ubuf->pagecount += list[i].size >> PAGE_SHIFT; |
dc4716d7 | 182 | if (ubuf->pagecount > pglimit) |
0d17455c | 183 | goto err; |
fbb0de79 | 184 | } |
33f35429 | 185 | ubuf->pages = kmalloc_array(ubuf->pagecount, sizeof(*ubuf->pages), |
fbb0de79 GH |
186 | GFP_KERNEL); |
187 | if (!ubuf->pages) { | |
188 | ret = -ENOMEM; | |
0d17455c | 189 | goto err; |
fbb0de79 GH |
190 | } |
191 | ||
192 | pgbuf = 0; | |
193 | for (i = 0; i < head->count; i++) { | |
7a1c67d7 | 194 | ret = -EBADFD; |
fbb0de79 GH |
195 | memfd = fget(list[i].memfd); |
196 | if (!memfd) | |
0d17455c | 197 | goto err; |
fbb0de79 | 198 | if (!shmem_mapping(file_inode(memfd)->i_mapping)) |
0d17455c | 199 | goto err; |
fbb0de79 | 200 | seals = memfd_fcntl(memfd, F_GET_SEALS, 0); |
7a1c67d7 GH |
201 | if (seals == -EINVAL) |
202 | goto err; | |
203 | ret = -EINVAL; | |
204 | if ((seals & SEALS_WANTED) != SEALS_WANTED || | |
fbb0de79 | 205 | (seals & SEALS_DENIED) != 0) |
0d17455c | 206 | goto err; |
fbb0de79 GH |
207 | pgoff = list[i].offset >> PAGE_SHIFT; |
208 | pgcnt = list[i].size >> PAGE_SHIFT; | |
209 | for (pgidx = 0; pgidx < pgcnt; pgidx++) { | |
210 | page = shmem_read_mapping_page( | |
211 | file_inode(memfd)->i_mapping, pgoff + pgidx); | |
212 | if (IS_ERR(page)) { | |
213 | ret = PTR_ERR(page); | |
0d17455c | 214 | goto err; |
fbb0de79 GH |
215 | } |
216 | ubuf->pages[pgbuf++] = page; | |
217 | } | |
218 | fput(memfd); | |
0d17455c | 219 | memfd = NULL; |
fbb0de79 | 220 | } |
fbb0de79 GH |
221 | |
222 | exp_info.ops = &udmabuf_ops; | |
223 | exp_info.size = ubuf->pagecount << PAGE_SHIFT; | |
224 | exp_info.priv = ubuf; | |
5c074eea | 225 | exp_info.flags = O_RDWR; |
fbb0de79 | 226 | |
c1bbed66 | 227 | ubuf->device = device; |
fbb0de79 GH |
228 | buf = dma_buf_export(&exp_info); |
229 | if (IS_ERR(buf)) { | |
230 | ret = PTR_ERR(buf); | |
0d17455c | 231 | goto err; |
fbb0de79 GH |
232 | } |
233 | ||
234 | flags = 0; | |
235 | if (head->flags & UDMABUF_FLAGS_CLOEXEC) | |
236 | flags |= O_CLOEXEC; | |
237 | return dma_buf_fd(buf, flags); | |
238 | ||
0d17455c | 239 | err: |
fbb0de79 GH |
240 | while (pgbuf > 0) |
241 | put_page(ubuf->pages[--pgbuf]); | |
683a0e63 GS |
242 | if (memfd) |
243 | fput(memfd); | |
fbb0de79 GH |
244 | kfree(ubuf->pages); |
245 | kfree(ubuf); | |
246 | return ret; | |
247 | } | |
248 | ||
249 | static long udmabuf_ioctl_create(struct file *filp, unsigned long arg) | |
250 | { | |
251 | struct udmabuf_create create; | |
252 | struct udmabuf_create_list head; | |
253 | struct udmabuf_create_item list; | |
254 | ||
255 | if (copy_from_user(&create, (void __user *)arg, | |
33f35429 | 256 | sizeof(create))) |
fbb0de79 GH |
257 | return -EFAULT; |
258 | ||
259 | head.flags = create.flags; | |
260 | head.count = 1; | |
261 | list.memfd = create.memfd; | |
262 | list.offset = create.offset; | |
263 | list.size = create.size; | |
264 | ||
c1bbed66 | 265 | return udmabuf_create(filp->private_data, &head, &list); |
fbb0de79 GH |
266 | } |
267 | ||
268 | static long udmabuf_ioctl_create_list(struct file *filp, unsigned long arg) | |
269 | { | |
270 | struct udmabuf_create_list head; | |
271 | struct udmabuf_create_item *list; | |
272 | int ret = -EINVAL; | |
273 | u32 lsize; | |
274 | ||
275 | if (copy_from_user(&head, (void __user *)arg, sizeof(head))) | |
276 | return -EFAULT; | |
dc4716d7 | 277 | if (head.count > list_limit) |
fbb0de79 GH |
278 | return -EINVAL; |
279 | lsize = sizeof(struct udmabuf_create_item) * head.count; | |
280 | list = memdup_user((void __user *)(arg + sizeof(head)), lsize); | |
281 | if (IS_ERR(list)) | |
282 | return PTR_ERR(list); | |
283 | ||
c1bbed66 | 284 | ret = udmabuf_create(filp->private_data, &head, list); |
fbb0de79 GH |
285 | kfree(list); |
286 | return ret; | |
287 | } | |
288 | ||
289 | static long udmabuf_ioctl(struct file *filp, unsigned int ioctl, | |
290 | unsigned long arg) | |
291 | { | |
292 | long ret; | |
293 | ||
294 | switch (ioctl) { | |
295 | case UDMABUF_CREATE: | |
296 | ret = udmabuf_ioctl_create(filp, arg); | |
297 | break; | |
298 | case UDMABUF_CREATE_LIST: | |
299 | ret = udmabuf_ioctl_create_list(filp, arg); | |
300 | break; | |
301 | default: | |
52499d9c | 302 | ret = -ENOTTY; |
fbb0de79 GH |
303 | break; |
304 | } | |
305 | return ret; | |
306 | } | |
307 | ||
308 | static const struct file_operations udmabuf_fops = { | |
309 | .owner = THIS_MODULE, | |
310 | .unlocked_ioctl = udmabuf_ioctl, | |
311 | }; | |
312 | ||
313 | static struct miscdevice udmabuf_misc = { | |
314 | .minor = MISC_DYNAMIC_MINOR, | |
315 | .name = "udmabuf", | |
316 | .fops = &udmabuf_fops, | |
317 | }; | |
318 | ||
319 | static int __init udmabuf_dev_init(void) | |
320 | { | |
321 | return misc_register(&udmabuf_misc); | |
322 | } | |
323 | ||
324 | static void __exit udmabuf_dev_exit(void) | |
325 | { | |
326 | misc_deregister(&udmabuf_misc); | |
327 | } | |
328 | ||
329 | module_init(udmabuf_dev_init) | |
330 | module_exit(udmabuf_dev_exit) | |
331 | ||
332 | MODULE_AUTHOR("Gerd Hoffmann <kraxel@redhat.com>"); | |
333 | MODULE_LICENSE("GPL v2"); |