drivers/dma-buf/udmabuf.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/cred.h>
   3 #include <linux/device.h>
   4 #include <linux/dma-buf.h>
   5 #include <linux/dma-resv.h>
   6 #include <linux/highmem.h>
   7 #include <linux/init.h>
   8 #include <linux/kernel.h>
   9 #include <linux/memfd.h>
  10 #include <linux/miscdevice.h>
  11 #include <linux/module.h>
  12 #include <linux/shmem_fs.h>
  13 #include <linux/slab.h>
  14 #include <linux/udmabuf.h>
  15 #include <linux/hugetlb.h>
  16 #include <linux/vmalloc.h>
  17 #include <linux/iosys-map.h>
  18
  19 static int list_limit = 1024;
  20 module_param(list_limit, int, 0644);
  21 MODULE_PARM_DESC(list_limit, "udmabuf_create_list->count limit. Default is 1024.");
  22
  23 static int size_limit_mb = 64;
  24 module_param(size_limit_mb, int, 0644);
  25 MODULE_PARM_DESC(size_limit_mb, "Max size of a dmabuf, in megabytes. Default is 64.");
  26
  27 struct udmabuf {
  28         pgoff_t pagecount;
  29         struct page **pages;
  30         struct sg_table *sg;
  31         struct miscdevice *device;
  32 };
  33
  34 static vm_fault_t udmabuf_vm_fault(struct vm_fault *vmf)
  35 {
  36         struct vm_area_struct *vma = vmf->vma;
  37         struct udmabuf *ubuf = vma->vm_private_data;
  38         pgoff_t pgoff = vmf->pgoff;
  39
  40         if (pgoff >= ubuf->pagecount)
  41                 return VM_FAULT_SIGBUS;
  42         vmf->page = ubuf->pages[pgoff];
  43         get_page(vmf->page);
  44         return 0;
  45 }
  46
  47 static const struct vm_operations_struct udmabuf_vm_ops = {
  48         .fault = udmabuf_vm_fault,
  49 };
  50
  51 static int mmap_udmabuf(struct dma_buf *buf, struct vm_area_struct *vma)
  52 {
  53         struct udmabuf *ubuf = buf->priv;
  54
  55         dma_resv_assert_held(buf->resv);
  56
  57         if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) == 0)
  58                 return -EINVAL;
  59
  60         vma->vm_ops = &udmabuf_vm_ops;
  61         vma->vm_private_data = ubuf;
  62         return 0;
  63 }
  64
  65 static int vmap_udmabuf(struct dma_buf *buf, struct iosys_map *map)
  66 {
  67         struct udmabuf *ubuf = buf->priv;
  68         void *vaddr;
  69
  70         dma_resv_assert_held(buf->resv);
  71
  72         vaddr = vm_map_ram(ubuf->pages, ubuf->pagecount, -1);
  73         if (!vaddr)
  74                 return -EINVAL;
  75
  76         iosys_map_set_vaddr(map, vaddr);
  77         return 0;
  78 }
  79
  80 static void vunmap_udmabuf(struct dma_buf *buf, struct iosys_map *map)
  81 {
  82         struct udmabuf *ubuf = buf->priv;
  83
  84         dma_resv_assert_held(buf->resv);
  85
  86         vm_unmap_ram(map->vaddr, ubuf->pagecount);
  87 }
  88
  89 static struct sg_table *get_sg_table(struct device *dev, struct dma_buf *buf,
  90                                      enum dma_data_direction direction)
  91 {
  92         struct udmabuf *ubuf = buf->priv;
  93         struct sg_table *sg;
  94         int ret;
  95
  96         sg = kzalloc(sizeof(*sg), GFP_KERNEL);
  97         if (!sg)
  98                 return ERR_PTR(-ENOMEM);
  99         ret = sg_alloc_table_from_pages(sg, ubuf->pages, ubuf->pagecount,
 100                                         0, ubuf->pagecount << PAGE_SHIFT,
 101                                         GFP_KERNEL);
 102         if (ret < 0)
 103                 goto err;
 104         ret = dma_map_sgtable(dev, sg, direction, 0);
 105         if (ret < 0)
 106                 goto err;
 107         return sg;
 108
 109 err:
 110         sg_free_table(sg);
 111         kfree(sg);
 112         return ERR_PTR(ret);
 113 }
 114
 115 static void put_sg_table(struct device *dev, struct sg_table *sg,
 116                          enum dma_data_direction direction)
 117 {
 118         dma_unmap_sgtable(dev, sg, direction, 0);
 119         sg_free_table(sg);
 120         kfree(sg);
 121 }
 122
 123 static struct sg_table *map_udmabuf(struct dma_buf_attachment *at,
 124                                     enum dma_data_direction direction)
 125 {
 126         return get_sg_table(at->dev, at->dmabuf, direction);
 127 }
 128
 129 static void unmap_udmabuf(struct dma_buf_attachment *at,
 130                           struct sg_table *sg,
 131                           enum dma_data_direction direction)
 132 {
 133         return put_sg_table(at->dev, sg, direction);
 134 }
 135
 136 static void release_udmabuf(struct dma_buf *buf)
 137 {
 138         struct udmabuf *ubuf = buf->priv;
 139         struct device *dev = ubuf->device->this_device;
 140         pgoff_t pg;
 141
 142         if (ubuf->sg)
 143                 put_sg_table(dev, ubuf->sg, DMA_BIDIRECTIONAL);
 144
 145         for (pg = 0; pg < ubuf->pagecount; pg++)
 146                 put_page(ubuf->pages[pg]);
 147         kfree(ubuf->pages);
 148         kfree(ubuf);
 149 }
 150
 151 static int begin_cpu_udmabuf(struct dma_buf *buf,
 152                              enum dma_data_direction direction)
 153 {
 154         struct udmabuf *ubuf = buf->priv;
 155         struct device *dev = ubuf->device->this_device;
 156         int ret = 0;
 157
 158         if (!ubuf->sg) {
 159                 ubuf->sg = get_sg_table(dev, buf, direction);
 160                 if (IS_ERR(ubuf->sg)) {
 161                         ret = PTR_ERR(ubuf->sg);
 162                         ubuf->sg = NULL;
 163                 }
 164         } else {
 165                 dma_sync_sg_for_cpu(dev, ubuf->sg->sgl, ubuf->sg->nents,
 166                                     direction);
 167         }
 168
 169         return ret;
 170 }
 171
 172 static int end_cpu_udmabuf(struct dma_buf *buf,
 173                            enum dma_data_direction direction)
 174 {
 175         struct udmabuf *ubuf = buf->priv;
 176         struct device *dev = ubuf->device->this_device;
 177
 178         if (!ubuf->sg)
 179                 return -EINVAL;
 180
 181         dma_sync_sg_for_device(dev, ubuf->sg->sgl, ubuf->sg->nents, direction);
 182         return 0;
 183 }
 184
 185 static const struct dma_buf_ops udmabuf_ops = {
 186         .cache_sgt_mapping = true,
 187         .map_dma_buf       = map_udmabuf,
 188         .unmap_dma_buf     = unmap_udmabuf,
 189         .release           = release_udmabuf,
 190         .mmap              = mmap_udmabuf,
 191         .vmap              = vmap_udmabuf,
 192         .vunmap            = vunmap_udmabuf,
 193         .begin_cpu_access  = begin_cpu_udmabuf,
 194         .end_cpu_access    = end_cpu_udmabuf,
 195 };
 196
 197 #define SEALS_WANTED (F_SEAL_SHRINK)
 198 #define SEALS_DENIED (F_SEAL_WRITE)
 199
 200 static long udmabuf_create(struct miscdevice *device,
 201                            struct udmabuf_create_list *head,
 202                            struct udmabuf_create_item *list)
 203 {
 204         DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
 205         struct file *memfd = NULL;
 206         struct address_space *mapping = NULL;
 207         struct udmabuf *ubuf;
 208         struct dma_buf *buf;
 209         pgoff_t pgoff, pgcnt, pgidx, pgbuf = 0, pglimit;
 210         struct page *page, *hpage = NULL;
 211         pgoff_t subpgoff, maxsubpgs;
 212         struct hstate *hpstate;
 213         int seals, ret = -EINVAL;
 214         u32 i, flags;
 215
 216         ubuf = kzalloc(sizeof(*ubuf), GFP_KERNEL);
 217         if (!ubuf)
 218                 return -ENOMEM;
 219
 220         pglimit = (size_limit_mb * 1024 * 1024) >> PAGE_SHIFT;
 221         for (i = 0; i < head->count; i++) {
 222                 if (!IS_ALIGNED(list[i].offset, PAGE_SIZE))
 223                         goto err;
 224                 if (!IS_ALIGNED(list[i].size, PAGE_SIZE))
 225                         goto err;
 226                 ubuf->pagecount += list[i].size >> PAGE_SHIFT;
 227                 if (ubuf->pagecount > pglimit)
 228                         goto err;
 229         }
 230
 231         if (!ubuf->pagecount)
 232                 goto err;
 233
 234         ubuf->pages = kmalloc_array(ubuf->pagecount, sizeof(*ubuf->pages),
 235                                     GFP_KERNEL);
 236         if (!ubuf->pages) {
 237                 ret = -ENOMEM;
 238                 goto err;
 239         }
 240
 241         pgbuf = 0;
 242         for (i = 0; i < head->count; i++) {
 243                 ret = -EBADFD;
 244                 memfd = fget(list[i].memfd);
 245                 if (!memfd)
 246                         goto err;
 247                 mapping = memfd->f_mapping;
 248                 if (!shmem_mapping(mapping) && !is_file_hugepages(memfd))
 249                         goto err;
 250                 seals = memfd_fcntl(memfd, F_GET_SEALS, 0);
 251                 if (seals == -EINVAL)
 252                         goto err;
 253                 ret = -EINVAL;
 254                 if ((seals & SEALS_WANTED) != SEALS_WANTED ||
 255                     (seals & SEALS_DENIED) != 0)
 256                         goto err;
 257                 pgoff = list[i].offset >> PAGE_SHIFT;
 258                 pgcnt = list[i].size   >> PAGE_SHIFT;
 259                 if (is_file_hugepages(memfd)) {
 260                         hpstate = hstate_file(memfd);
 261                         pgoff = list[i].offset >> huge_page_shift(hpstate);
 262                         subpgoff = (list[i].offset &
 263                                     ~huge_page_mask(hpstate)) >> PAGE_SHIFT;
 264                         maxsubpgs = huge_page_size(hpstate) >> PAGE_SHIFT;
 265                 }
 266                 for (pgidx = 0; pgidx < pgcnt; pgidx++) {
 267                         if (is_file_hugepages(memfd)) {
 268                                 if (!hpage) {
 269                                         hpage = find_get_page_flags(mapping, pgoff,
 270                                                                     FGP_ACCESSED);
 271                                         if (!hpage) {
 272                                                 ret = -EINVAL;
 273                                                 goto err;
 274                                         }
 275                                 }
 276                                 page = hpage + subpgoff;
 277                                 get_page(page);
 278                                 subpgoff++;
 279                                 if (subpgoff == maxsubpgs) {
 280                                         put_page(hpage);
 281                                         hpage = NULL;
 282                                         subpgoff = 0;
 283                                         pgoff++;
 284                                 }
 285                         } else {
 286                                 page = shmem_read_mapping_page(mapping,
 287                                                                pgoff + pgidx);
 288                                 if (IS_ERR(page)) {
 289                                         ret = PTR_ERR(page);
 290                                         goto err;
 291                                 }
 292                         }
 293                         ubuf->pages[pgbuf++] = page;
 294                 }
 295                 fput(memfd);
 296                 memfd = NULL;
 297                 if (hpage) {
 298                         put_page(hpage);
 299                         hpage = NULL;
 300                 }
 301         }
 302
 303         exp_info.ops  = &udmabuf_ops;
 304         exp_info.size = ubuf->pagecount << PAGE_SHIFT;
 305         exp_info.priv = ubuf;
 306         exp_info.flags = O_RDWR;
 307
 308         ubuf->device = device;
 309         buf = dma_buf_export(&exp_info);
 310         if (IS_ERR(buf)) {
 311                 ret = PTR_ERR(buf);
 312                 goto err;
 313         }
 314
 315         flags = 0;
 316         if (head->flags & UDMABUF_FLAGS_CLOEXEC)
 317                 flags |= O_CLOEXEC;
 318         return dma_buf_fd(buf, flags);
 319
 320 err:
 321         while (pgbuf > 0)
 322                 put_page(ubuf->pages[--pgbuf]);
 323         if (memfd)
 324                 fput(memfd);
 325         kfree(ubuf->pages);
 326         kfree(ubuf);
 327         return ret;
 328 }
 329
 330 static long udmabuf_ioctl_create(struct file *filp, unsigned long arg)
 331 {
 332         struct udmabuf_create create;
 333         struct udmabuf_create_list head;
 334         struct udmabuf_create_item list;
 335
 336         if (copy_from_user(&create, (void __user *)arg,
 337                            sizeof(create)))
 338                 return -EFAULT;
 339
 340         head.flags  = create.flags;
 341         head.count  = 1;
 342         list.memfd  = create.memfd;
 343         list.offset = create.offset;
 344         list.size   = create.size;
 345
 346         return udmabuf_create(filp->private_data, &head, &list);
 347 }
 348
 349 static long udmabuf_ioctl_create_list(struct file *filp, unsigned long arg)
 350 {
 351         struct udmabuf_create_list head;
 352         struct udmabuf_create_item *list;
 353         int ret = -EINVAL;
 354         u32 lsize;
 355
 356         if (copy_from_user(&head, (void __user *)arg, sizeof(head)))
 357                 return -EFAULT;
 358         if (head.count > list_limit)
 359                 return -EINVAL;
 360         lsize = sizeof(struct udmabuf_create_item) * head.count;
 361         list = memdup_user((void __user *)(arg + sizeof(head)), lsize);
 362         if (IS_ERR(list))
 363                 return PTR_ERR(list);
 364
 365         ret = udmabuf_create(filp->private_data, &head, list);
 366         kfree(list);
 367         return ret;
 368 }
 369
 370 static long udmabuf_ioctl(struct file *filp, unsigned int ioctl,
 371                           unsigned long arg)
 372 {
 373         long ret;
 374
 375         switch (ioctl) {
 376         case UDMABUF_CREATE:
 377                 ret = udmabuf_ioctl_create(filp, arg);
 378                 break;
 379         case UDMABUF_CREATE_LIST:
 380                 ret = udmabuf_ioctl_create_list(filp, arg);
 381                 break;
 382         default:
 383                 ret = -ENOTTY;
 384                 break;
 385         }
 386         return ret;
 387 }
 388
 389 static const struct file_operations udmabuf_fops = {
 390         .owner          = THIS_MODULE,
 391         .unlocked_ioctl = udmabuf_ioctl,
 392 #ifdef CONFIG_COMPAT
 393         .compat_ioctl   = udmabuf_ioctl,
 394 #endif
 395 };
 396
 397 static struct miscdevice udmabuf_misc = {
 398         .minor          = MISC_DYNAMIC_MINOR,
 399         .name           = "udmabuf",
 400         .fops           = &udmabuf_fops,
 401 };
 402
 403 static int __init udmabuf_dev_init(void)
 404 {
 405         int ret;
 406
 407         ret = misc_register(&udmabuf_misc);
 408         if (ret < 0) {
 409                 pr_err("Could not initialize udmabuf device\n");
 410                 return ret;
 411         }
 412
 413         ret = dma_coerce_mask_and_coherent(udmabuf_misc.this_device,
 414                                            DMA_BIT_MASK(64));
 415         if (ret < 0) {
 416                 pr_err("Could not setup DMA mask for udmabuf device\n");
 417                 misc_deregister(&udmabuf_misc);
 418                 return ret;
 419         }
 420
 421         return 0;
 422 }
 423
 424 static void __exit udmabuf_dev_exit(void)
 425 {
 426         misc_deregister(&udmabuf_misc);
 427 }
 428
 429 module_init(udmabuf_dev_init)
 430 module_exit(udmabuf_dev_exit)
 431
 432 MODULE_AUTHOR("Gerd Hoffmann <kraxel@redhat.com>");