drivers/dma/idxd/cdev.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
   3 #include <linux/init.h>
   4 #include <linux/kernel.h>
   5 #include <linux/module.h>
   6 #include <linux/pci.h>
   7 #include <linux/device.h>
   8 #include <linux/sched/task.h>
   9 #include <linux/io-64-nonatomic-lo-hi.h>
  10 #include <linux/cdev.h>
  11 #include <linux/fs.h>
  12 #include <linux/poll.h>
  13 #include <linux/iommu.h>
  14 #include <uapi/linux/idxd.h>
  15 #include "registers.h"
  16 #include "idxd.h"
  17
  18 struct idxd_cdev_context {
  19         const char *name;
  20         dev_t devt;
  21         struct ida minor_ida;
  22 };
  23
  24 /*
  25  * ictx is an array based off of accelerator types. enum idxd_type
  26  * is used as index
  27  */
  28 static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = {
  29         { .name = "dsa" },
  30         { .name = "iax" }
  31 };
  32
  33 struct idxd_user_context {
  34         struct idxd_wq *wq;
  35         struct task_struct *task;
  36         unsigned int pasid;
  37         unsigned int flags;
  38         struct iommu_sva *sva;
  39 };
  40
  41 static void idxd_cdev_dev_release(struct device *dev)
  42 {
  43         struct idxd_cdev *idxd_cdev = dev_to_cdev(dev);
  44         struct idxd_cdev_context *cdev_ctx;
  45         struct idxd_wq *wq = idxd_cdev->wq;
  46
  47         cdev_ctx = &ictx[wq->idxd->data->type];
  48         ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor);
  49         kfree(idxd_cdev);
  50 }
  51
  52 static struct device_type idxd_cdev_device_type = {
  53         .name = "idxd_cdev",
  54         .release = idxd_cdev_dev_release,
  55 };
  56
  57 static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode)
  58 {
  59         struct cdev *cdev = inode->i_cdev;
  60
  61         return container_of(cdev, struct idxd_cdev, cdev);
  62 }
  63
  64 static inline struct idxd_wq *inode_wq(struct inode *inode)
  65 {
  66         struct idxd_cdev *idxd_cdev = inode_idxd_cdev(inode);
  67
  68         return idxd_cdev->wq;
  69 }
  70
  71 static int idxd_cdev_open(struct inode *inode, struct file *filp)
  72 {
  73         struct idxd_user_context *ctx;
  74         struct idxd_device *idxd;
  75         struct idxd_wq *wq;
  76         struct device *dev;
  77         int rc = 0;
  78         struct iommu_sva *sva;
  79         unsigned int pasid;
  80
  81         wq = inode_wq(inode);
  82         idxd = wq->idxd;
  83         dev = &idxd->pdev->dev;
  84
  85         dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq));
  86
  87         ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
  88         if (!ctx)
  89                 return -ENOMEM;
  90
  91         mutex_lock(&wq->wq_lock);
  92
  93         if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) {
  94                 rc = -EBUSY;
  95                 goto failed;
  96         }
  97
  98         ctx->wq = wq;
  99         filp->private_data = ctx;
 100
 101         if (device_user_pasid_enabled(idxd)) {
 102                 sva = iommu_sva_bind_device(dev, current->mm);
 103                 if (IS_ERR(sva)) {
 104                         rc = PTR_ERR(sva);
 105                         dev_err(dev, "pasid allocation failed: %d\n", rc);
 106                         goto failed;
 107                 }
 108
 109                 pasid = iommu_sva_get_pasid(sva);
 110                 if (pasid == IOMMU_PASID_INVALID) {
 111                         iommu_sva_unbind_device(sva);
 112                         rc = -EINVAL;
 113                         goto failed;
 114                 }
 115
 116                 ctx->sva = sva;
 117                 ctx->pasid = pasid;
 118
 119                 if (wq_dedicated(wq)) {
 120                         rc = idxd_wq_set_pasid(wq, pasid);
 121                         if (rc < 0) {
 122                                 iommu_sva_unbind_device(sva);
 123                                 dev_err(dev, "wq set pasid failed: %d\n", rc);
 124                                 goto failed;
 125                         }
 126                 }
 127         }
 128
 129         idxd_wq_get(wq);
 130         mutex_unlock(&wq->wq_lock);
 131         return 0;
 132
 133  failed:
 134         mutex_unlock(&wq->wq_lock);
 135         kfree(ctx);
 136         return rc;
 137 }
 138
 139 static int idxd_cdev_release(struct inode *node, struct file *filep)
 140 {
 141         struct idxd_user_context *ctx = filep->private_data;
 142         struct idxd_wq *wq = ctx->wq;
 143         struct idxd_device *idxd = wq->idxd;
 144         struct device *dev = &idxd->pdev->dev;
 145         int rc;
 146
 147         dev_dbg(dev, "%s called\n", __func__);
 148         filep->private_data = NULL;
 149
 150         /* Wait for in-flight operations to complete. */
 151         if (wq_shared(wq)) {
 152                 idxd_device_drain_pasid(idxd, ctx->pasid);
 153         } else {
 154                 if (device_user_pasid_enabled(idxd)) {
 155                         /* The wq disable in the disable pasid function will drain the wq */
 156                         rc = idxd_wq_disable_pasid(wq);
 157                         if (rc < 0)
 158                                 dev_err(dev, "wq disable pasid failed.\n");
 159                 } else {
 160                         idxd_wq_drain(wq);
 161                 }
 162         }
 163
 164         if (ctx->sva)
 165                 iommu_sva_unbind_device(ctx->sva);
 166         kfree(ctx);
 167         mutex_lock(&wq->wq_lock);
 168         idxd_wq_put(wq);
 169         mutex_unlock(&wq->wq_lock);
 170         return 0;
 171 }
 172
 173 static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma,
 174                      const char *func)
 175 {
 176         struct device *dev = &wq->idxd->pdev->dev;
 177
 178         if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
 179                 dev_info_ratelimited(dev,
 180                                      "%s: %s: mapping too large: %lu\n",
 181                                      current->comm, func,
 182                                      vma->vm_end - vma->vm_start);
 183                 return -EINVAL;
 184         }
 185
 186         return 0;
 187 }
 188
 189 static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma)
 190 {
 191         struct idxd_user_context *ctx = filp->private_data;
 192         struct idxd_wq *wq = ctx->wq;
 193         struct idxd_device *idxd = wq->idxd;
 194         struct pci_dev *pdev = idxd->pdev;
 195         phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR);
 196         unsigned long pfn;
 197         int rc;
 198
 199         dev_dbg(&pdev->dev, "%s called\n", __func__);
 200         rc = check_vma(wq, vma, __func__);
 201         if (rc < 0)
 202                 return rc;
 203
 204         vma->vm_flags |= VM_DONTCOPY;
 205         pfn = (base + idxd_get_wq_portal_full_offset(wq->id,
 206                                 IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT;
 207         vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 208         vma->vm_private_data = ctx;
 209
 210         return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE,
 211                         vma->vm_page_prot);
 212 }
 213
 214 static __poll_t idxd_cdev_poll(struct file *filp,
 215                                struct poll_table_struct *wait)
 216 {
 217         struct idxd_user_context *ctx = filp->private_data;
 218         struct idxd_wq *wq = ctx->wq;
 219         struct idxd_device *idxd = wq->idxd;
 220         __poll_t out = 0;
 221
 222         poll_wait(filp, &wq->err_queue, wait);
 223         spin_lock(&idxd->dev_lock);
 224         if (idxd->sw_err.valid)
 225                 out = EPOLLIN | EPOLLRDNORM;
 226         spin_unlock(&idxd->dev_lock);
 227
 228         return out;
 229 }
 230
 231 static const struct file_operations idxd_cdev_fops = {
 232         .owner = THIS_MODULE,
 233         .open = idxd_cdev_open,
 234         .release = idxd_cdev_release,
 235         .mmap = idxd_cdev_mmap,
 236         .poll = idxd_cdev_poll,
 237 };
 238
 239 int idxd_cdev_get_major(struct idxd_device *idxd)
 240 {
 241         return MAJOR(ictx[idxd->data->type].devt);
 242 }
 243
 244 int idxd_wq_add_cdev(struct idxd_wq *wq)
 245 {
 246         struct idxd_device *idxd = wq->idxd;
 247         struct idxd_cdev *idxd_cdev;
 248         struct cdev *cdev;
 249         struct device *dev;
 250         struct idxd_cdev_context *cdev_ctx;
 251         int rc, minor;
 252
 253         idxd_cdev = kzalloc(sizeof(*idxd_cdev), GFP_KERNEL);
 254         if (!idxd_cdev)
 255                 return -ENOMEM;
 256
 257         idxd_cdev->idxd_dev.type = IDXD_DEV_CDEV;
 258         idxd_cdev->wq = wq;
 259         cdev = &idxd_cdev->cdev;
 260         dev = cdev_dev(idxd_cdev);
 261         cdev_ctx = &ictx[wq->idxd->data->type];
 262         minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL);
 263         if (minor < 0) {
 264                 kfree(idxd_cdev);
 265                 return minor;
 266         }
 267         idxd_cdev->minor = minor;
 268
 269         device_initialize(dev);
 270         dev->parent = wq_confdev(wq);
 271         dev->bus = &dsa_bus_type;
 272         dev->type = &idxd_cdev_device_type;
 273         dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor);
 274
 275         rc = dev_set_name(dev, "%s/wq%u.%u", idxd->data->name_prefix, idxd->id, wq->id);
 276         if (rc < 0)
 277                 goto err;
 278
 279         wq->idxd_cdev = idxd_cdev;
 280         cdev_init(cdev, &idxd_cdev_fops);
 281         rc = cdev_device_add(cdev, dev);
 282         if (rc) {
 283                 dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc);
 284                 goto err;
 285         }
 286
 287         return 0;
 288
 289  err:
 290         put_device(dev);
 291         wq->idxd_cdev = NULL;
 292         return rc;
 293 }
 294
 295 void idxd_wq_del_cdev(struct idxd_wq *wq)
 296 {
 297         struct idxd_cdev *idxd_cdev;
 298
 299         idxd_cdev = wq->idxd_cdev;
 300         wq->idxd_cdev = NULL;
 301         cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev));
 302         put_device(cdev_dev(idxd_cdev));
 303 }
 304
 305 static int idxd_user_drv_probe(struct idxd_dev *idxd_dev)
 306 {
 307         struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
 308         struct idxd_device *idxd = wq->idxd;
 309         int rc;
 310
 311         if (idxd->state != IDXD_DEV_ENABLED)
 312                 return -ENXIO;
 313
 314         /*
 315          * User type WQ is enabled only when SVA is enabled for two reasons:
 316          *   - If no IOMMU or IOMMU Passthrough without SVA, userspace
 317          *     can directly access physical address through the WQ.
 318          *   - The IDXD cdev driver does not provide any ways to pin
 319          *     user pages and translate the address from user VA to IOVA or
 320          *     PA without IOMMU SVA. Therefore the application has no way
 321          *     to instruct the device to perform DMA function. This makes
 322          *     the cdev not usable for normal application usage.
 323          */
 324         if (!device_user_pasid_enabled(idxd)) {
 325                 idxd->cmd_status = IDXD_SCMD_WQ_USER_NO_IOMMU;
 326                 dev_dbg(&idxd->pdev->dev,
 327                         "User type WQ cannot be enabled without SVA.\n");
 328
 329                 return -EOPNOTSUPP;
 330         }
 331
 332         mutex_lock(&wq->wq_lock);
 333         wq->type = IDXD_WQT_USER;
 334         rc = drv_enable_wq(wq);
 335         if (rc < 0)
 336                 goto err;
 337
 338         rc = idxd_wq_add_cdev(wq);
 339         if (rc < 0) {
 340                 idxd->cmd_status = IDXD_SCMD_CDEV_ERR;
 341                 goto err_cdev;
 342         }
 343
 344         idxd->cmd_status = 0;
 345         mutex_unlock(&wq->wq_lock);
 346         return 0;
 347
 348 err_cdev:
 349         drv_disable_wq(wq);
 350 err:
 351         wq->type = IDXD_WQT_NONE;
 352         mutex_unlock(&wq->wq_lock);
 353         return rc;
 354 }
 355
 356 static void idxd_user_drv_remove(struct idxd_dev *idxd_dev)
 357 {
 358         struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
 359
 360         mutex_lock(&wq->wq_lock);
 361         idxd_wq_del_cdev(wq);
 362         drv_disable_wq(wq);
 363         wq->type = IDXD_WQT_NONE;
 364         mutex_unlock(&wq->wq_lock);
 365 }
 366
 367 static enum idxd_dev_type dev_types[] = {
 368         IDXD_DEV_WQ,
 369         IDXD_DEV_NONE,
 370 };
 371
 372 struct idxd_device_driver idxd_user_drv = {
 373         .probe = idxd_user_drv_probe,
 374         .remove = idxd_user_drv_remove,
 375         .name = "user",
 376         .type = dev_types,
 377 };
 378 EXPORT_SYMBOL_GPL(idxd_user_drv);
 379
 380 int idxd_cdev_register(void)
 381 {
 382         int rc, i;
 383
 384         for (i = 0; i < IDXD_TYPE_MAX; i++) {
 385                 ida_init(&ictx[i].minor_ida);
 386                 rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK,
 387                                          ictx[i].name);
 388                 if (rc)
 389                         goto err_free_chrdev_region;
 390         }
 391
 392         return 0;
 393
 394 err_free_chrdev_region:
 395         for (i--; i >= 0; i--)
 396                 unregister_chrdev_region(ictx[i].devt, MINORMASK);
 397
 398         return rc;
 399 }
 400
 401 void idxd_cdev_remove(void)
 402 {
 403         int i;
 404
 405         for (i = 0; i < IDXD_TYPE_MAX; i++) {
 406                 unregister_chrdev_region(ictx[i].devt, MINORMASK);
 407                 ida_destroy(&ictx[i].minor_ida);
 408         }
 409 }