drivers/vfio/vfio_main.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * VFIO core
   4  *
   5  * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
   6  *     Author: Alex Williamson <alex.williamson@redhat.com>
   7  *
   8  * Derived from original vfio:
   9  * Copyright 2010 Cisco Systems, Inc.  All rights reserved.
  10  * Author: Tom Lyon, pugs@cisco.com
  11  */
  12
  13 #include <linux/cdev.h>
  14 #include <linux/compat.h>
  15 #include <linux/device.h>
  16 #include <linux/fs.h>
  17 #include <linux/idr.h>
  18 #include <linux/iommu.h>
  19 #if IS_ENABLED(CONFIG_KVM)
  20 #include <linux/kvm_host.h>
  21 #endif
  22 #include <linux/list.h>
  23 #include <linux/miscdevice.h>
  24 #include <linux/module.h>
  25 #include <linux/mount.h>
  26 #include <linux/mutex.h>
  27 #include <linux/pci.h>
  28 #include <linux/pseudo_fs.h>
  29 #include <linux/rwsem.h>
  30 #include <linux/sched.h>
  31 #include <linux/slab.h>
  32 #include <linux/stat.h>
  33 #include <linux/string.h>
  34 #include <linux/uaccess.h>
  35 #include <linux/vfio.h>
  36 #include <linux/wait.h>
  37 #include <linux/sched/signal.h>
  38 #include <linux/pm_runtime.h>
  39 #include <linux/interval_tree.h>
  40 #include <linux/iova_bitmap.h>
  41 #include <linux/iommufd.h>
  42 #include "vfio.h"
  43
  44 #define DRIVER_VERSION  "0.3"
  45 #define DRIVER_AUTHOR   "Alex Williamson <alex.williamson@redhat.com>"
  46 #define DRIVER_DESC     "VFIO - User Level meta-driver"
  47
  48 #define VFIO_MAGIC 0x5646494f /* "VFIO" */
  49
  50 static struct vfio {
  51         struct class                    *device_class;
  52         struct ida                      device_ida;
  53         struct vfsmount                 *vfs_mount;
  54         int                             fs_count;
  55 } vfio;
  56
  57 #ifdef CONFIG_VFIO_NOIOMMU
  58 bool vfio_noiommu __read_mostly;
  59 module_param_named(enable_unsafe_noiommu_mode,
  60                    vfio_noiommu, bool, S_IRUGO | S_IWUSR);
  61 MODULE_PARM_DESC(enable_unsafe_noiommu_mode, "Enable UNSAFE, no-IOMMU mode.  This mode provides no device isolation, no DMA translation, no host kernel protection, cannot be used for device assignment to virtual machines, requires RAWIO permissions, and will taint the kernel.  If you do not know what this is for, step away. (default: false)");
  62 #endif
  63
  64 static DEFINE_XARRAY(vfio_device_set_xa);
  65
  66 int vfio_assign_device_set(struct vfio_device *device, void *set_id)
  67 {
  68         unsigned long idx = (unsigned long)set_id;
  69         struct vfio_device_set *new_dev_set;
  70         struct vfio_device_set *dev_set;
  71
  72         if (WARN_ON(!set_id))
  73                 return -EINVAL;
  74
  75         /*
  76          * Atomically acquire a singleton object in the xarray for this set_id
  77          */
  78         xa_lock(&vfio_device_set_xa);
  79         dev_set = xa_load(&vfio_device_set_xa, idx);
  80         if (dev_set)
  81                 goto found_get_ref;
  82         xa_unlock(&vfio_device_set_xa);
  83
  84         new_dev_set = kzalloc(sizeof(*new_dev_set), GFP_KERNEL);
  85         if (!new_dev_set)
  86                 return -ENOMEM;
  87         mutex_init(&new_dev_set->lock);
  88         INIT_LIST_HEAD(&new_dev_set->device_list);
  89         new_dev_set->set_id = set_id;
  90
  91         xa_lock(&vfio_device_set_xa);
  92         dev_set = __xa_cmpxchg(&vfio_device_set_xa, idx, NULL, new_dev_set,
  93                                GFP_KERNEL);
  94         if (!dev_set) {
  95                 dev_set = new_dev_set;
  96                 goto found_get_ref;
  97         }
  98
  99         kfree(new_dev_set);
 100         if (xa_is_err(dev_set)) {
 101                 xa_unlock(&vfio_device_set_xa);
 102                 return xa_err(dev_set);
 103         }
 104
 105 found_get_ref:
 106         dev_set->device_count++;
 107         xa_unlock(&vfio_device_set_xa);
 108         mutex_lock(&dev_set->lock);
 109         device->dev_set = dev_set;
 110         list_add_tail(&device->dev_set_list, &dev_set->device_list);
 111         mutex_unlock(&dev_set->lock);
 112         return 0;
 113 }
 114 EXPORT_SYMBOL_GPL(vfio_assign_device_set);
 115
 116 static void vfio_release_device_set(struct vfio_device *device)
 117 {
 118         struct vfio_device_set *dev_set = device->dev_set;
 119
 120         if (!dev_set)
 121                 return;
 122
 123         mutex_lock(&dev_set->lock);
 124         list_del(&device->dev_set_list);
 125         mutex_unlock(&dev_set->lock);
 126
 127         xa_lock(&vfio_device_set_xa);
 128         if (!--dev_set->device_count) {
 129                 __xa_erase(&vfio_device_set_xa,
 130                            (unsigned long)dev_set->set_id);
 131                 mutex_destroy(&dev_set->lock);
 132                 kfree(dev_set);
 133         }
 134         xa_unlock(&vfio_device_set_xa);
 135 }
 136
 137 unsigned int vfio_device_set_open_count(struct vfio_device_set *dev_set)
 138 {
 139         struct vfio_device *cur;
 140         unsigned int open_count = 0;
 141
 142         lockdep_assert_held(&dev_set->lock);
 143
 144         list_for_each_entry(cur, &dev_set->device_list, dev_set_list)
 145                 open_count += cur->open_count;
 146         return open_count;
 147 }
 148 EXPORT_SYMBOL_GPL(vfio_device_set_open_count);
 149
 150 struct vfio_device *
 151 vfio_find_device_in_devset(struct vfio_device_set *dev_set,
 152                            struct device *dev)
 153 {
 154         struct vfio_device *cur;
 155
 156         lockdep_assert_held(&dev_set->lock);
 157
 158         list_for_each_entry(cur, &dev_set->device_list, dev_set_list)
 159                 if (cur->dev == dev)
 160                         return cur;
 161         return NULL;
 162 }
 163 EXPORT_SYMBOL_GPL(vfio_find_device_in_devset);
 164
 165 /*
 166  * Device objects - create, release, get, put, search
 167  */
 168 /* Device reference always implies a group reference */
 169 void vfio_device_put_registration(struct vfio_device *device)
 170 {
 171         if (refcount_dec_and_test(&device->refcount))
 172                 complete(&device->comp);
 173 }
 174
 175 bool vfio_device_try_get_registration(struct vfio_device *device)
 176 {
 177         return refcount_inc_not_zero(&device->refcount);
 178 }
 179
 180 /*
 181  * VFIO driver API
 182  */
 183 /* Release helper called by vfio_put_device() */
 184 static void vfio_device_release(struct device *dev)
 185 {
 186         struct vfio_device *device =
 187                         container_of(dev, struct vfio_device, device);
 188
 189         vfio_release_device_set(device);
 190         ida_free(&vfio.device_ida, device->index);
 191
 192         if (device->ops->release)
 193                 device->ops->release(device);
 194
 195         iput(device->inode);
 196         simple_release_fs(&vfio.vfs_mount, &vfio.fs_count);
 197         kvfree(device);
 198 }
 199
 200 static int vfio_init_device(struct vfio_device *device, struct device *dev,
 201                             const struct vfio_device_ops *ops);
 202
 203 /*
 204  * Allocate and initialize vfio_device so it can be registered to vfio
 205  * core.
 206  *
 207  * Drivers should use the wrapper vfio_alloc_device() for allocation.
 208  * @size is the size of the structure to be allocated, including any
 209  * private data used by the driver.
 210  *
 211  * Driver may provide an @init callback to cover device private data.
 212  *
 213  * Use vfio_put_device() to release the structure after success return.
 214  */
 215 struct vfio_device *_vfio_alloc_device(size_t size, struct device *dev,
 216                                        const struct vfio_device_ops *ops)
 217 {
 218         struct vfio_device *device;
 219         int ret;
 220
 221         if (WARN_ON(size < sizeof(struct vfio_device)))
 222                 return ERR_PTR(-EINVAL);
 223
 224         device = kvzalloc(size, GFP_KERNEL);
 225         if (!device)
 226                 return ERR_PTR(-ENOMEM);
 227
 228         ret = vfio_init_device(device, dev, ops);
 229         if (ret)
 230                 goto out_free;
 231         return device;
 232
 233 out_free:
 234         kvfree(device);
 235         return ERR_PTR(ret);
 236 }
 237 EXPORT_SYMBOL_GPL(_vfio_alloc_device);
 238
 239 static int vfio_fs_init_fs_context(struct fs_context *fc)
 240 {
 241         return init_pseudo(fc, VFIO_MAGIC) ? 0 : -ENOMEM;
 242 }
 243
 244 static struct file_system_type vfio_fs_type = {
 245         .name = "vfio",
 246         .owner = THIS_MODULE,
 247         .init_fs_context = vfio_fs_init_fs_context,
 248         .kill_sb = kill_anon_super,
 249 };
 250
 251 static struct inode *vfio_fs_inode_new(void)
 252 {
 253         struct inode *inode;
 254         int ret;
 255
 256         ret = simple_pin_fs(&vfio_fs_type, &vfio.vfs_mount, &vfio.fs_count);
 257         if (ret)
 258                 return ERR_PTR(ret);
 259
 260         inode = alloc_anon_inode(vfio.vfs_mount->mnt_sb);
 261         if (IS_ERR(inode))
 262                 simple_release_fs(&vfio.vfs_mount, &vfio.fs_count);
 263
 264         return inode;
 265 }
 266
 267 /*
 268  * Initialize a vfio_device so it can be registered to vfio core.
 269  */
 270 static int vfio_init_device(struct vfio_device *device, struct device *dev,
 271                             const struct vfio_device_ops *ops)
 272 {
 273         int ret;
 274
 275         ret = ida_alloc_max(&vfio.device_ida, MINORMASK, GFP_KERNEL);
 276         if (ret < 0) {
 277                 dev_dbg(dev, "Error to alloc index\n");
 278                 return ret;
 279         }
 280
 281         device->index = ret;
 282         init_completion(&device->comp);
 283         device->dev = dev;
 284         device->ops = ops;
 285         device->inode = vfio_fs_inode_new();
 286         if (IS_ERR(device->inode)) {
 287                 ret = PTR_ERR(device->inode);
 288                 goto out_inode;
 289         }
 290
 291         if (ops->init) {
 292                 ret = ops->init(device);
 293                 if (ret)
 294                         goto out_uninit;
 295         }
 296
 297         device_initialize(&device->device);
 298         device->device.release = vfio_device_release;
 299         device->device.class = vfio.device_class;
 300         device->device.parent = device->dev;
 301         return 0;
 302
 303 out_uninit:
 304         iput(device->inode);
 305         simple_release_fs(&vfio.vfs_mount, &vfio.fs_count);
 306 out_inode:
 307         vfio_release_device_set(device);
 308         ida_free(&vfio.device_ida, device->index);
 309         return ret;
 310 }
 311
 312 static int __vfio_register_dev(struct vfio_device *device,
 313                                enum vfio_group_type type)
 314 {
 315         int ret;
 316
 317         if (WARN_ON(IS_ENABLED(CONFIG_IOMMUFD) &&
 318                     (!device->ops->bind_iommufd ||
 319                      !device->ops->unbind_iommufd ||
 320                      !device->ops->attach_ioas ||
 321                      !device->ops->detach_ioas)))
 322                 return -EINVAL;
 323
 324         /*
 325          * If the driver doesn't specify a set then the device is added to a
 326          * singleton set just for itself.
 327          */
 328         if (!device->dev_set)
 329                 vfio_assign_device_set(device, device);
 330
 331         ret = dev_set_name(&device->device, "vfio%d", device->index);
 332         if (ret)
 333                 return ret;
 334
 335         ret = vfio_device_set_group(device, type);
 336         if (ret)
 337                 return ret;
 338
 339         /*
 340          * VFIO always sets IOMMU_CACHE because we offer no way for userspace to
 341          * restore cache coherency. It has to be checked here because it is only
 342          * valid for cases where we are using iommu groups.
 343          */
 344         if (type == VFIO_IOMMU && !vfio_device_is_noiommu(device) &&
 345             !device_iommu_capable(device->dev, IOMMU_CAP_CACHE_COHERENCY)) {
 346                 ret = -EINVAL;
 347                 goto err_out;
 348         }
 349
 350         ret = vfio_device_add(device);
 351         if (ret)
 352                 goto err_out;
 353
 354         /* Refcounting can't start until the driver calls register */
 355         refcount_set(&device->refcount, 1);
 356
 357         vfio_device_group_register(device);
 358         vfio_device_debugfs_init(device);
 359
 360         return 0;
 361 err_out:
 362         vfio_device_remove_group(device);
 363         return ret;
 364 }
 365
 366 int vfio_register_group_dev(struct vfio_device *device)
 367 {
 368         return __vfio_register_dev(device, VFIO_IOMMU);
 369 }
 370 EXPORT_SYMBOL_GPL(vfio_register_group_dev);
 371
 372 /*
 373  * Register a virtual device without IOMMU backing.  The user of this
 374  * device must not be able to directly trigger unmediated DMA.
 375  */
 376 int vfio_register_emulated_iommu_dev(struct vfio_device *device)
 377 {
 378         return __vfio_register_dev(device, VFIO_EMULATED_IOMMU);
 379 }
 380 EXPORT_SYMBOL_GPL(vfio_register_emulated_iommu_dev);
 381
 382 /*
 383  * Decrement the device reference count and wait for the device to be
 384  * removed.  Open file descriptors for the device... */
 385 void vfio_unregister_group_dev(struct vfio_device *device)
 386 {
 387         unsigned int i = 0;
 388         bool interrupted = false;
 389         long rc;
 390
 391         /*
 392          * Prevent new device opened by userspace via the
 393          * VFIO_GROUP_GET_DEVICE_FD in the group path.
 394          */
 395         vfio_device_group_unregister(device);
 396
 397         /*
 398          * Balances vfio_device_add() in register path, also prevents
 399          * new device opened by userspace in the cdev path.
 400          */
 401         vfio_device_del(device);
 402
 403         vfio_device_put_registration(device);
 404         rc = try_wait_for_completion(&device->comp);
 405         while (rc <= 0) {
 406                 if (device->ops->request)
 407                         device->ops->request(device, i++);
 408
 409                 if (interrupted) {
 410                         rc = wait_for_completion_timeout(&device->comp,
 411                                                          HZ * 10);
 412                 } else {
 413                         rc = wait_for_completion_interruptible_timeout(
 414                                 &device->comp, HZ * 10);
 415                         if (rc < 0) {
 416                                 interrupted = true;
 417                                 dev_warn(device->dev,
 418                                          "Device is currently in use, task"
 419                                          " \"%s\" (%d) "
 420                                          "blocked until device is released",
 421                                          current->comm, task_pid_nr(current));
 422                         }
 423                 }
 424         }
 425
 426         vfio_device_debugfs_exit(device);
 427         /* Balances vfio_device_set_group in register path */
 428         vfio_device_remove_group(device);
 429 }
 430 EXPORT_SYMBOL_GPL(vfio_unregister_group_dev);
 431
 432 #if IS_ENABLED(CONFIG_KVM)
 433 void vfio_device_get_kvm_safe(struct vfio_device *device, struct kvm *kvm)
 434 {
 435         void (*pfn)(struct kvm *kvm);
 436         bool (*fn)(struct kvm *kvm);
 437         bool ret;
 438
 439         lockdep_assert_held(&device->dev_set->lock);
 440
 441         if (!kvm)
 442                 return;
 443
 444         pfn = symbol_get(kvm_put_kvm);
 445         if (WARN_ON(!pfn))
 446                 return;
 447
 448         fn = symbol_get(kvm_get_kvm_safe);
 449         if (WARN_ON(!fn)) {
 450                 symbol_put(kvm_put_kvm);
 451                 return;
 452         }
 453
 454         ret = fn(kvm);
 455         symbol_put(kvm_get_kvm_safe);
 456         if (!ret) {
 457                 symbol_put(kvm_put_kvm);
 458                 return;
 459         }
 460
 461         device->put_kvm = pfn;
 462         device->kvm = kvm;
 463 }
 464
 465 void vfio_device_put_kvm(struct vfio_device *device)
 466 {
 467         lockdep_assert_held(&device->dev_set->lock);
 468
 469         if (!device->kvm)
 470                 return;
 471
 472         if (WARN_ON(!device->put_kvm))
 473                 goto clear;
 474
 475         device->put_kvm(device->kvm);
 476         device->put_kvm = NULL;
 477         symbol_put(kvm_put_kvm);
 478
 479 clear:
 480         device->kvm = NULL;
 481 }
 482 #endif
 483
 484 /* true if the vfio_device has open_device() called but not close_device() */
 485 static bool vfio_assert_device_open(struct vfio_device *device)
 486 {
 487         return !WARN_ON_ONCE(!READ_ONCE(device->open_count));
 488 }
 489
 490 struct vfio_device_file *
 491 vfio_allocate_device_file(struct vfio_device *device)
 492 {
 493         struct vfio_device_file *df;
 494
 495         df = kzalloc(sizeof(*df), GFP_KERNEL_ACCOUNT);
 496         if (!df)
 497                 return ERR_PTR(-ENOMEM);
 498
 499         df->device = device;
 500         spin_lock_init(&df->kvm_ref_lock);
 501
 502         return df;
 503 }
 504
 505 static int vfio_df_device_first_open(struct vfio_device_file *df)
 506 {
 507         struct vfio_device *device = df->device;
 508         struct iommufd_ctx *iommufd = df->iommufd;
 509         int ret;
 510
 511         lockdep_assert_held(&device->dev_set->lock);
 512
 513         if (!try_module_get(device->dev->driver->owner))
 514                 return -ENODEV;
 515
 516         if (iommufd)
 517                 ret = vfio_df_iommufd_bind(df);
 518         else
 519                 ret = vfio_device_group_use_iommu(device);
 520         if (ret)
 521                 goto err_module_put;
 522
 523         if (device->ops->open_device) {
 524                 ret = device->ops->open_device(device);
 525                 if (ret)
 526                         goto err_unuse_iommu;
 527         }
 528         return 0;
 529
 530 err_unuse_iommu:
 531         if (iommufd)
 532                 vfio_df_iommufd_unbind(df);
 533         else
 534                 vfio_device_group_unuse_iommu(device);
 535 err_module_put:
 536         module_put(device->dev->driver->owner);
 537         return ret;
 538 }
 539
 540 static void vfio_df_device_last_close(struct vfio_device_file *df)
 541 {
 542         struct vfio_device *device = df->device;
 543         struct iommufd_ctx *iommufd = df->iommufd;
 544
 545         lockdep_assert_held(&device->dev_set->lock);
 546
 547         if (device->ops->close_device)
 548                 device->ops->close_device(device);
 549         if (iommufd)
 550                 vfio_df_iommufd_unbind(df);
 551         else
 552                 vfio_device_group_unuse_iommu(device);
 553         module_put(device->dev->driver->owner);
 554 }
 555
 556 int vfio_df_open(struct vfio_device_file *df)
 557 {
 558         struct vfio_device *device = df->device;
 559         int ret = 0;
 560
 561         lockdep_assert_held(&device->dev_set->lock);
 562
 563         /*
 564          * Only the group path allows the device to be opened multiple
 565          * times.  The device cdev path doesn't have a secure way for it.
 566          */
 567         if (device->open_count != 0 && !df->group)
 568                 return -EINVAL;
 569
 570         device->open_count++;
 571         if (device->open_count == 1) {
 572                 ret = vfio_df_device_first_open(df);
 573                 if (ret)
 574                         device->open_count--;
 575         }
 576
 577         return ret;
 578 }
 579
 580 void vfio_df_close(struct vfio_device_file *df)
 581 {
 582         struct vfio_device *device = df->device;
 583
 584         lockdep_assert_held(&device->dev_set->lock);
 585
 586         if (!vfio_assert_device_open(device))
 587                 return;
 588         if (device->open_count == 1)
 589                 vfio_df_device_last_close(df);
 590         device->open_count--;
 591 }
 592
 593 /*
 594  * Wrapper around pm_runtime_resume_and_get().
 595  * Return error code on failure or 0 on success.
 596  */
 597 static inline int vfio_device_pm_runtime_get(struct vfio_device *device)
 598 {
 599         struct device *dev = device->dev;
 600
 601         if (dev->driver && dev->driver->pm) {
 602                 int ret;
 603
 604                 ret = pm_runtime_resume_and_get(dev);
 605                 if (ret) {
 606                         dev_info_ratelimited(dev,
 607                                 "vfio: runtime resume failed %d\n", ret);
 608                         return -EIO;
 609                 }
 610         }
 611
 612         return 0;
 613 }
 614
 615 /*
 616  * Wrapper around pm_runtime_put().
 617  */
 618 static inline void vfio_device_pm_runtime_put(struct vfio_device *device)
 619 {
 620         struct device *dev = device->dev;
 621
 622         if (dev->driver && dev->driver->pm)
 623                 pm_runtime_put(dev);
 624 }
 625
 626 /*
 627  * VFIO Device fd
 628  */
 629 static int vfio_device_fops_release(struct inode *inode, struct file *filep)
 630 {
 631         struct vfio_device_file *df = filep->private_data;
 632         struct vfio_device *device = df->device;
 633
 634         if (df->group)
 635                 vfio_df_group_close(df);
 636         else
 637                 vfio_df_unbind_iommufd(df);
 638
 639         vfio_device_put_registration(device);
 640
 641         kfree(df);
 642
 643         return 0;
 644 }
 645
 646 /*
 647  * vfio_mig_get_next_state - Compute the next step in the FSM
 648  * @cur_fsm - The current state the device is in
 649  * @new_fsm - The target state to reach
 650  * @next_fsm - Pointer to the next step to get to new_fsm
 651  *
 652  * Return 0 upon success, otherwise -errno
 653  * Upon success the next step in the state progression between cur_fsm and
 654  * new_fsm will be set in next_fsm.
 655  *
 656  * This breaks down requests for combination transitions into smaller steps and
 657  * returns the next step to get to new_fsm. The function may need to be called
 658  * multiple times before reaching new_fsm.
 659  *
 660  */
 661 int vfio_mig_get_next_state(struct vfio_device *device,
 662                             enum vfio_device_mig_state cur_fsm,
 663                             enum vfio_device_mig_state new_fsm,
 664                             enum vfio_device_mig_state *next_fsm)
 665 {
 666         enum { VFIO_DEVICE_NUM_STATES = VFIO_DEVICE_STATE_PRE_COPY_P2P + 1 };
 667         /*
 668          * The coding in this table requires the driver to implement the
 669          * following FSM arcs:
 670          *         RESUMING -> STOP
 671          *         STOP -> RESUMING
 672          *         STOP -> STOP_COPY
 673          *         STOP_COPY -> STOP
 674          *
 675          * If P2P is supported then the driver must also implement these FSM
 676          * arcs:
 677          *         RUNNING -> RUNNING_P2P
 678          *         RUNNING_P2P -> RUNNING
 679          *         RUNNING_P2P -> STOP
 680          *         STOP -> RUNNING_P2P
 681          *
 682          * If precopy is supported then the driver must support these additional
 683          * FSM arcs:
 684          *         RUNNING -> PRE_COPY
 685          *         PRE_COPY -> RUNNING
 686          *         PRE_COPY -> STOP_COPY
 687          * However, if precopy and P2P are supported together then the driver
 688          * must support these additional arcs beyond the P2P arcs above:
 689          *         PRE_COPY -> RUNNING
 690          *         PRE_COPY -> PRE_COPY_P2P
 691          *         PRE_COPY_P2P -> PRE_COPY
 692          *         PRE_COPY_P2P -> RUNNING_P2P
 693          *         PRE_COPY_P2P -> STOP_COPY
 694          *         RUNNING -> PRE_COPY
 695          *         RUNNING_P2P -> PRE_COPY_P2P
 696          *
 697          * Without P2P and precopy the driver must implement:
 698          *         RUNNING -> STOP
 699          *         STOP -> RUNNING
 700          *
 701          * The coding will step through multiple states for some combination
 702          * transitions; if all optional features are supported, this means the
 703          * following ones:
 704          *         PRE_COPY -> PRE_COPY_P2P -> STOP_COPY
 705          *         PRE_COPY -> RUNNING -> RUNNING_P2P
 706          *         PRE_COPY -> RUNNING -> RUNNING_P2P -> STOP
 707          *         PRE_COPY -> RUNNING -> RUNNING_P2P -> STOP -> RESUMING
 708          *         PRE_COPY_P2P -> RUNNING_P2P -> RUNNING
 709          *         PRE_COPY_P2P -> RUNNING_P2P -> STOP
 710          *         PRE_COPY_P2P -> RUNNING_P2P -> STOP -> RESUMING
 711          *         RESUMING -> STOP -> RUNNING_P2P
 712          *         RESUMING -> STOP -> RUNNING_P2P -> PRE_COPY_P2P
 713          *         RESUMING -> STOP -> RUNNING_P2P -> RUNNING
 714          *         RESUMING -> STOP -> RUNNING_P2P -> RUNNING -> PRE_COPY
 715          *         RESUMING -> STOP -> STOP_COPY
 716          *         RUNNING -> RUNNING_P2P -> PRE_COPY_P2P
 717          *         RUNNING -> RUNNING_P2P -> STOP
 718          *         RUNNING -> RUNNING_P2P -> STOP -> RESUMING
 719          *         RUNNING -> RUNNING_P2P -> STOP -> STOP_COPY
 720          *         RUNNING_P2P -> RUNNING -> PRE_COPY
 721          *         RUNNING_P2P -> STOP -> RESUMING
 722          *         RUNNING_P2P -> STOP -> STOP_COPY
 723          *         STOP -> RUNNING_P2P -> PRE_COPY_P2P
 724          *         STOP -> RUNNING_P2P -> RUNNING
 725          *         STOP -> RUNNING_P2P -> RUNNING -> PRE_COPY
 726          *         STOP_COPY -> STOP -> RESUMING
 727          *         STOP_COPY -> STOP -> RUNNING_P2P
 728          *         STOP_COPY -> STOP -> RUNNING_P2P -> RUNNING
 729          *
 730          *  The following transitions are blocked:
 731          *         STOP_COPY -> PRE_COPY
 732          *         STOP_COPY -> PRE_COPY_P2P
 733          */
 734         static const u8 vfio_from_fsm_table[VFIO_DEVICE_NUM_STATES][VFIO_DEVICE_NUM_STATES] = {
 735                 [VFIO_DEVICE_STATE_STOP] = {
 736                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
 737                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING_P2P,
 738                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_RUNNING_P2P,
 739                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 740                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY,
 741                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING,
 742                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 743                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 744                 },
 745                 [VFIO_DEVICE_STATE_RUNNING] = {
 746                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING_P2P,
 747                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING,
 748                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY,
 749                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 750                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_RUNNING_P2P,
 751                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING_P2P,
 752                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 753                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 754                 },
 755                 [VFIO_DEVICE_STATE_PRE_COPY] = {
 756                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING,
 757                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING,
 758                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY,
 759                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P,
 760                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_PRE_COPY_P2P,
 761                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING,
 762                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING,
 763                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 764                 },
 765                 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = {
 766                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING_P2P,
 767                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING_P2P,
 768                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY,
 769                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P,
 770                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY,
 771                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING_P2P,
 772                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 773                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 774                 },
 775                 [VFIO_DEVICE_STATE_STOP_COPY] = {
 776                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
 777                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP,
 778                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_ERROR,
 779                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_ERROR,
 780                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY,
 781                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP,
 782                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP,
 783                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 784                 },
 785                 [VFIO_DEVICE_STATE_RESUMING] = {
 786                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
 787                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP,
 788                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_STOP,
 789                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_STOP,
 790                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP,
 791                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING,
 792                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP,
 793                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 794                 },
 795                 [VFIO_DEVICE_STATE_RUNNING_P2P] = {
 796                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
 797                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING,
 798                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_RUNNING,
 799                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P,
 800                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP,
 801                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP,
 802                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 803                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 804                 },
 805                 [VFIO_DEVICE_STATE_ERROR] = {
 806                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_ERROR,
 807                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_ERROR,
 808                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_ERROR,
 809                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_ERROR,
 810                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_ERROR,
 811                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_ERROR,
 812                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_ERROR,
 813                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 814                 },
 815         };
 816
 817         static const unsigned int state_flags_table[VFIO_DEVICE_NUM_STATES] = {
 818                 [VFIO_DEVICE_STATE_STOP] = VFIO_MIGRATION_STOP_COPY,
 819                 [VFIO_DEVICE_STATE_RUNNING] = VFIO_MIGRATION_STOP_COPY,
 820                 [VFIO_DEVICE_STATE_PRE_COPY] =
 821                         VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_PRE_COPY,
 822                 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_MIGRATION_STOP_COPY |
 823                                                    VFIO_MIGRATION_P2P |
 824                                                    VFIO_MIGRATION_PRE_COPY,
 825                 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_MIGRATION_STOP_COPY,
 826                 [VFIO_DEVICE_STATE_RESUMING] = VFIO_MIGRATION_STOP_COPY,
 827                 [VFIO_DEVICE_STATE_RUNNING_P2P] =
 828                         VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P,
 829                 [VFIO_DEVICE_STATE_ERROR] = ~0U,
 830         };
 831
 832         if (WARN_ON(cur_fsm >= ARRAY_SIZE(vfio_from_fsm_table) ||
 833                     (state_flags_table[cur_fsm] & device->migration_flags) !=
 834                         state_flags_table[cur_fsm]))
 835                 return -EINVAL;
 836
 837         if (new_fsm >= ARRAY_SIZE(vfio_from_fsm_table) ||
 838            (state_flags_table[new_fsm] & device->migration_flags) !=
 839                         state_flags_table[new_fsm])
 840                 return -EINVAL;
 841
 842         /*
 843          * Arcs touching optional and unsupported states are skipped over. The
 844          * driver will instead see an arc from the original state to the next
 845          * logical state, as per the above comment.
 846          */
 847         *next_fsm = vfio_from_fsm_table[cur_fsm][new_fsm];
 848         while ((state_flags_table[*next_fsm] & device->migration_flags) !=
 849                         state_flags_table[*next_fsm])
 850                 *next_fsm = vfio_from_fsm_table[*next_fsm][new_fsm];
 851
 852         return (*next_fsm != VFIO_DEVICE_STATE_ERROR) ? 0 : -EINVAL;
 853 }
 854 EXPORT_SYMBOL_GPL(vfio_mig_get_next_state);
 855
 856 /*
 857  * Convert the drivers's struct file into a FD number and return it to userspace
 858  */
 859 static int vfio_ioct_mig_return_fd(struct file *filp, void __user *arg,
 860                                    struct vfio_device_feature_mig_state *mig)
 861 {
 862         int ret;
 863         int fd;
 864
 865         fd = get_unused_fd_flags(O_CLOEXEC);
 866         if (fd < 0) {
 867                 ret = fd;
 868                 goto out_fput;
 869         }
 870
 871         mig->data_fd = fd;
 872         if (copy_to_user(arg, mig, sizeof(*mig))) {
 873                 ret = -EFAULT;
 874                 goto out_put_unused;
 875         }
 876         fd_install(fd, filp);
 877         return 0;
 878
 879 out_put_unused:
 880         put_unused_fd(fd);
 881 out_fput:
 882         fput(filp);
 883         return ret;
 884 }
 885
 886 static int
 887 vfio_ioctl_device_feature_mig_device_state(struct vfio_device *device,
 888                                            u32 flags, void __user *arg,
 889                                            size_t argsz)
 890 {
 891         size_t minsz =
 892                 offsetofend(struct vfio_device_feature_mig_state, data_fd);
 893         struct vfio_device_feature_mig_state mig;
 894         struct file *filp = NULL;
 895         int ret;
 896
 897         if (!device->mig_ops)
 898                 return -ENOTTY;
 899
 900         ret = vfio_check_feature(flags, argsz,
 901                                  VFIO_DEVICE_FEATURE_SET |
 902                                  VFIO_DEVICE_FEATURE_GET,
 903                                  sizeof(mig));
 904         if (ret != 1)
 905                 return ret;
 906
 907         if (copy_from_user(&mig, arg, minsz))
 908                 return -EFAULT;
 909
 910         if (flags & VFIO_DEVICE_FEATURE_GET) {
 911                 enum vfio_device_mig_state curr_state;
 912
 913                 ret = device->mig_ops->migration_get_state(device,
 914                                                            &curr_state);
 915                 if (ret)
 916                         return ret;
 917                 mig.device_state = curr_state;
 918                 goto out_copy;
 919         }
 920
 921         /* Handle the VFIO_DEVICE_FEATURE_SET */
 922         filp = device->mig_ops->migration_set_state(device, mig.device_state);
 923         if (IS_ERR(filp) || !filp)
 924                 goto out_copy;
 925
 926         return vfio_ioct_mig_return_fd(filp, arg, &mig);
 927 out_copy:
 928         mig.data_fd = -1;
 929         if (copy_to_user(arg, &mig, sizeof(mig)))
 930                 return -EFAULT;
 931         if (IS_ERR(filp))
 932                 return PTR_ERR(filp);
 933         return 0;
 934 }
 935
 936 static int
 937 vfio_ioctl_device_feature_migration_data_size(struct vfio_device *device,
 938                                               u32 flags, void __user *arg,
 939                                               size_t argsz)
 940 {
 941         struct vfio_device_feature_mig_data_size data_size = {};
 942         unsigned long stop_copy_length;
 943         int ret;
 944
 945         if (!device->mig_ops)
 946                 return -ENOTTY;
 947
 948         ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET,
 949                                  sizeof(data_size));
 950         if (ret != 1)
 951                 return ret;
 952
 953         ret = device->mig_ops->migration_get_data_size(device, &stop_copy_length);
 954         if (ret)
 955                 return ret;
 956
 957         data_size.stop_copy_length = stop_copy_length;
 958         if (copy_to_user(arg, &data_size, sizeof(data_size)))
 959                 return -EFAULT;
 960
 961         return 0;
 962 }
 963
 964 static int vfio_ioctl_device_feature_migration(struct vfio_device *device,
 965                                                u32 flags, void __user *arg,
 966                                                size_t argsz)
 967 {
 968         struct vfio_device_feature_migration mig = {
 969                 .flags = device->migration_flags,
 970         };
 971         int ret;
 972
 973         if (!device->mig_ops)
 974                 return -ENOTTY;
 975
 976         ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET,
 977                                  sizeof(mig));
 978         if (ret != 1)
 979                 return ret;
 980         if (copy_to_user(arg, &mig, sizeof(mig)))
 981                 return -EFAULT;
 982         return 0;
 983 }
 984
 985 void vfio_combine_iova_ranges(struct rb_root_cached *root, u32 cur_nodes,
 986                               u32 req_nodes)
 987 {
 988         struct interval_tree_node *prev, *curr, *comb_start, *comb_end;
 989         unsigned long min_gap, curr_gap;
 990
 991         /* Special shortcut when a single range is required */
 992         if (req_nodes == 1) {
 993                 unsigned long last;
 994
 995                 comb_start = interval_tree_iter_first(root, 0, ULONG_MAX);
 996
 997                 /* Empty list */
 998                 if (WARN_ON_ONCE(!comb_start))
 999                         return;
1000
1001                 curr = comb_start;
1002                 while (curr) {
1003                         last = curr->last;
1004                         prev = curr;
1005                         curr = interval_tree_iter_next(curr, 0, ULONG_MAX);
1006                         if (prev != comb_start)
1007                                 interval_tree_remove(prev, root);
1008                 }
1009                 comb_start->last = last;
1010                 return;
1011         }
1012
1013         /* Combine ranges which have the smallest gap */
1014         while (cur_nodes > req_nodes) {
1015                 prev = NULL;
1016                 min_gap = ULONG_MAX;
1017                 curr = interval_tree_iter_first(root, 0, ULONG_MAX);
1018                 while (curr) {
1019                         if (prev) {
1020                                 curr_gap = curr->start - prev->last;
1021                                 if (curr_gap < min_gap) {
1022                                         min_gap = curr_gap;
1023                                         comb_start = prev;
1024                                         comb_end = curr;
1025                                 }
1026                         }
1027                         prev = curr;
1028                         curr = interval_tree_iter_next(curr, 0, ULONG_MAX);
1029                 }
1030
1031                 /* Empty list or no nodes to combine */
1032                 if (WARN_ON_ONCE(min_gap == ULONG_MAX))
1033                         break;
1034
1035                 comb_start->last = comb_end->last;
1036                 interval_tree_remove(comb_end, root);
1037                 cur_nodes--;
1038         }
1039 }
1040 EXPORT_SYMBOL_GPL(vfio_combine_iova_ranges);
1041
1042 /* Ranges should fit into a single kernel page */
1043 #define LOG_MAX_RANGES \
1044         (PAGE_SIZE / sizeof(struct vfio_device_feature_dma_logging_range))
1045
1046 static int
1047 vfio_ioctl_device_feature_logging_start(struct vfio_device *device,
1048                                         u32 flags, void __user *arg,
1049                                         size_t argsz)
1050 {
1051         size_t minsz =
1052                 offsetofend(struct vfio_device_feature_dma_logging_control,
1053                             ranges);
1054         struct vfio_device_feature_dma_logging_range __user *ranges;
1055         struct vfio_device_feature_dma_logging_control control;
1056         struct vfio_device_feature_dma_logging_range range;
1057         struct rb_root_cached root = RB_ROOT_CACHED;
1058         struct interval_tree_node *nodes;
1059         u64 iova_end;
1060         u32 nnodes;
1061         int i, ret;
1062
1063         if (!device->log_ops)
1064                 return -ENOTTY;
1065
1066         ret = vfio_check_feature(flags, argsz,
1067                                  VFIO_DEVICE_FEATURE_SET,
1068                                  sizeof(control));
1069         if (ret != 1)
1070                 return ret;
1071
1072         if (copy_from_user(&control, arg, minsz))
1073                 return -EFAULT;
1074
1075         nnodes = control.num_ranges;
1076         if (!nnodes)
1077                 return -EINVAL;
1078
1079         if (nnodes > LOG_MAX_RANGES)
1080                 return -E2BIG;
1081
1082         ranges = u64_to_user_ptr(control.ranges);
1083         nodes = kmalloc_array(nnodes, sizeof(struct interval_tree_node),
1084                               GFP_KERNEL);
1085         if (!nodes)
1086                 return -ENOMEM;
1087
1088         for (i = 0; i < nnodes; i++) {
1089                 if (copy_from_user(&range, &ranges[i], sizeof(range))) {
1090                         ret = -EFAULT;
1091                         goto end;
1092                 }
1093                 if (!IS_ALIGNED(range.iova, control.page_size) ||
1094                     !IS_ALIGNED(range.length, control.page_size)) {
1095                         ret = -EINVAL;
1096                         goto end;
1097                 }
1098
1099                 if (check_add_overflow(range.iova, range.length, &iova_end) ||
1100                     iova_end > ULONG_MAX) {
1101                         ret = -EOVERFLOW;
1102                         goto end;
1103                 }
1104
1105                 nodes[i].start = range.iova;
1106                 nodes[i].last = range.iova + range.length - 1;
1107                 if (interval_tree_iter_first(&root, nodes[i].start,
1108                                              nodes[i].last)) {
1109                         /* Range overlapping */
1110                         ret = -EINVAL;
1111                         goto end;
1112                 }
1113                 interval_tree_insert(nodes + i, &root);
1114         }
1115
1116         ret = device->log_ops->log_start(device, &root, nnodes,
1117                                          &control.page_size);
1118         if (ret)
1119                 goto end;
1120
1121         if (copy_to_user(arg, &control, sizeof(control))) {
1122                 ret = -EFAULT;
1123                 device->log_ops->log_stop(device);
1124         }
1125
1126 end:
1127         kfree(nodes);
1128         return ret;
1129 }
1130
1131 static int
1132 vfio_ioctl_device_feature_logging_stop(struct vfio_device *device,
1133                                        u32 flags, void __user *arg,
1134                                        size_t argsz)
1135 {
1136         int ret;
1137
1138         if (!device->log_ops)
1139                 return -ENOTTY;
1140
1141         ret = vfio_check_feature(flags, argsz,
1142                                  VFIO_DEVICE_FEATURE_SET, 0);
1143         if (ret != 1)
1144                 return ret;
1145
1146         return device->log_ops->log_stop(device);
1147 }
1148
1149 static int vfio_device_log_read_and_clear(struct iova_bitmap *iter,
1150                                           unsigned long iova, size_t length,
1151                                           void *opaque)
1152 {
1153         struct vfio_device *device = opaque;
1154
1155         return device->log_ops->log_read_and_clear(device, iova, length, iter);
1156 }
1157
1158 static int
1159 vfio_ioctl_device_feature_logging_report(struct vfio_device *device,
1160                                          u32 flags, void __user *arg,
1161                                          size_t argsz)
1162 {
1163         size_t minsz =
1164                 offsetofend(struct vfio_device_feature_dma_logging_report,
1165                             bitmap);
1166         struct vfio_device_feature_dma_logging_report report;
1167         struct iova_bitmap *iter;
1168         u64 iova_end;
1169         int ret;
1170
1171         if (!device->log_ops)
1172                 return -ENOTTY;
1173
1174         ret = vfio_check_feature(flags, argsz,
1175                                  VFIO_DEVICE_FEATURE_GET,
1176                                  sizeof(report));
1177         if (ret != 1)
1178                 return ret;
1179
1180         if (copy_from_user(&report, arg, minsz))
1181                 return -EFAULT;
1182
1183         if (report.page_size < SZ_4K || !is_power_of_2(report.page_size))
1184                 return -EINVAL;
1185
1186         if (check_add_overflow(report.iova, report.length, &iova_end) ||
1187             iova_end > ULONG_MAX)
1188                 return -EOVERFLOW;
1189
1190         iter = iova_bitmap_alloc(report.iova, report.length,
1191                                  report.page_size,
1192                                  u64_to_user_ptr(report.bitmap));
1193         if (IS_ERR(iter))
1194                 return PTR_ERR(iter);
1195
1196         ret = iova_bitmap_for_each(iter, device,
1197                                    vfio_device_log_read_and_clear);
1198
1199         iova_bitmap_free(iter);
1200         return ret;
1201 }
1202
1203 static int vfio_ioctl_device_feature(struct vfio_device *device,
1204                                      struct vfio_device_feature __user *arg)
1205 {
1206         size_t minsz = offsetofend(struct vfio_device_feature, flags);
1207         struct vfio_device_feature feature;
1208
1209         if (copy_from_user(&feature, arg, minsz))
1210                 return -EFAULT;
1211
1212         if (feature.argsz < minsz)
1213                 return -EINVAL;
1214
1215         /* Check unknown flags */
1216         if (feature.flags &
1217             ~(VFIO_DEVICE_FEATURE_MASK | VFIO_DEVICE_FEATURE_SET |
1218               VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_PROBE))
1219                 return -EINVAL;
1220
1221         /* GET & SET are mutually exclusive except with PROBE */
1222         if (!(feature.flags & VFIO_DEVICE_FEATURE_PROBE) &&
1223             (feature.flags & VFIO_DEVICE_FEATURE_SET) &&
1224             (feature.flags & VFIO_DEVICE_FEATURE_GET))
1225                 return -EINVAL;
1226
1227         switch (feature.flags & VFIO_DEVICE_FEATURE_MASK) {
1228         case VFIO_DEVICE_FEATURE_MIGRATION:
1229                 return vfio_ioctl_device_feature_migration(
1230                         device, feature.flags, arg->data,
1231                         feature.argsz - minsz);
1232         case VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE:
1233                 return vfio_ioctl_device_feature_mig_device_state(
1234                         device, feature.flags, arg->data,
1235                         feature.argsz - minsz);
1236         case VFIO_DEVICE_FEATURE_DMA_LOGGING_START:
1237                 return vfio_ioctl_device_feature_logging_start(
1238                         device, feature.flags, arg->data,
1239                         feature.argsz - minsz);
1240         case VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP:
1241                 return vfio_ioctl_device_feature_logging_stop(
1242                         device, feature.flags, arg->data,
1243                         feature.argsz - minsz);
1244         case VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT:
1245                 return vfio_ioctl_device_feature_logging_report(
1246                         device, feature.flags, arg->data,
1247                         feature.argsz - minsz);
1248         case VFIO_DEVICE_FEATURE_MIG_DATA_SIZE:
1249                 return vfio_ioctl_device_feature_migration_data_size(
1250                         device, feature.flags, arg->data,
1251                         feature.argsz - minsz);
1252         default:
1253                 if (unlikely(!device->ops->device_feature))
1254                         return -EINVAL;
1255                 return device->ops->device_feature(device, feature.flags,
1256                                                    arg->data,
1257                                                    feature.argsz - minsz);
1258         }
1259 }
1260
1261 static long vfio_device_fops_unl_ioctl(struct file *filep,
1262                                        unsigned int cmd, unsigned long arg)
1263 {
1264         struct vfio_device_file *df = filep->private_data;
1265         struct vfio_device *device = df->device;
1266         void __user *uptr = (void __user *)arg;
1267         int ret;
1268
1269         if (cmd == VFIO_DEVICE_BIND_IOMMUFD)
1270                 return vfio_df_ioctl_bind_iommufd(df, uptr);
1271
1272         /* Paired with smp_store_release() following vfio_df_open() */
1273         if (!smp_load_acquire(&df->access_granted))
1274                 return -EINVAL;
1275
1276         ret = vfio_device_pm_runtime_get(device);
1277         if (ret)
1278                 return ret;
1279
1280         /* cdev only ioctls */
1281         if (IS_ENABLED(CONFIG_VFIO_DEVICE_CDEV) && !df->group) {
1282                 switch (cmd) {
1283                 case VFIO_DEVICE_ATTACH_IOMMUFD_PT:
1284                         ret = vfio_df_ioctl_attach_pt(df, uptr);
1285                         goto out;
1286
1287                 case VFIO_DEVICE_DETACH_IOMMUFD_PT:
1288                         ret = vfio_df_ioctl_detach_pt(df, uptr);
1289                         goto out;
1290                 }
1291         }
1292
1293         switch (cmd) {
1294         case VFIO_DEVICE_FEATURE:
1295                 ret = vfio_ioctl_device_feature(device, uptr);
1296                 break;
1297
1298         default:
1299                 if (unlikely(!device->ops->ioctl))
1300                         ret = -EINVAL;
1301                 else
1302                         ret = device->ops->ioctl(device, cmd, arg);
1303                 break;
1304         }
1305 out:
1306         vfio_device_pm_runtime_put(device);
1307         return ret;
1308 }
1309
1310 static ssize_t vfio_device_fops_read(struct file *filep, char __user *buf,
1311                                      size_t count, loff_t *ppos)
1312 {
1313         struct vfio_device_file *df = filep->private_data;
1314         struct vfio_device *device = df->device;
1315
1316         /* Paired with smp_store_release() following vfio_df_open() */
1317         if (!smp_load_acquire(&df->access_granted))
1318                 return -EINVAL;
1319
1320         if (unlikely(!device->ops->read))
1321                 return -EINVAL;
1322
1323         return device->ops->read(device, buf, count, ppos);
1324 }
1325
1326 static ssize_t vfio_device_fops_write(struct file *filep,
1327                                       const char __user *buf,
1328                                       size_t count, loff_t *ppos)
1329 {
1330         struct vfio_device_file *df = filep->private_data;
1331         struct vfio_device *device = df->device;
1332
1333         /* Paired with smp_store_release() following vfio_df_open() */
1334         if (!smp_load_acquire(&df->access_granted))
1335                 return -EINVAL;
1336
1337         if (unlikely(!device->ops->write))
1338                 return -EINVAL;
1339
1340         return device->ops->write(device, buf, count, ppos);
1341 }
1342
1343 static int vfio_device_fops_mmap(struct file *filep, struct vm_area_struct *vma)
1344 {
1345         struct vfio_device_file *df = filep->private_data;
1346         struct vfio_device *device = df->device;
1347
1348         /* Paired with smp_store_release() following vfio_df_open() */
1349         if (!smp_load_acquire(&df->access_granted))
1350                 return -EINVAL;
1351
1352         if (unlikely(!device->ops->mmap))
1353                 return -EINVAL;
1354
1355         return device->ops->mmap(device, vma);
1356 }
1357
1358 const struct file_operations vfio_device_fops = {
1359         .owner          = THIS_MODULE,
1360         .open           = vfio_device_fops_cdev_open,
1361         .release        = vfio_device_fops_release,
1362         .read           = vfio_device_fops_read,
1363         .write          = vfio_device_fops_write,
1364         .unlocked_ioctl = vfio_device_fops_unl_ioctl,
1365         .compat_ioctl   = compat_ptr_ioctl,
1366         .mmap           = vfio_device_fops_mmap,
1367 };
1368
1369 static struct vfio_device *vfio_device_from_file(struct file *file)
1370 {
1371         struct vfio_device_file *df = file->private_data;
1372
1373         if (file->f_op != &vfio_device_fops)
1374                 return NULL;
1375         return df->device;
1376 }
1377
1378 /**
1379  * vfio_file_is_valid - True if the file is valid vfio file
1380  * @file: VFIO group file or VFIO device file
1381  */
1382 bool vfio_file_is_valid(struct file *file)
1383 {
1384         return vfio_group_from_file(file) ||
1385                vfio_device_from_file(file);
1386 }
1387 EXPORT_SYMBOL_GPL(vfio_file_is_valid);
1388
1389 /**
1390  * vfio_file_enforced_coherent - True if the DMA associated with the VFIO file
1391  *        is always CPU cache coherent
1392  * @file: VFIO group file or VFIO device file
1393  *
1394  * Enforced coherency means that the IOMMU ignores things like the PCIe no-snoop
1395  * bit in DMA transactions. A return of false indicates that the user has
1396  * rights to access additional instructions such as wbinvd on x86.
1397  */
1398 bool vfio_file_enforced_coherent(struct file *file)
1399 {
1400         struct vfio_device *device;
1401         struct vfio_group *group;
1402
1403         group = vfio_group_from_file(file);
1404         if (group)
1405                 return vfio_group_enforced_coherent(group);
1406
1407         device = vfio_device_from_file(file);
1408         if (device)
1409                 return device_iommu_capable(device->dev,
1410                                             IOMMU_CAP_ENFORCE_CACHE_COHERENCY);
1411
1412         return true;
1413 }
1414 EXPORT_SYMBOL_GPL(vfio_file_enforced_coherent);
1415
1416 static void vfio_device_file_set_kvm(struct file *file, struct kvm *kvm)
1417 {
1418         struct vfio_device_file *df = file->private_data;
1419
1420         /*
1421          * The kvm is first recorded in the vfio_device_file, and will
1422          * be propagated to vfio_device::kvm when the file is bound to
1423          * iommufd successfully in the vfio device cdev path.
1424          */
1425         spin_lock(&df->kvm_ref_lock);
1426         df->kvm = kvm;
1427         spin_unlock(&df->kvm_ref_lock);
1428 }
1429
1430 /**
1431  * vfio_file_set_kvm - Link a kvm with VFIO drivers
1432  * @file: VFIO group file or VFIO device file
1433  * @kvm: KVM to link
1434  *
1435  * When a VFIO device is first opened the KVM will be available in
1436  * device->kvm if one was associated with the file.
1437  */
1438 void vfio_file_set_kvm(struct file *file, struct kvm *kvm)
1439 {
1440         struct vfio_group *group;
1441
1442         group = vfio_group_from_file(file);
1443         if (group)
1444                 vfio_group_set_kvm(group, kvm);
1445
1446         if (vfio_device_from_file(file))
1447                 vfio_device_file_set_kvm(file, kvm);
1448 }
1449 EXPORT_SYMBOL_GPL(vfio_file_set_kvm);
1450
1451 /*
1452  * Sub-module support
1453  */
1454 /*
1455  * Helper for managing a buffer of info chain capabilities, allocate or
1456  * reallocate a buffer with additional @size, filling in @id and @version
1457  * of the capability.  A pointer to the new capability is returned.
1458  *
1459  * NB. The chain is based at the head of the buffer, so new entries are
1460  * added to the tail, vfio_info_cap_shift() should be called to fixup the
1461  * next offsets prior to copying to the user buffer.
1462  */
1463 struct vfio_info_cap_header *vfio_info_cap_add(struct vfio_info_cap *caps,
1464                                                size_t size, u16 id, u16 version)
1465 {
1466         void *buf;
1467         struct vfio_info_cap_header *header, *tmp;
1468
1469         /* Ensure that the next capability struct will be aligned */
1470         size = ALIGN(size, sizeof(u64));
1471
1472         buf = krealloc(caps->buf, caps->size + size, GFP_KERNEL);
1473         if (!buf) {
1474                 kfree(caps->buf);
1475                 caps->buf = NULL;
1476                 caps->size = 0;
1477                 return ERR_PTR(-ENOMEM);
1478         }
1479
1480         caps->buf = buf;
1481         header = buf + caps->size;
1482
1483         /* Eventually copied to user buffer, zero */
1484         memset(header, 0, size);
1485
1486         header->id = id;
1487         header->version = version;
1488
1489         /* Add to the end of the capability chain */
1490         for (tmp = buf; tmp->next; tmp = buf + tmp->next)
1491                 ; /* nothing */
1492
1493         tmp->next = caps->size;
1494         caps->size += size;
1495
1496         return header;
1497 }
1498 EXPORT_SYMBOL_GPL(vfio_info_cap_add);
1499
1500 void vfio_info_cap_shift(struct vfio_info_cap *caps, size_t offset)
1501 {
1502         struct vfio_info_cap_header *tmp;
1503         void *buf = (void *)caps->buf;
1504
1505         /* Capability structs should start with proper alignment */
1506         WARN_ON(!IS_ALIGNED(offset, sizeof(u64)));
1507
1508         for (tmp = buf; tmp->next; tmp = buf + tmp->next - offset)
1509                 tmp->next += offset;
1510 }
1511 EXPORT_SYMBOL(vfio_info_cap_shift);
1512
1513 int vfio_info_add_capability(struct vfio_info_cap *caps,
1514                              struct vfio_info_cap_header *cap, size_t size)
1515 {
1516         struct vfio_info_cap_header *header;
1517
1518         header = vfio_info_cap_add(caps, size, cap->id, cap->version);
1519         if (IS_ERR(header))
1520                 return PTR_ERR(header);
1521
1522         memcpy(header + 1, cap + 1, size - sizeof(*header));
1523
1524         return 0;
1525 }
1526 EXPORT_SYMBOL(vfio_info_add_capability);
1527
1528 int vfio_set_irqs_validate_and_prepare(struct vfio_irq_set *hdr, int num_irqs,
1529                                        int max_irq_type, size_t *data_size)
1530 {
1531         unsigned long minsz;
1532         size_t size;
1533
1534         minsz = offsetofend(struct vfio_irq_set, count);
1535
1536         if ((hdr->argsz < minsz) || (hdr->index >= max_irq_type) ||
1537             (hdr->count >= (U32_MAX - hdr->start)) ||
1538             (hdr->flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK |
1539                                 VFIO_IRQ_SET_ACTION_TYPE_MASK)))
1540                 return -EINVAL;
1541
1542         if (data_size)
1543                 *data_size = 0;
1544
1545         if (hdr->start >= num_irqs || hdr->start + hdr->count > num_irqs)
1546                 return -EINVAL;
1547
1548         switch (hdr->flags & VFIO_IRQ_SET_DATA_TYPE_MASK) {
1549         case VFIO_IRQ_SET_DATA_NONE:
1550                 size = 0;
1551                 break;
1552         case VFIO_IRQ_SET_DATA_BOOL:
1553                 size = sizeof(uint8_t);
1554                 break;
1555         case VFIO_IRQ_SET_DATA_EVENTFD:
1556                 size = sizeof(int32_t);
1557                 break;
1558         default:
1559                 return -EINVAL;
1560         }
1561
1562         if (size) {
1563                 if (hdr->argsz - minsz < hdr->count * size)
1564                         return -EINVAL;
1565
1566                 if (!data_size)
1567                         return -EINVAL;
1568
1569                 *data_size = hdr->count * size;
1570         }
1571
1572         return 0;
1573 }
1574 EXPORT_SYMBOL(vfio_set_irqs_validate_and_prepare);
1575
1576 /*
1577  * Pin contiguous user pages and return their associated host pages for local
1578  * domain only.
1579  * @device [in]  : device
1580  * @iova [in]    : starting IOVA of user pages to be pinned.
1581  * @npage [in]   : count of pages to be pinned.  This count should not
1582  *                 be greater than VFIO_PIN_PAGES_MAX_ENTRIES.
1583  * @prot [in]    : protection flags
1584  * @pages[out]   : array of host pages
1585  * Return error or number of pages pinned.
1586  *
1587  * A driver may only call this function if the vfio_device was created
1588  * by vfio_register_emulated_iommu_dev() due to vfio_device_container_pin_pages().
1589  */
1590 int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova,
1591                    int npage, int prot, struct page **pages)
1592 {
1593         /* group->container cannot change while a vfio device is open */
1594         if (!pages || !npage || WARN_ON(!vfio_assert_device_open(device)))
1595                 return -EINVAL;
1596         if (!device->ops->dma_unmap)
1597                 return -EINVAL;
1598         if (vfio_device_has_container(device))
1599                 return vfio_device_container_pin_pages(device, iova,
1600                                                        npage, prot, pages);
1601         if (device->iommufd_access) {
1602                 int ret;
1603
1604                 if (iova > ULONG_MAX)
1605                         return -EINVAL;
1606                 /*
1607                  * VFIO ignores the sub page offset, npages is from the start of
1608                  * a PAGE_SIZE chunk of IOVA. The caller is expected to recover
1609                  * the sub page offset by doing:
1610                  *     pages[0] + (iova % PAGE_SIZE)
1611                  */
1612                 ret = iommufd_access_pin_pages(
1613                         device->iommufd_access, ALIGN_DOWN(iova, PAGE_SIZE),
1614                         npage * PAGE_SIZE, pages,
1615                         (prot & IOMMU_WRITE) ? IOMMUFD_ACCESS_RW_WRITE : 0);
1616                 if (ret)
1617                         return ret;
1618                 return npage;
1619         }
1620         return -EINVAL;
1621 }
1622 EXPORT_SYMBOL(vfio_pin_pages);
1623
1624 /*
1625  * Unpin contiguous host pages for local domain only.
1626  * @device [in]  : device
1627  * @iova [in]    : starting address of user pages to be unpinned.
1628  * @npage [in]   : count of pages to be unpinned.  This count should not
1629  *                 be greater than VFIO_PIN_PAGES_MAX_ENTRIES.
1630  */
1631 void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage)
1632 {
1633         if (WARN_ON(!vfio_assert_device_open(device)))
1634                 return;
1635         if (WARN_ON(!device->ops->dma_unmap))
1636                 return;
1637
1638         if (vfio_device_has_container(device)) {
1639                 vfio_device_container_unpin_pages(device, iova, npage);
1640                 return;
1641         }
1642         if (device->iommufd_access) {
1643                 if (WARN_ON(iova > ULONG_MAX))
1644                         return;
1645                 iommufd_access_unpin_pages(device->iommufd_access,
1646                                            ALIGN_DOWN(iova, PAGE_SIZE),
1647                                            npage * PAGE_SIZE);
1648                 return;
1649         }
1650 }
1651 EXPORT_SYMBOL(vfio_unpin_pages);
1652
1653 /*
1654  * This interface allows the CPUs to perform some sort of virtual DMA on
1655  * behalf of the device.
1656  *
1657  * CPUs read/write from/into a range of IOVAs pointing to user space memory
1658  * into/from a kernel buffer.
1659  *
1660  * As the read/write of user space memory is conducted via the CPUs and is
1661  * not a real device DMA, it is not necessary to pin the user space memory.
1662  *
1663  * @device [in]         : VFIO device
1664  * @iova [in]           : base IOVA of a user space buffer
1665  * @data [in]           : pointer to kernel buffer
1666  * @len [in]            : kernel buffer length
1667  * @write               : indicate read or write
1668  * Return error code on failure or 0 on success.
1669  */
1670 int vfio_dma_rw(struct vfio_device *device, dma_addr_t iova, void *data,
1671                 size_t len, bool write)
1672 {
1673         if (!data || len <= 0 || !vfio_assert_device_open(device))
1674                 return -EINVAL;
1675
1676         if (vfio_device_has_container(device))
1677                 return vfio_device_container_dma_rw(device, iova,
1678                                                     data, len, write);
1679
1680         if (device->iommufd_access) {
1681                 unsigned int flags = 0;
1682
1683                 if (iova > ULONG_MAX)
1684                         return -EINVAL;
1685
1686                 /* VFIO historically tries to auto-detect a kthread */
1687                 if (!current->mm)
1688                         flags |= IOMMUFD_ACCESS_RW_KTHREAD;
1689                 if (write)
1690                         flags |= IOMMUFD_ACCESS_RW_WRITE;
1691                 return iommufd_access_rw(device->iommufd_access, iova, data,
1692                                          len, flags);
1693         }
1694         return -EINVAL;
1695 }
1696 EXPORT_SYMBOL(vfio_dma_rw);
1697
1698 /*
1699  * Module/class support
1700  */
1701 static int __init vfio_init(void)
1702 {
1703         int ret;
1704
1705         ida_init(&vfio.device_ida);
1706
1707         ret = vfio_group_init();
1708         if (ret)
1709                 return ret;
1710
1711         ret = vfio_virqfd_init();
1712         if (ret)
1713                 goto err_virqfd;
1714
1715         /* /sys/class/vfio-dev/vfioX */
1716         vfio.device_class = class_create("vfio-dev");
1717         if (IS_ERR(vfio.device_class)) {
1718                 ret = PTR_ERR(vfio.device_class);
1719                 goto err_dev_class;
1720         }
1721
1722         ret = vfio_cdev_init(vfio.device_class);
1723         if (ret)
1724                 goto err_alloc_dev_chrdev;
1725
1726         vfio_debugfs_create_root();
1727         pr_info(DRIVER_DESC " version: " DRIVER_VERSION "\n");
1728         return 0;
1729
1730 err_alloc_dev_chrdev:
1731         class_destroy(vfio.device_class);
1732         vfio.device_class = NULL;
1733 err_dev_class:
1734         vfio_virqfd_exit();
1735 err_virqfd:
1736         vfio_group_cleanup();
1737         return ret;
1738 }
1739
1740 static void __exit vfio_cleanup(void)
1741 {
1742         vfio_debugfs_remove_root();
1743         ida_destroy(&vfio.device_ida);
1744         vfio_cdev_cleanup();
1745         class_destroy(vfio.device_class);
1746         vfio.device_class = NULL;
1747         vfio_virqfd_exit();
1748         vfio_group_cleanup();
1749         xa_destroy(&vfio_device_set_xa);
1750 }
1751
1752 module_init(vfio_init);
1753 module_exit(vfio_cleanup);
1754
1755 MODULE_IMPORT_NS("IOMMUFD");
1756 MODULE_VERSION(DRIVER_VERSION);
1757 MODULE_LICENSE("GPL v2");
1758 MODULE_AUTHOR(DRIVER_AUTHOR);
1759 MODULE_DESCRIPTION(DRIVER_DESC);
1760 MODULE_SOFTDEP("post: vfio_iommu_type1 vfio_iommu_spapr_tce");