drivers/vhost/vdpa.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2018-2020 Intel Corporation.
   4  * Copyright (C) 2020 Red Hat, Inc.
   5  *
   6  * Author: Tiwei Bie <tiwei.bie@intel.com>
   7  *         Jason Wang <jasowang@redhat.com>
   8  *
   9  * Thanks Michael S. Tsirkin for the valuable comments and
  10  * suggestions.  And thanks to Cunming Liang and Zhihong Wang for all
  11  * their supports.
  12  */
  13
  14 #include <linux/kernel.h>
  15 #include <linux/module.h>
  16 #include <linux/cdev.h>
  17 #include <linux/device.h>
  18 #include <linux/mm.h>
  19 #include <linux/slab.h>
  20 #include <linux/iommu.h>
  21 #include <linux/uuid.h>
  22 #include <linux/vdpa.h>
  23 #include <linux/nospec.h>
  24 #include <linux/vhost.h>
  25
  26 #include "vhost.h"
  27
  28 enum {
  29         VHOST_VDPA_BACKEND_FEATURES =
  30         (1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2) |
  31         (1ULL << VHOST_BACKEND_F_IOTLB_BATCH) |
  32         (1ULL << VHOST_BACKEND_F_IOTLB_ASID),
  33 };
  34
  35 #define VHOST_VDPA_DEV_MAX (1U << MINORBITS)
  36
  37 #define VHOST_VDPA_IOTLB_BUCKETS 16
  38
  39 struct vhost_vdpa_as {
  40         struct hlist_node hash_link;
  41         struct vhost_iotlb iotlb;
  42         u32 id;
  43 };
  44
  45 struct vhost_vdpa {
  46         struct vhost_dev vdev;
  47         struct iommu_domain *domain;
  48         struct vhost_virtqueue *vqs;
  49         struct completion completion;
  50         struct vdpa_device *vdpa;
  51         struct hlist_head as[VHOST_VDPA_IOTLB_BUCKETS];
  52         struct device dev;
  53         struct cdev cdev;
  54         atomic_t opened;
  55         u32 nvqs;
  56         int virtio_id;
  57         int minor;
  58         struct eventfd_ctx *config_ctx;
  59         int in_batch;
  60         struct vdpa_iova_range range;
  61         u32 batch_asid;
  62 };
  63
  64 static DEFINE_IDA(vhost_vdpa_ida);
  65
  66 static dev_t vhost_vdpa_major;
  67
  68 static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v,
  69                                    struct vhost_iotlb *iotlb, u64 start,
  70                                    u64 last, u32 asid);
  71
  72 static inline u32 iotlb_to_asid(struct vhost_iotlb *iotlb)
  73 {
  74         struct vhost_vdpa_as *as = container_of(iotlb, struct
  75                                                 vhost_vdpa_as, iotlb);
  76         return as->id;
  77 }
  78
  79 static struct vhost_vdpa_as *asid_to_as(struct vhost_vdpa *v, u32 asid)
  80 {
  81         struct hlist_head *head = &v->as[asid % VHOST_VDPA_IOTLB_BUCKETS];
  82         struct vhost_vdpa_as *as;
  83
  84         hlist_for_each_entry(as, head, hash_link)
  85                 if (as->id == asid)
  86                         return as;
  87
  88         return NULL;
  89 }
  90
  91 static struct vhost_iotlb *asid_to_iotlb(struct vhost_vdpa *v, u32 asid)
  92 {
  93         struct vhost_vdpa_as *as = asid_to_as(v, asid);
  94
  95         if (!as)
  96                 return NULL;
  97
  98         return &as->iotlb;
  99 }
 100
 101 static struct vhost_vdpa_as *vhost_vdpa_alloc_as(struct vhost_vdpa *v, u32 asid)
 102 {
 103         struct hlist_head *head = &v->as[asid % VHOST_VDPA_IOTLB_BUCKETS];
 104         struct vhost_vdpa_as *as;
 105
 106         if (asid_to_as(v, asid))
 107                 return NULL;
 108
 109         if (asid >= v->vdpa->nas)
 110                 return NULL;
 111
 112         as = kmalloc(sizeof(*as), GFP_KERNEL);
 113         if (!as)
 114                 return NULL;
 115
 116         vhost_iotlb_init(&as->iotlb, 0, 0);
 117         as->id = asid;
 118         hlist_add_head(&as->hash_link, head);
 119
 120         return as;
 121 }
 122
 123 static struct vhost_vdpa_as *vhost_vdpa_find_alloc_as(struct vhost_vdpa *v,
 124                                                       u32 asid)
 125 {
 126         struct vhost_vdpa_as *as = asid_to_as(v, asid);
 127
 128         if (as)
 129                 return as;
 130
 131         return vhost_vdpa_alloc_as(v, asid);
 132 }
 133
 134 static int vhost_vdpa_remove_as(struct vhost_vdpa *v, u32 asid)
 135 {
 136         struct vhost_vdpa_as *as = asid_to_as(v, asid);
 137
 138         if (!as)
 139                 return -EINVAL;
 140
 141         hlist_del(&as->hash_link);
 142         vhost_vdpa_iotlb_unmap(v, &as->iotlb, 0ULL, 0ULL - 1, asid);
 143         kfree(as);
 144
 145         return 0;
 146 }
 147
 148 static void handle_vq_kick(struct vhost_work *work)
 149 {
 150         struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
 151                                                   poll.work);
 152         struct vhost_vdpa *v = container_of(vq->dev, struct vhost_vdpa, vdev);
 153         const struct vdpa_config_ops *ops = v->vdpa->config;
 154
 155         ops->kick_vq(v->vdpa, vq - v->vqs);
 156 }
 157
 158 static irqreturn_t vhost_vdpa_virtqueue_cb(void *private)
 159 {
 160         struct vhost_virtqueue *vq = private;
 161         struct eventfd_ctx *call_ctx = vq->call_ctx.ctx;
 162
 163         if (call_ctx)
 164                 eventfd_signal(call_ctx, 1);
 165
 166         return IRQ_HANDLED;
 167 }
 168
 169 static irqreturn_t vhost_vdpa_config_cb(void *private)
 170 {
 171         struct vhost_vdpa *v = private;
 172         struct eventfd_ctx *config_ctx = v->config_ctx;
 173
 174         if (config_ctx)
 175                 eventfd_signal(config_ctx, 1);
 176
 177         return IRQ_HANDLED;
 178 }
 179
 180 static void vhost_vdpa_setup_vq_irq(struct vhost_vdpa *v, u16 qid)
 181 {
 182         struct vhost_virtqueue *vq = &v->vqs[qid];
 183         const struct vdpa_config_ops *ops = v->vdpa->config;
 184         struct vdpa_device *vdpa = v->vdpa;
 185         int ret, irq;
 186
 187         if (!ops->get_vq_irq)
 188                 return;
 189
 190         irq = ops->get_vq_irq(vdpa, qid);
 191         if (irq < 0)
 192                 return;
 193
 194         irq_bypass_unregister_producer(&vq->call_ctx.producer);
 195         if (!vq->call_ctx.ctx)
 196                 return;
 197
 198         vq->call_ctx.producer.token = vq->call_ctx.ctx;
 199         vq->call_ctx.producer.irq = irq;
 200         ret = irq_bypass_register_producer(&vq->call_ctx.producer);
 201         if (unlikely(ret))
 202                 dev_info(&v->dev, "vq %u, irq bypass producer (token %p) registration fails, ret =  %d\n",
 203                          qid, vq->call_ctx.producer.token, ret);
 204 }
 205
 206 static void vhost_vdpa_unsetup_vq_irq(struct vhost_vdpa *v, u16 qid)
 207 {
 208         struct vhost_virtqueue *vq = &v->vqs[qid];
 209
 210         irq_bypass_unregister_producer(&vq->call_ctx.producer);
 211 }
 212
 213 static int vhost_vdpa_reset(struct vhost_vdpa *v)
 214 {
 215         struct vdpa_device *vdpa = v->vdpa;
 216
 217         v->in_batch = 0;
 218
 219         return vdpa_reset(vdpa);
 220 }
 221
 222 static long vhost_vdpa_bind_mm(struct vhost_vdpa *v)
 223 {
 224         struct vdpa_device *vdpa = v->vdpa;
 225         const struct vdpa_config_ops *ops = vdpa->config;
 226
 227         if (!vdpa->use_va || !ops->bind_mm)
 228                 return 0;
 229
 230         return ops->bind_mm(vdpa, v->vdev.mm);
 231 }
 232
 233 static void vhost_vdpa_unbind_mm(struct vhost_vdpa *v)
 234 {
 235         struct vdpa_device *vdpa = v->vdpa;
 236         const struct vdpa_config_ops *ops = vdpa->config;
 237
 238         if (!vdpa->use_va || !ops->unbind_mm)
 239                 return;
 240
 241         ops->unbind_mm(vdpa);
 242 }
 243
 244 static long vhost_vdpa_get_device_id(struct vhost_vdpa *v, u8 __user *argp)
 245 {
 246         struct vdpa_device *vdpa = v->vdpa;
 247         const struct vdpa_config_ops *ops = vdpa->config;
 248         u32 device_id;
 249
 250         device_id = ops->get_device_id(vdpa);
 251
 252         if (copy_to_user(argp, &device_id, sizeof(device_id)))
 253                 return -EFAULT;
 254
 255         return 0;
 256 }
 257
 258 static long vhost_vdpa_get_status(struct vhost_vdpa *v, u8 __user *statusp)
 259 {
 260         struct vdpa_device *vdpa = v->vdpa;
 261         const struct vdpa_config_ops *ops = vdpa->config;
 262         u8 status;
 263
 264         status = ops->get_status(vdpa);
 265
 266         if (copy_to_user(statusp, &status, sizeof(status)))
 267                 return -EFAULT;
 268
 269         return 0;
 270 }
 271
 272 static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp)
 273 {
 274         struct vdpa_device *vdpa = v->vdpa;
 275         const struct vdpa_config_ops *ops = vdpa->config;
 276         u8 status, status_old;
 277         u32 nvqs = v->nvqs;
 278         int ret;
 279         u16 i;
 280
 281         if (copy_from_user(&status, statusp, sizeof(status)))
 282                 return -EFAULT;
 283
 284         status_old = ops->get_status(vdpa);
 285
 286         /*
 287          * Userspace shouldn't remove status bits unless reset the
 288          * status to 0.
 289          */
 290         if (status != 0 && (status_old & ~status) != 0)
 291                 return -EINVAL;
 292
 293         if ((status_old & VIRTIO_CONFIG_S_DRIVER_OK) && !(status & VIRTIO_CONFIG_S_DRIVER_OK))
 294                 for (i = 0; i < nvqs; i++)
 295                         vhost_vdpa_unsetup_vq_irq(v, i);
 296
 297         if (status == 0) {
 298                 ret = vdpa_reset(vdpa);
 299                 if (ret)
 300                         return ret;
 301         } else
 302                 vdpa_set_status(vdpa, status);
 303
 304         if ((status & VIRTIO_CONFIG_S_DRIVER_OK) && !(status_old & VIRTIO_CONFIG_S_DRIVER_OK))
 305                 for (i = 0; i < nvqs; i++)
 306                         vhost_vdpa_setup_vq_irq(v, i);
 307
 308         return 0;
 309 }
 310
 311 static int vhost_vdpa_config_validate(struct vhost_vdpa *v,
 312                                       struct vhost_vdpa_config *c)
 313 {
 314         struct vdpa_device *vdpa = v->vdpa;
 315         size_t size = vdpa->config->get_config_size(vdpa);
 316
 317         if (c->len == 0 || c->off > size)
 318                 return -EINVAL;
 319
 320         if (c->len > size - c->off)
 321                 return -E2BIG;
 322
 323         return 0;
 324 }
 325
 326 static long vhost_vdpa_get_config(struct vhost_vdpa *v,
 327                                   struct vhost_vdpa_config __user *c)
 328 {
 329         struct vdpa_device *vdpa = v->vdpa;
 330         struct vhost_vdpa_config config;
 331         unsigned long size = offsetof(struct vhost_vdpa_config, buf);
 332         u8 *buf;
 333
 334         if (copy_from_user(&config, c, size))
 335                 return -EFAULT;
 336         if (vhost_vdpa_config_validate(v, &config))
 337                 return -EINVAL;
 338         buf = kvzalloc(config.len, GFP_KERNEL);
 339         if (!buf)
 340                 return -ENOMEM;
 341
 342         vdpa_get_config(vdpa, config.off, buf, config.len);
 343
 344         if (copy_to_user(c->buf, buf, config.len)) {
 345                 kvfree(buf);
 346                 return -EFAULT;
 347         }
 348
 349         kvfree(buf);
 350         return 0;
 351 }
 352
 353 static long vhost_vdpa_set_config(struct vhost_vdpa *v,
 354                                   struct vhost_vdpa_config __user *c)
 355 {
 356         struct vdpa_device *vdpa = v->vdpa;
 357         struct vhost_vdpa_config config;
 358         unsigned long size = offsetof(struct vhost_vdpa_config, buf);
 359         u8 *buf;
 360
 361         if (copy_from_user(&config, c, size))
 362                 return -EFAULT;
 363         if (vhost_vdpa_config_validate(v, &config))
 364                 return -EINVAL;
 365
 366         buf = vmemdup_user(c->buf, config.len);
 367         if (IS_ERR(buf))
 368                 return PTR_ERR(buf);
 369
 370         vdpa_set_config(vdpa, config.off, buf, config.len);
 371
 372         kvfree(buf);
 373         return 0;
 374 }
 375
 376 static bool vhost_vdpa_can_suspend(const struct vhost_vdpa *v)
 377 {
 378         struct vdpa_device *vdpa = v->vdpa;
 379         const struct vdpa_config_ops *ops = vdpa->config;
 380
 381         return ops->suspend;
 382 }
 383
 384 static bool vhost_vdpa_can_resume(const struct vhost_vdpa *v)
 385 {
 386         struct vdpa_device *vdpa = v->vdpa;
 387         const struct vdpa_config_ops *ops = vdpa->config;
 388
 389         return ops->resume;
 390 }
 391
 392 static long vhost_vdpa_get_features(struct vhost_vdpa *v, u64 __user *featurep)
 393 {
 394         struct vdpa_device *vdpa = v->vdpa;
 395         const struct vdpa_config_ops *ops = vdpa->config;
 396         u64 features;
 397
 398         features = ops->get_device_features(vdpa);
 399
 400         if (copy_to_user(featurep, &features, sizeof(features)))
 401                 return -EFAULT;
 402
 403         return 0;
 404 }
 405
 406 static long vhost_vdpa_set_features(struct vhost_vdpa *v, u64 __user *featurep)
 407 {
 408         struct vdpa_device *vdpa = v->vdpa;
 409         const struct vdpa_config_ops *ops = vdpa->config;
 410         u64 features;
 411
 412         /*
 413          * It's not allowed to change the features after they have
 414          * been negotiated.
 415          */
 416         if (ops->get_status(vdpa) & VIRTIO_CONFIG_S_FEATURES_OK)
 417                 return -EBUSY;
 418
 419         if (copy_from_user(&features, featurep, sizeof(features)))
 420                 return -EFAULT;
 421
 422         if (vdpa_set_features(vdpa, features))
 423                 return -EINVAL;
 424
 425         return 0;
 426 }
 427
 428 static long vhost_vdpa_get_vring_num(struct vhost_vdpa *v, u16 __user *argp)
 429 {
 430         struct vdpa_device *vdpa = v->vdpa;
 431         const struct vdpa_config_ops *ops = vdpa->config;
 432         u16 num;
 433
 434         num = ops->get_vq_num_max(vdpa);
 435
 436         if (copy_to_user(argp, &num, sizeof(num)))
 437                 return -EFAULT;
 438
 439         return 0;
 440 }
 441
 442 static void vhost_vdpa_config_put(struct vhost_vdpa *v)
 443 {
 444         if (v->config_ctx) {
 445                 eventfd_ctx_put(v->config_ctx);
 446                 v->config_ctx = NULL;
 447         }
 448 }
 449
 450 static long vhost_vdpa_set_config_call(struct vhost_vdpa *v, u32 __user *argp)
 451 {
 452         struct vdpa_callback cb;
 453         int fd;
 454         struct eventfd_ctx *ctx;
 455
 456         cb.callback = vhost_vdpa_config_cb;
 457         cb.private = v;
 458         if (copy_from_user(&fd, argp, sizeof(fd)))
 459                 return  -EFAULT;
 460
 461         ctx = fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(fd);
 462         swap(ctx, v->config_ctx);
 463
 464         if (!IS_ERR_OR_NULL(ctx))
 465                 eventfd_ctx_put(ctx);
 466
 467         if (IS_ERR(v->config_ctx)) {
 468                 long ret = PTR_ERR(v->config_ctx);
 469
 470                 v->config_ctx = NULL;
 471                 return ret;
 472         }
 473
 474         v->vdpa->config->set_config_cb(v->vdpa, &cb);
 475
 476         return 0;
 477 }
 478
 479 static long vhost_vdpa_get_iova_range(struct vhost_vdpa *v, u32 __user *argp)
 480 {
 481         struct vhost_vdpa_iova_range range = {
 482                 .first = v->range.first,
 483                 .last = v->range.last,
 484         };
 485
 486         if (copy_to_user(argp, &range, sizeof(range)))
 487                 return -EFAULT;
 488         return 0;
 489 }
 490
 491 static long vhost_vdpa_get_config_size(struct vhost_vdpa *v, u32 __user *argp)
 492 {
 493         struct vdpa_device *vdpa = v->vdpa;
 494         const struct vdpa_config_ops *ops = vdpa->config;
 495         u32 size;
 496
 497         size = ops->get_config_size(vdpa);
 498
 499         if (copy_to_user(argp, &size, sizeof(size)))
 500                 return -EFAULT;
 501
 502         return 0;
 503 }
 504
 505 static long vhost_vdpa_get_vqs_count(struct vhost_vdpa *v, u32 __user *argp)
 506 {
 507         struct vdpa_device *vdpa = v->vdpa;
 508
 509         if (copy_to_user(argp, &vdpa->nvqs, sizeof(vdpa->nvqs)))
 510                 return -EFAULT;
 511
 512         return 0;
 513 }
 514
 515 /* After a successful return of ioctl the device must not process more
 516  * virtqueue descriptors. The device can answer to read or writes of config
 517  * fields as if it were not suspended. In particular, writing to "queue_enable"
 518  * with a value of 1 will not make the device start processing buffers.
 519  */
 520 static long vhost_vdpa_suspend(struct vhost_vdpa *v)
 521 {
 522         struct vdpa_device *vdpa = v->vdpa;
 523         const struct vdpa_config_ops *ops = vdpa->config;
 524
 525         if (!ops->suspend)
 526                 return -EOPNOTSUPP;
 527
 528         return ops->suspend(vdpa);
 529 }
 530
 531 /* After a successful return of this ioctl the device resumes processing
 532  * virtqueue descriptors. The device becomes fully operational the same way it
 533  * was before it was suspended.
 534  */
 535 static long vhost_vdpa_resume(struct vhost_vdpa *v)
 536 {
 537         struct vdpa_device *vdpa = v->vdpa;
 538         const struct vdpa_config_ops *ops = vdpa->config;
 539
 540         if (!ops->resume)
 541                 return -EOPNOTSUPP;
 542
 543         return ops->resume(vdpa);
 544 }
 545
 546 static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
 547                                    void __user *argp)
 548 {
 549         struct vdpa_device *vdpa = v->vdpa;
 550         const struct vdpa_config_ops *ops = vdpa->config;
 551         struct vdpa_vq_state vq_state;
 552         struct vdpa_callback cb;
 553         struct vhost_virtqueue *vq;
 554         struct vhost_vring_state s;
 555         u32 idx;
 556         long r;
 557
 558         r = get_user(idx, (u32 __user *)argp);
 559         if (r < 0)
 560                 return r;
 561
 562         if (idx >= v->nvqs)
 563                 return -ENOBUFS;
 564
 565         idx = array_index_nospec(idx, v->nvqs);
 566         vq = &v->vqs[idx];
 567
 568         switch (cmd) {
 569         case VHOST_VDPA_SET_VRING_ENABLE:
 570                 if (copy_from_user(&s, argp, sizeof(s)))
 571                         return -EFAULT;
 572                 ops->set_vq_ready(vdpa, idx, s.num);
 573                 return 0;
 574         case VHOST_VDPA_GET_VRING_GROUP:
 575                 if (!ops->get_vq_group)
 576                         return -EOPNOTSUPP;
 577                 s.index = idx;
 578                 s.num = ops->get_vq_group(vdpa, idx);
 579                 if (s.num >= vdpa->ngroups)
 580                         return -EIO;
 581                 else if (copy_to_user(argp, &s, sizeof(s)))
 582                         return -EFAULT;
 583                 return 0;
 584         case VHOST_VDPA_SET_GROUP_ASID:
 585                 if (copy_from_user(&s, argp, sizeof(s)))
 586                         return -EFAULT;
 587                 if (s.num >= vdpa->nas)
 588                         return -EINVAL;
 589                 if (!ops->set_group_asid)
 590                         return -EOPNOTSUPP;
 591                 return ops->set_group_asid(vdpa, idx, s.num);
 592         case VHOST_GET_VRING_BASE:
 593                 r = ops->get_vq_state(v->vdpa, idx, &vq_state);
 594                 if (r)
 595                         return r;
 596
 597                 vq->last_avail_idx = vq_state.split.avail_index;
 598                 break;
 599         }
 600
 601         r = vhost_vring_ioctl(&v->vdev, cmd, argp);
 602         if (r)
 603                 return r;
 604
 605         switch (cmd) {
 606         case VHOST_SET_VRING_ADDR:
 607                 if (ops->set_vq_address(vdpa, idx,
 608                                         (u64)(uintptr_t)vq->desc,
 609                                         (u64)(uintptr_t)vq->avail,
 610                                         (u64)(uintptr_t)vq->used))
 611                         r = -EINVAL;
 612                 break;
 613
 614         case VHOST_SET_VRING_BASE:
 615                 vq_state.split.avail_index = vq->last_avail_idx;
 616                 if (ops->set_vq_state(vdpa, idx, &vq_state))
 617                         r = -EINVAL;
 618                 break;
 619
 620         case VHOST_SET_VRING_CALL:
 621                 if (vq->call_ctx.ctx) {
 622                         cb.callback = vhost_vdpa_virtqueue_cb;
 623                         cb.private = vq;
 624                         cb.trigger = vq->call_ctx.ctx;
 625                 } else {
 626                         cb.callback = NULL;
 627                         cb.private = NULL;
 628                         cb.trigger = NULL;
 629                 }
 630                 ops->set_vq_cb(vdpa, idx, &cb);
 631                 vhost_vdpa_setup_vq_irq(v, idx);
 632                 break;
 633
 634         case VHOST_SET_VRING_NUM:
 635                 ops->set_vq_num(vdpa, idx, vq->num);
 636                 break;
 637         }
 638
 639         return r;
 640 }
 641
 642 static long vhost_vdpa_unlocked_ioctl(struct file *filep,
 643                                       unsigned int cmd, unsigned long arg)
 644 {
 645         struct vhost_vdpa *v = filep->private_data;
 646         struct vhost_dev *d = &v->vdev;
 647         void __user *argp = (void __user *)arg;
 648         u64 __user *featurep = argp;
 649         u64 features;
 650         long r = 0;
 651
 652         if (cmd == VHOST_SET_BACKEND_FEATURES) {
 653                 if (copy_from_user(&features, featurep, sizeof(features)))
 654                         return -EFAULT;
 655                 if (features & ~(VHOST_VDPA_BACKEND_FEATURES |
 656                                  BIT_ULL(VHOST_BACKEND_F_SUSPEND) |
 657                                  BIT_ULL(VHOST_BACKEND_F_RESUME)))
 658                         return -EOPNOTSUPP;
 659                 if ((features & BIT_ULL(VHOST_BACKEND_F_SUSPEND)) &&
 660                      !vhost_vdpa_can_suspend(v))
 661                         return -EOPNOTSUPP;
 662                 if ((features & BIT_ULL(VHOST_BACKEND_F_RESUME)) &&
 663                      !vhost_vdpa_can_resume(v))
 664                         return -EOPNOTSUPP;
 665                 vhost_set_backend_features(&v->vdev, features);
 666                 return 0;
 667         }
 668
 669         mutex_lock(&d->mutex);
 670
 671         switch (cmd) {
 672         case VHOST_VDPA_GET_DEVICE_ID:
 673                 r = vhost_vdpa_get_device_id(v, argp);
 674                 break;
 675         case VHOST_VDPA_GET_STATUS:
 676                 r = vhost_vdpa_get_status(v, argp);
 677                 break;
 678         case VHOST_VDPA_SET_STATUS:
 679                 r = vhost_vdpa_set_status(v, argp);
 680                 break;
 681         case VHOST_VDPA_GET_CONFIG:
 682                 r = vhost_vdpa_get_config(v, argp);
 683                 break;
 684         case VHOST_VDPA_SET_CONFIG:
 685                 r = vhost_vdpa_set_config(v, argp);
 686                 break;
 687         case VHOST_GET_FEATURES:
 688                 r = vhost_vdpa_get_features(v, argp);
 689                 break;
 690         case VHOST_SET_FEATURES:
 691                 r = vhost_vdpa_set_features(v, argp);
 692                 break;
 693         case VHOST_VDPA_GET_VRING_NUM:
 694                 r = vhost_vdpa_get_vring_num(v, argp);
 695                 break;
 696         case VHOST_VDPA_GET_GROUP_NUM:
 697                 if (copy_to_user(argp, &v->vdpa->ngroups,
 698                                  sizeof(v->vdpa->ngroups)))
 699                         r = -EFAULT;
 700                 break;
 701         case VHOST_VDPA_GET_AS_NUM:
 702                 if (copy_to_user(argp, &v->vdpa->nas, sizeof(v->vdpa->nas)))
 703                         r = -EFAULT;
 704                 break;
 705         case VHOST_SET_LOG_BASE:
 706         case VHOST_SET_LOG_FD:
 707                 r = -ENOIOCTLCMD;
 708                 break;
 709         case VHOST_VDPA_SET_CONFIG_CALL:
 710                 r = vhost_vdpa_set_config_call(v, argp);
 711                 break;
 712         case VHOST_GET_BACKEND_FEATURES:
 713                 features = VHOST_VDPA_BACKEND_FEATURES;
 714                 if (vhost_vdpa_can_suspend(v))
 715                         features |= BIT_ULL(VHOST_BACKEND_F_SUSPEND);
 716                 if (vhost_vdpa_can_resume(v))
 717                         features |= BIT_ULL(VHOST_BACKEND_F_RESUME);
 718                 if (copy_to_user(featurep, &features, sizeof(features)))
 719                         r = -EFAULT;
 720                 break;
 721         case VHOST_VDPA_GET_IOVA_RANGE:
 722                 r = vhost_vdpa_get_iova_range(v, argp);
 723                 break;
 724         case VHOST_VDPA_GET_CONFIG_SIZE:
 725                 r = vhost_vdpa_get_config_size(v, argp);
 726                 break;
 727         case VHOST_VDPA_GET_VQS_COUNT:
 728                 r = vhost_vdpa_get_vqs_count(v, argp);
 729                 break;
 730         case VHOST_VDPA_SUSPEND:
 731                 r = vhost_vdpa_suspend(v);
 732                 break;
 733         case VHOST_VDPA_RESUME:
 734                 r = vhost_vdpa_resume(v);
 735                 break;
 736         default:
 737                 r = vhost_dev_ioctl(&v->vdev, cmd, argp);
 738                 if (r == -ENOIOCTLCMD)
 739                         r = vhost_vdpa_vring_ioctl(v, cmd, argp);
 740                 break;
 741         }
 742
 743         if (r)
 744                 goto out;
 745
 746         switch (cmd) {
 747         case VHOST_SET_OWNER:
 748                 r = vhost_vdpa_bind_mm(v);
 749                 if (r)
 750                         vhost_dev_reset_owner(d, NULL);
 751                 break;
 752         }
 753 out:
 754         mutex_unlock(&d->mutex);
 755         return r;
 756 }
 757 static void vhost_vdpa_general_unmap(struct vhost_vdpa *v,
 758                                      struct vhost_iotlb_map *map, u32 asid)
 759 {
 760         struct vdpa_device *vdpa = v->vdpa;
 761         const struct vdpa_config_ops *ops = vdpa->config;
 762         if (ops->dma_map) {
 763                 ops->dma_unmap(vdpa, asid, map->start, map->size);
 764         } else if (ops->set_map == NULL) {
 765                 iommu_unmap(v->domain, map->start, map->size);
 766         }
 767 }
 768
 769 static void vhost_vdpa_pa_unmap(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
 770                                 u64 start, u64 last, u32 asid)
 771 {
 772         struct vhost_dev *dev = &v->vdev;
 773         struct vhost_iotlb_map *map;
 774         struct page *page;
 775         unsigned long pfn, pinned;
 776
 777         while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) {
 778                 pinned = PFN_DOWN(map->size);
 779                 for (pfn = PFN_DOWN(map->addr);
 780                      pinned > 0; pfn++, pinned--) {
 781                         page = pfn_to_page(pfn);
 782                         if (map->perm & VHOST_ACCESS_WO)
 783                                 set_page_dirty_lock(page);
 784                         unpin_user_page(page);
 785                 }
 786                 atomic64_sub(PFN_DOWN(map->size), &dev->mm->pinned_vm);
 787                 vhost_vdpa_general_unmap(v, map, asid);
 788                 vhost_iotlb_map_free(iotlb, map);
 789         }
 790 }
 791
 792 static void vhost_vdpa_va_unmap(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
 793                                 u64 start, u64 last, u32 asid)
 794 {
 795         struct vhost_iotlb_map *map;
 796         struct vdpa_map_file *map_file;
 797
 798         while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) {
 799                 map_file = (struct vdpa_map_file *)map->opaque;
 800                 fput(map_file->file);
 801                 kfree(map_file);
 802                 vhost_vdpa_general_unmap(v, map, asid);
 803                 vhost_iotlb_map_free(iotlb, map);
 804         }
 805 }
 806
 807 static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v,
 808                                    struct vhost_iotlb *iotlb, u64 start,
 809                                    u64 last, u32 asid)
 810 {
 811         struct vdpa_device *vdpa = v->vdpa;
 812
 813         if (vdpa->use_va)
 814                 return vhost_vdpa_va_unmap(v, iotlb, start, last, asid);
 815
 816         return vhost_vdpa_pa_unmap(v, iotlb, start, last, asid);
 817 }
 818
 819 static int perm_to_iommu_flags(u32 perm)
 820 {
 821         int flags = 0;
 822
 823         switch (perm) {
 824         case VHOST_ACCESS_WO:
 825                 flags |= IOMMU_WRITE;
 826                 break;
 827         case VHOST_ACCESS_RO:
 828                 flags |= IOMMU_READ;
 829                 break;
 830         case VHOST_ACCESS_RW:
 831                 flags |= (IOMMU_WRITE | IOMMU_READ);
 832                 break;
 833         default:
 834                 WARN(1, "invalidate vhost IOTLB permission\n");
 835                 break;
 836         }
 837
 838         return flags | IOMMU_CACHE;
 839 }
 840
 841 static int vhost_vdpa_map(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
 842                           u64 iova, u64 size, u64 pa, u32 perm, void *opaque)
 843 {
 844         struct vhost_dev *dev = &v->vdev;
 845         struct vdpa_device *vdpa = v->vdpa;
 846         const struct vdpa_config_ops *ops = vdpa->config;
 847         u32 asid = iotlb_to_asid(iotlb);
 848         int r = 0;
 849
 850         r = vhost_iotlb_add_range_ctx(iotlb, iova, iova + size - 1,
 851                                       pa, perm, opaque);
 852         if (r)
 853                 return r;
 854
 855         if (ops->dma_map) {
 856                 r = ops->dma_map(vdpa, asid, iova, size, pa, perm, opaque);
 857         } else if (ops->set_map) {
 858                 if (!v->in_batch)
 859                         r = ops->set_map(vdpa, asid, iotlb);
 860         } else {
 861                 r = iommu_map(v->domain, iova, pa, size,
 862                               perm_to_iommu_flags(perm), GFP_KERNEL);
 863         }
 864         if (r) {
 865                 vhost_iotlb_del_range(iotlb, iova, iova + size - 1);
 866                 return r;
 867         }
 868
 869         if (!vdpa->use_va)
 870                 atomic64_add(PFN_DOWN(size), &dev->mm->pinned_vm);
 871
 872         return 0;
 873 }
 874
 875 static void vhost_vdpa_unmap(struct vhost_vdpa *v,
 876                              struct vhost_iotlb *iotlb,
 877                              u64 iova, u64 size)
 878 {
 879         struct vdpa_device *vdpa = v->vdpa;
 880         const struct vdpa_config_ops *ops = vdpa->config;
 881         u32 asid = iotlb_to_asid(iotlb);
 882
 883         vhost_vdpa_iotlb_unmap(v, iotlb, iova, iova + size - 1, asid);
 884
 885         if (ops->set_map) {
 886                 if (!v->in_batch)
 887                         ops->set_map(vdpa, asid, iotlb);
 888         }
 889
 890 }
 891
 892 static int vhost_vdpa_va_map(struct vhost_vdpa *v,
 893                              struct vhost_iotlb *iotlb,
 894                              u64 iova, u64 size, u64 uaddr, u32 perm)
 895 {
 896         struct vhost_dev *dev = &v->vdev;
 897         u64 offset, map_size, map_iova = iova;
 898         struct vdpa_map_file *map_file;
 899         struct vm_area_struct *vma;
 900         int ret = 0;
 901
 902         mmap_read_lock(dev->mm);
 903
 904         while (size) {
 905                 vma = find_vma(dev->mm, uaddr);
 906                 if (!vma) {
 907                         ret = -EINVAL;
 908                         break;
 909                 }
 910                 map_size = min(size, vma->vm_end - uaddr);
 911                 if (!(vma->vm_file && (vma->vm_flags & VM_SHARED) &&
 912                         !(vma->vm_flags & (VM_IO | VM_PFNMAP))))
 913                         goto next;
 914
 915                 map_file = kzalloc(sizeof(*map_file), GFP_KERNEL);
 916                 if (!map_file) {
 917                         ret = -ENOMEM;
 918                         break;
 919                 }
 920                 offset = (vma->vm_pgoff << PAGE_SHIFT) + uaddr - vma->vm_start;
 921                 map_file->offset = offset;
 922                 map_file->file = get_file(vma->vm_file);
 923                 ret = vhost_vdpa_map(v, iotlb, map_iova, map_size, uaddr,
 924                                      perm, map_file);
 925                 if (ret) {
 926                         fput(map_file->file);
 927                         kfree(map_file);
 928                         break;
 929                 }
 930 next:
 931                 size -= map_size;
 932                 uaddr += map_size;
 933                 map_iova += map_size;
 934         }
 935         if (ret)
 936                 vhost_vdpa_unmap(v, iotlb, iova, map_iova - iova);
 937
 938         mmap_read_unlock(dev->mm);
 939
 940         return ret;
 941 }
 942
 943 static int vhost_vdpa_pa_map(struct vhost_vdpa *v,
 944                              struct vhost_iotlb *iotlb,
 945                              u64 iova, u64 size, u64 uaddr, u32 perm)
 946 {
 947         struct vhost_dev *dev = &v->vdev;
 948         struct page **page_list;
 949         unsigned long list_size = PAGE_SIZE / sizeof(struct page *);
 950         unsigned int gup_flags = FOLL_LONGTERM;
 951         unsigned long npages, cur_base, map_pfn, last_pfn = 0;
 952         unsigned long lock_limit, sz2pin, nchunks, i;
 953         u64 start = iova;
 954         long pinned;
 955         int ret = 0;
 956
 957         /* Limit the use of memory for bookkeeping */
 958         page_list = (struct page **) __get_free_page(GFP_KERNEL);
 959         if (!page_list)
 960                 return -ENOMEM;
 961
 962         if (perm & VHOST_ACCESS_WO)
 963                 gup_flags |= FOLL_WRITE;
 964
 965         npages = PFN_UP(size + (iova & ~PAGE_MASK));
 966         if (!npages) {
 967                 ret = -EINVAL;
 968                 goto free;
 969         }
 970
 971         mmap_read_lock(dev->mm);
 972
 973         lock_limit = PFN_DOWN(rlimit(RLIMIT_MEMLOCK));
 974         if (npages + atomic64_read(&dev->mm->pinned_vm) > lock_limit) {
 975                 ret = -ENOMEM;
 976                 goto unlock;
 977         }
 978
 979         cur_base = uaddr & PAGE_MASK;
 980         iova &= PAGE_MASK;
 981         nchunks = 0;
 982
 983         while (npages) {
 984                 sz2pin = min_t(unsigned long, npages, list_size);
 985                 pinned = pin_user_pages(cur_base, sz2pin,
 986                                         gup_flags, page_list, NULL);
 987                 if (sz2pin != pinned) {
 988                         if (pinned < 0) {
 989                                 ret = pinned;
 990                         } else {
 991                                 unpin_user_pages(page_list, pinned);
 992                                 ret = -ENOMEM;
 993                         }
 994                         goto out;
 995                 }
 996                 nchunks++;
 997
 998                 if (!last_pfn)
 999                         map_pfn = page_to_pfn(page_list[0]);
1000
1001                 for (i = 0; i < pinned; i++) {
1002                         unsigned long this_pfn = page_to_pfn(page_list[i]);
1003                         u64 csize;
1004
1005                         if (last_pfn && (this_pfn != last_pfn + 1)) {
1006                                 /* Pin a contiguous chunk of memory */
1007                                 csize = PFN_PHYS(last_pfn - map_pfn + 1);
1008                                 ret = vhost_vdpa_map(v, iotlb, iova, csize,
1009                                                      PFN_PHYS(map_pfn),
1010                                                      perm, NULL);
1011                                 if (ret) {
1012                                         /*
1013                                          * Unpin the pages that are left unmapped
1014                                          * from this point on in the current
1015                                          * page_list. The remaining outstanding
1016                                          * ones which may stride across several
1017                                          * chunks will be covered in the common
1018                                          * error path subsequently.
1019                                          */
1020                                         unpin_user_pages(&page_list[i],
1021                                                          pinned - i);
1022                                         goto out;
1023                                 }
1024
1025                                 map_pfn = this_pfn;
1026                                 iova += csize;
1027                                 nchunks = 0;
1028                         }
1029
1030                         last_pfn = this_pfn;
1031                 }
1032
1033                 cur_base += PFN_PHYS(pinned);
1034                 npages -= pinned;
1035         }
1036
1037         /* Pin the rest chunk */
1038         ret = vhost_vdpa_map(v, iotlb, iova, PFN_PHYS(last_pfn - map_pfn + 1),
1039                              PFN_PHYS(map_pfn), perm, NULL);
1040 out:
1041         if (ret) {
1042                 if (nchunks) {
1043                         unsigned long pfn;
1044
1045                         /*
1046                          * Unpin the outstanding pages which are yet to be
1047                          * mapped but haven't due to vdpa_map() or
1048                          * pin_user_pages() failure.
1049                          *
1050                          * Mapped pages are accounted in vdpa_map(), hence
1051                          * the corresponding unpinning will be handled by
1052                          * vdpa_unmap().
1053                          */
1054                         WARN_ON(!last_pfn);
1055                         for (pfn = map_pfn; pfn <= last_pfn; pfn++)
1056                                 unpin_user_page(pfn_to_page(pfn));
1057                 }
1058                 vhost_vdpa_unmap(v, iotlb, start, size);
1059         }
1060 unlock:
1061         mmap_read_unlock(dev->mm);
1062 free:
1063         free_page((unsigned long)page_list);
1064         return ret;
1065
1066 }
1067
1068 static int vhost_vdpa_process_iotlb_update(struct vhost_vdpa *v,
1069                                            struct vhost_iotlb *iotlb,
1070                                            struct vhost_iotlb_msg *msg)
1071 {
1072         struct vdpa_device *vdpa = v->vdpa;
1073
1074         if (msg->iova < v->range.first || !msg->size ||
1075             msg->iova > U64_MAX - msg->size + 1 ||
1076             msg->iova + msg->size - 1 > v->range.last)
1077                 return -EINVAL;
1078
1079         if (vhost_iotlb_itree_first(iotlb, msg->iova,
1080                                     msg->iova + msg->size - 1))
1081                 return -EEXIST;
1082
1083         if (vdpa->use_va)
1084                 return vhost_vdpa_va_map(v, iotlb, msg->iova, msg->size,
1085                                          msg->uaddr, msg->perm);
1086
1087         return vhost_vdpa_pa_map(v, iotlb, msg->iova, msg->size, msg->uaddr,
1088                                  msg->perm);
1089 }
1090
1091 static int vhost_vdpa_process_iotlb_msg(struct vhost_dev *dev, u32 asid,
1092                                         struct vhost_iotlb_msg *msg)
1093 {
1094         struct vhost_vdpa *v = container_of(dev, struct vhost_vdpa, vdev);
1095         struct vdpa_device *vdpa = v->vdpa;
1096         const struct vdpa_config_ops *ops = vdpa->config;
1097         struct vhost_iotlb *iotlb = NULL;
1098         struct vhost_vdpa_as *as = NULL;
1099         int r = 0;
1100
1101         mutex_lock(&dev->mutex);
1102
1103         r = vhost_dev_check_owner(dev);
1104         if (r)
1105                 goto unlock;
1106
1107         if (msg->type == VHOST_IOTLB_UPDATE ||
1108             msg->type == VHOST_IOTLB_BATCH_BEGIN) {
1109                 as = vhost_vdpa_find_alloc_as(v, asid);
1110                 if (!as) {
1111                         dev_err(&v->dev, "can't find and alloc asid %d\n",
1112                                 asid);
1113                         r = -EINVAL;
1114                         goto unlock;
1115                 }
1116                 iotlb = &as->iotlb;
1117         } else
1118                 iotlb = asid_to_iotlb(v, asid);
1119
1120         if ((v->in_batch && v->batch_asid != asid) || !iotlb) {
1121                 if (v->in_batch && v->batch_asid != asid) {
1122                         dev_info(&v->dev, "batch id %d asid %d\n",
1123                                  v->batch_asid, asid);
1124                 }
1125                 if (!iotlb)
1126                         dev_err(&v->dev, "no iotlb for asid %d\n", asid);
1127                 r = -EINVAL;
1128                 goto unlock;
1129         }
1130
1131         switch (msg->type) {
1132         case VHOST_IOTLB_UPDATE:
1133                 r = vhost_vdpa_process_iotlb_update(v, iotlb, msg);
1134                 break;
1135         case VHOST_IOTLB_INVALIDATE:
1136                 vhost_vdpa_unmap(v, iotlb, msg->iova, msg->size);
1137                 break;
1138         case VHOST_IOTLB_BATCH_BEGIN:
1139                 v->batch_asid = asid;
1140                 v->in_batch = true;
1141                 break;
1142         case VHOST_IOTLB_BATCH_END:
1143                 if (v->in_batch && ops->set_map)
1144                         ops->set_map(vdpa, asid, iotlb);
1145                 v->in_batch = false;
1146                 break;
1147         default:
1148                 r = -EINVAL;
1149                 break;
1150         }
1151 unlock:
1152         mutex_unlock(&dev->mutex);
1153
1154         return r;
1155 }
1156
1157 static ssize_t vhost_vdpa_chr_write_iter(struct kiocb *iocb,
1158                                          struct iov_iter *from)
1159 {
1160         struct file *file = iocb->ki_filp;
1161         struct vhost_vdpa *v = file->private_data;
1162         struct vhost_dev *dev = &v->vdev;
1163
1164         return vhost_chr_write_iter(dev, from);
1165 }
1166
1167 static int vhost_vdpa_alloc_domain(struct vhost_vdpa *v)
1168 {
1169         struct vdpa_device *vdpa = v->vdpa;
1170         const struct vdpa_config_ops *ops = vdpa->config;
1171         struct device *dma_dev = vdpa_get_dma_dev(vdpa);
1172         const struct bus_type *bus;
1173         int ret;
1174
1175         /* Device want to do DMA by itself */
1176         if (ops->set_map || ops->dma_map)
1177                 return 0;
1178
1179         bus = dma_dev->bus;
1180         if (!bus)
1181                 return -EFAULT;
1182
1183         if (!device_iommu_capable(dma_dev, IOMMU_CAP_CACHE_COHERENCY)) {
1184                 dev_warn_once(&v->dev,
1185                               "Failed to allocate domain, device is not IOMMU cache coherent capable\n");
1186                 return -ENOTSUPP;
1187         }
1188
1189         v->domain = iommu_domain_alloc(bus);
1190         if (!v->domain)
1191                 return -EIO;
1192
1193         ret = iommu_attach_device(v->domain, dma_dev);
1194         if (ret)
1195                 goto err_attach;
1196
1197         return 0;
1198
1199 err_attach:
1200         iommu_domain_free(v->domain);
1201         v->domain = NULL;
1202         return ret;
1203 }
1204
1205 static void vhost_vdpa_free_domain(struct vhost_vdpa *v)
1206 {
1207         struct vdpa_device *vdpa = v->vdpa;
1208         struct device *dma_dev = vdpa_get_dma_dev(vdpa);
1209
1210         if (v->domain) {
1211                 iommu_detach_device(v->domain, dma_dev);
1212                 iommu_domain_free(v->domain);
1213         }
1214
1215         v->domain = NULL;
1216 }
1217
1218 static void vhost_vdpa_set_iova_range(struct vhost_vdpa *v)
1219 {
1220         struct vdpa_iova_range *range = &v->range;
1221         struct vdpa_device *vdpa = v->vdpa;
1222         const struct vdpa_config_ops *ops = vdpa->config;
1223
1224         if (ops->get_iova_range) {
1225                 *range = ops->get_iova_range(vdpa);
1226         } else if (v->domain && v->domain->geometry.force_aperture) {
1227                 range->first = v->domain->geometry.aperture_start;
1228                 range->last = v->domain->geometry.aperture_end;
1229         } else {
1230                 range->first = 0;
1231                 range->last = ULLONG_MAX;
1232         }
1233 }
1234
1235 static void vhost_vdpa_cleanup(struct vhost_vdpa *v)
1236 {
1237         struct vhost_vdpa_as *as;
1238         u32 asid;
1239
1240         for (asid = 0; asid < v->vdpa->nas; asid++) {
1241                 as = asid_to_as(v, asid);
1242                 if (as)
1243                         vhost_vdpa_remove_as(v, asid);
1244         }
1245
1246         vhost_vdpa_free_domain(v);
1247         vhost_dev_cleanup(&v->vdev);
1248         kfree(v->vdev.vqs);
1249 }
1250
1251 static int vhost_vdpa_open(struct inode *inode, struct file *filep)
1252 {
1253         struct vhost_vdpa *v;
1254         struct vhost_dev *dev;
1255         struct vhost_virtqueue **vqs;
1256         int r, opened;
1257         u32 i, nvqs;
1258
1259         v = container_of(inode->i_cdev, struct vhost_vdpa, cdev);
1260
1261         opened = atomic_cmpxchg(&v->opened, 0, 1);
1262         if (opened)
1263                 return -EBUSY;
1264
1265         nvqs = v->nvqs;
1266         r = vhost_vdpa_reset(v);
1267         if (r)
1268                 goto err;
1269
1270         vqs = kmalloc_array(nvqs, sizeof(*vqs), GFP_KERNEL);
1271         if (!vqs) {
1272                 r = -ENOMEM;
1273                 goto err;
1274         }
1275
1276         dev = &v->vdev;
1277         for (i = 0; i < nvqs; i++) {
1278                 vqs[i] = &v->vqs[i];
1279                 vqs[i]->handle_kick = handle_vq_kick;
1280         }
1281         vhost_dev_init(dev, vqs, nvqs, 0, 0, 0, false,
1282                        vhost_vdpa_process_iotlb_msg);
1283
1284         r = vhost_vdpa_alloc_domain(v);
1285         if (r)
1286                 goto err_alloc_domain;
1287
1288         vhost_vdpa_set_iova_range(v);
1289
1290         filep->private_data = v;
1291
1292         return 0;
1293
1294 err_alloc_domain:
1295         vhost_vdpa_cleanup(v);
1296 err:
1297         atomic_dec(&v->opened);
1298         return r;
1299 }
1300
1301 static void vhost_vdpa_clean_irq(struct vhost_vdpa *v)
1302 {
1303         u32 i;
1304
1305         for (i = 0; i < v->nvqs; i++)
1306                 vhost_vdpa_unsetup_vq_irq(v, i);
1307 }
1308
1309 static int vhost_vdpa_release(struct inode *inode, struct file *filep)
1310 {
1311         struct vhost_vdpa *v = filep->private_data;
1312         struct vhost_dev *d = &v->vdev;
1313
1314         mutex_lock(&d->mutex);
1315         filep->private_data = NULL;
1316         vhost_vdpa_clean_irq(v);
1317         vhost_vdpa_reset(v);
1318         vhost_dev_stop(&v->vdev);
1319         vhost_vdpa_unbind_mm(v);
1320         vhost_vdpa_config_put(v);
1321         vhost_vdpa_cleanup(v);
1322         mutex_unlock(&d->mutex);
1323
1324         atomic_dec(&v->opened);
1325         complete(&v->completion);
1326
1327         return 0;
1328 }
1329
1330 #ifdef CONFIG_MMU
1331 static vm_fault_t vhost_vdpa_fault(struct vm_fault *vmf)
1332 {
1333         struct vhost_vdpa *v = vmf->vma->vm_file->private_data;
1334         struct vdpa_device *vdpa = v->vdpa;
1335         const struct vdpa_config_ops *ops = vdpa->config;
1336         struct vdpa_notification_area notify;
1337         struct vm_area_struct *vma = vmf->vma;
1338         u16 index = vma->vm_pgoff;
1339
1340         notify = ops->get_vq_notification(vdpa, index);
1341
1342         vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1343         if (remap_pfn_range(vma, vmf->address & PAGE_MASK,
1344                             PFN_DOWN(notify.addr), PAGE_SIZE,
1345                             vma->vm_page_prot))
1346                 return VM_FAULT_SIGBUS;
1347
1348         return VM_FAULT_NOPAGE;
1349 }
1350
1351 static const struct vm_operations_struct vhost_vdpa_vm_ops = {
1352         .fault = vhost_vdpa_fault,
1353 };
1354
1355 static int vhost_vdpa_mmap(struct file *file, struct vm_area_struct *vma)
1356 {
1357         struct vhost_vdpa *v = vma->vm_file->private_data;
1358         struct vdpa_device *vdpa = v->vdpa;
1359         const struct vdpa_config_ops *ops = vdpa->config;
1360         struct vdpa_notification_area notify;
1361         unsigned long index = vma->vm_pgoff;
1362
1363         if (vma->vm_end - vma->vm_start != PAGE_SIZE)
1364                 return -EINVAL;
1365         if ((vma->vm_flags & VM_SHARED) == 0)
1366                 return -EINVAL;
1367         if (vma->vm_flags & VM_READ)
1368                 return -EINVAL;
1369         if (index > 65535)
1370                 return -EINVAL;
1371         if (!ops->get_vq_notification)
1372                 return -ENOTSUPP;
1373
1374         /* To be safe and easily modelled by userspace, We only
1375          * support the doorbell which sits on the page boundary and
1376          * does not share the page with other registers.
1377          */
1378         notify = ops->get_vq_notification(vdpa, index);
1379         if (notify.addr & (PAGE_SIZE - 1))
1380                 return -EINVAL;
1381         if (vma->vm_end - vma->vm_start != notify.size)
1382                 return -ENOTSUPP;
1383
1384         vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
1385         vma->vm_ops = &vhost_vdpa_vm_ops;
1386         return 0;
1387 }
1388 #endif /* CONFIG_MMU */
1389
1390 static const struct file_operations vhost_vdpa_fops = {
1391         .owner          = THIS_MODULE,
1392         .open           = vhost_vdpa_open,
1393         .release        = vhost_vdpa_release,
1394         .write_iter     = vhost_vdpa_chr_write_iter,
1395         .unlocked_ioctl = vhost_vdpa_unlocked_ioctl,
1396 #ifdef CONFIG_MMU
1397         .mmap           = vhost_vdpa_mmap,
1398 #endif /* CONFIG_MMU */
1399         .compat_ioctl   = compat_ptr_ioctl,
1400 };
1401
1402 static void vhost_vdpa_release_dev(struct device *device)
1403 {
1404         struct vhost_vdpa *v =
1405                container_of(device, struct vhost_vdpa, dev);
1406
1407         ida_simple_remove(&vhost_vdpa_ida, v->minor);
1408         kfree(v->vqs);
1409         kfree(v);
1410 }
1411
1412 static int vhost_vdpa_probe(struct vdpa_device *vdpa)
1413 {
1414         const struct vdpa_config_ops *ops = vdpa->config;
1415         struct vhost_vdpa *v;
1416         int minor;
1417         int i, r;
1418
1419         /* We can't support platform IOMMU device with more than 1
1420          * group or as
1421          */
1422         if (!ops->set_map && !ops->dma_map &&
1423             (vdpa->ngroups > 1 || vdpa->nas > 1))
1424                 return -EOPNOTSUPP;
1425
1426         v = kzalloc(sizeof(*v), GFP_KERNEL | __GFP_RETRY_MAYFAIL);
1427         if (!v)
1428                 return -ENOMEM;
1429
1430         minor = ida_simple_get(&vhost_vdpa_ida, 0,
1431                                VHOST_VDPA_DEV_MAX, GFP_KERNEL);
1432         if (minor < 0) {
1433                 kfree(v);
1434                 return minor;
1435         }
1436
1437         atomic_set(&v->opened, 0);
1438         v->minor = minor;
1439         v->vdpa = vdpa;
1440         v->nvqs = vdpa->nvqs;
1441         v->virtio_id = ops->get_device_id(vdpa);
1442
1443         device_initialize(&v->dev);
1444         v->dev.release = vhost_vdpa_release_dev;
1445         v->dev.parent = &vdpa->dev;
1446         v->dev.devt = MKDEV(MAJOR(vhost_vdpa_major), minor);
1447         v->vqs = kmalloc_array(v->nvqs, sizeof(struct vhost_virtqueue),
1448                                GFP_KERNEL);
1449         if (!v->vqs) {
1450                 r = -ENOMEM;
1451                 goto err;
1452         }
1453
1454         r = dev_set_name(&v->dev, "vhost-vdpa-%u", minor);
1455         if (r)
1456                 goto err;
1457
1458         cdev_init(&v->cdev, &vhost_vdpa_fops);
1459         v->cdev.owner = THIS_MODULE;
1460
1461         r = cdev_device_add(&v->cdev, &v->dev);
1462         if (r)
1463                 goto err;
1464
1465         init_completion(&v->completion);
1466         vdpa_set_drvdata(vdpa, v);
1467
1468         for (i = 0; i < VHOST_VDPA_IOTLB_BUCKETS; i++)
1469                 INIT_HLIST_HEAD(&v->as[i]);
1470
1471         return 0;
1472
1473 err:
1474         put_device(&v->dev);
1475         ida_simple_remove(&vhost_vdpa_ida, v->minor);
1476         return r;
1477 }
1478
1479 static void vhost_vdpa_remove(struct vdpa_device *vdpa)
1480 {
1481         struct vhost_vdpa *v = vdpa_get_drvdata(vdpa);
1482         int opened;
1483
1484         cdev_device_del(&v->cdev, &v->dev);
1485
1486         do {
1487                 opened = atomic_cmpxchg(&v->opened, 0, 1);
1488                 if (!opened)
1489                         break;
1490                 wait_for_completion(&v->completion);
1491         } while (1);
1492
1493         put_device(&v->dev);
1494 }
1495
1496 static struct vdpa_driver vhost_vdpa_driver = {
1497         .driver = {
1498                 .name   = "vhost_vdpa",
1499         },
1500         .probe  = vhost_vdpa_probe,
1501         .remove = vhost_vdpa_remove,
1502 };
1503
1504 static int __init vhost_vdpa_init(void)
1505 {
1506         int r;
1507
1508         r = alloc_chrdev_region(&vhost_vdpa_major, 0, VHOST_VDPA_DEV_MAX,
1509                                 "vhost-vdpa");
1510         if (r)
1511                 goto err_alloc_chrdev;
1512
1513         r = vdpa_register_driver(&vhost_vdpa_driver);
1514         if (r)
1515                 goto err_vdpa_register_driver;
1516
1517         return 0;
1518
1519 err_vdpa_register_driver:
1520         unregister_chrdev_region(vhost_vdpa_major, VHOST_VDPA_DEV_MAX);
1521 err_alloc_chrdev:
1522         return r;
1523 }
1524 module_init(vhost_vdpa_init);
1525
1526 static void __exit vhost_vdpa_exit(void)
1527 {
1528         vdpa_unregister_driver(&vhost_vdpa_driver);
1529         unregister_chrdev_region(vhost_vdpa_major, VHOST_VDPA_DEV_MAX);
1530 }
1531 module_exit(vhost_vdpa_exit);
1532
1533 MODULE_VERSION("0.0.1");
1534 MODULE_LICENSE("GPL v2");
1535 MODULE_AUTHOR("Intel Corporation");
1536 MODULE_DESCRIPTION("vDPA-based vhost backend for virtio");