drivers/gpu/drm/i915/gvt/kvmgt.c

   1 /*
   2  * KVMGT - the implementation of Intel mediated pass-through framework for KVM
   3  *
   4  * Copyright(c) 2014-2016 Intel Corporation. All rights reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  22  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  23  * SOFTWARE.
  24  *
  25  * Authors:
  26  *    Kevin Tian <kevin.tian@intel.com>
  27  *    Jike Song <jike.song@intel.com>
  28  *    Xiaoguang Chen <xiaoguang.chen@intel.com>
  29  */
  30
  31 #include <linux/init.h>
  32 #include <linux/device.h>
  33 #include <linux/mm.h>
  34 #include <linux/mmu_context.h>
  35 #include <linux/types.h>
  36 #include <linux/list.h>
  37 #include <linux/rbtree.h>
  38 #include <linux/spinlock.h>
  39 #include <linux/eventfd.h>
  40 #include <linux/uuid.h>
  41 #include <linux/kvm_host.h>
  42 #include <linux/vfio.h>
  43 #include <linux/mdev.h>
  44 #include <linux/debugfs.h>
  45
  46 #include "i915_drv.h"
  47 #include "gvt.h"
  48
  49 static const struct intel_gvt_ops *intel_gvt_ops;
  50
  51 /* helper macros copied from vfio-pci */
  52 #define VFIO_PCI_OFFSET_SHIFT   40
  53 #define VFIO_PCI_OFFSET_TO_INDEX(off)   (off >> VFIO_PCI_OFFSET_SHIFT)
  54 #define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT)
  55 #define VFIO_PCI_OFFSET_MASK    (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1)
  56
  57 #define OPREGION_SIGNATURE "IntelGraphicsMem"
  58
  59 struct vfio_region;
  60 struct intel_vgpu_regops {
  61         size_t (*rw)(struct intel_vgpu *vgpu, char *buf,
  62                         size_t count, loff_t *ppos, bool iswrite);
  63         void (*release)(struct intel_vgpu *vgpu,
  64                         struct vfio_region *region);
  65 };
  66
  67 struct vfio_region {
  68         u32                             type;
  69         u32                             subtype;
  70         size_t                          size;
  71         u32                             flags;
  72         const struct intel_vgpu_regops  *ops;
  73         void                            *data;
  74 };
  75
  76 struct kvmgt_pgfn {
  77         gfn_t gfn;
  78         struct hlist_node hnode;
  79 };
  80
  81 struct kvmgt_guest_info {
  82         struct kvm *kvm;
  83         struct intel_vgpu *vgpu;
  84         struct kvm_page_track_notifier_node track_node;
  85 #define NR_BKT (1 << 18)
  86         struct hlist_head ptable[NR_BKT];
  87 #undef NR_BKT
  88         struct dentry *debugfs_cache_entries;
  89 };
  90
  91 struct gvt_dma {
  92         struct intel_vgpu *vgpu;
  93         struct rb_node gfn_node;
  94         struct rb_node dma_addr_node;
  95         gfn_t gfn;
  96         dma_addr_t dma_addr;
  97         unsigned long size;
  98         struct kref ref;
  99 };
 100
 101 static inline bool handle_valid(unsigned long handle)
 102 {
 103         return !!(handle & ~0xff);
 104 }
 105
 106 static int kvmgt_guest_init(struct mdev_device *mdev);
 107 static void intel_vgpu_release_work(struct work_struct *work);
 108 static bool kvmgt_guest_exit(struct kvmgt_guest_info *info);
 109
 110 static void gvt_unpin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn,
 111                 unsigned long size)
 112 {
 113         int total_pages;
 114         int npage;
 115         int ret;
 116
 117         total_pages = roundup(size, PAGE_SIZE) / PAGE_SIZE;
 118
 119         for (npage = 0; npage < total_pages; npage++) {
 120                 unsigned long cur_gfn = gfn + npage;
 121
 122                 ret = vfio_unpin_pages(mdev_dev(vgpu->vdev.mdev), &cur_gfn, 1);
 123                 WARN_ON(ret != 1);
 124         }
 125 }
 126
 127 /* Pin a normal or compound guest page for dma. */
 128 static int gvt_pin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn,
 129                 unsigned long size, struct page **page)
 130 {
 131         unsigned long base_pfn = 0;
 132         int total_pages;
 133         int npage;
 134         int ret;
 135
 136         total_pages = roundup(size, PAGE_SIZE) / PAGE_SIZE;
 137         /*
 138          * We pin the pages one-by-one to avoid allocating a big arrary
 139          * on stack to hold pfns.
 140          */
 141         for (npage = 0; npage < total_pages; npage++) {
 142                 unsigned long cur_gfn = gfn + npage;
 143                 unsigned long pfn;
 144
 145                 ret = vfio_pin_pages(mdev_dev(vgpu->vdev.mdev), &cur_gfn, 1,
 146                                      IOMMU_READ | IOMMU_WRITE, &pfn);
 147                 if (ret != 1) {
 148                         gvt_vgpu_err("vfio_pin_pages failed for gfn 0x%lx, ret %d\n",
 149                                      cur_gfn, ret);
 150                         goto err;
 151                 }
 152
 153                 if (!pfn_valid(pfn)) {
 154                         gvt_vgpu_err("pfn 0x%lx is not mem backed\n", pfn);
 155                         npage++;
 156                         ret = -EFAULT;
 157                         goto err;
 158                 }
 159
 160                 if (npage == 0)
 161                         base_pfn = pfn;
 162                 else if (base_pfn + npage != pfn) {
 163                         gvt_vgpu_err("The pages are not continuous\n");
 164                         ret = -EINVAL;
 165                         npage++;
 166                         goto err;
 167                 }
 168         }
 169
 170         *page = pfn_to_page(base_pfn);
 171         return 0;
 172 err:
 173         gvt_unpin_guest_page(vgpu, gfn, npage * PAGE_SIZE);
 174         return ret;
 175 }
 176
 177 static int gvt_dma_map_page(struct intel_vgpu *vgpu, unsigned long gfn,
 178                 dma_addr_t *dma_addr, unsigned long size)
 179 {
 180         struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev;
 181         struct page *page = NULL;
 182         int ret;
 183
 184         ret = gvt_pin_guest_page(vgpu, gfn, size, &page);
 185         if (ret)
 186                 return ret;
 187
 188         /* Setup DMA mapping. */
 189         *dma_addr = dma_map_page(dev, page, 0, size, PCI_DMA_BIDIRECTIONAL);
 190         ret = dma_mapping_error(dev, *dma_addr);
 191         if (ret) {
 192                 gvt_vgpu_err("DMA mapping failed for pfn 0x%lx, ret %d\n",
 193                              page_to_pfn(page), ret);
 194                 gvt_unpin_guest_page(vgpu, gfn, size);
 195         }
 196
 197         return ret;
 198 }
 199
 200 static void gvt_dma_unmap_page(struct intel_vgpu *vgpu, unsigned long gfn,
 201                 dma_addr_t dma_addr, unsigned long size)
 202 {
 203         struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev;
 204
 205         dma_unmap_page(dev, dma_addr, size, PCI_DMA_BIDIRECTIONAL);
 206         gvt_unpin_guest_page(vgpu, gfn, size);
 207 }
 208
 209 static struct gvt_dma *__gvt_cache_find_dma_addr(struct intel_vgpu *vgpu,
 210                 dma_addr_t dma_addr)
 211 {
 212         struct rb_node *node = vgpu->vdev.dma_addr_cache.rb_node;
 213         struct gvt_dma *itr;
 214
 215         while (node) {
 216                 itr = rb_entry(node, struct gvt_dma, dma_addr_node);
 217
 218                 if (dma_addr < itr->dma_addr)
 219                         node = node->rb_left;
 220                 else if (dma_addr > itr->dma_addr)
 221                         node = node->rb_right;
 222                 else
 223                         return itr;
 224         }
 225         return NULL;
 226 }
 227
 228 static struct gvt_dma *__gvt_cache_find_gfn(struct intel_vgpu *vgpu, gfn_t gfn)
 229 {
 230         struct rb_node *node = vgpu->vdev.gfn_cache.rb_node;
 231         struct gvt_dma *itr;
 232
 233         while (node) {
 234                 itr = rb_entry(node, struct gvt_dma, gfn_node);
 235
 236                 if (gfn < itr->gfn)
 237                         node = node->rb_left;
 238                 else if (gfn > itr->gfn)
 239                         node = node->rb_right;
 240                 else
 241                         return itr;
 242         }
 243         return NULL;
 244 }
 245
 246 static int __gvt_cache_add(struct intel_vgpu *vgpu, gfn_t gfn,
 247                 dma_addr_t dma_addr, unsigned long size)
 248 {
 249         struct gvt_dma *new, *itr;
 250         struct rb_node **link, *parent = NULL;
 251
 252         new = kzalloc(sizeof(struct gvt_dma), GFP_KERNEL);
 253         if (!new)
 254                 return -ENOMEM;
 255
 256         new->vgpu = vgpu;
 257         new->gfn = gfn;
 258         new->dma_addr = dma_addr;
 259         new->size = size;
 260         kref_init(&new->ref);
 261
 262         /* gfn_cache maps gfn to struct gvt_dma. */
 263         link = &vgpu->vdev.gfn_cache.rb_node;
 264         while (*link) {
 265                 parent = *link;
 266                 itr = rb_entry(parent, struct gvt_dma, gfn_node);
 267
 268                 if (gfn < itr->gfn)
 269                         link = &parent->rb_left;
 270                 else
 271                         link = &parent->rb_right;
 272         }
 273         rb_link_node(&new->gfn_node, parent, link);
 274         rb_insert_color(&new->gfn_node, &vgpu->vdev.gfn_cache);
 275
 276         /* dma_addr_cache maps dma addr to struct gvt_dma. */
 277         parent = NULL;
 278         link = &vgpu->vdev.dma_addr_cache.rb_node;
 279         while (*link) {
 280                 parent = *link;
 281                 itr = rb_entry(parent, struct gvt_dma, dma_addr_node);
 282
 283                 if (dma_addr < itr->dma_addr)
 284                         link = &parent->rb_left;
 285                 else
 286                         link = &parent->rb_right;
 287         }
 288         rb_link_node(&new->dma_addr_node, parent, link);
 289         rb_insert_color(&new->dma_addr_node, &vgpu->vdev.dma_addr_cache);
 290
 291         vgpu->vdev.nr_cache_entries++;
 292         return 0;
 293 }
 294
 295 static void __gvt_cache_remove_entry(struct intel_vgpu *vgpu,
 296                                 struct gvt_dma *entry)
 297 {
 298         rb_erase(&entry->gfn_node, &vgpu->vdev.gfn_cache);
 299         rb_erase(&entry->dma_addr_node, &vgpu->vdev.dma_addr_cache);
 300         kfree(entry);
 301         vgpu->vdev.nr_cache_entries--;
 302 }
 303
 304 static void gvt_cache_destroy(struct intel_vgpu *vgpu)
 305 {
 306         struct gvt_dma *dma;
 307         struct rb_node *node = NULL;
 308
 309         for (;;) {
 310                 mutex_lock(&vgpu->vdev.cache_lock);
 311                 node = rb_first(&vgpu->vdev.gfn_cache);
 312                 if (!node) {
 313                         mutex_unlock(&vgpu->vdev.cache_lock);
 314                         break;
 315                 }
 316                 dma = rb_entry(node, struct gvt_dma, gfn_node);
 317                 gvt_dma_unmap_page(vgpu, dma->gfn, dma->dma_addr, dma->size);
 318                 __gvt_cache_remove_entry(vgpu, dma);
 319                 mutex_unlock(&vgpu->vdev.cache_lock);
 320         }
 321 }
 322
 323 static void gvt_cache_init(struct intel_vgpu *vgpu)
 324 {
 325         vgpu->vdev.gfn_cache = RB_ROOT;
 326         vgpu->vdev.dma_addr_cache = RB_ROOT;
 327         vgpu->vdev.nr_cache_entries = 0;
 328         mutex_init(&vgpu->vdev.cache_lock);
 329 }
 330
 331 static void kvmgt_protect_table_init(struct kvmgt_guest_info *info)
 332 {
 333         hash_init(info->ptable);
 334 }
 335
 336 static void kvmgt_protect_table_destroy(struct kvmgt_guest_info *info)
 337 {
 338         struct kvmgt_pgfn *p;
 339         struct hlist_node *tmp;
 340         int i;
 341
 342         hash_for_each_safe(info->ptable, i, tmp, p, hnode) {
 343                 hash_del(&p->hnode);
 344                 kfree(p);
 345         }
 346 }
 347
 348 static struct kvmgt_pgfn *
 349 __kvmgt_protect_table_find(struct kvmgt_guest_info *info, gfn_t gfn)
 350 {
 351         struct kvmgt_pgfn *p, *res = NULL;
 352
 353         hash_for_each_possible(info->ptable, p, hnode, gfn) {
 354                 if (gfn == p->gfn) {
 355                         res = p;
 356                         break;
 357                 }
 358         }
 359
 360         return res;
 361 }
 362
 363 static bool kvmgt_gfn_is_write_protected(struct kvmgt_guest_info *info,
 364                                 gfn_t gfn)
 365 {
 366         struct kvmgt_pgfn *p;
 367
 368         p = __kvmgt_protect_table_find(info, gfn);
 369         return !!p;
 370 }
 371
 372 static void kvmgt_protect_table_add(struct kvmgt_guest_info *info, gfn_t gfn)
 373 {
 374         struct kvmgt_pgfn *p;
 375
 376         if (kvmgt_gfn_is_write_protected(info, gfn))
 377                 return;
 378
 379         p = kzalloc(sizeof(struct kvmgt_pgfn), GFP_ATOMIC);
 380         if (WARN(!p, "gfn: 0x%llx\n", gfn))
 381                 return;
 382
 383         p->gfn = gfn;
 384         hash_add(info->ptable, &p->hnode, gfn);
 385 }
 386
 387 static void kvmgt_protect_table_del(struct kvmgt_guest_info *info,
 388                                 gfn_t gfn)
 389 {
 390         struct kvmgt_pgfn *p;
 391
 392         p = __kvmgt_protect_table_find(info, gfn);
 393         if (p) {
 394                 hash_del(&p->hnode);
 395                 kfree(p);
 396         }
 397 }
 398
 399 static size_t intel_vgpu_reg_rw_opregion(struct intel_vgpu *vgpu, char *buf,
 400                 size_t count, loff_t *ppos, bool iswrite)
 401 {
 402         unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) -
 403                         VFIO_PCI_NUM_REGIONS;
 404         void *base = vgpu->vdev.region[i].data;
 405         loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
 406
 407         if (pos >= vgpu->vdev.region[i].size || iswrite) {
 408                 gvt_vgpu_err("invalid op or offset for Intel vgpu OpRegion\n");
 409                 return -EINVAL;
 410         }
 411         count = min(count, (size_t)(vgpu->vdev.region[i].size - pos));
 412         memcpy(buf, base + pos, count);
 413
 414         return count;
 415 }
 416
 417 static void intel_vgpu_reg_release_opregion(struct intel_vgpu *vgpu,
 418                 struct vfio_region *region)
 419 {
 420 }
 421
 422 static const struct intel_vgpu_regops intel_vgpu_regops_opregion = {
 423         .rw = intel_vgpu_reg_rw_opregion,
 424         .release = intel_vgpu_reg_release_opregion,
 425 };
 426
 427 static int intel_vgpu_register_reg(struct intel_vgpu *vgpu,
 428                 unsigned int type, unsigned int subtype,
 429                 const struct intel_vgpu_regops *ops,
 430                 size_t size, u32 flags, void *data)
 431 {
 432         struct vfio_region *region;
 433
 434         region = krealloc(vgpu->vdev.region,
 435                         (vgpu->vdev.num_regions + 1) * sizeof(*region),
 436                         GFP_KERNEL);
 437         if (!region)
 438                 return -ENOMEM;
 439
 440         vgpu->vdev.region = region;
 441         vgpu->vdev.region[vgpu->vdev.num_regions].type = type;
 442         vgpu->vdev.region[vgpu->vdev.num_regions].subtype = subtype;
 443         vgpu->vdev.region[vgpu->vdev.num_regions].ops = ops;
 444         vgpu->vdev.region[vgpu->vdev.num_regions].size = size;
 445         vgpu->vdev.region[vgpu->vdev.num_regions].flags = flags;
 446         vgpu->vdev.region[vgpu->vdev.num_regions].data = data;
 447         vgpu->vdev.num_regions++;
 448         return 0;
 449 }
 450
 451 static int kvmgt_get_vfio_device(void *p_vgpu)
 452 {
 453         struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu;
 454
 455         vgpu->vdev.vfio_device = vfio_device_get_from_dev(
 456                 mdev_dev(vgpu->vdev.mdev));
 457         if (!vgpu->vdev.vfio_device) {
 458                 gvt_vgpu_err("failed to get vfio device\n");
 459                 return -ENODEV;
 460         }
 461         return 0;
 462 }
 463
 464
 465 static int kvmgt_set_opregion(void *p_vgpu)
 466 {
 467         struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu;
 468         void *base;
 469         int ret;
 470
 471         /* Each vgpu has its own opregion, although VFIO would create another
 472          * one later. This one is used to expose opregion to VFIO. And the
 473          * other one created by VFIO later, is used by guest actually.
 474          */
 475         base = vgpu_opregion(vgpu)->va;
 476         if (!base)
 477                 return -ENOMEM;
 478
 479         if (memcmp(base, OPREGION_SIGNATURE, 16)) {
 480                 memunmap(base);
 481                 return -EINVAL;
 482         }
 483
 484         ret = intel_vgpu_register_reg(vgpu,
 485                         PCI_VENDOR_ID_INTEL | VFIO_REGION_TYPE_PCI_VENDOR_TYPE,
 486                         VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION,
 487                         &intel_vgpu_regops_opregion, OPREGION_SIZE,
 488                         VFIO_REGION_INFO_FLAG_READ, base);
 489
 490         return ret;
 491 }
 492
 493 static void kvmgt_put_vfio_device(void *vgpu)
 494 {
 495         if (WARN_ON(!((struct intel_vgpu *)vgpu)->vdev.vfio_device))
 496                 return;
 497
 498         vfio_device_put(((struct intel_vgpu *)vgpu)->vdev.vfio_device);
 499 }
 500
 501 static int intel_vgpu_create(struct kobject *kobj, struct mdev_device *mdev)
 502 {
 503         struct intel_vgpu *vgpu = NULL;
 504         struct intel_vgpu_type *type;
 505         struct device *pdev;
 506         void *gvt;
 507         int ret;
 508
 509         pdev = mdev_parent_dev(mdev);
 510         gvt = kdev_to_i915(pdev)->gvt;
 511
 512         type = intel_gvt_ops->gvt_find_vgpu_type(gvt, kobject_name(kobj));
 513         if (!type) {
 514                 gvt_vgpu_err("failed to find type %s to create\n",
 515                                                 kobject_name(kobj));
 516                 ret = -EINVAL;
 517                 goto out;
 518         }
 519
 520         vgpu = intel_gvt_ops->vgpu_create(gvt, type);
 521         if (IS_ERR_OR_NULL(vgpu)) {
 522                 ret = vgpu == NULL ? -EFAULT : PTR_ERR(vgpu);
 523                 gvt_err("failed to create intel vgpu: %d\n", ret);
 524                 goto out;
 525         }
 526
 527         INIT_WORK(&vgpu->vdev.release_work, intel_vgpu_release_work);
 528
 529         vgpu->vdev.mdev = mdev;
 530         mdev_set_drvdata(mdev, vgpu);
 531
 532         gvt_dbg_core("intel_vgpu_create succeeded for mdev: %s\n",
 533                      dev_name(mdev_dev(mdev)));
 534         ret = 0;
 535
 536 out:
 537         return ret;
 538 }
 539
 540 static int intel_vgpu_remove(struct mdev_device *mdev)
 541 {
 542         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
 543
 544         if (handle_valid(vgpu->handle))
 545                 return -EBUSY;
 546
 547         intel_gvt_ops->vgpu_destroy(vgpu);
 548         return 0;
 549 }
 550
 551 static int intel_vgpu_iommu_notifier(struct notifier_block *nb,
 552                                      unsigned long action, void *data)
 553 {
 554         struct intel_vgpu *vgpu = container_of(nb,
 555                                         struct intel_vgpu,
 556                                         vdev.iommu_notifier);
 557
 558         if (action == VFIO_IOMMU_NOTIFY_DMA_UNMAP) {
 559                 struct vfio_iommu_type1_dma_unmap *unmap = data;
 560                 struct gvt_dma *entry;
 561                 unsigned long iov_pfn, end_iov_pfn;
 562
 563                 iov_pfn = unmap->iova >> PAGE_SHIFT;
 564                 end_iov_pfn = iov_pfn + unmap->size / PAGE_SIZE;
 565
 566                 mutex_lock(&vgpu->vdev.cache_lock);
 567                 for (; iov_pfn < end_iov_pfn; iov_pfn++) {
 568                         entry = __gvt_cache_find_gfn(vgpu, iov_pfn);
 569                         if (!entry)
 570                                 continue;
 571
 572                         gvt_dma_unmap_page(vgpu, entry->gfn, entry->dma_addr,
 573                                            entry->size);
 574                         __gvt_cache_remove_entry(vgpu, entry);
 575                 }
 576                 mutex_unlock(&vgpu->vdev.cache_lock);
 577         }
 578
 579         return NOTIFY_OK;
 580 }
 581
 582 static int intel_vgpu_group_notifier(struct notifier_block *nb,
 583                                      unsigned long action, void *data)
 584 {
 585         struct intel_vgpu *vgpu = container_of(nb,
 586                                         struct intel_vgpu,
 587                                         vdev.group_notifier);
 588
 589         /* the only action we care about */
 590         if (action == VFIO_GROUP_NOTIFY_SET_KVM) {
 591                 vgpu->vdev.kvm = data;
 592
 593                 if (!data)
 594                         schedule_work(&vgpu->vdev.release_work);
 595         }
 596
 597         return NOTIFY_OK;
 598 }
 599
 600 static int intel_vgpu_open(struct mdev_device *mdev)
 601 {
 602         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
 603         unsigned long events;
 604         int ret;
 605
 606         vgpu->vdev.iommu_notifier.notifier_call = intel_vgpu_iommu_notifier;
 607         vgpu->vdev.group_notifier.notifier_call = intel_vgpu_group_notifier;
 608
 609         events = VFIO_IOMMU_NOTIFY_DMA_UNMAP;
 610         ret = vfio_register_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY, &events,
 611                                 &vgpu->vdev.iommu_notifier);
 612         if (ret != 0) {
 613                 gvt_vgpu_err("vfio_register_notifier for iommu failed: %d\n",
 614                         ret);
 615                 goto out;
 616         }
 617
 618         events = VFIO_GROUP_NOTIFY_SET_KVM;
 619         ret = vfio_register_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY, &events,
 620                                 &vgpu->vdev.group_notifier);
 621         if (ret != 0) {
 622                 gvt_vgpu_err("vfio_register_notifier for group failed: %d\n",
 623                         ret);
 624                 goto undo_iommu;
 625         }
 626
 627         ret = kvmgt_guest_init(mdev);
 628         if (ret)
 629                 goto undo_group;
 630
 631         intel_gvt_ops->vgpu_activate(vgpu);
 632
 633         atomic_set(&vgpu->vdev.released, 0);
 634         return ret;
 635
 636 undo_group:
 637         vfio_unregister_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY,
 638                                         &vgpu->vdev.group_notifier);
 639
 640 undo_iommu:
 641         vfio_unregister_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY,
 642                                         &vgpu->vdev.iommu_notifier);
 643 out:
 644         return ret;
 645 }
 646
 647 static void intel_vgpu_release_msi_eventfd_ctx(struct intel_vgpu *vgpu)
 648 {
 649         struct eventfd_ctx *trigger;
 650
 651         trigger = vgpu->vdev.msi_trigger;
 652         if (trigger) {
 653                 eventfd_ctx_put(trigger);
 654                 vgpu->vdev.msi_trigger = NULL;
 655         }
 656 }
 657
 658 static void __intel_vgpu_release(struct intel_vgpu *vgpu)
 659 {
 660         struct kvmgt_guest_info *info;
 661         int ret;
 662
 663         if (!handle_valid(vgpu->handle))
 664                 return;
 665
 666         if (atomic_cmpxchg(&vgpu->vdev.released, 0, 1))
 667                 return;
 668
 669         intel_gvt_ops->vgpu_deactivate(vgpu);
 670
 671         ret = vfio_unregister_notifier(mdev_dev(vgpu->vdev.mdev), VFIO_IOMMU_NOTIFY,
 672                                         &vgpu->vdev.iommu_notifier);
 673         WARN(ret, "vfio_unregister_notifier for iommu failed: %d\n", ret);
 674
 675         ret = vfio_unregister_notifier(mdev_dev(vgpu->vdev.mdev), VFIO_GROUP_NOTIFY,
 676                                         &vgpu->vdev.group_notifier);
 677         WARN(ret, "vfio_unregister_notifier for group failed: %d\n", ret);
 678
 679         info = (struct kvmgt_guest_info *)vgpu->handle;
 680         kvmgt_guest_exit(info);
 681
 682         intel_vgpu_release_msi_eventfd_ctx(vgpu);
 683
 684         vgpu->vdev.kvm = NULL;
 685         vgpu->handle = 0;
 686 }
 687
 688 static void intel_vgpu_release(struct mdev_device *mdev)
 689 {
 690         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
 691
 692         __intel_vgpu_release(vgpu);
 693 }
 694
 695 static void intel_vgpu_release_work(struct work_struct *work)
 696 {
 697         struct intel_vgpu *vgpu = container_of(work, struct intel_vgpu,
 698                                         vdev.release_work);
 699
 700         __intel_vgpu_release(vgpu);
 701 }
 702
 703 static uint64_t intel_vgpu_get_bar_addr(struct intel_vgpu *vgpu, int bar)
 704 {
 705         u32 start_lo, start_hi;
 706         u32 mem_type;
 707
 708         start_lo = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &
 709                         PCI_BASE_ADDRESS_MEM_MASK;
 710         mem_type = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &
 711                         PCI_BASE_ADDRESS_MEM_TYPE_MASK;
 712
 713         switch (mem_type) {
 714         case PCI_BASE_ADDRESS_MEM_TYPE_64:
 715                 start_hi = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space
 716                                                 + bar + 4));
 717                 break;
 718         case PCI_BASE_ADDRESS_MEM_TYPE_32:
 719         case PCI_BASE_ADDRESS_MEM_TYPE_1M:
 720                 /* 1M mem BAR treated as 32-bit BAR */
 721         default:
 722                 /* mem unknown type treated as 32-bit BAR */
 723                 start_hi = 0;
 724                 break;
 725         }
 726
 727         return ((u64)start_hi << 32) | start_lo;
 728 }
 729
 730 static int intel_vgpu_bar_rw(struct intel_vgpu *vgpu, int bar, uint64_t off,
 731                              void *buf, unsigned int count, bool is_write)
 732 {
 733         uint64_t bar_start = intel_vgpu_get_bar_addr(vgpu, bar);
 734         int ret;
 735
 736         if (is_write)
 737                 ret = intel_gvt_ops->emulate_mmio_write(vgpu,
 738                                         bar_start + off, buf, count);
 739         else
 740                 ret = intel_gvt_ops->emulate_mmio_read(vgpu,
 741                                         bar_start + off, buf, count);
 742         return ret;
 743 }
 744
 745 static inline bool intel_vgpu_in_aperture(struct intel_vgpu *vgpu, uint64_t off)
 746 {
 747         return off >= vgpu_aperture_offset(vgpu) &&
 748                off < vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu);
 749 }
 750
 751 static int intel_vgpu_aperture_rw(struct intel_vgpu *vgpu, uint64_t off,
 752                 void *buf, unsigned long count, bool is_write)
 753 {
 754         void *aperture_va;
 755
 756         if (!intel_vgpu_in_aperture(vgpu, off) ||
 757             !intel_vgpu_in_aperture(vgpu, off + count)) {
 758                 gvt_vgpu_err("Invalid aperture offset %llu\n", off);
 759                 return -EINVAL;
 760         }
 761
 762         aperture_va = io_mapping_map_wc(&vgpu->gvt->dev_priv->ggtt.iomap,
 763                                         ALIGN_DOWN(off, PAGE_SIZE),
 764                                         count + offset_in_page(off));
 765         if (!aperture_va)
 766                 return -EIO;
 767
 768         if (is_write)
 769                 memcpy(aperture_va + offset_in_page(off), buf, count);
 770         else
 771                 memcpy(buf, aperture_va + offset_in_page(off), count);
 772
 773         io_mapping_unmap(aperture_va);
 774
 775         return 0;
 776 }
 777
 778 static ssize_t intel_vgpu_rw(struct mdev_device *mdev, char *buf,
 779                         size_t count, loff_t *ppos, bool is_write)
 780 {
 781         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
 782         unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
 783         uint64_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
 784         int ret = -EINVAL;
 785
 786
 787         if (index >= VFIO_PCI_NUM_REGIONS + vgpu->vdev.num_regions) {
 788                 gvt_vgpu_err("invalid index: %u\n", index);
 789                 return -EINVAL;
 790         }
 791
 792         switch (index) {
 793         case VFIO_PCI_CONFIG_REGION_INDEX:
 794                 if (is_write)
 795                         ret = intel_gvt_ops->emulate_cfg_write(vgpu, pos,
 796                                                 buf, count);
 797                 else
 798                         ret = intel_gvt_ops->emulate_cfg_read(vgpu, pos,
 799                                                 buf, count);
 800                 break;
 801         case VFIO_PCI_BAR0_REGION_INDEX:
 802                 ret = intel_vgpu_bar_rw(vgpu, PCI_BASE_ADDRESS_0, pos,
 803                                         buf, count, is_write);
 804                 break;
 805         case VFIO_PCI_BAR2_REGION_INDEX:
 806                 ret = intel_vgpu_aperture_rw(vgpu, pos, buf, count, is_write);
 807                 break;
 808         case VFIO_PCI_BAR1_REGION_INDEX:
 809         case VFIO_PCI_BAR3_REGION_INDEX:
 810         case VFIO_PCI_BAR4_REGION_INDEX:
 811         case VFIO_PCI_BAR5_REGION_INDEX:
 812         case VFIO_PCI_VGA_REGION_INDEX:
 813         case VFIO_PCI_ROM_REGION_INDEX:
 814                 break;
 815         default:
 816                 if (index >= VFIO_PCI_NUM_REGIONS + vgpu->vdev.num_regions)
 817                         return -EINVAL;
 818
 819                 index -= VFIO_PCI_NUM_REGIONS;
 820                 return vgpu->vdev.region[index].ops->rw(vgpu, buf, count,
 821                                 ppos, is_write);
 822         }
 823
 824         return ret == 0 ? count : ret;
 825 }
 826
 827 static bool gtt_entry(struct mdev_device *mdev, loff_t *ppos)
 828 {
 829         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
 830         unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
 831         struct intel_gvt *gvt = vgpu->gvt;
 832         int offset;
 833
 834         /* Only allow MMIO GGTT entry access */
 835         if (index != PCI_BASE_ADDRESS_0)
 836                 return false;
 837
 838         offset = (u64)(*ppos & VFIO_PCI_OFFSET_MASK) -
 839                 intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_0);
 840
 841         return (offset >= gvt->device_info.gtt_start_offset &&
 842                 offset < gvt->device_info.gtt_start_offset + gvt_ggtt_sz(gvt)) ?
 843                         true : false;
 844 }
 845
 846 static ssize_t intel_vgpu_read(struct mdev_device *mdev, char __user *buf,
 847                         size_t count, loff_t *ppos)
 848 {
 849         unsigned int done = 0;
 850         int ret;
 851
 852         while (count) {
 853                 size_t filled;
 854
 855                 /* Only support GGTT entry 8 bytes read */
 856                 if (count >= 8 && !(*ppos % 8) &&
 857                         gtt_entry(mdev, ppos)) {
 858                         u64 val;
 859
 860                         ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
 861                                         ppos, false);
 862                         if (ret <= 0)
 863                                 goto read_err;
 864
 865                         if (copy_to_user(buf, &val, sizeof(val)))
 866                                 goto read_err;
 867
 868                         filled = 8;
 869                 } else if (count >= 4 && !(*ppos % 4)) {
 870                         u32 val;
 871
 872                         ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
 873                                         ppos, false);
 874                         if (ret <= 0)
 875                                 goto read_err;
 876
 877                         if (copy_to_user(buf, &val, sizeof(val)))
 878                                 goto read_err;
 879
 880                         filled = 4;
 881                 } else if (count >= 2 && !(*ppos % 2)) {
 882                         u16 val;
 883
 884                         ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
 885                                         ppos, false);
 886                         if (ret <= 0)
 887                                 goto read_err;
 888
 889                         if (copy_to_user(buf, &val, sizeof(val)))
 890                                 goto read_err;
 891
 892                         filled = 2;
 893                 } else {
 894                         u8 val;
 895
 896                         ret = intel_vgpu_rw(mdev, &val, sizeof(val), ppos,
 897                                         false);
 898                         if (ret <= 0)
 899                                 goto read_err;
 900
 901                         if (copy_to_user(buf, &val, sizeof(val)))
 902                                 goto read_err;
 903
 904                         filled = 1;
 905                 }
 906
 907                 count -= filled;
 908                 done += filled;
 909                 *ppos += filled;
 910                 buf += filled;
 911         }
 912
 913         return done;
 914
 915 read_err:
 916         return -EFAULT;
 917 }
 918
 919 static ssize_t intel_vgpu_write(struct mdev_device *mdev,
 920                                 const char __user *buf,
 921                                 size_t count, loff_t *ppos)
 922 {
 923         unsigned int done = 0;
 924         int ret;
 925
 926         while (count) {
 927                 size_t filled;
 928
 929                 /* Only support GGTT entry 8 bytes write */
 930                 if (count >= 8 && !(*ppos % 8) &&
 931                         gtt_entry(mdev, ppos)) {
 932                         u64 val;
 933
 934                         if (copy_from_user(&val, buf, sizeof(val)))
 935                                 goto write_err;
 936
 937                         ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
 938                                         ppos, true);
 939                         if (ret <= 0)
 940                                 goto write_err;
 941
 942                         filled = 8;
 943                 } else if (count >= 4 && !(*ppos % 4)) {
 944                         u32 val;
 945
 946                         if (copy_from_user(&val, buf, sizeof(val)))
 947                                 goto write_err;
 948
 949                         ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
 950                                         ppos, true);
 951                         if (ret <= 0)
 952                                 goto write_err;
 953
 954                         filled = 4;
 955                 } else if (count >= 2 && !(*ppos % 2)) {
 956                         u16 val;
 957
 958                         if (copy_from_user(&val, buf, sizeof(val)))
 959                                 goto write_err;
 960
 961                         ret = intel_vgpu_rw(mdev, (char *)&val,
 962                                         sizeof(val), ppos, true);
 963                         if (ret <= 0)
 964                                 goto write_err;
 965
 966                         filled = 2;
 967                 } else {
 968                         u8 val;
 969
 970                         if (copy_from_user(&val, buf, sizeof(val)))
 971                                 goto write_err;
 972
 973                         ret = intel_vgpu_rw(mdev, &val, sizeof(val),
 974                                         ppos, true);
 975                         if (ret <= 0)
 976                                 goto write_err;
 977
 978                         filled = 1;
 979                 }
 980
 981                 count -= filled;
 982                 done += filled;
 983                 *ppos += filled;
 984                 buf += filled;
 985         }
 986
 987         return done;
 988 write_err:
 989         return -EFAULT;
 990 }
 991
 992 static int intel_vgpu_mmap(struct mdev_device *mdev, struct vm_area_struct *vma)
 993 {
 994         unsigned int index;
 995         u64 virtaddr;
 996         unsigned long req_size, pgoff = 0;
 997         pgprot_t pg_prot;
 998         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
 999
1000         index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
1001         if (index >= VFIO_PCI_ROM_REGION_INDEX)
1002                 return -EINVAL;
1003
1004         if (vma->vm_end < vma->vm_start)
1005                 return -EINVAL;
1006         if ((vma->vm_flags & VM_SHARED) == 0)
1007                 return -EINVAL;
1008         if (index != VFIO_PCI_BAR2_REGION_INDEX)
1009                 return -EINVAL;
1010
1011         pg_prot = vma->vm_page_prot;
1012         virtaddr = vma->vm_start;
1013         req_size = vma->vm_end - vma->vm_start;
1014         pgoff = vgpu_aperture_pa_base(vgpu) >> PAGE_SHIFT;
1015
1016         return remap_pfn_range(vma, virtaddr, pgoff, req_size, pg_prot);
1017 }
1018
1019 static int intel_vgpu_get_irq_count(struct intel_vgpu *vgpu, int type)
1020 {
1021         if (type == VFIO_PCI_INTX_IRQ_INDEX || type == VFIO_PCI_MSI_IRQ_INDEX)
1022                 return 1;
1023
1024         return 0;
1025 }
1026
1027 static int intel_vgpu_set_intx_mask(struct intel_vgpu *vgpu,
1028                         unsigned int index, unsigned int start,
1029                         unsigned int count, uint32_t flags,
1030                         void *data)
1031 {
1032         return 0;
1033 }
1034
1035 static int intel_vgpu_set_intx_unmask(struct intel_vgpu *vgpu,
1036                         unsigned int index, unsigned int start,
1037                         unsigned int count, uint32_t flags, void *data)
1038 {
1039         return 0;
1040 }
1041
1042 static int intel_vgpu_set_intx_trigger(struct intel_vgpu *vgpu,
1043                 unsigned int index, unsigned int start, unsigned int count,
1044                 uint32_t flags, void *data)
1045 {
1046         return 0;
1047 }
1048
1049 static int intel_vgpu_set_msi_trigger(struct intel_vgpu *vgpu,
1050                 unsigned int index, unsigned int start, unsigned int count,
1051                 uint32_t flags, void *data)
1052 {
1053         struct eventfd_ctx *trigger;
1054
1055         if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
1056                 int fd = *(int *)data;
1057
1058                 trigger = eventfd_ctx_fdget(fd);
1059                 if (IS_ERR(trigger)) {
1060                         gvt_vgpu_err("eventfd_ctx_fdget failed\n");
1061                         return PTR_ERR(trigger);
1062                 }
1063                 vgpu->vdev.msi_trigger = trigger;
1064         } else if ((flags & VFIO_IRQ_SET_DATA_NONE) && !count)
1065                 intel_vgpu_release_msi_eventfd_ctx(vgpu);
1066
1067         return 0;
1068 }
1069
1070 static int intel_vgpu_set_irqs(struct intel_vgpu *vgpu, uint32_t flags,
1071                 unsigned int index, unsigned int start, unsigned int count,
1072                 void *data)
1073 {
1074         int (*func)(struct intel_vgpu *vgpu, unsigned int index,
1075                         unsigned int start, unsigned int count, uint32_t flags,
1076                         void *data) = NULL;
1077
1078         switch (index) {
1079         case VFIO_PCI_INTX_IRQ_INDEX:
1080                 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
1081                 case VFIO_IRQ_SET_ACTION_MASK:
1082                         func = intel_vgpu_set_intx_mask;
1083                         break;
1084                 case VFIO_IRQ_SET_ACTION_UNMASK:
1085                         func = intel_vgpu_set_intx_unmask;
1086                         break;
1087                 case VFIO_IRQ_SET_ACTION_TRIGGER:
1088                         func = intel_vgpu_set_intx_trigger;
1089                         break;
1090                 }
1091                 break;
1092         case VFIO_PCI_MSI_IRQ_INDEX:
1093                 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
1094                 case VFIO_IRQ_SET_ACTION_MASK:
1095                 case VFIO_IRQ_SET_ACTION_UNMASK:
1096                         /* XXX Need masking support exported */
1097                         break;
1098                 case VFIO_IRQ_SET_ACTION_TRIGGER:
1099                         func = intel_vgpu_set_msi_trigger;
1100                         break;
1101                 }
1102                 break;
1103         }
1104
1105         if (!func)
1106                 return -ENOTTY;
1107
1108         return func(vgpu, index, start, count, flags, data);
1109 }
1110
1111 static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd,
1112                              unsigned long arg)
1113 {
1114         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
1115         unsigned long minsz;
1116
1117         gvt_dbg_core("vgpu%d ioctl, cmd: %d\n", vgpu->id, cmd);
1118
1119         if (cmd == VFIO_DEVICE_GET_INFO) {
1120                 struct vfio_device_info info;
1121
1122                 minsz = offsetofend(struct vfio_device_info, num_irqs);
1123
1124                 if (copy_from_user(&info, (void __user *)arg, minsz))
1125                         return -EFAULT;
1126
1127                 if (info.argsz < minsz)
1128                         return -EINVAL;
1129
1130                 info.flags = VFIO_DEVICE_FLAGS_PCI;
1131                 info.flags |= VFIO_DEVICE_FLAGS_RESET;
1132                 info.num_regions = VFIO_PCI_NUM_REGIONS +
1133                                 vgpu->vdev.num_regions;
1134                 info.num_irqs = VFIO_PCI_NUM_IRQS;
1135
1136                 return copy_to_user((void __user *)arg, &info, minsz) ?
1137                         -EFAULT : 0;
1138
1139         } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
1140                 struct vfio_region_info info;
1141                 struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
1142                 int i, ret;
1143                 struct vfio_region_info_cap_sparse_mmap *sparse = NULL;
1144                 size_t size;
1145                 int nr_areas = 1;
1146                 int cap_type_id;
1147
1148                 minsz = offsetofend(struct vfio_region_info, offset);
1149
1150                 if (copy_from_user(&info, (void __user *)arg, minsz))
1151                         return -EFAULT;
1152
1153                 if (info.argsz < minsz)
1154                         return -EINVAL;
1155
1156                 switch (info.index) {
1157                 case VFIO_PCI_CONFIG_REGION_INDEX:
1158                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1159                         info.size = vgpu->gvt->device_info.cfg_space_size;
1160                         info.flags = VFIO_REGION_INFO_FLAG_READ |
1161                                      VFIO_REGION_INFO_FLAG_WRITE;
1162                         break;
1163                 case VFIO_PCI_BAR0_REGION_INDEX:
1164                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1165                         info.size = vgpu->cfg_space.bar[info.index].size;
1166                         if (!info.size) {
1167                                 info.flags = 0;
1168                                 break;
1169                         }
1170
1171                         info.flags = VFIO_REGION_INFO_FLAG_READ |
1172                                      VFIO_REGION_INFO_FLAG_WRITE;
1173                         break;
1174                 case VFIO_PCI_BAR1_REGION_INDEX:
1175                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1176                         info.size = 0;
1177                         info.flags = 0;
1178                         break;
1179                 case VFIO_PCI_BAR2_REGION_INDEX:
1180                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1181                         info.flags = VFIO_REGION_INFO_FLAG_CAPS |
1182                                         VFIO_REGION_INFO_FLAG_MMAP |
1183                                         VFIO_REGION_INFO_FLAG_READ |
1184                                         VFIO_REGION_INFO_FLAG_WRITE;
1185                         info.size = gvt_aperture_sz(vgpu->gvt);
1186
1187                         size = sizeof(*sparse) +
1188                                         (nr_areas * sizeof(*sparse->areas));
1189                         sparse = kzalloc(size, GFP_KERNEL);
1190                         if (!sparse)
1191                                 return -ENOMEM;
1192
1193                         sparse->header.id = VFIO_REGION_INFO_CAP_SPARSE_MMAP;
1194                         sparse->header.version = 1;
1195                         sparse->nr_areas = nr_areas;
1196                         cap_type_id = VFIO_REGION_INFO_CAP_SPARSE_MMAP;
1197                         sparse->areas[0].offset =
1198                                         PAGE_ALIGN(vgpu_aperture_offset(vgpu));
1199                         sparse->areas[0].size = vgpu_aperture_sz(vgpu);
1200                         break;
1201
1202                 case VFIO_PCI_BAR3_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
1203                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1204                         info.size = 0;
1205                         info.flags = 0;
1206
1207                         gvt_dbg_core("get region info bar:%d\n", info.index);
1208                         break;
1209
1210                 case VFIO_PCI_ROM_REGION_INDEX:
1211                 case VFIO_PCI_VGA_REGION_INDEX:
1212                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1213                         info.size = 0;
1214                         info.flags = 0;
1215
1216                         gvt_dbg_core("get region info index:%d\n", info.index);
1217                         break;
1218                 default:
1219                         {
1220                                 struct vfio_region_info_cap_type cap_type = {
1221                                         .header.id = VFIO_REGION_INFO_CAP_TYPE,
1222                                         .header.version = 1 };
1223
1224                                 if (info.index >= VFIO_PCI_NUM_REGIONS +
1225                                                 vgpu->vdev.num_regions)
1226                                         return -EINVAL;
1227
1228                                 i = info.index - VFIO_PCI_NUM_REGIONS;
1229
1230                                 info.offset =
1231                                         VFIO_PCI_INDEX_TO_OFFSET(info.index);
1232                                 info.size = vgpu->vdev.region[i].size;
1233                                 info.flags = vgpu->vdev.region[i].flags;
1234
1235                                 cap_type.type = vgpu->vdev.region[i].type;
1236                                 cap_type.subtype = vgpu->vdev.region[i].subtype;
1237
1238                                 ret = vfio_info_add_capability(&caps,
1239                                                         &cap_type.header,
1240                                                         sizeof(cap_type));
1241                                 if (ret)
1242                                         return ret;
1243                         }
1244                 }
1245
1246                 if ((info.flags & VFIO_REGION_INFO_FLAG_CAPS) && sparse) {
1247                         switch (cap_type_id) {
1248                         case VFIO_REGION_INFO_CAP_SPARSE_MMAP:
1249                                 ret = vfio_info_add_capability(&caps,
1250                                         &sparse->header, sizeof(*sparse) +
1251                                         (sparse->nr_areas *
1252                                                 sizeof(*sparse->areas)));
1253                                 kfree(sparse);
1254                                 if (ret)
1255                                         return ret;
1256                                 break;
1257                         default:
1258                                 return -EINVAL;
1259                         }
1260                 }
1261
1262                 if (caps.size) {
1263                         info.flags |= VFIO_REGION_INFO_FLAG_CAPS;
1264                         if (info.argsz < sizeof(info) + caps.size) {
1265                                 info.argsz = sizeof(info) + caps.size;
1266                                 info.cap_offset = 0;
1267                         } else {
1268                                 vfio_info_cap_shift(&caps, sizeof(info));
1269                                 if (copy_to_user((void __user *)arg +
1270                                                   sizeof(info), caps.buf,
1271                                                   caps.size)) {
1272                                         kfree(caps.buf);
1273                                         return -EFAULT;
1274                                 }
1275                                 info.cap_offset = sizeof(info);
1276                         }
1277
1278                         kfree(caps.buf);
1279                 }
1280
1281                 return copy_to_user((void __user *)arg, &info, minsz) ?
1282                         -EFAULT : 0;
1283         } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
1284                 struct vfio_irq_info info;
1285
1286                 minsz = offsetofend(struct vfio_irq_info, count);
1287
1288                 if (copy_from_user(&info, (void __user *)arg, minsz))
1289                         return -EFAULT;
1290
1291                 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS)
1292                         return -EINVAL;
1293
1294                 switch (info.index) {
1295                 case VFIO_PCI_INTX_IRQ_INDEX:
1296                 case VFIO_PCI_MSI_IRQ_INDEX:
1297                         break;
1298                 default:
1299                         return -EINVAL;
1300                 }
1301
1302                 info.flags = VFIO_IRQ_INFO_EVENTFD;
1303
1304                 info.count = intel_vgpu_get_irq_count(vgpu, info.index);
1305
1306                 if (info.index == VFIO_PCI_INTX_IRQ_INDEX)
1307                         info.flags |= (VFIO_IRQ_INFO_MASKABLE |
1308                                        VFIO_IRQ_INFO_AUTOMASKED);
1309                 else
1310                         info.flags |= VFIO_IRQ_INFO_NORESIZE;
1311
1312                 return copy_to_user((void __user *)arg, &info, minsz) ?
1313                         -EFAULT : 0;
1314         } else if (cmd == VFIO_DEVICE_SET_IRQS) {
1315                 struct vfio_irq_set hdr;
1316                 u8 *data = NULL;
1317                 int ret = 0;
1318                 size_t data_size = 0;
1319
1320                 minsz = offsetofend(struct vfio_irq_set, count);
1321
1322                 if (copy_from_user(&hdr, (void __user *)arg, minsz))
1323                         return -EFAULT;
1324
1325                 if (!(hdr.flags & VFIO_IRQ_SET_DATA_NONE)) {
1326                         int max = intel_vgpu_get_irq_count(vgpu, hdr.index);
1327
1328                         ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
1329                                                 VFIO_PCI_NUM_IRQS, &data_size);
1330                         if (ret) {
1331                                 gvt_vgpu_err("intel:vfio_set_irqs_validate_and_prepare failed\n");
1332                                 return -EINVAL;
1333                         }
1334                         if (data_size) {
1335                                 data = memdup_user((void __user *)(arg + minsz),
1336                                                    data_size);
1337                                 if (IS_ERR(data))
1338                                         return PTR_ERR(data);
1339                         }
1340                 }
1341
1342                 ret = intel_vgpu_set_irqs(vgpu, hdr.flags, hdr.index,
1343                                         hdr.start, hdr.count, data);
1344                 kfree(data);
1345
1346                 return ret;
1347         } else if (cmd == VFIO_DEVICE_RESET) {
1348                 intel_gvt_ops->vgpu_reset(vgpu);
1349                 return 0;
1350         } else if (cmd == VFIO_DEVICE_QUERY_GFX_PLANE) {
1351                 struct vfio_device_gfx_plane_info dmabuf;
1352                 int ret = 0;
1353
1354                 minsz = offsetofend(struct vfio_device_gfx_plane_info,
1355                                     dmabuf_id);
1356                 if (copy_from_user(&dmabuf, (void __user *)arg, minsz))
1357                         return -EFAULT;
1358                 if (dmabuf.argsz < minsz)
1359                         return -EINVAL;
1360
1361                 ret = intel_gvt_ops->vgpu_query_plane(vgpu, &dmabuf);
1362                 if (ret != 0)
1363                         return ret;
1364
1365                 return copy_to_user((void __user *)arg, &dmabuf, minsz) ?
1366                                                                 -EFAULT : 0;
1367         } else if (cmd == VFIO_DEVICE_GET_GFX_DMABUF) {
1368                 __u32 dmabuf_id;
1369                 __s32 dmabuf_fd;
1370
1371                 if (get_user(dmabuf_id, (__u32 __user *)arg))
1372                         return -EFAULT;
1373
1374                 dmabuf_fd = intel_gvt_ops->vgpu_get_dmabuf(vgpu, dmabuf_id);
1375                 return dmabuf_fd;
1376
1377         }
1378
1379         return -ENOTTY;
1380 }
1381
1382 static ssize_t
1383 vgpu_id_show(struct device *dev, struct device_attribute *attr,
1384              char *buf)
1385 {
1386         struct mdev_device *mdev = mdev_from_dev(dev);
1387
1388         if (mdev) {
1389                 struct intel_vgpu *vgpu = (struct intel_vgpu *)
1390                         mdev_get_drvdata(mdev);
1391                 return sprintf(buf, "%d\n", vgpu->id);
1392         }
1393         return sprintf(buf, "\n");
1394 }
1395
1396 static ssize_t
1397 hw_id_show(struct device *dev, struct device_attribute *attr,
1398            char *buf)
1399 {
1400         struct mdev_device *mdev = mdev_from_dev(dev);
1401
1402         if (mdev) {
1403                 struct intel_vgpu *vgpu = (struct intel_vgpu *)
1404                         mdev_get_drvdata(mdev);
1405                 return sprintf(buf, "%u\n",
1406                                vgpu->submission.shadow_ctx->hw_id);
1407         }
1408         return sprintf(buf, "\n");
1409 }
1410
1411 static DEVICE_ATTR_RO(vgpu_id);
1412 static DEVICE_ATTR_RO(hw_id);
1413
1414 static struct attribute *intel_vgpu_attrs[] = {
1415         &dev_attr_vgpu_id.attr,
1416         &dev_attr_hw_id.attr,
1417         NULL
1418 };
1419
1420 static const struct attribute_group intel_vgpu_group = {
1421         .name = "intel_vgpu",
1422         .attrs = intel_vgpu_attrs,
1423 };
1424
1425 static const struct attribute_group *intel_vgpu_groups[] = {
1426         &intel_vgpu_group,
1427         NULL,
1428 };
1429
1430 static struct mdev_parent_ops intel_vgpu_ops = {
1431         .mdev_attr_groups       = intel_vgpu_groups,
1432         .create                 = intel_vgpu_create,
1433         .remove                 = intel_vgpu_remove,
1434
1435         .open                   = intel_vgpu_open,
1436         .release                = intel_vgpu_release,
1437
1438         .read                   = intel_vgpu_read,
1439         .write                  = intel_vgpu_write,
1440         .mmap                   = intel_vgpu_mmap,
1441         .ioctl                  = intel_vgpu_ioctl,
1442 };
1443
1444 static int kvmgt_host_init(struct device *dev, void *gvt, const void *ops)
1445 {
1446         struct attribute **kvm_type_attrs;
1447         struct attribute_group **kvm_vgpu_type_groups;
1448
1449         intel_gvt_ops = ops;
1450         if (!intel_gvt_ops->get_gvt_attrs(&kvm_type_attrs,
1451                         &kvm_vgpu_type_groups))
1452                 return -EFAULT;
1453         intel_vgpu_ops.supported_type_groups = kvm_vgpu_type_groups;
1454
1455         return mdev_register_device(dev, &intel_vgpu_ops);
1456 }
1457
1458 static void kvmgt_host_exit(struct device *dev, void *gvt)
1459 {
1460         mdev_unregister_device(dev);
1461 }
1462
1463 static int kvmgt_page_track_add(unsigned long handle, u64 gfn)
1464 {
1465         struct kvmgt_guest_info *info;
1466         struct kvm *kvm;
1467         struct kvm_memory_slot *slot;
1468         int idx;
1469
1470         if (!handle_valid(handle))
1471                 return -ESRCH;
1472
1473         info = (struct kvmgt_guest_info *)handle;
1474         kvm = info->kvm;
1475
1476         idx = srcu_read_lock(&kvm->srcu);
1477         slot = gfn_to_memslot(kvm, gfn);
1478         if (!slot) {
1479                 srcu_read_unlock(&kvm->srcu, idx);
1480                 return -EINVAL;
1481         }
1482
1483         spin_lock(&kvm->mmu_lock);
1484
1485         if (kvmgt_gfn_is_write_protected(info, gfn))
1486                 goto out;
1487
1488         kvm_slot_page_track_add_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE);
1489         kvmgt_protect_table_add(info, gfn);
1490
1491 out:
1492         spin_unlock(&kvm->mmu_lock);
1493         srcu_read_unlock(&kvm->srcu, idx);
1494         return 0;
1495 }
1496
1497 static int kvmgt_page_track_remove(unsigned long handle, u64 gfn)
1498 {
1499         struct kvmgt_guest_info *info;
1500         struct kvm *kvm;
1501         struct kvm_memory_slot *slot;
1502         int idx;
1503
1504         if (!handle_valid(handle))
1505                 return 0;
1506
1507         info = (struct kvmgt_guest_info *)handle;
1508         kvm = info->kvm;
1509
1510         idx = srcu_read_lock(&kvm->srcu);
1511         slot = gfn_to_memslot(kvm, gfn);
1512         if (!slot) {
1513                 srcu_read_unlock(&kvm->srcu, idx);
1514                 return -EINVAL;
1515         }
1516
1517         spin_lock(&kvm->mmu_lock);
1518
1519         if (!kvmgt_gfn_is_write_protected(info, gfn))
1520                 goto out;
1521
1522         kvm_slot_page_track_remove_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE);
1523         kvmgt_protect_table_del(info, gfn);
1524
1525 out:
1526         spin_unlock(&kvm->mmu_lock);
1527         srcu_read_unlock(&kvm->srcu, idx);
1528         return 0;
1529 }
1530
1531 static void kvmgt_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa,
1532                 const u8 *val, int len,
1533                 struct kvm_page_track_notifier_node *node)
1534 {
1535         struct kvmgt_guest_info *info = container_of(node,
1536                                         struct kvmgt_guest_info, track_node);
1537
1538         if (kvmgt_gfn_is_write_protected(info, gpa_to_gfn(gpa)))
1539                 intel_gvt_ops->write_protect_handler(info->vgpu, gpa,
1540                                                      (void *)val, len);
1541 }
1542
1543 static void kvmgt_page_track_flush_slot(struct kvm *kvm,
1544                 struct kvm_memory_slot *slot,
1545                 struct kvm_page_track_notifier_node *node)
1546 {
1547         int i;
1548         gfn_t gfn;
1549         struct kvmgt_guest_info *info = container_of(node,
1550                                         struct kvmgt_guest_info, track_node);
1551
1552         spin_lock(&kvm->mmu_lock);
1553         for (i = 0; i < slot->npages; i++) {
1554                 gfn = slot->base_gfn + i;
1555                 if (kvmgt_gfn_is_write_protected(info, gfn)) {
1556                         kvm_slot_page_track_remove_page(kvm, slot, gfn,
1557                                                 KVM_PAGE_TRACK_WRITE);
1558                         kvmgt_protect_table_del(info, gfn);
1559                 }
1560         }
1561         spin_unlock(&kvm->mmu_lock);
1562 }
1563
1564 static bool __kvmgt_vgpu_exist(struct intel_vgpu *vgpu, struct kvm *kvm)
1565 {
1566         struct intel_vgpu *itr;
1567         struct kvmgt_guest_info *info;
1568         int id;
1569         bool ret = false;
1570
1571         mutex_lock(&vgpu->gvt->lock);
1572         for_each_active_vgpu(vgpu->gvt, itr, id) {
1573                 if (!handle_valid(itr->handle))
1574                         continue;
1575
1576                 info = (struct kvmgt_guest_info *)itr->handle;
1577                 if (kvm && kvm == info->kvm) {
1578                         ret = true;
1579                         goto out;
1580                 }
1581         }
1582 out:
1583         mutex_unlock(&vgpu->gvt->lock);
1584         return ret;
1585 }
1586
1587 static int kvmgt_guest_init(struct mdev_device *mdev)
1588 {
1589         struct kvmgt_guest_info *info;
1590         struct intel_vgpu *vgpu;
1591         struct kvm *kvm;
1592
1593         vgpu = mdev_get_drvdata(mdev);
1594         if (handle_valid(vgpu->handle))
1595                 return -EEXIST;
1596
1597         kvm = vgpu->vdev.kvm;
1598         if (!kvm || kvm->mm != current->mm) {
1599                 gvt_vgpu_err("KVM is required to use Intel vGPU\n");
1600                 return -ESRCH;
1601         }
1602
1603         if (__kvmgt_vgpu_exist(vgpu, kvm))
1604                 return -EEXIST;
1605
1606         info = vzalloc(sizeof(struct kvmgt_guest_info));
1607         if (!info)
1608                 return -ENOMEM;
1609
1610         vgpu->handle = (unsigned long)info;
1611         info->vgpu = vgpu;
1612         info->kvm = kvm;
1613         kvm_get_kvm(info->kvm);
1614
1615         kvmgt_protect_table_init(info);
1616         gvt_cache_init(vgpu);
1617
1618         mutex_init(&vgpu->dmabuf_lock);
1619         init_completion(&vgpu->vblank_done);
1620
1621         info->track_node.track_write = kvmgt_page_track_write;
1622         info->track_node.track_flush_slot = kvmgt_page_track_flush_slot;
1623         kvm_page_track_register_notifier(kvm, &info->track_node);
1624
1625         info->debugfs_cache_entries = debugfs_create_ulong(
1626                                                 "kvmgt_nr_cache_entries",
1627                                                 0444, vgpu->debugfs,
1628                                                 &vgpu->vdev.nr_cache_entries);
1629         if (!info->debugfs_cache_entries)
1630                 gvt_vgpu_err("Cannot create kvmgt debugfs entry\n");
1631
1632         return 0;
1633 }
1634
1635 static bool kvmgt_guest_exit(struct kvmgt_guest_info *info)
1636 {
1637         debugfs_remove(info->debugfs_cache_entries);
1638
1639         kvm_page_track_unregister_notifier(info->kvm, &info->track_node);
1640         kvm_put_kvm(info->kvm);
1641         kvmgt_protect_table_destroy(info);
1642         gvt_cache_destroy(info->vgpu);
1643         vfree(info);
1644
1645         return true;
1646 }
1647
1648 static int kvmgt_attach_vgpu(void *vgpu, unsigned long *handle)
1649 {
1650         /* nothing to do here */
1651         return 0;
1652 }
1653
1654 static void kvmgt_detach_vgpu(unsigned long handle)
1655 {
1656         /* nothing to do here */
1657 }
1658
1659 static int kvmgt_inject_msi(unsigned long handle, u32 addr, u16 data)
1660 {
1661         struct kvmgt_guest_info *info;
1662         struct intel_vgpu *vgpu;
1663
1664         if (!handle_valid(handle))
1665                 return -ESRCH;
1666
1667         info = (struct kvmgt_guest_info *)handle;
1668         vgpu = info->vgpu;
1669
1670         /*
1671          * When guest is poweroff, msi_trigger is set to NULL, but vgpu's
1672          * config and mmio register isn't restored to default during guest
1673          * poweroff. If this vgpu is still used in next vm, this vgpu's pipe
1674          * may be enabled, then once this vgpu is active, it will get inject
1675          * vblank interrupt request. But msi_trigger is null until msi is
1676          * enabled by guest. so if msi_trigger is null, success is still
1677          * returned and don't inject interrupt into guest.
1678          */
1679         if (vgpu->vdev.msi_trigger == NULL)
1680                 return 0;
1681
1682         if (eventfd_signal(vgpu->vdev.msi_trigger, 1) == 1)
1683                 return 0;
1684
1685         return -EFAULT;
1686 }
1687
1688 static unsigned long kvmgt_gfn_to_pfn(unsigned long handle, unsigned long gfn)
1689 {
1690         struct kvmgt_guest_info *info;
1691         kvm_pfn_t pfn;
1692
1693         if (!handle_valid(handle))
1694                 return INTEL_GVT_INVALID_ADDR;
1695
1696         info = (struct kvmgt_guest_info *)handle;
1697
1698         pfn = gfn_to_pfn(info->kvm, gfn);
1699         if (is_error_noslot_pfn(pfn))
1700                 return INTEL_GVT_INVALID_ADDR;
1701
1702         return pfn;
1703 }
1704
1705 static int kvmgt_dma_map_guest_page(unsigned long handle, unsigned long gfn,
1706                 unsigned long size, dma_addr_t *dma_addr)
1707 {
1708         struct kvmgt_guest_info *info;
1709         struct intel_vgpu *vgpu;
1710         struct gvt_dma *entry;
1711         int ret;
1712
1713         if (!handle_valid(handle))
1714                 return -EINVAL;
1715
1716         info = (struct kvmgt_guest_info *)handle;
1717         vgpu = info->vgpu;
1718
1719         mutex_lock(&info->vgpu->vdev.cache_lock);
1720
1721         entry = __gvt_cache_find_gfn(info->vgpu, gfn);
1722         if (!entry) {
1723                 ret = gvt_dma_map_page(vgpu, gfn, dma_addr, size);
1724                 if (ret)
1725                         goto err_unlock;
1726
1727                 ret = __gvt_cache_add(info->vgpu, gfn, *dma_addr, size);
1728                 if (ret)
1729                         goto err_unmap;
1730         } else {
1731                 kref_get(&entry->ref);
1732                 *dma_addr = entry->dma_addr;
1733         }
1734
1735         mutex_unlock(&info->vgpu->vdev.cache_lock);
1736         return 0;
1737
1738 err_unmap:
1739         gvt_dma_unmap_page(vgpu, gfn, *dma_addr, size);
1740 err_unlock:
1741         mutex_unlock(&info->vgpu->vdev.cache_lock);
1742         return ret;
1743 }
1744
1745 static void __gvt_dma_release(struct kref *ref)
1746 {
1747         struct gvt_dma *entry = container_of(ref, typeof(*entry), ref);
1748
1749         gvt_dma_unmap_page(entry->vgpu, entry->gfn, entry->dma_addr,
1750                            entry->size);
1751         __gvt_cache_remove_entry(entry->vgpu, entry);
1752 }
1753
1754 static void kvmgt_dma_unmap_guest_page(unsigned long handle, dma_addr_t dma_addr)
1755 {
1756         struct kvmgt_guest_info *info;
1757         struct gvt_dma *entry;
1758
1759         if (!handle_valid(handle))
1760                 return;
1761
1762         info = (struct kvmgt_guest_info *)handle;
1763
1764         mutex_lock(&info->vgpu->vdev.cache_lock);
1765         entry = __gvt_cache_find_dma_addr(info->vgpu, dma_addr);
1766         if (entry)
1767                 kref_put(&entry->ref, __gvt_dma_release);
1768         mutex_unlock(&info->vgpu->vdev.cache_lock);
1769 }
1770
1771 static int kvmgt_rw_gpa(unsigned long handle, unsigned long gpa,
1772                         void *buf, unsigned long len, bool write)
1773 {
1774         struct kvmgt_guest_info *info;
1775         struct kvm *kvm;
1776         int idx, ret;
1777         bool kthread = current->mm == NULL;
1778
1779         if (!handle_valid(handle))
1780                 return -ESRCH;
1781
1782         info = (struct kvmgt_guest_info *)handle;
1783         kvm = info->kvm;
1784
1785         if (kthread)
1786                 use_mm(kvm->mm);
1787
1788         idx = srcu_read_lock(&kvm->srcu);
1789         ret = write ? kvm_write_guest(kvm, gpa, buf, len) :
1790                       kvm_read_guest(kvm, gpa, buf, len);
1791         srcu_read_unlock(&kvm->srcu, idx);
1792
1793         if (kthread)
1794                 unuse_mm(kvm->mm);
1795
1796         return ret;
1797 }
1798
1799 static int kvmgt_read_gpa(unsigned long handle, unsigned long gpa,
1800                         void *buf, unsigned long len)
1801 {
1802         return kvmgt_rw_gpa(handle, gpa, buf, len, false);
1803 }
1804
1805 static int kvmgt_write_gpa(unsigned long handle, unsigned long gpa,
1806                         void *buf, unsigned long len)
1807 {
1808         return kvmgt_rw_gpa(handle, gpa, buf, len, true);
1809 }
1810
1811 static unsigned long kvmgt_virt_to_pfn(void *addr)
1812 {
1813         return PFN_DOWN(__pa(addr));
1814 }
1815
1816 static bool kvmgt_is_valid_gfn(unsigned long handle, unsigned long gfn)
1817 {
1818         struct kvmgt_guest_info *info;
1819         struct kvm *kvm;
1820
1821         if (!handle_valid(handle))
1822                 return false;
1823
1824         info = (struct kvmgt_guest_info *)handle;
1825         kvm = info->kvm;
1826
1827         return kvm_is_visible_gfn(kvm, gfn);
1828
1829 }
1830
1831 struct intel_gvt_mpt kvmgt_mpt = {
1832         .host_init = kvmgt_host_init,
1833         .host_exit = kvmgt_host_exit,
1834         .attach_vgpu = kvmgt_attach_vgpu,
1835         .detach_vgpu = kvmgt_detach_vgpu,
1836         .inject_msi = kvmgt_inject_msi,
1837         .from_virt_to_mfn = kvmgt_virt_to_pfn,
1838         .enable_page_track = kvmgt_page_track_add,
1839         .disable_page_track = kvmgt_page_track_remove,
1840         .read_gpa = kvmgt_read_gpa,
1841         .write_gpa = kvmgt_write_gpa,
1842         .gfn_to_mfn = kvmgt_gfn_to_pfn,
1843         .dma_map_guest_page = kvmgt_dma_map_guest_page,
1844         .dma_unmap_guest_page = kvmgt_dma_unmap_guest_page,
1845         .set_opregion = kvmgt_set_opregion,
1846         .get_vfio_device = kvmgt_get_vfio_device,
1847         .put_vfio_device = kvmgt_put_vfio_device,
1848         .is_valid_gfn = kvmgt_is_valid_gfn,
1849 };
1850 EXPORT_SYMBOL_GPL(kvmgt_mpt);
1851
1852 static int __init kvmgt_init(void)
1853 {
1854         return 0;
1855 }
1856
1857 static void __exit kvmgt_exit(void)
1858 {
1859 }
1860
1861 module_init(kvmgt_init);
1862 module_exit(kvmgt_exit);
1863
1864 MODULE_LICENSE("GPL and additional rights");
1865 MODULE_AUTHOR("Intel Corporation");