virt/kvm/arm/vgic/vgic-its.c

   1 /*
   2  * GICv3 ITS emulation
   3  *
   4  * Copyright (C) 2015,2016 ARM Ltd.
   5  * Author: Andre Przywara <andre.przywara@arm.com>
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 #include <linux/cpu.h>
  21 #include <linux/kvm.h>
  22 #include <linux/kvm_host.h>
  23 #include <linux/interrupt.h>
  24 #include <linux/list.h>
  25 #include <linux/uaccess.h>
  26 #include <linux/list_sort.h>
  27
  28 #include <linux/irqchip/arm-gic-v3.h>
  29
  30 #include <asm/kvm_emulate.h>
  31 #include <asm/kvm_arm.h>
  32 #include <asm/kvm_mmu.h>
  33
  34 #include "vgic.h"
  35 #include "vgic-mmio.h"
  36
  37 static int vgic_its_save_tables_v0(struct vgic_its *its);
  38 static int vgic_its_restore_tables_v0(struct vgic_its *its);
  39 static int vgic_its_commit_v0(struct vgic_its *its);
  40 static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq,
  41                              struct kvm_vcpu *filter_vcpu, bool needs_inv);
  42
  43 /*
  44  * Creates a new (reference to a) struct vgic_irq for a given LPI.
  45  * If this LPI is already mapped on another ITS, we increase its refcount
  46  * and return a pointer to the existing structure.
  47  * If this is a "new" LPI, we allocate and initialize a new struct vgic_irq.
  48  * This function returns a pointer to the _unlocked_ structure.
  49  */
  50 static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid,
  51                                      struct kvm_vcpu *vcpu)
  52 {
  53         struct vgic_dist *dist = &kvm->arch.vgic;
  54         struct vgic_irq *irq = vgic_get_irq(kvm, NULL, intid), *oldirq;
  55         int ret;
  56
  57         /* In this case there is no put, since we keep the reference. */
  58         if (irq)
  59                 return irq;
  60
  61         irq = kzalloc(sizeof(struct vgic_irq), GFP_KERNEL);
  62         if (!irq)
  63                 return ERR_PTR(-ENOMEM);
  64
  65         INIT_LIST_HEAD(&irq->lpi_list);
  66         INIT_LIST_HEAD(&irq->ap_list);
  67         spin_lock_init(&irq->irq_lock);
  68
  69         irq->config = VGIC_CONFIG_EDGE;
  70         kref_init(&irq->refcount);
  71         irq->intid = intid;
  72         irq->target_vcpu = vcpu;
  73
  74         spin_lock(&dist->lpi_list_lock);
  75
  76         /*
  77          * There could be a race with another vgic_add_lpi(), so we need to
  78          * check that we don't add a second list entry with the same LPI.
  79          */
  80         list_for_each_entry(oldirq, &dist->lpi_list_head, lpi_list) {
  81                 if (oldirq->intid != intid)
  82                         continue;
  83
  84                 /* Someone was faster with adding this LPI, lets use that. */
  85                 kfree(irq);
  86                 irq = oldirq;
  87
  88                 /*
  89                  * This increases the refcount, the caller is expected to
  90                  * call vgic_put_irq() on the returned pointer once it's
  91                  * finished with the IRQ.
  92                  */
  93                 vgic_get_irq_kref(irq);
  94
  95                 goto out_unlock;
  96         }
  97
  98         list_add_tail(&irq->lpi_list, &dist->lpi_list_head);
  99         dist->lpi_list_count++;
 100
 101 out_unlock:
 102         spin_unlock(&dist->lpi_list_lock);
 103
 104         /*
 105          * We "cache" the configuration table entries in our struct vgic_irq's.
 106          * However we only have those structs for mapped IRQs, so we read in
 107          * the respective config data from memory here upon mapping the LPI.
 108          */
 109         ret = update_lpi_config(kvm, irq, NULL, false);
 110         if (ret)
 111                 return ERR_PTR(ret);
 112
 113         ret = vgic_v3_lpi_sync_pending_status(kvm, irq);
 114         if (ret)
 115                 return ERR_PTR(ret);
 116
 117         return irq;
 118 }
 119
 120 struct its_device {
 121         struct list_head dev_list;
 122
 123         /* the head for the list of ITTEs */
 124         struct list_head itt_head;
 125         u32 num_eventid_bits;
 126         gpa_t itt_addr;
 127         u32 device_id;
 128 };
 129
 130 #define COLLECTION_NOT_MAPPED ((u32)~0)
 131
 132 struct its_collection {
 133         struct list_head coll_list;
 134
 135         u32 collection_id;
 136         u32 target_addr;
 137 };
 138
 139 #define its_is_collection_mapped(coll) ((coll) && \
 140                                 ((coll)->target_addr != COLLECTION_NOT_MAPPED))
 141
 142 struct its_ite {
 143         struct list_head ite_list;
 144
 145         struct vgic_irq *irq;
 146         struct its_collection *collection;
 147         u32 event_id;
 148 };
 149
 150 /**
 151  * struct vgic_its_abi - ITS abi ops and settings
 152  * @cte_esz: collection table entry size
 153  * @dte_esz: device table entry size
 154  * @ite_esz: interrupt translation table entry size
 155  * @save tables: save the ITS tables into guest RAM
 156  * @restore_tables: restore the ITS internal structs from tables
 157  *  stored in guest RAM
 158  * @commit: initialize the registers which expose the ABI settings,
 159  *  especially the entry sizes
 160  */
 161 struct vgic_its_abi {
 162         int cte_esz;
 163         int dte_esz;
 164         int ite_esz;
 165         int (*save_tables)(struct vgic_its *its);
 166         int (*restore_tables)(struct vgic_its *its);
 167         int (*commit)(struct vgic_its *its);
 168 };
 169
 170 static const struct vgic_its_abi its_table_abi_versions[] = {
 171         [0] = {.cte_esz = 8, .dte_esz = 8, .ite_esz = 8,
 172          .save_tables = vgic_its_save_tables_v0,
 173          .restore_tables = vgic_its_restore_tables_v0,
 174          .commit = vgic_its_commit_v0,
 175         },
 176 };
 177
 178 #define NR_ITS_ABIS     ARRAY_SIZE(its_table_abi_versions)
 179
 180 inline const struct vgic_its_abi *vgic_its_get_abi(struct vgic_its *its)
 181 {
 182         return &its_table_abi_versions[its->abi_rev];
 183 }
 184
 185 int vgic_its_set_abi(struct vgic_its *its, int rev)
 186 {
 187         const struct vgic_its_abi *abi;
 188
 189         its->abi_rev = rev;
 190         abi = vgic_its_get_abi(its);
 191         return abi->commit(its);
 192 }
 193
 194 /*
 195  * Find and returns a device in the device table for an ITS.
 196  * Must be called with the its_lock mutex held.
 197  */
 198 static struct its_device *find_its_device(struct vgic_its *its, u32 device_id)
 199 {
 200         struct its_device *device;
 201
 202         list_for_each_entry(device, &its->device_list, dev_list)
 203                 if (device_id == device->device_id)
 204                         return device;
 205
 206         return NULL;
 207 }
 208
 209 /*
 210  * Find and returns an interrupt translation table entry (ITTE) for a given
 211  * Device ID/Event ID pair on an ITS.
 212  * Must be called with the its_lock mutex held.
 213  */
 214 static struct its_ite *find_ite(struct vgic_its *its, u32 device_id,
 215                                   u32 event_id)
 216 {
 217         struct its_device *device;
 218         struct its_ite *ite;
 219
 220         device = find_its_device(its, device_id);
 221         if (device == NULL)
 222                 return NULL;
 223
 224         list_for_each_entry(ite, &device->itt_head, ite_list)
 225                 if (ite->event_id == event_id)
 226                         return ite;
 227
 228         return NULL;
 229 }
 230
 231 /* To be used as an iterator this macro misses the enclosing parentheses */
 232 #define for_each_lpi_its(dev, ite, its) \
 233         list_for_each_entry(dev, &(its)->device_list, dev_list) \
 234                 list_for_each_entry(ite, &(dev)->itt_head, ite_list)
 235
 236 /*
 237  * We only implement 48 bits of PA at the moment, although the ITS
 238  * supports more. Let's be restrictive here.
 239  */
 240 #define BASER_ADDRESS(x)        ((x) & GENMASK_ULL(47, 16))
 241 #define CBASER_ADDRESS(x)       ((x) & GENMASK_ULL(47, 12))
 242
 243 #define GIC_LPI_OFFSET 8192
 244
 245 #define VITS_TYPER_IDBITS 16
 246 #define VITS_TYPER_DEVBITS 16
 247 #define VITS_DTE_MAX_DEVID_OFFSET       (BIT(14) - 1)
 248 #define VITS_ITE_MAX_EVENTID_OFFSET     (BIT(16) - 1)
 249
 250 /*
 251  * Finds and returns a collection in the ITS collection table.
 252  * Must be called with the its_lock mutex held.
 253  */
 254 static struct its_collection *find_collection(struct vgic_its *its, int coll_id)
 255 {
 256         struct its_collection *collection;
 257
 258         list_for_each_entry(collection, &its->collection_list, coll_list) {
 259                 if (coll_id == collection->collection_id)
 260                         return collection;
 261         }
 262
 263         return NULL;
 264 }
 265
 266 #define LPI_PROP_ENABLE_BIT(p)  ((p) & LPI_PROP_ENABLED)
 267 #define LPI_PROP_PRIORITY(p)    ((p) & 0xfc)
 268
 269 /*
 270  * Reads the configuration data for a given LPI from guest memory and
 271  * updates the fields in struct vgic_irq.
 272  * If filter_vcpu is not NULL, applies only if the IRQ is targeting this
 273  * VCPU. Unconditionally applies if filter_vcpu is NULL.
 274  */
 275 static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq,
 276                              struct kvm_vcpu *filter_vcpu, bool needs_inv)
 277 {
 278         u64 propbase = GICR_PROPBASER_ADDRESS(kvm->arch.vgic.propbaser);
 279         u8 prop;
 280         int ret;
 281         unsigned long flags;
 282
 283         ret = kvm_read_guest(kvm, propbase + irq->intid - GIC_LPI_OFFSET,
 284                              &prop, 1);
 285
 286         if (ret)
 287                 return ret;
 288
 289         spin_lock_irqsave(&irq->irq_lock, flags);
 290
 291         if (!filter_vcpu || filter_vcpu == irq->target_vcpu) {
 292                 irq->priority = LPI_PROP_PRIORITY(prop);
 293                 irq->enabled = LPI_PROP_ENABLE_BIT(prop);
 294
 295                 if (!irq->hw) {
 296                         vgic_queue_irq_unlock(kvm, irq, flags);
 297                         return 0;
 298                 }
 299         }
 300
 301         spin_unlock_irqrestore(&irq->irq_lock, flags);
 302
 303         if (irq->hw)
 304                 return its_prop_update_vlpi(irq->host_irq, prop, needs_inv);
 305
 306         return 0;
 307 }
 308
 309 /*
 310  * Create a snapshot of the current LPIs targeting @vcpu, so that we can
 311  * enumerate those LPIs without holding any lock.
 312  * Returns their number and puts the kmalloc'ed array into intid_ptr.
 313  */
 314 static int vgic_copy_lpi_list(struct kvm_vcpu *vcpu, u32 **intid_ptr)
 315 {
 316         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 317         struct vgic_irq *irq;
 318         u32 *intids;
 319         int irq_count = dist->lpi_list_count, i = 0;
 320
 321         /*
 322          * We use the current value of the list length, which may change
 323          * after the kmalloc. We don't care, because the guest shouldn't
 324          * change anything while the command handling is still running,
 325          * and in the worst case we would miss a new IRQ, which one wouldn't
 326          * expect to be covered by this command anyway.
 327          */
 328         intids = kmalloc_array(irq_count, sizeof(intids[0]), GFP_KERNEL);
 329         if (!intids)
 330                 return -ENOMEM;
 331
 332         spin_lock(&dist->lpi_list_lock);
 333         list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) {
 334                 /* We don't need to "get" the IRQ, as we hold the list lock. */
 335                 if (irq->target_vcpu != vcpu)
 336                         continue;
 337                 intids[i++] = irq->intid;
 338         }
 339         spin_unlock(&dist->lpi_list_lock);
 340
 341         *intid_ptr = intids;
 342         return i;
 343 }
 344
 345 static int update_affinity(struct vgic_irq *irq, struct kvm_vcpu *vcpu)
 346 {
 347         int ret = 0;
 348
 349         spin_lock(&irq->irq_lock);
 350         irq->target_vcpu = vcpu;
 351         spin_unlock(&irq->irq_lock);
 352
 353         if (irq->hw) {
 354                 struct its_vlpi_map map;
 355
 356                 ret = its_get_vlpi(irq->host_irq, &map);
 357                 if (ret)
 358                         return ret;
 359
 360                 map.vpe = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe;
 361
 362                 ret = its_map_vlpi(irq->host_irq, &map);
 363         }
 364
 365         return ret;
 366 }
 367
 368 /*
 369  * Promotes the ITS view of affinity of an ITTE (which redistributor this LPI
 370  * is targeting) to the VGIC's view, which deals with target VCPUs.
 371  * Needs to be called whenever either the collection for a LPIs has
 372  * changed or the collection itself got retargeted.
 373  */
 374 static void update_affinity_ite(struct kvm *kvm, struct its_ite *ite)
 375 {
 376         struct kvm_vcpu *vcpu;
 377
 378         if (!its_is_collection_mapped(ite->collection))
 379                 return;
 380
 381         vcpu = kvm_get_vcpu(kvm, ite->collection->target_addr);
 382         update_affinity(ite->irq, vcpu);
 383 }
 384
 385 /*
 386  * Updates the target VCPU for every LPI targeting this collection.
 387  * Must be called with the its_lock mutex held.
 388  */
 389 static void update_affinity_collection(struct kvm *kvm, struct vgic_its *its,
 390                                        struct its_collection *coll)
 391 {
 392         struct its_device *device;
 393         struct its_ite *ite;
 394
 395         for_each_lpi_its(device, ite, its) {
 396                 if (!ite->collection || coll != ite->collection)
 397                         continue;
 398
 399                 update_affinity_ite(kvm, ite);
 400         }
 401 }
 402
 403 static u32 max_lpis_propbaser(u64 propbaser)
 404 {
 405         int nr_idbits = (propbaser & 0x1f) + 1;
 406
 407         return 1U << min(nr_idbits, INTERRUPT_ID_BITS_ITS);
 408 }
 409
 410 /*
 411  * Sync the pending table pending bit of LPIs targeting @vcpu
 412  * with our own data structures. This relies on the LPI being
 413  * mapped before.
 414  */
 415 static int its_sync_lpi_pending_table(struct kvm_vcpu *vcpu)
 416 {
 417         gpa_t pendbase = GICR_PENDBASER_ADDRESS(vcpu->arch.vgic_cpu.pendbaser);
 418         struct vgic_irq *irq;
 419         int last_byte_offset = -1;
 420         int ret = 0;
 421         u32 *intids;
 422         int nr_irqs, i;
 423         unsigned long flags;
 424         u8 pendmask;
 425
 426         nr_irqs = vgic_copy_lpi_list(vcpu, &intids);
 427         if (nr_irqs < 0)
 428                 return nr_irqs;
 429
 430         for (i = 0; i < nr_irqs; i++) {
 431                 int byte_offset, bit_nr;
 432
 433                 byte_offset = intids[i] / BITS_PER_BYTE;
 434                 bit_nr = intids[i] % BITS_PER_BYTE;
 435
 436                 /*
 437                  * For contiguously allocated LPIs chances are we just read
 438                  * this very same byte in the last iteration. Reuse that.
 439                  */
 440                 if (byte_offset != last_byte_offset) {
 441                         ret = kvm_read_guest(vcpu->kvm, pendbase + byte_offset,
 442                                              &pendmask, 1);
 443                         if (ret) {
 444                                 kfree(intids);
 445                                 return ret;
 446                         }
 447                         last_byte_offset = byte_offset;
 448                 }
 449
 450                 irq = vgic_get_irq(vcpu->kvm, NULL, intids[i]);
 451                 spin_lock_irqsave(&irq->irq_lock, flags);
 452                 irq->pending_latch = pendmask & (1U << bit_nr);
 453                 vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
 454                 vgic_put_irq(vcpu->kvm, irq);
 455         }
 456
 457         kfree(intids);
 458
 459         return ret;
 460 }
 461
 462 static unsigned long vgic_mmio_read_its_typer(struct kvm *kvm,
 463                                               struct vgic_its *its,
 464                                               gpa_t addr, unsigned int len)
 465 {
 466         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
 467         u64 reg = GITS_TYPER_PLPIS;
 468
 469         /*
 470          * We use linear CPU numbers for redistributor addressing,
 471          * so GITS_TYPER.PTA is 0.
 472          * Also we force all PROPBASER registers to be the same, so
 473          * CommonLPIAff is 0 as well.
 474          * To avoid memory waste in the guest, we keep the number of IDBits and
 475          * DevBits low - as least for the time being.
 476          */
 477         reg |= GIC_ENCODE_SZ(VITS_TYPER_DEVBITS, 5) << GITS_TYPER_DEVBITS_SHIFT;
 478         reg |= GIC_ENCODE_SZ(VITS_TYPER_IDBITS, 5) << GITS_TYPER_IDBITS_SHIFT;
 479         reg |= GIC_ENCODE_SZ(abi->ite_esz, 4) << GITS_TYPER_ITT_ENTRY_SIZE_SHIFT;
 480
 481         return extract_bytes(reg, addr & 7, len);
 482 }
 483
 484 static unsigned long vgic_mmio_read_its_iidr(struct kvm *kvm,
 485                                              struct vgic_its *its,
 486                                              gpa_t addr, unsigned int len)
 487 {
 488         u32 val;
 489
 490         val = (its->abi_rev << GITS_IIDR_REV_SHIFT) & GITS_IIDR_REV_MASK;
 491         val |= (PRODUCT_ID_KVM << GITS_IIDR_PRODUCTID_SHIFT) | IMPLEMENTER_ARM;
 492         return val;
 493 }
 494
 495 static int vgic_mmio_uaccess_write_its_iidr(struct kvm *kvm,
 496                                             struct vgic_its *its,
 497                                             gpa_t addr, unsigned int len,
 498                                             unsigned long val)
 499 {
 500         u32 rev = GITS_IIDR_REV(val);
 501
 502         if (rev >= NR_ITS_ABIS)
 503                 return -EINVAL;
 504         return vgic_its_set_abi(its, rev);
 505 }
 506
 507 static unsigned long vgic_mmio_read_its_idregs(struct kvm *kvm,
 508                                                struct vgic_its *its,
 509                                                gpa_t addr, unsigned int len)
 510 {
 511         switch (addr & 0xffff) {
 512         case GITS_PIDR0:
 513                 return 0x92;    /* part number, bits[7:0] */
 514         case GITS_PIDR1:
 515                 return 0xb4;    /* part number, bits[11:8] */
 516         case GITS_PIDR2:
 517                 return GIC_PIDR2_ARCH_GICv3 | 0x0b;
 518         case GITS_PIDR4:
 519                 return 0x40;    /* This is a 64K software visible page */
 520         /* The following are the ID registers for (any) GIC. */
 521         case GITS_CIDR0:
 522                 return 0x0d;
 523         case GITS_CIDR1:
 524                 return 0xf0;
 525         case GITS_CIDR2:
 526                 return 0x05;
 527         case GITS_CIDR3:
 528                 return 0xb1;
 529         }
 530
 531         return 0;
 532 }
 533
 534 int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its,
 535                          u32 devid, u32 eventid, struct vgic_irq **irq)
 536 {
 537         struct kvm_vcpu *vcpu;
 538         struct its_ite *ite;
 539
 540         if (!its->enabled)
 541                 return -EBUSY;
 542
 543         ite = find_ite(its, devid, eventid);
 544         if (!ite || !its_is_collection_mapped(ite->collection))
 545                 return E_ITS_INT_UNMAPPED_INTERRUPT;
 546
 547         vcpu = kvm_get_vcpu(kvm, ite->collection->target_addr);
 548         if (!vcpu)
 549                 return E_ITS_INT_UNMAPPED_INTERRUPT;
 550
 551         if (!vcpu->arch.vgic_cpu.lpis_enabled)
 552                 return -EBUSY;
 553
 554         *irq = ite->irq;
 555         return 0;
 556 }
 557
 558 struct vgic_its *vgic_msi_to_its(struct kvm *kvm, struct kvm_msi *msi)
 559 {
 560         u64 address;
 561         struct kvm_io_device *kvm_io_dev;
 562         struct vgic_io_device *iodev;
 563
 564         if (!vgic_has_its(kvm))
 565                 return ERR_PTR(-ENODEV);
 566
 567         if (!(msi->flags & KVM_MSI_VALID_DEVID))
 568                 return ERR_PTR(-EINVAL);
 569
 570         address = (u64)msi->address_hi << 32 | msi->address_lo;
 571
 572         kvm_io_dev = kvm_io_bus_get_dev(kvm, KVM_MMIO_BUS, address);
 573         if (!kvm_io_dev)
 574                 return ERR_PTR(-EINVAL);
 575
 576         if (kvm_io_dev->ops != &kvm_io_gic_ops)
 577                 return ERR_PTR(-EINVAL);
 578
 579         iodev = container_of(kvm_io_dev, struct vgic_io_device, dev);
 580         if (iodev->iodev_type != IODEV_ITS)
 581                 return ERR_PTR(-EINVAL);
 582
 583         return iodev->its;
 584 }
 585
 586 /*
 587  * Find the target VCPU and the LPI number for a given devid/eventid pair
 588  * and make this IRQ pending, possibly injecting it.
 589  * Must be called with the its_lock mutex held.
 590  * Returns 0 on success, a positive error value for any ITS mapping
 591  * related errors and negative error values for generic errors.
 592  */
 593 static int vgic_its_trigger_msi(struct kvm *kvm, struct vgic_its *its,
 594                                 u32 devid, u32 eventid)
 595 {
 596         struct vgic_irq *irq = NULL;
 597         unsigned long flags;
 598         int err;
 599
 600         err = vgic_its_resolve_lpi(kvm, its, devid, eventid, &irq);
 601         if (err)
 602                 return err;
 603
 604         if (irq->hw)
 605                 return irq_set_irqchip_state(irq->host_irq,
 606                                              IRQCHIP_STATE_PENDING, true);
 607
 608         spin_lock_irqsave(&irq->irq_lock, flags);
 609         irq->pending_latch = true;
 610         vgic_queue_irq_unlock(kvm, irq, flags);
 611
 612         return 0;
 613 }
 614
 615 /*
 616  * Queries the KVM IO bus framework to get the ITS pointer from the given
 617  * doorbell address.
 618  * We then call vgic_its_trigger_msi() with the decoded data.
 619  * According to the KVM_SIGNAL_MSI API description returns 1 on success.
 620  */
 621 int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi)
 622 {
 623         struct vgic_its *its;
 624         int ret;
 625
 626         its = vgic_msi_to_its(kvm, msi);
 627         if (IS_ERR(its))
 628                 return PTR_ERR(its);
 629
 630         mutex_lock(&its->its_lock);
 631         ret = vgic_its_trigger_msi(kvm, its, msi->devid, msi->data);
 632         mutex_unlock(&its->its_lock);
 633
 634         if (ret < 0)
 635                 return ret;
 636
 637         /*
 638          * KVM_SIGNAL_MSI demands a return value > 0 for success and 0
 639          * if the guest has blocked the MSI. So we map any LPI mapping
 640          * related error to that.
 641          */
 642         if (ret)
 643                 return 0;
 644         else
 645                 return 1;
 646 }
 647
 648 /* Requires the its_lock to be held. */
 649 static void its_free_ite(struct kvm *kvm, struct its_ite *ite)
 650 {
 651         list_del(&ite->ite_list);
 652
 653         /* This put matches the get in vgic_add_lpi. */
 654         if (ite->irq) {
 655                 if (ite->irq->hw)
 656                         WARN_ON(its_unmap_vlpi(ite->irq->host_irq));
 657
 658                 vgic_put_irq(kvm, ite->irq);
 659         }
 660
 661         kfree(ite);
 662 }
 663
 664 static u64 its_cmd_mask_field(u64 *its_cmd, int word, int shift, int size)
 665 {
 666         return (le64_to_cpu(its_cmd[word]) >> shift) & (BIT_ULL(size) - 1);
 667 }
 668
 669 #define its_cmd_get_command(cmd)        its_cmd_mask_field(cmd, 0,  0,  8)
 670 #define its_cmd_get_deviceid(cmd)       its_cmd_mask_field(cmd, 0, 32, 32)
 671 #define its_cmd_get_size(cmd)           (its_cmd_mask_field(cmd, 1,  0,  5) + 1)
 672 #define its_cmd_get_id(cmd)             its_cmd_mask_field(cmd, 1,  0, 32)
 673 #define its_cmd_get_physical_id(cmd)    its_cmd_mask_field(cmd, 1, 32, 32)
 674 #define its_cmd_get_collection(cmd)     its_cmd_mask_field(cmd, 2,  0, 16)
 675 #define its_cmd_get_ittaddr(cmd)        (its_cmd_mask_field(cmd, 2,  8, 44) << 8)
 676 #define its_cmd_get_target_addr(cmd)    its_cmd_mask_field(cmd, 2, 16, 32)
 677 #define its_cmd_get_validbit(cmd)       its_cmd_mask_field(cmd, 2, 63,  1)
 678
 679 /*
 680  * The DISCARD command frees an Interrupt Translation Table Entry (ITTE).
 681  * Must be called with the its_lock mutex held.
 682  */
 683 static int vgic_its_cmd_handle_discard(struct kvm *kvm, struct vgic_its *its,
 684                                        u64 *its_cmd)
 685 {
 686         u32 device_id = its_cmd_get_deviceid(its_cmd);
 687         u32 event_id = its_cmd_get_id(its_cmd);
 688         struct its_ite *ite;
 689
 690
 691         ite = find_ite(its, device_id, event_id);
 692         if (ite && ite->collection) {
 693                 /*
 694                  * Though the spec talks about removing the pending state, we
 695                  * don't bother here since we clear the ITTE anyway and the
 696                  * pending state is a property of the ITTE struct.
 697                  */
 698                 its_free_ite(kvm, ite);
 699                 return 0;
 700         }
 701
 702         return E_ITS_DISCARD_UNMAPPED_INTERRUPT;
 703 }
 704
 705 /*
 706  * The MOVI command moves an ITTE to a different collection.
 707  * Must be called with the its_lock mutex held.
 708  */
 709 static int vgic_its_cmd_handle_movi(struct kvm *kvm, struct vgic_its *its,
 710                                     u64 *its_cmd)
 711 {
 712         u32 device_id = its_cmd_get_deviceid(its_cmd);
 713         u32 event_id = its_cmd_get_id(its_cmd);
 714         u32 coll_id = its_cmd_get_collection(its_cmd);
 715         struct kvm_vcpu *vcpu;
 716         struct its_ite *ite;
 717         struct its_collection *collection;
 718
 719         ite = find_ite(its, device_id, event_id);
 720         if (!ite)
 721                 return E_ITS_MOVI_UNMAPPED_INTERRUPT;
 722
 723         if (!its_is_collection_mapped(ite->collection))
 724                 return E_ITS_MOVI_UNMAPPED_COLLECTION;
 725
 726         collection = find_collection(its, coll_id);
 727         if (!its_is_collection_mapped(collection))
 728                 return E_ITS_MOVI_UNMAPPED_COLLECTION;
 729
 730         ite->collection = collection;
 731         vcpu = kvm_get_vcpu(kvm, collection->target_addr);
 732
 733         return update_affinity(ite->irq, vcpu);
 734 }
 735
 736 /*
 737  * Check whether an ID can be stored into the corresponding guest table.
 738  * For a direct table this is pretty easy, but gets a bit nasty for
 739  * indirect tables. We check whether the resulting guest physical address
 740  * is actually valid (covered by a memslot and guest accessible).
 741  * For this we have to read the respective first level entry.
 742  */
 743 static bool vgic_its_check_id(struct vgic_its *its, u64 baser, u32 id,
 744                               gpa_t *eaddr)
 745 {
 746         int l1_tbl_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
 747         u64 indirect_ptr, type = GITS_BASER_TYPE(baser);
 748         int esz = GITS_BASER_ENTRY_SIZE(baser);
 749         int index;
 750         gfn_t gfn;
 751
 752         switch (type) {
 753         case GITS_BASER_TYPE_DEVICE:
 754                 if (id >= BIT_ULL(VITS_TYPER_DEVBITS))
 755                         return false;
 756                 break;
 757         case GITS_BASER_TYPE_COLLECTION:
 758                 /* as GITS_TYPER.CIL == 0, ITS supports 16-bit collection ID */
 759                 if (id >= BIT_ULL(16))
 760                         return false;
 761                 break;
 762         default:
 763                 return false;
 764         }
 765
 766         if (!(baser & GITS_BASER_INDIRECT)) {
 767                 phys_addr_t addr;
 768
 769                 if (id >= (l1_tbl_size / esz))
 770                         return false;
 771
 772                 addr = BASER_ADDRESS(baser) + id * esz;
 773                 gfn = addr >> PAGE_SHIFT;
 774
 775                 if (eaddr)
 776                         *eaddr = addr;
 777                 return kvm_is_visible_gfn(its->dev->kvm, gfn);
 778         }
 779
 780         /* calculate and check the index into the 1st level */
 781         index = id / (SZ_64K / esz);
 782         if (index >= (l1_tbl_size / sizeof(u64)))
 783                 return false;
 784
 785         /* Each 1st level entry is represented by a 64-bit value. */
 786         if (kvm_read_guest(its->dev->kvm,
 787                            BASER_ADDRESS(baser) + index * sizeof(indirect_ptr),
 788                            &indirect_ptr, sizeof(indirect_ptr)))
 789                 return false;
 790
 791         indirect_ptr = le64_to_cpu(indirect_ptr);
 792
 793         /* check the valid bit of the first level entry */
 794         if (!(indirect_ptr & BIT_ULL(63)))
 795                 return false;
 796
 797         /*
 798          * Mask the guest physical address and calculate the frame number.
 799          * Any address beyond our supported 48 bits of PA will be caught
 800          * by the actual check in the final step.
 801          */
 802         indirect_ptr &= GENMASK_ULL(51, 16);
 803
 804         /* Find the address of the actual entry */
 805         index = id % (SZ_64K / esz);
 806         indirect_ptr += index * esz;
 807         gfn = indirect_ptr >> PAGE_SHIFT;
 808
 809         if (eaddr)
 810                 *eaddr = indirect_ptr;
 811         return kvm_is_visible_gfn(its->dev->kvm, gfn);
 812 }
 813
 814 static int vgic_its_alloc_collection(struct vgic_its *its,
 815                                      struct its_collection **colp,
 816                                      u32 coll_id)
 817 {
 818         struct its_collection *collection;
 819
 820         if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL))
 821                 return E_ITS_MAPC_COLLECTION_OOR;
 822
 823         collection = kzalloc(sizeof(*collection), GFP_KERNEL);
 824         if (!collection)
 825                 return -ENOMEM;
 826
 827         collection->collection_id = coll_id;
 828         collection->target_addr = COLLECTION_NOT_MAPPED;
 829
 830         list_add_tail(&collection->coll_list, &its->collection_list);
 831         *colp = collection;
 832
 833         return 0;
 834 }
 835
 836 static void vgic_its_free_collection(struct vgic_its *its, u32 coll_id)
 837 {
 838         struct its_collection *collection;
 839         struct its_device *device;
 840         struct its_ite *ite;
 841
 842         /*
 843          * Clearing the mapping for that collection ID removes the
 844          * entry from the list. If there wasn't any before, we can
 845          * go home early.
 846          */
 847         collection = find_collection(its, coll_id);
 848         if (!collection)
 849                 return;
 850
 851         for_each_lpi_its(device, ite, its)
 852                 if (ite->collection &&
 853                     ite->collection->collection_id == coll_id)
 854                         ite->collection = NULL;
 855
 856         list_del(&collection->coll_list);
 857         kfree(collection);
 858 }
 859
 860 /* Must be called with its_lock mutex held */
 861 static struct its_ite *vgic_its_alloc_ite(struct its_device *device,
 862                                           struct its_collection *collection,
 863                                           u32 event_id)
 864 {
 865         struct its_ite *ite;
 866
 867         ite = kzalloc(sizeof(*ite), GFP_KERNEL);
 868         if (!ite)
 869                 return ERR_PTR(-ENOMEM);
 870
 871         ite->event_id   = event_id;
 872         ite->collection = collection;
 873
 874         list_add_tail(&ite->ite_list, &device->itt_head);
 875         return ite;
 876 }
 877
 878 /*
 879  * The MAPTI and MAPI commands map LPIs to ITTEs.
 880  * Must be called with its_lock mutex held.
 881  */
 882 static int vgic_its_cmd_handle_mapi(struct kvm *kvm, struct vgic_its *its,
 883                                     u64 *its_cmd)
 884 {
 885         u32 device_id = its_cmd_get_deviceid(its_cmd);
 886         u32 event_id = its_cmd_get_id(its_cmd);
 887         u32 coll_id = its_cmd_get_collection(its_cmd);
 888         struct its_ite *ite;
 889         struct kvm_vcpu *vcpu = NULL;
 890         struct its_device *device;
 891         struct its_collection *collection, *new_coll = NULL;
 892         struct vgic_irq *irq;
 893         int lpi_nr;
 894
 895         device = find_its_device(its, device_id);
 896         if (!device)
 897                 return E_ITS_MAPTI_UNMAPPED_DEVICE;
 898
 899         if (event_id >= BIT_ULL(device->num_eventid_bits))
 900                 return E_ITS_MAPTI_ID_OOR;
 901
 902         if (its_cmd_get_command(its_cmd) == GITS_CMD_MAPTI)
 903                 lpi_nr = its_cmd_get_physical_id(its_cmd);
 904         else
 905                 lpi_nr = event_id;
 906         if (lpi_nr < GIC_LPI_OFFSET ||
 907             lpi_nr >= max_lpis_propbaser(kvm->arch.vgic.propbaser))
 908                 return E_ITS_MAPTI_PHYSICALID_OOR;
 909
 910         /* If there is an existing mapping, behavior is UNPREDICTABLE. */
 911         if (find_ite(its, device_id, event_id))
 912                 return 0;
 913
 914         collection = find_collection(its, coll_id);
 915         if (!collection) {
 916                 int ret = vgic_its_alloc_collection(its, &collection, coll_id);
 917                 if (ret)
 918                         return ret;
 919                 new_coll = collection;
 920         }
 921
 922         ite = vgic_its_alloc_ite(device, collection, event_id);
 923         if (IS_ERR(ite)) {
 924                 if (new_coll)
 925                         vgic_its_free_collection(its, coll_id);
 926                 return PTR_ERR(ite);
 927         }
 928
 929         if (its_is_collection_mapped(collection))
 930                 vcpu = kvm_get_vcpu(kvm, collection->target_addr);
 931
 932         irq = vgic_add_lpi(kvm, lpi_nr, vcpu);
 933         if (IS_ERR(irq)) {
 934                 if (new_coll)
 935                         vgic_its_free_collection(its, coll_id);
 936                 its_free_ite(kvm, ite);
 937                 return PTR_ERR(irq);
 938         }
 939         ite->irq = irq;
 940
 941         return 0;
 942 }
 943
 944 /* Requires the its_lock to be held. */
 945 static void vgic_its_free_device(struct kvm *kvm, struct its_device *device)
 946 {
 947         struct its_ite *ite, *temp;
 948
 949         /*
 950          * The spec says that unmapping a device with still valid
 951          * ITTEs associated is UNPREDICTABLE. We remove all ITTEs,
 952          * since we cannot leave the memory unreferenced.
 953          */
 954         list_for_each_entry_safe(ite, temp, &device->itt_head, ite_list)
 955                 its_free_ite(kvm, ite);
 956
 957         list_del(&device->dev_list);
 958         kfree(device);
 959 }
 960
 961 /* its lock must be held */
 962 static void vgic_its_free_device_list(struct kvm *kvm, struct vgic_its *its)
 963 {
 964         struct its_device *cur, *temp;
 965
 966         list_for_each_entry_safe(cur, temp, &its->device_list, dev_list)
 967                 vgic_its_free_device(kvm, cur);
 968 }
 969
 970 /* its lock must be held */
 971 static void vgic_its_free_collection_list(struct kvm *kvm, struct vgic_its *its)
 972 {
 973         struct its_collection *cur, *temp;
 974
 975         list_for_each_entry_safe(cur, temp, &its->collection_list, coll_list)
 976                 vgic_its_free_collection(its, cur->collection_id);
 977 }
 978
 979 /* Must be called with its_lock mutex held */
 980 static struct its_device *vgic_its_alloc_device(struct vgic_its *its,
 981                                                 u32 device_id, gpa_t itt_addr,
 982                                                 u8 num_eventid_bits)
 983 {
 984         struct its_device *device;
 985
 986         device = kzalloc(sizeof(*device), GFP_KERNEL);
 987         if (!device)
 988                 return ERR_PTR(-ENOMEM);
 989
 990         device->device_id = device_id;
 991         device->itt_addr = itt_addr;
 992         device->num_eventid_bits = num_eventid_bits;
 993         INIT_LIST_HEAD(&device->itt_head);
 994
 995         list_add_tail(&device->dev_list, &its->device_list);
 996         return device;
 997 }
 998
 999 /*
1000  * MAPD maps or unmaps a device ID to Interrupt Translation Tables (ITTs).
1001  * Must be called with the its_lock mutex held.
1002  */
1003 static int vgic_its_cmd_handle_mapd(struct kvm *kvm, struct vgic_its *its,
1004                                     u64 *its_cmd)
1005 {
1006         u32 device_id = its_cmd_get_deviceid(its_cmd);
1007         bool valid = its_cmd_get_validbit(its_cmd);
1008         u8 num_eventid_bits = its_cmd_get_size(its_cmd);
1009         gpa_t itt_addr = its_cmd_get_ittaddr(its_cmd);
1010         struct its_device *device;
1011
1012         if (!vgic_its_check_id(its, its->baser_device_table, device_id, NULL))
1013                 return E_ITS_MAPD_DEVICE_OOR;
1014
1015         if (valid && num_eventid_bits > VITS_TYPER_IDBITS)
1016                 return E_ITS_MAPD_ITTSIZE_OOR;
1017
1018         device = find_its_device(its, device_id);
1019
1020         /*
1021          * The spec says that calling MAPD on an already mapped device
1022          * invalidates all cached data for this device. We implement this
1023          * by removing the mapping and re-establishing it.
1024          */
1025         if (device)
1026                 vgic_its_free_device(kvm, device);
1027
1028         /*
1029          * The spec does not say whether unmapping a not-mapped device
1030          * is an error, so we are done in any case.
1031          */
1032         if (!valid)
1033                 return 0;
1034
1035         device = vgic_its_alloc_device(its, device_id, itt_addr,
1036                                        num_eventid_bits);
1037         if (IS_ERR(device))
1038                 return PTR_ERR(device);
1039
1040         return 0;
1041 }
1042
1043 /*
1044  * The MAPC command maps collection IDs to redistributors.
1045  * Must be called with the its_lock mutex held.
1046  */
1047 static int vgic_its_cmd_handle_mapc(struct kvm *kvm, struct vgic_its *its,
1048                                     u64 *its_cmd)
1049 {
1050         u16 coll_id;
1051         u32 target_addr;
1052         struct its_collection *collection;
1053         bool valid;
1054
1055         valid = its_cmd_get_validbit(its_cmd);
1056         coll_id = its_cmd_get_collection(its_cmd);
1057         target_addr = its_cmd_get_target_addr(its_cmd);
1058
1059         if (target_addr >= atomic_read(&kvm->online_vcpus))
1060                 return E_ITS_MAPC_PROCNUM_OOR;
1061
1062         if (!valid) {
1063                 vgic_its_free_collection(its, coll_id);
1064         } else {
1065                 collection = find_collection(its, coll_id);
1066
1067                 if (!collection) {
1068                         int ret;
1069
1070                         ret = vgic_its_alloc_collection(its, &collection,
1071                                                         coll_id);
1072                         if (ret)
1073                                 return ret;
1074                         collection->target_addr = target_addr;
1075                 } else {
1076                         collection->target_addr = target_addr;
1077                         update_affinity_collection(kvm, its, collection);
1078                 }
1079         }
1080
1081         return 0;
1082 }
1083
1084 /*
1085  * The CLEAR command removes the pending state for a particular LPI.
1086  * Must be called with the its_lock mutex held.
1087  */
1088 static int vgic_its_cmd_handle_clear(struct kvm *kvm, struct vgic_its *its,
1089                                      u64 *its_cmd)
1090 {
1091         u32 device_id = its_cmd_get_deviceid(its_cmd);
1092         u32 event_id = its_cmd_get_id(its_cmd);
1093         struct its_ite *ite;
1094
1095
1096         ite = find_ite(its, device_id, event_id);
1097         if (!ite)
1098                 return E_ITS_CLEAR_UNMAPPED_INTERRUPT;
1099
1100         ite->irq->pending_latch = false;
1101
1102         if (ite->irq->hw)
1103                 return irq_set_irqchip_state(ite->irq->host_irq,
1104                                              IRQCHIP_STATE_PENDING, false);
1105
1106         return 0;
1107 }
1108
1109 /*
1110  * The INV command syncs the configuration bits from the memory table.
1111  * Must be called with the its_lock mutex held.
1112  */
1113 static int vgic_its_cmd_handle_inv(struct kvm *kvm, struct vgic_its *its,
1114                                    u64 *its_cmd)
1115 {
1116         u32 device_id = its_cmd_get_deviceid(its_cmd);
1117         u32 event_id = its_cmd_get_id(its_cmd);
1118         struct its_ite *ite;
1119
1120
1121         ite = find_ite(its, device_id, event_id);
1122         if (!ite)
1123                 return E_ITS_INV_UNMAPPED_INTERRUPT;
1124
1125         return update_lpi_config(kvm, ite->irq, NULL, true);
1126 }
1127
1128 /*
1129  * The INVALL command requests flushing of all IRQ data in this collection.
1130  * Find the VCPU mapped to that collection, then iterate over the VM's list
1131  * of mapped LPIs and update the configuration for each IRQ which targets
1132  * the specified vcpu. The configuration will be read from the in-memory
1133  * configuration table.
1134  * Must be called with the its_lock mutex held.
1135  */
1136 static int vgic_its_cmd_handle_invall(struct kvm *kvm, struct vgic_its *its,
1137                                       u64 *its_cmd)
1138 {
1139         u32 coll_id = its_cmd_get_collection(its_cmd);
1140         struct its_collection *collection;
1141         struct kvm_vcpu *vcpu;
1142         struct vgic_irq *irq;
1143         u32 *intids;
1144         int irq_count, i;
1145
1146         collection = find_collection(its, coll_id);
1147         if (!its_is_collection_mapped(collection))
1148                 return E_ITS_INVALL_UNMAPPED_COLLECTION;
1149
1150         vcpu = kvm_get_vcpu(kvm, collection->target_addr);
1151
1152         irq_count = vgic_copy_lpi_list(vcpu, &intids);
1153         if (irq_count < 0)
1154                 return irq_count;
1155
1156         for (i = 0; i < irq_count; i++) {
1157                 irq = vgic_get_irq(kvm, NULL, intids[i]);
1158                 if (!irq)
1159                         continue;
1160                 update_lpi_config(kvm, irq, vcpu, false);
1161                 vgic_put_irq(kvm, irq);
1162         }
1163
1164         kfree(intids);
1165
1166         if (vcpu->arch.vgic_cpu.vgic_v3.its_vpe.its_vm)
1167                 its_invall_vpe(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe);
1168
1169         return 0;
1170 }
1171
1172 /*
1173  * The MOVALL command moves the pending state of all IRQs targeting one
1174  * redistributor to another. We don't hold the pending state in the VCPUs,
1175  * but in the IRQs instead, so there is really not much to do for us here.
1176  * However the spec says that no IRQ must target the old redistributor
1177  * afterwards, so we make sure that no LPI is using the associated target_vcpu.
1178  * This command affects all LPIs in the system that target that redistributor.
1179  */
1180 static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its,
1181                                       u64 *its_cmd)
1182 {
1183         u32 target1_addr = its_cmd_get_target_addr(its_cmd);
1184         u32 target2_addr = its_cmd_mask_field(its_cmd, 3, 16, 32);
1185         struct kvm_vcpu *vcpu1, *vcpu2;
1186         struct vgic_irq *irq;
1187         u32 *intids;
1188         int irq_count, i;
1189
1190         if (target1_addr >= atomic_read(&kvm->online_vcpus) ||
1191             target2_addr >= atomic_read(&kvm->online_vcpus))
1192                 return E_ITS_MOVALL_PROCNUM_OOR;
1193
1194         if (target1_addr == target2_addr)
1195                 return 0;
1196
1197         vcpu1 = kvm_get_vcpu(kvm, target1_addr);
1198         vcpu2 = kvm_get_vcpu(kvm, target2_addr);
1199
1200         irq_count = vgic_copy_lpi_list(vcpu1, &intids);
1201         if (irq_count < 0)
1202                 return irq_count;
1203
1204         for (i = 0; i < irq_count; i++) {
1205                 irq = vgic_get_irq(kvm, NULL, intids[i]);
1206
1207                 update_affinity(irq, vcpu2);
1208
1209                 vgic_put_irq(kvm, irq);
1210         }
1211
1212         kfree(intids);
1213         return 0;
1214 }
1215
1216 /*
1217  * The INT command injects the LPI associated with that DevID/EvID pair.
1218  * Must be called with the its_lock mutex held.
1219  */
1220 static int vgic_its_cmd_handle_int(struct kvm *kvm, struct vgic_its *its,
1221                                    u64 *its_cmd)
1222 {
1223         u32 msi_data = its_cmd_get_id(its_cmd);
1224         u64 msi_devid = its_cmd_get_deviceid(its_cmd);
1225
1226         return vgic_its_trigger_msi(kvm, its, msi_devid, msi_data);
1227 }
1228
1229 /*
1230  * This function is called with the its_cmd lock held, but the ITS data
1231  * structure lock dropped.
1232  */
1233 static int vgic_its_handle_command(struct kvm *kvm, struct vgic_its *its,
1234                                    u64 *its_cmd)
1235 {
1236         int ret = -ENODEV;
1237
1238         mutex_lock(&its->its_lock);
1239         switch (its_cmd_get_command(its_cmd)) {
1240         case GITS_CMD_MAPD:
1241                 ret = vgic_its_cmd_handle_mapd(kvm, its, its_cmd);
1242                 break;
1243         case GITS_CMD_MAPC:
1244                 ret = vgic_its_cmd_handle_mapc(kvm, its, its_cmd);
1245                 break;
1246         case GITS_CMD_MAPI:
1247                 ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1248                 break;
1249         case GITS_CMD_MAPTI:
1250                 ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1251                 break;
1252         case GITS_CMD_MOVI:
1253                 ret = vgic_its_cmd_handle_movi(kvm, its, its_cmd);
1254                 break;
1255         case GITS_CMD_DISCARD:
1256                 ret = vgic_its_cmd_handle_discard(kvm, its, its_cmd);
1257                 break;
1258         case GITS_CMD_CLEAR:
1259                 ret = vgic_its_cmd_handle_clear(kvm, its, its_cmd);
1260                 break;
1261         case GITS_CMD_MOVALL:
1262                 ret = vgic_its_cmd_handle_movall(kvm, its, its_cmd);
1263                 break;
1264         case GITS_CMD_INT:
1265                 ret = vgic_its_cmd_handle_int(kvm, its, its_cmd);
1266                 break;
1267         case GITS_CMD_INV:
1268                 ret = vgic_its_cmd_handle_inv(kvm, its, its_cmd);
1269                 break;
1270         case GITS_CMD_INVALL:
1271                 ret = vgic_its_cmd_handle_invall(kvm, its, its_cmd);
1272                 break;
1273         case GITS_CMD_SYNC:
1274                 /* we ignore this command: we are in sync all of the time */
1275                 ret = 0;
1276                 break;
1277         }
1278         mutex_unlock(&its->its_lock);
1279
1280         return ret;
1281 }
1282
1283 static u64 vgic_sanitise_its_baser(u64 reg)
1284 {
1285         reg = vgic_sanitise_field(reg, GITS_BASER_SHAREABILITY_MASK,
1286                                   GITS_BASER_SHAREABILITY_SHIFT,
1287                                   vgic_sanitise_shareability);
1288         reg = vgic_sanitise_field(reg, GITS_BASER_INNER_CACHEABILITY_MASK,
1289                                   GITS_BASER_INNER_CACHEABILITY_SHIFT,
1290                                   vgic_sanitise_inner_cacheability);
1291         reg = vgic_sanitise_field(reg, GITS_BASER_OUTER_CACHEABILITY_MASK,
1292                                   GITS_BASER_OUTER_CACHEABILITY_SHIFT,
1293                                   vgic_sanitise_outer_cacheability);
1294
1295         /* Bits 15:12 contain bits 51:48 of the PA, which we don't support. */
1296         reg &= ~GENMASK_ULL(15, 12);
1297
1298         /* We support only one (ITS) page size: 64K */
1299         reg = (reg & ~GITS_BASER_PAGE_SIZE_MASK) | GITS_BASER_PAGE_SIZE_64K;
1300
1301         return reg;
1302 }
1303
1304 static u64 vgic_sanitise_its_cbaser(u64 reg)
1305 {
1306         reg = vgic_sanitise_field(reg, GITS_CBASER_SHAREABILITY_MASK,
1307                                   GITS_CBASER_SHAREABILITY_SHIFT,
1308                                   vgic_sanitise_shareability);
1309         reg = vgic_sanitise_field(reg, GITS_CBASER_INNER_CACHEABILITY_MASK,
1310                                   GITS_CBASER_INNER_CACHEABILITY_SHIFT,
1311                                   vgic_sanitise_inner_cacheability);
1312         reg = vgic_sanitise_field(reg, GITS_CBASER_OUTER_CACHEABILITY_MASK,
1313                                   GITS_CBASER_OUTER_CACHEABILITY_SHIFT,
1314                                   vgic_sanitise_outer_cacheability);
1315
1316         /*
1317          * Sanitise the physical address to be 64k aligned.
1318          * Also limit the physical addresses to 48 bits.
1319          */
1320         reg &= ~(GENMASK_ULL(51, 48) | GENMASK_ULL(15, 12));
1321
1322         return reg;
1323 }
1324
1325 static unsigned long vgic_mmio_read_its_cbaser(struct kvm *kvm,
1326                                                struct vgic_its *its,
1327                                                gpa_t addr, unsigned int len)
1328 {
1329         return extract_bytes(its->cbaser, addr & 7, len);
1330 }
1331
1332 static void vgic_mmio_write_its_cbaser(struct kvm *kvm, struct vgic_its *its,
1333                                        gpa_t addr, unsigned int len,
1334                                        unsigned long val)
1335 {
1336         /* When GITS_CTLR.Enable is 1, this register is RO. */
1337         if (its->enabled)
1338                 return;
1339
1340         mutex_lock(&its->cmd_lock);
1341         its->cbaser = update_64bit_reg(its->cbaser, addr & 7, len, val);
1342         its->cbaser = vgic_sanitise_its_cbaser(its->cbaser);
1343         its->creadr = 0;
1344         /*
1345          * CWRITER is architecturally UNKNOWN on reset, but we need to reset
1346          * it to CREADR to make sure we start with an empty command buffer.
1347          */
1348         its->cwriter = its->creadr;
1349         mutex_unlock(&its->cmd_lock);
1350 }
1351
1352 #define ITS_CMD_BUFFER_SIZE(baser)      ((((baser) & 0xff) + 1) << 12)
1353 #define ITS_CMD_SIZE                    32
1354 #define ITS_CMD_OFFSET(reg)             ((reg) & GENMASK(19, 5))
1355
1356 /* Must be called with the cmd_lock held. */
1357 static void vgic_its_process_commands(struct kvm *kvm, struct vgic_its *its)
1358 {
1359         gpa_t cbaser;
1360         u64 cmd_buf[4];
1361
1362         /* Commands are only processed when the ITS is enabled. */
1363         if (!its->enabled)
1364                 return;
1365
1366         cbaser = CBASER_ADDRESS(its->cbaser);
1367
1368         while (its->cwriter != its->creadr) {
1369                 int ret = kvm_read_guest(kvm, cbaser + its->creadr,
1370                                          cmd_buf, ITS_CMD_SIZE);
1371                 /*
1372                  * If kvm_read_guest() fails, this could be due to the guest
1373                  * programming a bogus value in CBASER or something else going
1374                  * wrong from which we cannot easily recover.
1375                  * According to section 6.3.2 in the GICv3 spec we can just
1376                  * ignore that command then.
1377                  */
1378                 if (!ret)
1379                         vgic_its_handle_command(kvm, its, cmd_buf);
1380
1381                 its->creadr += ITS_CMD_SIZE;
1382                 if (its->creadr == ITS_CMD_BUFFER_SIZE(its->cbaser))
1383                         its->creadr = 0;
1384         }
1385 }
1386
1387 /*
1388  * By writing to CWRITER the guest announces new commands to be processed.
1389  * To avoid any races in the first place, we take the its_cmd lock, which
1390  * protects our ring buffer variables, so that there is only one user
1391  * per ITS handling commands at a given time.
1392  */
1393 static void vgic_mmio_write_its_cwriter(struct kvm *kvm, struct vgic_its *its,
1394                                         gpa_t addr, unsigned int len,
1395                                         unsigned long val)
1396 {
1397         u64 reg;
1398
1399         if (!its)
1400                 return;
1401
1402         mutex_lock(&its->cmd_lock);
1403
1404         reg = update_64bit_reg(its->cwriter, addr & 7, len, val);
1405         reg = ITS_CMD_OFFSET(reg);
1406         if (reg >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
1407                 mutex_unlock(&its->cmd_lock);
1408                 return;
1409         }
1410         its->cwriter = reg;
1411
1412         vgic_its_process_commands(kvm, its);
1413
1414         mutex_unlock(&its->cmd_lock);
1415 }
1416
1417 static unsigned long vgic_mmio_read_its_cwriter(struct kvm *kvm,
1418                                                 struct vgic_its *its,
1419                                                 gpa_t addr, unsigned int len)
1420 {
1421         return extract_bytes(its->cwriter, addr & 0x7, len);
1422 }
1423
1424 static unsigned long vgic_mmio_read_its_creadr(struct kvm *kvm,
1425                                                struct vgic_its *its,
1426                                                gpa_t addr, unsigned int len)
1427 {
1428         return extract_bytes(its->creadr, addr & 0x7, len);
1429 }
1430
1431 static int vgic_mmio_uaccess_write_its_creadr(struct kvm *kvm,
1432                                               struct vgic_its *its,
1433                                               gpa_t addr, unsigned int len,
1434                                               unsigned long val)
1435 {
1436         u32 cmd_offset;
1437         int ret = 0;
1438
1439         mutex_lock(&its->cmd_lock);
1440
1441         if (its->enabled) {
1442                 ret = -EBUSY;
1443                 goto out;
1444         }
1445
1446         cmd_offset = ITS_CMD_OFFSET(val);
1447         if (cmd_offset >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
1448                 ret = -EINVAL;
1449                 goto out;
1450         }
1451
1452         its->creadr = cmd_offset;
1453 out:
1454         mutex_unlock(&its->cmd_lock);
1455         return ret;
1456 }
1457
1458 #define BASER_INDEX(addr) (((addr) / sizeof(u64)) & 0x7)
1459 static unsigned long vgic_mmio_read_its_baser(struct kvm *kvm,
1460                                               struct vgic_its *its,
1461                                               gpa_t addr, unsigned int len)
1462 {
1463         u64 reg;
1464
1465         switch (BASER_INDEX(addr)) {
1466         case 0:
1467                 reg = its->baser_device_table;
1468                 break;
1469         case 1:
1470                 reg = its->baser_coll_table;
1471                 break;
1472         default:
1473                 reg = 0;
1474                 break;
1475         }
1476
1477         return extract_bytes(reg, addr & 7, len);
1478 }
1479
1480 #define GITS_BASER_RO_MASK      (GENMASK_ULL(52, 48) | GENMASK_ULL(58, 56))
1481 static void vgic_mmio_write_its_baser(struct kvm *kvm,
1482                                       struct vgic_its *its,
1483                                       gpa_t addr, unsigned int len,
1484                                       unsigned long val)
1485 {
1486         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
1487         u64 entry_size, table_type;
1488         u64 reg, *regptr, clearbits = 0;
1489
1490         /* When GITS_CTLR.Enable is 1, we ignore write accesses. */
1491         if (its->enabled)
1492                 return;
1493
1494         switch (BASER_INDEX(addr)) {
1495         case 0:
1496                 regptr = &its->baser_device_table;
1497                 entry_size = abi->dte_esz;
1498                 table_type = GITS_BASER_TYPE_DEVICE;
1499                 break;
1500         case 1:
1501                 regptr = &its->baser_coll_table;
1502                 entry_size = abi->cte_esz;
1503                 table_type = GITS_BASER_TYPE_COLLECTION;
1504                 clearbits = GITS_BASER_INDIRECT;
1505                 break;
1506         default:
1507                 return;
1508         }
1509
1510         reg = update_64bit_reg(*regptr, addr & 7, len, val);
1511         reg &= ~GITS_BASER_RO_MASK;
1512         reg &= ~clearbits;
1513
1514         reg |= (entry_size - 1) << GITS_BASER_ENTRY_SIZE_SHIFT;
1515         reg |= table_type << GITS_BASER_TYPE_SHIFT;
1516         reg = vgic_sanitise_its_baser(reg);
1517
1518         *regptr = reg;
1519
1520         if (!(reg & GITS_BASER_VALID)) {
1521                 /* Take the its_lock to prevent a race with a save/restore */
1522                 mutex_lock(&its->its_lock);
1523                 switch (table_type) {
1524                 case GITS_BASER_TYPE_DEVICE:
1525                         vgic_its_free_device_list(kvm, its);
1526                         break;
1527                 case GITS_BASER_TYPE_COLLECTION:
1528                         vgic_its_free_collection_list(kvm, its);
1529                         break;
1530                 }
1531                 mutex_unlock(&its->its_lock);
1532         }
1533 }
1534
1535 static unsigned long vgic_mmio_read_its_ctlr(struct kvm *vcpu,
1536                                              struct vgic_its *its,
1537                                              gpa_t addr, unsigned int len)
1538 {
1539         u32 reg = 0;
1540
1541         mutex_lock(&its->cmd_lock);
1542         if (its->creadr == its->cwriter)
1543                 reg |= GITS_CTLR_QUIESCENT;
1544         if (its->enabled)
1545                 reg |= GITS_CTLR_ENABLE;
1546         mutex_unlock(&its->cmd_lock);
1547
1548         return reg;
1549 }
1550
1551 static void vgic_mmio_write_its_ctlr(struct kvm *kvm, struct vgic_its *its,
1552                                      gpa_t addr, unsigned int len,
1553                                      unsigned long val)
1554 {
1555         mutex_lock(&its->cmd_lock);
1556
1557         /*
1558          * It is UNPREDICTABLE to enable the ITS if any of the CBASER or
1559          * device/collection BASER are invalid
1560          */
1561         if (!its->enabled && (val & GITS_CTLR_ENABLE) &&
1562                 (!(its->baser_device_table & GITS_BASER_VALID) ||
1563                  !(its->baser_coll_table & GITS_BASER_VALID) ||
1564                  !(its->cbaser & GITS_CBASER_VALID)))
1565                 goto out;
1566
1567         its->enabled = !!(val & GITS_CTLR_ENABLE);
1568
1569         /*
1570          * Try to process any pending commands. This function bails out early
1571          * if the ITS is disabled or no commands have been queued.
1572          */
1573         vgic_its_process_commands(kvm, its);
1574
1575 out:
1576         mutex_unlock(&its->cmd_lock);
1577 }
1578
1579 #define REGISTER_ITS_DESC(off, rd, wr, length, acc)             \
1580 {                                                               \
1581         .reg_offset = off,                                      \
1582         .len = length,                                          \
1583         .access_flags = acc,                                    \
1584         .its_read = rd,                                         \
1585         .its_write = wr,                                        \
1586 }
1587
1588 #define REGISTER_ITS_DESC_UACCESS(off, rd, wr, uwr, length, acc)\
1589 {                                                               \
1590         .reg_offset = off,                                      \
1591         .len = length,                                          \
1592         .access_flags = acc,                                    \
1593         .its_read = rd,                                         \
1594         .its_write = wr,                                        \
1595         .uaccess_its_write = uwr,                               \
1596 }
1597
1598 static void its_mmio_write_wi(struct kvm *kvm, struct vgic_its *its,
1599                               gpa_t addr, unsigned int len, unsigned long val)
1600 {
1601         /* Ignore */
1602 }
1603
1604 static struct vgic_register_region its_registers[] = {
1605         REGISTER_ITS_DESC(GITS_CTLR,
1606                 vgic_mmio_read_its_ctlr, vgic_mmio_write_its_ctlr, 4,
1607                 VGIC_ACCESS_32bit),
1608         REGISTER_ITS_DESC_UACCESS(GITS_IIDR,
1609                 vgic_mmio_read_its_iidr, its_mmio_write_wi,
1610                 vgic_mmio_uaccess_write_its_iidr, 4,
1611                 VGIC_ACCESS_32bit),
1612         REGISTER_ITS_DESC(GITS_TYPER,
1613                 vgic_mmio_read_its_typer, its_mmio_write_wi, 8,
1614                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1615         REGISTER_ITS_DESC(GITS_CBASER,
1616                 vgic_mmio_read_its_cbaser, vgic_mmio_write_its_cbaser, 8,
1617                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1618         REGISTER_ITS_DESC(GITS_CWRITER,
1619                 vgic_mmio_read_its_cwriter, vgic_mmio_write_its_cwriter, 8,
1620                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1621         REGISTER_ITS_DESC_UACCESS(GITS_CREADR,
1622                 vgic_mmio_read_its_creadr, its_mmio_write_wi,
1623                 vgic_mmio_uaccess_write_its_creadr, 8,
1624                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1625         REGISTER_ITS_DESC(GITS_BASER,
1626                 vgic_mmio_read_its_baser, vgic_mmio_write_its_baser, 0x40,
1627                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1628         REGISTER_ITS_DESC(GITS_IDREGS_BASE,
1629                 vgic_mmio_read_its_idregs, its_mmio_write_wi, 0x30,
1630                 VGIC_ACCESS_32bit),
1631 };
1632
1633 /* This is called on setting the LPI enable bit in the redistributor. */
1634 void vgic_enable_lpis(struct kvm_vcpu *vcpu)
1635 {
1636         if (!(vcpu->arch.vgic_cpu.pendbaser & GICR_PENDBASER_PTZ))
1637                 its_sync_lpi_pending_table(vcpu);
1638 }
1639
1640 static int vgic_register_its_iodev(struct kvm *kvm, struct vgic_its *its,
1641                                    u64 addr)
1642 {
1643         struct vgic_io_device *iodev = &its->iodev;
1644         int ret;
1645
1646         mutex_lock(&kvm->slots_lock);
1647         if (!IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {
1648                 ret = -EBUSY;
1649                 goto out;
1650         }
1651
1652         its->vgic_its_base = addr;
1653         iodev->regions = its_registers;
1654         iodev->nr_regions = ARRAY_SIZE(its_registers);
1655         kvm_iodevice_init(&iodev->dev, &kvm_io_gic_ops);
1656
1657         iodev->base_addr = its->vgic_its_base;
1658         iodev->iodev_type = IODEV_ITS;
1659         iodev->its = its;
1660         ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, iodev->base_addr,
1661                                       KVM_VGIC_V3_ITS_SIZE, &iodev->dev);
1662 out:
1663         mutex_unlock(&kvm->slots_lock);
1664
1665         return ret;
1666 }
1667
1668 #define INITIAL_BASER_VALUE                                               \
1669         (GIC_BASER_CACHEABILITY(GITS_BASER, INNER, RaWb)                | \
1670          GIC_BASER_CACHEABILITY(GITS_BASER, OUTER, SameAsInner)         | \
1671          GIC_BASER_SHAREABILITY(GITS_BASER, InnerShareable)             | \
1672          GITS_BASER_PAGE_SIZE_64K)
1673
1674 #define INITIAL_PROPBASER_VALUE                                           \
1675         (GIC_BASER_CACHEABILITY(GICR_PROPBASER, INNER, RaWb)            | \
1676          GIC_BASER_CACHEABILITY(GICR_PROPBASER, OUTER, SameAsInner)     | \
1677          GIC_BASER_SHAREABILITY(GICR_PROPBASER, InnerShareable))
1678
1679 static int vgic_its_create(struct kvm_device *dev, u32 type)
1680 {
1681         struct vgic_its *its;
1682
1683         if (type != KVM_DEV_TYPE_ARM_VGIC_ITS)
1684                 return -ENODEV;
1685
1686         its = kzalloc(sizeof(struct vgic_its), GFP_KERNEL);
1687         if (!its)
1688                 return -ENOMEM;
1689
1690         if (vgic_initialized(dev->kvm)) {
1691                 int ret = vgic_v4_init(dev->kvm);
1692                 if (ret < 0) {
1693                         kfree(its);
1694                         return ret;
1695                 }
1696         }
1697
1698         mutex_init(&its->its_lock);
1699         mutex_init(&its->cmd_lock);
1700
1701         its->vgic_its_base = VGIC_ADDR_UNDEF;
1702
1703         INIT_LIST_HEAD(&its->device_list);
1704         INIT_LIST_HEAD(&its->collection_list);
1705
1706         dev->kvm->arch.vgic.msis_require_devid = true;
1707         dev->kvm->arch.vgic.has_its = true;
1708         its->enabled = false;
1709         its->dev = dev;
1710
1711         its->baser_device_table = INITIAL_BASER_VALUE                   |
1712                 ((u64)GITS_BASER_TYPE_DEVICE << GITS_BASER_TYPE_SHIFT);
1713         its->baser_coll_table = INITIAL_BASER_VALUE |
1714                 ((u64)GITS_BASER_TYPE_COLLECTION << GITS_BASER_TYPE_SHIFT);
1715         dev->kvm->arch.vgic.propbaser = INITIAL_PROPBASER_VALUE;
1716
1717         dev->private = its;
1718
1719         return vgic_its_set_abi(its, NR_ITS_ABIS - 1);
1720 }
1721
1722 static void vgic_its_destroy(struct kvm_device *kvm_dev)
1723 {
1724         struct kvm *kvm = kvm_dev->kvm;
1725         struct vgic_its *its = kvm_dev->private;
1726
1727         mutex_lock(&its->its_lock);
1728
1729         vgic_its_free_device_list(kvm, its);
1730         vgic_its_free_collection_list(kvm, its);
1731
1732         mutex_unlock(&its->its_lock);
1733         kfree(its);
1734 }
1735
1736 int vgic_its_has_attr_regs(struct kvm_device *dev,
1737                            struct kvm_device_attr *attr)
1738 {
1739         const struct vgic_register_region *region;
1740         gpa_t offset = attr->attr;
1741         int align;
1742
1743         align = (offset < GITS_TYPER) || (offset >= GITS_PIDR4) ? 0x3 : 0x7;
1744
1745         if (offset & align)
1746                 return -EINVAL;
1747
1748         region = vgic_find_mmio_region(its_registers,
1749                                        ARRAY_SIZE(its_registers),
1750                                        offset);
1751         if (!region)
1752                 return -ENXIO;
1753
1754         return 0;
1755 }
1756
1757 int vgic_its_attr_regs_access(struct kvm_device *dev,
1758                               struct kvm_device_attr *attr,
1759                               u64 *reg, bool is_write)
1760 {
1761         const struct vgic_register_region *region;
1762         struct vgic_its *its;
1763         gpa_t addr, offset;
1764         unsigned int len;
1765         int align, ret = 0;
1766
1767         its = dev->private;
1768         offset = attr->attr;
1769
1770         /*
1771          * Although the spec supports upper/lower 32-bit accesses to
1772          * 64-bit ITS registers, the userspace ABI requires 64-bit
1773          * accesses to all 64-bit wide registers. We therefore only
1774          * support 32-bit accesses to GITS_CTLR, GITS_IIDR and GITS ID
1775          * registers
1776          */
1777         if ((offset < GITS_TYPER) || (offset >= GITS_PIDR4))
1778                 align = 0x3;
1779         else
1780                 align = 0x7;
1781
1782         if (offset & align)
1783                 return -EINVAL;
1784
1785         mutex_lock(&dev->kvm->lock);
1786
1787         if (IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {
1788                 ret = -ENXIO;
1789                 goto out;
1790         }
1791
1792         region = vgic_find_mmio_region(its_registers,
1793                                        ARRAY_SIZE(its_registers),
1794                                        offset);
1795         if (!region) {
1796                 ret = -ENXIO;
1797                 goto out;
1798         }
1799
1800         if (!lock_all_vcpus(dev->kvm)) {
1801                 ret = -EBUSY;
1802                 goto out;
1803         }
1804
1805         addr = its->vgic_its_base + offset;
1806
1807         len = region->access_flags & VGIC_ACCESS_64bit ? 8 : 4;
1808
1809         if (is_write) {
1810                 if (region->uaccess_its_write)
1811                         ret = region->uaccess_its_write(dev->kvm, its, addr,
1812                                                         len, *reg);
1813                 else
1814                         region->its_write(dev->kvm, its, addr, len, *reg);
1815         } else {
1816                 *reg = region->its_read(dev->kvm, its, addr, len);
1817         }
1818         unlock_all_vcpus(dev->kvm);
1819 out:
1820         mutex_unlock(&dev->kvm->lock);
1821         return ret;
1822 }
1823
1824 static u32 compute_next_devid_offset(struct list_head *h,
1825                                      struct its_device *dev)
1826 {
1827         struct its_device *next;
1828         u32 next_offset;
1829
1830         if (list_is_last(&dev->dev_list, h))
1831                 return 0;
1832         next = list_next_entry(dev, dev_list);
1833         next_offset = next->device_id - dev->device_id;
1834
1835         return min_t(u32, next_offset, VITS_DTE_MAX_DEVID_OFFSET);
1836 }
1837
1838 static u32 compute_next_eventid_offset(struct list_head *h, struct its_ite *ite)
1839 {
1840         struct its_ite *next;
1841         u32 next_offset;
1842
1843         if (list_is_last(&ite->ite_list, h))
1844                 return 0;
1845         next = list_next_entry(ite, ite_list);
1846         next_offset = next->event_id - ite->event_id;
1847
1848         return min_t(u32, next_offset, VITS_ITE_MAX_EVENTID_OFFSET);
1849 }
1850
1851 /**
1852  * entry_fn_t - Callback called on a table entry restore path
1853  * @its: its handle
1854  * @id: id of the entry
1855  * @entry: pointer to the entry
1856  * @opaque: pointer to an opaque data
1857  *
1858  * Return: < 0 on error, 0 if last element was identified, id offset to next
1859  * element otherwise
1860  */
1861 typedef int (*entry_fn_t)(struct vgic_its *its, u32 id, void *entry,
1862                           void *opaque);
1863
1864 /**
1865  * scan_its_table - Scan a contiguous table in guest RAM and applies a function
1866  * to each entry
1867  *
1868  * @its: its handle
1869  * @base: base gpa of the table
1870  * @size: size of the table in bytes
1871  * @esz: entry size in bytes
1872  * @start_id: the ID of the first entry in the table
1873  * (non zero for 2d level tables)
1874  * @fn: function to apply on each entry
1875  *
1876  * Return: < 0 on error, 0 if last element was identified, 1 otherwise
1877  * (the last element may not be found on second level tables)
1878  */
1879 static int scan_its_table(struct vgic_its *its, gpa_t base, int size, int esz,
1880                           int start_id, entry_fn_t fn, void *opaque)
1881 {
1882         struct kvm *kvm = its->dev->kvm;
1883         unsigned long len = size;
1884         int id = start_id;
1885         gpa_t gpa = base;
1886         char entry[esz];
1887         int ret;
1888
1889         memset(entry, 0, esz);
1890
1891         while (len > 0) {
1892                 int next_offset;
1893                 size_t byte_offset;
1894
1895                 ret = kvm_read_guest(kvm, gpa, entry, esz);
1896                 if (ret)
1897                         return ret;
1898
1899                 next_offset = fn(its, id, entry, opaque);
1900                 if (next_offset <= 0)
1901                         return next_offset;
1902
1903                 byte_offset = next_offset * esz;
1904                 id += next_offset;
1905                 gpa += byte_offset;
1906                 len -= byte_offset;
1907         }
1908         return 1;
1909 }
1910
1911 /**
1912  * vgic_its_save_ite - Save an interrupt translation entry at @gpa
1913  */
1914 static int vgic_its_save_ite(struct vgic_its *its, struct its_device *dev,
1915                               struct its_ite *ite, gpa_t gpa, int ite_esz)
1916 {
1917         struct kvm *kvm = its->dev->kvm;
1918         u32 next_offset;
1919         u64 val;
1920
1921         next_offset = compute_next_eventid_offset(&dev->itt_head, ite);
1922         val = ((u64)next_offset << KVM_ITS_ITE_NEXT_SHIFT) |
1923                ((u64)ite->irq->intid << KVM_ITS_ITE_PINTID_SHIFT) |
1924                 ite->collection->collection_id;
1925         val = cpu_to_le64(val);
1926         return kvm_write_guest(kvm, gpa, &val, ite_esz);
1927 }
1928
1929 /**
1930  * vgic_its_restore_ite - restore an interrupt translation entry
1931  * @event_id: id used for indexing
1932  * @ptr: pointer to the ITE entry
1933  * @opaque: pointer to the its_device
1934  */
1935 static int vgic_its_restore_ite(struct vgic_its *its, u32 event_id,
1936                                 void *ptr, void *opaque)
1937 {
1938         struct its_device *dev = (struct its_device *)opaque;
1939         struct its_collection *collection;
1940         struct kvm *kvm = its->dev->kvm;
1941         struct kvm_vcpu *vcpu = NULL;
1942         u64 val;
1943         u64 *p = (u64 *)ptr;
1944         struct vgic_irq *irq;
1945         u32 coll_id, lpi_id;
1946         struct its_ite *ite;
1947         u32 offset;
1948
1949         val = *p;
1950
1951         val = le64_to_cpu(val);
1952
1953         coll_id = val & KVM_ITS_ITE_ICID_MASK;
1954         lpi_id = (val & KVM_ITS_ITE_PINTID_MASK) >> KVM_ITS_ITE_PINTID_SHIFT;
1955
1956         if (!lpi_id)
1957                 return 1; /* invalid entry, no choice but to scan next entry */
1958
1959         if (lpi_id < VGIC_MIN_LPI)
1960                 return -EINVAL;
1961
1962         offset = val >> KVM_ITS_ITE_NEXT_SHIFT;
1963         if (event_id + offset >= BIT_ULL(dev->num_eventid_bits))
1964                 return -EINVAL;
1965
1966         collection = find_collection(its, coll_id);
1967         if (!collection)
1968                 return -EINVAL;
1969
1970         ite = vgic_its_alloc_ite(dev, collection, event_id);
1971         if (IS_ERR(ite))
1972                 return PTR_ERR(ite);
1973
1974         if (its_is_collection_mapped(collection))
1975                 vcpu = kvm_get_vcpu(kvm, collection->target_addr);
1976
1977         irq = vgic_add_lpi(kvm, lpi_id, vcpu);
1978         if (IS_ERR(irq))
1979                 return PTR_ERR(irq);
1980         ite->irq = irq;
1981
1982         return offset;
1983 }
1984
1985 static int vgic_its_ite_cmp(void *priv, struct list_head *a,
1986                             struct list_head *b)
1987 {
1988         struct its_ite *itea = container_of(a, struct its_ite, ite_list);
1989         struct its_ite *iteb = container_of(b, struct its_ite, ite_list);
1990
1991         if (itea->event_id < iteb->event_id)
1992                 return -1;
1993         else
1994                 return 1;
1995 }
1996
1997 static int vgic_its_save_itt(struct vgic_its *its, struct its_device *device)
1998 {
1999         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2000         gpa_t base = device->itt_addr;
2001         struct its_ite *ite;
2002         int ret;
2003         int ite_esz = abi->ite_esz;
2004
2005         list_sort(NULL, &device->itt_head, vgic_its_ite_cmp);
2006
2007         list_for_each_entry(ite, &device->itt_head, ite_list) {
2008                 gpa_t gpa = base + ite->event_id * ite_esz;
2009
2010                 /*
2011                  * If an LPI carries the HW bit, this means that this
2012                  * interrupt is controlled by GICv4, and we do not
2013                  * have direct access to that state. Let's simply fail
2014                  * the save operation...
2015                  */
2016                 if (ite->irq->hw)
2017                         return -EACCES;
2018
2019                 ret = vgic_its_save_ite(its, device, ite, gpa, ite_esz);
2020                 if (ret)
2021                         return ret;
2022         }
2023         return 0;
2024 }
2025
2026 /**
2027  * vgic_its_restore_itt - restore the ITT of a device
2028  *
2029  * @its: its handle
2030  * @dev: device handle
2031  *
2032  * Return 0 on success, < 0 on error
2033  */
2034 static int vgic_its_restore_itt(struct vgic_its *its, struct its_device *dev)
2035 {
2036         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2037         gpa_t base = dev->itt_addr;
2038         int ret;
2039         int ite_esz = abi->ite_esz;
2040         size_t max_size = BIT_ULL(dev->num_eventid_bits) * ite_esz;
2041
2042         ret = scan_its_table(its, base, max_size, ite_esz, 0,
2043                              vgic_its_restore_ite, dev);
2044
2045         /* scan_its_table returns +1 if all ITEs are invalid */
2046         if (ret > 0)
2047                 ret = 0;
2048
2049         return ret;
2050 }
2051
2052 /**
2053  * vgic_its_save_dte - Save a device table entry at a given GPA
2054  *
2055  * @its: ITS handle
2056  * @dev: ITS device
2057  * @ptr: GPA
2058  */
2059 static int vgic_its_save_dte(struct vgic_its *its, struct its_device *dev,
2060                              gpa_t ptr, int dte_esz)
2061 {
2062         struct kvm *kvm = its->dev->kvm;
2063         u64 val, itt_addr_field;
2064         u32 next_offset;
2065
2066         itt_addr_field = dev->itt_addr >> 8;
2067         next_offset = compute_next_devid_offset(&its->device_list, dev);
2068         val = (1ULL << KVM_ITS_DTE_VALID_SHIFT |
2069                ((u64)next_offset << KVM_ITS_DTE_NEXT_SHIFT) |
2070                (itt_addr_field << KVM_ITS_DTE_ITTADDR_SHIFT) |
2071                 (dev->num_eventid_bits - 1));
2072         val = cpu_to_le64(val);
2073         return kvm_write_guest(kvm, ptr, &val, dte_esz);
2074 }
2075
2076 /**
2077  * vgic_its_restore_dte - restore a device table entry
2078  *
2079  * @its: its handle
2080  * @id: device id the DTE corresponds to
2081  * @ptr: kernel VA where the 8 byte DTE is located
2082  * @opaque: unused
2083  *
2084  * Return: < 0 on error, 0 if the dte is the last one, id offset to the
2085  * next dte otherwise
2086  */
2087 static int vgic_its_restore_dte(struct vgic_its *its, u32 id,
2088                                 void *ptr, void *opaque)
2089 {
2090         struct its_device *dev;
2091         gpa_t itt_addr;
2092         u8 num_eventid_bits;
2093         u64 entry = *(u64 *)ptr;
2094         bool valid;
2095         u32 offset;
2096         int ret;
2097
2098         entry = le64_to_cpu(entry);
2099
2100         valid = entry >> KVM_ITS_DTE_VALID_SHIFT;
2101         num_eventid_bits = (entry & KVM_ITS_DTE_SIZE_MASK) + 1;
2102         itt_addr = ((entry & KVM_ITS_DTE_ITTADDR_MASK)
2103                         >> KVM_ITS_DTE_ITTADDR_SHIFT) << 8;
2104
2105         if (!valid)
2106                 return 1;
2107
2108         /* dte entry is valid */
2109         offset = (entry & KVM_ITS_DTE_NEXT_MASK) >> KVM_ITS_DTE_NEXT_SHIFT;
2110
2111         dev = vgic_its_alloc_device(its, id, itt_addr, num_eventid_bits);
2112         if (IS_ERR(dev))
2113                 return PTR_ERR(dev);
2114
2115         ret = vgic_its_restore_itt(its, dev);
2116         if (ret) {
2117                 vgic_its_free_device(its->dev->kvm, dev);
2118                 return ret;
2119         }
2120
2121         return offset;
2122 }
2123
2124 static int vgic_its_device_cmp(void *priv, struct list_head *a,
2125                                struct list_head *b)
2126 {
2127         struct its_device *deva = container_of(a, struct its_device, dev_list);
2128         struct its_device *devb = container_of(b, struct its_device, dev_list);
2129
2130         if (deva->device_id < devb->device_id)
2131                 return -1;
2132         else
2133                 return 1;
2134 }
2135
2136 /**
2137  * vgic_its_save_device_tables - Save the device table and all ITT
2138  * into guest RAM
2139  *
2140  * L1/L2 handling is hidden by vgic_its_check_id() helper which directly
2141  * returns the GPA of the device entry
2142  */
2143 static int vgic_its_save_device_tables(struct vgic_its *its)
2144 {
2145         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2146         u64 baser = its->baser_device_table;
2147         struct its_device *dev;
2148         int dte_esz = abi->dte_esz;
2149
2150         if (!(baser & GITS_BASER_VALID))
2151                 return 0;
2152
2153         list_sort(NULL, &its->device_list, vgic_its_device_cmp);
2154
2155         list_for_each_entry(dev, &its->device_list, dev_list) {
2156                 int ret;
2157                 gpa_t eaddr;
2158
2159                 if (!vgic_its_check_id(its, baser,
2160                                        dev->device_id, &eaddr))
2161                         return -EINVAL;
2162
2163                 ret = vgic_its_save_itt(its, dev);
2164                 if (ret)
2165                         return ret;
2166
2167                 ret = vgic_its_save_dte(its, dev, eaddr, dte_esz);
2168                 if (ret)
2169                         return ret;
2170         }
2171         return 0;
2172 }
2173
2174 /**
2175  * handle_l1_dte - callback used for L1 device table entries (2 stage case)
2176  *
2177  * @its: its handle
2178  * @id: index of the entry in the L1 table
2179  * @addr: kernel VA
2180  * @opaque: unused
2181  *
2182  * L1 table entries are scanned by steps of 1 entry
2183  * Return < 0 if error, 0 if last dte was found when scanning the L2
2184  * table, +1 otherwise (meaning next L1 entry must be scanned)
2185  */
2186 static int handle_l1_dte(struct vgic_its *its, u32 id, void *addr,
2187                          void *opaque)
2188 {
2189         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2190         int l2_start_id = id * (SZ_64K / abi->dte_esz);
2191         u64 entry = *(u64 *)addr;
2192         int dte_esz = abi->dte_esz;
2193         gpa_t gpa;
2194         int ret;
2195
2196         entry = le64_to_cpu(entry);
2197
2198         if (!(entry & KVM_ITS_L1E_VALID_MASK))
2199                 return 1;
2200
2201         gpa = entry & KVM_ITS_L1E_ADDR_MASK;
2202
2203         ret = scan_its_table(its, gpa, SZ_64K, dte_esz,
2204                              l2_start_id, vgic_its_restore_dte, NULL);
2205
2206         return ret;
2207 }
2208
2209 /**
2210  * vgic_its_restore_device_tables - Restore the device table and all ITT
2211  * from guest RAM to internal data structs
2212  */
2213 static int vgic_its_restore_device_tables(struct vgic_its *its)
2214 {
2215         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2216         u64 baser = its->baser_device_table;
2217         int l1_esz, ret;
2218         int l1_tbl_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
2219         gpa_t l1_gpa;
2220
2221         if (!(baser & GITS_BASER_VALID))
2222                 return 0;
2223
2224         l1_gpa = BASER_ADDRESS(baser);
2225
2226         if (baser & GITS_BASER_INDIRECT) {
2227                 l1_esz = GITS_LVL1_ENTRY_SIZE;
2228                 ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0,
2229                                      handle_l1_dte, NULL);
2230         } else {
2231                 l1_esz = abi->dte_esz;
2232                 ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0,
2233                                      vgic_its_restore_dte, NULL);
2234         }
2235
2236         /* scan_its_table returns +1 if all entries are invalid */
2237         if (ret > 0)
2238                 ret = 0;
2239
2240         return ret;
2241 }
2242
2243 static int vgic_its_save_cte(struct vgic_its *its,
2244                              struct its_collection *collection,
2245                              gpa_t gpa, int esz)
2246 {
2247         u64 val;
2248
2249         val = (1ULL << KVM_ITS_CTE_VALID_SHIFT |
2250                ((u64)collection->target_addr << KVM_ITS_CTE_RDBASE_SHIFT) |
2251                collection->collection_id);
2252         val = cpu_to_le64(val);
2253         return kvm_write_guest(its->dev->kvm, gpa, &val, esz);
2254 }
2255
2256 static int vgic_its_restore_cte(struct vgic_its *its, gpa_t gpa, int esz)
2257 {
2258         struct its_collection *collection;
2259         struct kvm *kvm = its->dev->kvm;
2260         u32 target_addr, coll_id;
2261         u64 val;
2262         int ret;
2263
2264         BUG_ON(esz > sizeof(val));
2265         ret = kvm_read_guest(kvm, gpa, &val, esz);
2266         if (ret)
2267                 return ret;
2268         val = le64_to_cpu(val);
2269         if (!(val & KVM_ITS_CTE_VALID_MASK))
2270                 return 0;
2271
2272         target_addr = (u32)(val >> KVM_ITS_CTE_RDBASE_SHIFT);
2273         coll_id = val & KVM_ITS_CTE_ICID_MASK;
2274
2275         if (target_addr >= atomic_read(&kvm->online_vcpus))
2276                 return -EINVAL;
2277
2278         collection = find_collection(its, coll_id);
2279         if (collection)
2280                 return -EEXIST;
2281         ret = vgic_its_alloc_collection(its, &collection, coll_id);
2282         if (ret)
2283                 return ret;
2284         collection->target_addr = target_addr;
2285         return 1;
2286 }
2287
2288 /**
2289  * vgic_its_save_collection_table - Save the collection table into
2290  * guest RAM
2291  */
2292 static int vgic_its_save_collection_table(struct vgic_its *its)
2293 {
2294         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2295         u64 baser = its->baser_coll_table;
2296         gpa_t gpa = BASER_ADDRESS(baser);
2297         struct its_collection *collection;
2298         u64 val;
2299         size_t max_size, filled = 0;
2300         int ret, cte_esz = abi->cte_esz;
2301
2302         if (!(baser & GITS_BASER_VALID))
2303                 return 0;
2304
2305         max_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
2306
2307         list_for_each_entry(collection, &its->collection_list, coll_list) {
2308                 ret = vgic_its_save_cte(its, collection, gpa, cte_esz);
2309                 if (ret)
2310                         return ret;
2311                 gpa += cte_esz;
2312                 filled += cte_esz;
2313         }
2314
2315         if (filled == max_size)
2316                 return 0;
2317
2318         /*
2319          * table is not fully filled, add a last dummy element
2320          * with valid bit unset
2321          */
2322         val = 0;
2323         BUG_ON(cte_esz > sizeof(val));
2324         ret = kvm_write_guest(its->dev->kvm, gpa, &val, cte_esz);
2325         return ret;
2326 }
2327
2328 /**
2329  * vgic_its_restore_collection_table - reads the collection table
2330  * in guest memory and restores the ITS internal state. Requires the
2331  * BASER registers to be restored before.
2332  */
2333 static int vgic_its_restore_collection_table(struct vgic_its *its)
2334 {
2335         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2336         u64 baser = its->baser_coll_table;
2337         int cte_esz = abi->cte_esz;
2338         size_t max_size, read = 0;
2339         gpa_t gpa;
2340         int ret;
2341
2342         if (!(baser & GITS_BASER_VALID))
2343                 return 0;
2344
2345         gpa = BASER_ADDRESS(baser);
2346
2347         max_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
2348
2349         while (read < max_size) {
2350                 ret = vgic_its_restore_cte(its, gpa, cte_esz);
2351                 if (ret <= 0)
2352                         break;
2353                 gpa += cte_esz;
2354                 read += cte_esz;
2355         }
2356
2357         if (ret > 0)
2358                 return 0;
2359
2360         return ret;
2361 }
2362
2363 /**
2364  * vgic_its_save_tables_v0 - Save the ITS tables into guest ARM
2365  * according to v0 ABI
2366  */
2367 static int vgic_its_save_tables_v0(struct vgic_its *its)
2368 {
2369         int ret;
2370
2371         ret = vgic_its_save_device_tables(its);
2372         if (ret)
2373                 return ret;
2374
2375         return vgic_its_save_collection_table(its);
2376 }
2377
2378 /**
2379  * vgic_its_restore_tables_v0 - Restore the ITS tables from guest RAM
2380  * to internal data structs according to V0 ABI
2381  *
2382  */
2383 static int vgic_its_restore_tables_v0(struct vgic_its *its)
2384 {
2385         int ret;
2386
2387         ret = vgic_its_restore_collection_table(its);
2388         if (ret)
2389                 return ret;
2390
2391         return vgic_its_restore_device_tables(its);
2392 }
2393
2394 static int vgic_its_commit_v0(struct vgic_its *its)
2395 {
2396         const struct vgic_its_abi *abi;
2397
2398         abi = vgic_its_get_abi(its);
2399         its->baser_coll_table &= ~GITS_BASER_ENTRY_SIZE_MASK;
2400         its->baser_device_table &= ~GITS_BASER_ENTRY_SIZE_MASK;
2401
2402         its->baser_coll_table |= (GIC_ENCODE_SZ(abi->cte_esz, 5)
2403                                         << GITS_BASER_ENTRY_SIZE_SHIFT);
2404
2405         its->baser_device_table |= (GIC_ENCODE_SZ(abi->dte_esz, 5)
2406                                         << GITS_BASER_ENTRY_SIZE_SHIFT);
2407         return 0;
2408 }
2409
2410 static void vgic_its_reset(struct kvm *kvm, struct vgic_its *its)
2411 {
2412         /* We need to keep the ABI specific field values */
2413         its->baser_coll_table &= ~GITS_BASER_VALID;
2414         its->baser_device_table &= ~GITS_BASER_VALID;
2415         its->cbaser = 0;
2416         its->creadr = 0;
2417         its->cwriter = 0;
2418         its->enabled = 0;
2419         vgic_its_free_device_list(kvm, its);
2420         vgic_its_free_collection_list(kvm, its);
2421 }
2422
2423 static int vgic_its_has_attr(struct kvm_device *dev,
2424                              struct kvm_device_attr *attr)
2425 {
2426         switch (attr->group) {
2427         case KVM_DEV_ARM_VGIC_GRP_ADDR:
2428                 switch (attr->attr) {
2429                 case KVM_VGIC_ITS_ADDR_TYPE:
2430                         return 0;
2431                 }
2432                 break;
2433         case KVM_DEV_ARM_VGIC_GRP_CTRL:
2434                 switch (attr->attr) {
2435                 case KVM_DEV_ARM_VGIC_CTRL_INIT:
2436                         return 0;
2437                 case KVM_DEV_ARM_ITS_CTRL_RESET:
2438                         return 0;
2439                 case KVM_DEV_ARM_ITS_SAVE_TABLES:
2440                         return 0;
2441                 case KVM_DEV_ARM_ITS_RESTORE_TABLES:
2442                         return 0;
2443                 }
2444                 break;
2445         case KVM_DEV_ARM_VGIC_GRP_ITS_REGS:
2446                 return vgic_its_has_attr_regs(dev, attr);
2447         }
2448         return -ENXIO;
2449 }
2450
2451 static int vgic_its_ctrl(struct kvm *kvm, struct vgic_its *its, u64 attr)
2452 {
2453         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2454         int ret = 0;
2455
2456         if (attr == KVM_DEV_ARM_VGIC_CTRL_INIT) /* Nothing to do */
2457                 return 0;
2458
2459         mutex_lock(&kvm->lock);
2460         mutex_lock(&its->its_lock);
2461
2462         if (!lock_all_vcpus(kvm)) {
2463                 mutex_unlock(&its->its_lock);
2464                 mutex_unlock(&kvm->lock);
2465                 return -EBUSY;
2466         }
2467
2468         switch (attr) {
2469         case KVM_DEV_ARM_ITS_CTRL_RESET:
2470                 vgic_its_reset(kvm, its);
2471                 break;
2472         case KVM_DEV_ARM_ITS_SAVE_TABLES:
2473                 ret = abi->save_tables(its);
2474                 break;
2475         case KVM_DEV_ARM_ITS_RESTORE_TABLES:
2476                 ret = abi->restore_tables(its);
2477                 break;
2478         }
2479
2480         unlock_all_vcpus(kvm);
2481         mutex_unlock(&its->its_lock);
2482         mutex_unlock(&kvm->lock);
2483         return ret;
2484 }
2485
2486 static int vgic_its_set_attr(struct kvm_device *dev,
2487                              struct kvm_device_attr *attr)
2488 {
2489         struct vgic_its *its = dev->private;
2490         int ret;
2491
2492         switch (attr->group) {
2493         case KVM_DEV_ARM_VGIC_GRP_ADDR: {
2494                 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2495                 unsigned long type = (unsigned long)attr->attr;
2496                 u64 addr;
2497
2498                 if (type != KVM_VGIC_ITS_ADDR_TYPE)
2499                         return -ENODEV;
2500
2501                 if (copy_from_user(&addr, uaddr, sizeof(addr)))
2502                         return -EFAULT;
2503
2504                 ret = vgic_check_ioaddr(dev->kvm, &its->vgic_its_base,
2505                                         addr, SZ_64K);
2506                 if (ret)
2507                         return ret;
2508
2509                 return vgic_register_its_iodev(dev->kvm, its, addr);
2510         }
2511         case KVM_DEV_ARM_VGIC_GRP_CTRL:
2512                 return vgic_its_ctrl(dev->kvm, its, attr->attr);
2513         case KVM_DEV_ARM_VGIC_GRP_ITS_REGS: {
2514                 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2515                 u64 reg;
2516
2517                 if (get_user(reg, uaddr))
2518                         return -EFAULT;
2519
2520                 return vgic_its_attr_regs_access(dev, attr, &reg, true);
2521         }
2522         }
2523         return -ENXIO;
2524 }
2525
2526 static int vgic_its_get_attr(struct kvm_device *dev,
2527                              struct kvm_device_attr *attr)
2528 {
2529         switch (attr->group) {
2530         case KVM_DEV_ARM_VGIC_GRP_ADDR: {
2531                 struct vgic_its *its = dev->private;
2532                 u64 addr = its->vgic_its_base;
2533                 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2534                 unsigned long type = (unsigned long)attr->attr;
2535
2536                 if (type != KVM_VGIC_ITS_ADDR_TYPE)
2537                         return -ENODEV;
2538
2539                 if (copy_to_user(uaddr, &addr, sizeof(addr)))
2540                         return -EFAULT;
2541                 break;
2542         }
2543         case KVM_DEV_ARM_VGIC_GRP_ITS_REGS: {
2544                 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2545                 u64 reg;
2546                 int ret;
2547
2548                 ret = vgic_its_attr_regs_access(dev, attr, &reg, false);
2549                 if (ret)
2550                         return ret;
2551                 return put_user(reg, uaddr);
2552         }
2553         default:
2554                 return -ENXIO;
2555         }
2556
2557         return 0;
2558 }
2559
2560 static struct kvm_device_ops kvm_arm_vgic_its_ops = {
2561         .name = "kvm-arm-vgic-its",
2562         .create = vgic_its_create,
2563         .destroy = vgic_its_destroy,
2564         .set_attr = vgic_its_set_attr,
2565         .get_attr = vgic_its_get_attr,
2566         .has_attr = vgic_its_has_attr,
2567 };
2568
2569 int kvm_vgic_register_its_device(void)
2570 {
2571         return kvm_register_device_ops(&kvm_arm_vgic_its_ops,
2572                                        KVM_DEV_TYPE_ARM_VGIC_ITS);
2573 }