virt/kvm/arm/vgic/vgic-its.c

   1 /*
   2  * GICv3 ITS emulation
   3  *
   4  * Copyright (C) 2015,2016 ARM Ltd.
   5  * Author: Andre Przywara <andre.przywara@arm.com>
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 #include <linux/cpu.h>
  21 #include <linux/kvm.h>
  22 #include <linux/kvm_host.h>
  23 #include <linux/interrupt.h>
  24 #include <linux/list.h>
  25 #include <linux/uaccess.h>
  26 #include <linux/list_sort.h>
  27
  28 #include <linux/irqchip/arm-gic-v3.h>
  29
  30 #include <asm/kvm_emulate.h>
  31 #include <asm/kvm_arm.h>
  32 #include <asm/kvm_mmu.h>
  33
  34 #include "vgic.h"
  35 #include "vgic-mmio.h"
  36
  37 static int vgic_its_save_tables_v0(struct vgic_its *its);
  38 static int vgic_its_restore_tables_v0(struct vgic_its *its);
  39 static int vgic_its_commit_v0(struct vgic_its *its);
  40 static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq,
  41                              struct kvm_vcpu *filter_vcpu);
  42
  43 /*
  44  * Creates a new (reference to a) struct vgic_irq for a given LPI.
  45  * If this LPI is already mapped on another ITS, we increase its refcount
  46  * and return a pointer to the existing structure.
  47  * If this is a "new" LPI, we allocate and initialize a new struct vgic_irq.
  48  * This function returns a pointer to the _unlocked_ structure.
  49  */
  50 static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid,
  51                                      struct kvm_vcpu *vcpu)
  52 {
  53         struct vgic_dist *dist = &kvm->arch.vgic;
  54         struct vgic_irq *irq = vgic_get_irq(kvm, NULL, intid), *oldirq;
  55         int ret;
  56
  57         /* In this case there is no put, since we keep the reference. */
  58         if (irq)
  59                 return irq;
  60
  61         irq = kzalloc(sizeof(struct vgic_irq), GFP_KERNEL);
  62         if (!irq)
  63                 return ERR_PTR(-ENOMEM);
  64
  65         INIT_LIST_HEAD(&irq->lpi_list);
  66         INIT_LIST_HEAD(&irq->ap_list);
  67         spin_lock_init(&irq->irq_lock);
  68
  69         irq->config = VGIC_CONFIG_EDGE;
  70         kref_init(&irq->refcount);
  71         irq->intid = intid;
  72         irq->target_vcpu = vcpu;
  73
  74         spin_lock(&dist->lpi_list_lock);
  75
  76         /*
  77          * There could be a race with another vgic_add_lpi(), so we need to
  78          * check that we don't add a second list entry with the same LPI.
  79          */
  80         list_for_each_entry(oldirq, &dist->lpi_list_head, lpi_list) {
  81                 if (oldirq->intid != intid)
  82                         continue;
  83
  84                 /* Someone was faster with adding this LPI, lets use that. */
  85                 kfree(irq);
  86                 irq = oldirq;
  87
  88                 /*
  89                  * This increases the refcount, the caller is expected to
  90                  * call vgic_put_irq() on the returned pointer once it's
  91                  * finished with the IRQ.
  92                  */
  93                 vgic_get_irq_kref(irq);
  94
  95                 goto out_unlock;
  96         }
  97
  98         list_add_tail(&irq->lpi_list, &dist->lpi_list_head);
  99         dist->lpi_list_count++;
 100
 101 out_unlock:
 102         spin_unlock(&dist->lpi_list_lock);
 103
 104         /*
 105          * We "cache" the configuration table entries in our struct vgic_irq's.
 106          * However we only have those structs for mapped IRQs, so we read in
 107          * the respective config data from memory here upon mapping the LPI.
 108          */
 109         ret = update_lpi_config(kvm, irq, NULL);
 110         if (ret)
 111                 return ERR_PTR(ret);
 112
 113         ret = vgic_v3_lpi_sync_pending_status(kvm, irq);
 114         if (ret)
 115                 return ERR_PTR(ret);
 116
 117         return irq;
 118 }
 119
 120 struct its_device {
 121         struct list_head dev_list;
 122
 123         /* the head for the list of ITTEs */
 124         struct list_head itt_head;
 125         u32 num_eventid_bits;
 126         gpa_t itt_addr;
 127         u32 device_id;
 128 };
 129
 130 #define COLLECTION_NOT_MAPPED ((u32)~0)
 131
 132 struct its_collection {
 133         struct list_head coll_list;
 134
 135         u32 collection_id;
 136         u32 target_addr;
 137 };
 138
 139 #define its_is_collection_mapped(coll) ((coll) && \
 140                                 ((coll)->target_addr != COLLECTION_NOT_MAPPED))
 141
 142 struct its_ite {
 143         struct list_head ite_list;
 144
 145         struct vgic_irq *irq;
 146         struct its_collection *collection;
 147         u32 event_id;
 148 };
 149
 150 /**
 151  * struct vgic_its_abi - ITS abi ops and settings
 152  * @cte_esz: collection table entry size
 153  * @dte_esz: device table entry size
 154  * @ite_esz: interrupt translation table entry size
 155  * @save tables: save the ITS tables into guest RAM
 156  * @restore_tables: restore the ITS internal structs from tables
 157  *  stored in guest RAM
 158  * @commit: initialize the registers which expose the ABI settings,
 159  *  especially the entry sizes
 160  */
 161 struct vgic_its_abi {
 162         int cte_esz;
 163         int dte_esz;
 164         int ite_esz;
 165         int (*save_tables)(struct vgic_its *its);
 166         int (*restore_tables)(struct vgic_its *its);
 167         int (*commit)(struct vgic_its *its);
 168 };
 169
 170 static const struct vgic_its_abi its_table_abi_versions[] = {
 171         [0] = {.cte_esz = 8, .dte_esz = 8, .ite_esz = 8,
 172          .save_tables = vgic_its_save_tables_v0,
 173          .restore_tables = vgic_its_restore_tables_v0,
 174          .commit = vgic_its_commit_v0,
 175         },
 176 };
 177
 178 #define NR_ITS_ABIS     ARRAY_SIZE(its_table_abi_versions)
 179
 180 inline const struct vgic_its_abi *vgic_its_get_abi(struct vgic_its *its)
 181 {
 182         return &its_table_abi_versions[its->abi_rev];
 183 }
 184
 185 int vgic_its_set_abi(struct vgic_its *its, int rev)
 186 {
 187         const struct vgic_its_abi *abi;
 188
 189         its->abi_rev = rev;
 190         abi = vgic_its_get_abi(its);
 191         return abi->commit(its);
 192 }
 193
 194 /*
 195  * Find and returns a device in the device table for an ITS.
 196  * Must be called with the its_lock mutex held.
 197  */
 198 static struct its_device *find_its_device(struct vgic_its *its, u32 device_id)
 199 {
 200         struct its_device *device;
 201
 202         list_for_each_entry(device, &its->device_list, dev_list)
 203                 if (device_id == device->device_id)
 204                         return device;
 205
 206         return NULL;
 207 }
 208
 209 /*
 210  * Find and returns an interrupt translation table entry (ITTE) for a given
 211  * Device ID/Event ID pair on an ITS.
 212  * Must be called with the its_lock mutex held.
 213  */
 214 static struct its_ite *find_ite(struct vgic_its *its, u32 device_id,
 215                                   u32 event_id)
 216 {
 217         struct its_device *device;
 218         struct its_ite *ite;
 219
 220         device = find_its_device(its, device_id);
 221         if (device == NULL)
 222                 return NULL;
 223
 224         list_for_each_entry(ite, &device->itt_head, ite_list)
 225                 if (ite->event_id == event_id)
 226                         return ite;
 227
 228         return NULL;
 229 }
 230
 231 /* To be used as an iterator this macro misses the enclosing parentheses */
 232 #define for_each_lpi_its(dev, ite, its) \
 233         list_for_each_entry(dev, &(its)->device_list, dev_list) \
 234                 list_for_each_entry(ite, &(dev)->itt_head, ite_list)
 235
 236 /*
 237  * We only implement 48 bits of PA at the moment, although the ITS
 238  * supports more. Let's be restrictive here.
 239  */
 240 #define BASER_ADDRESS(x)        ((x) & GENMASK_ULL(47, 16))
 241 #define CBASER_ADDRESS(x)       ((x) & GENMASK_ULL(47, 12))
 242
 243 #define GIC_LPI_OFFSET 8192
 244
 245 #define VITS_TYPER_IDBITS 16
 246 #define VITS_TYPER_DEVBITS 16
 247 #define VITS_DTE_MAX_DEVID_OFFSET       (BIT(14) - 1)
 248 #define VITS_ITE_MAX_EVENTID_OFFSET     (BIT(16) - 1)
 249
 250 /*
 251  * Finds and returns a collection in the ITS collection table.
 252  * Must be called with the its_lock mutex held.
 253  */
 254 static struct its_collection *find_collection(struct vgic_its *its, int coll_id)
 255 {
 256         struct its_collection *collection;
 257
 258         list_for_each_entry(collection, &its->collection_list, coll_list) {
 259                 if (coll_id == collection->collection_id)
 260                         return collection;
 261         }
 262
 263         return NULL;
 264 }
 265
 266 #define LPI_PROP_ENABLE_BIT(p)  ((p) & LPI_PROP_ENABLED)
 267 #define LPI_PROP_PRIORITY(p)    ((p) & 0xfc)
 268
 269 /*
 270  * Reads the configuration data for a given LPI from guest memory and
 271  * updates the fields in struct vgic_irq.
 272  * If filter_vcpu is not NULL, applies only if the IRQ is targeting this
 273  * VCPU. Unconditionally applies if filter_vcpu is NULL.
 274  */
 275 static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq,
 276                              struct kvm_vcpu *filter_vcpu)
 277 {
 278         u64 propbase = GICR_PROPBASER_ADDRESS(kvm->arch.vgic.propbaser);
 279         u8 prop;
 280         int ret;
 281         unsigned long flags;
 282
 283         ret = kvm_read_guest(kvm, propbase + irq->intid - GIC_LPI_OFFSET,
 284                              &prop, 1);
 285
 286         if (ret)
 287                 return ret;
 288
 289         spin_lock_irqsave(&irq->irq_lock, flags);
 290
 291         if (!filter_vcpu || filter_vcpu == irq->target_vcpu) {
 292                 irq->priority = LPI_PROP_PRIORITY(prop);
 293                 irq->enabled = LPI_PROP_ENABLE_BIT(prop);
 294
 295                 vgic_queue_irq_unlock(kvm, irq, flags);
 296         } else {
 297                 spin_unlock_irqrestore(&irq->irq_lock, flags);
 298         }
 299
 300         return 0;
 301 }
 302
 303 /*
 304  * Create a snapshot of the current LPIs targeting @vcpu, so that we can
 305  * enumerate those LPIs without holding any lock.
 306  * Returns their number and puts the kmalloc'ed array into intid_ptr.
 307  */
 308 static int vgic_copy_lpi_list(struct kvm_vcpu *vcpu, u32 **intid_ptr)
 309 {
 310         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 311         struct vgic_irq *irq;
 312         u32 *intids;
 313         int irq_count = dist->lpi_list_count, i = 0;
 314
 315         /*
 316          * We use the current value of the list length, which may change
 317          * after the kmalloc. We don't care, because the guest shouldn't
 318          * change anything while the command handling is still running,
 319          * and in the worst case we would miss a new IRQ, which one wouldn't
 320          * expect to be covered by this command anyway.
 321          */
 322         intids = kmalloc_array(irq_count, sizeof(intids[0]), GFP_KERNEL);
 323         if (!intids)
 324                 return -ENOMEM;
 325
 326         spin_lock(&dist->lpi_list_lock);
 327         list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) {
 328                 /* We don't need to "get" the IRQ, as we hold the list lock. */
 329                 if (irq->target_vcpu != vcpu)
 330                         continue;
 331                 intids[i++] = irq->intid;
 332         }
 333         spin_unlock(&dist->lpi_list_lock);
 334
 335         *intid_ptr = intids;
 336         return i;
 337 }
 338
 339 /*
 340  * Promotes the ITS view of affinity of an ITTE (which redistributor this LPI
 341  * is targeting) to the VGIC's view, which deals with target VCPUs.
 342  * Needs to be called whenever either the collection for a LPIs has
 343  * changed or the collection itself got retargeted.
 344  */
 345 static void update_affinity_ite(struct kvm *kvm, struct its_ite *ite)
 346 {
 347         struct kvm_vcpu *vcpu;
 348
 349         if (!its_is_collection_mapped(ite->collection))
 350                 return;
 351
 352         vcpu = kvm_get_vcpu(kvm, ite->collection->target_addr);
 353
 354         spin_lock(&ite->irq->irq_lock);
 355         ite->irq->target_vcpu = vcpu;
 356         spin_unlock(&ite->irq->irq_lock);
 357 }
 358
 359 /*
 360  * Updates the target VCPU for every LPI targeting this collection.
 361  * Must be called with the its_lock mutex held.
 362  */
 363 static void update_affinity_collection(struct kvm *kvm, struct vgic_its *its,
 364                                        struct its_collection *coll)
 365 {
 366         struct its_device *device;
 367         struct its_ite *ite;
 368
 369         for_each_lpi_its(device, ite, its) {
 370                 if (!ite->collection || coll != ite->collection)
 371                         continue;
 372
 373                 update_affinity_ite(kvm, ite);
 374         }
 375 }
 376
 377 static u32 max_lpis_propbaser(u64 propbaser)
 378 {
 379         int nr_idbits = (propbaser & 0x1f) + 1;
 380
 381         return 1U << min(nr_idbits, INTERRUPT_ID_BITS_ITS);
 382 }
 383
 384 /*
 385  * Sync the pending table pending bit of LPIs targeting @vcpu
 386  * with our own data structures. This relies on the LPI being
 387  * mapped before.
 388  */
 389 static int its_sync_lpi_pending_table(struct kvm_vcpu *vcpu)
 390 {
 391         gpa_t pendbase = GICR_PENDBASER_ADDRESS(vcpu->arch.vgic_cpu.pendbaser);
 392         struct vgic_irq *irq;
 393         int last_byte_offset = -1;
 394         int ret = 0;
 395         u32 *intids;
 396         int nr_irqs, i;
 397         unsigned long flags;
 398
 399         nr_irqs = vgic_copy_lpi_list(vcpu, &intids);
 400         if (nr_irqs < 0)
 401                 return nr_irqs;
 402
 403         for (i = 0; i < nr_irqs; i++) {
 404                 int byte_offset, bit_nr;
 405                 u8 pendmask;
 406
 407                 byte_offset = intids[i] / BITS_PER_BYTE;
 408                 bit_nr = intids[i] % BITS_PER_BYTE;
 409
 410                 /*
 411                  * For contiguously allocated LPIs chances are we just read
 412                  * this very same byte in the last iteration. Reuse that.
 413                  */
 414                 if (byte_offset != last_byte_offset) {
 415                         ret = kvm_read_guest(vcpu->kvm, pendbase + byte_offset,
 416                                              &pendmask, 1);
 417                         if (ret) {
 418                                 kfree(intids);
 419                                 return ret;
 420                         }
 421                         last_byte_offset = byte_offset;
 422                 }
 423
 424                 irq = vgic_get_irq(vcpu->kvm, NULL, intids[i]);
 425                 spin_lock_irqsave(&irq->irq_lock, flags);
 426                 irq->pending_latch = pendmask & (1U << bit_nr);
 427                 vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
 428                 vgic_put_irq(vcpu->kvm, irq);
 429         }
 430
 431         kfree(intids);
 432
 433         return ret;
 434 }
 435
 436 static unsigned long vgic_mmio_read_its_typer(struct kvm *kvm,
 437                                               struct vgic_its *its,
 438                                               gpa_t addr, unsigned int len)
 439 {
 440         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
 441         u64 reg = GITS_TYPER_PLPIS;
 442
 443         /*
 444          * We use linear CPU numbers for redistributor addressing,
 445          * so GITS_TYPER.PTA is 0.
 446          * Also we force all PROPBASER registers to be the same, so
 447          * CommonLPIAff is 0 as well.
 448          * To avoid memory waste in the guest, we keep the number of IDBits and
 449          * DevBits low - as least for the time being.
 450          */
 451         reg |= GIC_ENCODE_SZ(VITS_TYPER_DEVBITS, 5) << GITS_TYPER_DEVBITS_SHIFT;
 452         reg |= GIC_ENCODE_SZ(VITS_TYPER_IDBITS, 5) << GITS_TYPER_IDBITS_SHIFT;
 453         reg |= GIC_ENCODE_SZ(abi->ite_esz, 4) << GITS_TYPER_ITT_ENTRY_SIZE_SHIFT;
 454
 455         return extract_bytes(reg, addr & 7, len);
 456 }
 457
 458 static unsigned long vgic_mmio_read_its_iidr(struct kvm *kvm,
 459                                              struct vgic_its *its,
 460                                              gpa_t addr, unsigned int len)
 461 {
 462         u32 val;
 463
 464         val = (its->abi_rev << GITS_IIDR_REV_SHIFT) & GITS_IIDR_REV_MASK;
 465         val |= (PRODUCT_ID_KVM << GITS_IIDR_PRODUCTID_SHIFT) | IMPLEMENTER_ARM;
 466         return val;
 467 }
 468
 469 static int vgic_mmio_uaccess_write_its_iidr(struct kvm *kvm,
 470                                             struct vgic_its *its,
 471                                             gpa_t addr, unsigned int len,
 472                                             unsigned long val)
 473 {
 474         u32 rev = GITS_IIDR_REV(val);
 475
 476         if (rev >= NR_ITS_ABIS)
 477                 return -EINVAL;
 478         return vgic_its_set_abi(its, rev);
 479 }
 480
 481 static unsigned long vgic_mmio_read_its_idregs(struct kvm *kvm,
 482                                                struct vgic_its *its,
 483                                                gpa_t addr, unsigned int len)
 484 {
 485         switch (addr & 0xffff) {
 486         case GITS_PIDR0:
 487                 return 0x92;    /* part number, bits[7:0] */
 488         case GITS_PIDR1:
 489                 return 0xb4;    /* part number, bits[11:8] */
 490         case GITS_PIDR2:
 491                 return GIC_PIDR2_ARCH_GICv3 | 0x0b;
 492         case GITS_PIDR4:
 493                 return 0x40;    /* This is a 64K software visible page */
 494         /* The following are the ID registers for (any) GIC. */
 495         case GITS_CIDR0:
 496                 return 0x0d;
 497         case GITS_CIDR1:
 498                 return 0xf0;
 499         case GITS_CIDR2:
 500                 return 0x05;
 501         case GITS_CIDR3:
 502                 return 0xb1;
 503         }
 504
 505         return 0;
 506 }
 507
 508 /*
 509  * Find the target VCPU and the LPI number for a given devid/eventid pair
 510  * and make this IRQ pending, possibly injecting it.
 511  * Must be called with the its_lock mutex held.
 512  * Returns 0 on success, a positive error value for any ITS mapping
 513  * related errors and negative error values for generic errors.
 514  */
 515 static int vgic_its_trigger_msi(struct kvm *kvm, struct vgic_its *its,
 516                                 u32 devid, u32 eventid)
 517 {
 518         struct kvm_vcpu *vcpu;
 519         struct its_ite *ite;
 520         unsigned long flags;
 521
 522         if (!its->enabled)
 523                 return -EBUSY;
 524
 525         ite = find_ite(its, devid, eventid);
 526         if (!ite || !its_is_collection_mapped(ite->collection))
 527                 return E_ITS_INT_UNMAPPED_INTERRUPT;
 528
 529         vcpu = kvm_get_vcpu(kvm, ite->collection->target_addr);
 530         if (!vcpu)
 531                 return E_ITS_INT_UNMAPPED_INTERRUPT;
 532
 533         if (!vcpu->arch.vgic_cpu.lpis_enabled)
 534                 return -EBUSY;
 535
 536         spin_lock_irqsave(&ite->irq->irq_lock, flags);
 537         ite->irq->pending_latch = true;
 538         vgic_queue_irq_unlock(kvm, ite->irq, flags);
 539
 540         return 0;
 541 }
 542
 543 static struct vgic_io_device *vgic_get_its_iodev(struct kvm_io_device *dev)
 544 {
 545         struct vgic_io_device *iodev;
 546
 547         if (dev->ops != &kvm_io_gic_ops)
 548                 return NULL;
 549
 550         iodev = container_of(dev, struct vgic_io_device, dev);
 551
 552         if (iodev->iodev_type != IODEV_ITS)
 553                 return NULL;
 554
 555         return iodev;
 556 }
 557
 558 /*
 559  * Queries the KVM IO bus framework to get the ITS pointer from the given
 560  * doorbell address.
 561  * We then call vgic_its_trigger_msi() with the decoded data.
 562  * According to the KVM_SIGNAL_MSI API description returns 1 on success.
 563  */
 564 int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi)
 565 {
 566         u64 address;
 567         struct kvm_io_device *kvm_io_dev;
 568         struct vgic_io_device *iodev;
 569         int ret;
 570
 571         if (!vgic_has_its(kvm))
 572                 return -ENODEV;
 573
 574         if (!(msi->flags & KVM_MSI_VALID_DEVID))
 575                 return -EINVAL;
 576
 577         address = (u64)msi->address_hi << 32 | msi->address_lo;
 578
 579         kvm_io_dev = kvm_io_bus_get_dev(kvm, KVM_MMIO_BUS, address);
 580         if (!kvm_io_dev)
 581                 return -EINVAL;
 582
 583         iodev = vgic_get_its_iodev(kvm_io_dev);
 584         if (!iodev)
 585                 return -EINVAL;
 586
 587         mutex_lock(&iodev->its->its_lock);
 588         ret = vgic_its_trigger_msi(kvm, iodev->its, msi->devid, msi->data);
 589         mutex_unlock(&iodev->its->its_lock);
 590
 591         if (ret < 0)
 592                 return ret;
 593
 594         /*
 595          * KVM_SIGNAL_MSI demands a return value > 0 for success and 0
 596          * if the guest has blocked the MSI. So we map any LPI mapping
 597          * related error to that.
 598          */
 599         if (ret)
 600                 return 0;
 601         else
 602                 return 1;
 603 }
 604
 605 /* Requires the its_lock to be held. */
 606 static void its_free_ite(struct kvm *kvm, struct its_ite *ite)
 607 {
 608         list_del(&ite->ite_list);
 609
 610         /* This put matches the get in vgic_add_lpi. */
 611         if (ite->irq)
 612                 vgic_put_irq(kvm, ite->irq);
 613
 614         kfree(ite);
 615 }
 616
 617 static u64 its_cmd_mask_field(u64 *its_cmd, int word, int shift, int size)
 618 {
 619         return (le64_to_cpu(its_cmd[word]) >> shift) & (BIT_ULL(size) - 1);
 620 }
 621
 622 #define its_cmd_get_command(cmd)        its_cmd_mask_field(cmd, 0,  0,  8)
 623 #define its_cmd_get_deviceid(cmd)       its_cmd_mask_field(cmd, 0, 32, 32)
 624 #define its_cmd_get_size(cmd)           (its_cmd_mask_field(cmd, 1,  0,  5) + 1)
 625 #define its_cmd_get_id(cmd)             its_cmd_mask_field(cmd, 1,  0, 32)
 626 #define its_cmd_get_physical_id(cmd)    its_cmd_mask_field(cmd, 1, 32, 32)
 627 #define its_cmd_get_collection(cmd)     its_cmd_mask_field(cmd, 2,  0, 16)
 628 #define its_cmd_get_ittaddr(cmd)        (its_cmd_mask_field(cmd, 2,  8, 44) << 8)
 629 #define its_cmd_get_target_addr(cmd)    its_cmd_mask_field(cmd, 2, 16, 32)
 630 #define its_cmd_get_validbit(cmd)       its_cmd_mask_field(cmd, 2, 63,  1)
 631
 632 /*
 633  * The DISCARD command frees an Interrupt Translation Table Entry (ITTE).
 634  * Must be called with the its_lock mutex held.
 635  */
 636 static int vgic_its_cmd_handle_discard(struct kvm *kvm, struct vgic_its *its,
 637                                        u64 *its_cmd)
 638 {
 639         u32 device_id = its_cmd_get_deviceid(its_cmd);
 640         u32 event_id = its_cmd_get_id(its_cmd);
 641         struct its_ite *ite;
 642
 643
 644         ite = find_ite(its, device_id, event_id);
 645         if (ite && ite->collection) {
 646                 /*
 647                  * Though the spec talks about removing the pending state, we
 648                  * don't bother here since we clear the ITTE anyway and the
 649                  * pending state is a property of the ITTE struct.
 650                  */
 651                 its_free_ite(kvm, ite);
 652                 return 0;
 653         }
 654
 655         return E_ITS_DISCARD_UNMAPPED_INTERRUPT;
 656 }
 657
 658 /*
 659  * The MOVI command moves an ITTE to a different collection.
 660  * Must be called with the its_lock mutex held.
 661  */
 662 static int vgic_its_cmd_handle_movi(struct kvm *kvm, struct vgic_its *its,
 663                                     u64 *its_cmd)
 664 {
 665         u32 device_id = its_cmd_get_deviceid(its_cmd);
 666         u32 event_id = its_cmd_get_id(its_cmd);
 667         u32 coll_id = its_cmd_get_collection(its_cmd);
 668         struct kvm_vcpu *vcpu;
 669         struct its_ite *ite;
 670         struct its_collection *collection;
 671
 672         ite = find_ite(its, device_id, event_id);
 673         if (!ite)
 674                 return E_ITS_MOVI_UNMAPPED_INTERRUPT;
 675
 676         if (!its_is_collection_mapped(ite->collection))
 677                 return E_ITS_MOVI_UNMAPPED_COLLECTION;
 678
 679         collection = find_collection(its, coll_id);
 680         if (!its_is_collection_mapped(collection))
 681                 return E_ITS_MOVI_UNMAPPED_COLLECTION;
 682
 683         ite->collection = collection;
 684         vcpu = kvm_get_vcpu(kvm, collection->target_addr);
 685
 686         spin_lock(&ite->irq->irq_lock);
 687         ite->irq->target_vcpu = vcpu;
 688         spin_unlock(&ite->irq->irq_lock);
 689
 690         return 0;
 691 }
 692
 693 /*
 694  * Check whether an ID can be stored into the corresponding guest table.
 695  * For a direct table this is pretty easy, but gets a bit nasty for
 696  * indirect tables. We check whether the resulting guest physical address
 697  * is actually valid (covered by a memslot and guest accessible).
 698  * For this we have to read the respective first level entry.
 699  */
 700 static bool vgic_its_check_id(struct vgic_its *its, u64 baser, u32 id,
 701                               gpa_t *eaddr)
 702 {
 703         int l1_tbl_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
 704         u64 indirect_ptr, type = GITS_BASER_TYPE(baser);
 705         int esz = GITS_BASER_ENTRY_SIZE(baser);
 706         int index;
 707         gfn_t gfn;
 708
 709         switch (type) {
 710         case GITS_BASER_TYPE_DEVICE:
 711                 if (id >= BIT_ULL(VITS_TYPER_DEVBITS))
 712                         return false;
 713                 break;
 714         case GITS_BASER_TYPE_COLLECTION:
 715                 /* as GITS_TYPER.CIL == 0, ITS supports 16-bit collection ID */
 716                 if (id >= BIT_ULL(16))
 717                         return false;
 718                 break;
 719         default:
 720                 return false;
 721         }
 722
 723         if (!(baser & GITS_BASER_INDIRECT)) {
 724                 phys_addr_t addr;
 725
 726                 if (id >= (l1_tbl_size / esz))
 727                         return false;
 728
 729                 addr = BASER_ADDRESS(baser) + id * esz;
 730                 gfn = addr >> PAGE_SHIFT;
 731
 732                 if (eaddr)
 733                         *eaddr = addr;
 734                 return kvm_is_visible_gfn(its->dev->kvm, gfn);
 735         }
 736
 737         /* calculate and check the index into the 1st level */
 738         index = id / (SZ_64K / esz);
 739         if (index >= (l1_tbl_size / sizeof(u64)))
 740                 return false;
 741
 742         /* Each 1st level entry is represented by a 64-bit value. */
 743         if (kvm_read_guest(its->dev->kvm,
 744                            BASER_ADDRESS(baser) + index * sizeof(indirect_ptr),
 745                            &indirect_ptr, sizeof(indirect_ptr)))
 746                 return false;
 747
 748         indirect_ptr = le64_to_cpu(indirect_ptr);
 749
 750         /* check the valid bit of the first level entry */
 751         if (!(indirect_ptr & BIT_ULL(63)))
 752                 return false;
 753
 754         /*
 755          * Mask the guest physical address and calculate the frame number.
 756          * Any address beyond our supported 48 bits of PA will be caught
 757          * by the actual check in the final step.
 758          */
 759         indirect_ptr &= GENMASK_ULL(51, 16);
 760
 761         /* Find the address of the actual entry */
 762         index = id % (SZ_64K / esz);
 763         indirect_ptr += index * esz;
 764         gfn = indirect_ptr >> PAGE_SHIFT;
 765
 766         if (eaddr)
 767                 *eaddr = indirect_ptr;
 768         return kvm_is_visible_gfn(its->dev->kvm, gfn);
 769 }
 770
 771 static int vgic_its_alloc_collection(struct vgic_its *its,
 772                                      struct its_collection **colp,
 773                                      u32 coll_id)
 774 {
 775         struct its_collection *collection;
 776
 777         if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL))
 778                 return E_ITS_MAPC_COLLECTION_OOR;
 779
 780         collection = kzalloc(sizeof(*collection), GFP_KERNEL);
 781
 782         collection->collection_id = coll_id;
 783         collection->target_addr = COLLECTION_NOT_MAPPED;
 784
 785         list_add_tail(&collection->coll_list, &its->collection_list);
 786         *colp = collection;
 787
 788         return 0;
 789 }
 790
 791 static void vgic_its_free_collection(struct vgic_its *its, u32 coll_id)
 792 {
 793         struct its_collection *collection;
 794         struct its_device *device;
 795         struct its_ite *ite;
 796
 797         /*
 798          * Clearing the mapping for that collection ID removes the
 799          * entry from the list. If there wasn't any before, we can
 800          * go home early.
 801          */
 802         collection = find_collection(its, coll_id);
 803         if (!collection)
 804                 return;
 805
 806         for_each_lpi_its(device, ite, its)
 807                 if (ite->collection &&
 808                     ite->collection->collection_id == coll_id)
 809                         ite->collection = NULL;
 810
 811         list_del(&collection->coll_list);
 812         kfree(collection);
 813 }
 814
 815 /* Must be called with its_lock mutex held */
 816 static struct its_ite *vgic_its_alloc_ite(struct its_device *device,
 817                                           struct its_collection *collection,
 818                                           u32 event_id)
 819 {
 820         struct its_ite *ite;
 821
 822         ite = kzalloc(sizeof(*ite), GFP_KERNEL);
 823         if (!ite)
 824                 return ERR_PTR(-ENOMEM);
 825
 826         ite->event_id   = event_id;
 827         ite->collection = collection;
 828
 829         list_add_tail(&ite->ite_list, &device->itt_head);
 830         return ite;
 831 }
 832
 833 /*
 834  * The MAPTI and MAPI commands map LPIs to ITTEs.
 835  * Must be called with its_lock mutex held.
 836  */
 837 static int vgic_its_cmd_handle_mapi(struct kvm *kvm, struct vgic_its *its,
 838                                     u64 *its_cmd)
 839 {
 840         u32 device_id = its_cmd_get_deviceid(its_cmd);
 841         u32 event_id = its_cmd_get_id(its_cmd);
 842         u32 coll_id = its_cmd_get_collection(its_cmd);
 843         struct its_ite *ite;
 844         struct kvm_vcpu *vcpu = NULL;
 845         struct its_device *device;
 846         struct its_collection *collection, *new_coll = NULL;
 847         struct vgic_irq *irq;
 848         int lpi_nr;
 849
 850         device = find_its_device(its, device_id);
 851         if (!device)
 852                 return E_ITS_MAPTI_UNMAPPED_DEVICE;
 853
 854         if (event_id >= BIT_ULL(device->num_eventid_bits))
 855                 return E_ITS_MAPTI_ID_OOR;
 856
 857         if (its_cmd_get_command(its_cmd) == GITS_CMD_MAPTI)
 858                 lpi_nr = its_cmd_get_physical_id(its_cmd);
 859         else
 860                 lpi_nr = event_id;
 861         if (lpi_nr < GIC_LPI_OFFSET ||
 862             lpi_nr >= max_lpis_propbaser(kvm->arch.vgic.propbaser))
 863                 return E_ITS_MAPTI_PHYSICALID_OOR;
 864
 865         /* If there is an existing mapping, behavior is UNPREDICTABLE. */
 866         if (find_ite(its, device_id, event_id))
 867                 return 0;
 868
 869         collection = find_collection(its, coll_id);
 870         if (!collection) {
 871                 int ret = vgic_its_alloc_collection(its, &collection, coll_id);
 872                 if (ret)
 873                         return ret;
 874                 new_coll = collection;
 875         }
 876
 877         ite = vgic_its_alloc_ite(device, collection, event_id);
 878         if (IS_ERR(ite)) {
 879                 if (new_coll)
 880                         vgic_its_free_collection(its, coll_id);
 881                 return PTR_ERR(ite);
 882         }
 883
 884         if (its_is_collection_mapped(collection))
 885                 vcpu = kvm_get_vcpu(kvm, collection->target_addr);
 886
 887         irq = vgic_add_lpi(kvm, lpi_nr, vcpu);
 888         if (IS_ERR(irq)) {
 889                 if (new_coll)
 890                         vgic_its_free_collection(its, coll_id);
 891                 its_free_ite(kvm, ite);
 892                 return PTR_ERR(irq);
 893         }
 894         ite->irq = irq;
 895
 896         return 0;
 897 }
 898
 899 /* Requires the its_lock to be held. */
 900 static void vgic_its_free_device(struct kvm *kvm, struct its_device *device)
 901 {
 902         struct its_ite *ite, *temp;
 903
 904         /*
 905          * The spec says that unmapping a device with still valid
 906          * ITTEs associated is UNPREDICTABLE. We remove all ITTEs,
 907          * since we cannot leave the memory unreferenced.
 908          */
 909         list_for_each_entry_safe(ite, temp, &device->itt_head, ite_list)
 910                 its_free_ite(kvm, ite);
 911
 912         list_del(&device->dev_list);
 913         kfree(device);
 914 }
 915
 916 /* its lock must be held */
 917 static void vgic_its_free_device_list(struct kvm *kvm, struct vgic_its *its)
 918 {
 919         struct its_device *cur, *temp;
 920
 921         list_for_each_entry_safe(cur, temp, &its->device_list, dev_list)
 922                 vgic_its_free_device(kvm, cur);
 923 }
 924
 925 /* its lock must be held */
 926 static void vgic_its_free_collection_list(struct kvm *kvm, struct vgic_its *its)
 927 {
 928         struct its_collection *cur, *temp;
 929
 930         list_for_each_entry_safe(cur, temp, &its->collection_list, coll_list)
 931                 vgic_its_free_collection(its, cur->collection_id);
 932 }
 933
 934 /* Must be called with its_lock mutex held */
 935 static struct its_device *vgic_its_alloc_device(struct vgic_its *its,
 936                                                 u32 device_id, gpa_t itt_addr,
 937                                                 u8 num_eventid_bits)
 938 {
 939         struct its_device *device;
 940
 941         device = kzalloc(sizeof(*device), GFP_KERNEL);
 942         if (!device)
 943                 return ERR_PTR(-ENOMEM);
 944
 945         device->device_id = device_id;
 946         device->itt_addr = itt_addr;
 947         device->num_eventid_bits = num_eventid_bits;
 948         INIT_LIST_HEAD(&device->itt_head);
 949
 950         list_add_tail(&device->dev_list, &its->device_list);
 951         return device;
 952 }
 953
 954 /*
 955  * MAPD maps or unmaps a device ID to Interrupt Translation Tables (ITTs).
 956  * Must be called with the its_lock mutex held.
 957  */
 958 static int vgic_its_cmd_handle_mapd(struct kvm *kvm, struct vgic_its *its,
 959                                     u64 *its_cmd)
 960 {
 961         u32 device_id = its_cmd_get_deviceid(its_cmd);
 962         bool valid = its_cmd_get_validbit(its_cmd);
 963         u8 num_eventid_bits = its_cmd_get_size(its_cmd);
 964         gpa_t itt_addr = its_cmd_get_ittaddr(its_cmd);
 965         struct its_device *device;
 966
 967         if (!vgic_its_check_id(its, its->baser_device_table, device_id, NULL))
 968                 return E_ITS_MAPD_DEVICE_OOR;
 969
 970         if (valid && num_eventid_bits > VITS_TYPER_IDBITS)
 971                 return E_ITS_MAPD_ITTSIZE_OOR;
 972
 973         device = find_its_device(its, device_id);
 974
 975         /*
 976          * The spec says that calling MAPD on an already mapped device
 977          * invalidates all cached data for this device. We implement this
 978          * by removing the mapping and re-establishing it.
 979          */
 980         if (device)
 981                 vgic_its_free_device(kvm, device);
 982
 983         /*
 984          * The spec does not say whether unmapping a not-mapped device
 985          * is an error, so we are done in any case.
 986          */
 987         if (!valid)
 988                 return 0;
 989
 990         device = vgic_its_alloc_device(its, device_id, itt_addr,
 991                                        num_eventid_bits);
 992         if (IS_ERR(device))
 993                 return PTR_ERR(device);
 994
 995         return 0;
 996 }
 997
 998 /*
 999  * The MAPC command maps collection IDs to redistributors.
1000  * Must be called with the its_lock mutex held.
1001  */
1002 static int vgic_its_cmd_handle_mapc(struct kvm *kvm, struct vgic_its *its,
1003                                     u64 *its_cmd)
1004 {
1005         u16 coll_id;
1006         u32 target_addr;
1007         struct its_collection *collection;
1008         bool valid;
1009
1010         valid = its_cmd_get_validbit(its_cmd);
1011         coll_id = its_cmd_get_collection(its_cmd);
1012         target_addr = its_cmd_get_target_addr(its_cmd);
1013
1014         if (target_addr >= atomic_read(&kvm->online_vcpus))
1015                 return E_ITS_MAPC_PROCNUM_OOR;
1016
1017         if (!valid) {
1018                 vgic_its_free_collection(its, coll_id);
1019         } else {
1020                 collection = find_collection(its, coll_id);
1021
1022                 if (!collection) {
1023                         int ret;
1024
1025                         ret = vgic_its_alloc_collection(its, &collection,
1026                                                         coll_id);
1027                         if (ret)
1028                                 return ret;
1029                         collection->target_addr = target_addr;
1030                 } else {
1031                         collection->target_addr = target_addr;
1032                         update_affinity_collection(kvm, its, collection);
1033                 }
1034         }
1035
1036         return 0;
1037 }
1038
1039 /*
1040  * The CLEAR command removes the pending state for a particular LPI.
1041  * Must be called with the its_lock mutex held.
1042  */
1043 static int vgic_its_cmd_handle_clear(struct kvm *kvm, struct vgic_its *its,
1044                                      u64 *its_cmd)
1045 {
1046         u32 device_id = its_cmd_get_deviceid(its_cmd);
1047         u32 event_id = its_cmd_get_id(its_cmd);
1048         struct its_ite *ite;
1049
1050
1051         ite = find_ite(its, device_id, event_id);
1052         if (!ite)
1053                 return E_ITS_CLEAR_UNMAPPED_INTERRUPT;
1054
1055         ite->irq->pending_latch = false;
1056
1057         return 0;
1058 }
1059
1060 /*
1061  * The INV command syncs the configuration bits from the memory table.
1062  * Must be called with the its_lock mutex held.
1063  */
1064 static int vgic_its_cmd_handle_inv(struct kvm *kvm, struct vgic_its *its,
1065                                    u64 *its_cmd)
1066 {
1067         u32 device_id = its_cmd_get_deviceid(its_cmd);
1068         u32 event_id = its_cmd_get_id(its_cmd);
1069         struct its_ite *ite;
1070
1071
1072         ite = find_ite(its, device_id, event_id);
1073         if (!ite)
1074                 return E_ITS_INV_UNMAPPED_INTERRUPT;
1075
1076         return update_lpi_config(kvm, ite->irq, NULL);
1077 }
1078
1079 /*
1080  * The INVALL command requests flushing of all IRQ data in this collection.
1081  * Find the VCPU mapped to that collection, then iterate over the VM's list
1082  * of mapped LPIs and update the configuration for each IRQ which targets
1083  * the specified vcpu. The configuration will be read from the in-memory
1084  * configuration table.
1085  * Must be called with the its_lock mutex held.
1086  */
1087 static int vgic_its_cmd_handle_invall(struct kvm *kvm, struct vgic_its *its,
1088                                       u64 *its_cmd)
1089 {
1090         u32 coll_id = its_cmd_get_collection(its_cmd);
1091         struct its_collection *collection;
1092         struct kvm_vcpu *vcpu;
1093         struct vgic_irq *irq;
1094         u32 *intids;
1095         int irq_count, i;
1096
1097         collection = find_collection(its, coll_id);
1098         if (!its_is_collection_mapped(collection))
1099                 return E_ITS_INVALL_UNMAPPED_COLLECTION;
1100
1101         vcpu = kvm_get_vcpu(kvm, collection->target_addr);
1102
1103         irq_count = vgic_copy_lpi_list(vcpu, &intids);
1104         if (irq_count < 0)
1105                 return irq_count;
1106
1107         for (i = 0; i < irq_count; i++) {
1108                 irq = vgic_get_irq(kvm, NULL, intids[i]);
1109                 if (!irq)
1110                         continue;
1111                 update_lpi_config(kvm, irq, vcpu);
1112                 vgic_put_irq(kvm, irq);
1113         }
1114
1115         kfree(intids);
1116
1117         return 0;
1118 }
1119
1120 /*
1121  * The MOVALL command moves the pending state of all IRQs targeting one
1122  * redistributor to another. We don't hold the pending state in the VCPUs,
1123  * but in the IRQs instead, so there is really not much to do for us here.
1124  * However the spec says that no IRQ must target the old redistributor
1125  * afterwards, so we make sure that no LPI is using the associated target_vcpu.
1126  * This command affects all LPIs in the system that target that redistributor.
1127  */
1128 static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its,
1129                                       u64 *its_cmd)
1130 {
1131         struct vgic_dist *dist = &kvm->arch.vgic;
1132         u32 target1_addr = its_cmd_get_target_addr(its_cmd);
1133         u32 target2_addr = its_cmd_mask_field(its_cmd, 3, 16, 32);
1134         struct kvm_vcpu *vcpu1, *vcpu2;
1135         struct vgic_irq *irq;
1136
1137         if (target1_addr >= atomic_read(&kvm->online_vcpus) ||
1138             target2_addr >= atomic_read(&kvm->online_vcpus))
1139                 return E_ITS_MOVALL_PROCNUM_OOR;
1140
1141         if (target1_addr == target2_addr)
1142                 return 0;
1143
1144         vcpu1 = kvm_get_vcpu(kvm, target1_addr);
1145         vcpu2 = kvm_get_vcpu(kvm, target2_addr);
1146
1147         spin_lock(&dist->lpi_list_lock);
1148
1149         list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) {
1150                 spin_lock(&irq->irq_lock);
1151
1152                 if (irq->target_vcpu == vcpu1)
1153                         irq->target_vcpu = vcpu2;
1154
1155                 spin_unlock(&irq->irq_lock);
1156         }
1157
1158         spin_unlock(&dist->lpi_list_lock);
1159
1160         return 0;
1161 }
1162
1163 /*
1164  * The INT command injects the LPI associated with that DevID/EvID pair.
1165  * Must be called with the its_lock mutex held.
1166  */
1167 static int vgic_its_cmd_handle_int(struct kvm *kvm, struct vgic_its *its,
1168                                    u64 *its_cmd)
1169 {
1170         u32 msi_data = its_cmd_get_id(its_cmd);
1171         u64 msi_devid = its_cmd_get_deviceid(its_cmd);
1172
1173         return vgic_its_trigger_msi(kvm, its, msi_devid, msi_data);
1174 }
1175
1176 /*
1177  * This function is called with the its_cmd lock held, but the ITS data
1178  * structure lock dropped.
1179  */
1180 static int vgic_its_handle_command(struct kvm *kvm, struct vgic_its *its,
1181                                    u64 *its_cmd)
1182 {
1183         int ret = -ENODEV;
1184
1185         mutex_lock(&its->its_lock);
1186         switch (its_cmd_get_command(its_cmd)) {
1187         case GITS_CMD_MAPD:
1188                 ret = vgic_its_cmd_handle_mapd(kvm, its, its_cmd);
1189                 break;
1190         case GITS_CMD_MAPC:
1191                 ret = vgic_its_cmd_handle_mapc(kvm, its, its_cmd);
1192                 break;
1193         case GITS_CMD_MAPI:
1194                 ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1195                 break;
1196         case GITS_CMD_MAPTI:
1197                 ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1198                 break;
1199         case GITS_CMD_MOVI:
1200                 ret = vgic_its_cmd_handle_movi(kvm, its, its_cmd);
1201                 break;
1202         case GITS_CMD_DISCARD:
1203                 ret = vgic_its_cmd_handle_discard(kvm, its, its_cmd);
1204                 break;
1205         case GITS_CMD_CLEAR:
1206                 ret = vgic_its_cmd_handle_clear(kvm, its, its_cmd);
1207                 break;
1208         case GITS_CMD_MOVALL:
1209                 ret = vgic_its_cmd_handle_movall(kvm, its, its_cmd);
1210                 break;
1211         case GITS_CMD_INT:
1212                 ret = vgic_its_cmd_handle_int(kvm, its, its_cmd);
1213                 break;
1214         case GITS_CMD_INV:
1215                 ret = vgic_its_cmd_handle_inv(kvm, its, its_cmd);
1216                 break;
1217         case GITS_CMD_INVALL:
1218                 ret = vgic_its_cmd_handle_invall(kvm, its, its_cmd);
1219                 break;
1220         case GITS_CMD_SYNC:
1221                 /* we ignore this command: we are in sync all of the time */
1222                 ret = 0;
1223                 break;
1224         }
1225         mutex_unlock(&its->its_lock);
1226
1227         return ret;
1228 }
1229
1230 static u64 vgic_sanitise_its_baser(u64 reg)
1231 {
1232         reg = vgic_sanitise_field(reg, GITS_BASER_SHAREABILITY_MASK,
1233                                   GITS_BASER_SHAREABILITY_SHIFT,
1234                                   vgic_sanitise_shareability);
1235         reg = vgic_sanitise_field(reg, GITS_BASER_INNER_CACHEABILITY_MASK,
1236                                   GITS_BASER_INNER_CACHEABILITY_SHIFT,
1237                                   vgic_sanitise_inner_cacheability);
1238         reg = vgic_sanitise_field(reg, GITS_BASER_OUTER_CACHEABILITY_MASK,
1239                                   GITS_BASER_OUTER_CACHEABILITY_SHIFT,
1240                                   vgic_sanitise_outer_cacheability);
1241
1242         /* Bits 15:12 contain bits 51:48 of the PA, which we don't support. */
1243         reg &= ~GENMASK_ULL(15, 12);
1244
1245         /* We support only one (ITS) page size: 64K */
1246         reg = (reg & ~GITS_BASER_PAGE_SIZE_MASK) | GITS_BASER_PAGE_SIZE_64K;
1247
1248         return reg;
1249 }
1250
1251 static u64 vgic_sanitise_its_cbaser(u64 reg)
1252 {
1253         reg = vgic_sanitise_field(reg, GITS_CBASER_SHAREABILITY_MASK,
1254                                   GITS_CBASER_SHAREABILITY_SHIFT,
1255                                   vgic_sanitise_shareability);
1256         reg = vgic_sanitise_field(reg, GITS_CBASER_INNER_CACHEABILITY_MASK,
1257                                   GITS_CBASER_INNER_CACHEABILITY_SHIFT,
1258                                   vgic_sanitise_inner_cacheability);
1259         reg = vgic_sanitise_field(reg, GITS_CBASER_OUTER_CACHEABILITY_MASK,
1260                                   GITS_CBASER_OUTER_CACHEABILITY_SHIFT,
1261                                   vgic_sanitise_outer_cacheability);
1262
1263         /*
1264          * Sanitise the physical address to be 64k aligned.
1265          * Also limit the physical addresses to 48 bits.
1266          */
1267         reg &= ~(GENMASK_ULL(51, 48) | GENMASK_ULL(15, 12));
1268
1269         return reg;
1270 }
1271
1272 static unsigned long vgic_mmio_read_its_cbaser(struct kvm *kvm,
1273                                                struct vgic_its *its,
1274                                                gpa_t addr, unsigned int len)
1275 {
1276         return extract_bytes(its->cbaser, addr & 7, len);
1277 }
1278
1279 static void vgic_mmio_write_its_cbaser(struct kvm *kvm, struct vgic_its *its,
1280                                        gpa_t addr, unsigned int len,
1281                                        unsigned long val)
1282 {
1283         /* When GITS_CTLR.Enable is 1, this register is RO. */
1284         if (its->enabled)
1285                 return;
1286
1287         mutex_lock(&its->cmd_lock);
1288         its->cbaser = update_64bit_reg(its->cbaser, addr & 7, len, val);
1289         its->cbaser = vgic_sanitise_its_cbaser(its->cbaser);
1290         its->creadr = 0;
1291         /*
1292          * CWRITER is architecturally UNKNOWN on reset, but we need to reset
1293          * it to CREADR to make sure we start with an empty command buffer.
1294          */
1295         its->cwriter = its->creadr;
1296         mutex_unlock(&its->cmd_lock);
1297 }
1298
1299 #define ITS_CMD_BUFFER_SIZE(baser)      ((((baser) & 0xff) + 1) << 12)
1300 #define ITS_CMD_SIZE                    32
1301 #define ITS_CMD_OFFSET(reg)             ((reg) & GENMASK(19, 5))
1302
1303 /* Must be called with the cmd_lock held. */
1304 static void vgic_its_process_commands(struct kvm *kvm, struct vgic_its *its)
1305 {
1306         gpa_t cbaser;
1307         u64 cmd_buf[4];
1308
1309         /* Commands are only processed when the ITS is enabled. */
1310         if (!its->enabled)
1311                 return;
1312
1313         cbaser = CBASER_ADDRESS(its->cbaser);
1314
1315         while (its->cwriter != its->creadr) {
1316                 int ret = kvm_read_guest(kvm, cbaser + its->creadr,
1317                                          cmd_buf, ITS_CMD_SIZE);
1318                 /*
1319                  * If kvm_read_guest() fails, this could be due to the guest
1320                  * programming a bogus value in CBASER or something else going
1321                  * wrong from which we cannot easily recover.
1322                  * According to section 6.3.2 in the GICv3 spec we can just
1323                  * ignore that command then.
1324                  */
1325                 if (!ret)
1326                         vgic_its_handle_command(kvm, its, cmd_buf);
1327
1328                 its->creadr += ITS_CMD_SIZE;
1329                 if (its->creadr == ITS_CMD_BUFFER_SIZE(its->cbaser))
1330                         its->creadr = 0;
1331         }
1332 }
1333
1334 /*
1335  * By writing to CWRITER the guest announces new commands to be processed.
1336  * To avoid any races in the first place, we take the its_cmd lock, which
1337  * protects our ring buffer variables, so that there is only one user
1338  * per ITS handling commands at a given time.
1339  */
1340 static void vgic_mmio_write_its_cwriter(struct kvm *kvm, struct vgic_its *its,
1341                                         gpa_t addr, unsigned int len,
1342                                         unsigned long val)
1343 {
1344         u64 reg;
1345
1346         if (!its)
1347                 return;
1348
1349         mutex_lock(&its->cmd_lock);
1350
1351         reg = update_64bit_reg(its->cwriter, addr & 7, len, val);
1352         reg = ITS_CMD_OFFSET(reg);
1353         if (reg >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
1354                 mutex_unlock(&its->cmd_lock);
1355                 return;
1356         }
1357         its->cwriter = reg;
1358
1359         vgic_its_process_commands(kvm, its);
1360
1361         mutex_unlock(&its->cmd_lock);
1362 }
1363
1364 static unsigned long vgic_mmio_read_its_cwriter(struct kvm *kvm,
1365                                                 struct vgic_its *its,
1366                                                 gpa_t addr, unsigned int len)
1367 {
1368         return extract_bytes(its->cwriter, addr & 0x7, len);
1369 }
1370
1371 static unsigned long vgic_mmio_read_its_creadr(struct kvm *kvm,
1372                                                struct vgic_its *its,
1373                                                gpa_t addr, unsigned int len)
1374 {
1375         return extract_bytes(its->creadr, addr & 0x7, len);
1376 }
1377
1378 static int vgic_mmio_uaccess_write_its_creadr(struct kvm *kvm,
1379                                               struct vgic_its *its,
1380                                               gpa_t addr, unsigned int len,
1381                                               unsigned long val)
1382 {
1383         u32 cmd_offset;
1384         int ret = 0;
1385
1386         mutex_lock(&its->cmd_lock);
1387
1388         if (its->enabled) {
1389                 ret = -EBUSY;
1390                 goto out;
1391         }
1392
1393         cmd_offset = ITS_CMD_OFFSET(val);
1394         if (cmd_offset >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
1395                 ret = -EINVAL;
1396                 goto out;
1397         }
1398
1399         its->creadr = cmd_offset;
1400 out:
1401         mutex_unlock(&its->cmd_lock);
1402         return ret;
1403 }
1404
1405 #define BASER_INDEX(addr) (((addr) / sizeof(u64)) & 0x7)
1406 static unsigned long vgic_mmio_read_its_baser(struct kvm *kvm,
1407                                               struct vgic_its *its,
1408                                               gpa_t addr, unsigned int len)
1409 {
1410         u64 reg;
1411
1412         switch (BASER_INDEX(addr)) {
1413         case 0:
1414                 reg = its->baser_device_table;
1415                 break;
1416         case 1:
1417                 reg = its->baser_coll_table;
1418                 break;
1419         default:
1420                 reg = 0;
1421                 break;
1422         }
1423
1424         return extract_bytes(reg, addr & 7, len);
1425 }
1426
1427 #define GITS_BASER_RO_MASK      (GENMASK_ULL(52, 48) | GENMASK_ULL(58, 56))
1428 static void vgic_mmio_write_its_baser(struct kvm *kvm,
1429                                       struct vgic_its *its,
1430                                       gpa_t addr, unsigned int len,
1431                                       unsigned long val)
1432 {
1433         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
1434         u64 entry_size, table_type;
1435         u64 reg, *regptr, clearbits = 0;
1436
1437         /* When GITS_CTLR.Enable is 1, we ignore write accesses. */
1438         if (its->enabled)
1439                 return;
1440
1441         switch (BASER_INDEX(addr)) {
1442         case 0:
1443                 regptr = &its->baser_device_table;
1444                 entry_size = abi->dte_esz;
1445                 table_type = GITS_BASER_TYPE_DEVICE;
1446                 break;
1447         case 1:
1448                 regptr = &its->baser_coll_table;
1449                 entry_size = abi->cte_esz;
1450                 table_type = GITS_BASER_TYPE_COLLECTION;
1451                 clearbits = GITS_BASER_INDIRECT;
1452                 break;
1453         default:
1454                 return;
1455         }
1456
1457         reg = update_64bit_reg(*regptr, addr & 7, len, val);
1458         reg &= ~GITS_BASER_RO_MASK;
1459         reg &= ~clearbits;
1460
1461         reg |= (entry_size - 1) << GITS_BASER_ENTRY_SIZE_SHIFT;
1462         reg |= table_type << GITS_BASER_TYPE_SHIFT;
1463         reg = vgic_sanitise_its_baser(reg);
1464
1465         *regptr = reg;
1466
1467         if (!(reg & GITS_BASER_VALID)) {
1468                 /* Take the its_lock to prevent a race with a save/restore */
1469                 mutex_lock(&its->its_lock);
1470                 switch (table_type) {
1471                 case GITS_BASER_TYPE_DEVICE:
1472                         vgic_its_free_device_list(kvm, its);
1473                         break;
1474                 case GITS_BASER_TYPE_COLLECTION:
1475                         vgic_its_free_collection_list(kvm, its);
1476                         break;
1477                 }
1478                 mutex_unlock(&its->its_lock);
1479         }
1480 }
1481
1482 static unsigned long vgic_mmio_read_its_ctlr(struct kvm *vcpu,
1483                                              struct vgic_its *its,
1484                                              gpa_t addr, unsigned int len)
1485 {
1486         u32 reg = 0;
1487
1488         mutex_lock(&its->cmd_lock);
1489         if (its->creadr == its->cwriter)
1490                 reg |= GITS_CTLR_QUIESCENT;
1491         if (its->enabled)
1492                 reg |= GITS_CTLR_ENABLE;
1493         mutex_unlock(&its->cmd_lock);
1494
1495         return reg;
1496 }
1497
1498 static void vgic_mmio_write_its_ctlr(struct kvm *kvm, struct vgic_its *its,
1499                                      gpa_t addr, unsigned int len,
1500                                      unsigned long val)
1501 {
1502         mutex_lock(&its->cmd_lock);
1503
1504         /*
1505          * It is UNPREDICTABLE to enable the ITS if any of the CBASER or
1506          * device/collection BASER are invalid
1507          */
1508         if (!its->enabled && (val & GITS_CTLR_ENABLE) &&
1509                 (!(its->baser_device_table & GITS_BASER_VALID) ||
1510                  !(its->baser_coll_table & GITS_BASER_VALID) ||
1511                  !(its->cbaser & GITS_CBASER_VALID)))
1512                 goto out;
1513
1514         its->enabled = !!(val & GITS_CTLR_ENABLE);
1515
1516         /*
1517          * Try to process any pending commands. This function bails out early
1518          * if the ITS is disabled or no commands have been queued.
1519          */
1520         vgic_its_process_commands(kvm, its);
1521
1522 out:
1523         mutex_unlock(&its->cmd_lock);
1524 }
1525
1526 #define REGISTER_ITS_DESC(off, rd, wr, length, acc)             \
1527 {                                                               \
1528         .reg_offset = off,                                      \
1529         .len = length,                                          \
1530         .access_flags = acc,                                    \
1531         .its_read = rd,                                         \
1532         .its_write = wr,                                        \
1533 }
1534
1535 #define REGISTER_ITS_DESC_UACCESS(off, rd, wr, uwr, length, acc)\
1536 {                                                               \
1537         .reg_offset = off,                                      \
1538         .len = length,                                          \
1539         .access_flags = acc,                                    \
1540         .its_read = rd,                                         \
1541         .its_write = wr,                                        \
1542         .uaccess_its_write = uwr,                               \
1543 }
1544
1545 static void its_mmio_write_wi(struct kvm *kvm, struct vgic_its *its,
1546                               gpa_t addr, unsigned int len, unsigned long val)
1547 {
1548         /* Ignore */
1549 }
1550
1551 static struct vgic_register_region its_registers[] = {
1552         REGISTER_ITS_DESC(GITS_CTLR,
1553                 vgic_mmio_read_its_ctlr, vgic_mmio_write_its_ctlr, 4,
1554                 VGIC_ACCESS_32bit),
1555         REGISTER_ITS_DESC_UACCESS(GITS_IIDR,
1556                 vgic_mmio_read_its_iidr, its_mmio_write_wi,
1557                 vgic_mmio_uaccess_write_its_iidr, 4,
1558                 VGIC_ACCESS_32bit),
1559         REGISTER_ITS_DESC(GITS_TYPER,
1560                 vgic_mmio_read_its_typer, its_mmio_write_wi, 8,
1561                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1562         REGISTER_ITS_DESC(GITS_CBASER,
1563                 vgic_mmio_read_its_cbaser, vgic_mmio_write_its_cbaser, 8,
1564                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1565         REGISTER_ITS_DESC(GITS_CWRITER,
1566                 vgic_mmio_read_its_cwriter, vgic_mmio_write_its_cwriter, 8,
1567                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1568         REGISTER_ITS_DESC_UACCESS(GITS_CREADR,
1569                 vgic_mmio_read_its_creadr, its_mmio_write_wi,
1570                 vgic_mmio_uaccess_write_its_creadr, 8,
1571                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1572         REGISTER_ITS_DESC(GITS_BASER,
1573                 vgic_mmio_read_its_baser, vgic_mmio_write_its_baser, 0x40,
1574                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1575         REGISTER_ITS_DESC(GITS_IDREGS_BASE,
1576                 vgic_mmio_read_its_idregs, its_mmio_write_wi, 0x30,
1577                 VGIC_ACCESS_32bit),
1578 };
1579
1580 /* This is called on setting the LPI enable bit in the redistributor. */
1581 void vgic_enable_lpis(struct kvm_vcpu *vcpu)
1582 {
1583         if (!(vcpu->arch.vgic_cpu.pendbaser & GICR_PENDBASER_PTZ))
1584                 its_sync_lpi_pending_table(vcpu);
1585 }
1586
1587 static int vgic_register_its_iodev(struct kvm *kvm, struct vgic_its *its,
1588                                    u64 addr)
1589 {
1590         struct vgic_io_device *iodev = &its->iodev;
1591         int ret;
1592
1593         mutex_lock(&kvm->slots_lock);
1594         if (!IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {
1595                 ret = -EBUSY;
1596                 goto out;
1597         }
1598
1599         its->vgic_its_base = addr;
1600         iodev->regions = its_registers;
1601         iodev->nr_regions = ARRAY_SIZE(its_registers);
1602         kvm_iodevice_init(&iodev->dev, &kvm_io_gic_ops);
1603
1604         iodev->base_addr = its->vgic_its_base;
1605         iodev->iodev_type = IODEV_ITS;
1606         iodev->its = its;
1607         ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, iodev->base_addr,
1608                                       KVM_VGIC_V3_ITS_SIZE, &iodev->dev);
1609 out:
1610         mutex_unlock(&kvm->slots_lock);
1611
1612         return ret;
1613 }
1614
1615 #define INITIAL_BASER_VALUE                                               \
1616         (GIC_BASER_CACHEABILITY(GITS_BASER, INNER, RaWb)                | \
1617          GIC_BASER_CACHEABILITY(GITS_BASER, OUTER, SameAsInner)         | \
1618          GIC_BASER_SHAREABILITY(GITS_BASER, InnerShareable)             | \
1619          GITS_BASER_PAGE_SIZE_64K)
1620
1621 #define INITIAL_PROPBASER_VALUE                                           \
1622         (GIC_BASER_CACHEABILITY(GICR_PROPBASER, INNER, RaWb)            | \
1623          GIC_BASER_CACHEABILITY(GICR_PROPBASER, OUTER, SameAsInner)     | \
1624          GIC_BASER_SHAREABILITY(GICR_PROPBASER, InnerShareable))
1625
1626 static int vgic_its_create(struct kvm_device *dev, u32 type)
1627 {
1628         struct vgic_its *its;
1629
1630         if (type != KVM_DEV_TYPE_ARM_VGIC_ITS)
1631                 return -ENODEV;
1632
1633         its = kzalloc(sizeof(struct vgic_its), GFP_KERNEL);
1634         if (!its)
1635                 return -ENOMEM;
1636
1637         mutex_init(&its->its_lock);
1638         mutex_init(&its->cmd_lock);
1639
1640         its->vgic_its_base = VGIC_ADDR_UNDEF;
1641
1642         INIT_LIST_HEAD(&its->device_list);
1643         INIT_LIST_HEAD(&its->collection_list);
1644
1645         dev->kvm->arch.vgic.msis_require_devid = true;
1646         dev->kvm->arch.vgic.has_its = true;
1647         its->enabled = false;
1648         its->dev = dev;
1649
1650         its->baser_device_table = INITIAL_BASER_VALUE                   |
1651                 ((u64)GITS_BASER_TYPE_DEVICE << GITS_BASER_TYPE_SHIFT);
1652         its->baser_coll_table = INITIAL_BASER_VALUE |
1653                 ((u64)GITS_BASER_TYPE_COLLECTION << GITS_BASER_TYPE_SHIFT);
1654         dev->kvm->arch.vgic.propbaser = INITIAL_PROPBASER_VALUE;
1655
1656         dev->private = its;
1657
1658         return vgic_its_set_abi(its, NR_ITS_ABIS - 1);
1659 }
1660
1661 static void vgic_its_destroy(struct kvm_device *kvm_dev)
1662 {
1663         struct kvm *kvm = kvm_dev->kvm;
1664         struct vgic_its *its = kvm_dev->private;
1665
1666         mutex_lock(&its->its_lock);
1667
1668         vgic_its_free_device_list(kvm, its);
1669         vgic_its_free_collection_list(kvm, its);
1670
1671         mutex_unlock(&its->its_lock);
1672         kfree(its);
1673 }
1674
1675 int vgic_its_has_attr_regs(struct kvm_device *dev,
1676                            struct kvm_device_attr *attr)
1677 {
1678         const struct vgic_register_region *region;
1679         gpa_t offset = attr->attr;
1680         int align;
1681
1682         align = (offset < GITS_TYPER) || (offset >= GITS_PIDR4) ? 0x3 : 0x7;
1683
1684         if (offset & align)
1685                 return -EINVAL;
1686
1687         region = vgic_find_mmio_region(its_registers,
1688                                        ARRAY_SIZE(its_registers),
1689                                        offset);
1690         if (!region)
1691                 return -ENXIO;
1692
1693         return 0;
1694 }
1695
1696 int vgic_its_attr_regs_access(struct kvm_device *dev,
1697                               struct kvm_device_attr *attr,
1698                               u64 *reg, bool is_write)
1699 {
1700         const struct vgic_register_region *region;
1701         struct vgic_its *its;
1702         gpa_t addr, offset;
1703         unsigned int len;
1704         int align, ret = 0;
1705
1706         its = dev->private;
1707         offset = attr->attr;
1708
1709         /*
1710          * Although the spec supports upper/lower 32-bit accesses to
1711          * 64-bit ITS registers, the userspace ABI requires 64-bit
1712          * accesses to all 64-bit wide registers. We therefore only
1713          * support 32-bit accesses to GITS_CTLR, GITS_IIDR and GITS ID
1714          * registers
1715          */
1716         if ((offset < GITS_TYPER) || (offset >= GITS_PIDR4))
1717                 align = 0x3;
1718         else
1719                 align = 0x7;
1720
1721         if (offset & align)
1722                 return -EINVAL;
1723
1724         mutex_lock(&dev->kvm->lock);
1725
1726         if (IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {
1727                 ret = -ENXIO;
1728                 goto out;
1729         }
1730
1731         region = vgic_find_mmio_region(its_registers,
1732                                        ARRAY_SIZE(its_registers),
1733                                        offset);
1734         if (!region) {
1735                 ret = -ENXIO;
1736                 goto out;
1737         }
1738
1739         if (!lock_all_vcpus(dev->kvm)) {
1740                 ret = -EBUSY;
1741                 goto out;
1742         }
1743
1744         addr = its->vgic_its_base + offset;
1745
1746         len = region->access_flags & VGIC_ACCESS_64bit ? 8 : 4;
1747
1748         if (is_write) {
1749                 if (region->uaccess_its_write)
1750                         ret = region->uaccess_its_write(dev->kvm, its, addr,
1751                                                         len, *reg);
1752                 else
1753                         region->its_write(dev->kvm, its, addr, len, *reg);
1754         } else {
1755                 *reg = region->its_read(dev->kvm, its, addr, len);
1756         }
1757         unlock_all_vcpus(dev->kvm);
1758 out:
1759         mutex_unlock(&dev->kvm->lock);
1760         return ret;
1761 }
1762
1763 static u32 compute_next_devid_offset(struct list_head *h,
1764                                      struct its_device *dev)
1765 {
1766         struct its_device *next;
1767         u32 next_offset;
1768
1769         if (list_is_last(&dev->dev_list, h))
1770                 return 0;
1771         next = list_next_entry(dev, dev_list);
1772         next_offset = next->device_id - dev->device_id;
1773
1774         return min_t(u32, next_offset, VITS_DTE_MAX_DEVID_OFFSET);
1775 }
1776
1777 static u32 compute_next_eventid_offset(struct list_head *h, struct its_ite *ite)
1778 {
1779         struct its_ite *next;
1780         u32 next_offset;
1781
1782         if (list_is_last(&ite->ite_list, h))
1783                 return 0;
1784         next = list_next_entry(ite, ite_list);
1785         next_offset = next->event_id - ite->event_id;
1786
1787         return min_t(u32, next_offset, VITS_ITE_MAX_EVENTID_OFFSET);
1788 }
1789
1790 /**
1791  * entry_fn_t - Callback called on a table entry restore path
1792  * @its: its handle
1793  * @id: id of the entry
1794  * @entry: pointer to the entry
1795  * @opaque: pointer to an opaque data
1796  *
1797  * Return: < 0 on error, 0 if last element was identified, id offset to next
1798  * element otherwise
1799  */
1800 typedef int (*entry_fn_t)(struct vgic_its *its, u32 id, void *entry,
1801                           void *opaque);
1802
1803 /**
1804  * scan_its_table - Scan a contiguous table in guest RAM and applies a function
1805  * to each entry
1806  *
1807  * @its: its handle
1808  * @base: base gpa of the table
1809  * @size: size of the table in bytes
1810  * @esz: entry size in bytes
1811  * @start_id: the ID of the first entry in the table
1812  * (non zero for 2d level tables)
1813  * @fn: function to apply on each entry
1814  *
1815  * Return: < 0 on error, 0 if last element was identified, 1 otherwise
1816  * (the last element may not be found on second level tables)
1817  */
1818 static int scan_its_table(struct vgic_its *its, gpa_t base, int size, int esz,
1819                           int start_id, entry_fn_t fn, void *opaque)
1820 {
1821         struct kvm *kvm = its->dev->kvm;
1822         unsigned long len = size;
1823         int id = start_id;
1824         gpa_t gpa = base;
1825         char entry[esz];
1826         int ret;
1827
1828         memset(entry, 0, esz);
1829
1830         while (len > 0) {
1831                 int next_offset;
1832                 size_t byte_offset;
1833
1834                 ret = kvm_read_guest(kvm, gpa, entry, esz);
1835                 if (ret)
1836                         return ret;
1837
1838                 next_offset = fn(its, id, entry, opaque);
1839                 if (next_offset <= 0)
1840                         return next_offset;
1841
1842                 byte_offset = next_offset * esz;
1843                 id += next_offset;
1844                 gpa += byte_offset;
1845                 len -= byte_offset;
1846         }
1847         return 1;
1848 }
1849
1850 /**
1851  * vgic_its_save_ite - Save an interrupt translation entry at @gpa
1852  */
1853 static int vgic_its_save_ite(struct vgic_its *its, struct its_device *dev,
1854                               struct its_ite *ite, gpa_t gpa, int ite_esz)
1855 {
1856         struct kvm *kvm = its->dev->kvm;
1857         u32 next_offset;
1858         u64 val;
1859
1860         next_offset = compute_next_eventid_offset(&dev->itt_head, ite);
1861         val = ((u64)next_offset << KVM_ITS_ITE_NEXT_SHIFT) |
1862                ((u64)ite->irq->intid << KVM_ITS_ITE_PINTID_SHIFT) |
1863                 ite->collection->collection_id;
1864         val = cpu_to_le64(val);
1865         return kvm_write_guest(kvm, gpa, &val, ite_esz);
1866 }
1867
1868 /**
1869  * vgic_its_restore_ite - restore an interrupt translation entry
1870  * @event_id: id used for indexing
1871  * @ptr: pointer to the ITE entry
1872  * @opaque: pointer to the its_device
1873  */
1874 static int vgic_its_restore_ite(struct vgic_its *its, u32 event_id,
1875                                 void *ptr, void *opaque)
1876 {
1877         struct its_device *dev = (struct its_device *)opaque;
1878         struct its_collection *collection;
1879         struct kvm *kvm = its->dev->kvm;
1880         struct kvm_vcpu *vcpu = NULL;
1881         u64 val;
1882         u64 *p = (u64 *)ptr;
1883         struct vgic_irq *irq;
1884         u32 coll_id, lpi_id;
1885         struct its_ite *ite;
1886         u32 offset;
1887
1888         val = *p;
1889
1890         val = le64_to_cpu(val);
1891
1892         coll_id = val & KVM_ITS_ITE_ICID_MASK;
1893         lpi_id = (val & KVM_ITS_ITE_PINTID_MASK) >> KVM_ITS_ITE_PINTID_SHIFT;
1894
1895         if (!lpi_id)
1896                 return 1; /* invalid entry, no choice but to scan next entry */
1897
1898         if (lpi_id < VGIC_MIN_LPI)
1899                 return -EINVAL;
1900
1901         offset = val >> KVM_ITS_ITE_NEXT_SHIFT;
1902         if (event_id + offset >= BIT_ULL(dev->num_eventid_bits))
1903                 return -EINVAL;
1904
1905         collection = find_collection(its, coll_id);
1906         if (!collection)
1907                 return -EINVAL;
1908
1909         ite = vgic_its_alloc_ite(dev, collection, event_id);
1910         if (IS_ERR(ite))
1911                 return PTR_ERR(ite);
1912
1913         if (its_is_collection_mapped(collection))
1914                 vcpu = kvm_get_vcpu(kvm, collection->target_addr);
1915
1916         irq = vgic_add_lpi(kvm, lpi_id, vcpu);
1917         if (IS_ERR(irq))
1918                 return PTR_ERR(irq);
1919         ite->irq = irq;
1920
1921         return offset;
1922 }
1923
1924 static int vgic_its_ite_cmp(void *priv, struct list_head *a,
1925                             struct list_head *b)
1926 {
1927         struct its_ite *itea = container_of(a, struct its_ite, ite_list);
1928         struct its_ite *iteb = container_of(b, struct its_ite, ite_list);
1929
1930         if (itea->event_id < iteb->event_id)
1931                 return -1;
1932         else
1933                 return 1;
1934 }
1935
1936 static int vgic_its_save_itt(struct vgic_its *its, struct its_device *device)
1937 {
1938         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
1939         gpa_t base = device->itt_addr;
1940         struct its_ite *ite;
1941         int ret;
1942         int ite_esz = abi->ite_esz;
1943
1944         list_sort(NULL, &device->itt_head, vgic_its_ite_cmp);
1945
1946         list_for_each_entry(ite, &device->itt_head, ite_list) {
1947                 gpa_t gpa = base + ite->event_id * ite_esz;
1948
1949                 ret = vgic_its_save_ite(its, device, ite, gpa, ite_esz);
1950                 if (ret)
1951                         return ret;
1952         }
1953         return 0;
1954 }
1955
1956 /**
1957  * vgic_its_restore_itt - restore the ITT of a device
1958  *
1959  * @its: its handle
1960  * @dev: device handle
1961  *
1962  * Return 0 on success, < 0 on error
1963  */
1964 static int vgic_its_restore_itt(struct vgic_its *its, struct its_device *dev)
1965 {
1966         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
1967         gpa_t base = dev->itt_addr;
1968         int ret;
1969         int ite_esz = abi->ite_esz;
1970         size_t max_size = BIT_ULL(dev->num_eventid_bits) * ite_esz;
1971
1972         ret = scan_its_table(its, base, max_size, ite_esz, 0,
1973                              vgic_its_restore_ite, dev);
1974
1975         /* scan_its_table returns +1 if all ITEs are invalid */
1976         if (ret > 0)
1977                 ret = 0;
1978
1979         return ret;
1980 }
1981
1982 /**
1983  * vgic_its_save_dte - Save a device table entry at a given GPA
1984  *
1985  * @its: ITS handle
1986  * @dev: ITS device
1987  * @ptr: GPA
1988  */
1989 static int vgic_its_save_dte(struct vgic_its *its, struct its_device *dev,
1990                              gpa_t ptr, int dte_esz)
1991 {
1992         struct kvm *kvm = its->dev->kvm;
1993         u64 val, itt_addr_field;
1994         u32 next_offset;
1995
1996         itt_addr_field = dev->itt_addr >> 8;
1997         next_offset = compute_next_devid_offset(&its->device_list, dev);
1998         val = (1ULL << KVM_ITS_DTE_VALID_SHIFT |
1999                ((u64)next_offset << KVM_ITS_DTE_NEXT_SHIFT) |
2000                (itt_addr_field << KVM_ITS_DTE_ITTADDR_SHIFT) |
2001                 (dev->num_eventid_bits - 1));
2002         val = cpu_to_le64(val);
2003         return kvm_write_guest(kvm, ptr, &val, dte_esz);
2004 }
2005
2006 /**
2007  * vgic_its_restore_dte - restore a device table entry
2008  *
2009  * @its: its handle
2010  * @id: device id the DTE corresponds to
2011  * @ptr: kernel VA where the 8 byte DTE is located
2012  * @opaque: unused
2013  *
2014  * Return: < 0 on error, 0 if the dte is the last one, id offset to the
2015  * next dte otherwise
2016  */
2017 static int vgic_its_restore_dte(struct vgic_its *its, u32 id,
2018                                 void *ptr, void *opaque)
2019 {
2020         struct its_device *dev;
2021         gpa_t itt_addr;
2022         u8 num_eventid_bits;
2023         u64 entry = *(u64 *)ptr;
2024         bool valid;
2025         u32 offset;
2026         int ret;
2027
2028         entry = le64_to_cpu(entry);
2029
2030         valid = entry >> KVM_ITS_DTE_VALID_SHIFT;
2031         num_eventid_bits = (entry & KVM_ITS_DTE_SIZE_MASK) + 1;
2032         itt_addr = ((entry & KVM_ITS_DTE_ITTADDR_MASK)
2033                         >> KVM_ITS_DTE_ITTADDR_SHIFT) << 8;
2034
2035         if (!valid)
2036                 return 1;
2037
2038         /* dte entry is valid */
2039         offset = (entry & KVM_ITS_DTE_NEXT_MASK) >> KVM_ITS_DTE_NEXT_SHIFT;
2040
2041         dev = vgic_its_alloc_device(its, id, itt_addr, num_eventid_bits);
2042         if (IS_ERR(dev))
2043                 return PTR_ERR(dev);
2044
2045         ret = vgic_its_restore_itt(its, dev);
2046         if (ret) {
2047                 vgic_its_free_device(its->dev->kvm, dev);
2048                 return ret;
2049         }
2050
2051         return offset;
2052 }
2053
2054 static int vgic_its_device_cmp(void *priv, struct list_head *a,
2055                                struct list_head *b)
2056 {
2057         struct its_device *deva = container_of(a, struct its_device, dev_list);
2058         struct its_device *devb = container_of(b, struct its_device, dev_list);
2059
2060         if (deva->device_id < devb->device_id)
2061                 return -1;
2062         else
2063                 return 1;
2064 }
2065
2066 /**
2067  * vgic_its_save_device_tables - Save the device table and all ITT
2068  * into guest RAM
2069  *
2070  * L1/L2 handling is hidden by vgic_its_check_id() helper which directly
2071  * returns the GPA of the device entry
2072  */
2073 static int vgic_its_save_device_tables(struct vgic_its *its)
2074 {
2075         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2076         u64 baser = its->baser_device_table;
2077         struct its_device *dev;
2078         int dte_esz = abi->dte_esz;
2079
2080         if (!(baser & GITS_BASER_VALID))
2081                 return 0;
2082
2083         list_sort(NULL, &its->device_list, vgic_its_device_cmp);
2084
2085         list_for_each_entry(dev, &its->device_list, dev_list) {
2086                 int ret;
2087                 gpa_t eaddr;
2088
2089                 if (!vgic_its_check_id(its, baser,
2090                                        dev->device_id, &eaddr))
2091                         return -EINVAL;
2092
2093                 ret = vgic_its_save_itt(its, dev);
2094                 if (ret)
2095                         return ret;
2096
2097                 ret = vgic_its_save_dte(its, dev, eaddr, dte_esz);
2098                 if (ret)
2099                         return ret;
2100         }
2101         return 0;
2102 }
2103
2104 /**
2105  * handle_l1_dte - callback used for L1 device table entries (2 stage case)
2106  *
2107  * @its: its handle
2108  * @id: index of the entry in the L1 table
2109  * @addr: kernel VA
2110  * @opaque: unused
2111  *
2112  * L1 table entries are scanned by steps of 1 entry
2113  * Return < 0 if error, 0 if last dte was found when scanning the L2
2114  * table, +1 otherwise (meaning next L1 entry must be scanned)
2115  */
2116 static int handle_l1_dte(struct vgic_its *its, u32 id, void *addr,
2117                          void *opaque)
2118 {
2119         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2120         int l2_start_id = id * (SZ_64K / abi->dte_esz);
2121         u64 entry = *(u64 *)addr;
2122         int dte_esz = abi->dte_esz;
2123         gpa_t gpa;
2124         int ret;
2125
2126         entry = le64_to_cpu(entry);
2127
2128         if (!(entry & KVM_ITS_L1E_VALID_MASK))
2129                 return 1;
2130
2131         gpa = entry & KVM_ITS_L1E_ADDR_MASK;
2132
2133         ret = scan_its_table(its, gpa, SZ_64K, dte_esz,
2134                              l2_start_id, vgic_its_restore_dte, NULL);
2135
2136         return ret;
2137 }
2138
2139 /**
2140  * vgic_its_restore_device_tables - Restore the device table and all ITT
2141  * from guest RAM to internal data structs
2142  */
2143 static int vgic_its_restore_device_tables(struct vgic_its *its)
2144 {
2145         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2146         u64 baser = its->baser_device_table;
2147         int l1_esz, ret;
2148         int l1_tbl_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
2149         gpa_t l1_gpa;
2150
2151         if (!(baser & GITS_BASER_VALID))
2152                 return 0;
2153
2154         l1_gpa = BASER_ADDRESS(baser);
2155
2156         if (baser & GITS_BASER_INDIRECT) {
2157                 l1_esz = GITS_LVL1_ENTRY_SIZE;
2158                 ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0,
2159                                      handle_l1_dte, NULL);
2160         } else {
2161                 l1_esz = abi->dte_esz;
2162                 ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0,
2163                                      vgic_its_restore_dte, NULL);
2164         }
2165
2166         /* scan_its_table returns +1 if all entries are invalid */
2167         if (ret > 0)
2168                 ret = 0;
2169
2170         return ret;
2171 }
2172
2173 static int vgic_its_save_cte(struct vgic_its *its,
2174                              struct its_collection *collection,
2175                              gpa_t gpa, int esz)
2176 {
2177         u64 val;
2178
2179         val = (1ULL << KVM_ITS_CTE_VALID_SHIFT |
2180                ((u64)collection->target_addr << KVM_ITS_CTE_RDBASE_SHIFT) |
2181                collection->collection_id);
2182         val = cpu_to_le64(val);
2183         return kvm_write_guest(its->dev->kvm, gpa, &val, esz);
2184 }
2185
2186 static int vgic_its_restore_cte(struct vgic_its *its, gpa_t gpa, int esz)
2187 {
2188         struct its_collection *collection;
2189         struct kvm *kvm = its->dev->kvm;
2190         u32 target_addr, coll_id;
2191         u64 val;
2192         int ret;
2193
2194         BUG_ON(esz > sizeof(val));
2195         ret = kvm_read_guest(kvm, gpa, &val, esz);
2196         if (ret)
2197                 return ret;
2198         val = le64_to_cpu(val);
2199         if (!(val & KVM_ITS_CTE_VALID_MASK))
2200                 return 0;
2201
2202         target_addr = (u32)(val >> KVM_ITS_CTE_RDBASE_SHIFT);
2203         coll_id = val & KVM_ITS_CTE_ICID_MASK;
2204
2205         if (target_addr >= atomic_read(&kvm->online_vcpus))
2206                 return -EINVAL;
2207
2208         collection = find_collection(its, coll_id);
2209         if (collection)
2210                 return -EEXIST;
2211         ret = vgic_its_alloc_collection(its, &collection, coll_id);
2212         if (ret)
2213                 return ret;
2214         collection->target_addr = target_addr;
2215         return 1;
2216 }
2217
2218 /**
2219  * vgic_its_save_collection_table - Save the collection table into
2220  * guest RAM
2221  */
2222 static int vgic_its_save_collection_table(struct vgic_its *its)
2223 {
2224         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2225         u64 baser = its->baser_coll_table;
2226         gpa_t gpa = BASER_ADDRESS(baser);
2227         struct its_collection *collection;
2228         u64 val;
2229         size_t max_size, filled = 0;
2230         int ret, cte_esz = abi->cte_esz;
2231
2232         if (!(baser & GITS_BASER_VALID))
2233                 return 0;
2234
2235         max_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
2236
2237         list_for_each_entry(collection, &its->collection_list, coll_list) {
2238                 ret = vgic_its_save_cte(its, collection, gpa, cte_esz);
2239                 if (ret)
2240                         return ret;
2241                 gpa += cte_esz;
2242                 filled += cte_esz;
2243         }
2244
2245         if (filled == max_size)
2246                 return 0;
2247
2248         /*
2249          * table is not fully filled, add a last dummy element
2250          * with valid bit unset
2251          */
2252         val = 0;
2253         BUG_ON(cte_esz > sizeof(val));
2254         ret = kvm_write_guest(its->dev->kvm, gpa, &val, cte_esz);
2255         return ret;
2256 }
2257
2258 /**
2259  * vgic_its_restore_collection_table - reads the collection table
2260  * in guest memory and restores the ITS internal state. Requires the
2261  * BASER registers to be restored before.
2262  */
2263 static int vgic_its_restore_collection_table(struct vgic_its *its)
2264 {
2265         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2266         u64 baser = its->baser_coll_table;
2267         int cte_esz = abi->cte_esz;
2268         size_t max_size, read = 0;
2269         gpa_t gpa;
2270         int ret;
2271
2272         if (!(baser & GITS_BASER_VALID))
2273                 return 0;
2274
2275         gpa = BASER_ADDRESS(baser);
2276
2277         max_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
2278
2279         while (read < max_size) {
2280                 ret = vgic_its_restore_cte(its, gpa, cte_esz);
2281                 if (ret <= 0)
2282                         break;
2283                 gpa += cte_esz;
2284                 read += cte_esz;
2285         }
2286
2287         if (ret > 0)
2288                 return 0;
2289
2290         return ret;
2291 }
2292
2293 /**
2294  * vgic_its_save_tables_v0 - Save the ITS tables into guest ARM
2295  * according to v0 ABI
2296  */
2297 static int vgic_its_save_tables_v0(struct vgic_its *its)
2298 {
2299         int ret;
2300
2301         ret = vgic_its_save_device_tables(its);
2302         if (ret)
2303                 return ret;
2304
2305         return vgic_its_save_collection_table(its);
2306 }
2307
2308 /**
2309  * vgic_its_restore_tables_v0 - Restore the ITS tables from guest RAM
2310  * to internal data structs according to V0 ABI
2311  *
2312  */
2313 static int vgic_its_restore_tables_v0(struct vgic_its *its)
2314 {
2315         int ret;
2316
2317         ret = vgic_its_restore_collection_table(its);
2318         if (ret)
2319                 return ret;
2320
2321         return vgic_its_restore_device_tables(its);
2322 }
2323
2324 static int vgic_its_commit_v0(struct vgic_its *its)
2325 {
2326         const struct vgic_its_abi *abi;
2327
2328         abi = vgic_its_get_abi(its);
2329         its->baser_coll_table &= ~GITS_BASER_ENTRY_SIZE_MASK;
2330         its->baser_device_table &= ~GITS_BASER_ENTRY_SIZE_MASK;
2331
2332         its->baser_coll_table |= (GIC_ENCODE_SZ(abi->cte_esz, 5)
2333                                         << GITS_BASER_ENTRY_SIZE_SHIFT);
2334
2335         its->baser_device_table |= (GIC_ENCODE_SZ(abi->dte_esz, 5)
2336                                         << GITS_BASER_ENTRY_SIZE_SHIFT);
2337         return 0;
2338 }
2339
2340 static void vgic_its_reset(struct kvm *kvm, struct vgic_its *its)
2341 {
2342         /* We need to keep the ABI specific field values */
2343         its->baser_coll_table &= ~GITS_BASER_VALID;
2344         its->baser_device_table &= ~GITS_BASER_VALID;
2345         its->cbaser = 0;
2346         its->creadr = 0;
2347         its->cwriter = 0;
2348         its->enabled = 0;
2349         vgic_its_free_device_list(kvm, its);
2350         vgic_its_free_collection_list(kvm, its);
2351 }
2352
2353 static int vgic_its_has_attr(struct kvm_device *dev,
2354                              struct kvm_device_attr *attr)
2355 {
2356         switch (attr->group) {
2357         case KVM_DEV_ARM_VGIC_GRP_ADDR:
2358                 switch (attr->attr) {
2359                 case KVM_VGIC_ITS_ADDR_TYPE:
2360                         return 0;
2361                 }
2362                 break;
2363         case KVM_DEV_ARM_VGIC_GRP_CTRL:
2364                 switch (attr->attr) {
2365                 case KVM_DEV_ARM_VGIC_CTRL_INIT:
2366                         return 0;
2367                 case KVM_DEV_ARM_ITS_CTRL_RESET:
2368                         return 0;
2369                 case KVM_DEV_ARM_ITS_SAVE_TABLES:
2370                         return 0;
2371                 case KVM_DEV_ARM_ITS_RESTORE_TABLES:
2372                         return 0;
2373                 }
2374                 break;
2375         case KVM_DEV_ARM_VGIC_GRP_ITS_REGS:
2376                 return vgic_its_has_attr_regs(dev, attr);
2377         }
2378         return -ENXIO;
2379 }
2380
2381 static int vgic_its_ctrl(struct kvm *kvm, struct vgic_its *its, u64 attr)
2382 {
2383         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2384         int ret = 0;
2385
2386         if (attr == KVM_DEV_ARM_VGIC_CTRL_INIT) /* Nothing to do */
2387                 return 0;
2388
2389         mutex_lock(&kvm->lock);
2390         mutex_lock(&its->its_lock);
2391
2392         if (!lock_all_vcpus(kvm)) {
2393                 mutex_unlock(&its->its_lock);
2394                 mutex_unlock(&kvm->lock);
2395                 return -EBUSY;
2396         }
2397
2398         switch (attr) {
2399         case KVM_DEV_ARM_ITS_CTRL_RESET:
2400                 vgic_its_reset(kvm, its);
2401                 break;
2402         case KVM_DEV_ARM_ITS_SAVE_TABLES:
2403                 ret = abi->save_tables(its);
2404                 break;
2405         case KVM_DEV_ARM_ITS_RESTORE_TABLES:
2406                 ret = abi->restore_tables(its);
2407                 break;
2408         }
2409
2410         unlock_all_vcpus(kvm);
2411         mutex_unlock(&its->its_lock);
2412         mutex_unlock(&kvm->lock);
2413         return ret;
2414 }
2415
2416 static int vgic_its_set_attr(struct kvm_device *dev,
2417                              struct kvm_device_attr *attr)
2418 {
2419         struct vgic_its *its = dev->private;
2420         int ret;
2421
2422         switch (attr->group) {
2423         case KVM_DEV_ARM_VGIC_GRP_ADDR: {
2424                 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2425                 unsigned long type = (unsigned long)attr->attr;
2426                 u64 addr;
2427
2428                 if (type != KVM_VGIC_ITS_ADDR_TYPE)
2429                         return -ENODEV;
2430
2431                 if (copy_from_user(&addr, uaddr, sizeof(addr)))
2432                         return -EFAULT;
2433
2434                 ret = vgic_check_ioaddr(dev->kvm, &its->vgic_its_base,
2435                                         addr, SZ_64K);
2436                 if (ret)
2437                         return ret;
2438
2439                 return vgic_register_its_iodev(dev->kvm, its, addr);
2440         }
2441         case KVM_DEV_ARM_VGIC_GRP_CTRL:
2442                 return vgic_its_ctrl(dev->kvm, its, attr->attr);
2443         case KVM_DEV_ARM_VGIC_GRP_ITS_REGS: {
2444                 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2445                 u64 reg;
2446
2447                 if (get_user(reg, uaddr))
2448                         return -EFAULT;
2449
2450                 return vgic_its_attr_regs_access(dev, attr, &reg, true);
2451         }
2452         }
2453         return -ENXIO;
2454 }
2455
2456 static int vgic_its_get_attr(struct kvm_device *dev,
2457                              struct kvm_device_attr *attr)
2458 {
2459         switch (attr->group) {
2460         case KVM_DEV_ARM_VGIC_GRP_ADDR: {
2461                 struct vgic_its *its = dev->private;
2462                 u64 addr = its->vgic_its_base;
2463                 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2464                 unsigned long type = (unsigned long)attr->attr;
2465
2466                 if (type != KVM_VGIC_ITS_ADDR_TYPE)
2467                         return -ENODEV;
2468
2469                 if (copy_to_user(uaddr, &addr, sizeof(addr)))
2470                         return -EFAULT;
2471                 break;
2472         }
2473         case KVM_DEV_ARM_VGIC_GRP_ITS_REGS: {
2474                 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2475                 u64 reg;
2476                 int ret;
2477
2478                 ret = vgic_its_attr_regs_access(dev, attr, &reg, false);
2479                 if (ret)
2480                         return ret;
2481                 return put_user(reg, uaddr);
2482         }
2483         default:
2484                 return -ENXIO;
2485         }
2486
2487         return 0;
2488 }
2489
2490 static struct kvm_device_ops kvm_arm_vgic_its_ops = {
2491         .name = "kvm-arm-vgic-its",
2492         .create = vgic_its_create,
2493         .destroy = vgic_its_destroy,
2494         .set_attr = vgic_its_set_attr,
2495         .get_attr = vgic_its_get_attr,
2496         .has_attr = vgic_its_has_attr,
2497 };
2498
2499 int kvm_vgic_register_its_device(void)
2500 {
2501         return kvm_register_device_ops(&kvm_arm_vgic_its_ops,
2502                                        KVM_DEV_TYPE_ARM_VGIC_ITS);
2503 }