drivers/vfio/pci/vfio_pci_rdwr.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * VFIO PCI I/O Port & MMIO access
   4  *
   5  * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
   6  *     Author: Alex Williamson <alex.williamson@redhat.com>
   7  *
   8  * Derived from original vfio:
   9  * Copyright 2010 Cisco Systems, Inc.  All rights reserved.
  10  * Author: Tom Lyon, pugs@cisco.com
  11  */
  12
  13 #include <linux/fs.h>
  14 #include <linux/pci.h>
  15 #include <linux/uaccess.h>
  16 #include <linux/io.h>
  17 #include <linux/vfio.h>
  18 #include <linux/vgaarb.h>
  19
  20 #include "vfio_pci_priv.h"
  21
  22 #ifdef __LITTLE_ENDIAN
  23 #define vfio_ioread64   ioread64
  24 #define vfio_iowrite64  iowrite64
  25 #define vfio_ioread32   ioread32
  26 #define vfio_iowrite32  iowrite32
  27 #define vfio_ioread16   ioread16
  28 #define vfio_iowrite16  iowrite16
  29 #else
  30 #define vfio_ioread64   ioread64be
  31 #define vfio_iowrite64  iowrite64be
  32 #define vfio_ioread32   ioread32be
  33 #define vfio_iowrite32  iowrite32be
  34 #define vfio_ioread16   ioread16be
  35 #define vfio_iowrite16  iowrite16be
  36 #endif
  37 #define vfio_ioread8    ioread8
  38 #define vfio_iowrite8   iowrite8
  39
  40 #define VFIO_IOWRITE(size) \
  41 int vfio_pci_core_iowrite##size(struct vfio_pci_core_device *vdev,      \
  42                         bool test_mem, u##size val, void __iomem *io)   \
  43 {                                                                       \
  44         if (test_mem) {                                                 \
  45                 down_read(&vdev->memory_lock);                          \
  46                 if (!__vfio_pci_memory_enabled(vdev)) {                 \
  47                         up_read(&vdev->memory_lock);                    \
  48                         return -EIO;                                    \
  49                 }                                                       \
  50         }                                                               \
  51                                                                         \
  52         vfio_iowrite##size(val, io);                                    \
  53                                                                         \
  54         if (test_mem)                                                   \
  55                 up_read(&vdev->memory_lock);                            \
  56                                                                         \
  57         return 0;                                                       \
  58 }                                                                       \
  59 EXPORT_SYMBOL_GPL(vfio_pci_core_iowrite##size);
  60
  61 VFIO_IOWRITE(8)
  62 VFIO_IOWRITE(16)
  63 VFIO_IOWRITE(32)
  64 #ifdef iowrite64
  65 VFIO_IOWRITE(64)
  66 #endif
  67
  68 #define VFIO_IOREAD(size) \
  69 int vfio_pci_core_ioread##size(struct vfio_pci_core_device *vdev,       \
  70                         bool test_mem, u##size *val, void __iomem *io)  \
  71 {                                                                       \
  72         if (test_mem) {                                                 \
  73                 down_read(&vdev->memory_lock);                          \
  74                 if (!__vfio_pci_memory_enabled(vdev)) {                 \
  75                         up_read(&vdev->memory_lock);                    \
  76                         return -EIO;                                    \
  77                 }                                                       \
  78         }                                                               \
  79                                                                         \
  80         *val = vfio_ioread##size(io);                                   \
  81                                                                         \
  82         if (test_mem)                                                   \
  83                 up_read(&vdev->memory_lock);                            \
  84                                                                         \
  85         return 0;                                                       \
  86 }                                                                       \
  87 EXPORT_SYMBOL_GPL(vfio_pci_core_ioread##size);
  88
  89 VFIO_IOREAD(8)
  90 VFIO_IOREAD(16)
  91 VFIO_IOREAD(32)
  92
  93 /*
  94  * Read or write from an __iomem region (MMIO or I/O port) with an excluded
  95  * range which is inaccessible.  The excluded range drops writes and fills
  96  * reads with -1.  This is intended for handling MSI-X vector tables and
  97  * leftover space for ROM BARs.
  98  */
  99 ssize_t vfio_pci_core_do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem,
 100                                void __iomem *io, char __user *buf,
 101                                loff_t off, size_t count, size_t x_start,
 102                                size_t x_end, bool iswrite)
 103 {
 104         ssize_t done = 0;
 105         int ret;
 106
 107         while (count) {
 108                 size_t fillable, filled;
 109
 110                 if (off < x_start)
 111                         fillable = min(count, (size_t)(x_start - off));
 112                 else if (off >= x_end)
 113                         fillable = count;
 114                 else
 115                         fillable = 0;
 116
 117                 if (fillable >= 4 && !(off % 4)) {
 118                         u32 val;
 119
 120                         if (iswrite) {
 121                                 if (copy_from_user(&val, buf, 4))
 122                                         return -EFAULT;
 123
 124                                 ret = vfio_pci_core_iowrite32(vdev, test_mem,
 125                                                               val, io + off);
 126                                 if (ret)
 127                                         return ret;
 128                         } else {
 129                                 ret = vfio_pci_core_ioread32(vdev, test_mem,
 130                                                              &val, io + off);
 131                                 if (ret)
 132                                         return ret;
 133
 134                                 if (copy_to_user(buf, &val, 4))
 135                                         return -EFAULT;
 136                         }
 137
 138                         filled = 4;
 139                 } else if (fillable >= 2 && !(off % 2)) {
 140                         u16 val;
 141
 142                         if (iswrite) {
 143                                 if (copy_from_user(&val, buf, 2))
 144                                         return -EFAULT;
 145
 146                                 ret = vfio_pci_core_iowrite16(vdev, test_mem,
 147                                                               val, io + off);
 148                                 if (ret)
 149                                         return ret;
 150                         } else {
 151                                 ret = vfio_pci_core_ioread16(vdev, test_mem,
 152                                                              &val, io + off);
 153                                 if (ret)
 154                                         return ret;
 155
 156                                 if (copy_to_user(buf, &val, 2))
 157                                         return -EFAULT;
 158                         }
 159
 160                         filled = 2;
 161                 } else if (fillable) {
 162                         u8 val;
 163
 164                         if (iswrite) {
 165                                 if (copy_from_user(&val, buf, 1))
 166                                         return -EFAULT;
 167
 168                                 ret = vfio_pci_core_iowrite8(vdev, test_mem,
 169                                                              val, io + off);
 170                                 if (ret)
 171                                         return ret;
 172                         } else {
 173                                 ret = vfio_pci_core_ioread8(vdev, test_mem,
 174                                                             &val, io + off);
 175                                 if (ret)
 176                                         return ret;
 177
 178                                 if (copy_to_user(buf, &val, 1))
 179                                         return -EFAULT;
 180                         }
 181
 182                         filled = 1;
 183                 } else {
 184                         /* Fill reads with -1, drop writes */
 185                         filled = min(count, (size_t)(x_end - off));
 186                         if (!iswrite) {
 187                                 u8 val = 0xFF;
 188                                 size_t i;
 189
 190                                 for (i = 0; i < filled; i++)
 191                                         if (copy_to_user(buf + i, &val, 1))
 192                                                 return -EFAULT;
 193                         }
 194                 }
 195
 196                 count -= filled;
 197                 done += filled;
 198                 off += filled;
 199                 buf += filled;
 200         }
 201
 202         return done;
 203 }
 204 EXPORT_SYMBOL_GPL(vfio_pci_core_do_io_rw);
 205
 206 int vfio_pci_core_setup_barmap(struct vfio_pci_core_device *vdev, int bar)
 207 {
 208         struct pci_dev *pdev = vdev->pdev;
 209         int ret;
 210         void __iomem *io;
 211
 212         if (vdev->barmap[bar])
 213                 return 0;
 214
 215         ret = pci_request_selected_regions(pdev, 1 << bar, "vfio");
 216         if (ret)
 217                 return ret;
 218
 219         io = pci_iomap(pdev, bar, 0);
 220         if (!io) {
 221                 pci_release_selected_regions(pdev, 1 << bar);
 222                 return -ENOMEM;
 223         }
 224
 225         vdev->barmap[bar] = io;
 226
 227         return 0;
 228 }
 229 EXPORT_SYMBOL_GPL(vfio_pci_core_setup_barmap);
 230
 231 ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf,
 232                         size_t count, loff_t *ppos, bool iswrite)
 233 {
 234         struct pci_dev *pdev = vdev->pdev;
 235         loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
 236         int bar = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
 237         size_t x_start = 0, x_end = 0;
 238         resource_size_t end;
 239         void __iomem *io;
 240         struct resource *res = &vdev->pdev->resource[bar];
 241         ssize_t done;
 242
 243         if (pci_resource_start(pdev, bar))
 244                 end = pci_resource_len(pdev, bar);
 245         else if (bar == PCI_ROM_RESOURCE &&
 246                  pdev->resource[bar].flags & IORESOURCE_ROM_SHADOW)
 247                 end = 0x20000;
 248         else
 249                 return -EINVAL;
 250
 251         if (pos >= end)
 252                 return -EINVAL;
 253
 254         count = min(count, (size_t)(end - pos));
 255
 256         if (bar == PCI_ROM_RESOURCE) {
 257                 /*
 258                  * The ROM can fill less space than the BAR, so we start the
 259                  * excluded range at the end of the actual ROM.  This makes
 260                  * filling large ROM BARs much faster.
 261                  */
 262                 io = pci_map_rom(pdev, &x_start);
 263                 if (!io) {
 264                         done = -ENOMEM;
 265                         goto out;
 266                 }
 267                 x_end = end;
 268         } else {
 269                 int ret = vfio_pci_core_setup_barmap(vdev, bar);
 270                 if (ret) {
 271                         done = ret;
 272                         goto out;
 273                 }
 274
 275                 io = vdev->barmap[bar];
 276         }
 277
 278         if (bar == vdev->msix_bar) {
 279                 x_start = vdev->msix_offset;
 280                 x_end = vdev->msix_offset + vdev->msix_size;
 281         }
 282
 283         done = vfio_pci_core_do_io_rw(vdev, res->flags & IORESOURCE_MEM, io, buf, pos,
 284                                       count, x_start, x_end, iswrite);
 285
 286         if (done >= 0)
 287                 *ppos += done;
 288
 289         if (bar == PCI_ROM_RESOURCE)
 290                 pci_unmap_rom(pdev, io);
 291 out:
 292         return done;
 293 }
 294
 295 #ifdef CONFIG_VFIO_PCI_VGA
 296 ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev, char __user *buf,
 297                                size_t count, loff_t *ppos, bool iswrite)
 298 {
 299         int ret;
 300         loff_t off, pos = *ppos & VFIO_PCI_OFFSET_MASK;
 301         void __iomem *iomem = NULL;
 302         unsigned int rsrc;
 303         bool is_ioport;
 304         ssize_t done;
 305
 306         if (!vdev->has_vga)
 307                 return -EINVAL;
 308
 309         if (pos > 0xbfffful)
 310                 return -EINVAL;
 311
 312         switch ((u32)pos) {
 313         case 0xa0000 ... 0xbffff:
 314                 count = min(count, (size_t)(0xc0000 - pos));
 315                 iomem = ioremap(0xa0000, 0xbffff - 0xa0000 + 1);
 316                 off = pos - 0xa0000;
 317                 rsrc = VGA_RSRC_LEGACY_MEM;
 318                 is_ioport = false;
 319                 break;
 320         case 0x3b0 ... 0x3bb:
 321                 count = min(count, (size_t)(0x3bc - pos));
 322                 iomem = ioport_map(0x3b0, 0x3bb - 0x3b0 + 1);
 323                 off = pos - 0x3b0;
 324                 rsrc = VGA_RSRC_LEGACY_IO;
 325                 is_ioport = true;
 326                 break;
 327         case 0x3c0 ... 0x3df:
 328                 count = min(count, (size_t)(0x3e0 - pos));
 329                 iomem = ioport_map(0x3c0, 0x3df - 0x3c0 + 1);
 330                 off = pos - 0x3c0;
 331                 rsrc = VGA_RSRC_LEGACY_IO;
 332                 is_ioport = true;
 333                 break;
 334         default:
 335                 return -EINVAL;
 336         }
 337
 338         if (!iomem)
 339                 return -ENOMEM;
 340
 341         ret = vga_get_interruptible(vdev->pdev, rsrc);
 342         if (ret) {
 343                 is_ioport ? ioport_unmap(iomem) : iounmap(iomem);
 344                 return ret;
 345         }
 346
 347         /*
 348          * VGA MMIO is a legacy, non-BAR resource that hopefully allows
 349          * probing, so we don't currently worry about access in relation
 350          * to the memory enable bit in the command register.
 351          */
 352         done = vfio_pci_core_do_io_rw(vdev, false, iomem, buf, off, count,
 353                                       0, 0, iswrite);
 354
 355         vga_put(vdev->pdev, rsrc);
 356
 357         is_ioport ? ioport_unmap(iomem) : iounmap(iomem);
 358
 359         if (done >= 0)
 360                 *ppos += done;
 361
 362         return done;
 363 }
 364 #endif
 365
 366 static void vfio_pci_ioeventfd_do_write(struct vfio_pci_ioeventfd *ioeventfd,
 367                                         bool test_mem)
 368 {
 369         switch (ioeventfd->count) {
 370         case 1:
 371                 vfio_pci_core_iowrite8(ioeventfd->vdev, test_mem,
 372                                        ioeventfd->data, ioeventfd->addr);
 373                 break;
 374         case 2:
 375                 vfio_pci_core_iowrite16(ioeventfd->vdev, test_mem,
 376                                         ioeventfd->data, ioeventfd->addr);
 377                 break;
 378         case 4:
 379                 vfio_pci_core_iowrite32(ioeventfd->vdev, test_mem,
 380                                         ioeventfd->data, ioeventfd->addr);
 381                 break;
 382 #ifdef iowrite64
 383         case 8:
 384                 vfio_pci_core_iowrite64(ioeventfd->vdev, test_mem,
 385                                         ioeventfd->data, ioeventfd->addr);
 386                 break;
 387 #endif
 388         }
 389 }
 390
 391 static int vfio_pci_ioeventfd_handler(void *opaque, void *unused)
 392 {
 393         struct vfio_pci_ioeventfd *ioeventfd = opaque;
 394         struct vfio_pci_core_device *vdev = ioeventfd->vdev;
 395
 396         if (ioeventfd->test_mem) {
 397                 if (!down_read_trylock(&vdev->memory_lock))
 398                         return 1; /* Lock contended, use thread */
 399                 if (!__vfio_pci_memory_enabled(vdev)) {
 400                         up_read(&vdev->memory_lock);
 401                         return 0;
 402                 }
 403         }
 404
 405         vfio_pci_ioeventfd_do_write(ioeventfd, false);
 406
 407         if (ioeventfd->test_mem)
 408                 up_read(&vdev->memory_lock);
 409
 410         return 0;
 411 }
 412
 413 static void vfio_pci_ioeventfd_thread(void *opaque, void *unused)
 414 {
 415         struct vfio_pci_ioeventfd *ioeventfd = opaque;
 416
 417         vfio_pci_ioeventfd_do_write(ioeventfd, ioeventfd->test_mem);
 418 }
 419
 420 int vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset,
 421                        uint64_t data, int count, int fd)
 422 {
 423         struct pci_dev *pdev = vdev->pdev;
 424         loff_t pos = offset & VFIO_PCI_OFFSET_MASK;
 425         int ret, bar = VFIO_PCI_OFFSET_TO_INDEX(offset);
 426         struct vfio_pci_ioeventfd *ioeventfd;
 427
 428         /* Only support ioeventfds into BARs */
 429         if (bar > VFIO_PCI_BAR5_REGION_INDEX)
 430                 return -EINVAL;
 431
 432         if (pos + count > pci_resource_len(pdev, bar))
 433                 return -EINVAL;
 434
 435         /* Disallow ioeventfds working around MSI-X table writes */
 436         if (bar == vdev->msix_bar &&
 437             !(pos + count <= vdev->msix_offset ||
 438               pos >= vdev->msix_offset + vdev->msix_size))
 439                 return -EINVAL;
 440
 441 #ifndef iowrite64
 442         if (count == 8)
 443                 return -EINVAL;
 444 #endif
 445
 446         ret = vfio_pci_core_setup_barmap(vdev, bar);
 447         if (ret)
 448                 return ret;
 449
 450         mutex_lock(&vdev->ioeventfds_lock);
 451
 452         list_for_each_entry(ioeventfd, &vdev->ioeventfds_list, next) {
 453                 if (ioeventfd->pos == pos && ioeventfd->bar == bar &&
 454                     ioeventfd->data == data && ioeventfd->count == count) {
 455                         if (fd == -1) {
 456                                 vfio_virqfd_disable(&ioeventfd->virqfd);
 457                                 list_del(&ioeventfd->next);
 458                                 vdev->ioeventfds_nr--;
 459                                 kfree(ioeventfd);
 460                                 ret = 0;
 461                         } else
 462                                 ret = -EEXIST;
 463
 464                         goto out_unlock;
 465                 }
 466         }
 467
 468         if (fd < 0) {
 469                 ret = -ENODEV;
 470                 goto out_unlock;
 471         }
 472
 473         if (vdev->ioeventfds_nr >= VFIO_PCI_IOEVENTFD_MAX) {
 474                 ret = -ENOSPC;
 475                 goto out_unlock;
 476         }
 477
 478         ioeventfd = kzalloc(sizeof(*ioeventfd), GFP_KERNEL_ACCOUNT);
 479         if (!ioeventfd) {
 480                 ret = -ENOMEM;
 481                 goto out_unlock;
 482         }
 483
 484         ioeventfd->vdev = vdev;
 485         ioeventfd->addr = vdev->barmap[bar] + pos;
 486         ioeventfd->data = data;
 487         ioeventfd->pos = pos;
 488         ioeventfd->bar = bar;
 489         ioeventfd->count = count;
 490         ioeventfd->test_mem = vdev->pdev->resource[bar].flags & IORESOURCE_MEM;
 491
 492         ret = vfio_virqfd_enable(ioeventfd, vfio_pci_ioeventfd_handler,
 493                                  vfio_pci_ioeventfd_thread, NULL,
 494                                  &ioeventfd->virqfd, fd);
 495         if (ret) {
 496                 kfree(ioeventfd);
 497                 goto out_unlock;
 498         }
 499
 500         list_add(&ioeventfd->next, &vdev->ioeventfds_list);
 501         vdev->ioeventfds_nr++;
 502
 503 out_unlock:
 504         mutex_unlock(&vdev->ioeventfds_lock);
 505
 506         return ret;
 507 }