tools/testing/selftests/iommu/iommufd.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES */
   3 #include <stdlib.h>
   4 #include <sys/mman.h>
   5 #include <sys/eventfd.h>
   6
   7 #define __EXPORTED_HEADERS__
   8 #include <linux/vfio.h>
   9
  10 #include "iommufd_utils.h"
  11
  12 static unsigned long HUGEPAGE_SIZE;
  13
  14 #define MOCK_PAGE_SIZE (PAGE_SIZE / 2)
  15 #define MOCK_HUGE_PAGE_SIZE (512 * MOCK_PAGE_SIZE)
  16
  17 static unsigned long get_huge_page_size(void)
  18 {
  19         char buf[80];
  20         int ret;
  21         int fd;
  22
  23         fd = open("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size",
  24                   O_RDONLY);
  25         if (fd < 0)
  26                 return 2 * 1024 * 1024;
  27
  28         ret = read(fd, buf, sizeof(buf));
  29         close(fd);
  30         if (ret <= 0 || ret == sizeof(buf))
  31                 return 2 * 1024 * 1024;
  32         buf[ret] = 0;
  33         return strtoul(buf, NULL, 10);
  34 }
  35
  36 static __attribute__((constructor)) void setup_sizes(void)
  37 {
  38         void *vrc;
  39         int rc;
  40
  41         PAGE_SIZE = sysconf(_SC_PAGE_SIZE);
  42         HUGEPAGE_SIZE = get_huge_page_size();
  43
  44         BUFFER_SIZE = PAGE_SIZE * 16;
  45         rc = posix_memalign(&buffer, HUGEPAGE_SIZE, BUFFER_SIZE);
  46         assert(!rc);
  47         assert(buffer);
  48         assert((uintptr_t)buffer % HUGEPAGE_SIZE == 0);
  49         vrc = mmap(buffer, BUFFER_SIZE, PROT_READ | PROT_WRITE,
  50                    MAP_SHARED | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
  51         assert(vrc == buffer);
  52 }
  53
  54 FIXTURE(iommufd)
  55 {
  56         int fd;
  57 };
  58
  59 FIXTURE_SETUP(iommufd)
  60 {
  61         self->fd = open("/dev/iommu", O_RDWR);
  62         ASSERT_NE(-1, self->fd);
  63 }
  64
  65 FIXTURE_TEARDOWN(iommufd)
  66 {
  67         teardown_iommufd(self->fd, _metadata);
  68 }
  69
  70 TEST_F(iommufd, simple_close)
  71 {
  72 }
  73
  74 TEST_F(iommufd, cmd_fail)
  75 {
  76         struct iommu_destroy cmd = { .size = sizeof(cmd), .id = 0 };
  77
  78         /* object id is invalid */
  79         EXPECT_ERRNO(ENOENT, _test_ioctl_destroy(self->fd, 0));
  80         /* Bad pointer */
  81         EXPECT_ERRNO(EFAULT, ioctl(self->fd, IOMMU_DESTROY, NULL));
  82         /* Unknown ioctl */
  83         EXPECT_ERRNO(ENOTTY,
  84                      ioctl(self->fd, _IO(IOMMUFD_TYPE, IOMMUFD_CMD_BASE - 1),
  85                            &cmd));
  86 }
  87
  88 TEST_F(iommufd, cmd_length)
  89 {
  90 #define TEST_LENGTH(_struct, _ioctl, _last)                              \
  91         {                                                                \
  92                 size_t min_size = offsetofend(struct _struct, _last);    \
  93                 struct {                                                 \
  94                         struct _struct cmd;                              \
  95                         uint8_t extra;                                   \
  96                 } cmd = { .cmd = { .size = min_size - 1 },               \
  97                           .extra = UINT8_MAX };                          \
  98                 int old_errno;                                           \
  99                 int rc;                                                  \
 100                                                                          \
 101                 EXPECT_ERRNO(EINVAL, ioctl(self->fd, _ioctl, &cmd));     \
 102                 cmd.cmd.size = sizeof(struct _struct) + 1;               \
 103                 EXPECT_ERRNO(E2BIG, ioctl(self->fd, _ioctl, &cmd));      \
 104                 cmd.cmd.size = sizeof(struct _struct);                   \
 105                 rc = ioctl(self->fd, _ioctl, &cmd);                      \
 106                 old_errno = errno;                                       \
 107                 cmd.cmd.size = sizeof(struct _struct) + 1;               \
 108                 cmd.extra = 0;                                           \
 109                 if (rc) {                                                \
 110                         EXPECT_ERRNO(old_errno,                          \
 111                                      ioctl(self->fd, _ioctl, &cmd));     \
 112                 } else {                                                 \
 113                         ASSERT_EQ(0, ioctl(self->fd, _ioctl, &cmd));     \
 114                 }                                                        \
 115         }
 116
 117         TEST_LENGTH(iommu_destroy, IOMMU_DESTROY, id);
 118         TEST_LENGTH(iommu_hw_info, IOMMU_GET_HW_INFO, __reserved);
 119         TEST_LENGTH(iommu_hwpt_alloc, IOMMU_HWPT_ALLOC, __reserved);
 120         TEST_LENGTH(iommu_hwpt_invalidate, IOMMU_HWPT_INVALIDATE, __reserved);
 121         TEST_LENGTH(iommu_ioas_alloc, IOMMU_IOAS_ALLOC, out_ioas_id);
 122         TEST_LENGTH(iommu_ioas_iova_ranges, IOMMU_IOAS_IOVA_RANGES,
 123                     out_iova_alignment);
 124         TEST_LENGTH(iommu_ioas_allow_iovas, IOMMU_IOAS_ALLOW_IOVAS,
 125                     allowed_iovas);
 126         TEST_LENGTH(iommu_ioas_map, IOMMU_IOAS_MAP, iova);
 127         TEST_LENGTH(iommu_ioas_copy, IOMMU_IOAS_COPY, src_iova);
 128         TEST_LENGTH(iommu_ioas_unmap, IOMMU_IOAS_UNMAP, length);
 129         TEST_LENGTH(iommu_option, IOMMU_OPTION, val64);
 130         TEST_LENGTH(iommu_vfio_ioas, IOMMU_VFIO_IOAS, __reserved);
 131 #undef TEST_LENGTH
 132 }
 133
 134 TEST_F(iommufd, cmd_ex_fail)
 135 {
 136         struct {
 137                 struct iommu_destroy cmd;
 138                 __u64 future;
 139         } cmd = { .cmd = { .size = sizeof(cmd), .id = 0 } };
 140
 141         /* object id is invalid and command is longer */
 142         EXPECT_ERRNO(ENOENT, ioctl(self->fd, IOMMU_DESTROY, &cmd));
 143         /* future area is non-zero */
 144         cmd.future = 1;
 145         EXPECT_ERRNO(E2BIG, ioctl(self->fd, IOMMU_DESTROY, &cmd));
 146         /* Original command "works" */
 147         cmd.cmd.size = sizeof(cmd.cmd);
 148         EXPECT_ERRNO(ENOENT, ioctl(self->fd, IOMMU_DESTROY, &cmd));
 149         /* Short command fails */
 150         cmd.cmd.size = sizeof(cmd.cmd) - 1;
 151         EXPECT_ERRNO(EINVAL, ioctl(self->fd, IOMMU_DESTROY, &cmd));
 152 }
 153
 154 TEST_F(iommufd, global_options)
 155 {
 156         struct iommu_option cmd = {
 157                 .size = sizeof(cmd),
 158                 .option_id = IOMMU_OPTION_RLIMIT_MODE,
 159                 .op = IOMMU_OPTION_OP_GET,
 160                 .val64 = 1,
 161         };
 162
 163         cmd.option_id = IOMMU_OPTION_RLIMIT_MODE;
 164         ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
 165         ASSERT_EQ(0, cmd.val64);
 166
 167         /* This requires root */
 168         cmd.op = IOMMU_OPTION_OP_SET;
 169         cmd.val64 = 1;
 170         ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
 171         cmd.val64 = 2;
 172         EXPECT_ERRNO(EINVAL, ioctl(self->fd, IOMMU_OPTION, &cmd));
 173
 174         cmd.op = IOMMU_OPTION_OP_GET;
 175         ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
 176         ASSERT_EQ(1, cmd.val64);
 177
 178         cmd.op = IOMMU_OPTION_OP_SET;
 179         cmd.val64 = 0;
 180         ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
 181
 182         cmd.op = IOMMU_OPTION_OP_GET;
 183         cmd.option_id = IOMMU_OPTION_HUGE_PAGES;
 184         EXPECT_ERRNO(ENOENT, ioctl(self->fd, IOMMU_OPTION, &cmd));
 185         cmd.op = IOMMU_OPTION_OP_SET;
 186         EXPECT_ERRNO(ENOENT, ioctl(self->fd, IOMMU_OPTION, &cmd));
 187 }
 188
 189 FIXTURE(iommufd_ioas)
 190 {
 191         int fd;
 192         uint32_t ioas_id;
 193         uint32_t stdev_id;
 194         uint32_t hwpt_id;
 195         uint32_t device_id;
 196         uint64_t base_iova;
 197 };
 198
 199 FIXTURE_VARIANT(iommufd_ioas)
 200 {
 201         unsigned int mock_domains;
 202         unsigned int memory_limit;
 203 };
 204
 205 FIXTURE_SETUP(iommufd_ioas)
 206 {
 207         unsigned int i;
 208
 209
 210         self->fd = open("/dev/iommu", O_RDWR);
 211         ASSERT_NE(-1, self->fd);
 212         test_ioctl_ioas_alloc(&self->ioas_id);
 213
 214         if (!variant->memory_limit) {
 215                 test_ioctl_set_default_memory_limit();
 216         } else {
 217                 test_ioctl_set_temp_memory_limit(variant->memory_limit);
 218         }
 219
 220         for (i = 0; i != variant->mock_domains; i++) {
 221                 test_cmd_mock_domain(self->ioas_id, &self->stdev_id,
 222                                      &self->hwpt_id, &self->device_id);
 223                 self->base_iova = MOCK_APERTURE_START;
 224         }
 225 }
 226
 227 FIXTURE_TEARDOWN(iommufd_ioas)
 228 {
 229         test_ioctl_set_default_memory_limit();
 230         teardown_iommufd(self->fd, _metadata);
 231 }
 232
 233 FIXTURE_VARIANT_ADD(iommufd_ioas, no_domain)
 234 {
 235 };
 236
 237 FIXTURE_VARIANT_ADD(iommufd_ioas, mock_domain)
 238 {
 239         .mock_domains = 1,
 240 };
 241
 242 FIXTURE_VARIANT_ADD(iommufd_ioas, two_mock_domain)
 243 {
 244         .mock_domains = 2,
 245 };
 246
 247 FIXTURE_VARIANT_ADD(iommufd_ioas, mock_domain_limit)
 248 {
 249         .mock_domains = 1,
 250         .memory_limit = 16,
 251 };
 252
 253 TEST_F(iommufd_ioas, ioas_auto_destroy)
 254 {
 255 }
 256
 257 TEST_F(iommufd_ioas, ioas_destroy)
 258 {
 259         if (self->stdev_id) {
 260                 /* IOAS cannot be freed while a device has a HWPT using it */
 261                 EXPECT_ERRNO(EBUSY,
 262                              _test_ioctl_destroy(self->fd, self->ioas_id));
 263         } else {
 264                 /* Can allocate and manually free an IOAS table */
 265                 test_ioctl_destroy(self->ioas_id);
 266         }
 267 }
 268
 269 TEST_F(iommufd_ioas, alloc_hwpt_nested)
 270 {
 271         const uint32_t min_data_len =
 272                 offsetofend(struct iommu_hwpt_selftest, iotlb);
 273         struct iommu_hwpt_selftest data = {
 274                 .iotlb = IOMMU_TEST_IOTLB_DEFAULT,
 275         };
 276         struct iommu_hwpt_invalidate_selftest inv_reqs[2] = {};
 277         uint32_t nested_hwpt_id[2] = {};
 278         uint32_t num_inv;
 279         uint32_t parent_hwpt_id = 0;
 280         uint32_t parent_hwpt_id_not_work = 0;
 281         uint32_t test_hwpt_id = 0;
 282
 283         if (self->device_id) {
 284                 /* Negative tests */
 285                 test_err_hwpt_alloc(ENOENT, self->ioas_id, self->device_id, 0,
 286                                     &test_hwpt_id);
 287                 test_err_hwpt_alloc(EINVAL, self->device_id, self->device_id, 0,
 288                                     &test_hwpt_id);
 289
 290                 test_cmd_hwpt_alloc(self->device_id, self->ioas_id,
 291                                     IOMMU_HWPT_ALLOC_NEST_PARENT,
 292                                     &parent_hwpt_id);
 293
 294                 test_cmd_hwpt_alloc(self->device_id, self->ioas_id, 0,
 295                                     &parent_hwpt_id_not_work);
 296
 297                 /* Negative nested tests */
 298                 test_err_hwpt_alloc_nested(EINVAL, self->device_id,
 299                                            parent_hwpt_id, 0,
 300                                            &nested_hwpt_id[0],
 301                                            IOMMU_HWPT_DATA_NONE, &data,
 302                                            sizeof(data));
 303                 test_err_hwpt_alloc_nested(EOPNOTSUPP, self->device_id,
 304                                            parent_hwpt_id, 0,
 305                                            &nested_hwpt_id[0],
 306                                            IOMMU_HWPT_DATA_SELFTEST + 1, &data,
 307                                            sizeof(data));
 308                 test_err_hwpt_alloc_nested(EINVAL, self->device_id,
 309                                            parent_hwpt_id, 0,
 310                                            &nested_hwpt_id[0],
 311                                            IOMMU_HWPT_DATA_SELFTEST, &data,
 312                                            min_data_len - 1);
 313                 test_err_hwpt_alloc_nested(EFAULT, self->device_id,
 314                                            parent_hwpt_id, 0,
 315                                            &nested_hwpt_id[0],
 316                                            IOMMU_HWPT_DATA_SELFTEST, NULL,
 317                                            sizeof(data));
 318                 test_err_hwpt_alloc_nested(
 319                         EOPNOTSUPP, self->device_id, parent_hwpt_id,
 320                         IOMMU_HWPT_ALLOC_NEST_PARENT, &nested_hwpt_id[0],
 321                         IOMMU_HWPT_DATA_SELFTEST, &data, sizeof(data));
 322                 test_err_hwpt_alloc_nested(EINVAL, self->device_id,
 323                                            parent_hwpt_id_not_work, 0,
 324                                            &nested_hwpt_id[0],
 325                                            IOMMU_HWPT_DATA_SELFTEST, &data,
 326                                            sizeof(data));
 327
 328                 /* Allocate two nested hwpts sharing one common parent hwpt */
 329                 test_cmd_hwpt_alloc_nested(self->device_id, parent_hwpt_id, 0,
 330                                            &nested_hwpt_id[0],
 331                                            IOMMU_HWPT_DATA_SELFTEST, &data,
 332                                            sizeof(data));
 333                 test_cmd_hwpt_alloc_nested(self->device_id, parent_hwpt_id, 0,
 334                                            &nested_hwpt_id[1],
 335                                            IOMMU_HWPT_DATA_SELFTEST, &data,
 336                                            sizeof(data));
 337                 test_cmd_hwpt_check_iotlb_all(nested_hwpt_id[0],
 338                                               IOMMU_TEST_IOTLB_DEFAULT);
 339                 test_cmd_hwpt_check_iotlb_all(nested_hwpt_id[1],
 340                                               IOMMU_TEST_IOTLB_DEFAULT);
 341
 342                 /* Negative test: a nested hwpt on top of a nested hwpt */
 343                 test_err_hwpt_alloc_nested(EINVAL, self->device_id,
 344                                            nested_hwpt_id[0], 0, &test_hwpt_id,
 345                                            IOMMU_HWPT_DATA_SELFTEST, &data,
 346                                            sizeof(data));
 347                 /* Negative test: parent hwpt now cannot be freed */
 348                 EXPECT_ERRNO(EBUSY,
 349                              _test_ioctl_destroy(self->fd, parent_hwpt_id));
 350
 351                 /* hwpt_invalidate only supports a user-managed hwpt (nested) */
 352                 num_inv = 1;
 353                 test_err_hwpt_invalidate(ENOENT, parent_hwpt_id, inv_reqs,
 354                                          IOMMU_HWPT_INVALIDATE_DATA_SELFTEST,
 355                                          sizeof(*inv_reqs), &num_inv);
 356                 assert(!num_inv);
 357
 358                 /* Check data_type by passing zero-length array */
 359                 num_inv = 0;
 360                 test_cmd_hwpt_invalidate(nested_hwpt_id[0], inv_reqs,
 361                                          IOMMU_HWPT_INVALIDATE_DATA_SELFTEST,
 362                                          sizeof(*inv_reqs), &num_inv);
 363                 assert(!num_inv);
 364
 365                 /* Negative test: Invalid data_type */
 366                 num_inv = 1;
 367                 test_err_hwpt_invalidate(EINVAL, nested_hwpt_id[0], inv_reqs,
 368                                          IOMMU_HWPT_INVALIDATE_DATA_SELFTEST_INVALID,
 369                                          sizeof(*inv_reqs), &num_inv);
 370                 assert(!num_inv);
 371
 372                 /* Negative test: structure size sanity */
 373                 num_inv = 1;
 374                 test_err_hwpt_invalidate(EINVAL, nested_hwpt_id[0], inv_reqs,
 375                                          IOMMU_HWPT_INVALIDATE_DATA_SELFTEST,
 376                                          sizeof(*inv_reqs) + 1, &num_inv);
 377                 assert(!num_inv);
 378
 379                 num_inv = 1;
 380                 test_err_hwpt_invalidate(EINVAL, nested_hwpt_id[0], inv_reqs,
 381                                          IOMMU_HWPT_INVALIDATE_DATA_SELFTEST,
 382                                          1, &num_inv);
 383                 assert(!num_inv);
 384
 385                 /* Negative test: invalid flag is passed */
 386                 num_inv = 1;
 387                 inv_reqs[0].flags = 0xffffffff;
 388                 test_err_hwpt_invalidate(EOPNOTSUPP, nested_hwpt_id[0], inv_reqs,
 389                                          IOMMU_HWPT_INVALIDATE_DATA_SELFTEST,
 390                                          sizeof(*inv_reqs), &num_inv);
 391                 assert(!num_inv);
 392
 393                 /* Negative test: invalid data_uptr when array is not empty */
 394                 num_inv = 1;
 395                 inv_reqs[0].flags = 0;
 396                 test_err_hwpt_invalidate(EINVAL, nested_hwpt_id[0], NULL,
 397                                          IOMMU_HWPT_INVALIDATE_DATA_SELFTEST,
 398                                          sizeof(*inv_reqs), &num_inv);
 399                 assert(!num_inv);
 400
 401                 /* Negative test: invalid entry_len when array is not empty */
 402                 num_inv = 1;
 403                 inv_reqs[0].flags = 0;
 404                 test_err_hwpt_invalidate(EINVAL, nested_hwpt_id[0], inv_reqs,
 405                                          IOMMU_HWPT_INVALIDATE_DATA_SELFTEST,
 406                                          0, &num_inv);
 407                 assert(!num_inv);
 408
 409                 /* Negative test: invalid iotlb_id */
 410                 num_inv = 1;
 411                 inv_reqs[0].flags = 0;
 412                 inv_reqs[0].iotlb_id = MOCK_NESTED_DOMAIN_IOTLB_ID_MAX + 1;
 413                 test_err_hwpt_invalidate(EINVAL, nested_hwpt_id[0], inv_reqs,
 414                                          IOMMU_HWPT_INVALIDATE_DATA_SELFTEST,
 415                                          sizeof(*inv_reqs), &num_inv);
 416                 assert(!num_inv);
 417
 418                 /*
 419                  * Invalidate the 1st iotlb entry but fail the 2nd request
 420                  * due to invalid flags configuration in the 2nd request.
 421                  */
 422                 num_inv = 2;
 423                 inv_reqs[0].flags = 0;
 424                 inv_reqs[0].iotlb_id = 0;
 425                 inv_reqs[1].flags = 0xffffffff;
 426                 inv_reqs[1].iotlb_id = 1;
 427                 test_err_hwpt_invalidate(EOPNOTSUPP, nested_hwpt_id[0], inv_reqs,
 428                                          IOMMU_HWPT_INVALIDATE_DATA_SELFTEST,
 429                                          sizeof(*inv_reqs), &num_inv);
 430                 assert(num_inv == 1);
 431                 test_cmd_hwpt_check_iotlb(nested_hwpt_id[0], 0, 0);
 432                 test_cmd_hwpt_check_iotlb(nested_hwpt_id[0], 1,
 433                                           IOMMU_TEST_IOTLB_DEFAULT);
 434                 test_cmd_hwpt_check_iotlb(nested_hwpt_id[0], 2,
 435                                           IOMMU_TEST_IOTLB_DEFAULT);
 436                 test_cmd_hwpt_check_iotlb(nested_hwpt_id[0], 3,
 437                                           IOMMU_TEST_IOTLB_DEFAULT);
 438
 439                 /*
 440                  * Invalidate the 1st iotlb entry but fail the 2nd request
 441                  * due to invalid iotlb_id configuration in the 2nd request.
 442                  */
 443                 num_inv = 2;
 444                 inv_reqs[0].flags = 0;
 445                 inv_reqs[0].iotlb_id = 0;
 446                 inv_reqs[1].flags = 0;
 447                 inv_reqs[1].iotlb_id = MOCK_NESTED_DOMAIN_IOTLB_ID_MAX + 1;
 448                 test_err_hwpt_invalidate(EINVAL, nested_hwpt_id[0], inv_reqs,
 449                                          IOMMU_HWPT_INVALIDATE_DATA_SELFTEST,
 450                                          sizeof(*inv_reqs), &num_inv);
 451                 assert(num_inv == 1);
 452                 test_cmd_hwpt_check_iotlb(nested_hwpt_id[0], 0, 0);
 453                 test_cmd_hwpt_check_iotlb(nested_hwpt_id[0], 1,
 454                                           IOMMU_TEST_IOTLB_DEFAULT);
 455                 test_cmd_hwpt_check_iotlb(nested_hwpt_id[0], 2,
 456                                           IOMMU_TEST_IOTLB_DEFAULT);
 457                 test_cmd_hwpt_check_iotlb(nested_hwpt_id[0], 3,
 458                                           IOMMU_TEST_IOTLB_DEFAULT);
 459
 460                 /* Invalidate the 2nd iotlb entry and verify */
 461                 num_inv = 1;
 462                 inv_reqs[0].flags = 0;
 463                 inv_reqs[0].iotlb_id = 1;
 464                 test_cmd_hwpt_invalidate(nested_hwpt_id[0], inv_reqs,
 465                                          IOMMU_HWPT_INVALIDATE_DATA_SELFTEST,
 466                                          sizeof(*inv_reqs), &num_inv);
 467                 assert(num_inv == 1);
 468                 test_cmd_hwpt_check_iotlb(nested_hwpt_id[0], 0, 0);
 469                 test_cmd_hwpt_check_iotlb(nested_hwpt_id[0], 1, 0);
 470                 test_cmd_hwpt_check_iotlb(nested_hwpt_id[0], 2,
 471                                           IOMMU_TEST_IOTLB_DEFAULT);
 472                 test_cmd_hwpt_check_iotlb(nested_hwpt_id[0], 3,
 473                                           IOMMU_TEST_IOTLB_DEFAULT);
 474
 475                 /* Invalidate the 3rd and 4th iotlb entries and verify */
 476                 num_inv = 2;
 477                 inv_reqs[0].flags = 0;
 478                 inv_reqs[0].iotlb_id = 2;
 479                 inv_reqs[1].flags = 0;
 480                 inv_reqs[1].iotlb_id = 3;
 481                 test_cmd_hwpt_invalidate(nested_hwpt_id[0], inv_reqs,
 482                                          IOMMU_HWPT_INVALIDATE_DATA_SELFTEST,
 483                                          sizeof(*inv_reqs), &num_inv);
 484                 assert(num_inv == 2);
 485                 test_cmd_hwpt_check_iotlb_all(nested_hwpt_id[0], 0);
 486
 487                 /* Invalidate all iotlb entries for nested_hwpt_id[1] and verify */
 488                 num_inv = 1;
 489                 inv_reqs[0].flags = IOMMU_TEST_INVALIDATE_FLAG_ALL;
 490                 test_cmd_hwpt_invalidate(nested_hwpt_id[1], inv_reqs,
 491                                          IOMMU_HWPT_INVALIDATE_DATA_SELFTEST,
 492                                          sizeof(*inv_reqs), &num_inv);
 493                 assert(num_inv == 1);
 494                 test_cmd_hwpt_check_iotlb_all(nested_hwpt_id[1], 0);
 495
 496                 /* Attach device to nested_hwpt_id[0] that then will be busy */
 497                 test_cmd_mock_domain_replace(self->stdev_id, nested_hwpt_id[0]);
 498                 EXPECT_ERRNO(EBUSY,
 499                              _test_ioctl_destroy(self->fd, nested_hwpt_id[0]));
 500
 501                 /* Switch from nested_hwpt_id[0] to nested_hwpt_id[1] */
 502                 test_cmd_mock_domain_replace(self->stdev_id, nested_hwpt_id[1]);
 503                 EXPECT_ERRNO(EBUSY,
 504                              _test_ioctl_destroy(self->fd, nested_hwpt_id[1]));
 505                 test_ioctl_destroy(nested_hwpt_id[0]);
 506
 507                 /* Detach from nested_hwpt_id[1] and destroy it */
 508                 test_cmd_mock_domain_replace(self->stdev_id, parent_hwpt_id);
 509                 test_ioctl_destroy(nested_hwpt_id[1]);
 510
 511                 /* Detach from the parent hw_pagetable and destroy it */
 512                 test_cmd_mock_domain_replace(self->stdev_id, self->ioas_id);
 513                 test_ioctl_destroy(parent_hwpt_id);
 514                 test_ioctl_destroy(parent_hwpt_id_not_work);
 515         } else {
 516                 test_err_hwpt_alloc(ENOENT, self->device_id, self->ioas_id, 0,
 517                                     &parent_hwpt_id);
 518                 test_err_hwpt_alloc_nested(ENOENT, self->device_id,
 519                                            parent_hwpt_id, 0,
 520                                            &nested_hwpt_id[0],
 521                                            IOMMU_HWPT_DATA_SELFTEST, &data,
 522                                            sizeof(data));
 523                 test_err_hwpt_alloc_nested(ENOENT, self->device_id,
 524                                            parent_hwpt_id, 0,
 525                                            &nested_hwpt_id[1],
 526                                            IOMMU_HWPT_DATA_SELFTEST, &data,
 527                                            sizeof(data));
 528                 test_err_mock_domain_replace(ENOENT, self->stdev_id,
 529                                              nested_hwpt_id[0]);
 530                 test_err_mock_domain_replace(ENOENT, self->stdev_id,
 531                                              nested_hwpt_id[1]);
 532         }
 533 }
 534
 535 TEST_F(iommufd_ioas, hwpt_attach)
 536 {
 537         /* Create a device attached directly to a hwpt */
 538         if (self->stdev_id) {
 539                 test_cmd_mock_domain(self->hwpt_id, NULL, NULL, NULL);
 540         } else {
 541                 test_err_mock_domain(ENOENT, self->hwpt_id, NULL, NULL);
 542         }
 543 }
 544
 545 TEST_F(iommufd_ioas, ioas_area_destroy)
 546 {
 547         /* Adding an area does not change ability to destroy */
 548         test_ioctl_ioas_map_fixed(buffer, PAGE_SIZE, self->base_iova);
 549         if (self->stdev_id)
 550                 EXPECT_ERRNO(EBUSY,
 551                              _test_ioctl_destroy(self->fd, self->ioas_id));
 552         else
 553                 test_ioctl_destroy(self->ioas_id);
 554 }
 555
 556 TEST_F(iommufd_ioas, ioas_area_auto_destroy)
 557 {
 558         int i;
 559
 560         /* Can allocate and automatically free an IOAS table with many areas */
 561         for (i = 0; i != 10; i++) {
 562                 test_ioctl_ioas_map_fixed(buffer, PAGE_SIZE,
 563                                           self->base_iova + i * PAGE_SIZE);
 564         }
 565 }
 566
 567 TEST_F(iommufd_ioas, get_hw_info)
 568 {
 569         struct iommu_test_hw_info buffer_exact;
 570         struct iommu_test_hw_info_buffer_larger {
 571                 struct iommu_test_hw_info info;
 572                 uint64_t trailing_bytes;
 573         } buffer_larger;
 574         struct iommu_test_hw_info_buffer_smaller {
 575                 __u32 flags;
 576         } buffer_smaller;
 577
 578         if (self->device_id) {
 579                 /* Provide a zero-size user_buffer */
 580                 test_cmd_get_hw_info(self->device_id, NULL, 0);
 581                 /* Provide a user_buffer with exact size */
 582                 test_cmd_get_hw_info(self->device_id, &buffer_exact, sizeof(buffer_exact));
 583                 /*
 584                  * Provide a user_buffer with size larger than the exact size to check if
 585                  * kernel zero the trailing bytes.
 586                  */
 587                 test_cmd_get_hw_info(self->device_id, &buffer_larger, sizeof(buffer_larger));
 588                 /*
 589                  * Provide a user_buffer with size smaller than the exact size to check if
 590                  * the fields within the size range still gets updated.
 591                  */
 592                 test_cmd_get_hw_info(self->device_id, &buffer_smaller, sizeof(buffer_smaller));
 593         } else {
 594                 test_err_get_hw_info(ENOENT, self->device_id,
 595                                      &buffer_exact, sizeof(buffer_exact));
 596                 test_err_get_hw_info(ENOENT, self->device_id,
 597                                      &buffer_larger, sizeof(buffer_larger));
 598         }
 599 }
 600
 601 TEST_F(iommufd_ioas, area)
 602 {
 603         int i;
 604
 605         /* Unmap fails if nothing is mapped */
 606         for (i = 0; i != 10; i++)
 607                 test_err_ioctl_ioas_unmap(ENOENT, i * PAGE_SIZE, PAGE_SIZE);
 608
 609         /* Unmap works */
 610         for (i = 0; i != 10; i++)
 611                 test_ioctl_ioas_map_fixed(buffer, PAGE_SIZE,
 612                                           self->base_iova + i * PAGE_SIZE);
 613         for (i = 0; i != 10; i++)
 614                 test_ioctl_ioas_unmap(self->base_iova + i * PAGE_SIZE,
 615                                       PAGE_SIZE);
 616
 617         /* Split fails */
 618         test_ioctl_ioas_map_fixed(buffer, PAGE_SIZE * 2,
 619                                   self->base_iova + 16 * PAGE_SIZE);
 620         test_err_ioctl_ioas_unmap(ENOENT, self->base_iova + 16 * PAGE_SIZE,
 621                                   PAGE_SIZE);
 622         test_err_ioctl_ioas_unmap(ENOENT, self->base_iova + 17 * PAGE_SIZE,
 623                                   PAGE_SIZE);
 624
 625         /* Over map fails */
 626         test_err_ioctl_ioas_map_fixed(EEXIST, buffer, PAGE_SIZE * 2,
 627                                       self->base_iova + 16 * PAGE_SIZE);
 628         test_err_ioctl_ioas_map_fixed(EEXIST, buffer, PAGE_SIZE,
 629                                       self->base_iova + 16 * PAGE_SIZE);
 630         test_err_ioctl_ioas_map_fixed(EEXIST, buffer, PAGE_SIZE,
 631                                       self->base_iova + 17 * PAGE_SIZE);
 632         test_err_ioctl_ioas_map_fixed(EEXIST, buffer, PAGE_SIZE * 2,
 633                                       self->base_iova + 15 * PAGE_SIZE);
 634         test_err_ioctl_ioas_map_fixed(EEXIST, buffer, PAGE_SIZE * 3,
 635                                       self->base_iova + 15 * PAGE_SIZE);
 636
 637         /* unmap all works */
 638         test_ioctl_ioas_unmap(0, UINT64_MAX);
 639
 640         /* Unmap all succeeds on an empty IOAS */
 641         test_ioctl_ioas_unmap(0, UINT64_MAX);
 642 }
 643
 644 TEST_F(iommufd_ioas, unmap_fully_contained_areas)
 645 {
 646         uint64_t unmap_len;
 647         int i;
 648
 649         /* Give no_domain some space to rewind base_iova */
 650         self->base_iova += 4 * PAGE_SIZE;
 651
 652         for (i = 0; i != 4; i++)
 653                 test_ioctl_ioas_map_fixed(buffer, 8 * PAGE_SIZE,
 654                                           self->base_iova + i * 16 * PAGE_SIZE);
 655
 656         /* Unmap not fully contained area doesn't work */
 657         test_err_ioctl_ioas_unmap(ENOENT, self->base_iova - 4 * PAGE_SIZE,
 658                                   8 * PAGE_SIZE);
 659         test_err_ioctl_ioas_unmap(ENOENT,
 660                                   self->base_iova + 3 * 16 * PAGE_SIZE +
 661                                           8 * PAGE_SIZE - 4 * PAGE_SIZE,
 662                                   8 * PAGE_SIZE);
 663
 664         /* Unmap fully contained areas works */
 665         ASSERT_EQ(0, _test_ioctl_ioas_unmap(self->fd, self->ioas_id,
 666                                             self->base_iova - 4 * PAGE_SIZE,
 667                                             3 * 16 * PAGE_SIZE + 8 * PAGE_SIZE +
 668                                                     4 * PAGE_SIZE,
 669                                             &unmap_len));
 670         ASSERT_EQ(32 * PAGE_SIZE, unmap_len);
 671 }
 672
 673 TEST_F(iommufd_ioas, area_auto_iova)
 674 {
 675         struct iommu_test_cmd test_cmd = {
 676                 .size = sizeof(test_cmd),
 677                 .op = IOMMU_TEST_OP_ADD_RESERVED,
 678                 .id = self->ioas_id,
 679                 .add_reserved = { .start = PAGE_SIZE * 4,
 680                                   .length = PAGE_SIZE * 100 },
 681         };
 682         struct iommu_iova_range ranges[1] = {};
 683         struct iommu_ioas_allow_iovas allow_cmd = {
 684                 .size = sizeof(allow_cmd),
 685                 .ioas_id = self->ioas_id,
 686                 .num_iovas = 1,
 687                 .allowed_iovas = (uintptr_t)ranges,
 688         };
 689         __u64 iovas[10];
 690         int i;
 691
 692         /* Simple 4k pages */
 693         for (i = 0; i != 10; i++)
 694                 test_ioctl_ioas_map(buffer, PAGE_SIZE, &iovas[i]);
 695         for (i = 0; i != 10; i++)
 696                 test_ioctl_ioas_unmap(iovas[i], PAGE_SIZE);
 697
 698         /* Kernel automatically aligns IOVAs properly */
 699         for (i = 0; i != 10; i++) {
 700                 size_t length = PAGE_SIZE * (i + 1);
 701
 702                 if (self->stdev_id) {
 703                         test_ioctl_ioas_map(buffer, length, &iovas[i]);
 704                 } else {
 705                         test_ioctl_ioas_map((void *)(1UL << 31), length,
 706                                             &iovas[i]);
 707                 }
 708                 EXPECT_EQ(0, iovas[i] % (1UL << (ffs(length) - 1)));
 709         }
 710         for (i = 0; i != 10; i++)
 711                 test_ioctl_ioas_unmap(iovas[i], PAGE_SIZE * (i + 1));
 712
 713         /* Avoids a reserved region */
 714         ASSERT_EQ(0,
 715                   ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ADD_RESERVED),
 716                         &test_cmd));
 717         for (i = 0; i != 10; i++) {
 718                 size_t length = PAGE_SIZE * (i + 1);
 719
 720                 test_ioctl_ioas_map(buffer, length, &iovas[i]);
 721                 EXPECT_EQ(0, iovas[i] % (1UL << (ffs(length) - 1)));
 722                 EXPECT_EQ(false,
 723                           iovas[i] > test_cmd.add_reserved.start &&
 724                                   iovas[i] <
 725                                           test_cmd.add_reserved.start +
 726                                                   test_cmd.add_reserved.length);
 727         }
 728         for (i = 0; i != 10; i++)
 729                 test_ioctl_ioas_unmap(iovas[i], PAGE_SIZE * (i + 1));
 730
 731         /* Allowed region intersects with a reserved region */
 732         ranges[0].start = PAGE_SIZE;
 733         ranges[0].last = PAGE_SIZE * 600;
 734         EXPECT_ERRNO(EADDRINUSE,
 735                      ioctl(self->fd, IOMMU_IOAS_ALLOW_IOVAS, &allow_cmd));
 736
 737         /* Allocate from an allowed region */
 738         if (self->stdev_id) {
 739                 ranges[0].start = MOCK_APERTURE_START + PAGE_SIZE;
 740                 ranges[0].last = MOCK_APERTURE_START + PAGE_SIZE * 600 - 1;
 741         } else {
 742                 ranges[0].start = PAGE_SIZE * 200;
 743                 ranges[0].last = PAGE_SIZE * 600 - 1;
 744         }
 745         ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_ALLOW_IOVAS, &allow_cmd));
 746         for (i = 0; i != 10; i++) {
 747                 size_t length = PAGE_SIZE * (i + 1);
 748
 749                 test_ioctl_ioas_map(buffer, length, &iovas[i]);
 750                 EXPECT_EQ(0, iovas[i] % (1UL << (ffs(length) - 1)));
 751                 EXPECT_EQ(true, iovas[i] >= ranges[0].start);
 752                 EXPECT_EQ(true, iovas[i] <= ranges[0].last);
 753                 EXPECT_EQ(true, iovas[i] + length > ranges[0].start);
 754                 EXPECT_EQ(true, iovas[i] + length <= ranges[0].last + 1);
 755         }
 756         for (i = 0; i != 10; i++)
 757                 test_ioctl_ioas_unmap(iovas[i], PAGE_SIZE * (i + 1));
 758 }
 759
 760 TEST_F(iommufd_ioas, area_allowed)
 761 {
 762         struct iommu_test_cmd test_cmd = {
 763                 .size = sizeof(test_cmd),
 764                 .op = IOMMU_TEST_OP_ADD_RESERVED,
 765                 .id = self->ioas_id,
 766                 .add_reserved = { .start = PAGE_SIZE * 4,
 767                                   .length = PAGE_SIZE * 100 },
 768         };
 769         struct iommu_iova_range ranges[1] = {};
 770         struct iommu_ioas_allow_iovas allow_cmd = {
 771                 .size = sizeof(allow_cmd),
 772                 .ioas_id = self->ioas_id,
 773                 .num_iovas = 1,
 774                 .allowed_iovas = (uintptr_t)ranges,
 775         };
 776
 777         /* Reserved intersects an allowed */
 778         allow_cmd.num_iovas = 1;
 779         ranges[0].start = self->base_iova;
 780         ranges[0].last = ranges[0].start + PAGE_SIZE * 600;
 781         ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_ALLOW_IOVAS, &allow_cmd));
 782         test_cmd.add_reserved.start = ranges[0].start + PAGE_SIZE;
 783         test_cmd.add_reserved.length = PAGE_SIZE;
 784         EXPECT_ERRNO(EADDRINUSE,
 785                      ioctl(self->fd,
 786                            _IOMMU_TEST_CMD(IOMMU_TEST_OP_ADD_RESERVED),
 787                            &test_cmd));
 788         allow_cmd.num_iovas = 0;
 789         ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_ALLOW_IOVAS, &allow_cmd));
 790
 791         /* Allowed intersects a reserved */
 792         ASSERT_EQ(0,
 793                   ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ADD_RESERVED),
 794                         &test_cmd));
 795         allow_cmd.num_iovas = 1;
 796         ranges[0].start = self->base_iova;
 797         ranges[0].last = ranges[0].start + PAGE_SIZE * 600;
 798         EXPECT_ERRNO(EADDRINUSE,
 799                      ioctl(self->fd, IOMMU_IOAS_ALLOW_IOVAS, &allow_cmd));
 800 }
 801
 802 TEST_F(iommufd_ioas, copy_area)
 803 {
 804         struct iommu_ioas_copy copy_cmd = {
 805                 .size = sizeof(copy_cmd),
 806                 .flags = IOMMU_IOAS_MAP_FIXED_IOVA,
 807                 .dst_ioas_id = self->ioas_id,
 808                 .src_ioas_id = self->ioas_id,
 809                 .length = PAGE_SIZE,
 810         };
 811
 812         test_ioctl_ioas_map_fixed(buffer, PAGE_SIZE, self->base_iova);
 813
 814         /* Copy inside a single IOAS */
 815         copy_cmd.src_iova = self->base_iova;
 816         copy_cmd.dst_iova = self->base_iova + PAGE_SIZE;
 817         ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_COPY, &copy_cmd));
 818
 819         /* Copy between IOAS's */
 820         copy_cmd.src_iova = self->base_iova;
 821         copy_cmd.dst_iova = 0;
 822         test_ioctl_ioas_alloc(&copy_cmd.dst_ioas_id);
 823         ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_COPY, &copy_cmd));
 824 }
 825
 826 TEST_F(iommufd_ioas, iova_ranges)
 827 {
 828         struct iommu_test_cmd test_cmd = {
 829                 .size = sizeof(test_cmd),
 830                 .op = IOMMU_TEST_OP_ADD_RESERVED,
 831                 .id = self->ioas_id,
 832                 .add_reserved = { .start = PAGE_SIZE, .length = PAGE_SIZE },
 833         };
 834         struct iommu_iova_range *ranges = buffer;
 835         struct iommu_ioas_iova_ranges ranges_cmd = {
 836                 .size = sizeof(ranges_cmd),
 837                 .ioas_id = self->ioas_id,
 838                 .num_iovas = BUFFER_SIZE / sizeof(*ranges),
 839                 .allowed_iovas = (uintptr_t)ranges,
 840         };
 841
 842         /* Range can be read */
 843         ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_IOVA_RANGES, &ranges_cmd));
 844         EXPECT_EQ(1, ranges_cmd.num_iovas);
 845         if (!self->stdev_id) {
 846                 EXPECT_EQ(0, ranges[0].start);
 847                 EXPECT_EQ(SIZE_MAX, ranges[0].last);
 848                 EXPECT_EQ(1, ranges_cmd.out_iova_alignment);
 849         } else {
 850                 EXPECT_EQ(MOCK_APERTURE_START, ranges[0].start);
 851                 EXPECT_EQ(MOCK_APERTURE_LAST, ranges[0].last);
 852                 EXPECT_EQ(MOCK_PAGE_SIZE, ranges_cmd.out_iova_alignment);
 853         }
 854
 855         /* Buffer too small */
 856         memset(ranges, 0, BUFFER_SIZE);
 857         ranges_cmd.num_iovas = 0;
 858         EXPECT_ERRNO(EMSGSIZE,
 859                      ioctl(self->fd, IOMMU_IOAS_IOVA_RANGES, &ranges_cmd));
 860         EXPECT_EQ(1, ranges_cmd.num_iovas);
 861         EXPECT_EQ(0, ranges[0].start);
 862         EXPECT_EQ(0, ranges[0].last);
 863
 864         /* 2 ranges */
 865         ASSERT_EQ(0,
 866                   ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ADD_RESERVED),
 867                         &test_cmd));
 868         ranges_cmd.num_iovas = BUFFER_SIZE / sizeof(*ranges);
 869         ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_IOVA_RANGES, &ranges_cmd));
 870         if (!self->stdev_id) {
 871                 EXPECT_EQ(2, ranges_cmd.num_iovas);
 872                 EXPECT_EQ(0, ranges[0].start);
 873                 EXPECT_EQ(PAGE_SIZE - 1, ranges[0].last);
 874                 EXPECT_EQ(PAGE_SIZE * 2, ranges[1].start);
 875                 EXPECT_EQ(SIZE_MAX, ranges[1].last);
 876         } else {
 877                 EXPECT_EQ(1, ranges_cmd.num_iovas);
 878                 EXPECT_EQ(MOCK_APERTURE_START, ranges[0].start);
 879                 EXPECT_EQ(MOCK_APERTURE_LAST, ranges[0].last);
 880         }
 881
 882         /* Buffer too small */
 883         memset(ranges, 0, BUFFER_SIZE);
 884         ranges_cmd.num_iovas = 1;
 885         if (!self->stdev_id) {
 886                 EXPECT_ERRNO(EMSGSIZE, ioctl(self->fd, IOMMU_IOAS_IOVA_RANGES,
 887                                              &ranges_cmd));
 888                 EXPECT_EQ(2, ranges_cmd.num_iovas);
 889                 EXPECT_EQ(0, ranges[0].start);
 890                 EXPECT_EQ(PAGE_SIZE - 1, ranges[0].last);
 891         } else {
 892                 ASSERT_EQ(0,
 893                           ioctl(self->fd, IOMMU_IOAS_IOVA_RANGES, &ranges_cmd));
 894                 EXPECT_EQ(1, ranges_cmd.num_iovas);
 895                 EXPECT_EQ(MOCK_APERTURE_START, ranges[0].start);
 896                 EXPECT_EQ(MOCK_APERTURE_LAST, ranges[0].last);
 897         }
 898         EXPECT_EQ(0, ranges[1].start);
 899         EXPECT_EQ(0, ranges[1].last);
 900 }
 901
 902 TEST_F(iommufd_ioas, access_domain_destory)
 903 {
 904         struct iommu_test_cmd access_cmd = {
 905                 .size = sizeof(access_cmd),
 906                 .op = IOMMU_TEST_OP_ACCESS_PAGES,
 907                 .access_pages = { .iova = self->base_iova + PAGE_SIZE,
 908                                   .length = PAGE_SIZE},
 909         };
 910         size_t buf_size = 2 * HUGEPAGE_SIZE;
 911         uint8_t *buf;
 912
 913         buf = mmap(0, buf_size, PROT_READ | PROT_WRITE,
 914                    MAP_SHARED | MAP_ANONYMOUS | MAP_HUGETLB | MAP_POPULATE, -1,
 915                    0);
 916         ASSERT_NE(MAP_FAILED, buf);
 917         test_ioctl_ioas_map_fixed(buf, buf_size, self->base_iova);
 918
 919         test_cmd_create_access(self->ioas_id, &access_cmd.id,
 920                                MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES);
 921         access_cmd.access_pages.uptr = (uintptr_t)buf + PAGE_SIZE;
 922         ASSERT_EQ(0,
 923                   ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES),
 924                         &access_cmd));
 925
 926         /* Causes a complicated unpin across a huge page boundary */
 927         if (self->stdev_id)
 928                 test_ioctl_destroy(self->stdev_id);
 929
 930         test_cmd_destroy_access_pages(
 931                 access_cmd.id, access_cmd.access_pages.out_access_pages_id);
 932         test_cmd_destroy_access(access_cmd.id);
 933         ASSERT_EQ(0, munmap(buf, buf_size));
 934 }
 935
 936 TEST_F(iommufd_ioas, access_pin)
 937 {
 938         struct iommu_test_cmd access_cmd = {
 939                 .size = sizeof(access_cmd),
 940                 .op = IOMMU_TEST_OP_ACCESS_PAGES,
 941                 .access_pages = { .iova = MOCK_APERTURE_START,
 942                                   .length = BUFFER_SIZE,
 943                                   .uptr = (uintptr_t)buffer },
 944         };
 945         struct iommu_test_cmd check_map_cmd = {
 946                 .size = sizeof(check_map_cmd),
 947                 .op = IOMMU_TEST_OP_MD_CHECK_MAP,
 948                 .check_map = { .iova = MOCK_APERTURE_START,
 949                                .length = BUFFER_SIZE,
 950                                .uptr = (uintptr_t)buffer },
 951         };
 952         uint32_t access_pages_id;
 953         unsigned int npages;
 954
 955         test_cmd_create_access(self->ioas_id, &access_cmd.id,
 956                                MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES);
 957
 958         for (npages = 1; npages < BUFFER_SIZE / PAGE_SIZE; npages++) {
 959                 uint32_t mock_stdev_id;
 960                 uint32_t mock_hwpt_id;
 961
 962                 access_cmd.access_pages.length = npages * PAGE_SIZE;
 963
 964                 /* Single map/unmap */
 965                 test_ioctl_ioas_map_fixed(buffer, BUFFER_SIZE,
 966                                           MOCK_APERTURE_START);
 967                 ASSERT_EQ(0, ioctl(self->fd,
 968                                    _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES),
 969                                    &access_cmd));
 970                 test_cmd_destroy_access_pages(
 971                         access_cmd.id,
 972                         access_cmd.access_pages.out_access_pages_id);
 973
 974                 /* Double user */
 975                 ASSERT_EQ(0, ioctl(self->fd,
 976                                    _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES),
 977                                    &access_cmd));
 978                 access_pages_id = access_cmd.access_pages.out_access_pages_id;
 979                 ASSERT_EQ(0, ioctl(self->fd,
 980                                    _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES),
 981                                    &access_cmd));
 982                 test_cmd_destroy_access_pages(
 983                         access_cmd.id,
 984                         access_cmd.access_pages.out_access_pages_id);
 985                 test_cmd_destroy_access_pages(access_cmd.id, access_pages_id);
 986
 987                 /* Add/remove a domain with a user */
 988                 ASSERT_EQ(0, ioctl(self->fd,
 989                                    _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES),
 990                                    &access_cmd));
 991                 test_cmd_mock_domain(self->ioas_id, &mock_stdev_id,
 992                                      &mock_hwpt_id, NULL);
 993                 check_map_cmd.id = mock_hwpt_id;
 994                 ASSERT_EQ(0, ioctl(self->fd,
 995                                    _IOMMU_TEST_CMD(IOMMU_TEST_OP_MD_CHECK_MAP),
 996                                    &check_map_cmd));
 997
 998                 test_ioctl_destroy(mock_stdev_id);
 999                 test_cmd_destroy_access_pages(
1000                         access_cmd.id,
1001                         access_cmd.access_pages.out_access_pages_id);
1002
1003                 test_ioctl_ioas_unmap(MOCK_APERTURE_START, BUFFER_SIZE);
1004         }
1005         test_cmd_destroy_access(access_cmd.id);
1006 }
1007
1008 TEST_F(iommufd_ioas, access_pin_unmap)
1009 {
1010         struct iommu_test_cmd access_pages_cmd = {
1011                 .size = sizeof(access_pages_cmd),
1012                 .op = IOMMU_TEST_OP_ACCESS_PAGES,
1013                 .access_pages = { .iova = MOCK_APERTURE_START,
1014                                   .length = BUFFER_SIZE,
1015                                   .uptr = (uintptr_t)buffer },
1016         };
1017
1018         test_cmd_create_access(self->ioas_id, &access_pages_cmd.id,
1019                                MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES);
1020         test_ioctl_ioas_map_fixed(buffer, BUFFER_SIZE, MOCK_APERTURE_START);
1021         ASSERT_EQ(0,
1022                   ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES),
1023                         &access_pages_cmd));
1024
1025         /* Trigger the unmap op */
1026         test_ioctl_ioas_unmap(MOCK_APERTURE_START, BUFFER_SIZE);
1027
1028         /* kernel removed the item for us */
1029         test_err_destroy_access_pages(
1030                 ENOENT, access_pages_cmd.id,
1031                 access_pages_cmd.access_pages.out_access_pages_id);
1032 }
1033
1034 static void check_access_rw(struct __test_metadata *_metadata, int fd,
1035                             unsigned int access_id, uint64_t iova,
1036                             unsigned int def_flags)
1037 {
1038         uint16_t tmp[32];
1039         struct iommu_test_cmd access_cmd = {
1040                 .size = sizeof(access_cmd),
1041                 .op = IOMMU_TEST_OP_ACCESS_RW,
1042                 .id = access_id,
1043                 .access_rw = { .uptr = (uintptr_t)tmp },
1044         };
1045         uint16_t *buffer16 = buffer;
1046         unsigned int i;
1047         void *tmp2;
1048
1049         for (i = 0; i != BUFFER_SIZE / sizeof(*buffer16); i++)
1050                 buffer16[i] = rand();
1051
1052         for (access_cmd.access_rw.iova = iova + PAGE_SIZE - 50;
1053              access_cmd.access_rw.iova < iova + PAGE_SIZE + 50;
1054              access_cmd.access_rw.iova++) {
1055                 for (access_cmd.access_rw.length = 1;
1056                      access_cmd.access_rw.length < sizeof(tmp);
1057                      access_cmd.access_rw.length++) {
1058                         access_cmd.access_rw.flags = def_flags;
1059                         ASSERT_EQ(0, ioctl(fd,
1060                                            _IOMMU_TEST_CMD(
1061                                                    IOMMU_TEST_OP_ACCESS_RW),
1062                                            &access_cmd));
1063                         ASSERT_EQ(0,
1064                                   memcmp(buffer + (access_cmd.access_rw.iova -
1065                                                    iova),
1066                                          tmp, access_cmd.access_rw.length));
1067
1068                         for (i = 0; i != ARRAY_SIZE(tmp); i++)
1069                                 tmp[i] = rand();
1070                         access_cmd.access_rw.flags = def_flags |
1071                                                      MOCK_ACCESS_RW_WRITE;
1072                         ASSERT_EQ(0, ioctl(fd,
1073                                            _IOMMU_TEST_CMD(
1074                                                    IOMMU_TEST_OP_ACCESS_RW),
1075                                            &access_cmd));
1076                         ASSERT_EQ(0,
1077                                   memcmp(buffer + (access_cmd.access_rw.iova -
1078                                                    iova),
1079                                          tmp, access_cmd.access_rw.length));
1080                 }
1081         }
1082
1083         /* Multi-page test */
1084         tmp2 = malloc(BUFFER_SIZE);
1085         ASSERT_NE(NULL, tmp2);
1086         access_cmd.access_rw.iova = iova;
1087         access_cmd.access_rw.length = BUFFER_SIZE;
1088         access_cmd.access_rw.flags = def_flags;
1089         access_cmd.access_rw.uptr = (uintptr_t)tmp2;
1090         ASSERT_EQ(0, ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_RW),
1091                            &access_cmd));
1092         ASSERT_EQ(0, memcmp(buffer, tmp2, access_cmd.access_rw.length));
1093         free(tmp2);
1094 }
1095
1096 TEST_F(iommufd_ioas, access_rw)
1097 {
1098         __u32 access_id;
1099         __u64 iova;
1100
1101         test_cmd_create_access(self->ioas_id, &access_id, 0);
1102         test_ioctl_ioas_map(buffer, BUFFER_SIZE, &iova);
1103         check_access_rw(_metadata, self->fd, access_id, iova, 0);
1104         check_access_rw(_metadata, self->fd, access_id, iova,
1105                         MOCK_ACCESS_RW_SLOW_PATH);
1106         test_ioctl_ioas_unmap(iova, BUFFER_SIZE);
1107         test_cmd_destroy_access(access_id);
1108 }
1109
1110 TEST_F(iommufd_ioas, access_rw_unaligned)
1111 {
1112         __u32 access_id;
1113         __u64 iova;
1114
1115         test_cmd_create_access(self->ioas_id, &access_id, 0);
1116
1117         /* Unaligned pages */
1118         iova = self->base_iova + MOCK_PAGE_SIZE;
1119         test_ioctl_ioas_map_fixed(buffer, BUFFER_SIZE, iova);
1120         check_access_rw(_metadata, self->fd, access_id, iova, 0);
1121         test_ioctl_ioas_unmap(iova, BUFFER_SIZE);
1122         test_cmd_destroy_access(access_id);
1123 }
1124
1125 TEST_F(iommufd_ioas, fork_gone)
1126 {
1127         __u32 access_id;
1128         pid_t child;
1129
1130         test_cmd_create_access(self->ioas_id, &access_id, 0);
1131
1132         /* Create a mapping with a different mm */
1133         child = fork();
1134         if (!child) {
1135                 test_ioctl_ioas_map_fixed(buffer, BUFFER_SIZE,
1136                                           MOCK_APERTURE_START);
1137                 exit(0);
1138         }
1139         ASSERT_NE(-1, child);
1140         ASSERT_EQ(child, waitpid(child, NULL, 0));
1141
1142         if (self->stdev_id) {
1143                 /*
1144                  * If a domain already existed then everything was pinned within
1145                  * the fork, so this copies from one domain to another.
1146                  */
1147                 test_cmd_mock_domain(self->ioas_id, NULL, NULL, NULL);
1148                 check_access_rw(_metadata, self->fd, access_id,
1149                                 MOCK_APERTURE_START, 0);
1150
1151         } else {
1152                 /*
1153                  * Otherwise we need to actually pin pages which can't happen
1154                  * since the fork is gone.
1155                  */
1156                 test_err_mock_domain(EFAULT, self->ioas_id, NULL, NULL);
1157         }
1158
1159         test_cmd_destroy_access(access_id);
1160 }
1161
1162 TEST_F(iommufd_ioas, fork_present)
1163 {
1164         __u32 access_id;
1165         int pipefds[2];
1166         uint64_t tmp;
1167         pid_t child;
1168         int efd;
1169
1170         test_cmd_create_access(self->ioas_id, &access_id, 0);
1171
1172         ASSERT_EQ(0, pipe2(pipefds, O_CLOEXEC));
1173         efd = eventfd(0, EFD_CLOEXEC);
1174         ASSERT_NE(-1, efd);
1175
1176         /* Create a mapping with a different mm */
1177         child = fork();
1178         if (!child) {
1179                 __u64 iova;
1180                 uint64_t one = 1;
1181
1182                 close(pipefds[1]);
1183                 test_ioctl_ioas_map_fixed(buffer, BUFFER_SIZE,
1184                                           MOCK_APERTURE_START);
1185                 if (write(efd, &one, sizeof(one)) != sizeof(one))
1186                         exit(100);
1187                 if (read(pipefds[0], &iova, 1) != 1)
1188                         exit(100);
1189                 exit(0);
1190         }
1191         close(pipefds[0]);
1192         ASSERT_NE(-1, child);
1193         ASSERT_EQ(8, read(efd, &tmp, sizeof(tmp)));
1194
1195         /* Read pages from the remote process */
1196         test_cmd_mock_domain(self->ioas_id, NULL, NULL, NULL);
1197         check_access_rw(_metadata, self->fd, access_id, MOCK_APERTURE_START, 0);
1198
1199         ASSERT_EQ(0, close(pipefds[1]));
1200         ASSERT_EQ(child, waitpid(child, NULL, 0));
1201
1202         test_cmd_destroy_access(access_id);
1203 }
1204
1205 TEST_F(iommufd_ioas, ioas_option_huge_pages)
1206 {
1207         struct iommu_option cmd = {
1208                 .size = sizeof(cmd),
1209                 .option_id = IOMMU_OPTION_HUGE_PAGES,
1210                 .op = IOMMU_OPTION_OP_GET,
1211                 .val64 = 3,
1212                 .object_id = self->ioas_id,
1213         };
1214
1215         ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
1216         ASSERT_EQ(1, cmd.val64);
1217
1218         cmd.op = IOMMU_OPTION_OP_SET;
1219         cmd.val64 = 0;
1220         ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
1221
1222         cmd.op = IOMMU_OPTION_OP_GET;
1223         cmd.val64 = 3;
1224         ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
1225         ASSERT_EQ(0, cmd.val64);
1226
1227         cmd.op = IOMMU_OPTION_OP_SET;
1228         cmd.val64 = 2;
1229         EXPECT_ERRNO(EINVAL, ioctl(self->fd, IOMMU_OPTION, &cmd));
1230
1231         cmd.op = IOMMU_OPTION_OP_SET;
1232         cmd.val64 = 1;
1233         ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
1234 }
1235
1236 TEST_F(iommufd_ioas, ioas_iova_alloc)
1237 {
1238         unsigned int length;
1239         __u64 iova;
1240
1241         for (length = 1; length != PAGE_SIZE * 2; length++) {
1242                 if (variant->mock_domains && (length % MOCK_PAGE_SIZE)) {
1243                         test_err_ioctl_ioas_map(EINVAL, buffer, length, &iova);
1244                 } else {
1245                         test_ioctl_ioas_map(buffer, length, &iova);
1246                         test_ioctl_ioas_unmap(iova, length);
1247                 }
1248         }
1249 }
1250
1251 TEST_F(iommufd_ioas, ioas_align_change)
1252 {
1253         struct iommu_option cmd = {
1254                 .size = sizeof(cmd),
1255                 .option_id = IOMMU_OPTION_HUGE_PAGES,
1256                 .op = IOMMU_OPTION_OP_SET,
1257                 .object_id = self->ioas_id,
1258                 /* 0 means everything must be aligned to PAGE_SIZE */
1259                 .val64 = 0,
1260         };
1261
1262         /*
1263          * We cannot upgrade the alignment using OPTION_HUGE_PAGES when a domain
1264          * and map are present.
1265          */
1266         if (variant->mock_domains)
1267                 return;
1268
1269         /*
1270          * We can upgrade to PAGE_SIZE alignment when things are aligned right
1271          */
1272         test_ioctl_ioas_map_fixed(buffer, PAGE_SIZE, MOCK_APERTURE_START);
1273         ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
1274
1275         /* Misalignment is rejected at map time */
1276         test_err_ioctl_ioas_map_fixed(EINVAL, buffer + MOCK_PAGE_SIZE,
1277                                       PAGE_SIZE,
1278                                       MOCK_APERTURE_START + PAGE_SIZE);
1279         ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
1280
1281         /* Reduce alignment */
1282         cmd.val64 = 1;
1283         ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
1284
1285         /* Confirm misalignment is rejected during alignment upgrade */
1286         test_ioctl_ioas_map_fixed(buffer + MOCK_PAGE_SIZE, PAGE_SIZE,
1287                                   MOCK_APERTURE_START + PAGE_SIZE);
1288         cmd.val64 = 0;
1289         EXPECT_ERRNO(EADDRINUSE, ioctl(self->fd, IOMMU_OPTION, &cmd));
1290
1291         test_ioctl_ioas_unmap(MOCK_APERTURE_START + PAGE_SIZE, PAGE_SIZE);
1292         test_ioctl_ioas_unmap(MOCK_APERTURE_START, PAGE_SIZE);
1293 }
1294
1295 TEST_F(iommufd_ioas, copy_sweep)
1296 {
1297         struct iommu_ioas_copy copy_cmd = {
1298                 .size = sizeof(copy_cmd),
1299                 .flags = IOMMU_IOAS_MAP_FIXED_IOVA,
1300                 .src_ioas_id = self->ioas_id,
1301                 .dst_iova = MOCK_APERTURE_START,
1302                 .length = MOCK_PAGE_SIZE,
1303         };
1304         unsigned int dst_ioas_id;
1305         uint64_t last_iova;
1306         uint64_t iova;
1307
1308         test_ioctl_ioas_alloc(&dst_ioas_id);
1309         copy_cmd.dst_ioas_id = dst_ioas_id;
1310
1311         if (variant->mock_domains)
1312                 last_iova = MOCK_APERTURE_START + BUFFER_SIZE - 1;
1313         else
1314                 last_iova = MOCK_APERTURE_START + BUFFER_SIZE - 2;
1315
1316         test_ioctl_ioas_map_fixed(buffer, last_iova - MOCK_APERTURE_START + 1,
1317                                   MOCK_APERTURE_START);
1318
1319         for (iova = MOCK_APERTURE_START - PAGE_SIZE; iova <= last_iova;
1320              iova += 511) {
1321                 copy_cmd.src_iova = iova;
1322                 if (iova < MOCK_APERTURE_START ||
1323                     iova + copy_cmd.length - 1 > last_iova) {
1324                         EXPECT_ERRNO(ENOENT, ioctl(self->fd, IOMMU_IOAS_COPY,
1325                                                    &copy_cmd));
1326                 } else {
1327                         ASSERT_EQ(0,
1328                                   ioctl(self->fd, IOMMU_IOAS_COPY, &copy_cmd));
1329                         test_ioctl_ioas_unmap_id(dst_ioas_id, copy_cmd.dst_iova,
1330                                                  copy_cmd.length);
1331                 }
1332         }
1333
1334         test_ioctl_destroy(dst_ioas_id);
1335 }
1336
1337 FIXTURE(iommufd_mock_domain)
1338 {
1339         int fd;
1340         uint32_t ioas_id;
1341         uint32_t hwpt_id;
1342         uint32_t hwpt_ids[2];
1343         uint32_t stdev_ids[2];
1344         uint32_t idev_ids[2];
1345         int mmap_flags;
1346         size_t mmap_buf_size;
1347 };
1348
1349 FIXTURE_VARIANT(iommufd_mock_domain)
1350 {
1351         unsigned int mock_domains;
1352         bool hugepages;
1353 };
1354
1355 FIXTURE_SETUP(iommufd_mock_domain)
1356 {
1357         unsigned int i;
1358
1359         self->fd = open("/dev/iommu", O_RDWR);
1360         ASSERT_NE(-1, self->fd);
1361         test_ioctl_ioas_alloc(&self->ioas_id);
1362
1363         ASSERT_GE(ARRAY_SIZE(self->hwpt_ids), variant->mock_domains);
1364
1365         for (i = 0; i != variant->mock_domains; i++)
1366                 test_cmd_mock_domain(self->ioas_id, &self->stdev_ids[i],
1367                                      &self->hwpt_ids[i], &self->idev_ids[i]);
1368         self->hwpt_id = self->hwpt_ids[0];
1369
1370         self->mmap_flags = MAP_SHARED | MAP_ANONYMOUS;
1371         self->mmap_buf_size = PAGE_SIZE * 8;
1372         if (variant->hugepages) {
1373                 /*
1374                  * MAP_POPULATE will cause the kernel to fail mmap if THPs are
1375                  * not available.
1376                  */
1377                 self->mmap_flags |= MAP_HUGETLB | MAP_POPULATE;
1378                 self->mmap_buf_size = HUGEPAGE_SIZE * 2;
1379         }
1380 }
1381
1382 FIXTURE_TEARDOWN(iommufd_mock_domain)
1383 {
1384         teardown_iommufd(self->fd, _metadata);
1385 }
1386
1387 FIXTURE_VARIANT_ADD(iommufd_mock_domain, one_domain)
1388 {
1389         .mock_domains = 1,
1390         .hugepages = false,
1391 };
1392
1393 FIXTURE_VARIANT_ADD(iommufd_mock_domain, two_domains)
1394 {
1395         .mock_domains = 2,
1396         .hugepages = false,
1397 };
1398
1399 FIXTURE_VARIANT_ADD(iommufd_mock_domain, one_domain_hugepage)
1400 {
1401         .mock_domains = 1,
1402         .hugepages = true,
1403 };
1404
1405 FIXTURE_VARIANT_ADD(iommufd_mock_domain, two_domains_hugepage)
1406 {
1407         .mock_domains = 2,
1408         .hugepages = true,
1409 };
1410
1411 /* Have the kernel check that the user pages made it to the iommu_domain */
1412 #define check_mock_iova(_ptr, _iova, _length)                                \
1413         ({                                                                   \
1414                 struct iommu_test_cmd check_map_cmd = {                      \
1415                         .size = sizeof(check_map_cmd),                       \
1416                         .op = IOMMU_TEST_OP_MD_CHECK_MAP,                    \
1417                         .id = self->hwpt_id,                                 \
1418                         .check_map = { .iova = _iova,                        \
1419                                        .length = _length,                    \
1420                                        .uptr = (uintptr_t)(_ptr) },          \
1421                 };                                                           \
1422                 ASSERT_EQ(0,                                                 \
1423                           ioctl(self->fd,                                    \
1424                                 _IOMMU_TEST_CMD(IOMMU_TEST_OP_MD_CHECK_MAP), \
1425                                 &check_map_cmd));                            \
1426                 if (self->hwpt_ids[1]) {                                     \
1427                         check_map_cmd.id = self->hwpt_ids[1];                \
1428                         ASSERT_EQ(0,                                         \
1429                                   ioctl(self->fd,                            \
1430                                         _IOMMU_TEST_CMD(                     \
1431                                                 IOMMU_TEST_OP_MD_CHECK_MAP), \
1432                                         &check_map_cmd));                    \
1433                 }                                                            \
1434         })
1435
1436 TEST_F(iommufd_mock_domain, basic)
1437 {
1438         size_t buf_size = self->mmap_buf_size;
1439         uint8_t *buf;
1440         __u64 iova;
1441
1442         /* Simple one page map */
1443         test_ioctl_ioas_map(buffer, PAGE_SIZE, &iova);
1444         check_mock_iova(buffer, iova, PAGE_SIZE);
1445
1446         buf = mmap(0, buf_size, PROT_READ | PROT_WRITE, self->mmap_flags, -1,
1447                    0);
1448         ASSERT_NE(MAP_FAILED, buf);
1449
1450         /* EFAULT half way through mapping */
1451         ASSERT_EQ(0, munmap(buf + buf_size / 2, buf_size / 2));
1452         test_err_ioctl_ioas_map(EFAULT, buf, buf_size, &iova);
1453
1454         /* EFAULT on first page */
1455         ASSERT_EQ(0, munmap(buf, buf_size / 2));
1456         test_err_ioctl_ioas_map(EFAULT, buf, buf_size, &iova);
1457 }
1458
1459 TEST_F(iommufd_mock_domain, ro_unshare)
1460 {
1461         uint8_t *buf;
1462         __u64 iova;
1463         int fd;
1464
1465         fd = open("/proc/self/exe", O_RDONLY);
1466         ASSERT_NE(-1, fd);
1467
1468         buf = mmap(0, PAGE_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
1469         ASSERT_NE(MAP_FAILED, buf);
1470         close(fd);
1471
1472         /*
1473          * There have been lots of changes to the "unshare" mechanism in
1474          * get_user_pages(), make sure it works right. The write to the page
1475          * after we map it for reading should not change the assigned PFN.
1476          */
1477         ASSERT_EQ(0,
1478                   _test_ioctl_ioas_map(self->fd, self->ioas_id, buf, PAGE_SIZE,
1479                                        &iova, IOMMU_IOAS_MAP_READABLE));
1480         check_mock_iova(buf, iova, PAGE_SIZE);
1481         memset(buf, 1, PAGE_SIZE);
1482         check_mock_iova(buf, iova, PAGE_SIZE);
1483         ASSERT_EQ(0, munmap(buf, PAGE_SIZE));
1484 }
1485
1486 TEST_F(iommufd_mock_domain, all_aligns)
1487 {
1488         size_t test_step = variant->hugepages ? (self->mmap_buf_size / 16) :
1489                                                 MOCK_PAGE_SIZE;
1490         size_t buf_size = self->mmap_buf_size;
1491         unsigned int start;
1492         unsigned int end;
1493         uint8_t *buf;
1494
1495         buf = mmap(0, buf_size, PROT_READ | PROT_WRITE, self->mmap_flags, -1,
1496                    0);
1497         ASSERT_NE(MAP_FAILED, buf);
1498         check_refs(buf, buf_size, 0);
1499
1500         /*
1501          * Map every combination of page size and alignment within a big region,
1502          * less for hugepage case as it takes so long to finish.
1503          */
1504         for (start = 0; start < buf_size; start += test_step) {
1505                 if (variant->hugepages)
1506                         end = buf_size;
1507                 else
1508                         end = start + MOCK_PAGE_SIZE;
1509                 for (; end < buf_size; end += MOCK_PAGE_SIZE) {
1510                         size_t length = end - start;
1511                         __u64 iova;
1512
1513                         test_ioctl_ioas_map(buf + start, length, &iova);
1514                         check_mock_iova(buf + start, iova, length);
1515                         check_refs(buf + start / PAGE_SIZE * PAGE_SIZE,
1516                                    end / PAGE_SIZE * PAGE_SIZE -
1517                                            start / PAGE_SIZE * PAGE_SIZE,
1518                                    1);
1519
1520                         test_ioctl_ioas_unmap(iova, length);
1521                 }
1522         }
1523         check_refs(buf, buf_size, 0);
1524         ASSERT_EQ(0, munmap(buf, buf_size));
1525 }
1526
1527 TEST_F(iommufd_mock_domain, all_aligns_copy)
1528 {
1529         size_t test_step = variant->hugepages ? self->mmap_buf_size / 16 :
1530                                                 MOCK_PAGE_SIZE;
1531         size_t buf_size = self->mmap_buf_size;
1532         unsigned int start;
1533         unsigned int end;
1534         uint8_t *buf;
1535
1536         buf = mmap(0, buf_size, PROT_READ | PROT_WRITE, self->mmap_flags, -1,
1537                    0);
1538         ASSERT_NE(MAP_FAILED, buf);
1539         check_refs(buf, buf_size, 0);
1540
1541         /*
1542          * Map every combination of page size and alignment within a big region,
1543          * less for hugepage case as it takes so long to finish.
1544          */
1545         for (start = 0; start < buf_size; start += test_step) {
1546                 if (variant->hugepages)
1547                         end = buf_size;
1548                 else
1549                         end = start + MOCK_PAGE_SIZE;
1550                 for (; end < buf_size; end += MOCK_PAGE_SIZE) {
1551                         size_t length = end - start;
1552                         unsigned int old_id;
1553                         uint32_t mock_stdev_id;
1554                         __u64 iova;
1555
1556                         test_ioctl_ioas_map(buf + start, length, &iova);
1557
1558                         /* Add and destroy a domain while the area exists */
1559                         old_id = self->hwpt_ids[1];
1560                         test_cmd_mock_domain(self->ioas_id, &mock_stdev_id,
1561                                              &self->hwpt_ids[1], NULL);
1562
1563                         check_mock_iova(buf + start, iova, length);
1564                         check_refs(buf + start / PAGE_SIZE * PAGE_SIZE,
1565                                    end / PAGE_SIZE * PAGE_SIZE -
1566                                            start / PAGE_SIZE * PAGE_SIZE,
1567                                    1);
1568
1569                         test_ioctl_destroy(mock_stdev_id);
1570                         self->hwpt_ids[1] = old_id;
1571
1572                         test_ioctl_ioas_unmap(iova, length);
1573                 }
1574         }
1575         check_refs(buf, buf_size, 0);
1576         ASSERT_EQ(0, munmap(buf, buf_size));
1577 }
1578
1579 TEST_F(iommufd_mock_domain, user_copy)
1580 {
1581         struct iommu_test_cmd access_cmd = {
1582                 .size = sizeof(access_cmd),
1583                 .op = IOMMU_TEST_OP_ACCESS_PAGES,
1584                 .access_pages = { .length = BUFFER_SIZE,
1585                                   .uptr = (uintptr_t)buffer },
1586         };
1587         struct iommu_ioas_copy copy_cmd = {
1588                 .size = sizeof(copy_cmd),
1589                 .flags = IOMMU_IOAS_MAP_FIXED_IOVA,
1590                 .dst_ioas_id = self->ioas_id,
1591                 .dst_iova = MOCK_APERTURE_START,
1592                 .length = BUFFER_SIZE,
1593         };
1594         struct iommu_ioas_unmap unmap_cmd = {
1595                 .size = sizeof(unmap_cmd),
1596                 .ioas_id = self->ioas_id,
1597                 .iova = MOCK_APERTURE_START,
1598                 .length = BUFFER_SIZE,
1599         };
1600         unsigned int new_ioas_id, ioas_id;
1601
1602         /* Pin the pages in an IOAS with no domains then copy to an IOAS with domains */
1603         test_ioctl_ioas_alloc(&ioas_id);
1604         test_ioctl_ioas_map_id(ioas_id, buffer, BUFFER_SIZE,
1605                                &copy_cmd.src_iova);
1606
1607         test_cmd_create_access(ioas_id, &access_cmd.id,
1608                                MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES);
1609
1610         access_cmd.access_pages.iova = copy_cmd.src_iova;
1611         ASSERT_EQ(0,
1612                   ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES),
1613                         &access_cmd));
1614         copy_cmd.src_ioas_id = ioas_id;
1615         ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_COPY, &copy_cmd));
1616         check_mock_iova(buffer, MOCK_APERTURE_START, BUFFER_SIZE);
1617
1618         /* Now replace the ioas with a new one */
1619         test_ioctl_ioas_alloc(&new_ioas_id);
1620         test_ioctl_ioas_map_id(new_ioas_id, buffer, BUFFER_SIZE,
1621                                &copy_cmd.src_iova);
1622         test_cmd_access_replace_ioas(access_cmd.id, new_ioas_id);
1623
1624         /* Destroy the old ioas and cleanup copied mapping */
1625         ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_UNMAP, &unmap_cmd));
1626         test_ioctl_destroy(ioas_id);
1627
1628         /* Then run the same test again with the new ioas */
1629         access_cmd.access_pages.iova = copy_cmd.src_iova;
1630         ASSERT_EQ(0,
1631                   ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES),
1632                         &access_cmd));
1633         copy_cmd.src_ioas_id = new_ioas_id;
1634         ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_COPY, &copy_cmd));
1635         check_mock_iova(buffer, MOCK_APERTURE_START, BUFFER_SIZE);
1636
1637         test_cmd_destroy_access_pages(
1638                 access_cmd.id, access_cmd.access_pages.out_access_pages_id);
1639         test_cmd_destroy_access(access_cmd.id);
1640
1641         test_ioctl_destroy(new_ioas_id);
1642 }
1643
1644 TEST_F(iommufd_mock_domain, replace)
1645 {
1646         uint32_t ioas_id;
1647
1648         test_ioctl_ioas_alloc(&ioas_id);
1649
1650         test_cmd_mock_domain_replace(self->stdev_ids[0], ioas_id);
1651
1652         /*
1653          * Replacing the IOAS causes the prior HWPT to be deallocated, thus we
1654          * should get enoent when we try to use it.
1655          */
1656         if (variant->mock_domains == 1)
1657                 test_err_mock_domain_replace(ENOENT, self->stdev_ids[0],
1658                                              self->hwpt_ids[0]);
1659
1660         test_cmd_mock_domain_replace(self->stdev_ids[0], ioas_id);
1661         if (variant->mock_domains >= 2) {
1662                 test_cmd_mock_domain_replace(self->stdev_ids[0],
1663                                              self->hwpt_ids[1]);
1664                 test_cmd_mock_domain_replace(self->stdev_ids[0],
1665                                              self->hwpt_ids[1]);
1666                 test_cmd_mock_domain_replace(self->stdev_ids[0],
1667                                              self->hwpt_ids[0]);
1668         }
1669
1670         test_cmd_mock_domain_replace(self->stdev_ids[0], self->ioas_id);
1671         test_ioctl_destroy(ioas_id);
1672 }
1673
1674 TEST_F(iommufd_mock_domain, alloc_hwpt)
1675 {
1676         int i;
1677
1678         for (i = 0; i != variant->mock_domains; i++) {
1679                 uint32_t hwpt_id[2];
1680                 uint32_t stddev_id;
1681
1682                 test_err_hwpt_alloc(EOPNOTSUPP,
1683                                     self->idev_ids[i], self->ioas_id,
1684                                     ~IOMMU_HWPT_ALLOC_NEST_PARENT, &hwpt_id[0]);
1685                 test_cmd_hwpt_alloc(self->idev_ids[i], self->ioas_id,
1686                                     0, &hwpt_id[0]);
1687                 test_cmd_hwpt_alloc(self->idev_ids[i], self->ioas_id,
1688                                     IOMMU_HWPT_ALLOC_NEST_PARENT, &hwpt_id[1]);
1689
1690                 /* Do a hw_pagetable rotation test */
1691                 test_cmd_mock_domain_replace(self->stdev_ids[i], hwpt_id[0]);
1692                 EXPECT_ERRNO(EBUSY, _test_ioctl_destroy(self->fd, hwpt_id[0]));
1693                 test_cmd_mock_domain_replace(self->stdev_ids[i], hwpt_id[1]);
1694                 EXPECT_ERRNO(EBUSY, _test_ioctl_destroy(self->fd, hwpt_id[1]));
1695                 test_cmd_mock_domain_replace(self->stdev_ids[i], self->ioas_id);
1696                 test_ioctl_destroy(hwpt_id[1]);
1697
1698                 test_cmd_mock_domain(hwpt_id[0], &stddev_id, NULL, NULL);
1699                 test_ioctl_destroy(stddev_id);
1700                 test_ioctl_destroy(hwpt_id[0]);
1701         }
1702 }
1703
1704 FIXTURE(iommufd_dirty_tracking)
1705 {
1706         int fd;
1707         uint32_t ioas_id;
1708         uint32_t hwpt_id;
1709         uint32_t stdev_id;
1710         uint32_t idev_id;
1711         unsigned long page_size;
1712         unsigned long bitmap_size;
1713         void *bitmap;
1714         void *buffer;
1715 };
1716
1717 FIXTURE_VARIANT(iommufd_dirty_tracking)
1718 {
1719         unsigned long buffer_size;
1720         bool hugepages;
1721 };
1722
1723 FIXTURE_SETUP(iommufd_dirty_tracking)
1724 {
1725         int mmap_flags;
1726         void *vrc;
1727         int rc;
1728
1729         self->fd = open("/dev/iommu", O_RDWR);
1730         ASSERT_NE(-1, self->fd);
1731
1732         rc = posix_memalign(&self->buffer, HUGEPAGE_SIZE, variant->buffer_size);
1733         if (rc || !self->buffer) {
1734                 SKIP(return, "Skipping buffer_size=%lu due to errno=%d",
1735                            variant->buffer_size, rc);
1736         }
1737
1738         mmap_flags = MAP_SHARED | MAP_ANONYMOUS | MAP_FIXED;
1739         if (variant->hugepages) {
1740                 /*
1741                  * MAP_POPULATE will cause the kernel to fail mmap if THPs are
1742                  * not available.
1743                  */
1744                 mmap_flags |= MAP_HUGETLB | MAP_POPULATE;
1745         }
1746         assert((uintptr_t)self->buffer % HUGEPAGE_SIZE == 0);
1747         vrc = mmap(self->buffer, variant->buffer_size, PROT_READ | PROT_WRITE,
1748                    mmap_flags, -1, 0);
1749         assert(vrc == self->buffer);
1750
1751         self->page_size = MOCK_PAGE_SIZE;
1752         self->bitmap_size =
1753                 variant->buffer_size / self->page_size / BITS_PER_BYTE;
1754
1755         /* Provision with an extra (PAGE_SIZE) for the unaligned case */
1756         rc = posix_memalign(&self->bitmap, PAGE_SIZE,
1757                             self->bitmap_size + PAGE_SIZE);
1758         assert(!rc);
1759         assert(self->bitmap);
1760         assert((uintptr_t)self->bitmap % PAGE_SIZE == 0);
1761
1762         test_ioctl_ioas_alloc(&self->ioas_id);
1763         /* Enable 1M mock IOMMU hugepages */
1764         if (variant->hugepages) {
1765                 test_cmd_mock_domain_flags(self->ioas_id,
1766                                            MOCK_FLAGS_DEVICE_HUGE_IOVA,
1767                                            &self->stdev_id, &self->hwpt_id,
1768                                            &self->idev_id);
1769         } else {
1770                 test_cmd_mock_domain(self->ioas_id, &self->stdev_id,
1771                                      &self->hwpt_id, &self->idev_id);
1772         }
1773 }
1774
1775 FIXTURE_TEARDOWN(iommufd_dirty_tracking)
1776 {
1777         munmap(self->buffer, variant->buffer_size);
1778         munmap(self->bitmap, self->bitmap_size);
1779         teardown_iommufd(self->fd, _metadata);
1780 }
1781
1782 FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty128k)
1783 {
1784         /* one u32 index bitmap */
1785         .buffer_size = 128UL * 1024UL,
1786 };
1787
1788 FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty256k)
1789 {
1790         /* one u64 index bitmap */
1791         .buffer_size = 256UL * 1024UL,
1792 };
1793
1794 FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty640k)
1795 {
1796         /* two u64 index and trailing end bitmap */
1797         .buffer_size = 640UL * 1024UL,
1798 };
1799
1800 FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty128M)
1801 {
1802         /* 4K bitmap (128M IOVA range) */
1803         .buffer_size = 128UL * 1024UL * 1024UL,
1804 };
1805
1806 FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty128M_huge)
1807 {
1808         /* 4K bitmap (128M IOVA range) */
1809         .buffer_size = 128UL * 1024UL * 1024UL,
1810         .hugepages = true,
1811 };
1812
1813 FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty256M)
1814 {
1815         /* 8K bitmap (256M IOVA range) */
1816         .buffer_size = 256UL * 1024UL * 1024UL,
1817 };
1818
1819 FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty256M_huge)
1820 {
1821         /* 8K bitmap (256M IOVA range) */
1822         .buffer_size = 256UL * 1024UL * 1024UL,
1823         .hugepages = true,
1824 };
1825
1826 TEST_F(iommufd_dirty_tracking, enforce_dirty)
1827 {
1828         uint32_t ioas_id, stddev_id, idev_id;
1829         uint32_t hwpt_id, _hwpt_id;
1830         uint32_t dev_flags;
1831
1832         /* Regular case */
1833         dev_flags = MOCK_FLAGS_DEVICE_NO_DIRTY;
1834         test_cmd_hwpt_alloc(self->idev_id, self->ioas_id,
1835                             IOMMU_HWPT_ALLOC_DIRTY_TRACKING, &hwpt_id);
1836         test_cmd_mock_domain(hwpt_id, &stddev_id, NULL, NULL);
1837         test_err_mock_domain_flags(EINVAL, hwpt_id, dev_flags, &stddev_id,
1838                                    NULL);
1839         test_ioctl_destroy(stddev_id);
1840         test_ioctl_destroy(hwpt_id);
1841
1842         /* IOMMU device does not support dirty tracking */
1843         test_ioctl_ioas_alloc(&ioas_id);
1844         test_cmd_mock_domain_flags(ioas_id, dev_flags, &stddev_id, &_hwpt_id,
1845                                    &idev_id);
1846         test_err_hwpt_alloc(EOPNOTSUPP, idev_id, ioas_id,
1847                             IOMMU_HWPT_ALLOC_DIRTY_TRACKING, &hwpt_id);
1848         test_ioctl_destroy(stddev_id);
1849 }
1850
1851 TEST_F(iommufd_dirty_tracking, set_dirty_tracking)
1852 {
1853         uint32_t stddev_id;
1854         uint32_t hwpt_id;
1855
1856         test_cmd_hwpt_alloc(self->idev_id, self->ioas_id,
1857                             IOMMU_HWPT_ALLOC_DIRTY_TRACKING, &hwpt_id);
1858         test_cmd_mock_domain(hwpt_id, &stddev_id, NULL, NULL);
1859         test_cmd_set_dirty_tracking(hwpt_id, true);
1860         test_cmd_set_dirty_tracking(hwpt_id, false);
1861
1862         test_ioctl_destroy(stddev_id);
1863         test_ioctl_destroy(hwpt_id);
1864 }
1865
1866 TEST_F(iommufd_dirty_tracking, device_dirty_capability)
1867 {
1868         uint32_t caps = 0;
1869         uint32_t stddev_id;
1870         uint32_t hwpt_id;
1871
1872         test_cmd_hwpt_alloc(self->idev_id, self->ioas_id, 0, &hwpt_id);
1873         test_cmd_mock_domain(hwpt_id, &stddev_id, NULL, NULL);
1874         test_cmd_get_hw_capabilities(self->idev_id, caps,
1875                                      IOMMU_HW_CAP_DIRTY_TRACKING);
1876         ASSERT_EQ(IOMMU_HW_CAP_DIRTY_TRACKING,
1877                   caps & IOMMU_HW_CAP_DIRTY_TRACKING);
1878
1879         test_ioctl_destroy(stddev_id);
1880         test_ioctl_destroy(hwpt_id);
1881 }
1882
1883 TEST_F(iommufd_dirty_tracking, get_dirty_bitmap)
1884 {
1885         uint32_t page_size = MOCK_PAGE_SIZE;
1886         uint32_t hwpt_id;
1887         uint32_t ioas_id;
1888
1889         if (variant->hugepages)
1890                 page_size = MOCK_HUGE_PAGE_SIZE;
1891
1892         test_ioctl_ioas_alloc(&ioas_id);
1893         test_ioctl_ioas_map_fixed_id(ioas_id, self->buffer,
1894                                      variant->buffer_size, MOCK_APERTURE_START);
1895
1896         test_cmd_hwpt_alloc(self->idev_id, ioas_id,
1897                             IOMMU_HWPT_ALLOC_DIRTY_TRACKING, &hwpt_id);
1898
1899         test_cmd_set_dirty_tracking(hwpt_id, true);
1900
1901         test_mock_dirty_bitmaps(hwpt_id, variant->buffer_size,
1902                                 MOCK_APERTURE_START, self->page_size, page_size,
1903                                 self->bitmap, self->bitmap_size, 0, _metadata);
1904
1905         /* PAGE_SIZE unaligned bitmap */
1906         test_mock_dirty_bitmaps(hwpt_id, variant->buffer_size,
1907                                 MOCK_APERTURE_START, self->page_size, page_size,
1908                                 self->bitmap + MOCK_PAGE_SIZE,
1909                                 self->bitmap_size, 0, _metadata);
1910
1911         /* u64 unaligned bitmap */
1912         test_mock_dirty_bitmaps(hwpt_id, variant->buffer_size,
1913                                 MOCK_APERTURE_START, self->page_size, page_size,
1914                                 self->bitmap + 0xff1, self->bitmap_size, 0,
1915                                 _metadata);
1916
1917         test_ioctl_destroy(hwpt_id);
1918 }
1919
1920 TEST_F(iommufd_dirty_tracking, get_dirty_bitmap_no_clear)
1921 {
1922         uint32_t page_size = MOCK_PAGE_SIZE;
1923         uint32_t hwpt_id;
1924         uint32_t ioas_id;
1925
1926         if (variant->hugepages)
1927                 page_size = MOCK_HUGE_PAGE_SIZE;
1928
1929         test_ioctl_ioas_alloc(&ioas_id);
1930         test_ioctl_ioas_map_fixed_id(ioas_id, self->buffer,
1931                                      variant->buffer_size, MOCK_APERTURE_START);
1932
1933         test_cmd_hwpt_alloc(self->idev_id, ioas_id,
1934                             IOMMU_HWPT_ALLOC_DIRTY_TRACKING, &hwpt_id);
1935
1936         test_cmd_set_dirty_tracking(hwpt_id, true);
1937
1938         test_mock_dirty_bitmaps(hwpt_id, variant->buffer_size,
1939                                 MOCK_APERTURE_START, self->page_size, page_size,
1940                                 self->bitmap, self->bitmap_size,
1941                                 IOMMU_HWPT_GET_DIRTY_BITMAP_NO_CLEAR,
1942                                 _metadata);
1943
1944         /* Unaligned bitmap */
1945         test_mock_dirty_bitmaps(hwpt_id, variant->buffer_size,
1946                                 MOCK_APERTURE_START, self->page_size, page_size,
1947                                 self->bitmap + MOCK_PAGE_SIZE,
1948                                 self->bitmap_size,
1949                                 IOMMU_HWPT_GET_DIRTY_BITMAP_NO_CLEAR,
1950                                 _metadata);
1951
1952         /* u64 unaligned bitmap */
1953         test_mock_dirty_bitmaps(hwpt_id, variant->buffer_size,
1954                                 MOCK_APERTURE_START, self->page_size, page_size,
1955                                 self->bitmap + 0xff1, self->bitmap_size,
1956                                 IOMMU_HWPT_GET_DIRTY_BITMAP_NO_CLEAR,
1957                                 _metadata);
1958
1959         test_ioctl_destroy(hwpt_id);
1960 }
1961
1962 /* VFIO compatibility IOCTLs */
1963
1964 TEST_F(iommufd, simple_ioctls)
1965 {
1966         ASSERT_EQ(VFIO_API_VERSION, ioctl(self->fd, VFIO_GET_API_VERSION));
1967         ASSERT_EQ(1, ioctl(self->fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1v2_IOMMU));
1968 }
1969
1970 TEST_F(iommufd, unmap_cmd)
1971 {
1972         struct vfio_iommu_type1_dma_unmap unmap_cmd = {
1973                 .iova = MOCK_APERTURE_START,
1974                 .size = PAGE_SIZE,
1975         };
1976
1977         unmap_cmd.argsz = 1;
1978         EXPECT_ERRNO(EINVAL, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd));
1979
1980         unmap_cmd.argsz = sizeof(unmap_cmd);
1981         unmap_cmd.flags = 1 << 31;
1982         EXPECT_ERRNO(EINVAL, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd));
1983
1984         unmap_cmd.flags = 0;
1985         EXPECT_ERRNO(ENODEV, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd));
1986 }
1987
1988 TEST_F(iommufd, map_cmd)
1989 {
1990         struct vfio_iommu_type1_dma_map map_cmd = {
1991                 .iova = MOCK_APERTURE_START,
1992                 .size = PAGE_SIZE,
1993                 .vaddr = (__u64)buffer,
1994         };
1995
1996         map_cmd.argsz = 1;
1997         EXPECT_ERRNO(EINVAL, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd));
1998
1999         map_cmd.argsz = sizeof(map_cmd);
2000         map_cmd.flags = 1 << 31;
2001         EXPECT_ERRNO(EINVAL, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd));
2002
2003         /* Requires a domain to be attached */
2004         map_cmd.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE;
2005         EXPECT_ERRNO(ENODEV, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd));
2006 }
2007
2008 TEST_F(iommufd, info_cmd)
2009 {
2010         struct vfio_iommu_type1_info info_cmd = {};
2011
2012         /* Invalid argsz */
2013         info_cmd.argsz = 1;
2014         EXPECT_ERRNO(EINVAL, ioctl(self->fd, VFIO_IOMMU_GET_INFO, &info_cmd));
2015
2016         info_cmd.argsz = sizeof(info_cmd);
2017         EXPECT_ERRNO(ENODEV, ioctl(self->fd, VFIO_IOMMU_GET_INFO, &info_cmd));
2018 }
2019
2020 TEST_F(iommufd, set_iommu_cmd)
2021 {
2022         /* Requires a domain to be attached */
2023         EXPECT_ERRNO(ENODEV,
2024                      ioctl(self->fd, VFIO_SET_IOMMU, VFIO_TYPE1v2_IOMMU));
2025         EXPECT_ERRNO(ENODEV, ioctl(self->fd, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU));
2026 }
2027
2028 TEST_F(iommufd, vfio_ioas)
2029 {
2030         struct iommu_vfio_ioas vfio_ioas_cmd = {
2031                 .size = sizeof(vfio_ioas_cmd),
2032                 .op = IOMMU_VFIO_IOAS_GET,
2033         };
2034         __u32 ioas_id;
2035
2036         /* ENODEV if there is no compat ioas */
2037         EXPECT_ERRNO(ENODEV, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd));
2038
2039         /* Invalid id for set */
2040         vfio_ioas_cmd.op = IOMMU_VFIO_IOAS_SET;
2041         EXPECT_ERRNO(ENOENT, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd));
2042
2043         /* Valid id for set*/
2044         test_ioctl_ioas_alloc(&ioas_id);
2045         vfio_ioas_cmd.ioas_id = ioas_id;
2046         ASSERT_EQ(0, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd));
2047
2048         /* Same id comes back from get */
2049         vfio_ioas_cmd.op = IOMMU_VFIO_IOAS_GET;
2050         ASSERT_EQ(0, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd));
2051         ASSERT_EQ(ioas_id, vfio_ioas_cmd.ioas_id);
2052
2053         /* Clear works */
2054         vfio_ioas_cmd.op = IOMMU_VFIO_IOAS_CLEAR;
2055         ASSERT_EQ(0, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd));
2056         vfio_ioas_cmd.op = IOMMU_VFIO_IOAS_GET;
2057         EXPECT_ERRNO(ENODEV, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd));
2058 }
2059
2060 FIXTURE(vfio_compat_mock_domain)
2061 {
2062         int fd;
2063         uint32_t ioas_id;
2064 };
2065
2066 FIXTURE_VARIANT(vfio_compat_mock_domain)
2067 {
2068         unsigned int version;
2069 };
2070
2071 FIXTURE_SETUP(vfio_compat_mock_domain)
2072 {
2073         struct iommu_vfio_ioas vfio_ioas_cmd = {
2074                 .size = sizeof(vfio_ioas_cmd),
2075                 .op = IOMMU_VFIO_IOAS_SET,
2076         };
2077
2078         self->fd = open("/dev/iommu", O_RDWR);
2079         ASSERT_NE(-1, self->fd);
2080
2081         /* Create what VFIO would consider a group */
2082         test_ioctl_ioas_alloc(&self->ioas_id);
2083         test_cmd_mock_domain(self->ioas_id, NULL, NULL, NULL);
2084
2085         /* Attach it to the vfio compat */
2086         vfio_ioas_cmd.ioas_id = self->ioas_id;
2087         ASSERT_EQ(0, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd));
2088         ASSERT_EQ(0, ioctl(self->fd, VFIO_SET_IOMMU, variant->version));
2089 }
2090
2091 FIXTURE_TEARDOWN(vfio_compat_mock_domain)
2092 {
2093         teardown_iommufd(self->fd, _metadata);
2094 }
2095
2096 FIXTURE_VARIANT_ADD(vfio_compat_mock_domain, Ver1v2)
2097 {
2098         .version = VFIO_TYPE1v2_IOMMU,
2099 };
2100
2101 FIXTURE_VARIANT_ADD(vfio_compat_mock_domain, Ver1v0)
2102 {
2103         .version = VFIO_TYPE1_IOMMU,
2104 };
2105
2106 TEST_F(vfio_compat_mock_domain, simple_close)
2107 {
2108 }
2109
2110 TEST_F(vfio_compat_mock_domain, option_huge_pages)
2111 {
2112         struct iommu_option cmd = {
2113                 .size = sizeof(cmd),
2114                 .option_id = IOMMU_OPTION_HUGE_PAGES,
2115                 .op = IOMMU_OPTION_OP_GET,
2116                 .val64 = 3,
2117                 .object_id = self->ioas_id,
2118         };
2119
2120         ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd));
2121         if (variant->version == VFIO_TYPE1_IOMMU) {
2122                 ASSERT_EQ(0, cmd.val64);
2123         } else {
2124                 ASSERT_EQ(1, cmd.val64);
2125         }
2126 }
2127
2128 /*
2129  * Execute an ioctl command stored in buffer and check that the result does not
2130  * overflow memory.
2131  */
2132 static bool is_filled(const void *buf, uint8_t c, size_t len)
2133 {
2134         const uint8_t *cbuf = buf;
2135
2136         for (; len; cbuf++, len--)
2137                 if (*cbuf != c)
2138                         return false;
2139         return true;
2140 }
2141
2142 #define ioctl_check_buf(fd, cmd)                                         \
2143         ({                                                               \
2144                 size_t _cmd_len = *(__u32 *)buffer;                      \
2145                                                                          \
2146                 memset(buffer + _cmd_len, 0xAA, BUFFER_SIZE - _cmd_len); \
2147                 ASSERT_EQ(0, ioctl(fd, cmd, buffer));                    \
2148                 ASSERT_EQ(true, is_filled(buffer + _cmd_len, 0xAA,       \
2149                                           BUFFER_SIZE - _cmd_len));      \
2150         })
2151
2152 static void check_vfio_info_cap_chain(struct __test_metadata *_metadata,
2153                                       struct vfio_iommu_type1_info *info_cmd)
2154 {
2155         const struct vfio_info_cap_header *cap;
2156
2157         ASSERT_GE(info_cmd->argsz, info_cmd->cap_offset + sizeof(*cap));
2158         cap = buffer + info_cmd->cap_offset;
2159         while (true) {
2160                 size_t cap_size;
2161
2162                 if (cap->next)
2163                         cap_size = (buffer + cap->next) - (void *)cap;
2164                 else
2165                         cap_size = (buffer + info_cmd->argsz) - (void *)cap;
2166
2167                 switch (cap->id) {
2168                 case VFIO_IOMMU_TYPE1_INFO_CAP_IOVA_RANGE: {
2169                         struct vfio_iommu_type1_info_cap_iova_range *data =
2170                                 (void *)cap;
2171
2172                         ASSERT_EQ(1, data->header.version);
2173                         ASSERT_EQ(1, data->nr_iovas);
2174                         EXPECT_EQ(MOCK_APERTURE_START,
2175                                   data->iova_ranges[0].start);
2176                         EXPECT_EQ(MOCK_APERTURE_LAST, data->iova_ranges[0].end);
2177                         break;
2178                 }
2179                 case VFIO_IOMMU_TYPE1_INFO_DMA_AVAIL: {
2180                         struct vfio_iommu_type1_info_dma_avail *data =
2181                                 (void *)cap;
2182
2183                         ASSERT_EQ(1, data->header.version);
2184                         ASSERT_EQ(sizeof(*data), cap_size);
2185                         break;
2186                 }
2187                 default:
2188                         ASSERT_EQ(false, true);
2189                         break;
2190                 }
2191                 if (!cap->next)
2192                         break;
2193
2194                 ASSERT_GE(info_cmd->argsz, cap->next + sizeof(*cap));
2195                 ASSERT_GE(buffer + cap->next, (void *)cap);
2196                 cap = buffer + cap->next;
2197         }
2198 }
2199
2200 TEST_F(vfio_compat_mock_domain, get_info)
2201 {
2202         struct vfio_iommu_type1_info *info_cmd = buffer;
2203         unsigned int i;
2204         size_t caplen;
2205
2206         /* Pre-cap ABI */
2207         *info_cmd = (struct vfio_iommu_type1_info){
2208                 .argsz = offsetof(struct vfio_iommu_type1_info, cap_offset),
2209         };
2210         ioctl_check_buf(self->fd, VFIO_IOMMU_GET_INFO);
2211         ASSERT_NE(0, info_cmd->iova_pgsizes);
2212         ASSERT_EQ(VFIO_IOMMU_INFO_PGSIZES | VFIO_IOMMU_INFO_CAPS,
2213                   info_cmd->flags);
2214
2215         /* Read the cap chain size */
2216         *info_cmd = (struct vfio_iommu_type1_info){
2217                 .argsz = sizeof(*info_cmd),
2218         };
2219         ioctl_check_buf(self->fd, VFIO_IOMMU_GET_INFO);
2220         ASSERT_NE(0, info_cmd->iova_pgsizes);
2221         ASSERT_EQ(VFIO_IOMMU_INFO_PGSIZES | VFIO_IOMMU_INFO_CAPS,
2222                   info_cmd->flags);
2223         ASSERT_EQ(0, info_cmd->cap_offset);
2224         ASSERT_LT(sizeof(*info_cmd), info_cmd->argsz);
2225
2226         /* Read the caps, kernel should never create a corrupted caps */
2227         caplen = info_cmd->argsz;
2228         for (i = sizeof(*info_cmd); i < caplen; i++) {
2229                 *info_cmd = (struct vfio_iommu_type1_info){
2230                         .argsz = i,
2231                 };
2232                 ioctl_check_buf(self->fd, VFIO_IOMMU_GET_INFO);
2233                 ASSERT_EQ(VFIO_IOMMU_INFO_PGSIZES | VFIO_IOMMU_INFO_CAPS,
2234                           info_cmd->flags);
2235                 if (!info_cmd->cap_offset)
2236                         continue;
2237                 check_vfio_info_cap_chain(_metadata, info_cmd);
2238         }
2239 }
2240
2241 static void shuffle_array(unsigned long *array, size_t nelms)
2242 {
2243         unsigned int i;
2244
2245         /* Shuffle */
2246         for (i = 0; i != nelms; i++) {
2247                 unsigned long tmp = array[i];
2248                 unsigned int other = rand() % (nelms - i);
2249
2250                 array[i] = array[other];
2251                 array[other] = tmp;
2252         }
2253 }
2254
2255 TEST_F(vfio_compat_mock_domain, map)
2256 {
2257         struct vfio_iommu_type1_dma_map map_cmd = {
2258                 .argsz = sizeof(map_cmd),
2259                 .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE,
2260                 .vaddr = (uintptr_t)buffer,
2261                 .size = BUFFER_SIZE,
2262                 .iova = MOCK_APERTURE_START,
2263         };
2264         struct vfio_iommu_type1_dma_unmap unmap_cmd = {
2265                 .argsz = sizeof(unmap_cmd),
2266                 .size = BUFFER_SIZE,
2267                 .iova = MOCK_APERTURE_START,
2268         };
2269         unsigned long pages_iova[BUFFER_SIZE / PAGE_SIZE];
2270         unsigned int i;
2271
2272         /* Simple map/unmap */
2273         ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd));
2274         ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd));
2275         ASSERT_EQ(BUFFER_SIZE, unmap_cmd.size);
2276
2277         /* UNMAP_FLAG_ALL requires 0 iova/size */
2278         ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd));
2279         unmap_cmd.flags = VFIO_DMA_UNMAP_FLAG_ALL;
2280         EXPECT_ERRNO(EINVAL, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd));
2281
2282         unmap_cmd.iova = 0;
2283         unmap_cmd.size = 0;
2284         ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd));
2285         ASSERT_EQ(BUFFER_SIZE, unmap_cmd.size);
2286
2287         /* Small pages */
2288         for (i = 0; i != ARRAY_SIZE(pages_iova); i++) {
2289                 map_cmd.iova = pages_iova[i] =
2290                         MOCK_APERTURE_START + i * PAGE_SIZE;
2291                 map_cmd.vaddr = (uintptr_t)buffer + i * PAGE_SIZE;
2292                 map_cmd.size = PAGE_SIZE;
2293                 ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd));
2294         }
2295         shuffle_array(pages_iova, ARRAY_SIZE(pages_iova));
2296
2297         unmap_cmd.flags = 0;
2298         unmap_cmd.size = PAGE_SIZE;
2299         for (i = 0; i != ARRAY_SIZE(pages_iova); i++) {
2300                 unmap_cmd.iova = pages_iova[i];
2301                 ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd));
2302         }
2303 }
2304
2305 TEST_F(vfio_compat_mock_domain, huge_map)
2306 {
2307         size_t buf_size = HUGEPAGE_SIZE * 2;
2308         struct vfio_iommu_type1_dma_map map_cmd = {
2309                 .argsz = sizeof(map_cmd),
2310                 .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE,
2311                 .size = buf_size,
2312                 .iova = MOCK_APERTURE_START,
2313         };
2314         struct vfio_iommu_type1_dma_unmap unmap_cmd = {
2315                 .argsz = sizeof(unmap_cmd),
2316         };
2317         unsigned long pages_iova[16];
2318         unsigned int i;
2319         void *buf;
2320
2321         /* Test huge pages and splitting */
2322         buf = mmap(0, buf_size, PROT_READ | PROT_WRITE,
2323                    MAP_SHARED | MAP_ANONYMOUS | MAP_HUGETLB | MAP_POPULATE, -1,
2324                    0);
2325         ASSERT_NE(MAP_FAILED, buf);
2326         map_cmd.vaddr = (uintptr_t)buf;
2327         ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd));
2328
2329         unmap_cmd.size = buf_size / ARRAY_SIZE(pages_iova);
2330         for (i = 0; i != ARRAY_SIZE(pages_iova); i++)
2331                 pages_iova[i] = MOCK_APERTURE_START + (i * unmap_cmd.size);
2332         shuffle_array(pages_iova, ARRAY_SIZE(pages_iova));
2333
2334         /* type1 mode can cut up larger mappings, type1v2 always fails */
2335         for (i = 0; i != ARRAY_SIZE(pages_iova); i++) {
2336                 unmap_cmd.iova = pages_iova[i];
2337                 unmap_cmd.size = buf_size / ARRAY_SIZE(pages_iova);
2338                 if (variant->version == VFIO_TYPE1_IOMMU) {
2339                         ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA,
2340                                            &unmap_cmd));
2341                 } else {
2342                         EXPECT_ERRNO(ENOENT,
2343                                      ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA,
2344                                            &unmap_cmd));
2345                 }
2346         }
2347 }
2348
2349 TEST_HARNESS_MAIN