drivers/gpu/drm/i915/gem/selftests/huge_pages.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2017 Intel Corporation
   5  */
   6
   7 #include <linux/prime_numbers.h>
   8
   9 #include "i915_selftest.h"
  10
  11 #include "gem/i915_gem_internal.h"
  12 #include "gem/i915_gem_lmem.h"
  13 #include "gem/i915_gem_pm.h"
  14 #include "gem/i915_gem_region.h"
  15
  16 #include "gt/intel_gt.h"
  17
  18 #include "igt_gem_utils.h"
  19 #include "mock_context.h"
  20
  21 #include "selftests/mock_drm.h"
  22 #include "selftests/mock_gem_device.h"
  23 #include "selftests/mock_region.h"
  24 #include "selftests/i915_random.h"
  25
  26 static struct i915_gem_context *hugepage_ctx(struct drm_i915_private *i915,
  27                                              struct file *file)
  28 {
  29         struct i915_gem_context *ctx = live_context(i915, file);
  30         struct i915_address_space *vm;
  31
  32         if (IS_ERR(ctx))
  33                 return ctx;
  34
  35         vm = ctx->vm;
  36         if (vm)
  37                 WRITE_ONCE(vm->scrub_64K, true);
  38
  39         return ctx;
  40 }
  41
  42 static const unsigned int page_sizes[] = {
  43         I915_GTT_PAGE_SIZE_2M,
  44         I915_GTT_PAGE_SIZE_64K,
  45         I915_GTT_PAGE_SIZE_4K,
  46 };
  47
  48 static unsigned int get_largest_page_size(struct drm_i915_private *i915,
  49                                           u64 rem)
  50 {
  51         int i;
  52
  53         for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) {
  54                 unsigned int page_size = page_sizes[i];
  55
  56                 if (HAS_PAGE_SIZES(i915, page_size) && rem >= page_size)
  57                         return page_size;
  58         }
  59
  60         return 0;
  61 }
  62
  63 static void huge_pages_free_pages(struct sg_table *st)
  64 {
  65         struct scatterlist *sg;
  66
  67         for (sg = st->sgl; sg; sg = __sg_next(sg)) {
  68                 if (sg_page(sg))
  69                         __free_pages(sg_page(sg), get_order(sg->length));
  70         }
  71
  72         sg_free_table(st);
  73         kfree(st);
  74 }
  75
  76 static int get_huge_pages(struct drm_i915_gem_object *obj)
  77 {
  78 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY)
  79         unsigned int page_mask = obj->mm.page_mask;
  80         struct sg_table *st;
  81         struct scatterlist *sg;
  82         unsigned int sg_page_sizes;
  83         u64 rem;
  84
  85         st = kmalloc(sizeof(*st), GFP);
  86         if (!st)
  87                 return -ENOMEM;
  88
  89         if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) {
  90                 kfree(st);
  91                 return -ENOMEM;
  92         }
  93
  94         rem = obj->base.size;
  95         sg = st->sgl;
  96         st->nents = 0;
  97         sg_page_sizes = 0;
  98
  99         /*
 100          * Our goal here is simple, we want to greedily fill the object from
 101          * largest to smallest page-size, while ensuring that we use *every*
 102          * page-size as per the given page-mask.
 103          */
 104         do {
 105                 unsigned int bit = ilog2(page_mask);
 106                 unsigned int page_size = BIT(bit);
 107                 int order = get_order(page_size);
 108
 109                 do {
 110                         struct page *page;
 111
 112                         GEM_BUG_ON(order >= MAX_ORDER);
 113                         page = alloc_pages(GFP | __GFP_ZERO, order);
 114                         if (!page)
 115                                 goto err;
 116
 117                         sg_set_page(sg, page, page_size, 0);
 118                         sg_page_sizes |= page_size;
 119                         st->nents++;
 120
 121                         rem -= page_size;
 122                         if (!rem) {
 123                                 sg_mark_end(sg);
 124                                 break;
 125                         }
 126
 127                         sg = __sg_next(sg);
 128                 } while ((rem - ((page_size-1) & page_mask)) >= page_size);
 129
 130                 page_mask &= (page_size-1);
 131         } while (page_mask);
 132
 133         if (i915_gem_gtt_prepare_pages(obj, st))
 134                 goto err;
 135
 136         GEM_BUG_ON(sg_page_sizes != obj->mm.page_mask);
 137         __i915_gem_object_set_pages(obj, st, sg_page_sizes);
 138
 139         return 0;
 140
 141 err:
 142         sg_set_page(sg, NULL, 0, 0);
 143         sg_mark_end(sg);
 144         huge_pages_free_pages(st);
 145
 146         return -ENOMEM;
 147 }
 148
 149 static void put_huge_pages(struct drm_i915_gem_object *obj,
 150                            struct sg_table *pages)
 151 {
 152         i915_gem_gtt_finish_pages(obj, pages);
 153         huge_pages_free_pages(pages);
 154
 155         obj->mm.dirty = false;
 156
 157         __start_cpu_write(obj);
 158 }
 159
 160 static const struct drm_i915_gem_object_ops huge_page_ops = {
 161         .name = "huge-gem",
 162         .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
 163         .get_pages = get_huge_pages,
 164         .put_pages = put_huge_pages,
 165 };
 166
 167 static struct drm_i915_gem_object *
 168 huge_pages_object(struct drm_i915_private *i915,
 169                   u64 size,
 170                   unsigned int page_mask)
 171 {
 172         static struct lock_class_key lock_class;
 173         struct drm_i915_gem_object *obj;
 174         unsigned int cache_level;
 175
 176         GEM_BUG_ON(!size);
 177         GEM_BUG_ON(!IS_ALIGNED(size, BIT(__ffs(page_mask))));
 178
 179         if (size >> PAGE_SHIFT > INT_MAX)
 180                 return ERR_PTR(-E2BIG);
 181
 182         if (overflows_type(size, obj->base.size))
 183                 return ERR_PTR(-E2BIG);
 184
 185         obj = i915_gem_object_alloc();
 186         if (!obj)
 187                 return ERR_PTR(-ENOMEM);
 188
 189         drm_gem_private_object_init(&i915->drm, &obj->base, size);
 190         i915_gem_object_init(obj, &huge_page_ops, &lock_class, 0);
 191         obj->mem_flags |= I915_BO_FLAG_STRUCT_PAGE;
 192         i915_gem_object_set_volatile(obj);
 193
 194         obj->write_domain = I915_GEM_DOMAIN_CPU;
 195         obj->read_domains = I915_GEM_DOMAIN_CPU;
 196
 197         cache_level = HAS_LLC(i915) ? I915_CACHE_LLC : I915_CACHE_NONE;
 198         i915_gem_object_set_cache_coherency(obj, cache_level);
 199
 200         obj->mm.page_mask = page_mask;
 201
 202         return obj;
 203 }
 204
 205 static int fake_get_huge_pages(struct drm_i915_gem_object *obj)
 206 {
 207         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 208         const u64 max_len = rounddown_pow_of_two(UINT_MAX);
 209         struct sg_table *st;
 210         struct scatterlist *sg;
 211         unsigned int sg_page_sizes;
 212         u64 rem;
 213
 214         st = kmalloc(sizeof(*st), GFP);
 215         if (!st)
 216                 return -ENOMEM;
 217
 218         if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) {
 219                 kfree(st);
 220                 return -ENOMEM;
 221         }
 222
 223         /* Use optimal page sized chunks to fill in the sg table */
 224         rem = obj->base.size;
 225         sg = st->sgl;
 226         st->nents = 0;
 227         sg_page_sizes = 0;
 228         do {
 229                 unsigned int page_size = get_largest_page_size(i915, rem);
 230                 unsigned int len = min(page_size * div_u64(rem, page_size),
 231                                        max_len);
 232
 233                 GEM_BUG_ON(!page_size);
 234
 235                 sg->offset = 0;
 236                 sg->length = len;
 237                 sg_dma_len(sg) = len;
 238                 sg_dma_address(sg) = page_size;
 239
 240                 sg_page_sizes |= len;
 241
 242                 st->nents++;
 243
 244                 rem -= len;
 245                 if (!rem) {
 246                         sg_mark_end(sg);
 247                         break;
 248                 }
 249
 250                 sg = sg_next(sg);
 251         } while (1);
 252
 253         i915_sg_trim(st);
 254
 255         __i915_gem_object_set_pages(obj, st, sg_page_sizes);
 256
 257         return 0;
 258 }
 259
 260 static int fake_get_huge_pages_single(struct drm_i915_gem_object *obj)
 261 {
 262         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 263         struct sg_table *st;
 264         struct scatterlist *sg;
 265         unsigned int page_size;
 266
 267         st = kmalloc(sizeof(*st), GFP);
 268         if (!st)
 269                 return -ENOMEM;
 270
 271         if (sg_alloc_table(st, 1, GFP)) {
 272                 kfree(st);
 273                 return -ENOMEM;
 274         }
 275
 276         sg = st->sgl;
 277         st->nents = 1;
 278
 279         page_size = get_largest_page_size(i915, obj->base.size);
 280         GEM_BUG_ON(!page_size);
 281
 282         sg->offset = 0;
 283         sg->length = obj->base.size;
 284         sg_dma_len(sg) = obj->base.size;
 285         sg_dma_address(sg) = page_size;
 286
 287         __i915_gem_object_set_pages(obj, st, sg->length);
 288
 289         return 0;
 290 #undef GFP
 291 }
 292
 293 static void fake_free_huge_pages(struct drm_i915_gem_object *obj,
 294                                  struct sg_table *pages)
 295 {
 296         sg_free_table(pages);
 297         kfree(pages);
 298 }
 299
 300 static void fake_put_huge_pages(struct drm_i915_gem_object *obj,
 301                                 struct sg_table *pages)
 302 {
 303         fake_free_huge_pages(obj, pages);
 304         obj->mm.dirty = false;
 305 }
 306
 307 static const struct drm_i915_gem_object_ops fake_ops = {
 308         .name = "fake-gem",
 309         .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
 310         .get_pages = fake_get_huge_pages,
 311         .put_pages = fake_put_huge_pages,
 312 };
 313
 314 static const struct drm_i915_gem_object_ops fake_ops_single = {
 315         .name = "fake-gem",
 316         .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
 317         .get_pages = fake_get_huge_pages_single,
 318         .put_pages = fake_put_huge_pages,
 319 };
 320
 321 static struct drm_i915_gem_object *
 322 fake_huge_pages_object(struct drm_i915_private *i915, u64 size, bool single)
 323 {
 324         static struct lock_class_key lock_class;
 325         struct drm_i915_gem_object *obj;
 326
 327         GEM_BUG_ON(!size);
 328         GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
 329
 330         if (size >> PAGE_SHIFT > UINT_MAX)
 331                 return ERR_PTR(-E2BIG);
 332
 333         if (overflows_type(size, obj->base.size))
 334                 return ERR_PTR(-E2BIG);
 335
 336         obj = i915_gem_object_alloc();
 337         if (!obj)
 338                 return ERR_PTR(-ENOMEM);
 339
 340         drm_gem_private_object_init(&i915->drm, &obj->base, size);
 341
 342         if (single)
 343                 i915_gem_object_init(obj, &fake_ops_single, &lock_class, 0);
 344         else
 345                 i915_gem_object_init(obj, &fake_ops, &lock_class, 0);
 346
 347         i915_gem_object_set_volatile(obj);
 348
 349         obj->write_domain = I915_GEM_DOMAIN_CPU;
 350         obj->read_domains = I915_GEM_DOMAIN_CPU;
 351         obj->cache_level = I915_CACHE_NONE;
 352
 353         return obj;
 354 }
 355
 356 static int igt_check_page_sizes(struct i915_vma *vma)
 357 {
 358         struct drm_i915_private *i915 = vma->vm->i915;
 359         unsigned int supported = INTEL_INFO(i915)->page_sizes;
 360         struct drm_i915_gem_object *obj = vma->obj;
 361         int err;
 362
 363         /* We have to wait for the async bind to complete before our asserts */
 364         err = i915_vma_sync(vma);
 365         if (err)
 366                 return err;
 367
 368         if (!HAS_PAGE_SIZES(i915, vma->page_sizes.sg)) {
 369                 pr_err("unsupported page_sizes.sg=%u, supported=%u\n",
 370                        vma->page_sizes.sg & ~supported, supported);
 371                 err = -EINVAL;
 372         }
 373
 374         if (!HAS_PAGE_SIZES(i915, vma->resource->page_sizes_gtt)) {
 375                 pr_err("unsupported page_sizes.gtt=%u, supported=%u\n",
 376                        vma->resource->page_sizes_gtt & ~supported, supported);
 377                 err = -EINVAL;
 378         }
 379
 380         if (vma->page_sizes.phys != obj->mm.page_sizes.phys) {
 381                 pr_err("vma->page_sizes.phys(%u) != obj->mm.page_sizes.phys(%u)\n",
 382                        vma->page_sizes.phys, obj->mm.page_sizes.phys);
 383                 err = -EINVAL;
 384         }
 385
 386         if (vma->page_sizes.sg != obj->mm.page_sizes.sg) {
 387                 pr_err("vma->page_sizes.sg(%u) != obj->mm.page_sizes.sg(%u)\n",
 388                        vma->page_sizes.sg, obj->mm.page_sizes.sg);
 389                 err = -EINVAL;
 390         }
 391
 392         /*
 393          * The dma-api is like a box of chocolates when it comes to the
 394          * alignment of dma addresses, however for LMEM we have total control
 395          * and so can guarantee alignment, likewise when we allocate our blocks
 396          * they should appear in descending order, and if we know that we align
 397          * to the largest page size for the GTT address, we should be able to
 398          * assert that if we see 2M physical pages then we should also get 2M
 399          * GTT pages. If we don't then something might be wrong in our
 400          * construction of the backing pages.
 401          *
 402          * Maintaining alignment is required to utilise huge pages in the ppGGT.
 403          */
 404         if (i915_gem_object_is_lmem(obj) &&
 405             IS_ALIGNED(vma->node.start, SZ_2M) &&
 406             vma->page_sizes.sg & SZ_2M &&
 407             vma->resource->page_sizes_gtt < SZ_2M) {
 408                 pr_err("gtt pages mismatch for LMEM, expected 2M GTT pages, sg(%u), gtt(%u)\n",
 409                        vma->page_sizes.sg, vma->resource->page_sizes_gtt);
 410                 err = -EINVAL;
 411         }
 412
 413         return err;
 414 }
 415
 416 static int igt_mock_exhaust_device_supported_pages(void *arg)
 417 {
 418         struct i915_ppgtt *ppgtt = arg;
 419         struct drm_i915_private *i915 = ppgtt->vm.i915;
 420         unsigned int saved_mask = INTEL_INFO(i915)->page_sizes;
 421         struct drm_i915_gem_object *obj;
 422         struct i915_vma *vma;
 423         int i, j, single;
 424         int err;
 425
 426         /*
 427          * Sanity check creating objects with every valid page support
 428          * combination for our mock device.
 429          */
 430
 431         for (i = 1; i < BIT(ARRAY_SIZE(page_sizes)); i++) {
 432                 unsigned int combination = SZ_4K; /* Required for ppGTT */
 433
 434                 for (j = 0; j < ARRAY_SIZE(page_sizes); j++) {
 435                         if (i & BIT(j))
 436                                 combination |= page_sizes[j];
 437                 }
 438
 439                 mkwrite_device_info(i915)->page_sizes = combination;
 440
 441                 for (single = 0; single <= 1; ++single) {
 442                         obj = fake_huge_pages_object(i915, combination, !!single);
 443                         if (IS_ERR(obj)) {
 444                                 err = PTR_ERR(obj);
 445                                 goto out_device;
 446                         }
 447
 448                         if (obj->base.size != combination) {
 449                                 pr_err("obj->base.size=%zu, expected=%u\n",
 450                                        obj->base.size, combination);
 451                                 err = -EINVAL;
 452                                 goto out_put;
 453                         }
 454
 455                         vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 456                         if (IS_ERR(vma)) {
 457                                 err = PTR_ERR(vma);
 458                                 goto out_put;
 459                         }
 460
 461                         err = i915_vma_pin(vma, 0, 0, PIN_USER);
 462                         if (err)
 463                                 goto out_put;
 464
 465                         err = igt_check_page_sizes(vma);
 466
 467                         if (vma->page_sizes.sg != combination) {
 468                                 pr_err("page_sizes.sg=%u, expected=%u\n",
 469                                        vma->page_sizes.sg, combination);
 470                                 err = -EINVAL;
 471                         }
 472
 473                         i915_vma_unpin(vma);
 474                         i915_gem_object_put(obj);
 475
 476                         if (err)
 477                                 goto out_device;
 478                 }
 479         }
 480
 481         goto out_device;
 482
 483 out_put:
 484         i915_gem_object_put(obj);
 485 out_device:
 486         mkwrite_device_info(i915)->page_sizes = saved_mask;
 487
 488         return err;
 489 }
 490
 491 static int igt_mock_memory_region_huge_pages(void *arg)
 492 {
 493         const unsigned int flags[] = { 0, I915_BO_ALLOC_CONTIGUOUS };
 494         struct i915_ppgtt *ppgtt = arg;
 495         struct drm_i915_private *i915 = ppgtt->vm.i915;
 496         unsigned long supported = INTEL_INFO(i915)->page_sizes;
 497         struct intel_memory_region *mem;
 498         struct drm_i915_gem_object *obj;
 499         struct i915_vma *vma;
 500         int bit;
 501         int err = 0;
 502
 503         mem = mock_region_create(i915, 0, SZ_2G, I915_GTT_PAGE_SIZE_4K, 0, 0);
 504         if (IS_ERR(mem)) {
 505                 pr_err("%s failed to create memory region\n", __func__);
 506                 return PTR_ERR(mem);
 507         }
 508
 509         for_each_set_bit(bit, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
 510                 unsigned int page_size = BIT(bit);
 511                 resource_size_t phys;
 512                 int i;
 513
 514                 for (i = 0; i < ARRAY_SIZE(flags); ++i) {
 515                         obj = i915_gem_object_create_region(mem,
 516                                                             page_size, page_size,
 517                                                             flags[i]);
 518                         if (IS_ERR(obj)) {
 519                                 err = PTR_ERR(obj);
 520                                 goto out_region;
 521                         }
 522
 523                         vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 524                         if (IS_ERR(vma)) {
 525                                 err = PTR_ERR(vma);
 526                                 goto out_put;
 527                         }
 528
 529                         err = i915_vma_pin(vma, 0, 0, PIN_USER);
 530                         if (err)
 531                                 goto out_put;
 532
 533                         err = igt_check_page_sizes(vma);
 534                         if (err)
 535                                 goto out_unpin;
 536
 537                         phys = i915_gem_object_get_dma_address(obj, 0);
 538                         if (!IS_ALIGNED(phys, page_size)) {
 539                                 pr_err("%s addr misaligned(%pa) page_size=%u\n",
 540                                        __func__, &phys, page_size);
 541                                 err = -EINVAL;
 542                                 goto out_unpin;
 543                         }
 544
 545                         if (vma->resource->page_sizes_gtt != page_size) {
 546                                 pr_err("%s page_sizes.gtt=%u, expected=%u\n",
 547                                        __func__, vma->resource->page_sizes_gtt,
 548                                        page_size);
 549                                 err = -EINVAL;
 550                                 goto out_unpin;
 551                         }
 552
 553                         i915_vma_unpin(vma);
 554                         __i915_gem_object_put_pages(obj);
 555                         i915_gem_object_put(obj);
 556                 }
 557         }
 558
 559         goto out_region;
 560
 561 out_unpin:
 562         i915_vma_unpin(vma);
 563 out_put:
 564         i915_gem_object_put(obj);
 565 out_region:
 566         intel_memory_region_destroy(mem);
 567         return err;
 568 }
 569
 570 static int igt_mock_ppgtt_misaligned_dma(void *arg)
 571 {
 572         struct i915_ppgtt *ppgtt = arg;
 573         struct drm_i915_private *i915 = ppgtt->vm.i915;
 574         unsigned long supported = INTEL_INFO(i915)->page_sizes;
 575         struct drm_i915_gem_object *obj;
 576         int bit;
 577         int err;
 578
 579         /*
 580          * Sanity check dma misalignment for huge pages -- the dma addresses we
 581          * insert into the paging structures need to always respect the page
 582          * size alignment.
 583          */
 584
 585         bit = ilog2(I915_GTT_PAGE_SIZE_64K);
 586
 587         for_each_set_bit_from(bit, &supported,
 588                               ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
 589                 IGT_TIMEOUT(end_time);
 590                 unsigned int page_size = BIT(bit);
 591                 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
 592                 unsigned int offset;
 593                 unsigned int size =
 594                         round_up(page_size, I915_GTT_PAGE_SIZE_2M) << 1;
 595                 struct i915_vma *vma;
 596
 597                 obj = fake_huge_pages_object(i915, size, true);
 598                 if (IS_ERR(obj))
 599                         return PTR_ERR(obj);
 600
 601                 if (obj->base.size != size) {
 602                         pr_err("obj->base.size=%zu, expected=%u\n",
 603                                obj->base.size, size);
 604                         err = -EINVAL;
 605                         goto out_put;
 606                 }
 607
 608                 err = i915_gem_object_pin_pages_unlocked(obj);
 609                 if (err)
 610                         goto out_put;
 611
 612                 /* Force the page size for this object */
 613                 obj->mm.page_sizes.sg = page_size;
 614
 615                 vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 616                 if (IS_ERR(vma)) {
 617                         err = PTR_ERR(vma);
 618                         goto out_unpin;
 619                 }
 620
 621                 err = i915_vma_pin(vma, 0, 0, flags);
 622                 if (err)
 623                         goto out_unpin;
 624
 625
 626                 err = igt_check_page_sizes(vma);
 627
 628                 if (vma->resource->page_sizes_gtt != page_size) {
 629                         pr_err("page_sizes.gtt=%u, expected %u\n",
 630                                vma->resource->page_sizes_gtt, page_size);
 631                         err = -EINVAL;
 632                 }
 633
 634                 i915_vma_unpin(vma);
 635
 636                 if (err)
 637                         goto out_unpin;
 638
 639                 /*
 640                  * Try all the other valid offsets until the next
 641                  * boundary -- should always fall back to using 4K
 642                  * pages.
 643                  */
 644                 for (offset = 4096; offset < page_size; offset += 4096) {
 645                         err = i915_vma_unbind_unlocked(vma);
 646                         if (err)
 647                                 goto out_unpin;
 648
 649                         err = i915_vma_pin(vma, 0, 0, flags | offset);
 650                         if (err)
 651                                 goto out_unpin;
 652
 653                         err = igt_check_page_sizes(vma);
 654
 655                         if (vma->resource->page_sizes_gtt != I915_GTT_PAGE_SIZE_4K) {
 656                                 pr_err("page_sizes.gtt=%u, expected %llu\n",
 657                                        vma->resource->page_sizes_gtt,
 658                                        I915_GTT_PAGE_SIZE_4K);
 659                                 err = -EINVAL;
 660                         }
 661
 662                         i915_vma_unpin(vma);
 663
 664                         if (err)
 665                                 goto out_unpin;
 666
 667                         if (igt_timeout(end_time,
 668                                         "%s timed out at offset %x with page-size %x\n",
 669                                         __func__, offset, page_size))
 670                                 break;
 671                 }
 672
 673                 i915_gem_object_lock(obj, NULL);
 674                 i915_gem_object_unpin_pages(obj);
 675                 __i915_gem_object_put_pages(obj);
 676                 i915_gem_object_unlock(obj);
 677                 i915_gem_object_put(obj);
 678         }
 679
 680         return 0;
 681
 682 out_unpin:
 683         i915_gem_object_lock(obj, NULL);
 684         i915_gem_object_unpin_pages(obj);
 685         i915_gem_object_unlock(obj);
 686 out_put:
 687         i915_gem_object_put(obj);
 688
 689         return err;
 690 }
 691
 692 static void close_object_list(struct list_head *objects,
 693                               struct i915_ppgtt *ppgtt)
 694 {
 695         struct drm_i915_gem_object *obj, *on;
 696
 697         list_for_each_entry_safe(obj, on, objects, st_link) {
 698                 list_del(&obj->st_link);
 699                 i915_gem_object_lock(obj, NULL);
 700                 i915_gem_object_unpin_pages(obj);
 701                 __i915_gem_object_put_pages(obj);
 702                 i915_gem_object_unlock(obj);
 703                 i915_gem_object_put(obj);
 704         }
 705 }
 706
 707 static int igt_mock_ppgtt_huge_fill(void *arg)
 708 {
 709         struct i915_ppgtt *ppgtt = arg;
 710         struct drm_i915_private *i915 = ppgtt->vm.i915;
 711         unsigned long max_pages = ppgtt->vm.total >> PAGE_SHIFT;
 712         unsigned long page_num;
 713         bool single = false;
 714         LIST_HEAD(objects);
 715         IGT_TIMEOUT(end_time);
 716         int err = -ENODEV;
 717
 718         for_each_prime_number_from(page_num, 1, max_pages) {
 719                 struct drm_i915_gem_object *obj;
 720                 u64 size = page_num << PAGE_SHIFT;
 721                 struct i915_vma *vma;
 722                 unsigned int expected_gtt = 0;
 723                 int i;
 724
 725                 obj = fake_huge_pages_object(i915, size, single);
 726                 if (IS_ERR(obj)) {
 727                         err = PTR_ERR(obj);
 728                         break;
 729                 }
 730
 731                 if (obj->base.size != size) {
 732                         pr_err("obj->base.size=%zd, expected=%llu\n",
 733                                obj->base.size, size);
 734                         i915_gem_object_put(obj);
 735                         err = -EINVAL;
 736                         break;
 737                 }
 738
 739                 err = i915_gem_object_pin_pages_unlocked(obj);
 740                 if (err) {
 741                         i915_gem_object_put(obj);
 742                         break;
 743                 }
 744
 745                 list_add(&obj->st_link, &objects);
 746
 747                 vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 748                 if (IS_ERR(vma)) {
 749                         err = PTR_ERR(vma);
 750                         break;
 751                 }
 752
 753                 err = i915_vma_pin(vma, 0, 0, PIN_USER);
 754                 if (err)
 755                         break;
 756
 757                 err = igt_check_page_sizes(vma);
 758                 if (err) {
 759                         i915_vma_unpin(vma);
 760                         break;
 761                 }
 762
 763                 /*
 764                  * Figure out the expected gtt page size knowing that we go from
 765                  * largest to smallest page size sg chunks, and that we align to
 766                  * the largest page size.
 767                  */
 768                 for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) {
 769                         unsigned int page_size = page_sizes[i];
 770
 771                         if (HAS_PAGE_SIZES(i915, page_size) &&
 772                             size >= page_size) {
 773                                 expected_gtt |= page_size;
 774                                 size &= page_size-1;
 775                         }
 776                 }
 777
 778                 GEM_BUG_ON(!expected_gtt);
 779                 GEM_BUG_ON(size);
 780
 781                 if (expected_gtt & I915_GTT_PAGE_SIZE_4K)
 782                         expected_gtt &= ~I915_GTT_PAGE_SIZE_64K;
 783
 784                 i915_vma_unpin(vma);
 785
 786                 if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) {
 787                         if (!IS_ALIGNED(vma->node.start,
 788                                         I915_GTT_PAGE_SIZE_2M)) {
 789                                 pr_err("node.start(%llx) not aligned to 2M\n",
 790                                        vma->node.start);
 791                                 err = -EINVAL;
 792                                 break;
 793                         }
 794
 795                         if (!IS_ALIGNED(vma->node.size,
 796                                         I915_GTT_PAGE_SIZE_2M)) {
 797                                 pr_err("node.size(%llx) not aligned to 2M\n",
 798                                        vma->node.size);
 799                                 err = -EINVAL;
 800                                 break;
 801                         }
 802                 }
 803
 804                 if (vma->resource->page_sizes_gtt != expected_gtt) {
 805                         pr_err("gtt=%u, expected=%u, size=%zd, single=%s\n",
 806                                vma->resource->page_sizes_gtt, expected_gtt,
 807                                obj->base.size, yesno(!!single));
 808                         err = -EINVAL;
 809                         break;
 810                 }
 811
 812                 if (igt_timeout(end_time,
 813                                 "%s timed out at size %zd\n",
 814                                 __func__, obj->base.size))
 815                         break;
 816
 817                 single = !single;
 818         }
 819
 820         close_object_list(&objects, ppgtt);
 821
 822         if (err == -ENOMEM || err == -ENOSPC)
 823                 err = 0;
 824
 825         return err;
 826 }
 827
 828 static int igt_mock_ppgtt_64K(void *arg)
 829 {
 830         struct i915_ppgtt *ppgtt = arg;
 831         struct drm_i915_private *i915 = ppgtt->vm.i915;
 832         struct drm_i915_gem_object *obj;
 833         const struct object_info {
 834                 unsigned int size;
 835                 unsigned int gtt;
 836                 unsigned int offset;
 837         } objects[] = {
 838                 /* Cases with forced padding/alignment */
 839                 {
 840                         .size = SZ_64K,
 841                         .gtt = I915_GTT_PAGE_SIZE_64K,
 842                         .offset = 0,
 843                 },
 844                 {
 845                         .size = SZ_64K + SZ_4K,
 846                         .gtt = I915_GTT_PAGE_SIZE_4K,
 847                         .offset = 0,
 848                 },
 849                 {
 850                         .size = SZ_64K - SZ_4K,
 851                         .gtt = I915_GTT_PAGE_SIZE_4K,
 852                         .offset = 0,
 853                 },
 854                 {
 855                         .size = SZ_2M,
 856                         .gtt = I915_GTT_PAGE_SIZE_64K,
 857                         .offset = 0,
 858                 },
 859                 {
 860                         .size = SZ_2M - SZ_4K,
 861                         .gtt = I915_GTT_PAGE_SIZE_4K,
 862                         .offset = 0,
 863                 },
 864                 {
 865                         .size = SZ_2M + SZ_4K,
 866                         .gtt = I915_GTT_PAGE_SIZE_64K | I915_GTT_PAGE_SIZE_4K,
 867                         .offset = 0,
 868                 },
 869                 {
 870                         .size = SZ_2M + SZ_64K,
 871                         .gtt = I915_GTT_PAGE_SIZE_64K,
 872                         .offset = 0,
 873                 },
 874                 {
 875                         .size = SZ_2M - SZ_64K,
 876                         .gtt = I915_GTT_PAGE_SIZE_64K,
 877                         .offset = 0,
 878                 },
 879                 /* Try without any forced padding/alignment */
 880                 {
 881                         .size = SZ_64K,
 882                         .offset = SZ_2M,
 883                         .gtt = I915_GTT_PAGE_SIZE_4K,
 884                 },
 885                 {
 886                         .size = SZ_128K,
 887                         .offset = SZ_2M - SZ_64K,
 888                         .gtt = I915_GTT_PAGE_SIZE_4K,
 889                 },
 890         };
 891         struct i915_vma *vma;
 892         int i, single;
 893         int err;
 894
 895         /*
 896          * Sanity check some of the trickiness with 64K pages -- either we can
 897          * safely mark the whole page-table(2M block) as 64K, or we have to
 898          * always fallback to 4K.
 899          */
 900
 901         if (!HAS_PAGE_SIZES(i915, I915_GTT_PAGE_SIZE_64K))
 902                 return 0;
 903
 904         for (i = 0; i < ARRAY_SIZE(objects); ++i) {
 905                 unsigned int size = objects[i].size;
 906                 unsigned int expected_gtt = objects[i].gtt;
 907                 unsigned int offset = objects[i].offset;
 908                 unsigned int flags = PIN_USER;
 909
 910                 for (single = 0; single <= 1; single++) {
 911                         obj = fake_huge_pages_object(i915, size, !!single);
 912                         if (IS_ERR(obj))
 913                                 return PTR_ERR(obj);
 914
 915                         err = i915_gem_object_pin_pages_unlocked(obj);
 916                         if (err)
 917                                 goto out_object_put;
 918
 919                         /*
 920                          * Disable 2M pages -- We only want to use 64K/4K pages
 921                          * for this test.
 922                          */
 923                         obj->mm.page_sizes.sg &= ~I915_GTT_PAGE_SIZE_2M;
 924
 925                         vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 926                         if (IS_ERR(vma)) {
 927                                 err = PTR_ERR(vma);
 928                                 goto out_object_unpin;
 929                         }
 930
 931                         if (offset)
 932                                 flags |= PIN_OFFSET_FIXED | offset;
 933
 934                         err = i915_vma_pin(vma, 0, 0, flags);
 935                         if (err)
 936                                 goto out_object_unpin;
 937
 938                         err = igt_check_page_sizes(vma);
 939                         if (err)
 940                                 goto out_vma_unpin;
 941
 942                         if (!offset && vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) {
 943                                 if (!IS_ALIGNED(vma->node.start,
 944                                                 I915_GTT_PAGE_SIZE_2M)) {
 945                                         pr_err("node.start(%llx) not aligned to 2M\n",
 946                                                vma->node.start);
 947                                         err = -EINVAL;
 948                                         goto out_vma_unpin;
 949                                 }
 950
 951                                 if (!IS_ALIGNED(vma->node.size,
 952                                                 I915_GTT_PAGE_SIZE_2M)) {
 953                                         pr_err("node.size(%llx) not aligned to 2M\n",
 954                                                vma->node.size);
 955                                         err = -EINVAL;
 956                                         goto out_vma_unpin;
 957                                 }
 958                         }
 959
 960                         if (vma->resource->page_sizes_gtt != expected_gtt) {
 961                                 pr_err("gtt=%u, expected=%u, i=%d, single=%s\n",
 962                                        vma->resource->page_sizes_gtt,
 963                                        expected_gtt, i, yesno(!!single));
 964                                 err = -EINVAL;
 965                                 goto out_vma_unpin;
 966                         }
 967
 968                         i915_vma_unpin(vma);
 969                         i915_gem_object_lock(obj, NULL);
 970                         i915_gem_object_unpin_pages(obj);
 971                         __i915_gem_object_put_pages(obj);
 972                         i915_gem_object_unlock(obj);
 973                         i915_gem_object_put(obj);
 974
 975                         i915_gem_drain_freed_objects(i915);
 976                 }
 977         }
 978
 979         return 0;
 980
 981 out_vma_unpin:
 982         i915_vma_unpin(vma);
 983 out_object_unpin:
 984         i915_gem_object_lock(obj, NULL);
 985         i915_gem_object_unpin_pages(obj);
 986         i915_gem_object_unlock(obj);
 987 out_object_put:
 988         i915_gem_object_put(obj);
 989
 990         return err;
 991 }
 992
 993 static int gpu_write(struct intel_context *ce,
 994                      struct i915_vma *vma,
 995                      u32 dw,
 996                      u32 val)
 997 {
 998         int err;
 999
1000         i915_gem_object_lock(vma->obj, NULL);
1001         err = i915_gem_object_set_to_gtt_domain(vma->obj, true);
1002         i915_gem_object_unlock(vma->obj);
1003         if (err)
1004                 return err;
1005
1006         return igt_gpu_fill_dw(ce, vma, dw * sizeof(u32),
1007                                vma->size >> PAGE_SHIFT, val);
1008 }
1009
1010 static int
1011 __cpu_check_shmem(struct drm_i915_gem_object *obj, u32 dword, u32 val)
1012 {
1013         unsigned int needs_flush;
1014         unsigned long n;
1015         int err;
1016
1017         i915_gem_object_lock(obj, NULL);
1018         err = i915_gem_object_prepare_read(obj, &needs_flush);
1019         if (err)
1020                 goto err_unlock;
1021
1022         for (n = 0; n < obj->base.size >> PAGE_SHIFT; ++n) {
1023                 u32 *ptr = kmap_atomic(i915_gem_object_get_page(obj, n));
1024
1025                 if (needs_flush & CLFLUSH_BEFORE)
1026                         drm_clflush_virt_range(ptr, PAGE_SIZE);
1027
1028                 if (ptr[dword] != val) {
1029                         pr_err("n=%lu ptr[%u]=%u, val=%u\n",
1030                                n, dword, ptr[dword], val);
1031                         kunmap_atomic(ptr);
1032                         err = -EINVAL;
1033                         break;
1034                 }
1035
1036                 kunmap_atomic(ptr);
1037         }
1038
1039         i915_gem_object_finish_access(obj);
1040 err_unlock:
1041         i915_gem_object_unlock(obj);
1042
1043         return err;
1044 }
1045
1046 static int __cpu_check_vmap(struct drm_i915_gem_object *obj, u32 dword, u32 val)
1047 {
1048         unsigned long n = obj->base.size >> PAGE_SHIFT;
1049         u32 *ptr;
1050         int err;
1051
1052         err = i915_gem_object_wait(obj, 0, MAX_SCHEDULE_TIMEOUT);
1053         if (err)
1054                 return err;
1055
1056         ptr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
1057         if (IS_ERR(ptr))
1058                 return PTR_ERR(ptr);
1059
1060         ptr += dword;
1061         while (n--) {
1062                 if (*ptr != val) {
1063                         pr_err("base[%u]=%08x, val=%08x\n",
1064                                dword, *ptr, val);
1065                         err = -EINVAL;
1066                         break;
1067                 }
1068
1069                 ptr += PAGE_SIZE / sizeof(*ptr);
1070         }
1071
1072         i915_gem_object_unpin_map(obj);
1073         return err;
1074 }
1075
1076 static int cpu_check(struct drm_i915_gem_object *obj, u32 dword, u32 val)
1077 {
1078         if (i915_gem_object_has_struct_page(obj))
1079                 return __cpu_check_shmem(obj, dword, val);
1080         else
1081                 return __cpu_check_vmap(obj, dword, val);
1082 }
1083
1084 static int __igt_write_huge(struct intel_context *ce,
1085                             struct drm_i915_gem_object *obj,
1086                             u64 size, u64 offset,
1087                             u32 dword, u32 val)
1088 {
1089         unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
1090         struct i915_vma *vma;
1091         int err;
1092
1093         vma = i915_vma_instance(obj, ce->vm, NULL);
1094         if (IS_ERR(vma))
1095                 return PTR_ERR(vma);
1096
1097         err = i915_vma_pin(vma, size, 0, flags | offset);
1098         if (err) {
1099                 /*
1100                  * The ggtt may have some pages reserved so
1101                  * refrain from erroring out.
1102                  */
1103                 if (err == -ENOSPC && i915_is_ggtt(ce->vm))
1104                         err = 0;
1105
1106                 return err;
1107         }
1108
1109         err = igt_check_page_sizes(vma);
1110         if (err)
1111                 goto out_vma_unpin;
1112
1113         err = gpu_write(ce, vma, dword, val);
1114         if (err) {
1115                 pr_err("gpu-write failed at offset=%llx\n", offset);
1116                 goto out_vma_unpin;
1117         }
1118
1119         err = cpu_check(obj, dword, val);
1120         if (err) {
1121                 pr_err("cpu-check failed at offset=%llx\n", offset);
1122                 goto out_vma_unpin;
1123         }
1124
1125 out_vma_unpin:
1126         i915_vma_unpin(vma);
1127         return err;
1128 }
1129
1130 static int igt_write_huge(struct drm_i915_private *i915,
1131                           struct drm_i915_gem_object *obj)
1132 {
1133         struct i915_gem_engines *engines;
1134         struct i915_gem_engines_iter it;
1135         struct intel_context *ce;
1136         I915_RND_STATE(prng);
1137         IGT_TIMEOUT(end_time);
1138         unsigned int max_page_size;
1139         unsigned int count;
1140         struct i915_gem_context *ctx;
1141         struct file *file;
1142         u64 max;
1143         u64 num;
1144         u64 size;
1145         int *order;
1146         int i, n;
1147         int err = 0;
1148
1149         file = mock_file(i915);
1150         if (IS_ERR(file))
1151                 return PTR_ERR(file);
1152
1153         ctx = hugepage_ctx(i915, file);
1154         if (IS_ERR(ctx)) {
1155                 err = PTR_ERR(ctx);
1156                 goto out;
1157         }
1158
1159         GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
1160
1161         size = obj->base.size;
1162         if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K)
1163                 size = round_up(size, I915_GTT_PAGE_SIZE_2M);
1164
1165         n = 0;
1166         count = 0;
1167         max = U64_MAX;
1168         for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
1169                 count++;
1170                 if (!intel_engine_can_store_dword(ce->engine))
1171                         continue;
1172
1173                 max = min(max, ce->vm->total);
1174                 n++;
1175         }
1176         i915_gem_context_unlock_engines(ctx);
1177         if (!n)
1178                 goto out;
1179
1180         /*
1181          * To keep things interesting when alternating between engines in our
1182          * randomized order, lets also make feeding to the same engine a few
1183          * times in succession a possibility by enlarging the permutation array.
1184          */
1185         order = i915_random_order(count * count, &prng);
1186         if (!order)
1187                 return -ENOMEM;
1188
1189         max_page_size = rounddown_pow_of_two(obj->mm.page_sizes.sg);
1190         max = div_u64(max - size, max_page_size);
1191
1192         /*
1193          * Try various offsets in an ascending/descending fashion until we
1194          * timeout -- we want to avoid issues hidden by effectively always using
1195          * offset = 0.
1196          */
1197         i = 0;
1198         engines = i915_gem_context_lock_engines(ctx);
1199         for_each_prime_number_from(num, 0, max) {
1200                 u64 offset_low = num * max_page_size;
1201                 u64 offset_high = (max - num) * max_page_size;
1202                 u32 dword = offset_in_page(num) / 4;
1203                 struct intel_context *ce;
1204
1205                 ce = engines->engines[order[i] % engines->num_engines];
1206                 i = (i + 1) % (count * count);
1207                 if (!ce || !intel_engine_can_store_dword(ce->engine))
1208                         continue;
1209
1210                 /*
1211                  * In order to utilize 64K pages we need to both pad the vma
1212                  * size and ensure the vma offset is at the start of the pt
1213                  * boundary, however to improve coverage we opt for testing both
1214                  * aligned and unaligned offsets.
1215                  */
1216                 if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K)
1217                         offset_low = round_down(offset_low,
1218                                                 I915_GTT_PAGE_SIZE_2M);
1219
1220                 err = __igt_write_huge(ce, obj, size, offset_low,
1221                                        dword, num + 1);
1222                 if (err)
1223                         break;
1224
1225                 err = __igt_write_huge(ce, obj, size, offset_high,
1226                                        dword, num + 1);
1227                 if (err)
1228                         break;
1229
1230                 if (igt_timeout(end_time,
1231                                 "%s timed out on %s, offset_low=%llx offset_high=%llx, max_page_size=%x\n",
1232                                 __func__, ce->engine->name, offset_low, offset_high,
1233                                 max_page_size))
1234                         break;
1235         }
1236         i915_gem_context_unlock_engines(ctx);
1237
1238         kfree(order);
1239
1240 out:
1241         fput(file);
1242         return err;
1243 }
1244
1245 typedef struct drm_i915_gem_object *
1246 (*igt_create_fn)(struct drm_i915_private *i915, u32 size, u32 flags);
1247
1248 static inline bool igt_can_allocate_thp(struct drm_i915_private *i915)
1249 {
1250         return i915->mm.gemfs && has_transparent_hugepage();
1251 }
1252
1253 static struct drm_i915_gem_object *
1254 igt_create_shmem(struct drm_i915_private *i915, u32 size, u32 flags)
1255 {
1256         if (!igt_can_allocate_thp(i915)) {
1257                 pr_info("%s missing THP support, skipping\n", __func__);
1258                 return ERR_PTR(-ENODEV);
1259         }
1260
1261         return i915_gem_object_create_shmem(i915, size);
1262 }
1263
1264 static struct drm_i915_gem_object *
1265 igt_create_internal(struct drm_i915_private *i915, u32 size, u32 flags)
1266 {
1267         return i915_gem_object_create_internal(i915, size);
1268 }
1269
1270 static struct drm_i915_gem_object *
1271 igt_create_system(struct drm_i915_private *i915, u32 size, u32 flags)
1272 {
1273         return huge_pages_object(i915, size, size);
1274 }
1275
1276 static struct drm_i915_gem_object *
1277 igt_create_local(struct drm_i915_private *i915, u32 size, u32 flags)
1278 {
1279         return i915_gem_object_create_lmem(i915, size, flags);
1280 }
1281
1282 static u32 igt_random_size(struct rnd_state *prng,
1283                            u32 min_page_size,
1284                            u32 max_page_size)
1285 {
1286         u64 mask;
1287         u32 size;
1288
1289         GEM_BUG_ON(!is_power_of_2(min_page_size));
1290         GEM_BUG_ON(!is_power_of_2(max_page_size));
1291         GEM_BUG_ON(min_page_size < PAGE_SIZE);
1292         GEM_BUG_ON(min_page_size > max_page_size);
1293
1294         mask = ((max_page_size << 1ULL) - 1) & PAGE_MASK;
1295         size = prandom_u32_state(prng) & mask;
1296         if (size < min_page_size)
1297                 size |= min_page_size;
1298
1299         return size;
1300 }
1301
1302 static int igt_ppgtt_smoke_huge(void *arg)
1303 {
1304         struct drm_i915_private *i915 = arg;
1305         struct drm_i915_gem_object *obj;
1306         I915_RND_STATE(prng);
1307         struct {
1308                 igt_create_fn fn;
1309                 u32 min;
1310                 u32 max;
1311         } backends[] = {
1312                 { igt_create_internal, SZ_64K, SZ_2M,  },
1313                 { igt_create_shmem,    SZ_64K, SZ_32M, },
1314                 { igt_create_local,    SZ_64K, SZ_1G,  },
1315         };
1316         int err;
1317         int i;
1318
1319         /*
1320          * Sanity check that the HW uses huge pages correctly through our
1321          * various backends -- ensure that our writes land in the right place.
1322          */
1323
1324         for (i = 0; i < ARRAY_SIZE(backends); ++i) {
1325                 u32 min = backends[i].min;
1326                 u32 max = backends[i].max;
1327                 u32 size = max;
1328
1329 try_again:
1330                 size = igt_random_size(&prng, min, rounddown_pow_of_two(size));
1331
1332                 obj = backends[i].fn(i915, size, 0);
1333                 if (IS_ERR(obj)) {
1334                         err = PTR_ERR(obj);
1335                         if (err == -E2BIG) {
1336                                 size >>= 1;
1337                                 goto try_again;
1338                         } else if (err == -ENODEV) {
1339                                 err = 0;
1340                                 continue;
1341                         }
1342
1343                         return err;
1344                 }
1345
1346                 err = i915_gem_object_pin_pages_unlocked(obj);
1347                 if (err) {
1348                         if (err == -ENXIO || err == -E2BIG || err == -ENOMEM) {
1349                                 i915_gem_object_put(obj);
1350                                 size >>= 1;
1351                                 goto try_again;
1352                         }
1353                         goto out_put;
1354                 }
1355
1356                 if (obj->mm.page_sizes.phys < min) {
1357                         pr_info("%s unable to allocate huge-page(s) with size=%u, i=%d\n",
1358                                 __func__, size, i);
1359                         err = -ENOMEM;
1360                         goto out_unpin;
1361                 }
1362
1363                 err = igt_write_huge(i915, obj);
1364                 if (err) {
1365                         pr_err("%s write-huge failed with size=%u, i=%d\n",
1366                                __func__, size, i);
1367                 }
1368 out_unpin:
1369                 i915_gem_object_lock(obj, NULL);
1370                 i915_gem_object_unpin_pages(obj);
1371                 __i915_gem_object_put_pages(obj);
1372                 i915_gem_object_unlock(obj);
1373 out_put:
1374                 i915_gem_object_put(obj);
1375
1376                 if (err == -ENOMEM || err == -ENXIO)
1377                         err = 0;
1378
1379                 if (err)
1380                         break;
1381
1382                 cond_resched();
1383         }
1384
1385         return err;
1386 }
1387
1388 static int igt_ppgtt_sanity_check(void *arg)
1389 {
1390         struct drm_i915_private *i915 = arg;
1391         unsigned int supported = INTEL_INFO(i915)->page_sizes;
1392         struct {
1393                 igt_create_fn fn;
1394                 unsigned int flags;
1395         } backends[] = {
1396                 { igt_create_system, 0,                        },
1397                 { igt_create_local,  0,                        },
1398                 { igt_create_local,  I915_BO_ALLOC_CONTIGUOUS, },
1399         };
1400         struct {
1401                 u32 size;
1402                 u32 pages;
1403         } combos[] = {
1404                 { SZ_64K,               SZ_64K          },
1405                 { SZ_2M,                SZ_2M           },
1406                 { SZ_2M,                SZ_64K          },
1407                 { SZ_2M - SZ_64K,       SZ_64K          },
1408                 { SZ_2M - SZ_4K,        SZ_64K | SZ_4K  },
1409                 { SZ_2M + SZ_4K,        SZ_64K | SZ_4K  },
1410                 { SZ_2M + SZ_4K,        SZ_2M  | SZ_4K  },
1411                 { SZ_2M + SZ_64K,       SZ_2M  | SZ_64K },
1412         };
1413         int i, j;
1414         int err;
1415
1416         if (supported == I915_GTT_PAGE_SIZE_4K)
1417                 return 0;
1418
1419         /*
1420          * Sanity check that the HW behaves with a limited set of combinations.
1421          * We already have a bunch of randomised testing, which should give us
1422          * a decent amount of variation between runs, however we should keep
1423          * this to limit the chances of introducing a temporary regression, by
1424          * testing the most obvious cases that might make something blow up.
1425          */
1426
1427         for (i = 0; i < ARRAY_SIZE(backends); ++i) {
1428                 for (j = 0; j < ARRAY_SIZE(combos); ++j) {
1429                         struct drm_i915_gem_object *obj;
1430                         u32 size = combos[j].size;
1431                         u32 pages = combos[j].pages;
1432
1433                         obj = backends[i].fn(i915, size, backends[i].flags);
1434                         if (IS_ERR(obj)) {
1435                                 err = PTR_ERR(obj);
1436                                 if (err == -ENODEV) {
1437                                         pr_info("Device lacks local memory, skipping\n");
1438                                         err = 0;
1439                                         break;
1440                                 }
1441
1442                                 return err;
1443                         }
1444
1445                         err = i915_gem_object_pin_pages_unlocked(obj);
1446                         if (err) {
1447                                 i915_gem_object_put(obj);
1448                                 goto out;
1449                         }
1450
1451                         GEM_BUG_ON(pages > obj->base.size);
1452                         pages = pages & supported;
1453
1454                         if (pages)
1455                                 obj->mm.page_sizes.sg = pages;
1456
1457                         err = igt_write_huge(i915, obj);
1458
1459                         i915_gem_object_lock(obj, NULL);
1460                         i915_gem_object_unpin_pages(obj);
1461                         __i915_gem_object_put_pages(obj);
1462                         i915_gem_object_unlock(obj);
1463                         i915_gem_object_put(obj);
1464
1465                         if (err) {
1466                                 pr_err("%s write-huge failed with size=%u pages=%u i=%d, j=%d\n",
1467                                        __func__, size, pages, i, j);
1468                                 goto out;
1469                         }
1470                 }
1471
1472                 cond_resched();
1473         }
1474
1475 out:
1476         if (err == -ENOMEM)
1477                 err = 0;
1478
1479         return err;
1480 }
1481
1482 static int igt_ppgtt_compact(void *arg)
1483 {
1484         struct drm_i915_private *i915 = arg;
1485         struct drm_i915_gem_object *obj;
1486         int err;
1487
1488         /*
1489          * Simple test to catch issues with compact 64K pages -- since the pt is
1490          * compacted to 256B that gives us 32 entries per pt, however since the
1491          * backing page for the pt is 4K, any extra entries we might incorrectly
1492          * write out should be ignored by the HW. If ever hit such a case this
1493          * test should catch it since some of our writes would land in scratch.
1494          */
1495
1496         if (!HAS_64K_PAGES(i915)) {
1497                 pr_info("device lacks compact 64K page support, skipping\n");
1498                 return 0;
1499         }
1500
1501         if (!HAS_LMEM(i915)) {
1502                 pr_info("device lacks LMEM support, skipping\n");
1503                 return 0;
1504         }
1505
1506         /* We want the range to cover multiple page-table boundaries. */
1507         obj = i915_gem_object_create_lmem(i915, SZ_4M, 0);
1508         if (IS_ERR(obj))
1509                 return PTR_ERR(obj);
1510
1511         err = i915_gem_object_pin_pages_unlocked(obj);
1512         if (err)
1513                 goto out_put;
1514
1515         if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_64K) {
1516                 pr_info("LMEM compact unable to allocate huge-page(s)\n");
1517                 goto out_unpin;
1518         }
1519
1520         /*
1521          * Disable 2M GTT pages by forcing the page-size to 64K for the GTT
1522          * insertion.
1523          */
1524         obj->mm.page_sizes.sg = I915_GTT_PAGE_SIZE_64K;
1525
1526         err = igt_write_huge(i915, obj);
1527         if (err)
1528                 pr_err("LMEM compact write-huge failed\n");
1529
1530 out_unpin:
1531         i915_gem_object_unpin_pages(obj);
1532 out_put:
1533         i915_gem_object_put(obj);
1534
1535         if (err == -ENOMEM)
1536                 err = 0;
1537
1538         return err;
1539 }
1540
1541 static int igt_tmpfs_fallback(void *arg)
1542 {
1543         struct drm_i915_private *i915 = arg;
1544         struct i915_address_space *vm;
1545         struct i915_gem_context *ctx;
1546         struct vfsmount *gemfs = i915->mm.gemfs;
1547         struct drm_i915_gem_object *obj;
1548         struct i915_vma *vma;
1549         struct file *file;
1550         u32 *vaddr;
1551         int err = 0;
1552
1553         file = mock_file(i915);
1554         if (IS_ERR(file))
1555                 return PTR_ERR(file);
1556
1557         ctx = hugepage_ctx(i915, file);
1558         if (IS_ERR(ctx)) {
1559                 err = PTR_ERR(ctx);
1560                 goto out;
1561         }
1562         vm = i915_gem_context_get_eb_vm(ctx);
1563
1564         /*
1565          * Make sure that we don't burst into a ball of flames upon falling back
1566          * to tmpfs, which we rely on if on the off-chance we encouter a failure
1567          * when setting up gemfs.
1568          */
1569
1570         i915->mm.gemfs = NULL;
1571
1572         obj = i915_gem_object_create_shmem(i915, PAGE_SIZE);
1573         if (IS_ERR(obj)) {
1574                 err = PTR_ERR(obj);
1575                 goto out_restore;
1576         }
1577
1578         vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
1579         if (IS_ERR(vaddr)) {
1580                 err = PTR_ERR(vaddr);
1581                 goto out_put;
1582         }
1583         *vaddr = 0xdeadbeaf;
1584
1585         __i915_gem_object_flush_map(obj, 0, 64);
1586         i915_gem_object_unpin_map(obj);
1587
1588         vma = i915_vma_instance(obj, vm, NULL);
1589         if (IS_ERR(vma)) {
1590                 err = PTR_ERR(vma);
1591                 goto out_put;
1592         }
1593
1594         err = i915_vma_pin(vma, 0, 0, PIN_USER);
1595         if (err)
1596                 goto out_put;
1597
1598         err = igt_check_page_sizes(vma);
1599
1600         i915_vma_unpin(vma);
1601 out_put:
1602         i915_gem_object_put(obj);
1603 out_restore:
1604         i915->mm.gemfs = gemfs;
1605
1606         i915_vm_put(vm);
1607 out:
1608         fput(file);
1609         return err;
1610 }
1611
1612 static int igt_shrink_thp(void *arg)
1613 {
1614         struct drm_i915_private *i915 = arg;
1615         struct i915_address_space *vm;
1616         struct i915_gem_context *ctx;
1617         struct drm_i915_gem_object *obj;
1618         struct i915_gem_engines_iter it;
1619         struct intel_context *ce;
1620         struct i915_vma *vma;
1621         struct file *file;
1622         unsigned int flags = PIN_USER;
1623         unsigned int n;
1624         bool should_swap;
1625         int err;
1626
1627         if (!igt_can_allocate_thp(i915)) {
1628                 pr_info("missing THP support, skipping\n");
1629                 return 0;
1630         }
1631
1632         file = mock_file(i915);
1633         if (IS_ERR(file))
1634                 return PTR_ERR(file);
1635
1636         ctx = hugepage_ctx(i915, file);
1637         if (IS_ERR(ctx)) {
1638                 err = PTR_ERR(ctx);
1639                 goto out;
1640         }
1641         vm = i915_gem_context_get_eb_vm(ctx);
1642
1643         /*
1644          * Sanity check shrinking huge-paged object -- make sure nothing blows
1645          * up.
1646          */
1647
1648         obj = i915_gem_object_create_shmem(i915, SZ_2M);
1649         if (IS_ERR(obj)) {
1650                 err = PTR_ERR(obj);
1651                 goto out_vm;
1652         }
1653
1654         vma = i915_vma_instance(obj, vm, NULL);
1655         if (IS_ERR(vma)) {
1656                 err = PTR_ERR(vma);
1657                 goto out_put;
1658         }
1659
1660         err = i915_vma_pin(vma, 0, 0, flags);
1661         if (err)
1662                 goto out_put;
1663
1664         if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_2M) {
1665                 pr_info("failed to allocate THP, finishing test early\n");
1666                 goto out_unpin;
1667         }
1668
1669         err = igt_check_page_sizes(vma);
1670         if (err)
1671                 goto out_unpin;
1672
1673         n = 0;
1674
1675         for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
1676                 if (!intel_engine_can_store_dword(ce->engine))
1677                         continue;
1678
1679                 err = gpu_write(ce, vma, n++, 0xdeadbeaf);
1680                 if (err)
1681                         break;
1682         }
1683         i915_gem_context_unlock_engines(ctx);
1684         /*
1685          * Nuke everything *before* we unpin the pages so we can be reasonably
1686          * sure that when later checking get_nr_swap_pages() that some random
1687          * leftover object doesn't steal the remaining swap space.
1688          */
1689         i915_gem_shrink(NULL, i915, -1UL, NULL,
1690                         I915_SHRINK_BOUND |
1691                         I915_SHRINK_UNBOUND |
1692                         I915_SHRINK_ACTIVE);
1693         i915_vma_unpin(vma);
1694         if (err)
1695                 goto out_put;
1696
1697         /*
1698          * Now that the pages are *unpinned* shrinking should invoke
1699          * shmem to truncate our pages, if we have available swap.
1700          */
1701         should_swap = get_nr_swap_pages() > 0;
1702         i915_gem_shrink(NULL, i915, -1UL, NULL,
1703                         I915_SHRINK_BOUND |
1704                         I915_SHRINK_UNBOUND |
1705                         I915_SHRINK_ACTIVE |
1706                         I915_SHRINK_WRITEBACK);
1707         if (should_swap == i915_gem_object_has_pages(obj)) {
1708                 pr_err("unexpected pages mismatch, should_swap=%s\n",
1709                        yesno(should_swap));
1710                 err = -EINVAL;
1711                 goto out_put;
1712         }
1713
1714         if (should_swap == (obj->mm.page_sizes.sg || obj->mm.page_sizes.phys)) {
1715                 pr_err("unexpected residual page-size bits, should_swap=%s\n",
1716                        yesno(should_swap));
1717                 err = -EINVAL;
1718                 goto out_put;
1719         }
1720
1721         err = i915_vma_pin(vma, 0, 0, flags);
1722         if (err)
1723                 goto out_put;
1724
1725         while (n--) {
1726                 err = cpu_check(obj, n, 0xdeadbeaf);
1727                 if (err)
1728                         break;
1729         }
1730
1731 out_unpin:
1732         i915_vma_unpin(vma);
1733 out_put:
1734         i915_gem_object_put(obj);
1735 out_vm:
1736         i915_vm_put(vm);
1737 out:
1738         fput(file);
1739         return err;
1740 }
1741
1742 int i915_gem_huge_page_mock_selftests(void)
1743 {
1744         static const struct i915_subtest tests[] = {
1745                 SUBTEST(igt_mock_exhaust_device_supported_pages),
1746                 SUBTEST(igt_mock_memory_region_huge_pages),
1747                 SUBTEST(igt_mock_ppgtt_misaligned_dma),
1748                 SUBTEST(igt_mock_ppgtt_huge_fill),
1749                 SUBTEST(igt_mock_ppgtt_64K),
1750         };
1751         struct drm_i915_private *dev_priv;
1752         struct i915_ppgtt *ppgtt;
1753         int err;
1754
1755         dev_priv = mock_gem_device();
1756         if (!dev_priv)
1757                 return -ENOMEM;
1758
1759         /* Pretend to be a device which supports the 48b PPGTT */
1760         mkwrite_device_info(dev_priv)->ppgtt_type = INTEL_PPGTT_FULL;
1761         mkwrite_device_info(dev_priv)->ppgtt_size = 48;
1762
1763         ppgtt = i915_ppgtt_create(to_gt(dev_priv), 0);
1764         if (IS_ERR(ppgtt)) {
1765                 err = PTR_ERR(ppgtt);
1766                 goto out_unlock;
1767         }
1768
1769         if (!i915_vm_is_4lvl(&ppgtt->vm)) {
1770                 pr_err("failed to create 48b PPGTT\n");
1771                 err = -EINVAL;
1772                 goto out_put;
1773         }
1774
1775         /* If we were ever hit this then it's time to mock the 64K scratch */
1776         if (!i915_vm_has_scratch_64K(&ppgtt->vm)) {
1777                 pr_err("PPGTT missing 64K scratch page\n");
1778                 err = -EINVAL;
1779                 goto out_put;
1780         }
1781
1782         err = i915_subtests(tests, ppgtt);
1783
1784 out_put:
1785         i915_vm_put(&ppgtt->vm);
1786 out_unlock:
1787         mock_destroy_device(dev_priv);
1788         return err;
1789 }
1790
1791 int i915_gem_huge_page_live_selftests(struct drm_i915_private *i915)
1792 {
1793         static const struct i915_subtest tests[] = {
1794                 SUBTEST(igt_shrink_thp),
1795                 SUBTEST(igt_tmpfs_fallback),
1796                 SUBTEST(igt_ppgtt_smoke_huge),
1797                 SUBTEST(igt_ppgtt_sanity_check),
1798                 SUBTEST(igt_ppgtt_compact),
1799         };
1800
1801         if (!HAS_PPGTT(i915)) {
1802                 pr_info("PPGTT not supported, skipping live-selftests\n");
1803                 return 0;
1804         }
1805
1806         if (intel_gt_is_wedged(to_gt(i915)))
1807                 return 0;
1808
1809         return i915_live_subtests(tests, i915);
1810 }