drivers/gpu/drm/i915/selftests/huge_pages.c

   1 /*
   2  * Copyright © 2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include "../i915_selftest.h"
  26
  27 #include <linux/prime_numbers.h>
  28
  29 #include "mock_drm.h"
  30
  31 static const unsigned int page_sizes[] = {
  32         I915_GTT_PAGE_SIZE_2M,
  33         I915_GTT_PAGE_SIZE_64K,
  34         I915_GTT_PAGE_SIZE_4K,
  35 };
  36
  37 static unsigned int get_largest_page_size(struct drm_i915_private *i915,
  38                                           u64 rem)
  39 {
  40         int i;
  41
  42         for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) {
  43                 unsigned int page_size = page_sizes[i];
  44
  45                 if (HAS_PAGE_SIZES(i915, page_size) && rem >= page_size)
  46                         return page_size;
  47         }
  48
  49         return 0;
  50 }
  51
  52 static void huge_pages_free_pages(struct sg_table *st)
  53 {
  54         struct scatterlist *sg;
  55
  56         for (sg = st->sgl; sg; sg = __sg_next(sg)) {
  57                 if (sg_page(sg))
  58                         __free_pages(sg_page(sg), get_order(sg->length));
  59         }
  60
  61         sg_free_table(st);
  62         kfree(st);
  63 }
  64
  65 static int get_huge_pages(struct drm_i915_gem_object *obj)
  66 {
  67 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY)
  68         unsigned int page_mask = obj->mm.page_mask;
  69         struct sg_table *st;
  70         struct scatterlist *sg;
  71         unsigned int sg_page_sizes;
  72         u64 rem;
  73
  74         st = kmalloc(sizeof(*st), GFP);
  75         if (!st)
  76                 return -ENOMEM;
  77
  78         if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) {
  79                 kfree(st);
  80                 return -ENOMEM;
  81         }
  82
  83         rem = obj->base.size;
  84         sg = st->sgl;
  85         st->nents = 0;
  86         sg_page_sizes = 0;
  87
  88         /*
  89          * Our goal here is simple, we want to greedily fill the object from
  90          * largest to smallest page-size, while ensuring that we use *every*
  91          * page-size as per the given page-mask.
  92          */
  93         do {
  94                 unsigned int bit = ilog2(page_mask);
  95                 unsigned int page_size = BIT(bit);
  96                 int order = get_order(page_size);
  97
  98                 do {
  99                         struct page *page;
 100
 101                         GEM_BUG_ON(order >= MAX_ORDER);
 102                         page = alloc_pages(GFP | __GFP_ZERO, order);
 103                         if (!page)
 104                                 goto err;
 105
 106                         sg_set_page(sg, page, page_size, 0);
 107                         sg_page_sizes |= page_size;
 108                         st->nents++;
 109
 110                         rem -= page_size;
 111                         if (!rem) {
 112                                 sg_mark_end(sg);
 113                                 break;
 114                         }
 115
 116                         sg = __sg_next(sg);
 117                 } while ((rem - ((page_size-1) & page_mask)) >= page_size);
 118
 119                 page_mask &= (page_size-1);
 120         } while (page_mask);
 121
 122         if (i915_gem_gtt_prepare_pages(obj, st))
 123                 goto err;
 124
 125         obj->mm.madv = I915_MADV_DONTNEED;
 126
 127         GEM_BUG_ON(sg_page_sizes != obj->mm.page_mask);
 128         __i915_gem_object_set_pages(obj, st, sg_page_sizes);
 129
 130         return 0;
 131
 132 err:
 133         sg_set_page(sg, NULL, 0, 0);
 134         sg_mark_end(sg);
 135         huge_pages_free_pages(st);
 136
 137         return -ENOMEM;
 138 }
 139
 140 static void put_huge_pages(struct drm_i915_gem_object *obj,
 141                            struct sg_table *pages)
 142 {
 143         i915_gem_gtt_finish_pages(obj, pages);
 144         huge_pages_free_pages(pages);
 145
 146         obj->mm.dirty = false;
 147         obj->mm.madv = I915_MADV_WILLNEED;
 148 }
 149
 150 static const struct drm_i915_gem_object_ops huge_page_ops = {
 151         .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
 152                  I915_GEM_OBJECT_IS_SHRINKABLE,
 153         .get_pages = get_huge_pages,
 154         .put_pages = put_huge_pages,
 155 };
 156
 157 static struct drm_i915_gem_object *
 158 huge_pages_object(struct drm_i915_private *i915,
 159                   u64 size,
 160                   unsigned int page_mask)
 161 {
 162         struct drm_i915_gem_object *obj;
 163
 164         GEM_BUG_ON(!size);
 165         GEM_BUG_ON(!IS_ALIGNED(size, BIT(__ffs(page_mask))));
 166
 167         if (size >> PAGE_SHIFT > INT_MAX)
 168                 return ERR_PTR(-E2BIG);
 169
 170         if (overflows_type(size, obj->base.size))
 171                 return ERR_PTR(-E2BIG);
 172
 173         obj = i915_gem_object_alloc(i915);
 174         if (!obj)
 175                 return ERR_PTR(-ENOMEM);
 176
 177         drm_gem_private_object_init(&i915->drm, &obj->base, size);
 178         i915_gem_object_init(obj, &huge_page_ops);
 179
 180         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
 181         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
 182         obj->cache_level = I915_CACHE_NONE;
 183
 184         obj->mm.page_mask = page_mask;
 185
 186         return obj;
 187 }
 188
 189 static int fake_get_huge_pages(struct drm_i915_gem_object *obj)
 190 {
 191         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 192         const u64 max_len = rounddown_pow_of_two(UINT_MAX);
 193         struct sg_table *st;
 194         struct scatterlist *sg;
 195         unsigned int sg_page_sizes;
 196         u64 rem;
 197
 198         st = kmalloc(sizeof(*st), GFP);
 199         if (!st)
 200                 return -ENOMEM;
 201
 202         if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) {
 203                 kfree(st);
 204                 return -ENOMEM;
 205         }
 206
 207         /* Use optimal page sized chunks to fill in the sg table */
 208         rem = obj->base.size;
 209         sg = st->sgl;
 210         st->nents = 0;
 211         sg_page_sizes = 0;
 212         do {
 213                 unsigned int page_size = get_largest_page_size(i915, rem);
 214                 unsigned int len = min(page_size * div_u64(rem, page_size),
 215                                        max_len);
 216
 217                 GEM_BUG_ON(!page_size);
 218
 219                 sg->offset = 0;
 220                 sg->length = len;
 221                 sg_dma_len(sg) = len;
 222                 sg_dma_address(sg) = page_size;
 223
 224                 sg_page_sizes |= len;
 225
 226                 st->nents++;
 227
 228                 rem -= len;
 229                 if (!rem) {
 230                         sg_mark_end(sg);
 231                         break;
 232                 }
 233
 234                 sg = sg_next(sg);
 235         } while (1);
 236
 237         obj->mm.madv = I915_MADV_DONTNEED;
 238
 239         __i915_gem_object_set_pages(obj, st, sg_page_sizes);
 240
 241         return 0;
 242 }
 243
 244 static int fake_get_huge_pages_single(struct drm_i915_gem_object *obj)
 245 {
 246         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 247         struct sg_table *st;
 248         struct scatterlist *sg;
 249         unsigned int page_size;
 250
 251         st = kmalloc(sizeof(*st), GFP);
 252         if (!st)
 253                 return -ENOMEM;
 254
 255         if (sg_alloc_table(st, 1, GFP)) {
 256                 kfree(st);
 257                 return -ENOMEM;
 258         }
 259
 260         sg = st->sgl;
 261         st->nents = 1;
 262
 263         page_size = get_largest_page_size(i915, obj->base.size);
 264         GEM_BUG_ON(!page_size);
 265
 266         sg->offset = 0;
 267         sg->length = obj->base.size;
 268         sg_dma_len(sg) = obj->base.size;
 269         sg_dma_address(sg) = page_size;
 270
 271         obj->mm.madv = I915_MADV_DONTNEED;
 272
 273         __i915_gem_object_set_pages(obj, st, sg->length);
 274
 275         return 0;
 276 #undef GFP
 277 }
 278
 279 static void fake_free_huge_pages(struct drm_i915_gem_object *obj,
 280                                  struct sg_table *pages)
 281 {
 282         sg_free_table(pages);
 283         kfree(pages);
 284 }
 285
 286 static void fake_put_huge_pages(struct drm_i915_gem_object *obj,
 287                                 struct sg_table *pages)
 288 {
 289         fake_free_huge_pages(obj, pages);
 290         obj->mm.dirty = false;
 291         obj->mm.madv = I915_MADV_WILLNEED;
 292 }
 293
 294 static const struct drm_i915_gem_object_ops fake_ops = {
 295         .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
 296         .get_pages = fake_get_huge_pages,
 297         .put_pages = fake_put_huge_pages,
 298 };
 299
 300 static const struct drm_i915_gem_object_ops fake_ops_single = {
 301         .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
 302         .get_pages = fake_get_huge_pages_single,
 303         .put_pages = fake_put_huge_pages,
 304 };
 305
 306 static struct drm_i915_gem_object *
 307 fake_huge_pages_object(struct drm_i915_private *i915, u64 size, bool single)
 308 {
 309         struct drm_i915_gem_object *obj;
 310
 311         GEM_BUG_ON(!size);
 312         GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
 313
 314         if (size >> PAGE_SHIFT > UINT_MAX)
 315                 return ERR_PTR(-E2BIG);
 316
 317         if (overflows_type(size, obj->base.size))
 318                 return ERR_PTR(-E2BIG);
 319
 320         obj = i915_gem_object_alloc(i915);
 321         if (!obj)
 322                 return ERR_PTR(-ENOMEM);
 323
 324         drm_gem_private_object_init(&i915->drm, &obj->base, size);
 325
 326         if (single)
 327                 i915_gem_object_init(obj, &fake_ops_single);
 328         else
 329                 i915_gem_object_init(obj, &fake_ops);
 330
 331         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
 332         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
 333         obj->cache_level = I915_CACHE_NONE;
 334
 335         return obj;
 336 }
 337
 338 static int igt_check_page_sizes(struct i915_vma *vma)
 339 {
 340         struct drm_i915_private *i915 = to_i915(vma->obj->base.dev);
 341         unsigned int supported = INTEL_INFO(i915)->page_sizes;
 342         struct drm_i915_gem_object *obj = vma->obj;
 343         int err = 0;
 344
 345         if (!HAS_PAGE_SIZES(i915, vma->page_sizes.sg)) {
 346                 pr_err("unsupported page_sizes.sg=%u, supported=%u\n",
 347                        vma->page_sizes.sg & ~supported, supported);
 348                 err = -EINVAL;
 349         }
 350
 351         if (!HAS_PAGE_SIZES(i915, vma->page_sizes.gtt)) {
 352                 pr_err("unsupported page_sizes.gtt=%u, supported=%u\n",
 353                        vma->page_sizes.gtt & ~supported, supported);
 354                 err = -EINVAL;
 355         }
 356
 357         if (vma->page_sizes.phys != obj->mm.page_sizes.phys) {
 358                 pr_err("vma->page_sizes.phys(%u) != obj->mm.page_sizes.phys(%u)\n",
 359                        vma->page_sizes.phys, obj->mm.page_sizes.phys);
 360                 err = -EINVAL;
 361         }
 362
 363         if (vma->page_sizes.sg != obj->mm.page_sizes.sg) {
 364                 pr_err("vma->page_sizes.sg(%u) != obj->mm.page_sizes.sg(%u)\n",
 365                        vma->page_sizes.sg, obj->mm.page_sizes.sg);
 366                 err = -EINVAL;
 367         }
 368
 369         if (obj->mm.page_sizes.gtt) {
 370                 pr_err("obj->page_sizes.gtt(%u) should never be set\n",
 371                        obj->mm.page_sizes.gtt);
 372                 err = -EINVAL;
 373         }
 374
 375         return err;
 376 }
 377
 378 static int igt_mock_exhaust_device_supported_pages(void *arg)
 379 {
 380         struct i915_hw_ppgtt *ppgtt = arg;
 381         struct drm_i915_private *i915 = ppgtt->base.i915;
 382         unsigned int saved_mask = INTEL_INFO(i915)->page_sizes;
 383         struct drm_i915_gem_object *obj;
 384         struct i915_vma *vma;
 385         int i, j, single;
 386         int err;
 387
 388         /*
 389          * Sanity check creating objects with every valid page support
 390          * combination for our mock device.
 391          */
 392
 393         for (i = 1; i < BIT(ARRAY_SIZE(page_sizes)); i++) {
 394                 unsigned int combination = 0;
 395
 396                 for (j = 0; j < ARRAY_SIZE(page_sizes); j++) {
 397                         if (i & BIT(j))
 398                                 combination |= page_sizes[j];
 399                 }
 400
 401                 mkwrite_device_info(i915)->page_sizes = combination;
 402
 403                 for (single = 0; single <= 1; ++single) {
 404                         obj = fake_huge_pages_object(i915, combination, !!single);
 405                         if (IS_ERR(obj)) {
 406                                 err = PTR_ERR(obj);
 407                                 goto out_device;
 408                         }
 409
 410                         if (obj->base.size != combination) {
 411                                 pr_err("obj->base.size=%zu, expected=%u\n",
 412                                        obj->base.size, combination);
 413                                 err = -EINVAL;
 414                                 goto out_put;
 415                         }
 416
 417                         vma = i915_vma_instance(obj, &ppgtt->base, NULL);
 418                         if (IS_ERR(vma)) {
 419                                 err = PTR_ERR(vma);
 420                                 goto out_put;
 421                         }
 422
 423                         err = i915_vma_pin(vma, 0, 0, PIN_USER);
 424                         if (err)
 425                                 goto out_close;
 426
 427                         err = igt_check_page_sizes(vma);
 428
 429                         if (vma->page_sizes.sg != combination) {
 430                                 pr_err("page_sizes.sg=%u, expected=%u\n",
 431                                        vma->page_sizes.sg, combination);
 432                                 err = -EINVAL;
 433                         }
 434
 435                         i915_vma_unpin(vma);
 436                         i915_vma_close(vma);
 437
 438                         i915_gem_object_put(obj);
 439
 440                         if (err)
 441                                 goto out_device;
 442                 }
 443         }
 444
 445         goto out_device;
 446
 447 out_close:
 448         i915_vma_close(vma);
 449 out_put:
 450         i915_gem_object_put(obj);
 451 out_device:
 452         mkwrite_device_info(i915)->page_sizes = saved_mask;
 453
 454         return err;
 455 }
 456
 457 static int igt_mock_ppgtt_misaligned_dma(void *arg)
 458 {
 459         struct i915_hw_ppgtt *ppgtt = arg;
 460         struct drm_i915_private *i915 = ppgtt->base.i915;
 461         unsigned long supported = INTEL_INFO(i915)->page_sizes;
 462         struct drm_i915_gem_object *obj;
 463         int bit;
 464         int err;
 465
 466         /*
 467          * Sanity check dma misalignment for huge pages -- the dma addresses we
 468          * insert into the paging structures need to always respect the page
 469          * size alignment.
 470          */
 471
 472         bit = ilog2(I915_GTT_PAGE_SIZE_64K);
 473
 474         for_each_set_bit_from(bit, &supported,
 475                               ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
 476                 IGT_TIMEOUT(end_time);
 477                 unsigned int page_size = BIT(bit);
 478                 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
 479                 unsigned int offset;
 480                 unsigned int size =
 481                         round_up(page_size, I915_GTT_PAGE_SIZE_2M) << 1;
 482                 struct i915_vma *vma;
 483
 484                 obj = fake_huge_pages_object(i915, size, true);
 485                 if (IS_ERR(obj))
 486                         return PTR_ERR(obj);
 487
 488                 if (obj->base.size != size) {
 489                         pr_err("obj->base.size=%zu, expected=%u\n",
 490                                obj->base.size, size);
 491                         err = -EINVAL;
 492                         goto out_put;
 493                 }
 494
 495                 err = i915_gem_object_pin_pages(obj);
 496                 if (err)
 497                         goto out_put;
 498
 499                 /* Force the page size for this object */
 500                 obj->mm.page_sizes.sg = page_size;
 501
 502                 vma = i915_vma_instance(obj, &ppgtt->base, NULL);
 503                 if (IS_ERR(vma)) {
 504                         err = PTR_ERR(vma);
 505                         goto out_unpin;
 506                 }
 507
 508                 err = i915_vma_pin(vma, 0, 0, flags);
 509                 if (err) {
 510                         i915_vma_close(vma);
 511                         goto out_unpin;
 512                 }
 513
 514
 515                 err = igt_check_page_sizes(vma);
 516
 517                 if (vma->page_sizes.gtt != page_size) {
 518                         pr_err("page_sizes.gtt=%u, expected %u\n",
 519                                vma->page_sizes.gtt, page_size);
 520                         err = -EINVAL;
 521                 }
 522
 523                 i915_vma_unpin(vma);
 524
 525                 if (err) {
 526                         i915_vma_close(vma);
 527                         goto out_unpin;
 528                 }
 529
 530                 /*
 531                  * Try all the other valid offsets until the next
 532                  * boundary -- should always fall back to using 4K
 533                  * pages.
 534                  */
 535                 for (offset = 4096; offset < page_size; offset += 4096) {
 536                         err = i915_vma_unbind(vma);
 537                         if (err) {
 538                                 i915_vma_close(vma);
 539                                 goto out_unpin;
 540                         }
 541
 542                         err = i915_vma_pin(vma, 0, 0, flags | offset);
 543                         if (err) {
 544                                 i915_vma_close(vma);
 545                                 goto out_unpin;
 546                         }
 547
 548                         err = igt_check_page_sizes(vma);
 549
 550                         if (vma->page_sizes.gtt != I915_GTT_PAGE_SIZE_4K) {
 551                                 pr_err("page_sizes.gtt=%u, expected %lu\n",
 552                                        vma->page_sizes.gtt, I915_GTT_PAGE_SIZE_4K);
 553                                 err = -EINVAL;
 554                         }
 555
 556                         i915_vma_unpin(vma);
 557
 558                         if (err) {
 559                                 i915_vma_close(vma);
 560                                 goto out_unpin;
 561                         }
 562
 563                         if (igt_timeout(end_time,
 564                                         "%s timed out at offset %x with page-size %x\n",
 565                                         __func__, offset, page_size))
 566                                 break;
 567                 }
 568
 569                 i915_vma_close(vma);
 570
 571                 i915_gem_object_unpin_pages(obj);
 572                 i915_gem_object_put(obj);
 573         }
 574
 575         return 0;
 576
 577 out_unpin:
 578         i915_gem_object_unpin_pages(obj);
 579 out_put:
 580         i915_gem_object_put(obj);
 581
 582         return err;
 583 }
 584
 585 static void close_object_list(struct list_head *objects,
 586                               struct i915_hw_ppgtt *ppgtt)
 587 {
 588         struct drm_i915_gem_object *obj, *on;
 589
 590         list_for_each_entry_safe(obj, on, objects, st_link) {
 591                 struct i915_vma *vma;
 592
 593                 vma = i915_vma_instance(obj, &ppgtt->base, NULL);
 594                 if (!IS_ERR(vma))
 595                         i915_vma_close(vma);
 596
 597                 list_del(&obj->st_link);
 598                 i915_gem_object_unpin_pages(obj);
 599                 i915_gem_object_put(obj);
 600         }
 601 }
 602
 603 static int igt_mock_ppgtt_huge_fill(void *arg)
 604 {
 605         struct i915_hw_ppgtt *ppgtt = arg;
 606         struct drm_i915_private *i915 = ppgtt->base.i915;
 607         unsigned long max_pages = ppgtt->base.total >> PAGE_SHIFT;
 608         unsigned long page_num;
 609         bool single = false;
 610         LIST_HEAD(objects);
 611         IGT_TIMEOUT(end_time);
 612         int err = -ENODEV;
 613
 614         for_each_prime_number_from(page_num, 1, max_pages) {
 615                 struct drm_i915_gem_object *obj;
 616                 u64 size = page_num << PAGE_SHIFT;
 617                 struct i915_vma *vma;
 618                 unsigned int expected_gtt = 0;
 619                 int i;
 620
 621                 obj = fake_huge_pages_object(i915, size, single);
 622                 if (IS_ERR(obj)) {
 623                         err = PTR_ERR(obj);
 624                         break;
 625                 }
 626
 627                 if (obj->base.size != size) {
 628                         pr_err("obj->base.size=%zd, expected=%llu\n",
 629                                obj->base.size, size);
 630                         i915_gem_object_put(obj);
 631                         err = -EINVAL;
 632                         break;
 633                 }
 634
 635                 err = i915_gem_object_pin_pages(obj);
 636                 if (err) {
 637                         i915_gem_object_put(obj);
 638                         break;
 639                 }
 640
 641                 list_add(&obj->st_link, &objects);
 642
 643                 vma = i915_vma_instance(obj, &ppgtt->base, NULL);
 644                 if (IS_ERR(vma)) {
 645                         err = PTR_ERR(vma);
 646                         break;
 647                 }
 648
 649                 err = i915_vma_pin(vma, 0, 0, PIN_USER);
 650                 if (err)
 651                         break;
 652
 653                 err = igt_check_page_sizes(vma);
 654                 if (err) {
 655                         i915_vma_unpin(vma);
 656                         break;
 657                 }
 658
 659                 /*
 660                  * Figure out the expected gtt page size knowing that we go from
 661                  * largest to smallest page size sg chunks, and that we align to
 662                  * the largest page size.
 663                  */
 664                 for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) {
 665                         unsigned int page_size = page_sizes[i];
 666
 667                         if (HAS_PAGE_SIZES(i915, page_size) &&
 668                             size >= page_size) {
 669                                 expected_gtt |= page_size;
 670                                 size &= page_size-1;
 671                         }
 672                 }
 673
 674                 GEM_BUG_ON(!expected_gtt);
 675                 GEM_BUG_ON(size);
 676
 677                 if (expected_gtt & I915_GTT_PAGE_SIZE_4K)
 678                         expected_gtt &= ~I915_GTT_PAGE_SIZE_64K;
 679
 680                 i915_vma_unpin(vma);
 681
 682                 if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) {
 683                         if (!IS_ALIGNED(vma->node.start,
 684                                         I915_GTT_PAGE_SIZE_2M)) {
 685                                 pr_err("node.start(%llx) not aligned to 2M\n",
 686                                        vma->node.start);
 687                                 err = -EINVAL;
 688                                 break;
 689                         }
 690
 691                         if (!IS_ALIGNED(vma->node.size,
 692                                         I915_GTT_PAGE_SIZE_2M)) {
 693                                 pr_err("node.size(%llx) not aligned to 2M\n",
 694                                        vma->node.size);
 695                                 err = -EINVAL;
 696                                 break;
 697                         }
 698                 }
 699
 700                 if (vma->page_sizes.gtt != expected_gtt) {
 701                         pr_err("gtt=%u, expected=%u, size=%zd, single=%s\n",
 702                                vma->page_sizes.gtt, expected_gtt,
 703                                obj->base.size, yesno(!!single));
 704                         err = -EINVAL;
 705                         break;
 706                 }
 707
 708                 if (igt_timeout(end_time,
 709                                 "%s timed out at size %zd\n",
 710                                 __func__, obj->base.size))
 711                         break;
 712
 713                 single = !single;
 714         }
 715
 716         close_object_list(&objects, ppgtt);
 717
 718         if (err == -ENOMEM || err == -ENOSPC)
 719                 err = 0;
 720
 721         return err;
 722 }
 723
 724 static int igt_mock_ppgtt_64K(void *arg)
 725 {
 726         struct i915_hw_ppgtt *ppgtt = arg;
 727         struct drm_i915_private *i915 = ppgtt->base.i915;
 728         struct drm_i915_gem_object *obj;
 729         const struct object_info {
 730                 unsigned int size;
 731                 unsigned int gtt;
 732                 unsigned int offset;
 733         } objects[] = {
 734                 /* Cases with forced padding/alignment */
 735                 {
 736                         .size = SZ_64K,
 737                         .gtt = I915_GTT_PAGE_SIZE_64K,
 738                         .offset = 0,
 739                 },
 740                 {
 741                         .size = SZ_64K + SZ_4K,
 742                         .gtt = I915_GTT_PAGE_SIZE_4K,
 743                         .offset = 0,
 744                 },
 745                 {
 746                         .size = SZ_64K - SZ_4K,
 747                         .gtt = I915_GTT_PAGE_SIZE_4K,
 748                         .offset = 0,
 749                 },
 750                 {
 751                         .size = SZ_2M,
 752                         .gtt = I915_GTT_PAGE_SIZE_64K,
 753                         .offset = 0,
 754                 },
 755                 {
 756                         .size = SZ_2M - SZ_4K,
 757                         .gtt = I915_GTT_PAGE_SIZE_4K,
 758                         .offset = 0,
 759                 },
 760                 {
 761                         .size = SZ_2M + SZ_4K,
 762                         .gtt = I915_GTT_PAGE_SIZE_64K | I915_GTT_PAGE_SIZE_4K,
 763                         .offset = 0,
 764                 },
 765                 {
 766                         .size = SZ_2M + SZ_64K,
 767                         .gtt = I915_GTT_PAGE_SIZE_64K,
 768                         .offset = 0,
 769                 },
 770                 {
 771                         .size = SZ_2M - SZ_64K,
 772                         .gtt = I915_GTT_PAGE_SIZE_64K,
 773                         .offset = 0,
 774                 },
 775                 /* Try without any forced padding/alignment */
 776                 {
 777                         .size = SZ_64K,
 778                         .offset = SZ_2M,
 779                         .gtt = I915_GTT_PAGE_SIZE_4K,
 780                 },
 781                 {
 782                         .size = SZ_128K,
 783                         .offset = SZ_2M - SZ_64K,
 784                         .gtt = I915_GTT_PAGE_SIZE_4K,
 785                 },
 786         };
 787         struct i915_vma *vma;
 788         int i, single;
 789         int err;
 790
 791         /*
 792          * Sanity check some of the trickiness with 64K pages -- either we can
 793          * safely mark the whole page-table(2M block) as 64K, or we have to
 794          * always fallback to 4K.
 795          */
 796
 797         if (!HAS_PAGE_SIZES(i915, I915_GTT_PAGE_SIZE_64K))
 798                 return 0;
 799
 800         for (i = 0; i < ARRAY_SIZE(objects); ++i) {
 801                 unsigned int size = objects[i].size;
 802                 unsigned int expected_gtt = objects[i].gtt;
 803                 unsigned int offset = objects[i].offset;
 804                 unsigned int flags = PIN_USER;
 805
 806                 for (single = 0; single <= 1; single++) {
 807                         obj = fake_huge_pages_object(i915, size, !!single);
 808                         if (IS_ERR(obj))
 809                                 return PTR_ERR(obj);
 810
 811                         err = i915_gem_object_pin_pages(obj);
 812                         if (err)
 813                                 goto out_object_put;
 814
 815                         /*
 816                          * Disable 2M pages -- We only want to use 64K/4K pages
 817                          * for this test.
 818                          */
 819                         obj->mm.page_sizes.sg &= ~I915_GTT_PAGE_SIZE_2M;
 820
 821                         vma = i915_vma_instance(obj, &ppgtt->base, NULL);
 822                         if (IS_ERR(vma)) {
 823                                 err = PTR_ERR(vma);
 824                                 goto out_object_unpin;
 825                         }
 826
 827                         if (offset)
 828                                 flags |= PIN_OFFSET_FIXED | offset;
 829
 830                         err = i915_vma_pin(vma, 0, 0, flags);
 831                         if (err)
 832                                 goto out_vma_close;
 833
 834                         err = igt_check_page_sizes(vma);
 835                         if (err)
 836                                 goto out_vma_unpin;
 837
 838                         if (!offset && vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) {
 839                                 if (!IS_ALIGNED(vma->node.start,
 840                                                 I915_GTT_PAGE_SIZE_2M)) {
 841                                         pr_err("node.start(%llx) not aligned to 2M\n",
 842                                                vma->node.start);
 843                                         err = -EINVAL;
 844                                         goto out_vma_unpin;
 845                                 }
 846
 847                                 if (!IS_ALIGNED(vma->node.size,
 848                                                 I915_GTT_PAGE_SIZE_2M)) {
 849                                         pr_err("node.size(%llx) not aligned to 2M\n",
 850                                                vma->node.size);
 851                                         err = -EINVAL;
 852                                         goto out_vma_unpin;
 853                                 }
 854                         }
 855
 856                         if (vma->page_sizes.gtt != expected_gtt) {
 857                                 pr_err("gtt=%u, expected=%u, i=%d, single=%s\n",
 858                                        vma->page_sizes.gtt, expected_gtt, i,
 859                                        yesno(!!single));
 860                                 err = -EINVAL;
 861                                 goto out_vma_unpin;
 862                         }
 863
 864                         i915_vma_unpin(vma);
 865                         i915_vma_close(vma);
 866
 867                         i915_gem_object_unpin_pages(obj);
 868                         i915_gem_object_put(obj);
 869                 }
 870         }
 871
 872         return 0;
 873
 874 out_vma_unpin:
 875         i915_vma_unpin(vma);
 876 out_vma_close:
 877         i915_vma_close(vma);
 878 out_object_unpin:
 879         i915_gem_object_unpin_pages(obj);
 880 out_object_put:
 881         i915_gem_object_put(obj);
 882
 883         return err;
 884 }
 885
 886 static struct i915_vma *
 887 gpu_write_dw(struct i915_vma *vma, u64 offset, u32 val)
 888 {
 889         struct drm_i915_private *i915 = to_i915(vma->obj->base.dev);
 890         const int gen = INTEL_GEN(vma->vm->i915);
 891         unsigned int count = vma->size >> PAGE_SHIFT;
 892         struct drm_i915_gem_object *obj;
 893         struct i915_vma *batch;
 894         unsigned int size;
 895         u32 *cmd;
 896         int n;
 897         int err;
 898
 899         size = (1 + 4 * count) * sizeof(u32);
 900         size = round_up(size, PAGE_SIZE);
 901         obj = i915_gem_object_create_internal(i915, size);
 902         if (IS_ERR(obj))
 903                 return ERR_CAST(obj);
 904
 905         cmd = i915_gem_object_pin_map(obj, I915_MAP_WB);
 906         if (IS_ERR(cmd)) {
 907                 err = PTR_ERR(cmd);
 908                 goto err;
 909         }
 910
 911         offset += vma->node.start;
 912
 913         for (n = 0; n < count; n++) {
 914                 if (gen >= 8) {
 915                         *cmd++ = MI_STORE_DWORD_IMM_GEN4;
 916                         *cmd++ = lower_32_bits(offset);
 917                         *cmd++ = upper_32_bits(offset);
 918                         *cmd++ = val;
 919                 } else if (gen >= 4) {
 920                         *cmd++ = MI_STORE_DWORD_IMM_GEN4 |
 921                                 (gen < 6 ? 1 << 22 : 0);
 922                         *cmd++ = 0;
 923                         *cmd++ = offset;
 924                         *cmd++ = val;
 925                 } else {
 926                         *cmd++ = MI_STORE_DWORD_IMM | 1 << 22;
 927                         *cmd++ = offset;
 928                         *cmd++ = val;
 929                 }
 930
 931                 offset += PAGE_SIZE;
 932         }
 933
 934         *cmd = MI_BATCH_BUFFER_END;
 935
 936         i915_gem_object_unpin_map(obj);
 937
 938         err = i915_gem_object_set_to_gtt_domain(obj, false);
 939         if (err)
 940                 goto err;
 941
 942         batch = i915_vma_instance(obj, vma->vm, NULL);
 943         if (IS_ERR(batch)) {
 944                 err = PTR_ERR(batch);
 945                 goto err;
 946         }
 947
 948         err = i915_vma_pin(batch, 0, 0, PIN_USER);
 949         if (err)
 950                 goto err;
 951
 952         return batch;
 953
 954 err:
 955         i915_gem_object_put(obj);
 956
 957         return ERR_PTR(err);
 958 }
 959
 960 static int gpu_write(struct i915_vma *vma,
 961                      struct i915_gem_context *ctx,
 962                      struct intel_engine_cs *engine,
 963                      u32 dword,
 964                      u32 value)
 965 {
 966         struct drm_i915_gem_request *rq;
 967         struct i915_vma *batch;
 968         int flags = 0;
 969         int err;
 970
 971         GEM_BUG_ON(!intel_engine_can_store_dword(engine));
 972
 973         err = i915_gem_object_set_to_gtt_domain(vma->obj, true);
 974         if (err)
 975                 return err;
 976
 977         rq = i915_gem_request_alloc(engine, ctx);
 978         if (IS_ERR(rq))
 979                 return PTR_ERR(rq);
 980
 981         batch = gpu_write_dw(vma, dword * sizeof(u32), value);
 982         if (IS_ERR(batch)) {
 983                 err = PTR_ERR(batch);
 984                 goto err_request;
 985         }
 986
 987         i915_vma_move_to_active(batch, rq, 0);
 988         i915_gem_object_set_active_reference(batch->obj);
 989         i915_vma_unpin(batch);
 990         i915_vma_close(batch);
 991
 992         err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
 993         if (err)
 994                 goto err_request;
 995
 996         err = i915_switch_context(rq);
 997         if (err)
 998                 goto err_request;
 999
1000         err = rq->engine->emit_bb_start(rq,
1001                                         batch->node.start, batch->node.size,
1002                                         flags);
1003         if (err)
1004                 goto err_request;
1005
1006         i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
1007
1008         reservation_object_lock(vma->resv, NULL);
1009         reservation_object_add_excl_fence(vma->resv, &rq->fence);
1010         reservation_object_unlock(vma->resv);
1011
1012 err_request:
1013         __i915_add_request(rq, err == 0);
1014
1015         return err;
1016 }
1017
1018 static int cpu_check(struct drm_i915_gem_object *obj, u32 dword, u32 val)
1019 {
1020         unsigned int needs_flush;
1021         unsigned long n;
1022         int err;
1023
1024         err = i915_gem_obj_prepare_shmem_read(obj, &needs_flush);
1025         if (err)
1026                 return err;
1027
1028         for (n = 0; n < obj->base.size >> PAGE_SHIFT; ++n) {
1029                 u32 *ptr = kmap_atomic(i915_gem_object_get_page(obj, n));
1030
1031                 if (needs_flush & CLFLUSH_BEFORE)
1032                         drm_clflush_virt_range(ptr, PAGE_SIZE);
1033
1034                 if (ptr[dword] != val) {
1035                         pr_err("n=%lu ptr[%u]=%u, val=%u\n",
1036                                n, dword, ptr[dword], val);
1037                         kunmap_atomic(ptr);
1038                         err = -EINVAL;
1039                         break;
1040                 }
1041
1042                 kunmap_atomic(ptr);
1043         }
1044
1045         i915_gem_obj_finish_shmem_access(obj);
1046
1047         return err;
1048 }
1049
1050 static int igt_write_huge(struct i915_gem_context *ctx,
1051                           struct drm_i915_gem_object *obj)
1052 {
1053         struct drm_i915_private *i915 = to_i915(obj->base.dev);
1054         struct i915_address_space *vm = ctx->ppgtt ? &ctx->ppgtt->base : &i915->ggtt.base;
1055         struct intel_engine_cs *engine;
1056         struct i915_vma *vma;
1057         unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
1058         unsigned int max_page_size;
1059         unsigned int id;
1060         u64 max;
1061         u64 num;
1062         u64 size;
1063         int err = 0;
1064
1065         GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
1066
1067         size = obj->base.size;
1068         if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K)
1069                 size = round_up(size, I915_GTT_PAGE_SIZE_2M);
1070
1071         max_page_size = rounddown_pow_of_two(obj->mm.page_sizes.sg);
1072         max = div_u64((vm->total - size), max_page_size);
1073
1074         vma = i915_vma_instance(obj, vm, NULL);
1075         if (IS_ERR(vma))
1076                 return PTR_ERR(vma);
1077
1078         for_each_engine(engine, i915, id) {
1079                 IGT_TIMEOUT(end_time);
1080
1081                 if (!intel_engine_can_store_dword(engine)) {
1082                         pr_info("store-dword-imm not supported on engine=%u\n",
1083                                 id);
1084                         continue;
1085                 }
1086
1087                 /*
1088                  * Try various offsets until we timeout -- we want to avoid
1089                  * issues hidden by effectively always using offset = 0.
1090                  */
1091                 for_each_prime_number_from(num, 0, max) {
1092                         u64 offset = num * max_page_size;
1093                         u32 dword;
1094
1095                         err = i915_vma_unbind(vma);
1096                         if (err)
1097                                 goto out_vma_close;
1098
1099                         err = i915_vma_pin(vma, size, max_page_size, flags | offset);
1100                         if (err) {
1101                                 /*
1102                                  * The ggtt may have some pages reserved so
1103                                  * refrain from erroring out.
1104                                  */
1105                                 if (err == -ENOSPC && i915_is_ggtt(vm)) {
1106                                         err = 0;
1107                                         continue;
1108                                 }
1109
1110                                 goto out_vma_close;
1111                         }
1112
1113                         err = igt_check_page_sizes(vma);
1114                         if (err)
1115                                 goto out_vma_unpin;
1116
1117                         dword = offset_in_page(num) / 4;
1118
1119                         err = gpu_write(vma, ctx, engine, dword, num + 1);
1120                         if (err) {
1121                                 pr_err("gpu-write failed at offset=%llx", offset);
1122                                 goto out_vma_unpin;
1123                         }
1124
1125                         err = cpu_check(obj, dword, num + 1);
1126                         if (err) {
1127                                 pr_err("cpu-check failed at offset=%llx", offset);
1128                                 goto out_vma_unpin;
1129                         }
1130
1131                         i915_vma_unpin(vma);
1132
1133                         if (num > 0 &&
1134                             igt_timeout(end_time,
1135                                         "%s timed out on engine=%u at offset=%llx, max_page_size=%x\n",
1136                                         __func__, id, offset, max_page_size))
1137                                 break;
1138                 }
1139         }
1140
1141 out_vma_unpin:
1142         if (i915_vma_is_pinned(vma))
1143                 i915_vma_unpin(vma);
1144 out_vma_close:
1145         i915_vma_close(vma);
1146
1147         return err;
1148 }
1149
1150 static int igt_ppgtt_exhaust_huge(void *arg)
1151 {
1152         struct i915_gem_context *ctx = arg;
1153         struct drm_i915_private *i915 = ctx->i915;
1154         unsigned long supported = INTEL_INFO(i915)->page_sizes;
1155         static unsigned int pages[ARRAY_SIZE(page_sizes)];
1156         struct drm_i915_gem_object *obj;
1157         unsigned int size_mask;
1158         unsigned int page_mask;
1159         int n, i;
1160         int err = -ENODEV;
1161
1162         /*
1163          * Sanity check creating objects with a varying mix of page sizes --
1164          * ensuring that our writes lands in the right place.
1165          */
1166
1167         n = 0;
1168         for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1)
1169                 pages[n++] = BIT(i);
1170
1171         for (size_mask = 2; size_mask < BIT(n); size_mask++) {
1172                 unsigned int size = 0;
1173
1174                 for (i = 0; i < n; i++) {
1175                         if (size_mask & BIT(i))
1176                                 size |= pages[i];
1177                 }
1178
1179                 /*
1180                  * For our page mask we want to enumerate all the page-size
1181                  * combinations which will fit into our chosen object size.
1182                  */
1183                 for (page_mask = 2; page_mask <= size_mask; page_mask++) {
1184                         unsigned int page_sizes = 0;
1185
1186                         for (i = 0; i < n; i++) {
1187                                 if (page_mask & BIT(i))
1188                                         page_sizes |= pages[i];
1189                         }
1190
1191                         /*
1192                          * Ensure that we can actually fill the given object
1193                          * with our chosen page mask.
1194                          */
1195                         if (!IS_ALIGNED(size, BIT(__ffs(page_sizes))))
1196                                 continue;
1197
1198                         obj = huge_pages_object(i915, size, page_sizes);
1199                         if (IS_ERR(obj)) {
1200                                 err = PTR_ERR(obj);
1201                                 goto out_device;
1202                         }
1203
1204                         err = i915_gem_object_pin_pages(obj);
1205                         if (err) {
1206                                 i915_gem_object_put(obj);
1207
1208                                 if (err == -ENOMEM) {
1209                                         pr_info("unable to get pages, size=%u, pages=%u\n",
1210                                                 size, page_sizes);
1211                                         err = 0;
1212                                         break;
1213                                 }
1214
1215                                 pr_err("pin_pages failed, size=%u, pages=%u\n",
1216                                        size_mask, page_mask);
1217
1218                                 goto out_device;
1219                         }
1220
1221                         /* Force the page-size for the gtt insertion */
1222                         obj->mm.page_sizes.sg = page_sizes;
1223
1224                         err = igt_write_huge(ctx, obj);
1225                         if (err) {
1226                                 pr_err("exhaust write-huge failed with size=%u\n",
1227                                        size);
1228                                 goto out_unpin;
1229                         }
1230
1231                         i915_gem_object_unpin_pages(obj);
1232                         i915_gem_object_put(obj);
1233                 }
1234         }
1235
1236         goto out_device;
1237
1238 out_unpin:
1239         i915_gem_object_unpin_pages(obj);
1240         i915_gem_object_put(obj);
1241 out_device:
1242         mkwrite_device_info(i915)->page_sizes = supported;
1243
1244         return err;
1245 }
1246
1247 static int igt_ppgtt_internal_huge(void *arg)
1248 {
1249         struct i915_gem_context *ctx = arg;
1250         struct drm_i915_private *i915 = ctx->i915;
1251         struct drm_i915_gem_object *obj;
1252         static const unsigned int sizes[] = {
1253                 SZ_64K,
1254                 SZ_128K,
1255                 SZ_256K,
1256                 SZ_512K,
1257                 SZ_1M,
1258                 SZ_2M,
1259         };
1260         int i;
1261         int err;
1262
1263         /*
1264          * Sanity check that the HW uses huge pages correctly through internal
1265          * -- ensure that our writes land in the right place.
1266          */
1267
1268         for (i = 0; i < ARRAY_SIZE(sizes); ++i) {
1269                 unsigned int size = sizes[i];
1270
1271                 obj = i915_gem_object_create_internal(i915, size);
1272                 if (IS_ERR(obj))
1273                         return PTR_ERR(obj);
1274
1275                 err = i915_gem_object_pin_pages(obj);
1276                 if (err)
1277                         goto out_put;
1278
1279                 if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_64K) {
1280                         pr_info("internal unable to allocate huge-page(s) with size=%u\n",
1281                                 size);
1282                         goto out_unpin;
1283                 }
1284
1285                 err = igt_write_huge(ctx, obj);
1286                 if (err) {
1287                         pr_err("internal write-huge failed with size=%u\n",
1288                                size);
1289                         goto out_unpin;
1290                 }
1291
1292                 i915_gem_object_unpin_pages(obj);
1293                 i915_gem_object_put(obj);
1294         }
1295
1296         return 0;
1297
1298 out_unpin:
1299         i915_gem_object_unpin_pages(obj);
1300 out_put:
1301         i915_gem_object_put(obj);
1302
1303         return err;
1304 }
1305
1306 static inline bool igt_can_allocate_thp(struct drm_i915_private *i915)
1307 {
1308         return i915->mm.gemfs && has_transparent_hugepage();
1309 }
1310
1311 static int igt_ppgtt_gemfs_huge(void *arg)
1312 {
1313         struct i915_gem_context *ctx = arg;
1314         struct drm_i915_private *i915 = ctx->i915;
1315         struct drm_i915_gem_object *obj;
1316         static const unsigned int sizes[] = {
1317                 SZ_2M,
1318                 SZ_4M,
1319                 SZ_8M,
1320                 SZ_16M,
1321                 SZ_32M,
1322         };
1323         int i;
1324         int err;
1325
1326         /*
1327          * Sanity check that the HW uses huge pages correctly through gemfs --
1328          * ensure that our writes land in the right place.
1329          */
1330
1331         if (!igt_can_allocate_thp(i915)) {
1332                 pr_info("missing THP support, skipping\n");
1333                 return 0;
1334         }
1335
1336         for (i = 0; i < ARRAY_SIZE(sizes); ++i) {
1337                 unsigned int size = sizes[i];
1338
1339                 obj = i915_gem_object_create(i915, size);
1340                 if (IS_ERR(obj))
1341                         return PTR_ERR(obj);
1342
1343                 err = i915_gem_object_pin_pages(obj);
1344                 if (err)
1345                         goto out_put;
1346
1347                 if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_2M) {
1348                         pr_info("finishing test early, gemfs unable to allocate huge-page(s) with size=%u\n",
1349                                 size);
1350                         goto out_unpin;
1351                 }
1352
1353                 err = igt_write_huge(ctx, obj);
1354                 if (err) {
1355                         pr_err("gemfs write-huge failed with size=%u\n",
1356                                size);
1357                         goto out_unpin;
1358                 }
1359
1360                 i915_gem_object_unpin_pages(obj);
1361                 i915_gem_object_put(obj);
1362         }
1363
1364         return 0;
1365
1366 out_unpin:
1367         i915_gem_object_unpin_pages(obj);
1368 out_put:
1369         i915_gem_object_put(obj);
1370
1371         return err;
1372 }
1373
1374 static int igt_ppgtt_pin_update(void *arg)
1375 {
1376         struct i915_gem_context *ctx = arg;
1377         struct drm_i915_private *dev_priv = ctx->i915;
1378         unsigned long supported = INTEL_INFO(dev_priv)->page_sizes;
1379         struct i915_hw_ppgtt *ppgtt = ctx->ppgtt;
1380         struct drm_i915_gem_object *obj;
1381         struct i915_vma *vma;
1382         unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
1383         int first, last;
1384         int err;
1385
1386         /*
1387          * Make sure there's no funny business when doing a PIN_UPDATE -- in the
1388          * past we had a subtle issue with being able to incorrectly do multiple
1389          * alloc va ranges on the same object when doing a PIN_UPDATE, which
1390          * resulted in some pretty nasty bugs, though only when using
1391          * huge-gtt-pages.
1392          */
1393
1394         if (!USES_FULL_48BIT_PPGTT(dev_priv)) {
1395                 pr_info("48b PPGTT not supported, skipping\n");
1396                 return 0;
1397         }
1398
1399         first = ilog2(I915_GTT_PAGE_SIZE_64K);
1400         last = ilog2(I915_GTT_PAGE_SIZE_2M);
1401
1402         for_each_set_bit_from(first, &supported, last + 1) {
1403                 unsigned int page_size = BIT(first);
1404
1405                 obj = i915_gem_object_create_internal(dev_priv, page_size);
1406                 if (IS_ERR(obj))
1407                         return PTR_ERR(obj);
1408
1409                 vma = i915_vma_instance(obj, &ppgtt->base, NULL);
1410                 if (IS_ERR(vma)) {
1411                         err = PTR_ERR(vma);
1412                         goto out_put;
1413                 }
1414
1415                 err = i915_vma_pin(vma, SZ_2M, 0, flags);
1416                 if (err)
1417                         goto out_close;
1418
1419                 if (vma->page_sizes.sg < page_size) {
1420                         pr_info("Unable to allocate page-size %x, finishing test early\n",
1421                                 page_size);
1422                         goto out_unpin;
1423                 }
1424
1425                 err = igt_check_page_sizes(vma);
1426                 if (err)
1427                         goto out_unpin;
1428
1429                 if (vma->page_sizes.gtt != page_size) {
1430                         dma_addr_t addr = i915_gem_object_get_dma_address(obj, 0);
1431
1432                         /*
1433                          * The only valid reason for this to ever fail would be
1434                          * if the dma-mapper screwed us over when we did the
1435                          * dma_map_sg(), since it has the final say over the dma
1436                          * address.
1437                          */
1438                         if (IS_ALIGNED(addr, page_size)) {
1439                                 pr_err("page_sizes.gtt=%u, expected=%u\n",
1440                                        vma->page_sizes.gtt, page_size);
1441                                 err = -EINVAL;
1442                         } else {
1443                                 pr_info("dma address misaligned, finishing test early\n");
1444                         }
1445
1446                         goto out_unpin;
1447                 }
1448
1449                 err = i915_vma_bind(vma, I915_CACHE_NONE, PIN_UPDATE);
1450                 if (err)
1451                         goto out_unpin;
1452
1453                 i915_vma_unpin(vma);
1454                 i915_vma_close(vma);
1455
1456                 i915_gem_object_put(obj);
1457         }
1458
1459         obj = i915_gem_object_create_internal(dev_priv, PAGE_SIZE);
1460         if (IS_ERR(obj))
1461                 return PTR_ERR(obj);
1462
1463         vma = i915_vma_instance(obj, &ppgtt->base, NULL);
1464         if (IS_ERR(vma)) {
1465                 err = PTR_ERR(vma);
1466                 goto out_put;
1467         }
1468
1469         err = i915_vma_pin(vma, 0, 0, flags);
1470         if (err)
1471                 goto out_close;
1472
1473         /*
1474          * Make sure we don't end up with something like where the pde is still
1475          * pointing to the 2M page, and the pt we just filled-in is dangling --
1476          * we can check this by writing to the first page where it would then
1477          * land in the now stale 2M page.
1478          */
1479
1480         err = gpu_write(vma, ctx, dev_priv->engine[RCS], 0, 0xdeadbeaf);
1481         if (err)
1482                 goto out_unpin;
1483
1484         err = cpu_check(obj, 0, 0xdeadbeaf);
1485
1486 out_unpin:
1487         i915_vma_unpin(vma);
1488 out_close:
1489         i915_vma_close(vma);
1490 out_put:
1491         i915_gem_object_put(obj);
1492
1493         return err;
1494 }
1495
1496 static int igt_tmpfs_fallback(void *arg)
1497 {
1498         struct i915_gem_context *ctx = arg;
1499         struct drm_i915_private *i915 = ctx->i915;
1500         struct vfsmount *gemfs = i915->mm.gemfs;
1501         struct i915_address_space *vm = ctx->ppgtt ? &ctx->ppgtt->base : &i915->ggtt.base;
1502         struct drm_i915_gem_object *obj;
1503         struct i915_vma *vma;
1504         u32 *vaddr;
1505         int err = 0;
1506
1507         /*
1508          * Make sure that we don't burst into a ball of flames upon falling back
1509          * to tmpfs, which we rely on if on the off-chance we encouter a failure
1510          * when setting up gemfs.
1511          */
1512
1513         i915->mm.gemfs = NULL;
1514
1515         obj = i915_gem_object_create(i915, PAGE_SIZE);
1516         if (IS_ERR(obj)) {
1517                 err = PTR_ERR(obj);
1518                 goto out_restore;
1519         }
1520
1521         vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
1522         if (IS_ERR(vaddr)) {
1523                 err = PTR_ERR(vaddr);
1524                 goto out_put;
1525         }
1526         *vaddr = 0xdeadbeaf;
1527
1528         i915_gem_object_unpin_map(obj);
1529
1530         vma = i915_vma_instance(obj, vm, NULL);
1531         if (IS_ERR(vma)) {
1532                 err = PTR_ERR(vma);
1533                 goto out_put;
1534         }
1535
1536         err = i915_vma_pin(vma, 0, 0, PIN_USER);
1537         if (err)
1538                 goto out_close;
1539
1540         err = igt_check_page_sizes(vma);
1541
1542         i915_vma_unpin(vma);
1543 out_close:
1544         i915_vma_close(vma);
1545 out_put:
1546         i915_gem_object_put(obj);
1547 out_restore:
1548         i915->mm.gemfs = gemfs;
1549
1550         return err;
1551 }
1552
1553 static int igt_shrink_thp(void *arg)
1554 {
1555         struct i915_gem_context *ctx = arg;
1556         struct drm_i915_private *i915 = ctx->i915;
1557         struct i915_address_space *vm = ctx->ppgtt ? &ctx->ppgtt->base : &i915->ggtt.base;
1558         struct drm_i915_gem_object *obj;
1559         struct i915_vma *vma;
1560         unsigned int flags = PIN_USER;
1561         int err;
1562
1563         /*
1564          * Sanity check shrinking huge-paged object -- make sure nothing blows
1565          * up.
1566          */
1567
1568         if (!igt_can_allocate_thp(i915)) {
1569                 pr_info("missing THP support, skipping\n");
1570                 return 0;
1571         }
1572
1573         obj = i915_gem_object_create(i915, SZ_2M);
1574         if (IS_ERR(obj))
1575                 return PTR_ERR(obj);
1576
1577         vma = i915_vma_instance(obj, vm, NULL);
1578         if (IS_ERR(vma)) {
1579                 err = PTR_ERR(vma);
1580                 goto out_put;
1581         }
1582
1583         err = i915_vma_pin(vma, 0, 0, flags);
1584         if (err)
1585                 goto out_close;
1586
1587         if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_2M) {
1588                 pr_info("failed to allocate THP, finishing test early\n");
1589                 goto out_unpin;
1590         }
1591
1592         err = igt_check_page_sizes(vma);
1593         if (err)
1594                 goto out_unpin;
1595
1596         err = gpu_write(vma, ctx, i915->engine[RCS], 0, 0xdeadbeaf);
1597         if (err)
1598                 goto out_unpin;
1599
1600         i915_vma_unpin(vma);
1601
1602         /*
1603          * Now that the pages are *unpinned* shrink-all should invoke
1604          * shmem to truncate our pages.
1605          */
1606         i915_gem_shrink_all(i915);
1607         if (!IS_ERR_OR_NULL(obj->mm.pages)) {
1608                 pr_err("shrink-all didn't truncate the pages\n");
1609                 err = -EINVAL;
1610                 goto out_close;
1611         }
1612
1613         if (obj->mm.page_sizes.sg || obj->mm.page_sizes.phys) {
1614                 pr_err("residual page-size bits left\n");
1615                 err = -EINVAL;
1616                 goto out_close;
1617         }
1618
1619         err = i915_vma_pin(vma, 0, 0, flags);
1620         if (err)
1621                 goto out_close;
1622
1623         err = cpu_check(obj, 0, 0xdeadbeaf);
1624
1625 out_unpin:
1626         i915_vma_unpin(vma);
1627 out_close:
1628         i915_vma_close(vma);
1629 out_put:
1630         i915_gem_object_put(obj);
1631
1632         return err;
1633 }
1634
1635 int i915_gem_huge_page_mock_selftests(void)
1636 {
1637         static const struct i915_subtest tests[] = {
1638                 SUBTEST(igt_mock_exhaust_device_supported_pages),
1639                 SUBTEST(igt_mock_ppgtt_misaligned_dma),
1640                 SUBTEST(igt_mock_ppgtt_huge_fill),
1641                 SUBTEST(igt_mock_ppgtt_64K),
1642         };
1643         int saved_ppgtt = i915_modparams.enable_ppgtt;
1644         struct drm_i915_private *dev_priv;
1645         struct pci_dev *pdev;
1646         struct i915_hw_ppgtt *ppgtt;
1647         int err;
1648
1649         dev_priv = mock_gem_device();
1650         if (!dev_priv)
1651                 return -ENOMEM;
1652
1653         /* Pretend to be a device which supports the 48b PPGTT */
1654         i915_modparams.enable_ppgtt = 3;
1655
1656         pdev = dev_priv->drm.pdev;
1657         dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(39));
1658
1659         mutex_lock(&dev_priv->drm.struct_mutex);
1660         ppgtt = i915_ppgtt_create(dev_priv, ERR_PTR(-ENODEV), "mock");
1661         if (IS_ERR(ppgtt)) {
1662                 err = PTR_ERR(ppgtt);
1663                 goto out_unlock;
1664         }
1665
1666         if (!i915_vm_is_48bit(&ppgtt->base)) {
1667                 pr_err("failed to create 48b PPGTT\n");
1668                 err = -EINVAL;
1669                 goto out_close;
1670         }
1671
1672         /* If we were ever hit this then it's time to mock the 64K scratch */
1673         if (!i915_vm_has_scratch_64K(&ppgtt->base)) {
1674                 pr_err("PPGTT missing 64K scratch page\n");
1675                 err = -EINVAL;
1676                 goto out_close;
1677         }
1678
1679         err = i915_subtests(tests, ppgtt);
1680
1681 out_close:
1682         i915_ppgtt_close(&ppgtt->base);
1683         i915_ppgtt_put(ppgtt);
1684
1685 out_unlock:
1686         mutex_unlock(&dev_priv->drm.struct_mutex);
1687
1688         i915_modparams.enable_ppgtt = saved_ppgtt;
1689
1690         drm_dev_unref(&dev_priv->drm);
1691
1692         return err;
1693 }
1694
1695 int i915_gem_huge_page_live_selftests(struct drm_i915_private *dev_priv)
1696 {
1697         static const struct i915_subtest tests[] = {
1698                 SUBTEST(igt_shrink_thp),
1699                 SUBTEST(igt_ppgtt_pin_update),
1700                 SUBTEST(igt_tmpfs_fallback),
1701                 SUBTEST(igt_ppgtt_exhaust_huge),
1702                 SUBTEST(igt_ppgtt_gemfs_huge),
1703                 SUBTEST(igt_ppgtt_internal_huge),
1704         };
1705         struct drm_file *file;
1706         struct i915_gem_context *ctx;
1707         int err;
1708
1709         if (!USES_PPGTT(dev_priv)) {
1710                 pr_info("PPGTT not supported, skipping live-selftests\n");
1711                 return 0;
1712         }
1713
1714         file = mock_file(dev_priv);
1715         if (IS_ERR(file))
1716                 return PTR_ERR(file);
1717
1718         mutex_lock(&dev_priv->drm.struct_mutex);
1719
1720         ctx = live_context(dev_priv, file);
1721         if (IS_ERR(ctx)) {
1722                 err = PTR_ERR(ctx);
1723                 goto out_unlock;
1724         }
1725
1726         err = i915_subtests(tests, ctx);
1727
1728 out_unlock:
1729         mutex_unlock(&dev_priv->drm.struct_mutex);
1730
1731         mock_file_free(dev_priv, file);
1732
1733         return err;
1734 }