drivers/gpu/drm/i915/i915_gem_gtt.c

   1 /*
   2  * Copyright © 2010 Daniel Vetter
   3  * Copyright © 2011-2014 Intel Corporation
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <linux/seq_file.h>
  27 #include <linux/stop_machine.h>
  28 #include <drm/drmP.h>
  29 #include <drm/i915_drm.h>
  30 #include "i915_drv.h"
  31 #include "i915_vgpu.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include "intel_frontbuffer.h"
  35
  36 #define I915_GFP_DMA (GFP_KERNEL | __GFP_HIGHMEM)
  37
  38 /**
  39  * DOC: Global GTT views
  40  *
  41  * Background and previous state
  42  *
  43  * Historically objects could exists (be bound) in global GTT space only as
  44  * singular instances with a view representing all of the object's backing pages
  45  * in a linear fashion. This view will be called a normal view.
  46  *
  47  * To support multiple views of the same object, where the number of mapped
  48  * pages is not equal to the backing store, or where the layout of the pages
  49  * is not linear, concept of a GGTT view was added.
  50  *
  51  * One example of an alternative view is a stereo display driven by a single
  52  * image. In this case we would have a framebuffer looking like this
  53  * (2x2 pages):
  54  *
  55  *    12
  56  *    34
  57  *
  58  * Above would represent a normal GGTT view as normally mapped for GPU or CPU
  59  * rendering. In contrast, fed to the display engine would be an alternative
  60  * view which could look something like this:
  61  *
  62  *   1212
  63  *   3434
  64  *
  65  * In this example both the size and layout of pages in the alternative view is
  66  * different from the normal view.
  67  *
  68  * Implementation and usage
  69  *
  70  * GGTT views are implemented using VMAs and are distinguished via enum
  71  * i915_ggtt_view_type and struct i915_ggtt_view.
  72  *
  73  * A new flavour of core GEM functions which work with GGTT bound objects were
  74  * added with the _ggtt_ infix, and sometimes with _view postfix to avoid
  75  * renaming  in large amounts of code. They take the struct i915_ggtt_view
  76  * parameter encapsulating all metadata required to implement a view.
  77  *
  78  * As a helper for callers which are only interested in the normal view,
  79  * globally const i915_ggtt_view_normal singleton instance exists. All old core
  80  * GEM API functions, the ones not taking the view parameter, are operating on,
  81  * or with the normal GGTT view.
  82  *
  83  * Code wanting to add or use a new GGTT view needs to:
  84  *
  85  * 1. Add a new enum with a suitable name.
  86  * 2. Extend the metadata in the i915_ggtt_view structure if required.
  87  * 3. Add support to i915_get_vma_pages().
  88  *
  89  * New views are required to build a scatter-gather table from within the
  90  * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and
  91  * exists for the lifetime of an VMA.
  92  *
  93  * Core API is designed to have copy semantics which means that passed in
  94  * struct i915_ggtt_view does not need to be persistent (left around after
  95  * calling the core API functions).
  96  *
  97  */
  98
  99 static int
 100 i915_get_ggtt_vma_pages(struct i915_vma *vma);
 101
 102 const struct i915_ggtt_view i915_ggtt_view_normal = {
 103         .type = I915_GGTT_VIEW_NORMAL,
 104 };
 105 const struct i915_ggtt_view i915_ggtt_view_rotated = {
 106         .type = I915_GGTT_VIEW_ROTATED,
 107 };
 108
 109 int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv,
 110                                 int enable_ppgtt)
 111 {
 112         bool has_aliasing_ppgtt;
 113         bool has_full_ppgtt;
 114         bool has_full_48bit_ppgtt;
 115
 116         has_aliasing_ppgtt = dev_priv->info.has_aliasing_ppgtt;
 117         has_full_ppgtt = dev_priv->info.has_full_ppgtt;
 118         has_full_48bit_ppgtt = dev_priv->info.has_full_48bit_ppgtt;
 119
 120         if (intel_vgpu_active(dev_priv)) {
 121                 /* emulation is too hard */
 122                 has_full_ppgtt = false;
 123                 has_full_48bit_ppgtt = false;
 124         }
 125
 126         if (!has_aliasing_ppgtt)
 127                 return 0;
 128
 129         /*
 130          * We don't allow disabling PPGTT for gen9+ as it's a requirement for
 131          * execlists, the sole mechanism available to submit work.
 132          */
 133         if (enable_ppgtt == 0 && INTEL_GEN(dev_priv) < 9)
 134                 return 0;
 135
 136         if (enable_ppgtt == 1)
 137                 return 1;
 138
 139         if (enable_ppgtt == 2 && has_full_ppgtt)
 140                 return 2;
 141
 142         if (enable_ppgtt == 3 && has_full_48bit_ppgtt)
 143                 return 3;
 144
 145 #ifdef CONFIG_INTEL_IOMMU
 146         /* Disable ppgtt on SNB if VT-d is on. */
 147         if (IS_GEN6(dev_priv) && intel_iommu_gfx_mapped) {
 148                 DRM_INFO("Disabling PPGTT because VT-d is on\n");
 149                 return 0;
 150         }
 151 #endif
 152
 153         /* Early VLV doesn't have this */
 154         if (IS_VALLEYVIEW(dev_priv) && dev_priv->drm.pdev->revision < 0xb) {
 155                 DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n");
 156                 return 0;
 157         }
 158
 159         if (INTEL_GEN(dev_priv) >= 8 && i915.enable_execlists && has_full_ppgtt)
 160                 return has_full_48bit_ppgtt ? 3 : 2;
 161         else
 162                 return has_aliasing_ppgtt ? 1 : 0;
 163 }
 164
 165 static int ppgtt_bind_vma(struct i915_vma *vma,
 166                           enum i915_cache_level cache_level,
 167                           u32 unused)
 168 {
 169         u32 pte_flags = 0;
 170
 171         vma->pages = vma->obj->mm.pages;
 172
 173         /* Currently applicable only to VLV */
 174         if (vma->obj->gt_ro)
 175                 pte_flags |= PTE_READ_ONLY;
 176
 177         vma->vm->insert_entries(vma->vm, vma->pages, vma->node.start,
 178                                 cache_level, pte_flags);
 179
 180         return 0;
 181 }
 182
 183 static void ppgtt_unbind_vma(struct i915_vma *vma)
 184 {
 185         vma->vm->clear_range(vma->vm,
 186                              vma->node.start,
 187                              vma->size);
 188 }
 189
 190 static gen8_pte_t gen8_pte_encode(dma_addr_t addr,
 191                                   enum i915_cache_level level)
 192 {
 193         gen8_pte_t pte = _PAGE_PRESENT | _PAGE_RW;
 194         pte |= addr;
 195
 196         switch (level) {
 197         case I915_CACHE_NONE:
 198                 pte |= PPAT_UNCACHED_INDEX;
 199                 break;
 200         case I915_CACHE_WT:
 201                 pte |= PPAT_DISPLAY_ELLC_INDEX;
 202                 break;
 203         default:
 204                 pte |= PPAT_CACHED_INDEX;
 205                 break;
 206         }
 207
 208         return pte;
 209 }
 210
 211 static gen8_pde_t gen8_pde_encode(const dma_addr_t addr,
 212                                   const enum i915_cache_level level)
 213 {
 214         gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW;
 215         pde |= addr;
 216         if (level != I915_CACHE_NONE)
 217                 pde |= PPAT_CACHED_PDE_INDEX;
 218         else
 219                 pde |= PPAT_UNCACHED_INDEX;
 220         return pde;
 221 }
 222
 223 #define gen8_pdpe_encode gen8_pde_encode
 224 #define gen8_pml4e_encode gen8_pde_encode
 225
 226 static gen6_pte_t snb_pte_encode(dma_addr_t addr,
 227                                  enum i915_cache_level level,
 228                                  u32 unused)
 229 {
 230         gen6_pte_t pte = GEN6_PTE_VALID;
 231         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 232
 233         switch (level) {
 234         case I915_CACHE_L3_LLC:
 235         case I915_CACHE_LLC:
 236                 pte |= GEN6_PTE_CACHE_LLC;
 237                 break;
 238         case I915_CACHE_NONE:
 239                 pte |= GEN6_PTE_UNCACHED;
 240                 break;
 241         default:
 242                 MISSING_CASE(level);
 243         }
 244
 245         return pte;
 246 }
 247
 248 static gen6_pte_t ivb_pte_encode(dma_addr_t addr,
 249                                  enum i915_cache_level level,
 250                                  u32 unused)
 251 {
 252         gen6_pte_t pte = GEN6_PTE_VALID;
 253         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 254
 255         switch (level) {
 256         case I915_CACHE_L3_LLC:
 257                 pte |= GEN7_PTE_CACHE_L3_LLC;
 258                 break;
 259         case I915_CACHE_LLC:
 260                 pte |= GEN6_PTE_CACHE_LLC;
 261                 break;
 262         case I915_CACHE_NONE:
 263                 pte |= GEN6_PTE_UNCACHED;
 264                 break;
 265         default:
 266                 MISSING_CASE(level);
 267         }
 268
 269         return pte;
 270 }
 271
 272 static gen6_pte_t byt_pte_encode(dma_addr_t addr,
 273                                  enum i915_cache_level level,
 274                                  u32 flags)
 275 {
 276         gen6_pte_t pte = GEN6_PTE_VALID;
 277         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 278
 279         if (!(flags & PTE_READ_ONLY))
 280                 pte |= BYT_PTE_WRITEABLE;
 281
 282         if (level != I915_CACHE_NONE)
 283                 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
 284
 285         return pte;
 286 }
 287
 288 static gen6_pte_t hsw_pte_encode(dma_addr_t addr,
 289                                  enum i915_cache_level level,
 290                                  u32 unused)
 291 {
 292         gen6_pte_t pte = GEN6_PTE_VALID;
 293         pte |= HSW_PTE_ADDR_ENCODE(addr);
 294
 295         if (level != I915_CACHE_NONE)
 296                 pte |= HSW_WB_LLC_AGE3;
 297
 298         return pte;
 299 }
 300
 301 static gen6_pte_t iris_pte_encode(dma_addr_t addr,
 302                                   enum i915_cache_level level,
 303                                   u32 unused)
 304 {
 305         gen6_pte_t pte = GEN6_PTE_VALID;
 306         pte |= HSW_PTE_ADDR_ENCODE(addr);
 307
 308         switch (level) {
 309         case I915_CACHE_NONE:
 310                 break;
 311         case I915_CACHE_WT:
 312                 pte |= HSW_WT_ELLC_LLC_AGE3;
 313                 break;
 314         default:
 315                 pte |= HSW_WB_ELLC_LLC_AGE3;
 316                 break;
 317         }
 318
 319         return pte;
 320 }
 321
 322 static int __setup_page_dma(struct drm_i915_private *dev_priv,
 323                             struct i915_page_dma *p, gfp_t flags)
 324 {
 325         struct device *kdev = &dev_priv->drm.pdev->dev;
 326
 327         p->page = alloc_page(flags);
 328         if (!p->page)
 329                 return -ENOMEM;
 330
 331         p->daddr = dma_map_page(kdev,
 332                                 p->page, 0, 4096, PCI_DMA_BIDIRECTIONAL);
 333
 334         if (dma_mapping_error(kdev, p->daddr)) {
 335                 __free_page(p->page);
 336                 return -EINVAL;
 337         }
 338
 339         return 0;
 340 }
 341
 342 static int setup_page_dma(struct drm_i915_private *dev_priv,
 343                           struct i915_page_dma *p)
 344 {
 345         return __setup_page_dma(dev_priv, p, I915_GFP_DMA);
 346 }
 347
 348 static void cleanup_page_dma(struct drm_i915_private *dev_priv,
 349                              struct i915_page_dma *p)
 350 {
 351         struct pci_dev *pdev = dev_priv->drm.pdev;
 352
 353         if (WARN_ON(!p->page))
 354                 return;
 355
 356         dma_unmap_page(&pdev->dev, p->daddr, 4096, PCI_DMA_BIDIRECTIONAL);
 357         __free_page(p->page);
 358         memset(p, 0, sizeof(*p));
 359 }
 360
 361 static void *kmap_page_dma(struct i915_page_dma *p)
 362 {
 363         return kmap_atomic(p->page);
 364 }
 365
 366 /* We use the flushing unmap only with ppgtt structures:
 367  * page directories, page tables and scratch pages.
 368  */
 369 static void kunmap_page_dma(struct drm_i915_private *dev_priv, void *vaddr)
 370 {
 371         /* There are only few exceptions for gen >=6. chv and bxt.
 372          * And we are not sure about the latter so play safe for now.
 373          */
 374         if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv))
 375                 drm_clflush_virt_range(vaddr, PAGE_SIZE);
 376
 377         kunmap_atomic(vaddr);
 378 }
 379
 380 #define kmap_px(px) kmap_page_dma(px_base(px))
 381 #define kunmap_px(ppgtt, vaddr) \
 382                 kunmap_page_dma((ppgtt)->base.i915, (vaddr))
 383
 384 #define setup_px(dev_priv, px) setup_page_dma((dev_priv), px_base(px))
 385 #define cleanup_px(dev_priv, px) cleanup_page_dma((dev_priv), px_base(px))
 386 #define fill_px(dev_priv, px, v) fill_page_dma((dev_priv), px_base(px), (v))
 387 #define fill32_px(dev_priv, px, v) \
 388                 fill_page_dma_32((dev_priv), px_base(px), (v))
 389
 390 static void fill_page_dma(struct drm_i915_private *dev_priv,
 391                           struct i915_page_dma *p, const uint64_t val)
 392 {
 393         int i;
 394         uint64_t * const vaddr = kmap_page_dma(p);
 395
 396         for (i = 0; i < 512; i++)
 397                 vaddr[i] = val;
 398
 399         kunmap_page_dma(dev_priv, vaddr);
 400 }
 401
 402 static void fill_page_dma_32(struct drm_i915_private *dev_priv,
 403                              struct i915_page_dma *p, const uint32_t val32)
 404 {
 405         uint64_t v = val32;
 406
 407         v = v << 32 | val32;
 408
 409         fill_page_dma(dev_priv, p, v);
 410 }
 411
 412 static int
 413 setup_scratch_page(struct drm_i915_private *dev_priv,
 414                    struct i915_page_dma *scratch,
 415                    gfp_t gfp)
 416 {
 417         return __setup_page_dma(dev_priv, scratch, gfp | __GFP_ZERO);
 418 }
 419
 420 static void cleanup_scratch_page(struct drm_i915_private *dev_priv,
 421                                  struct i915_page_dma *scratch)
 422 {
 423         cleanup_page_dma(dev_priv, scratch);
 424 }
 425
 426 static struct i915_page_table *alloc_pt(struct drm_i915_private *dev_priv)
 427 {
 428         struct i915_page_table *pt;
 429         const size_t count = INTEL_GEN(dev_priv) >= 8 ? GEN8_PTES : GEN6_PTES;
 430         int ret = -ENOMEM;
 431
 432         pt = kzalloc(sizeof(*pt), GFP_KERNEL);
 433         if (!pt)
 434                 return ERR_PTR(-ENOMEM);
 435
 436         pt->used_ptes = kcalloc(BITS_TO_LONGS(count), sizeof(*pt->used_ptes),
 437                                 GFP_KERNEL);
 438
 439         if (!pt->used_ptes)
 440                 goto fail_bitmap;
 441
 442         ret = setup_px(dev_priv, pt);
 443         if (ret)
 444                 goto fail_page_m;
 445
 446         return pt;
 447
 448 fail_page_m:
 449         kfree(pt->used_ptes);
 450 fail_bitmap:
 451         kfree(pt);
 452
 453         return ERR_PTR(ret);
 454 }
 455
 456 static void free_pt(struct drm_i915_private *dev_priv,
 457                     struct i915_page_table *pt)
 458 {
 459         cleanup_px(dev_priv, pt);
 460         kfree(pt->used_ptes);
 461         kfree(pt);
 462 }
 463
 464 static void gen8_initialize_pt(struct i915_address_space *vm,
 465                                struct i915_page_table *pt)
 466 {
 467         gen8_pte_t scratch_pte;
 468
 469         scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
 470                                       I915_CACHE_LLC);
 471
 472         fill_px(vm->i915, pt, scratch_pte);
 473 }
 474
 475 static void gen6_initialize_pt(struct i915_address_space *vm,
 476                                struct i915_page_table *pt)
 477 {
 478         gen6_pte_t scratch_pte;
 479
 480         WARN_ON(vm->scratch_page.daddr == 0);
 481
 482         scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
 483                                      I915_CACHE_LLC, 0);
 484
 485         fill32_px(vm->i915, pt, scratch_pte);
 486 }
 487
 488 static struct i915_page_directory *alloc_pd(struct drm_i915_private *dev_priv)
 489 {
 490         struct i915_page_directory *pd;
 491         int ret = -ENOMEM;
 492
 493         pd = kzalloc(sizeof(*pd), GFP_KERNEL);
 494         if (!pd)
 495                 return ERR_PTR(-ENOMEM);
 496
 497         pd->used_pdes = kcalloc(BITS_TO_LONGS(I915_PDES),
 498                                 sizeof(*pd->used_pdes), GFP_KERNEL);
 499         if (!pd->used_pdes)
 500                 goto fail_bitmap;
 501
 502         ret = setup_px(dev_priv, pd);
 503         if (ret)
 504                 goto fail_page_m;
 505
 506         return pd;
 507
 508 fail_page_m:
 509         kfree(pd->used_pdes);
 510 fail_bitmap:
 511         kfree(pd);
 512
 513         return ERR_PTR(ret);
 514 }
 515
 516 static void free_pd(struct drm_i915_private *dev_priv,
 517                     struct i915_page_directory *pd)
 518 {
 519         if (px_page(pd)) {
 520                 cleanup_px(dev_priv, pd);
 521                 kfree(pd->used_pdes);
 522                 kfree(pd);
 523         }
 524 }
 525
 526 static void gen8_initialize_pd(struct i915_address_space *vm,
 527                                struct i915_page_directory *pd)
 528 {
 529         gen8_pde_t scratch_pde;
 530
 531         scratch_pde = gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC);
 532
 533         fill_px(vm->i915, pd, scratch_pde);
 534 }
 535
 536 static int __pdp_init(struct drm_i915_private *dev_priv,
 537                       struct i915_page_directory_pointer *pdp)
 538 {
 539         size_t pdpes = I915_PDPES_PER_PDP(dev_priv);
 540
 541         pdp->used_pdpes = kcalloc(BITS_TO_LONGS(pdpes),
 542                                   sizeof(unsigned long),
 543                                   GFP_KERNEL);
 544         if (!pdp->used_pdpes)
 545                 return -ENOMEM;
 546
 547         pdp->page_directory = kcalloc(pdpes, sizeof(*pdp->page_directory),
 548                                       GFP_KERNEL);
 549         if (!pdp->page_directory) {
 550                 kfree(pdp->used_pdpes);
 551                 /* the PDP might be the statically allocated top level. Keep it
 552                  * as clean as possible */
 553                 pdp->used_pdpes = NULL;
 554                 return -ENOMEM;
 555         }
 556
 557         return 0;
 558 }
 559
 560 static void __pdp_fini(struct i915_page_directory_pointer *pdp)
 561 {
 562         kfree(pdp->used_pdpes);
 563         kfree(pdp->page_directory);
 564         pdp->page_directory = NULL;
 565 }
 566
 567 static struct
 568 i915_page_directory_pointer *alloc_pdp(struct drm_i915_private *dev_priv)
 569 {
 570         struct i915_page_directory_pointer *pdp;
 571         int ret = -ENOMEM;
 572
 573         WARN_ON(!USES_FULL_48BIT_PPGTT(dev_priv));
 574
 575         pdp = kzalloc(sizeof(*pdp), GFP_KERNEL);
 576         if (!pdp)
 577                 return ERR_PTR(-ENOMEM);
 578
 579         ret = __pdp_init(dev_priv, pdp);
 580         if (ret)
 581                 goto fail_bitmap;
 582
 583         ret = setup_px(dev_priv, pdp);
 584         if (ret)
 585                 goto fail_page_m;
 586
 587         return pdp;
 588
 589 fail_page_m:
 590         __pdp_fini(pdp);
 591 fail_bitmap:
 592         kfree(pdp);
 593
 594         return ERR_PTR(ret);
 595 }
 596
 597 static void free_pdp(struct drm_i915_private *dev_priv,
 598                      struct i915_page_directory_pointer *pdp)
 599 {
 600         __pdp_fini(pdp);
 601         if (USES_FULL_48BIT_PPGTT(dev_priv)) {
 602                 cleanup_px(dev_priv, pdp);
 603                 kfree(pdp);
 604         }
 605 }
 606
 607 static void gen8_initialize_pdp(struct i915_address_space *vm,
 608                                 struct i915_page_directory_pointer *pdp)
 609 {
 610         gen8_ppgtt_pdpe_t scratch_pdpe;
 611
 612         scratch_pdpe = gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC);
 613
 614         fill_px(vm->i915, pdp, scratch_pdpe);
 615 }
 616
 617 static void gen8_initialize_pml4(struct i915_address_space *vm,
 618                                  struct i915_pml4 *pml4)
 619 {
 620         gen8_ppgtt_pml4e_t scratch_pml4e;
 621
 622         scratch_pml4e = gen8_pml4e_encode(px_dma(vm->scratch_pdp),
 623                                           I915_CACHE_LLC);
 624
 625         fill_px(vm->i915, pml4, scratch_pml4e);
 626 }
 627
 628 static void
 629 gen8_setup_page_directory(struct i915_hw_ppgtt *ppgtt,
 630                           struct i915_page_directory_pointer *pdp,
 631                           struct i915_page_directory *pd,
 632                           int index)
 633 {
 634         gen8_ppgtt_pdpe_t *page_directorypo;
 635
 636         if (!USES_FULL_48BIT_PPGTT(to_i915(ppgtt->base.dev)))
 637                 return;
 638
 639         page_directorypo = kmap_px(pdp);
 640         page_directorypo[index] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC);
 641         kunmap_px(ppgtt, page_directorypo);
 642 }
 643
 644 static void
 645 gen8_setup_page_directory_pointer(struct i915_hw_ppgtt *ppgtt,
 646                                   struct i915_pml4 *pml4,
 647                                   struct i915_page_directory_pointer *pdp,
 648                                   int index)
 649 {
 650         gen8_ppgtt_pml4e_t *pagemap = kmap_px(pml4);
 651
 652         WARN_ON(!USES_FULL_48BIT_PPGTT(to_i915(ppgtt->base.dev)));
 653         pagemap[index] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC);
 654         kunmap_px(ppgtt, pagemap);
 655 }
 656
 657 /* Broadwell Page Directory Pointer Descriptors */
 658 static int gen8_write_pdp(struct drm_i915_gem_request *req,
 659                           unsigned entry,
 660                           dma_addr_t addr)
 661 {
 662         struct intel_ring *ring = req->ring;
 663         struct intel_engine_cs *engine = req->engine;
 664         int ret;
 665
 666         BUG_ON(entry >= 4);
 667
 668         ret = intel_ring_begin(req, 6);
 669         if (ret)
 670                 return ret;
 671
 672         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
 673         intel_ring_emit_reg(ring, GEN8_RING_PDP_UDW(engine, entry));
 674         intel_ring_emit(ring, upper_32_bits(addr));
 675         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
 676         intel_ring_emit_reg(ring, GEN8_RING_PDP_LDW(engine, entry));
 677         intel_ring_emit(ring, lower_32_bits(addr));
 678         intel_ring_advance(ring);
 679
 680         return 0;
 681 }
 682
 683 static int gen8_legacy_mm_switch(struct i915_hw_ppgtt *ppgtt,
 684                                  struct drm_i915_gem_request *req)
 685 {
 686         int i, ret;
 687
 688         for (i = GEN8_LEGACY_PDPES - 1; i >= 0; i--) {
 689                 const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
 690
 691                 ret = gen8_write_pdp(req, i, pd_daddr);
 692                 if (ret)
 693                         return ret;
 694         }
 695
 696         return 0;
 697 }
 698
 699 static int gen8_48b_mm_switch(struct i915_hw_ppgtt *ppgtt,
 700                               struct drm_i915_gem_request *req)
 701 {
 702         return gen8_write_pdp(req, 0, px_dma(&ppgtt->pml4));
 703 }
 704
 705 /* PDE TLBs are a pain to invalidate on GEN8+. When we modify
 706  * the page table structures, we mark them dirty so that
 707  * context switching/execlist queuing code takes extra steps
 708  * to ensure that tlbs are flushed.
 709  */
 710 static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt)
 711 {
 712         ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.i915)->ring_mask;
 713 }
 714
 715 /* Removes entries from a single page table, releasing it if it's empty.
 716  * Caller can use the return value to update higher-level entries.
 717  */
 718 static bool gen8_ppgtt_clear_pt(struct i915_address_space *vm,
 719                                 struct i915_page_table *pt,
 720                                 uint64_t start,
 721                                 uint64_t length)
 722 {
 723         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 724         unsigned int num_entries = gen8_pte_count(start, length);
 725         unsigned int pte = gen8_pte_index(start);
 726         unsigned int pte_end = pte + num_entries;
 727         gen8_pte_t *pt_vaddr;
 728         gen8_pte_t scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
 729                                                  I915_CACHE_LLC);
 730
 731         if (WARN_ON(!px_page(pt)))
 732                 return false;
 733
 734         GEM_BUG_ON(pte_end > GEN8_PTES);
 735
 736         bitmap_clear(pt->used_ptes, pte, num_entries);
 737
 738         if (bitmap_empty(pt->used_ptes, GEN8_PTES))
 739                 return true;
 740
 741         pt_vaddr = kmap_px(pt);
 742
 743         while (pte < pte_end)
 744                 pt_vaddr[pte++] = scratch_pte;
 745
 746         kunmap_px(ppgtt, pt_vaddr);
 747
 748         return false;
 749 }
 750
 751 /* Removes entries from a single page dir, releasing it if it's empty.
 752  * Caller can use the return value to update higher-level entries
 753  */
 754 static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm,
 755                                 struct i915_page_directory *pd,
 756                                 uint64_t start,
 757                                 uint64_t length)
 758 {
 759         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 760         struct i915_page_table *pt;
 761         uint64_t pde;
 762         gen8_pde_t *pde_vaddr;
 763         gen8_pde_t scratch_pde = gen8_pde_encode(px_dma(vm->scratch_pt),
 764                                                  I915_CACHE_LLC);
 765
 766         gen8_for_each_pde(pt, pd, start, length, pde) {
 767                 if (WARN_ON(!pd->page_table[pde]))
 768                         break;
 769
 770                 if (gen8_ppgtt_clear_pt(vm, pt, start, length)) {
 771                         __clear_bit(pde, pd->used_pdes);
 772                         pde_vaddr = kmap_px(pd);
 773                         pde_vaddr[pde] = scratch_pde;
 774                         kunmap_px(ppgtt, pde_vaddr);
 775                         free_pt(vm->i915, pt);
 776                 }
 777         }
 778
 779         if (bitmap_empty(pd->used_pdes, I915_PDES))
 780                 return true;
 781
 782         return false;
 783 }
 784
 785 /* Removes entries from a single page dir pointer, releasing it if it's empty.
 786  * Caller can use the return value to update higher-level entries
 787  */
 788 static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
 789                                  struct i915_page_directory_pointer *pdp,
 790                                  uint64_t start,
 791                                  uint64_t length)
 792 {
 793         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 794         struct i915_page_directory *pd;
 795         uint64_t pdpe;
 796         gen8_ppgtt_pdpe_t *pdpe_vaddr;
 797         gen8_ppgtt_pdpe_t scratch_pdpe =
 798                 gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC);
 799
 800         gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
 801                 if (WARN_ON(!pdp->page_directory[pdpe]))
 802                         break;
 803
 804                 if (gen8_ppgtt_clear_pd(vm, pd, start, length)) {
 805                         __clear_bit(pdpe, pdp->used_pdpes);
 806                         if (USES_FULL_48BIT_PPGTT(dev_priv)) {
 807                                 pdpe_vaddr = kmap_px(pdp);
 808                                 pdpe_vaddr[pdpe] = scratch_pdpe;
 809                                 kunmap_px(ppgtt, pdpe_vaddr);
 810                         }
 811                         free_pd(vm->i915, pd);
 812                 }
 813         }
 814
 815         mark_tlbs_dirty(ppgtt);
 816
 817         if (bitmap_empty(pdp->used_pdpes, I915_PDPES_PER_PDP(dev_priv)))
 818                 return true;
 819
 820         return false;
 821 }
 822
 823 /* Removes entries from a single pml4.
 824  * This is the top-level structure in 4-level page tables used on gen8+.
 825  * Empty entries are always scratch pml4e.
 826  */
 827 static void gen8_ppgtt_clear_pml4(struct i915_address_space *vm,
 828                                   struct i915_pml4 *pml4,
 829                                   uint64_t start,
 830                                   uint64_t length)
 831 {
 832         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 833         struct i915_page_directory_pointer *pdp;
 834         uint64_t pml4e;
 835         gen8_ppgtt_pml4e_t *pml4e_vaddr;
 836         gen8_ppgtt_pml4e_t scratch_pml4e =
 837                 gen8_pml4e_encode(px_dma(vm->scratch_pdp), I915_CACHE_LLC);
 838
 839         GEM_BUG_ON(!USES_FULL_48BIT_PPGTT(vm->i915));
 840
 841         gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
 842                 if (WARN_ON(!pml4->pdps[pml4e]))
 843                         break;
 844
 845                 if (gen8_ppgtt_clear_pdp(vm, pdp, start, length)) {
 846                         __clear_bit(pml4e, pml4->used_pml4es);
 847                         pml4e_vaddr = kmap_px(pml4);
 848                         pml4e_vaddr[pml4e] = scratch_pml4e;
 849                         kunmap_px(ppgtt, pml4e_vaddr);
 850                         free_pdp(vm->i915, pdp);
 851                 }
 852         }
 853 }
 854
 855 static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
 856                                    uint64_t start, uint64_t length)
 857 {
 858         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 859
 860         if (USES_FULL_48BIT_PPGTT(vm->i915))
 861                 gen8_ppgtt_clear_pml4(vm, &ppgtt->pml4, start, length);
 862         else
 863                 gen8_ppgtt_clear_pdp(vm, &ppgtt->pdp, start, length);
 864 }
 865
 866 static void
 867 gen8_ppgtt_insert_pte_entries(struct i915_address_space *vm,
 868                               struct i915_page_directory_pointer *pdp,
 869                               struct sg_page_iter *sg_iter,
 870                               uint64_t start,
 871                               enum i915_cache_level cache_level)
 872 {
 873         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 874         gen8_pte_t *pt_vaddr;
 875         unsigned pdpe = gen8_pdpe_index(start);
 876         unsigned pde = gen8_pde_index(start);
 877         unsigned pte = gen8_pte_index(start);
 878
 879         pt_vaddr = NULL;
 880
 881         while (__sg_page_iter_next(sg_iter)) {
 882                 if (pt_vaddr == NULL) {
 883                         struct i915_page_directory *pd = pdp->page_directory[pdpe];
 884                         struct i915_page_table *pt = pd->page_table[pde];
 885                         pt_vaddr = kmap_px(pt);
 886                 }
 887
 888                 pt_vaddr[pte] =
 889                         gen8_pte_encode(sg_page_iter_dma_address(sg_iter),
 890                                         cache_level);
 891                 if (++pte == GEN8_PTES) {
 892                         kunmap_px(ppgtt, pt_vaddr);
 893                         pt_vaddr = NULL;
 894                         if (++pde == I915_PDES) {
 895                                 if (++pdpe == I915_PDPES_PER_PDP(vm->i915))
 896                                         break;
 897                                 pde = 0;
 898                         }
 899                         pte = 0;
 900                 }
 901         }
 902
 903         if (pt_vaddr)
 904                 kunmap_px(ppgtt, pt_vaddr);
 905 }
 906
 907 static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
 908                                       struct sg_table *pages,
 909                                       uint64_t start,
 910                                       enum i915_cache_level cache_level,
 911                                       u32 unused)
 912 {
 913         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 914         struct sg_page_iter sg_iter;
 915
 916         __sg_page_iter_start(&sg_iter, pages->sgl, sg_nents(pages->sgl), 0);
 917
 918         if (!USES_FULL_48BIT_PPGTT(vm->i915)) {
 919                 gen8_ppgtt_insert_pte_entries(vm, &ppgtt->pdp, &sg_iter, start,
 920                                               cache_level);
 921         } else {
 922                 struct i915_page_directory_pointer *pdp;
 923                 uint64_t pml4e;
 924                 uint64_t length = (uint64_t)pages->orig_nents << PAGE_SHIFT;
 925
 926                 gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, pml4e) {
 927                         gen8_ppgtt_insert_pte_entries(vm, pdp, &sg_iter,
 928                                                       start, cache_level);
 929                 }
 930         }
 931 }
 932
 933 static void gen8_free_page_tables(struct drm_i915_private *dev_priv,
 934                                   struct i915_page_directory *pd)
 935 {
 936         int i;
 937
 938         if (!px_page(pd))
 939                 return;
 940
 941         for_each_set_bit(i, pd->used_pdes, I915_PDES) {
 942                 if (WARN_ON(!pd->page_table[i]))
 943                         continue;
 944
 945                 free_pt(dev_priv, pd->page_table[i]);
 946                 pd->page_table[i] = NULL;
 947         }
 948 }
 949
 950 static int gen8_init_scratch(struct i915_address_space *vm)
 951 {
 952         struct drm_i915_private *dev_priv = vm->i915;
 953         int ret;
 954
 955         ret = setup_scratch_page(dev_priv, &vm->scratch_page, I915_GFP_DMA);
 956         if (ret)
 957                 return ret;
 958
 959         vm->scratch_pt = alloc_pt(dev_priv);
 960         if (IS_ERR(vm->scratch_pt)) {
 961                 ret = PTR_ERR(vm->scratch_pt);
 962                 goto free_scratch_page;
 963         }
 964
 965         vm->scratch_pd = alloc_pd(dev_priv);
 966         if (IS_ERR(vm->scratch_pd)) {
 967                 ret = PTR_ERR(vm->scratch_pd);
 968                 goto free_pt;
 969         }
 970
 971         if (USES_FULL_48BIT_PPGTT(dev_priv)) {
 972                 vm->scratch_pdp = alloc_pdp(dev_priv);
 973                 if (IS_ERR(vm->scratch_pdp)) {
 974                         ret = PTR_ERR(vm->scratch_pdp);
 975                         goto free_pd;
 976                 }
 977         }
 978
 979         gen8_initialize_pt(vm, vm->scratch_pt);
 980         gen8_initialize_pd(vm, vm->scratch_pd);
 981         if (USES_FULL_48BIT_PPGTT(dev_priv))
 982                 gen8_initialize_pdp(vm, vm->scratch_pdp);
 983
 984         return 0;
 985
 986 free_pd:
 987         free_pd(dev_priv, vm->scratch_pd);
 988 free_pt:
 989         free_pt(dev_priv, vm->scratch_pt);
 990 free_scratch_page:
 991         cleanup_scratch_page(dev_priv, &vm->scratch_page);
 992
 993         return ret;
 994 }
 995
 996 static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create)
 997 {
 998         enum vgt_g2v_type msg;
 999         struct drm_i915_private *dev_priv = ppgtt->base.i915;
1000         int i;
1001
1002         if (USES_FULL_48BIT_PPGTT(dev_priv)) {
1003                 u64 daddr = px_dma(&ppgtt->pml4);
1004
1005                 I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr));
1006                 I915_WRITE(vgtif_reg(pdp[0].hi), upper_32_bits(daddr));
1007
1008                 msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE :
1009                                 VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY);
1010         } else {
1011                 for (i = 0; i < GEN8_LEGACY_PDPES; i++) {
1012                         u64 daddr = i915_page_dir_dma_addr(ppgtt, i);
1013
1014                         I915_WRITE(vgtif_reg(pdp[i].lo), lower_32_bits(daddr));
1015                         I915_WRITE(vgtif_reg(pdp[i].hi), upper_32_bits(daddr));
1016                 }
1017
1018                 msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE :
1019                                 VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY);
1020         }
1021
1022         I915_WRITE(vgtif_reg(g2v_notify), msg);
1023
1024         return 0;
1025 }
1026
1027 static void gen8_free_scratch(struct i915_address_space *vm)
1028 {
1029         struct drm_i915_private *dev_priv = vm->i915;
1030
1031         if (USES_FULL_48BIT_PPGTT(dev_priv))
1032                 free_pdp(dev_priv, vm->scratch_pdp);
1033         free_pd(dev_priv, vm->scratch_pd);
1034         free_pt(dev_priv, vm->scratch_pt);
1035         cleanup_scratch_page(dev_priv, &vm->scratch_page);
1036 }
1037
1038 static void gen8_ppgtt_cleanup_3lvl(struct drm_i915_private *dev_priv,
1039                                     struct i915_page_directory_pointer *pdp)
1040 {
1041         int i;
1042
1043         for_each_set_bit(i, pdp->used_pdpes, I915_PDPES_PER_PDP(dev_priv)) {
1044                 if (WARN_ON(!pdp->page_directory[i]))
1045                         continue;
1046
1047                 gen8_free_page_tables(dev_priv, pdp->page_directory[i]);
1048                 free_pd(dev_priv, pdp->page_directory[i]);
1049         }
1050
1051         free_pdp(dev_priv, pdp);
1052 }
1053
1054 static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt *ppgtt)
1055 {
1056         struct drm_i915_private *dev_priv = ppgtt->base.i915;
1057         int i;
1058
1059         for_each_set_bit(i, ppgtt->pml4.used_pml4es, GEN8_PML4ES_PER_PML4) {
1060                 if (WARN_ON(!ppgtt->pml4.pdps[i]))
1061                         continue;
1062
1063                 gen8_ppgtt_cleanup_3lvl(dev_priv, ppgtt->pml4.pdps[i]);
1064         }
1065
1066         cleanup_px(dev_priv, &ppgtt->pml4);
1067 }
1068
1069 static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
1070 {
1071         struct drm_i915_private *dev_priv = vm->i915;
1072         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1073
1074         if (intel_vgpu_active(dev_priv))
1075                 gen8_ppgtt_notify_vgt(ppgtt, false);
1076
1077         if (!USES_FULL_48BIT_PPGTT(dev_priv))
1078                 gen8_ppgtt_cleanup_3lvl(dev_priv, &ppgtt->pdp);
1079         else
1080                 gen8_ppgtt_cleanup_4lvl(ppgtt);
1081
1082         gen8_free_scratch(vm);
1083 }
1084
1085 /**
1086  * gen8_ppgtt_alloc_pagetabs() - Allocate page tables for VA range.
1087  * @vm: Master vm structure.
1088  * @pd: Page directory for this address range.
1089  * @start:      Starting virtual address to begin allocations.
1090  * @length:     Size of the allocations.
1091  * @new_pts:    Bitmap set by function with new allocations. Likely used by the
1092  *              caller to free on error.
1093  *
1094  * Allocate the required number of page tables. Extremely similar to
1095  * gen8_ppgtt_alloc_page_directories(). The main difference is here we are limited by
1096  * the page directory boundary (instead of the page directory pointer). That
1097  * boundary is 1GB virtual. Therefore, unlike gen8_ppgtt_alloc_page_directories(), it is
1098  * possible, and likely that the caller will need to use multiple calls of this
1099  * function to achieve the appropriate allocation.
1100  *
1101  * Return: 0 if success; negative error code otherwise.
1102  */
1103 static int gen8_ppgtt_alloc_pagetabs(struct i915_address_space *vm,
1104                                      struct i915_page_directory *pd,
1105                                      uint64_t start,
1106                                      uint64_t length,
1107                                      unsigned long *new_pts)
1108 {
1109         struct drm_i915_private *dev_priv = vm->i915;
1110         struct i915_page_table *pt;
1111         uint32_t pde;
1112
1113         gen8_for_each_pde(pt, pd, start, length, pde) {
1114                 /* Don't reallocate page tables */
1115                 if (test_bit(pde, pd->used_pdes)) {
1116                         /* Scratch is never allocated this way */
1117                         WARN_ON(pt == vm->scratch_pt);
1118                         continue;
1119                 }
1120
1121                 pt = alloc_pt(dev_priv);
1122                 if (IS_ERR(pt))
1123                         goto unwind_out;
1124
1125                 gen8_initialize_pt(vm, pt);
1126                 pd->page_table[pde] = pt;
1127                 __set_bit(pde, new_pts);
1128                 trace_i915_page_table_entry_alloc(vm, pde, start, GEN8_PDE_SHIFT);
1129         }
1130
1131         return 0;
1132
1133 unwind_out:
1134         for_each_set_bit(pde, new_pts, I915_PDES)
1135                 free_pt(dev_priv, pd->page_table[pde]);
1136
1137         return -ENOMEM;
1138 }
1139
1140 /**
1141  * gen8_ppgtt_alloc_page_directories() - Allocate page directories for VA range.
1142  * @vm: Master vm structure.
1143  * @pdp:        Page directory pointer for this address range.
1144  * @start:      Starting virtual address to begin allocations.
1145  * @length:     Size of the allocations.
1146  * @new_pds:    Bitmap set by function with new allocations. Likely used by the
1147  *              caller to free on error.
1148  *
1149  * Allocate the required number of page directories starting at the pde index of
1150  * @start, and ending at the pde index @start + @length. This function will skip
1151  * over already allocated page directories within the range, and only allocate
1152  * new ones, setting the appropriate pointer within the pdp as well as the
1153  * correct position in the bitmap @new_pds.
1154  *
1155  * The function will only allocate the pages within the range for a give page
1156  * directory pointer. In other words, if @start + @length straddles a virtually
1157  * addressed PDP boundary (512GB for 4k pages), there will be more allocations
1158  * required by the caller, This is not currently possible, and the BUG in the
1159  * code will prevent it.
1160  *
1161  * Return: 0 if success; negative error code otherwise.
1162  */
1163 static int
1164 gen8_ppgtt_alloc_page_directories(struct i915_address_space *vm,
1165                                   struct i915_page_directory_pointer *pdp,
1166                                   uint64_t start,
1167                                   uint64_t length,
1168                                   unsigned long *new_pds)
1169 {
1170         struct drm_i915_private *dev_priv = vm->i915;
1171         struct i915_page_directory *pd;
1172         uint32_t pdpe;
1173         uint32_t pdpes = I915_PDPES_PER_PDP(dev_priv);
1174
1175         WARN_ON(!bitmap_empty(new_pds, pdpes));
1176
1177         gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1178                 if (test_bit(pdpe, pdp->used_pdpes))
1179                         continue;
1180
1181                 pd = alloc_pd(dev_priv);
1182                 if (IS_ERR(pd))
1183                         goto unwind_out;
1184
1185                 gen8_initialize_pd(vm, pd);
1186                 pdp->page_directory[pdpe] = pd;
1187                 __set_bit(pdpe, new_pds);
1188                 trace_i915_page_directory_entry_alloc(vm, pdpe, start, GEN8_PDPE_SHIFT);
1189         }
1190
1191         return 0;
1192
1193 unwind_out:
1194         for_each_set_bit(pdpe, new_pds, pdpes)
1195                 free_pd(dev_priv, pdp->page_directory[pdpe]);
1196
1197         return -ENOMEM;
1198 }
1199
1200 /**
1201  * gen8_ppgtt_alloc_page_dirpointers() - Allocate pdps for VA range.
1202  * @vm: Master vm structure.
1203  * @pml4:       Page map level 4 for this address range.
1204  * @start:      Starting virtual address to begin allocations.
1205  * @length:     Size of the allocations.
1206  * @new_pdps:   Bitmap set by function with new allocations. Likely used by the
1207  *              caller to free on error.
1208  *
1209  * Allocate the required number of page directory pointers. Extremely similar to
1210  * gen8_ppgtt_alloc_page_directories() and gen8_ppgtt_alloc_pagetabs().
1211  * The main difference is here we are limited by the pml4 boundary (instead of
1212  * the page directory pointer).
1213  *
1214  * Return: 0 if success; negative error code otherwise.
1215  */
1216 static int
1217 gen8_ppgtt_alloc_page_dirpointers(struct i915_address_space *vm,
1218                                   struct i915_pml4 *pml4,
1219                                   uint64_t start,
1220                                   uint64_t length,
1221                                   unsigned long *new_pdps)
1222 {
1223         struct drm_i915_private *dev_priv = vm->i915;
1224         struct i915_page_directory_pointer *pdp;
1225         uint32_t pml4e;
1226
1227         WARN_ON(!bitmap_empty(new_pdps, GEN8_PML4ES_PER_PML4));
1228
1229         gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
1230                 if (!test_bit(pml4e, pml4->used_pml4es)) {
1231                         pdp = alloc_pdp(dev_priv);
1232                         if (IS_ERR(pdp))
1233                                 goto unwind_out;
1234
1235                         gen8_initialize_pdp(vm, pdp);
1236                         pml4->pdps[pml4e] = pdp;
1237                         __set_bit(pml4e, new_pdps);
1238                         trace_i915_page_directory_pointer_entry_alloc(vm,
1239                                                                       pml4e,
1240                                                                       start,
1241                                                                       GEN8_PML4E_SHIFT);
1242                 }
1243         }
1244
1245         return 0;
1246
1247 unwind_out:
1248         for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4)
1249                 free_pdp(dev_priv, pml4->pdps[pml4e]);
1250
1251         return -ENOMEM;
1252 }
1253
1254 static void
1255 free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long *new_pts)
1256 {
1257         kfree(new_pts);
1258         kfree(new_pds);
1259 }
1260
1261 /* Fills in the page directory bitmap, and the array of page tables bitmap. Both
1262  * of these are based on the number of PDPEs in the system.
1263  */
1264 static
1265 int __must_check alloc_gen8_temp_bitmaps(unsigned long **new_pds,
1266                                          unsigned long **new_pts,
1267                                          uint32_t pdpes)
1268 {
1269         unsigned long *pds;
1270         unsigned long *pts;
1271
1272         pds = kcalloc(BITS_TO_LONGS(pdpes), sizeof(unsigned long), GFP_TEMPORARY);
1273         if (!pds)
1274                 return -ENOMEM;
1275
1276         pts = kcalloc(pdpes, BITS_TO_LONGS(I915_PDES) * sizeof(unsigned long),
1277                       GFP_TEMPORARY);
1278         if (!pts)
1279                 goto err_out;
1280
1281         *new_pds = pds;
1282         *new_pts = pts;
1283
1284         return 0;
1285
1286 err_out:
1287         free_gen8_temp_bitmaps(pds, pts);
1288         return -ENOMEM;
1289 }
1290
1291 static int gen8_alloc_va_range_3lvl(struct i915_address_space *vm,
1292                                     struct i915_page_directory_pointer *pdp,
1293                                     uint64_t start,
1294                                     uint64_t length)
1295 {
1296         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1297         unsigned long *new_page_dirs, *new_page_tables;
1298         struct drm_i915_private *dev_priv = vm->i915;
1299         struct i915_page_directory *pd;
1300         const uint64_t orig_start = start;
1301         const uint64_t orig_length = length;
1302         uint32_t pdpe;
1303         uint32_t pdpes = I915_PDPES_PER_PDP(dev_priv);
1304         int ret;
1305
1306         ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes);
1307         if (ret)
1308                 return ret;
1309
1310         /* Do the allocations first so we can easily bail out */
1311         ret = gen8_ppgtt_alloc_page_directories(vm, pdp, start, length,
1312                                                 new_page_dirs);
1313         if (ret) {
1314                 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
1315                 return ret;
1316         }
1317
1318         /* For every page directory referenced, allocate page tables */
1319         gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1320                 ret = gen8_ppgtt_alloc_pagetabs(vm, pd, start, length,
1321                                                 new_page_tables + pdpe * BITS_TO_LONGS(I915_PDES));
1322                 if (ret)
1323                         goto err_out;
1324         }
1325
1326         start = orig_start;
1327         length = orig_length;
1328
1329         /* Allocations have completed successfully, so set the bitmaps, and do
1330          * the mappings. */
1331         gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1332                 gen8_pde_t *const page_directory = kmap_px(pd);
1333                 struct i915_page_table *pt;
1334                 uint64_t pd_len = length;
1335                 uint64_t pd_start = start;
1336                 uint32_t pde;
1337
1338                 /* Every pd should be allocated, we just did that above. */
1339                 WARN_ON(!pd);
1340
1341                 gen8_for_each_pde(pt, pd, pd_start, pd_len, pde) {
1342                         /* Same reasoning as pd */
1343                         WARN_ON(!pt);
1344                         WARN_ON(!pd_len);
1345                         WARN_ON(!gen8_pte_count(pd_start, pd_len));
1346
1347                         /* Set our used ptes within the page table */
1348                         bitmap_set(pt->used_ptes,
1349                                    gen8_pte_index(pd_start),
1350                                    gen8_pte_count(pd_start, pd_len));
1351
1352                         /* Our pde is now pointing to the pagetable, pt */
1353                         __set_bit(pde, pd->used_pdes);
1354
1355                         /* Map the PDE to the page table */
1356                         page_directory[pde] = gen8_pde_encode(px_dma(pt),
1357                                                               I915_CACHE_LLC);
1358                         trace_i915_page_table_entry_map(&ppgtt->base, pde, pt,
1359                                                         gen8_pte_index(start),
1360                                                         gen8_pte_count(start, length),
1361                                                         GEN8_PTES);
1362
1363                         /* NB: We haven't yet mapped ptes to pages. At this
1364                          * point we're still relying on insert_entries() */
1365                 }
1366
1367                 kunmap_px(ppgtt, page_directory);
1368                 __set_bit(pdpe, pdp->used_pdpes);
1369                 gen8_setup_page_directory(ppgtt, pdp, pd, pdpe);
1370         }
1371
1372         free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
1373         mark_tlbs_dirty(ppgtt);
1374         return 0;
1375
1376 err_out:
1377         while (pdpe--) {
1378                 unsigned long temp;
1379
1380                 for_each_set_bit(temp, new_page_tables + pdpe *
1381                                 BITS_TO_LONGS(I915_PDES), I915_PDES)
1382                         free_pt(dev_priv,
1383                                 pdp->page_directory[pdpe]->page_table[temp]);
1384         }
1385
1386         for_each_set_bit(pdpe, new_page_dirs, pdpes)
1387                 free_pd(dev_priv, pdp->page_directory[pdpe]);
1388
1389         free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
1390         mark_tlbs_dirty(ppgtt);
1391         return ret;
1392 }
1393
1394 static int gen8_alloc_va_range_4lvl(struct i915_address_space *vm,
1395                                     struct i915_pml4 *pml4,
1396                                     uint64_t start,
1397                                     uint64_t length)
1398 {
1399         DECLARE_BITMAP(new_pdps, GEN8_PML4ES_PER_PML4);
1400         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1401         struct i915_page_directory_pointer *pdp;
1402         uint64_t pml4e;
1403         int ret = 0;
1404
1405         /* Do the pml4 allocations first, so we don't need to track the newly
1406          * allocated tables below the pdp */
1407         bitmap_zero(new_pdps, GEN8_PML4ES_PER_PML4);
1408
1409         /* The pagedirectory and pagetable allocations are done in the shared 3
1410          * and 4 level code. Just allocate the pdps.
1411          */
1412         ret = gen8_ppgtt_alloc_page_dirpointers(vm, pml4, start, length,
1413                                                 new_pdps);
1414         if (ret)
1415                 return ret;
1416
1417         WARN(bitmap_weight(new_pdps, GEN8_PML4ES_PER_PML4) > 2,
1418              "The allocation has spanned more than 512GB. "
1419              "It is highly likely this is incorrect.");
1420
1421         gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
1422                 WARN_ON(!pdp);
1423
1424                 ret = gen8_alloc_va_range_3lvl(vm, pdp, start, length);
1425                 if (ret)
1426                         goto err_out;
1427
1428                 gen8_setup_page_directory_pointer(ppgtt, pml4, pdp, pml4e);
1429         }
1430
1431         bitmap_or(pml4->used_pml4es, new_pdps, pml4->used_pml4es,
1432                   GEN8_PML4ES_PER_PML4);
1433
1434         return 0;
1435
1436 err_out:
1437         for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4)
1438                 gen8_ppgtt_cleanup_3lvl(vm->i915, pml4->pdps[pml4e]);
1439
1440         return ret;
1441 }
1442
1443 static int gen8_alloc_va_range(struct i915_address_space *vm,
1444                                uint64_t start, uint64_t length)
1445 {
1446         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1447
1448         if (USES_FULL_48BIT_PPGTT(vm->i915))
1449                 return gen8_alloc_va_range_4lvl(vm, &ppgtt->pml4, start, length);
1450         else
1451                 return gen8_alloc_va_range_3lvl(vm, &ppgtt->pdp, start, length);
1452 }
1453
1454 static void gen8_dump_pdp(struct i915_page_directory_pointer *pdp,
1455                           uint64_t start, uint64_t length,
1456                           gen8_pte_t scratch_pte,
1457                           struct seq_file *m)
1458 {
1459         struct i915_page_directory *pd;
1460         uint32_t pdpe;
1461
1462         gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1463                 struct i915_page_table *pt;
1464                 uint64_t pd_len = length;
1465                 uint64_t pd_start = start;
1466                 uint32_t pde;
1467
1468                 if (!test_bit(pdpe, pdp->used_pdpes))
1469                         continue;
1470
1471                 seq_printf(m, "\tPDPE #%d\n", pdpe);
1472                 gen8_for_each_pde(pt, pd, pd_start, pd_len, pde) {
1473                         uint32_t  pte;
1474                         gen8_pte_t *pt_vaddr;
1475
1476                         if (!test_bit(pde, pd->used_pdes))
1477                                 continue;
1478
1479                         pt_vaddr = kmap_px(pt);
1480                         for (pte = 0; pte < GEN8_PTES; pte += 4) {
1481                                 uint64_t va =
1482                                         (pdpe << GEN8_PDPE_SHIFT) |
1483                                         (pde << GEN8_PDE_SHIFT) |
1484                                         (pte << GEN8_PTE_SHIFT);
1485                                 int i;
1486                                 bool found = false;
1487
1488                                 for (i = 0; i < 4; i++)
1489                                         if (pt_vaddr[pte + i] != scratch_pte)
1490                                                 found = true;
1491                                 if (!found)
1492                                         continue;
1493
1494                                 seq_printf(m, "\t\t0x%llx [%03d,%03d,%04d]: =", va, pdpe, pde, pte);
1495                                 for (i = 0; i < 4; i++) {
1496                                         if (pt_vaddr[pte + i] != scratch_pte)
1497                                                 seq_printf(m, " %llx", pt_vaddr[pte + i]);
1498                                         else
1499                                                 seq_puts(m, "  SCRATCH ");
1500                                 }
1501                                 seq_puts(m, "\n");
1502                         }
1503                         /* don't use kunmap_px, it could trigger
1504                          * an unnecessary flush.
1505                          */
1506                         kunmap_atomic(pt_vaddr);
1507                 }
1508         }
1509 }
1510
1511 static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
1512 {
1513         struct i915_address_space *vm = &ppgtt->base;
1514         uint64_t start = ppgtt->base.start;
1515         uint64_t length = ppgtt->base.total;
1516         gen8_pte_t scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
1517                                                  I915_CACHE_LLC);
1518
1519         if (!USES_FULL_48BIT_PPGTT(vm->i915)) {
1520                 gen8_dump_pdp(&ppgtt->pdp, start, length, scratch_pte, m);
1521         } else {
1522                 uint64_t pml4e;
1523                 struct i915_pml4 *pml4 = &ppgtt->pml4;
1524                 struct i915_page_directory_pointer *pdp;
1525
1526                 gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
1527                         if (!test_bit(pml4e, pml4->used_pml4es))
1528                                 continue;
1529
1530                         seq_printf(m, "    PML4E #%llu\n", pml4e);
1531                         gen8_dump_pdp(pdp, start, length, scratch_pte, m);
1532                 }
1533         }
1534 }
1535
1536 static int gen8_preallocate_top_level_pdps(struct i915_hw_ppgtt *ppgtt)
1537 {
1538         unsigned long *new_page_dirs, *new_page_tables;
1539         uint32_t pdpes = I915_PDPES_PER_PDP(to_i915(ppgtt->base.dev));
1540         int ret;
1541
1542         /* We allocate temp bitmap for page tables for no gain
1543          * but as this is for init only, lets keep the things simple
1544          */
1545         ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes);
1546         if (ret)
1547                 return ret;
1548
1549         /* Allocate for all pdps regardless of how the ppgtt
1550          * was defined.
1551          */
1552         ret = gen8_ppgtt_alloc_page_directories(&ppgtt->base, &ppgtt->pdp,
1553                                                 0, 1ULL << 32,
1554                                                 new_page_dirs);
1555         if (!ret)
1556                 *ppgtt->pdp.used_pdpes = *new_page_dirs;
1557
1558         free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
1559
1560         return ret;
1561 }
1562
1563 /*
1564  * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
1565  * with a net effect resembling a 2-level page table in normal x86 terms. Each
1566  * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address
1567  * space.
1568  *
1569  */
1570 static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
1571 {
1572         struct drm_i915_private *dev_priv = ppgtt->base.i915;
1573         int ret;
1574
1575         ret = gen8_init_scratch(&ppgtt->base);
1576         if (ret)
1577                 return ret;
1578
1579         ppgtt->base.start = 0;
1580         ppgtt->base.cleanup = gen8_ppgtt_cleanup;
1581         ppgtt->base.allocate_va_range = gen8_alloc_va_range;
1582         ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
1583         ppgtt->base.clear_range = gen8_ppgtt_clear_range;
1584         ppgtt->base.unbind_vma = ppgtt_unbind_vma;
1585         ppgtt->base.bind_vma = ppgtt_bind_vma;
1586         ppgtt->debug_dump = gen8_dump_ppgtt;
1587
1588         if (USES_FULL_48BIT_PPGTT(dev_priv)) {
1589                 ret = setup_px(dev_priv, &ppgtt->pml4);
1590                 if (ret)
1591                         goto free_scratch;
1592
1593                 gen8_initialize_pml4(&ppgtt->base, &ppgtt->pml4);
1594
1595                 ppgtt->base.total = 1ULL << 48;
1596                 ppgtt->switch_mm = gen8_48b_mm_switch;
1597         } else {
1598                 ret = __pdp_init(dev_priv, &ppgtt->pdp);
1599                 if (ret)
1600                         goto free_scratch;
1601
1602                 ppgtt->base.total = 1ULL << 32;
1603                 ppgtt->switch_mm = gen8_legacy_mm_switch;
1604                 trace_i915_page_directory_pointer_entry_alloc(&ppgtt->base,
1605                                                               0, 0,
1606                                                               GEN8_PML4E_SHIFT);
1607
1608                 if (intel_vgpu_active(dev_priv)) {
1609                         ret = gen8_preallocate_top_level_pdps(ppgtt);
1610                         if (ret)
1611                                 goto free_scratch;
1612                 }
1613         }
1614
1615         if (intel_vgpu_active(dev_priv))
1616                 gen8_ppgtt_notify_vgt(ppgtt, true);
1617
1618         return 0;
1619
1620 free_scratch:
1621         gen8_free_scratch(&ppgtt->base);
1622         return ret;
1623 }
1624
1625 static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
1626 {
1627         struct i915_address_space *vm = &ppgtt->base;
1628         struct i915_page_table *unused;
1629         gen6_pte_t scratch_pte;
1630         uint32_t pd_entry;
1631         uint32_t  pte, pde;
1632         uint32_t start = ppgtt->base.start, length = ppgtt->base.total;
1633
1634         scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
1635                                      I915_CACHE_LLC, 0);
1636
1637         gen6_for_each_pde(unused, &ppgtt->pd, start, length, pde) {
1638                 u32 expected;
1639                 gen6_pte_t *pt_vaddr;
1640                 const dma_addr_t pt_addr = px_dma(ppgtt->pd.page_table[pde]);
1641                 pd_entry = readl(ppgtt->pd_addr + pde);
1642                 expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID);
1643
1644                 if (pd_entry != expected)
1645                         seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n",
1646                                    pde,
1647                                    pd_entry,
1648                                    expected);
1649                 seq_printf(m, "\tPDE: %x\n", pd_entry);
1650
1651                 pt_vaddr = kmap_px(ppgtt->pd.page_table[pde]);
1652
1653                 for (pte = 0; pte < GEN6_PTES; pte+=4) {
1654                         unsigned long va =
1655                                 (pde * PAGE_SIZE * GEN6_PTES) +
1656                                 (pte * PAGE_SIZE);
1657                         int i;
1658                         bool found = false;
1659                         for (i = 0; i < 4; i++)
1660                                 if (pt_vaddr[pte + i] != scratch_pte)
1661                                         found = true;
1662                         if (!found)
1663                                 continue;
1664
1665                         seq_printf(m, "\t\t0x%lx [%03d,%04d]: =", va, pde, pte);
1666                         for (i = 0; i < 4; i++) {
1667                                 if (pt_vaddr[pte + i] != scratch_pte)
1668                                         seq_printf(m, " %08x", pt_vaddr[pte + i]);
1669                                 else
1670                                         seq_puts(m, "  SCRATCH ");
1671                         }
1672                         seq_puts(m, "\n");
1673                 }
1674                 kunmap_px(ppgtt, pt_vaddr);
1675         }
1676 }
1677
1678 /* Write pde (index) from the page directory @pd to the page table @pt */
1679 static void gen6_write_pde(struct i915_page_directory *pd,
1680                             const int pde, struct i915_page_table *pt)
1681 {
1682         /* Caller needs to make sure the write completes if necessary */
1683         struct i915_hw_ppgtt *ppgtt =
1684                 container_of(pd, struct i915_hw_ppgtt, pd);
1685         u32 pd_entry;
1686
1687         pd_entry = GEN6_PDE_ADDR_ENCODE(px_dma(pt));
1688         pd_entry |= GEN6_PDE_VALID;
1689
1690         writel(pd_entry, ppgtt->pd_addr + pde);
1691 }
1692
1693 /* Write all the page tables found in the ppgtt structure to incrementing page
1694  * directories. */
1695 static void gen6_write_page_range(struct drm_i915_private *dev_priv,
1696                                   struct i915_page_directory *pd,
1697                                   uint32_t start, uint32_t length)
1698 {
1699         struct i915_ggtt *ggtt = &dev_priv->ggtt;
1700         struct i915_page_table *pt;
1701         uint32_t pde;
1702
1703         gen6_for_each_pde(pt, pd, start, length, pde)
1704                 gen6_write_pde(pd, pde, pt);
1705
1706         /* Make sure write is complete before other code can use this page
1707          * table. Also require for WC mapped PTEs */
1708         readl(ggtt->gsm);
1709 }
1710
1711 static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt)
1712 {
1713         BUG_ON(ppgtt->pd.base.ggtt_offset & 0x3f);
1714
1715         return (ppgtt->pd.base.ggtt_offset / 64) << 16;
1716 }
1717
1718 static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
1719                          struct drm_i915_gem_request *req)
1720 {
1721         struct intel_ring *ring = req->ring;
1722         struct intel_engine_cs *engine = req->engine;
1723         int ret;
1724
1725         /* NB: TLBs must be flushed and invalidated before a switch */
1726         ret = engine->emit_flush(req, EMIT_INVALIDATE | EMIT_FLUSH);
1727         if (ret)
1728                 return ret;
1729
1730         ret = intel_ring_begin(req, 6);
1731         if (ret)
1732                 return ret;
1733
1734         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
1735         intel_ring_emit_reg(ring, RING_PP_DIR_DCLV(engine));
1736         intel_ring_emit(ring, PP_DIR_DCLV_2G);
1737         intel_ring_emit_reg(ring, RING_PP_DIR_BASE(engine));
1738         intel_ring_emit(ring, get_pd_offset(ppgtt));
1739         intel_ring_emit(ring, MI_NOOP);
1740         intel_ring_advance(ring);
1741
1742         return 0;
1743 }
1744
1745 static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
1746                           struct drm_i915_gem_request *req)
1747 {
1748         struct intel_ring *ring = req->ring;
1749         struct intel_engine_cs *engine = req->engine;
1750         int ret;
1751
1752         /* NB: TLBs must be flushed and invalidated before a switch */
1753         ret = engine->emit_flush(req, EMIT_INVALIDATE | EMIT_FLUSH);
1754         if (ret)
1755                 return ret;
1756
1757         ret = intel_ring_begin(req, 6);
1758         if (ret)
1759                 return ret;
1760
1761         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
1762         intel_ring_emit_reg(ring, RING_PP_DIR_DCLV(engine));
1763         intel_ring_emit(ring, PP_DIR_DCLV_2G);
1764         intel_ring_emit_reg(ring, RING_PP_DIR_BASE(engine));
1765         intel_ring_emit(ring, get_pd_offset(ppgtt));
1766         intel_ring_emit(ring, MI_NOOP);
1767         intel_ring_advance(ring);
1768
1769         /* XXX: RCS is the only one to auto invalidate the TLBs? */
1770         if (engine->id != RCS) {
1771                 ret = engine->emit_flush(req, EMIT_INVALIDATE | EMIT_FLUSH);
1772                 if (ret)
1773                         return ret;
1774         }
1775
1776         return 0;
1777 }
1778
1779 static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
1780                           struct drm_i915_gem_request *req)
1781 {
1782         struct intel_engine_cs *engine = req->engine;
1783         struct drm_i915_private *dev_priv = req->i915;
1784
1785         I915_WRITE(RING_PP_DIR_DCLV(engine), PP_DIR_DCLV_2G);
1786         I915_WRITE(RING_PP_DIR_BASE(engine), get_pd_offset(ppgtt));
1787         return 0;
1788 }
1789
1790 static void gen8_ppgtt_enable(struct drm_i915_private *dev_priv)
1791 {
1792         struct intel_engine_cs *engine;
1793         enum intel_engine_id id;
1794
1795         for_each_engine(engine, dev_priv, id) {
1796                 u32 four_level = USES_FULL_48BIT_PPGTT(dev_priv) ?
1797                                  GEN8_GFX_PPGTT_48B : 0;
1798                 I915_WRITE(RING_MODE_GEN7(engine),
1799                            _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE | four_level));
1800         }
1801 }
1802
1803 static void gen7_ppgtt_enable(struct drm_i915_private *dev_priv)
1804 {
1805         struct intel_engine_cs *engine;
1806         uint32_t ecochk, ecobits;
1807         enum intel_engine_id id;
1808
1809         ecobits = I915_READ(GAC_ECO_BITS);
1810         I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
1811
1812         ecochk = I915_READ(GAM_ECOCHK);
1813         if (IS_HASWELL(dev_priv)) {
1814                 ecochk |= ECOCHK_PPGTT_WB_HSW;
1815         } else {
1816                 ecochk |= ECOCHK_PPGTT_LLC_IVB;
1817                 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
1818         }
1819         I915_WRITE(GAM_ECOCHK, ecochk);
1820
1821         for_each_engine(engine, dev_priv, id) {
1822                 /* GFX_MODE is per-ring on gen7+ */
1823                 I915_WRITE(RING_MODE_GEN7(engine),
1824                            _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
1825         }
1826 }
1827
1828 static void gen6_ppgtt_enable(struct drm_i915_private *dev_priv)
1829 {
1830         uint32_t ecochk, gab_ctl, ecobits;
1831
1832         ecobits = I915_READ(GAC_ECO_BITS);
1833         I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
1834                    ECOBITS_PPGTT_CACHE64B);
1835
1836         gab_ctl = I915_READ(GAB_CTL);
1837         I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
1838
1839         ecochk = I915_READ(GAM_ECOCHK);
1840         I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);
1841
1842         I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
1843 }
1844
1845 /* PPGTT support for Sandybdrige/Gen6 and later */
1846 static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
1847                                    uint64_t start,
1848                                    uint64_t length)
1849 {
1850         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1851         gen6_pte_t *pt_vaddr, scratch_pte;
1852         unsigned first_entry = start >> PAGE_SHIFT;
1853         unsigned num_entries = length >> PAGE_SHIFT;
1854         unsigned act_pt = first_entry / GEN6_PTES;
1855         unsigned first_pte = first_entry % GEN6_PTES;
1856         unsigned last_pte, i;
1857
1858         scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
1859                                      I915_CACHE_LLC, 0);
1860
1861         while (num_entries) {
1862                 last_pte = first_pte + num_entries;
1863                 if (last_pte > GEN6_PTES)
1864                         last_pte = GEN6_PTES;
1865
1866                 pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]);
1867
1868                 for (i = first_pte; i < last_pte; i++)
1869                         pt_vaddr[i] = scratch_pte;
1870
1871                 kunmap_px(ppgtt, pt_vaddr);
1872
1873                 num_entries -= last_pte - first_pte;
1874                 first_pte = 0;
1875                 act_pt++;
1876         }
1877 }
1878
1879 static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
1880                                       struct sg_table *pages,
1881                                       uint64_t start,
1882                                       enum i915_cache_level cache_level, u32 flags)
1883 {
1884         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1885         unsigned first_entry = start >> PAGE_SHIFT;
1886         unsigned act_pt = first_entry / GEN6_PTES;
1887         unsigned act_pte = first_entry % GEN6_PTES;
1888         gen6_pte_t *pt_vaddr = NULL;
1889         struct sgt_iter sgt_iter;
1890         dma_addr_t addr;
1891
1892         for_each_sgt_dma(addr, sgt_iter, pages) {
1893                 if (pt_vaddr == NULL)
1894                         pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]);
1895
1896                 pt_vaddr[act_pte] =
1897                         vm->pte_encode(addr, cache_level, flags);
1898
1899                 if (++act_pte == GEN6_PTES) {
1900                         kunmap_px(ppgtt, pt_vaddr);
1901                         pt_vaddr = NULL;
1902                         act_pt++;
1903                         act_pte = 0;
1904                 }
1905         }
1906
1907         if (pt_vaddr)
1908                 kunmap_px(ppgtt, pt_vaddr);
1909 }
1910
1911 static int gen6_alloc_va_range(struct i915_address_space *vm,
1912                                uint64_t start_in, uint64_t length_in)
1913 {
1914         DECLARE_BITMAP(new_page_tables, I915_PDES);
1915         struct drm_i915_private *dev_priv = vm->i915;
1916         struct i915_ggtt *ggtt = &dev_priv->ggtt;
1917         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1918         struct i915_page_table *pt;
1919         uint32_t start, length, start_save, length_save;
1920         uint32_t pde;
1921         int ret;
1922
1923         start = start_save = start_in;
1924         length = length_save = length_in;
1925
1926         bitmap_zero(new_page_tables, I915_PDES);
1927
1928         /* The allocation is done in two stages so that we can bail out with
1929          * minimal amount of pain. The first stage finds new page tables that
1930          * need allocation. The second stage marks use ptes within the page
1931          * tables.
1932          */
1933         gen6_for_each_pde(pt, &ppgtt->pd, start, length, pde) {
1934                 if (pt != vm->scratch_pt) {
1935                         WARN_ON(bitmap_empty(pt->used_ptes, GEN6_PTES));
1936                         continue;
1937                 }
1938
1939                 /* We've already allocated a page table */
1940                 WARN_ON(!bitmap_empty(pt->used_ptes, GEN6_PTES));
1941
1942                 pt = alloc_pt(dev_priv);
1943                 if (IS_ERR(pt)) {
1944                         ret = PTR_ERR(pt);
1945                         goto unwind_out;
1946                 }
1947
1948                 gen6_initialize_pt(vm, pt);
1949
1950                 ppgtt->pd.page_table[pde] = pt;
1951                 __set_bit(pde, new_page_tables);
1952                 trace_i915_page_table_entry_alloc(vm, pde, start, GEN6_PDE_SHIFT);
1953         }
1954
1955         start = start_save;
1956         length = length_save;
1957
1958         gen6_for_each_pde(pt, &ppgtt->pd, start, length, pde) {
1959                 DECLARE_BITMAP(tmp_bitmap, GEN6_PTES);
1960
1961                 bitmap_zero(tmp_bitmap, GEN6_PTES);
1962                 bitmap_set(tmp_bitmap, gen6_pte_index(start),
1963                            gen6_pte_count(start, length));
1964
1965                 if (__test_and_clear_bit(pde, new_page_tables))
1966                         gen6_write_pde(&ppgtt->pd, pde, pt);
1967
1968                 trace_i915_page_table_entry_map(vm, pde, pt,
1969                                          gen6_pte_index(start),
1970                                          gen6_pte_count(start, length),
1971                                          GEN6_PTES);
1972                 bitmap_or(pt->used_ptes, tmp_bitmap, pt->used_ptes,
1973                                 GEN6_PTES);
1974         }
1975
1976         WARN_ON(!bitmap_empty(new_page_tables, I915_PDES));
1977
1978         /* Make sure write is complete before other code can use this page
1979          * table. Also require for WC mapped PTEs */
1980         readl(ggtt->gsm);
1981
1982         mark_tlbs_dirty(ppgtt);
1983         return 0;
1984
1985 unwind_out:
1986         for_each_set_bit(pde, new_page_tables, I915_PDES) {
1987                 struct i915_page_table *pt = ppgtt->pd.page_table[pde];
1988
1989                 ppgtt->pd.page_table[pde] = vm->scratch_pt;
1990                 free_pt(dev_priv, pt);
1991         }
1992
1993         mark_tlbs_dirty(ppgtt);
1994         return ret;
1995 }
1996
1997 static int gen6_init_scratch(struct i915_address_space *vm)
1998 {
1999         struct drm_i915_private *dev_priv = vm->i915;
2000         int ret;
2001
2002         ret = setup_scratch_page(dev_priv, &vm->scratch_page, I915_GFP_DMA);
2003         if (ret)
2004                 return ret;
2005
2006         vm->scratch_pt = alloc_pt(dev_priv);
2007         if (IS_ERR(vm->scratch_pt)) {
2008                 cleanup_scratch_page(dev_priv, &vm->scratch_page);
2009                 return PTR_ERR(vm->scratch_pt);
2010         }
2011
2012         gen6_initialize_pt(vm, vm->scratch_pt);
2013
2014         return 0;
2015 }
2016
2017 static void gen6_free_scratch(struct i915_address_space *vm)
2018 {
2019         struct drm_i915_private *dev_priv = vm->i915;
2020
2021         free_pt(dev_priv, vm->scratch_pt);
2022         cleanup_scratch_page(dev_priv, &vm->scratch_page);
2023 }
2024
2025 static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
2026 {
2027         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
2028         struct i915_page_directory *pd = &ppgtt->pd;
2029         struct drm_i915_private *dev_priv = vm->i915;
2030         struct i915_page_table *pt;
2031         uint32_t pde;
2032
2033         drm_mm_remove_node(&ppgtt->node);
2034
2035         gen6_for_all_pdes(pt, pd, pde)
2036                 if (pt != vm->scratch_pt)
2037                         free_pt(dev_priv, pt);
2038
2039         gen6_free_scratch(vm);
2040 }
2041
2042 static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt)
2043 {
2044         struct i915_address_space *vm = &ppgtt->base;
2045         struct drm_i915_private *dev_priv = ppgtt->base.i915;
2046         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2047         bool retried = false;
2048         int ret;
2049
2050         /* PPGTT PDEs reside in the GGTT and consists of 512 entries. The
2051          * allocator works in address space sizes, so it's multiplied by page
2052          * size. We allocate at the top of the GTT to avoid fragmentation.
2053          */
2054         BUG_ON(!drm_mm_initialized(&ggtt->base.mm));
2055
2056         ret = gen6_init_scratch(vm);
2057         if (ret)
2058                 return ret;
2059
2060 alloc:
2061         ret = drm_mm_insert_node_in_range_generic(&ggtt->base.mm, &ppgtt->node,
2062                                                   GEN6_PD_SIZE, GEN6_PD_ALIGN,
2063                                                   I915_COLOR_UNEVICTABLE,
2064                                                   0, ggtt->base.total,
2065                                                   DRM_MM_TOPDOWN);
2066         if (ret == -ENOSPC && !retried) {
2067                 ret = i915_gem_evict_something(&ggtt->base,
2068                                                GEN6_PD_SIZE, GEN6_PD_ALIGN,
2069                                                I915_COLOR_UNEVICTABLE,
2070                                                0, ggtt->base.total,
2071                                                0);
2072                 if (ret)
2073                         goto err_out;
2074
2075                 retried = true;
2076                 goto alloc;
2077         }
2078
2079         if (ret)
2080                 goto err_out;
2081
2082
2083         if (ppgtt->node.start < ggtt->mappable_end)
2084                 DRM_DEBUG("Forced to use aperture for PDEs\n");
2085
2086         return 0;
2087
2088 err_out:
2089         gen6_free_scratch(vm);
2090         return ret;
2091 }
2092
2093 static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)
2094 {
2095         return gen6_ppgtt_allocate_page_directories(ppgtt);
2096 }
2097
2098 static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt,
2099                                   uint64_t start, uint64_t length)
2100 {
2101         struct i915_page_table *unused;
2102         uint32_t pde;
2103
2104         gen6_for_each_pde(unused, &ppgtt->pd, start, length, pde)
2105                 ppgtt->pd.page_table[pde] = ppgtt->base.scratch_pt;
2106 }
2107
2108 static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
2109 {
2110         struct drm_i915_private *dev_priv = ppgtt->base.i915;
2111         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2112         int ret;
2113
2114         ppgtt->base.pte_encode = ggtt->base.pte_encode;
2115         if (intel_vgpu_active(dev_priv) || IS_GEN6(dev_priv))
2116                 ppgtt->switch_mm = gen6_mm_switch;
2117         else if (IS_HASWELL(dev_priv))
2118                 ppgtt->switch_mm = hsw_mm_switch;
2119         else if (IS_GEN7(dev_priv))
2120                 ppgtt->switch_mm = gen7_mm_switch;
2121         else
2122                 BUG();
2123
2124         ret = gen6_ppgtt_alloc(ppgtt);
2125         if (ret)
2126                 return ret;
2127
2128         ppgtt->base.allocate_va_range = gen6_alloc_va_range;
2129         ppgtt->base.clear_range = gen6_ppgtt_clear_range;
2130         ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
2131         ppgtt->base.unbind_vma = ppgtt_unbind_vma;
2132         ppgtt->base.bind_vma = ppgtt_bind_vma;
2133         ppgtt->base.cleanup = gen6_ppgtt_cleanup;
2134         ppgtt->base.start = 0;
2135         ppgtt->base.total = I915_PDES * GEN6_PTES * PAGE_SIZE;
2136         ppgtt->debug_dump = gen6_dump_ppgtt;
2137
2138         ppgtt->pd.base.ggtt_offset =
2139                 ppgtt->node.start / PAGE_SIZE * sizeof(gen6_pte_t);
2140
2141         ppgtt->pd_addr = (gen6_pte_t __iomem *)ggtt->gsm +
2142                 ppgtt->pd.base.ggtt_offset / sizeof(gen6_pte_t);
2143
2144         gen6_scratch_va_range(ppgtt, 0, ppgtt->base.total);
2145
2146         gen6_write_page_range(dev_priv, &ppgtt->pd, 0, ppgtt->base.total);
2147
2148         DRM_DEBUG_DRIVER("Allocated pde space (%lldM) at GTT entry: %llx\n",
2149                          ppgtt->node.size >> 20,
2150                          ppgtt->node.start / PAGE_SIZE);
2151
2152         DRM_DEBUG("Adding PPGTT at offset %x\n",
2153                   ppgtt->pd.base.ggtt_offset << 10);
2154
2155         return 0;
2156 }
2157
2158 static int __hw_ppgtt_init(struct i915_hw_ppgtt *ppgtt,
2159                            struct drm_i915_private *dev_priv)
2160 {
2161         ppgtt->base.i915 = dev_priv;
2162
2163         if (INTEL_INFO(dev_priv)->gen < 8)
2164                 return gen6_ppgtt_init(ppgtt);
2165         else
2166                 return gen8_ppgtt_init(ppgtt);
2167 }
2168
2169 static void i915_address_space_init(struct i915_address_space *vm,
2170                                     struct drm_i915_private *dev_priv,
2171                                     const char *name)
2172 {
2173         i915_gem_timeline_init(dev_priv, &vm->timeline, name);
2174         drm_mm_init(&vm->mm, vm->start, vm->total);
2175         INIT_LIST_HEAD(&vm->active_list);
2176         INIT_LIST_HEAD(&vm->inactive_list);
2177         INIT_LIST_HEAD(&vm->unbound_list);
2178         list_add_tail(&vm->global_link, &dev_priv->vm_list);
2179 }
2180
2181 static void i915_address_space_fini(struct i915_address_space *vm)
2182 {
2183         i915_gem_timeline_fini(&vm->timeline);
2184         drm_mm_takedown(&vm->mm);
2185         list_del(&vm->global_link);
2186 }
2187
2188 static void gtt_write_workarounds(struct drm_i915_private *dev_priv)
2189 {
2190         /* This function is for gtt related workarounds. This function is
2191          * called on driver load and after a GPU reset, so you can place
2192          * workarounds here even if they get overwritten by GPU reset.
2193          */
2194         /* WaIncreaseDefaultTLBEntries:chv,bdw,skl,bxt */
2195         if (IS_BROADWELL(dev_priv))
2196                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_BDW);
2197         else if (IS_CHERRYVIEW(dev_priv))
2198                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_CHV);
2199         else if (IS_SKYLAKE(dev_priv))
2200                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_SKL);
2201         else if (IS_BROXTON(dev_priv))
2202                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_BXT);
2203 }
2204
2205 static int i915_ppgtt_init(struct i915_hw_ppgtt *ppgtt,
2206                            struct drm_i915_private *dev_priv,
2207                            struct drm_i915_file_private *file_priv,
2208                            const char *name)
2209 {
2210         int ret;
2211
2212         ret = __hw_ppgtt_init(ppgtt, dev_priv);
2213         if (ret == 0) {
2214                 kref_init(&ppgtt->ref);
2215                 i915_address_space_init(&ppgtt->base, dev_priv, name);
2216                 ppgtt->base.file = file_priv;
2217         }
2218
2219         return ret;
2220 }
2221
2222 int i915_ppgtt_init_hw(struct drm_i915_private *dev_priv)
2223 {
2224         gtt_write_workarounds(dev_priv);
2225
2226         /* In the case of execlists, PPGTT is enabled by the context descriptor
2227          * and the PDPs are contained within the context itself.  We don't
2228          * need to do anything here. */
2229         if (i915.enable_execlists)
2230                 return 0;
2231
2232         if (!USES_PPGTT(dev_priv))
2233                 return 0;
2234
2235         if (IS_GEN6(dev_priv))
2236                 gen6_ppgtt_enable(dev_priv);
2237         else if (IS_GEN7(dev_priv))
2238                 gen7_ppgtt_enable(dev_priv);
2239         else if (INTEL_GEN(dev_priv) >= 8)
2240                 gen8_ppgtt_enable(dev_priv);
2241         else
2242                 MISSING_CASE(INTEL_GEN(dev_priv));
2243
2244         return 0;
2245 }
2246
2247 struct i915_hw_ppgtt *
2248 i915_ppgtt_create(struct drm_i915_private *dev_priv,
2249                   struct drm_i915_file_private *fpriv,
2250                   const char *name)
2251 {
2252         struct i915_hw_ppgtt *ppgtt;
2253         int ret;
2254
2255         ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
2256         if (!ppgtt)
2257                 return ERR_PTR(-ENOMEM);
2258
2259         ret = i915_ppgtt_init(ppgtt, dev_priv, fpriv, name);
2260         if (ret) {
2261                 kfree(ppgtt);
2262                 return ERR_PTR(ret);
2263         }
2264
2265         trace_i915_ppgtt_create(&ppgtt->base);
2266
2267         return ppgtt;
2268 }
2269
2270 void i915_ppgtt_release(struct kref *kref)
2271 {
2272         struct i915_hw_ppgtt *ppgtt =
2273                 container_of(kref, struct i915_hw_ppgtt, ref);
2274
2275         trace_i915_ppgtt_release(&ppgtt->base);
2276
2277         /* vmas should already be unbound and destroyed */
2278         WARN_ON(!list_empty(&ppgtt->base.active_list));
2279         WARN_ON(!list_empty(&ppgtt->base.inactive_list));
2280         WARN_ON(!list_empty(&ppgtt->base.unbound_list));
2281
2282         i915_address_space_fini(&ppgtt->base);
2283
2284         ppgtt->base.cleanup(&ppgtt->base);
2285         kfree(ppgtt);
2286 }
2287
2288 /* Certain Gen5 chipsets require require idling the GPU before
2289  * unmapping anything from the GTT when VT-d is enabled.
2290  */
2291 static bool needs_idle_maps(struct drm_i915_private *dev_priv)
2292 {
2293 #ifdef CONFIG_INTEL_IOMMU
2294         /* Query intel_iommu to see if we need the workaround. Presumably that
2295          * was loaded first.
2296          */
2297         if (IS_GEN5(dev_priv) && IS_MOBILE(dev_priv) && intel_iommu_gfx_mapped)
2298                 return true;
2299 #endif
2300         return false;
2301 }
2302
2303 void i915_check_and_clear_faults(struct drm_i915_private *dev_priv)
2304 {
2305         struct intel_engine_cs *engine;
2306         enum intel_engine_id id;
2307
2308         if (INTEL_INFO(dev_priv)->gen < 6)
2309                 return;
2310
2311         for_each_engine(engine, dev_priv, id) {
2312                 u32 fault_reg;
2313                 fault_reg = I915_READ(RING_FAULT_REG(engine));
2314                 if (fault_reg & RING_FAULT_VALID) {
2315                         DRM_DEBUG_DRIVER("Unexpected fault\n"
2316                                          "\tAddr: 0x%08lx\n"
2317                                          "\tAddress space: %s\n"
2318                                          "\tSource ID: %d\n"
2319                                          "\tType: %d\n",
2320                                          fault_reg & PAGE_MASK,
2321                                          fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
2322                                          RING_FAULT_SRCID(fault_reg),
2323                                          RING_FAULT_FAULT_TYPE(fault_reg));
2324                         I915_WRITE(RING_FAULT_REG(engine),
2325                                    fault_reg & ~RING_FAULT_VALID);
2326                 }
2327         }
2328
2329         /* Engine specific init may not have been done till this point. */
2330         if (dev_priv->engine[RCS])
2331                 POSTING_READ(RING_FAULT_REG(dev_priv->engine[RCS]));
2332 }
2333
2334 static void i915_ggtt_flush(struct drm_i915_private *dev_priv)
2335 {
2336         if (INTEL_INFO(dev_priv)->gen < 6) {
2337                 intel_gtt_chipset_flush();
2338         } else {
2339                 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
2340                 POSTING_READ(GFX_FLSH_CNTL_GEN6);
2341         }
2342 }
2343
2344 void i915_gem_suspend_gtt_mappings(struct drm_i915_private *dev_priv)
2345 {
2346         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2347
2348         /* Don't bother messing with faults pre GEN6 as we have little
2349          * documentation supporting that it's a good idea.
2350          */
2351         if (INTEL_GEN(dev_priv) < 6)
2352                 return;
2353
2354         i915_check_and_clear_faults(dev_priv);
2355
2356         ggtt->base.clear_range(&ggtt->base, ggtt->base.start, ggtt->base.total);
2357
2358         i915_ggtt_flush(dev_priv);
2359 }
2360
2361 int i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj,
2362                                struct sg_table *pages)
2363 {
2364         do {
2365                 if (dma_map_sg(&obj->base.dev->pdev->dev,
2366                                pages->sgl, pages->nents,
2367                                PCI_DMA_BIDIRECTIONAL))
2368                         return 0;
2369
2370                 /* If the DMA remap fails, one cause can be that we have
2371                  * too many objects pinned in a small remapping table,
2372                  * such as swiotlb. Incrementally purge all other objects and
2373                  * try again - if there are no more pages to remove from
2374                  * the DMA remapper, i915_gem_shrink will return 0.
2375                  */
2376                 GEM_BUG_ON(obj->mm.pages == pages);
2377         } while (i915_gem_shrink(to_i915(obj->base.dev),
2378                                  obj->base.size >> PAGE_SHIFT,
2379                                  I915_SHRINK_BOUND |
2380                                  I915_SHRINK_UNBOUND |
2381                                  I915_SHRINK_ACTIVE));
2382
2383         return -ENOSPC;
2384 }
2385
2386 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
2387 {
2388         writeq(pte, addr);
2389 }
2390
2391 static void gen8_ggtt_insert_page(struct i915_address_space *vm,
2392                                   dma_addr_t addr,
2393                                   uint64_t offset,
2394                                   enum i915_cache_level level,
2395                                   u32 unused)
2396 {
2397         struct drm_i915_private *dev_priv = vm->i915;
2398         gen8_pte_t __iomem *pte =
2399                 (gen8_pte_t __iomem *)dev_priv->ggtt.gsm +
2400                 (offset >> PAGE_SHIFT);
2401
2402         gen8_set_pte(pte, gen8_pte_encode(addr, level));
2403
2404         I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
2405         POSTING_READ(GFX_FLSH_CNTL_GEN6);
2406 }
2407
2408 static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
2409                                      struct sg_table *st,
2410                                      uint64_t start,
2411                                      enum i915_cache_level level, u32 unused)
2412 {
2413         struct drm_i915_private *dev_priv = vm->i915;
2414         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2415         struct sgt_iter sgt_iter;
2416         gen8_pte_t __iomem *gtt_entries;
2417         gen8_pte_t gtt_entry;
2418         dma_addr_t addr;
2419         int i = 0;
2420
2421         gtt_entries = (gen8_pte_t __iomem *)ggtt->gsm + (start >> PAGE_SHIFT);
2422
2423         for_each_sgt_dma(addr, sgt_iter, st) {
2424                 gtt_entry = gen8_pte_encode(addr, level);
2425                 gen8_set_pte(&gtt_entries[i++], gtt_entry);
2426         }
2427
2428         /*
2429          * XXX: This serves as a posting read to make sure that the PTE has
2430          * actually been updated. There is some concern that even though
2431          * registers and PTEs are within the same BAR that they are potentially
2432          * of NUMA access patterns. Therefore, even with the way we assume
2433          * hardware should work, we must keep this posting read for paranoia.
2434          */
2435         if (i != 0)
2436                 WARN_ON(readq(&gtt_entries[i-1]) != gtt_entry);
2437
2438         /* This next bit makes the above posting read even more important. We
2439          * want to flush the TLBs only after we're certain all the PTE updates
2440          * have finished.
2441          */
2442         I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
2443         POSTING_READ(GFX_FLSH_CNTL_GEN6);
2444 }
2445
2446 struct insert_entries {
2447         struct i915_address_space *vm;
2448         struct sg_table *st;
2449         uint64_t start;
2450         enum i915_cache_level level;
2451         u32 flags;
2452 };
2453
2454 static int gen8_ggtt_insert_entries__cb(void *_arg)
2455 {
2456         struct insert_entries *arg = _arg;
2457         gen8_ggtt_insert_entries(arg->vm, arg->st,
2458                                  arg->start, arg->level, arg->flags);
2459         return 0;
2460 }
2461
2462 static void gen8_ggtt_insert_entries__BKL(struct i915_address_space *vm,
2463                                           struct sg_table *st,
2464                                           uint64_t start,
2465                                           enum i915_cache_level level,
2466                                           u32 flags)
2467 {
2468         struct insert_entries arg = { vm, st, start, level, flags };
2469         stop_machine(gen8_ggtt_insert_entries__cb, &arg, NULL);
2470 }
2471
2472 static void gen6_ggtt_insert_page(struct i915_address_space *vm,
2473                                   dma_addr_t addr,
2474                                   uint64_t offset,
2475                                   enum i915_cache_level level,
2476                                   u32 flags)
2477 {
2478         struct drm_i915_private *dev_priv = vm->i915;
2479         gen6_pte_t __iomem *pte =
2480                 (gen6_pte_t __iomem *)dev_priv->ggtt.gsm +
2481                 (offset >> PAGE_SHIFT);
2482
2483         iowrite32(vm->pte_encode(addr, level, flags), pte);
2484
2485         I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
2486         POSTING_READ(GFX_FLSH_CNTL_GEN6);
2487 }
2488
2489 /*
2490  * Binds an object into the global gtt with the specified cache level. The object
2491  * will be accessible to the GPU via commands whose operands reference offsets
2492  * within the global GTT as well as accessible by the GPU through the GMADR
2493  * mapped BAR (dev_priv->mm.gtt->gtt).
2494  */
2495 static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
2496                                      struct sg_table *st,
2497                                      uint64_t start,
2498                                      enum i915_cache_level level, u32 flags)
2499 {
2500         struct drm_i915_private *dev_priv = vm->i915;
2501         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2502         struct sgt_iter sgt_iter;
2503         gen6_pte_t __iomem *gtt_entries;
2504         gen6_pte_t gtt_entry;
2505         dma_addr_t addr;
2506         int i = 0;
2507
2508         gtt_entries = (gen6_pte_t __iomem *)ggtt->gsm + (start >> PAGE_SHIFT);
2509
2510         for_each_sgt_dma(addr, sgt_iter, st) {
2511                 gtt_entry = vm->pte_encode(addr, level, flags);
2512                 iowrite32(gtt_entry, &gtt_entries[i++]);
2513         }
2514
2515         /* XXX: This serves as a posting read to make sure that the PTE has
2516          * actually been updated. There is some concern that even though
2517          * registers and PTEs are within the same BAR that they are potentially
2518          * of NUMA access patterns. Therefore, even with the way we assume
2519          * hardware should work, we must keep this posting read for paranoia.
2520          */
2521         if (i != 0)
2522                 WARN_ON(readl(&gtt_entries[i-1]) != gtt_entry);
2523
2524         /* This next bit makes the above posting read even more important. We
2525          * want to flush the TLBs only after we're certain all the PTE updates
2526          * have finished.
2527          */
2528         I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
2529         POSTING_READ(GFX_FLSH_CNTL_GEN6);
2530 }
2531
2532 static void nop_clear_range(struct i915_address_space *vm,
2533                             uint64_t start, uint64_t length)
2534 {
2535 }
2536
2537 static void gen8_ggtt_clear_range(struct i915_address_space *vm,
2538                                   uint64_t start, uint64_t length)
2539 {
2540         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2541         unsigned first_entry = start >> PAGE_SHIFT;
2542         unsigned num_entries = length >> PAGE_SHIFT;
2543         gen8_pte_t scratch_pte, __iomem *gtt_base =
2544                 (gen8_pte_t __iomem *)ggtt->gsm + first_entry;
2545         const int max_entries = ggtt_total_entries(ggtt) - first_entry;
2546         int i;
2547
2548         if (WARN(num_entries > max_entries,
2549                  "First entry = %d; Num entries = %d (max=%d)\n",
2550                  first_entry, num_entries, max_entries))
2551                 num_entries = max_entries;
2552
2553         scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
2554                                       I915_CACHE_LLC);
2555         for (i = 0; i < num_entries; i++)
2556                 gen8_set_pte(&gtt_base[i], scratch_pte);
2557         readl(gtt_base);
2558 }
2559
2560 static void gen6_ggtt_clear_range(struct i915_address_space *vm,
2561                                   uint64_t start,
2562                                   uint64_t length)
2563 {
2564         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2565         unsigned first_entry = start >> PAGE_SHIFT;
2566         unsigned num_entries = length >> PAGE_SHIFT;
2567         gen6_pte_t scratch_pte, __iomem *gtt_base =
2568                 (gen6_pte_t __iomem *)ggtt->gsm + first_entry;
2569         const int max_entries = ggtt_total_entries(ggtt) - first_entry;
2570         int i;
2571
2572         if (WARN(num_entries > max_entries,
2573                  "First entry = %d; Num entries = %d (max=%d)\n",
2574                  first_entry, num_entries, max_entries))
2575                 num_entries = max_entries;
2576
2577         scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
2578                                      I915_CACHE_LLC, 0);
2579
2580         for (i = 0; i < num_entries; i++)
2581                 iowrite32(scratch_pte, &gtt_base[i]);
2582         readl(gtt_base);
2583 }
2584
2585 static void i915_ggtt_insert_page(struct i915_address_space *vm,
2586                                   dma_addr_t addr,
2587                                   uint64_t offset,
2588                                   enum i915_cache_level cache_level,
2589                                   u32 unused)
2590 {
2591         unsigned int flags = (cache_level == I915_CACHE_NONE) ?
2592                 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
2593
2594         intel_gtt_insert_page(addr, offset >> PAGE_SHIFT, flags);
2595 }
2596
2597 static void i915_ggtt_insert_entries(struct i915_address_space *vm,
2598                                      struct sg_table *pages,
2599                                      uint64_t start,
2600                                      enum i915_cache_level cache_level, u32 unused)
2601 {
2602         unsigned int flags = (cache_level == I915_CACHE_NONE) ?
2603                 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
2604
2605         intel_gtt_insert_sg_entries(pages, start >> PAGE_SHIFT, flags);
2606
2607 }
2608
2609 static void i915_ggtt_clear_range(struct i915_address_space *vm,
2610                                   uint64_t start,
2611                                   uint64_t length)
2612 {
2613         intel_gtt_clear_range(start >> PAGE_SHIFT, length >> PAGE_SHIFT);
2614 }
2615
2616 static int ggtt_bind_vma(struct i915_vma *vma,
2617                          enum i915_cache_level cache_level,
2618                          u32 flags)
2619 {
2620         struct drm_i915_private *i915 = vma->vm->i915;
2621         struct drm_i915_gem_object *obj = vma->obj;
2622         u32 pte_flags = 0;
2623         int ret;
2624
2625         ret = i915_get_ggtt_vma_pages(vma);
2626         if (ret)
2627                 return ret;
2628
2629         /* Currently applicable only to VLV */
2630         if (obj->gt_ro)
2631                 pte_flags |= PTE_READ_ONLY;
2632
2633         intel_runtime_pm_get(i915);
2634         vma->vm->insert_entries(vma->vm, vma->pages, vma->node.start,
2635                                 cache_level, pte_flags);
2636         intel_runtime_pm_put(i915);
2637
2638         /*
2639          * Without aliasing PPGTT there's no difference between
2640          * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally
2641          * upgrade to both bound if we bind either to avoid double-binding.
2642          */
2643         vma->flags |= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
2644
2645         return 0;
2646 }
2647
2648 static int aliasing_gtt_bind_vma(struct i915_vma *vma,
2649                                  enum i915_cache_level cache_level,
2650                                  u32 flags)
2651 {
2652         struct drm_i915_private *i915 = vma->vm->i915;
2653         u32 pte_flags;
2654         int ret;
2655
2656         ret = i915_get_ggtt_vma_pages(vma);
2657         if (ret)
2658                 return ret;
2659
2660         /* Currently applicable only to VLV */
2661         pte_flags = 0;
2662         if (vma->obj->gt_ro)
2663                 pte_flags |= PTE_READ_ONLY;
2664
2665
2666         if (flags & I915_VMA_GLOBAL_BIND) {
2667                 intel_runtime_pm_get(i915);
2668                 vma->vm->insert_entries(vma->vm,
2669                                         vma->pages, vma->node.start,
2670                                         cache_level, pte_flags);
2671                 intel_runtime_pm_put(i915);
2672         }
2673
2674         if (flags & I915_VMA_LOCAL_BIND) {
2675                 struct i915_hw_ppgtt *appgtt = i915->mm.aliasing_ppgtt;
2676                 appgtt->base.insert_entries(&appgtt->base,
2677                                             vma->pages, vma->node.start,
2678                                             cache_level, pte_flags);
2679         }
2680
2681         return 0;
2682 }
2683
2684 static void ggtt_unbind_vma(struct i915_vma *vma)
2685 {
2686         struct drm_i915_private *i915 = vma->vm->i915;
2687         struct i915_hw_ppgtt *appgtt = i915->mm.aliasing_ppgtt;
2688         const u64 size = min(vma->size, vma->node.size);
2689
2690         if (vma->flags & I915_VMA_GLOBAL_BIND) {
2691                 intel_runtime_pm_get(i915);
2692                 vma->vm->clear_range(vma->vm,
2693                                      vma->node.start, size);
2694                 intel_runtime_pm_put(i915);
2695         }
2696
2697         if (vma->flags & I915_VMA_LOCAL_BIND && appgtt)
2698                 appgtt->base.clear_range(&appgtt->base,
2699                                          vma->node.start, size);
2700 }
2701
2702 void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj,
2703                                struct sg_table *pages)
2704 {
2705         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2706         struct device *kdev = &dev_priv->drm.pdev->dev;
2707         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2708
2709         if (unlikely(ggtt->do_idle_maps)) {
2710                 if (i915_gem_wait_for_idle(dev_priv, I915_WAIT_LOCKED)) {
2711                         DRM_ERROR("Failed to wait for idle; VT'd may hang.\n");
2712                         /* Wait a bit, in hopes it avoids the hang */
2713                         udelay(10);
2714                 }
2715         }
2716
2717         dma_unmap_sg(kdev, pages->sgl, pages->nents, PCI_DMA_BIDIRECTIONAL);
2718 }
2719
2720 static void i915_gtt_color_adjust(const struct drm_mm_node *node,
2721                                   unsigned long color,
2722                                   u64 *start,
2723                                   u64 *end)
2724 {
2725         if (node->color != color)
2726                 *start += 4096;
2727
2728         node = list_next_entry(node, node_list);
2729         if (node->allocated && node->color != color)
2730                 *end -= 4096;
2731 }
2732
2733 int i915_gem_init_ggtt(struct drm_i915_private *dev_priv)
2734 {
2735         /* Let GEM Manage all of the aperture.
2736          *
2737          * However, leave one page at the end still bound to the scratch page.
2738          * There are a number of places where the hardware apparently prefetches
2739          * past the end of the object, and we've seen multiple hangs with the
2740          * GPU head pointer stuck in a batchbuffer bound at the last page of the
2741          * aperture.  One page should be enough to keep any prefetching inside
2742          * of the aperture.
2743          */
2744         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2745         unsigned long hole_start, hole_end;
2746         struct i915_hw_ppgtt *ppgtt;
2747         struct drm_mm_node *entry;
2748         int ret;
2749
2750         ret = intel_vgt_balloon(dev_priv);
2751         if (ret)
2752                 return ret;
2753
2754         /* Reserve a mappable slot for our lockless error capture */
2755         ret = drm_mm_insert_node_in_range_generic(&ggtt->base.mm,
2756                                                   &ggtt->error_capture,
2757                                                   4096, 0,
2758                                                   I915_COLOR_UNEVICTABLE,
2759                                                   0, ggtt->mappable_end,
2760                                                   0, 0);
2761         if (ret)
2762                 return ret;
2763
2764         /* Clear any non-preallocated blocks */
2765         drm_mm_for_each_hole(entry, &ggtt->base.mm, hole_start, hole_end) {
2766                 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
2767                               hole_start, hole_end);
2768                 ggtt->base.clear_range(&ggtt->base, hole_start,
2769                                        hole_end - hole_start);
2770         }
2771
2772         /* And finally clear the reserved guard page */
2773         ggtt->base.clear_range(&ggtt->base,
2774                                ggtt->base.total - PAGE_SIZE, PAGE_SIZE);
2775
2776         if (USES_PPGTT(dev_priv) && !USES_FULL_PPGTT(dev_priv)) {
2777                 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
2778                 if (!ppgtt) {
2779                         ret = -ENOMEM;
2780                         goto err;
2781                 }
2782
2783                 ret = __hw_ppgtt_init(ppgtt, dev_priv);
2784                 if (ret)
2785                         goto err_ppgtt;
2786
2787                 if (ppgtt->base.allocate_va_range) {
2788                         ret = ppgtt->base.allocate_va_range(&ppgtt->base, 0,
2789                                                             ppgtt->base.total);
2790                         if (ret)
2791                                 goto err_ppgtt_cleanup;
2792                 }
2793
2794                 ppgtt->base.clear_range(&ppgtt->base,
2795                                         ppgtt->base.start,
2796                                         ppgtt->base.total);
2797
2798                 dev_priv->mm.aliasing_ppgtt = ppgtt;
2799                 WARN_ON(ggtt->base.bind_vma != ggtt_bind_vma);
2800                 ggtt->base.bind_vma = aliasing_gtt_bind_vma;
2801         }
2802
2803         return 0;
2804
2805 err_ppgtt_cleanup:
2806         ppgtt->base.cleanup(&ppgtt->base);
2807 err_ppgtt:
2808         kfree(ppgtt);
2809 err:
2810         drm_mm_remove_node(&ggtt->error_capture);
2811         return ret;
2812 }
2813
2814 /**
2815  * i915_ggtt_cleanup_hw - Clean up GGTT hardware initialization
2816  * @dev_priv: i915 device
2817  */
2818 void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv)
2819 {
2820         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2821
2822         if (dev_priv->mm.aliasing_ppgtt) {
2823                 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
2824                 ppgtt->base.cleanup(&ppgtt->base);
2825                 kfree(ppgtt);
2826         }
2827
2828         i915_gem_cleanup_stolen(&dev_priv->drm);
2829
2830         if (drm_mm_node_allocated(&ggtt->error_capture))
2831                 drm_mm_remove_node(&ggtt->error_capture);
2832
2833         if (drm_mm_initialized(&ggtt->base.mm)) {
2834                 intel_vgt_deballoon(dev_priv);
2835
2836                 mutex_lock(&dev_priv->drm.struct_mutex);
2837                 i915_address_space_fini(&ggtt->base);
2838                 mutex_unlock(&dev_priv->drm.struct_mutex);
2839         }
2840
2841         ggtt->base.cleanup(&ggtt->base);
2842
2843         arch_phys_wc_del(ggtt->mtrr);
2844         io_mapping_fini(&ggtt->mappable);
2845 }
2846
2847 static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
2848 {
2849         snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
2850         snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
2851         return snb_gmch_ctl << 20;
2852 }
2853
2854 static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
2855 {
2856         bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
2857         bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
2858         if (bdw_gmch_ctl)
2859                 bdw_gmch_ctl = 1 << bdw_gmch_ctl;
2860
2861 #ifdef CONFIG_X86_32
2862         /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * PAGE_SIZE */
2863         if (bdw_gmch_ctl > 4)
2864                 bdw_gmch_ctl = 4;
2865 #endif
2866
2867         return bdw_gmch_ctl << 20;
2868 }
2869
2870 static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
2871 {
2872         gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
2873         gmch_ctrl &= SNB_GMCH_GGMS_MASK;
2874
2875         if (gmch_ctrl)
2876                 return 1 << (20 + gmch_ctrl);
2877
2878         return 0;
2879 }
2880
2881 static size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
2882 {
2883         snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
2884         snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
2885         return snb_gmch_ctl << 25; /* 32 MB units */
2886 }
2887
2888 static size_t gen8_get_stolen_size(u16 bdw_gmch_ctl)
2889 {
2890         bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
2891         bdw_gmch_ctl &= BDW_GMCH_GMS_MASK;
2892         return bdw_gmch_ctl << 25; /* 32 MB units */
2893 }
2894
2895 static size_t chv_get_stolen_size(u16 gmch_ctrl)
2896 {
2897         gmch_ctrl >>= SNB_GMCH_GMS_SHIFT;
2898         gmch_ctrl &= SNB_GMCH_GMS_MASK;
2899
2900         /*
2901          * 0x0  to 0x10: 32MB increments starting at 0MB
2902          * 0x11 to 0x16: 4MB increments starting at 8MB
2903          * 0x17 to 0x1d: 4MB increments start at 36MB
2904          */
2905         if (gmch_ctrl < 0x11)
2906                 return gmch_ctrl << 25;
2907         else if (gmch_ctrl < 0x17)
2908                 return (gmch_ctrl - 0x11 + 2) << 22;
2909         else
2910                 return (gmch_ctrl - 0x17 + 9) << 22;
2911 }
2912
2913 static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl)
2914 {
2915         gen9_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
2916         gen9_gmch_ctl &= BDW_GMCH_GMS_MASK;
2917
2918         if (gen9_gmch_ctl < 0xf0)
2919                 return gen9_gmch_ctl << 25; /* 32 MB units */
2920         else
2921                 /* 4MB increments starting at 0xf0 for 4MB */
2922                 return (gen9_gmch_ctl - 0xf0 + 1) << 22;
2923 }
2924
2925 static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
2926 {
2927         struct drm_i915_private *dev_priv = ggtt->base.i915;
2928         struct pci_dev *pdev = dev_priv->drm.pdev;
2929         phys_addr_t phys_addr;
2930         int ret;
2931
2932         /* For Modern GENs the PTEs and register space are split in the BAR */
2933         phys_addr = pci_resource_start(pdev, 0) + pci_resource_len(pdev, 0) / 2;
2934
2935         /*
2936          * On BXT writes larger than 64 bit to the GTT pagetable range will be
2937          * dropped. For WC mappings in general we have 64 byte burst writes
2938          * when the WC buffer is flushed, so we can't use it, but have to
2939          * resort to an uncached mapping. The WC issue is easily caught by the
2940          * readback check when writing GTT PTE entries.
2941          */
2942         if (IS_GEN9_LP(dev_priv))
2943                 ggtt->gsm = ioremap_nocache(phys_addr, size);
2944         else
2945                 ggtt->gsm = ioremap_wc(phys_addr, size);
2946         if (!ggtt->gsm) {
2947                 DRM_ERROR("Failed to map the ggtt page table\n");
2948                 return -ENOMEM;
2949         }
2950
2951         ret = setup_scratch_page(dev_priv, &ggtt->base.scratch_page, GFP_DMA32);
2952         if (ret) {
2953                 DRM_ERROR("Scratch setup failed\n");
2954                 /* iounmap will also get called at remove, but meh */
2955                 iounmap(ggtt->gsm);
2956                 return ret;
2957         }
2958
2959         return 0;
2960 }
2961
2962 /* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
2963  * bits. When using advanced contexts each context stores its own PAT, but
2964  * writing this data shouldn't be harmful even in those cases. */
2965 static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv)
2966 {
2967         uint64_t pat;
2968
2969         pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC)     | /* for normal objects, no eLLC */
2970               GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */
2971               GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */
2972               GEN8_PPAT(3, GEN8_PPAT_UC)                     | /* Uncached objects, mostly for scanout */
2973               GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) |
2974               GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) |
2975               GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) |
2976               GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
2977
2978         if (!USES_PPGTT(dev_priv))
2979                 /* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry,
2980                  * so RTL will always use the value corresponding to
2981                  * pat_sel = 000".
2982                  * So let's disable cache for GGTT to avoid screen corruptions.
2983                  * MOCS still can be used though.
2984                  * - System agent ggtt writes (i.e. cpu gtt mmaps) already work
2985                  * before this patch, i.e. the same uncached + snooping access
2986                  * like on gen6/7 seems to be in effect.
2987                  * - So this just fixes blitter/render access. Again it looks
2988                  * like it's not just uncached access, but uncached + snooping.
2989                  * So we can still hold onto all our assumptions wrt cpu
2990                  * clflushing on LLC machines.
2991                  */
2992                 pat = GEN8_PPAT(0, GEN8_PPAT_UC);
2993
2994         /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b
2995          * write would work. */
2996         I915_WRITE(GEN8_PRIVATE_PAT_LO, pat);
2997         I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32);
2998 }
2999
3000 static void chv_setup_private_ppat(struct drm_i915_private *dev_priv)
3001 {
3002         uint64_t pat;
3003
3004         /*
3005          * Map WB on BDW to snooped on CHV.
3006          *
3007          * Only the snoop bit has meaning for CHV, the rest is
3008          * ignored.
3009          *
3010          * The hardware will never snoop for certain types of accesses:
3011          * - CPU GTT (GMADR->GGTT->no snoop->memory)
3012          * - PPGTT page tables
3013          * - some other special cycles
3014          *
3015          * As with BDW, we also need to consider the following for GT accesses:
3016          * "For GGTT, there is NO pat_sel[2:0] from the entry,
3017          * so RTL will always use the value corresponding to
3018          * pat_sel = 000".
3019          * Which means we must set the snoop bit in PAT entry 0
3020          * in order to keep the global status page working.
3021          */
3022         pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) |
3023               GEN8_PPAT(1, 0) |
3024               GEN8_PPAT(2, 0) |
3025               GEN8_PPAT(3, 0) |
3026               GEN8_PPAT(4, CHV_PPAT_SNOOP) |
3027               GEN8_PPAT(5, CHV_PPAT_SNOOP) |
3028               GEN8_PPAT(6, CHV_PPAT_SNOOP) |
3029               GEN8_PPAT(7, CHV_PPAT_SNOOP);
3030
3031         I915_WRITE(GEN8_PRIVATE_PAT_LO, pat);
3032         I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32);
3033 }
3034
3035 static void gen6_gmch_remove(struct i915_address_space *vm)
3036 {
3037         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
3038
3039         iounmap(ggtt->gsm);
3040         cleanup_scratch_page(vm->i915, &vm->scratch_page);
3041 }
3042
3043 static int gen8_gmch_probe(struct i915_ggtt *ggtt)
3044 {
3045         struct drm_i915_private *dev_priv = ggtt->base.i915;
3046         struct pci_dev *pdev = dev_priv->drm.pdev;
3047         unsigned int size;
3048         u16 snb_gmch_ctl;
3049
3050         /* TODO: We're not aware of mappable constraints on gen8 yet */
3051         ggtt->mappable_base = pci_resource_start(pdev, 2);
3052         ggtt->mappable_end = pci_resource_len(pdev, 2);
3053
3054         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(39)))
3055                 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(39));
3056
3057         pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
3058
3059         if (INTEL_GEN(dev_priv) >= 9) {
3060                 ggtt->stolen_size = gen9_get_stolen_size(snb_gmch_ctl);
3061                 size = gen8_get_total_gtt_size(snb_gmch_ctl);
3062         } else if (IS_CHERRYVIEW(dev_priv)) {
3063                 ggtt->stolen_size = chv_get_stolen_size(snb_gmch_ctl);
3064                 size = chv_get_total_gtt_size(snb_gmch_ctl);
3065         } else {
3066                 ggtt->stolen_size = gen8_get_stolen_size(snb_gmch_ctl);
3067                 size = gen8_get_total_gtt_size(snb_gmch_ctl);
3068         }
3069
3070         ggtt->base.total = (size / sizeof(gen8_pte_t)) << PAGE_SHIFT;
3071
3072         if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv))
3073                 chv_setup_private_ppat(dev_priv);
3074         else
3075                 bdw_setup_private_ppat(dev_priv);
3076
3077         ggtt->base.cleanup = gen6_gmch_remove;
3078         ggtt->base.bind_vma = ggtt_bind_vma;
3079         ggtt->base.unbind_vma = ggtt_unbind_vma;
3080         ggtt->base.insert_page = gen8_ggtt_insert_page;
3081         ggtt->base.clear_range = nop_clear_range;
3082         if (!USES_FULL_PPGTT(dev_priv) || intel_scanout_needs_vtd_wa(dev_priv))
3083                 ggtt->base.clear_range = gen8_ggtt_clear_range;
3084
3085         ggtt->base.insert_entries = gen8_ggtt_insert_entries;
3086         if (IS_CHERRYVIEW(dev_priv))
3087                 ggtt->base.insert_entries = gen8_ggtt_insert_entries__BKL;
3088
3089         return ggtt_probe_common(ggtt, size);
3090 }
3091
3092 static int gen6_gmch_probe(struct i915_ggtt *ggtt)
3093 {
3094         struct drm_i915_private *dev_priv = ggtt->base.i915;
3095         struct pci_dev *pdev = dev_priv->drm.pdev;
3096         unsigned int size;
3097         u16 snb_gmch_ctl;
3098
3099         ggtt->mappable_base = pci_resource_start(pdev, 2);
3100         ggtt->mappable_end = pci_resource_len(pdev, 2);
3101
3102         /* 64/512MB is the current min/max we actually know of, but this is just
3103          * a coarse sanity check.
3104          */
3105         if (ggtt->mappable_end < (64<<20) || ggtt->mappable_end > (512<<20)) {
3106                 DRM_ERROR("Unknown GMADR size (%llx)\n", ggtt->mappable_end);
3107                 return -ENXIO;
3108         }
3109
3110         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(40)))
3111                 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(40));
3112         pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
3113
3114         ggtt->stolen_size = gen6_get_stolen_size(snb_gmch_ctl);
3115
3116         size = gen6_get_total_gtt_size(snb_gmch_ctl);
3117         ggtt->base.total = (size / sizeof(gen6_pte_t)) << PAGE_SHIFT;
3118
3119         ggtt->base.clear_range = gen6_ggtt_clear_range;
3120         ggtt->base.insert_page = gen6_ggtt_insert_page;
3121         ggtt->base.insert_entries = gen6_ggtt_insert_entries;
3122         ggtt->base.bind_vma = ggtt_bind_vma;
3123         ggtt->base.unbind_vma = ggtt_unbind_vma;
3124         ggtt->base.cleanup = gen6_gmch_remove;
3125
3126         if (HAS_EDRAM(dev_priv))
3127                 ggtt->base.pte_encode = iris_pte_encode;
3128         else if (IS_HASWELL(dev_priv))
3129                 ggtt->base.pte_encode = hsw_pte_encode;
3130         else if (IS_VALLEYVIEW(dev_priv))
3131                 ggtt->base.pte_encode = byt_pte_encode;
3132         else if (INTEL_GEN(dev_priv) >= 7)
3133                 ggtt->base.pte_encode = ivb_pte_encode;
3134         else
3135                 ggtt->base.pte_encode = snb_pte_encode;
3136
3137         return ggtt_probe_common(ggtt, size);
3138 }
3139
3140 static void i915_gmch_remove(struct i915_address_space *vm)
3141 {
3142         intel_gmch_remove();
3143 }
3144
3145 static int i915_gmch_probe(struct i915_ggtt *ggtt)
3146 {
3147         struct drm_i915_private *dev_priv = ggtt->base.i915;
3148         int ret;
3149
3150         ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->drm.pdev, NULL);
3151         if (!ret) {
3152                 DRM_ERROR("failed to set up gmch\n");
3153                 return -EIO;
3154         }
3155
3156         intel_gtt_get(&ggtt->base.total,
3157                       &ggtt->stolen_size,
3158                       &ggtt->mappable_base,
3159                       &ggtt->mappable_end);
3160
3161         ggtt->do_idle_maps = needs_idle_maps(dev_priv);
3162         ggtt->base.insert_page = i915_ggtt_insert_page;
3163         ggtt->base.insert_entries = i915_ggtt_insert_entries;
3164         ggtt->base.clear_range = i915_ggtt_clear_range;
3165         ggtt->base.bind_vma = ggtt_bind_vma;
3166         ggtt->base.unbind_vma = ggtt_unbind_vma;
3167         ggtt->base.cleanup = i915_gmch_remove;
3168
3169         if (unlikely(ggtt->do_idle_maps))
3170                 DRM_INFO("applying Ironlake quirks for intel_iommu\n");
3171
3172         return 0;
3173 }
3174
3175 /**
3176  * i915_ggtt_probe_hw - Probe GGTT hardware location
3177  * @dev_priv: i915 device
3178  */
3179 int i915_ggtt_probe_hw(struct drm_i915_private *dev_priv)
3180 {
3181         struct i915_ggtt *ggtt = &dev_priv->ggtt;
3182         int ret;
3183
3184         ggtt->base.i915 = dev_priv;
3185
3186         if (INTEL_GEN(dev_priv) <= 5)
3187                 ret = i915_gmch_probe(ggtt);
3188         else if (INTEL_GEN(dev_priv) < 8)
3189                 ret = gen6_gmch_probe(ggtt);
3190         else
3191                 ret = gen8_gmch_probe(ggtt);
3192         if (ret)
3193                 return ret;
3194
3195         /* Trim the GGTT to fit the GuC mappable upper range (when enabled).
3196          * This is easier than doing range restriction on the fly, as we
3197          * currently don't have any bits spare to pass in this upper
3198          * restriction!
3199          */
3200         if (HAS_GUC(dev_priv) && i915.enable_guc_loading) {
3201                 ggtt->base.total = min_t(u64, ggtt->base.total, GUC_GGTT_TOP);
3202                 ggtt->mappable_end = min(ggtt->mappable_end, ggtt->base.total);
3203         }
3204
3205         if ((ggtt->base.total - 1) >> 32) {
3206                 DRM_ERROR("We never expected a Global GTT with more than 32bits"
3207                           " of address space! Found %lldM!\n",
3208                           ggtt->base.total >> 20);
3209                 ggtt->base.total = 1ULL << 32;
3210                 ggtt->mappable_end = min(ggtt->mappable_end, ggtt->base.total);
3211         }
3212
3213         if (ggtt->mappable_end > ggtt->base.total) {
3214                 DRM_ERROR("mappable aperture extends past end of GGTT,"
3215                           " aperture=%llx, total=%llx\n",
3216                           ggtt->mappable_end, ggtt->base.total);
3217                 ggtt->mappable_end = ggtt->base.total;
3218         }
3219
3220         /* GMADR is the PCI mmio aperture into the global GTT. */
3221         DRM_INFO("Memory usable by graphics device = %lluM\n",
3222                  ggtt->base.total >> 20);
3223         DRM_DEBUG_DRIVER("GMADR size = %lldM\n", ggtt->mappable_end >> 20);
3224         DRM_DEBUG_DRIVER("GTT stolen size = %uM\n", ggtt->stolen_size >> 20);
3225 #ifdef CONFIG_INTEL_IOMMU
3226         if (intel_iommu_gfx_mapped)
3227                 DRM_INFO("VT-d active for gfx access\n");
3228 #endif
3229
3230         return 0;
3231 }
3232
3233 /**
3234  * i915_ggtt_init_hw - Initialize GGTT hardware
3235  * @dev_priv: i915 device
3236  */
3237 int i915_ggtt_init_hw(struct drm_i915_private *dev_priv)
3238 {
3239         struct i915_ggtt *ggtt = &dev_priv->ggtt;
3240         int ret;
3241
3242         INIT_LIST_HEAD(&dev_priv->vm_list);
3243
3244         /* Subtract the guard page before address space initialization to
3245          * shrink the range used by drm_mm.
3246          */
3247         mutex_lock(&dev_priv->drm.struct_mutex);
3248         ggtt->base.total -= PAGE_SIZE;
3249         i915_address_space_init(&ggtt->base, dev_priv, "[global]");
3250         ggtt->base.total += PAGE_SIZE;
3251         if (!HAS_LLC(dev_priv))
3252                 ggtt->base.mm.color_adjust = i915_gtt_color_adjust;
3253         mutex_unlock(&dev_priv->drm.struct_mutex);
3254
3255         if (!io_mapping_init_wc(&dev_priv->ggtt.mappable,
3256                                 dev_priv->ggtt.mappable_base,
3257                                 dev_priv->ggtt.mappable_end)) {
3258                 ret = -EIO;
3259                 goto out_gtt_cleanup;
3260         }
3261
3262         ggtt->mtrr = arch_phys_wc_add(ggtt->mappable_base, ggtt->mappable_end);
3263
3264         /*
3265          * Initialise stolen early so that we may reserve preallocated
3266          * objects for the BIOS to KMS transition.
3267          */
3268         ret = i915_gem_init_stolen(dev_priv);
3269         if (ret)
3270                 goto out_gtt_cleanup;
3271
3272         return 0;
3273
3274 out_gtt_cleanup:
3275         ggtt->base.cleanup(&ggtt->base);
3276         return ret;
3277 }
3278
3279 int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv)
3280 {
3281         if (INTEL_GEN(dev_priv) < 6 && !intel_enable_gtt())
3282                 return -EIO;
3283
3284         return 0;
3285 }
3286
3287 void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv)
3288 {
3289         struct i915_ggtt *ggtt = &dev_priv->ggtt;
3290         struct drm_i915_gem_object *obj, *on;
3291
3292         i915_check_and_clear_faults(dev_priv);
3293
3294         /* First fill our portion of the GTT with scratch pages */
3295         ggtt->base.clear_range(&ggtt->base, ggtt->base.start, ggtt->base.total);
3296
3297         ggtt->base.closed = true; /* skip rewriting PTE on VMA unbind */
3298
3299         /* clflush objects bound into the GGTT and rebind them. */
3300         list_for_each_entry_safe(obj, on,
3301                                  &dev_priv->mm.bound_list, global_link) {
3302                 bool ggtt_bound = false;
3303                 struct i915_vma *vma;
3304
3305                 list_for_each_entry(vma, &obj->vma_list, obj_link) {
3306                         if (vma->vm != &ggtt->base)
3307                                 continue;
3308
3309                         if (!i915_vma_unbind(vma))
3310                                 continue;
3311
3312                         WARN_ON(i915_vma_bind(vma, obj->cache_level,
3313                                               PIN_UPDATE));
3314                         ggtt_bound = true;
3315                 }
3316
3317                 if (ggtt_bound)
3318                         WARN_ON(i915_gem_object_set_to_gtt_domain(obj, false));
3319         }
3320
3321         ggtt->base.closed = false;
3322
3323         if (INTEL_GEN(dev_priv) >= 8) {
3324                 if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv))
3325                         chv_setup_private_ppat(dev_priv);
3326                 else
3327                         bdw_setup_private_ppat(dev_priv);
3328
3329                 return;
3330         }
3331
3332         if (USES_PPGTT(dev_priv)) {
3333                 struct i915_address_space *vm;
3334
3335                 list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
3336                         /* TODO: Perhaps it shouldn't be gen6 specific */
3337
3338                         struct i915_hw_ppgtt *ppgtt;
3339
3340                         if (i915_is_ggtt(vm))
3341                                 ppgtt = dev_priv->mm.aliasing_ppgtt;
3342                         else
3343                                 ppgtt = i915_vm_to_ppgtt(vm);
3344
3345                         gen6_write_page_range(dev_priv, &ppgtt->pd,
3346                                               0, ppgtt->base.total);
3347                 }
3348         }
3349
3350         i915_ggtt_flush(dev_priv);
3351 }
3352
3353 struct i915_vma *
3354 i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
3355                     struct i915_address_space *vm,
3356                     const struct i915_ggtt_view *view)
3357 {
3358         struct rb_node *rb;
3359
3360         rb = obj->vma_tree.rb_node;
3361         while (rb) {
3362                 struct i915_vma *vma = rb_entry(rb, struct i915_vma, obj_node);
3363                 long cmp;
3364
3365                 cmp = i915_vma_compare(vma, vm, view);
3366                 if (cmp == 0)
3367                         return vma;
3368
3369                 if (cmp < 0)
3370                         rb = rb->rb_right;
3371                 else
3372                         rb = rb->rb_left;
3373         }
3374
3375         return NULL;
3376 }
3377
3378 struct i915_vma *
3379 i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
3380                                   struct i915_address_space *vm,
3381                                   const struct i915_ggtt_view *view)
3382 {
3383         struct i915_vma *vma;
3384
3385         lockdep_assert_held(&obj->base.dev->struct_mutex);
3386         GEM_BUG_ON(view && !i915_is_ggtt(vm));
3387
3388         vma = i915_gem_obj_to_vma(obj, vm, view);
3389         if (!vma) {
3390                 vma = i915_vma_create(obj, vm, view);
3391                 GEM_BUG_ON(vma != i915_gem_obj_to_vma(obj, vm, view));
3392         }
3393
3394         GEM_BUG_ON(i915_vma_is_closed(vma));
3395         return vma;
3396 }
3397
3398 static struct scatterlist *
3399 rotate_pages(const dma_addr_t *in, unsigned int offset,
3400              unsigned int width, unsigned int height,
3401              unsigned int stride,
3402              struct sg_table *st, struct scatterlist *sg)
3403 {
3404         unsigned int column, row;
3405         unsigned int src_idx;
3406
3407         for (column = 0; column < width; column++) {
3408                 src_idx = stride * (height - 1) + column;
3409                 for (row = 0; row < height; row++) {
3410                         st->nents++;
3411                         /* We don't need the pages, but need to initialize
3412                          * the entries so the sg list can be happily traversed.
3413                          * The only thing we need are DMA addresses.
3414                          */
3415                         sg_set_page(sg, NULL, PAGE_SIZE, 0);
3416                         sg_dma_address(sg) = in[offset + src_idx];
3417                         sg_dma_len(sg) = PAGE_SIZE;
3418                         sg = sg_next(sg);
3419                         src_idx -= stride;
3420                 }
3421         }
3422
3423         return sg;
3424 }
3425
3426 static struct sg_table *
3427 intel_rotate_fb_obj_pages(const struct intel_rotation_info *rot_info,
3428                           struct drm_i915_gem_object *obj)
3429 {
3430         const size_t n_pages = obj->base.size / PAGE_SIZE;
3431         unsigned int size = intel_rotation_info_size(rot_info);
3432         struct sgt_iter sgt_iter;
3433         dma_addr_t dma_addr;
3434         unsigned long i;
3435         dma_addr_t *page_addr_list;
3436         struct sg_table *st;
3437         struct scatterlist *sg;
3438         int ret = -ENOMEM;
3439
3440         /* Allocate a temporary list of source pages for random access. */
3441         page_addr_list = drm_malloc_gfp(n_pages,
3442                                         sizeof(dma_addr_t),
3443                                         GFP_TEMPORARY);
3444         if (!page_addr_list)
3445                 return ERR_PTR(ret);
3446
3447         /* Allocate target SG list. */
3448         st = kmalloc(sizeof(*st), GFP_KERNEL);
3449         if (!st)
3450                 goto err_st_alloc;
3451
3452         ret = sg_alloc_table(st, size, GFP_KERNEL);
3453         if (ret)
3454                 goto err_sg_alloc;
3455
3456         /* Populate source page list from the object. */
3457         i = 0;
3458         for_each_sgt_dma(dma_addr, sgt_iter, obj->mm.pages)
3459                 page_addr_list[i++] = dma_addr;
3460
3461         GEM_BUG_ON(i != n_pages);
3462         st->nents = 0;
3463         sg = st->sgl;
3464
3465         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++) {
3466                 sg = rotate_pages(page_addr_list, rot_info->plane[i].offset,
3467                                   rot_info->plane[i].width, rot_info->plane[i].height,
3468                                   rot_info->plane[i].stride, st, sg);
3469         }
3470
3471         DRM_DEBUG_KMS("Created rotated page mapping for object size %zu (%ux%u tiles, %u pages)\n",
3472                       obj->base.size, rot_info->plane[0].width, rot_info->plane[0].height, size);
3473
3474         drm_free_large(page_addr_list);
3475
3476         return st;
3477
3478 err_sg_alloc:
3479         kfree(st);
3480 err_st_alloc:
3481         drm_free_large(page_addr_list);
3482
3483         DRM_DEBUG_KMS("Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n",
3484                       obj->base.size, rot_info->plane[0].width, rot_info->plane[0].height, size);
3485
3486         return ERR_PTR(ret);
3487 }
3488
3489 static struct sg_table *
3490 intel_partial_pages(const struct i915_ggtt_view *view,
3491                     struct drm_i915_gem_object *obj)
3492 {
3493         struct sg_table *st;
3494         struct scatterlist *sg, *iter;
3495         unsigned int count = view->params.partial.size;
3496         unsigned int offset;
3497         int ret = -ENOMEM;
3498
3499         st = kmalloc(sizeof(*st), GFP_KERNEL);
3500         if (!st)
3501                 goto err_st_alloc;
3502
3503         ret = sg_alloc_table(st, count, GFP_KERNEL);
3504         if (ret)
3505                 goto err_sg_alloc;
3506
3507         iter = i915_gem_object_get_sg(obj,
3508                                       view->params.partial.offset,
3509                                       &offset);
3510         GEM_BUG_ON(!iter);
3511
3512         sg = st->sgl;
3513         st->nents = 0;
3514         do {
3515                 unsigned int len;
3516
3517                 len = min(iter->length - (offset << PAGE_SHIFT),
3518                           count << PAGE_SHIFT);
3519                 sg_set_page(sg, NULL, len, 0);
3520                 sg_dma_address(sg) =
3521                         sg_dma_address(iter) + (offset << PAGE_SHIFT);
3522                 sg_dma_len(sg) = len;
3523
3524                 st->nents++;
3525                 count -= len >> PAGE_SHIFT;
3526                 if (count == 0) {
3527                         sg_mark_end(sg);
3528                         return st;
3529                 }
3530
3531                 sg = __sg_next(sg);
3532                 iter = __sg_next(iter);
3533                 offset = 0;
3534         } while (1);
3535
3536 err_sg_alloc:
3537         kfree(st);
3538 err_st_alloc:
3539         return ERR_PTR(ret);
3540 }
3541
3542 static int
3543 i915_get_ggtt_vma_pages(struct i915_vma *vma)
3544 {
3545         int ret = 0;
3546
3547         /* The vma->pages are only valid within the lifespan of the borrowed
3548          * obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so
3549          * must be the vma->pages. A simple rule is that vma->pages must only
3550          * be accessed when the obj->mm.pages are pinned.
3551          */
3552         GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj));
3553
3554         if (vma->pages)
3555                 return 0;
3556
3557         if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL)
3558                 vma->pages = vma->obj->mm.pages;
3559         else if (vma->ggtt_view.type == I915_GGTT_VIEW_ROTATED)
3560                 vma->pages =
3561                         intel_rotate_fb_obj_pages(&vma->ggtt_view.params.rotated, vma->obj);
3562         else if (vma->ggtt_view.type == I915_GGTT_VIEW_PARTIAL)
3563                 vma->pages = intel_partial_pages(&vma->ggtt_view, vma->obj);
3564         else
3565                 WARN_ONCE(1, "GGTT view %u not implemented!\n",
3566                           vma->ggtt_view.type);
3567
3568         if (!vma->pages) {
3569                 DRM_ERROR("Failed to get pages for GGTT view type %u!\n",
3570                           vma->ggtt_view.type);
3571                 ret = -EINVAL;
3572         } else if (IS_ERR(vma->pages)) {
3573                 ret = PTR_ERR(vma->pages);
3574                 vma->pages = NULL;
3575                 DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n",
3576                           vma->ggtt_view.type, ret);
3577         }
3578
3579         return ret;
3580 }
3581