drivers/gpu/drm/i915/gt/intel_ggtt.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2020 Intel Corporation
   4  */
   5
   6 #include <linux/types.h>
   7 #include <asm/set_memory.h>
   8 #include <asm/smp.h>
   9
  10 #include <drm/i915_drm.h>
  11
  12 #include "gem/i915_gem_lmem.h"
  13
  14 #include "intel_gt.h"
  15 #include "intel_gt_gmch.h"
  16 #include "intel_gt_regs.h"
  17 #include "i915_drv.h"
  18 #include "i915_scatterlist.h"
  19 #include "i915_utils.h"
  20 #include "i915_vgpu.h"
  21
  22 #include "intel_gtt.h"
  23 #include "gen8_ppgtt.h"
  24
  25 static void i915_ggtt_color_adjust(const struct drm_mm_node *node,
  26                                    unsigned long color,
  27                                    u64 *start,
  28                                    u64 *end)
  29 {
  30         if (i915_node_color_differs(node, color))
  31                 *start += I915_GTT_PAGE_SIZE;
  32
  33         /*
  34          * Also leave a space between the unallocated reserved node after the
  35          * GTT and any objects within the GTT, i.e. we use the color adjustment
  36          * to insert a guard page to prevent prefetches crossing over the
  37          * GTT boundary.
  38          */
  39         node = list_next_entry(node, node_list);
  40         if (node->color != color)
  41                 *end -= I915_GTT_PAGE_SIZE;
  42 }
  43
  44 static int ggtt_init_hw(struct i915_ggtt *ggtt)
  45 {
  46         struct drm_i915_private *i915 = ggtt->vm.i915;
  47
  48         i915_address_space_init(&ggtt->vm, VM_CLASS_GGTT);
  49
  50         ggtt->vm.is_ggtt = true;
  51
  52         /* Only VLV supports read-only GGTT mappings */
  53         ggtt->vm.has_read_only = IS_VALLEYVIEW(i915);
  54
  55         if (!HAS_LLC(i915) && !HAS_PPGTT(i915))
  56                 ggtt->vm.mm.color_adjust = i915_ggtt_color_adjust;
  57
  58         if (ggtt->mappable_end) {
  59                 if (!io_mapping_init_wc(&ggtt->iomap,
  60                                         ggtt->gmadr.start,
  61                                         ggtt->mappable_end)) {
  62                         ggtt->vm.cleanup(&ggtt->vm);
  63                         return -EIO;
  64                 }
  65
  66                 ggtt->mtrr = arch_phys_wc_add(ggtt->gmadr.start,
  67                                               ggtt->mappable_end);
  68         }
  69
  70         intel_ggtt_init_fences(ggtt);
  71
  72         return 0;
  73 }
  74
  75 /**
  76  * i915_ggtt_init_hw - Initialize GGTT hardware
  77  * @i915: i915 device
  78  */
  79 int i915_ggtt_init_hw(struct drm_i915_private *i915)
  80 {
  81         int ret;
  82
  83         /*
  84          * Note that we use page colouring to enforce a guard page at the
  85          * end of the address space. This is required as the CS may prefetch
  86          * beyond the end of the batch buffer, across the page boundary,
  87          * and beyond the end of the GTT if we do not provide a guard.
  88          */
  89         ret = ggtt_init_hw(to_gt(i915)->ggtt);
  90         if (ret)
  91                 return ret;
  92
  93         return 0;
  94 }
  95
  96 /**
  97  * i915_ggtt_suspend_vm - Suspend the memory mappings for a GGTT or DPT VM
  98  * @vm: The VM to suspend the mappings for
  99  *
 100  * Suspend the memory mappings for all objects mapped to HW via the GGTT or a
 101  * DPT page table.
 102  */
 103 void i915_ggtt_suspend_vm(struct i915_address_space *vm)
 104 {
 105         struct i915_vma *vma, *vn;
 106         int save_skip_rewrite;
 107
 108         drm_WARN_ON(&vm->i915->drm, !vm->is_ggtt && !vm->is_dpt);
 109
 110 retry:
 111         i915_gem_drain_freed_objects(vm->i915);
 112
 113         mutex_lock(&vm->mutex);
 114
 115         /*
 116          * Skip rewriting PTE on VMA unbind.
 117          * FIXME: Use an argument to i915_vma_unbind() instead?
 118          */
 119         save_skip_rewrite = vm->skip_pte_rewrite;
 120         vm->skip_pte_rewrite = true;
 121
 122         list_for_each_entry_safe(vma, vn, &vm->bound_list, vm_link) {
 123                 struct drm_i915_gem_object *obj = vma->obj;
 124
 125                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
 126
 127                 if (i915_vma_is_pinned(vma) || !i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
 128                         continue;
 129
 130                 /* unlikely to race when GPU is idle, so no worry about slowpath.. */
 131                 if (WARN_ON(!i915_gem_object_trylock(obj, NULL))) {
 132                         /*
 133                          * No dead objects should appear here, GPU should be
 134                          * completely idle, and userspace suspended
 135                          */
 136                         i915_gem_object_get(obj);
 137
 138                         mutex_unlock(&vm->mutex);
 139
 140                         i915_gem_object_lock(obj, NULL);
 141                         GEM_WARN_ON(i915_vma_unbind(vma));
 142                         i915_gem_object_unlock(obj);
 143                         i915_gem_object_put(obj);
 144
 145                         vm->skip_pte_rewrite = save_skip_rewrite;
 146                         goto retry;
 147                 }
 148
 149                 if (!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) {
 150                         i915_vma_wait_for_bind(vma);
 151
 152                         __i915_vma_evict(vma, false);
 153                         drm_mm_remove_node(&vma->node);
 154                 }
 155
 156                 i915_gem_object_unlock(obj);
 157         }
 158
 159         vm->clear_range(vm, 0, vm->total);
 160
 161         vm->skip_pte_rewrite = save_skip_rewrite;
 162
 163         mutex_unlock(&vm->mutex);
 164 }
 165
 166 void i915_ggtt_suspend(struct i915_ggtt *ggtt)
 167 {
 168         i915_ggtt_suspend_vm(&ggtt->vm);
 169         ggtt->invalidate(ggtt);
 170
 171         intel_gt_check_and_clear_faults(ggtt->vm.gt);
 172 }
 173
 174 void gen6_ggtt_invalidate(struct i915_ggtt *ggtt)
 175 {
 176         struct intel_uncore *uncore = ggtt->vm.gt->uncore;
 177
 178         spin_lock_irq(&uncore->lock);
 179         intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
 180         intel_uncore_read_fw(uncore, GFX_FLSH_CNTL_GEN6);
 181         spin_unlock_irq(&uncore->lock);
 182 }
 183
 184 void gen8_ggtt_invalidate(struct i915_ggtt *ggtt)
 185 {
 186         struct intel_uncore *uncore = ggtt->vm.gt->uncore;
 187
 188         /*
 189          * Note that as an uncached mmio write, this will flush the
 190          * WCB of the writes into the GGTT before it triggers the invalidate.
 191          */
 192         intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
 193 }
 194
 195 static void guc_ggtt_invalidate(struct i915_ggtt *ggtt)
 196 {
 197         struct intel_uncore *uncore = ggtt->vm.gt->uncore;
 198         struct drm_i915_private *i915 = ggtt->vm.i915;
 199
 200         gen8_ggtt_invalidate(ggtt);
 201
 202         if (GRAPHICS_VER(i915) >= 12)
 203                 intel_uncore_write_fw(uncore, GEN12_GUC_TLB_INV_CR,
 204                                       GEN12_GUC_TLB_INV_CR_INVALIDATE);
 205         else
 206                 intel_uncore_write_fw(uncore, GEN8_GTCR, GEN8_GTCR_INVALIDATE);
 207 }
 208
 209 u64 gen8_ggtt_pte_encode(dma_addr_t addr,
 210                          enum i915_cache_level level,
 211                          u32 flags)
 212 {
 213         gen8_pte_t pte = addr | GEN8_PAGE_PRESENT;
 214
 215         if (flags & PTE_LM)
 216                 pte |= GEN12_GGTT_PTE_LM;
 217
 218         return pte;
 219 }
 220
 221 void intel_ggtt_bind_vma(struct i915_address_space *vm,
 222                           struct i915_vm_pt_stash *stash,
 223                           struct i915_vma_resource *vma_res,
 224                           enum i915_cache_level cache_level,
 225                           u32 flags)
 226 {
 227         u32 pte_flags;
 228
 229         if (vma_res->bound_flags & (~flags & I915_VMA_BIND_MASK))
 230                 return;
 231
 232         vma_res->bound_flags |= flags;
 233
 234         /* Applicable to VLV (gen8+ do not support RO in the GGTT) */
 235         pte_flags = 0;
 236         if (vma_res->bi.readonly)
 237                 pte_flags |= PTE_READ_ONLY;
 238         if (vma_res->bi.lmem)
 239                 pte_flags |= PTE_LM;
 240
 241         vm->insert_entries(vm, vma_res, cache_level, pte_flags);
 242         vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE;
 243 }
 244
 245 void intel_ggtt_unbind_vma(struct i915_address_space *vm,
 246                             struct i915_vma_resource *vma_res)
 247 {
 248         vm->clear_range(vm, vma_res->start, vma_res->vma_size);
 249 }
 250
 251 static int ggtt_reserve_guc_top(struct i915_ggtt *ggtt)
 252 {
 253         u64 size;
 254         int ret;
 255
 256         if (!intel_uc_uses_guc(&ggtt->vm.gt->uc))
 257                 return 0;
 258
 259         GEM_BUG_ON(ggtt->vm.total <= GUC_GGTT_TOP);
 260         size = ggtt->vm.total - GUC_GGTT_TOP;
 261
 262         ret = i915_gem_gtt_reserve(&ggtt->vm, NULL, &ggtt->uc_fw, size,
 263                                    GUC_GGTT_TOP, I915_COLOR_UNEVICTABLE,
 264                                    PIN_NOEVICT);
 265         if (ret)
 266                 drm_dbg(&ggtt->vm.i915->drm,
 267                         "Failed to reserve top of GGTT for GuC\n");
 268
 269         return ret;
 270 }
 271
 272 static void ggtt_release_guc_top(struct i915_ggtt *ggtt)
 273 {
 274         if (drm_mm_node_allocated(&ggtt->uc_fw))
 275                 drm_mm_remove_node(&ggtt->uc_fw);
 276 }
 277
 278 static void cleanup_init_ggtt(struct i915_ggtt *ggtt)
 279 {
 280         ggtt_release_guc_top(ggtt);
 281         if (drm_mm_node_allocated(&ggtt->error_capture))
 282                 drm_mm_remove_node(&ggtt->error_capture);
 283         mutex_destroy(&ggtt->error_mutex);
 284 }
 285
 286 static int init_ggtt(struct i915_ggtt *ggtt)
 287 {
 288         /*
 289          * Let GEM Manage all of the aperture.
 290          *
 291          * However, leave one page at the end still bound to the scratch page.
 292          * There are a number of places where the hardware apparently prefetches
 293          * past the end of the object, and we've seen multiple hangs with the
 294          * GPU head pointer stuck in a batchbuffer bound at the last page of the
 295          * aperture.  One page should be enough to keep any prefetching inside
 296          * of the aperture.
 297          */
 298         unsigned long hole_start, hole_end;
 299         struct drm_mm_node *entry;
 300         int ret;
 301
 302         /*
 303          * GuC requires all resources that we're sharing with it to be placed in
 304          * non-WOPCM memory. If GuC is not present or not in use we still need a
 305          * small bias as ring wraparound at offset 0 sometimes hangs. No idea
 306          * why.
 307          */
 308         ggtt->pin_bias = max_t(u32, I915_GTT_PAGE_SIZE,
 309                                intel_wopcm_guc_size(&ggtt->vm.i915->wopcm));
 310
 311         ret = intel_vgt_balloon(ggtt);
 312         if (ret)
 313                 return ret;
 314
 315         mutex_init(&ggtt->error_mutex);
 316         if (ggtt->mappable_end) {
 317                 /*
 318                  * Reserve a mappable slot for our lockless error capture.
 319                  *
 320                  * We strongly prefer taking address 0x0 in order to protect
 321                  * other critical buffers against accidental overwrites,
 322                  * as writing to address 0 is a very common mistake.
 323                  *
 324                  * Since 0 may already be in use by the system (e.g. the BIOS
 325                  * framebuffer), we let the reservation fail quietly and hope
 326                  * 0 remains reserved always.
 327                  *
 328                  * If we fail to reserve 0, and then fail to find any space
 329                  * for an error-capture, remain silent. We can afford not
 330                  * to reserve an error_capture node as we have fallback
 331                  * paths, and we trust that 0 will remain reserved. However,
 332                  * the only likely reason for failure to insert is a driver
 333                  * bug, which we expect to cause other failures...
 334                  */
 335                 ggtt->error_capture.size = I915_GTT_PAGE_SIZE;
 336                 ggtt->error_capture.color = I915_COLOR_UNEVICTABLE;
 337                 if (drm_mm_reserve_node(&ggtt->vm.mm, &ggtt->error_capture))
 338                         drm_mm_insert_node_in_range(&ggtt->vm.mm,
 339                                                     &ggtt->error_capture,
 340                                                     ggtt->error_capture.size, 0,
 341                                                     ggtt->error_capture.color,
 342                                                     0, ggtt->mappable_end,
 343                                                     DRM_MM_INSERT_LOW);
 344         }
 345         if (drm_mm_node_allocated(&ggtt->error_capture))
 346                 drm_dbg(&ggtt->vm.i915->drm,
 347                         "Reserved GGTT:[%llx, %llx] for use by error capture\n",
 348                         ggtt->error_capture.start,
 349                         ggtt->error_capture.start + ggtt->error_capture.size);
 350
 351         /*
 352          * The upper portion of the GuC address space has a sizeable hole
 353          * (several MB) that is inaccessible by GuC. Reserve this range within
 354          * GGTT as it can comfortably hold GuC/HuC firmware images.
 355          */
 356         ret = ggtt_reserve_guc_top(ggtt);
 357         if (ret)
 358                 goto err;
 359
 360         /* Clear any non-preallocated blocks */
 361         drm_mm_for_each_hole(entry, &ggtt->vm.mm, hole_start, hole_end) {
 362                 drm_dbg(&ggtt->vm.i915->drm,
 363                         "clearing unused GTT space: [%lx, %lx]\n",
 364                         hole_start, hole_end);
 365                 ggtt->vm.clear_range(&ggtt->vm, hole_start,
 366                                      hole_end - hole_start);
 367         }
 368
 369         /* And finally clear the reserved guard page */
 370         ggtt->vm.clear_range(&ggtt->vm, ggtt->vm.total - PAGE_SIZE, PAGE_SIZE);
 371
 372         return 0;
 373
 374 err:
 375         cleanup_init_ggtt(ggtt);
 376         return ret;
 377 }
 378
 379 static void aliasing_gtt_bind_vma(struct i915_address_space *vm,
 380                                   struct i915_vm_pt_stash *stash,
 381                                   struct i915_vma_resource *vma_res,
 382                                   enum i915_cache_level cache_level,
 383                                   u32 flags)
 384 {
 385         u32 pte_flags;
 386
 387         /* Currently applicable only to VLV */
 388         pte_flags = 0;
 389         if (vma_res->bi.readonly)
 390                 pte_flags |= PTE_READ_ONLY;
 391
 392         if (flags & I915_VMA_LOCAL_BIND)
 393                 ppgtt_bind_vma(&i915_vm_to_ggtt(vm)->alias->vm,
 394                                stash, vma_res, cache_level, flags);
 395
 396         if (flags & I915_VMA_GLOBAL_BIND)
 397                 vm->insert_entries(vm, vma_res, cache_level, pte_flags);
 398
 399         vma_res->bound_flags |= flags;
 400 }
 401
 402 static void aliasing_gtt_unbind_vma(struct i915_address_space *vm,
 403                                     struct i915_vma_resource *vma_res)
 404 {
 405         if (vma_res->bound_flags & I915_VMA_GLOBAL_BIND)
 406                 vm->clear_range(vm, vma_res->start, vma_res->vma_size);
 407
 408         if (vma_res->bound_flags & I915_VMA_LOCAL_BIND)
 409                 ppgtt_unbind_vma(&i915_vm_to_ggtt(vm)->alias->vm, vma_res);
 410 }
 411
 412 static int init_aliasing_ppgtt(struct i915_ggtt *ggtt)
 413 {
 414         struct i915_vm_pt_stash stash = {};
 415         struct i915_ppgtt *ppgtt;
 416         int err;
 417
 418         ppgtt = i915_ppgtt_create(ggtt->vm.gt, 0);
 419         if (IS_ERR(ppgtt))
 420                 return PTR_ERR(ppgtt);
 421
 422         if (GEM_WARN_ON(ppgtt->vm.total < ggtt->vm.total)) {
 423                 err = -ENODEV;
 424                 goto err_ppgtt;
 425         }
 426
 427         err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, ggtt->vm.total);
 428         if (err)
 429                 goto err_ppgtt;
 430
 431         i915_gem_object_lock(ppgtt->vm.scratch[0], NULL);
 432         err = i915_vm_map_pt_stash(&ppgtt->vm, &stash);
 433         i915_gem_object_unlock(ppgtt->vm.scratch[0]);
 434         if (err)
 435                 goto err_stash;
 436
 437         /*
 438          * Note we only pre-allocate as far as the end of the global
 439          * GTT. On 48b / 4-level page-tables, the difference is very,
 440          * very significant! We have to preallocate as GVT/vgpu does
 441          * not like the page directory disappearing.
 442          */
 443         ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, ggtt->vm.total);
 444
 445         ggtt->alias = ppgtt;
 446         ggtt->vm.bind_async_flags |= ppgtt->vm.bind_async_flags;
 447
 448         GEM_BUG_ON(ggtt->vm.vma_ops.bind_vma != intel_ggtt_bind_vma);
 449         ggtt->vm.vma_ops.bind_vma = aliasing_gtt_bind_vma;
 450
 451         GEM_BUG_ON(ggtt->vm.vma_ops.unbind_vma != intel_ggtt_unbind_vma);
 452         ggtt->vm.vma_ops.unbind_vma = aliasing_gtt_unbind_vma;
 453
 454         i915_vm_free_pt_stash(&ppgtt->vm, &stash);
 455         return 0;
 456
 457 err_stash:
 458         i915_vm_free_pt_stash(&ppgtt->vm, &stash);
 459 err_ppgtt:
 460         i915_vm_put(&ppgtt->vm);
 461         return err;
 462 }
 463
 464 static void fini_aliasing_ppgtt(struct i915_ggtt *ggtt)
 465 {
 466         struct i915_ppgtt *ppgtt;
 467
 468         ppgtt = fetch_and_zero(&ggtt->alias);
 469         if (!ppgtt)
 470                 return;
 471
 472         i915_vm_put(&ppgtt->vm);
 473
 474         ggtt->vm.vma_ops.bind_vma   = intel_ggtt_bind_vma;
 475         ggtt->vm.vma_ops.unbind_vma = intel_ggtt_unbind_vma;
 476 }
 477
 478 int i915_init_ggtt(struct drm_i915_private *i915)
 479 {
 480         int ret;
 481
 482         ret = init_ggtt(to_gt(i915)->ggtt);
 483         if (ret)
 484                 return ret;
 485
 486         if (INTEL_PPGTT(i915) == INTEL_PPGTT_ALIASING) {
 487                 ret = init_aliasing_ppgtt(to_gt(i915)->ggtt);
 488                 if (ret)
 489                         cleanup_init_ggtt(to_gt(i915)->ggtt);
 490         }
 491
 492         return 0;
 493 }
 494
 495 static void ggtt_cleanup_hw(struct i915_ggtt *ggtt)
 496 {
 497         struct i915_vma *vma, *vn;
 498
 499         flush_workqueue(ggtt->vm.i915->wq);
 500         i915_gem_drain_freed_objects(ggtt->vm.i915);
 501
 502         mutex_lock(&ggtt->vm.mutex);
 503
 504         ggtt->vm.skip_pte_rewrite = true;
 505
 506         list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) {
 507                 struct drm_i915_gem_object *obj = vma->obj;
 508                 bool trylock;
 509
 510                 trylock = i915_gem_object_trylock(obj, NULL);
 511                 WARN_ON(!trylock);
 512
 513                 WARN_ON(__i915_vma_unbind(vma));
 514                 if (trylock)
 515                         i915_gem_object_unlock(obj);
 516         }
 517
 518         if (drm_mm_node_allocated(&ggtt->error_capture))
 519                 drm_mm_remove_node(&ggtt->error_capture);
 520         mutex_destroy(&ggtt->error_mutex);
 521
 522         ggtt_release_guc_top(ggtt);
 523         intel_vgt_deballoon(ggtt);
 524
 525         ggtt->vm.cleanup(&ggtt->vm);
 526
 527         mutex_unlock(&ggtt->vm.mutex);
 528         i915_address_space_fini(&ggtt->vm);
 529
 530         arch_phys_wc_del(ggtt->mtrr);
 531
 532         if (ggtt->iomap.size)
 533                 io_mapping_fini(&ggtt->iomap);
 534 }
 535
 536 /**
 537  * i915_ggtt_driver_release - Clean up GGTT hardware initialization
 538  * @i915: i915 device
 539  */
 540 void i915_ggtt_driver_release(struct drm_i915_private *i915)
 541 {
 542         struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 543
 544         fini_aliasing_ppgtt(ggtt);
 545
 546         intel_ggtt_fini_fences(ggtt);
 547         ggtt_cleanup_hw(ggtt);
 548 }
 549
 550 /**
 551  * i915_ggtt_driver_late_release - Cleanup of GGTT that needs to be done after
 552  * all free objects have been drained.
 553  * @i915: i915 device
 554  */
 555 void i915_ggtt_driver_late_release(struct drm_i915_private *i915)
 556 {
 557         struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 558
 559         GEM_WARN_ON(kref_read(&ggtt->vm.resv_ref) != 1);
 560         dma_resv_fini(&ggtt->vm._resv);
 561 }
 562
 563 struct resource intel_pci_resource(struct pci_dev *pdev, int bar)
 564 {
 565         return (struct resource)DEFINE_RES_MEM(pci_resource_start(pdev, bar),
 566                                                pci_resource_len(pdev, bar));
 567 }
 568
 569 static int ggtt_probe_hw(struct i915_ggtt *ggtt, struct intel_gt *gt)
 570 {
 571         struct drm_i915_private *i915 = gt->i915;
 572         int ret;
 573
 574         ggtt->vm.gt = gt;
 575         ggtt->vm.i915 = i915;
 576         ggtt->vm.dma = i915->drm.dev;
 577         dma_resv_init(&ggtt->vm._resv);
 578
 579         if (GRAPHICS_VER(i915) <= 5)
 580                 ret = intel_gt_gmch_gen5_probe(ggtt);
 581         else if (GRAPHICS_VER(i915) < 8)
 582                 ret = intel_gt_gmch_gen6_probe(ggtt);
 583         else
 584                 ret = intel_gt_gmch_gen8_probe(ggtt);
 585         if (ret) {
 586                 dma_resv_fini(&ggtt->vm._resv);
 587                 return ret;
 588         }
 589
 590         if ((ggtt->vm.total - 1) >> 32) {
 591                 drm_err(&i915->drm,
 592                         "We never expected a Global GTT with more than 32bits"
 593                         " of address space! Found %lldM!\n",
 594                         ggtt->vm.total >> 20);
 595                 ggtt->vm.total = 1ULL << 32;
 596                 ggtt->mappable_end =
 597                         min_t(u64, ggtt->mappable_end, ggtt->vm.total);
 598         }
 599
 600         if (ggtt->mappable_end > ggtt->vm.total) {
 601                 drm_err(&i915->drm,
 602                         "mappable aperture extends past end of GGTT,"
 603                         " aperture=%pa, total=%llx\n",
 604                         &ggtt->mappable_end, ggtt->vm.total);
 605                 ggtt->mappable_end = ggtt->vm.total;
 606         }
 607
 608         /* GMADR is the PCI mmio aperture into the global GTT. */
 609         drm_dbg(&i915->drm, "GGTT size = %lluM\n", ggtt->vm.total >> 20);
 610         drm_dbg(&i915->drm, "GMADR size = %lluM\n",
 611                 (u64)ggtt->mappable_end >> 20);
 612         drm_dbg(&i915->drm, "DSM size = %lluM\n",
 613                 (u64)resource_size(&intel_graphics_stolen_res) >> 20);
 614
 615         return 0;
 616 }
 617
 618 /**
 619  * i915_ggtt_probe_hw - Probe GGTT hardware location
 620  * @i915: i915 device
 621  */
 622 int i915_ggtt_probe_hw(struct drm_i915_private *i915)
 623 {
 624         int ret;
 625
 626         ret = ggtt_probe_hw(to_gt(i915)->ggtt, to_gt(i915));
 627         if (ret)
 628                 return ret;
 629
 630         if (i915_vtd_active(i915))
 631                 drm_info(&i915->drm, "VT-d active for gfx access\n");
 632
 633         return 0;
 634 }
 635
 636 int i915_ggtt_enable_hw(struct drm_i915_private *i915)
 637 {
 638         return intel_gt_gmch_gen5_enable_hw(i915);
 639 }
 640
 641 void i915_ggtt_enable_guc(struct i915_ggtt *ggtt)
 642 {
 643         GEM_BUG_ON(ggtt->invalidate != gen8_ggtt_invalidate);
 644
 645         ggtt->invalidate = guc_ggtt_invalidate;
 646
 647         ggtt->invalidate(ggtt);
 648 }
 649
 650 void i915_ggtt_disable_guc(struct i915_ggtt *ggtt)
 651 {
 652         /* XXX Temporary pardon for error unload */
 653         if (ggtt->invalidate == gen8_ggtt_invalidate)
 654                 return;
 655
 656         /* We should only be called after i915_ggtt_enable_guc() */
 657         GEM_BUG_ON(ggtt->invalidate != guc_ggtt_invalidate);
 658
 659         ggtt->invalidate = gen8_ggtt_invalidate;
 660
 661         ggtt->invalidate(ggtt);
 662 }
 663
 664 /**
 665  * i915_ggtt_resume_vm - Restore the memory mappings for a GGTT or DPT VM
 666  * @vm: The VM to restore the mappings for
 667  *
 668  * Restore the memory mappings for all objects mapped to HW via the GGTT or a
 669  * DPT page table.
 670  *
 671  * Returns %true if restoring the mapping for any object that was in a write
 672  * domain before suspend.
 673  */
 674 bool i915_ggtt_resume_vm(struct i915_address_space *vm)
 675 {
 676         struct i915_vma *vma;
 677         bool write_domain_objs = false;
 678
 679         drm_WARN_ON(&vm->i915->drm, !vm->is_ggtt && !vm->is_dpt);
 680
 681         /* First fill our portion of the GTT with scratch pages */
 682         vm->clear_range(vm, 0, vm->total);
 683
 684         /* clflush objects bound into the GGTT and rebind them. */
 685         list_for_each_entry(vma, &vm->bound_list, vm_link) {
 686                 struct drm_i915_gem_object *obj = vma->obj;
 687                 unsigned int was_bound =
 688                         atomic_read(&vma->flags) & I915_VMA_BIND_MASK;
 689
 690                 GEM_BUG_ON(!was_bound);
 691                 vma->ops->bind_vma(vm, NULL, vma->resource,
 692                                    obj ? obj->cache_level : 0,
 693                                    was_bound);
 694                 if (obj) { /* only used during resume => exclusive access */
 695                         write_domain_objs |= fetch_and_zero(&obj->write_domain);
 696                         obj->read_domains |= I915_GEM_DOMAIN_GTT;
 697                 }
 698         }
 699
 700         return write_domain_objs;
 701 }
 702
 703 void i915_ggtt_resume(struct i915_ggtt *ggtt)
 704 {
 705         bool flush;
 706
 707         intel_gt_check_and_clear_faults(ggtt->vm.gt);
 708
 709         flush = i915_ggtt_resume_vm(&ggtt->vm);
 710
 711         ggtt->invalidate(ggtt);
 712
 713         if (flush)
 714                 wbinvd_on_all_cpus();
 715
 716         if (GRAPHICS_VER(ggtt->vm.i915) >= 8)
 717                 setup_private_pat(ggtt->vm.gt->uncore);
 718
 719         intel_ggtt_restore_fences(ggtt);
 720 }