projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915: Set GAC_ECO_BITS register on Gen7+
[linux-2.6-block.git] / drivers / gpu / drm / i915 / i915_gem_gtt.c
CommitLineData
76aaf220
DV		 1/*
2 * Copyright © 2010 Daniel Vetter
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
760285e7
DH		 25#include <drm/drmP.h>
26#include <drm/i915_drm.h>
76aaf220
DV		 27#include "i915_drv.h"
28#include "i915_trace.h"
29#include "intel_drv.h"
30
e7c2b58b 31typedef uint32_t gen6_gtt_pte_t;
f61c0609 32
26b1ff35
BW		 33/* PPGTT stuff */
34#define GEN6_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0xff0))
35
36#define GEN6_PDE_VALID (1 << 0)
37/* gen6+ has bit 11-4 for physical addr bit 39-32 */
38#define GEN6_PDE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr)
39
40#define GEN6_PTE_VALID (1 << 0)
41#define GEN6_PTE_UNCACHED (1 << 1)
42#define HSW_PTE_UNCACHED (0)
43#define GEN6_PTE_CACHE_LLC (2 << 1)
44#define GEN6_PTE_CACHE_LLC_MLC (3 << 1)
45#define GEN6_PTE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr)
46
e7c2b58b 47static inline gen6_gtt_pte_t gen6_pte_encode(struct drm_device *dev,
c81dbe05
BW		 48 dma_addr_t addr,
49 enum i915_cache_level level)
54d12527 50{
e7c2b58b 51 gen6_gtt_pte_t pte = GEN6_PTE_VALID;
54d12527 52 pte |= GEN6_PTE_ADDR_ENCODE(addr);
e7210c3c
BW		 53
54 switch (level) {
55 case I915_CACHE_LLC_MLC:
56 /* Haswell doesn't set L3 this way */
57 if (IS_HASWELL(dev))
58 pte |= GEN6_PTE_CACHE_LLC;
59 else
60 pte |= GEN6_PTE_CACHE_LLC_MLC;
61 break;
62 case I915_CACHE_LLC:
63 pte |= GEN6_PTE_CACHE_LLC;
64 break;
65 case I915_CACHE_NONE:
66 if (IS_HASWELL(dev))
67 pte |= HSW_PTE_UNCACHED;
68 else
69 pte |= GEN6_PTE_UNCACHED;
70 break;
71 default:
72 BUG();
73 }
74
54d12527
BW		 75 return pte;
76}
77
b7c36d25 78static int gen6_ppgtt_enable(struct drm_device *dev)
6197349b
BW		 79{
80 drm_i915_private_t *dev_priv = dev->dev_private;
81 uint32_t pd_offset;
82 struct intel_ring_buffer *ring;
83 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
84 gen6_gtt_pte_t __iomem *pd_addr;
85 uint32_t pd_entry;
86 int i;
87
88 pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm +
89 ppgtt->pd_offset / sizeof(gen6_gtt_pte_t);
90 for (i = 0; i < ppgtt->num_pd_entries; i++) {
91 dma_addr_t pt_addr;
92
93 pt_addr = ppgtt->pt_dma_addr[i];
94 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
95 pd_entry |= GEN6_PDE_VALID;
96
97 writel(pd_entry, pd_addr + i);
98 }
99 readl(pd_addr);
100
101 pd_offset = ppgtt->pd_offset;
102 pd_offset /= 64; /* in cachelines, */
103 pd_offset <<= 16;
104
105 if (INTEL_INFO(dev)->gen == 6) {
106 uint32_t ecochk, gab_ctl, ecobits;
107
108 ecobits = I915_READ(GAC_ECO_BITS);
3b9d7888
VS		 109 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
110 ECOBITS_PPGTT_CACHE64B);
6197349b
BW		 111
112 gab_ctl = I915_READ(GAB_CTL);
113 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
114
115 ecochk = I915_READ(GAM_ECOCHK);
116 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT |
117 ECOCHK_PPGTT_CACHE64B);
118 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
119 } else if (INTEL_INFO(dev)->gen >= 7) {
a65c2fcd
VS		 120 uint32_t ecobits;
121
122 ecobits = I915_READ(GAC_ECO_BITS);
123 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
124
6197349b
BW		 125 I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B);
126 /* GFX_MODE is per-ring on gen7+ */
127 }
128
129 for_each_ring(ring, dev_priv, i) {
130 if (INTEL_INFO(dev)->gen >= 7)
131 I915_WRITE(RING_MODE_GEN7(ring),
132 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
133
134 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
135 I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
136 }
b7c36d25 137 return 0;
6197349b
BW		 138}
139
1d2a314c 140/* PPGTT support for Sandybdrige/Gen6 and later */
def886c3 141static void gen6_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt,
1d2a314c
DV		 142 unsigned first_entry,
143 unsigned num_entries)
144{
e7c2b58b 145 gen6_gtt_pte_t *pt_vaddr, scratch_pte;
a15326a5 146 unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
7bddb01f
DV		 147 unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
148 unsigned last_pte, i;
1d2a314c 149
960e3e42
DV		 150 scratch_pte = gen6_pte_encode(ppgtt->dev,
151 ppgtt->scratch_page_dma_addr,
152 I915_CACHE_LLC);
1d2a314c 153
7bddb01f
DV		 154 while (num_entries) {
155 last_pte = first_pte + num_entries;
156 if (last_pte > I915_PPGTT_PT_ENTRIES)
157 last_pte = I915_PPGTT_PT_ENTRIES;
158
a15326a5 159 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pt]);
1d2a314c 160
7bddb01f
DV		 161 for (i = first_pte; i < last_pte; i++)
162 pt_vaddr[i] = scratch_pte;
1d2a314c
DV		 163
164 kunmap_atomic(pt_vaddr);
1d2a314c 165
7bddb01f
DV		 166 num_entries -= last_pte - first_pte;
167 first_pte = 0;
a15326a5 168 act_pt++;
7bddb01f 169 }
1d2a314c
DV		 170}
171
def886c3
DV		 172static void gen6_ppgtt_insert_entries(struct i915_hw_ppgtt *ppgtt,
173 struct sg_table *pages,
174 unsigned first_entry,
175 enum i915_cache_level cache_level)
176{
e7c2b58b 177 gen6_gtt_pte_t *pt_vaddr;
a15326a5 178 unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
6e995e23
ID		 179 unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES;
180 struct sg_page_iter sg_iter;
181
a15326a5 182 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pt]);
6e995e23
ID		 183 for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
184 dma_addr_t page_addr;
185
2db76d7c 186 page_addr = sg_page_iter_dma_address(&sg_iter);
6e995e23 187 pt_vaddr[act_pte] = gen6_pte_encode(ppgtt->dev, page_addr,
6ddc4fc7 188 cache_level);
6e995e23
ID		 189 if (++act_pte == I915_PPGTT_PT_ENTRIES) {
190 kunmap_atomic(pt_vaddr);
a15326a5
DV		 191 act_pt++;
192 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pt]);
6e995e23 193 act_pte = 0;
def886c3 194
def886c3 195 }
def886c3 196 }
6e995e23 197 kunmap_atomic(pt_vaddr);
def886c3
DV		 198}
199
3440d265 200static void gen6_ppgtt_cleanup(struct i915_hw_ppgtt *ppgtt)
1d2a314c 201{
3440d265
DV		 202 int i;
203
204 if (ppgtt->pt_dma_addr) {
205 for (i = 0; i < ppgtt->num_pd_entries; i++)
206 pci_unmap_page(ppgtt->dev->pdev,
207 ppgtt->pt_dma_addr[i],
208 4096, PCI_DMA_BIDIRECTIONAL);
209 }
210
211 kfree(ppgtt->pt_dma_addr);
212 for (i = 0; i < ppgtt->num_pd_entries; i++)
213 __free_page(ppgtt->pt_pages[i]);
214 kfree(ppgtt->pt_pages);
215 kfree(ppgtt);
216}
217
218static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
219{
220 struct drm_device *dev = ppgtt->dev;
1d2a314c 221 struct drm_i915_private *dev_priv = dev->dev_private;
1d2a314c 222 unsigned first_pd_entry_in_global_pt;
1d2a314c
DV		 223 int i;
224 int ret = -ENOMEM;
225
226 /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024
227 * entries. For aliasing ppgtt support we just steal them at the end for
228 * now. */
a54c0c27
BW		 229 first_pd_entry_in_global_pt =
230 gtt_total_entries(dev_priv->gtt) - I915_PPGTT_PD_ENTRIES;
1d2a314c 231
1d2a314c 232 ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES;
6197349b 233 ppgtt->enable = gen6_ppgtt_enable;
def886c3
DV		 234 ppgtt->clear_range = gen6_ppgtt_clear_range;
235 ppgtt->insert_entries = gen6_ppgtt_insert_entries;
3440d265 236 ppgtt->cleanup = gen6_ppgtt_cleanup;
1d2a314c
DV		 237 ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries,
238 GFP_KERNEL);
239 if (!ppgtt->pt_pages)
3440d265 240 return -ENOMEM;
1d2a314c
DV		 241
242 for (i = 0; i < ppgtt->num_pd_entries; i++) {
243 ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
244 if (!ppgtt->pt_pages[i])
245 goto err_pt_alloc;
246 }
247
8d2e6308
BW		 248 ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t) *ppgtt->num_pd_entries,
249 GFP_KERNEL);
250 if (!ppgtt->pt_dma_addr)
251 goto err_pt_alloc;
1d2a314c 252
8d2e6308
BW		 253 for (i = 0; i < ppgtt->num_pd_entries; i++) {
254 dma_addr_t pt_addr;
211c568b 255
8d2e6308
BW		 256 pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096,
257 PCI_DMA_BIDIRECTIONAL);
1d2a314c 258
8d2e6308
BW		 259 if (pci_dma_mapping_error(dev->pdev, pt_addr)) {
260 ret = -EIO;
261 goto err_pd_pin;
1d2a314c 262
211c568b 263 }
8d2e6308 264 ppgtt->pt_dma_addr[i] = pt_addr;
1d2a314c 265 }
1d2a314c 266
def886c3
DV		 267 ppgtt->clear_range(ppgtt, 0,
268 ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES);
1d2a314c 269
e7c2b58b 270 ppgtt->pd_offset = first_pd_entry_in_global_pt * sizeof(gen6_gtt_pte_t);
1d2a314c 271
1d2a314c
DV		 272 return 0;
273
274err_pd_pin:
275 if (ppgtt->pt_dma_addr) {
276 for (i--; i >= 0; i--)
277 pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
278 4096, PCI_DMA_BIDIRECTIONAL);
279 }
280err_pt_alloc:
281 kfree(ppgtt->pt_dma_addr);
282 for (i = 0; i < ppgtt->num_pd_entries; i++) {
283 if (ppgtt->pt_pages[i])
284 __free_page(ppgtt->pt_pages[i]);
285 }
286 kfree(ppgtt->pt_pages);
3440d265
DV		 287
288 return ret;
289}
290
291static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
292{
293 struct drm_i915_private *dev_priv = dev->dev_private;
294 struct i915_hw_ppgtt *ppgtt;
295 int ret;
296
297 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
298 if (!ppgtt)
299 return -ENOMEM;
300
301 ppgtt->dev = dev;
1e7d12d4 302 ppgtt->scratch_page_dma_addr = dev_priv->gtt.scratch_page_dma;
3440d265 303
3ed124b2
BW		 304 if (INTEL_INFO(dev)->gen < 8)
305 ret = gen6_ppgtt_init(ppgtt);
306 else
307 BUG();
308
3440d265
DV		 309 if (ret)
310 kfree(ppgtt);
311 else
312 dev_priv->mm.aliasing_ppgtt = ppgtt;
1d2a314c
DV		 313
314 return ret;
315}
316
317void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
318{
319 struct drm_i915_private *dev_priv = dev->dev_private;
320 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
1d2a314c
DV		 321
322 if (!ppgtt)
323 return;
324
3440d265 325 ppgtt->cleanup(ppgtt);
5963cf04 326 dev_priv->mm.aliasing_ppgtt = NULL;
1d2a314c
DV		 327}
328
7bddb01f
DV		 329void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
330 struct drm_i915_gem_object *obj,
331 enum i915_cache_level cache_level)
332{
def886c3
DV		 333 ppgtt->insert_entries(ppgtt, obj->pages,
334 obj->gtt_space->start >> PAGE_SHIFT,
335 cache_level);
7bddb01f
DV		 336}
337
338void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
339 struct drm_i915_gem_object *obj)
340{
def886c3
DV		 341 ppgtt->clear_range(ppgtt,
342 obj->gtt_space->start >> PAGE_SHIFT,
343 obj->base.size >> PAGE_SHIFT);
7bddb01f
DV		 344}
345
a81cc00c
BW		 346extern int intel_iommu_gfx_mapped;
347/* Certain Gen5 chipsets require require idling the GPU before
348 * unmapping anything from the GTT when VT-d is enabled.
349 */
350static inline bool needs_idle_maps(struct drm_device *dev)
351{
352#ifdef CONFIG_INTEL_IOMMU
353 /* Query intel_iommu to see if we need the workaround. Presumably that
354 * was loaded first.
355 */
356 if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped)
357 return true;
358#endif
359 return false;
360}
361
5c042287
BW		 362static bool do_idling(struct drm_i915_private *dev_priv)
363{
364 bool ret = dev_priv->mm.interruptible;
365
a81cc00c 366 if (unlikely(dev_priv->gtt.do_idle_maps)) {
5c042287 367 dev_priv->mm.interruptible = false;
b2da9fe5 368 if (i915_gpu_idle(dev_priv->dev)) {
5c042287
BW		 369 DRM_ERROR("Couldn't idle GPU\n");
370 /* Wait a bit, in hopes it avoids the hang */
371 udelay(10);
372 }
373 }
374
375 return ret;
376}
377
378static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
379{
a81cc00c 380 if (unlikely(dev_priv->gtt.do_idle_maps))
5c042287
BW		 381 dev_priv->mm.interruptible = interruptible;
382}
383
76aaf220
DV		 384void i915_gem_restore_gtt_mappings(struct drm_device *dev)
385{
386 struct drm_i915_private *dev_priv = dev->dev_private;
05394f39 387 struct drm_i915_gem_object *obj;
76aaf220 388
bee4a186 389 /* First fill our portion of the GTT with scratch pages */
7faf1ab2
DV		 390 dev_priv->gtt.gtt_clear_range(dev, dev_priv->gtt.start / PAGE_SIZE,
391 dev_priv->gtt.total / PAGE_SIZE);
bee4a186 392
6c085a72 393 list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) {
a8e93126 394 i915_gem_clflush_object(obj);
74163907 395 i915_gem_gtt_bind_object(obj, obj->cache_level);
76aaf220
DV		 396 }
397
e76e9aeb 398 i915_gem_chipset_flush(dev);
76aaf220 399}
7c2e6fdf 400
74163907 401int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
7c2e6fdf 402{
9da3da66 403 if (obj->has_dma_mapping)
74163907 404 return 0;
9da3da66
CW		 405
406 if (!dma_map_sg(&obj->base.dev->pdev->dev,
407 obj->pages->sgl, obj->pages->nents,
408 PCI_DMA_BIDIRECTIONAL))
409 return -ENOSPC;
410
411 return 0;
7c2e6fdf
DV		 412}
413
e76e9aeb
BW		 414/*
415 * Binds an object into the global gtt with the specified cache level. The object
416 * will be accessible to the GPU via commands whose operands reference offsets
417 * within the global GTT as well as accessible by the GPU through the GMADR
418 * mapped BAR (dev_priv->mm.gtt->gtt).
419 */
7faf1ab2
DV		 420static void gen6_ggtt_insert_entries(struct drm_device *dev,
421 struct sg_table *st,
422 unsigned int first_entry,
423 enum i915_cache_level level)
e76e9aeb 424{
e76e9aeb 425 struct drm_i915_private *dev_priv = dev->dev_private;
e7c2b58b
BW		 426 gen6_gtt_pte_t __iomem *gtt_entries =
427 (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
6e995e23
ID		 428 int i = 0;
429 struct sg_page_iter sg_iter;
e76e9aeb
BW		 430 dma_addr_t addr;
431
6e995e23 432 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
2db76d7c 433 addr = sg_page_iter_dma_address(&sg_iter);
6e995e23
ID		 434 iowrite32(gen6_pte_encode(dev, addr, level), &gtt_entries[i]);
435 i++;
e76e9aeb
BW		 436 }
437
e76e9aeb
BW		 438 /* XXX: This serves as a posting read to make sure that the PTE has
439 * actually been updated. There is some concern that even though
440 * registers and PTEs are within the same BAR that they are potentially
441 * of NUMA access patterns. Therefore, even with the way we assume
442 * hardware should work, we must keep this posting read for paranoia.
443 */
444 if (i != 0)
960e3e42
DV		 445 WARN_ON(readl(&gtt_entries[i-1])
446 != gen6_pte_encode(dev, addr, level));
0f9b91c7
BW		 447
448 /* This next bit makes the above posting read even more important. We
449 * want to flush the TLBs only after we're certain all the PTE updates
450 * have finished.
451 */
452 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
453 POSTING_READ(GFX_FLSH_CNTL_GEN6);
e76e9aeb
BW		 454}
455
7faf1ab2
DV		 456static void gen6_ggtt_clear_range(struct drm_device *dev,
457 unsigned int first_entry,
458 unsigned int num_entries)
459{
460 struct drm_i915_private *dev_priv = dev->dev_private;
e7c2b58b
BW		 461 gen6_gtt_pte_t scratch_pte, __iomem *gtt_base =
462 (gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
a54c0c27 463 const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
7faf1ab2
DV		 464 int i;
465
466 if (WARN(num_entries > max_entries,
467 "First entry = %d; Num entries = %d (max=%d)\n",
468 first_entry, num_entries, max_entries))
469 num_entries = max_entries;
470
960e3e42
DV		 471 scratch_pte = gen6_pte_encode(dev, dev_priv->gtt.scratch_page_dma,
472 I915_CACHE_LLC);
7faf1ab2
DV		 473 for (i = 0; i < num_entries; i++)
474 iowrite32(scratch_pte, &gtt_base[i]);
475 readl(gtt_base);
476}
477
478
479static void i915_ggtt_insert_entries(struct drm_device *dev,
480 struct sg_table *st,
481 unsigned int pg_start,
482 enum i915_cache_level cache_level)
483{
484 unsigned int flags = (cache_level == I915_CACHE_NONE) ?
485 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
486
487 intel_gtt_insert_sg_entries(st, pg_start, flags);
488
489}
490
491static void i915_ggtt_clear_range(struct drm_device *dev,
492 unsigned int first_entry,
493 unsigned int num_entries)
494{
495 intel_gtt_clear_range(first_entry, num_entries);
496}
497
498
74163907
DV		 499void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
500 enum i915_cache_level cache_level)
d5bd1449
CW		 501{
502 struct drm_device *dev = obj->base.dev;
7faf1ab2
DV		 503 struct drm_i915_private *dev_priv = dev->dev_private;
504
505 dev_priv->gtt.gtt_insert_entries(dev, obj->pages,
506 obj->gtt_space->start >> PAGE_SHIFT,
507 cache_level);
d5bd1449 508
74898d7e 509 obj->has_global_gtt_mapping = 1;
d5bd1449
CW		 510}
511
05394f39 512void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
74163907 513{
7faf1ab2
DV		 514 struct drm_device *dev = obj->base.dev;
515 struct drm_i915_private *dev_priv = dev->dev_private;
516
517 dev_priv->gtt.gtt_clear_range(obj->base.dev,
518 obj->gtt_space->start >> PAGE_SHIFT,
519 obj->base.size >> PAGE_SHIFT);
74898d7e
DV		 520
521 obj->has_global_gtt_mapping = 0;
74163907
DV		 522}
523
524void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
7c2e6fdf 525{
5c042287
BW		 526 struct drm_device *dev = obj->base.dev;
527 struct drm_i915_private *dev_priv = dev->dev_private;
528 bool interruptible;
529
530 interruptible = do_idling(dev_priv);
531
9da3da66
CW		 532 if (!obj->has_dma_mapping)
533 dma_unmap_sg(&dev->pdev->dev,
534 obj->pages->sgl, obj->pages->nents,
535 PCI_DMA_BIDIRECTIONAL);
5c042287
BW		 536
537 undo_idling(dev_priv, interruptible);
7c2e6fdf 538}
644ec02b 539
42d6ab48
CW		 540static void i915_gtt_color_adjust(struct drm_mm_node *node,
541 unsigned long color,
542 unsigned long *start,
543 unsigned long *end)
544{
545 if (node->color != color)
546 *start += 4096;
547
548 if (!list_empty(&node->node_list)) {
549 node = list_entry(node->node_list.next,
550 struct drm_mm_node,
551 node_list);
552 if (node->allocated && node->color != color)
553 *end -= 4096;
554 }
555}
d7e5008f
BW		 556void i915_gem_setup_global_gtt(struct drm_device *dev,
557 unsigned long start,
558 unsigned long mappable_end,
559 unsigned long end)
644ec02b 560{
e78891ca
BW		 561 /* Let GEM Manage all of the aperture.
562 *
563 * However, leave one page at the end still bound to the scratch page.
564 * There are a number of places where the hardware apparently prefetches
565 * past the end of the object, and we've seen multiple hangs with the
566 * GPU head pointer stuck in a batchbuffer bound at the last page of the
567 * aperture. One page should be enough to keep any prefetching inside
568 * of the aperture.
569 */
644ec02b 570 drm_i915_private_t *dev_priv = dev->dev_private;
ed2f3452
CW		 571 struct drm_mm_node *entry;
572 struct drm_i915_gem_object *obj;
573 unsigned long hole_start, hole_end;
644ec02b 574
35451cb6
BW		 575 BUG_ON(mappable_end > end);
576
ed2f3452 577 /* Subtract the guard page ... */
d1dd20a9 578 drm_mm_init(&dev_priv->mm.gtt_space, start, end - start - PAGE_SIZE);
42d6ab48
CW		 579 if (!HAS_LLC(dev))
580 dev_priv->mm.gtt_space.color_adjust = i915_gtt_color_adjust;
644ec02b 581
ed2f3452
CW		 582 /* Mark any preallocated objects as occupied */
583 list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) {
584 DRM_DEBUG_KMS("reserving preallocated space: %x + %zx\n",
585 obj->gtt_offset, obj->base.size);
586
587 BUG_ON(obj->gtt_space != I915_GTT_RESERVED);
588 obj->gtt_space = drm_mm_create_block(&dev_priv->mm.gtt_space,
589 obj->gtt_offset,
590 obj->base.size,
591 false);
592 obj->has_global_gtt_mapping = 1;
593 }
594
5d4545ae 595 dev_priv->gtt.start = start;
5d4545ae 596 dev_priv->gtt.total = end - start;
644ec02b 597
ed2f3452
CW		 598 /* Clear any non-preallocated blocks */
599 drm_mm_for_each_hole(entry, &dev_priv->mm.gtt_space,
600 hole_start, hole_end) {
601 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
602 hole_start, hole_end);
7faf1ab2
DV		 603 dev_priv->gtt.gtt_clear_range(dev, hole_start / PAGE_SIZE,
604 (hole_end-hole_start) / PAGE_SIZE);
ed2f3452
CW		 605 }
606
607 /* And finally clear the reserved guard page */
7faf1ab2 608 dev_priv->gtt.gtt_clear_range(dev, end / PAGE_SIZE - 1, 1);
e76e9aeb
BW		 609}
610
d7e5008f
BW		 611static bool
612intel_enable_ppgtt(struct drm_device *dev)
613{
614 if (i915_enable_ppgtt >= 0)
615 return i915_enable_ppgtt;
616
617#ifdef CONFIG_INTEL_IOMMU
618 /* Disable ppgtt on SNB if VT-d is on. */
619 if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
620 return false;
621#endif
622
623 return true;
624}
625
626void i915_gem_init_global_gtt(struct drm_device *dev)
627{
628 struct drm_i915_private *dev_priv = dev->dev_private;
629 unsigned long gtt_size, mappable_size;
d7e5008f 630
a54c0c27 631 gtt_size = dev_priv->gtt.total;
93d18799 632 mappable_size = dev_priv->gtt.mappable_end;
d7e5008f
BW		 633
634 if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
e78891ca 635 int ret;
3eb1c005
BW		 636
637 if (INTEL_INFO(dev)->gen <= 7) {
638 /* PPGTT pdes are stolen from global gtt ptes, so shrink the
639 * aperture accordingly when using aliasing ppgtt. */
640 gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
641 }
d7e5008f
BW		 642
643 i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
644
645 ret = i915_gem_init_aliasing_ppgtt(dev);
e78891ca 646 if (!ret)
d7e5008f 647 return;
e78891ca
BW		 648
649 DRM_ERROR("Aliased PPGTT setup failed %d\n", ret);
650 drm_mm_takedown(&dev_priv->mm.gtt_space);
651 gtt_size += I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
d7e5008f 652 }
e78891ca 653 i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
e76e9aeb
BW		 654}
655
656static int setup_scratch_page(struct drm_device *dev)
657{
658 struct drm_i915_private *dev_priv = dev->dev_private;
659 struct page *page;
660 dma_addr_t dma_addr;
661
662 page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
663 if (page == NULL)
664 return -ENOMEM;
665 get_page(page);
666 set_pages_uc(page, 1);
667
668#ifdef CONFIG_INTEL_IOMMU
669 dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE,
670 PCI_DMA_BIDIRECTIONAL);
671 if (pci_dma_mapping_error(dev->pdev, dma_addr))
672 return -EINVAL;
673#else
674 dma_addr = page_to_phys(page);
675#endif
9c61a32d
BW		 676 dev_priv->gtt.scratch_page = page;
677 dev_priv->gtt.scratch_page_dma = dma_addr;
e76e9aeb
BW		 678
679 return 0;
680}
681
682static void teardown_scratch_page(struct drm_device *dev)
683{
684 struct drm_i915_private *dev_priv = dev->dev_private;
9c61a32d
BW		 685 set_pages_wb(dev_priv->gtt.scratch_page, 1);
686 pci_unmap_page(dev->pdev, dev_priv->gtt.scratch_page_dma,
e76e9aeb 687 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
9c61a32d
BW		 688 put_page(dev_priv->gtt.scratch_page);
689 __free_page(dev_priv->gtt.scratch_page);
e76e9aeb
BW		 690}
691
692static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
693{
694 snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
695 snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
696 return snb_gmch_ctl << 20;
697}
698
baa09f5f 699static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
e76e9aeb
BW		 700{
701 snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
702 snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
703 return snb_gmch_ctl << 25; /* 32 MB units */
704}
705
baa09f5f 706static inline size_t gen7_get_stolen_size(u16 snb_gmch_ctl)
03752f5b
BW		 707{
708 static const int stolen_decoder[] = {
709 0, 0, 0, 0, 0, 32, 48, 64, 128, 256, 96, 160, 224, 352};
710 snb_gmch_ctl >>= IVB_GMCH_GMS_SHIFT;
711 snb_gmch_ctl &= IVB_GMCH_GMS_MASK;
712 return stolen_decoder[snb_gmch_ctl] << 20;
713}
714
baa09f5f
BW		 715static int gen6_gmch_probe(struct drm_device *dev,
716 size_t *gtt_total,
41907ddc
BW		 717 size_t *stolen,
718 phys_addr_t *mappable_base,
719 unsigned long *mappable_end)
e76e9aeb
BW		 720{
721 struct drm_i915_private *dev_priv = dev->dev_private;
722 phys_addr_t gtt_bus_addr;
baa09f5f 723 unsigned int gtt_size;
e76e9aeb 724 u16 snb_gmch_ctl;
e76e9aeb
BW		 725 int ret;
726
41907ddc
BW		 727 *mappable_base = pci_resource_start(dev->pdev, 2);
728 *mappable_end = pci_resource_len(dev->pdev, 2);
729
baa09f5f
BW		 730 /* 64/512MB is the current min/max we actually know of, but this is just
731 * a coarse sanity check.
e76e9aeb 732 */
41907ddc 733 if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) {
baa09f5f
BW		 734 DRM_ERROR("Unknown GMADR size (%lx)\n",
735 dev_priv->gtt.mappable_end);
736 return -ENXIO;
e76e9aeb
BW		 737 }
738
e76e9aeb
BW		 739 if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40)))
740 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40));
e76e9aeb 741 pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
baa09f5f 742 gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl);
e76e9aeb 743
086ddcce 744 if (IS_GEN7(dev) && !IS_VALLEYVIEW(dev))
baa09f5f
BW		 745 *stolen = gen7_get_stolen_size(snb_gmch_ctl);
746 else
747 *stolen = gen6_get_stolen_size(snb_gmch_ctl);
e76e9aeb 748
e7c2b58b 749 *gtt_total = (gtt_size / sizeof(gen6_gtt_pte_t)) << PAGE_SHIFT;
e76e9aeb 750
a93e4161
BW		 751 /* For Modern GENs the PTEs and register space are split in the BAR */
752 gtt_bus_addr = pci_resource_start(dev->pdev, 0) +
753 (pci_resource_len(dev->pdev, 0) / 2);
754
baa09f5f 755 dev_priv->gtt.gsm = ioremap_wc(gtt_bus_addr, gtt_size);
5d4545ae 756 if (!dev_priv->gtt.gsm) {
e76e9aeb 757 DRM_ERROR("Failed to map the gtt page table\n");
baa09f5f 758 return -ENOMEM;
e76e9aeb
BW		 759 }
760
baa09f5f
BW		 761 ret = setup_scratch_page(dev);
762 if (ret)
763 DRM_ERROR("Scratch setup failed\n");
e76e9aeb 764
7faf1ab2
DV		 765 dev_priv->gtt.gtt_clear_range = gen6_ggtt_clear_range;
766 dev_priv->gtt.gtt_insert_entries = gen6_ggtt_insert_entries;
767
e76e9aeb
BW		 768 return ret;
769}
770
d93c6233 771static void gen6_gmch_remove(struct drm_device *dev)
e76e9aeb
BW		 772{
773 struct drm_i915_private *dev_priv = dev->dev_private;
5d4545ae 774 iounmap(dev_priv->gtt.gsm);
baa09f5f 775 teardown_scratch_page(dev_priv->dev);
644ec02b 776}
baa09f5f
BW		 777
778static int i915_gmch_probe(struct drm_device *dev,
779 size_t *gtt_total,
41907ddc
BW		 780 size_t *stolen,
781 phys_addr_t *mappable_base,
782 unsigned long *mappable_end)
baa09f5f
BW		 783{
784 struct drm_i915_private *dev_priv = dev->dev_private;
785 int ret;
786
baa09f5f
BW		 787 ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL);
788 if (!ret) {
789 DRM_ERROR("failed to set up gmch\n");
790 return -EIO;
791 }
792
41907ddc 793 intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end);
baa09f5f
BW		 794
795 dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev);
796 dev_priv->gtt.gtt_clear_range = i915_ggtt_clear_range;
797 dev_priv->gtt.gtt_insert_entries = i915_ggtt_insert_entries;
798
799 return 0;
800}
801
802static void i915_gmch_remove(struct drm_device *dev)
803{
804 intel_gmch_remove();
805}
806
807int i915_gem_gtt_init(struct drm_device *dev)
808{
809 struct drm_i915_private *dev_priv = dev->dev_private;
810 struct i915_gtt *gtt = &dev_priv->gtt;
811 unsigned long gtt_size;
812 int ret;
813
baa09f5f
BW		 814 if (INTEL_INFO(dev)->gen <= 5) {
815 dev_priv->gtt.gtt_probe = i915_gmch_probe;
816 dev_priv->gtt.gtt_remove = i915_gmch_remove;
817 } else {
818 dev_priv->gtt.gtt_probe = gen6_gmch_probe;
819 dev_priv->gtt.gtt_remove = gen6_gmch_remove;
820 }
821
822 ret = dev_priv->gtt.gtt_probe(dev, &dev_priv->gtt.total,
41907ddc
BW		 823 &dev_priv->gtt.stolen_size,
824 &gtt->mappable_base,
825 &gtt->mappable_end);
a54c0c27 826 if (ret)
baa09f5f 827 return ret;
baa09f5f 828
e7c2b58b 829 gtt_size = (dev_priv->gtt.total >> PAGE_SHIFT) * sizeof(gen6_gtt_pte_t);
baa09f5f
BW		 830
831 /* GMADR is the PCI mmio aperture into the global GTT. */
832 DRM_INFO("Memory usable by graphics device = %zdM\n",
833 dev_priv->gtt.total >> 20);
834 DRM_DEBUG_DRIVER("GMADR size = %ldM\n",
835 dev_priv->gtt.mappable_end >> 20);
836 DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n",
837 dev_priv->gtt.stolen_size >> 20);
838
839 return 0;
840}
Linux 6.x block layer and io_uring tree(s)