Commit | Line | Data |
---|---|---|
2c86e55d MA |
1 | // SPDX-License-Identifier: MIT |
2 | /* | |
3 | * Copyright © 2020 Intel Corporation | |
4 | */ | |
5 | ||
6 | #include <linux/log2.h> | |
7 | ||
8 | #include "gen6_ppgtt.h" | |
9 | #include "i915_scatterlist.h" | |
10 | #include "i915_trace.h" | |
11 | #include "i915_vgpu.h" | |
12 | #include "intel_gt.h" | |
13 | ||
14 | /* Write pde (index) from the page directory @pd to the page table @pt */ | |
9834dfef CW |
15 | static void gen6_write_pde(const struct gen6_ppgtt *ppgtt, |
16 | const unsigned int pde, | |
17 | const struct i915_page_table *pt) | |
2c86e55d | 18 | { |
89351925 CW |
19 | dma_addr_t addr = pt ? px_dma(pt) : px_dma(ppgtt->base.vm.scratch[1]); |
20 | ||
2c86e55d | 21 | /* Caller needs to make sure the write completes if necessary */ |
89351925 | 22 | iowrite32(GEN6_PDE_ADDR_ENCODE(addr) | GEN6_PDE_VALID, |
2c86e55d MA |
23 | ppgtt->pd_addr + pde); |
24 | } | |
25 | ||
26 | void gen7_ppgtt_enable(struct intel_gt *gt) | |
27 | { | |
28 | struct drm_i915_private *i915 = gt->i915; | |
29 | struct intel_uncore *uncore = gt->uncore; | |
2c86e55d MA |
30 | u32 ecochk; |
31 | ||
32 | intel_uncore_rmw(uncore, GAC_ECO_BITS, 0, ECOBITS_PPGTT_CACHE64B); | |
33 | ||
34 | ecochk = intel_uncore_read(uncore, GAM_ECOCHK); | |
35 | if (IS_HASWELL(i915)) { | |
36 | ecochk |= ECOCHK_PPGTT_WB_HSW; | |
37 | } else { | |
38 | ecochk |= ECOCHK_PPGTT_LLC_IVB; | |
39 | ecochk &= ~ECOCHK_PPGTT_GFDT_IVB; | |
40 | } | |
41 | intel_uncore_write(uncore, GAM_ECOCHK, ecochk); | |
2c86e55d MA |
42 | } |
43 | ||
44 | void gen6_ppgtt_enable(struct intel_gt *gt) | |
45 | { | |
46 | struct intel_uncore *uncore = gt->uncore; | |
47 | ||
48 | intel_uncore_rmw(uncore, | |
49 | GAC_ECO_BITS, | |
50 | 0, | |
51 | ECOBITS_SNB_BIT | ECOBITS_PPGTT_CACHE64B); | |
52 | ||
53 | intel_uncore_rmw(uncore, | |
54 | GAB_CTL, | |
55 | 0, | |
56 | GAB_CTL_CONT_AFTER_PAGEFAULT); | |
57 | ||
58 | intel_uncore_rmw(uncore, | |
59 | GAM_ECOCHK, | |
60 | 0, | |
61 | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B); | |
62 | ||
63 | if (HAS_PPGTT(uncore->i915)) /* may be disabled for VT-d */ | |
64 | intel_uncore_write(uncore, | |
65 | GFX_MODE, | |
66 | _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); | |
67 | } | |
68 | ||
69 | /* PPGTT support for Sandybdrige/Gen6 and later */ | |
70 | static void gen6_ppgtt_clear_range(struct i915_address_space *vm, | |
71 | u64 start, u64 length) | |
72 | { | |
73 | struct gen6_ppgtt * const ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm)); | |
74 | const unsigned int first_entry = start / I915_GTT_PAGE_SIZE; | |
89351925 | 75 | const gen6_pte_t scratch_pte = vm->scratch[0]->encode; |
2c86e55d MA |
76 | unsigned int pde = first_entry / GEN6_PTES; |
77 | unsigned int pte = first_entry % GEN6_PTES; | |
78 | unsigned int num_entries = length / I915_GTT_PAGE_SIZE; | |
79 | ||
80 | while (num_entries) { | |
81 | struct i915_page_table * const pt = | |
82 | i915_pt_entry(ppgtt->base.pd, pde++); | |
83 | const unsigned int count = min(num_entries, GEN6_PTES - pte); | |
84 | gen6_pte_t *vaddr; | |
85 | ||
2c86e55d MA |
86 | num_entries -= count; |
87 | ||
88 | GEM_BUG_ON(count > atomic_read(&pt->used)); | |
89 | if (!atomic_sub_return(count, &pt->used)) | |
90 | ppgtt->scan_for_unused_pt = true; | |
91 | ||
92 | /* | |
93 | * Note that the hw doesn't support removing PDE on the fly | |
94 | * (they are cached inside the context with no means to | |
95 | * invalidate the cache), so we can only reset the PTE | |
96 | * entries back to scratch. | |
97 | */ | |
98 | ||
99 | vaddr = kmap_atomic_px(pt); | |
100 | memset32(vaddr + pte, scratch_pte, count); | |
101 | kunmap_atomic(vaddr); | |
102 | ||
103 | pte = 0; | |
104 | } | |
105 | } | |
106 | ||
107 | static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, | |
108 | struct i915_vma *vma, | |
109 | enum i915_cache_level cache_level, | |
110 | u32 flags) | |
111 | { | |
112 | struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); | |
113 | struct i915_page_directory * const pd = ppgtt->pd; | |
114 | unsigned int first_entry = vma->node.start / I915_GTT_PAGE_SIZE; | |
115 | unsigned int act_pt = first_entry / GEN6_PTES; | |
116 | unsigned int act_pte = first_entry % GEN6_PTES; | |
117 | const u32 pte_encode = vm->pte_encode(0, cache_level, flags); | |
118 | struct sgt_dma iter = sgt_dma(vma); | |
119 | gen6_pte_t *vaddr; | |
120 | ||
89351925 | 121 | GEM_BUG_ON(!pd->entry[act_pt]); |
2c86e55d MA |
122 | |
123 | vaddr = kmap_atomic_px(i915_pt_entry(pd, act_pt)); | |
124 | do { | |
8a473dba | 125 | GEM_BUG_ON(sg_dma_len(iter.sg) < I915_GTT_PAGE_SIZE); |
2c86e55d MA |
126 | vaddr[act_pte] = pte_encode | GEN6_PTE_ADDR_ENCODE(iter.dma); |
127 | ||
128 | iter.dma += I915_GTT_PAGE_SIZE; | |
129 | if (iter.dma == iter.max) { | |
130 | iter.sg = __sg_next(iter.sg); | |
8a473dba | 131 | if (!iter.sg || sg_dma_len(iter.sg) == 0) |
2c86e55d MA |
132 | break; |
133 | ||
134 | iter.dma = sg_dma_address(iter.sg); | |
8a473dba | 135 | iter.max = iter.dma + sg_dma_len(iter.sg); |
2c86e55d MA |
136 | } |
137 | ||
138 | if (++act_pte == GEN6_PTES) { | |
139 | kunmap_atomic(vaddr); | |
140 | vaddr = kmap_atomic_px(i915_pt_entry(pd, ++act_pt)); | |
141 | act_pte = 0; | |
142 | } | |
143 | } while (1); | |
144 | kunmap_atomic(vaddr); | |
145 | ||
146 | vma->page_sizes.gtt = I915_GTT_PAGE_SIZE; | |
147 | } | |
148 | ||
149 | static void gen6_flush_pd(struct gen6_ppgtt *ppgtt, u64 start, u64 end) | |
150 | { | |
151 | struct i915_page_directory * const pd = ppgtt->base.pd; | |
152 | struct i915_page_table *pt; | |
153 | unsigned int pde; | |
154 | ||
155 | start = round_down(start, SZ_64K); | |
156 | end = round_up(end, SZ_64K) - start; | |
157 | ||
158 | mutex_lock(&ppgtt->flush); | |
159 | ||
160 | gen6_for_each_pde(pt, pd, start, end, pde) | |
161 | gen6_write_pde(ppgtt, pde, pt); | |
162 | ||
163 | mb(); | |
164 | ioread32(ppgtt->pd_addr + pde - 1); | |
165 | gen6_ggtt_invalidate(ppgtt->base.vm.gt->ggtt); | |
166 | mb(); | |
167 | ||
168 | mutex_unlock(&ppgtt->flush); | |
169 | } | |
170 | ||
cd0452aa CW |
171 | static void gen6_alloc_va_range(struct i915_address_space *vm, |
172 | struct i915_vm_pt_stash *stash, | |
173 | u64 start, u64 length) | |
2c86e55d MA |
174 | { |
175 | struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm)); | |
176 | struct i915_page_directory * const pd = ppgtt->base.pd; | |
cd0452aa | 177 | struct i915_page_table *pt; |
b297bde1 | 178 | bool flush = false; |
2c86e55d MA |
179 | u64 from = start; |
180 | unsigned int pde; | |
2c86e55d | 181 | |
2c86e55d MA |
182 | spin_lock(&pd->lock); |
183 | gen6_for_each_pde(pt, pd, start, length, pde) { | |
184 | const unsigned int count = gen6_pte_count(start, length); | |
185 | ||
89351925 | 186 | if (!pt) { |
2c86e55d MA |
187 | spin_unlock(&pd->lock); |
188 | ||
cd0452aa | 189 | pt = stash->pt[0]; |
89351925 CW |
190 | __i915_gem_object_pin_pages(pt->base); |
191 | i915_gem_object_make_unshrinkable(pt->base); | |
2c86e55d | 192 | |
89351925 | 193 | fill32_px(pt, vm->scratch[0]->encode); |
2c86e55d MA |
194 | |
195 | spin_lock(&pd->lock); | |
89351925 | 196 | if (!pd->entry[pde]) { |
cd0452aa CW |
197 | stash->pt[0] = pt->stash; |
198 | atomic_set(&pt->used, 0); | |
2c86e55d MA |
199 | pd->entry[pde] = pt; |
200 | } else { | |
2c86e55d MA |
201 | pt = pd->entry[pde]; |
202 | } | |
b297bde1 CW |
203 | |
204 | flush = true; | |
2c86e55d MA |
205 | } |
206 | ||
207 | atomic_add(count, &pt->used); | |
208 | } | |
209 | spin_unlock(&pd->lock); | |
210 | ||
b297bde1 CW |
211 | if (flush && i915_vma_is_bound(ppgtt->vma, I915_VMA_GLOBAL_BIND)) { |
212 | intel_wakeref_t wakeref; | |
213 | ||
214 | with_intel_runtime_pm(&vm->i915->runtime_pm, wakeref) | |
215 | gen6_flush_pd(ppgtt, from, start); | |
216 | } | |
2c86e55d MA |
217 | } |
218 | ||
219 | static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt) | |
220 | { | |
221 | struct i915_address_space * const vm = &ppgtt->base.vm; | |
2c86e55d MA |
222 | int ret; |
223 | ||
89351925 | 224 | ret = setup_scratch_page(vm); |
2c86e55d MA |
225 | if (ret) |
226 | return ret; | |
227 | ||
89351925 CW |
228 | vm->scratch[0]->encode = |
229 | vm->pte_encode(px_dma(vm->scratch[0]), | |
2c86e55d MA |
230 | I915_CACHE_NONE, PTE_READ_ONLY); |
231 | ||
89351925 | 232 | vm->scratch[1] = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K); |
fa812ce9 CW |
233 | if (IS_ERR(vm->scratch[1])) { |
234 | ret = PTR_ERR(vm->scratch[1]); | |
235 | goto err_scratch0; | |
236 | } | |
89351925 CW |
237 | |
238 | ret = pin_pt_dma(vm, vm->scratch[1]); | |
fa812ce9 CW |
239 | if (ret) |
240 | goto err_scratch1; | |
2c86e55d | 241 | |
89351925 | 242 | fill32_px(vm->scratch[1], vm->scratch[0]->encode); |
2c86e55d MA |
243 | |
244 | return 0; | |
fa812ce9 CW |
245 | |
246 | err_scratch1: | |
247 | i915_gem_object_put(vm->scratch[1]); | |
248 | err_scratch0: | |
249 | i915_gem_object_put(vm->scratch[0]); | |
250 | return ret; | |
2c86e55d MA |
251 | } |
252 | ||
253 | static void gen6_ppgtt_free_pd(struct gen6_ppgtt *ppgtt) | |
254 | { | |
255 | struct i915_page_directory * const pd = ppgtt->base.pd; | |
2c86e55d MA |
256 | struct i915_page_table *pt; |
257 | u32 pde; | |
258 | ||
259 | gen6_for_all_pdes(pt, pd, pde) | |
89351925 | 260 | if (pt) |
82adf901 | 261 | free_pt(&ppgtt->base.vm, pt); |
2c86e55d MA |
262 | } |
263 | ||
264 | static void gen6_ppgtt_cleanup(struct i915_address_space *vm) | |
265 | { | |
266 | struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm)); | |
267 | ||
268 | __i915_vma_put(ppgtt->vma); | |
269 | ||
270 | gen6_ppgtt_free_pd(ppgtt); | |
271 | free_scratch(vm); | |
272 | ||
273 | mutex_destroy(&ppgtt->flush); | |
274 | mutex_destroy(&ppgtt->pin_mutex); | |
82adf901 CW |
275 | |
276 | free_pd(&ppgtt->base.vm, ppgtt->base.pd); | |
2c86e55d MA |
277 | } |
278 | ||
279 | static int pd_vma_set_pages(struct i915_vma *vma) | |
280 | { | |
281 | vma->pages = ERR_PTR(-ENODEV); | |
282 | return 0; | |
283 | } | |
284 | ||
285 | static void pd_vma_clear_pages(struct i915_vma *vma) | |
286 | { | |
287 | GEM_BUG_ON(!vma->pages); | |
288 | ||
289 | vma->pages = NULL; | |
290 | } | |
291 | ||
cd0452aa CW |
292 | static void pd_vma_bind(struct i915_address_space *vm, |
293 | struct i915_vm_pt_stash *stash, | |
294 | struct i915_vma *vma, | |
295 | enum i915_cache_level cache_level, | |
296 | u32 unused) | |
2c86e55d | 297 | { |
12b07256 | 298 | struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); |
2c86e55d MA |
299 | struct gen6_ppgtt *ppgtt = vma->private; |
300 | u32 ggtt_offset = i915_ggtt_offset(vma) / I915_GTT_PAGE_SIZE; | |
301 | ||
89351925 | 302 | ppgtt->pp_dir = ggtt_offset * sizeof(gen6_pte_t) << 10; |
2c86e55d MA |
303 | ppgtt->pd_addr = (gen6_pte_t __iomem *)ggtt->gsm + ggtt_offset; |
304 | ||
305 | gen6_flush_pd(ppgtt, 0, ppgtt->base.vm.total); | |
2c86e55d MA |
306 | } |
307 | ||
12b07256 | 308 | static void pd_vma_unbind(struct i915_address_space *vm, struct i915_vma *vma) |
2c86e55d MA |
309 | { |
310 | struct gen6_ppgtt *ppgtt = vma->private; | |
311 | struct i915_page_directory * const pd = ppgtt->base.pd; | |
2c86e55d MA |
312 | struct i915_page_table *pt; |
313 | unsigned int pde; | |
314 | ||
315 | if (!ppgtt->scan_for_unused_pt) | |
316 | return; | |
317 | ||
318 | /* Free all no longer used page tables */ | |
319 | gen6_for_all_pdes(pt, ppgtt->base.pd, pde) { | |
89351925 | 320 | if (!pt || atomic_read(&pt->used)) |
2c86e55d MA |
321 | continue; |
322 | ||
82adf901 | 323 | free_pt(&ppgtt->base.vm, pt); |
89351925 | 324 | pd->entry[pde] = NULL; |
2c86e55d MA |
325 | } |
326 | ||
327 | ppgtt->scan_for_unused_pt = false; | |
328 | } | |
329 | ||
330 | static const struct i915_vma_ops pd_vma_ops = { | |
331 | .set_pages = pd_vma_set_pages, | |
332 | .clear_pages = pd_vma_clear_pages, | |
333 | .bind_vma = pd_vma_bind, | |
334 | .unbind_vma = pd_vma_unbind, | |
335 | }; | |
336 | ||
337 | static struct i915_vma *pd_vma_create(struct gen6_ppgtt *ppgtt, int size) | |
338 | { | |
339 | struct i915_ggtt *ggtt = ppgtt->base.vm.gt->ggtt; | |
340 | struct i915_vma *vma; | |
341 | ||
342 | GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); | |
343 | GEM_BUG_ON(size > ggtt->vm.total); | |
344 | ||
345 | vma = i915_vma_alloc(); | |
346 | if (!vma) | |
347 | return ERR_PTR(-ENOMEM); | |
348 | ||
349 | i915_active_init(&vma->active, NULL, NULL); | |
350 | ||
351 | kref_init(&vma->ref); | |
352 | mutex_init(&vma->pages_mutex); | |
353 | vma->vm = i915_vm_get(&ggtt->vm); | |
354 | vma->ops = &pd_vma_ops; | |
355 | vma->private = ppgtt; | |
356 | ||
357 | vma->size = size; | |
358 | vma->fence_size = size; | |
359 | atomic_set(&vma->flags, I915_VMA_GGTT); | |
360 | vma->ggtt_view.type = I915_GGTT_VIEW_ROTATED; /* prevent fencing */ | |
361 | ||
362 | INIT_LIST_HEAD(&vma->obj_link); | |
363 | INIT_LIST_HEAD(&vma->closed_link); | |
364 | ||
365 | return vma; | |
366 | } | |
367 | ||
47b08693 | 368 | int gen6_ppgtt_pin(struct i915_ppgtt *base, struct i915_gem_ww_ctx *ww) |
2c86e55d MA |
369 | { |
370 | struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(base); | |
371 | int err; | |
372 | ||
373 | GEM_BUG_ON(!atomic_read(&ppgtt->base.vm.open)); | |
374 | ||
375 | /* | |
376 | * Workaround the limited maximum vma->pin_count and the aliasing_ppgtt | |
377 | * which will be pinned into every active context. | |
378 | * (When vma->pin_count becomes atomic, I expect we will naturally | |
379 | * need a larger, unpacked, type and kill this redundancy.) | |
380 | */ | |
381 | if (atomic_add_unless(&ppgtt->pin_count, 1, 0)) | |
382 | return 0; | |
383 | ||
384 | if (mutex_lock_interruptible(&ppgtt->pin_mutex)) | |
385 | return -EINTR; | |
386 | ||
387 | /* | |
388 | * PPGTT PDEs reside in the GGTT and consists of 512 entries. The | |
389 | * allocator works in address space sizes, so it's multiplied by page | |
390 | * size. We allocate at the top of the GTT to avoid fragmentation. | |
391 | */ | |
392 | err = 0; | |
393 | if (!atomic_read(&ppgtt->pin_count)) | |
47b08693 | 394 | err = i915_ggtt_pin(ppgtt->vma, ww, GEN6_PD_ALIGN, PIN_HIGH); |
2c86e55d MA |
395 | if (!err) |
396 | atomic_inc(&ppgtt->pin_count); | |
397 | mutex_unlock(&ppgtt->pin_mutex); | |
398 | ||
399 | return err; | |
400 | } | |
401 | ||
402 | void gen6_ppgtt_unpin(struct i915_ppgtt *base) | |
403 | { | |
404 | struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(base); | |
405 | ||
406 | GEM_BUG_ON(!atomic_read(&ppgtt->pin_count)); | |
407 | if (atomic_dec_and_test(&ppgtt->pin_count)) | |
408 | i915_vma_unpin(ppgtt->vma); | |
409 | } | |
410 | ||
411 | void gen6_ppgtt_unpin_all(struct i915_ppgtt *base) | |
412 | { | |
413 | struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(base); | |
414 | ||
415 | if (!atomic_read(&ppgtt->pin_count)) | |
416 | return; | |
417 | ||
418 | i915_vma_unpin(ppgtt->vma); | |
419 | atomic_set(&ppgtt->pin_count, 0); | |
420 | } | |
421 | ||
422 | struct i915_ppgtt *gen6_ppgtt_create(struct intel_gt *gt) | |
423 | { | |
424 | struct i915_ggtt * const ggtt = gt->ggtt; | |
425 | struct gen6_ppgtt *ppgtt; | |
426 | int err; | |
427 | ||
428 | ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); | |
429 | if (!ppgtt) | |
430 | return ERR_PTR(-ENOMEM); | |
431 | ||
432 | mutex_init(&ppgtt->flush); | |
433 | mutex_init(&ppgtt->pin_mutex); | |
434 | ||
435 | ppgtt_init(&ppgtt->base, gt); | |
cd0452aa | 436 | ppgtt->base.vm.pd_shift = ilog2(SZ_4K * SZ_4K / sizeof(gen6_pte_t)); |
2c86e55d MA |
437 | ppgtt->base.vm.top = 1; |
438 | ||
439 | ppgtt->base.vm.bind_async_flags = I915_VMA_LOCAL_BIND; | |
440 | ppgtt->base.vm.allocate_va_range = gen6_alloc_va_range; | |
441 | ppgtt->base.vm.clear_range = gen6_ppgtt_clear_range; | |
442 | ppgtt->base.vm.insert_entries = gen6_ppgtt_insert_entries; | |
443 | ppgtt->base.vm.cleanup = gen6_ppgtt_cleanup; | |
444 | ||
89351925 | 445 | ppgtt->base.vm.alloc_pt_dma = alloc_pt_dma; |
2c86e55d MA |
446 | ppgtt->base.vm.pte_encode = ggtt->vm.pte_encode; |
447 | ||
82adf901 | 448 | ppgtt->base.pd = __alloc_pd(I915_PDES); |
2c86e55d MA |
449 | if (!ppgtt->base.pd) { |
450 | err = -ENOMEM; | |
451 | goto err_free; | |
452 | } | |
453 | ||
454 | err = gen6_ppgtt_init_scratch(ppgtt); | |
455 | if (err) | |
456 | goto err_pd; | |
457 | ||
458 | ppgtt->vma = pd_vma_create(ppgtt, GEN6_PD_SIZE); | |
459 | if (IS_ERR(ppgtt->vma)) { | |
460 | err = PTR_ERR(ppgtt->vma); | |
461 | goto err_scratch; | |
462 | } | |
463 | ||
464 | return &ppgtt->base; | |
465 | ||
466 | err_scratch: | |
467 | free_scratch(&ppgtt->base.vm); | |
468 | err_pd: | |
82adf901 | 469 | free_pd(&ppgtt->base.vm, ppgtt->base.pd); |
2c86e55d MA |
470 | err_free: |
471 | mutex_destroy(&ppgtt->pin_mutex); | |
472 | kfree(ppgtt); | |
473 | return ERR_PTR(err); | |
474 | } |