Commit | Line | Data |
---|---|---|
f033428d CW |
1 | /* |
2 | * SPDX-License-Identifier: MIT | |
3 | * | |
4 | * Copyright © 2014-2016 Intel Corporation | |
5 | */ | |
6 | ||
5f2ec909 JN |
7 | #include <drm/drm_cache.h> |
8 | ||
4bedceae CW |
9 | #include "gt/intel_gt.h" |
10 | #include "gt/intel_gt_pm.h" | |
11 | ||
f033428d CW |
12 | #include "i915_drv.h" |
13 | #include "i915_gem_object.h" | |
37d63f8f | 14 | #include "i915_scatterlist.h" |
01377a0d | 15 | #include "i915_gem_lmem.h" |
cc662126 | 16 | #include "i915_gem_mman.h" |
f033428d CW |
17 | |
18 | void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, | |
19 | struct sg_table *pages, | |
20 | unsigned int sg_page_sizes) | |
21 | { | |
22 | struct drm_i915_private *i915 = to_i915(obj->base.dev); | |
23 | unsigned long supported = INTEL_INFO(i915)->page_sizes; | |
0175969e | 24 | bool shrinkable; |
f033428d CW |
25 | int i; |
26 | ||
a3258dbd | 27 | assert_object_held_shared(obj); |
f033428d | 28 | |
7c98501a MA |
29 | if (i915_gem_object_is_volatile(obj)) |
30 | obj->mm.madv = I915_MADV_DONTNEED; | |
31 | ||
f033428d CW |
32 | /* Make the pages coherent with the GPU (flushing any swapin). */ |
33 | if (obj->cache_dirty) { | |
068b1bd0 | 34 | WARN_ON_ONCE(IS_DGFX(i915)); |
f033428d CW |
35 | obj->write_domain = 0; |
36 | if (i915_gem_object_has_struct_page(obj)) | |
37 | drm_clflush_sg(pages); | |
38 | obj->cache_dirty = false; | |
39 | } | |
40 | ||
41 | obj->mm.get_page.sg_pos = pages->sgl; | |
42 | obj->mm.get_page.sg_idx = 0; | |
934941ed TU |
43 | obj->mm.get_dma_page.sg_pos = pages->sgl; |
44 | obj->mm.get_dma_page.sg_idx = 0; | |
f033428d CW |
45 | |
46 | obj->mm.pages = pages; | |
47 | ||
f033428d CW |
48 | GEM_BUG_ON(!sg_page_sizes); |
49 | obj->mm.page_sizes.phys = sg_page_sizes; | |
50 | ||
51 | /* | |
52 | * Calculate the supported page-sizes which fit into the given | |
53 | * sg_page_sizes. This will give us the page-sizes which we may be able | |
54 | * to use opportunistically when later inserting into the GTT. For | |
55 | * example if phys=2G, then in theory we should be able to use 1G, 2M, | |
56 | * 64K or 4K pages, although in practice this will depend on a number of | |
57 | * other factors. | |
58 | */ | |
59 | obj->mm.page_sizes.sg = 0; | |
60 | for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) { | |
61 | if (obj->mm.page_sizes.phys & ~0u << i) | |
62 | obj->mm.page_sizes.sg |= BIT(i); | |
63 | } | |
64 | GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg)); | |
65 | ||
0175969e CW |
66 | shrinkable = i915_gem_object_is_shrinkable(obj); |
67 | ||
68 | if (i915_gem_object_is_tiled(obj) && | |
69 | i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) { | |
70 | GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj)); | |
71 | i915_gem_object_set_tiling_quirk(obj); | |
8777d17b CW |
72 | GEM_BUG_ON(!list_empty(&obj->mm.link)); |
73 | atomic_inc(&obj->mm.shrink_pin); | |
0175969e CW |
74 | shrinkable = false; |
75 | } | |
76 | ||
ebd4a8ec | 77 | if (shrinkable && !i915_gem_object_has_self_managed_shrink_list(obj)) { |
ecab9be1 | 78 | struct list_head *list; |
a8cff4c8 CW |
79 | unsigned long flags; |
80 | ||
cf41a8f1 | 81 | assert_object_held(obj); |
a8cff4c8 CW |
82 | spin_lock_irqsave(&i915->mm.obj_lock, flags); |
83 | ||
d82b4b26 CW |
84 | i915->mm.shrink_count++; |
85 | i915->mm.shrink_memory += obj->base.size; | |
ecab9be1 CW |
86 | |
87 | if (obj->mm.madv != I915_MADV_WILLNEED) | |
88 | list = &i915->mm.purge_list; | |
89 | else | |
90 | list = &i915->mm.shrink_list; | |
91 | list_add_tail(&obj->mm.link, list); | |
a8cff4c8 | 92 | |
99013b10 | 93 | atomic_set(&obj->mm.shrink_pin, 0); |
a8cff4c8 | 94 | spin_unlock_irqrestore(&i915->mm.obj_lock, flags); |
d82b4b26 | 95 | } |
f033428d CW |
96 | } |
97 | ||
98 | int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj) | |
99 | { | |
d0bf4582 | 100 | struct drm_i915_private *i915 = to_i915(obj->base.dev); |
f033428d CW |
101 | int err; |
102 | ||
a3258dbd TH |
103 | assert_object_held_shared(obj); |
104 | ||
f033428d | 105 | if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) { |
d0bf4582 WK |
106 | drm_dbg(&i915->drm, |
107 | "Attempting to obtain a purgeable object\n"); | |
f033428d CW |
108 | return -EFAULT; |
109 | } | |
110 | ||
111 | err = obj->ops->get_pages(obj); | |
112 | GEM_BUG_ON(!err && !i915_gem_object_has_pages(obj)); | |
113 | ||
114 | return err; | |
115 | } | |
116 | ||
117 | /* Ensure that the associated pages are gathered from the backing storage | |
118 | * and pinned into our object. i915_gem_object_pin_pages() may be called | |
119 | * multiple times before they are released by a single call to | |
120 | * i915_gem_object_unpin_pages() - once the pages are no longer referenced | |
121 | * either as a result of memory pressure (reaping pages under the shrinker) | |
122 | * or as the object is itself released. | |
123 | */ | |
124 | int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj) | |
125 | { | |
126 | int err; | |
127 | ||
cf41a8f1 | 128 | assert_object_held(obj); |
f033428d | 129 | |
a3258dbd TH |
130 | assert_object_held_shared(obj); |
131 | ||
f033428d CW |
132 | if (unlikely(!i915_gem_object_has_pages(obj))) { |
133 | GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj)); | |
134 | ||
135 | err = ____i915_gem_object_get_pages(obj); | |
136 | if (err) | |
cf41a8f1 | 137 | return err; |
f033428d CW |
138 | |
139 | smp_mb__before_atomic(); | |
140 | } | |
141 | atomic_inc(&obj->mm.pages_pin_count); | |
142 | ||
cf41a8f1 | 143 | return 0; |
f033428d CW |
144 | } |
145 | ||
c858ffa1 ML |
146 | int i915_gem_object_pin_pages_unlocked(struct drm_i915_gem_object *obj) |
147 | { | |
148 | struct i915_gem_ww_ctx ww; | |
149 | int err; | |
150 | ||
151 | i915_gem_ww_ctx_init(&ww, true); | |
152 | retry: | |
153 | err = i915_gem_object_lock(obj, &ww); | |
154 | if (!err) | |
155 | err = i915_gem_object_pin_pages(obj); | |
156 | ||
157 | if (err == -EDEADLK) { | |
158 | err = i915_gem_ww_ctx_backoff(&ww); | |
159 | if (!err) | |
160 | goto retry; | |
161 | } | |
162 | i915_gem_ww_ctx_fini(&ww); | |
163 | return err; | |
164 | } | |
165 | ||
f033428d | 166 | /* Immediately discard the backing storage */ |
7ae03459 | 167 | int i915_gem_object_truncate(struct drm_i915_gem_object *obj) |
f033428d | 168 | { |
f033428d | 169 | if (obj->ops->truncate) |
7ae03459 MA |
170 | return obj->ops->truncate(obj); |
171 | ||
172 | return 0; | |
f033428d CW |
173 | } |
174 | ||
f033428d CW |
175 | static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj) |
176 | { | |
177 | struct radix_tree_iter iter; | |
178 | void __rcu **slot; | |
179 | ||
180 | rcu_read_lock(); | |
181 | radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0) | |
182 | radix_tree_delete(&obj->mm.get_page.radix, iter.index); | |
934941ed TU |
183 | radix_tree_for_each_slot(slot, &obj->mm.get_dma_page.radix, &iter, 0) |
184 | radix_tree_delete(&obj->mm.get_dma_page.radix, iter.index); | |
f033428d CW |
185 | rcu_read_unlock(); |
186 | } | |
187 | ||
01377a0d AJ |
188 | static void unmap_object(struct drm_i915_gem_object *obj, void *ptr) |
189 | { | |
6056e500 | 190 | if (is_vmalloc_addr(ptr)) |
01377a0d | 191 | vunmap(ptr); |
01377a0d AJ |
192 | } |
193 | ||
5d36acb7 CW |
194 | static void flush_tlb_invalidate(struct drm_i915_gem_object *obj) |
195 | { | |
196 | struct drm_i915_private *i915 = to_i915(obj->base.dev); | |
197 | struct intel_gt *gt = to_gt(i915); | |
198 | ||
199 | if (!obj->mm.tlb) | |
200 | return; | |
201 | ||
202 | intel_gt_invalidate_tlb(gt, obj->mm.tlb); | |
203 | obj->mm.tlb = 0; | |
204 | } | |
205 | ||
f033428d CW |
206 | struct sg_table * |
207 | __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj) | |
208 | { | |
f033428d CW |
209 | struct sg_table *pages; |
210 | ||
a3258dbd TH |
211 | assert_object_held_shared(obj); |
212 | ||
f033428d CW |
213 | pages = fetch_and_zero(&obj->mm.pages); |
214 | if (IS_ERR_OR_NULL(pages)) | |
215 | return pages; | |
216 | ||
7c98501a MA |
217 | if (i915_gem_object_is_volatile(obj)) |
218 | obj->mm.madv = I915_MADV_WILLNEED; | |
219 | ||
ebd4a8ec MA |
220 | if (!i915_gem_object_has_self_managed_shrink_list(obj)) |
221 | i915_gem_object_make_unshrinkable(obj); | |
f033428d CW |
222 | |
223 | if (obj->mm.mapping) { | |
01377a0d | 224 | unmap_object(obj, page_mask_bits(obj->mm.mapping)); |
f033428d CW |
225 | obj->mm.mapping = NULL; |
226 | } | |
227 | ||
228 | __i915_gem_object_reset_page_iter(obj); | |
229 | obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0; | |
230 | ||
5d36acb7 | 231 | flush_tlb_invalidate(obj); |
7938d615 | 232 | |
f033428d CW |
233 | return pages; |
234 | } | |
235 | ||
cf41a8f1 | 236 | int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj) |
f033428d CW |
237 | { |
238 | struct sg_table *pages; | |
f033428d CW |
239 | |
240 | if (i915_gem_object_has_pinned_pages(obj)) | |
241 | return -EBUSY; | |
242 | ||
a3258dbd TH |
243 | /* May be called by shrinker from within get_pages() (on another bo) */ |
244 | assert_object_held_shared(obj); | |
245 | ||
cc662126 AJ |
246 | i915_gem_object_release_mmap_offset(obj); |
247 | ||
f033428d CW |
248 | /* |
249 | * ->put_pages might need to allocate memory for the bit17 swizzle | |
250 | * array, hence protect them from being reaped by removing them from gtt | |
251 | * lists early. | |
252 | */ | |
253 | pages = __i915_gem_object_unset_pages(obj); | |
254 | ||
255 | /* | |
256 | * XXX Temporary hijinx to avoid updating all backends to handle | |
257 | * NULL pages. In the future, when we have more asynchronous | |
258 | * get_pages backends we should be better able to handle the | |
259 | * cancellation of the async task in a more uniform manner. | |
260 | */ | |
abd2f577 | 261 | if (!IS_ERR_OR_NULL(pages)) |
f033428d CW |
262 | obj->ops->put_pages(obj, pages); |
263 | ||
abd2f577 ML |
264 | return 0; |
265 | } | |
266 | ||
f033428d | 267 | /* The 'mapping' part of i915_gem_object_pin_map() below */ |
534a6687 | 268 | static void *i915_gem_object_map_page(struct drm_i915_gem_object *obj, |
cb2ce93e | 269 | enum i915_map_type type) |
f033428d | 270 | { |
534a6687 CH |
271 | unsigned long n_pages = obj->base.size >> PAGE_SHIFT, i; |
272 | struct page *stack[32], **pages = stack, *page; | |
273 | struct sgt_iter iter; | |
f033428d | 274 | pgprot_t pgprot; |
534a6687 | 275 | void *vaddr; |
01377a0d | 276 | |
534a6687 CH |
277 | switch (type) { |
278 | default: | |
279 | MISSING_CASE(type); | |
280 | fallthrough; /* to use PAGE_KERNEL anyway */ | |
281 | case I915_MAP_WB: | |
4caf017e CW |
282 | /* |
283 | * On 32b, highmem using a finite set of indirect PTE (i.e. | |
284 | * vmap) to provide virtual mappings of the high pages. | |
285 | * As these are finite, map_new_virtual() must wait for some | |
286 | * other kmap() to finish when it runs out. If we map a large | |
287 | * number of objects, there is no method for it to tell us | |
288 | * to release the mappings, and we deadlock. | |
289 | * | |
290 | * However, if we make an explicit vmap of the page, that | |
291 | * uses a larger vmalloc arena, and also has the ability | |
292 | * to tell us to release unwanted mappings. Most importantly, | |
293 | * it will fail and propagate an error instead of waiting | |
294 | * forever. | |
295 | * | |
296 | * So if the page is beyond the 32b boundary, make an explicit | |
46ce3a62 | 297 | * vmap. |
4caf017e | 298 | */ |
534a6687 CH |
299 | if (n_pages == 1 && !PageHighMem(sg_page(obj->mm.pages->sgl))) |
300 | return page_address(sg_page(obj->mm.pages->sgl)); | |
f033428d CW |
301 | pgprot = PAGE_KERNEL; |
302 | break; | |
303 | case I915_MAP_WC: | |
304 | pgprot = pgprot_writecombine(PAGE_KERNEL_IO); | |
305 | break; | |
306 | } | |
f033428d | 307 | |
534a6687 CH |
308 | if (n_pages > ARRAY_SIZE(stack)) { |
309 | /* Too big for stack -- allocate temporary array instead */ | |
310 | pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL); | |
311 | if (!pages) | |
cb2ce93e | 312 | return ERR_PTR(-ENOMEM); |
534a6687 | 313 | } |
6056e500 | 314 | |
534a6687 CH |
315 | i = 0; |
316 | for_each_sgt_page(page, iter, obj->mm.pages) | |
317 | pages[i++] = page; | |
318 | vaddr = vmap(pages, n_pages, 0, pgprot); | |
319 | if (pages != stack) | |
320 | kvfree(pages); | |
37df0edf CW |
321 | |
322 | return vaddr ?: ERR_PTR(-ENOMEM); | |
534a6687 | 323 | } |
6056e500 | 324 | |
534a6687 | 325 | static void *i915_gem_object_map_pfn(struct drm_i915_gem_object *obj, |
cb2ce93e | 326 | enum i915_map_type type) |
534a6687 CH |
327 | { |
328 | resource_size_t iomap = obj->mm.region->iomap.base - | |
329 | obj->mm.region->region.start; | |
330 | unsigned long n_pfn = obj->base.size >> PAGE_SHIFT; | |
331 | unsigned long stack[32], *pfns = stack, i; | |
332 | struct sgt_iter iter; | |
333 | dma_addr_t addr; | |
334 | void *vaddr; | |
335 | ||
b3f450d9 | 336 | GEM_BUG_ON(type != I915_MAP_WC); |
6056e500 | 337 | |
534a6687 CH |
338 | if (n_pfn > ARRAY_SIZE(stack)) { |
339 | /* Too big for stack -- allocate temporary array instead */ | |
340 | pfns = kvmalloc_array(n_pfn, sizeof(*pfns), GFP_KERNEL); | |
341 | if (!pfns) | |
cb2ce93e | 342 | return ERR_PTR(-ENOMEM); |
6056e500 CW |
343 | } |
344 | ||
534a6687 CH |
345 | i = 0; |
346 | for_each_sgt_daddr(addr, iter, obj->mm.pages) | |
347 | pfns[i++] = (iomap + addr) >> PAGE_SHIFT; | |
348 | vaddr = vmap_pfn(pfns, n_pfn, pgprot_writecombine(PAGE_KERNEL_IO)); | |
349 | if (pfns != stack) | |
350 | kvfree(pfns); | |
37df0edf CW |
351 | |
352 | return vaddr ?: ERR_PTR(-ENOMEM); | |
f033428d CW |
353 | } |
354 | ||
355 | /* get, pin, and map the pages of the object into kernel space */ | |
356 | void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj, | |
357 | enum i915_map_type type) | |
358 | { | |
359 | enum i915_map_type has_type; | |
360 | bool pinned; | |
361 | void *ptr; | |
362 | int err; | |
363 | ||
c471748d | 364 | if (!i915_gem_object_has_struct_page(obj) && |
0ff37575 | 365 | !i915_gem_object_has_iomem(obj)) |
f033428d CW |
366 | return ERR_PTR(-ENXIO); |
367 | ||
30b9d1b3 MA |
368 | if (WARN_ON_ONCE(obj->flags & I915_BO_ALLOC_GPU_ONLY)) |
369 | return ERR_PTR(-EINVAL); | |
370 | ||
cf41a8f1 | 371 | assert_object_held(obj); |
f033428d CW |
372 | |
373 | pinned = !(type & I915_MAP_OVERRIDE); | |
374 | type &= ~I915_MAP_OVERRIDE; | |
375 | ||
376 | if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) { | |
377 | if (unlikely(!i915_gem_object_has_pages(obj))) { | |
378 | GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj)); | |
379 | ||
380 | err = ____i915_gem_object_get_pages(obj); | |
cf41a8f1 ML |
381 | if (err) |
382 | return ERR_PTR(err); | |
f033428d CW |
383 | |
384 | smp_mb__before_atomic(); | |
385 | } | |
386 | atomic_inc(&obj->mm.pages_pin_count); | |
387 | pinned = false; | |
388 | } | |
389 | GEM_BUG_ON(!i915_gem_object_has_pages(obj)); | |
390 | ||
b3f450d9 MA |
391 | /* |
392 | * For discrete our CPU mappings needs to be consistent in order to | |
393 | * function correctly on !x86. When mapping things through TTM, we use | |
394 | * the same rules to determine the caching type. | |
395 | * | |
396 | * The caching rules, starting from DG1: | |
397 | * | |
398 | * - If the object can be placed in device local-memory, then the | |
399 | * pages should be allocated and mapped as write-combined only. | |
400 | * | |
401 | * - Everything else is always allocated and mapped as write-back, | |
402 | * with the guarantee that everything is also coherent with the | |
403 | * GPU. | |
404 | * | |
405 | * Internal users of lmem are already expected to get this right, so no | |
406 | * fudging needed there. | |
407 | */ | |
408 | if (i915_gem_object_placement_possible(obj, INTEL_MEMORY_LOCAL)) { | |
409 | if (type != I915_MAP_WC && !obj->mm.n_placements) { | |
410 | ptr = ERR_PTR(-ENODEV); | |
411 | goto err_unpin; | |
412 | } | |
413 | ||
414 | type = I915_MAP_WC; | |
415 | } else if (IS_DGFX(to_i915(obj->base.dev))) { | |
416 | type = I915_MAP_WB; | |
417 | } | |
418 | ||
f033428d CW |
419 | ptr = page_unpack_bits(obj->mm.mapping, &has_type); |
420 | if (ptr && has_type != type) { | |
421 | if (pinned) { | |
cb2ce93e | 422 | ptr = ERR_PTR(-EBUSY); |
f033428d CW |
423 | goto err_unpin; |
424 | } | |
425 | ||
01377a0d | 426 | unmap_object(obj, ptr); |
f033428d CW |
427 | |
428 | ptr = obj->mm.mapping = NULL; | |
429 | } | |
430 | ||
431 | if (!ptr) { | |
f6c466b8 ML |
432 | err = i915_gem_object_wait_moving_fence(obj, true); |
433 | if (err) { | |
434 | ptr = ERR_PTR(err); | |
435 | goto err_unpin; | |
436 | } | |
437 | ||
bdd8b6c9 | 438 | if (GEM_WARN_ON(type == I915_MAP_WC && !pat_enabled())) |
cb2ce93e | 439 | ptr = ERR_PTR(-ENODEV); |
534a6687 CH |
440 | else if (i915_gem_object_has_struct_page(obj)) |
441 | ptr = i915_gem_object_map_page(obj, type); | |
442 | else | |
443 | ptr = i915_gem_object_map_pfn(obj, type); | |
cb2ce93e | 444 | if (IS_ERR(ptr)) |
f033428d | 445 | goto err_unpin; |
f033428d CW |
446 | |
447 | obj->mm.mapping = page_pack_bits(ptr, type); | |
448 | } | |
449 | ||
f033428d CW |
450 | return ptr; |
451 | ||
452 | err_unpin: | |
453 | atomic_dec(&obj->mm.pages_pin_count); | |
cf41a8f1 | 454 | return ptr; |
f033428d CW |
455 | } |
456 | ||
74827b53 ML |
457 | void *i915_gem_object_pin_map_unlocked(struct drm_i915_gem_object *obj, |
458 | enum i915_map_type type) | |
459 | { | |
460 | void *ret; | |
461 | ||
462 | i915_gem_object_lock(obj, NULL); | |
463 | ret = i915_gem_object_pin_map(obj, type); | |
464 | i915_gem_object_unlock(obj); | |
465 | ||
466 | return ret; | |
467 | } | |
468 | ||
f033428d CW |
469 | void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj, |
470 | unsigned long offset, | |
471 | unsigned long size) | |
472 | { | |
473 | enum i915_map_type has_type; | |
474 | void *ptr; | |
475 | ||
476 | GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); | |
477 | GEM_BUG_ON(range_overflows_t(typeof(obj->base.size), | |
478 | offset, size, obj->base.size)); | |
479 | ||
9bad40a2 | 480 | wmb(); /* let all previous writes be visible to coherent partners */ |
f033428d CW |
481 | obj->mm.dirty = true; |
482 | ||
483 | if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE) | |
484 | return; | |
485 | ||
486 | ptr = page_unpack_bits(obj->mm.mapping, &has_type); | |
487 | if (has_type == I915_MAP_WC) | |
488 | return; | |
489 | ||
490 | drm_clflush_virt_range(ptr + offset, size); | |
491 | if (size == obj->base.size) { | |
492 | obj->write_domain &= ~I915_GEM_DOMAIN_CPU; | |
493 | obj->cache_dirty = false; | |
494 | } | |
495 | } | |
496 | ||
89d19b2b CW |
497 | void __i915_gem_object_release_map(struct drm_i915_gem_object *obj) |
498 | { | |
499 | GEM_BUG_ON(!obj->mm.mapping); | |
500 | ||
501 | /* | |
502 | * We allow removing the mapping from underneath pinned pages! | |
503 | * | |
504 | * Furthermore, since this is an unsafe operation reserved only | |
505 | * for construction time manipulation, we ignore locking prudence. | |
506 | */ | |
507 | unmap_object(obj, page_mask_bits(fetch_and_zero(&obj->mm.mapping))); | |
508 | ||
509 | i915_gem_object_unpin_map(obj); | |
510 | } | |
511 | ||
f033428d | 512 | struct scatterlist * |
934941ed TU |
513 | __i915_gem_object_get_sg(struct drm_i915_gem_object *obj, |
514 | struct i915_gem_object_page_iter *iter, | |
515 | unsigned int n, | |
0edbb9ba | 516 | unsigned int *offset, |
7d6a276e | 517 | bool dma) |
f033428d | 518 | { |
f033428d CW |
519 | struct scatterlist *sg; |
520 | unsigned int idx, count; | |
521 | ||
522 | might_sleep(); | |
523 | GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT); | |
d1487389 TH |
524 | if (!i915_gem_object_has_pinned_pages(obj)) |
525 | assert_object_held(obj); | |
f033428d CW |
526 | |
527 | /* As we iterate forward through the sg, we record each entry in a | |
528 | * radixtree for quick repeated (backwards) lookups. If we have seen | |
529 | * this index previously, we will have an entry for it. | |
530 | * | |
531 | * Initial lookup is O(N), but this is amortized to O(1) for | |
532 | * sequential page access (where each new request is consecutive | |
533 | * to the previous one). Repeated lookups are O(lg(obj->base.size)), | |
534 | * i.e. O(1) with a large constant! | |
535 | */ | |
536 | if (n < READ_ONCE(iter->sg_idx)) | |
537 | goto lookup; | |
538 | ||
539 | mutex_lock(&iter->lock); | |
540 | ||
541 | /* We prefer to reuse the last sg so that repeated lookup of this | |
542 | * (or the subsequent) sg are fast - comparing against the last | |
543 | * sg is faster than going through the radixtree. | |
544 | */ | |
545 | ||
546 | sg = iter->sg_pos; | |
547 | idx = iter->sg_idx; | |
934941ed | 548 | count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg); |
f033428d CW |
549 | |
550 | while (idx + count <= n) { | |
551 | void *entry; | |
552 | unsigned long i; | |
553 | int ret; | |
554 | ||
555 | /* If we cannot allocate and insert this entry, or the | |
556 | * individual pages from this range, cancel updating the | |
557 | * sg_idx so that on this lookup we are forced to linearly | |
558 | * scan onwards, but on future lookups we will try the | |
559 | * insertion again (in which case we need to be careful of | |
560 | * the error return reporting that we have already inserted | |
561 | * this index). | |
562 | */ | |
563 | ret = radix_tree_insert(&iter->radix, idx, sg); | |
564 | if (ret && ret != -EEXIST) | |
565 | goto scan; | |
566 | ||
567 | entry = xa_mk_value(idx); | |
568 | for (i = 1; i < count; i++) { | |
569 | ret = radix_tree_insert(&iter->radix, idx + i, entry); | |
570 | if (ret && ret != -EEXIST) | |
571 | goto scan; | |
572 | } | |
573 | ||
574 | idx += count; | |
575 | sg = ____sg_next(sg); | |
934941ed | 576 | count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg); |
f033428d CW |
577 | } |
578 | ||
579 | scan: | |
580 | iter->sg_pos = sg; | |
581 | iter->sg_idx = idx; | |
582 | ||
583 | mutex_unlock(&iter->lock); | |
584 | ||
585 | if (unlikely(n < idx)) /* insertion completed by another thread */ | |
586 | goto lookup; | |
587 | ||
7d6a276e | 588 | /* In case we failed to insert the entry into the radixtree, we need |
f033428d CW |
589 | * to look beyond the current sg. |
590 | */ | |
591 | while (idx + count <= n) { | |
592 | idx += count; | |
593 | sg = ____sg_next(sg); | |
934941ed | 594 | count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg); |
f033428d CW |
595 | } |
596 | ||
597 | *offset = n - idx; | |
598 | return sg; | |
599 | ||
600 | lookup: | |
601 | rcu_read_lock(); | |
602 | ||
603 | sg = radix_tree_lookup(&iter->radix, n); | |
604 | GEM_BUG_ON(!sg); | |
605 | ||
606 | /* If this index is in the middle of multi-page sg entry, | |
607 | * the radix tree will contain a value entry that points | |
608 | * to the start of that range. We will return the pointer to | |
609 | * the base page and the offset of this page within the | |
610 | * sg entry's range. | |
611 | */ | |
612 | *offset = 0; | |
613 | if (unlikely(xa_is_value(sg))) { | |
614 | unsigned long base = xa_to_value(sg); | |
615 | ||
616 | sg = radix_tree_lookup(&iter->radix, base); | |
617 | GEM_BUG_ON(!sg); | |
618 | ||
619 | *offset = n - base; | |
620 | } | |
621 | ||
622 | rcu_read_unlock(); | |
623 | ||
624 | return sg; | |
625 | } | |
626 | ||
627 | struct page * | |
628 | i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n) | |
629 | { | |
630 | struct scatterlist *sg; | |
631 | unsigned int offset; | |
632 | ||
633 | GEM_BUG_ON(!i915_gem_object_has_struct_page(obj)); | |
634 | ||
7d6a276e | 635 | sg = i915_gem_object_get_sg(obj, n, &offset); |
f033428d CW |
636 | return nth_page(sg_page(sg), offset); |
637 | } | |
638 | ||
4993a8a3 DA |
639 | /* Like i915_gem_object_get_page(), but mark the returned page dirty */ |
640 | struct page * | |
641 | i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, | |
642 | unsigned int n) | |
643 | { | |
644 | struct page *page; | |
645 | ||
646 | page = i915_gem_object_get_page(obj, n); | |
647 | if (!obj->mm.dirty) | |
648 | set_page_dirty(page); | |
649 | ||
650 | return page; | |
651 | } | |
652 | ||
f033428d CW |
653 | dma_addr_t |
654 | i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj, | |
655 | unsigned long n, | |
656 | unsigned int *len) | |
657 | { | |
658 | struct scatterlist *sg; | |
659 | unsigned int offset; | |
660 | ||
7d6a276e | 661 | sg = i915_gem_object_get_sg_dma(obj, n, &offset); |
f033428d CW |
662 | |
663 | if (len) | |
664 | *len = sg_dma_len(sg) - (offset << PAGE_SHIFT); | |
665 | ||
666 | return sg_dma_address(sg) + (offset << PAGE_SHIFT); | |
667 | } | |
668 | ||
669 | dma_addr_t | |
670 | i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, | |
671 | unsigned long n) | |
672 | { | |
673 | return i915_gem_object_get_dma_address_len(obj, n, NULL); | |
674 | } |