Commit | Line | Data |
---|---|---|
213d5092 TH |
1 | // SPDX-License-Identifier: MIT |
2 | /* | |
3 | * Copyright © 2021 Intel Corporation | |
4 | */ | |
5 | ||
82508de2 JN |
6 | #include <linux/shmem_fs.h> |
7 | ||
213d5092 | 8 | #include <drm/ttm/ttm_placement.h> |
a3185f91 | 9 | #include <drm/ttm/ttm_tt.h> |
93735059 | 10 | #include <drm/drm_buddy.h> |
213d5092 TH |
11 | |
12 | #include "i915_drv.h" | |
93735059 | 13 | #include "i915_ttm_buddy_manager.h" |
213d5092 TH |
14 | #include "intel_memory_region.h" |
15 | #include "intel_region_ttm.h" | |
16 | ||
c56ce956 | 17 | #include "gem/i915_gem_mman.h" |
213d5092 TH |
18 | #include "gem/i915_gem_object.h" |
19 | #include "gem/i915_gem_region.h" | |
20 | #include "gem/i915_gem_ttm.h" | |
3589fdbd | 21 | #include "gem/i915_gem_ttm_move.h" |
c56ce956 | 22 | #include "gem/i915_gem_ttm_pm.h" |
76a6d563 | 23 | #include "gt/intel_gpu_commands.h" |
213d5092 | 24 | |
213d5092 TH |
25 | #define I915_TTM_PRIO_PURGE 0 |
26 | #define I915_TTM_PRIO_NO_PAGES 1 | |
27 | #define I915_TTM_PRIO_HAS_PAGES 2 | |
93735059 | 28 | #define I915_TTM_PRIO_NEEDS_CPU_ACCESS 3 |
213d5092 | 29 | |
38f28c06 TH |
30 | /* |
31 | * Size of struct ttm_place vector in on-stack struct ttm_placement allocs | |
32 | */ | |
33 | #define I915_TTM_MAX_PLACEMENTS INTEL_REGION_UNKNOWN | |
34 | ||
213d5092 TH |
35 | /** |
36 | * struct i915_ttm_tt - TTM page vector with additional private information | |
37 | * @ttm: The base TTM page vector. | |
38 | * @dev: The struct device used for dma mapping and unmapping. | |
cad7109a | 39 | * @cached_rsgt: The cached scatter-gather table. |
7ae03459 MA |
40 | * @is_shmem: Set if using shmem. |
41 | * @filp: The shmem file, if using shmem backend. | |
213d5092 TH |
42 | * |
43 | * Note that DMA may be going on right up to the point where the page- | |
44 | * vector is unpopulated in delayed destroy. Hence keep the | |
45 | * scatter-gather table mapped and cached up to that point. This is | |
46 | * different from the cached gem object io scatter-gather table which | |
47 | * doesn't have an associated dma mapping. | |
48 | */ | |
49 | struct i915_ttm_tt { | |
50 | struct ttm_tt ttm; | |
51 | struct device *dev; | |
cad7109a | 52 | struct i915_refct_sgt cached_rsgt; |
7ae03459 MA |
53 | |
54 | bool is_shmem; | |
55 | struct file *filp; | |
213d5092 TH |
56 | }; |
57 | ||
38f28c06 TH |
58 | static const struct ttm_place sys_placement_flags = { |
59 | .fpfn = 0, | |
60 | .lpfn = 0, | |
61 | .mem_type = I915_PL_SYSTEM, | |
62 | .flags = 0, | |
213d5092 TH |
63 | }; |
64 | ||
65 | static struct ttm_placement i915_sys_placement = { | |
66 | .num_placement = 1, | |
38f28c06 | 67 | .placement = &sys_placement_flags, |
213d5092 TH |
68 | }; |
69 | ||
c56ce956 TH |
70 | /** |
71 | * i915_ttm_sys_placement - Return the struct ttm_placement to be | |
72 | * used for an object in system memory. | |
73 | * | |
74 | * Rather than making the struct extern, use this | |
75 | * function. | |
76 | * | |
77 | * Return: A pointer to a static variable for sys placement. | |
78 | */ | |
79 | struct ttm_placement *i915_ttm_sys_placement(void) | |
80 | { | |
81 | return &i915_sys_placement; | |
82 | } | |
83 | ||
b07a6483 TH |
84 | static int i915_ttm_err_to_gem(int err) |
85 | { | |
86 | /* Fastpath */ | |
87 | if (likely(!err)) | |
88 | return 0; | |
89 | ||
90 | switch (err) { | |
91 | case -EBUSY: | |
92 | /* | |
93 | * TTM likes to convert -EDEADLK to -EBUSY, and wants us to | |
94 | * restart the operation, since we don't record the contending | |
95 | * lock. We use -EAGAIN to restart. | |
96 | */ | |
97 | return -EAGAIN; | |
98 | case -ENOSPC: | |
99 | /* | |
100 | * Memory type / region is full, and we can't evict. | |
101 | * Except possibly system, that returns -ENOMEM; | |
102 | */ | |
103 | return -ENXIO; | |
104 | default: | |
105 | break; | |
106 | } | |
107 | ||
108 | return err; | |
109 | } | |
110 | ||
38f28c06 TH |
111 | static enum ttm_caching |
112 | i915_ttm_select_tt_caching(const struct drm_i915_gem_object *obj) | |
113 | { | |
114 | /* | |
2eda4fc6 MA |
115 | * Objects only allowed in system get cached cpu-mappings, or when |
116 | * evicting lmem-only buffers to system for swapping. Other objects get | |
117 | * WC mapping for now. Even if in system. | |
38f28c06 | 118 | */ |
2eda4fc6 | 119 | if (obj->mm.n_placements <= 1) |
38f28c06 TH |
120 | return ttm_cached; |
121 | ||
122 | return ttm_write_combined; | |
123 | } | |
124 | ||
125 | static void | |
126 | i915_ttm_place_from_region(const struct intel_memory_region *mr, | |
beb6a229 | 127 | struct ttm_place *place, |
ecbf2060 MA |
128 | resource_size_t offset, |
129 | resource_size_t size, | |
beb6a229 | 130 | unsigned int flags) |
38f28c06 TH |
131 | { |
132 | memset(place, 0, sizeof(*place)); | |
133 | place->mem_type = intel_region_to_ttm_type(mr); | |
beb6a229 | 134 | |
66ddc693 MA |
135 | if (mr->type == INTEL_MEMORY_SYSTEM) |
136 | return; | |
137 | ||
beb6a229 | 138 | if (flags & I915_BO_ALLOC_CONTIGUOUS) |
30b9d1b3 | 139 | place->flags |= TTM_PL_FLAG_CONTIGUOUS; |
ecbf2060 | 140 | if (offset != I915_BO_INVALID_OFFSET) { |
6949aa0e | 141 | WARN_ON(overflows_type(offset >> PAGE_SHIFT, place->fpfn)); |
ecbf2060 | 142 | place->fpfn = offset >> PAGE_SHIFT; |
6949aa0e | 143 | WARN_ON(overflows_type(place->fpfn + (size >> PAGE_SHIFT), place->lpfn)); |
ecbf2060 | 144 | place->lpfn = place->fpfn + (size >> PAGE_SHIFT); |
3c0fa9f4 | 145 | } else if (resource_size(&mr->io) && resource_size(&mr->io) < mr->total) { |
30b9d1b3 MA |
146 | if (flags & I915_BO_ALLOC_GPU_ONLY) { |
147 | place->flags |= TTM_PL_FLAG_TOPDOWN; | |
148 | } else { | |
149 | place->fpfn = 0; | |
3c0fa9f4 VS |
150 | WARN_ON(overflows_type(resource_size(&mr->io) >> PAGE_SHIFT, place->lpfn)); |
151 | place->lpfn = resource_size(&mr->io) >> PAGE_SHIFT; | |
30b9d1b3 | 152 | } |
3312a4ac | 153 | } |
38f28c06 TH |
154 | } |
155 | ||
156 | static void | |
157 | i915_ttm_placement_from_obj(const struct drm_i915_gem_object *obj, | |
a78a8da5 | 158 | struct ttm_place *places, |
38f28c06 TH |
159 | struct ttm_placement *placement) |
160 | { | |
161 | unsigned int num_allowed = obj->mm.n_placements; | |
beb6a229 | 162 | unsigned int flags = obj->flags; |
38f28c06 TH |
163 | unsigned int i; |
164 | ||
38f28c06 | 165 | i915_ttm_place_from_region(num_allowed ? obj->mm.placements[0] : |
a78a8da5 | 166 | obj->mm.region, &places[0], obj->bo_offset, |
ecbf2060 | 167 | obj->base.size, flags); |
38f28c06 TH |
168 | |
169 | /* Cache this on object? */ | |
a78a8da5 SA |
170 | for (i = 0; i < num_allowed; ++i) { |
171 | i915_ttm_place_from_region(obj->mm.placements[i], | |
172 | &places[i + 1], obj->bo_offset, | |
173 | obj->base.size, flags); | |
174 | places[i + 1].flags |= TTM_PL_FLAG_FALLBACK; | |
38f28c06 TH |
175 | } |
176 | ||
a78a8da5 SA |
177 | placement->num_placement = num_allowed + 1; |
178 | placement->placement = places; | |
38f28c06 TH |
179 | } |
180 | ||
7ae03459 MA |
181 | static int i915_ttm_tt_shmem_populate(struct ttm_device *bdev, |
182 | struct ttm_tt *ttm, | |
183 | struct ttm_operation_ctx *ctx) | |
184 | { | |
185 | struct drm_i915_private *i915 = container_of(bdev, typeof(*i915), bdev); | |
186 | struct intel_memory_region *mr = i915->mm.regions[INTEL_MEMORY_SYSTEM]; | |
187 | struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); | |
78a07fe7 | 188 | const unsigned int max_segment = i915_sg_segment_size(i915->drm.dev); |
5719d4fe | 189 | const size_t size = (size_t)ttm->num_pages << PAGE_SHIFT; |
7ae03459 MA |
190 | struct file *filp = i915_tt->filp; |
191 | struct sgt_iter sgt_iter; | |
192 | struct sg_table *st; | |
193 | struct page *page; | |
194 | unsigned long i; | |
195 | int err; | |
196 | ||
197 | if (!filp) { | |
198 | struct address_space *mapping; | |
199 | gfp_t mask; | |
200 | ||
201 | filp = shmem_file_setup("i915-shmem-tt", size, VM_NORESERVE); | |
202 | if (IS_ERR(filp)) | |
203 | return PTR_ERR(filp); | |
204 | ||
205 | mask = GFP_HIGHUSER | __GFP_RECLAIMABLE; | |
206 | ||
207 | mapping = filp->f_mapping; | |
208 | mapping_set_gfp_mask(mapping, mask); | |
209 | GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM)); | |
210 | ||
211 | i915_tt->filp = filp; | |
212 | } | |
213 | ||
cad7109a TH |
214 | st = &i915_tt->cached_rsgt.table; |
215 | err = shmem_sg_alloc_table(i915, st, size, mr, filp->f_mapping, | |
216 | max_segment); | |
217 | if (err) | |
218 | return err; | |
7ae03459 | 219 | |
cad7109a TH |
220 | err = dma_map_sgtable(i915_tt->dev, st, DMA_BIDIRECTIONAL, |
221 | DMA_ATTR_SKIP_CPU_SYNC); | |
222 | if (err) | |
7ae03459 | 223 | goto err_free_st; |
7ae03459 MA |
224 | |
225 | i = 0; | |
226 | for_each_sgt_page(page, sgt_iter, st) | |
227 | ttm->pages[i++] = page; | |
228 | ||
229 | if (ttm->page_flags & TTM_TT_FLAG_SWAPPED) | |
230 | ttm->page_flags &= ~TTM_TT_FLAG_SWAPPED; | |
231 | ||
7ae03459 MA |
232 | return 0; |
233 | ||
234 | err_free_st: | |
cad7109a TH |
235 | shmem_sg_free_table(st, filp->f_mapping, false, false); |
236 | ||
7ae03459 MA |
237 | return err; |
238 | } | |
239 | ||
240 | static void i915_ttm_tt_shmem_unpopulate(struct ttm_tt *ttm) | |
241 | { | |
242 | struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); | |
243 | bool backup = ttm->page_flags & TTM_TT_FLAG_SWAPPED; | |
cad7109a TH |
244 | struct sg_table *st = &i915_tt->cached_rsgt.table; |
245 | ||
246 | shmem_sg_free_table(st, file_inode(i915_tt->filp)->i_mapping, | |
247 | backup, backup); | |
248 | } | |
7ae03459 | 249 | |
cad7109a TH |
250 | static void i915_ttm_tt_release(struct kref *ref) |
251 | { | |
252 | struct i915_ttm_tt *i915_tt = | |
253 | container_of(ref, typeof(*i915_tt), cached_rsgt.kref); | |
254 | struct sg_table *st = &i915_tt->cached_rsgt.table; | |
7ae03459 | 255 | |
cad7109a TH |
256 | GEM_WARN_ON(st->sgl); |
257 | ||
258 | kfree(i915_tt); | |
7ae03459 MA |
259 | } |
260 | ||
cad7109a TH |
261 | static const struct i915_refct_sgt_ops tt_rsgt_ops = { |
262 | .release = i915_ttm_tt_release | |
263 | }; | |
264 | ||
213d5092 TH |
265 | static struct ttm_tt *i915_ttm_tt_create(struct ttm_buffer_object *bo, |
266 | uint32_t page_flags) | |
267 | { | |
76a6d563 R |
268 | struct drm_i915_private *i915 = container_of(bo->bdev, typeof(*i915), |
269 | bdev); | |
213d5092 | 270 | struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); |
76a6d563 | 271 | unsigned long ccs_pages = 0; |
6385eb7a | 272 | enum ttm_caching caching; |
213d5092 TH |
273 | struct i915_ttm_tt *i915_tt; |
274 | int ret; | |
275 | ||
6667d78a | 276 | if (i915_ttm_is_ghost_object(bo)) |
6385eb7a TH |
277 | return NULL; |
278 | ||
213d5092 TH |
279 | i915_tt = kzalloc(sizeof(*i915_tt), GFP_KERNEL); |
280 | if (!i915_tt) | |
281 | return NULL; | |
282 | ||
516198d3 CK |
283 | if (obj->flags & I915_BO_ALLOC_CPU_CLEAR && (!bo->resource || |
284 | ttm_manager_type(bo->bdev, bo->resource->mem_type)->use_tt)) | |
43d46f0b | 285 | page_flags |= TTM_TT_FLAG_ZERO_ALLOC; |
213d5092 | 286 | |
6385eb7a | 287 | caching = i915_ttm_select_tt_caching(obj); |
7ae03459 MA |
288 | if (i915_gem_object_is_shrinkable(obj) && caching == ttm_cached) { |
289 | page_flags |= TTM_TT_FLAG_EXTERNAL | | |
290 | TTM_TT_FLAG_EXTERNAL_MAPPABLE; | |
291 | i915_tt->is_shmem = true; | |
213d5092 TH |
292 | } |
293 | ||
873fef88 | 294 | if (i915_gem_object_needs_ccs_pages(obj)) |
76a6d563 R |
295 | ccs_pages = DIV_ROUND_UP(DIV_ROUND_UP(bo->base.size, |
296 | NUM_BYTES_PER_CCS_BYTE), | |
297 | PAGE_SIZE); | |
298 | ||
299 | ret = ttm_tt_init(&i915_tt->ttm, bo, page_flags, caching, ccs_pages); | |
7ae03459 MA |
300 | if (ret) |
301 | goto err_free; | |
302 | ||
cad7109a TH |
303 | __i915_refct_sgt_init(&i915_tt->cached_rsgt, bo->base.size, |
304 | &tt_rsgt_ops); | |
305 | ||
213d5092 TH |
306 | i915_tt->dev = obj->base.dev->dev; |
307 | ||
308 | return &i915_tt->ttm; | |
7ae03459 MA |
309 | |
310 | err_free: | |
311 | kfree(i915_tt); | |
312 | return NULL; | |
313 | } | |
314 | ||
315 | static int i915_ttm_tt_populate(struct ttm_device *bdev, | |
316 | struct ttm_tt *ttm, | |
317 | struct ttm_operation_ctx *ctx) | |
318 | { | |
319 | struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); | |
320 | ||
321 | if (i915_tt->is_shmem) | |
322 | return i915_ttm_tt_shmem_populate(bdev, ttm, ctx); | |
323 | ||
324 | return ttm_pool_alloc(&bdev->pool, ttm, ctx); | |
213d5092 TH |
325 | } |
326 | ||
327 | static void i915_ttm_tt_unpopulate(struct ttm_device *bdev, struct ttm_tt *ttm) | |
328 | { | |
329 | struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); | |
cad7109a TH |
330 | struct sg_table *st = &i915_tt->cached_rsgt.table; |
331 | ||
332 | if (st->sgl) | |
333 | dma_unmap_sgtable(i915_tt->dev, st, DMA_BIDIRECTIONAL, 0); | |
213d5092 | 334 | |
7ae03459 MA |
335 | if (i915_tt->is_shmem) { |
336 | i915_ttm_tt_shmem_unpopulate(ttm); | |
337 | } else { | |
cad7109a | 338 | sg_free_table(st); |
7ae03459 | 339 | ttm_pool_free(&bdev->pool, ttm); |
213d5092 | 340 | } |
213d5092 TH |
341 | } |
342 | ||
343 | static void i915_ttm_tt_destroy(struct ttm_device *bdev, struct ttm_tt *ttm) | |
344 | { | |
345 | struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); | |
346 | ||
7ae03459 MA |
347 | if (i915_tt->filp) |
348 | fput(i915_tt->filp); | |
349 | ||
c865204e | 350 | ttm_tt_fini(ttm); |
cad7109a | 351 | i915_refct_sgt_put(&i915_tt->cached_rsgt); |
213d5092 TH |
352 | } |
353 | ||
354 | static bool i915_ttm_eviction_valuable(struct ttm_buffer_object *bo, | |
355 | const struct ttm_place *place) | |
356 | { | |
357 | struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); | |
358 | ||
6667d78a | 359 | if (i915_ttm_is_ghost_object(bo)) |
6385eb7a TH |
360 | return false; |
361 | ||
7ae03459 MA |
362 | /* |
363 | * EXTERNAL objects should never be swapped out by TTM, instead we need | |
364 | * to handle that ourselves. TTM will already skip such objects for us, | |
365 | * but we would like to avoid grabbing locks for no good reason. | |
366 | */ | |
367 | if (bo->ttm && bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL) | |
6164807d | 368 | return false; |
7ae03459 | 369 | |
213d5092 | 370 | /* Will do for now. Our pinned objects are still on TTM's LRU lists */ |
93735059 MA |
371 | if (!i915_gem_object_evictable(obj)) |
372 | return false; | |
373 | ||
92b2b55e | 374 | return ttm_bo_eviction_valuable(bo, place); |
213d5092 TH |
375 | } |
376 | ||
377 | static void i915_ttm_evict_flags(struct ttm_buffer_object *bo, | |
378 | struct ttm_placement *placement) | |
379 | { | |
380 | *placement = i915_sys_placement; | |
381 | } | |
382 | ||
3589fdbd TH |
383 | /** |
384 | * i915_ttm_free_cached_io_rsgt - Free object cached LMEM information | |
385 | * @obj: The GEM object | |
386 | * This function frees any LMEM-related information that is cached on | |
387 | * the object. For example the radix tree for fast page lookup and the | |
388 | * cached refcounted sg-table | |
389 | */ | |
390 | void i915_ttm_free_cached_io_rsgt(struct drm_i915_gem_object *obj) | |
213d5092 | 391 | { |
cf3e3e86 ML |
392 | struct radix_tree_iter iter; |
393 | void __rcu **slot; | |
394 | ||
cad7109a | 395 | if (!obj->ttm.cached_io_rsgt) |
cf3e3e86 ML |
396 | return; |
397 | ||
398 | rcu_read_lock(); | |
399 | radix_tree_for_each_slot(slot, &obj->ttm.get_io_page.radix, &iter, 0) | |
400 | radix_tree_delete(&obj->ttm.get_io_page.radix, iter.index); | |
401 | rcu_read_unlock(); | |
402 | ||
cad7109a TH |
403 | i915_refct_sgt_put(obj->ttm.cached_io_rsgt); |
404 | obj->ttm.cached_io_rsgt = NULL; | |
213d5092 TH |
405 | } |
406 | ||
3589fdbd TH |
407 | /** |
408 | * i915_ttm_purge - Clear an object of its memory | |
409 | * @obj: The object | |
410 | * | |
411 | * This function is called to clear an object of it's memory when it is | |
412 | * marked as not needed anymore. | |
413 | * | |
414 | * Return: 0 on success, negative error code on failure. | |
415 | */ | |
416 | int i915_ttm_purge(struct drm_i915_gem_object *obj) | |
213d5092 TH |
417 | { |
418 | struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); | |
7ae03459 MA |
419 | struct i915_ttm_tt *i915_tt = |
420 | container_of(bo->ttm, typeof(*i915_tt), ttm); | |
213d5092 TH |
421 | struct ttm_operation_ctx ctx = { |
422 | .interruptible = true, | |
423 | .no_wait_gpu = false, | |
424 | }; | |
425 | struct ttm_placement place = {}; | |
426 | int ret; | |
427 | ||
428 | if (obj->mm.madv == __I915_MADV_PURGED) | |
7ae03459 | 429 | return 0; |
213d5092 | 430 | |
213d5092 | 431 | ret = ttm_bo_validate(bo, &place, &ctx); |
7ae03459 MA |
432 | if (ret) |
433 | return ret; | |
434 | ||
435 | if (bo->ttm && i915_tt->filp) { | |
436 | /* | |
437 | * The below fput(which eventually calls shmem_truncate) might | |
438 | * be delayed by worker, so when directly called to purge the | |
439 | * pages(like by the shrinker) we should try to be more | |
440 | * aggressive and release the pages immediately. | |
441 | */ | |
442 | shmem_truncate_range(file_inode(i915_tt->filp), | |
443 | 0, (loff_t)-1); | |
444 | fput(fetch_and_zero(&i915_tt->filp)); | |
213d5092 | 445 | } |
7ae03459 MA |
446 | |
447 | obj->write_domain = 0; | |
448 | obj->read_domains = 0; | |
449 | i915_ttm_adjust_gem_after_move(obj); | |
cad7109a | 450 | i915_ttm_free_cached_io_rsgt(obj); |
7ae03459 | 451 | obj->mm.madv = __I915_MADV_PURGED; |
3589fdbd | 452 | |
7ae03459 MA |
453 | return 0; |
454 | } | |
455 | ||
ffa3fe08 | 456 | static int i915_ttm_shrink(struct drm_i915_gem_object *obj, unsigned int flags) |
7ae03459 MA |
457 | { |
458 | struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); | |
459 | struct i915_ttm_tt *i915_tt = | |
460 | container_of(bo->ttm, typeof(*i915_tt), ttm); | |
461 | struct ttm_operation_ctx ctx = { | |
462 | .interruptible = true, | |
ffa3fe08 | 463 | .no_wait_gpu = flags & I915_GEM_OBJECT_SHRINK_NO_GPU_WAIT, |
7ae03459 MA |
464 | }; |
465 | struct ttm_placement place = {}; | |
466 | int ret; | |
467 | ||
58c7ee06 | 468 | if (!bo->ttm || i915_ttm_cpu_maps_iomem(bo->resource)) |
7ae03459 MA |
469 | return 0; |
470 | ||
471 | GEM_BUG_ON(!i915_tt->is_shmem); | |
472 | ||
473 | if (!i915_tt->filp) | |
474 | return 0; | |
475 | ||
004746e4 TH |
476 | ret = ttm_bo_wait_ctx(bo, &ctx); |
477 | if (ret) | |
478 | return ret; | |
479 | ||
7ae03459 MA |
480 | switch (obj->mm.madv) { |
481 | case I915_MADV_DONTNEED: | |
482 | return i915_ttm_purge(obj); | |
483 | case __I915_MADV_PURGED: | |
484 | return 0; | |
485 | } | |
486 | ||
487 | if (bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED) | |
488 | return 0; | |
489 | ||
490 | bo->ttm->page_flags |= TTM_TT_FLAG_SWAPPED; | |
491 | ret = ttm_bo_validate(bo, &place, &ctx); | |
492 | if (ret) { | |
493 | bo->ttm->page_flags &= ~TTM_TT_FLAG_SWAPPED; | |
494 | return ret; | |
495 | } | |
496 | ||
ffa3fe08 | 497 | if (flags & I915_GEM_OBJECT_SHRINK_WRITEBACK) |
7ae03459 MA |
498 | __shmem_writeback(obj->base.size, i915_tt->filp->f_mapping); |
499 | ||
500 | return 0; | |
213d5092 TH |
501 | } |
502 | ||
213d5092 TH |
503 | static void i915_ttm_delete_mem_notify(struct ttm_buffer_object *bo) |
504 | { | |
505 | struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); | |
ad74457a | 506 | |
58c7ee06 MA |
507 | /* |
508 | * This gets called twice by ttm, so long as we have a ttm resource or | |
509 | * ttm_tt then we can still safely call this. Due to pipeline-gutting, | |
510 | * we maybe have NULL bo->resource, but in that case we should always | |
511 | * have a ttm alive (like if the pages are swapped out). | |
512 | */ | |
513 | if ((bo->resource || bo->ttm) && !i915_ttm_is_ghost_object(bo)) { | |
068396bb | 514 | __i915_gem_object_pages_fini(obj); |
cad7109a | 515 | i915_ttm_free_cached_io_rsgt(obj); |
213d5092 TH |
516 | } |
517 | } | |
518 | ||
cad7109a | 519 | static struct i915_refct_sgt *i915_ttm_tt_get_st(struct ttm_tt *ttm) |
213d5092 TH |
520 | { |
521 | struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); | |
213d5092 TH |
522 | struct sg_table *st; |
523 | int ret; | |
524 | ||
cad7109a TH |
525 | if (i915_tt->cached_rsgt.table.sgl) |
526 | return i915_refct_sgt_get(&i915_tt->cached_rsgt); | |
213d5092 | 527 | |
cad7109a | 528 | st = &i915_tt->cached_rsgt.table; |
23852bec LT |
529 | ret = sg_alloc_table_from_pages_segment(st, |
530 | ttm->pages, ttm->num_pages, | |
531 | 0, (unsigned long)ttm->num_pages << PAGE_SHIFT, | |
78a07fe7 | 532 | i915_sg_segment_size(i915_tt->dev), GFP_KERNEL); |
23852bec | 533 | if (ret) { |
cad7109a | 534 | st->sgl = NULL; |
23852bec | 535 | return ERR_PTR(ret); |
213d5092 TH |
536 | } |
537 | ||
538 | ret = dma_map_sgtable(i915_tt->dev, st, DMA_BIDIRECTIONAL, 0); | |
539 | if (ret) { | |
540 | sg_free_table(st); | |
213d5092 TH |
541 | return ERR_PTR(ret); |
542 | } | |
543 | ||
cad7109a | 544 | return i915_refct_sgt_get(&i915_tt->cached_rsgt); |
213d5092 TH |
545 | } |
546 | ||
3589fdbd TH |
547 | /** |
548 | * i915_ttm_resource_get_st - Get a refcounted sg-table pointing to the | |
549 | * resource memory | |
550 | * @obj: The GEM object used for sg-table caching | |
551 | * @res: The struct ttm_resource for which an sg-table is requested. | |
552 | * | |
553 | * This function returns a refcounted sg-table representing the memory | |
554 | * pointed to by @res. If @res is the object's current resource it may also | |
555 | * cache the sg_table on the object or attempt to access an already cached | |
556 | * sg-table. The refcounted sg-table needs to be put when no-longer in use. | |
557 | * | |
558 | * Return: A valid pointer to a struct i915_refct_sgt or error pointer on | |
559 | * failure. | |
560 | */ | |
561 | struct i915_refct_sgt * | |
213d5092 TH |
562 | i915_ttm_resource_get_st(struct drm_i915_gem_object *obj, |
563 | struct ttm_resource *res) | |
564 | { | |
565 | struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); | |
9306b2b2 | 566 | u32 page_alignment; |
213d5092 | 567 | |
3589fdbd | 568 | if (!i915_ttm_gtt_binds_lmem(res)) |
213d5092 TH |
569 | return i915_ttm_tt_get_st(bo->ttm); |
570 | ||
bc99f120 MA |
571 | page_alignment = bo->page_alignment << PAGE_SHIFT; |
572 | if (!page_alignment) | |
573 | page_alignment = obj->mm.region->min_page_size; | |
574 | ||
3c2b8f32 TH |
575 | /* |
576 | * If CPU mapping differs, we need to add the ttm_tt pages to | |
577 | * the resulting st. Might make sense for GGTT. | |
578 | */ | |
3589fdbd | 579 | GEM_WARN_ON(!i915_ttm_cpu_maps_iomem(res)); |
cad7109a TH |
580 | if (bo->resource == res) { |
581 | if (!obj->ttm.cached_io_rsgt) { | |
582 | struct i915_refct_sgt *rsgt; | |
583 | ||
584 | rsgt = intel_region_ttm_resource_to_rsgt(obj->mm.region, | |
bc99f120 MA |
585 | res, |
586 | page_alignment); | |
cad7109a TH |
587 | if (IS_ERR(rsgt)) |
588 | return rsgt; | |
589 | ||
590 | obj->ttm.cached_io_rsgt = rsgt; | |
591 | } | |
592 | return i915_refct_sgt_get(obj->ttm.cached_io_rsgt); | |
593 | } | |
594 | ||
bc99f120 MA |
595 | return intel_region_ttm_resource_to_rsgt(obj->mm.region, res, |
596 | page_alignment); | |
213d5092 TH |
597 | } |
598 | ||
6ef295e3 MA |
599 | static int i915_ttm_truncate(struct drm_i915_gem_object *obj) |
600 | { | |
601 | struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); | |
58377de4 | 602 | long err; |
6ef295e3 MA |
603 | |
604 | WARN_ON_ONCE(obj->mm.madv == I915_MADV_WILLNEED); | |
605 | ||
58377de4 CK |
606 | err = dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, |
607 | true, 15 * HZ); | |
608 | if (err < 0) | |
5524b5e5 | 609 | return err; |
58377de4 CK |
610 | if (err == 0) |
611 | return -EBUSY; | |
5524b5e5 | 612 | |
6ef295e3 MA |
613 | err = i915_ttm_move_notify(bo); |
614 | if (err) | |
615 | return err; | |
616 | ||
617 | return i915_ttm_purge(obj); | |
618 | } | |
619 | ||
3589fdbd | 620 | static void i915_ttm_swap_notify(struct ttm_buffer_object *bo) |
213d5092 TH |
621 | { |
622 | struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); | |
6385eb7a TH |
623 | int ret; |
624 | ||
6667d78a | 625 | if (i915_ttm_is_ghost_object(bo)) |
6385eb7a | 626 | return; |
213d5092 | 627 | |
6385eb7a | 628 | ret = i915_ttm_move_notify(bo); |
3589fdbd TH |
629 | GEM_WARN_ON(ret); |
630 | GEM_WARN_ON(obj->ttm.cached_io_rsgt); | |
631 | if (!ret && obj->mm.madv != I915_MADV_WILLNEED) | |
213d5092 | 632 | i915_ttm_purge(obj); |
213d5092 TH |
633 | } |
634 | ||
bfe53be2 MA |
635 | /** |
636 | * i915_ttm_resource_mappable - Return true if the ttm resource is CPU | |
637 | * accessible. | |
638 | * @res: The TTM resource to check. | |
639 | * | |
640 | * This is interesting on small-BAR systems where we may encounter lmem objects | |
641 | * that can't be accessed via the CPU. | |
642 | */ | |
643 | bool i915_ttm_resource_mappable(struct ttm_resource *res) | |
503725c2 MA |
644 | { |
645 | struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res); | |
646 | ||
647 | if (!i915_ttm_cpu_maps_iomem(res)) | |
648 | return true; | |
649 | ||
e3c92eb4 | 650 | return bman_res->used_visible_size == PFN_UP(bman_res->base.size); |
503725c2 MA |
651 | } |
652 | ||
cf3e3e86 ML |
653 | static int i915_ttm_io_mem_reserve(struct ttm_device *bdev, struct ttm_resource *mem) |
654 | { | |
bfe53be2 MA |
655 | struct drm_i915_gem_object *obj = i915_ttm_to_gem(mem->bo); |
656 | bool unknown_state; | |
657 | ||
6667d78a | 658 | if (i915_ttm_is_ghost_object(mem->bo)) |
bfe53be2 MA |
659 | return -EINVAL; |
660 | ||
661 | if (!kref_get_unless_zero(&obj->base.refcount)) | |
662 | return -EINVAL; | |
663 | ||
664 | assert_object_held(obj); | |
665 | ||
666 | unknown_state = i915_gem_object_has_unknown_state(obj); | |
667 | i915_gem_object_put(obj); | |
668 | if (unknown_state) | |
669 | return -EINVAL; | |
670 | ||
3589fdbd | 671 | if (!i915_ttm_cpu_maps_iomem(mem)) |
cf3e3e86 ML |
672 | return 0; |
673 | ||
503725c2 MA |
674 | if (!i915_ttm_resource_mappable(mem)) |
675 | return -EINVAL; | |
676 | ||
cf3e3e86 ML |
677 | mem->bus.caching = ttm_write_combined; |
678 | mem->bus.is_iomem = true; | |
679 | ||
680 | return 0; | |
681 | } | |
682 | ||
683 | static unsigned long i915_ttm_io_mem_pfn(struct ttm_buffer_object *bo, | |
684 | unsigned long page_offset) | |
685 | { | |
686 | struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); | |
cf3e3e86 | 687 | struct scatterlist *sg; |
6385eb7a | 688 | unsigned long base; |
cf3e3e86 ML |
689 | unsigned int ofs; |
690 | ||
6667d78a | 691 | GEM_BUG_ON(i915_ttm_is_ghost_object(bo)); |
cf3e3e86 ML |
692 | GEM_WARN_ON(bo->ttm); |
693 | ||
6385eb7a | 694 | base = obj->mm.region->iomap.base - obj->mm.region->region.start; |
f47e6306 | 695 | sg = i915_gem_object_page_iter_get_sg(obj, &obj->ttm.get_io_page, page_offset, &ofs); |
cf3e3e86 ML |
696 | |
697 | return ((base + sg_dma_address(sg)) >> PAGE_SHIFT) + ofs; | |
698 | } | |
699 | ||
26b15eb0 MA |
700 | static int i915_ttm_access_memory(struct ttm_buffer_object *bo, |
701 | unsigned long offset, void *buf, | |
702 | int len, int write) | |
703 | { | |
704 | struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); | |
705 | resource_size_t iomap = obj->mm.region->iomap.base - | |
706 | obj->mm.region->region.start; | |
707 | unsigned long page = offset >> PAGE_SHIFT; | |
708 | unsigned long bytes_left = len; | |
709 | ||
710 | /* | |
711 | * TODO: For now just let it fail if the resource is non-mappable, | |
712 | * otherwise we need to perform the memcpy from the gpu here, without | |
713 | * interfering with the object (like moving the entire thing). | |
714 | */ | |
715 | if (!i915_ttm_resource_mappable(bo->resource)) | |
716 | return -EIO; | |
717 | ||
718 | offset -= page << PAGE_SHIFT; | |
719 | do { | |
720 | unsigned long bytes = min(bytes_left, PAGE_SIZE - offset); | |
721 | void __iomem *ptr; | |
722 | dma_addr_t daddr; | |
723 | ||
724 | daddr = i915_gem_object_get_dma_address(obj, page); | |
725 | ptr = ioremap_wc(iomap + daddr + offset, bytes); | |
726 | if (!ptr) | |
727 | return -EIO; | |
728 | ||
729 | if (write) | |
730 | memcpy_toio(ptr, buf, bytes); | |
731 | else | |
732 | memcpy_fromio(buf, ptr, bytes); | |
733 | iounmap(ptr); | |
734 | ||
735 | page++; | |
736 | buf += bytes; | |
737 | bytes_left -= bytes; | |
738 | offset = 0; | |
739 | } while (bytes_left); | |
740 | ||
741 | return len; | |
742 | } | |
743 | ||
6385eb7a TH |
744 | /* |
745 | * All callbacks need to take care not to downcast a struct ttm_buffer_object | |
746 | * without checking its subclass, since it might be a TTM ghost object. | |
747 | */ | |
213d5092 TH |
748 | static struct ttm_device_funcs i915_ttm_bo_driver = { |
749 | .ttm_tt_create = i915_ttm_tt_create, | |
7ae03459 | 750 | .ttm_tt_populate = i915_ttm_tt_populate, |
213d5092 TH |
751 | .ttm_tt_unpopulate = i915_ttm_tt_unpopulate, |
752 | .ttm_tt_destroy = i915_ttm_tt_destroy, | |
753 | .eviction_valuable = i915_ttm_eviction_valuable, | |
754 | .evict_flags = i915_ttm_evict_flags, | |
755 | .move = i915_ttm_move, | |
756 | .swap_notify = i915_ttm_swap_notify, | |
757 | .delete_mem_notify = i915_ttm_delete_mem_notify, | |
cf3e3e86 ML |
758 | .io_mem_reserve = i915_ttm_io_mem_reserve, |
759 | .io_mem_pfn = i915_ttm_io_mem_pfn, | |
26b15eb0 | 760 | .access_memory = i915_ttm_access_memory, |
213d5092 TH |
761 | }; |
762 | ||
763 | /** | |
764 | * i915_ttm_driver - Return a pointer to the TTM device funcs | |
765 | * | |
766 | * Return: Pointer to statically allocated TTM device funcs. | |
767 | */ | |
768 | struct ttm_device_funcs *i915_ttm_driver(void) | |
769 | { | |
770 | return &i915_ttm_bo_driver; | |
771 | } | |
772 | ||
b6e913e1 TH |
773 | static int __i915_ttm_get_pages(struct drm_i915_gem_object *obj, |
774 | struct ttm_placement *placement) | |
213d5092 TH |
775 | { |
776 | struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); | |
777 | struct ttm_operation_ctx ctx = { | |
778 | .interruptible = true, | |
779 | .no_wait_gpu = false, | |
780 | }; | |
787db3bb DG |
781 | struct ttm_placement initial_placement; |
782 | struct ttm_place initial_place; | |
213d5092 TH |
783 | int ret; |
784 | ||
b07a6483 | 785 | /* First try only the requested placement. No eviction. */ |
787db3bb DG |
786 | initial_placement.num_placement = 1; |
787 | memcpy(&initial_place, placement->placement, sizeof(struct ttm_place)); | |
788 | initial_place.flags |= TTM_PL_FLAG_DESIRED; | |
789 | initial_placement.placement = &initial_place; | |
790 | ret = ttm_bo_validate(bo, &initial_placement, &ctx); | |
b07a6483 TH |
791 | if (ret) { |
792 | ret = i915_ttm_err_to_gem(ret); | |
793 | /* | |
794 | * Anything that wants to restart the operation gets to | |
795 | * do that. | |
796 | */ | |
797 | if (ret == -EDEADLK || ret == -EINTR || ret == -ERESTARTSYS || | |
798 | ret == -EAGAIN) | |
799 | return ret; | |
213d5092 | 800 | |
b07a6483 TH |
801 | /* |
802 | * If the initial attempt fails, allow all accepted placements, | |
803 | * evicting if necessary. | |
804 | */ | |
b6e913e1 | 805 | ret = ttm_bo_validate(bo, placement, &ctx); |
b07a6483 TH |
806 | if (ret) |
807 | return i915_ttm_err_to_gem(ret); | |
808 | } | |
213d5092 | 809 | |
3c2b8f32 TH |
810 | if (bo->ttm && !ttm_tt_is_populated(bo->ttm)) { |
811 | ret = ttm_tt_populate(bo->bdev, bo->ttm, &ctx); | |
812 | if (ret) | |
813 | return ret; | |
814 | ||
815 | i915_ttm_adjust_domains_after_move(obj); | |
816 | i915_ttm_adjust_gem_after_move(obj); | |
817 | } | |
818 | ||
75e38285 | 819 | if (!i915_gem_object_has_pages(obj)) { |
cad7109a TH |
820 | struct i915_refct_sgt *rsgt = |
821 | i915_ttm_resource_get_st(obj, bo->resource); | |
822 | ||
823 | if (IS_ERR(rsgt)) | |
824 | return PTR_ERR(rsgt); | |
213d5092 | 825 | |
cad7109a TH |
826 | GEM_BUG_ON(obj->mm.rsgt); |
827 | obj->mm.rsgt = rsgt; | |
8c949515 | 828 | __i915_gem_object_set_pages(obj, &rsgt->table); |
75e38285 | 829 | } |
213d5092 | 830 | |
76a6d563 | 831 | GEM_BUG_ON(bo->ttm && ((obj->base.size >> PAGE_SHIFT) < bo->ttm->num_pages)); |
ebd4a8ec | 832 | i915_ttm_adjust_lru(obj); |
213d5092 TH |
833 | return ret; |
834 | } | |
835 | ||
b6e913e1 TH |
836 | static int i915_ttm_get_pages(struct drm_i915_gem_object *obj) |
837 | { | |
a78a8da5 | 838 | struct ttm_place places[I915_TTM_MAX_PLACEMENTS + 1]; |
b6e913e1 TH |
839 | struct ttm_placement placement; |
840 | ||
c3bfba9a CW |
841 | /* restricted by sg_alloc_table */ |
842 | if (overflows_type(obj->base.size >> PAGE_SHIFT, unsigned int)) | |
843 | return -E2BIG; | |
844 | ||
b6e913e1 TH |
845 | GEM_BUG_ON(obj->mm.n_placements > I915_TTM_MAX_PLACEMENTS); |
846 | ||
847 | /* Move to the requested placement. */ | |
a78a8da5 | 848 | i915_ttm_placement_from_obj(obj, places, &placement); |
b6e913e1 TH |
849 | |
850 | return __i915_ttm_get_pages(obj, &placement); | |
851 | } | |
852 | ||
853 | /** | |
854 | * DOC: Migration vs eviction | |
855 | * | |
856 | * GEM migration may not be the same as TTM migration / eviction. If | |
857 | * the TTM core decides to evict an object it may be evicted to a | |
858 | * TTM memory type that is not in the object's allowable GEM regions, or | |
859 | * in fact theoretically to a TTM memory type that doesn't correspond to | |
860 | * a GEM memory region. In that case the object's GEM region is not | |
861 | * updated, and the data is migrated back to the GEM region at | |
862 | * get_pages time. TTM may however set up CPU ptes to the object even | |
863 | * when it is evicted. | |
864 | * Gem forced migration using the i915_ttm_migrate() op, is allowed even | |
865 | * to regions that are not in the object's list of allowable placements. | |
866 | */ | |
503725c2 MA |
867 | static int __i915_ttm_migrate(struct drm_i915_gem_object *obj, |
868 | struct intel_memory_region *mr, | |
869 | unsigned int flags) | |
b6e913e1 TH |
870 | { |
871 | struct ttm_place requested; | |
872 | struct ttm_placement placement; | |
873 | int ret; | |
874 | ||
ecbf2060 MA |
875 | i915_ttm_place_from_region(mr, &requested, obj->bo_offset, |
876 | obj->base.size, flags); | |
b6e913e1 | 877 | placement.num_placement = 1; |
b6e913e1 | 878 | placement.placement = &requested; |
b6e913e1 TH |
879 | |
880 | ret = __i915_ttm_get_pages(obj, &placement); | |
881 | if (ret) | |
882 | return ret; | |
883 | ||
884 | /* | |
885 | * Reinitialize the region bindings. This is primarily | |
886 | * required for objects where the new region is not in | |
887 | * its allowable placements. | |
888 | */ | |
889 | if (obj->mm.region != mr) { | |
890 | i915_gem_object_release_memory_region(obj); | |
891 | i915_gem_object_init_memory_region(obj, mr); | |
892 | } | |
893 | ||
894 | return 0; | |
895 | } | |
896 | ||
503725c2 | 897 | static int i915_ttm_migrate(struct drm_i915_gem_object *obj, |
695ddc93 MA |
898 | struct intel_memory_region *mr, |
899 | unsigned int flags) | |
503725c2 | 900 | { |
695ddc93 | 901 | return __i915_ttm_migrate(obj, mr, flags); |
503725c2 MA |
902 | } |
903 | ||
213d5092 TH |
904 | static void i915_ttm_put_pages(struct drm_i915_gem_object *obj, |
905 | struct sg_table *st) | |
906 | { | |
907 | /* | |
908 | * We're currently not called from a shrinker, so put_pages() | |
909 | * typically means the object is about to destroyed, or called | |
910 | * from move_notify(). So just avoid doing much for now. | |
911 | * If the object is not destroyed next, The TTM eviction logic | |
912 | * and shrinkers will move it out if needed. | |
913 | */ | |
cad7109a TH |
914 | |
915 | if (obj->mm.rsgt) | |
916 | i915_refct_sgt_put(fetch_and_zero(&obj->mm.rsgt)); | |
213d5092 TH |
917 | } |
918 | ||
3589fdbd TH |
919 | /** |
920 | * i915_ttm_adjust_lru - Adjust an object's position on relevant LRU lists. | |
921 | * @obj: The object | |
922 | */ | |
923 | void i915_ttm_adjust_lru(struct drm_i915_gem_object *obj) | |
213d5092 TH |
924 | { |
925 | struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); | |
7ae03459 MA |
926 | struct i915_ttm_tt *i915_tt = |
927 | container_of(bo->ttm, typeof(*i915_tt), ttm); | |
ebd4a8ec MA |
928 | bool shrinkable = |
929 | bo->ttm && i915_tt->filp && ttm_tt_is_populated(bo->ttm); | |
213d5092 TH |
930 | |
931 | /* | |
932 | * Don't manipulate the TTM LRUs while in TTM bo destruction. | |
933 | * We're called through i915_ttm_delete_mem_notify(). | |
934 | */ | |
935 | if (!kref_read(&bo->kref)) | |
936 | return; | |
937 | ||
ebd4a8ec MA |
938 | /* |
939 | * We skip managing the shrinker LRU in set_pages() and just manage | |
940 | * everything here. This does at least solve the issue with having | |
941 | * temporary shmem mappings(like with evicted lmem) not being visible to | |
942 | * the shrinker. Only our shmem objects are shrinkable, everything else | |
943 | * we keep as unshrinkable. | |
944 | * | |
945 | * To make sure everything plays nice we keep an extra shrink pin in TTM | |
946 | * if the underlying pages are not currently shrinkable. Once we release | |
947 | * our pin, like when the pages are moved to shmem, the pages will then | |
948 | * be added to the shrinker LRU, assuming the caller isn't also holding | |
949 | * a pin. | |
950 | * | |
951 | * TODO: consider maybe also bumping the shrinker list here when we have | |
952 | * already unpinned it, which should give us something more like an LRU. | |
d3cb30f8 TH |
953 | * |
954 | * TODO: There is a small window of opportunity for this function to | |
955 | * get called from eviction after we've dropped the last GEM refcount, | |
956 | * but before the TTM deleted flag is set on the object. Avoid | |
957 | * adjusting the shrinker list in such cases, since the object is | |
958 | * not available to the shrinker anyway due to its zero refcount. | |
959 | * To fix this properly we should move to a TTM shrinker LRU list for | |
960 | * these objects. | |
ebd4a8ec | 961 | */ |
d3cb30f8 TH |
962 | if (kref_get_unless_zero(&obj->base.refcount)) { |
963 | if (shrinkable != obj->mm.ttm_shrinkable) { | |
964 | if (shrinkable) { | |
965 | if (obj->mm.madv == I915_MADV_WILLNEED) | |
966 | __i915_gem_object_make_shrinkable(obj); | |
967 | else | |
968 | __i915_gem_object_make_purgeable(obj); | |
969 | } else { | |
970 | i915_gem_object_make_unshrinkable(obj); | |
971 | } | |
972 | ||
973 | obj->mm.ttm_shrinkable = shrinkable; | |
ebd4a8ec | 974 | } |
d3cb30f8 | 975 | i915_gem_object_put(obj); |
ebd4a8ec MA |
976 | } |
977 | ||
213d5092 TH |
978 | /* |
979 | * Put on the correct LRU list depending on the MADV status | |
980 | */ | |
981 | spin_lock(&bo->bdev->lru_lock); | |
ebd4a8ec | 982 | if (shrinkable) { |
7ae03459 MA |
983 | /* Try to keep shmem_tt from being considered for shrinking. */ |
984 | bo->priority = TTM_MAX_BO_PRIORITY - 1; | |
985 | } else if (obj->mm.madv != I915_MADV_WILLNEED) { | |
213d5092 TH |
986 | bo->priority = I915_TTM_PRIO_PURGE; |
987 | } else if (!i915_gem_object_has_pages(obj)) { | |
ba2c5d15 | 988 | bo->priority = I915_TTM_PRIO_NO_PAGES; |
213d5092 | 989 | } else { |
93735059 MA |
990 | struct ttm_resource_manager *man = |
991 | ttm_manager_type(bo->bdev, bo->resource->mem_type); | |
992 | ||
993 | /* | |
994 | * If we need to place an LMEM resource which doesn't need CPU | |
995 | * access then we should try not to victimize mappable objects | |
996 | * first, since we likely end up stealing more of the mappable | |
997 | * portion. And likewise when we try to find space for a mappble | |
998 | * object, we know not to ever victimize objects that don't | |
999 | * occupy any mappable pages. | |
1000 | */ | |
1001 | if (i915_ttm_cpu_maps_iomem(bo->resource) && | |
1002 | i915_ttm_buddy_man_visible_size(man) < man->size && | |
1003 | !(obj->flags & I915_BO_ALLOC_GPU_ONLY)) | |
1004 | bo->priority = I915_TTM_PRIO_NEEDS_CPU_ACCESS; | |
1005 | else | |
1006 | bo->priority = I915_TTM_PRIO_HAS_PAGES; | |
213d5092 TH |
1007 | } |
1008 | ||
fee2ede1 | 1009 | ttm_bo_move_to_lru_tail(bo); |
213d5092 TH |
1010 | spin_unlock(&bo->bdev->lru_lock); |
1011 | } | |
1012 | ||
1013 | /* | |
1014 | * TTM-backed gem object destruction requires some clarification. | |
1015 | * Basically we have two possibilities here. We can either rely on the | |
1016 | * i915 delayed destruction and put the TTM object when the object | |
1017 | * is idle. This would be detected by TTM which would bypass the | |
1018 | * TTM delayed destroy handling. The other approach is to put the TTM | |
1019 | * object early and rely on the TTM destroyed handling, and then free | |
1020 | * the leftover parts of the GEM object once TTM's destroyed list handling is | |
1021 | * complete. For now, we rely on the latter for two reasons: | |
1022 | * a) TTM can evict an object even when it's on the delayed destroy list, | |
1023 | * which in theory allows for complete eviction. | |
1024 | * b) There is work going on in TTM to allow freeing an object even when | |
1025 | * it's not idle, and using the TTM destroyed list handling could help us | |
1026 | * benefit from that. | |
1027 | */ | |
1028 | static void i915_ttm_delayed_free(struct drm_i915_gem_object *obj) | |
1029 | { | |
068396bb TH |
1030 | GEM_BUG_ON(!obj->ttm.created); |
1031 | ||
1032 | ttm_bo_put(i915_gem_to_ttm(obj)); | |
213d5092 TH |
1033 | } |
1034 | ||
cf3e3e86 ML |
1035 | static vm_fault_t vm_fault_ttm(struct vm_fault *vmf) |
1036 | { | |
1037 | struct vm_area_struct *area = vmf->vma; | |
6385eb7a | 1038 | struct ttm_buffer_object *bo = area->vm_private_data; |
ebd4a8ec | 1039 | struct drm_device *dev = bo->base.dev; |
6667d78a | 1040 | struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); |
ad74457a | 1041 | intel_wakeref_t wakeref = 0; |
ebd4a8ec MA |
1042 | vm_fault_t ret; |
1043 | int idx; | |
cf3e3e86 ML |
1044 | |
1045 | /* Sanity check that we allow writing into this object */ | |
1046 | if (unlikely(i915_gem_object_is_readonly(obj) && | |
1047 | area->vm_flags & VM_WRITE)) | |
1048 | return VM_FAULT_SIGBUS; | |
1049 | ||
ebd4a8ec MA |
1050 | ret = ttm_bo_vm_reserve(bo, vmf); |
1051 | if (ret) | |
1052 | return ret; | |
1053 | ||
03ee5956 MA |
1054 | if (obj->mm.madv != I915_MADV_WILLNEED) { |
1055 | dma_resv_unlock(bo->base.resv); | |
1056 | return VM_FAULT_SIGBUS; | |
1057 | } | |
1058 | ||
516198d3 CK |
1059 | /* |
1060 | * This must be swapped out with shmem ttm_tt (pipeline-gutting). | |
1061 | * Calling ttm_bo_validate() here with TTM_PL_SYSTEM should only go as | |
1062 | * far as far doing a ttm_bo_move_null(), which should skip all the | |
1063 | * other junk. | |
1064 | */ | |
1065 | if (!bo->resource) { | |
1066 | struct ttm_operation_ctx ctx = { | |
1067 | .interruptible = true, | |
1068 | .no_wait_gpu = true, /* should be idle already */ | |
1069 | }; | |
fde789e8 | 1070 | int err; |
516198d3 CK |
1071 | |
1072 | GEM_BUG_ON(!bo->ttm || !(bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED)); | |
1073 | ||
fde789e8 MA |
1074 | err = ttm_bo_validate(bo, i915_ttm_sys_placement(), &ctx); |
1075 | if (err) { | |
516198d3 CK |
1076 | dma_resv_unlock(bo->base.resv); |
1077 | return VM_FAULT_SIGBUS; | |
1078 | } | |
1079 | } else if (!i915_ttm_resource_mappable(bo->resource)) { | |
503725c2 MA |
1080 | int err = -ENODEV; |
1081 | int i; | |
1082 | ||
1083 | for (i = 0; i < obj->mm.n_placements; i++) { | |
1084 | struct intel_memory_region *mr = obj->mm.placements[i]; | |
1085 | unsigned int flags; | |
1086 | ||
3c0fa9f4 | 1087 | if (!resource_size(&mr->io) && mr->type != INTEL_MEMORY_SYSTEM) |
503725c2 MA |
1088 | continue; |
1089 | ||
1090 | flags = obj->flags; | |
1091 | flags &= ~I915_BO_ALLOC_GPU_ONLY; | |
1092 | err = __i915_ttm_migrate(obj, mr, flags); | |
1093 | if (!err) | |
1094 | break; | |
1095 | } | |
1096 | ||
1097 | if (err) { | |
47cdb66a ND |
1098 | drm_dbg_ratelimited(dev, |
1099 | "Unable to make resource CPU accessible(err = %pe)\n", | |
1100 | ERR_PTR(err)); | |
503725c2 | 1101 | dma_resv_unlock(bo->base.resv); |
ad74457a AG |
1102 | ret = VM_FAULT_SIGBUS; |
1103 | goto out_rpm; | |
503725c2 MA |
1104 | } |
1105 | } | |
1106 | ||
625b7446 MA |
1107 | if (i915_ttm_cpu_maps_iomem(bo->resource)) |
1108 | wakeref = intel_runtime_pm_get(&to_i915(obj->base.dev)->runtime_pm); | |
1109 | ||
ebd4a8ec MA |
1110 | if (drm_dev_enter(dev, &idx)) { |
1111 | ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot, | |
be373fad | 1112 | TTM_BO_VM_NUM_PREFAULT); |
ebd4a8ec MA |
1113 | drm_dev_exit(idx); |
1114 | } else { | |
1115 | ret = ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot); | |
1116 | } | |
ad74457a | 1117 | |
ebd4a8ec | 1118 | if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) |
ad74457a AG |
1119 | goto out_rpm; |
1120 | ||
1cacd689 AG |
1121 | /* |
1122 | * ttm_bo_vm_reserve() already has dma_resv_lock. | |
1123 | * userfault_count is protected by dma_resv lock and rpm wakeref. | |
1124 | */ | |
ad74457a AG |
1125 | if (ret == VM_FAULT_NOPAGE && wakeref && !obj->userfault_count) { |
1126 | obj->userfault_count = 1; | |
1cacd689 | 1127 | spin_lock(&to_i915(obj->base.dev)->runtime_pm.lmem_userfault_lock); |
e66c8dcf | 1128 | list_add(&obj->userfault_link, &to_i915(obj->base.dev)->runtime_pm.lmem_userfault_list); |
1cacd689 | 1129 | spin_unlock(&to_i915(obj->base.dev)->runtime_pm.lmem_userfault_lock); |
ccb0e027 MA |
1130 | |
1131 | GEM_WARN_ON(!i915_ttm_cpu_maps_iomem(bo->resource)); | |
ad74457a AG |
1132 | } |
1133 | ||
1134 | if (wakeref & CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND) | |
e66c8dcf | 1135 | intel_wakeref_auto(&to_i915(obj->base.dev)->runtime_pm.userfault_wakeref, |
ad74457a | 1136 | msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)); |
ebd4a8ec MA |
1137 | |
1138 | i915_ttm_adjust_lru(obj); | |
1139 | ||
1140 | dma_resv_unlock(bo->base.resv); | |
ad74457a AG |
1141 | |
1142 | out_rpm: | |
1143 | if (wakeref) | |
1144 | intel_runtime_pm_put(&to_i915(obj->base.dev)->runtime_pm, wakeref); | |
1145 | ||
ebd4a8ec | 1146 | return ret; |
cf3e3e86 ML |
1147 | } |
1148 | ||
1149 | static int | |
1150 | vm_access_ttm(struct vm_area_struct *area, unsigned long addr, | |
1151 | void *buf, int len, int write) | |
1152 | { | |
1153 | struct drm_i915_gem_object *obj = | |
1154 | i915_ttm_to_gem(area->vm_private_data); | |
1155 | ||
1156 | if (i915_gem_object_is_readonly(obj) && write) | |
1157 | return -EACCES; | |
1158 | ||
1159 | return ttm_bo_vm_access(area, addr, buf, len, write); | |
1160 | } | |
1161 | ||
1162 | static void ttm_vm_open(struct vm_area_struct *vma) | |
1163 | { | |
1164 | struct drm_i915_gem_object *obj = | |
1165 | i915_ttm_to_gem(vma->vm_private_data); | |
1166 | ||
6667d78a | 1167 | GEM_BUG_ON(i915_ttm_is_ghost_object(vma->vm_private_data)); |
cf3e3e86 ML |
1168 | i915_gem_object_get(obj); |
1169 | } | |
1170 | ||
1171 | static void ttm_vm_close(struct vm_area_struct *vma) | |
1172 | { | |
1173 | struct drm_i915_gem_object *obj = | |
1174 | i915_ttm_to_gem(vma->vm_private_data); | |
1175 | ||
6667d78a | 1176 | GEM_BUG_ON(i915_ttm_is_ghost_object(vma->vm_private_data)); |
cf3e3e86 ML |
1177 | i915_gem_object_put(obj); |
1178 | } | |
1179 | ||
1180 | static const struct vm_operations_struct vm_ops_ttm = { | |
1181 | .fault = vm_fault_ttm, | |
1182 | .access = vm_access_ttm, | |
1183 | .open = ttm_vm_open, | |
1184 | .close = ttm_vm_close, | |
1185 | }; | |
1186 | ||
1187 | static u64 i915_ttm_mmap_offset(struct drm_i915_gem_object *obj) | |
1188 | { | |
1189 | /* The ttm_bo must be allocated with I915_BO_ALLOC_USER */ | |
1190 | GEM_BUG_ON(!drm_mm_node_allocated(&obj->base.vma_node.vm_node)); | |
1191 | ||
1192 | return drm_vma_node_offset_addr(&obj->base.vma_node); | |
1193 | } | |
1194 | ||
8ee262ba MA |
1195 | static void i915_ttm_unmap_virtual(struct drm_i915_gem_object *obj) |
1196 | { | |
1cacd689 AG |
1197 | struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); |
1198 | intel_wakeref_t wakeref = 0; | |
1199 | ||
1200 | assert_object_held_shared(obj); | |
1201 | ||
1202 | if (i915_ttm_cpu_maps_iomem(bo->resource)) { | |
1203 | wakeref = intel_runtime_pm_get(&to_i915(obj->base.dev)->runtime_pm); | |
1204 | ||
1205 | /* userfault_count is protected by obj lock and rpm wakeref. */ | |
1206 | if (obj->userfault_count) { | |
1207 | spin_lock(&to_i915(obj->base.dev)->runtime_pm.lmem_userfault_lock); | |
1208 | list_del(&obj->userfault_link); | |
1209 | spin_unlock(&to_i915(obj->base.dev)->runtime_pm.lmem_userfault_lock); | |
1210 | obj->userfault_count = 0; | |
1211 | } | |
1212 | } | |
1213 | ||
ccb0e027 MA |
1214 | GEM_WARN_ON(obj->userfault_count); |
1215 | ||
8ee262ba | 1216 | ttm_bo_unmap_virtual(i915_gem_to_ttm(obj)); |
1cacd689 AG |
1217 | |
1218 | if (wakeref) | |
1219 | intel_runtime_pm_put(&to_i915(obj->base.dev)->runtime_pm, wakeref); | |
8ee262ba MA |
1220 | } |
1221 | ||
4bc2d574 | 1222 | static const struct drm_i915_gem_object_ops i915_gem_ttm_obj_ops = { |
213d5092 | 1223 | .name = "i915_gem_object_ttm", |
5d12ffe6 MA |
1224 | .flags = I915_GEM_OBJECT_IS_SHRINKABLE | |
1225 | I915_GEM_OBJECT_SELF_MANAGED_SHRINK_LIST, | |
213d5092 TH |
1226 | |
1227 | .get_pages = i915_ttm_get_pages, | |
1228 | .put_pages = i915_ttm_put_pages, | |
6ef295e3 | 1229 | .truncate = i915_ttm_truncate, |
ffa3fe08 | 1230 | .shrink = i915_ttm_shrink, |
7ae03459 | 1231 | |
213d5092 TH |
1232 | .adjust_lru = i915_ttm_adjust_lru, |
1233 | .delayed_free = i915_ttm_delayed_free, | |
b6e913e1 | 1234 | .migrate = i915_ttm_migrate, |
7ae03459 | 1235 | |
cf3e3e86 | 1236 | .mmap_offset = i915_ttm_mmap_offset, |
8ee262ba | 1237 | .unmap_virtual = i915_ttm_unmap_virtual, |
cf3e3e86 | 1238 | .mmap_ops = &vm_ops_ttm, |
213d5092 TH |
1239 | }; |
1240 | ||
1241 | void i915_ttm_bo_destroy(struct ttm_buffer_object *bo) | |
1242 | { | |
1243 | struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); | |
1244 | ||
1245 | i915_gem_object_release_memory_region(obj); | |
cf3e3e86 | 1246 | mutex_destroy(&obj->ttm.get_io_page.lock); |
48b09612 | 1247 | |
068396bb | 1248 | if (obj->ttm.created) { |
ebd4a8ec MA |
1249 | /* |
1250 | * We freely manage the shrinker LRU outide of the mm.pages life | |
1251 | * cycle. As a result when destroying the object we should be | |
1252 | * extra paranoid and ensure we remove it from the LRU, before | |
1253 | * we free the object. | |
1254 | * | |
1255 | * Touching the ttm_shrinkable outside of the object lock here | |
1256 | * should be safe now that the last GEM object ref was dropped. | |
1257 | */ | |
1258 | if (obj->mm.ttm_shrinkable) | |
1259 | i915_gem_object_make_unshrinkable(obj); | |
1260 | ||
068396bb TH |
1261 | i915_ttm_backup_free(obj); |
1262 | ||
1263 | /* This releases all gem object bindings to the backend. */ | |
1264 | __i915_gem_free_object(obj); | |
1265 | ||
213d5092 | 1266 | call_rcu(&obj->rcu, __i915_gem_free_object_rcu); |
068396bb TH |
1267 | } else { |
1268 | __i915_gem_object_fini(obj); | |
1269 | } | |
213d5092 TH |
1270 | } |
1271 | ||
98a1dacc | 1272 | /* |
213d5092 TH |
1273 | * __i915_gem_ttm_object_init - Initialize a ttm-backed i915 gem object |
1274 | * @mem: The initial memory region for the object. | |
1275 | * @obj: The gem object. | |
1276 | * @size: Object size in bytes. | |
1277 | * @flags: gem object flags. | |
1278 | * | |
1279 | * Return: 0 on success, negative error code on failure. | |
1280 | */ | |
1281 | int __i915_gem_ttm_object_init(struct intel_memory_region *mem, | |
1282 | struct drm_i915_gem_object *obj, | |
9b78b5da | 1283 | resource_size_t offset, |
213d5092 | 1284 | resource_size_t size, |
d22632c8 | 1285 | resource_size_t page_size, |
213d5092 TH |
1286 | unsigned int flags) |
1287 | { | |
1288 | static struct lock_class_key lock_class; | |
1289 | struct drm_i915_private *i915 = mem->i915; | |
3c2b8f32 TH |
1290 | struct ttm_operation_ctx ctx = { |
1291 | .interruptible = true, | |
1292 | .no_wait_gpu = false, | |
1293 | }; | |
213d5092 | 1294 | enum ttm_bo_type bo_type; |
213d5092 TH |
1295 | int ret; |
1296 | ||
213d5092 TH |
1297 | drm_gem_private_object_init(&i915->drm, &obj->base, size); |
1298 | i915_gem_object_init(obj, &i915_gem_ttm_obj_ops, &lock_class, flags); | |
068396bb | 1299 | |
ecbf2060 MA |
1300 | obj->bo_offset = offset; |
1301 | ||
068396bb | 1302 | /* Don't put on a region list until we're either locked or fully initialized. */ |
8b1f7f92 | 1303 | obj->mm.region = mem; |
068396bb TH |
1304 | INIT_LIST_HEAD(&obj->mm.region_link); |
1305 | ||
cf3e3e86 ML |
1306 | INIT_RADIX_TREE(&obj->ttm.get_io_page.radix, GFP_KERNEL | __GFP_NOWARN); |
1307 | mutex_init(&obj->ttm.get_io_page.lock); | |
213d5092 TH |
1308 | bo_type = (obj->flags & I915_BO_ALLOC_USER) ? ttm_bo_type_device : |
1309 | ttm_bo_type_kernel; | |
1310 | ||
3c2b8f32 TH |
1311 | obj->base.vma_node.driver_private = i915_gem_to_ttm(obj); |
1312 | ||
d22632c8 MA |
1313 | /* Forcing the page size is kernel internal only */ |
1314 | GEM_BUG_ON(page_size && obj->mm.n_placements); | |
1315 | ||
ebd4a8ec MA |
1316 | /* |
1317 | * Keep an extra shrink pin to prevent the object from being made | |
1318 | * shrinkable too early. If the ttm_tt is ever allocated in shmem, we | |
1319 | * drop the pin. The TTM backend manages the shrinker LRU itself, | |
1320 | * outside of the normal mm.pages life cycle. | |
1321 | */ | |
1322 | i915_gem_object_make_unshrinkable(obj); | |
1323 | ||
213d5092 TH |
1324 | /* |
1325 | * If this function fails, it will call the destructor, but | |
1326 | * our caller still owns the object. So no freeing in the | |
1327 | * destructor until obj->ttm.created is true. | |
1328 | * Similarly, in delayed_destroy, we can't call ttm_bo_put() | |
1329 | * until successful initialization. | |
1330 | */ | |
347987a2 CK |
1331 | ret = ttm_bo_init_reserved(&i915->bdev, i915_gem_to_ttm(obj), bo_type, |
1332 | &i915_sys_placement, page_size >> PAGE_SHIFT, | |
3c2b8f32 | 1333 | &ctx, NULL, NULL, i915_ttm_bo_destroy); |
18f968cb GM |
1334 | |
1335 | /* | |
1336 | * XXX: The ttm_bo_init_reserved() functions returns -ENOSPC if the size | |
1337 | * is too big to add vma. The direct function that returns -ENOSPC is | |
1338 | * drm_mm_insert_node_in_range(). To handle the same error as other code | |
1339 | * that returns -E2BIG when the size is too large, it converts -ENOSPC to | |
1340 | * -E2BIG. | |
1341 | */ | |
1342 | if (size >> PAGE_SHIFT > INT_MAX && ret == -ENOSPC) | |
1343 | ret = -E2BIG; | |
1344 | ||
3c2b8f32 TH |
1345 | if (ret) |
1346 | return i915_ttm_err_to_gem(ret); | |
213d5092 | 1347 | |
3c2b8f32 | 1348 | obj->ttm.created = true; |
068396bb TH |
1349 | i915_gem_object_release_memory_region(obj); |
1350 | i915_gem_object_init_memory_region(obj, mem); | |
3c2b8f32 TH |
1351 | i915_ttm_adjust_domains_after_move(obj); |
1352 | i915_ttm_adjust_gem_after_move(obj); | |
1353 | i915_gem_object_unlock(obj); | |
213d5092 | 1354 | |
3c2b8f32 | 1355 | return 0; |
213d5092 | 1356 | } |
32b7cf51 TH |
1357 | |
1358 | static const struct intel_memory_region_ops ttm_system_region_ops = { | |
1359 | .init_object = __i915_gem_ttm_object_init, | |
8b1f7f92 | 1360 | .release = intel_region_ttm_fini, |
32b7cf51 TH |
1361 | }; |
1362 | ||
1363 | struct intel_memory_region * | |
1364 | i915_gem_ttm_system_setup(struct drm_i915_private *i915, | |
1365 | u16 type, u16 instance) | |
1366 | { | |
1367 | struct intel_memory_region *mr; | |
1368 | ||
1369 | mr = intel_memory_region_create(i915, 0, | |
1370 | totalram_pages() << PAGE_SHIFT, | |
235582ca | 1371 | PAGE_SIZE, 0, 0, |
32b7cf51 TH |
1372 | type, instance, |
1373 | &ttm_system_region_ops); | |
1374 | if (IS_ERR(mr)) | |
1375 | return mr; | |
1376 | ||
1377 | intel_memory_region_set_name(mr, "system-ttm"); | |
1378 | return mr; | |
213d5092 | 1379 | } |