Commit | Line | Data |
---|---|---|
213d5092 TH |
1 | // SPDX-License-Identifier: MIT |
2 | /* | |
3 | * Copyright © 2021 Intel Corporation | |
4 | */ | |
5 | ||
82508de2 JN |
6 | #include <linux/shmem_fs.h> |
7 | ||
213d5092 TH |
8 | #include <drm/ttm/ttm_bo_driver.h> |
9 | #include <drm/ttm/ttm_placement.h> | |
93735059 | 10 | #include <drm/drm_buddy.h> |
213d5092 TH |
11 | |
12 | #include "i915_drv.h" | |
93735059 | 13 | #include "i915_ttm_buddy_manager.h" |
213d5092 TH |
14 | #include "intel_memory_region.h" |
15 | #include "intel_region_ttm.h" | |
16 | ||
c56ce956 | 17 | #include "gem/i915_gem_mman.h" |
213d5092 TH |
18 | #include "gem/i915_gem_object.h" |
19 | #include "gem/i915_gem_region.h" | |
20 | #include "gem/i915_gem_ttm.h" | |
3589fdbd | 21 | #include "gem/i915_gem_ttm_move.h" |
c56ce956 | 22 | #include "gem/i915_gem_ttm_pm.h" |
76a6d563 | 23 | #include "gt/intel_gpu_commands.h" |
213d5092 | 24 | |
213d5092 TH |
25 | #define I915_TTM_PRIO_PURGE 0 |
26 | #define I915_TTM_PRIO_NO_PAGES 1 | |
27 | #define I915_TTM_PRIO_HAS_PAGES 2 | |
93735059 | 28 | #define I915_TTM_PRIO_NEEDS_CPU_ACCESS 3 |
213d5092 | 29 | |
38f28c06 TH |
30 | /* |
31 | * Size of struct ttm_place vector in on-stack struct ttm_placement allocs | |
32 | */ | |
33 | #define I915_TTM_MAX_PLACEMENTS INTEL_REGION_UNKNOWN | |
34 | ||
213d5092 TH |
35 | /** |
36 | * struct i915_ttm_tt - TTM page vector with additional private information | |
37 | * @ttm: The base TTM page vector. | |
38 | * @dev: The struct device used for dma mapping and unmapping. | |
cad7109a | 39 | * @cached_rsgt: The cached scatter-gather table. |
7ae03459 MA |
40 | * @is_shmem: Set if using shmem. |
41 | * @filp: The shmem file, if using shmem backend. | |
213d5092 TH |
42 | * |
43 | * Note that DMA may be going on right up to the point where the page- | |
44 | * vector is unpopulated in delayed destroy. Hence keep the | |
45 | * scatter-gather table mapped and cached up to that point. This is | |
46 | * different from the cached gem object io scatter-gather table which | |
47 | * doesn't have an associated dma mapping. | |
48 | */ | |
49 | struct i915_ttm_tt { | |
50 | struct ttm_tt ttm; | |
51 | struct device *dev; | |
cad7109a | 52 | struct i915_refct_sgt cached_rsgt; |
7ae03459 MA |
53 | |
54 | bool is_shmem; | |
55 | struct file *filp; | |
213d5092 TH |
56 | }; |
57 | ||
38f28c06 TH |
58 | static const struct ttm_place sys_placement_flags = { |
59 | .fpfn = 0, | |
60 | .lpfn = 0, | |
61 | .mem_type = I915_PL_SYSTEM, | |
62 | .flags = 0, | |
213d5092 TH |
63 | }; |
64 | ||
65 | static struct ttm_placement i915_sys_placement = { | |
66 | .num_placement = 1, | |
38f28c06 | 67 | .placement = &sys_placement_flags, |
213d5092 | 68 | .num_busy_placement = 1, |
38f28c06 | 69 | .busy_placement = &sys_placement_flags, |
213d5092 TH |
70 | }; |
71 | ||
c56ce956 TH |
72 | /** |
73 | * i915_ttm_sys_placement - Return the struct ttm_placement to be | |
74 | * used for an object in system memory. | |
75 | * | |
76 | * Rather than making the struct extern, use this | |
77 | * function. | |
78 | * | |
79 | * Return: A pointer to a static variable for sys placement. | |
80 | */ | |
81 | struct ttm_placement *i915_ttm_sys_placement(void) | |
82 | { | |
83 | return &i915_sys_placement; | |
84 | } | |
85 | ||
b07a6483 TH |
86 | static int i915_ttm_err_to_gem(int err) |
87 | { | |
88 | /* Fastpath */ | |
89 | if (likely(!err)) | |
90 | return 0; | |
91 | ||
92 | switch (err) { | |
93 | case -EBUSY: | |
94 | /* | |
95 | * TTM likes to convert -EDEADLK to -EBUSY, and wants us to | |
96 | * restart the operation, since we don't record the contending | |
97 | * lock. We use -EAGAIN to restart. | |
98 | */ | |
99 | return -EAGAIN; | |
100 | case -ENOSPC: | |
101 | /* | |
102 | * Memory type / region is full, and we can't evict. | |
103 | * Except possibly system, that returns -ENOMEM; | |
104 | */ | |
105 | return -ENXIO; | |
106 | default: | |
107 | break; | |
108 | } | |
109 | ||
110 | return err; | |
111 | } | |
112 | ||
38f28c06 TH |
113 | static enum ttm_caching |
114 | i915_ttm_select_tt_caching(const struct drm_i915_gem_object *obj) | |
115 | { | |
116 | /* | |
2eda4fc6 MA |
117 | * Objects only allowed in system get cached cpu-mappings, or when |
118 | * evicting lmem-only buffers to system for swapping. Other objects get | |
119 | * WC mapping for now. Even if in system. | |
38f28c06 | 120 | */ |
2eda4fc6 | 121 | if (obj->mm.n_placements <= 1) |
38f28c06 TH |
122 | return ttm_cached; |
123 | ||
124 | return ttm_write_combined; | |
125 | } | |
126 | ||
127 | static void | |
128 | i915_ttm_place_from_region(const struct intel_memory_region *mr, | |
beb6a229 | 129 | struct ttm_place *place, |
ecbf2060 MA |
130 | resource_size_t offset, |
131 | resource_size_t size, | |
beb6a229 | 132 | unsigned int flags) |
38f28c06 TH |
133 | { |
134 | memset(place, 0, sizeof(*place)); | |
135 | place->mem_type = intel_region_to_ttm_type(mr); | |
beb6a229 | 136 | |
66ddc693 MA |
137 | if (mr->type == INTEL_MEMORY_SYSTEM) |
138 | return; | |
139 | ||
beb6a229 | 140 | if (flags & I915_BO_ALLOC_CONTIGUOUS) |
30b9d1b3 | 141 | place->flags |= TTM_PL_FLAG_CONTIGUOUS; |
ecbf2060 MA |
142 | if (offset != I915_BO_INVALID_OFFSET) { |
143 | place->fpfn = offset >> PAGE_SHIFT; | |
144 | place->lpfn = place->fpfn + (size >> PAGE_SHIFT); | |
145 | } else if (mr->io_size && mr->io_size < mr->total) { | |
30b9d1b3 MA |
146 | if (flags & I915_BO_ALLOC_GPU_ONLY) { |
147 | place->flags |= TTM_PL_FLAG_TOPDOWN; | |
148 | } else { | |
149 | place->fpfn = 0; | |
150 | place->lpfn = mr->io_size >> PAGE_SHIFT; | |
151 | } | |
3312a4ac | 152 | } |
38f28c06 TH |
153 | } |
154 | ||
155 | static void | |
156 | i915_ttm_placement_from_obj(const struct drm_i915_gem_object *obj, | |
157 | struct ttm_place *requested, | |
158 | struct ttm_place *busy, | |
159 | struct ttm_placement *placement) | |
160 | { | |
161 | unsigned int num_allowed = obj->mm.n_placements; | |
beb6a229 | 162 | unsigned int flags = obj->flags; |
38f28c06 TH |
163 | unsigned int i; |
164 | ||
165 | placement->num_placement = 1; | |
166 | i915_ttm_place_from_region(num_allowed ? obj->mm.placements[0] : | |
ecbf2060 MA |
167 | obj->mm.region, requested, obj->bo_offset, |
168 | obj->base.size, flags); | |
38f28c06 TH |
169 | |
170 | /* Cache this on object? */ | |
171 | placement->num_busy_placement = num_allowed; | |
172 | for (i = 0; i < placement->num_busy_placement; ++i) | |
ecbf2060 MA |
173 | i915_ttm_place_from_region(obj->mm.placements[i], busy + i, |
174 | obj->bo_offset, obj->base.size, flags); | |
38f28c06 TH |
175 | |
176 | if (num_allowed == 0) { | |
177 | *busy = *requested; | |
178 | placement->num_busy_placement = 1; | |
179 | } | |
180 | ||
181 | placement->placement = requested; | |
182 | placement->busy_placement = busy; | |
183 | } | |
184 | ||
7ae03459 MA |
185 | static int i915_ttm_tt_shmem_populate(struct ttm_device *bdev, |
186 | struct ttm_tt *ttm, | |
187 | struct ttm_operation_ctx *ctx) | |
188 | { | |
189 | struct drm_i915_private *i915 = container_of(bdev, typeof(*i915), bdev); | |
190 | struct intel_memory_region *mr = i915->mm.regions[INTEL_MEMORY_SYSTEM]; | |
191 | struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); | |
78a07fe7 | 192 | const unsigned int max_segment = i915_sg_segment_size(i915->drm.dev); |
5719d4fe | 193 | const size_t size = (size_t)ttm->num_pages << PAGE_SHIFT; |
7ae03459 MA |
194 | struct file *filp = i915_tt->filp; |
195 | struct sgt_iter sgt_iter; | |
196 | struct sg_table *st; | |
197 | struct page *page; | |
198 | unsigned long i; | |
199 | int err; | |
200 | ||
201 | if (!filp) { | |
202 | struct address_space *mapping; | |
203 | gfp_t mask; | |
204 | ||
205 | filp = shmem_file_setup("i915-shmem-tt", size, VM_NORESERVE); | |
206 | if (IS_ERR(filp)) | |
207 | return PTR_ERR(filp); | |
208 | ||
209 | mask = GFP_HIGHUSER | __GFP_RECLAIMABLE; | |
210 | ||
211 | mapping = filp->f_mapping; | |
212 | mapping_set_gfp_mask(mapping, mask); | |
213 | GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM)); | |
214 | ||
215 | i915_tt->filp = filp; | |
216 | } | |
217 | ||
cad7109a TH |
218 | st = &i915_tt->cached_rsgt.table; |
219 | err = shmem_sg_alloc_table(i915, st, size, mr, filp->f_mapping, | |
220 | max_segment); | |
221 | if (err) | |
222 | return err; | |
7ae03459 | 223 | |
cad7109a TH |
224 | err = dma_map_sgtable(i915_tt->dev, st, DMA_BIDIRECTIONAL, |
225 | DMA_ATTR_SKIP_CPU_SYNC); | |
226 | if (err) | |
7ae03459 | 227 | goto err_free_st; |
7ae03459 MA |
228 | |
229 | i = 0; | |
230 | for_each_sgt_page(page, sgt_iter, st) | |
231 | ttm->pages[i++] = page; | |
232 | ||
233 | if (ttm->page_flags & TTM_TT_FLAG_SWAPPED) | |
234 | ttm->page_flags &= ~TTM_TT_FLAG_SWAPPED; | |
235 | ||
7ae03459 MA |
236 | return 0; |
237 | ||
238 | err_free_st: | |
cad7109a TH |
239 | shmem_sg_free_table(st, filp->f_mapping, false, false); |
240 | ||
7ae03459 MA |
241 | return err; |
242 | } | |
243 | ||
244 | static void i915_ttm_tt_shmem_unpopulate(struct ttm_tt *ttm) | |
245 | { | |
246 | struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); | |
247 | bool backup = ttm->page_flags & TTM_TT_FLAG_SWAPPED; | |
cad7109a TH |
248 | struct sg_table *st = &i915_tt->cached_rsgt.table; |
249 | ||
250 | shmem_sg_free_table(st, file_inode(i915_tt->filp)->i_mapping, | |
251 | backup, backup); | |
252 | } | |
7ae03459 | 253 | |
cad7109a TH |
254 | static void i915_ttm_tt_release(struct kref *ref) |
255 | { | |
256 | struct i915_ttm_tt *i915_tt = | |
257 | container_of(ref, typeof(*i915_tt), cached_rsgt.kref); | |
258 | struct sg_table *st = &i915_tt->cached_rsgt.table; | |
7ae03459 | 259 | |
cad7109a TH |
260 | GEM_WARN_ON(st->sgl); |
261 | ||
262 | kfree(i915_tt); | |
7ae03459 MA |
263 | } |
264 | ||
cad7109a TH |
265 | static const struct i915_refct_sgt_ops tt_rsgt_ops = { |
266 | .release = i915_ttm_tt_release | |
267 | }; | |
268 | ||
213d5092 TH |
269 | static struct ttm_tt *i915_ttm_tt_create(struct ttm_buffer_object *bo, |
270 | uint32_t page_flags) | |
271 | { | |
76a6d563 R |
272 | struct drm_i915_private *i915 = container_of(bo->bdev, typeof(*i915), |
273 | bdev); | |
213d5092 TH |
274 | struct ttm_resource_manager *man = |
275 | ttm_manager_type(bo->bdev, bo->resource->mem_type); | |
276 | struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); | |
76a6d563 | 277 | unsigned long ccs_pages = 0; |
6385eb7a | 278 | enum ttm_caching caching; |
213d5092 TH |
279 | struct i915_ttm_tt *i915_tt; |
280 | int ret; | |
281 | ||
6667d78a | 282 | if (i915_ttm_is_ghost_object(bo)) |
6385eb7a TH |
283 | return NULL; |
284 | ||
213d5092 TH |
285 | i915_tt = kzalloc(sizeof(*i915_tt), GFP_KERNEL); |
286 | if (!i915_tt) | |
287 | return NULL; | |
288 | ||
289 | if (obj->flags & I915_BO_ALLOC_CPU_CLEAR && | |
290 | man->use_tt) | |
43d46f0b | 291 | page_flags |= TTM_TT_FLAG_ZERO_ALLOC; |
213d5092 | 292 | |
6385eb7a | 293 | caching = i915_ttm_select_tt_caching(obj); |
7ae03459 MA |
294 | if (i915_gem_object_is_shrinkable(obj) && caching == ttm_cached) { |
295 | page_flags |= TTM_TT_FLAG_EXTERNAL | | |
296 | TTM_TT_FLAG_EXTERNAL_MAPPABLE; | |
297 | i915_tt->is_shmem = true; | |
213d5092 TH |
298 | } |
299 | ||
873fef88 | 300 | if (i915_gem_object_needs_ccs_pages(obj)) |
76a6d563 R |
301 | ccs_pages = DIV_ROUND_UP(DIV_ROUND_UP(bo->base.size, |
302 | NUM_BYTES_PER_CCS_BYTE), | |
303 | PAGE_SIZE); | |
304 | ||
305 | ret = ttm_tt_init(&i915_tt->ttm, bo, page_flags, caching, ccs_pages); | |
7ae03459 MA |
306 | if (ret) |
307 | goto err_free; | |
308 | ||
cad7109a TH |
309 | __i915_refct_sgt_init(&i915_tt->cached_rsgt, bo->base.size, |
310 | &tt_rsgt_ops); | |
311 | ||
213d5092 TH |
312 | i915_tt->dev = obj->base.dev->dev; |
313 | ||
314 | return &i915_tt->ttm; | |
7ae03459 MA |
315 | |
316 | err_free: | |
317 | kfree(i915_tt); | |
318 | return NULL; | |
319 | } | |
320 | ||
321 | static int i915_ttm_tt_populate(struct ttm_device *bdev, | |
322 | struct ttm_tt *ttm, | |
323 | struct ttm_operation_ctx *ctx) | |
324 | { | |
325 | struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); | |
326 | ||
327 | if (i915_tt->is_shmem) | |
328 | return i915_ttm_tt_shmem_populate(bdev, ttm, ctx); | |
329 | ||
330 | return ttm_pool_alloc(&bdev->pool, ttm, ctx); | |
213d5092 TH |
331 | } |
332 | ||
333 | static void i915_ttm_tt_unpopulate(struct ttm_device *bdev, struct ttm_tt *ttm) | |
334 | { | |
335 | struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); | |
cad7109a TH |
336 | struct sg_table *st = &i915_tt->cached_rsgt.table; |
337 | ||
338 | if (st->sgl) | |
339 | dma_unmap_sgtable(i915_tt->dev, st, DMA_BIDIRECTIONAL, 0); | |
213d5092 | 340 | |
7ae03459 MA |
341 | if (i915_tt->is_shmem) { |
342 | i915_ttm_tt_shmem_unpopulate(ttm); | |
343 | } else { | |
cad7109a | 344 | sg_free_table(st); |
7ae03459 | 345 | ttm_pool_free(&bdev->pool, ttm); |
213d5092 | 346 | } |
213d5092 TH |
347 | } |
348 | ||
349 | static void i915_ttm_tt_destroy(struct ttm_device *bdev, struct ttm_tt *ttm) | |
350 | { | |
351 | struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); | |
352 | ||
7ae03459 MA |
353 | if (i915_tt->filp) |
354 | fput(i915_tt->filp); | |
355 | ||
c865204e | 356 | ttm_tt_fini(ttm); |
cad7109a | 357 | i915_refct_sgt_put(&i915_tt->cached_rsgt); |
213d5092 TH |
358 | } |
359 | ||
360 | static bool i915_ttm_eviction_valuable(struct ttm_buffer_object *bo, | |
361 | const struct ttm_place *place) | |
362 | { | |
363 | struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); | |
364 | ||
6667d78a | 365 | if (i915_ttm_is_ghost_object(bo)) |
6385eb7a TH |
366 | return false; |
367 | ||
7ae03459 MA |
368 | /* |
369 | * EXTERNAL objects should never be swapped out by TTM, instead we need | |
370 | * to handle that ourselves. TTM will already skip such objects for us, | |
371 | * but we would like to avoid grabbing locks for no good reason. | |
372 | */ | |
373 | if (bo->ttm && bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL) | |
6164807d | 374 | return false; |
7ae03459 | 375 | |
213d5092 | 376 | /* Will do for now. Our pinned objects are still on TTM's LRU lists */ |
93735059 MA |
377 | if (!i915_gem_object_evictable(obj)) |
378 | return false; | |
379 | ||
92b2b55e | 380 | return ttm_bo_eviction_valuable(bo, place); |
213d5092 TH |
381 | } |
382 | ||
383 | static void i915_ttm_evict_flags(struct ttm_buffer_object *bo, | |
384 | struct ttm_placement *placement) | |
385 | { | |
386 | *placement = i915_sys_placement; | |
387 | } | |
388 | ||
3589fdbd TH |
389 | /** |
390 | * i915_ttm_free_cached_io_rsgt - Free object cached LMEM information | |
391 | * @obj: The GEM object | |
392 | * This function frees any LMEM-related information that is cached on | |
393 | * the object. For example the radix tree for fast page lookup and the | |
394 | * cached refcounted sg-table | |
395 | */ | |
396 | void i915_ttm_free_cached_io_rsgt(struct drm_i915_gem_object *obj) | |
213d5092 | 397 | { |
cf3e3e86 ML |
398 | struct radix_tree_iter iter; |
399 | void __rcu **slot; | |
400 | ||
cad7109a | 401 | if (!obj->ttm.cached_io_rsgt) |
cf3e3e86 ML |
402 | return; |
403 | ||
404 | rcu_read_lock(); | |
405 | radix_tree_for_each_slot(slot, &obj->ttm.get_io_page.radix, &iter, 0) | |
406 | radix_tree_delete(&obj->ttm.get_io_page.radix, iter.index); | |
407 | rcu_read_unlock(); | |
408 | ||
cad7109a TH |
409 | i915_refct_sgt_put(obj->ttm.cached_io_rsgt); |
410 | obj->ttm.cached_io_rsgt = NULL; | |
213d5092 TH |
411 | } |
412 | ||
3589fdbd TH |
413 | /** |
414 | * i915_ttm_purge - Clear an object of its memory | |
415 | * @obj: The object | |
416 | * | |
417 | * This function is called to clear an object of it's memory when it is | |
418 | * marked as not needed anymore. | |
419 | * | |
420 | * Return: 0 on success, negative error code on failure. | |
421 | */ | |
422 | int i915_ttm_purge(struct drm_i915_gem_object *obj) | |
213d5092 TH |
423 | { |
424 | struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); | |
7ae03459 MA |
425 | struct i915_ttm_tt *i915_tt = |
426 | container_of(bo->ttm, typeof(*i915_tt), ttm); | |
213d5092 TH |
427 | struct ttm_operation_ctx ctx = { |
428 | .interruptible = true, | |
429 | .no_wait_gpu = false, | |
430 | }; | |
431 | struct ttm_placement place = {}; | |
432 | int ret; | |
433 | ||
434 | if (obj->mm.madv == __I915_MADV_PURGED) | |
7ae03459 | 435 | return 0; |
213d5092 | 436 | |
213d5092 | 437 | ret = ttm_bo_validate(bo, &place, &ctx); |
7ae03459 MA |
438 | if (ret) |
439 | return ret; | |
440 | ||
441 | if (bo->ttm && i915_tt->filp) { | |
442 | /* | |
443 | * The below fput(which eventually calls shmem_truncate) might | |
444 | * be delayed by worker, so when directly called to purge the | |
445 | * pages(like by the shrinker) we should try to be more | |
446 | * aggressive and release the pages immediately. | |
447 | */ | |
448 | shmem_truncate_range(file_inode(i915_tt->filp), | |
449 | 0, (loff_t)-1); | |
450 | fput(fetch_and_zero(&i915_tt->filp)); | |
213d5092 | 451 | } |
7ae03459 MA |
452 | |
453 | obj->write_domain = 0; | |
454 | obj->read_domains = 0; | |
455 | i915_ttm_adjust_gem_after_move(obj); | |
cad7109a | 456 | i915_ttm_free_cached_io_rsgt(obj); |
7ae03459 | 457 | obj->mm.madv = __I915_MADV_PURGED; |
3589fdbd | 458 | |
7ae03459 MA |
459 | return 0; |
460 | } | |
461 | ||
ffa3fe08 | 462 | static int i915_ttm_shrink(struct drm_i915_gem_object *obj, unsigned int flags) |
7ae03459 MA |
463 | { |
464 | struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); | |
465 | struct i915_ttm_tt *i915_tt = | |
466 | container_of(bo->ttm, typeof(*i915_tt), ttm); | |
467 | struct ttm_operation_ctx ctx = { | |
468 | .interruptible = true, | |
ffa3fe08 | 469 | .no_wait_gpu = flags & I915_GEM_OBJECT_SHRINK_NO_GPU_WAIT, |
7ae03459 MA |
470 | }; |
471 | struct ttm_placement place = {}; | |
472 | int ret; | |
473 | ||
474 | if (!bo->ttm || bo->resource->mem_type != TTM_PL_SYSTEM) | |
475 | return 0; | |
476 | ||
477 | GEM_BUG_ON(!i915_tt->is_shmem); | |
478 | ||
479 | if (!i915_tt->filp) | |
480 | return 0; | |
481 | ||
004746e4 TH |
482 | ret = ttm_bo_wait_ctx(bo, &ctx); |
483 | if (ret) | |
484 | return ret; | |
485 | ||
7ae03459 MA |
486 | switch (obj->mm.madv) { |
487 | case I915_MADV_DONTNEED: | |
488 | return i915_ttm_purge(obj); | |
489 | case __I915_MADV_PURGED: | |
490 | return 0; | |
491 | } | |
492 | ||
493 | if (bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED) | |
494 | return 0; | |
495 | ||
496 | bo->ttm->page_flags |= TTM_TT_FLAG_SWAPPED; | |
497 | ret = ttm_bo_validate(bo, &place, &ctx); | |
498 | if (ret) { | |
499 | bo->ttm->page_flags &= ~TTM_TT_FLAG_SWAPPED; | |
500 | return ret; | |
501 | } | |
502 | ||
ffa3fe08 | 503 | if (flags & I915_GEM_OBJECT_SHRINK_WRITEBACK) |
7ae03459 MA |
504 | __shmem_writeback(obj->base.size, i915_tt->filp->f_mapping); |
505 | ||
506 | return 0; | |
213d5092 TH |
507 | } |
508 | ||
213d5092 TH |
509 | static void i915_ttm_delete_mem_notify(struct ttm_buffer_object *bo) |
510 | { | |
511 | struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); | |
ad74457a | 512 | |
6667d78a | 513 | if (bo->resource && !i915_ttm_is_ghost_object(bo)) { |
068396bb | 514 | __i915_gem_object_pages_fini(obj); |
cad7109a | 515 | i915_ttm_free_cached_io_rsgt(obj); |
213d5092 TH |
516 | } |
517 | } | |
518 | ||
cad7109a | 519 | static struct i915_refct_sgt *i915_ttm_tt_get_st(struct ttm_tt *ttm) |
213d5092 TH |
520 | { |
521 | struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); | |
213d5092 TH |
522 | struct sg_table *st; |
523 | int ret; | |
524 | ||
cad7109a TH |
525 | if (i915_tt->cached_rsgt.table.sgl) |
526 | return i915_refct_sgt_get(&i915_tt->cached_rsgt); | |
213d5092 | 527 | |
cad7109a | 528 | st = &i915_tt->cached_rsgt.table; |
23852bec LT |
529 | ret = sg_alloc_table_from_pages_segment(st, |
530 | ttm->pages, ttm->num_pages, | |
531 | 0, (unsigned long)ttm->num_pages << PAGE_SHIFT, | |
78a07fe7 | 532 | i915_sg_segment_size(i915_tt->dev), GFP_KERNEL); |
23852bec | 533 | if (ret) { |
cad7109a | 534 | st->sgl = NULL; |
23852bec | 535 | return ERR_PTR(ret); |
213d5092 TH |
536 | } |
537 | ||
538 | ret = dma_map_sgtable(i915_tt->dev, st, DMA_BIDIRECTIONAL, 0); | |
539 | if (ret) { | |
540 | sg_free_table(st); | |
213d5092 TH |
541 | return ERR_PTR(ret); |
542 | } | |
543 | ||
cad7109a | 544 | return i915_refct_sgt_get(&i915_tt->cached_rsgt); |
213d5092 TH |
545 | } |
546 | ||
3589fdbd TH |
547 | /** |
548 | * i915_ttm_resource_get_st - Get a refcounted sg-table pointing to the | |
549 | * resource memory | |
550 | * @obj: The GEM object used for sg-table caching | |
551 | * @res: The struct ttm_resource for which an sg-table is requested. | |
552 | * | |
553 | * This function returns a refcounted sg-table representing the memory | |
554 | * pointed to by @res. If @res is the object's current resource it may also | |
555 | * cache the sg_table on the object or attempt to access an already cached | |
556 | * sg-table. The refcounted sg-table needs to be put when no-longer in use. | |
557 | * | |
558 | * Return: A valid pointer to a struct i915_refct_sgt or error pointer on | |
559 | * failure. | |
560 | */ | |
561 | struct i915_refct_sgt * | |
213d5092 TH |
562 | i915_ttm_resource_get_st(struct drm_i915_gem_object *obj, |
563 | struct ttm_resource *res) | |
564 | { | |
565 | struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); | |
9306b2b2 | 566 | u32 page_alignment; |
213d5092 | 567 | |
3589fdbd | 568 | if (!i915_ttm_gtt_binds_lmem(res)) |
213d5092 TH |
569 | return i915_ttm_tt_get_st(bo->ttm); |
570 | ||
bc99f120 MA |
571 | page_alignment = bo->page_alignment << PAGE_SHIFT; |
572 | if (!page_alignment) | |
573 | page_alignment = obj->mm.region->min_page_size; | |
574 | ||
3c2b8f32 TH |
575 | /* |
576 | * If CPU mapping differs, we need to add the ttm_tt pages to | |
577 | * the resulting st. Might make sense for GGTT. | |
578 | */ | |
3589fdbd | 579 | GEM_WARN_ON(!i915_ttm_cpu_maps_iomem(res)); |
cad7109a TH |
580 | if (bo->resource == res) { |
581 | if (!obj->ttm.cached_io_rsgt) { | |
582 | struct i915_refct_sgt *rsgt; | |
583 | ||
584 | rsgt = intel_region_ttm_resource_to_rsgt(obj->mm.region, | |
bc99f120 MA |
585 | res, |
586 | page_alignment); | |
cad7109a TH |
587 | if (IS_ERR(rsgt)) |
588 | return rsgt; | |
589 | ||
590 | obj->ttm.cached_io_rsgt = rsgt; | |
591 | } | |
592 | return i915_refct_sgt_get(obj->ttm.cached_io_rsgt); | |
593 | } | |
594 | ||
bc99f120 MA |
595 | return intel_region_ttm_resource_to_rsgt(obj->mm.region, res, |
596 | page_alignment); | |
213d5092 TH |
597 | } |
598 | ||
6ef295e3 MA |
599 | static int i915_ttm_truncate(struct drm_i915_gem_object *obj) |
600 | { | |
601 | struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); | |
602 | int err; | |
603 | ||
604 | WARN_ON_ONCE(obj->mm.madv == I915_MADV_WILLNEED); | |
605 | ||
606 | err = i915_ttm_move_notify(bo); | |
607 | if (err) | |
608 | return err; | |
609 | ||
610 | return i915_ttm_purge(obj); | |
611 | } | |
612 | ||
3589fdbd | 613 | static void i915_ttm_swap_notify(struct ttm_buffer_object *bo) |
213d5092 TH |
614 | { |
615 | struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); | |
6385eb7a TH |
616 | int ret; |
617 | ||
6667d78a | 618 | if (i915_ttm_is_ghost_object(bo)) |
6385eb7a | 619 | return; |
213d5092 | 620 | |
6385eb7a | 621 | ret = i915_ttm_move_notify(bo); |
3589fdbd TH |
622 | GEM_WARN_ON(ret); |
623 | GEM_WARN_ON(obj->ttm.cached_io_rsgt); | |
624 | if (!ret && obj->mm.madv != I915_MADV_WILLNEED) | |
213d5092 | 625 | i915_ttm_purge(obj); |
213d5092 TH |
626 | } |
627 | ||
bfe53be2 MA |
628 | /** |
629 | * i915_ttm_resource_mappable - Return true if the ttm resource is CPU | |
630 | * accessible. | |
631 | * @res: The TTM resource to check. | |
632 | * | |
633 | * This is interesting on small-BAR systems where we may encounter lmem objects | |
634 | * that can't be accessed via the CPU. | |
635 | */ | |
636 | bool i915_ttm_resource_mappable(struct ttm_resource *res) | |
503725c2 MA |
637 | { |
638 | struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res); | |
639 | ||
640 | if (!i915_ttm_cpu_maps_iomem(res)) | |
641 | return true; | |
642 | ||
e3c92eb4 | 643 | return bman_res->used_visible_size == PFN_UP(bman_res->base.size); |
503725c2 MA |
644 | } |
645 | ||
cf3e3e86 ML |
646 | static int i915_ttm_io_mem_reserve(struct ttm_device *bdev, struct ttm_resource *mem) |
647 | { | |
bfe53be2 MA |
648 | struct drm_i915_gem_object *obj = i915_ttm_to_gem(mem->bo); |
649 | bool unknown_state; | |
650 | ||
6667d78a | 651 | if (i915_ttm_is_ghost_object(mem->bo)) |
bfe53be2 MA |
652 | return -EINVAL; |
653 | ||
654 | if (!kref_get_unless_zero(&obj->base.refcount)) | |
655 | return -EINVAL; | |
656 | ||
657 | assert_object_held(obj); | |
658 | ||
659 | unknown_state = i915_gem_object_has_unknown_state(obj); | |
660 | i915_gem_object_put(obj); | |
661 | if (unknown_state) | |
662 | return -EINVAL; | |
663 | ||
3589fdbd | 664 | if (!i915_ttm_cpu_maps_iomem(mem)) |
cf3e3e86 ML |
665 | return 0; |
666 | ||
503725c2 MA |
667 | if (!i915_ttm_resource_mappable(mem)) |
668 | return -EINVAL; | |
669 | ||
cf3e3e86 ML |
670 | mem->bus.caching = ttm_write_combined; |
671 | mem->bus.is_iomem = true; | |
672 | ||
673 | return 0; | |
674 | } | |
675 | ||
676 | static unsigned long i915_ttm_io_mem_pfn(struct ttm_buffer_object *bo, | |
677 | unsigned long page_offset) | |
678 | { | |
679 | struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); | |
cf3e3e86 | 680 | struct scatterlist *sg; |
6385eb7a | 681 | unsigned long base; |
cf3e3e86 ML |
682 | unsigned int ofs; |
683 | ||
6667d78a | 684 | GEM_BUG_ON(i915_ttm_is_ghost_object(bo)); |
cf3e3e86 ML |
685 | GEM_WARN_ON(bo->ttm); |
686 | ||
6385eb7a | 687 | base = obj->mm.region->iomap.base - obj->mm.region->region.start; |
7d6a276e | 688 | sg = __i915_gem_object_get_sg(obj, &obj->ttm.get_io_page, page_offset, &ofs, true); |
cf3e3e86 ML |
689 | |
690 | return ((base + sg_dma_address(sg)) >> PAGE_SHIFT) + ofs; | |
691 | } | |
692 | ||
26b15eb0 MA |
693 | static int i915_ttm_access_memory(struct ttm_buffer_object *bo, |
694 | unsigned long offset, void *buf, | |
695 | int len, int write) | |
696 | { | |
697 | struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); | |
698 | resource_size_t iomap = obj->mm.region->iomap.base - | |
699 | obj->mm.region->region.start; | |
700 | unsigned long page = offset >> PAGE_SHIFT; | |
701 | unsigned long bytes_left = len; | |
702 | ||
703 | /* | |
704 | * TODO: For now just let it fail if the resource is non-mappable, | |
705 | * otherwise we need to perform the memcpy from the gpu here, without | |
706 | * interfering with the object (like moving the entire thing). | |
707 | */ | |
708 | if (!i915_ttm_resource_mappable(bo->resource)) | |
709 | return -EIO; | |
710 | ||
711 | offset -= page << PAGE_SHIFT; | |
712 | do { | |
713 | unsigned long bytes = min(bytes_left, PAGE_SIZE - offset); | |
714 | void __iomem *ptr; | |
715 | dma_addr_t daddr; | |
716 | ||
717 | daddr = i915_gem_object_get_dma_address(obj, page); | |
718 | ptr = ioremap_wc(iomap + daddr + offset, bytes); | |
719 | if (!ptr) | |
720 | return -EIO; | |
721 | ||
722 | if (write) | |
723 | memcpy_toio(ptr, buf, bytes); | |
724 | else | |
725 | memcpy_fromio(buf, ptr, bytes); | |
726 | iounmap(ptr); | |
727 | ||
728 | page++; | |
729 | buf += bytes; | |
730 | bytes_left -= bytes; | |
731 | offset = 0; | |
732 | } while (bytes_left); | |
733 | ||
734 | return len; | |
735 | } | |
736 | ||
6385eb7a TH |
737 | /* |
738 | * All callbacks need to take care not to downcast a struct ttm_buffer_object | |
739 | * without checking its subclass, since it might be a TTM ghost object. | |
740 | */ | |
213d5092 TH |
741 | static struct ttm_device_funcs i915_ttm_bo_driver = { |
742 | .ttm_tt_create = i915_ttm_tt_create, | |
7ae03459 | 743 | .ttm_tt_populate = i915_ttm_tt_populate, |
213d5092 TH |
744 | .ttm_tt_unpopulate = i915_ttm_tt_unpopulate, |
745 | .ttm_tt_destroy = i915_ttm_tt_destroy, | |
746 | .eviction_valuable = i915_ttm_eviction_valuable, | |
747 | .evict_flags = i915_ttm_evict_flags, | |
748 | .move = i915_ttm_move, | |
749 | .swap_notify = i915_ttm_swap_notify, | |
750 | .delete_mem_notify = i915_ttm_delete_mem_notify, | |
cf3e3e86 ML |
751 | .io_mem_reserve = i915_ttm_io_mem_reserve, |
752 | .io_mem_pfn = i915_ttm_io_mem_pfn, | |
26b15eb0 | 753 | .access_memory = i915_ttm_access_memory, |
213d5092 TH |
754 | }; |
755 | ||
756 | /** | |
757 | * i915_ttm_driver - Return a pointer to the TTM device funcs | |
758 | * | |
759 | * Return: Pointer to statically allocated TTM device funcs. | |
760 | */ | |
761 | struct ttm_device_funcs *i915_ttm_driver(void) | |
762 | { | |
763 | return &i915_ttm_bo_driver; | |
764 | } | |
765 | ||
b6e913e1 TH |
766 | static int __i915_ttm_get_pages(struct drm_i915_gem_object *obj, |
767 | struct ttm_placement *placement) | |
213d5092 TH |
768 | { |
769 | struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); | |
770 | struct ttm_operation_ctx ctx = { | |
771 | .interruptible = true, | |
772 | .no_wait_gpu = false, | |
773 | }; | |
b07a6483 | 774 | int real_num_busy; |
213d5092 TH |
775 | int ret; |
776 | ||
b07a6483 | 777 | /* First try only the requested placement. No eviction. */ |
b6e913e1 TH |
778 | real_num_busy = fetch_and_zero(&placement->num_busy_placement); |
779 | ret = ttm_bo_validate(bo, placement, &ctx); | |
b07a6483 TH |
780 | if (ret) { |
781 | ret = i915_ttm_err_to_gem(ret); | |
782 | /* | |
783 | * Anything that wants to restart the operation gets to | |
784 | * do that. | |
785 | */ | |
786 | if (ret == -EDEADLK || ret == -EINTR || ret == -ERESTARTSYS || | |
787 | ret == -EAGAIN) | |
788 | return ret; | |
213d5092 | 789 | |
b07a6483 TH |
790 | /* |
791 | * If the initial attempt fails, allow all accepted placements, | |
792 | * evicting if necessary. | |
793 | */ | |
b6e913e1 TH |
794 | placement->num_busy_placement = real_num_busy; |
795 | ret = ttm_bo_validate(bo, placement, &ctx); | |
b07a6483 TH |
796 | if (ret) |
797 | return i915_ttm_err_to_gem(ret); | |
798 | } | |
213d5092 | 799 | |
3c2b8f32 TH |
800 | if (bo->ttm && !ttm_tt_is_populated(bo->ttm)) { |
801 | ret = ttm_tt_populate(bo->bdev, bo->ttm, &ctx); | |
802 | if (ret) | |
803 | return ret; | |
804 | ||
805 | i915_ttm_adjust_domains_after_move(obj); | |
806 | i915_ttm_adjust_gem_after_move(obj); | |
807 | } | |
808 | ||
75e38285 | 809 | if (!i915_gem_object_has_pages(obj)) { |
cad7109a TH |
810 | struct i915_refct_sgt *rsgt = |
811 | i915_ttm_resource_get_st(obj, bo->resource); | |
812 | ||
813 | if (IS_ERR(rsgt)) | |
814 | return PTR_ERR(rsgt); | |
213d5092 | 815 | |
cad7109a TH |
816 | GEM_BUG_ON(obj->mm.rsgt); |
817 | obj->mm.rsgt = rsgt; | |
818 | __i915_gem_object_set_pages(obj, &rsgt->table, | |
819 | i915_sg_dma_sizes(rsgt->table.sgl)); | |
75e38285 | 820 | } |
213d5092 | 821 | |
76a6d563 | 822 | GEM_BUG_ON(bo->ttm && ((obj->base.size >> PAGE_SHIFT) < bo->ttm->num_pages)); |
ebd4a8ec | 823 | i915_ttm_adjust_lru(obj); |
213d5092 TH |
824 | return ret; |
825 | } | |
826 | ||
b6e913e1 TH |
827 | static int i915_ttm_get_pages(struct drm_i915_gem_object *obj) |
828 | { | |
829 | struct ttm_place requested, busy[I915_TTM_MAX_PLACEMENTS]; | |
830 | struct ttm_placement placement; | |
831 | ||
832 | GEM_BUG_ON(obj->mm.n_placements > I915_TTM_MAX_PLACEMENTS); | |
833 | ||
834 | /* Move to the requested placement. */ | |
835 | i915_ttm_placement_from_obj(obj, &requested, busy, &placement); | |
836 | ||
837 | return __i915_ttm_get_pages(obj, &placement); | |
838 | } | |
839 | ||
840 | /** | |
841 | * DOC: Migration vs eviction | |
842 | * | |
843 | * GEM migration may not be the same as TTM migration / eviction. If | |
844 | * the TTM core decides to evict an object it may be evicted to a | |
845 | * TTM memory type that is not in the object's allowable GEM regions, or | |
846 | * in fact theoretically to a TTM memory type that doesn't correspond to | |
847 | * a GEM memory region. In that case the object's GEM region is not | |
848 | * updated, and the data is migrated back to the GEM region at | |
849 | * get_pages time. TTM may however set up CPU ptes to the object even | |
850 | * when it is evicted. | |
851 | * Gem forced migration using the i915_ttm_migrate() op, is allowed even | |
852 | * to regions that are not in the object's list of allowable placements. | |
853 | */ | |
503725c2 MA |
854 | static int __i915_ttm_migrate(struct drm_i915_gem_object *obj, |
855 | struct intel_memory_region *mr, | |
856 | unsigned int flags) | |
b6e913e1 TH |
857 | { |
858 | struct ttm_place requested; | |
859 | struct ttm_placement placement; | |
860 | int ret; | |
861 | ||
ecbf2060 MA |
862 | i915_ttm_place_from_region(mr, &requested, obj->bo_offset, |
863 | obj->base.size, flags); | |
b6e913e1 TH |
864 | placement.num_placement = 1; |
865 | placement.num_busy_placement = 1; | |
866 | placement.placement = &requested; | |
867 | placement.busy_placement = &requested; | |
868 | ||
869 | ret = __i915_ttm_get_pages(obj, &placement); | |
870 | if (ret) | |
871 | return ret; | |
872 | ||
873 | /* | |
874 | * Reinitialize the region bindings. This is primarily | |
875 | * required for objects where the new region is not in | |
876 | * its allowable placements. | |
877 | */ | |
878 | if (obj->mm.region != mr) { | |
879 | i915_gem_object_release_memory_region(obj); | |
880 | i915_gem_object_init_memory_region(obj, mr); | |
881 | } | |
882 | ||
883 | return 0; | |
884 | } | |
885 | ||
503725c2 | 886 | static int i915_ttm_migrate(struct drm_i915_gem_object *obj, |
695ddc93 MA |
887 | struct intel_memory_region *mr, |
888 | unsigned int flags) | |
503725c2 | 889 | { |
695ddc93 | 890 | return __i915_ttm_migrate(obj, mr, flags); |
503725c2 MA |
891 | } |
892 | ||
213d5092 TH |
893 | static void i915_ttm_put_pages(struct drm_i915_gem_object *obj, |
894 | struct sg_table *st) | |
895 | { | |
896 | /* | |
897 | * We're currently not called from a shrinker, so put_pages() | |
898 | * typically means the object is about to destroyed, or called | |
899 | * from move_notify(). So just avoid doing much for now. | |
900 | * If the object is not destroyed next, The TTM eviction logic | |
901 | * and shrinkers will move it out if needed. | |
902 | */ | |
cad7109a TH |
903 | |
904 | if (obj->mm.rsgt) | |
905 | i915_refct_sgt_put(fetch_and_zero(&obj->mm.rsgt)); | |
213d5092 TH |
906 | } |
907 | ||
3589fdbd TH |
908 | /** |
909 | * i915_ttm_adjust_lru - Adjust an object's position on relevant LRU lists. | |
910 | * @obj: The object | |
911 | */ | |
912 | void i915_ttm_adjust_lru(struct drm_i915_gem_object *obj) | |
213d5092 TH |
913 | { |
914 | struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); | |
7ae03459 MA |
915 | struct i915_ttm_tt *i915_tt = |
916 | container_of(bo->ttm, typeof(*i915_tt), ttm); | |
ebd4a8ec MA |
917 | bool shrinkable = |
918 | bo->ttm && i915_tt->filp && ttm_tt_is_populated(bo->ttm); | |
213d5092 TH |
919 | |
920 | /* | |
921 | * Don't manipulate the TTM LRUs while in TTM bo destruction. | |
922 | * We're called through i915_ttm_delete_mem_notify(). | |
923 | */ | |
924 | if (!kref_read(&bo->kref)) | |
925 | return; | |
926 | ||
ebd4a8ec MA |
927 | /* |
928 | * We skip managing the shrinker LRU in set_pages() and just manage | |
929 | * everything here. This does at least solve the issue with having | |
930 | * temporary shmem mappings(like with evicted lmem) not being visible to | |
931 | * the shrinker. Only our shmem objects are shrinkable, everything else | |
932 | * we keep as unshrinkable. | |
933 | * | |
934 | * To make sure everything plays nice we keep an extra shrink pin in TTM | |
935 | * if the underlying pages are not currently shrinkable. Once we release | |
936 | * our pin, like when the pages are moved to shmem, the pages will then | |
937 | * be added to the shrinker LRU, assuming the caller isn't also holding | |
938 | * a pin. | |
939 | * | |
940 | * TODO: consider maybe also bumping the shrinker list here when we have | |
941 | * already unpinned it, which should give us something more like an LRU. | |
d3cb30f8 TH |
942 | * |
943 | * TODO: There is a small window of opportunity for this function to | |
944 | * get called from eviction after we've dropped the last GEM refcount, | |
945 | * but before the TTM deleted flag is set on the object. Avoid | |
946 | * adjusting the shrinker list in such cases, since the object is | |
947 | * not available to the shrinker anyway due to its zero refcount. | |
948 | * To fix this properly we should move to a TTM shrinker LRU list for | |
949 | * these objects. | |
ebd4a8ec | 950 | */ |
d3cb30f8 TH |
951 | if (kref_get_unless_zero(&obj->base.refcount)) { |
952 | if (shrinkable != obj->mm.ttm_shrinkable) { | |
953 | if (shrinkable) { | |
954 | if (obj->mm.madv == I915_MADV_WILLNEED) | |
955 | __i915_gem_object_make_shrinkable(obj); | |
956 | else | |
957 | __i915_gem_object_make_purgeable(obj); | |
958 | } else { | |
959 | i915_gem_object_make_unshrinkable(obj); | |
960 | } | |
961 | ||
962 | obj->mm.ttm_shrinkable = shrinkable; | |
ebd4a8ec | 963 | } |
d3cb30f8 | 964 | i915_gem_object_put(obj); |
ebd4a8ec MA |
965 | } |
966 | ||
213d5092 TH |
967 | /* |
968 | * Put on the correct LRU list depending on the MADV status | |
969 | */ | |
970 | spin_lock(&bo->bdev->lru_lock); | |
ebd4a8ec | 971 | if (shrinkable) { |
7ae03459 MA |
972 | /* Try to keep shmem_tt from being considered for shrinking. */ |
973 | bo->priority = TTM_MAX_BO_PRIORITY - 1; | |
974 | } else if (obj->mm.madv != I915_MADV_WILLNEED) { | |
213d5092 TH |
975 | bo->priority = I915_TTM_PRIO_PURGE; |
976 | } else if (!i915_gem_object_has_pages(obj)) { | |
ba2c5d15 | 977 | bo->priority = I915_TTM_PRIO_NO_PAGES; |
213d5092 | 978 | } else { |
93735059 MA |
979 | struct ttm_resource_manager *man = |
980 | ttm_manager_type(bo->bdev, bo->resource->mem_type); | |
981 | ||
982 | /* | |
983 | * If we need to place an LMEM resource which doesn't need CPU | |
984 | * access then we should try not to victimize mappable objects | |
985 | * first, since we likely end up stealing more of the mappable | |
986 | * portion. And likewise when we try to find space for a mappble | |
987 | * object, we know not to ever victimize objects that don't | |
988 | * occupy any mappable pages. | |
989 | */ | |
990 | if (i915_ttm_cpu_maps_iomem(bo->resource) && | |
991 | i915_ttm_buddy_man_visible_size(man) < man->size && | |
992 | !(obj->flags & I915_BO_ALLOC_GPU_ONLY)) | |
993 | bo->priority = I915_TTM_PRIO_NEEDS_CPU_ACCESS; | |
994 | else | |
995 | bo->priority = I915_TTM_PRIO_HAS_PAGES; | |
213d5092 TH |
996 | } |
997 | ||
fee2ede1 | 998 | ttm_bo_move_to_lru_tail(bo); |
213d5092 TH |
999 | spin_unlock(&bo->bdev->lru_lock); |
1000 | } | |
1001 | ||
1002 | /* | |
1003 | * TTM-backed gem object destruction requires some clarification. | |
1004 | * Basically we have two possibilities here. We can either rely on the | |
1005 | * i915 delayed destruction and put the TTM object when the object | |
1006 | * is idle. This would be detected by TTM which would bypass the | |
1007 | * TTM delayed destroy handling. The other approach is to put the TTM | |
1008 | * object early and rely on the TTM destroyed handling, and then free | |
1009 | * the leftover parts of the GEM object once TTM's destroyed list handling is | |
1010 | * complete. For now, we rely on the latter for two reasons: | |
1011 | * a) TTM can evict an object even when it's on the delayed destroy list, | |
1012 | * which in theory allows for complete eviction. | |
1013 | * b) There is work going on in TTM to allow freeing an object even when | |
1014 | * it's not idle, and using the TTM destroyed list handling could help us | |
1015 | * benefit from that. | |
1016 | */ | |
1017 | static void i915_ttm_delayed_free(struct drm_i915_gem_object *obj) | |
1018 | { | |
068396bb TH |
1019 | GEM_BUG_ON(!obj->ttm.created); |
1020 | ||
1021 | ttm_bo_put(i915_gem_to_ttm(obj)); | |
213d5092 TH |
1022 | } |
1023 | ||
cf3e3e86 ML |
1024 | static vm_fault_t vm_fault_ttm(struct vm_fault *vmf) |
1025 | { | |
1026 | struct vm_area_struct *area = vmf->vma; | |
6385eb7a | 1027 | struct ttm_buffer_object *bo = area->vm_private_data; |
ebd4a8ec | 1028 | struct drm_device *dev = bo->base.dev; |
6667d78a | 1029 | struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); |
ad74457a | 1030 | intel_wakeref_t wakeref = 0; |
ebd4a8ec MA |
1031 | vm_fault_t ret; |
1032 | int idx; | |
cf3e3e86 | 1033 | |
6667d78a | 1034 | if (i915_ttm_is_ghost_object(bo)) |
6385eb7a TH |
1035 | return VM_FAULT_SIGBUS; |
1036 | ||
cf3e3e86 ML |
1037 | /* Sanity check that we allow writing into this object */ |
1038 | if (unlikely(i915_gem_object_is_readonly(obj) && | |
1039 | area->vm_flags & VM_WRITE)) | |
1040 | return VM_FAULT_SIGBUS; | |
1041 | ||
ebd4a8ec MA |
1042 | ret = ttm_bo_vm_reserve(bo, vmf); |
1043 | if (ret) | |
1044 | return ret; | |
1045 | ||
03ee5956 MA |
1046 | if (obj->mm.madv != I915_MADV_WILLNEED) { |
1047 | dma_resv_unlock(bo->base.resv); | |
1048 | return VM_FAULT_SIGBUS; | |
1049 | } | |
1050 | ||
ad74457a AG |
1051 | if (i915_ttm_cpu_maps_iomem(bo->resource)) |
1052 | wakeref = intel_runtime_pm_get(&to_i915(obj->base.dev)->runtime_pm); | |
1053 | ||
503725c2 MA |
1054 | if (!i915_ttm_resource_mappable(bo->resource)) { |
1055 | int err = -ENODEV; | |
1056 | int i; | |
1057 | ||
1058 | for (i = 0; i < obj->mm.n_placements; i++) { | |
1059 | struct intel_memory_region *mr = obj->mm.placements[i]; | |
1060 | unsigned int flags; | |
1061 | ||
1062 | if (!mr->io_size && mr->type != INTEL_MEMORY_SYSTEM) | |
1063 | continue; | |
1064 | ||
1065 | flags = obj->flags; | |
1066 | flags &= ~I915_BO_ALLOC_GPU_ONLY; | |
1067 | err = __i915_ttm_migrate(obj, mr, flags); | |
1068 | if (!err) | |
1069 | break; | |
1070 | } | |
1071 | ||
1072 | if (err) { | |
e5cedf98 ND |
1073 | drm_dbg(dev, "Unable to make resource CPU accessible(err = %pe)\n", |
1074 | ERR_PTR(err)); | |
503725c2 | 1075 | dma_resv_unlock(bo->base.resv); |
ad74457a AG |
1076 | ret = VM_FAULT_SIGBUS; |
1077 | goto out_rpm; | |
503725c2 MA |
1078 | } |
1079 | } | |
1080 | ||
ebd4a8ec MA |
1081 | if (drm_dev_enter(dev, &idx)) { |
1082 | ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot, | |
be373fad | 1083 | TTM_BO_VM_NUM_PREFAULT); |
ebd4a8ec MA |
1084 | drm_dev_exit(idx); |
1085 | } else { | |
1086 | ret = ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot); | |
1087 | } | |
ad74457a | 1088 | |
ebd4a8ec | 1089 | if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) |
ad74457a AG |
1090 | goto out_rpm; |
1091 | ||
1cacd689 AG |
1092 | /* |
1093 | * ttm_bo_vm_reserve() already has dma_resv_lock. | |
1094 | * userfault_count is protected by dma_resv lock and rpm wakeref. | |
1095 | */ | |
ad74457a AG |
1096 | if (ret == VM_FAULT_NOPAGE && wakeref && !obj->userfault_count) { |
1097 | obj->userfault_count = 1; | |
1cacd689 | 1098 | spin_lock(&to_i915(obj->base.dev)->runtime_pm.lmem_userfault_lock); |
e66c8dcf | 1099 | list_add(&obj->userfault_link, &to_i915(obj->base.dev)->runtime_pm.lmem_userfault_list); |
1cacd689 | 1100 | spin_unlock(&to_i915(obj->base.dev)->runtime_pm.lmem_userfault_lock); |
ad74457a AG |
1101 | } |
1102 | ||
1103 | if (wakeref & CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND) | |
e66c8dcf | 1104 | intel_wakeref_auto(&to_i915(obj->base.dev)->runtime_pm.userfault_wakeref, |
ad74457a | 1105 | msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)); |
ebd4a8ec MA |
1106 | |
1107 | i915_ttm_adjust_lru(obj); | |
1108 | ||
1109 | dma_resv_unlock(bo->base.resv); | |
ad74457a AG |
1110 | |
1111 | out_rpm: | |
1112 | if (wakeref) | |
1113 | intel_runtime_pm_put(&to_i915(obj->base.dev)->runtime_pm, wakeref); | |
1114 | ||
ebd4a8ec | 1115 | return ret; |
cf3e3e86 ML |
1116 | } |
1117 | ||
1118 | static int | |
1119 | vm_access_ttm(struct vm_area_struct *area, unsigned long addr, | |
1120 | void *buf, int len, int write) | |
1121 | { | |
1122 | struct drm_i915_gem_object *obj = | |
1123 | i915_ttm_to_gem(area->vm_private_data); | |
1124 | ||
1125 | if (i915_gem_object_is_readonly(obj) && write) | |
1126 | return -EACCES; | |
1127 | ||
1128 | return ttm_bo_vm_access(area, addr, buf, len, write); | |
1129 | } | |
1130 | ||
1131 | static void ttm_vm_open(struct vm_area_struct *vma) | |
1132 | { | |
1133 | struct drm_i915_gem_object *obj = | |
1134 | i915_ttm_to_gem(vma->vm_private_data); | |
1135 | ||
6667d78a | 1136 | GEM_BUG_ON(i915_ttm_is_ghost_object(vma->vm_private_data)); |
cf3e3e86 ML |
1137 | i915_gem_object_get(obj); |
1138 | } | |
1139 | ||
1140 | static void ttm_vm_close(struct vm_area_struct *vma) | |
1141 | { | |
1142 | struct drm_i915_gem_object *obj = | |
1143 | i915_ttm_to_gem(vma->vm_private_data); | |
1144 | ||
6667d78a | 1145 | GEM_BUG_ON(i915_ttm_is_ghost_object(vma->vm_private_data)); |
cf3e3e86 ML |
1146 | i915_gem_object_put(obj); |
1147 | } | |
1148 | ||
1149 | static const struct vm_operations_struct vm_ops_ttm = { | |
1150 | .fault = vm_fault_ttm, | |
1151 | .access = vm_access_ttm, | |
1152 | .open = ttm_vm_open, | |
1153 | .close = ttm_vm_close, | |
1154 | }; | |
1155 | ||
1156 | static u64 i915_ttm_mmap_offset(struct drm_i915_gem_object *obj) | |
1157 | { | |
1158 | /* The ttm_bo must be allocated with I915_BO_ALLOC_USER */ | |
1159 | GEM_BUG_ON(!drm_mm_node_allocated(&obj->base.vma_node.vm_node)); | |
1160 | ||
1161 | return drm_vma_node_offset_addr(&obj->base.vma_node); | |
1162 | } | |
1163 | ||
8ee262ba MA |
1164 | static void i915_ttm_unmap_virtual(struct drm_i915_gem_object *obj) |
1165 | { | |
1cacd689 AG |
1166 | struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); |
1167 | intel_wakeref_t wakeref = 0; | |
1168 | ||
1169 | assert_object_held_shared(obj); | |
1170 | ||
1171 | if (i915_ttm_cpu_maps_iomem(bo->resource)) { | |
1172 | wakeref = intel_runtime_pm_get(&to_i915(obj->base.dev)->runtime_pm); | |
1173 | ||
1174 | /* userfault_count is protected by obj lock and rpm wakeref. */ | |
1175 | if (obj->userfault_count) { | |
1176 | spin_lock(&to_i915(obj->base.dev)->runtime_pm.lmem_userfault_lock); | |
1177 | list_del(&obj->userfault_link); | |
1178 | spin_unlock(&to_i915(obj->base.dev)->runtime_pm.lmem_userfault_lock); | |
1179 | obj->userfault_count = 0; | |
1180 | } | |
1181 | } | |
1182 | ||
8ee262ba | 1183 | ttm_bo_unmap_virtual(i915_gem_to_ttm(obj)); |
1cacd689 AG |
1184 | |
1185 | if (wakeref) | |
1186 | intel_runtime_pm_put(&to_i915(obj->base.dev)->runtime_pm, wakeref); | |
8ee262ba MA |
1187 | } |
1188 | ||
4bc2d574 | 1189 | static const struct drm_i915_gem_object_ops i915_gem_ttm_obj_ops = { |
213d5092 | 1190 | .name = "i915_gem_object_ttm", |
5d12ffe6 MA |
1191 | .flags = I915_GEM_OBJECT_IS_SHRINKABLE | |
1192 | I915_GEM_OBJECT_SELF_MANAGED_SHRINK_LIST, | |
213d5092 TH |
1193 | |
1194 | .get_pages = i915_ttm_get_pages, | |
1195 | .put_pages = i915_ttm_put_pages, | |
6ef295e3 | 1196 | .truncate = i915_ttm_truncate, |
ffa3fe08 | 1197 | .shrink = i915_ttm_shrink, |
7ae03459 | 1198 | |
213d5092 TH |
1199 | .adjust_lru = i915_ttm_adjust_lru, |
1200 | .delayed_free = i915_ttm_delayed_free, | |
b6e913e1 | 1201 | .migrate = i915_ttm_migrate, |
7ae03459 | 1202 | |
cf3e3e86 | 1203 | .mmap_offset = i915_ttm_mmap_offset, |
8ee262ba | 1204 | .unmap_virtual = i915_ttm_unmap_virtual, |
cf3e3e86 | 1205 | .mmap_ops = &vm_ops_ttm, |
213d5092 TH |
1206 | }; |
1207 | ||
1208 | void i915_ttm_bo_destroy(struct ttm_buffer_object *bo) | |
1209 | { | |
1210 | struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); | |
1211 | ||
1212 | i915_gem_object_release_memory_region(obj); | |
cf3e3e86 | 1213 | mutex_destroy(&obj->ttm.get_io_page.lock); |
48b09612 | 1214 | |
068396bb | 1215 | if (obj->ttm.created) { |
ebd4a8ec MA |
1216 | /* |
1217 | * We freely manage the shrinker LRU outide of the mm.pages life | |
1218 | * cycle. As a result when destroying the object we should be | |
1219 | * extra paranoid and ensure we remove it from the LRU, before | |
1220 | * we free the object. | |
1221 | * | |
1222 | * Touching the ttm_shrinkable outside of the object lock here | |
1223 | * should be safe now that the last GEM object ref was dropped. | |
1224 | */ | |
1225 | if (obj->mm.ttm_shrinkable) | |
1226 | i915_gem_object_make_unshrinkable(obj); | |
1227 | ||
068396bb TH |
1228 | i915_ttm_backup_free(obj); |
1229 | ||
1230 | /* This releases all gem object bindings to the backend. */ | |
1231 | __i915_gem_free_object(obj); | |
1232 | ||
213d5092 | 1233 | call_rcu(&obj->rcu, __i915_gem_free_object_rcu); |
068396bb TH |
1234 | } else { |
1235 | __i915_gem_object_fini(obj); | |
1236 | } | |
213d5092 TH |
1237 | } |
1238 | ||
1239 | /** | |
1240 | * __i915_gem_ttm_object_init - Initialize a ttm-backed i915 gem object | |
1241 | * @mem: The initial memory region for the object. | |
1242 | * @obj: The gem object. | |
1243 | * @size: Object size in bytes. | |
1244 | * @flags: gem object flags. | |
1245 | * | |
1246 | * Return: 0 on success, negative error code on failure. | |
1247 | */ | |
1248 | int __i915_gem_ttm_object_init(struct intel_memory_region *mem, | |
1249 | struct drm_i915_gem_object *obj, | |
9b78b5da | 1250 | resource_size_t offset, |
213d5092 | 1251 | resource_size_t size, |
d22632c8 | 1252 | resource_size_t page_size, |
213d5092 TH |
1253 | unsigned int flags) |
1254 | { | |
1255 | static struct lock_class_key lock_class; | |
1256 | struct drm_i915_private *i915 = mem->i915; | |
3c2b8f32 TH |
1257 | struct ttm_operation_ctx ctx = { |
1258 | .interruptible = true, | |
1259 | .no_wait_gpu = false, | |
1260 | }; | |
213d5092 | 1261 | enum ttm_bo_type bo_type; |
213d5092 TH |
1262 | int ret; |
1263 | ||
213d5092 TH |
1264 | drm_gem_private_object_init(&i915->drm, &obj->base, size); |
1265 | i915_gem_object_init(obj, &i915_gem_ttm_obj_ops, &lock_class, flags); | |
068396bb | 1266 | |
ecbf2060 MA |
1267 | obj->bo_offset = offset; |
1268 | ||
068396bb | 1269 | /* Don't put on a region list until we're either locked or fully initialized. */ |
8b1f7f92 | 1270 | obj->mm.region = mem; |
068396bb TH |
1271 | INIT_LIST_HEAD(&obj->mm.region_link); |
1272 | ||
cf3e3e86 ML |
1273 | INIT_RADIX_TREE(&obj->ttm.get_io_page.radix, GFP_KERNEL | __GFP_NOWARN); |
1274 | mutex_init(&obj->ttm.get_io_page.lock); | |
213d5092 TH |
1275 | bo_type = (obj->flags & I915_BO_ALLOC_USER) ? ttm_bo_type_device : |
1276 | ttm_bo_type_kernel; | |
1277 | ||
3c2b8f32 TH |
1278 | obj->base.vma_node.driver_private = i915_gem_to_ttm(obj); |
1279 | ||
d22632c8 MA |
1280 | /* Forcing the page size is kernel internal only */ |
1281 | GEM_BUG_ON(page_size && obj->mm.n_placements); | |
1282 | ||
ebd4a8ec MA |
1283 | /* |
1284 | * Keep an extra shrink pin to prevent the object from being made | |
1285 | * shrinkable too early. If the ttm_tt is ever allocated in shmem, we | |
1286 | * drop the pin. The TTM backend manages the shrinker LRU itself, | |
1287 | * outside of the normal mm.pages life cycle. | |
1288 | */ | |
1289 | i915_gem_object_make_unshrinkable(obj); | |
1290 | ||
213d5092 TH |
1291 | /* |
1292 | * If this function fails, it will call the destructor, but | |
1293 | * our caller still owns the object. So no freeing in the | |
1294 | * destructor until obj->ttm.created is true. | |
1295 | * Similarly, in delayed_destroy, we can't call ttm_bo_put() | |
1296 | * until successful initialization. | |
1297 | */ | |
347987a2 CK |
1298 | ret = ttm_bo_init_reserved(&i915->bdev, i915_gem_to_ttm(obj), bo_type, |
1299 | &i915_sys_placement, page_size >> PAGE_SHIFT, | |
3c2b8f32 TH |
1300 | &ctx, NULL, NULL, i915_ttm_bo_destroy); |
1301 | if (ret) | |
1302 | return i915_ttm_err_to_gem(ret); | |
213d5092 | 1303 | |
3c2b8f32 | 1304 | obj->ttm.created = true; |
068396bb TH |
1305 | i915_gem_object_release_memory_region(obj); |
1306 | i915_gem_object_init_memory_region(obj, mem); | |
3c2b8f32 TH |
1307 | i915_ttm_adjust_domains_after_move(obj); |
1308 | i915_ttm_adjust_gem_after_move(obj); | |
1309 | i915_gem_object_unlock(obj); | |
213d5092 | 1310 | |
3c2b8f32 | 1311 | return 0; |
213d5092 | 1312 | } |
32b7cf51 TH |
1313 | |
1314 | static const struct intel_memory_region_ops ttm_system_region_ops = { | |
1315 | .init_object = __i915_gem_ttm_object_init, | |
8b1f7f92 | 1316 | .release = intel_region_ttm_fini, |
32b7cf51 TH |
1317 | }; |
1318 | ||
1319 | struct intel_memory_region * | |
1320 | i915_gem_ttm_system_setup(struct drm_i915_private *i915, | |
1321 | u16 type, u16 instance) | |
1322 | { | |
1323 | struct intel_memory_region *mr; | |
1324 | ||
1325 | mr = intel_memory_region_create(i915, 0, | |
1326 | totalram_pages() << PAGE_SHIFT, | |
235582ca | 1327 | PAGE_SIZE, 0, 0, |
32b7cf51 TH |
1328 | type, instance, |
1329 | &ttm_system_region_ops); | |
1330 | if (IS_ERR(mr)) | |
1331 | return mr; | |
1332 | ||
1333 | intel_memory_region_set_name(mr, "system-ttm"); | |
1334 | return mr; | |
213d5092 | 1335 | } |