Commit | Line | Data |
---|---|---|
b47eb4a2 CW |
1 | /* |
2 | * Copyright © 2008-2010 Intel Corporation | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice (including the next | |
12 | * paragraph) shall be included in all copies or substantial portions of the | |
13 | * Software. | |
14 | * | |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS | |
21 | * IN THE SOFTWARE. | |
22 | * | |
23 | * Authors: | |
24 | * Eric Anholt <eric@anholt.net> | |
25 | * Chris Wilson <chris@chris-wilson.co.uuk> | |
26 | * | |
27 | */ | |
28 | ||
10be98a7 | 29 | #include "gem/i915_gem_context.h" |
b97060a9 | 30 | #include "gt/intel_gt.h" |
66101975 | 31 | #include "gt/intel_gt_requests.h" |
10be98a7 | 32 | |
74e21ac2 | 33 | #include "i915_drv.h" |
2ef97818 | 34 | #include "i915_gem_evict.h" |
db53a302 | 35 | #include "i915_trace.h" |
b47eb4a2 | 36 | |
9c1477e8 CW |
37 | I915_SELFTEST_DECLARE(static struct igt_evict_ctl { |
38 | bool fail_if_busy:1; | |
39 | } igt_evict_ctl;) | |
40 | ||
7e00897b ML |
41 | static bool dying_vma(struct i915_vma *vma) |
42 | { | |
43 | return !kref_read(&vma->obj->base.refcount); | |
44 | } | |
45 | ||
0f857158 | 46 | static int ggtt_flush(struct i915_address_space *vm) |
2889caa9 | 47 | { |
0f857158 AI |
48 | struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); |
49 | struct intel_gt *gt; | |
50 | int ret = 0; | |
51 | ||
52 | list_for_each_entry(gt, &ggtt->gt_list, ggtt_link) { | |
53 | /* | |
54 | * Not everything in the GGTT is tracked via vma (otherwise we | |
55 | * could evict as required with minimal stalling) so we are forced | |
56 | * to idle the GPU and explicitly retire outstanding requests in | |
57 | * the hopes that we can then remove contexts and the like only | |
58 | * bound by their active reference. | |
59 | */ | |
60 | ret = intel_gt_wait_for_idle(gt, MAX_SCHEDULE_TIMEOUT); | |
61 | if (ret) | |
62 | return ret; | |
63 | } | |
64 | return ret; | |
2889caa9 CW |
65 | } |
66 | ||
7e00897b ML |
67 | static bool grab_vma(struct i915_vma *vma, struct i915_gem_ww_ctx *ww) |
68 | { | |
69 | /* | |
70 | * We add the extra refcount so the object doesn't drop to zero until | |
71 | * after ungrab_vma(), this way trylock is always paired with unlock. | |
72 | */ | |
73 | if (i915_gem_object_get_rcu(vma->obj)) { | |
74 | if (!i915_gem_object_trylock(vma->obj, ww)) { | |
75 | i915_gem_object_put(vma->obj); | |
76 | return false; | |
77 | } | |
78 | } else { | |
79 | /* Dead objects don't need pins */ | |
80 | atomic_and(~I915_VMA_PIN_MASK, &vma->flags); | |
81 | } | |
82 | ||
83 | return true; | |
84 | } | |
85 | ||
86 | static void ungrab_vma(struct i915_vma *vma) | |
87 | { | |
88 | if (dying_vma(vma)) | |
89 | return; | |
90 | ||
91 | i915_gem_object_unlock(vma->obj); | |
92 | i915_gem_object_put(vma->obj); | |
93 | } | |
94 | ||
cd377ea9 | 95 | static bool |
9a71e277 | 96 | mark_free(struct drm_mm_scan *scan, |
7e00897b | 97 | struct i915_gem_ww_ctx *ww, |
9a71e277 CW |
98 | struct i915_vma *vma, |
99 | unsigned int flags, | |
100 | struct list_head *unwind) | |
b47eb4a2 | 101 | { |
20dfbde4 | 102 | if (i915_vma_is_pinned(vma)) |
1b50247a CW |
103 | return false; |
104 | ||
7e00897b ML |
105 | if (!grab_vma(vma, ww)) |
106 | return false; | |
107 | ||
8c45cec4 | 108 | list_add(&vma->evict_link, unwind); |
9a71e277 | 109 | return drm_mm_scan_add_block(scan, &vma->node); |
b47eb4a2 CW |
110 | } |
111 | ||
e956996c CW |
112 | static bool defer_evict(struct i915_vma *vma) |
113 | { | |
114 | if (i915_vma_is_active(vma)) | |
115 | return true; | |
116 | ||
117 | if (i915_vma_is_scanout(vma)) | |
118 | return true; | |
119 | ||
120 | return false; | |
121 | } | |
122 | ||
c2c1d491 DV |
123 | /** |
124 | * i915_gem_evict_something - Evict vmas to make room for binding a new one | |
c2c1d491 | 125 | * @vm: address space to evict from |
7e00897b | 126 | * @ww: An optional struct i915_gem_ww_ctx. |
7838a63a | 127 | * @min_size: size of the desired free space |
c2c1d491 | 128 | * @alignment: alignment constraint of the desired free space |
33dd8899 | 129 | * @color: color for the desired space |
7838a63a DV |
130 | * @start: start (inclusive) of the range from which to evict objects |
131 | * @end: end (exclusive) of the range from which to evict objects | |
132 | * @flags: additional flags to control the eviction algorithm | |
c2c1d491 DV |
133 | * |
134 | * This function will try to evict vmas until a free space satisfying the | |
135 | * requirements is found. Callers must check first whether any such hole exists | |
136 | * already before calling this function. | |
137 | * | |
138 | * This function is used by the object/vma binding code. | |
139 | * | |
eb0b44ad DV |
140 | * Since this function is only used to free up virtual address space it only |
141 | * ignores pinned vmas, and not object where the backing storage itself is | |
142 | * pinned. Hence obj->pages_pin_count does not protect against eviction. | |
143 | * | |
c2c1d491 DV |
144 | * To clarify: This is for freeing up virtual address space, not for freeing |
145 | * memory in e.g. the shrinker. | |
146 | */ | |
b47eb4a2 | 147 | int |
e522ac23 | 148 | i915_gem_evict_something(struct i915_address_space *vm, |
7e00897b | 149 | struct i915_gem_ww_ctx *ww, |
2ffffd0f | 150 | u64 min_size, u64 alignment, |
33dd8899 | 151 | unsigned long color, |
2ffffd0f | 152 | u64 start, u64 end, |
1ec9e26d | 153 | unsigned flags) |
b47eb4a2 | 154 | { |
9a71e277 | 155 | struct drm_mm_scan scan; |
9332f3b1 | 156 | struct list_head eviction_list; |
9332f3b1 | 157 | struct i915_vma *vma, *next; |
3fa489da | 158 | struct drm_mm_node *node; |
4e64e553 | 159 | enum drm_mm_insert_mode mode; |
499197dc | 160 | struct i915_vma *active; |
0f857158 | 161 | struct intel_gt *gt; |
9332f3b1 | 162 | int ret; |
b47eb4a2 | 163 | |
2850748e | 164 | lockdep_assert_held(&vm->mutex); |
e522ac23 | 165 | trace_i915_gem_evict(vm, min_size, alignment, flags); |
db53a302 | 166 | |
cd377ea9 | 167 | /* |
499197dc CW |
168 | * The goal is to evict objects and amalgamate space in rough LRU order. |
169 | * Since both active and inactive objects reside on the same list, | |
170 | * in a mix of creation and last scanned order, as we process the list | |
171 | * we sort it into inactive/active, which keeps the active portion | |
172 | * in a rough MRU order. | |
cd377ea9 CW |
173 | * |
174 | * The retirement sequence is thus: | |
499197dc CW |
175 | * 1. Inactive objects (already retired, random order) |
176 | * 2. Active objects (will stall on unbinding, oldest scanned first) | |
cd377ea9 | 177 | */ |
4e64e553 CW |
178 | mode = DRM_MM_INSERT_BEST; |
179 | if (flags & PIN_HIGH) | |
180 | mode = DRM_MM_INSERT_HIGH; | |
181 | if (flags & PIN_MAPPABLE) | |
182 | mode = DRM_MM_INSERT_LOW; | |
2c4b3895 | 183 | drm_mm_scan_init_with_range(&scan, &vm->mm, |
33dd8899 | 184 | min_size, alignment, color, |
4e64e553 | 185 | start, end, mode); |
cd377ea9 | 186 | |
0f857158 AI |
187 | if (i915_is_ggtt(vm)) { |
188 | struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); | |
189 | ||
190 | list_for_each_entry(gt, &ggtt->gt_list, ggtt_link) | |
191 | intel_gt_retire_requests(gt); | |
192 | } else { | |
193 | intel_gt_retire_requests(vm->gt); | |
194 | } | |
f33a8a51 | 195 | |
9332f3b1 | 196 | search_again: |
499197dc | 197 | active = NULL; |
9332f3b1 | 198 | INIT_LIST_HEAD(&eviction_list); |
499197dc | 199 | list_for_each_entry_safe(vma, next, &vm->bound_list, vm_link) { |
73e28cc4 CW |
200 | if (vma == active) { /* now seen this vma twice */ |
201 | if (flags & PIN_NONBLOCK) | |
202 | break; | |
203 | ||
204 | active = ERR_PTR(-EAGAIN); | |
205 | } | |
206 | ||
499197dc CW |
207 | /* |
208 | * We keep this list in a rough least-recently scanned order | |
209 | * of active elements (inactive elements are cheap to reap). | |
210 | * New entries are added to the end, and we move anything we | |
211 | * scan to the end. The assumption is that the working set | |
212 | * of applications is either steady state (and thanks to the | |
213 | * userspace bo cache it almost always is) or volatile and | |
214 | * frequently replaced after a frame, which are self-evicting! | |
215 | * Given that assumption, the MRU order of the scan list is | |
216 | * fairly static, and keeping it in least-recently scan order | |
217 | * is suitable. | |
218 | * | |
219 | * To notice when we complete one full cycle, we record the | |
220 | * first active element seen, before moving it to the tail. | |
221 | */ | |
e956996c | 222 | if (active != ERR_PTR(-EAGAIN) && defer_evict(vma)) { |
73e28cc4 CW |
223 | if (!active) |
224 | active = vma; | |
499197dc | 225 | |
73e28cc4 CW |
226 | list_move_tail(&vma->vm_link, &vm->bound_list); |
227 | continue; | |
499197dc CW |
228 | } |
229 | ||
7e00897b | 230 | if (mark_free(&scan, ww, vma, flags, &eviction_list)) |
499197dc CW |
231 | goto found; |
232 | } | |
cd377ea9 CW |
233 | |
234 | /* Nothing found, clean up and bail out! */ | |
8c45cec4 | 235 | list_for_each_entry_safe(vma, next, &eviction_list, evict_link) { |
9a71e277 | 236 | ret = drm_mm_scan_remove_block(&scan, &vma->node); |
cd377ea9 | 237 | BUG_ON(ret); |
7e00897b | 238 | ungrab_vma(vma); |
cd377ea9 CW |
239 | } |
240 | ||
55b4f1ce CW |
241 | /* |
242 | * Can we unpin some objects such as idle hw contents, | |
9332f3b1 CW |
243 | * or pending flips? But since only the GGTT has global entries |
244 | * such as scanouts, rinbuffers and contexts, we can skip the | |
245 | * purge when inspecting per-process local address spaces. | |
cd377ea9 | 246 | */ |
9332f3b1 | 247 | if (!i915_is_ggtt(vm) || flags & PIN_NONBLOCK) |
74e21ac2 | 248 | return -ENOSPC; |
ad071acb | 249 | |
55b4f1ce CW |
250 | /* |
251 | * Not everything in the GGTT is tracked via VMA using | |
252 | * i915_vma_move_to_active(), otherwise we could evict as required | |
253 | * with minimal stalling. Instead we are forced to idle the GPU and | |
254 | * explicitly retire outstanding requests which will then remove | |
255 | * the pinning for active objects such as contexts and ring, | |
256 | * enabling us to evict them on the next iteration. | |
257 | * | |
258 | * To ensure that all user contexts are evictable, we perform | |
259 | * a switch to the perma-pinned kernel context. This all also gives | |
260 | * us a termination condition, when the last retired context is | |
261 | * the kernel's there is no more we can evict. | |
262 | */ | |
79ffac85 CW |
263 | if (I915_SELFTEST_ONLY(igt_evict_ctl.fail_if_busy)) |
264 | return -EBUSY; | |
9c1477e8 | 265 | |
0f857158 | 266 | ret = ggtt_flush(vm); |
79ffac85 CW |
267 | if (ret) |
268 | return ret; | |
74e21ac2 | 269 | |
79ffac85 | 270 | cond_resched(); |
9332f3b1 | 271 | |
79ffac85 CW |
272 | flags |= PIN_NONBLOCK; |
273 | goto search_again; | |
cd377ea9 CW |
274 | |
275 | found: | |
e39a0150 | 276 | /* drm_mm doesn't allow any other other operations while |
9332f3b1 CW |
277 | * scanning, therefore store to-be-evicted objects on a |
278 | * temporary list and take a reference for all before | |
279 | * calling unbind (which may remove the active reference | |
280 | * of any of our objects, thus corrupting the list). | |
281 | */ | |
8c45cec4 | 282 | list_for_each_entry_safe(vma, next, &eviction_list, evict_link) { |
7e00897b | 283 | if (drm_mm_scan_remove_block(&scan, &vma->node)) { |
20dfbde4 | 284 | __i915_vma_pin(vma); |
7e00897b | 285 | } else { |
8c45cec4 | 286 | list_del(&vma->evict_link); |
7e00897b ML |
287 | ungrab_vma(vma); |
288 | } | |
cd377ea9 | 289 | } |
b47eb4a2 | 290 | |
cd377ea9 | 291 | /* Unbinding will emit any required flushes */ |
121dfbb2 | 292 | ret = 0; |
8c45cec4 | 293 | list_for_each_entry_safe(vma, next, &eviction_list, evict_link) { |
20dfbde4 | 294 | __i915_vma_unpin(vma); |
e39a0150 | 295 | if (ret == 0) |
2850748e | 296 | ret = __i915_vma_unbind(vma); |
7e00897b | 297 | ungrab_vma(vma); |
b47eb4a2 | 298 | } |
3fa489da CW |
299 | |
300 | while (ret == 0 && (node = drm_mm_scan_color_evict(&scan))) { | |
301 | vma = container_of(node, struct i915_vma, node); | |
53dd7028 MA |
302 | |
303 | /* If we find any non-objects (!vma), we cannot evict them */ | |
7e00897b ML |
304 | if (vma->node.color != I915_COLOR_UNEVICTABLE && |
305 | grab_vma(vma, ww)) { | |
53dd7028 | 306 | ret = __i915_vma_unbind(vma); |
7e00897b ML |
307 | ungrab_vma(vma); |
308 | } else { | |
309 | ret = -ENOSPC; | |
310 | } | |
3fa489da CW |
311 | } |
312 | ||
e39a0150 | 313 | return ret; |
b47eb4a2 CW |
314 | } |
315 | ||
172ae5b4 | 316 | /** |
e9d2871f | 317 | * i915_gem_evict_for_node - Evict vmas to make room for binding a new one |
625d988a | 318 | * @vm: address space to evict from |
7e00897b | 319 | * @ww: An optional struct i915_gem_ww_ctx. |
625d988a | 320 | * @target: range (and color) to evict for |
172ae5b4 CW |
321 | * @flags: additional flags to control the eviction algorithm |
322 | * | |
323 | * This function will try to evict vmas that overlap the target node. | |
324 | * | |
325 | * To clarify: This is for freeing up virtual address space, not for freeing | |
326 | * memory in e.g. the shrinker. | |
327 | */ | |
625d988a | 328 | int i915_gem_evict_for_node(struct i915_address_space *vm, |
7e00897b | 329 | struct i915_gem_ww_ctx *ww, |
625d988a CW |
330 | struct drm_mm_node *target, |
331 | unsigned int flags) | |
506a8e87 | 332 | { |
172ae5b4 CW |
333 | LIST_HEAD(eviction_list); |
334 | struct drm_mm_node *node; | |
625d988a CW |
335 | u64 start = target->start; |
336 | u64 end = start + target->size; | |
172ae5b4 | 337 | struct i915_vma *vma, *next; |
172ae5b4 | 338 | int ret = 0; |
506a8e87 | 339 | |
2850748e | 340 | lockdep_assert_held(&vm->mutex); |
a6508ded CW |
341 | GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE)); |
342 | GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE)); | |
343 | ||
625d988a | 344 | trace_i915_gem_evict_node(vm, target, flags); |
172ae5b4 | 345 | |
f33a8a51 CW |
346 | /* |
347 | * Retire before we search the active list. Although we have | |
7155b057 CW |
348 | * reasonable accuracy in our retirement lists, we may have |
349 | * a stray pin (preventing eviction) that can only be resolved by | |
350 | * retiring. | |
351 | */ | |
0f857158 AI |
352 | if (i915_is_ggtt(vm)) { |
353 | struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); | |
354 | struct intel_gt *gt; | |
355 | ||
356 | list_for_each_entry(gt, &ggtt->gt_list, ggtt_link) | |
357 | intel_gt_retire_requests(gt); | |
358 | } else { | |
359 | intel_gt_retire_requests(vm->gt); | |
360 | } | |
7155b057 | 361 | |
33dd8899 | 362 | if (i915_vm_has_cache_coloring(vm)) { |
172ae5b4 | 363 | /* Expand search to cover neighbouring guard pages (or lack!) */ |
381b943b | 364 | if (start) |
f51455d4 | 365 | start -= I915_GTT_PAGE_SIZE; |
a6508ded CW |
366 | |
367 | /* Always look at the page afterwards to avoid the end-of-GTT */ | |
368 | end += I915_GTT_PAGE_SIZE; | |
172ae5b4 | 369 | } |
a6508ded | 370 | GEM_BUG_ON(start >= end); |
4c7d62c6 | 371 | |
625d988a | 372 | drm_mm_for_each_node_in_range(node, &vm->mm, start, end) { |
172ae5b4 CW |
373 | /* If we find any non-objects (!vma), we cannot evict them */ |
374 | if (node->color == I915_COLOR_UNEVICTABLE) { | |
375 | ret = -ENOSPC; | |
506a8e87 | 376 | break; |
172ae5b4 | 377 | } |
506a8e87 | 378 | |
b290a78b | 379 | GEM_BUG_ON(!drm_mm_node_allocated(node)); |
506a8e87 CW |
380 | vma = container_of(node, typeof(*vma), node); |
381 | ||
2920bb94 CW |
382 | /* |
383 | * If we are using coloring to insert guard pages between | |
172ae5b4 CW |
384 | * different cache domains within the address space, we have |
385 | * to check whether the objects on either side of our range | |
386 | * abutt and conflict. If they are in conflict, then we evict | |
387 | * those as well to make room for our guard pages. | |
388 | */ | |
33dd8899 | 389 | if (i915_vm_has_cache_coloring(vm)) { |
fe65cbdb MA |
390 | if (node->start + node->size == target->start) { |
391 | if (node->color == target->color) | |
172ae5b4 CW |
392 | continue; |
393 | } | |
fe65cbdb MA |
394 | if (node->start == target->start + target->size) { |
395 | if (node->color == target->color) | |
172ae5b4 CW |
396 | continue; |
397 | } | |
398 | } | |
506a8e87 | 399 | |
2920bb94 | 400 | if (i915_vma_is_pinned(vma)) { |
172ae5b4 CW |
401 | ret = -ENOSPC; |
402 | break; | |
403 | } | |
506a8e87 | 404 | |
2920bb94 | 405 | if (flags & PIN_NONBLOCK && i915_vma_is_active(vma)) { |
172ae5b4 | 406 | ret = -ENOSPC; |
172ae5b4 | 407 | break; |
506a8e87 CW |
408 | } |
409 | ||
7e00897b ML |
410 | if (!grab_vma(vma, ww)) { |
411 | ret = -ENOSPC; | |
412 | break; | |
413 | } | |
414 | ||
2920bb94 CW |
415 | /* |
416 | * Never show fear in the face of dragons! | |
172ae5b4 CW |
417 | * |
418 | * We cannot directly remove this node from within this | |
419 | * iterator and as with i915_gem_evict_something() we employ | |
420 | * the vma pin_count in order to prevent the action of | |
421 | * unbinding one vma from freeing (by dropping its active | |
422 | * reference) another in our eviction list. | |
423 | */ | |
424 | __i915_vma_pin(vma); | |
8c45cec4 | 425 | list_add(&vma->evict_link, &eviction_list); |
172ae5b4 CW |
426 | } |
427 | ||
8c45cec4 | 428 | list_for_each_entry_safe(vma, next, &eviction_list, evict_link) { |
172ae5b4 CW |
429 | __i915_vma_unpin(vma); |
430 | if (ret == 0) | |
2850748e | 431 | ret = __i915_vma_unbind(vma); |
7e00897b ML |
432 | |
433 | ungrab_vma(vma); | |
506a8e87 CW |
434 | } |
435 | ||
172ae5b4 | 436 | return ret; |
506a8e87 CW |
437 | } |
438 | ||
68c8c17f | 439 | /** |
c2c1d491 | 440 | * i915_gem_evict_vm - Evict all idle vmas from a vm |
c2c1d491 | 441 | * @vm: Address space to cleanse |
6945c53b ML |
442 | * @ww: An optional struct i915_gem_ww_ctx. If not NULL, i915_gem_evict_vm |
443 | * will be able to evict vma's locked by the ww as well. | |
801fa7a8 MA |
444 | * @busy_bo: Optional pointer to struct drm_i915_gem_object. If not NULL, then |
445 | * in the event i915_gem_evict_vm() is unable to trylock an object for eviction, | |
446 | * then @busy_bo will point to it. -EBUSY is also returned. The caller must drop | |
447 | * the vm->mutex, before trying again to acquire the contended lock. The caller | |
448 | * also owns a reference to the object. | |
68c8c17f | 449 | * |
2889caa9 | 450 | * This function evicts all vmas from a vm. |
68c8c17f | 451 | * |
c2c1d491 DV |
452 | * This is used by the execbuf code as a last-ditch effort to defragment the |
453 | * address space. | |
454 | * | |
455 | * To clarify: This is for freeing up virtual address space, not for freeing | |
456 | * memory in e.g. the shrinker. | |
68c8c17f | 457 | */ |
801fa7a8 MA |
458 | int i915_gem_evict_vm(struct i915_address_space *vm, struct i915_gem_ww_ctx *ww, |
459 | struct drm_i915_gem_object **busy_bo) | |
7b796122 | 460 | { |
a725d711 | 461 | int ret = 0; |
7b796122 | 462 | |
2850748e | 463 | lockdep_assert_held(&vm->mutex); |
bcccff84 BW |
464 | trace_i915_gem_evict_vm(vm); |
465 | ||
2889caa9 CW |
466 | /* Switch back to the default context in order to unpin |
467 | * the existing context objects. However, such objects only | |
468 | * pin themselves inside the global GTT and performing the | |
469 | * switch otherwise is ineffective. | |
470 | */ | |
471 | if (i915_is_ggtt(vm)) { | |
0f857158 | 472 | ret = ggtt_flush(vm); |
7b796122 BW |
473 | if (ret) |
474 | return ret; | |
7b796122 BW |
475 | } |
476 | ||
a725d711 CW |
477 | do { |
478 | struct i915_vma *vma, *vn; | |
479 | LIST_HEAD(eviction_list); | |
6945c53b | 480 | LIST_HEAD(locked_eviction_list); |
7b796122 | 481 | |
a725d711 CW |
482 | list_for_each_entry(vma, &vm->bound_list, vm_link) { |
483 | if (i915_vma_is_pinned(vma)) | |
484 | continue; | |
485 | ||
6945c53b ML |
486 | /* |
487 | * If we already own the lock, trylock fails. In case | |
488 | * the resv is shared among multiple objects, we still | |
489 | * need the object ref. | |
490 | */ | |
801fa7a8 | 491 | if (!i915_gem_object_get_rcu(vma->obj) || |
6945c53b ML |
492 | (ww && (dma_resv_locking_ctx(vma->obj->base.resv) == &ww->ctx))) { |
493 | __i915_vma_pin(vma); | |
494 | list_add(&vma->evict_link, &locked_eviction_list); | |
495 | continue; | |
496 | } | |
497 | ||
801fa7a8 MA |
498 | if (!i915_gem_object_trylock(vma->obj, ww)) { |
499 | if (busy_bo) { | |
500 | *busy_bo = vma->obj; /* holds ref */ | |
501 | ret = -EBUSY; | |
502 | break; | |
503 | } | |
504 | i915_gem_object_put(vma->obj); | |
6945c53b | 505 | continue; |
801fa7a8 | 506 | } |
6945c53b | 507 | |
a725d711 CW |
508 | __i915_vma_pin(vma); |
509 | list_add(&vma->evict_link, &eviction_list); | |
510 | } | |
6945c53b | 511 | if (list_empty(&eviction_list) && list_empty(&locked_eviction_list)) |
a725d711 CW |
512 | break; |
513 | ||
6945c53b ML |
514 | /* Unbind locked objects first, before unlocking the eviction_list */ |
515 | list_for_each_entry_safe(vma, vn, &locked_eviction_list, evict_link) { | |
516 | __i915_vma_unpin(vma); | |
517 | ||
801fa7a8 | 518 | if (ret == 0) { |
6945c53b | 519 | ret = __i915_vma_unbind(vma); |
801fa7a8 MA |
520 | if (ret != -EINTR) /* "Get me out of here!" */ |
521 | ret = 0; | |
522 | } | |
523 | if (!dying_vma(vma)) | |
524 | i915_gem_object_put(vma->obj); | |
6945c53b ML |
525 | } |
526 | ||
a725d711 CW |
527 | list_for_each_entry_safe(vma, vn, &eviction_list, evict_link) { |
528 | __i915_vma_unpin(vma); | |
801fa7a8 | 529 | if (ret == 0) { |
a725d711 | 530 | ret = __i915_vma_unbind(vma); |
801fa7a8 MA |
531 | if (ret != -EINTR) /* "Get me out of here!" */ |
532 | ret = 0; | |
533 | } | |
6945c53b ML |
534 | |
535 | i915_gem_object_unlock(vma->obj); | |
801fa7a8 | 536 | i915_gem_object_put(vma->obj); |
a725d711 CW |
537 | } |
538 | } while (ret == 0); | |
2889caa9 | 539 | |
2889caa9 | 540 | return ret; |
7b796122 | 541 | } |
f40a7b75 CW |
542 | |
543 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) | |
544 | #include "selftests/i915_gem_evict.c" | |
545 | #endif |