Commit | Line | Data |
---|---|---|
b42fe9ca JL |
1 | /* |
2 | * Copyright © 2016 Intel Corporation | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice (including the next | |
12 | * paragraph) shall be included in all copies or substantial portions of the | |
13 | * Software. | |
14 | * | |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS | |
21 | * IN THE SOFTWARE. | |
22 | * | |
23 | */ | |
24 | ||
25 | #include "i915_vma.h" | |
26 | ||
27 | #include "i915_drv.h" | |
28 | #include "intel_ringbuffer.h" | |
29 | #include "intel_frontbuffer.h" | |
30 | ||
31 | #include <drm/drm_gem.h> | |
32 | ||
33 | static void | |
34 | i915_vma_retire(struct i915_gem_active *active, | |
35 | struct drm_i915_gem_request *rq) | |
36 | { | |
37 | const unsigned int idx = rq->engine->id; | |
38 | struct i915_vma *vma = | |
39 | container_of(active, struct i915_vma, last_read[idx]); | |
40 | struct drm_i915_gem_object *obj = vma->obj; | |
41 | ||
42 | GEM_BUG_ON(!i915_vma_has_active_engine(vma, idx)); | |
43 | ||
44 | i915_vma_clear_active(vma, idx); | |
45 | if (i915_vma_is_active(vma)) | |
46 | return; | |
47 | ||
44a0ec0d | 48 | GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); |
b42fe9ca JL |
49 | list_move_tail(&vma->vm_link, &vma->vm->inactive_list); |
50 | if (unlikely(i915_vma_is_closed(vma) && !i915_vma_is_pinned(vma))) | |
51 | WARN_ON(i915_vma_unbind(vma)); | |
52 | ||
53 | GEM_BUG_ON(!i915_gem_object_is_active(obj)); | |
54 | if (--obj->active_count) | |
55 | return; | |
56 | ||
1ab22356 CW |
57 | /* Prune the shared fence arrays iff completely idle (inc. external) */ |
58 | if (reservation_object_trylock(obj->resv)) { | |
59 | if (reservation_object_test_signaled_rcu(obj->resv, true)) | |
60 | reservation_object_add_excl_fence(obj->resv, NULL); | |
61 | reservation_object_unlock(obj->resv); | |
62 | } | |
63 | ||
b42fe9ca JL |
64 | /* Bump our place on the bound list to keep it roughly in LRU order |
65 | * so that we don't steal from recently used but inactive objects | |
66 | * (unless we are forced to ofc!) | |
67 | */ | |
f2123818 | 68 | spin_lock(&rq->i915->mm.obj_lock); |
b42fe9ca | 69 | if (obj->bind_count) |
f2123818 CW |
70 | list_move_tail(&obj->mm.link, &rq->i915->mm.bound_list); |
71 | spin_unlock(&rq->i915->mm.obj_lock); | |
b42fe9ca JL |
72 | |
73 | obj->mm.dirty = true; /* be paranoid */ | |
74 | ||
75 | if (i915_gem_object_has_active_reference(obj)) { | |
76 | i915_gem_object_clear_active_reference(obj); | |
77 | i915_gem_object_put(obj); | |
78 | } | |
79 | } | |
80 | ||
b42fe9ca | 81 | static struct i915_vma * |
a01cb37a CW |
82 | vma_create(struct drm_i915_gem_object *obj, |
83 | struct i915_address_space *vm, | |
84 | const struct i915_ggtt_view *view) | |
b42fe9ca JL |
85 | { |
86 | struct i915_vma *vma; | |
87 | struct rb_node *rb, **p; | |
88 | int i; | |
89 | ||
e1cc3db0 CW |
90 | /* The aliasing_ppgtt should never be used directly! */ |
91 | GEM_BUG_ON(vm == &vm->i915->mm.aliasing_ppgtt->base); | |
92 | ||
1fcdaa7e | 93 | vma = kmem_cache_zalloc(vm->i915->vmas, GFP_KERNEL); |
b42fe9ca JL |
94 | if (vma == NULL) |
95 | return ERR_PTR(-ENOMEM); | |
96 | ||
b42fe9ca JL |
97 | for (i = 0; i < ARRAY_SIZE(vma->last_read); i++) |
98 | init_request_active(&vma->last_read[i], i915_vma_retire); | |
b42fe9ca | 99 | init_request_active(&vma->last_fence, NULL); |
b42fe9ca JL |
100 | vma->vm = vm; |
101 | vma->obj = obj; | |
95ff7c7d | 102 | vma->resv = obj->resv; |
b42fe9ca | 103 | vma->size = obj->base.size; |
f51455d4 | 104 | vma->display_alignment = I915_GTT_MIN_ALIGNMENT; |
b42fe9ca | 105 | |
7c518460 | 106 | if (view && view->type != I915_GGTT_VIEW_NORMAL) { |
b42fe9ca JL |
107 | vma->ggtt_view = *view; |
108 | if (view->type == I915_GGTT_VIEW_PARTIAL) { | |
07e19ea4 | 109 | GEM_BUG_ON(range_overflows_t(u64, |
8bab1193 CW |
110 | view->partial.offset, |
111 | view->partial.size, | |
07e19ea4 | 112 | obj->base.size >> PAGE_SHIFT)); |
8bab1193 | 113 | vma->size = view->partial.size; |
b42fe9ca | 114 | vma->size <<= PAGE_SHIFT; |
07e19ea4 | 115 | GEM_BUG_ON(vma->size >= obj->base.size); |
b42fe9ca | 116 | } else if (view->type == I915_GGTT_VIEW_ROTATED) { |
8bab1193 | 117 | vma->size = intel_rotation_info_size(&view->rotated); |
b42fe9ca JL |
118 | vma->size <<= PAGE_SHIFT; |
119 | } | |
120 | } | |
121 | ||
1fcdaa7e CW |
122 | if (unlikely(vma->size > vm->total)) |
123 | goto err_vma; | |
124 | ||
b00ddb27 CW |
125 | GEM_BUG_ON(!IS_ALIGNED(vma->size, I915_GTT_PAGE_SIZE)); |
126 | ||
b42fe9ca | 127 | if (i915_is_ggtt(vm)) { |
1fcdaa7e CW |
128 | if (unlikely(overflows_type(vma->size, u32))) |
129 | goto err_vma; | |
130 | ||
91d4e0aa CW |
131 | vma->fence_size = i915_gem_fence_size(vm->i915, vma->size, |
132 | i915_gem_object_get_tiling(obj), | |
133 | i915_gem_object_get_stride(obj)); | |
1fcdaa7e CW |
134 | if (unlikely(vma->fence_size < vma->size || /* overflow */ |
135 | vma->fence_size > vm->total)) | |
136 | goto err_vma; | |
137 | ||
f51455d4 | 138 | GEM_BUG_ON(!IS_ALIGNED(vma->fence_size, I915_GTT_MIN_ALIGNMENT)); |
944397f0 | 139 | |
91d4e0aa CW |
140 | vma->fence_alignment = i915_gem_fence_alignment(vm->i915, vma->size, |
141 | i915_gem_object_get_tiling(obj), | |
142 | i915_gem_object_get_stride(obj)); | |
944397f0 CW |
143 | GEM_BUG_ON(!is_power_of_2(vma->fence_alignment)); |
144 | ||
e2189dd0 CW |
145 | /* |
146 | * We put the GGTT vma at the start of the vma-list, followed | |
147 | * by the ppGGTT vma. This allows us to break early when | |
148 | * iterating over only the GGTT vma for an object, see | |
149 | * for_each_ggtt_vma() | |
150 | */ | |
b42fe9ca JL |
151 | vma->flags |= I915_VMA_GGTT; |
152 | list_add(&vma->obj_link, &obj->vma_list); | |
153 | } else { | |
154 | i915_ppgtt_get(i915_vm_to_ppgtt(vm)); | |
155 | list_add_tail(&vma->obj_link, &obj->vma_list); | |
156 | } | |
157 | ||
158 | rb = NULL; | |
159 | p = &obj->vma_tree.rb_node; | |
160 | while (*p) { | |
161 | struct i915_vma *pos; | |
162 | ||
163 | rb = *p; | |
164 | pos = rb_entry(rb, struct i915_vma, obj_node); | |
165 | if (i915_vma_compare(pos, vm, view) < 0) | |
166 | p = &rb->rb_right; | |
167 | else | |
168 | p = &rb->rb_left; | |
169 | } | |
170 | rb_link_node(&vma->obj_node, rb, p); | |
171 | rb_insert_color(&vma->obj_node, &obj->vma_tree); | |
1fcdaa7e | 172 | list_add(&vma->vm_link, &vm->unbound_list); |
b42fe9ca JL |
173 | |
174 | return vma; | |
1fcdaa7e CW |
175 | |
176 | err_vma: | |
177 | kmem_cache_free(vm->i915->vmas, vma); | |
178 | return ERR_PTR(-E2BIG); | |
b42fe9ca JL |
179 | } |
180 | ||
481a6f7d CW |
181 | static struct i915_vma * |
182 | vma_lookup(struct drm_i915_gem_object *obj, | |
183 | struct i915_address_space *vm, | |
184 | const struct i915_ggtt_view *view) | |
718659a6 CW |
185 | { |
186 | struct rb_node *rb; | |
187 | ||
718659a6 CW |
188 | rb = obj->vma_tree.rb_node; |
189 | while (rb) { | |
190 | struct i915_vma *vma = rb_entry(rb, struct i915_vma, obj_node); | |
191 | long cmp; | |
192 | ||
193 | cmp = i915_vma_compare(vma, vm, view); | |
194 | if (cmp == 0) | |
195 | return vma; | |
196 | ||
197 | if (cmp < 0) | |
198 | rb = rb->rb_right; | |
199 | else | |
200 | rb = rb->rb_left; | |
201 | } | |
202 | ||
203 | return NULL; | |
204 | } | |
205 | ||
718659a6 CW |
206 | /** |
207 | * i915_vma_instance - return the singleton instance of the VMA | |
208 | * @obj: parent &struct drm_i915_gem_object to be mapped | |
209 | * @vm: address space in which the mapping is located | |
210 | * @view: additional mapping requirements | |
211 | * | |
212 | * i915_vma_instance() looks up an existing VMA of the @obj in the @vm with | |
213 | * the same @view characteristics. If a match is not found, one is created. | |
214 | * Once created, the VMA is kept until either the object is freed, or the | |
215 | * address space is closed. | |
216 | * | |
217 | * Must be called with struct_mutex held. | |
218 | * | |
219 | * Returns the vma, or an error pointer. | |
220 | */ | |
221 | struct i915_vma * | |
222 | i915_vma_instance(struct drm_i915_gem_object *obj, | |
223 | struct i915_address_space *vm, | |
224 | const struct i915_ggtt_view *view) | |
225 | { | |
226 | struct i915_vma *vma; | |
227 | ||
228 | lockdep_assert_held(&obj->base.dev->struct_mutex); | |
229 | GEM_BUG_ON(view && !i915_is_ggtt(vm)); | |
230 | GEM_BUG_ON(vm->closed); | |
231 | ||
481a6f7d | 232 | vma = vma_lookup(obj, vm, view); |
718659a6 | 233 | if (!vma) |
a01cb37a | 234 | vma = vma_create(obj, vm, view); |
718659a6 CW |
235 | |
236 | GEM_BUG_ON(!IS_ERR(vma) && i915_vma_is_closed(vma)); | |
4ea9527c | 237 | GEM_BUG_ON(!IS_ERR(vma) && i915_vma_compare(vma, vm, view)); |
481a6f7d | 238 | GEM_BUG_ON(!IS_ERR(vma) && vma_lookup(obj, vm, view) != vma); |
718659a6 CW |
239 | return vma; |
240 | } | |
241 | ||
b42fe9ca JL |
242 | /** |
243 | * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space. | |
244 | * @vma: VMA to map | |
245 | * @cache_level: mapping cache level | |
246 | * @flags: flags like global or local mapping | |
247 | * | |
248 | * DMA addresses are taken from the scatter-gather table of this object (or of | |
249 | * this VMA in case of non-default GGTT views) and PTE entries set up. | |
250 | * Note that DMA addresses are also the only part of the SG table we care about. | |
251 | */ | |
252 | int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level, | |
253 | u32 flags) | |
254 | { | |
255 | u32 bind_flags; | |
256 | u32 vma_flags; | |
257 | int ret; | |
258 | ||
aa149431 CW |
259 | GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); |
260 | GEM_BUG_ON(vma->size > vma->node.size); | |
261 | ||
262 | if (GEM_WARN_ON(range_overflows(vma->node.start, | |
263 | vma->node.size, | |
264 | vma->vm->total))) | |
265 | return -ENODEV; | |
266 | ||
267 | if (GEM_WARN_ON(!flags)) | |
b42fe9ca JL |
268 | return -EINVAL; |
269 | ||
270 | bind_flags = 0; | |
271 | if (flags & PIN_GLOBAL) | |
272 | bind_flags |= I915_VMA_GLOBAL_BIND; | |
273 | if (flags & PIN_USER) | |
274 | bind_flags |= I915_VMA_LOCAL_BIND; | |
275 | ||
276 | vma_flags = vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND); | |
277 | if (flags & PIN_UPDATE) | |
278 | bind_flags |= vma_flags; | |
279 | else | |
280 | bind_flags &= ~vma_flags; | |
281 | if (bind_flags == 0) | |
282 | return 0; | |
283 | ||
fa3f46af MA |
284 | GEM_BUG_ON(!vma->pages); |
285 | ||
6146e6da | 286 | trace_i915_vma_bind(vma, bind_flags); |
b42fe9ca JL |
287 | ret = vma->vm->bind_vma(vma, cache_level, bind_flags); |
288 | if (ret) | |
289 | return ret; | |
290 | ||
291 | vma->flags |= bind_flags; | |
292 | return 0; | |
293 | } | |
294 | ||
295 | void __iomem *i915_vma_pin_iomap(struct i915_vma *vma) | |
296 | { | |
297 | void __iomem *ptr; | |
b4563f59 | 298 | int err; |
b42fe9ca JL |
299 | |
300 | /* Access through the GTT requires the device to be awake. */ | |
49d73912 | 301 | assert_rpm_wakelock_held(vma->vm->i915); |
b42fe9ca | 302 | |
49d73912 | 303 | lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); |
b4563f59 CW |
304 | if (WARN_ON(!i915_vma_is_map_and_fenceable(vma))) { |
305 | err = -ENODEV; | |
306 | goto err; | |
307 | } | |
b42fe9ca JL |
308 | |
309 | GEM_BUG_ON(!i915_vma_is_ggtt(vma)); | |
310 | GEM_BUG_ON((vma->flags & I915_VMA_GLOBAL_BIND) == 0); | |
311 | ||
312 | ptr = vma->iomap; | |
313 | if (ptr == NULL) { | |
73ebd503 | 314 | ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->iomap, |
b42fe9ca JL |
315 | vma->node.start, |
316 | vma->node.size); | |
b4563f59 CW |
317 | if (ptr == NULL) { |
318 | err = -ENOMEM; | |
319 | goto err; | |
320 | } | |
b42fe9ca JL |
321 | |
322 | vma->iomap = ptr; | |
323 | } | |
324 | ||
325 | __i915_vma_pin(vma); | |
b4563f59 | 326 | |
3bd40735 | 327 | err = i915_vma_pin_fence(vma); |
b4563f59 CW |
328 | if (err) |
329 | goto err_unpin; | |
330 | ||
7125397b | 331 | i915_vma_set_ggtt_write(vma); |
b42fe9ca | 332 | return ptr; |
b4563f59 CW |
333 | |
334 | err_unpin: | |
335 | __i915_vma_unpin(vma); | |
336 | err: | |
337 | return IO_ERR_PTR(err); | |
338 | } | |
339 | ||
7125397b CW |
340 | void i915_vma_flush_writes(struct i915_vma *vma) |
341 | { | |
342 | if (!i915_vma_has_ggtt_write(vma)) | |
343 | return; | |
344 | ||
345 | i915_gem_flush_ggtt_writes(vma->vm->i915); | |
346 | ||
347 | i915_vma_unset_ggtt_write(vma); | |
348 | } | |
349 | ||
b4563f59 CW |
350 | void i915_vma_unpin_iomap(struct i915_vma *vma) |
351 | { | |
352 | lockdep_assert_held(&vma->obj->base.dev->struct_mutex); | |
353 | ||
354 | GEM_BUG_ON(vma->iomap == NULL); | |
355 | ||
7125397b CW |
356 | i915_vma_flush_writes(vma); |
357 | ||
b4563f59 CW |
358 | i915_vma_unpin_fence(vma); |
359 | i915_vma_unpin(vma); | |
b42fe9ca JL |
360 | } |
361 | ||
362 | void i915_vma_unpin_and_release(struct i915_vma **p_vma) | |
363 | { | |
364 | struct i915_vma *vma; | |
365 | struct drm_i915_gem_object *obj; | |
366 | ||
367 | vma = fetch_and_zero(p_vma); | |
368 | if (!vma) | |
369 | return; | |
370 | ||
371 | obj = vma->obj; | |
372 | ||
373 | i915_vma_unpin(vma); | |
374 | i915_vma_close(vma); | |
375 | ||
376 | __i915_gem_object_release_unless_active(obj); | |
377 | } | |
378 | ||
782a3e9e CW |
379 | bool i915_vma_misplaced(const struct i915_vma *vma, |
380 | u64 size, u64 alignment, u64 flags) | |
b42fe9ca JL |
381 | { |
382 | if (!drm_mm_node_allocated(&vma->node)) | |
383 | return false; | |
384 | ||
385 | if (vma->node.size < size) | |
386 | return true; | |
387 | ||
f51455d4 CW |
388 | GEM_BUG_ON(alignment && !is_power_of_2(alignment)); |
389 | if (alignment && !IS_ALIGNED(vma->node.start, alignment)) | |
b42fe9ca JL |
390 | return true; |
391 | ||
392 | if (flags & PIN_MAPPABLE && !i915_vma_is_map_and_fenceable(vma)) | |
393 | return true; | |
394 | ||
395 | if (flags & PIN_OFFSET_BIAS && | |
396 | vma->node.start < (flags & PIN_OFFSET_MASK)) | |
397 | return true; | |
398 | ||
399 | if (flags & PIN_OFFSET_FIXED && | |
400 | vma->node.start != (flags & PIN_OFFSET_MASK)) | |
401 | return true; | |
402 | ||
403 | return false; | |
404 | } | |
405 | ||
406 | void __i915_vma_set_map_and_fenceable(struct i915_vma *vma) | |
407 | { | |
b42fe9ca | 408 | bool mappable, fenceable; |
b42fe9ca | 409 | |
944397f0 CW |
410 | GEM_BUG_ON(!i915_vma_is_ggtt(vma)); |
411 | GEM_BUG_ON(!vma->fence_size); | |
b42fe9ca JL |
412 | |
413 | /* | |
414 | * Explicitly disable for rotated VMA since the display does not | |
415 | * need the fence and the VMA is not accessible to other users. | |
416 | */ | |
944397f0 CW |
417 | if (vma->ggtt_view.type == I915_GGTT_VIEW_ROTATED) |
418 | return; | |
419 | ||
420 | fenceable = (vma->node.size >= vma->fence_size && | |
f51455d4 | 421 | IS_ALIGNED(vma->node.start, vma->fence_alignment)); |
944397f0 CW |
422 | |
423 | mappable = vma->node.start + vma->fence_size <= i915_vm_to_ggtt(vma->vm)->mappable_end; | |
424 | ||
425 | if (mappable && fenceable) | |
b42fe9ca JL |
426 | vma->flags |= I915_VMA_CAN_FENCE; |
427 | else | |
428 | vma->flags &= ~I915_VMA_CAN_FENCE; | |
429 | } | |
430 | ||
7d1d9aea | 431 | static bool color_differs(struct drm_mm_node *node, unsigned long color) |
b42fe9ca | 432 | { |
7d1d9aea CW |
433 | return node->allocated && node->color != color; |
434 | } | |
435 | ||
436 | bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long cache_level) | |
437 | { | |
438 | struct drm_mm_node *node = &vma->node; | |
b42fe9ca JL |
439 | struct drm_mm_node *other; |
440 | ||
441 | /* | |
442 | * On some machines we have to be careful when putting differing types | |
443 | * of snoopable memory together to avoid the prefetcher crossing memory | |
444 | * domains and dying. During vm initialisation, we decide whether or not | |
445 | * these constraints apply and set the drm_mm.color_adjust | |
446 | * appropriately. | |
447 | */ | |
448 | if (vma->vm->mm.color_adjust == NULL) | |
449 | return true; | |
450 | ||
7d1d9aea CW |
451 | /* Only valid to be called on an already inserted vma */ |
452 | GEM_BUG_ON(!drm_mm_node_allocated(node)); | |
453 | GEM_BUG_ON(list_empty(&node->node_list)); | |
b42fe9ca | 454 | |
7d1d9aea | 455 | other = list_prev_entry(node, node_list); |
ef426c10 | 456 | if (color_differs(other, cache_level) && !drm_mm_hole_follows(other)) |
b42fe9ca JL |
457 | return false; |
458 | ||
7d1d9aea | 459 | other = list_next_entry(node, node_list); |
ef426c10 | 460 | if (color_differs(other, cache_level) && !drm_mm_hole_follows(node)) |
b42fe9ca JL |
461 | return false; |
462 | ||
463 | return true; | |
464 | } | |
465 | ||
466 | /** | |
467 | * i915_vma_insert - finds a slot for the vma in its address space | |
468 | * @vma: the vma | |
469 | * @size: requested size in bytes (can be larger than the VMA) | |
470 | * @alignment: required alignment | |
471 | * @flags: mask of PIN_* flags to use | |
472 | * | |
473 | * First we try to allocate some free space that meets the requirements for | |
474 | * the VMA. Failiing that, if the flags permit, it will evict an old VMA, | |
475 | * preferrably the oldest idle entry to make room for the new VMA. | |
476 | * | |
477 | * Returns: | |
478 | * 0 on success, negative error code otherwise. | |
479 | */ | |
480 | static int | |
481 | i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) | |
482 | { | |
49d73912 | 483 | struct drm_i915_private *dev_priv = vma->vm->i915; |
b42fe9ca JL |
484 | struct drm_i915_gem_object *obj = vma->obj; |
485 | u64 start, end; | |
486 | int ret; | |
487 | ||
010e3e68 | 488 | GEM_BUG_ON(i915_vma_is_closed(vma)); |
b42fe9ca JL |
489 | GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND)); |
490 | GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); | |
491 | ||
492 | size = max(size, vma->size); | |
944397f0 CW |
493 | alignment = max(alignment, vma->display_alignment); |
494 | if (flags & PIN_MAPPABLE) { | |
495 | size = max_t(typeof(size), size, vma->fence_size); | |
496 | alignment = max_t(typeof(alignment), | |
497 | alignment, vma->fence_alignment); | |
498 | } | |
b42fe9ca | 499 | |
f51455d4 CW |
500 | GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); |
501 | GEM_BUG_ON(!IS_ALIGNED(alignment, I915_GTT_MIN_ALIGNMENT)); | |
502 | GEM_BUG_ON(!is_power_of_2(alignment)); | |
503 | ||
b42fe9ca | 504 | start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0; |
f51455d4 | 505 | GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE)); |
b42fe9ca JL |
506 | |
507 | end = vma->vm->total; | |
508 | if (flags & PIN_MAPPABLE) | |
509 | end = min_t(u64, end, dev_priv->ggtt.mappable_end); | |
510 | if (flags & PIN_ZONE_4G) | |
f51455d4 CW |
511 | end = min_t(u64, end, (1ULL << 32) - I915_GTT_PAGE_SIZE); |
512 | GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE)); | |
b42fe9ca JL |
513 | |
514 | /* If binding the object/GGTT view requires more space than the entire | |
515 | * aperture has, reject it early before evicting everything in a vain | |
516 | * attempt to find space. | |
517 | */ | |
518 | if (size > end) { | |
519 | DRM_DEBUG("Attempting to bind an object larger than the aperture: request=%llu [object=%zd] > %s aperture=%llu\n", | |
520 | size, obj->base.size, | |
521 | flags & PIN_MAPPABLE ? "mappable" : "total", | |
522 | end); | |
2889caa9 | 523 | return -ENOSPC; |
b42fe9ca JL |
524 | } |
525 | ||
526 | ret = i915_gem_object_pin_pages(obj); | |
527 | if (ret) | |
528 | return ret; | |
529 | ||
fa3f46af MA |
530 | GEM_BUG_ON(vma->pages); |
531 | ||
532 | ret = vma->vm->set_pages(vma); | |
533 | if (ret) | |
534 | goto err_unpin; | |
535 | ||
b42fe9ca JL |
536 | if (flags & PIN_OFFSET_FIXED) { |
537 | u64 offset = flags & PIN_OFFSET_MASK; | |
f51455d4 | 538 | if (!IS_ALIGNED(offset, alignment) || |
e8f9ae9b | 539 | range_overflows(offset, size, end)) { |
b42fe9ca | 540 | ret = -EINVAL; |
fa3f46af | 541 | goto err_clear; |
b42fe9ca JL |
542 | } |
543 | ||
625d988a CW |
544 | ret = i915_gem_gtt_reserve(vma->vm, &vma->node, |
545 | size, offset, obj->cache_level, | |
546 | flags); | |
547 | if (ret) | |
fa3f46af | 548 | goto err_clear; |
b42fe9ca | 549 | } else { |
7464284b MA |
550 | /* |
551 | * We only support huge gtt pages through the 48b PPGTT, | |
552 | * however we also don't want to force any alignment for | |
553 | * objects which need to be tightly packed into the low 32bits. | |
554 | * | |
555 | * Note that we assume that GGTT are limited to 4GiB for the | |
556 | * forseeable future. See also i915_ggtt_offset(). | |
557 | */ | |
558 | if (upper_32_bits(end - 1) && | |
559 | vma->page_sizes.sg > I915_GTT_PAGE_SIZE) { | |
855822be MA |
560 | /* |
561 | * We can't mix 64K and 4K PTEs in the same page-table | |
562 | * (2M block), and so to avoid the ugliness and | |
563 | * complexity of coloring we opt for just aligning 64K | |
564 | * objects to 2M. | |
565 | */ | |
7464284b | 566 | u64 page_alignment = |
855822be MA |
567 | rounddown_pow_of_two(vma->page_sizes.sg | |
568 | I915_GTT_PAGE_SIZE_2M); | |
7464284b | 569 | |
bef27bdb CW |
570 | /* |
571 | * Check we don't expand for the limited Global GTT | |
572 | * (mappable aperture is even more precious!). This | |
573 | * also checks that we exclude the aliasing-ppgtt. | |
574 | */ | |
575 | GEM_BUG_ON(i915_vma_is_ggtt(vma)); | |
576 | ||
7464284b | 577 | alignment = max(alignment, page_alignment); |
855822be MA |
578 | |
579 | if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) | |
580 | size = round_up(size, I915_GTT_PAGE_SIZE_2M); | |
7464284b MA |
581 | } |
582 | ||
e007b19d CW |
583 | ret = i915_gem_gtt_insert(vma->vm, &vma->node, |
584 | size, alignment, obj->cache_level, | |
585 | start, end, flags); | |
586 | if (ret) | |
fa3f46af | 587 | goto err_clear; |
b42fe9ca JL |
588 | |
589 | GEM_BUG_ON(vma->node.start < start); | |
590 | GEM_BUG_ON(vma->node.start + vma->node.size > end); | |
591 | } | |
44a0ec0d | 592 | GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); |
b42fe9ca JL |
593 | GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level)); |
594 | ||
b42fe9ca | 595 | list_move_tail(&vma->vm_link, &vma->vm->inactive_list); |
f2123818 CW |
596 | |
597 | spin_lock(&dev_priv->mm.obj_lock); | |
598 | list_move_tail(&obj->mm.link, &dev_priv->mm.bound_list); | |
b42fe9ca | 599 | obj->bind_count++; |
f2123818 CW |
600 | spin_unlock(&dev_priv->mm.obj_lock); |
601 | ||
b42fe9ca JL |
602 | GEM_BUG_ON(atomic_read(&obj->mm.pages_pin_count) < obj->bind_count); |
603 | ||
604 | return 0; | |
605 | ||
fa3f46af MA |
606 | err_clear: |
607 | vma->vm->clear_pages(vma); | |
b42fe9ca JL |
608 | err_unpin: |
609 | i915_gem_object_unpin_pages(obj); | |
610 | return ret; | |
611 | } | |
612 | ||
31c7effa CW |
613 | static void |
614 | i915_vma_remove(struct i915_vma *vma) | |
615 | { | |
f2123818 | 616 | struct drm_i915_private *i915 = vma->vm->i915; |
31c7effa CW |
617 | struct drm_i915_gem_object *obj = vma->obj; |
618 | ||
619 | GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); | |
620 | GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND)); | |
621 | ||
fa3f46af MA |
622 | vma->vm->clear_pages(vma); |
623 | ||
31c7effa CW |
624 | drm_mm_remove_node(&vma->node); |
625 | list_move_tail(&vma->vm_link, &vma->vm->unbound_list); | |
626 | ||
627 | /* Since the unbound list is global, only move to that list if | |
628 | * no more VMAs exist. | |
629 | */ | |
f2123818 | 630 | spin_lock(&i915->mm.obj_lock); |
31c7effa | 631 | if (--obj->bind_count == 0) |
f2123818 CW |
632 | list_move_tail(&obj->mm.link, &i915->mm.unbound_list); |
633 | spin_unlock(&i915->mm.obj_lock); | |
31c7effa CW |
634 | |
635 | /* And finally now the object is completely decoupled from this vma, | |
636 | * we can drop its hold on the backing storage and allow it to be | |
637 | * reaped by the shrinker. | |
638 | */ | |
639 | i915_gem_object_unpin_pages(obj); | |
640 | GEM_BUG_ON(atomic_read(&obj->mm.pages_pin_count) < obj->bind_count); | |
641 | } | |
642 | ||
b42fe9ca JL |
643 | int __i915_vma_do_pin(struct i915_vma *vma, |
644 | u64 size, u64 alignment, u64 flags) | |
645 | { | |
31c7effa | 646 | const unsigned int bound = vma->flags; |
b42fe9ca JL |
647 | int ret; |
648 | ||
49d73912 | 649 | lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); |
b42fe9ca JL |
650 | GEM_BUG_ON((flags & (PIN_GLOBAL | PIN_USER)) == 0); |
651 | GEM_BUG_ON((flags & PIN_GLOBAL) && !i915_vma_is_ggtt(vma)); | |
652 | ||
653 | if (WARN_ON(bound & I915_VMA_PIN_OVERFLOW)) { | |
654 | ret = -EBUSY; | |
31c7effa | 655 | goto err_unpin; |
b42fe9ca JL |
656 | } |
657 | ||
658 | if ((bound & I915_VMA_BIND_MASK) == 0) { | |
659 | ret = i915_vma_insert(vma, size, alignment, flags); | |
660 | if (ret) | |
31c7effa | 661 | goto err_unpin; |
b42fe9ca | 662 | } |
d36caeea | 663 | GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); |
b42fe9ca JL |
664 | |
665 | ret = i915_vma_bind(vma, vma->obj->cache_level, flags); | |
666 | if (ret) | |
31c7effa | 667 | goto err_remove; |
b42fe9ca | 668 | |
d36caeea CW |
669 | GEM_BUG_ON((vma->flags & I915_VMA_BIND_MASK) == 0); |
670 | ||
b42fe9ca JL |
671 | if ((bound ^ vma->flags) & I915_VMA_GLOBAL_BIND) |
672 | __i915_vma_set_map_and_fenceable(vma); | |
673 | ||
674 | GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags)); | |
675 | return 0; | |
676 | ||
31c7effa CW |
677 | err_remove: |
678 | if ((bound & I915_VMA_BIND_MASK) == 0) { | |
31c7effa | 679 | i915_vma_remove(vma); |
fa3f46af | 680 | GEM_BUG_ON(vma->pages); |
d36caeea | 681 | GEM_BUG_ON(vma->flags & I915_VMA_BIND_MASK); |
31c7effa CW |
682 | } |
683 | err_unpin: | |
b42fe9ca JL |
684 | __i915_vma_unpin(vma); |
685 | return ret; | |
686 | } | |
687 | ||
b8e5d2ef | 688 | static void i915_vma_destroy(struct i915_vma *vma) |
b42fe9ca | 689 | { |
7a3bc034 CW |
690 | int i; |
691 | ||
b42fe9ca JL |
692 | GEM_BUG_ON(vma->node.allocated); |
693 | GEM_BUG_ON(i915_vma_is_active(vma)); | |
694 | GEM_BUG_ON(!i915_vma_is_closed(vma)); | |
695 | GEM_BUG_ON(vma->fence); | |
696 | ||
7a3bc034 CW |
697 | for (i = 0; i < ARRAY_SIZE(vma->last_read); i++) |
698 | GEM_BUG_ON(i915_gem_active_isset(&vma->last_read[i])); | |
699 | GEM_BUG_ON(i915_gem_active_isset(&vma->last_fence)); | |
700 | ||
010e3e68 | 701 | list_del(&vma->obj_link); |
b42fe9ca | 702 | list_del(&vma->vm_link); |
010e3e68 | 703 | |
b42fe9ca JL |
704 | if (!i915_vma_is_ggtt(vma)) |
705 | i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm)); | |
706 | ||
707 | kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma); | |
708 | } | |
709 | ||
710 | void i915_vma_close(struct i915_vma *vma) | |
711 | { | |
712 | GEM_BUG_ON(i915_vma_is_closed(vma)); | |
713 | vma->flags |= I915_VMA_CLOSED; | |
714 | ||
b42fe9ca JL |
715 | rb_erase(&vma->obj_node, &vma->obj->vma_tree); |
716 | ||
717 | if (!i915_vma_is_active(vma) && !i915_vma_is_pinned(vma)) | |
718 | WARN_ON(i915_vma_unbind(vma)); | |
719 | } | |
720 | ||
721 | static void __i915_vma_iounmap(struct i915_vma *vma) | |
722 | { | |
723 | GEM_BUG_ON(i915_vma_is_pinned(vma)); | |
724 | ||
725 | if (vma->iomap == NULL) | |
726 | return; | |
727 | ||
728 | io_mapping_unmap(vma->iomap); | |
729 | vma->iomap = NULL; | |
730 | } | |
731 | ||
a65adaf8 CW |
732 | void i915_vma_revoke_mmap(struct i915_vma *vma) |
733 | { | |
734 | struct drm_vma_offset_node *node = &vma->obj->base.vma_node; | |
735 | u64 vma_offset; | |
736 | ||
737 | lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); | |
738 | ||
739 | if (!i915_vma_has_userfault(vma)) | |
740 | return; | |
741 | ||
742 | GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma)); | |
743 | GEM_BUG_ON(!vma->obj->userfault_count); | |
744 | ||
745 | vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT; | |
746 | unmap_mapping_range(vma->vm->i915->drm.anon_inode->i_mapping, | |
747 | drm_vma_node_offset_addr(node) + vma_offset, | |
748 | vma->size, | |
749 | 1); | |
750 | ||
751 | i915_vma_unset_userfault(vma); | |
752 | if (!--vma->obj->userfault_count) | |
753 | list_del(&vma->obj->userfault_link); | |
754 | } | |
755 | ||
b42fe9ca JL |
756 | int i915_vma_unbind(struct i915_vma *vma) |
757 | { | |
758 | struct drm_i915_gem_object *obj = vma->obj; | |
759 | unsigned long active; | |
760 | int ret; | |
761 | ||
762 | lockdep_assert_held(&obj->base.dev->struct_mutex); | |
763 | ||
764 | /* First wait upon any activity as retiring the request may | |
765 | * have side-effects such as unpinning or even unbinding this vma. | |
766 | */ | |
7f017b19 | 767 | might_sleep(); |
b42fe9ca JL |
768 | active = i915_vma_get_active(vma); |
769 | if (active) { | |
770 | int idx; | |
771 | ||
772 | /* When a closed VMA is retired, it is unbound - eek. | |
773 | * In order to prevent it from being recursively closed, | |
774 | * take a pin on the vma so that the second unbind is | |
775 | * aborted. | |
776 | * | |
777 | * Even more scary is that the retire callback may free | |
778 | * the object (last active vma). To prevent the explosion | |
779 | * we defer the actual object free to a worker that can | |
780 | * only proceed once it acquires the struct_mutex (which | |
781 | * we currently hold, therefore it cannot free this object | |
782 | * before we are finished). | |
783 | */ | |
784 | __i915_vma_pin(vma); | |
785 | ||
786 | for_each_active(active, idx) { | |
787 | ret = i915_gem_active_retire(&vma->last_read[idx], | |
49d73912 | 788 | &vma->vm->i915->drm.struct_mutex); |
b42fe9ca JL |
789 | if (ret) |
790 | break; | |
791 | } | |
792 | ||
760a898d CW |
793 | if (!ret) { |
794 | ret = i915_gem_active_retire(&vma->last_fence, | |
795 | &vma->vm->i915->drm.struct_mutex); | |
796 | } | |
797 | ||
b42fe9ca JL |
798 | __i915_vma_unpin(vma); |
799 | if (ret) | |
800 | return ret; | |
b42fe9ca | 801 | } |
7a3bc034 | 802 | GEM_BUG_ON(i915_vma_is_active(vma)); |
b42fe9ca JL |
803 | |
804 | if (i915_vma_is_pinned(vma)) | |
805 | return -EBUSY; | |
806 | ||
807 | if (!drm_mm_node_allocated(&vma->node)) | |
808 | goto destroy; | |
809 | ||
810 | GEM_BUG_ON(obj->bind_count == 0); | |
811 | GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); | |
812 | ||
813 | if (i915_vma_is_map_and_fenceable(vma)) { | |
7125397b CW |
814 | /* |
815 | * Check that we have flushed all writes through the GGTT | |
816 | * before the unbind, other due to non-strict nature of those | |
817 | * indirect writes they may end up referencing the GGTT PTE | |
818 | * after the unbind. | |
819 | */ | |
820 | i915_vma_flush_writes(vma); | |
821 | GEM_BUG_ON(i915_vma_has_ggtt_write(vma)); | |
822 | ||
b42fe9ca JL |
823 | /* release the fence reg _after_ flushing */ |
824 | ret = i915_vma_put_fence(vma); | |
825 | if (ret) | |
826 | return ret; | |
827 | ||
828 | /* Force a pagefault for domain tracking on next user access */ | |
a65adaf8 | 829 | i915_vma_revoke_mmap(vma); |
b42fe9ca JL |
830 | |
831 | __i915_vma_iounmap(vma); | |
832 | vma->flags &= ~I915_VMA_CAN_FENCE; | |
833 | } | |
a65adaf8 CW |
834 | GEM_BUG_ON(vma->fence); |
835 | GEM_BUG_ON(i915_vma_has_userfault(vma)); | |
b42fe9ca JL |
836 | |
837 | if (likely(!vma->vm->closed)) { | |
838 | trace_i915_vma_unbind(vma); | |
839 | vma->vm->unbind_vma(vma); | |
840 | } | |
841 | vma->flags &= ~(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND); | |
842 | ||
31c7effa | 843 | i915_vma_remove(vma); |
b42fe9ca JL |
844 | |
845 | destroy: | |
846 | if (unlikely(i915_vma_is_closed(vma))) | |
847 | i915_vma_destroy(vma); | |
848 | ||
849 | return 0; | |
850 | } | |
851 | ||
e3c7a1c5 CW |
852 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) |
853 | #include "selftests/i915_vma.c" | |
854 | #endif |