[linux-2.6-block.git] / drivers / gpu / drm / i915 / i915_gem_evict.c

/*
 * Copyright © 2008-2010 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Eric Anholt <eric@anholt.net>
 *    Chris Wilson <chris@chris-wilson.co.uuk>
 *
 */

#include <drm/drmP.h>
#include <drm/i915_drm.h>

#include "i915_drv.h"
#include "intel_drv.h"
#include "i915_trace.h"

static int switch_to_pinned_context(struct drm_i915_private *dev_priv)
{
	struct intel_engine_cs *engine;

	if (i915.enable_execlists)
		return 0;

	for_each_engine(engine, dev_priv) {
		struct drm_i915_gem_request *req;
		int ret;

		if (engine->last_context == NULL)
			continue;

		if (engine->last_context == dev_priv->kernel_context)
			continue;

		req = i915_gem_request_alloc(engine, dev_priv->kernel_context);
		if (IS_ERR(req))
			return PTR_ERR(req);

		ret = i915_switch_context(req);
		i915_add_request_no_flush(req);
		if (ret)
			return ret;
	}

	return 0;
}


static bool
mark_free(struct i915_vma *vma, struct list_head *unwind)
{
	if (vma->pin_count)
		return false;

	if (WARN_ON(!list_empty(&vma->exec_list)))
		return false;

	list_add(&vma->exec_list, unwind);
	return drm_mm_scan_add_block(&vma->node);
}

/**
 * i915_gem_evict_something - Evict vmas to make room for binding a new one
 * @dev: drm_device
 * @vm: address space to evict from
 * @min_size: size of the desired free space
 * @alignment: alignment constraint of the desired free space
 * @cache_level: cache_level for the desired space
 * @start: start (inclusive) of the range from which to evict objects
 * @end: end (exclusive) of the range from which to evict objects
 * @flags: additional flags to control the eviction algorithm
 *
 * This function will try to evict vmas until a free space satisfying the
 * requirements is found. Callers must check first whether any such hole exists
 * already before calling this function.
 *
 * This function is used by the object/vma binding code.
 *
 * Since this function is only used to free up virtual address space it only
 * ignores pinned vmas, and not object where the backing storage itself is
 * pinned. Hence obj->pages_pin_count does not protect against eviction.
 *
 * To clarify: This is for freeing up virtual address space, not for freeing
 * memory in e.g. the shrinker.
 */
int
i915_gem_evict_something(struct drm_device *dev, struct i915_address_space *vm,
			 int min_size, unsigned alignment, unsigned cache_level,
			 unsigned long start, unsigned long end,
			 unsigned flags)
{
	struct list_head eviction_list, unwind_list;
	struct i915_vma *vma;
	int ret = 0;
	int pass = 0;

	trace_i915_gem_evict(dev, min_size, alignment, flags);

	/*
	 * The goal is to evict objects and amalgamate space in LRU order.
	 * The oldest idle objects reside on the inactive list, which is in
	 * retirement order. The next objects to retire are those on the (per
	 * ring) active list that do not have an outstanding flush. Once the
	 * hardware reports completion (the seqno is updated after the
	 * batchbuffer has been finished) the clean buffer objects would
	 * be retired to the inactive list. Any dirty objects would be added
	 * to the tail of the flushing list. So after processing the clean
	 * active objects we need to emit a MI_FLUSH to retire the flushing
	 * list, hence the retirement order of the flushing list is in
	 * advance of the dirty objects on the active lists.
	 *
	 * The retirement sequence is thus:
	 *   1. Inactive objects (already retired)
	 *   2. Clean active objects
	 *   3. Flushing list
	 *   4. Dirty active objects.
	 *
	 * On each list, the oldest objects lie at the HEAD with the freshest
	 * object on the TAIL.
	 */

	INIT_LIST_HEAD(&unwind_list);
	if (start != 0 || end != vm->total) {
		drm_mm_init_scan_with_range(&vm->mm, min_size,
					    alignment, cache_level,
					    start, end);
	} else
		drm_mm_init_scan(&vm->mm, min_size, alignment, cache_level);

search_again:
	/* First see if there is a large enough contiguous idle region... */
	list_for_each_entry(vma, &vm->inactive_list, vm_link) {
		if (mark_free(vma, &unwind_list))
			goto found;
	}

	if (flags & PIN_NONBLOCK)
		goto none;

	/* Now merge in the soon-to-be-expired objects... */
	list_for_each_entry(vma, &vm->active_list, vm_link) {
		if (mark_free(vma, &unwind_list))
			goto found;
	}

none:
	/* Nothing found, clean up and bail out! */
	while (!list_empty(&unwind_list)) {
		vma = list_first_entry(&unwind_list,
				       struct i915_vma,
				       exec_list);
		ret = drm_mm_scan_remove_block(&vma->node);
		BUG_ON(ret);

		list_del_init(&vma->exec_list);
	}

	/* Can we unpin some objects such as idle hw contents,
	 * or pending flips?
	 */
	if (flags & PIN_NONBLOCK)
		return -ENOSPC;

	/* Only idle the GPU and repeat the search once */
	if (pass++ == 0) {
		struct drm_i915_private *dev_priv = to_i915(dev);

		if (i915_is_ggtt(vm)) {
			ret = switch_to_pinned_context(dev_priv);
			if (ret)
				return ret;
		}

		ret = i915_gem_wait_for_idle(dev_priv);
		if (ret)
			return ret;

		i915_gem_retire_requests(dev_priv);
		goto search_again;
	}

	/* If we still have pending pageflip completions, drop
	 * back to userspace to give our workqueues time to
	 * acquire our locks and unpin the old scanouts.
	 */
	return intel_has_pending_fb_unpin(dev) ? -EAGAIN : -ENOSPC;

found:
	/* drm_mm doesn't allow any other other operations while
	 * scanning, therefore store to be evicted objects on a
	 * temporary list. */
	INIT_LIST_HEAD(&eviction_list);
	while (!list_empty(&unwind_list)) {
		vma = list_first_entry(&unwind_list,
				       struct i915_vma,
				       exec_list);
		if (drm_mm_scan_remove_block(&vma->node)) {
			list_move(&vma->exec_list, &eviction_list);
			drm_gem_object_reference(&vma->obj->base);
			continue;
		}
		list_del_init(&vma->exec_list);
	}

	/* Unbinding will emit any required flushes */
	while (!list_empty(&eviction_list)) {
		struct drm_gem_object *obj;
		vma = list_first_entry(&eviction_list,
				       struct i915_vma,
				       exec_list);

		obj =  &vma->obj->base;
		list_del_init(&vma->exec_list);
		if (ret == 0)
			ret = i915_vma_unbind(vma);

		drm_gem_object_unreference(obj);
	}

	return ret;
}

int
i915_gem_evict_for_vma(struct i915_vma *target)
{
	struct drm_mm_node *node, *next;

	list_for_each_entry_safe(node, next,
			&target->vm->mm.head_node.node_list,
			node_list) {
		struct i915_vma *vma;
		int ret;

		if (node->start + node->size <= target->node.start)
			continue;
		if (node->start >= target->node.start + target->node.size)
			break;

		vma = container_of(node, typeof(*vma), node);

		if (vma->pin_count) {
			if (!vma->exec_entry || (vma->pin_count > 1))
				/* Object is pinned for some other use */
				return -EBUSY;

			/* We need to evict a buffer in the same batch */
			if (vma->exec_entry->flags & EXEC_OBJECT_PINNED)
				/* Overlapping fixed objects in the same batch */
				return -EINVAL;

			return -ENOSPC;
		}

		ret = i915_vma_unbind(vma);
		if (ret)
			return ret;
	}

	return 0;
}

/**
 * i915_gem_evict_vm - Evict all idle vmas from a vm
 * @vm: Address space to cleanse
 * @do_idle: Boolean directing whether to idle first.
 *
 * This function evicts all idles vmas from a vm. If all unpinned vmas should be
 * evicted the @do_idle needs to be set to true.
 *
 * This is used by the execbuf code as a last-ditch effort to defragment the
 * address space.
 *
 * To clarify: This is for freeing up virtual address space, not for freeing
 * memory in e.g. the shrinker.
 */
int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle)
{
	struct i915_vma *vma, *next;
	int ret;

	WARN_ON(!mutex_is_locked(&vm->dev->struct_mutex));
	trace_i915_gem_evict_vm(vm);

	if (do_idle) {
		struct drm_i915_private *dev_priv = to_i915(vm->dev);

		if (i915_is_ggtt(vm)) {
			ret = switch_to_pinned_context(dev_priv);
			if (ret)
				return ret;
		}

		ret = i915_gem_wait_for_idle(dev_priv);
		if (ret)
			return ret;

		i915_gem_retire_requests(dev_priv);

		WARN_ON(!list_empty(&vm->active_list));
	}

	list_for_each_entry_safe(vma, next, &vm->inactive_list, vm_link)
		if (vma->pin_count == 0)
			WARN_ON(i915_vma_unbind(vma));

	return 0;
}
Commit	Line	Data
b47eb4a2 CW	1	/*
	2	* Copyright © 2008-2010 Intel Corporation
	3	*
	4	* Permission is hereby granted, free of charge, to any person obtaining a
	5	* copy of this software and associated documentation files (the "Software"),
	6	* to deal in the Software without restriction, including without limitation
	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	* and/or sell copies of the Software, and to permit persons to whom the
	9	* Software is furnished to do so, subject to the following conditions:
	10	*
	11	* The above copyright notice and this permission notice (including the next
	12	* paragraph) shall be included in all copies or substantial portions of the
	13	* Software.
	14	*
	15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
	20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
	21	* IN THE SOFTWARE.
	22	*
	23	* Authors:
	24	* Eric Anholt <eric@anholt.net>
	25	* Chris Wilson <chris@chris-wilson.co.uuk>
	26	*
	27	*/
	28
760285e7	29	#include <drm/drmP.h>
760285e7	30	#include <drm/i915_drm.h>
74e21ac2 CW	31
	32	#include "i915_drv.h"
	33	#include "intel_drv.h"
db53a302	34	#include "i915_trace.h"
b47eb4a2	35
6e5a5beb CW	36	static int switch_to_pinned_context(struct drm_i915_private *dev_priv)
	37	{
	38	struct intel_engine_cs *engine;
	39
	40	if (i915.enable_execlists)
	41	return 0;
	42
	43	for_each_engine(engine, dev_priv) {
	44	struct drm_i915_gem_request *req;
	45	int ret;
	46
	47	if (engine->last_context == NULL)
	48	continue;
	49
	50	if (engine->last_context == dev_priv->kernel_context)
	51	continue;
	52
	53	req = i915_gem_request_alloc(engine, dev_priv->kernel_context);
	54	if (IS_ERR(req))
	55	return PTR_ERR(req);
	56
	57	ret = i915_switch_context(req);
	58	i915_add_request_no_flush(req);
	59	if (ret)
	60	return ret;
	61	}
	62
	63	return 0;
	64	}
	65
	66
cd377ea9	67	static bool
f6cd1f15	68	mark_free(struct i915_vma vma, struct list_head unwind)
b47eb4a2	69	{
3036537d	70	if (vma->pin_count)
1b50247a CW	71	return false;
1b50247a CW	72
b93dab6e DV	73	if (WARN_ON(!list_empty(&vma->exec_list)))
	74	return false;
	75
82a55ad1	76	list_add(&vma->exec_list, unwind);
2f633156	77	return drm_mm_scan_add_block(&vma->node);
b47eb4a2 CW	78	}
b47eb4a2 CW	79
c2c1d491 DV	80	/**
	81	* i915_gem_evict_something - Evict vmas to make room for binding a new one
	82	* @dev: drm_device
	83	* @vm: address space to evict from
7838a63a	84	* @min_size: size of the desired free space
c2c1d491 DV	85	* @alignment: alignment constraint of the desired free space
c2c1d491 DV	86	* @cache_level: cache_level for the desired space
7838a63a DV	87	* @start: start (inclusive) of the range from which to evict objects
	88	* @end: end (exclusive) of the range from which to evict objects
	89	* @flags: additional flags to control the eviction algorithm
c2c1d491 DV	90	*
	91	* This function will try to evict vmas until a free space satisfying the
	92	* requirements is found. Callers must check first whether any such hole exists
	93	* already before calling this function.
	94	*
	95	* This function is used by the object/vma binding code.
	96	*
eb0b44ad DV	97	* Since this function is only used to free up virtual address space it only
	98	* ignores pinned vmas, and not object where the backing storage itself is
	99	* pinned. Hence obj->pages_pin_count does not protect against eviction.
	100	*
c2c1d491 DV	101	* To clarify: This is for freeing up virtual address space, not for freeing
	102	* memory in e.g. the shrinker.
	103	*/
b47eb4a2	104	int
f6cd1f15 BW	105	i915_gem_evict_something(struct drm_device dev, struct i915_address_space vm,
f6cd1f15 BW	106	int min_size, unsigned alignment, unsigned cache_level,
d23db88c	107	unsigned long start, unsigned long end,
1ec9e26d	108	unsigned flags)
b47eb4a2	109	{
cd377ea9	110	struct list_head eviction_list, unwind_list;
2f633156	111	struct i915_vma *vma;
cd377ea9	112	int ret = 0;
74e21ac2	113	int pass = 0;
b47eb4a2	114
1ec9e26d	115	trace_i915_gem_evict(dev, min_size, alignment, flags);
db53a302	116
cd377ea9 CW	117	/*
	118	* The goal is to evict objects and amalgamate space in LRU order.
	119	* The oldest idle objects reside on the inactive list, which is in
	120	* retirement order. The next objects to retire are those on the (per
	121	* ring) active list that do not have an outstanding flush. Once the
	122	* hardware reports completion (the seqno is updated after the
	123	* batchbuffer has been finished) the clean buffer objects would
	124	* be retired to the inactive list. Any dirty objects would be added
	125	* to the tail of the flushing list. So after processing the clean
	126	* active objects we need to emit a MI_FLUSH to retire the flushing
	127	* list, hence the retirement order of the flushing list is in
	128	* advance of the dirty objects on the active lists.
	129	*
	130	* The retirement sequence is thus:
	131	* 1. Inactive objects (already retired)
	132	* 2. Clean active objects
	133	* 3. Flushing list
	134	* 4. Dirty active objects.
	135	*
	136	* On each list, the oldest objects lie at the HEAD with the freshest
	137	* object on the TAIL.
	138	*/
	139
	140	INIT_LIST_HEAD(&unwind_list);
d23db88c	141	if (start != 0 \|\| end != vm->total) {
5cef07e1	142	drm_mm_init_scan_with_range(&vm->mm, min_size,
d23db88c CW	143	alignment, cache_level,
d23db88c CW	144	start, end);
f6cd1f15	145	} else
5cef07e1	146	drm_mm_init_scan(&vm->mm, min_size, alignment, cache_level);
cd377ea9	147
ad071acb	148	search_again:
cd377ea9	149	/* First see if there is a large enough contiguous idle region... */
1c7f4bca	150	list_for_each_entry(vma, &vm->inactive_list, vm_link) {
f6cd1f15	151	if (mark_free(vma, &unwind_list))
cd377ea9 CW	152	goto found;
cd377ea9 CW	153	}
b47eb4a2	154
1ec9e26d	155	if (flags & PIN_NONBLOCK)
86a1ee26	156	goto none;
b47eb4a2	157
cd377ea9	158	/* Now merge in the soon-to-be-expired objects... */
1c7f4bca	159	list_for_each_entry(vma, &vm->active_list, vm_link) {
f6cd1f15	160	if (mark_free(vma, &unwind_list))
cd377ea9 CW	161	goto found;
	162	}
	163
86a1ee26	164	none:
cd377ea9	165	/* Nothing found, clean up and bail out! */
092de6f2	166	while (!list_empty(&unwind_list)) {
82a55ad1 BW	167	vma = list_first_entry(&unwind_list,
82a55ad1 BW	168	struct i915_vma,
092de6f2	169	exec_list);
2f633156	170	ret = drm_mm_scan_remove_block(&vma->node);
cd377ea9	171	BUG_ON(ret);
092de6f2	172
82a55ad1	173	list_del_init(&vma->exec_list);
cd377ea9 CW	174	}
cd377ea9 CW	175
ad071acb CW	176	/* Can we unpin some objects such as idle hw contents,
ad071acb CW	177	* or pending flips?
cd377ea9	178	*/
1ec9e26d	179	if (flags & PIN_NONBLOCK)
74e21ac2	180	return -ENOSPC;
ad071acb CW	181
ad071acb CW	182	/* Only idle the GPU and repeat the search once */
74e21ac2	183	if (pass++ == 0) {
6e5a5beb CW	184	struct drm_i915_private *dev_priv = to_i915(dev);
6e5a5beb CW	185
883445d4 CW	186	if (i915_is_ggtt(vm)) {
	187	ret = switch_to_pinned_context(dev_priv);
	188	if (ret)
	189	return ret;
	190	}
74e21ac2	191
6e5a5beb CW	192	ret = i915_gem_wait_for_idle(dev_priv);
	193	if (ret)
	194	return ret;
	195
	196	i915_gem_retire_requests(dev_priv);
74e21ac2 CW	197	goto search_again;
	198	}
	199
	200	/* If we still have pending pageflip completions, drop
	201	* back to userspace to give our workqueues time to
	202	* acquire our locks and unpin the old scanouts.
	203	*/
	204	return intel_has_pending_fb_unpin(dev) ? -EAGAIN : -ENOSPC;
cd377ea9 CW	205
cd377ea9 CW	206	found:
e39a0150 CW	207	/* drm_mm doesn't allow any other other operations while
	208	* scanning, therefore store to be evicted objects on a
	209	* temporary list. */
cd377ea9	210	INIT_LIST_HEAD(&eviction_list);
e39a0150	211	while (!list_empty(&unwind_list)) {
82a55ad1 BW	212	vma = list_first_entry(&unwind_list,
82a55ad1 BW	213	struct i915_vma,
432e58ed	214	exec_list);
2f633156	215	if (drm_mm_scan_remove_block(&vma->node)) {
82a55ad1 BW	216	list_move(&vma->exec_list, &eviction_list);
82a55ad1 BW	217	drm_gem_object_reference(&vma->obj->base);
e39a0150 CW	218	continue;
e39a0150 CW	219	}
82a55ad1	220	list_del_init(&vma->exec_list);
cd377ea9	221	}
b47eb4a2	222
cd377ea9	223	/* Unbinding will emit any required flushes */
e39a0150	224	while (!list_empty(&eviction_list)) {
8637b407	225	struct drm_gem_object *obj;
82a55ad1 BW	226	vma = list_first_entry(&eviction_list,
82a55ad1 BW	227	struct i915_vma,
432e58ed	228	exec_list);
8637b407 BW	229
	230	obj = &vma->obj->base;
	231	list_del_init(&vma->exec_list);
e39a0150	232	if (ret == 0)
82a55ad1	233	ret = i915_vma_unbind(vma);
092de6f2	234
8637b407	235	drm_gem_object_unreference(obj);
b47eb4a2	236	}
cd377ea9	237
e39a0150	238	return ret;
b47eb4a2 CW	239	}
b47eb4a2 CW	240
506a8e87 CW	241	int
	242	i915_gem_evict_for_vma(struct i915_vma *target)
	243	{
	244	struct drm_mm_node node, next;
	245
	246	list_for_each_entry_safe(node, next,
	247	&target->vm->mm.head_node.node_list,
	248	node_list) {
	249	struct i915_vma *vma;
	250	int ret;
	251
	252	if (node->start + node->size <= target->node.start)
	253	continue;
	254	if (node->start >= target->node.start + target->node.size)
	255	break;
	256
	257	vma = container_of(node, typeof(*vma), node);
	258
	259	if (vma->pin_count) {
	260	if (!vma->exec_entry \|\| (vma->pin_count > 1))
	261	/* Object is pinned for some other use */
	262	return -EBUSY;
	263
	264	/* We need to evict a buffer in the same batch */
	265	if (vma->exec_entry->flags & EXEC_OBJECT_PINNED)
	266	/* Overlapping fixed objects in the same batch */
	267	return -EINVAL;
	268
	269	return -ENOSPC;
	270	}
	271
	272	ret = i915_vma_unbind(vma);
	273	if (ret)
	274	return ret;
	275	}
	276
	277	return 0;
	278	}
	279
68c8c17f	280	/**
c2c1d491	281	* i915_gem_evict_vm - Evict all idle vmas from a vm
c2c1d491	282	* @vm: Address space to cleanse
68c8c17f BW	283	* @do_idle: Boolean directing whether to idle first.
68c8c17f BW	284	*
c2c1d491 DV	285	* This function evicts all idles vmas from a vm. If all unpinned vmas should be
c2c1d491 DV	286	* evicted the @do_idle needs to be set to true.
68c8c17f	287	*
c2c1d491 DV	288	* This is used by the execbuf code as a last-ditch effort to defragment the
	289	* address space.
	290	*
	291	* To clarify: This is for freeing up virtual address space, not for freeing
	292	* memory in e.g. the shrinker.
68c8c17f BW	293	*/
68c8c17f BW	294	int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle)
7b796122 BW	295	{
	296	struct i915_vma vma, next;
	297	int ret;
	298
b9b5dce5	299	WARN_ON(!mutex_is_locked(&vm->dev->struct_mutex));
bcccff84 BW	300	trace_i915_gem_evict_vm(vm);
bcccff84 BW	301
7b796122	302	if (do_idle) {
6e5a5beb CW	303	struct drm_i915_private *dev_priv = to_i915(vm->dev);
6e5a5beb CW	304
883445d4 CW	305	if (i915_is_ggtt(vm)) {
	306	ret = switch_to_pinned_context(dev_priv);
	307	if (ret)
	308	return ret;
	309	}
6e5a5beb CW	310
6e5a5beb CW	311	ret = i915_gem_wait_for_idle(dev_priv);
7b796122 BW	312	if (ret)
	313	return ret;
	314
6e5a5beb	315	i915_gem_retire_requests(dev_priv);
b9b5dce5 BW	316
b9b5dce5 BW	317	WARN_ON(!list_empty(&vm->active_list));
7b796122 BW	318	}
7b796122 BW	319
1c7f4bca	320	list_for_each_entry_safe(vma, next, &vm->inactive_list, vm_link)
d7f46fc4	321	if (vma->pin_count == 0)
7b796122 BW	322	WARN_ON(i915_vma_unbind(vma));
	323
	324	return 0;
	325	}