[linux-2.6-block.git] / drivers / gpu / drm / i915 / i915_active.c

/*
 * SPDX-License-Identifier: MIT
 *
 * Copyright © 2019 Intel Corporation
 */

#include "i915_drv.h"
#include "i915_active.h"

#define BKL(ref) (&(ref)->i915->drm.struct_mutex)

/*
 * Active refs memory management
 *
 * To be more economical with memory, we reap all the i915_active trees as
 * they idle (when we know the active requests are inactive) and allocate the
 * nodes from a local slab cache to hopefully reduce the fragmentation.
 */
static struct i915_global_active {
	struct kmem_cache *slab_cache;
} global;

struct active_node {
	struct i915_gem_active base;
	struct i915_active *ref;
	struct rb_node node;
	u64 timeline;
};

static void
__active_park(struct i915_active *ref)
{
	struct active_node *it, *n;

	rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) {
		GEM_BUG_ON(i915_gem_active_isset(&it->base));
		kmem_cache_free(global.slab_cache, it);
	}
	ref->tree = RB_ROOT;
}

static void
__active_retire(struct i915_active *ref)
{
	GEM_BUG_ON(!ref->count);
	if (--ref->count)
		return;

	/* return the unused nodes to our slabcache */
	__active_park(ref);

	ref->retire(ref);
}

static void
node_retire(struct i915_gem_active *base, struct i915_request *rq)
{
	__active_retire(container_of(base, struct active_node, base)->ref);
}

static void
last_retire(struct i915_gem_active *base, struct i915_request *rq)
{
	__active_retire(container_of(base, struct i915_active, last));
}

static struct i915_gem_active *
active_instance(struct i915_active *ref, u64 idx)
{
	struct active_node *node;
	struct rb_node **p, *parent;
	struct i915_request *old;

	/*
	 * We track the most recently used timeline to skip a rbtree search
	 * for the common case, under typical loads we never need the rbtree
	 * at all. We can reuse the last slot if it is empty, that is
	 * after the previous activity has been retired, or if it matches the
	 * current timeline.
	 *
	 * Note that we allow the timeline to be active simultaneously in
	 * the rbtree and the last cache. We do this to avoid having
	 * to search and replace the rbtree element for a new timeline, with
	 * the cost being that we must be aware that the ref may be retired
	 * twice for the same timeline (as the older rbtree element will be
	 * retired before the new request added to last).
	 */
	old = i915_gem_active_raw(&ref->last, BKL(ref));
	if (!old || old->fence.context == idx)
		goto out;

	/* Move the currently active fence into the rbtree */
	idx = old->fence.context;

	parent = NULL;
	p = &ref->tree.rb_node;
	while (*p) {
		parent = *p;

		node = rb_entry(parent, struct active_node, node);
		if (node->timeline == idx)
			goto replace;

		if (node->timeline < idx)
			p = &parent->rb_right;
		else
			p = &parent->rb_left;
	}

	node = kmem_cache_alloc(global.slab_cache, GFP_KERNEL);

	/* kmalloc may retire the ref->last (thanks shrinker)! */
	if (unlikely(!i915_gem_active_raw(&ref->last, BKL(ref)))) {
		kmem_cache_free(global.slab_cache, node);
		goto out;
	}

	if (unlikely(!node))
		return ERR_PTR(-ENOMEM);

	init_request_active(&node->base, node_retire);
	node->ref = ref;
	node->timeline = idx;

	rb_link_node(&node->node, parent, p);
	rb_insert_color(&node->node, &ref->tree);

replace:
	/*
	 * Overwrite the previous active slot in the rbtree with last,
	 * leaving last zeroed. If the previous slot is still active,
	 * we must be careful as we now only expect to receive one retire
	 * callback not two, and so much undo the active counting for the
	 * overwritten slot.
	 */
	if (i915_gem_active_isset(&node->base)) {
		/* Retire ourselves from the old rq->active_list */
		__list_del_entry(&node->base.link);
		ref->count--;
		GEM_BUG_ON(!ref->count);
	}
	GEM_BUG_ON(list_empty(&ref->last.link));
	list_replace_init(&ref->last.link, &node->base.link);
	node->base.request = fetch_and_zero(&ref->last.request);

out:
	return &ref->last;
}

void i915_active_init(struct drm_i915_private *i915,
		      struct i915_active *ref,
		      void (*retire)(struct i915_active *ref))
{
	ref->i915 = i915;
	ref->retire = retire;
	ref->tree = RB_ROOT;
	init_request_active(&ref->last, last_retire);
	ref->count = 0;
}

int i915_active_ref(struct i915_active *ref,
		    u64 timeline,
		    struct i915_request *rq)
{
	struct i915_gem_active *active;

	active = active_instance(ref, timeline);
	if (IS_ERR(active))
		return PTR_ERR(active);

	if (!i915_gem_active_isset(active))
		ref->count++;
	i915_gem_active_set(active, rq);

	GEM_BUG_ON(!ref->count);
	return 0;
}

bool i915_active_acquire(struct i915_active *ref)
{
	lockdep_assert_held(BKL(ref));
	return !ref->count++;
}

void i915_active_release(struct i915_active *ref)
{
	lockdep_assert_held(BKL(ref));
	__active_retire(ref);
}

int i915_active_wait(struct i915_active *ref)
{
	struct active_node *it, *n;
	int ret = 0;

	if (i915_active_acquire(ref))
		goto out_release;

	ret = i915_gem_active_retire(&ref->last, BKL(ref));
	if (ret)
		goto out_release;

	rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) {
		ret = i915_gem_active_retire(&it->base, BKL(ref));
		if (ret)
			break;
	}

out_release:
	i915_active_release(ref);
	return ret;
}

static int __i915_request_await_active(struct i915_request *rq,
				       struct i915_gem_active *active)
{
	struct i915_request *barrier =
		i915_gem_active_raw(active, &rq->i915->drm.struct_mutex);

	return barrier ? i915_request_await_dma_fence(rq, &barrier->fence) : 0;
}

int i915_request_await_active(struct i915_request *rq, struct i915_active *ref)
{
	struct active_node *it, *n;
	int ret;

	ret = __i915_request_await_active(rq, &ref->last);
	if (ret)
		return ret;

	rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) {
		ret = __i915_request_await_active(rq, &it->base);
		if (ret)
			return ret;
	}

	return 0;
}

#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
void i915_active_fini(struct i915_active *ref)
{
	GEM_BUG_ON(i915_gem_active_isset(&ref->last));
	GEM_BUG_ON(!RB_EMPTY_ROOT(&ref->tree));
	GEM_BUG_ON(ref->count);
}
#endif

#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
#include "selftests/i915_active.c"
#endif

int __init i915_global_active_init(void)
{
	global.slab_cache = KMEM_CACHE(active_node, SLAB_HWCACHE_ALIGN);
	if (!global.slab_cache)
		return -ENOMEM;

	return 0;
}

void __exit i915_global_active_exit(void)
{
	kmem_cache_destroy(global.slab_cache);
}
Commit	Line	Data
64d6c500 CW	1	/*
	2	* SPDX-License-Identifier: MIT
	3	*
	4	* Copyright © 2019 Intel Corporation
	5	*/
	6
	7	#include "i915_drv.h"
	8	#include "i915_active.h"
	9
	10	#define BKL(ref) (&(ref)->i915->drm.struct_mutex)
	11
5f5c139d CW	12	/*
	13	* Active refs memory management
	14	*
	15	* To be more economical with memory, we reap all the i915_active trees as
	16	* they idle (when we know the active requests are inactive) and allocate the
	17	* nodes from a local slab cache to hopefully reduce the fragmentation.
	18	*/
	19	static struct i915_global_active {
	20	struct kmem_cache *slab_cache;
	21	} global;
	22
64d6c500 CW	23	struct active_node {
	24	struct i915_gem_active base;
	25	struct i915_active *ref;
	26	struct rb_node node;
	27	u64 timeline;
	28	};
	29
a42375af CW	30	static void
	31	__active_park(struct i915_active *ref)
	32	{
	33	struct active_node it, n;
	34
	35	rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) {
	36	GEM_BUG_ON(i915_gem_active_isset(&it->base));
5f5c139d	37	kmem_cache_free(global.slab_cache, it);
a42375af CW	38	}
	39	ref->tree = RB_ROOT;
	40	}
	41
64d6c500 CW	42	static void
	43	__active_retire(struct i915_active *ref)
	44	{
	45	GEM_BUG_ON(!ref->count);
a42375af CW	46	if (--ref->count)
	47	return;
	48
	49	/* return the unused nodes to our slabcache */
	50	__active_park(ref);
	51
	52	ref->retire(ref);
64d6c500 CW	53	}
	54
	55	static void
	56	node_retire(struct i915_gem_active base, struct i915_request rq)
	57	{
	58	__active_retire(container_of(base, struct active_node, base)->ref);
	59	}
	60
	61	static void
	62	last_retire(struct i915_gem_active base, struct i915_request rq)
	63	{
	64	__active_retire(container_of(base, struct i915_active, last));
	65	}
	66
	67	static struct i915_gem_active *
	68	active_instance(struct i915_active *ref, u64 idx)
	69	{
	70	struct active_node *node;
	71	struct rb_node *p, parent;
	72	struct i915_request *old;
	73
	74	/*
	75	* We track the most recently used timeline to skip a rbtree search
	76	* for the common case, under typical loads we never need the rbtree
	77	* at all. We can reuse the last slot if it is empty, that is
	78	* after the previous activity has been retired, or if it matches the
	79	* current timeline.
	80	*
	81	* Note that we allow the timeline to be active simultaneously in
	82	* the rbtree and the last cache. We do this to avoid having
	83	* to search and replace the rbtree element for a new timeline, with
	84	* the cost being that we must be aware that the ref may be retired
	85	* twice for the same timeline (as the older rbtree element will be
	86	* retired before the new request added to last).
	87	*/
	88	old = i915_gem_active_raw(&ref->last, BKL(ref));
	89	if (!old \|\| old->fence.context == idx)
	90	goto out;
	91
	92	/* Move the currently active fence into the rbtree */
	93	idx = old->fence.context;
	94
	95	parent = NULL;
	96	p = &ref->tree.rb_node;
	97	while (*p) {
	98	parent = *p;
	99
	100	node = rb_entry(parent, struct active_node, node);
	101	if (node->timeline == idx)
	102	goto replace;
	103
	104	if (node->timeline < idx)
	105	p = &parent->rb_right;
	106	else
	107	p = &parent->rb_left;
	108	}
	109
5f5c139d	110	node = kmem_cache_alloc(global.slab_cache, GFP_KERNEL);
64d6c500 CW	111
	112	/* kmalloc may retire the ref->last (thanks shrinker)! */
	113	if (unlikely(!i915_gem_active_raw(&ref->last, BKL(ref)))) {
5f5c139d	114	kmem_cache_free(global.slab_cache, node);
64d6c500 CW	115	goto out;
	116	}
	117
	118	if (unlikely(!node))
	119	return ERR_PTR(-ENOMEM);
	120
	121	init_request_active(&node->base, node_retire);
	122	node->ref = ref;
	123	node->timeline = idx;
	124
	125	rb_link_node(&node->node, parent, p);
	126	rb_insert_color(&node->node, &ref->tree);
	127
	128	replace:
	129	/*
	130	* Overwrite the previous active slot in the rbtree with last,
	131	* leaving last zeroed. If the previous slot is still active,
	132	* we must be careful as we now only expect to receive one retire
	133	* callback not two, and so much undo the active counting for the
	134	* overwritten slot.
	135	*/
	136	if (i915_gem_active_isset(&node->base)) {
	137	/* Retire ourselves from the old rq->active_list */
	138	__list_del_entry(&node->base.link);
	139	ref->count--;
	140	GEM_BUG_ON(!ref->count);
	141	}
	142	GEM_BUG_ON(list_empty(&ref->last.link));
	143	list_replace_init(&ref->last.link, &node->base.link);
	144	node->base.request = fetch_and_zero(&ref->last.request);
	145
	146	out:
	147	return &ref->last;
	148	}
	149
	150	void i915_active_init(struct drm_i915_private *i915,
	151	struct i915_active *ref,
	152	void (retire)(struct i915_active ref))
	153	{
	154	ref->i915 = i915;
	155	ref->retire = retire;
	156	ref->tree = RB_ROOT;
	157	init_request_active(&ref->last, last_retire);
	158	ref->count = 0;
	159	}
	160
	161	int i915_active_ref(struct i915_active *ref,
	162	u64 timeline,
	163	struct i915_request *rq)
	164	{
	165	struct i915_gem_active *active;
	166
	167	active = active_instance(ref, timeline);
	168	if (IS_ERR(active))
	169	return PTR_ERR(active);
	170
	171	if (!i915_gem_active_isset(active))
	172	ref->count++;
	173	i915_gem_active_set(active, rq);
	174
	175	GEM_BUG_ON(!ref->count);
	176	return 0;
	177	}
	178
179	bool i915_active_acquire(struct i915_active *ref)
180	{
181	lockdep_assert_held(BKL(ref));
182	return !ref->count++;
183	}
184
185	void i915_active_release(struct i915_active *ref)
186	{
187	lockdep_assert_held(BKL(ref));
188	__active_retire(ref);
189	}
190
191	int i915_active_wait(struct i915_active *ref)
192	{
193	struct active_node it, n;
194	int ret = 0;
195
196	if (i915_active_acquire(ref))
197	goto out_release;
198
199	ret = i915_gem_active_retire(&ref->last, BKL(ref));
200	if (ret)
201	goto out_release;
202
203	rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) {
204	ret = i915_gem_active_retire(&it->base, BKL(ref));
205	if (ret)
206	break;
207	}
208
209	out_release:
210	i915_active_release(ref);
211	return ret;
212	}
213
214	static int __i915_request_await_active(struct i915_request *rq,
215	struct i915_gem_active *active)
216	{
217	struct i915_request *barrier =
218	i915_gem_active_raw(active, &rq->i915->drm.struct_mutex);
219
220	return barrier ? i915_request_await_dma_fence(rq, &barrier->fence) : 0;
221	}
222
223	int i915_request_await_active(struct i915_request rq, struct i915_active ref)
224	{
225	struct active_node it, n;
226	int ret;
227
228	ret = __i915_request_await_active(rq, &ref->last);
229	if (ret)
230	return ret;
231
232	rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) {
233	ret = __i915_request_await_active(rq, &it->base);
234	if (ret)
235	return ret;
236	}
237
238	return 0;
239	}
240
a42375af	241	#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
64d6c500 CW	242	void i915_active_fini(struct i915_active *ref)
64d6c500 CW	243	{
64d6c500	244	GEM_BUG_ON(i915_gem_active_isset(&ref->last));
a42375af CW	245	GEM_BUG_ON(!RB_EMPTY_ROOT(&ref->tree));
a42375af CW	246	GEM_BUG_ON(ref->count);
64d6c500	247	}
a42375af	248	#endif
64d6c500 CW	249
	250	#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
	251	#include "selftests/i915_active.c"
	252	#endif
5f5c139d CW	253
	254	int __init i915_global_active_init(void)
	255	{
	256	global.slab_cache = KMEM_CACHE(active_node, SLAB_HWCACHE_ALIGN);
	257	if (!global.slab_cache)
	258	return -ENOMEM;
	259
	260	return 0;
	261	}
	262
	263	void __exit i915_global_active_exit(void)
	264	{
	265	kmem_cache_destroy(global.slab_cache);
	266	}