[linux-2.6-block.git] / drivers / gpu / drm / i915 / i915_sw_fence.c

/*
 * SPDX-License-Identifier: MIT
 *
 * (C) Copyright 2016 Intel Corporation
 */

#include <linux/slab.h>
#include <linux/dma-fence.h>
#include <linux/irq_work.h>
#include <linux/reservation.h>

#include "i915_sw_fence.h"
#include "i915_selftest.h"

#define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */

static DEFINE_SPINLOCK(i915_sw_fence_lock);

enum {
	DEBUG_FENCE_IDLE = 0,
	DEBUG_FENCE_NOTIFY,
};

static void *i915_sw_fence_debug_hint(void *addr)
{
	return (void *)(((struct i915_sw_fence *)addr)->flags & I915_SW_FENCE_MASK);
}

#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS

static struct debug_obj_descr i915_sw_fence_debug_descr = {
	.name = "i915_sw_fence",
	.debug_hint = i915_sw_fence_debug_hint,
};

static inline void debug_fence_init(struct i915_sw_fence *fence)
{
	debug_object_init(fence, &i915_sw_fence_debug_descr);
}

static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)
{
	debug_object_init_on_stack(fence, &i915_sw_fence_debug_descr);
}

static inline void debug_fence_activate(struct i915_sw_fence *fence)
{
	debug_object_activate(fence, &i915_sw_fence_debug_descr);
}

static inline void debug_fence_set_state(struct i915_sw_fence *fence,
					 int old, int new)
{
	debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new);
}

static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
{
	debug_object_deactivate(fence, &i915_sw_fence_debug_descr);
}

static inline void debug_fence_destroy(struct i915_sw_fence *fence)
{
	debug_object_destroy(fence, &i915_sw_fence_debug_descr);
}

static inline void debug_fence_free(struct i915_sw_fence *fence)
{
	debug_object_free(fence, &i915_sw_fence_debug_descr);
	smp_wmb(); /* flush the change in state before reallocation */
}

static inline void debug_fence_assert(struct i915_sw_fence *fence)
{
	debug_object_assert_init(fence, &i915_sw_fence_debug_descr);
}

#else

static inline void debug_fence_init(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_activate(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_set_state(struct i915_sw_fence *fence,
					 int old, int new)
{
}

static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_destroy(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_free(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_assert(struct i915_sw_fence *fence)
{
}

#endif

static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
				  enum i915_sw_fence_notify state)
{
	i915_sw_fence_notify_t fn;

	fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK);
	return fn(fence, state);
}

#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
void i915_sw_fence_fini(struct i915_sw_fence *fence)
{
	debug_fence_free(fence);
}
#endif

static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,
					struct list_head *continuation)
{
	wait_queue_head_t *x = &fence->wait;
	wait_queue_entry_t *pos, *next;
	unsigned long flags;

	debug_fence_deactivate(fence);
	atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */

	/*
	 * To prevent unbounded recursion as we traverse the graph of
	 * i915_sw_fences, we move the entry list from this, the next ready
	 * fence, to the tail of the original fence's entry list
	 * (and so added to the list to be woken).
	 */

	spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation);
	if (continuation) {
		list_for_each_entry_safe(pos, next, &x->head, entry) {
			if (pos->func == autoremove_wake_function)
				pos->func(pos, TASK_NORMAL, 0, continuation);
			else
				list_move_tail(&pos->entry, continuation);
		}
	} else {
		LIST_HEAD(extra);

		do {
			list_for_each_entry_safe(pos, next, &x->head, entry)
				pos->func(pos, TASK_NORMAL, 0, &extra);

			if (list_empty(&extra))
				break;

			list_splice_tail_init(&extra, &x->head);
		} while (1);
	}
	spin_unlock_irqrestore(&x->lock, flags);

	debug_fence_assert(fence);
}

static void __i915_sw_fence_complete(struct i915_sw_fence *fence,
				     struct list_head *continuation)
{
	debug_fence_assert(fence);

	if (!atomic_dec_and_test(&fence->pending))
		return;

	debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY);

	if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)
		return;

	debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE);

	__i915_sw_fence_wake_up_all(fence, continuation);

	debug_fence_destroy(fence);
	__i915_sw_fence_notify(fence, FENCE_FREE);
}

void i915_sw_fence_complete(struct i915_sw_fence *fence)
{
	debug_fence_assert(fence);

	if (WARN_ON(i915_sw_fence_done(fence)))
		return;

	__i915_sw_fence_complete(fence, NULL);
}

void i915_sw_fence_await(struct i915_sw_fence *fence)
{
	debug_fence_assert(fence);
	WARN_ON(atomic_inc_return(&fence->pending) <= 1);
}

void __i915_sw_fence_init(struct i915_sw_fence *fence,
			  i915_sw_fence_notify_t fn,
			  const char *name,
			  struct lock_class_key *key)
{
	BUG_ON(!fn || (unsigned long)fn & ~I915_SW_FENCE_MASK);

	debug_fence_init(fence);

	__init_waitqueue_head(&fence->wait, name, key);
	atomic_set(&fence->pending, 1);
	fence->flags = (unsigned long)fn;
}

void i915_sw_fence_commit(struct i915_sw_fence *fence)
{
	debug_fence_activate(fence);
	i915_sw_fence_complete(fence);
}

static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key)
{
	list_del(&wq->entry);
	__i915_sw_fence_complete(wq->private, key);

	if (wq->flags & I915_SW_FENCE_FLAG_ALLOC)
		kfree(wq);
	return 0;
}

static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
				    const struct i915_sw_fence * const signaler)
{
	wait_queue_entry_t *wq;

	if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
		return false;

	if (fence == signaler)
		return true;

	list_for_each_entry(wq, &fence->wait.head, entry) {
		if (wq->func != i915_sw_fence_wake)
			continue;

		if (__i915_sw_fence_check_if_after(wq->private, signaler))
			return true;
	}

	return false;
}

static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)
{
	wait_queue_entry_t *wq;

	if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
		return;

	list_for_each_entry(wq, &fence->wait.head, entry) {
		if (wq->func != i915_sw_fence_wake)
			continue;

		__i915_sw_fence_clear_checked_bit(wq->private);
	}
}

static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
				  const struct i915_sw_fence * const signaler)
{
	unsigned long flags;
	bool err;

	if (!IS_ENABLED(CONFIG_DRM_I915_SW_FENCE_CHECK_DAG))
		return false;

	spin_lock_irqsave(&i915_sw_fence_lock, flags);
	err = __i915_sw_fence_check_if_after(fence, signaler);
	__i915_sw_fence_clear_checked_bit(fence);
	spin_unlock_irqrestore(&i915_sw_fence_lock, flags);

	return err;
}

static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
					  struct i915_sw_fence *signaler,
					  wait_queue_entry_t *wq, gfp_t gfp)
{
	unsigned long flags;
	int pending;

	debug_fence_assert(fence);
	might_sleep_if(gfpflags_allow_blocking(gfp));

	if (i915_sw_fence_done(signaler))
		return 0;

	debug_fence_assert(signaler);

	/* The dependency graph must be acyclic. */
	if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))
		return -EINVAL;

	pending = 0;
	if (!wq) {
		wq = kmalloc(sizeof(*wq), gfp);
		if (!wq) {
			if (!gfpflags_allow_blocking(gfp))
				return -ENOMEM;

			i915_sw_fence_wait(signaler);
			return 0;
		}

		pending |= I915_SW_FENCE_FLAG_ALLOC;
	}

	INIT_LIST_HEAD(&wq->entry);
	wq->flags = pending;
	wq->func = i915_sw_fence_wake;
	wq->private = fence;

	i915_sw_fence_await(fence);

	spin_lock_irqsave(&signaler->wait.lock, flags);
	if (likely(!i915_sw_fence_done(signaler))) {
		__add_wait_queue_entry_tail(&signaler->wait, wq);
		pending = 1;
	} else {
		i915_sw_fence_wake(wq, 0, 0, NULL);
		pending = 0;
	}
	spin_unlock_irqrestore(&signaler->wait.lock, flags);

	return pending;
}

int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
				 struct i915_sw_fence *signaler,
				 wait_queue_entry_t *wq)
{
	return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);
}

int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
				     struct i915_sw_fence *signaler,
				     gfp_t gfp)
{
	return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);
}

struct i915_sw_dma_fence_cb {
	struct dma_fence_cb base;
	struct i915_sw_fence *fence;
};

struct i915_sw_dma_fence_cb_timer {
	struct i915_sw_dma_fence_cb base;
	struct dma_fence *dma;
	struct timer_list timer;
	struct irq_work work;
	struct rcu_head rcu;
};

static void dma_i915_sw_fence_wake(struct dma_fence *dma,
				   struct dma_fence_cb *data)
{
	struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base);

	i915_sw_fence_complete(cb->fence);
	kfree(cb);
}

static void timer_i915_sw_fence_wake(struct timer_list *t)
{
	struct i915_sw_dma_fence_cb_timer *cb = from_timer(cb, t, timer);
	struct i915_sw_fence *fence;

	fence = xchg(&cb->base.fence, NULL);
	if (!fence)
		return;

	pr_notice("Asynchronous wait on fence %s:%s:%llx timed out (hint:%pS)\n",
		  cb->dma->ops->get_driver_name(cb->dma),
		  cb->dma->ops->get_timeline_name(cb->dma),
		  cb->dma->seqno,
		  i915_sw_fence_debug_hint(fence));

	i915_sw_fence_complete(fence);
}

static void dma_i915_sw_fence_wake_timer(struct dma_fence *dma,
					 struct dma_fence_cb *data)
{
	struct i915_sw_dma_fence_cb_timer *cb =
		container_of(data, typeof(*cb), base.base);
	struct i915_sw_fence *fence;

	fence = xchg(&cb->base.fence, NULL);
	if (fence)
		i915_sw_fence_complete(fence);

	irq_work_queue(&cb->work);
}

static void irq_i915_sw_fence_work(struct irq_work *wrk)
{
	struct i915_sw_dma_fence_cb_timer *cb =
		container_of(wrk, typeof(*cb), work);

	del_timer_sync(&cb->timer);
	dma_fence_put(cb->dma);

	kfree_rcu(cb, rcu);
}

int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
				  struct dma_fence *dma,
				  unsigned long timeout,
				  gfp_t gfp)
{
	struct i915_sw_dma_fence_cb *cb;
	dma_fence_func_t func;
	int ret;

	debug_fence_assert(fence);
	might_sleep_if(gfpflags_allow_blocking(gfp));

	if (dma_fence_is_signaled(dma))
		return 0;

	cb = kmalloc(timeout ?
		     sizeof(struct i915_sw_dma_fence_cb_timer) :
		     sizeof(struct i915_sw_dma_fence_cb),
		     gfp);
	if (!cb) {
		if (!gfpflags_allow_blocking(gfp))
			return -ENOMEM;

		return dma_fence_wait(dma, false);
	}

	cb->fence = fence;
	i915_sw_fence_await(fence);

	func = dma_i915_sw_fence_wake;
	if (timeout) {
		struct i915_sw_dma_fence_cb_timer *timer =
			container_of(cb, typeof(*timer), base);

		timer->dma = dma_fence_get(dma);
		init_irq_work(&timer->work, irq_i915_sw_fence_work);

		timer_setup(&timer->timer,
			    timer_i915_sw_fence_wake, TIMER_IRQSAFE);
		mod_timer(&timer->timer, round_jiffies_up(jiffies + timeout));

		func = dma_i915_sw_fence_wake_timer;
	}

	ret = dma_fence_add_callback(dma, &cb->base, func);
	if (ret == 0) {
		ret = 1;
	} else {
		func(dma, &cb->base);
		if (ret == -ENOENT) /* fence already signaled */
			ret = 0;
	}

	return ret;
}

int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
				    struct reservation_object *resv,
				    const struct dma_fence_ops *exclude,
				    bool write,
				    unsigned long timeout,
				    gfp_t gfp)
{
	struct dma_fence *excl;
	int ret = 0, pending;

	debug_fence_assert(fence);
	might_sleep_if(gfpflags_allow_blocking(gfp));

	if (write) {
		struct dma_fence **shared;
		unsigned int count, i;

		ret = reservation_object_get_fences_rcu(resv,
							&excl, &count, &shared);
		if (ret)
			return ret;

		for (i = 0; i < count; i++) {
			if (shared[i]->ops == exclude)
				continue;

			pending = i915_sw_fence_await_dma_fence(fence,
								shared[i],
								timeout,
								gfp);
			if (pending < 0) {
				ret = pending;
				break;
			}

			ret |= pending;
		}

		for (i = 0; i < count; i++)
			dma_fence_put(shared[i]);
		kfree(shared);
	} else {
		excl = reservation_object_get_excl_rcu(resv);
	}

	if (ret >= 0 && excl && excl->ops != exclude) {
		pending = i915_sw_fence_await_dma_fence(fence,
							excl,
							timeout,
							gfp);
		if (pending < 0)
			ret = pending;
		else
			ret |= pending;
	}

	dma_fence_put(excl);

	return ret;
}

#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
#include "selftests/lib_sw_fence.c"
#include "selftests/i915_sw_fence.c"
#endif
Commit	Line	Data
e68a139f	1	/*
635b3bc6	2	* SPDX-License-Identifier: MIT
e68a139f	3	*
635b3bc6	4	* (C) Copyright 2016 Intel Corporation
e68a139f CW	5	*/
	6
	7	#include <linux/slab.h>
f54d1867	8	#include <linux/dma-fence.h>
81c0ed21	9	#include <linux/irq_work.h>
e68a139f CW	10	#include <linux/reservation.h>
	11
	12	#include "i915_sw_fence.h"
47624cc3	13	#include "i915_selftest.h"
e68a139f	14
7e941861 CW	15	#define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */
7e941861 CW	16
e68a139f CW	17	static DEFINE_SPINLOCK(i915_sw_fence_lock);
e68a139f CW	18
fc158405 CW	19	enum {
	20	DEBUG_FENCE_IDLE = 0,
	21	DEBUG_FENCE_NOTIFY,
	22	};
	23
fc158405 CW	24	static void i915_sw_fence_debug_hint(void addr)
	25	{
	26	return (void )(((struct i915_sw_fence )addr)->flags & I915_SW_FENCE_MASK);
	27	}
	28
5791bad4 CW	29	#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
5791bad4 CW	30
fc158405 CW	31	static struct debug_obj_descr i915_sw_fence_debug_descr = {
	32	.name = "i915_sw_fence",
	33	.debug_hint = i915_sw_fence_debug_hint,
	34	};
	35
	36	static inline void debug_fence_init(struct i915_sw_fence *fence)
	37	{
	38	debug_object_init(fence, &i915_sw_fence_debug_descr);
	39	}
	40
214707fc CW	41	static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)
	42	{
	43	debug_object_init_on_stack(fence, &i915_sw_fence_debug_descr);
	44	}
	45
fc158405 CW	46	static inline void debug_fence_activate(struct i915_sw_fence *fence)
	47	{
	48	debug_object_activate(fence, &i915_sw_fence_debug_descr);
	49	}
	50
	51	static inline void debug_fence_set_state(struct i915_sw_fence *fence,
	52	int old, int new)
	53	{
	54	debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new);
	55	}
	56
	57	static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
	58	{
	59	debug_object_deactivate(fence, &i915_sw_fence_debug_descr);
	60	}
	61
	62	static inline void debug_fence_destroy(struct i915_sw_fence *fence)
	63	{
	64	debug_object_destroy(fence, &i915_sw_fence_debug_descr);
	65	}
	66
	67	static inline void debug_fence_free(struct i915_sw_fence *fence)
	68	{
	69	debug_object_free(fence, &i915_sw_fence_debug_descr);
6f13f29f	70	smp_wmb(); /* flush the change in state before reallocation */
fc158405 CW	71	}
	72
	73	static inline void debug_fence_assert(struct i915_sw_fence *fence)
	74	{
	75	debug_object_assert_init(fence, &i915_sw_fence_debug_descr);
	76	}
	77
	78	#else
	79
	80	static inline void debug_fence_init(struct i915_sw_fence *fence)
	81	{
	82	}
	83
214707fc CW	84	static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)
	85	{
	86	}
	87
fc158405 CW	88	static inline void debug_fence_activate(struct i915_sw_fence *fence)
	89	{
	90	}
	91
	92	static inline void debug_fence_set_state(struct i915_sw_fence *fence,
	93	int old, int new)
	94	{
	95	}
	96
	97	static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
	98	{
	99	}
	100
	101	static inline void debug_fence_destroy(struct i915_sw_fence *fence)
	102	{
	103	}
	104
	105	static inline void debug_fence_free(struct i915_sw_fence *fence)
	106	{
	107	}
	108
	109	static inline void debug_fence_assert(struct i915_sw_fence *fence)
	110	{
	111	}
	112
	113	#endif
	114
e68a139f CW	115	static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
	116	enum i915_sw_fence_notify state)
	117	{
	118	i915_sw_fence_notify_t fn;
	119
	120	fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK);
	121	return fn(fence, state);
	122	}
	123
fc158405 CW	124	#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
	125	void i915_sw_fence_fini(struct i915_sw_fence *fence)
	126	{
	127	debug_fence_free(fence);
	128	}
	129	#endif
	130
e68a139f CW	131	static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,
	132	struct list_head *continuation)
	133	{
	134	wait_queue_head_t *x = &fence->wait;
ac6424b9	135	wait_queue_entry_t pos, next;
e68a139f CW	136	unsigned long flags;
e68a139f CW	137
fc158405	138	debug_fence_deactivate(fence);
e68a139f CW	139	atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */
	140
	141	/*
	142	* To prevent unbounded recursion as we traverse the graph of
2055da97 IM	143	* i915_sw_fences, we move the entry list from this, the next ready
2055da97 IM	144	* fence, to the tail of the original fence's entry list
e68a139f CW	145	* (and so added to the list to be woken).
	146	*/
	147
	148	spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation);
	149	if (continuation) {
2055da97	150	list_for_each_entry_safe(pos, next, &x->head, entry) {
e68a139f CW	151	if (pos->func == autoremove_wake_function)
	152	pos->func(pos, TASK_NORMAL, 0, continuation);
	153	else
2055da97	154	list_move_tail(&pos->entry, continuation);
e68a139f CW	155	}
	156	} else {
	157	LIST_HEAD(extra);
	158
	159	do {
2055da97	160	list_for_each_entry_safe(pos, next, &x->head, entry)
e68a139f CW	161	pos->func(pos, TASK_NORMAL, 0, &extra);
	162
	163	if (list_empty(&extra))
	164	break;
	165
2055da97	166	list_splice_tail_init(&extra, &x->head);
e68a139f CW	167	} while (1);
	168	}
	169	spin_unlock_irqrestore(&x->lock, flags);
fc158405 CW	170
fc158405 CW	171	debug_fence_assert(fence);
e68a139f CW	172	}
	173
	174	static void __i915_sw_fence_complete(struct i915_sw_fence *fence,
	175	struct list_head *continuation)
	176	{
fc158405 CW	177	debug_fence_assert(fence);
fc158405 CW	178
e68a139f CW	179	if (!atomic_dec_and_test(&fence->pending))
	180	return;
	181
fc158405 CW	182	debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY);
fc158405 CW	183
9310cb7f	184	if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)
e68a139f CW	185	return;
e68a139f CW	186
fc158405 CW	187	debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE);
fc158405 CW	188
e68a139f	189	__i915_sw_fence_wake_up_all(fence, continuation);
9310cb7f CW	190
	191	debug_fence_destroy(fence);
	192	__i915_sw_fence_notify(fence, FENCE_FREE);
e68a139f CW	193	}
e68a139f CW	194
e8861964	195	void i915_sw_fence_complete(struct i915_sw_fence *fence)
e68a139f	196	{
fc158405 CW	197	debug_fence_assert(fence);
fc158405 CW	198
e68a139f CW	199	if (WARN_ON(i915_sw_fence_done(fence)))
	200	return;
	201
	202	__i915_sw_fence_complete(fence, NULL);
	203	}
	204
e8861964	205	void i915_sw_fence_await(struct i915_sw_fence *fence)
e68a139f	206	{
fc158405	207	debug_fence_assert(fence);
e68a139f CW	208	WARN_ON(atomic_inc_return(&fence->pending) <= 1);
	209	}
	210
556b7487 CW	211	void __i915_sw_fence_init(struct i915_sw_fence *fence,
	212	i915_sw_fence_notify_t fn,
	213	const char *name,
	214	struct lock_class_key *key)
e68a139f	215	{
9310cb7f	216	BUG_ON(!fn \|\| (unsigned long)fn & ~I915_SW_FENCE_MASK);
e68a139f	217
fc158405 CW	218	debug_fence_init(fence);
fc158405 CW	219
556b7487	220	__init_waitqueue_head(&fence->wait, name, key);
e68a139f CW	221	atomic_set(&fence->pending, 1);
	222	fence->flags = (unsigned long)fn;
	223	}
	224
fc158405 CW	225	void i915_sw_fence_commit(struct i915_sw_fence *fence)
	226	{
	227	debug_fence_activate(fence);
9310cb7f	228	i915_sw_fence_complete(fence);
fc158405 CW	229	}
fc158405 CW	230
ac6424b9	231	static int i915_sw_fence_wake(wait_queue_entry_t wq, unsigned mode, int flags, void key)
e68a139f	232	{
2055da97	233	list_del(&wq->entry);
e68a139f	234	__i915_sw_fence_complete(wq->private, key);
9310cb7f	235
7e941861 CW	236	if (wq->flags & I915_SW_FENCE_FLAG_ALLOC)
7e941861 CW	237	kfree(wq);
e68a139f CW	238	return 0;
	239	}
	240
	241	static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
	242	const struct i915_sw_fence * const signaler)
	243	{
ac6424b9	244	wait_queue_entry_t *wq;
e68a139f CW	245
	246	if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
	247	return false;
	248
	249	if (fence == signaler)
	250	return true;
	251
2055da97	252	list_for_each_entry(wq, &fence->wait.head, entry) {
e68a139f CW	253	if (wq->func != i915_sw_fence_wake)
	254	continue;
	255
	256	if (__i915_sw_fence_check_if_after(wq->private, signaler))
	257	return true;
	258	}
	259
	260	return false;
	261	}
	262
	263	static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)
	264	{
ac6424b9	265	wait_queue_entry_t *wq;
e68a139f CW	266
	267	if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
	268	return;
	269
2055da97	270	list_for_each_entry(wq, &fence->wait.head, entry) {
e68a139f CW	271	if (wq->func != i915_sw_fence_wake)
	272	continue;
	273
	274	__i915_sw_fence_clear_checked_bit(wq->private);
	275	}
	276	}
	277
	278	static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
	279	const struct i915_sw_fence * const signaler)
	280	{
	281	unsigned long flags;
	282	bool err;
	283
47624cc3	284	if (!IS_ENABLED(CONFIG_DRM_I915_SW_FENCE_CHECK_DAG))
e68a139f CW	285	return false;
	286
	287	spin_lock_irqsave(&i915_sw_fence_lock, flags);
	288	err = __i915_sw_fence_check_if_after(fence, signaler);
	289	__i915_sw_fence_clear_checked_bit(fence);
	290	spin_unlock_irqrestore(&i915_sw_fence_lock, flags);
	291
	292	return err;
	293	}
	294
7e941861 CW	295	static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
7e941861 CW	296	struct i915_sw_fence *signaler,
ac6424b9	297	wait_queue_entry_t *wq, gfp_t gfp)
e68a139f CW	298	{
	299	unsigned long flags;
	300	int pending;
	301
fc158405	302	debug_fence_assert(fence);
e30a7581	303	might_sleep_if(gfpflags_allow_blocking(gfp));
fc158405	304
e68a139f CW	305	if (i915_sw_fence_done(signaler))
	306	return 0;
	307
fc158405 CW	308	debug_fence_assert(signaler);
fc158405 CW	309
e68a139f CW	310	/* The dependency graph must be acyclic. */
	311	if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))
	312	return -EINVAL;
	313
7e941861 CW	314	pending = 0;
	315	if (!wq) {
	316	wq = kmalloc(sizeof(*wq), gfp);
	317	if (!wq) {
	318	if (!gfpflags_allow_blocking(gfp))
	319	return -ENOMEM;
	320
	321	i915_sw_fence_wait(signaler);
	322	return 0;
	323	}
	324
	325	pending \|= I915_SW_FENCE_FLAG_ALLOC;
	326	}
	327
2055da97	328	INIT_LIST_HEAD(&wq->entry);
7e941861	329	wq->flags = pending;
e68a139f	330	wq->func = i915_sw_fence_wake;
9310cb7f	331	wq->private = fence;
e68a139f CW	332
	333	i915_sw_fence_await(fence);
	334
	335	spin_lock_irqsave(&signaler->wait.lock, flags);
	336	if (likely(!i915_sw_fence_done(signaler))) {
ac6424b9	337	__add_wait_queue_entry_tail(&signaler->wait, wq);
e68a139f CW	338	pending = 1;
	339	} else {
	340	i915_sw_fence_wake(wq, 0, 0, NULL);
	341	pending = 0;
	342	}
	343	spin_unlock_irqrestore(&signaler->wait.lock, flags);
	344
	345	return pending;
	346	}
	347
7e941861 CW	348	int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
7e941861 CW	349	struct i915_sw_fence *signaler,
ac6424b9	350	wait_queue_entry_t *wq)
7e941861 CW	351	{
	352	return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);
	353	}
	354
	355	int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
	356	struct i915_sw_fence *signaler,
	357	gfp_t gfp)
	358	{
	359	return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);
	360	}
	361
f54d1867 CW	362	struct i915_sw_dma_fence_cb {
f54d1867 CW	363	struct dma_fence_cb base;
e68a139f	364	struct i915_sw_fence *fence;
f255c1e9 CW	365	};
	366
	367	struct i915_sw_dma_fence_cb_timer {
	368	struct i915_sw_dma_fence_cb base;
f54d1867	369	struct dma_fence *dma;
e68a139f	370	struct timer_list timer;
81c0ed21	371	struct irq_work work;
7d622351	372	struct rcu_head rcu;
e68a139f CW	373	};
e68a139f CW	374
f255c1e9 CW	375	static void dma_i915_sw_fence_wake(struct dma_fence *dma,
	376	struct dma_fence_cb *data)
	377	{
	378	struct i915_sw_dma_fence_cb cb = container_of(data, typeof(cb), base);
	379
	380	i915_sw_fence_complete(cb->fence);
	381	kfree(cb);
	382	}
	383
39cbf2aa	384	static void timer_i915_sw_fence_wake(struct timer_list *t)
e68a139f	385	{
f255c1e9	386	struct i915_sw_dma_fence_cb_timer *cb = from_timer(cb, t, timer);
81c0ed21 CW	387	struct i915_sw_fence *fence;
81c0ed21 CW	388
f255c1e9	389	fence = xchg(&cb->base.fence, NULL);
81c0ed21 CW	390	if (!fence)
81c0ed21 CW	391	return;
e68a139f	392
b312d8ca	393	pr_notice("Asynchronous wait on fence %s:%s:%llx timed out (hint:%pS)\n",
5791bad4 CW	394	cb->dma->ops->get_driver_name(cb->dma),
	395	cb->dma->ops->get_timeline_name(cb->dma),
	396	cb->dma->seqno,
	397	i915_sw_fence_debug_hint(fence));
e68a139f	398
81c0ed21	399	i915_sw_fence_complete(fence);
e68a139f CW	400	}
e68a139f CW	401
f255c1e9 CW	402	static void dma_i915_sw_fence_wake_timer(struct dma_fence *dma,
f255c1e9 CW	403	struct dma_fence_cb *data)
e68a139f	404	{
f255c1e9 CW	405	struct i915_sw_dma_fence_cb_timer *cb =
f255c1e9 CW	406	container_of(data, typeof(*cb), base.base);
81c0ed21 CW	407	struct i915_sw_fence *fence;
81c0ed21 CW	408
f255c1e9	409	fence = xchg(&cb->base.fence, NULL);
81c0ed21 CW	410	if (fence)
	411	i915_sw_fence_complete(fence);
	412
f255c1e9	413	irq_work_queue(&cb->work);
81c0ed21 CW	414	}
	415
	416	static void irq_i915_sw_fence_work(struct irq_work *wrk)
	417	{
f255c1e9 CW	418	struct i915_sw_dma_fence_cb_timer *cb =
f255c1e9 CW	419	container_of(wrk, typeof(*cb), work);
e68a139f CW	420
e68a139f CW	421	del_timer_sync(&cb->timer);
f54d1867	422	dma_fence_put(cb->dma);
e68a139f	423
7d622351	424	kfree_rcu(cb, rcu);
e68a139f CW	425	}
	426
	427	int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
f54d1867	428	struct dma_fence *dma,
e68a139f CW	429	unsigned long timeout,
	430	gfp_t gfp)
	431	{
f54d1867	432	struct i915_sw_dma_fence_cb *cb;
f255c1e9	433	dma_fence_func_t func;
e68a139f CW	434	int ret;
e68a139f CW	435
fc158405	436	debug_fence_assert(fence);
e30a7581	437	might_sleep_if(gfpflags_allow_blocking(gfp));
fc158405	438
f54d1867	439	if (dma_fence_is_signaled(dma))
e68a139f CW	440	return 0;
e68a139f CW	441
f255c1e9 CW	442	cb = kmalloc(timeout ?
	443	sizeof(struct i915_sw_dma_fence_cb_timer) :
	444	sizeof(struct i915_sw_dma_fence_cb),
	445	gfp);
e68a139f CW	446	if (!cb) {
	447	if (!gfpflags_allow_blocking(gfp))
	448	return -ENOMEM;
	449
f54d1867	450	return dma_fence_wait(dma, false);
e68a139f CW	451	}
e68a139f CW	452
9310cb7f	453	cb->fence = fence;
e68a139f CW	454	i915_sw_fence_await(fence);
e68a139f CW	455
f255c1e9	456	func = dma_i915_sw_fence_wake;
e68a139f	457	if (timeout) {
f255c1e9 CW	458	struct i915_sw_dma_fence_cb_timer *timer =
f255c1e9 CW	459	container_of(cb, typeof(*timer), base);
c32164b1	460
f255c1e9 CW	461	timer->dma = dma_fence_get(dma);
	462	init_irq_work(&timer->work, irq_i915_sw_fence_work);
	463
	464	timer_setup(&timer->timer,
c32164b1	465	timer_i915_sw_fence_wake, TIMER_IRQSAFE);
f255c1e9 CW	466	mod_timer(&timer->timer, round_jiffies_up(jiffies + timeout));
	467
	468	func = dma_i915_sw_fence_wake_timer;
e68a139f CW	469	}
e68a139f CW	470
f255c1e9	471	ret = dma_fence_add_callback(dma, &cb->base, func);
e68a139f CW	472	if (ret == 0) {
	473	ret = 1;
	474	} else {
f255c1e9	475	func(dma, &cb->base);
e68a139f CW	476	if (ret == -ENOENT) /* fence already signaled */
	477	ret = 0;
	478	}
	479
	480	return ret;
	481	}
	482
	483	int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
	484	struct reservation_object *resv,
f54d1867	485	const struct dma_fence_ops *exclude,
e68a139f CW	486	bool write,
	487	unsigned long timeout,
	488	gfp_t gfp)
	489	{
f54d1867	490	struct dma_fence *excl;
e68a139f CW	491	int ret = 0, pending;
e68a139f CW	492
fc158405	493	debug_fence_assert(fence);
e30a7581	494	might_sleep_if(gfpflags_allow_blocking(gfp));
fc158405	495
e68a139f	496	if (write) {
f54d1867	497	struct dma_fence **shared;
e68a139f CW	498	unsigned int count, i;
	499
	500	ret = reservation_object_get_fences_rcu(resv,
	501	&excl, &count, &shared);
	502	if (ret)
	503	return ret;
	504
	505	for (i = 0; i < count; i++) {
	506	if (shared[i]->ops == exclude)
	507	continue;
	508
	509	pending = i915_sw_fence_await_dma_fence(fence,
	510	shared[i],
	511	timeout,
	512	gfp);
	513	if (pending < 0) {
	514	ret = pending;
	515	break;
	516	}
	517
	518	ret \|= pending;
	519	}
	520
	521	for (i = 0; i < count; i++)
f54d1867	522	dma_fence_put(shared[i]);
e68a139f CW	523	kfree(shared);
	524	} else {
	525	excl = reservation_object_get_excl_rcu(resv);
	526	}
	527
	528	if (ret >= 0 && excl && excl->ops != exclude) {
	529	pending = i915_sw_fence_await_dma_fence(fence,
	530	excl,
	531	timeout,
	532	gfp);
	533	if (pending < 0)
	534	ret = pending;
	535	else
	536	ret \|= pending;
	537	}
	538
f54d1867	539	dma_fence_put(excl);
e68a139f CW	540
	541	return ret;
	542	}
47624cc3 CW	543
47624cc3 CW	544	#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
214707fc	545	#include "selftests/lib_sw_fence.c"
47624cc3 CW	546	#include "selftests/i915_sw_fence.c"
47624cc3 CW	547	#endif