[linux-2.6-block.git] / drivers / gpu / drm / i915 / i915_sw_fence.c

/*
 * SPDX-License-Identifier: MIT
 *
 * (C) Copyright 2016 Intel Corporation
 */

#include <linux/slab.h>
#include <linux/dma-fence.h>
#include <linux/irq_work.h>
#include <linux/dma-resv.h>

#include "i915_sw_fence.h"
#include "i915_selftest.h"

#if IS_ENABLED(CONFIG_DRM_I915_DEBUG)
#define I915_SW_FENCE_BUG_ON(expr) BUG_ON(expr)
#else
#define I915_SW_FENCE_BUG_ON(expr) BUILD_BUG_ON_INVALID(expr)
#endif

static DEFINE_SPINLOCK(i915_sw_fence_lock);

#define WQ_FLAG_BITS \
	BITS_PER_TYPE(typeof_member(struct wait_queue_entry, flags))

/* after WQ_FLAG_* for safety */
#define I915_SW_FENCE_FLAG_FENCE BIT(WQ_FLAG_BITS - 1)
#define I915_SW_FENCE_FLAG_ALLOC BIT(WQ_FLAG_BITS - 2)

enum {
	DEBUG_FENCE_IDLE = 0,
	DEBUG_FENCE_NOTIFY,
};

static void *i915_sw_fence_debug_hint(void *addr)
{
	return (void *)(((struct i915_sw_fence *)addr)->flags & I915_SW_FENCE_MASK);
}

#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS

static const struct debug_obj_descr i915_sw_fence_debug_descr = {
	.name = "i915_sw_fence",
	.debug_hint = i915_sw_fence_debug_hint,
};

static inline void debug_fence_init(struct i915_sw_fence *fence)
{
	debug_object_init(fence, &i915_sw_fence_debug_descr);
}

static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)
{
	debug_object_init_on_stack(fence, &i915_sw_fence_debug_descr);
}

static inline void debug_fence_activate(struct i915_sw_fence *fence)
{
	debug_object_activate(fence, &i915_sw_fence_debug_descr);
}

static inline void debug_fence_set_state(struct i915_sw_fence *fence,
					 int old, int new)
{
	debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new);
}

static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
{
	debug_object_deactivate(fence, &i915_sw_fence_debug_descr);
}

static inline void debug_fence_destroy(struct i915_sw_fence *fence)
{
	debug_object_destroy(fence, &i915_sw_fence_debug_descr);
}

static inline void debug_fence_free(struct i915_sw_fence *fence)
{
	debug_object_free(fence, &i915_sw_fence_debug_descr);
	smp_wmb(); /* flush the change in state before reallocation */
}

static inline void debug_fence_assert(struct i915_sw_fence *fence)
{
	debug_object_assert_init(fence, &i915_sw_fence_debug_descr);
}

#else

static inline void debug_fence_init(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_activate(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_set_state(struct i915_sw_fence *fence,
					 int old, int new)
{
}

static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_destroy(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_free(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_assert(struct i915_sw_fence *fence)
{
}

#endif

static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
				  enum i915_sw_fence_notify state)
{
	i915_sw_fence_notify_t fn;

	fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK);
	return fn(fence, state);
}

#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
void i915_sw_fence_fini(struct i915_sw_fence *fence)
{
	debug_fence_free(fence);
}
#endif

static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,
					struct list_head *continuation)
{
	wait_queue_head_t *x = &fence->wait;
	wait_queue_entry_t *pos, *next;
	unsigned long flags;

	debug_fence_deactivate(fence);
	atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */

	/*
	 * To prevent unbounded recursion as we traverse the graph of
	 * i915_sw_fences, we move the entry list from this, the next ready
	 * fence, to the tail of the original fence's entry list
	 * (and so added to the list to be woken).
	 */

	spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation);
	if (continuation) {
		list_for_each_entry_safe(pos, next, &x->head, entry) {
			if (pos->flags & I915_SW_FENCE_FLAG_FENCE)
				list_move_tail(&pos->entry, continuation);
			else
				pos->func(pos, TASK_NORMAL, 0, continuation);
		}
	} else {
		LIST_HEAD(extra);

		do {
			list_for_each_entry_safe(pos, next, &x->head, entry) {
				int wake_flags;

				wake_flags = 0;
				if (pos->flags & I915_SW_FENCE_FLAG_FENCE)
					wake_flags = fence->error;

				pos->func(pos, TASK_NORMAL, wake_flags, &extra);
			}

			if (list_empty(&extra))
				break;

			list_splice_tail_init(&extra, &x->head);
		} while (1);
	}
	spin_unlock_irqrestore(&x->lock, flags);

	debug_fence_assert(fence);
}

static void __i915_sw_fence_complete(struct i915_sw_fence *fence,
				     struct list_head *continuation)
{
	debug_fence_assert(fence);

	if (!atomic_dec_and_test(&fence->pending))
		return;

	debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY);

	if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)
		return;

	debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE);

	__i915_sw_fence_wake_up_all(fence, continuation);

	debug_fence_destroy(fence);
	__i915_sw_fence_notify(fence, FENCE_FREE);
}

void i915_sw_fence_complete(struct i915_sw_fence *fence)
{
	debug_fence_assert(fence);

	if (WARN_ON(i915_sw_fence_done(fence)))
		return;

	__i915_sw_fence_complete(fence, NULL);
}

bool i915_sw_fence_await(struct i915_sw_fence *fence)
{
	int pending;

	/*
	 * It is only safe to add a new await to the fence while it has
	 * not yet been signaled (i.e. there are still existing signalers).
	 */
	pending = atomic_read(&fence->pending);
	do {
		if (pending < 1)
			return false;
	} while (!atomic_try_cmpxchg(&fence->pending, &pending, pending + 1));

	return true;
}

void __i915_sw_fence_init(struct i915_sw_fence *fence,
			  i915_sw_fence_notify_t fn,
			  const char *name,
			  struct lock_class_key *key)
{
	BUG_ON(!fn || (unsigned long)fn & ~I915_SW_FENCE_MASK);

	__init_waitqueue_head(&fence->wait, name, key);
	fence->flags = (unsigned long)fn;

	i915_sw_fence_reinit(fence);
}

void i915_sw_fence_reinit(struct i915_sw_fence *fence)
{
	debug_fence_init(fence);

	atomic_set(&fence->pending, 1);
	fence->error = 0;

	I915_SW_FENCE_BUG_ON(!fence->flags);
	I915_SW_FENCE_BUG_ON(!list_empty(&fence->wait.head));
}

void i915_sw_fence_commit(struct i915_sw_fence *fence)
{
	debug_fence_activate(fence);
	i915_sw_fence_complete(fence);
}

static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key)
{
	i915_sw_fence_set_error_once(wq->private, flags);

	list_del(&wq->entry);
	__i915_sw_fence_complete(wq->private, key);

	if (wq->flags & I915_SW_FENCE_FLAG_ALLOC)
		kfree(wq);
	return 0;
}

static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
				    const struct i915_sw_fence * const signaler)
{
	wait_queue_entry_t *wq;

	if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
		return false;

	if (fence == signaler)
		return true;

	list_for_each_entry(wq, &fence->wait.head, entry) {
		if (wq->func != i915_sw_fence_wake)
			continue;

		if (__i915_sw_fence_check_if_after(wq->private, signaler))
			return true;
	}

	return false;
}

static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)
{
	wait_queue_entry_t *wq;

	if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
		return;

	list_for_each_entry(wq, &fence->wait.head, entry) {
		if (wq->func != i915_sw_fence_wake)
			continue;

		__i915_sw_fence_clear_checked_bit(wq->private);
	}
}

static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
				  const struct i915_sw_fence * const signaler)
{
	unsigned long flags;
	bool err;

	if (!IS_ENABLED(CONFIG_DRM_I915_SW_FENCE_CHECK_DAG))
		return false;

	spin_lock_irqsave(&i915_sw_fence_lock, flags);
	err = __i915_sw_fence_check_if_after(fence, signaler);
	__i915_sw_fence_clear_checked_bit(fence);
	spin_unlock_irqrestore(&i915_sw_fence_lock, flags);

	return err;
}

static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
					  struct i915_sw_fence *signaler,
					  wait_queue_entry_t *wq, gfp_t gfp)
{
	unsigned int pending;
	unsigned long flags;

	debug_fence_assert(fence);
	might_sleep_if(gfpflags_allow_blocking(gfp));

	if (i915_sw_fence_done(signaler)) {
		i915_sw_fence_set_error_once(fence, signaler->error);
		return 0;
	}

	debug_fence_assert(signaler);

	/* The dependency graph must be acyclic. */
	if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))
		return -EINVAL;

	pending = I915_SW_FENCE_FLAG_FENCE;
	if (!wq) {
		wq = kmalloc(sizeof(*wq), gfp);
		if (!wq) {
			if (!gfpflags_allow_blocking(gfp))
				return -ENOMEM;

			i915_sw_fence_wait(signaler);
			i915_sw_fence_set_error_once(fence, signaler->error);
			return 0;
		}

		pending |= I915_SW_FENCE_FLAG_ALLOC;
	}

	INIT_LIST_HEAD(&wq->entry);
	wq->flags = pending;
	wq->func = i915_sw_fence_wake;
	wq->private = fence;

	i915_sw_fence_await(fence);

	spin_lock_irqsave(&signaler->wait.lock, flags);
	if (likely(!i915_sw_fence_done(signaler))) {
		__add_wait_queue_entry_tail(&signaler->wait, wq);
		pending = 1;
	} else {
		i915_sw_fence_wake(wq, 0, signaler->error, NULL);
		pending = 0;
	}
	spin_unlock_irqrestore(&signaler->wait.lock, flags);

	return pending;
}

int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
				 struct i915_sw_fence *signaler,
				 wait_queue_entry_t *wq)
{
	return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);
}

int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
				     struct i915_sw_fence *signaler,
				     gfp_t gfp)
{
	return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);
}

struct i915_sw_dma_fence_cb_timer {
	struct i915_sw_dma_fence_cb base;
	struct dma_fence *dma;
	struct timer_list timer;
	struct irq_work work;
	struct rcu_head rcu;
};

static void dma_i915_sw_fence_wake(struct dma_fence *dma,
				   struct dma_fence_cb *data)
{
	struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base);

	i915_sw_fence_set_error_once(cb->fence, dma->error);
	i915_sw_fence_complete(cb->fence);
	kfree(cb);
}

static void timer_i915_sw_fence_wake(struct timer_list *t)
{
	struct i915_sw_dma_fence_cb_timer *cb = from_timer(cb, t, timer);
	struct i915_sw_fence *fence;

	fence = xchg(&cb->base.fence, NULL);
	if (!fence)
		return;

	pr_notice("Asynchronous wait on fence %s:%s:%llx timed out (hint:%ps)\n",
		  cb->dma->ops->get_driver_name(cb->dma),
		  cb->dma->ops->get_timeline_name(cb->dma),
		  cb->dma->seqno,
		  i915_sw_fence_debug_hint(fence));

	i915_sw_fence_set_error_once(fence, -ETIMEDOUT);
	i915_sw_fence_complete(fence);
}

static void dma_i915_sw_fence_wake_timer(struct dma_fence *dma,
					 struct dma_fence_cb *data)
{
	struct i915_sw_dma_fence_cb_timer *cb =
		container_of(data, typeof(*cb), base.base);
	struct i915_sw_fence *fence;

	fence = xchg(&cb->base.fence, NULL);
	if (fence) {
		i915_sw_fence_set_error_once(fence, dma->error);
		i915_sw_fence_complete(fence);
	}

	irq_work_queue(&cb->work);
}

static void irq_i915_sw_fence_work(struct irq_work *wrk)
{
	struct i915_sw_dma_fence_cb_timer *cb =
		container_of(wrk, typeof(*cb), work);

	del_timer_sync(&cb->timer);
	dma_fence_put(cb->dma);

	kfree_rcu(cb, rcu);
}

int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
				  struct dma_fence *dma,
				  unsigned long timeout,
				  gfp_t gfp)
{
	struct i915_sw_dma_fence_cb *cb;
	dma_fence_func_t func;
	int ret;

	debug_fence_assert(fence);
	might_sleep_if(gfpflags_allow_blocking(gfp));

	if (dma_fence_is_signaled(dma)) {
		i915_sw_fence_set_error_once(fence, dma->error);
		return 0;
	}

	cb = kmalloc(timeout ?
		     sizeof(struct i915_sw_dma_fence_cb_timer) :
		     sizeof(struct i915_sw_dma_fence_cb),
		     gfp);
	if (!cb) {
		if (!gfpflags_allow_blocking(gfp))
			return -ENOMEM;

		ret = dma_fence_wait(dma, false);
		if (ret)
			return ret;

		i915_sw_fence_set_error_once(fence, dma->error);
		return 0;
	}

	cb->fence = fence;
	i915_sw_fence_await(fence);

	func = dma_i915_sw_fence_wake;
	if (timeout) {
		struct i915_sw_dma_fence_cb_timer *timer =
			container_of(cb, typeof(*timer), base);

		timer->dma = dma_fence_get(dma);
		init_irq_work(&timer->work, irq_i915_sw_fence_work);

		timer_setup(&timer->timer,
			    timer_i915_sw_fence_wake, TIMER_IRQSAFE);
		mod_timer(&timer->timer, round_jiffies_up(jiffies + timeout));

		func = dma_i915_sw_fence_wake_timer;
	}

	ret = dma_fence_add_callback(dma, &cb->base, func);
	if (ret == 0) {
		ret = 1;
	} else {
		func(dma, &cb->base);
		if (ret == -ENOENT) /* fence already signaled */
			ret = 0;
	}

	return ret;
}

static void __dma_i915_sw_fence_wake(struct dma_fence *dma,
				     struct dma_fence_cb *data)
{
	struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base);

	i915_sw_fence_set_error_once(cb->fence, dma->error);
	i915_sw_fence_complete(cb->fence);
}

int __i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
				    struct dma_fence *dma,
				    struct i915_sw_dma_fence_cb *cb)
{
	int ret;

	debug_fence_assert(fence);

	if (dma_fence_is_signaled(dma)) {
		i915_sw_fence_set_error_once(fence, dma->error);
		return 0;
	}

	cb->fence = fence;
	i915_sw_fence_await(fence);

	ret = 1;
	if (dma_fence_add_callback(dma, &cb->base, __dma_i915_sw_fence_wake)) {
		/* fence already signaled */
		__dma_i915_sw_fence_wake(dma, &cb->base);
		ret = 0;
	}

	return ret;
}

int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
				    struct dma_resv *resv,
				    const struct dma_fence_ops *exclude,
				    bool write,
				    unsigned long timeout,
				    gfp_t gfp)
{
	struct dma_fence *excl;
	int ret = 0, pending;

	debug_fence_assert(fence);
	might_sleep_if(gfpflags_allow_blocking(gfp));

	if (write) {
		struct dma_fence **shared;
		unsigned int count, i;

		ret = dma_resv_get_fences_rcu(resv, &excl, &count, &shared);
		if (ret)
			return ret;

		for (i = 0; i < count; i++) {
			if (shared[i]->ops == exclude)
				continue;

			pending = i915_sw_fence_await_dma_fence(fence,
								shared[i],
								timeout,
								gfp);
			if (pending < 0) {
				ret = pending;
				break;
			}

			ret |= pending;
		}

		for (i = 0; i < count; i++)
			dma_fence_put(shared[i]);
		kfree(shared);
	} else {
		excl = dma_resv_get_excl_rcu(resv);
	}

	if (ret >= 0 && excl && excl->ops != exclude) {
		pending = i915_sw_fence_await_dma_fence(fence,
							excl,
							timeout,
							gfp);
		if (pending < 0)
			ret = pending;
		else
			ret |= pending;
	}

	dma_fence_put(excl);

	return ret;
}

#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
#include "selftests/lib_sw_fence.c"
#include "selftests/i915_sw_fence.c"
#endif
Commit	Line	Data
	1	/*
	2	* SPDX-License-Identifier: MIT
	3	*
	4	* (C) Copyright 2016 Intel Corporation
	5	*/
	6
	7	#include <linux/slab.h>
	8	#include <linux/dma-fence.h>
	9	#include <linux/irq_work.h>
	10	#include <linux/dma-resv.h>
	11
	12	#include "i915_sw_fence.h"
	13	#include "i915_selftest.h"
	14
	15	#if IS_ENABLED(CONFIG_DRM_I915_DEBUG)
	16	#define I915_SW_FENCE_BUG_ON(expr) BUG_ON(expr)
	17	#else
	18	#define I915_SW_FENCE_BUG_ON(expr) BUILD_BUG_ON_INVALID(expr)
	19	#endif
	20
	21	static DEFINE_SPINLOCK(i915_sw_fence_lock);
	22
	23	#define WQ_FLAG_BITS \
	24	BITS_PER_TYPE(typeof_member(struct wait_queue_entry, flags))
	25
	26	/* after WQ_FLAG_* for safety */
	27	#define I915_SW_FENCE_FLAG_FENCE BIT(WQ_FLAG_BITS - 1)
	28	#define I915_SW_FENCE_FLAG_ALLOC BIT(WQ_FLAG_BITS - 2)
	29
	30	enum {
	31	DEBUG_FENCE_IDLE = 0,
	32	DEBUG_FENCE_NOTIFY,
	33	};
	34
	35	static void i915_sw_fence_debug_hint(void addr)
	36	{
	37	return (void )(((struct i915_sw_fence )addr)->flags & I915_SW_FENCE_MASK);
	38	}
	39
	40	#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
	41
	42	static const struct debug_obj_descr i915_sw_fence_debug_descr = {
	43	.name = "i915_sw_fence",
	44	.debug_hint = i915_sw_fence_debug_hint,
	45	};
	46
	47	static inline void debug_fence_init(struct i915_sw_fence *fence)
	48	{
	49	debug_object_init(fence, &i915_sw_fence_debug_descr);
	50	}
	51
	52	static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)
	53	{
	54	debug_object_init_on_stack(fence, &i915_sw_fence_debug_descr);
	55	}
	56
	57	static inline void debug_fence_activate(struct i915_sw_fence *fence)
	58	{
	59	debug_object_activate(fence, &i915_sw_fence_debug_descr);
	60	}
	61
	62	static inline void debug_fence_set_state(struct i915_sw_fence *fence,
	63	int old, int new)
	64	{
	65	debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new);
	66	}
	67
	68	static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
	69	{
	70	debug_object_deactivate(fence, &i915_sw_fence_debug_descr);
	71	}
	72
	73	static inline void debug_fence_destroy(struct i915_sw_fence *fence)
	74	{
	75	debug_object_destroy(fence, &i915_sw_fence_debug_descr);
	76	}
	77
	78	static inline void debug_fence_free(struct i915_sw_fence *fence)
	79	{
	80	debug_object_free(fence, &i915_sw_fence_debug_descr);
	81	smp_wmb(); /* flush the change in state before reallocation */
	82	}
	83
	84	static inline void debug_fence_assert(struct i915_sw_fence *fence)
	85	{
	86	debug_object_assert_init(fence, &i915_sw_fence_debug_descr);
	87	}
	88
	89	#else
	90
	91	static inline void debug_fence_init(struct i915_sw_fence *fence)
	92	{
	93	}
	94
	95	static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)
	96	{
	97	}
	98
	99	static inline void debug_fence_activate(struct i915_sw_fence *fence)
	100	{
	101	}
	102
	103	static inline void debug_fence_set_state(struct i915_sw_fence *fence,
	104	int old, int new)
	105	{
	106	}
	107
	108	static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
	109	{
	110	}
	111
	112	static inline void debug_fence_destroy(struct i915_sw_fence *fence)
	113	{
	114	}
	115
	116	static inline void debug_fence_free(struct i915_sw_fence *fence)
	117	{
	118	}
	119
	120	static inline void debug_fence_assert(struct i915_sw_fence *fence)
	121	{
	122	}
	123
	124	#endif
	125
	126	static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
	127	enum i915_sw_fence_notify state)
	128	{
	129	i915_sw_fence_notify_t fn;
	130
	131	fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK);
	132	return fn(fence, state);
	133	}
	134
	135	#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
	136	void i915_sw_fence_fini(struct i915_sw_fence *fence)
	137	{
	138	debug_fence_free(fence);
	139	}
	140	#endif
	141
	142	static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,
	143	struct list_head *continuation)
	144	{
	145	wait_queue_head_t *x = &fence->wait;
	146	wait_queue_entry_t pos, next;
	147	unsigned long flags;
	148
	149	debug_fence_deactivate(fence);
	150	atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */
	151
	152	/*
	153	* To prevent unbounded recursion as we traverse the graph of
	154	* i915_sw_fences, we move the entry list from this, the next ready
	155	* fence, to the tail of the original fence's entry list
	156	* (and so added to the list to be woken).
	157	*/
	158
	159	spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation);
	160	if (continuation) {
	161	list_for_each_entry_safe(pos, next, &x->head, entry) {
	162	if (pos->flags & I915_SW_FENCE_FLAG_FENCE)
	163	list_move_tail(&pos->entry, continuation);
	164	else
	165	pos->func(pos, TASK_NORMAL, 0, continuation);
	166	}
	167	} else {
	168	LIST_HEAD(extra);
	169
	170	do {
	171	list_for_each_entry_safe(pos, next, &x->head, entry) {
	172	int wake_flags;
	173
	174	wake_flags = 0;
	175	if (pos->flags & I915_SW_FENCE_FLAG_FENCE)
	176	wake_flags = fence->error;
	177
	178	pos->func(pos, TASK_NORMAL, wake_flags, &extra);
	179	}
	180
	181	if (list_empty(&extra))
	182	break;
	183
	184	list_splice_tail_init(&extra, &x->head);
	185	} while (1);
	186	}
	187	spin_unlock_irqrestore(&x->lock, flags);
	188
	189	debug_fence_assert(fence);
	190	}
	191
	192	static void __i915_sw_fence_complete(struct i915_sw_fence *fence,
	193	struct list_head *continuation)
	194	{
	195	debug_fence_assert(fence);
	196
	197	if (!atomic_dec_and_test(&fence->pending))
	198	return;
	199
	200	debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY);
	201
	202	if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)
	203	return;
	204
	205	debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE);
	206
	207	__i915_sw_fence_wake_up_all(fence, continuation);
	208
	209	debug_fence_destroy(fence);
	210	__i915_sw_fence_notify(fence, FENCE_FREE);
	211	}
	212
	213	void i915_sw_fence_complete(struct i915_sw_fence *fence)
	214	{
	215	debug_fence_assert(fence);
	216
	217	if (WARN_ON(i915_sw_fence_done(fence)))
	218	return;
	219
	220	__i915_sw_fence_complete(fence, NULL);
	221	}
	222
	223	bool i915_sw_fence_await(struct i915_sw_fence *fence)
	224	{
	225	int pending;
	226
	227	/*
	228	* It is only safe to add a new await to the fence while it has
	229	* not yet been signaled (i.e. there are still existing signalers).
	230	*/
	231	pending = atomic_read(&fence->pending);
	232	do {
	233	if (pending < 1)
	234	return false;
	235	} while (!atomic_try_cmpxchg(&fence->pending, &pending, pending + 1));
	236
	237	return true;
	238	}
	239
	240	void __i915_sw_fence_init(struct i915_sw_fence *fence,
	241	i915_sw_fence_notify_t fn,
	242	const char *name,
	243	struct lock_class_key *key)
	244	{
	245	BUG_ON(!fn \|\| (unsigned long)fn & ~I915_SW_FENCE_MASK);
	246
	247	__init_waitqueue_head(&fence->wait, name, key);
	248	fence->flags = (unsigned long)fn;
	249
	250	i915_sw_fence_reinit(fence);
	251	}
	252
	253	void i915_sw_fence_reinit(struct i915_sw_fence *fence)
	254	{
	255	debug_fence_init(fence);
	256
	257	atomic_set(&fence->pending, 1);
	258	fence->error = 0;
	259
	260	I915_SW_FENCE_BUG_ON(!fence->flags);
	261	I915_SW_FENCE_BUG_ON(!list_empty(&fence->wait.head));
	262	}
	263
	264	void i915_sw_fence_commit(struct i915_sw_fence *fence)
	265	{
	266	debug_fence_activate(fence);
	267	i915_sw_fence_complete(fence);
	268	}
	269
	270	static int i915_sw_fence_wake(wait_queue_entry_t wq, unsigned mode, int flags, void key)
	271	{
	272	i915_sw_fence_set_error_once(wq->private, flags);
	273
	274	list_del(&wq->entry);
	275	__i915_sw_fence_complete(wq->private, key);
	276
	277	if (wq->flags & I915_SW_FENCE_FLAG_ALLOC)
	278	kfree(wq);
	279	return 0;
	280	}
	281
	282	static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
	283	const struct i915_sw_fence * const signaler)
	284	{
	285	wait_queue_entry_t *wq;
	286
	287	if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
	288	return false;
	289
	290	if (fence == signaler)
	291	return true;
	292
	293	list_for_each_entry(wq, &fence->wait.head, entry) {
	294	if (wq->func != i915_sw_fence_wake)
	295	continue;
	296
	297	if (__i915_sw_fence_check_if_after(wq->private, signaler))
	298	return true;
	299	}
	300
	301	return false;
	302	}
	303
	304	static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)
	305	{
	306	wait_queue_entry_t *wq;
	307
	308	if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
	309	return;
	310
	311	list_for_each_entry(wq, &fence->wait.head, entry) {
	312	if (wq->func != i915_sw_fence_wake)
	313	continue;
	314
	315	__i915_sw_fence_clear_checked_bit(wq->private);
	316	}
	317	}
	318
	319	static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
	320	const struct i915_sw_fence * const signaler)
	321	{
	322	unsigned long flags;
	323	bool err;
	324
	325	if (!IS_ENABLED(CONFIG_DRM_I915_SW_FENCE_CHECK_DAG))
	326	return false;
	327
	328	spin_lock_irqsave(&i915_sw_fence_lock, flags);
	329	err = __i915_sw_fence_check_if_after(fence, signaler);
	330	__i915_sw_fence_clear_checked_bit(fence);
	331	spin_unlock_irqrestore(&i915_sw_fence_lock, flags);
	332
	333	return err;
	334	}
	335
	336	static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
	337	struct i915_sw_fence *signaler,
	338	wait_queue_entry_t *wq, gfp_t gfp)
	339	{
	340	unsigned int pending;
	341	unsigned long flags;
	342
	343	debug_fence_assert(fence);
	344	might_sleep_if(gfpflags_allow_blocking(gfp));
	345
	346	if (i915_sw_fence_done(signaler)) {
	347	i915_sw_fence_set_error_once(fence, signaler->error);
	348	return 0;
	349	}
	350
	351	debug_fence_assert(signaler);
	352
	353	/* The dependency graph must be acyclic. */
	354	if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))
	355	return -EINVAL;
	356
	357	pending = I915_SW_FENCE_FLAG_FENCE;
	358	if (!wq) {
	359	wq = kmalloc(sizeof(*wq), gfp);
	360	if (!wq) {
	361	if (!gfpflags_allow_blocking(gfp))
	362	return -ENOMEM;
	363
	364	i915_sw_fence_wait(signaler);
	365	i915_sw_fence_set_error_once(fence, signaler->error);
	366	return 0;
	367	}
	368
	369	pending \|= I915_SW_FENCE_FLAG_ALLOC;
	370	}
	371
	372	INIT_LIST_HEAD(&wq->entry);
	373	wq->flags = pending;
	374	wq->func = i915_sw_fence_wake;
	375	wq->private = fence;
	376
	377	i915_sw_fence_await(fence);
	378
	379	spin_lock_irqsave(&signaler->wait.lock, flags);
	380	if (likely(!i915_sw_fence_done(signaler))) {
	381	__add_wait_queue_entry_tail(&signaler->wait, wq);
	382	pending = 1;
	383	} else {
	384	i915_sw_fence_wake(wq, 0, signaler->error, NULL);
	385	pending = 0;
	386	}
	387	spin_unlock_irqrestore(&signaler->wait.lock, flags);
	388
	389	return pending;
	390	}
	391
	392	int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
	393	struct i915_sw_fence *signaler,
	394	wait_queue_entry_t *wq)
	395	{
	396	return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);
	397	}
	398
	399	int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
	400	struct i915_sw_fence *signaler,
	401	gfp_t gfp)
	402	{
	403	return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);
	404	}
	405
	406	struct i915_sw_dma_fence_cb_timer {
	407	struct i915_sw_dma_fence_cb base;
	408	struct dma_fence *dma;
	409	struct timer_list timer;
	410	struct irq_work work;
	411	struct rcu_head rcu;
	412	};
	413
	414	static void dma_i915_sw_fence_wake(struct dma_fence *dma,
	415	struct dma_fence_cb *data)
	416	{
	417	struct i915_sw_dma_fence_cb cb = container_of(data, typeof(cb), base);
	418
	419	i915_sw_fence_set_error_once(cb->fence, dma->error);
	420	i915_sw_fence_complete(cb->fence);
	421	kfree(cb);
	422	}
	423
	424	static void timer_i915_sw_fence_wake(struct timer_list *t)
	425	{
	426	struct i915_sw_dma_fence_cb_timer *cb = from_timer(cb, t, timer);
	427	struct i915_sw_fence *fence;
	428
	429	fence = xchg(&cb->base.fence, NULL);
	430	if (!fence)
	431	return;
	432
	433	pr_notice("Asynchronous wait on fence %s:%s:%llx timed out (hint:%ps)\n",
	434	cb->dma->ops->get_driver_name(cb->dma),
	435	cb->dma->ops->get_timeline_name(cb->dma),
	436	cb->dma->seqno,
	437	i915_sw_fence_debug_hint(fence));
	438
	439	i915_sw_fence_set_error_once(fence, -ETIMEDOUT);
	440	i915_sw_fence_complete(fence);
	441	}
	442
	443	static void dma_i915_sw_fence_wake_timer(struct dma_fence *dma,
	444	struct dma_fence_cb *data)
	445	{
	446	struct i915_sw_dma_fence_cb_timer *cb =
	447	container_of(data, typeof(*cb), base.base);
	448	struct i915_sw_fence *fence;
	449
	450	fence = xchg(&cb->base.fence, NULL);
	451	if (fence) {
	452	i915_sw_fence_set_error_once(fence, dma->error);
	453	i915_sw_fence_complete(fence);
	454	}
	455
	456	irq_work_queue(&cb->work);
	457	}
	458
	459	static void irq_i915_sw_fence_work(struct irq_work *wrk)
	460	{
	461	struct i915_sw_dma_fence_cb_timer *cb =
	462	container_of(wrk, typeof(*cb), work);
	463
	464	del_timer_sync(&cb->timer);
	465	dma_fence_put(cb->dma);
	466
	467	kfree_rcu(cb, rcu);
	468	}
	469
	470	int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
	471	struct dma_fence *dma,
	472	unsigned long timeout,
	473	gfp_t gfp)
	474	{
	475	struct i915_sw_dma_fence_cb *cb;
	476	dma_fence_func_t func;
	477	int ret;
	478
	479	debug_fence_assert(fence);
	480	might_sleep_if(gfpflags_allow_blocking(gfp));
	481
	482	if (dma_fence_is_signaled(dma)) {
	483	i915_sw_fence_set_error_once(fence, dma->error);
	484	return 0;
	485	}
	486
	487	cb = kmalloc(timeout ?
	488	sizeof(struct i915_sw_dma_fence_cb_timer) :
	489	sizeof(struct i915_sw_dma_fence_cb),
	490	gfp);
	491	if (!cb) {
	492	if (!gfpflags_allow_blocking(gfp))
	493	return -ENOMEM;
	494
	495	ret = dma_fence_wait(dma, false);
	496	if (ret)
	497	return ret;
	498
	499	i915_sw_fence_set_error_once(fence, dma->error);
	500	return 0;
	501	}
	502
	503	cb->fence = fence;
	504	i915_sw_fence_await(fence);
	505
	506	func = dma_i915_sw_fence_wake;
	507	if (timeout) {
	508	struct i915_sw_dma_fence_cb_timer *timer =
	509	container_of(cb, typeof(*timer), base);
	510
	511	timer->dma = dma_fence_get(dma);
	512	init_irq_work(&timer->work, irq_i915_sw_fence_work);
	513
	514	timer_setup(&timer->timer,
	515	timer_i915_sw_fence_wake, TIMER_IRQSAFE);
	516	mod_timer(&timer->timer, round_jiffies_up(jiffies + timeout));
	517
	518	func = dma_i915_sw_fence_wake_timer;
	519	}
	520
	521	ret = dma_fence_add_callback(dma, &cb->base, func);
	522	if (ret == 0) {
	523	ret = 1;
	524	} else {
	525	func(dma, &cb->base);
	526	if (ret == -ENOENT) /* fence already signaled */
	527	ret = 0;
	528	}
	529
	530	return ret;
	531	}
	532
	533	static void __dma_i915_sw_fence_wake(struct dma_fence *dma,
	534	struct dma_fence_cb *data)
	535	{
	536	struct i915_sw_dma_fence_cb cb = container_of(data, typeof(cb), base);
	537
	538	i915_sw_fence_set_error_once(cb->fence, dma->error);
	539	i915_sw_fence_complete(cb->fence);
	540	}
	541
	542	int __i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
	543	struct dma_fence *dma,
	544	struct i915_sw_dma_fence_cb *cb)
	545	{
	546	int ret;
	547
	548	debug_fence_assert(fence);
	549
	550	if (dma_fence_is_signaled(dma)) {
	551	i915_sw_fence_set_error_once(fence, dma->error);
	552	return 0;
	553	}
	554
	555	cb->fence = fence;
	556	i915_sw_fence_await(fence);
	557
	558	ret = 1;
	559	if (dma_fence_add_callback(dma, &cb->base, __dma_i915_sw_fence_wake)) {
	560	/* fence already signaled */
	561	__dma_i915_sw_fence_wake(dma, &cb->base);
	562	ret = 0;
	563	}
	564
	565	return ret;
	566	}
	567
	568	int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
	569	struct dma_resv *resv,
	570	const struct dma_fence_ops *exclude,
	571	bool write,
	572	unsigned long timeout,
	573	gfp_t gfp)
	574	{
	575	struct dma_fence *excl;
	576	int ret = 0, pending;
	577
	578	debug_fence_assert(fence);
	579	might_sleep_if(gfpflags_allow_blocking(gfp));
	580
	581	if (write) {
	582	struct dma_fence **shared;
	583	unsigned int count, i;
	584
	585	ret = dma_resv_get_fences_rcu(resv, &excl, &count, &shared);
	586	if (ret)
	587	return ret;
	588
	589	for (i = 0; i < count; i++) {
	590	if (shared[i]->ops == exclude)
	591	continue;
	592
	593	pending = i915_sw_fence_await_dma_fence(fence,
	594	shared[i],
	595	timeout,
	596	gfp);
	597	if (pending < 0) {
	598	ret = pending;
	599	break;
	600	}
	601
	602	ret \|= pending;
	603	}
	604
	605	for (i = 0; i < count; i++)
	606	dma_fence_put(shared[i]);
	607	kfree(shared);
	608	} else {
	609	excl = dma_resv_get_excl_rcu(resv);
	610	}
	611
	612	if (ret >= 0 && excl && excl->ops != exclude) {
	613	pending = i915_sw_fence_await_dma_fence(fence,
	614	excl,
	615	timeout,
	616	gfp);
	617	if (pending < 0)
	618	ret = pending;
	619	else
	620	ret \|= pending;
	621	}
	622
	623	dma_fence_put(excl);
	624
	625	return ret;
	626	}
	627
	628	#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
	629	#include "selftests/lib_sw_fence.c"
	630	#include "selftests/i915_sw_fence.c"
	631	#endif