[linux-2.6-block.git] / drivers / gpu / drm / i915 / i915_sw_fence.c

/*
 * (C) Copyright 2016 Intel Corporation
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; version 2
 * of the License.
 */

#include <linux/slab.h>
#include <linux/dma-fence.h>
#include <linux/irq_work.h>
#include <linux/reservation.h>

#include "i915_sw_fence.h"
#include "i915_selftest.h"

#define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */

static DEFINE_SPINLOCK(i915_sw_fence_lock);

enum {
	DEBUG_FENCE_IDLE = 0,
	DEBUG_FENCE_NOTIFY,
};

#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS

static void *i915_sw_fence_debug_hint(void *addr)
{
	return (void *)(((struct i915_sw_fence *)addr)->flags & I915_SW_FENCE_MASK);
}

static struct debug_obj_descr i915_sw_fence_debug_descr = {
	.name = "i915_sw_fence",
	.debug_hint = i915_sw_fence_debug_hint,
};

static inline void debug_fence_init(struct i915_sw_fence *fence)
{
	debug_object_init(fence, &i915_sw_fence_debug_descr);
}

static inline void debug_fence_activate(struct i915_sw_fence *fence)
{
	debug_object_activate(fence, &i915_sw_fence_debug_descr);
}

static inline void debug_fence_set_state(struct i915_sw_fence *fence,
					 int old, int new)
{
	debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new);
}

static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
{
	debug_object_deactivate(fence, &i915_sw_fence_debug_descr);
}

static inline void debug_fence_destroy(struct i915_sw_fence *fence)
{
	debug_object_destroy(fence, &i915_sw_fence_debug_descr);
}

static inline void debug_fence_free(struct i915_sw_fence *fence)
{
	debug_object_free(fence, &i915_sw_fence_debug_descr);
	smp_wmb(); /* flush the change in state before reallocation */
}

static inline void debug_fence_assert(struct i915_sw_fence *fence)
{
	debug_object_assert_init(fence, &i915_sw_fence_debug_descr);
}

#else

static inline void debug_fence_init(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_activate(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_set_state(struct i915_sw_fence *fence,
					 int old, int new)
{
}

static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_destroy(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_free(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_assert(struct i915_sw_fence *fence)
{
}

#endif

static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
				  enum i915_sw_fence_notify state)
{
	i915_sw_fence_notify_t fn;

	fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK);
	return fn(fence, state);
}

#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
void i915_sw_fence_fini(struct i915_sw_fence *fence)
{
	debug_fence_free(fence);
}
#endif

static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,
					struct list_head *continuation)
{
	wait_queue_head_t *x = &fence->wait;
	wait_queue_entry_t *pos, *next;
	unsigned long flags;

	debug_fence_deactivate(fence);
	atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */

	/*
	 * To prevent unbounded recursion as we traverse the graph of
	 * i915_sw_fences, we move the entry list from this, the next ready
	 * fence, to the tail of the original fence's entry list
	 * (and so added to the list to be woken).
	 */

	spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation);
	if (continuation) {
		list_for_each_entry_safe(pos, next, &x->head, entry) {
			if (pos->func == autoremove_wake_function)
				pos->func(pos, TASK_NORMAL, 0, continuation);
			else
				list_move_tail(&pos->entry, continuation);
		}
	} else {
		LIST_HEAD(extra);

		do {
			list_for_each_entry_safe(pos, next, &x->head, entry)
				pos->func(pos, TASK_NORMAL, 0, &extra);

			if (list_empty(&extra))
				break;

			list_splice_tail_init(&extra, &x->head);
		} while (1);
	}
	spin_unlock_irqrestore(&x->lock, flags);

	debug_fence_assert(fence);
}

static void __i915_sw_fence_complete(struct i915_sw_fence *fence,
				     struct list_head *continuation)
{
	debug_fence_assert(fence);

	if (!atomic_dec_and_test(&fence->pending))
		return;

	debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY);

	if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)
		return;

	debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE);

	__i915_sw_fence_wake_up_all(fence, continuation);

	debug_fence_destroy(fence);
	__i915_sw_fence_notify(fence, FENCE_FREE);
}

static void i915_sw_fence_complete(struct i915_sw_fence *fence)
{
	debug_fence_assert(fence);

	if (WARN_ON(i915_sw_fence_done(fence)))
		return;

	__i915_sw_fence_complete(fence, NULL);
}

static void i915_sw_fence_await(struct i915_sw_fence *fence)
{
	debug_fence_assert(fence);
	WARN_ON(atomic_inc_return(&fence->pending) <= 1);
}

void __i915_sw_fence_init(struct i915_sw_fence *fence,
			  i915_sw_fence_notify_t fn,
			  const char *name,
			  struct lock_class_key *key)
{
	BUG_ON(!fn || (unsigned long)fn & ~I915_SW_FENCE_MASK);

	debug_fence_init(fence);

	__init_waitqueue_head(&fence->wait, name, key);
	atomic_set(&fence->pending, 1);
	fence->flags = (unsigned long)fn;
}

void i915_sw_fence_commit(struct i915_sw_fence *fence)
{
	debug_fence_activate(fence);
	i915_sw_fence_complete(fence);
}

static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key)
{
	list_del(&wq->entry);
	__i915_sw_fence_complete(wq->private, key);

	if (wq->flags & I915_SW_FENCE_FLAG_ALLOC)
		kfree(wq);
	return 0;
}

static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
				    const struct i915_sw_fence * const signaler)
{
	wait_queue_entry_t *wq;

	if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
		return false;

	if (fence == signaler)
		return true;

	list_for_each_entry(wq, &fence->wait.head, entry) {
		if (wq->func != i915_sw_fence_wake)
			continue;

		if (__i915_sw_fence_check_if_after(wq->private, signaler))
			return true;
	}

	return false;
}

static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)
{
	wait_queue_entry_t *wq;

	if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
		return;

	list_for_each_entry(wq, &fence->wait.head, entry) {
		if (wq->func != i915_sw_fence_wake)
			continue;

		__i915_sw_fence_clear_checked_bit(wq->private);
	}
}

static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
				  const struct i915_sw_fence * const signaler)
{
	unsigned long flags;
	bool err;

	if (!IS_ENABLED(CONFIG_DRM_I915_SW_FENCE_CHECK_DAG))
		return false;

	spin_lock_irqsave(&i915_sw_fence_lock, flags);
	err = __i915_sw_fence_check_if_after(fence, signaler);
	__i915_sw_fence_clear_checked_bit(fence);
	spin_unlock_irqrestore(&i915_sw_fence_lock, flags);

	return err;
}

static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
					  struct i915_sw_fence *signaler,
					  wait_queue_entry_t *wq, gfp_t gfp)
{
	unsigned long flags;
	int pending;

	debug_fence_assert(fence);

	if (i915_sw_fence_done(signaler))
		return 0;

	debug_fence_assert(signaler);

	/* The dependency graph must be acyclic. */
	if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))
		return -EINVAL;

	pending = 0;
	if (!wq) {
		wq = kmalloc(sizeof(*wq), gfp);
		if (!wq) {
			if (!gfpflags_allow_blocking(gfp))
				return -ENOMEM;

			i915_sw_fence_wait(signaler);
			return 0;
		}

		pending |= I915_SW_FENCE_FLAG_ALLOC;
	}

	INIT_LIST_HEAD(&wq->entry);
	wq->flags = pending;
	wq->func = i915_sw_fence_wake;
	wq->private = fence;

	i915_sw_fence_await(fence);

	spin_lock_irqsave(&signaler->wait.lock, flags);
	if (likely(!i915_sw_fence_done(signaler))) {
		__add_wait_queue_entry_tail(&signaler->wait, wq);
		pending = 1;
	} else {
		i915_sw_fence_wake(wq, 0, 0, NULL);
		pending = 0;
	}
	spin_unlock_irqrestore(&signaler->wait.lock, flags);

	return pending;
}

int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
				 struct i915_sw_fence *signaler,
				 wait_queue_entry_t *wq)
{
	return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);
}

int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
				     struct i915_sw_fence *signaler,
				     gfp_t gfp)
{
	return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);
}

struct i915_sw_dma_fence_cb {
	struct dma_fence_cb base;
	struct i915_sw_fence *fence;
	struct dma_fence *dma;
	struct timer_list timer;
	struct irq_work work;
};

static void timer_i915_sw_fence_wake(unsigned long data)
{
	struct i915_sw_dma_fence_cb *cb = (struct i915_sw_dma_fence_cb *)data;
	struct i915_sw_fence *fence;

	fence = xchg(&cb->fence, NULL);
	if (!fence)
		return;

	pr_warn("asynchronous wait on fence %s:%s:%x timed out\n",
		cb->dma->ops->get_driver_name(cb->dma),
		cb->dma->ops->get_timeline_name(cb->dma),
		cb->dma->seqno);

	i915_sw_fence_complete(fence);
}

static void dma_i915_sw_fence_wake(struct dma_fence *dma,
				   struct dma_fence_cb *data)
{
	struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base);
	struct i915_sw_fence *fence;

	fence = xchg(&cb->fence, NULL);
	if (fence)
		i915_sw_fence_complete(fence);

	irq_work_queue(&cb->work);
}

static void irq_i915_sw_fence_work(struct irq_work *wrk)
{
	struct i915_sw_dma_fence_cb *cb = container_of(wrk, typeof(*cb), work);

	del_timer_sync(&cb->timer);
	dma_fence_put(cb->dma);

	kfree(cb);
}

int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
				  struct dma_fence *dma,
				  unsigned long timeout,
				  gfp_t gfp)
{
	struct i915_sw_dma_fence_cb *cb;
	int ret;

	debug_fence_assert(fence);

	if (dma_fence_is_signaled(dma))
		return 0;

	cb = kmalloc(sizeof(*cb), gfp);
	if (!cb) {
		if (!gfpflags_allow_blocking(gfp))
			return -ENOMEM;

		return dma_fence_wait(dma, false);
	}

	cb->fence = fence;
	i915_sw_fence_await(fence);

	cb->dma = NULL;
	__setup_timer(&cb->timer,
		      timer_i915_sw_fence_wake, (unsigned long)cb,
		      TIMER_IRQSAFE);
	init_irq_work(&cb->work, irq_i915_sw_fence_work);
	if (timeout) {
		cb->dma = dma_fence_get(dma);
		mod_timer(&cb->timer, round_jiffies_up(jiffies + timeout));
	}

	ret = dma_fence_add_callback(dma, &cb->base, dma_i915_sw_fence_wake);
	if (ret == 0) {
		ret = 1;
	} else {
		dma_i915_sw_fence_wake(dma, &cb->base);
		if (ret == -ENOENT) /* fence already signaled */
			ret = 0;
	}

	return ret;
}

int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
				    struct reservation_object *resv,
				    const struct dma_fence_ops *exclude,
				    bool write,
				    unsigned long timeout,
				    gfp_t gfp)
{
	struct dma_fence *excl;
	int ret = 0, pending;

	debug_fence_assert(fence);

	if (write) {
		struct dma_fence **shared;
		unsigned int count, i;

		ret = reservation_object_get_fences_rcu(resv,
							&excl, &count, &shared);
		if (ret)
			return ret;

		for (i = 0; i < count; i++) {
			if (shared[i]->ops == exclude)
				continue;

			pending = i915_sw_fence_await_dma_fence(fence,
								shared[i],
								timeout,
								gfp);
			if (pending < 0) {
				ret = pending;
				break;
			}

			ret |= pending;
		}

		for (i = 0; i < count; i++)
			dma_fence_put(shared[i]);
		kfree(shared);
	} else {
		excl = reservation_object_get_excl_rcu(resv);
	}

	if (ret >= 0 && excl && excl->ops != exclude) {
		pending = i915_sw_fence_await_dma_fence(fence,
							excl,
							timeout,
							gfp);
		if (pending < 0)
			ret = pending;
		else
			ret |= pending;
	}

	dma_fence_put(excl);

	return ret;
}

#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
#include "selftests/i915_sw_fence.c"
#endif
Commit	Line	Data
e68a139f CW	1	/*
	2	* (C) Copyright 2016 Intel Corporation
	3	*
	4	* This program is free software; you can redistribute it and/or
	5	* modify it under the terms of the GNU General Public License
	6	* as published by the Free Software Foundation; version 2
	7	* of the License.
	8	*/
	9
	10	#include <linux/slab.h>
f54d1867	11	#include <linux/dma-fence.h>
81c0ed21	12	#include <linux/irq_work.h>
e68a139f CW	13	#include <linux/reservation.h>
	14
	15	#include "i915_sw_fence.h"
47624cc3	16	#include "i915_selftest.h"
e68a139f	17
7e941861 CW	18	#define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */
7e941861 CW	19
e68a139f CW	20	static DEFINE_SPINLOCK(i915_sw_fence_lock);
e68a139f CW	21
fc158405 CW	22	enum {
	23	DEBUG_FENCE_IDLE = 0,
	24	DEBUG_FENCE_NOTIFY,
	25	};
	26
	27	#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
	28
	29	static void i915_sw_fence_debug_hint(void addr)
	30	{
	31	return (void )(((struct i915_sw_fence )addr)->flags & I915_SW_FENCE_MASK);
	32	}
	33
	34	static struct debug_obj_descr i915_sw_fence_debug_descr = {
	35	.name = "i915_sw_fence",
	36	.debug_hint = i915_sw_fence_debug_hint,
	37	};
	38
	39	static inline void debug_fence_init(struct i915_sw_fence *fence)
	40	{
	41	debug_object_init(fence, &i915_sw_fence_debug_descr);
	42	}
	43
	44	static inline void debug_fence_activate(struct i915_sw_fence *fence)
	45	{
	46	debug_object_activate(fence, &i915_sw_fence_debug_descr);
	47	}
	48
	49	static inline void debug_fence_set_state(struct i915_sw_fence *fence,
	50	int old, int new)
	51	{
	52	debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new);
	53	}
	54
	55	static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
	56	{
	57	debug_object_deactivate(fence, &i915_sw_fence_debug_descr);
	58	}
	59
	60	static inline void debug_fence_destroy(struct i915_sw_fence *fence)
	61	{
	62	debug_object_destroy(fence, &i915_sw_fence_debug_descr);
	63	}
	64
	65	static inline void debug_fence_free(struct i915_sw_fence *fence)
	66	{
	67	debug_object_free(fence, &i915_sw_fence_debug_descr);
6f13f29f	68	smp_wmb(); /* flush the change in state before reallocation */
fc158405 CW	69	}
	70
	71	static inline void debug_fence_assert(struct i915_sw_fence *fence)
	72	{
	73	debug_object_assert_init(fence, &i915_sw_fence_debug_descr);
	74	}
	75
	76	#else
	77
	78	static inline void debug_fence_init(struct i915_sw_fence *fence)
	79	{
	80	}
	81
	82	static inline void debug_fence_activate(struct i915_sw_fence *fence)
	83	{
	84	}
	85
	86	static inline void debug_fence_set_state(struct i915_sw_fence *fence,
	87	int old, int new)
	88	{
	89	}
	90
	91	static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
	92	{
	93	}
	94
	95	static inline void debug_fence_destroy(struct i915_sw_fence *fence)
	96	{
	97	}
	98
	99	static inline void debug_fence_free(struct i915_sw_fence *fence)
	100	{
	101	}
	102
	103	static inline void debug_fence_assert(struct i915_sw_fence *fence)
	104	{
	105	}
	106
	107	#endif
	108
e68a139f CW	109	static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
	110	enum i915_sw_fence_notify state)
	111	{
	112	i915_sw_fence_notify_t fn;
	113
	114	fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK);
	115	return fn(fence, state);
	116	}
	117
fc158405 CW	118	#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
	119	void i915_sw_fence_fini(struct i915_sw_fence *fence)
	120	{
	121	debug_fence_free(fence);
	122	}
	123	#endif
	124
e68a139f CW	125	static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,
	126	struct list_head *continuation)
	127	{
	128	wait_queue_head_t *x = &fence->wait;
ac6424b9	129	wait_queue_entry_t pos, next;
e68a139f CW	130	unsigned long flags;
e68a139f CW	131
fc158405	132	debug_fence_deactivate(fence);
e68a139f CW	133	atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */
	134
	135	/*
	136	* To prevent unbounded recursion as we traverse the graph of
2055da97 IM	137	* i915_sw_fences, we move the entry list from this, the next ready
2055da97 IM	138	* fence, to the tail of the original fence's entry list
e68a139f CW	139	* (and so added to the list to be woken).
	140	*/
	141
	142	spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation);
	143	if (continuation) {
2055da97	144	list_for_each_entry_safe(pos, next, &x->head, entry) {
e68a139f CW	145	if (pos->func == autoremove_wake_function)
	146	pos->func(pos, TASK_NORMAL, 0, continuation);
	147	else
2055da97	148	list_move_tail(&pos->entry, continuation);
e68a139f CW	149	}
	150	} else {
	151	LIST_HEAD(extra);
	152
	153	do {
2055da97	154	list_for_each_entry_safe(pos, next, &x->head, entry)
e68a139f CW	155	pos->func(pos, TASK_NORMAL, 0, &extra);
	156
	157	if (list_empty(&extra))
	158	break;
	159
2055da97	160	list_splice_tail_init(&extra, &x->head);
e68a139f CW	161	} while (1);
	162	}
	163	spin_unlock_irqrestore(&x->lock, flags);
fc158405 CW	164
fc158405 CW	165	debug_fence_assert(fence);
e68a139f CW	166	}
	167
	168	static void __i915_sw_fence_complete(struct i915_sw_fence *fence,
	169	struct list_head *continuation)
	170	{
fc158405 CW	171	debug_fence_assert(fence);
fc158405 CW	172
e68a139f CW	173	if (!atomic_dec_and_test(&fence->pending))
	174	return;
	175
fc158405 CW	176	debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY);
fc158405 CW	177
9310cb7f	178	if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)
e68a139f CW	179	return;
e68a139f CW	180
fc158405 CW	181	debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE);
fc158405 CW	182
e68a139f	183	__i915_sw_fence_wake_up_all(fence, continuation);
9310cb7f CW	184
	185	debug_fence_destroy(fence);
	186	__i915_sw_fence_notify(fence, FENCE_FREE);
e68a139f CW	187	}
	188
	189	static void i915_sw_fence_complete(struct i915_sw_fence *fence)
	190	{
fc158405 CW	191	debug_fence_assert(fence);
fc158405 CW	192
e68a139f CW	193	if (WARN_ON(i915_sw_fence_done(fence)))
	194	return;
	195
	196	__i915_sw_fence_complete(fence, NULL);
	197	}
	198
	199	static void i915_sw_fence_await(struct i915_sw_fence *fence)
	200	{
fc158405	201	debug_fence_assert(fence);
e68a139f CW	202	WARN_ON(atomic_inc_return(&fence->pending) <= 1);
	203	}
	204
556b7487 CW	205	void __i915_sw_fence_init(struct i915_sw_fence *fence,
	206	i915_sw_fence_notify_t fn,
	207	const char *name,
	208	struct lock_class_key *key)
e68a139f	209	{
9310cb7f	210	BUG_ON(!fn \|\| (unsigned long)fn & ~I915_SW_FENCE_MASK);
e68a139f	211
fc158405 CW	212	debug_fence_init(fence);
fc158405 CW	213
556b7487	214	__init_waitqueue_head(&fence->wait, name, key);
e68a139f CW	215	atomic_set(&fence->pending, 1);
	216	fence->flags = (unsigned long)fn;
	217	}
	218
fc158405 CW	219	void i915_sw_fence_commit(struct i915_sw_fence *fence)
	220	{
	221	debug_fence_activate(fence);
9310cb7f	222	i915_sw_fence_complete(fence);
fc158405 CW	223	}
fc158405 CW	224
ac6424b9	225	static int i915_sw_fence_wake(wait_queue_entry_t wq, unsigned mode, int flags, void key)
e68a139f	226	{
2055da97	227	list_del(&wq->entry);
e68a139f	228	__i915_sw_fence_complete(wq->private, key);
9310cb7f	229
7e941861 CW	230	if (wq->flags & I915_SW_FENCE_FLAG_ALLOC)
7e941861 CW	231	kfree(wq);
e68a139f CW	232	return 0;
	233	}
	234
	235	static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
	236	const struct i915_sw_fence * const signaler)
	237	{
ac6424b9	238	wait_queue_entry_t *wq;
e68a139f CW	239
	240	if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
	241	return false;
	242
	243	if (fence == signaler)
	244	return true;
	245
2055da97	246	list_for_each_entry(wq, &fence->wait.head, entry) {
e68a139f CW	247	if (wq->func != i915_sw_fence_wake)
	248	continue;
	249
	250	if (__i915_sw_fence_check_if_after(wq->private, signaler))
	251	return true;
	252	}
	253
	254	return false;
	255	}
	256
	257	static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)
	258	{
ac6424b9	259	wait_queue_entry_t *wq;
e68a139f CW	260
	261	if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
	262	return;
	263
2055da97	264	list_for_each_entry(wq, &fence->wait.head, entry) {
e68a139f CW	265	if (wq->func != i915_sw_fence_wake)
	266	continue;
	267
	268	__i915_sw_fence_clear_checked_bit(wq->private);
	269	}
	270	}
	271
	272	static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
	273	const struct i915_sw_fence * const signaler)
	274	{
	275	unsigned long flags;
	276	bool err;
	277
47624cc3	278	if (!IS_ENABLED(CONFIG_DRM_I915_SW_FENCE_CHECK_DAG))
e68a139f CW	279	return false;
	280
	281	spin_lock_irqsave(&i915_sw_fence_lock, flags);
	282	err = __i915_sw_fence_check_if_after(fence, signaler);
	283	__i915_sw_fence_clear_checked_bit(fence);
	284	spin_unlock_irqrestore(&i915_sw_fence_lock, flags);
	285
	286	return err;
	287	}
	288
7e941861 CW	289	static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
7e941861 CW	290	struct i915_sw_fence *signaler,
ac6424b9	291	wait_queue_entry_t *wq, gfp_t gfp)
e68a139f CW	292	{
	293	unsigned long flags;
	294	int pending;
	295
fc158405 CW	296	debug_fence_assert(fence);
fc158405 CW	297
e68a139f CW	298	if (i915_sw_fence_done(signaler))
	299	return 0;
	300
fc158405 CW	301	debug_fence_assert(signaler);
fc158405 CW	302
e68a139f CW	303	/* The dependency graph must be acyclic. */
	304	if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))
	305	return -EINVAL;
	306
7e941861 CW	307	pending = 0;
	308	if (!wq) {
	309	wq = kmalloc(sizeof(*wq), gfp);
	310	if (!wq) {
	311	if (!gfpflags_allow_blocking(gfp))
	312	return -ENOMEM;
	313
	314	i915_sw_fence_wait(signaler);
	315	return 0;
	316	}
	317
	318	pending \|= I915_SW_FENCE_FLAG_ALLOC;
	319	}
	320
2055da97	321	INIT_LIST_HEAD(&wq->entry);
7e941861	322	wq->flags = pending;
e68a139f	323	wq->func = i915_sw_fence_wake;
9310cb7f	324	wq->private = fence;
e68a139f CW	325
	326	i915_sw_fence_await(fence);
	327
	328	spin_lock_irqsave(&signaler->wait.lock, flags);
	329	if (likely(!i915_sw_fence_done(signaler))) {
ac6424b9	330	__add_wait_queue_entry_tail(&signaler->wait, wq);
e68a139f CW	331	pending = 1;
	332	} else {
	333	i915_sw_fence_wake(wq, 0, 0, NULL);
	334	pending = 0;
	335	}
	336	spin_unlock_irqrestore(&signaler->wait.lock, flags);
	337
	338	return pending;
	339	}
	340
7e941861 CW	341	int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
7e941861 CW	342	struct i915_sw_fence *signaler,
ac6424b9	343	wait_queue_entry_t *wq)
7e941861 CW	344	{
	345	return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);
	346	}
	347
	348	int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
	349	struct i915_sw_fence *signaler,
	350	gfp_t gfp)
	351	{
	352	return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);
	353	}
	354
f54d1867 CW	355	struct i915_sw_dma_fence_cb {
f54d1867 CW	356	struct dma_fence_cb base;
e68a139f	357	struct i915_sw_fence *fence;
f54d1867	358	struct dma_fence *dma;
e68a139f	359	struct timer_list timer;
81c0ed21	360	struct irq_work work;
e68a139f CW	361	};
	362
	363	static void timer_i915_sw_fence_wake(unsigned long data)
	364	{
f54d1867	365	struct i915_sw_dma_fence_cb cb = (struct i915_sw_dma_fence_cb )data;
81c0ed21 CW	366	struct i915_sw_fence *fence;
	367
	368	fence = xchg(&cb->fence, NULL);
	369	if (!fence)
	370	return;
e68a139f	371
8dfe162a JP	372	pr_warn("asynchronous wait on fence %s:%s:%x timed out\n",
	373	cb->dma->ops->get_driver_name(cb->dma),
	374	cb->dma->ops->get_timeline_name(cb->dma),
	375	cb->dma->seqno);
e68a139f	376
81c0ed21	377	i915_sw_fence_complete(fence);
e68a139f CW	378	}
e68a139f CW	379
f54d1867 CW	380	static void dma_i915_sw_fence_wake(struct dma_fence *dma,
f54d1867 CW	381	struct dma_fence_cb *data)
e68a139f	382	{
f54d1867	383	struct i915_sw_dma_fence_cb cb = container_of(data, typeof(cb), base);
81c0ed21 CW	384	struct i915_sw_fence *fence;
	385
	386	fence = xchg(&cb->fence, NULL);
	387	if (fence)
	388	i915_sw_fence_complete(fence);
	389
	390	irq_work_queue(&cb->work);
	391	}
	392
	393	static void irq_i915_sw_fence_work(struct irq_work *wrk)
	394	{
	395	struct i915_sw_dma_fence_cb cb = container_of(wrk, typeof(cb), work);
e68a139f CW	396
e68a139f CW	397	del_timer_sync(&cb->timer);
f54d1867	398	dma_fence_put(cb->dma);
e68a139f CW	399
	400	kfree(cb);
	401	}
	402
	403	int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
f54d1867	404	struct dma_fence *dma,
e68a139f CW	405	unsigned long timeout,
	406	gfp_t gfp)
	407	{
f54d1867	408	struct i915_sw_dma_fence_cb *cb;
e68a139f CW	409	int ret;
e68a139f CW	410
fc158405 CW	411	debug_fence_assert(fence);
fc158405 CW	412
f54d1867	413	if (dma_fence_is_signaled(dma))
e68a139f CW	414	return 0;
	415
	416	cb = kmalloc(sizeof(*cb), gfp);
	417	if (!cb) {
	418	if (!gfpflags_allow_blocking(gfp))
	419	return -ENOMEM;
	420
f54d1867	421	return dma_fence_wait(dma, false);
e68a139f CW	422	}
e68a139f CW	423
9310cb7f	424	cb->fence = fence;
e68a139f CW	425	i915_sw_fence_await(fence);
	426
	427	cb->dma = NULL;
	428	__setup_timer(&cb->timer,
	429	timer_i915_sw_fence_wake, (unsigned long)cb,
	430	TIMER_IRQSAFE);
81c0ed21	431	init_irq_work(&cb->work, irq_i915_sw_fence_work);
e68a139f	432	if (timeout) {
f54d1867	433	cb->dma = dma_fence_get(dma);
e68a139f CW	434	mod_timer(&cb->timer, round_jiffies_up(jiffies + timeout));
	435	}
	436
f54d1867	437	ret = dma_fence_add_callback(dma, &cb->base, dma_i915_sw_fence_wake);
e68a139f CW	438	if (ret == 0) {
	439	ret = 1;
	440	} else {
	441	dma_i915_sw_fence_wake(dma, &cb->base);
	442	if (ret == -ENOENT) /* fence already signaled */
	443	ret = 0;
	444	}
	445
	446	return ret;
	447	}
	448
	449	int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
	450	struct reservation_object *resv,
f54d1867	451	const struct dma_fence_ops *exclude,
e68a139f CW	452	bool write,
	453	unsigned long timeout,
	454	gfp_t gfp)
	455	{
f54d1867	456	struct dma_fence *excl;
e68a139f CW	457	int ret = 0, pending;
e68a139f CW	458
fc158405 CW	459	debug_fence_assert(fence);
fc158405 CW	460
e68a139f	461	if (write) {
f54d1867	462	struct dma_fence **shared;
e68a139f CW	463	unsigned int count, i;
	464
	465	ret = reservation_object_get_fences_rcu(resv,
	466	&excl, &count, &shared);
	467	if (ret)
	468	return ret;
	469
	470	for (i = 0; i < count; i++) {
	471	if (shared[i]->ops == exclude)
	472	continue;
	473
	474	pending = i915_sw_fence_await_dma_fence(fence,
	475	shared[i],
	476	timeout,
	477	gfp);
	478	if (pending < 0) {
	479	ret = pending;
	480	break;
	481	}
	482
	483	ret \|= pending;
	484	}
	485
	486	for (i = 0; i < count; i++)
f54d1867	487	dma_fence_put(shared[i]);
e68a139f CW	488	kfree(shared);
	489	} else {
	490	excl = reservation_object_get_excl_rcu(resv);
	491	}
	492
	493	if (ret >= 0 && excl && excl->ops != exclude) {
	494	pending = i915_sw_fence_await_dma_fence(fence,
	495	excl,
	496	timeout,
	497	gfp);
	498	if (pending < 0)
	499	ret = pending;
	500	else
	501	ret \|= pending;
	502	}
	503
f54d1867	504	dma_fence_put(excl);
e68a139f CW	505
	506	return ret;
	507	}
47624cc3 CW	508
	509	#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
	510	#include "selftests/i915_sw_fence.c"
	511	#endif