[linux-2.6-block.git] / drivers / dma-buf / dma-fence-chain.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * fence-chain: chain fences together in a timeline
 *
 * Copyright (C) 2018 Advanced Micro Devices, Inc.
 * Authors:
 *	Christian König <christian.koenig@amd.com>
 */

#include <linux/dma-fence-chain.h>

static bool dma_fence_chain_enable_signaling(struct dma_fence *fence);

/**
 * dma_fence_chain_get_prev - use RCU to get a reference to the previous fence
 * @chain: chain node to get the previous node from
 *
 * Use dma_fence_get_rcu_safe to get a reference to the previous fence of the
 * chain node.
 */
static struct dma_fence *dma_fence_chain_get_prev(struct dma_fence_chain *chain)
{
	struct dma_fence *prev;

	rcu_read_lock();
	prev = dma_fence_get_rcu_safe(&chain->prev);
	rcu_read_unlock();
	return prev;
}

/**
 * dma_fence_chain_walk - chain walking function
 * @fence: current chain node
 *
 * Walk the chain to the next node. Returns the next fence or NULL if we are at
 * the end of the chain. Garbage collects chain nodes which are already
 * signaled.
 */
struct dma_fence *dma_fence_chain_walk(struct dma_fence *fence)
{
	struct dma_fence_chain *chain, *prev_chain;
	struct dma_fence *prev, *replacement, *tmp;

	chain = to_dma_fence_chain(fence);
	if (!chain) {
		dma_fence_put(fence);
		return NULL;
	}

	while ((prev = dma_fence_chain_get_prev(chain))) {

		prev_chain = to_dma_fence_chain(prev);
		if (prev_chain) {
			if (!dma_fence_is_signaled(prev_chain->fence))
				break;

			replacement = dma_fence_chain_get_prev(prev_chain);
		} else {
			if (!dma_fence_is_signaled(prev))
				break;

			replacement = NULL;
		}

		tmp = cmpxchg((void **)&chain->prev, (void *)prev, (void *)replacement);
		if (tmp == prev)
			dma_fence_put(tmp);
		else
			dma_fence_put(replacement);
		dma_fence_put(prev);
	}

	dma_fence_put(fence);
	return prev;
}
EXPORT_SYMBOL(dma_fence_chain_walk);

/**
 * dma_fence_chain_find_seqno - find fence chain node by seqno
 * @pfence: pointer to the chain node where to start
 * @seqno: the sequence number to search for
 *
 * Advance the fence pointer to the chain node which will signal this sequence
 * number. If no sequence number is provided then this is a no-op.
 *
 * Returns EINVAL if the fence is not a chain node or the sequence number has
 * not yet advanced far enough.
 */
int dma_fence_chain_find_seqno(struct dma_fence **pfence, uint64_t seqno)
{
	struct dma_fence_chain *chain;

	if (!seqno)
		return 0;

	chain = to_dma_fence_chain(*pfence);
	if (!chain || chain->base.seqno < seqno)
		return -EINVAL;

	dma_fence_chain_for_each(*pfence, &chain->base) {
		if ((*pfence)->context != chain->base.context ||
		    to_dma_fence_chain(*pfence)->prev_seqno < seqno)
			break;
	}
	dma_fence_put(&chain->base);

	return 0;
}
EXPORT_SYMBOL(dma_fence_chain_find_seqno);

static const char *dma_fence_chain_get_driver_name(struct dma_fence *fence)
{
        return "dma_fence_chain";
}

static const char *dma_fence_chain_get_timeline_name(struct dma_fence *fence)
{
        return "unbound";
}

static void dma_fence_chain_irq_work(struct irq_work *work)
{
	struct dma_fence_chain *chain;

	chain = container_of(work, typeof(*chain), work);

	/* Try to rearm the callback */
	if (!dma_fence_chain_enable_signaling(&chain->base))
		/* Ok, we are done. No more unsignaled fences left */
		dma_fence_signal(&chain->base);
	dma_fence_put(&chain->base);
}

static void dma_fence_chain_cb(struct dma_fence *f, struct dma_fence_cb *cb)
{
	struct dma_fence_chain *chain;

	chain = container_of(cb, typeof(*chain), cb);
	irq_work_queue(&chain->work);
	dma_fence_put(f);
}

static bool dma_fence_chain_enable_signaling(struct dma_fence *fence)
{
	struct dma_fence_chain *head = to_dma_fence_chain(fence);

	dma_fence_get(&head->base);
	dma_fence_chain_for_each(fence, &head->base) {
		struct dma_fence_chain *chain = to_dma_fence_chain(fence);
		struct dma_fence *f = chain ? chain->fence : fence;

		dma_fence_get(f);
		if (!dma_fence_add_callback(f, &head->cb, dma_fence_chain_cb)) {
			dma_fence_put(fence);
			return true;
		}
		dma_fence_put(f);
	}
	dma_fence_put(&head->base);
	return false;
}

static bool dma_fence_chain_signaled(struct dma_fence *fence)
{
	dma_fence_chain_for_each(fence, fence) {
		struct dma_fence_chain *chain = to_dma_fence_chain(fence);
		struct dma_fence *f = chain ? chain->fence : fence;

		if (!dma_fence_is_signaled(f)) {
			dma_fence_put(fence);
			return false;
		}
	}

	return true;
}

static void dma_fence_chain_release(struct dma_fence *fence)
{
	struct dma_fence_chain *chain = to_dma_fence_chain(fence);
	struct dma_fence *prev;

	/* Manually unlink the chain as much as possible to avoid recursion
	 * and potential stack overflow.
	 */
	while ((prev = rcu_dereference_protected(chain->prev, true))) {
		struct dma_fence_chain *prev_chain;

		if (kref_read(&prev->refcount) > 1)
		       break;

		prev_chain = to_dma_fence_chain(prev);
		if (!prev_chain)
			break;

		/* No need for atomic operations since we hold the last
		 * reference to prev_chain.
		 */
		chain->prev = prev_chain->prev;
		RCU_INIT_POINTER(prev_chain->prev, NULL);
		dma_fence_put(prev);
	}
	dma_fence_put(prev);

	dma_fence_put(chain->fence);
	dma_fence_free(fence);
}

const struct dma_fence_ops dma_fence_chain_ops = {
	.use_64bit_seqno = true,
	.get_driver_name = dma_fence_chain_get_driver_name,
	.get_timeline_name = dma_fence_chain_get_timeline_name,
	.enable_signaling = dma_fence_chain_enable_signaling,
	.signaled = dma_fence_chain_signaled,
	.release = dma_fence_chain_release,
};
EXPORT_SYMBOL(dma_fence_chain_ops);

/**
 * dma_fence_chain_init - initialize a fence chain
 * @chain: the chain node to initialize
 * @prev: the previous fence
 * @fence: the current fence
 *
 * Initialize a new chain node and either start a new chain or add the node to
 * the existing chain of the previous fence.
 */
void dma_fence_chain_init(struct dma_fence_chain *chain,
			  struct dma_fence *prev,
			  struct dma_fence *fence,
			  uint64_t seqno)
{
	struct dma_fence_chain *prev_chain = to_dma_fence_chain(prev);
	uint64_t context;

	spin_lock_init(&chain->lock);
	rcu_assign_pointer(chain->prev, prev);
	chain->fence = fence;
	chain->prev_seqno = 0;
	init_irq_work(&chain->work, dma_fence_chain_irq_work);

	/* Try to reuse the context of the previous chain node. */
	if (prev_chain && __dma_fence_is_later(seqno, prev->seqno, prev->ops)) {
		context = prev->context;
		chain->prev_seqno = prev->seqno;
	} else {
		context = dma_fence_context_alloc(1);
		/* Make sure that we always have a valid sequence number. */
		if (prev_chain)
			seqno = max(prev->seqno, seqno);
	}

	dma_fence_init(&chain->base, &dma_fence_chain_ops,
		       &chain->lock, context, seqno);
}
EXPORT_SYMBOL(dma_fence_chain_init);
Commit	Line	Data
1802d0be	1	// SPDX-License-Identifier: GPL-2.0-only
7bf60c52 CK	2	/*
	3	* fence-chain: chain fences together in a timeline
	4	*
	5	* Copyright (C) 2018 Advanced Micro Devices, Inc.
	6	* Authors:
	7	* Christian König <christian.koenig@amd.com>
7bf60c52 CK	8	*/
	9
	10	#include <linux/dma-fence-chain.h>
	11
	12	static bool dma_fence_chain_enable_signaling(struct dma_fence *fence);
	13
	14	/**
	15	* dma_fence_chain_get_prev - use RCU to get a reference to the previous fence
	16	* @chain: chain node to get the previous node from
	17	*
	18	* Use dma_fence_get_rcu_safe to get a reference to the previous fence of the
	19	* chain node.
	20	*/
	21	static struct dma_fence dma_fence_chain_get_prev(struct dma_fence_chain chain)
	22	{
	23	struct dma_fence *prev;
	24
	25	rcu_read_lock();
	26	prev = dma_fence_get_rcu_safe(&chain->prev);
	27	rcu_read_unlock();
	28	return prev;
	29	}
	30
	31	/**
	32	* dma_fence_chain_walk - chain walking function
	33	* @fence: current chain node
	34	*
	35	* Walk the chain to the next node. Returns the next fence or NULL if we are at
	36	* the end of the chain. Garbage collects chain nodes which are already
	37	* signaled.
	38	*/
	39	struct dma_fence dma_fence_chain_walk(struct dma_fence fence)
	40	{
	41	struct dma_fence_chain chain, prev_chain;
	42	struct dma_fence prev, replacement, *tmp;
	43
	44	chain = to_dma_fence_chain(fence);
	45	if (!chain) {
	46	dma_fence_put(fence);
	47	return NULL;
	48	}
	49
	50	while ((prev = dma_fence_chain_get_prev(chain))) {
	51
	52	prev_chain = to_dma_fence_chain(prev);
	53	if (prev_chain) {
	54	if (!dma_fence_is_signaled(prev_chain->fence))
	55	break;
	56
	57	replacement = dma_fence_chain_get_prev(prev_chain);
	58	} else {
	59	if (!dma_fence_is_signaled(prev))
	60	break;
	61
	62	replacement = NULL;
	63	}
	64
	65	tmp = cmpxchg((void *)&chain->prev, (void )prev, (void *)replacement);
	66	if (tmp == prev)
	67	dma_fence_put(tmp);
	68	else
	69	dma_fence_put(replacement);
	70	dma_fence_put(prev);
	71	}
72
73	dma_fence_put(fence);
74	return prev;
75	}
76	EXPORT_SYMBOL(dma_fence_chain_walk);
77
78	/**
79	* dma_fence_chain_find_seqno - find fence chain node by seqno
80	* @pfence: pointer to the chain node where to start
81	* @seqno: the sequence number to search for
82	*
83	* Advance the fence pointer to the chain node which will signal this sequence
84	* number. If no sequence number is provided then this is a no-op.
85	*
86	* Returns EINVAL if the fence is not a chain node or the sequence number has
87	* not yet advanced far enough.
88	*/
89	int dma_fence_chain_find_seqno(struct dma_fence **pfence, uint64_t seqno)
90	{
91	struct dma_fence_chain *chain;
92
93	if (!seqno)
94	return 0;
95
96	chain = to_dma_fence_chain(*pfence);
97	if (!chain \|\| chain->base.seqno < seqno)
98	return -EINVAL;
99
100	dma_fence_chain_for_each(*pfence, &chain->base) {
101	if ((*pfence)->context != chain->base.context \|\|
102	to_dma_fence_chain(*pfence)->prev_seqno < seqno)
103	break;
104	}
105	dma_fence_put(&chain->base);
106
107	return 0;
108	}
109	EXPORT_SYMBOL(dma_fence_chain_find_seqno);
110
111	static const char dma_fence_chain_get_driver_name(struct dma_fence fence)
112	{
113	return "dma_fence_chain";
114	}
115
116	static const char dma_fence_chain_get_timeline_name(struct dma_fence fence)
117	{
118	return "unbound";
119	}
120
121	static void dma_fence_chain_irq_work(struct irq_work *work)
122	{
123	struct dma_fence_chain *chain;
124
125	chain = container_of(work, typeof(*chain), work);
126
127	/* Try to rearm the callback */
128	if (!dma_fence_chain_enable_signaling(&chain->base))
129	/* Ok, we are done. No more unsignaled fences left */
130	dma_fence_signal(&chain->base);
131	dma_fence_put(&chain->base);
132	}
133
134	static void dma_fence_chain_cb(struct dma_fence f, struct dma_fence_cb cb)
135	{
136	struct dma_fence_chain *chain;
137
138	chain = container_of(cb, typeof(*chain), cb);
139	irq_work_queue(&chain->work);
140	dma_fence_put(f);
141	}
142
143	static bool dma_fence_chain_enable_signaling(struct dma_fence *fence)
144	{
145	struct dma_fence_chain *head = to_dma_fence_chain(fence);
146
147	dma_fence_get(&head->base);
148	dma_fence_chain_for_each(fence, &head->base) {
149	struct dma_fence_chain *chain = to_dma_fence_chain(fence);
150	struct dma_fence *f = chain ? chain->fence : fence;
151
152	dma_fence_get(f);
153	if (!dma_fence_add_callback(f, &head->cb, dma_fence_chain_cb)) {
154	dma_fence_put(fence);
155	return true;
156	}
157	dma_fence_put(f);
158	}
159	dma_fence_put(&head->base);
160	return false;
161	}
162
163	static bool dma_fence_chain_signaled(struct dma_fence *fence)
164	{
165	dma_fence_chain_for_each(fence, fence) {
166	struct dma_fence_chain *chain = to_dma_fence_chain(fence);
167	struct dma_fence *f = chain ? chain->fence : fence;
168
169	if (!dma_fence_is_signaled(f)) {
170	dma_fence_put(fence);
171	return false;
172	}
173	}
174
175	return true;
176	}
177
178	static void dma_fence_chain_release(struct dma_fence *fence)
179	{
180	struct dma_fence_chain *chain = to_dma_fence_chain(fence);
92cb3e59 CK	181	struct dma_fence *prev;
	182
	183	/* Manually unlink the chain as much as possible to avoid recursion
	184	* and potential stack overflow.
	185	*/
	186	while ((prev = rcu_dereference_protected(chain->prev, true))) {
	187	struct dma_fence_chain *prev_chain;
	188
	189	if (kref_read(&prev->refcount) > 1)
	190	break;
	191
	192	prev_chain = to_dma_fence_chain(prev);
	193	if (!prev_chain)
	194	break;
	195
	196	/* No need for atomic operations since we hold the last
	197	* reference to prev_chain.
	198	*/
	199	chain->prev = prev_chain->prev;
	200	RCU_INIT_POINTER(prev_chain->prev, NULL);
	201	dma_fence_put(prev);
	202	}
	203	dma_fence_put(prev);
7bf60c52	204
7bf60c52 CK	205	dma_fence_put(chain->fence);
	206	dma_fence_free(fence);
	207	}
	208
	209	const struct dma_fence_ops dma_fence_chain_ops = {
5e498abf	210	.use_64bit_seqno = true,
7bf60c52 CK	211	.get_driver_name = dma_fence_chain_get_driver_name,
	212	.get_timeline_name = dma_fence_chain_get_timeline_name,
	213	.enable_signaling = dma_fence_chain_enable_signaling,
	214	.signaled = dma_fence_chain_signaled,
	215	.release = dma_fence_chain_release,
	216	};
	217	EXPORT_SYMBOL(dma_fence_chain_ops);
	218
	219	/**
	220	* dma_fence_chain_init - initialize a fence chain
	221	* @chain: the chain node to initialize
	222	* @prev: the previous fence
	223	* @fence: the current fence
	224	*
	225	* Initialize a new chain node and either start a new chain or add the node to
	226	* the existing chain of the previous fence.
	227	*/
	228	void dma_fence_chain_init(struct dma_fence_chain *chain,
	229	struct dma_fence *prev,
	230	struct dma_fence *fence,
	231	uint64_t seqno)
	232	{
	233	struct dma_fence_chain *prev_chain = to_dma_fence_chain(prev);
	234	uint64_t context;
	235
	236	spin_lock_init(&chain->lock);
	237	rcu_assign_pointer(chain->prev, prev);
	238	chain->fence = fence;
	239	chain->prev_seqno = 0;
	240	init_irq_work(&chain->work, dma_fence_chain_irq_work);
	241
	242	/* Try to reuse the context of the previous chain node. */
5e498abf	243	if (prev_chain && __dma_fence_is_later(seqno, prev->seqno, prev->ops)) {
7bf60c52 CK	244	context = prev->context;
	245	chain->prev_seqno = prev->seqno;
	246	} else {
	247	context = dma_fence_context_alloc(1);
	248	/* Make sure that we always have a valid sequence number. */
	249	if (prev_chain)
	250	seqno = max(prev->seqno, seqno);
	251	}
	252
	253	dma_fence_init(&chain->base, &dma_fence_chain_ops,
	254	&chain->lock, context, seqno);
	255	}
	256	EXPORT_SYMBOL(dma_fence_chain_init);