[linux-2.6-block.git] / drivers / dma-buf / dma-fence-chain.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * fence-chain: chain fences together in a timeline
 *
 * Copyright (C) 2018 Advanced Micro Devices, Inc.
 * Authors:
 *	Christian König <christian.koenig@amd.com>
 */

#include <linux/dma-fence-chain.h>

static bool dma_fence_chain_enable_signaling(struct dma_fence *fence);

/**
 * dma_fence_chain_get_prev - use RCU to get a reference to the previous fence
 * @chain: chain node to get the previous node from
 *
 * Use dma_fence_get_rcu_safe to get a reference to the previous fence of the
 * chain node.
 */
static struct dma_fence *dma_fence_chain_get_prev(struct dma_fence_chain *chain)
{
	struct dma_fence *prev;

	rcu_read_lock();
	prev = dma_fence_get_rcu_safe(&chain->prev);
	rcu_read_unlock();
	return prev;
}

/**
 * dma_fence_chain_walk - chain walking function
 * @fence: current chain node
 *
 * Walk the chain to the next node. Returns the next fence or NULL if we are at
 * the end of the chain. Garbage collects chain nodes which are already
 * signaled.
 */
struct dma_fence *dma_fence_chain_walk(struct dma_fence *fence)
{
	struct dma_fence_chain *chain, *prev_chain;
	struct dma_fence *prev, *replacement, *tmp;

	chain = to_dma_fence_chain(fence);
	if (!chain) {
		dma_fence_put(fence);
		return NULL;
	}

	while ((prev = dma_fence_chain_get_prev(chain))) {

		prev_chain = to_dma_fence_chain(prev);
		if (prev_chain) {
			if (!dma_fence_is_signaled(prev_chain->fence))
				break;

			replacement = dma_fence_chain_get_prev(prev_chain);
		} else {
			if (!dma_fence_is_signaled(prev))
				break;

			replacement = NULL;
		}

		tmp = cmpxchg((struct dma_fence __force **)&chain->prev,
			      prev, replacement);
		if (tmp == prev)
			dma_fence_put(tmp);
		else
			dma_fence_put(replacement);
		dma_fence_put(prev);
	}

	dma_fence_put(fence);
	return prev;
}
EXPORT_SYMBOL(dma_fence_chain_walk);

/**
 * dma_fence_chain_find_seqno - find fence chain node by seqno
 * @pfence: pointer to the chain node where to start
 * @seqno: the sequence number to search for
 *
 * Advance the fence pointer to the chain node which will signal this sequence
 * number. If no sequence number is provided then this is a no-op.
 *
 * Returns EINVAL if the fence is not a chain node or the sequence number has
 * not yet advanced far enough.
 */
int dma_fence_chain_find_seqno(struct dma_fence **pfence, uint64_t seqno)
{
	struct dma_fence_chain *chain;

	if (!seqno)
		return 0;

	chain = to_dma_fence_chain(*pfence);
	if (!chain || chain->base.seqno < seqno)
		return -EINVAL;

	dma_fence_chain_for_each(*pfence, &chain->base) {
		if ((*pfence)->context != chain->base.context ||
		    to_dma_fence_chain(*pfence)->prev_seqno < seqno)
			break;
	}
	dma_fence_put(&chain->base);

	return 0;
}
EXPORT_SYMBOL(dma_fence_chain_find_seqno);

static const char *dma_fence_chain_get_driver_name(struct dma_fence *fence)
{
        return "dma_fence_chain";
}

static const char *dma_fence_chain_get_timeline_name(struct dma_fence *fence)
{
        return "unbound";
}

static void dma_fence_chain_irq_work(struct irq_work *work)
{
	struct dma_fence_chain *chain;

	chain = container_of(work, typeof(*chain), work);

	/* Try to rearm the callback */
	if (!dma_fence_chain_enable_signaling(&chain->base))
		/* Ok, we are done. No more unsignaled fences left */
		dma_fence_signal(&chain->base);
	dma_fence_put(&chain->base);
}

static void dma_fence_chain_cb(struct dma_fence *f, struct dma_fence_cb *cb)
{
	struct dma_fence_chain *chain;

	chain = container_of(cb, typeof(*chain), cb);
	irq_work_queue(&chain->work);
	dma_fence_put(f);
}

static bool dma_fence_chain_enable_signaling(struct dma_fence *fence)
{
	struct dma_fence_chain *head = to_dma_fence_chain(fence);

	dma_fence_get(&head->base);
	dma_fence_chain_for_each(fence, &head->base) {
		struct dma_fence_chain *chain = to_dma_fence_chain(fence);
		struct dma_fence *f = chain ? chain->fence : fence;

		dma_fence_get(f);
		if (!dma_fence_add_callback(f, &head->cb, dma_fence_chain_cb)) {
			dma_fence_put(fence);
			return true;
		}
		dma_fence_put(f);
	}
	dma_fence_put(&head->base);
	return false;
}

static bool dma_fence_chain_signaled(struct dma_fence *fence)
{
	dma_fence_chain_for_each(fence, fence) {
		struct dma_fence_chain *chain = to_dma_fence_chain(fence);
		struct dma_fence *f = chain ? chain->fence : fence;

		if (!dma_fence_is_signaled(f)) {
			dma_fence_put(fence);
			return false;
		}
	}

	return true;
}

static void dma_fence_chain_release(struct dma_fence *fence)
{
	struct dma_fence_chain *chain = to_dma_fence_chain(fence);
	struct dma_fence *prev;

	/* Manually unlink the chain as much as possible to avoid recursion
	 * and potential stack overflow.
	 */
	while ((prev = rcu_dereference_protected(chain->prev, true))) {
		struct dma_fence_chain *prev_chain;

		if (kref_read(&prev->refcount) > 1)
		       break;

		prev_chain = to_dma_fence_chain(prev);
		if (!prev_chain)
			break;

		/* No need for atomic operations since we hold the last
		 * reference to prev_chain.
		 */
		chain->prev = prev_chain->prev;
		RCU_INIT_POINTER(prev_chain->prev, NULL);
		dma_fence_put(prev);
	}
	dma_fence_put(prev);

	dma_fence_put(chain->fence);
	dma_fence_free(fence);
}

const struct dma_fence_ops dma_fence_chain_ops = {
	.use_64bit_seqno = true,
	.get_driver_name = dma_fence_chain_get_driver_name,
	.get_timeline_name = dma_fence_chain_get_timeline_name,
	.enable_signaling = dma_fence_chain_enable_signaling,
	.signaled = dma_fence_chain_signaled,
	.release = dma_fence_chain_release,
};
EXPORT_SYMBOL(dma_fence_chain_ops);

/**
 * dma_fence_chain_init - initialize a fence chain
 * @chain: the chain node to initialize
 * @prev: the previous fence
 * @fence: the current fence
 *
 * Initialize a new chain node and either start a new chain or add the node to
 * the existing chain of the previous fence.
 */
void dma_fence_chain_init(struct dma_fence_chain *chain,
			  struct dma_fence *prev,
			  struct dma_fence *fence,
			  uint64_t seqno)
{
	struct dma_fence_chain *prev_chain = to_dma_fence_chain(prev);
	uint64_t context;

	spin_lock_init(&chain->lock);
	rcu_assign_pointer(chain->prev, prev);
	chain->fence = fence;
	chain->prev_seqno = 0;
	init_irq_work(&chain->work, dma_fence_chain_irq_work);

	/* Try to reuse the context of the previous chain node. */
	if (prev_chain && __dma_fence_is_later(seqno, prev->seqno, prev->ops)) {
		context = prev->context;
		chain->prev_seqno = prev->seqno;
	} else {
		context = dma_fence_context_alloc(1);
		/* Make sure that we always have a valid sequence number. */
		if (prev_chain)
			seqno = max(prev->seqno, seqno);
	}

	dma_fence_init(&chain->base, &dma_fence_chain_ops,
		       &chain->lock, context, seqno);
}
EXPORT_SYMBOL(dma_fence_chain_init);
Commit	Line	Data
1802d0be	1	// SPDX-License-Identifier: GPL-2.0-only
7bf60c52 CK	2	/*
	3	* fence-chain: chain fences together in a timeline
	4	*
	5	* Copyright (C) 2018 Advanced Micro Devices, Inc.
	6	* Authors:
	7	* Christian König <christian.koenig@amd.com>
7bf60c52 CK	8	*/
	9
	10	#include <linux/dma-fence-chain.h>
	11
	12	static bool dma_fence_chain_enable_signaling(struct dma_fence *fence);
	13
	14	/**
	15	* dma_fence_chain_get_prev - use RCU to get a reference to the previous fence
	16	* @chain: chain node to get the previous node from
	17	*
	18	* Use dma_fence_get_rcu_safe to get a reference to the previous fence of the
	19	* chain node.
	20	*/
	21	static struct dma_fence dma_fence_chain_get_prev(struct dma_fence_chain chain)
	22	{
	23	struct dma_fence *prev;
	24
	25	rcu_read_lock();
	26	prev = dma_fence_get_rcu_safe(&chain->prev);
	27	rcu_read_unlock();
	28	return prev;
	29	}
	30
	31	/**
	32	* dma_fence_chain_walk - chain walking function
	33	* @fence: current chain node
	34	*
	35	* Walk the chain to the next node. Returns the next fence or NULL if we are at
	36	* the end of the chain. Garbage collects chain nodes which are already
	37	* signaled.
	38	*/
	39	struct dma_fence dma_fence_chain_walk(struct dma_fence fence)
	40	{
	41	struct dma_fence_chain chain, prev_chain;
	42	struct dma_fence prev, replacement, *tmp;
	43
	44	chain = to_dma_fence_chain(fence);
	45	if (!chain) {
	46	dma_fence_put(fence);
	47	return NULL;
	48	}
	49
	50	while ((prev = dma_fence_chain_get_prev(chain))) {
	51
	52	prev_chain = to_dma_fence_chain(prev);
	53	if (prev_chain) {
	54	if (!dma_fence_is_signaled(prev_chain->fence))
	55	break;
	56
	57	replacement = dma_fence_chain_get_prev(prev_chain);
	58	} else {
	59	if (!dma_fence_is_signaled(prev))
	60	break;
	61
	62	replacement = NULL;
	63	}
	64
29da47cf CW	65	tmp = cmpxchg((struct dma_fence __force **)&chain->prev,
29da47cf CW	66	prev, replacement);
7bf60c52 CK	67	if (tmp == prev)
	68	dma_fence_put(tmp);
	69	else
	70	dma_fence_put(replacement);
	71	dma_fence_put(prev);
	72	}
	73
	74	dma_fence_put(fence);
	75	return prev;
	76	}
	77	EXPORT_SYMBOL(dma_fence_chain_walk);
	78
	79	/**
	80	* dma_fence_chain_find_seqno - find fence chain node by seqno
	81	* @pfence: pointer to the chain node where to start
	82	* @seqno: the sequence number to search for
	83	*
	84	* Advance the fence pointer to the chain node which will signal this sequence
	85	* number. If no sequence number is provided then this is a no-op.
	86	*
	87	* Returns EINVAL if the fence is not a chain node or the sequence number has
	88	* not yet advanced far enough.
	89	*/
	90	int dma_fence_chain_find_seqno(struct dma_fence **pfence, uint64_t seqno)
	91	{
	92	struct dma_fence_chain *chain;
	93
	94	if (!seqno)
	95	return 0;
	96
	97	chain = to_dma_fence_chain(*pfence);
	98	if (!chain \|\| chain->base.seqno < seqno)
	99	return -EINVAL;
	100
	101	dma_fence_chain_for_each(*pfence, &chain->base) {
	102	if ((*pfence)->context != chain->base.context \|\|
	103	to_dma_fence_chain(*pfence)->prev_seqno < seqno)
	104	break;
	105	}
	106	dma_fence_put(&chain->base);
	107
	108	return 0;
	109	}
	110	EXPORT_SYMBOL(dma_fence_chain_find_seqno);
	111
	112	static const char dma_fence_chain_get_driver_name(struct dma_fence fence)
	113	{
	114	return "dma_fence_chain";
	115	}
	116
	117	static const char dma_fence_chain_get_timeline_name(struct dma_fence fence)
	118	{
	119	return "unbound";
	120	}
	121
	122	static void dma_fence_chain_irq_work(struct irq_work *work)
	123	{
	124	struct dma_fence_chain *chain;
	125
	126	chain = container_of(work, typeof(*chain), work);
	127
	128	/* Try to rearm the callback */
	129	if (!dma_fence_chain_enable_signaling(&chain->base))
	130	/* Ok, we are done. No more unsignaled fences left */
131	dma_fence_signal(&chain->base);
132	dma_fence_put(&chain->base);
133	}
134
135	static void dma_fence_chain_cb(struct dma_fence f, struct dma_fence_cb cb)
136	{
137	struct dma_fence_chain *chain;
138
139	chain = container_of(cb, typeof(*chain), cb);
140	irq_work_queue(&chain->work);
141	dma_fence_put(f);
142	}
143
144	static bool dma_fence_chain_enable_signaling(struct dma_fence *fence)
145	{
146	struct dma_fence_chain *head = to_dma_fence_chain(fence);
147
148	dma_fence_get(&head->base);
149	dma_fence_chain_for_each(fence, &head->base) {
150	struct dma_fence_chain *chain = to_dma_fence_chain(fence);
151	struct dma_fence *f = chain ? chain->fence : fence;
152
153	dma_fence_get(f);
154	if (!dma_fence_add_callback(f, &head->cb, dma_fence_chain_cb)) {
155	dma_fence_put(fence);
156	return true;
157	}
158	dma_fence_put(f);
159	}
160	dma_fence_put(&head->base);
161	return false;
162	}
163
164	static bool dma_fence_chain_signaled(struct dma_fence *fence)
165	{
166	dma_fence_chain_for_each(fence, fence) {
167	struct dma_fence_chain *chain = to_dma_fence_chain(fence);
168	struct dma_fence *f = chain ? chain->fence : fence;
169
170	if (!dma_fence_is_signaled(f)) {
171	dma_fence_put(fence);
172	return false;
173	}
174	}
175
176	return true;
177	}
178
179	static void dma_fence_chain_release(struct dma_fence *fence)
180	{
181	struct dma_fence_chain *chain = to_dma_fence_chain(fence);
92cb3e59 CK	182	struct dma_fence *prev;
	183
	184	/* Manually unlink the chain as much as possible to avoid recursion
	185	* and potential stack overflow.
	186	*/
	187	while ((prev = rcu_dereference_protected(chain->prev, true))) {
	188	struct dma_fence_chain *prev_chain;
	189
	190	if (kref_read(&prev->refcount) > 1)
	191	break;
	192
	193	prev_chain = to_dma_fence_chain(prev);
	194	if (!prev_chain)
	195	break;
	196
	197	/* No need for atomic operations since we hold the last
	198	* reference to prev_chain.
	199	*/
	200	chain->prev = prev_chain->prev;
	201	RCU_INIT_POINTER(prev_chain->prev, NULL);
	202	dma_fence_put(prev);
	203	}
	204	dma_fence_put(prev);
7bf60c52	205
7bf60c52 CK	206	dma_fence_put(chain->fence);
	207	dma_fence_free(fence);
	208	}
	209
	210	const struct dma_fence_ops dma_fence_chain_ops = {
5e498abf	211	.use_64bit_seqno = true,
7bf60c52 CK	212	.get_driver_name = dma_fence_chain_get_driver_name,
	213	.get_timeline_name = dma_fence_chain_get_timeline_name,
	214	.enable_signaling = dma_fence_chain_enable_signaling,
	215	.signaled = dma_fence_chain_signaled,
	216	.release = dma_fence_chain_release,
	217	};
	218	EXPORT_SYMBOL(dma_fence_chain_ops);
	219
	220	/**
	221	* dma_fence_chain_init - initialize a fence chain
	222	* @chain: the chain node to initialize
	223	* @prev: the previous fence
	224	* @fence: the current fence
	225	*
	226	* Initialize a new chain node and either start a new chain or add the node to
	227	* the existing chain of the previous fence.
	228	*/
	229	void dma_fence_chain_init(struct dma_fence_chain *chain,
	230	struct dma_fence *prev,
	231	struct dma_fence *fence,
	232	uint64_t seqno)
	233	{
	234	struct dma_fence_chain *prev_chain = to_dma_fence_chain(prev);
	235	uint64_t context;
	236
	237	spin_lock_init(&chain->lock);
	238	rcu_assign_pointer(chain->prev, prev);
	239	chain->fence = fence;
	240	chain->prev_seqno = 0;
	241	init_irq_work(&chain->work, dma_fence_chain_irq_work);
	242
	243	/* Try to reuse the context of the previous chain node. */
5e498abf	244	if (prev_chain && __dma_fence_is_later(seqno, prev->seqno, prev->ops)) {
7bf60c52 CK	245	context = prev->context;
	246	chain->prev_seqno = prev->seqno;
	247	} else {
	248	context = dma_fence_context_alloc(1);
	249	/* Make sure that we always have a valid sequence number. */
	250	if (prev_chain)
	251	seqno = max(prev->seqno, seqno);
	252	}
	253
	254	dma_fence_init(&chain->base, &dma_fence_chain_ops,
	255	&chain->lock, context, seqno);
	256	}
	257	EXPORT_SYMBOL(dma_fence_chain_init);