[linux-block.git] / drivers / gpu / host1x / intr.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Tegra host1x Interrupt Management
 *
 * Copyright (c) 2010-2013, NVIDIA Corporation.
 */

#include <linux/clk.h>
#include <linux/interrupt.h>
#include <linux/slab.h>
#include <linux/irq.h>

#include <trace/events/host1x.h>
#include "channel.h"
#include "dev.h"
#include "fence.h"
#include "intr.h"

/* Wait list management */

enum waitlist_state {
	WLS_PENDING,
	WLS_REMOVED,
	WLS_CANCELLED,
	WLS_HANDLED
};

static void waiter_release(struct kref *kref)
{
	kfree(container_of(kref, struct host1x_waitlist, refcount));
}

/*
 * add a waiter to a waiter queue, sorted by threshold
 * returns true if it was added at the head of the queue
 */
static bool add_waiter_to_queue(struct host1x_waitlist *waiter,
				struct list_head *queue)
{
	struct host1x_waitlist *pos;
	u32 thresh = waiter->thresh;

	list_for_each_entry_reverse(pos, queue, list)
		if ((s32)(pos->thresh - thresh) <= 0) {
			list_add(&waiter->list, &pos->list);
			return false;
		}

	list_add(&waiter->list, queue);
	return true;
}

/*
 * run through a waiter queue for a single sync point ID
 * and gather all completed waiters into lists by actions
 */
static void remove_completed_waiters(struct list_head *head, u32 sync,
			struct list_head completed[HOST1X_INTR_ACTION_COUNT])
{
	struct list_head *dest;
	struct host1x_waitlist *waiter, *next, *prev;

	list_for_each_entry_safe(waiter, next, head, list) {
		if ((s32)(waiter->thresh - sync) > 0)
			break;

		dest = completed + waiter->action;

		/* consolidate submit cleanups */
		if (waiter->action == HOST1X_INTR_ACTION_SUBMIT_COMPLETE &&
		    !list_empty(dest)) {
			prev = list_entry(dest->prev,
					  struct host1x_waitlist, list);
			if (prev->data == waiter->data) {
				prev->count++;
				dest = NULL;
			}
		}

		/* PENDING->REMOVED or CANCELLED->HANDLED */
		if (atomic_inc_return(&waiter->state) == WLS_HANDLED || !dest) {
			list_del(&waiter->list);
			kref_put(&waiter->refcount, waiter_release);
		} else
			list_move_tail(&waiter->list, dest);
	}
}

static void reset_threshold_interrupt(struct host1x *host,
				      struct list_head *head,
				      unsigned int id)
{
	u32 thresh =
		list_first_entry(head, struct host1x_waitlist, list)->thresh;

	host1x_hw_intr_set_syncpt_threshold(host, id, thresh);
	host1x_hw_intr_enable_syncpt_intr(host, id);
}

static void action_submit_complete(struct host1x_waitlist *waiter)
{
	struct host1x_channel *channel = waiter->data;

	host1x_cdma_update(&channel->cdma);

	/*  Add nr_completed to trace */
	trace_host1x_channel_submit_complete(dev_name(channel->dev),
					     waiter->count, waiter->thresh);
}

static void action_wakeup(struct host1x_waitlist *waiter)
{
	wait_queue_head_t *wq = waiter->data;

	wake_up(wq);
}

static void action_wakeup_interruptible(struct host1x_waitlist *waiter)
{
	wait_queue_head_t *wq = waiter->data;

	wake_up_interruptible(wq);
}

static void action_signal_fence(struct host1x_waitlist *waiter)
{
	struct host1x_syncpt_fence *f = waiter->data;

	host1x_fence_signal(f);
}

typedef void (*action_handler)(struct host1x_waitlist *waiter);

static const action_handler action_handlers[HOST1X_INTR_ACTION_COUNT] = {
	action_submit_complete,
	action_wakeup,
	action_wakeup_interruptible,
	action_signal_fence,
};

static void run_handlers(struct list_head completed[HOST1X_INTR_ACTION_COUNT])
{
	struct list_head *head = completed;
	unsigned int i;

	for (i = 0; i < HOST1X_INTR_ACTION_COUNT; ++i, ++head) {
		action_handler handler = action_handlers[i];
		struct host1x_waitlist *waiter, *next;

		list_for_each_entry_safe(waiter, next, head, list) {
			list_del(&waiter->list);
			handler(waiter);
			WARN_ON(atomic_xchg(&waiter->state, WLS_HANDLED) !=
				WLS_REMOVED);
			kref_put(&waiter->refcount, waiter_release);
		}
	}
}

/*
 * Remove & handle all waiters that have completed for the given syncpt
 */
static int process_wait_list(struct host1x *host,
			     struct host1x_syncpt *syncpt,
			     u32 threshold)
{
	struct list_head completed[HOST1X_INTR_ACTION_COUNT];
	unsigned int i;
	int empty;

	for (i = 0; i < HOST1X_INTR_ACTION_COUNT; ++i)
		INIT_LIST_HEAD(completed + i);

	spin_lock(&syncpt->intr.lock);

	remove_completed_waiters(&syncpt->intr.wait_head, threshold,
				 completed);

	empty = list_empty(&syncpt->intr.wait_head);
	if (empty)
		host1x_hw_intr_disable_syncpt_intr(host, syncpt->id);
	else
		reset_threshold_interrupt(host, &syncpt->intr.wait_head,
					  syncpt->id);

	spin_unlock(&syncpt->intr.lock);

	run_handlers(completed);

	return empty;
}

/*
 * Sync point threshold interrupt service thread function
 * Handles sync point threshold triggers, in thread context
 */

static void syncpt_thresh_work(struct work_struct *work)
{
	struct host1x_syncpt_intr *syncpt_intr =
		container_of(work, struct host1x_syncpt_intr, work);
	struct host1x_syncpt *syncpt =
		container_of(syncpt_intr, struct host1x_syncpt, intr);
	unsigned int id = syncpt->id;
	struct host1x *host = syncpt->host;

	(void)process_wait_list(host, syncpt,
				host1x_syncpt_load(host->syncpt + id));
}

int host1x_intr_add_action(struct host1x *host, struct host1x_syncpt *syncpt,
			   u32 thresh, enum host1x_intr_action action,
			   void *data, struct host1x_waitlist *waiter,
			   void **ref)
{
	int queue_was_empty;

	if (waiter == NULL) {
		pr_warn("%s: NULL waiter\n", __func__);
		return -EINVAL;
	}

	/* initialize a new waiter */
	INIT_LIST_HEAD(&waiter->list);
	kref_init(&waiter->refcount);
	if (ref)
		kref_get(&waiter->refcount);
	waiter->thresh = thresh;
	waiter->action = action;
	atomic_set(&waiter->state, WLS_PENDING);
	waiter->data = data;
	waiter->count = 1;

	spin_lock(&syncpt->intr.lock);

	queue_was_empty = list_empty(&syncpt->intr.wait_head);

	if (add_waiter_to_queue(waiter, &syncpt->intr.wait_head)) {
		/* added at head of list - new threshold value */
		host1x_hw_intr_set_syncpt_threshold(host, syncpt->id, thresh);

		/* added as first waiter - enable interrupt */
		if (queue_was_empty)
			host1x_hw_intr_enable_syncpt_intr(host, syncpt->id);
	}

	if (ref)
		*ref = waiter;

	spin_unlock(&syncpt->intr.lock);

	return 0;
}

void host1x_intr_put_ref(struct host1x *host, unsigned int id, void *ref,
			 bool flush)
{
	struct host1x_waitlist *waiter = ref;
	struct host1x_syncpt *syncpt;

	atomic_cmpxchg(&waiter->state, WLS_PENDING, WLS_CANCELLED);

	syncpt = host->syncpt + id;

	spin_lock(&syncpt->intr.lock);
	if (atomic_cmpxchg(&waiter->state, WLS_CANCELLED, WLS_HANDLED) ==
	    WLS_CANCELLED) {
		list_del(&waiter->list);
		kref_put(&waiter->refcount, waiter_release);
	}
	spin_unlock(&syncpt->intr.lock);

	if (flush) {
		/* Wait until any concurrently executing handler has finished. */
		while (atomic_read(&waiter->state) != WLS_HANDLED)
			schedule();
	}

	kref_put(&waiter->refcount, waiter_release);
}

int host1x_intr_init(struct host1x *host, unsigned int irq_sync)
{
	unsigned int id;
	u32 nb_pts = host1x_syncpt_nb_pts(host);

	mutex_init(&host->intr_mutex);
	host->intr_syncpt_irq = irq_sync;

	for (id = 0; id < nb_pts; ++id) {
		struct host1x_syncpt *syncpt = host->syncpt + id;

		spin_lock_init(&syncpt->intr.lock);
		INIT_LIST_HEAD(&syncpt->intr.wait_head);
		snprintf(syncpt->intr.thresh_irq_name,
			 sizeof(syncpt->intr.thresh_irq_name),
			 "host1x_sp_%02u", id);
	}

	return 0;
}

void host1x_intr_deinit(struct host1x *host)
{
}

void host1x_intr_start(struct host1x *host)
{
	u32 hz = clk_get_rate(host->clk);
	int err;

	mutex_lock(&host->intr_mutex);
	err = host1x_hw_intr_init_host_sync(host, DIV_ROUND_UP(hz, 1000000),
					    syncpt_thresh_work);
	if (err) {
		mutex_unlock(&host->intr_mutex);
		return;
	}
	mutex_unlock(&host->intr_mutex);
}

void host1x_intr_stop(struct host1x *host)
{
	unsigned int id;
	struct host1x_syncpt *syncpt = host->syncpt;
	u32 nb_pts = host1x_syncpt_nb_pts(host);

	mutex_lock(&host->intr_mutex);

	host1x_hw_intr_disable_all_syncpt_intrs(host);

	for (id = 0; id < nb_pts; ++id) {
		struct host1x_waitlist *waiter, *next;

		list_for_each_entry_safe(waiter, next,
			&syncpt[id].intr.wait_head, list) {
			if (atomic_cmpxchg(&waiter->state,
			    WLS_CANCELLED, WLS_HANDLED) == WLS_CANCELLED) {
				list_del(&waiter->list);
				kref_put(&waiter->refcount, waiter_release);
			}
		}

		if (!list_empty(&syncpt[id].intr.wait_head)) {
			/* output diagnostics */
			mutex_unlock(&host->intr_mutex);
			pr_warn("%s cannot stop syncpt intr id=%u\n",
				__func__, id);
			return;
		}
	}

	host1x_hw_intr_free_syncpt_irq(host);

	mutex_unlock(&host->intr_mutex);
}
Commit	Line	Data
9952f691	1	// SPDX-License-Identifier: GPL-2.0-only
7ede0b0b TB	2	/*
	3	* Tegra host1x Interrupt Management
	4	*
	5	* Copyright (c) 2010-2013, NVIDIA Corporation.
7ede0b0b TB	6	*/
	7
	8	#include <linux/clk.h>
	9	#include <linux/interrupt.h>
	10	#include <linux/slab.h>
	11	#include <linux/irq.h>
	12
6579324a TB	13	#include <trace/events/host1x.h>
6579324a TB	14	#include "channel.h"
7ede0b0b	15	#include "dev.h"
687db220	16	#include "fence.h"
7ede0b0b TB	17	#include "intr.h"
	18
	19	/* Wait list management */
	20
	21	enum waitlist_state {
	22	WLS_PENDING,
	23	WLS_REMOVED,
	24	WLS_CANCELLED,
	25	WLS_HANDLED
	26	};
	27
	28	static void waiter_release(struct kref *kref)
	29	{
	30	kfree(container_of(kref, struct host1x_waitlist, refcount));
	31	}
	32
	33	/*
	34	* add a waiter to a waiter queue, sorted by threshold
	35	* returns true if it was added at the head of the queue
	36	*/
	37	static bool add_waiter_to_queue(struct host1x_waitlist *waiter,
	38	struct list_head *queue)
	39	{
	40	struct host1x_waitlist *pos;
	41	u32 thresh = waiter->thresh;
	42
	43	list_for_each_entry_reverse(pos, queue, list)
	44	if ((s32)(pos->thresh - thresh) <= 0) {
	45	list_add(&waiter->list, &pos->list);
	46	return false;
	47	}
	48
	49	list_add(&waiter->list, queue);
	50	return true;
	51	}
	52
	53	/*
	54	* run through a waiter queue for a single sync point ID
	55	* and gather all completed waiters into lists by actions
	56	*/
	57	static void remove_completed_waiters(struct list_head *head, u32 sync,
	58	struct list_head completed[HOST1X_INTR_ACTION_COUNT])
	59	{
	60	struct list_head *dest;
6579324a	61	struct host1x_waitlist waiter, next, *prev;
7ede0b0b TB	62
	63	list_for_each_entry_safe(waiter, next, head, list) {
	64	if ((s32)(waiter->thresh - sync) > 0)
	65	break;
	66
	67	dest = completed + waiter->action;
	68
6579324a TB	69	/* consolidate submit cleanups */
	70	if (waiter->action == HOST1X_INTR_ACTION_SUBMIT_COMPLETE &&
	71	!list_empty(dest)) {
	72	prev = list_entry(dest->prev,
	73	struct host1x_waitlist, list);
	74	if (prev->data == waiter->data) {
	75	prev->count++;
	76	dest = NULL;
	77	}
	78	}
	79
7ede0b0b TB	80	/* PENDING->REMOVED or CANCELLED->HANDLED */
	81	if (atomic_inc_return(&waiter->state) == WLS_HANDLED \|\| !dest) {
	82	list_del(&waiter->list);
	83	kref_put(&waiter->refcount, waiter_release);
	84	} else
	85	list_move_tail(&waiter->list, dest);
	86	}
	87	}
	88
	89	static void reset_threshold_interrupt(struct host1x *host,
	90	struct list_head *head,
	91	unsigned int id)
	92	{
	93	u32 thresh =
	94	list_first_entry(head, struct host1x_waitlist, list)->thresh;
	95
	96	host1x_hw_intr_set_syncpt_threshold(host, id, thresh);
	97	host1x_hw_intr_enable_syncpt_intr(host, id);
	98	}
	99
6579324a TB	100	static void action_submit_complete(struct host1x_waitlist *waiter)
	101	{
	102	struct host1x_channel *channel = waiter->data;
	103
	104	host1x_cdma_update(&channel->cdma);
	105
	106	/* Add nr_completed to trace */
	107	trace_host1x_channel_submit_complete(dev_name(channel->dev),
	108	waiter->count, waiter->thresh);
6579324a TB	109	}
6579324a TB	110
7ede0b0b TB	111	static void action_wakeup(struct host1x_waitlist *waiter)
	112	{
	113	wait_queue_head_t *wq = waiter->data;
6df633d0	114
7ede0b0b TB	115	wake_up(wq);
	116	}
	117
	118	static void action_wakeup_interruptible(struct host1x_waitlist *waiter)
	119	{
	120	wait_queue_head_t *wq = waiter->data;
6df633d0	121
7ede0b0b TB	122	wake_up_interruptible(wq);
	123	}
	124
687db220 MP	125	static void action_signal_fence(struct host1x_waitlist *waiter)
	126	{
	127	struct host1x_syncpt_fence *f = waiter->data;
	128
	129	host1x_fence_signal(f);
	130	}
	131
7ede0b0b TB	132	typedef void (action_handler)(struct host1x_waitlist waiter);
7ede0b0b TB	133
9f2e57ca	134	static const action_handler action_handlers[HOST1X_INTR_ACTION_COUNT] = {
6579324a	135	action_submit_complete,
7ede0b0b TB	136	action_wakeup,
7ede0b0b TB	137	action_wakeup_interruptible,
687db220	138	action_signal_fence,
7ede0b0b TB	139	};
	140
	141	static void run_handlers(struct list_head completed[HOST1X_INTR_ACTION_COUNT])
	142	{
	143	struct list_head *head = completed;
d4ad3ad9	144	unsigned int i;
7ede0b0b TB	145
	146	for (i = 0; i < HOST1X_INTR_ACTION_COUNT; ++i, ++head) {
	147	action_handler handler = action_handlers[i];
	148	struct host1x_waitlist waiter, next;
	149
	150	list_for_each_entry_safe(waiter, next, head, list) {
	151	list_del(&waiter->list);
	152	handler(waiter);
	153	WARN_ON(atomic_xchg(&waiter->state, WLS_HANDLED) !=
	154	WLS_REMOVED);
	155	kref_put(&waiter->refcount, waiter_release);
	156	}
	157	}
	158	}
	159
	160	/*
	161	* Remove & handle all waiters that have completed for the given syncpt
	162	*/
	163	static int process_wait_list(struct host1x *host,
	164	struct host1x_syncpt *syncpt,
	165	u32 threshold)
	166	{
	167	struct list_head completed[HOST1X_INTR_ACTION_COUNT];
	168	unsigned int i;
	169	int empty;
	170
	171	for (i = 0; i < HOST1X_INTR_ACTION_COUNT; ++i)
	172	INIT_LIST_HEAD(completed + i);
	173
	174	spin_lock(&syncpt->intr.lock);
	175
	176	remove_completed_waiters(&syncpt->intr.wait_head, threshold,
	177	completed);
	178
	179	empty = list_empty(&syncpt->intr.wait_head);
	180	if (empty)
	181	host1x_hw_intr_disable_syncpt_intr(host, syncpt->id);
	182	else
	183	reset_threshold_interrupt(host, &syncpt->intr.wait_head,
	184	syncpt->id);
	185
	186	spin_unlock(&syncpt->intr.lock);
	187
	188	run_handlers(completed);
	189
	190	return empty;
	191	}
	192
	193	/*
	194	* Sync point threshold interrupt service thread function
	195	* Handles sync point threshold triggers, in thread context
	196	*/
	197
	198	static void syncpt_thresh_work(struct work_struct *work)
	199	{
	200	struct host1x_syncpt_intr *syncpt_intr =
	201	container_of(work, struct host1x_syncpt_intr, work);
	202	struct host1x_syncpt *syncpt =
	203	container_of(syncpt_intr, struct host1x_syncpt, intr);
	204	unsigned int id = syncpt->id;
	205	struct host1x *host = syncpt->host;
	206
	207	(void)process_wait_list(host, syncpt,
	208	host1x_syncpt_load(host->syncpt + id));
209	}
210
ac330f45 TR	211	int host1x_intr_add_action(struct host1x host, struct host1x_syncpt syncpt,
	212	u32 thresh, enum host1x_intr_action action,
	213	void data, struct host1x_waitlist waiter,
	214	void **ref)
7ede0b0b	215	{
7ede0b0b TB	216	int queue_was_empty;
	217
	218	if (waiter == NULL) {
	219	pr_warn("%s: NULL waiter\n", __func__);
	220	return -EINVAL;
	221	}
	222
	223	/* initialize a new waiter */
	224	INIT_LIST_HEAD(&waiter->list);
	225	kref_init(&waiter->refcount);
	226	if (ref)
	227	kref_get(&waiter->refcount);
	228	waiter->thresh = thresh;
	229	waiter->action = action;
	230	atomic_set(&waiter->state, WLS_PENDING);
	231	waiter->data = data;
	232	waiter->count = 1;
	233
7ede0b0b TB	234	spin_lock(&syncpt->intr.lock);
	235
	236	queue_was_empty = list_empty(&syncpt->intr.wait_head);
	237
	238	if (add_waiter_to_queue(waiter, &syncpt->intr.wait_head)) {
	239	/* added at head of list - new threshold value */
ac330f45	240	host1x_hw_intr_set_syncpt_threshold(host, syncpt->id, thresh);
7ede0b0b TB	241
	242	/* added as first waiter - enable interrupt */
	243	if (queue_was_empty)
ac330f45	244	host1x_hw_intr_enable_syncpt_intr(host, syncpt->id);
7ede0b0b TB	245	}
7ede0b0b TB	246
7ede0b0b TB	247	if (ref)
7ede0b0b TB	248	*ref = waiter;
5a8d95d2 MP	249
	250	spin_unlock(&syncpt->intr.lock);
	251
7ede0b0b TB	252	return 0;
	253	}
	254
ecfb888a MP	255	void host1x_intr_put_ref(struct host1x host, unsigned int id, void ref,
ecfb888a MP	256	bool flush)
7ede0b0b TB	257	{
	258	struct host1x_waitlist *waiter = ref;
	259	struct host1x_syncpt *syncpt;
	260
ecfb888a	261	atomic_cmpxchg(&waiter->state, WLS_PENDING, WLS_CANCELLED);
7ede0b0b TB	262
7ede0b0b TB	263	syncpt = host->syncpt + id;
ecfb888a MP	264
	265	spin_lock(&syncpt->intr.lock);
	266	if (atomic_cmpxchg(&waiter->state, WLS_CANCELLED, WLS_HANDLED) ==
	267	WLS_CANCELLED) {
	268	list_del(&waiter->list);
	269	kref_put(&waiter->refcount, waiter_release);
	270	}
	271	spin_unlock(&syncpt->intr.lock);
	272
	273	if (flush) {
	274	/* Wait until any concurrently executing handler has finished. */
	275	while (atomic_read(&waiter->state) != WLS_HANDLED)
	276	schedule();
	277	}
7ede0b0b TB	278
	279	kref_put(&waiter->refcount, waiter_release);
	280	}
	281
	282	int host1x_intr_init(struct host1x *host, unsigned int irq_sync)
	283	{
	284	unsigned int id;
	285	u32 nb_pts = host1x_syncpt_nb_pts(host);
	286
	287	mutex_init(&host->intr_mutex);
	288	host->intr_syncpt_irq = irq_sync;
7ede0b0b TB	289
	290	for (id = 0; id < nb_pts; ++id) {
	291	struct host1x_syncpt *syncpt = host->syncpt + id;
	292
	293	spin_lock_init(&syncpt->intr.lock);
	294	INIT_LIST_HEAD(&syncpt->intr.wait_head);
	295	snprintf(syncpt->intr.thresh_irq_name,
	296	sizeof(syncpt->intr.thresh_irq_name),
5c0d8d38	297	"host1x_sp_%02u", id);
7ede0b0b TB	298	}
7ede0b0b TB	299
7ede0b0b TB	300	return 0;
	301	}
	302
	303	void host1x_intr_deinit(struct host1x *host)
	304	{
7ede0b0b TB	305	}
	306
	307	void host1x_intr_start(struct host1x *host)
	308	{
	309	u32 hz = clk_get_rate(host->clk);
	310	int err;
	311
	312	mutex_lock(&host->intr_mutex);
	313	err = host1x_hw_intr_init_host_sync(host, DIV_ROUND_UP(hz, 1000000),
	314	syncpt_thresh_work);
	315	if (err) {
	316	mutex_unlock(&host->intr_mutex);
	317	return;
	318	}
	319	mutex_unlock(&host->intr_mutex);
	320	}
	321
	322	void host1x_intr_stop(struct host1x *host)
	323	{
	324	unsigned int id;
	325	struct host1x_syncpt *syncpt = host->syncpt;
	326	u32 nb_pts = host1x_syncpt_nb_pts(host);
	327
	328	mutex_lock(&host->intr_mutex);
	329
	330	host1x_hw_intr_disable_all_syncpt_intrs(host);
	331
	332	for (id = 0; id < nb_pts; ++id) {
	333	struct host1x_waitlist waiter, next;
	334
	335	list_for_each_entry_safe(waiter, next,
	336	&syncpt[id].intr.wait_head, list) {
	337	if (atomic_cmpxchg(&waiter->state,
	338	WLS_CANCELLED, WLS_HANDLED) == WLS_CANCELLED) {
	339	list_del(&waiter->list);
	340	kref_put(&waiter->refcount, waiter_release);
	341	}
	342	}
	343
	344	if (!list_empty(&syncpt[id].intr.wait_head)) {
	345	/* output diagnostics */
	346	mutex_unlock(&host->intr_mutex);
5c0d8d38	347	pr_warn("%s cannot stop syncpt intr id=%u\n",
7ede0b0b TB	348	__func__, id);
	349	return;
	350	}
	351	}
	352
	353	host1x_hw_intr_free_syncpt_irq(host);
	354
	355	mutex_unlock(&host->intr_mutex);
	356	}