[linux-2.6-block.git] / drivers / gpu / host1x / cdma.c

/*
 * Tegra host1x Command DMA
 *
 * Copyright (c) 2010-2013, NVIDIA Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */


#include <asm/cacheflush.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
#include <linux/kfifo.h>
#include <linux/slab.h>
#include <trace/events/host1x.h>

#include "cdma.h"
#include "channel.h"
#include "dev.h"
#include "debug.h"
#include "host1x_bo.h"
#include "job.h"

/*
 * push_buffer
 *
 * The push buffer is a circular array of words to be fetched by command DMA.
 * Note that it works slightly differently to the sync queue; fence == pos
 * means that the push buffer is full, not empty.
 */

#define HOST1X_PUSHBUFFER_SLOTS	512

/*
 * Clean up push buffer resources
 */
static void host1x_pushbuffer_destroy(struct push_buffer *pb)
{
	struct host1x_cdma *cdma = pb_to_cdma(pb);
	struct host1x *host1x = cdma_to_host1x(cdma);

	if (pb->phys != 0)
		dma_free_writecombine(host1x->dev, pb->size_bytes + 4,
				      pb->mapped, pb->phys);

	pb->mapped = NULL;
	pb->phys = 0;
}

/*
 * Init push buffer resources
 */
static int host1x_pushbuffer_init(struct push_buffer *pb)
{
	struct host1x_cdma *cdma = pb_to_cdma(pb);
	struct host1x *host1x = cdma_to_host1x(cdma);

	pb->mapped = NULL;
	pb->phys = 0;
	pb->size_bytes = HOST1X_PUSHBUFFER_SLOTS * 8;

	/* initialize buffer pointers */
	pb->fence = pb->size_bytes - 8;
	pb->pos = 0;

	/* allocate and map pushbuffer memory */
	pb->mapped = dma_alloc_writecombine(host1x->dev, pb->size_bytes + 4,
					    &pb->phys, GFP_KERNEL);
	if (!pb->mapped)
		goto fail;

	host1x_hw_pushbuffer_init(host1x, pb);

	return 0;

fail:
	host1x_pushbuffer_destroy(pb);
	return -ENOMEM;
}

/*
 * Push two words to the push buffer
 * Caller must ensure push buffer is not full
 */
static void host1x_pushbuffer_push(struct push_buffer *pb, u32 op1, u32 op2)
{
	u32 pos = pb->pos;
	u32 *p = (u32 *)((u32)pb->mapped + pos);
	WARN_ON(pos == pb->fence);
	*(p++) = op1;
	*(p++) = op2;
	pb->pos = (pos + 8) & (pb->size_bytes - 1);
}

/*
 * Pop a number of two word slots from the push buffer
 * Caller must ensure push buffer is not empty
 */
static void host1x_pushbuffer_pop(struct push_buffer *pb, unsigned int slots)
{
	/* Advance the next write position */
	pb->fence = (pb->fence + slots * 8) & (pb->size_bytes - 1);
}

/*
 * Return the number of two word slots free in the push buffer
 */
static u32 host1x_pushbuffer_space(struct push_buffer *pb)
{
	return ((pb->fence - pb->pos) & (pb->size_bytes - 1)) / 8;
}

/*
 * Sleep (if necessary) until the requested event happens
 *   - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty.
 *     - Returns 1
 *   - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer
 *     - Return the amount of space (> 0)
 * Must be called with the cdma lock held.
 */
unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma,
				     enum cdma_event event)
{
	for (;;) {
		unsigned int space;

		if (event == CDMA_EVENT_SYNC_QUEUE_EMPTY)
			space = list_empty(&cdma->sync_queue) ? 1 : 0;
		else if (event == CDMA_EVENT_PUSH_BUFFER_SPACE) {
			struct push_buffer *pb = &cdma->push_buffer;
			space = host1x_pushbuffer_space(pb);
		} else {
			WARN_ON(1);
			return -EINVAL;
		}

		if (space)
			return space;

		trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev),
				       event);

		/* If somebody has managed to already start waiting, yield */
		if (cdma->event != CDMA_EVENT_NONE) {
			mutex_unlock(&cdma->lock);
			schedule();
			mutex_lock(&cdma->lock);
			continue;
		}
		cdma->event = event;

		mutex_unlock(&cdma->lock);
		down(&cdma->sem);
		mutex_lock(&cdma->lock);
	}
	return 0;
}

/*
 * Start timer that tracks the time spent by the job.
 * Must be called with the cdma lock held.
 */
static void cdma_start_timer_locked(struct host1x_cdma *cdma,
				    struct host1x_job *job)
{
	struct host1x *host = cdma_to_host1x(cdma);

	if (cdma->timeout.client) {
		/* timer already started */
		return;
	}

	cdma->timeout.client = job->client;
	cdma->timeout.syncpt = host1x_syncpt_get(host, job->syncpt_id);
	cdma->timeout.syncpt_val = job->syncpt_end;
	cdma->timeout.start_ktime = ktime_get();

	schedule_delayed_work(&cdma->timeout.wq,
			      msecs_to_jiffies(job->timeout));
}

/*
 * Stop timer when a buffer submission completes.
 * Must be called with the cdma lock held.
 */
static void stop_cdma_timer_locked(struct host1x_cdma *cdma)
{
	cancel_delayed_work(&cdma->timeout.wq);
	cdma->timeout.client = 0;
}

/*
 * For all sync queue entries that have already finished according to the
 * current sync point registers:
 *  - unpin & unref their mems
 *  - pop their push buffer slots
 *  - remove them from the sync queue
 * This is normally called from the host code's worker thread, but can be
 * called manually if necessary.
 * Must be called with the cdma lock held.
 */
static void update_cdma_locked(struct host1x_cdma *cdma)
{
	bool signal = false;
	struct host1x *host1x = cdma_to_host1x(cdma);
	struct host1x_job *job, *n;

	/* If CDMA is stopped, queue is cleared and we can return */
	if (!cdma->running)
		return;

	/*
	 * Walk the sync queue, reading the sync point registers as necessary,
	 * to consume as many sync queue entries as possible without blocking
	 */
	list_for_each_entry_safe(job, n, &cdma->sync_queue, list) {
		struct host1x_syncpt *sp =
			host1x_syncpt_get(host1x, job->syncpt_id);

		/* Check whether this syncpt has completed, and bail if not */
		if (!host1x_syncpt_is_expired(sp, job->syncpt_end)) {
			/* Start timer on next pending syncpt */
			if (job->timeout)
				cdma_start_timer_locked(cdma, job);
			break;
		}

		/* Cancel timeout, when a buffer completes */
		if (cdma->timeout.client)
			stop_cdma_timer_locked(cdma);

		/* Unpin the memory */
		host1x_job_unpin(job);

		/* Pop push buffer slots */
		if (job->num_slots) {
			struct push_buffer *pb = &cdma->push_buffer;
			host1x_pushbuffer_pop(pb, job->num_slots);
			if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE)
				signal = true;
		}

		list_del(&job->list);
		host1x_job_put(job);
	}

	if (cdma->event == CDMA_EVENT_SYNC_QUEUE_EMPTY &&
	    list_empty(&cdma->sync_queue))
		signal = true;

	if (signal) {
		cdma->event = CDMA_EVENT_NONE;
		up(&cdma->sem);
	}
}

void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma,
				   struct device *dev)
{
	u32 restart_addr;
	u32 syncpt_incrs;
	struct host1x_job *job = NULL;
	u32 syncpt_val;
	struct host1x *host1x = cdma_to_host1x(cdma);

	syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);

	dev_dbg(dev, "%s: starting cleanup (thresh %d)\n",
		__func__, syncpt_val);

	/*
	 * Move the sync_queue read pointer to the first entry that hasn't
	 * completed based on the current HW syncpt value. It's likely there
	 * won't be any (i.e. we're still at the head), but covers the case
	 * where a syncpt incr happens just prior/during the teardown.
	 */

	dev_dbg(dev, "%s: skip completed buffers still in sync_queue\n",
		__func__);

	list_for_each_entry(job, &cdma->sync_queue, list) {
		if (syncpt_val < job->syncpt_end)
			break;

		host1x_job_dump(dev, job);
	}

	/*
	 * Walk the sync_queue, first incrementing with the CPU syncpts that
	 * are partially executed (the first buffer) or fully skipped while
	 * still in the current context (slots are also NOP-ed).
	 *
	 * At the point contexts are interleaved, syncpt increments must be
	 * done inline with the pushbuffer from a GATHER buffer to maintain
	 * the order (slots are modified to be a GATHER of syncpt incrs).
	 *
	 * Note: save in restart_addr the location where the timed out buffer
	 * started in the PB, so we can start the refetch from there (with the
	 * modified NOP-ed PB slots). This lets things appear to have completed
	 * properly for this buffer and resources are freed.
	 */

	dev_dbg(dev, "%s: perform CPU incr on pending same ctx buffers\n",
		__func__);

	if (!list_empty(&cdma->sync_queue))
		restart_addr = job->first_get;
	else
		restart_addr = cdma->last_pos;

	/* do CPU increments as long as this context continues */
	list_for_each_entry_from(job, &cdma->sync_queue, list) {
		/* different context, gets us out of this loop */
		if (job->client != cdma->timeout.client)
			break;

		/* won't need a timeout when replayed */
		job->timeout = 0;

		syncpt_incrs = job->syncpt_end - syncpt_val;
		dev_dbg(dev, "%s: CPU incr (%d)\n", __func__, syncpt_incrs);

		host1x_job_dump(dev, job);

		/* safe to use CPU to incr syncpts */
		host1x_hw_cdma_timeout_cpu_incr(host1x, cdma, job->first_get,
						syncpt_incrs, job->syncpt_end,
						job->num_slots);

		syncpt_val += syncpt_incrs;
	}

	/* The following sumbits from the same client may be dependent on the
	 * failed submit and therefore they may fail. Force a small timeout
	 * to make the queue cleanup faster */

	list_for_each_entry_from(job, &cdma->sync_queue, list)
		if (job->client == cdma->timeout.client)
			job->timeout = min_t(unsigned int, job->timeout, 500);

	dev_dbg(dev, "%s: finished sync_queue modification\n", __func__);

	/* roll back DMAGET and start up channel again */
	host1x_hw_cdma_resume(host1x, cdma, restart_addr);
}

/*
 * Create a cdma
 */
int host1x_cdma_init(struct host1x_cdma *cdma)
{
	int err;

	mutex_init(&cdma->lock);
	sema_init(&cdma->sem, 0);

	INIT_LIST_HEAD(&cdma->sync_queue);

	cdma->event = CDMA_EVENT_NONE;
	cdma->running = false;
	cdma->torndown = false;

	err = host1x_pushbuffer_init(&cdma->push_buffer);
	if (err)
		return err;
	return 0;
}

/*
 * Destroy a cdma
 */
int host1x_cdma_deinit(struct host1x_cdma *cdma)
{
	struct push_buffer *pb = &cdma->push_buffer;
	struct host1x *host1x = cdma_to_host1x(cdma);

	if (cdma->running) {
		pr_warn("%s: CDMA still running\n", __func__);
		return -EBUSY;
	}

	host1x_pushbuffer_destroy(pb);
	host1x_hw_cdma_timeout_destroy(host1x, cdma);

	return 0;
}

/*
 * Begin a cdma submit
 */
int host1x_cdma_begin(struct host1x_cdma *cdma, struct host1x_job *job)
{
	struct host1x *host1x = cdma_to_host1x(cdma);

	mutex_lock(&cdma->lock);

	if (job->timeout) {
		/* init state on first submit with timeout value */
		if (!cdma->timeout.initialized) {
			int err;
			err = host1x_hw_cdma_timeout_init(host1x, cdma,
							  job->syncpt_id);
			if (err) {
				mutex_unlock(&cdma->lock);
				return err;
			}
		}
	}
	if (!cdma->running)
		host1x_hw_cdma_start(host1x, cdma);

	cdma->slots_free = 0;
	cdma->slots_used = 0;
	cdma->first_get = cdma->push_buffer.pos;

	trace_host1x_cdma_begin(dev_name(job->channel->dev));
	return 0;
}

/*
 * Push two words into a push buffer slot
 * Blocks as necessary if the push buffer is full.
 */
void host1x_cdma_push(struct host1x_cdma *cdma, u32 op1, u32 op2)
{
	struct host1x *host1x = cdma_to_host1x(cdma);
	struct push_buffer *pb = &cdma->push_buffer;
	u32 slots_free = cdma->slots_free;

	if (host1x_debug_trace_cmdbuf)
		trace_host1x_cdma_push(dev_name(cdma_to_channel(cdma)->dev),
				       op1, op2);

	if (slots_free == 0) {
		host1x_hw_cdma_flush(host1x, cdma);
		slots_free = host1x_cdma_wait_locked(cdma,
						CDMA_EVENT_PUSH_BUFFER_SPACE);
	}
	cdma->slots_free = slots_free - 1;
	cdma->slots_used++;
	host1x_pushbuffer_push(pb, op1, op2);
}

/*
 * End a cdma submit
 * Kick off DMA, add job to the sync queue, and a number of slots to be freed
 * from the pushbuffer. The handles for a submit must all be pinned at the same
 * time, but they can be unpinned in smaller chunks.
 */
void host1x_cdma_end(struct host1x_cdma *cdma,
		     struct host1x_job *job)
{
	struct host1x *host1x = cdma_to_host1x(cdma);
	bool idle = list_empty(&cdma->sync_queue);

	host1x_hw_cdma_flush(host1x, cdma);

	job->first_get = cdma->first_get;
	job->num_slots = cdma->slots_used;
	host1x_job_get(job);
	list_add_tail(&job->list, &cdma->sync_queue);

	/* start timer on idle -> active transitions */
	if (job->timeout && idle)
		cdma_start_timer_locked(cdma, job);

	trace_host1x_cdma_end(dev_name(job->channel->dev));
	mutex_unlock(&cdma->lock);
}

/*
 * Update cdma state according to current sync point values
 */
void host1x_cdma_update(struct host1x_cdma *cdma)
{
	mutex_lock(&cdma->lock);
	update_cdma_locked(cdma);
	mutex_unlock(&cdma->lock);
}
Commit	Line	Data
6579324a TB	1	/*
	2	* Tegra host1x Command DMA
	3	*
	4	* Copyright (c) 2010-2013, NVIDIA Corporation.
	5	*
	6	* This program is free software; you can redistribute it and/or modify it
	7	* under the terms and conditions of the GNU General Public License,
	8	* version 2, as published by the Free Software Foundation.
	9	*
	10	* This program is distributed in the hope it will be useful, but WITHOUT
	11	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	12	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
	13	* more details.
	14	*
	15	* You should have received a copy of the GNU General Public License
	16	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	17	*/
	18
	19
	20	#include <asm/cacheflush.h>
	21	#include <linux/device.h>
	22	#include <linux/dma-mapping.h>
	23	#include <linux/interrupt.h>
	24	#include <linux/kernel.h>
	25	#include <linux/kfifo.h>
	26	#include <linux/slab.h>
	27	#include <trace/events/host1x.h>
	28
	29	#include "cdma.h"
	30	#include "channel.h"
	31	#include "dev.h"
	32	#include "debug.h"
	33	#include "host1x_bo.h"
	34	#include "job.h"
	35
	36	/*
	37	* push_buffer
	38	*
	39	* The push buffer is a circular array of words to be fetched by command DMA.
	40	* Note that it works slightly differently to the sync queue; fence == pos
	41	* means that the push buffer is full, not empty.
	42	*/
	43
	44	#define HOST1X_PUSHBUFFER_SLOTS 512
	45
	46	/*
	47	* Clean up push buffer resources
	48	*/
	49	static void host1x_pushbuffer_destroy(struct push_buffer *pb)
	50	{
	51	struct host1x_cdma *cdma = pb_to_cdma(pb);
	52	struct host1x *host1x = cdma_to_host1x(cdma);
	53
	54	if (pb->phys != 0)
	55	dma_free_writecombine(host1x->dev, pb->size_bytes + 4,
	56	pb->mapped, pb->phys);
	57
	58	pb->mapped = NULL;
	59	pb->phys = 0;
	60	}
	61
	62	/*
	63	* Init push buffer resources
	64	*/
65	static int host1x_pushbuffer_init(struct push_buffer *pb)
66	{
67	struct host1x_cdma *cdma = pb_to_cdma(pb);
68	struct host1x *host1x = cdma_to_host1x(cdma);
69
70	pb->mapped = NULL;
71	pb->phys = 0;
72	pb->size_bytes = HOST1X_PUSHBUFFER_SLOTS * 8;
73
74	/* initialize buffer pointers */
75	pb->fence = pb->size_bytes - 8;
76	pb->pos = 0;
77
78	/* allocate and map pushbuffer memory */
79	pb->mapped = dma_alloc_writecombine(host1x->dev, pb->size_bytes + 4,
80	&pb->phys, GFP_KERNEL);
81	if (!pb->mapped)
82	goto fail;
83
84	host1x_hw_pushbuffer_init(host1x, pb);
85
86	return 0;
87
88	fail:
89	host1x_pushbuffer_destroy(pb);
90	return -ENOMEM;
91	}
92
93	/*
94	* Push two words to the push buffer
95	* Caller must ensure push buffer is not full
96	*/
97	static void host1x_pushbuffer_push(struct push_buffer *pb, u32 op1, u32 op2)
98	{
99	u32 pos = pb->pos;
100	u32 p = (u32 )((u32)pb->mapped + pos);
101	WARN_ON(pos == pb->fence);
102	*(p++) = op1;
103	*(p++) = op2;
104	pb->pos = (pos + 8) & (pb->size_bytes - 1);
105	}
106
107	/*
108	* Pop a number of two word slots from the push buffer
109	* Caller must ensure push buffer is not empty
110	*/
111	static void host1x_pushbuffer_pop(struct push_buffer *pb, unsigned int slots)
112	{
113	/* Advance the next write position */
114	pb->fence = (pb->fence + slots * 8) & (pb->size_bytes - 1);
115	}
116
117	/*
118	* Return the number of two word slots free in the push buffer
119	*/
120	static u32 host1x_pushbuffer_space(struct push_buffer *pb)
121	{
122	return ((pb->fence - pb->pos) & (pb->size_bytes - 1)) / 8;
123	}
124
125	/*
126	* Sleep (if necessary) until the requested event happens
127	* - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty.
128	* - Returns 1
129	* - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer
130	* - Return the amount of space (> 0)
131	* Must be called with the cdma lock held.
132	*/
133	unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma,
134	enum cdma_event event)
135	{
136	for (;;) {
137	unsigned int space;
138
139	if (event == CDMA_EVENT_SYNC_QUEUE_EMPTY)
140	space = list_empty(&cdma->sync_queue) ? 1 : 0;
141	else if (event == CDMA_EVENT_PUSH_BUFFER_SPACE) {
142	struct push_buffer *pb = &cdma->push_buffer;
143	space = host1x_pushbuffer_space(pb);
144	} else {
145	WARN_ON(1);
146	return -EINVAL;
147	}
148
149	if (space)
150	return space;
151
152	trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev),
153	event);
154
155	/* If somebody has managed to already start waiting, yield */
156	if (cdma->event != CDMA_EVENT_NONE) {
157	mutex_unlock(&cdma->lock);
158	schedule();
159	mutex_lock(&cdma->lock);
160	continue;
161	}
162	cdma->event = event;
163
164	mutex_unlock(&cdma->lock);
165	down(&cdma->sem);
166	mutex_lock(&cdma->lock);
167	}
168	return 0;
169	}
170
171	/*
172	* Start timer that tracks the time spent by the job.
173	* Must be called with the cdma lock held.
174	*/
175	static void cdma_start_timer_locked(struct host1x_cdma *cdma,
176	struct host1x_job *job)
177	{
178	struct host1x *host = cdma_to_host1x(cdma);
179
180	if (cdma->timeout.client) {
181	/* timer already started */
182	return;
183	}
184
185	cdma->timeout.client = job->client;
186	cdma->timeout.syncpt = host1x_syncpt_get(host, job->syncpt_id);
187	cdma->timeout.syncpt_val = job->syncpt_end;
188	cdma->timeout.start_ktime = ktime_get();
189
190	schedule_delayed_work(&cdma->timeout.wq,
191	msecs_to_jiffies(job->timeout));
192	}
193
194	/*
195	* Stop timer when a buffer submission completes.
196	* Must be called with the cdma lock held.
197	*/
198	static void stop_cdma_timer_locked(struct host1x_cdma *cdma)
199	{
200	cancel_delayed_work(&cdma->timeout.wq);
201	cdma->timeout.client = 0;
202	}
203
204	/*
205	* For all sync queue entries that have already finished according to the
206	* current sync point registers:
207	* - unpin & unref their mems
208	* - pop their push buffer slots
209	* - remove them from the sync queue
210	* This is normally called from the host code's worker thread, but can be
211	* called manually if necessary.
212	* Must be called with the cdma lock held.
213	*/
214	static void update_cdma_locked(struct host1x_cdma *cdma)
215	{
216	bool signal = false;
217	struct host1x *host1x = cdma_to_host1x(cdma);
218	struct host1x_job job, n;
219
220	/* If CDMA is stopped, queue is cleared and we can return */
221	if (!cdma->running)
222	return;
223
224	/*
225	* Walk the sync queue, reading the sync point registers as necessary,
226	* to consume as many sync queue entries as possible without blocking
227	*/
228	list_for_each_entry_safe(job, n, &cdma->sync_queue, list) {
229	struct host1x_syncpt *sp =
230	host1x_syncpt_get(host1x, job->syncpt_id);
231
232	/* Check whether this syncpt has completed, and bail if not */
233	if (!host1x_syncpt_is_expired(sp, job->syncpt_end)) {
234	/* Start timer on next pending syncpt */
235	if (job->timeout)
236	cdma_start_timer_locked(cdma, job);
237	break;
238	}
239
240	/* Cancel timeout, when a buffer completes */
241	if (cdma->timeout.client)
242	stop_cdma_timer_locked(cdma);
243
244	/* Unpin the memory */
245	host1x_job_unpin(job);
246
247	/* Pop push buffer slots */
248	if (job->num_slots) {
249	struct push_buffer *pb = &cdma->push_buffer;
250	host1x_pushbuffer_pop(pb, job->num_slots);
251	if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE)
252	signal = true;
253	}
254
255	list_del(&job->list);
256	host1x_job_put(job);
257	}
258
259	if (cdma->event == CDMA_EVENT_SYNC_QUEUE_EMPTY &&
260	list_empty(&cdma->sync_queue))
261	signal = true;
262
263	if (signal) {
264	cdma->event = CDMA_EVENT_NONE;
265	up(&cdma->sem);
266	}
267	}
268
269	void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma,
270	struct device *dev)
271	{
272	u32 restart_addr;
273	u32 syncpt_incrs;
274	struct host1x_job *job = NULL;
275	u32 syncpt_val;
276	struct host1x *host1x = cdma_to_host1x(cdma);
277
278	syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);
279
280	dev_dbg(dev, "%s: starting cleanup (thresh %d)\n",
281	__func__, syncpt_val);
282
283	/*
284	* Move the sync_queue read pointer to the first entry that hasn't
285	* completed based on the current HW syncpt value. It's likely there
286	* won't be any (i.e. we're still at the head), but covers the case
287	* where a syncpt incr happens just prior/during the teardown.
288	*/
289
290	dev_dbg(dev, "%s: skip completed buffers still in sync_queue\n",
291	__func__);
292
293	list_for_each_entry(job, &cdma->sync_queue, list) {
294	if (syncpt_val < job->syncpt_end)
295	break;
296
297	host1x_job_dump(dev, job);
298	}
299
300	/*
301	* Walk the sync_queue, first incrementing with the CPU syncpts that
302	* are partially executed (the first buffer) or fully skipped while
303	* still in the current context (slots are also NOP-ed).
304	*
305	* At the point contexts are interleaved, syncpt increments must be
306	* done inline with the pushbuffer from a GATHER buffer to maintain
307	* the order (slots are modified to be a GATHER of syncpt incrs).
308	*
309	* Note: save in restart_addr the location where the timed out buffer
310	* started in the PB, so we can start the refetch from there (with the
311	* modified NOP-ed PB slots). This lets things appear to have completed
312	* properly for this buffer and resources are freed.
313	*/
314
315	dev_dbg(dev, "%s: perform CPU incr on pending same ctx buffers\n",
316	__func__);
317
318	if (!list_empty(&cdma->sync_queue))
319	restart_addr = job->first_get;
320	else
321	restart_addr = cdma->last_pos;
322
323	/* do CPU increments as long as this context continues */
324	list_for_each_entry_from(job, &cdma->sync_queue, list) {
325	/* different context, gets us out of this loop */
326	if (job->client != cdma->timeout.client)
327	break;
328
329	/* won't need a timeout when replayed */
330	job->timeout = 0;
331
332	syncpt_incrs = job->syncpt_end - syncpt_val;
333	dev_dbg(dev, "%s: CPU incr (%d)\n", __func__, syncpt_incrs);
334
335	host1x_job_dump(dev, job);
336
337	/* safe to use CPU to incr syncpts */
338	host1x_hw_cdma_timeout_cpu_incr(host1x, cdma, job->first_get,
339	syncpt_incrs, job->syncpt_end,
340	job->num_slots);
341
342	syncpt_val += syncpt_incrs;
343	}
344
345	/* The following sumbits from the same client may be dependent on the
346	* failed submit and therefore they may fail. Force a small timeout
347	* to make the queue cleanup faster */
348
349	list_for_each_entry_from(job, &cdma->sync_queue, list)
350	if (job->client == cdma->timeout.client)
351	job->timeout = min_t(unsigned int, job->timeout, 500);
352
353	dev_dbg(dev, "%s: finished sync_queue modification\n", __func__);
354
355	/* roll back DMAGET and start up channel again */
356	host1x_hw_cdma_resume(host1x, cdma, restart_addr);
357	}
358
359	/*
360	* Create a cdma
361	*/
362	int host1x_cdma_init(struct host1x_cdma *cdma)
363	{
364	int err;
365
366	mutex_init(&cdma->lock);
367	sema_init(&cdma->sem, 0);
368
369	INIT_LIST_HEAD(&cdma->sync_queue);
370
371	cdma->event = CDMA_EVENT_NONE;
372	cdma->running = false;
373	cdma->torndown = false;
374
375	err = host1x_pushbuffer_init(&cdma->push_buffer);
376	if (err)
377	return err;
378	return 0;
379	}
380
381	/*
382	* Destroy a cdma
383	*/
384	int host1x_cdma_deinit(struct host1x_cdma *cdma)
385	{
386	struct push_buffer *pb = &cdma->push_buffer;
387	struct host1x *host1x = cdma_to_host1x(cdma);
388
389	if (cdma->running) {
390	pr_warn("%s: CDMA still running\n", __func__);
391	return -EBUSY;
392	}
393
394	host1x_pushbuffer_destroy(pb);
395	host1x_hw_cdma_timeout_destroy(host1x, cdma);
396
397	return 0;
398	}
399
400	/*
401	* Begin a cdma submit
402	*/
403	int host1x_cdma_begin(struct host1x_cdma cdma, struct host1x_job job)
404	{
405	struct host1x *host1x = cdma_to_host1x(cdma);
406
407	mutex_lock(&cdma->lock);
408
409	if (job->timeout) {
410	/* init state on first submit with timeout value */
411	if (!cdma->timeout.initialized) {
412	int err;
413	err = host1x_hw_cdma_timeout_init(host1x, cdma,
414	job->syncpt_id);
415	if (err) {
416	mutex_unlock(&cdma->lock);
417	return err;
418	}
419	}
420	}
421	if (!cdma->running)
422	host1x_hw_cdma_start(host1x, cdma);
423
424	cdma->slots_free = 0;
425	cdma->slots_used = 0;
426	cdma->first_get = cdma->push_buffer.pos;
427
428	trace_host1x_cdma_begin(dev_name(job->channel->dev));
429	return 0;
430	}
431
432	/*
433	* Push two words into a push buffer slot
434	* Blocks as necessary if the push buffer is full.
435	*/
436	void host1x_cdma_push(struct host1x_cdma *cdma, u32 op1, u32 op2)
437	{
438	struct host1x *host1x = cdma_to_host1x(cdma);
439	struct push_buffer *pb = &cdma->push_buffer;
440	u32 slots_free = cdma->slots_free;
441
6236451d TB	442	if (host1x_debug_trace_cmdbuf)
	443	trace_host1x_cdma_push(dev_name(cdma_to_channel(cdma)->dev),
	444	op1, op2);
	445
6579324a TB	446	if (slots_free == 0) {
	447	host1x_hw_cdma_flush(host1x, cdma);
	448	slots_free = host1x_cdma_wait_locked(cdma,
	449	CDMA_EVENT_PUSH_BUFFER_SPACE);
	450	}
	451	cdma->slots_free = slots_free - 1;
	452	cdma->slots_used++;
	453	host1x_pushbuffer_push(pb, op1, op2);
	454	}
	455
	456	/*
	457	* End a cdma submit
	458	* Kick off DMA, add job to the sync queue, and a number of slots to be freed
	459	* from the pushbuffer. The handles for a submit must all be pinned at the same
	460	* time, but they can be unpinned in smaller chunks.
	461	*/
	462	void host1x_cdma_end(struct host1x_cdma *cdma,
	463	struct host1x_job *job)
	464	{
	465	struct host1x *host1x = cdma_to_host1x(cdma);
	466	bool idle = list_empty(&cdma->sync_queue);
	467
	468	host1x_hw_cdma_flush(host1x, cdma);
	469
	470	job->first_get = cdma->first_get;
	471	job->num_slots = cdma->slots_used;
	472	host1x_job_get(job);
	473	list_add_tail(&job->list, &cdma->sync_queue);
	474
	475	/* start timer on idle -> active transitions */
	476	if (job->timeout && idle)
	477	cdma_start_timer_locked(cdma, job);
	478
	479	trace_host1x_cdma_end(dev_name(job->channel->dev));
	480	mutex_unlock(&cdma->lock);
	481	}
	482
	483	/*
	484	* Update cdma state according to current sync point values
	485	*/
	486	void host1x_cdma_update(struct host1x_cdma *cdma)
	487	{
	488	mutex_lock(&cdma->lock);
	489	update_cdma_locked(cdma);
	490	mutex_unlock(&cdma->lock);
	491	}