[linux-block.git] / drivers / gpu / host1x / cdma.c

/*
 * Tegra host1x Command DMA
 *
 * Copyright (c) 2010-2013, NVIDIA Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */


#include <asm/cacheflush.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
#include <linux/host1x.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
#include <linux/kfifo.h>
#include <linux/slab.h>
#include <trace/events/host1x.h>

#include "cdma.h"
#include "channel.h"
#include "dev.h"
#include "debug.h"
#include "job.h"

/*
 * push_buffer
 *
 * The push buffer is a circular array of words to be fetched by command DMA.
 * Note that it works slightly differently to the sync queue; fence == pos
 * means that the push buffer is full, not empty.
 */

/*
 * Typically the commands written into the push buffer are a pair of words. We
 * use slots to represent each of these pairs and to simplify things. Note the
 * strange number of slots allocated here. 512 slots will fit exactly within a
 * single memory page. We also need one additional word at the end of the push
 * buffer for the RESTART opcode that will instruct the CDMA to jump back to
 * the beginning of the push buffer. With 512 slots, this means that we'll use
 * 2 memory pages and waste 4092 bytes of the second page that will never be
 * used.
 */
#define HOST1X_PUSHBUFFER_SLOTS	511

/*
 * Clean up push buffer resources
 */
static void host1x_pushbuffer_destroy(struct push_buffer *pb)
{
	struct host1x_cdma *cdma = pb_to_cdma(pb);
	struct host1x *host1x = cdma_to_host1x(cdma);

	if (!pb->mapped)
		return;

	if (host1x->domain) {
		iommu_unmap(host1x->domain, pb->dma, pb->alloc_size);
		free_iova(&host1x->iova, iova_pfn(&host1x->iova, pb->dma));
	}

	dma_free_wc(host1x->dev, pb->alloc_size, pb->mapped, pb->phys);

	pb->mapped = NULL;
	pb->phys = 0;
}

/*
 * Init push buffer resources
 */
static int host1x_pushbuffer_init(struct push_buffer *pb)
{
	struct host1x_cdma *cdma = pb_to_cdma(pb);
	struct host1x *host1x = cdma_to_host1x(cdma);
	struct iova *alloc;
	u32 size;
	int err;

	pb->mapped = NULL;
	pb->phys = 0;
	pb->size = HOST1X_PUSHBUFFER_SLOTS * 8;

	size = pb->size + 4;

	/* initialize buffer pointers */
	pb->fence = pb->size - 8;
	pb->pos = 0;

	if (host1x->domain) {
		unsigned long shift;

		size = iova_align(&host1x->iova, size);

		pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys,
					  GFP_KERNEL);
		if (!pb->mapped)
			return -ENOMEM;

		shift = iova_shift(&host1x->iova);
		alloc = alloc_iova(&host1x->iova, size >> shift,
				   host1x->iova_end >> shift, true);
		if (!alloc) {
			err = -ENOMEM;
			goto iommu_free_mem;
		}

		pb->dma = iova_dma_addr(&host1x->iova, alloc);
		err = iommu_map(host1x->domain, pb->dma, pb->phys, size,
				IOMMU_READ);
		if (err)
			goto iommu_free_iova;
	} else {
		pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys,
					  GFP_KERNEL);
		if (!pb->mapped)
			return -ENOMEM;

		pb->dma = pb->phys;
	}

	pb->alloc_size = size;

	host1x_hw_pushbuffer_init(host1x, pb);

	return 0;

iommu_free_iova:
	__free_iova(&host1x->iova, alloc);
iommu_free_mem:
	dma_free_wc(host1x->dev, size, pb->mapped, pb->phys);

	return err;
}

/*
 * Push two words to the push buffer
 * Caller must ensure push buffer is not full
 */
static void host1x_pushbuffer_push(struct push_buffer *pb, u32 op1, u32 op2)
{
	u32 *p = (u32 *)((void *)pb->mapped + pb->pos);

	WARN_ON(pb->pos == pb->fence);
	*(p++) = op1;
	*(p++) = op2;
	pb->pos += 8;

	if (pb->pos >= pb->size)
		pb->pos -= pb->size;
}

/*
 * Pop a number of two word slots from the push buffer
 * Caller must ensure push buffer is not empty
 */
static void host1x_pushbuffer_pop(struct push_buffer *pb, unsigned int slots)
{
	/* Advance the next write position */
	pb->fence += slots * 8;

	if (pb->fence >= pb->size)
		pb->fence -= pb->size;
}

/*
 * Return the number of two word slots free in the push buffer
 */
static u32 host1x_pushbuffer_space(struct push_buffer *pb)
{
	unsigned int fence = pb->fence;

	if (pb->fence < pb->pos)
		fence += pb->size;

	return (fence - pb->pos) / 8;
}

/*
 * Sleep (if necessary) until the requested event happens
 *   - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty.
 *     - Returns 1
 *   - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer
 *     - Return the amount of space (> 0)
 * Must be called with the cdma lock held.
 */
unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma,
				     enum cdma_event event)
{
	for (;;) {
		struct push_buffer *pb = &cdma->push_buffer;
		unsigned int space;

		switch (event) {
		case CDMA_EVENT_SYNC_QUEUE_EMPTY:
			space = list_empty(&cdma->sync_queue) ? 1 : 0;
			break;

		case CDMA_EVENT_PUSH_BUFFER_SPACE:
			space = host1x_pushbuffer_space(pb);
			break;

		default:
			WARN_ON(1);
			return -EINVAL;
		}

		if (space)
			return space;

		trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev),
				       event);

		/* If somebody has managed to already start waiting, yield */
		if (cdma->event != CDMA_EVENT_NONE) {
			mutex_unlock(&cdma->lock);
			schedule();
			mutex_lock(&cdma->lock);
			continue;
		}

		cdma->event = event;

		mutex_unlock(&cdma->lock);
		wait_for_completion(&cdma->complete);
		mutex_lock(&cdma->lock);
	}

	return 0;
}

/*
 * Sleep (if necessary) until the push buffer has enough free space.
 *
 * Must be called with the cdma lock held.
 */
int host1x_cdma_wait_pushbuffer_space(struct host1x *host1x,
				      struct host1x_cdma *cdma,
				      unsigned int needed)
{
	while (true) {
		struct push_buffer *pb = &cdma->push_buffer;
		unsigned int space;

		space = host1x_pushbuffer_space(pb);
		if (space >= needed)
			break;

		trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev),
				       CDMA_EVENT_PUSH_BUFFER_SPACE);

		host1x_hw_cdma_flush(host1x, cdma);

		/* If somebody has managed to already start waiting, yield */
		if (cdma->event != CDMA_EVENT_NONE) {
			mutex_unlock(&cdma->lock);
			schedule();
			mutex_lock(&cdma->lock);
			continue;
		}

		cdma->event = CDMA_EVENT_PUSH_BUFFER_SPACE;

		mutex_unlock(&cdma->lock);
		wait_for_completion(&cdma->complete);
		mutex_lock(&cdma->lock);
	}

	return 0;
}
/*
 * Start timer that tracks the time spent by the job.
 * Must be called with the cdma lock held.
 */
static void cdma_start_timer_locked(struct host1x_cdma *cdma,
				    struct host1x_job *job)
{
	struct host1x *host = cdma_to_host1x(cdma);

	if (cdma->timeout.client) {
		/* timer already started */
		return;
	}

	cdma->timeout.client = job->client;
	cdma->timeout.syncpt = host1x_syncpt_get(host, job->syncpt_id);
	cdma->timeout.syncpt_val = job->syncpt_end;
	cdma->timeout.start_ktime = ktime_get();

	schedule_delayed_work(&cdma->timeout.wq,
			      msecs_to_jiffies(job->timeout));
}

/*
 * Stop timer when a buffer submission completes.
 * Must be called with the cdma lock held.
 */
static void stop_cdma_timer_locked(struct host1x_cdma *cdma)
{
	cancel_delayed_work(&cdma->timeout.wq);
	cdma->timeout.client = NULL;
}

/*
 * For all sync queue entries that have already finished according to the
 * current sync point registers:
 *  - unpin & unref their mems
 *  - pop their push buffer slots
 *  - remove them from the sync queue
 * This is normally called from the host code's worker thread, but can be
 * called manually if necessary.
 * Must be called with the cdma lock held.
 */
static void update_cdma_locked(struct host1x_cdma *cdma)
{
	bool signal = false;
	struct host1x *host1x = cdma_to_host1x(cdma);
	struct host1x_job *job, *n;

	/* If CDMA is stopped, queue is cleared and we can return */
	if (!cdma->running)
		return;

	/*
	 * Walk the sync queue, reading the sync point registers as necessary,
	 * to consume as many sync queue entries as possible without blocking
	 */
	list_for_each_entry_safe(job, n, &cdma->sync_queue, list) {
		struct host1x_syncpt *sp =
			host1x_syncpt_get(host1x, job->syncpt_id);

		/* Check whether this syncpt has completed, and bail if not */
		if (!host1x_syncpt_is_expired(sp, job->syncpt_end)) {
			/* Start timer on next pending syncpt */
			if (job->timeout)
				cdma_start_timer_locked(cdma, job);

			break;
		}

		/* Cancel timeout, when a buffer completes */
		if (cdma->timeout.client)
			stop_cdma_timer_locked(cdma);

		/* Unpin the memory */
		host1x_job_unpin(job);

		/* Pop push buffer slots */
		if (job->num_slots) {
			struct push_buffer *pb = &cdma->push_buffer;

			host1x_pushbuffer_pop(pb, job->num_slots);

			if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE)
				signal = true;
		}

		list_del(&job->list);
		host1x_job_put(job);
	}

	if (cdma->event == CDMA_EVENT_SYNC_QUEUE_EMPTY &&
	    list_empty(&cdma->sync_queue))
		signal = true;

	if (signal) {
		cdma->event = CDMA_EVENT_NONE;
		complete(&cdma->complete);
	}
}

void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma,
				   struct device *dev)
{
	struct host1x *host1x = cdma_to_host1x(cdma);
	u32 restart_addr, syncpt_incrs, syncpt_val;
	struct host1x_job *job, *next_job = NULL;

	syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);

	dev_dbg(dev, "%s: starting cleanup (thresh %d)\n",
		__func__, syncpt_val);

	/*
	 * Move the sync_queue read pointer to the first entry that hasn't
	 * completed based on the current HW syncpt value. It's likely there
	 * won't be any (i.e. we're still at the head), but covers the case
	 * where a syncpt incr happens just prior/during the teardown.
	 */

	dev_dbg(dev, "%s: skip completed buffers still in sync_queue\n",
		__func__);

	list_for_each_entry(job, &cdma->sync_queue, list) {
		if (syncpt_val < job->syncpt_end) {

			if (!list_is_last(&job->list, &cdma->sync_queue))
				next_job = list_next_entry(job, list);

			goto syncpt_incr;
		}

		host1x_job_dump(dev, job);
	}

	/* all jobs have been completed */
	job = NULL;

syncpt_incr:

	/*
	 * Increment with CPU the remaining syncpts of a partially executed job.
	 *
	 * CDMA will continue execution starting with the next job or will get
	 * into idle state.
	 */
	if (next_job)
		restart_addr = next_job->first_get;
	else
		restart_addr = cdma->last_pos;

	/* do CPU increments for the remaining syncpts */
	if (job) {
		dev_dbg(dev, "%s: perform CPU incr on pending buffers\n",
			__func__);

		/* won't need a timeout when replayed */
		job->timeout = 0;

		syncpt_incrs = job->syncpt_end - syncpt_val;
		dev_dbg(dev, "%s: CPU incr (%d)\n", __func__, syncpt_incrs);

		host1x_job_dump(dev, job);

		/* safe to use CPU to incr syncpts */
		host1x_hw_cdma_timeout_cpu_incr(host1x, cdma, job->first_get,
						syncpt_incrs, job->syncpt_end,
						job->num_slots);

		dev_dbg(dev, "%s: finished sync_queue modification\n",
			__func__);
	}

	/* roll back DMAGET and start up channel again */
	host1x_hw_cdma_resume(host1x, cdma, restart_addr);
}

/*
 * Create a cdma
 */
int host1x_cdma_init(struct host1x_cdma *cdma)
{
	int err;

	mutex_init(&cdma->lock);
	init_completion(&cdma->complete);

	INIT_LIST_HEAD(&cdma->sync_queue);

	cdma->event = CDMA_EVENT_NONE;
	cdma->running = false;
	cdma->torndown = false;

	err = host1x_pushbuffer_init(&cdma->push_buffer);
	if (err)
		return err;

	return 0;
}

/*
 * Destroy a cdma
 */
int host1x_cdma_deinit(struct host1x_cdma *cdma)
{
	struct push_buffer *pb = &cdma->push_buffer;
	struct host1x *host1x = cdma_to_host1x(cdma);

	if (cdma->running) {
		pr_warn("%s: CDMA still running\n", __func__);
		return -EBUSY;
	}

	host1x_pushbuffer_destroy(pb);
	host1x_hw_cdma_timeout_destroy(host1x, cdma);

	return 0;
}

/*
 * Begin a cdma submit
 */
int host1x_cdma_begin(struct host1x_cdma *cdma, struct host1x_job *job)
{
	struct host1x *host1x = cdma_to_host1x(cdma);

	mutex_lock(&cdma->lock);

	if (job->timeout) {
		/* init state on first submit with timeout value */
		if (!cdma->timeout.initialized) {
			int err;

			err = host1x_hw_cdma_timeout_init(host1x, cdma,
							  job->syncpt_id);
			if (err) {
				mutex_unlock(&cdma->lock);
				return err;
			}
		}
	}

	if (!cdma->running)
		host1x_hw_cdma_start(host1x, cdma);

	cdma->slots_free = 0;
	cdma->slots_used = 0;
	cdma->first_get = cdma->push_buffer.pos;

	trace_host1x_cdma_begin(dev_name(job->channel->dev));
	return 0;
}

/*
 * Push two words into a push buffer slot
 * Blocks as necessary if the push buffer is full.
 */
void host1x_cdma_push(struct host1x_cdma *cdma, u32 op1, u32 op2)
{
	struct host1x *host1x = cdma_to_host1x(cdma);
	struct push_buffer *pb = &cdma->push_buffer;
	u32 slots_free = cdma->slots_free;

	if (host1x_debug_trace_cmdbuf)
		trace_host1x_cdma_push(dev_name(cdma_to_channel(cdma)->dev),
				       op1, op2);

	if (slots_free == 0) {
		host1x_hw_cdma_flush(host1x, cdma);
		slots_free = host1x_cdma_wait_locked(cdma,
						CDMA_EVENT_PUSH_BUFFER_SPACE);
	}

	cdma->slots_free = slots_free - 1;
	cdma->slots_used++;
	host1x_pushbuffer_push(pb, op1, op2);
}

/*
 * Push four words into two consecutive push buffer slots. Note that extra
 * care needs to be taken not to split the two slots across the end of the
 * push buffer. Otherwise the RESTART opcode at the end of the push buffer
 * that ensures processing will restart at the beginning will break up the
 * four words.
 *
 * Blocks as necessary if the push buffer is full.
 */
void host1x_cdma_push_wide(struct host1x_cdma *cdma, u32 op1, u32 op2,
			   u32 op3, u32 op4)
{
	struct host1x_channel *channel = cdma_to_channel(cdma);
	struct host1x *host1x = cdma_to_host1x(cdma);
	struct push_buffer *pb = &cdma->push_buffer;
	unsigned int needed = 2, extra = 0, i;
	unsigned int space = cdma->slots_free;

	if (host1x_debug_trace_cmdbuf)
		trace_host1x_cdma_push_wide(dev_name(channel->dev), op1, op2,
					    op3, op4);

	/* compute number of extra slots needed for padding */
	if (pb->pos + 16 > pb->size) {
		extra = (pb->size - pb->pos) / 8;
		needed += extra;
	}

	host1x_cdma_wait_pushbuffer_space(host1x, cdma, needed);
	space = host1x_pushbuffer_space(pb);

	cdma->slots_free = space - needed;
	cdma->slots_used += needed;

	/*
	 * Note that we rely on the fact that this is only used to submit wide
	 * gather opcodes, which consist of 3 words, and they are padded with
	 * a NOP to avoid having to deal with fractional slots (a slot always
	 * represents 2 words). The fourth opcode passed to this function will
	 * therefore always be a NOP.
	 *
	 * This works around a slight ambiguity when it comes to opcodes. For
	 * all current host1x incarnations the NOP opcode uses the exact same
	 * encoding (0x20000000), so we could hard-code the value here, but a
	 * new incarnation may change it and break that assumption.
	 */
	for (i = 0; i < extra; i++)
		host1x_pushbuffer_push(pb, op4, op4);

	host1x_pushbuffer_push(pb, op1, op2);
	host1x_pushbuffer_push(pb, op3, op4);
}

/*
 * End a cdma submit
 * Kick off DMA, add job to the sync queue, and a number of slots to be freed
 * from the pushbuffer. The handles for a submit must all be pinned at the same
 * time, but they can be unpinned in smaller chunks.
 */
void host1x_cdma_end(struct host1x_cdma *cdma,
		     struct host1x_job *job)
{
	struct host1x *host1x = cdma_to_host1x(cdma);
	bool idle = list_empty(&cdma->sync_queue);

	host1x_hw_cdma_flush(host1x, cdma);

	job->first_get = cdma->first_get;
	job->num_slots = cdma->slots_used;
	host1x_job_get(job);
	list_add_tail(&job->list, &cdma->sync_queue);

	/* start timer on idle -> active transitions */
	if (job->timeout && idle)
		cdma_start_timer_locked(cdma, job);

	trace_host1x_cdma_end(dev_name(job->channel->dev));
	mutex_unlock(&cdma->lock);
}

/*
 * Update cdma state according to current sync point values
 */
void host1x_cdma_update(struct host1x_cdma *cdma)
{
	mutex_lock(&cdma->lock);
	update_cdma_locked(cdma);
	mutex_unlock(&cdma->lock);
}
Commit	Line	Data
6579324a TB	1	/*
	2	* Tegra host1x Command DMA
	3	*
	4	* Copyright (c) 2010-2013, NVIDIA Corporation.
	5	*
	6	* This program is free software; you can redistribute it and/or modify it
	7	* under the terms and conditions of the GNU General Public License,
	8	* version 2, as published by the Free Software Foundation.
	9	*
	10	* This program is distributed in the hope it will be useful, but WITHOUT
	11	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	12	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
	13	* more details.
	14	*
	15	* You should have received a copy of the GNU General Public License
	16	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	17	*/
	18
	19
	20	#include <asm/cacheflush.h>
	21	#include <linux/device.h>
	22	#include <linux/dma-mapping.h>
35d747a8	23	#include <linux/host1x.h>
6579324a TB	24	#include <linux/interrupt.h>
	25	#include <linux/kernel.h>
	26	#include <linux/kfifo.h>
	27	#include <linux/slab.h>
	28	#include <trace/events/host1x.h>
	29
	30	#include "cdma.h"
	31	#include "channel.h"
	32	#include "dev.h"
	33	#include "debug.h"
6579324a TB	34	#include "job.h"
	35
	36	/*
	37	* push_buffer
	38	*
	39	* The push buffer is a circular array of words to be fetched by command DMA.
	40	* Note that it works slightly differently to the sync queue; fence == pos
	41	* means that the push buffer is full, not empty.
	42	*/
	43
e1f338c0 TR	44	/*
	45	* Typically the commands written into the push buffer are a pair of words. We
	46	* use slots to represent each of these pairs and to simplify things. Note the
	47	* strange number of slots allocated here. 512 slots will fit exactly within a
	48	* single memory page. We also need one additional word at the end of the push
	49	* buffer for the RESTART opcode that will instruct the CDMA to jump back to
	50	* the beginning of the push buffer. With 512 slots, this means that we'll use
	51	* 2 memory pages and waste 4092 bytes of the second page that will never be
	52	* used.
	53	*/
	54	#define HOST1X_PUSHBUFFER_SLOTS 511
6579324a TB	55
	56	/*
	57	* Clean up push buffer resources
	58	*/
	59	static void host1x_pushbuffer_destroy(struct push_buffer *pb)
	60	{
	61	struct host1x_cdma *cdma = pb_to_cdma(pb);
	62	struct host1x *host1x = cdma_to_host1x(cdma);
	63
2f8a6da8	64	if (!pb->mapped)
404bfb78 MP	65	return;
	66
	67	if (host1x->domain) {
	68	iommu_unmap(host1x->domain, pb->dma, pb->alloc_size);
	69	free_iova(&host1x->iova, iova_pfn(&host1x->iova, pb->dma));
	70	}
	71
	72	dma_free_wc(host1x->dev, pb->alloc_size, pb->mapped, pb->phys);
6579324a TB	73
	74	pb->mapped = NULL;
	75	pb->phys = 0;
	76	}
	77
	78	/*
	79	* Init push buffer resources
	80	*/
	81	static int host1x_pushbuffer_init(struct push_buffer *pb)
	82	{
	83	struct host1x_cdma *cdma = pb_to_cdma(pb);
	84	struct host1x *host1x = cdma_to_host1x(cdma);
404bfb78 MP	85	struct iova *alloc;
	86	u32 size;
	87	int err;
6579324a TB	88
	89	pb->mapped = NULL;
	90	pb->phys = 0;
404bfb78 MP	91	pb->size = HOST1X_PUSHBUFFER_SLOTS * 8;
	92
	93	size = pb->size + 4;
6579324a TB	94
6579324a TB	95	/* initialize buffer pointers */
404bfb78	96	pb->fence = pb->size - 8;
6579324a TB	97	pb->pos = 0;
6579324a TB	98
404bfb78 MP	99	if (host1x->domain) {
	100	unsigned long shift;
	101
	102	size = iova_align(&host1x->iova, size);
	103
	104	pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys,
	105	GFP_KERNEL);
	106	if (!pb->mapped)
	107	return -ENOMEM;
	108
	109	shift = iova_shift(&host1x->iova);
	110	alloc = alloc_iova(&host1x->iova, size >> shift,
	111	host1x->iova_end >> shift, true);
	112	if (!alloc) {
	113	err = -ENOMEM;
	114	goto iommu_free_mem;
	115	}
	116
	117	pb->dma = iova_dma_addr(&host1x->iova, alloc);
	118	err = iommu_map(host1x->domain, pb->dma, pb->phys, size,
	119	IOMMU_READ);
	120	if (err)
	121	goto iommu_free_iova;
	122	} else {
	123	pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys,
	124	GFP_KERNEL);
	125	if (!pb->mapped)
	126	return -ENOMEM;
	127
	128	pb->dma = pb->phys;
	129	}
	130
	131	pb->alloc_size = size;
6579324a TB	132
	133	host1x_hw_pushbuffer_init(host1x, pb);
	134
	135	return 0;
	136
404bfb78 MP	137	iommu_free_iova:
	138	__free_iova(&host1x->iova, alloc);
	139	iommu_free_mem:
27db6a00	140	dma_free_wc(host1x->dev, size, pb->mapped, pb->phys);
404bfb78 MP	141
404bfb78 MP	142	return err;
6579324a TB	143	}
	144
	145	/*
	146	* Push two words to the push buffer
	147	* Caller must ensure push buffer is not full
	148	*/
	149	static void host1x_pushbuffer_push(struct push_buffer *pb, u32 op1, u32 op2)
	150	{
ebb2475c TR	151	u32 p = (u32 )((void *)pb->mapped + pb->pos);
	152
	153	WARN_ON(pb->pos == pb->fence);
6579324a TB	154	*(p++) = op1;
6579324a TB	155	*(p++) = op2;
e1f338c0 TR	156	pb->pos += 8;
	157
	158	if (pb->pos >= pb->size)
	159	pb->pos -= pb->size;
6579324a TB	160	}
	161
	162	/*
	163	* Pop a number of two word slots from the push buffer
	164	* Caller must ensure push buffer is not empty
	165	*/
	166	static void host1x_pushbuffer_pop(struct push_buffer *pb, unsigned int slots)
	167	{
	168	/* Advance the next write position */
e1f338c0 TR	169	pb->fence += slots * 8;
	170
	171	if (pb->fence >= pb->size)
	172	pb->fence -= pb->size;
6579324a TB	173	}
	174
	175	/*
	176	* Return the number of two word slots free in the push buffer
	177	*/
	178	static u32 host1x_pushbuffer_space(struct push_buffer *pb)
	179	{
e1f338c0 TR	180	unsigned int fence = pb->fence;
	181
	182	if (pb->fence < pb->pos)
	183	fence += pb->size;
	184
	185	return (fence - pb->pos) / 8;
6579324a TB	186	}
	187
	188	/*
	189	* Sleep (if necessary) until the requested event happens
	190	* - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty.
	191	* - Returns 1
	192	* - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer
	193	* - Return the amount of space (> 0)
	194	* Must be called with the cdma lock held.
	195	*/
	196	unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma,
	197	enum cdma_event event)
	198	{
	199	for (;;) {
0b8070d1	200	struct push_buffer *pb = &cdma->push_buffer;
6579324a TB	201	unsigned int space;
6579324a TB	202
0b8070d1 TR	203	switch (event) {
0b8070d1 TR	204	case CDMA_EVENT_SYNC_QUEUE_EMPTY:
6579324a	205	space = list_empty(&cdma->sync_queue) ? 1 : 0;
0b8070d1 TR	206	break;
	207
	208	case CDMA_EVENT_PUSH_BUFFER_SPACE:
6579324a	209	space = host1x_pushbuffer_space(pb);
0b8070d1 TR	210	break;
	211
	212	default:
6579324a TB	213	WARN_ON(1);
	214	return -EINVAL;
	215	}
	216
	217	if (space)
	218	return space;
	219
	220	trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev),
	221	event);
	222
	223	/* If somebody has managed to already start waiting, yield */
	224	if (cdma->event != CDMA_EVENT_NONE) {
	225	mutex_unlock(&cdma->lock);
	226	schedule();
	227	mutex_lock(&cdma->lock);
	228	continue;
	229	}
0b8070d1	230
6579324a TB	231	cdma->event = event;
	232
	233	mutex_unlock(&cdma->lock);
0747a672	234	wait_for_completion(&cdma->complete);
6579324a TB	235	mutex_lock(&cdma->lock);
6579324a TB	236	}
0b8070d1	237
6579324a TB	238	return 0;
	239	}
	240
5a5fccbd TR	241	/*
	242	* Sleep (if necessary) until the push buffer has enough free space.
	243	*
	244	* Must be called with the cdma lock held.
	245	*/
	246	int host1x_cdma_wait_pushbuffer_space(struct host1x *host1x,
	247	struct host1x_cdma *cdma,
	248	unsigned int needed)
	249	{
	250	while (true) {
	251	struct push_buffer *pb = &cdma->push_buffer;
	252	unsigned int space;
	253
	254	space = host1x_pushbuffer_space(pb);
	255	if (space >= needed)
	256	break;
	257
	258	trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev),
	259	CDMA_EVENT_PUSH_BUFFER_SPACE);
	260
	261	host1x_hw_cdma_flush(host1x, cdma);
	262
	263	/* If somebody has managed to already start waiting, yield */
	264	if (cdma->event != CDMA_EVENT_NONE) {
	265	mutex_unlock(&cdma->lock);
	266	schedule();
	267	mutex_lock(&cdma->lock);
	268	continue;
	269	}
	270
	271	cdma->event = CDMA_EVENT_PUSH_BUFFER_SPACE;
	272
	273	mutex_unlock(&cdma->lock);
	274	wait_for_completion(&cdma->complete);
	275	mutex_lock(&cdma->lock);
	276	}
	277
	278	return 0;
	279	}
6579324a TB	280	/*
	281	* Start timer that tracks the time spent by the job.
	282	* Must be called with the cdma lock held.
	283	*/
	284	static void cdma_start_timer_locked(struct host1x_cdma *cdma,
	285	struct host1x_job *job)
	286	{
	287	struct host1x *host = cdma_to_host1x(cdma);
	288
	289	if (cdma->timeout.client) {
	290	/* timer already started */
	291	return;
	292	}
	293
	294	cdma->timeout.client = job->client;
	295	cdma->timeout.syncpt = host1x_syncpt_get(host, job->syncpt_id);
	296	cdma->timeout.syncpt_val = job->syncpt_end;
	297	cdma->timeout.start_ktime = ktime_get();
	298
	299	schedule_delayed_work(&cdma->timeout.wq,
	300	msecs_to_jiffies(job->timeout));
	301	}
	302
	303	/*
	304	* Stop timer when a buffer submission completes.
	305	* Must be called with the cdma lock held.
	306	*/
	307	static void stop_cdma_timer_locked(struct host1x_cdma *cdma)
	308	{
	309	cancel_delayed_work(&cdma->timeout.wq);
bf3d41cc	310	cdma->timeout.client = NULL;
6579324a TB	311	}
	312
	313	/*
	314	* For all sync queue entries that have already finished according to the
	315	* current sync point registers:
	316	* - unpin & unref their mems
	317	* - pop their push buffer slots
	318	* - remove them from the sync queue
	319	* This is normally called from the host code's worker thread, but can be
	320	* called manually if necessary.
	321	* Must be called with the cdma lock held.
	322	*/
	323	static void update_cdma_locked(struct host1x_cdma *cdma)
	324	{
	325	bool signal = false;
	326	struct host1x *host1x = cdma_to_host1x(cdma);
	327	struct host1x_job job, n;
	328
	329	/* If CDMA is stopped, queue is cleared and we can return */
	330	if (!cdma->running)
	331	return;
	332
	333	/*
	334	* Walk the sync queue, reading the sync point registers as necessary,
	335	* to consume as many sync queue entries as possible without blocking
	336	*/
	337	list_for_each_entry_safe(job, n, &cdma->sync_queue, list) {
	338	struct host1x_syncpt *sp =
	339	host1x_syncpt_get(host1x, job->syncpt_id);
	340
	341	/* Check whether this syncpt has completed, and bail if not */
	342	if (!host1x_syncpt_is_expired(sp, job->syncpt_end)) {
	343	/* Start timer on next pending syncpt */
	344	if (job->timeout)
	345	cdma_start_timer_locked(cdma, job);
0b8070d1	346
6579324a TB	347	break;
	348	}
	349
	350	/* Cancel timeout, when a buffer completes */
	351	if (cdma->timeout.client)
	352	stop_cdma_timer_locked(cdma);
	353
	354	/* Unpin the memory */
	355	host1x_job_unpin(job);
	356
	357	/* Pop push buffer slots */
	358	if (job->num_slots) {
	359	struct push_buffer *pb = &cdma->push_buffer;
6df633d0	360
6579324a	361	host1x_pushbuffer_pop(pb, job->num_slots);
0b8070d1	362
6579324a TB	363	if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE)
	364	signal = true;
	365	}
	366
	367	list_del(&job->list);
	368	host1x_job_put(job);
	369	}
	370
	371	if (cdma->event == CDMA_EVENT_SYNC_QUEUE_EMPTY &&
	372	list_empty(&cdma->sync_queue))
	373	signal = true;
	374
	375	if (signal) {
	376	cdma->event = CDMA_EVENT_NONE;
0747a672	377	complete(&cdma->complete);
6579324a TB	378	}
	379	}
	380
	381	void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma,
	382	struct device *dev)
	383	{
6579324a	384	struct host1x *host1x = cdma_to_host1x(cdma);
0b8070d1	385	u32 restart_addr, syncpt_incrs, syncpt_val;
79930baf	386	struct host1x_job job, next_job = NULL;
6579324a TB	387
	388	syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);
	389
	390	dev_dbg(dev, "%s: starting cleanup (thresh %d)\n",
	391	__func__, syncpt_val);
	392
	393	/*
	394	* Move the sync_queue read pointer to the first entry that hasn't
	395	* completed based on the current HW syncpt value. It's likely there
	396	* won't be any (i.e. we're still at the head), but covers the case
	397	* where a syncpt incr happens just prior/during the teardown.
	398	*/
	399
	400	dev_dbg(dev, "%s: skip completed buffers still in sync_queue\n",
	401	__func__);
	402
	403	list_for_each_entry(job, &cdma->sync_queue, list) {
79930baf DO	404	if (syncpt_val < job->syncpt_end) {
	405
	406	if (!list_is_last(&job->list, &cdma->sync_queue))
	407	next_job = list_next_entry(job, list);
	408
5d6f0436	409	goto syncpt_incr;
79930baf	410	}
6579324a TB	411
	412	host1x_job_dump(dev, job);
	413	}
	414
5d6f0436 DO	415	/* all jobs have been completed */
	416	job = NULL;
	417
	418	syncpt_incr:
	419
6579324a	420	/*
e8bad659	421	* Increment with CPU the remaining syncpts of a partially executed job.
6579324a	422	*
79930baf DO	423	* CDMA will continue execution starting with the next job or will get
79930baf DO	424	* into idle state.
6579324a	425	*/
79930baf DO	426	if (next_job)
79930baf DO	427	restart_addr = next_job->first_get;
6579324a TB	428	else
	429	restart_addr = cdma->last_pos;
	430
e8bad659 DO	431	/* do CPU increments for the remaining syncpts */
e8bad659 DO	432	if (job) {
5d6f0436 DO	433	dev_dbg(dev, "%s: perform CPU incr on pending buffers\n",
	434	__func__);
	435
6579324a TB	436	/* won't need a timeout when replayed */
	437	job->timeout = 0;
	438
	439	syncpt_incrs = job->syncpt_end - syncpt_val;
	440	dev_dbg(dev, "%s: CPU incr (%d)\n", __func__, syncpt_incrs);
	441
	442	host1x_job_dump(dev, job);
	443
	444	/* safe to use CPU to incr syncpts */
	445	host1x_hw_cdma_timeout_cpu_incr(host1x, cdma, job->first_get,
	446	syncpt_incrs, job->syncpt_end,
	447	job->num_slots);
6579324a	448
5d6f0436 DO	449	dev_dbg(dev, "%s: finished sync_queue modification\n",
	450	__func__);
	451	}
6579324a TB	452
	453	/* roll back DMAGET and start up channel again */
	454	host1x_hw_cdma_resume(host1x, cdma, restart_addr);
	455	}
	456
	457	/*
	458	* Create a cdma
	459	*/
	460	int host1x_cdma_init(struct host1x_cdma *cdma)
	461	{
	462	int err;
	463
	464	mutex_init(&cdma->lock);
0747a672	465	init_completion(&cdma->complete);
6579324a TB	466
	467	INIT_LIST_HEAD(&cdma->sync_queue);
	468
	469	cdma->event = CDMA_EVENT_NONE;
	470	cdma->running = false;
	471	cdma->torndown = false;
	472
	473	err = host1x_pushbuffer_init(&cdma->push_buffer);
	474	if (err)
	475	return err;
0b8070d1	476
6579324a TB	477	return 0;
	478	}
	479
	480	/*
	481	* Destroy a cdma
	482	*/
	483	int host1x_cdma_deinit(struct host1x_cdma *cdma)
	484	{
	485	struct push_buffer *pb = &cdma->push_buffer;
	486	struct host1x *host1x = cdma_to_host1x(cdma);
	487
	488	if (cdma->running) {
	489	pr_warn("%s: CDMA still running\n", __func__);
	490	return -EBUSY;
	491	}
	492
	493	host1x_pushbuffer_destroy(pb);
	494	host1x_hw_cdma_timeout_destroy(host1x, cdma);
	495
	496	return 0;
	497	}
	498
	499	/*
	500	* Begin a cdma submit
	501	*/
	502	int host1x_cdma_begin(struct host1x_cdma cdma, struct host1x_job job)
	503	{
	504	struct host1x *host1x = cdma_to_host1x(cdma);
	505
	506	mutex_lock(&cdma->lock);
	507
	508	if (job->timeout) {
	509	/* init state on first submit with timeout value */
	510	if (!cdma->timeout.initialized) {
	511	int err;
6df633d0	512
6579324a TB	513	err = host1x_hw_cdma_timeout_init(host1x, cdma,
	514	job->syncpt_id);
	515	if (err) {
	516	mutex_unlock(&cdma->lock);
	517	return err;
	518	}
	519	}
	520	}
0b8070d1	521
6579324a TB	522	if (!cdma->running)
	523	host1x_hw_cdma_start(host1x, cdma);
	524
	525	cdma->slots_free = 0;
	526	cdma->slots_used = 0;
	527	cdma->first_get = cdma->push_buffer.pos;
	528
	529	trace_host1x_cdma_begin(dev_name(job->channel->dev));
	530	return 0;
	531	}
	532
	533	/*
	534	* Push two words into a push buffer slot
	535	* Blocks as necessary if the push buffer is full.
	536	*/
	537	void host1x_cdma_push(struct host1x_cdma *cdma, u32 op1, u32 op2)
	538	{
	539	struct host1x *host1x = cdma_to_host1x(cdma);
	540	struct push_buffer *pb = &cdma->push_buffer;
	541	u32 slots_free = cdma->slots_free;
	542
6236451d TB	543	if (host1x_debug_trace_cmdbuf)
	544	trace_host1x_cdma_push(dev_name(cdma_to_channel(cdma)->dev),
	545	op1, op2);
	546
6579324a TB	547	if (slots_free == 0) {
	548	host1x_hw_cdma_flush(host1x, cdma);
	549	slots_free = host1x_cdma_wait_locked(cdma,
	550	CDMA_EVENT_PUSH_BUFFER_SPACE);
	551	}
0b8070d1	552
6579324a TB	553	cdma->slots_free = slots_free - 1;
	554	cdma->slots_used++;
	555	host1x_pushbuffer_push(pb, op1, op2);
	556	}
	557
5a5fccbd TR	558	/*
	559	* Push four words into two consecutive push buffer slots. Note that extra
	560	* care needs to be taken not to split the two slots across the end of the
	561	* push buffer. Otherwise the RESTART opcode at the end of the push buffer
	562	* that ensures processing will restart at the beginning will break up the
	563	* four words.
	564	*
	565	* Blocks as necessary if the push buffer is full.
	566	*/
	567	void host1x_cdma_push_wide(struct host1x_cdma *cdma, u32 op1, u32 op2,
	568	u32 op3, u32 op4)
	569	{
	570	struct host1x_channel *channel = cdma_to_channel(cdma);
	571	struct host1x *host1x = cdma_to_host1x(cdma);
	572	struct push_buffer *pb = &cdma->push_buffer;
	573	unsigned int needed = 2, extra = 0, i;
	574	unsigned int space = cdma->slots_free;
	575
	576	if (host1x_debug_trace_cmdbuf)
	577	trace_host1x_cdma_push_wide(dev_name(channel->dev), op1, op2,
	578	op3, op4);
	579
	580	/* compute number of extra slots needed for padding */
	581	if (pb->pos + 16 > pb->size) {
	582	extra = (pb->size - pb->pos) / 8;
	583	needed += extra;
	584	}
	585
	586	host1x_cdma_wait_pushbuffer_space(host1x, cdma, needed);
	587	space = host1x_pushbuffer_space(pb);
	588
	589	cdma->slots_free = space - needed;
	590	cdma->slots_used += needed;
	591
	592	/*
	593	* Note that we rely on the fact that this is only used to submit wide
	594	* gather opcodes, which consist of 3 words, and they are padded with
	595	* a NOP to avoid having to deal with fractional slots (a slot always
	596	* represents 2 words). The fourth opcode passed to this function will
	597	* therefore always be a NOP.
	598	*
	599	* This works around a slight ambiguity when it comes to opcodes. For
	600	* all current host1x incarnations the NOP opcode uses the exact same
	601	* encoding (0x20000000), so we could hard-code the value here, but a
	602	* new incarnation may change it and break that assumption.
	603	*/
	604	for (i = 0; i < extra; i++)
	605	host1x_pushbuffer_push(pb, op4, op4);
	606
	607	host1x_pushbuffer_push(pb, op1, op2);
	608	host1x_pushbuffer_push(pb, op3, op4);
	609	}
	610
6579324a TB	611	/*
	612	* End a cdma submit
	613	* Kick off DMA, add job to the sync queue, and a number of slots to be freed
	614	* from the pushbuffer. The handles for a submit must all be pinned at the same
	615	* time, but they can be unpinned in smaller chunks.
	616	*/
	617	void host1x_cdma_end(struct host1x_cdma *cdma,
	618	struct host1x_job *job)
	619	{
	620	struct host1x *host1x = cdma_to_host1x(cdma);
	621	bool idle = list_empty(&cdma->sync_queue);
	622
	623	host1x_hw_cdma_flush(host1x, cdma);
	624
	625	job->first_get = cdma->first_get;
	626	job->num_slots = cdma->slots_used;
	627	host1x_job_get(job);
	628	list_add_tail(&job->list, &cdma->sync_queue);
	629
	630	/* start timer on idle -> active transitions */
	631	if (job->timeout && idle)
	632	cdma_start_timer_locked(cdma, job);
	633
	634	trace_host1x_cdma_end(dev_name(job->channel->dev));
	635	mutex_unlock(&cdma->lock);
	636	}
	637
	638	/*
	639	* Update cdma state according to current sync point values
	640	*/
	641	void host1x_cdma_update(struct host1x_cdma *cdma)
	642	{
	643	mutex_lock(&cdma->lock);
	644	update_cdma_locked(cdma);
	645	mutex_unlock(&cdma->lock);
	646	}