[linux-block.git] / drivers / gpu / drm / nouveau / nouveau_fence.c

/*
 * Copyright (C) 2007 Ben Skeggs.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial
 * portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 */

#include <drm/drmP.h>

#include <linux/ktime.h>
#include <linux/hrtimer.h>

#include "nouveau_drm.h"
#include "nouveau_dma.h"
#include "nouveau_fence.h"

#include <engine/fifo.h>

struct fence_work {
	struct work_struct base;
	struct list_head head;
	void (*func)(void *);
	void *data;
};

static void
nouveau_fence_signal(struct nouveau_fence *fence)
{
	struct fence_work *work, *temp;

	list_for_each_entry_safe(work, temp, &fence->work, head) {
		schedule_work(&work->base);
		list_del(&work->head);
	}

	fence->channel = NULL;
	list_del(&fence->head);
}

void
nouveau_fence_context_del(struct nouveau_fence_chan *fctx)
{
	struct nouveau_fence *fence, *fnext;
	spin_lock(&fctx->lock);
	list_for_each_entry_safe(fence, fnext, &fctx->pending, head) {
		nouveau_fence_signal(fence);
	}
	spin_unlock(&fctx->lock);
}

void
nouveau_fence_context_new(struct nouveau_fence_chan *fctx)
{
	INIT_LIST_HEAD(&fctx->flip);
	INIT_LIST_HEAD(&fctx->pending);
	spin_lock_init(&fctx->lock);
}

static void
nouveau_fence_work_handler(struct work_struct *kwork)
{
	struct fence_work *work = container_of(kwork, typeof(*work), base);
	work->func(work->data);
	kfree(work);
}

void
nouveau_fence_work(struct nouveau_fence *fence,
		   void (*func)(void *), void *data)
{
	struct nouveau_channel *chan = fence->channel;
	struct nouveau_fence_chan *fctx;
	struct fence_work *work = NULL;

	if (nouveau_fence_done(fence)) {
		func(data);
		return;
	}

	fctx = chan->fence;
	work = kmalloc(sizeof(*work), GFP_KERNEL);
	if (!work) {
		WARN_ON(nouveau_fence_wait(fence, false, false));
		func(data);
		return;
	}

	spin_lock(&fctx->lock);
	if (!fence->channel) {
		spin_unlock(&fctx->lock);
		kfree(work);
		func(data);
		return;
	}

	INIT_WORK(&work->base, nouveau_fence_work_handler);
	work->func = func;
	work->data = data;
	list_add(&work->head, &fence->work);
	spin_unlock(&fctx->lock);
}

static void
nouveau_fence_update(struct nouveau_channel *chan)
{
	struct nouveau_fence_chan *fctx = chan->fence;
	struct nouveau_fence *fence, *fnext;

	spin_lock(&fctx->lock);
	list_for_each_entry_safe(fence, fnext, &fctx->pending, head) {
		if (fctx->read(chan) < fence->sequence)
			break;

		nouveau_fence_signal(fence);
		nouveau_fence_unref(&fence);
	}
	spin_unlock(&fctx->lock);
}

int
nouveau_fence_emit(struct nouveau_fence *fence, struct nouveau_channel *chan)
{
	struct nouveau_fence_chan *fctx = chan->fence;
	int ret;

	fence->channel  = chan;
	fence->timeout  = jiffies + (15 * DRM_HZ);
	fence->sequence = ++fctx->sequence;

	ret = fctx->emit(fence);
	if (!ret) {
		kref_get(&fence->kref);
		spin_lock(&fctx->lock);
		list_add_tail(&fence->head, &fctx->pending);
		spin_unlock(&fctx->lock);
	}

	return ret;
}

bool
nouveau_fence_done(struct nouveau_fence *fence)
{
	if (fence->channel)
		nouveau_fence_update(fence->channel);
	return !fence->channel;
}

struct nouveau_fence_uevent {
	struct nouveau_eventh handler;
	struct nouveau_fence_priv *priv;
};

static int
nouveau_fence_wait_uevent_handler(struct nouveau_eventh *event, int index)
{
	struct nouveau_fence_uevent *uevent =
		container_of(event, struct nouveau_fence_uevent, handler);
	wake_up_all(&uevent->priv->waiting);
	return NVKM_EVENT_KEEP;
}

static int
nouveau_fence_wait_uevent(struct nouveau_fence *fence, bool intr)

{
	struct nouveau_channel *chan = fence->channel;
	struct nouveau_fifo *pfifo = nouveau_fifo(chan->drm->device);
	struct nouveau_fence_priv *priv = chan->drm->fence;
	struct nouveau_fence_uevent uevent = {
		.handler.func = nouveau_fence_wait_uevent_handler,
		.priv = priv,
	};
	int ret = 0;

	nouveau_event_get(pfifo->uevent, 0, &uevent.handler);

	if (fence->timeout) {
		unsigned long timeout = fence->timeout - jiffies;

		if (time_before(jiffies, fence->timeout)) {
			if (intr) {
				ret = wait_event_interruptible_timeout(
						priv->waiting,
						nouveau_fence_done(fence),
						timeout);
			} else {
				ret = wait_event_timeout(priv->waiting,
						nouveau_fence_done(fence),
						timeout);
			}
		}

		if (ret >= 0) {
			fence->timeout = jiffies + ret;
			if (time_after_eq(jiffies, fence->timeout))
				ret = -EBUSY;
		}
	} else {
		if (intr) {
			ret = wait_event_interruptible(priv->waiting,
					nouveau_fence_done(fence));
		} else {
			wait_event(priv->waiting, nouveau_fence_done(fence));
		}
	}

	nouveau_event_put(pfifo->uevent, 0, &uevent.handler);
	if (unlikely(ret < 0))
		return ret;

	return 0;
}

int
nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)
{
	struct nouveau_channel *chan = fence->channel;
	struct nouveau_fence_priv *priv = chan ? chan->drm->fence : NULL;
	unsigned long sleep_time = NSEC_PER_MSEC / 1000;
	ktime_t t;
	int ret = 0;

	while (priv && priv->uevent && lazy && !nouveau_fence_done(fence)) {
		ret = nouveau_fence_wait_uevent(fence, intr);
		if (ret < 0)
			return ret;
	}

	while (!nouveau_fence_done(fence)) {
		if (fence->timeout && time_after_eq(jiffies, fence->timeout)) {
			ret = -EBUSY;
			break;
		}

		__set_current_state(intr ? TASK_INTERRUPTIBLE :
					   TASK_UNINTERRUPTIBLE);
		if (lazy) {
			t = ktime_set(0, sleep_time);
			schedule_hrtimeout(&t, HRTIMER_MODE_REL);
			sleep_time *= 2;
			if (sleep_time > NSEC_PER_MSEC)
				sleep_time = NSEC_PER_MSEC;
		}

		if (intr && signal_pending(current)) {
			ret = -ERESTARTSYS;
			break;
		}
	}

	__set_current_state(TASK_RUNNING);
	return ret;
}

int
nouveau_fence_sync(struct nouveau_fence *fence, struct nouveau_channel *chan)
{
	struct nouveau_fence_chan *fctx = chan->fence;
	struct nouveau_channel *prev;
	int ret = 0;

	prev = fence ? fence->channel : NULL;
	if (prev) {
		if (unlikely(prev != chan && !nouveau_fence_done(fence))) {
			ret = fctx->sync(fence, prev, chan);
			if (unlikely(ret))
				ret = nouveau_fence_wait(fence, true, false);
		}
	}

	return ret;
}

static void
nouveau_fence_del(struct kref *kref)
{
	struct nouveau_fence *fence = container_of(kref, typeof(*fence), kref);
	kfree(fence);
}

void
nouveau_fence_unref(struct nouveau_fence **pfence)
{
	if (*pfence)
		kref_put(&(*pfence)->kref, nouveau_fence_del);
	*pfence = NULL;
}

struct nouveau_fence *
nouveau_fence_ref(struct nouveau_fence *fence)
{
	kref_get(&fence->kref);
	return fence;
}

int
nouveau_fence_new(struct nouveau_channel *chan, bool sysmem,
		  struct nouveau_fence **pfence)
{
	struct nouveau_fence *fence;
	int ret = 0;

	if (unlikely(!chan->fence))
		return -ENODEV;

	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
	if (!fence)
		return -ENOMEM;

	INIT_LIST_HEAD(&fence->work);
	fence->sysmem = sysmem;
	kref_init(&fence->kref);

	ret = nouveau_fence_emit(fence, chan);
	if (ret)
		nouveau_fence_unref(&fence);

	*pfence = fence;
	return ret;
}
Commit	Line	Data
6ee73861 BS	1	/*
	2	* Copyright (C) 2007 Ben Skeggs.
	3	* All Rights Reserved.
	4	*
	5	* Permission is hereby granted, free of charge, to any person obtaining
	6	* a copy of this software and associated documentation files (the
	7	* "Software"), to deal in the Software without restriction, including
	8	* without limitation the rights to use, copy, modify, merge, publish,
	9	* distribute, sublicense, and/or sell copies of the Software, and to
	10	* permit persons to whom the Software is furnished to do so, subject to
	11	* the following conditions:
	12	*
	13	* The above copyright notice and this permission notice (including the
	14	* next paragraph) shall be included in all copies or substantial
	15	* portions of the Software.
	16	*
	17	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	18	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	19	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
	20	* IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
	21	* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
	22	* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
	23	* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
	24	*
	25	*/
	26
760285e7	27	#include <drm/drmP.h>
6ee73861	28
bd35fe5a MS	29	#include <linux/ktime.h>
	30	#include <linux/hrtimer.h>
	31
ebb945a9	32	#include "nouveau_drm.h"
6ee73861	33	#include "nouveau_dma.h"
ebb945a9	34	#include "nouveau_fence.h"
6ee73861	35
e18c080f BS	36	#include <engine/fifo.h>
e18c080f BS	37
c4c7044f BS	38	struct fence_work {
	39	struct work_struct base;
	40	struct list_head head;
	41	void (func)(void );
	42	void *data;
	43	};
	44
	45	static void
	46	nouveau_fence_signal(struct nouveau_fence *fence)
	47	{
	48	struct fence_work work, temp;
	49
	50	list_for_each_entry_safe(work, temp, &fence->work, head) {
	51	schedule_work(&work->base);
	52	list_del(&work->head);
	53	}
	54
	55	fence->channel = NULL;
	56	list_del(&fence->head);
	57	}
	58
5e120f6e BS	59	void
	60	nouveau_fence_context_del(struct nouveau_fence_chan *fctx)
	61	{
	62	struct nouveau_fence fence, fnext;
	63	spin_lock(&fctx->lock);
	64	list_for_each_entry_safe(fence, fnext, &fctx->pending, head) {
c4c7044f	65	nouveau_fence_signal(fence);
5e120f6e BS	66	}
	67	spin_unlock(&fctx->lock);
	68	}
	69
	70	void
	71	nouveau_fence_context_new(struct nouveau_fence_chan *fctx)
	72	{
f589be88	73	INIT_LIST_HEAD(&fctx->flip);
5e120f6e BS	74	INIT_LIST_HEAD(&fctx->pending);
	75	spin_lock_init(&fctx->lock);
	76	}
6ee73861	77
c4c7044f BS	78	static void
	79	nouveau_fence_work_handler(struct work_struct *kwork)
	80	{
	81	struct fence_work work = container_of(kwork, typeof(work), base);
	82	work->func(work->data);
	83	kfree(work);
	84	}
	85
	86	void
	87	nouveau_fence_work(struct nouveau_fence *fence,
	88	void (func)(void ), void *data)
	89	{
	90	struct nouveau_channel *chan = fence->channel;
	91	struct nouveau_fence_chan *fctx;
	92	struct fence_work *work = NULL;
	93
	94	if (nouveau_fence_done(fence)) {
	95	func(data);
	96	return;
	97	}
	98
	99	fctx = chan->fence;
	100	work = kmalloc(sizeof(*work), GFP_KERNEL);
	101	if (!work) {
	102	WARN_ON(nouveau_fence_wait(fence, false, false));
	103	func(data);
	104	return;
	105	}
	106
	107	spin_lock(&fctx->lock);
	108	if (!fence->channel) {
	109	spin_unlock(&fctx->lock);
	110	kfree(work);
	111	func(data);
	112	return;
	113	}
	114
	115	INIT_WORK(&work->base, nouveau_fence_work_handler);
	116	work->func = func;
	117	work->data = data;
	118	list_add(&work->head, &fence->work);
	119	spin_unlock(&fctx->lock);
	120	}
	121
ebb945a9	122	static void
6ee73861 BS	123	nouveau_fence_update(struct nouveau_channel *chan)
6ee73861 BS	124	{
e193b1d4	125	struct nouveau_fence_chan *fctx = chan->fence;
5e120f6e	126	struct nouveau_fence fence, fnext;
6ee73861	127
5e120f6e BS	128	spin_lock(&fctx->lock);
5e120f6e BS	129	list_for_each_entry_safe(fence, fnext, &fctx->pending, head) {
827520ce	130	if (fctx->read(chan) < fence->sequence)
b08abd4e BS	131	break;
b08abd4e BS	132
c4c7044f	133	nouveau_fence_signal(fence);
d375e7d5	134	nouveau_fence_unref(&fence);
6ee73861	135	}
5e120f6e	136	spin_unlock(&fctx->lock);
6ee73861 BS	137	}
	138
	139	int
d375e7d5	140	nouveau_fence_emit(struct nouveau_fence fence, struct nouveau_channel chan)
6ee73861	141	{
e193b1d4	142	struct nouveau_fence_chan *fctx = chan->fence;
6ee73861 BS	143	int ret;
6ee73861 BS	144
5e120f6e	145	fence->channel = chan;
06b237ef	146	fence->timeout = jiffies + (15 * DRM_HZ);
5e120f6e	147	fence->sequence = ++fctx->sequence;
6ee73861	148
827520ce	149	ret = fctx->emit(fence);
5e120f6e BS	150	if (!ret) {
	151	kref_get(&fence->kref);
	152	spin_lock(&fctx->lock);
	153	list_add_tail(&fence->head, &fctx->pending);
	154	spin_unlock(&fctx->lock);
529c4959	155	}
6ee73861	156
5e120f6e	157	return ret;
6ee73861 BS	158	}
6ee73861 BS	159
6ee73861	160	bool
d375e7d5	161	nouveau_fence_done(struct nouveau_fence *fence)
6ee73861	162	{
d375e7d5 BS	163	if (fence->channel)
	164	nouveau_fence_update(fence->channel);
	165	return !fence->channel;
6ee73861 BS	166	}
6ee73861 BS	167
e18c080f BS	168	struct nouveau_fence_uevent {
	169	struct nouveau_eventh handler;
	170	struct nouveau_fence_priv *priv;
	171	};
	172
	173	static int
	174	nouveau_fence_wait_uevent_handler(struct nouveau_eventh *event, int index)
	175	{
	176	struct nouveau_fence_uevent *uevent =
	177	container_of(event, struct nouveau_fence_uevent, handler);
	178	wake_up_all(&uevent->priv->waiting);
	179	return NVKM_EVENT_KEEP;
	180	}
	181
	182	static int
	183	nouveau_fence_wait_uevent(struct nouveau_fence *fence, bool intr)
	184
	185	{
	186	struct nouveau_channel *chan = fence->channel;
	187	struct nouveau_fifo *pfifo = nouveau_fifo(chan->drm->device);
	188	struct nouveau_fence_priv *priv = chan->drm->fence;
	189	struct nouveau_fence_uevent uevent = {
	190	.handler.func = nouveau_fence_wait_uevent_handler,
	191	.priv = priv,
	192	};
	193	int ret = 0;
	194
	195	nouveau_event_get(pfifo->uevent, 0, &uevent.handler);
	196
	197	if (fence->timeout) {
	198	unsigned long timeout = fence->timeout - jiffies;
	199
	200	if (time_before(jiffies, fence->timeout)) {
	201	if (intr) {
	202	ret = wait_event_interruptible_timeout(
	203	priv->waiting,
	204	nouveau_fence_done(fence),
	205	timeout);
	206	} else {
	207	ret = wait_event_timeout(priv->waiting,
	208	nouveau_fence_done(fence),
	209	timeout);
	210	}
	211	}
	212
	213	if (ret >= 0) {
	214	fence->timeout = jiffies + ret;
	215	if (time_after_eq(jiffies, fence->timeout))
	216	ret = -EBUSY;
	217	}
	218	} else {
	219	if (intr) {
	220	ret = wait_event_interruptible(priv->waiting,
	221	nouveau_fence_done(fence));
	222	} else {
	223	wait_event(priv->waiting, nouveau_fence_done(fence));
	224	}
	225	}
	226
	227	nouveau_event_put(pfifo->uevent, 0, &uevent.handler);
	228	if (unlikely(ret < 0))
	229	return ret;
	230
	231	return 0;
232	}
233
6ee73861	234	int
875ac34a	235	nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)
6ee73861	236	{
e18c080f BS	237	struct nouveau_channel *chan = fence->channel;
e18c080f BS	238	struct nouveau_fence_priv *priv = chan ? chan->drm->fence : NULL;
bd35fe5a MS	239	unsigned long sleep_time = NSEC_PER_MSEC / 1000;
bd35fe5a MS	240	ktime_t t;
6ee73861 BS	241	int ret = 0;
6ee73861 BS	242
e18c080f BS	243	while (priv && priv->uevent && lazy && !nouveau_fence_done(fence)) {
	244	ret = nouveau_fence_wait_uevent(fence, intr);
	245	if (ret < 0)
	246	return ret;
	247	}
	248
d375e7d5 BS	249	while (!nouveau_fence_done(fence)) {
d375e7d5 BS	250	if (fence->timeout && time_after_eq(jiffies, fence->timeout)) {
6ee73861 BS	251	ret = -EBUSY;
	252	break;
	253	}
	254
875ac34a BS	255	__set_current_state(intr ? TASK_INTERRUPTIBLE :
875ac34a BS	256	TASK_UNINTERRUPTIBLE);
bd35fe5a MS	257	if (lazy) {
	258	t = ktime_set(0, sleep_time);
	259	schedule_hrtimeout(&t, HRTIMER_MODE_REL);
	260	sleep_time *= 2;
	261	if (sleep_time > NSEC_PER_MSEC)
	262	sleep_time = NSEC_PER_MSEC;
	263	}
6ee73861 BS	264
6ee73861 BS	265	if (intr && signal_pending(current)) {
9ddc8c52	266	ret = -ERESTARTSYS;
6ee73861 BS	267	break;
	268	}
	269	}
	270
	271	__set_current_state(TASK_RUNNING);
d375e7d5 BS	272	return ret;
	273	}
	274
5e120f6e BS	275	int
	276	nouveau_fence_sync(struct nouveau_fence fence, struct nouveau_channel chan)
	277	{
827520ce	278	struct nouveau_fence_chan *fctx = chan->fence;
906c033e	279	struct nouveau_channel *prev;
5e120f6e BS	280	int ret = 0;
5e120f6e BS	281
ebb945a9	282	prev = fence ? fence->channel : NULL;
906c033e BS	283	if (prev) {
906c033e BS	284	if (unlikely(prev != chan && !nouveau_fence_done(fence))) {
827520ce	285	ret = fctx->sync(fence, prev, chan);
906c033e BS	286	if (unlikely(ret))
	287	ret = nouveau_fence_wait(fence, true, false);
	288	}
5e120f6e BS	289	}
	290
	291	return ret;
	292	}
	293
d375e7d5 BS	294	static void
	295	nouveau_fence_del(struct kref *kref)
	296	{
	297	struct nouveau_fence fence = container_of(kref, typeof(fence), kref);
	298	kfree(fence);
	299	}
	300
	301	void
	302	nouveau_fence_unref(struct nouveau_fence **pfence)
	303	{
	304	if (*pfence)
	305	kref_put(&(*pfence)->kref, nouveau_fence_del);
	306	*pfence = NULL;
	307	}
	308
	309	struct nouveau_fence *
	310	nouveau_fence_ref(struct nouveau_fence *fence)
	311	{
	312	kref_get(&fence->kref);
	313	return fence;
	314	}
	315
	316	int
264ce192 BS	317	nouveau_fence_new(struct nouveau_channel *chan, bool sysmem,
264ce192 BS	318	struct nouveau_fence **pfence)
d375e7d5 BS	319	{
	320	struct nouveau_fence *fence;
	321	int ret = 0;
6ee73861	322
e193b1d4	323	if (unlikely(!chan->fence))
5e120f6e BS	324	return -ENODEV;
5e120f6e BS	325
d375e7d5 BS	326	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
	327	if (!fence)
	328	return -ENOMEM;
264ce192	329
c4c7044f	330	INIT_LIST_HEAD(&fence->work);
264ce192	331	fence->sysmem = sysmem;
d375e7d5 BS	332	kref_init(&fence->kref);
d375e7d5 BS	333
b5d8f052 CD	334	ret = nouveau_fence_emit(fence, chan);
	335	if (ret)
	336	nouveau_fence_unref(&fence);
d375e7d5 BS	337
d375e7d5 BS	338	*pfence = fence;
6ee73861 BS	339	return ret;
6ee73861 BS	340	}