[linux-2.6-block.git] / drivers / gpu / drm / i915 / gem / i915_gem_wait.c

/*
 * SPDX-License-Identifier: MIT
 *
 * Copyright © 2016 Intel Corporation
 */

#include <linux/dma-fence-array.h>
#include <linux/jiffies.h>

#include "gt/intel_engine.h"

#include "i915_gem_ioctls.h"
#include "i915_gem_object.h"

static long
i915_gem_object_wait_fence(struct dma_fence *fence,
			   unsigned int flags,
			   long timeout)
{
	BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1);

	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
		return timeout;

	if (dma_fence_is_i915(fence))
		return i915_request_wait(to_request(fence), flags, timeout);

	return dma_fence_wait_timeout(fence,
				      flags & I915_WAIT_INTERRUPTIBLE,
				      timeout);
}

static long
i915_gem_object_wait_reservation(struct reservation_object *resv,
				 unsigned int flags,
				 long timeout)
{
	struct dma_fence *excl;
	bool prune_fences = false;

	if (flags & I915_WAIT_ALL) {
		struct dma_fence **shared;
		unsigned int count, i;
		int ret;

		ret = reservation_object_get_fences_rcu(resv,
							&excl, &count, &shared);
		if (ret)
			return ret;

		for (i = 0; i < count; i++) {
			timeout = i915_gem_object_wait_fence(shared[i],
							     flags, timeout);
			if (timeout < 0)
				break;

			dma_fence_put(shared[i]);
		}

		for (; i < count; i++)
			dma_fence_put(shared[i]);
		kfree(shared);

		/*
		 * If both shared fences and an exclusive fence exist,
		 * then by construction the shared fences must be later
		 * than the exclusive fence. If we successfully wait for
		 * all the shared fences, we know that the exclusive fence
		 * must all be signaled. If all the shared fences are
		 * signaled, we can prune the array and recover the
		 * floating references on the fences/requests.
		 */
		prune_fences = count && timeout >= 0;
	} else {
		excl = reservation_object_get_excl_rcu(resv);
	}

	if (excl && timeout >= 0)
		timeout = i915_gem_object_wait_fence(excl, flags, timeout);

	dma_fence_put(excl);

	/*
	 * Opportunistically prune the fences iff we know they have *all* been
	 * signaled.
	 */
	if (prune_fences && reservation_object_trylock(resv)) {
		if (reservation_object_test_signaled_rcu(resv, true))
			reservation_object_add_excl_fence(resv, NULL);
		reservation_object_unlock(resv);
	}

	return timeout;
}

static void __fence_set_priority(struct dma_fence *fence,
				 const struct i915_sched_attr *attr)
{
	struct i915_request *rq;
	struct intel_engine_cs *engine;

	if (dma_fence_is_signaled(fence) || !dma_fence_is_i915(fence))
		return;

	rq = to_request(fence);
	engine = rq->engine;

	local_bh_disable();
	rcu_read_lock(); /* RCU serialisation for set-wedged protection */
	if (engine->schedule)
		engine->schedule(rq, attr);
	rcu_read_unlock();
	local_bh_enable(); /* kick the tasklets if queues were reprioritised */
}

static void fence_set_priority(struct dma_fence *fence,
			       const struct i915_sched_attr *attr)
{
	/* Recurse once into a fence-array */
	if (dma_fence_is_array(fence)) {
		struct dma_fence_array *array = to_dma_fence_array(fence);
		int i;

		for (i = 0; i < array->num_fences; i++)
			__fence_set_priority(array->fences[i], attr);
	} else {
		__fence_set_priority(fence, attr);
	}
}

int
i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
			      unsigned int flags,
			      const struct i915_sched_attr *attr)
{
	struct dma_fence *excl;

	if (flags & I915_WAIT_ALL) {
		struct dma_fence **shared;
		unsigned int count, i;
		int ret;

		ret = reservation_object_get_fences_rcu(obj->base.resv,
							&excl, &count, &shared);
		if (ret)
			return ret;

		for (i = 0; i < count; i++) {
			fence_set_priority(shared[i], attr);
			dma_fence_put(shared[i]);
		}

		kfree(shared);
	} else {
		excl = reservation_object_get_excl_rcu(obj->base.resv);
	}

	if (excl) {
		fence_set_priority(excl, attr);
		dma_fence_put(excl);
	}
	return 0;
}

/**
 * Waits for rendering to the object to be completed
 * @obj: i915 gem object
 * @flags: how to wait (under a lock, for all rendering or just for writes etc)
 * @timeout: how long to wait
 */
int
i915_gem_object_wait(struct drm_i915_gem_object *obj,
		     unsigned int flags,
		     long timeout)
{
	might_sleep();
	GEM_BUG_ON(timeout < 0);

	timeout = i915_gem_object_wait_reservation(obj->base.resv,
						   flags, timeout);
	return timeout < 0 ? timeout : 0;
}

static inline unsigned long nsecs_to_jiffies_timeout(const u64 n)
{
	/* nsecs_to_jiffies64() does not guard against overflow */
	if (NSEC_PER_SEC % HZ &&
	    div_u64(n, NSEC_PER_SEC) >= MAX_JIFFY_OFFSET / HZ)
		return MAX_JIFFY_OFFSET;

	return min_t(u64, MAX_JIFFY_OFFSET, nsecs_to_jiffies64(n) + 1);
}

static unsigned long to_wait_timeout(s64 timeout_ns)
{
	if (timeout_ns < 0)
		return MAX_SCHEDULE_TIMEOUT;

	if (timeout_ns == 0)
		return 0;

	return nsecs_to_jiffies_timeout(timeout_ns);
}

/**
 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
 * @dev: drm device pointer
 * @data: ioctl data blob
 * @file: drm file pointer
 *
 * Returns 0 if successful, else an error is returned with the remaining time in
 * the timeout parameter.
 *  -ETIME: object is still busy after timeout
 *  -ERESTARTSYS: signal interrupted the wait
 *  -ENONENT: object doesn't exist
 * Also possible, but rare:
 *  -EAGAIN: incomplete, restart syscall
 *  -ENOMEM: damn
 *  -ENODEV: Internal IRQ fail
 *  -E?: The add request failed
 *
 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
 * non-zero timeout parameter the wait ioctl will wait for the given number of
 * nanoseconds on an object becoming unbusy. Since the wait itself does so
 * without holding struct_mutex the object may become re-busied before this
 * function completes. A similar but shorter * race condition exists in the busy
 * ioctl
 */
int
i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
{
	struct drm_i915_gem_wait *args = data;
	struct drm_i915_gem_object *obj;
	ktime_t start;
	long ret;

	if (args->flags != 0)
		return -EINVAL;

	obj = i915_gem_object_lookup(file, args->bo_handle);
	if (!obj)
		return -ENOENT;

	start = ktime_get();

	ret = i915_gem_object_wait(obj,
				   I915_WAIT_INTERRUPTIBLE |
				   I915_WAIT_PRIORITY |
				   I915_WAIT_ALL,
				   to_wait_timeout(args->timeout_ns));

	if (args->timeout_ns > 0) {
		args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start));
		if (args->timeout_ns < 0)
			args->timeout_ns = 0;

		/*
		 * Apparently ktime isn't accurate enough and occasionally has a
		 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
		 * things up to make the test happy. We allow up to 1 jiffy.
		 *
		 * This is a regression from the timespec->ktime conversion.
		 */
		if (ret == -ETIME && !nsecs_to_jiffies(args->timeout_ns))
			args->timeout_ns = 0;

		/* Asked to wait beyond the jiffie/scheduler precision? */
		if (ret == -ETIME && args->timeout_ns)
			ret = -EAGAIN;
	}

	i915_gem_object_put(obj);
	return ret;
}
Commit	Line	Data
d45a1a53 CW	1	/*
	2	* SPDX-License-Identifier: MIT
	3	*
	4	* Copyright © 2016 Intel Corporation
	5	*/
	6
	7	#include <linux/dma-fence-array.h>
	8	#include <linux/jiffies.h>
	9
	10	#include "gt/intel_engine.h"
	11
	12	#include "i915_gem_ioctls.h"
	13	#include "i915_gem_object.h"
	14
	15	static long
	16	i915_gem_object_wait_fence(struct dma_fence *fence,
	17	unsigned int flags,
	18	long timeout)
	19	{
	20	BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1);
	21
	22	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
	23	return timeout;
	24
	25	if (dma_fence_is_i915(fence))
	26	return i915_request_wait(to_request(fence), flags, timeout);
	27
	28	return dma_fence_wait_timeout(fence,
	29	flags & I915_WAIT_INTERRUPTIBLE,
	30	timeout);
	31	}
	32
	33	static long
	34	i915_gem_object_wait_reservation(struct reservation_object *resv,
	35	unsigned int flags,
	36	long timeout)
	37	{
d45a1a53 CW	38	struct dma_fence *excl;
	39	bool prune_fences = false;
	40
	41	if (flags & I915_WAIT_ALL) {
	42	struct dma_fence **shared;
	43	unsigned int count, i;
	44	int ret;
	45
	46	ret = reservation_object_get_fences_rcu(resv,
	47	&excl, &count, &shared);
	48	if (ret)
	49	return ret;
	50
	51	for (i = 0; i < count; i++) {
	52	timeout = i915_gem_object_wait_fence(shared[i],
	53	flags, timeout);
	54	if (timeout < 0)
	55	break;
	56
	57	dma_fence_put(shared[i]);
	58	}
	59
	60	for (; i < count; i++)
	61	dma_fence_put(shared[i]);
	62	kfree(shared);
	63
	64	/*
	65	* If both shared fences and an exclusive fence exist,
	66	* then by construction the shared fences must be later
	67	* than the exclusive fence. If we successfully wait for
	68	* all the shared fences, we know that the exclusive fence
	69	* must all be signaled. If all the shared fences are
	70	* signaled, we can prune the array and recover the
	71	* floating references on the fences/requests.
	72	*/
	73	prune_fences = count && timeout >= 0;
	74	} else {
	75	excl = reservation_object_get_excl_rcu(resv);
	76	}
	77
	78	if (excl && timeout >= 0)
	79	timeout = i915_gem_object_wait_fence(excl, flags, timeout);
	80
	81	dma_fence_put(excl);
	82
	83	/*
	84	* Opportunistically prune the fences iff we know they have all been
4c2488cf	85	* signaled.
d45a1a53	86	*/
4c2488cf CK	87	if (prune_fences && reservation_object_trylock(resv)) {
	88	if (reservation_object_test_signaled_rcu(resv, true))
	89	reservation_object_add_excl_fence(resv, NULL);
	90	reservation_object_unlock(resv);
d45a1a53 CW	91	}
	92
	93	return timeout;
	94	}
	95
	96	static void __fence_set_priority(struct dma_fence *fence,
	97	const struct i915_sched_attr *attr)
	98	{
	99	struct i915_request *rq;
	100	struct intel_engine_cs *engine;
	101
	102	if (dma_fence_is_signaled(fence) \|\| !dma_fence_is_i915(fence))
	103	return;
	104
	105	rq = to_request(fence);
	106	engine = rq->engine;
	107
	108	local_bh_disable();
	109	rcu_read_lock(); /* RCU serialisation for set-wedged protection */
	110	if (engine->schedule)
	111	engine->schedule(rq, attr);
	112	rcu_read_unlock();
	113	local_bh_enable(); /* kick the tasklets if queues were reprioritised */
	114	}
	115
	116	static void fence_set_priority(struct dma_fence *fence,
	117	const struct i915_sched_attr *attr)
	118	{
	119	/* Recurse once into a fence-array */
	120	if (dma_fence_is_array(fence)) {
	121	struct dma_fence_array *array = to_dma_fence_array(fence);
	122	int i;
	123
	124	for (i = 0; i < array->num_fences; i++)
	125	__fence_set_priority(array->fences[i], attr);
	126	} else {
	127	__fence_set_priority(fence, attr);
	128	}
	129	}
	130
	131	int
	132	i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
	133	unsigned int flags,
	134	const struct i915_sched_attr *attr)
	135	{
	136	struct dma_fence *excl;
	137
	138	if (flags & I915_WAIT_ALL) {
	139	struct dma_fence **shared;
	140	unsigned int count, i;
	141	int ret;
	142
ef78f7b1	143	ret = reservation_object_get_fences_rcu(obj->base.resv,
d45a1a53 CW	144	&excl, &count, &shared);
	145	if (ret)
	146	return ret;
	147
	148	for (i = 0; i < count; i++) {
	149	fence_set_priority(shared[i], attr);
	150	dma_fence_put(shared[i]);
	151	}
	152
	153	kfree(shared);
	154	} else {
ef78f7b1	155	excl = reservation_object_get_excl_rcu(obj->base.resv);
d45a1a53 CW	156	}
	157
	158	if (excl) {
	159	fence_set_priority(excl, attr);
	160	dma_fence_put(excl);
	161	}
	162	return 0;
	163	}
	164
	165	/**
	166	* Waits for rendering to the object to be completed
	167	* @obj: i915 gem object
	168	* @flags: how to wait (under a lock, for all rendering or just for writes etc)
	169	* @timeout: how long to wait
	170	*/
	171	int
	172	i915_gem_object_wait(struct drm_i915_gem_object *obj,
	173	unsigned int flags,
	174	long timeout)
	175	{
	176	might_sleep();
	177	GEM_BUG_ON(timeout < 0);
	178
ef78f7b1 CW	179	timeout = i915_gem_object_wait_reservation(obj->base.resv,
ef78f7b1 CW	180	flags, timeout);
d45a1a53 CW	181	return timeout < 0 ? timeout : 0;
	182	}
	183
	184	static inline unsigned long nsecs_to_jiffies_timeout(const u64 n)
	185	{
	186	/* nsecs_to_jiffies64() does not guard against overflow */
	187	if (NSEC_PER_SEC % HZ &&
	188	div_u64(n, NSEC_PER_SEC) >= MAX_JIFFY_OFFSET / HZ)
	189	return MAX_JIFFY_OFFSET;
	190
	191	return min_t(u64, MAX_JIFFY_OFFSET, nsecs_to_jiffies64(n) + 1);
	192	}
	193
	194	static unsigned long to_wait_timeout(s64 timeout_ns)
	195	{
	196	if (timeout_ns < 0)
	197	return MAX_SCHEDULE_TIMEOUT;
	198
	199	if (timeout_ns == 0)
	200	return 0;
	201
	202	return nsecs_to_jiffies_timeout(timeout_ns);
	203	}
	204
	205	/**
	206	* i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
	207	* @dev: drm device pointer
	208	* @data: ioctl data blob
	209	* @file: drm file pointer
	210	*
	211	* Returns 0 if successful, else an error is returned with the remaining time in
	212	* the timeout parameter.
	213	* -ETIME: object is still busy after timeout
	214	* -ERESTARTSYS: signal interrupted the wait
	215	* -ENONENT: object doesn't exist
	216	* Also possible, but rare:
	217	* -EAGAIN: incomplete, restart syscall
	218	* -ENOMEM: damn
	219	* -ENODEV: Internal IRQ fail
	220	* -E?: The add request failed
	221	*
	222	* The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
	223	* non-zero timeout parameter the wait ioctl will wait for the given number of
	224	* nanoseconds on an object becoming unbusy. Since the wait itself does so
	225	* without holding struct_mutex the object may become re-busied before this
	226	* function completes. A similar but shorter * race condition exists in the busy
	227	* ioctl
	228	*/
	229	int
	230	i915_gem_wait_ioctl(struct drm_device dev, void data, struct drm_file *file)
	231	{
	232	struct drm_i915_gem_wait *args = data;
	233	struct drm_i915_gem_object *obj;
	234	ktime_t start;
	235	long ret;
	236
	237	if (args->flags != 0)
	238	return -EINVAL;
	239
	240	obj = i915_gem_object_lookup(file, args->bo_handle);
	241	if (!obj)
	242	return -ENOENT;
	243
	244	start = ktime_get();
245
246	ret = i915_gem_object_wait(obj,
247	I915_WAIT_INTERRUPTIBLE \|
248	I915_WAIT_PRIORITY \|
249	I915_WAIT_ALL,
250	to_wait_timeout(args->timeout_ns));
251
252	if (args->timeout_ns > 0) {
253	args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start));
254	if (args->timeout_ns < 0)
255	args->timeout_ns = 0;
256
257	/*
258	* Apparently ktime isn't accurate enough and occasionally has a
259	* bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
260	* things up to make the test happy. We allow up to 1 jiffy.
261	*
262	* This is a regression from the timespec->ktime conversion.
263	*/
264	if (ret == -ETIME && !nsecs_to_jiffies(args->timeout_ns))
265	args->timeout_ns = 0;
266
267	/* Asked to wait beyond the jiffie/scheduler precision? */
268	if (ret == -ETIME && args->timeout_ns)
269	ret = -EAGAIN;
270	}
271
272	i915_gem_object_put(obj);
273	return ret;
274	}