[linux-block.git] / drivers / gpu / drm / xe / xe_sync.c

// SPDX-License-Identifier: MIT
/*
 * Copyright © 2021 Intel Corporation
 */

#include "xe_sync.h"

#include <linux/dma-fence-array.h>
#include <linux/kthread.h>
#include <linux/sched/mm.h>
#include <linux/uaccess.h>

#include <drm/drm_print.h>
#include <drm/drm_syncobj.h>
#include <drm/xe_drm.h>

#include "xe_device_types.h"
#include "xe_exec_queue.h"
#include "xe_macros.h"
#include "xe_sched_job_types.h"

struct xe_user_fence {
	struct xe_device *xe;
	struct kref refcount;
	struct dma_fence_cb cb;
	struct work_struct worker;
	struct mm_struct *mm;
	u64 __user *addr;
	u64 value;
	int signalled;
};

static void user_fence_destroy(struct kref *kref)
{
	struct xe_user_fence *ufence = container_of(kref, struct xe_user_fence,
						 refcount);

	mmdrop(ufence->mm);
	kfree(ufence);
}

static void user_fence_get(struct xe_user_fence *ufence)
{
	kref_get(&ufence->refcount);
}

static void user_fence_put(struct xe_user_fence *ufence)
{
	kref_put(&ufence->refcount, user_fence_destroy);
}

static struct xe_user_fence *user_fence_create(struct xe_device *xe, u64 addr,
					       u64 value)
{
	struct xe_user_fence *ufence;

	ufence = kmalloc(sizeof(*ufence), GFP_KERNEL);
	if (!ufence)
		return NULL;

	ufence->xe = xe;
	kref_init(&ufence->refcount);
	ufence->addr = u64_to_user_ptr(addr);
	ufence->value = value;
	ufence->mm = current->mm;
	mmgrab(ufence->mm);

	return ufence;
}

static void user_fence_worker(struct work_struct *w)
{
	struct xe_user_fence *ufence = container_of(w, struct xe_user_fence, worker);

	if (mmget_not_zero(ufence->mm)) {
		kthread_use_mm(ufence->mm);
		if (copy_to_user(ufence->addr, &ufence->value, sizeof(ufence->value)))
			XE_WARN_ON("Copy to user failed");
		kthread_unuse_mm(ufence->mm);
		mmput(ufence->mm);
	}

	wake_up_all(&ufence->xe->ufence_wq);
	WRITE_ONCE(ufence->signalled, 1);
	user_fence_put(ufence);
}

static void kick_ufence(struct xe_user_fence *ufence, struct dma_fence *fence)
{
	INIT_WORK(&ufence->worker, user_fence_worker);
	queue_work(ufence->xe->ordered_wq, &ufence->worker);
	dma_fence_put(fence);
}

static void user_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
{
	struct xe_user_fence *ufence = container_of(cb, struct xe_user_fence, cb);

	kick_ufence(ufence, fence);
}

int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef,
			struct xe_sync_entry *sync,
			struct drm_xe_sync __user *sync_user,
			unsigned int flags)
{
	struct drm_xe_sync sync_in;
	int err;
	bool exec = flags & SYNC_PARSE_FLAG_EXEC;
	bool in_lr_mode = flags & SYNC_PARSE_FLAG_LR_MODE;
	bool disallow_user_fence = flags & SYNC_PARSE_FLAG_DISALLOW_USER_FENCE;
	bool signal;

	if (copy_from_user(&sync_in, sync_user, sizeof(*sync_user)))
		return -EFAULT;

	if (XE_IOCTL_DBG(xe, sync_in.flags & ~DRM_XE_SYNC_FLAG_SIGNAL) ||
	    XE_IOCTL_DBG(xe, sync_in.reserved[0] || sync_in.reserved[1]))
		return -EINVAL;

	signal = sync_in.flags & DRM_XE_SYNC_FLAG_SIGNAL;
	switch (sync_in.type) {
	case DRM_XE_SYNC_TYPE_SYNCOBJ:
		if (XE_IOCTL_DBG(xe, in_lr_mode && signal))
			return -EOPNOTSUPP;

		if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr)))
			return -EINVAL;

		sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle);
		if (XE_IOCTL_DBG(xe, !sync->syncobj))
			return -ENOENT;

		if (!signal) {
			sync->fence = drm_syncobj_fence_get(sync->syncobj);
			if (XE_IOCTL_DBG(xe, !sync->fence))
				return -EINVAL;
		}
		break;

	case DRM_XE_SYNC_TYPE_TIMELINE_SYNCOBJ:
		if (XE_IOCTL_DBG(xe, in_lr_mode && signal))
			return -EOPNOTSUPP;

		if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr)))
			return -EINVAL;

		if (XE_IOCTL_DBG(xe, sync_in.timeline_value == 0))
			return -EINVAL;

		sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle);
		if (XE_IOCTL_DBG(xe, !sync->syncobj))
			return -ENOENT;

		if (signal) {
			sync->chain_fence = dma_fence_chain_alloc();
			if (!sync->chain_fence)
				return -ENOMEM;
		} else {
			sync->fence = drm_syncobj_fence_get(sync->syncobj);
			if (XE_IOCTL_DBG(xe, !sync->fence))
				return -EINVAL;

			err = dma_fence_chain_find_seqno(&sync->fence,
							 sync_in.timeline_value);
			if (err)
				return err;
		}
		break;

	case DRM_XE_SYNC_TYPE_USER_FENCE:
		if (XE_IOCTL_DBG(xe, disallow_user_fence))
			return -EOPNOTSUPP;

		if (XE_IOCTL_DBG(xe, !signal))
			return -EOPNOTSUPP;

		if (XE_IOCTL_DBG(xe, sync_in.addr & 0x7))
			return -EINVAL;

		if (exec) {
			sync->addr = sync_in.addr;
		} else {
			sync->ufence = user_fence_create(xe, sync_in.addr,
							 sync_in.timeline_value);
			if (XE_IOCTL_DBG(xe, !sync->ufence))
				return -ENOMEM;
		}

		break;

	default:
		return -EINVAL;
	}

	sync->type = sync_in.type;
	sync->flags = sync_in.flags;
	sync->timeline_value = sync_in.timeline_value;

	return 0;
}

int xe_sync_entry_wait(struct xe_sync_entry *sync)
{
	if (sync->fence)
		dma_fence_wait(sync->fence, true);

	return 0;
}

int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job)
{
	int err;

	if (sync->fence) {
		err = drm_sched_job_add_dependency(&job->drm,
						   dma_fence_get(sync->fence));
		if (err) {
			dma_fence_put(sync->fence);
			return err;
		}
	}

	return 0;
}

void xe_sync_entry_signal(struct xe_sync_entry *sync, struct xe_sched_job *job,
			  struct dma_fence *fence)
{
	if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL))
		return;

	if (sync->chain_fence) {
		drm_syncobj_add_point(sync->syncobj, sync->chain_fence,
				      fence, sync->timeline_value);
		/*
		 * The chain's ownership is transferred to the
		 * timeline.
		 */
		sync->chain_fence = NULL;
	} else if (sync->syncobj) {
		drm_syncobj_replace_fence(sync->syncobj, fence);
	} else if (sync->ufence) {
		int err;

		dma_fence_get(fence);
		user_fence_get(sync->ufence);
		err = dma_fence_add_callback(fence, &sync->ufence->cb,
					     user_fence_cb);
		if (err == -ENOENT) {
			kick_ufence(sync->ufence, fence);
		} else if (err) {
			XE_WARN_ON("failed to add user fence");
			user_fence_put(sync->ufence);
			dma_fence_put(fence);
		}
	} else if (sync->type == DRM_XE_SYNC_TYPE_USER_FENCE) {
		job->user_fence.used = true;
		job->user_fence.addr = sync->addr;
		job->user_fence.value = sync->timeline_value;
	}
}

void xe_sync_entry_cleanup(struct xe_sync_entry *sync)
{
	if (sync->syncobj)
		drm_syncobj_put(sync->syncobj);
	if (sync->fence)
		dma_fence_put(sync->fence);
	if (sync->chain_fence)
		dma_fence_put(&sync->chain_fence->base);
	if (sync->ufence)
		user_fence_put(sync->ufence);
}

/**
 * xe_sync_in_fence_get() - Get a fence from syncs, exec queue, and VM
 * @sync: input syncs
 * @num_sync: number of syncs
 * @q: exec queue
 * @vm: VM
 *
 * Get a fence from syncs, exec queue, and VM. If syncs contain in-fences create
 * and return a composite fence of all in-fences + last fence. If no in-fences
 * return last fence on  input exec queue. Caller must drop reference to
 * returned fence.
 *
 * Return: fence on success, ERR_PTR(-ENOMEM) on failure
 */
struct dma_fence *
xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync,
		     struct xe_exec_queue *q, struct xe_vm *vm)
{
	struct dma_fence **fences = NULL;
	struct dma_fence_array *cf = NULL;
	struct dma_fence *fence;
	int i, num_in_fence = 0, current_fence = 0;

	lockdep_assert_held(&vm->lock);

	/* Count in-fences */
	for (i = 0; i < num_sync; ++i) {
		if (sync[i].fence) {
			++num_in_fence;
			fence = sync[i].fence;
		}
	}

	/* Easy case... */
	if (!num_in_fence) {
		fence = xe_exec_queue_last_fence_get(q, vm);
		return fence;
	}

	/* Create composite fence */
	fences = kmalloc_array(num_in_fence + 1, sizeof(*fences), GFP_KERNEL);
	if (!fences)
		return ERR_PTR(-ENOMEM);
	for (i = 0; i < num_sync; ++i) {
		if (sync[i].fence) {
			dma_fence_get(sync[i].fence);
			fences[current_fence++] = sync[i].fence;
		}
	}
	fences[current_fence++] = xe_exec_queue_last_fence_get(q, vm);
	cf = dma_fence_array_create(num_in_fence, fences,
				    vm->composite_fence_ctx,
				    vm->composite_fence_seqno++,
				    false);
	if (!cf) {
		--vm->composite_fence_seqno;
		goto err_out;
	}

	return &cf->base;

err_out:
	while (current_fence)
		dma_fence_put(fences[--current_fence]);
	kfree(fences);
	kfree(cf);

	return ERR_PTR(-ENOMEM);
}

/**
 * xe_sync_ufence_get() - Get user fence from sync
 * @sync: input sync
 *
 * Get a user fence reference from sync.
 *
 * Return: xe_user_fence pointer with reference
 */
struct xe_user_fence *xe_sync_ufence_get(struct xe_sync_entry *sync)
{
	user_fence_get(sync->ufence);

	return sync->ufence;
}

/**
 * xe_sync_ufence_put() - Put user fence reference
 * @ufence: user fence reference
 *
 */
void xe_sync_ufence_put(struct xe_user_fence *ufence)
{
	user_fence_put(ufence);
}

/**
 * xe_sync_ufence_get_status() - Get user fence status
 * @ufence: user fence
 *
 * Return: 1 if signalled, 0 not signalled, <0 on error
 */
int xe_sync_ufence_get_status(struct xe_user_fence *ufence)
{
	return READ_ONCE(ufence->signalled);
}
Commit	Line	Data
dd08ebf6 MB	1	// SPDX-License-Identifier: MIT
	2	/*
	3	* Copyright © 2021 Intel Corporation
	4	*/
	5
	6	#include "xe_sync.h"
	7
eb9702ad	8	#include <linux/dma-fence-array.h>
dd08ebf6 MB	9	#include <linux/kthread.h>
	10	#include <linux/sched/mm.h>
	11	#include <linux/uaccess.h>
ea9f879d	12
dd08ebf6 MB	13	#include <drm/drm_print.h>
dd08ebf6 MB	14	#include <drm/drm_syncobj.h>
ea9f879d	15	#include <drm/xe_drm.h>
dd08ebf6 MB	16
dd08ebf6 MB	17	#include "xe_device_types.h"
eb9702ad	18	#include "xe_exec_queue.h"
dd08ebf6	19	#include "xe_macros.h"
ea9f879d	20	#include "xe_sched_job_types.h"
dd08ebf6	21
86b3cd6d	22	struct xe_user_fence {
dd08ebf6 MB	23	struct xe_device *xe;
	24	struct kref refcount;
	25	struct dma_fence_cb cb;
	26	struct work_struct worker;
	27	struct mm_struct *mm;
	28	u64 __user *addr;
	29	u64 value;
86b3cd6d	30	int signalled;
dd08ebf6 MB	31	};
	32
	33	static void user_fence_destroy(struct kref *kref)
	34	{
86b3cd6d	35	struct xe_user_fence *ufence = container_of(kref, struct xe_user_fence,
dd08ebf6 MB	36	refcount);
	37
	38	mmdrop(ufence->mm);
	39	kfree(ufence);
	40	}
	41
86b3cd6d	42	static void user_fence_get(struct xe_user_fence *ufence)
dd08ebf6 MB	43	{
	44	kref_get(&ufence->refcount);
	45	}
	46
86b3cd6d	47	static void user_fence_put(struct xe_user_fence *ufence)
dd08ebf6 MB	48	{
	49	kref_put(&ufence->refcount, user_fence_destroy);
	50	}
	51
86b3cd6d MK	52	static struct xe_user_fence user_fence_create(struct xe_device xe, u64 addr,
86b3cd6d MK	53	u64 value)
dd08ebf6	54	{
86b3cd6d	55	struct xe_user_fence *ufence;
dd08ebf6 MB	56
	57	ufence = kmalloc(sizeof(*ufence), GFP_KERNEL);
	58	if (!ufence)
	59	return NULL;
	60
	61	ufence->xe = xe;
	62	kref_init(&ufence->refcount);
	63	ufence->addr = u64_to_user_ptr(addr);
	64	ufence->value = value;
	65	ufence->mm = current->mm;
	66	mmgrab(ufence->mm);
	67
	68	return ufence;
	69	}
	70
	71	static void user_fence_worker(struct work_struct *w)
	72	{
86b3cd6d	73	struct xe_user_fence *ufence = container_of(w, struct xe_user_fence, worker);
dd08ebf6 MB	74
	75	if (mmget_not_zero(ufence->mm)) {
	76	kthread_use_mm(ufence->mm);
	77	if (copy_to_user(ufence->addr, &ufence->value, sizeof(ufence->value)))
	78	XE_WARN_ON("Copy to user failed");
	79	kthread_unuse_mm(ufence->mm);
	80	mmput(ufence->mm);
	81	}
	82
	83	wake_up_all(&ufence->xe->ufence_wq);
86b3cd6d	84	WRITE_ONCE(ufence->signalled, 1);
dd08ebf6 MB	85	user_fence_put(ufence);
	86	}
	87
86b3cd6d	88	static void kick_ufence(struct xe_user_fence ufence, struct dma_fence fence)
dd08ebf6 MB	89	{
	90	INIT_WORK(&ufence->worker, user_fence_worker);
	91	queue_work(ufence->xe->ordered_wq, &ufence->worker);
	92	dma_fence_put(fence);
	93	}
	94
	95	static void user_fence_cb(struct dma_fence fence, struct dma_fence_cb cb)
	96	{
86b3cd6d	97	struct xe_user_fence *ufence = container_of(cb, struct xe_user_fence, cb);
dd08ebf6 MB	98
	99	kick_ufence(ufence, fence);
	100	}
	101
	102	int xe_sync_entry_parse(struct xe_device xe, struct xe_file xef,
	103	struct xe_sync_entry *sync,
	104	struct drm_xe_sync __user *sync_user,
53bf60f6	105	unsigned int flags)
dd08ebf6 MB	106	{
	107	struct drm_xe_sync sync_in;
	108	int err;
53bf60f6 MB	109	bool exec = flags & SYNC_PARSE_FLAG_EXEC;
53bf60f6 MB	110	bool in_lr_mode = flags & SYNC_PARSE_FLAG_LR_MODE;
eb9702ad	111	bool disallow_user_fence = flags & SYNC_PARSE_FLAG_DISALLOW_USER_FENCE;
7cabe558	112	bool signal;
dd08ebf6 MB	113
	114	if (copy_from_user(&sync_in, sync_user, sizeof(*sync_user)))
	115	return -EFAULT;
	116
37d078e5	117	if (XE_IOCTL_DBG(xe, sync_in.flags & ~DRM_XE_SYNC_FLAG_SIGNAL) \|\|
b8c1ba83	118	XE_IOCTL_DBG(xe, sync_in.reserved[0] \|\| sync_in.reserved[1]))
dd08ebf6 MB	119	return -EINVAL;
dd08ebf6 MB	120
3ac4a789	121	signal = sync_in.flags & DRM_XE_SYNC_FLAG_SIGNAL;
37d078e5 RV	122	switch (sync_in.type) {
37d078e5 RV	123	case DRM_XE_SYNC_TYPE_SYNCOBJ:
fdb6a053	124	if (XE_IOCTL_DBG(xe, in_lr_mode && signal))
ee6ad137	125	return -EOPNOTSUPP;
dd08ebf6	126
b8c1ba83	127	if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr)))
dd08ebf6 MB	128	return -EINVAL;
	129
	130	sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle);
b8c1ba83	131	if (XE_IOCTL_DBG(xe, !sync->syncobj))
dd08ebf6 MB	132	return -ENOENT;
dd08ebf6 MB	133
7cabe558	134	if (!signal) {
dd08ebf6	135	sync->fence = drm_syncobj_fence_get(sync->syncobj);
b8c1ba83	136	if (XE_IOCTL_DBG(xe, !sync->fence))
dd08ebf6 MB	137	return -EINVAL;
	138	}
	139	break;
	140
37d078e5	141	case DRM_XE_SYNC_TYPE_TIMELINE_SYNCOBJ:
fdb6a053	142	if (XE_IOCTL_DBG(xe, in_lr_mode && signal))
ee6ad137	143	return -EOPNOTSUPP;
dd08ebf6	144
b8c1ba83	145	if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr)))
dd08ebf6 MB	146	return -EINVAL;
dd08ebf6 MB	147
b8c1ba83	148	if (XE_IOCTL_DBG(xe, sync_in.timeline_value == 0))
dd08ebf6 MB	149	return -EINVAL;
	150
	151	sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle);
b8c1ba83	152	if (XE_IOCTL_DBG(xe, !sync->syncobj))
dd08ebf6 MB	153	return -ENOENT;
dd08ebf6 MB	154
7cabe558	155	if (signal) {
dd08ebf6 MB	156	sync->chain_fence = dma_fence_chain_alloc();
	157	if (!sync->chain_fence)
	158	return -ENOMEM;
	159	} else {
	160	sync->fence = drm_syncobj_fence_get(sync->syncobj);
b8c1ba83	161	if (XE_IOCTL_DBG(xe, !sync->fence))
dd08ebf6 MB	162	return -EINVAL;
	163
	164	err = dma_fence_chain_find_seqno(&sync->fence,
	165	sync_in.timeline_value);
	166	if (err)
	167	return err;
	168	}
	169	break;
	170
37d078e5	171	case DRM_XE_SYNC_TYPE_USER_FENCE:
eb9702ad MB	172	if (XE_IOCTL_DBG(xe, disallow_user_fence))
	173	return -EOPNOTSUPP;
	174
b8c1ba83	175	if (XE_IOCTL_DBG(xe, !signal))
ee6ad137	176	return -EOPNOTSUPP;
dd08ebf6	177
b8c1ba83	178	if (XE_IOCTL_DBG(xe, sync_in.addr & 0x7))
dd08ebf6 MB	179	return -EINVAL;
	180
	181	if (exec) {
	182	sync->addr = sync_in.addr;
	183	} else {
	184	sync->ufence = user_fence_create(xe, sync_in.addr,
	185	sync_in.timeline_value);
b8c1ba83	186	if (XE_IOCTL_DBG(xe, !sync->ufence))
dd08ebf6 MB	187	return -ENOMEM;
	188	}
	189
	190	break;
	191
	192	default:
	193	return -EINVAL;
	194	}
	195
37d078e5	196	sync->type = sync_in.type;
dd08ebf6 MB	197	sync->flags = sync_in.flags;
	198	sync->timeline_value = sync_in.timeline_value;
	199
	200	return 0;
	201	}
	202
	203	int xe_sync_entry_wait(struct xe_sync_entry *sync)
	204	{
	205	if (sync->fence)
	206	dma_fence_wait(sync->fence, true);
	207
	208	return 0;
	209	}
	210
	211	int xe_sync_entry_add_deps(struct xe_sync_entry sync, struct xe_sched_job job)
	212	{
	213	int err;
	214
	215	if (sync->fence) {
	216	err = drm_sched_job_add_dependency(&job->drm,
	217	dma_fence_get(sync->fence));
	218	if (err) {
	219	dma_fence_put(sync->fence);
	220	return err;
	221	}
	222	}
	223
	224	return 0;
	225	}
	226
f3e9b1f4	227	void xe_sync_entry_signal(struct xe_sync_entry sync, struct xe_sched_job job,
dd08ebf6 MB	228	struct dma_fence *fence)
dd08ebf6 MB	229	{
3ac4a789	230	if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL))
f3e9b1f4	231	return;
dd08ebf6 MB	232
	233	if (sync->chain_fence) {
	234	drm_syncobj_add_point(sync->syncobj, sync->chain_fence,
	235	fence, sync->timeline_value);
	236	/*
	237	* The chain's ownership is transferred to the
	238	* timeline.
	239	*/
	240	sync->chain_fence = NULL;
	241	} else if (sync->syncobj) {
	242	drm_syncobj_replace_fence(sync->syncobj, fence);
	243	} else if (sync->ufence) {
	244	int err;
	245
	246	dma_fence_get(fence);
	247	user_fence_get(sync->ufence);
	248	err = dma_fence_add_callback(fence, &sync->ufence->cb,
	249	user_fence_cb);
	250	if (err == -ENOENT) {
	251	kick_ufence(sync->ufence, fence);
	252	} else if (err) {
	253	XE_WARN_ON("failed to add user fence");
	254	user_fence_put(sync->ufence);
	255	dma_fence_put(fence);
	256	}
37d078e5	257	} else if (sync->type == DRM_XE_SYNC_TYPE_USER_FENCE) {
dd08ebf6 MB	258	job->user_fence.used = true;
	259	job->user_fence.addr = sync->addr;
	260	job->user_fence.value = sync->timeline_value;
	261	}
dd08ebf6 MB	262	}
	263
	264	void xe_sync_entry_cleanup(struct xe_sync_entry *sync)
	265	{
	266	if (sync->syncobj)
	267	drm_syncobj_put(sync->syncobj);
	268	if (sync->fence)
	269	dma_fence_put(sync->fence);
	270	if (sync->chain_fence)
	271	dma_fence_put(&sync->chain_fence->base);
	272	if (sync->ufence)
	273	user_fence_put(sync->ufence);
	274	}
eb9702ad MB	275
	276	/**
	277	* xe_sync_in_fence_get() - Get a fence from syncs, exec queue, and VM
	278	* @sync: input syncs
	279	* @num_sync: number of syncs
	280	* @q: exec queue
	281	* @vm: VM
	282	*
	283	* Get a fence from syncs, exec queue, and VM. If syncs contain in-fences create
	284	* and return a composite fence of all in-fences + last fence. If no in-fences
	285	* return last fence on input exec queue. Caller must drop reference to
	286	* returned fence.
	287	*
	288	* Return: fence on success, ERR_PTR(-ENOMEM) on failure
	289	*/
	290	struct dma_fence *
	291	xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync,
	292	struct xe_exec_queue q, struct xe_vm vm)
	293	{
	294	struct dma_fence **fences = NULL;
	295	struct dma_fence_array *cf = NULL;
	296	struct dma_fence *fence;
	297	int i, num_in_fence = 0, current_fence = 0;
	298
	299	lockdep_assert_held(&vm->lock);
	300
	301	/* Count in-fences */
	302	for (i = 0; i < num_sync; ++i) {
	303	if (sync[i].fence) {
	304	++num_in_fence;
	305	fence = sync[i].fence;
	306	}
	307	}
	308
	309	/* Easy case... */
	310	if (!num_in_fence) {
	311	fence = xe_exec_queue_last_fence_get(q, vm);
eb9702ad MB	312	return fence;
	313	}
	314
	315	/* Create composite fence */
	316	fences = kmalloc_array(num_in_fence + 1, sizeof(*fences), GFP_KERNEL);
	317	if (!fences)
	318	return ERR_PTR(-ENOMEM);
	319	for (i = 0; i < num_sync; ++i) {
	320	if (sync[i].fence) {
	321	dma_fence_get(sync[i].fence);
	322	fences[current_fence++] = sync[i].fence;
	323	}
	324	}
	325	fences[current_fence++] = xe_exec_queue_last_fence_get(q, vm);
eb9702ad MB	326	cf = dma_fence_array_create(num_in_fence, fences,
	327	vm->composite_fence_ctx,
	328	vm->composite_fence_seqno++,
	329	false);
	330	if (!cf) {
	331	--vm->composite_fence_seqno;
	332	goto err_out;
	333	}
	334
	335	return &cf->base;
	336
	337	err_out:
	338	while (current_fence)
	339	dma_fence_put(fences[--current_fence]);
	340	kfree(fences);
	341	kfree(cf);
	342
	343	return ERR_PTR(-ENOMEM);
	344	}
86b3cd6d MK	345
	346	/**
	347	* xe_sync_ufence_get() - Get user fence from sync
	348	* @sync: input sync
	349	*
	350	* Get a user fence reference from sync.
	351	*
	352	* Return: xe_user_fence pointer with reference
	353	*/
	354	struct xe_user_fence xe_sync_ufence_get(struct xe_sync_entry sync)
	355	{
	356	user_fence_get(sync->ufence);
	357
	358	return sync->ufence;
	359	}
	360
	361	/**
	362	* xe_sync_ufence_put() - Put user fence reference
	363	* @ufence: user fence reference
	364	*
	365	*/
	366	void xe_sync_ufence_put(struct xe_user_fence *ufence)
	367	{
	368	user_fence_put(ufence);
	369	}
	370
	371	/**
	372	* xe_sync_ufence_get_status() - Get user fence status
	373	* @ufence: user fence
	374	*
	375	* Return: 1 if signalled, 0 not signalled, <0 on error
	376	*/
	377	int xe_sync_ufence_get_status(struct xe_user_fence *ufence)
	378	{
	379	return READ_ONCE(ufence->signalled);
	380	}