[linux-2.6-block.git] / io_uring / cancel.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/namei.h>
#include <linux/nospec.h>
#include <linux/io_uring.h>

#include <uapi/linux/io_uring.h>

#include "io_uring.h"
#include "tctx.h"
#include "poll.h"
#include "timeout.h"
#include "waitid.h"
#include "futex.h"
#include "cancel.h"

struct io_cancel {
	struct file			*file;
	u64				addr;
	u32				flags;
	s32				fd;
	u8				opcode;
};

#define CANCEL_FLAGS	(IORING_ASYNC_CANCEL_ALL | IORING_ASYNC_CANCEL_FD | \
			 IORING_ASYNC_CANCEL_ANY | IORING_ASYNC_CANCEL_FD_FIXED | \
			 IORING_ASYNC_CANCEL_USERDATA | IORING_ASYNC_CANCEL_OP)

/*
 * Returns true if the request matches the criteria outlined by 'cd'.
 */
bool io_cancel_req_match(struct io_kiocb *req, struct io_cancel_data *cd)
{
	bool match_user_data = cd->flags & IORING_ASYNC_CANCEL_USERDATA;

	if (req->ctx != cd->ctx)
		return false;

	if (!(cd->flags & (IORING_ASYNC_CANCEL_FD | IORING_ASYNC_CANCEL_OP)))
		match_user_data = true;

	if (cd->flags & IORING_ASYNC_CANCEL_ANY)
		goto check_seq;
	if (cd->flags & IORING_ASYNC_CANCEL_FD) {
		if (req->file != cd->file)
			return false;
	}
	if (cd->flags & IORING_ASYNC_CANCEL_OP) {
		if (req->opcode != cd->opcode)
			return false;
	}
	if (match_user_data && req->cqe.user_data != cd->data)
		return false;
	if (cd->flags & IORING_ASYNC_CANCEL_ALL) {
check_seq:
		if (cd->seq == req->work.cancel_seq)
			return false;
		req->work.cancel_seq = cd->seq;
	}

	return true;
}

static bool io_cancel_cb(struct io_wq_work *work, void *data)
{
	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
	struct io_cancel_data *cd = data;

	return io_cancel_req_match(req, cd);
}

static int io_async_cancel_one(struct io_uring_task *tctx,
			       struct io_cancel_data *cd)
{
	enum io_wq_cancel cancel_ret;
	int ret = 0;
	bool all;

	if (!tctx || !tctx->io_wq)
		return -ENOENT;

	all = cd->flags & (IORING_ASYNC_CANCEL_ALL|IORING_ASYNC_CANCEL_ANY);
	cancel_ret = io_wq_cancel_cb(tctx->io_wq, io_cancel_cb, cd, all);
	switch (cancel_ret) {
	case IO_WQ_CANCEL_OK:
		ret = 0;
		break;
	case IO_WQ_CANCEL_RUNNING:
		ret = -EALREADY;
		break;
	case IO_WQ_CANCEL_NOTFOUND:
		ret = -ENOENT;
		break;
	}

	return ret;
}

int io_try_cancel(struct io_uring_task *tctx, struct io_cancel_data *cd,
		  unsigned issue_flags)
{
	struct io_ring_ctx *ctx = cd->ctx;
	int ret;

	WARN_ON_ONCE(!io_wq_current_is_worker() && tctx != current->io_uring);

	ret = io_async_cancel_one(tctx, cd);
	/*
	 * Fall-through even for -EALREADY, as we may have poll armed
	 * that need unarming.
	 */
	if (!ret)
		return 0;

	ret = io_poll_cancel(ctx, cd, issue_flags);
	if (ret != -ENOENT)
		return ret;

	ret = io_waitid_cancel(ctx, cd, issue_flags);
	if (ret != -ENOENT)
		return ret;

	ret = io_futex_cancel(ctx, cd, issue_flags);
	if (ret != -ENOENT)
		return ret;

	spin_lock(&ctx->completion_lock);
	if (!(cd->flags & IORING_ASYNC_CANCEL_FD))
		ret = io_timeout_cancel(ctx, cd);
	spin_unlock(&ctx->completion_lock);
	return ret;
}

int io_async_cancel_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	struct io_cancel *cancel = io_kiocb_to_cmd(req, struct io_cancel);

	if (unlikely(req->flags & REQ_F_BUFFER_SELECT))
		return -EINVAL;
	if (sqe->off || sqe->splice_fd_in)
		return -EINVAL;

	cancel->addr = READ_ONCE(sqe->addr);
	cancel->flags = READ_ONCE(sqe->cancel_flags);
	if (cancel->flags & ~CANCEL_FLAGS)
		return -EINVAL;
	if (cancel->flags & IORING_ASYNC_CANCEL_FD) {
		if (cancel->flags & IORING_ASYNC_CANCEL_ANY)
			return -EINVAL;
		cancel->fd = READ_ONCE(sqe->fd);
	}
	if (cancel->flags & IORING_ASYNC_CANCEL_OP) {
		if (cancel->flags & IORING_ASYNC_CANCEL_ANY)
			return -EINVAL;
		cancel->opcode = READ_ONCE(sqe->len);
	}

	return 0;
}

static int __io_async_cancel(struct io_cancel_data *cd,
			     struct io_uring_task *tctx,
			     unsigned int issue_flags)
{
	bool all = cd->flags & (IORING_ASYNC_CANCEL_ALL|IORING_ASYNC_CANCEL_ANY);
	struct io_ring_ctx *ctx = cd->ctx;
	struct io_tctx_node *node;
	int ret, nr = 0;

	do {
		ret = io_try_cancel(tctx, cd, issue_flags);
		if (ret == -ENOENT)
			break;
		if (!all)
			return ret;
		nr++;
	} while (1);

	/* slow path, try all io-wq's */
	io_ring_submit_lock(ctx, issue_flags);
	ret = -ENOENT;
	list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
		struct io_uring_task *tctx = node->task->io_uring;

		ret = io_async_cancel_one(tctx, cd);
		if (ret != -ENOENT) {
			if (!all)
				break;
			nr++;
		}
	}
	io_ring_submit_unlock(ctx, issue_flags);
	return all ? nr : ret;
}

int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)
{
	struct io_cancel *cancel = io_kiocb_to_cmd(req, struct io_cancel);
	struct io_cancel_data cd = {
		.ctx	= req->ctx,
		.data	= cancel->addr,
		.flags	= cancel->flags,
		.opcode	= cancel->opcode,
		.seq	= atomic_inc_return(&req->ctx->cancel_seq),
	};
	struct io_uring_task *tctx = req->task->io_uring;
	int ret;

	if (cd.flags & IORING_ASYNC_CANCEL_FD) {
		if (req->flags & REQ_F_FIXED_FILE ||
		    cd.flags & IORING_ASYNC_CANCEL_FD_FIXED) {
			req->flags |= REQ_F_FIXED_FILE;
			req->file = io_file_get_fixed(req, cancel->fd,
							issue_flags);
		} else {
			req->file = io_file_get_normal(req, cancel->fd);
		}
		if (!req->file) {
			ret = -EBADF;
			goto done;
		}
		cd.file = req->file;
	}

	ret = __io_async_cancel(&cd, tctx, issue_flags);
done:
	if (ret < 0)
		req_set_fail(req);
	io_req_set_res(req, ret, 0);
	return IOU_OK;
}

void init_hash_table(struct io_hash_table *table, unsigned size)
{
	unsigned int i;

	for (i = 0; i < size; i++) {
		spin_lock_init(&table->hbs[i].lock);
		INIT_HLIST_HEAD(&table->hbs[i].list);
	}
}

static int __io_sync_cancel(struct io_uring_task *tctx,
			    struct io_cancel_data *cd, int fd)
{
	struct io_ring_ctx *ctx = cd->ctx;

	/* fixed must be grabbed every time since we drop the uring_lock */
	if ((cd->flags & IORING_ASYNC_CANCEL_FD) &&
	    (cd->flags & IORING_ASYNC_CANCEL_FD_FIXED)) {
		if (unlikely(fd >= ctx->nr_user_files))
			return -EBADF;
		fd = array_index_nospec(fd, ctx->nr_user_files);
		cd->file = io_file_from_index(&ctx->file_table, fd);
		if (!cd->file)
			return -EBADF;
	}

	return __io_async_cancel(cd, tctx, 0);
}

int io_sync_cancel(struct io_ring_ctx *ctx, void __user *arg)
	__must_hold(&ctx->uring_lock)
{
	struct io_cancel_data cd = {
		.ctx	= ctx,
		.seq	= atomic_inc_return(&ctx->cancel_seq),
	};
	ktime_t timeout = KTIME_MAX;
	struct io_uring_sync_cancel_reg sc;
	struct file *file = NULL;
	DEFINE_WAIT(wait);
	int ret, i;

	if (copy_from_user(&sc, arg, sizeof(sc)))
		return -EFAULT;
	if (sc.flags & ~CANCEL_FLAGS)
		return -EINVAL;
	for (i = 0; i < ARRAY_SIZE(sc.pad); i++)
		if (sc.pad[i])
			return -EINVAL;
	for (i = 0; i < ARRAY_SIZE(sc.pad2); i++)
		if (sc.pad2[i])
			return -EINVAL;

	cd.data = sc.addr;
	cd.flags = sc.flags;
	cd.opcode = sc.opcode;

	/* we can grab a normal file descriptor upfront */
	if ((cd.flags & IORING_ASYNC_CANCEL_FD) &&
	   !(cd.flags & IORING_ASYNC_CANCEL_FD_FIXED)) {
		file = fget(sc.fd);
		if (!file)
			return -EBADF;
		cd.file = file;
	}

	ret = __io_sync_cancel(current->io_uring, &cd, sc.fd);

	/* found something, done! */
	if (ret != -EALREADY)
		goto out;

	if (sc.timeout.tv_sec != -1UL || sc.timeout.tv_nsec != -1UL) {
		struct timespec64 ts = {
			.tv_sec		= sc.timeout.tv_sec,
			.tv_nsec	= sc.timeout.tv_nsec
		};

		timeout = ktime_add_ns(timespec64_to_ktime(ts), ktime_get_ns());
	}

	/*
	 * Keep looking until we get -ENOENT. we'll get woken everytime
	 * every time a request completes and will retry the cancelation.
	 */
	do {
		cd.seq = atomic_inc_return(&ctx->cancel_seq);

		prepare_to_wait(&ctx->cq_wait, &wait, TASK_INTERRUPTIBLE);

		ret = __io_sync_cancel(current->io_uring, &cd, sc.fd);

		mutex_unlock(&ctx->uring_lock);
		if (ret != -EALREADY)
			break;

		ret = io_run_task_work_sig(ctx);
		if (ret < 0)
			break;
		ret = schedule_hrtimeout(&timeout, HRTIMER_MODE_ABS);
		if (!ret) {
			ret = -ETIME;
			break;
		}
		mutex_lock(&ctx->uring_lock);
	} while (1);

	finish_wait(&ctx->cq_wait, &wait);
	mutex_lock(&ctx->uring_lock);

	if (ret == -ENOENT || ret > 0)
		ret = 0;
out:
	if (file)
		fput(file);
	return ret;
}
Commit	Line	Data
7aaff708 JA	1	// SPDX-License-Identifier: GPL-2.0
	2	#include <linux/kernel.h>
	3	#include <linux/errno.h>
	4	#include <linux/fs.h>
	5	#include <linux/file.h>
	6	#include <linux/mm.h>
	7	#include <linux/slab.h>
	8	#include <linux/namei.h>
78a861b9	9	#include <linux/nospec.h>
7aaff708 JA	10	#include <linux/io_uring.h>
	11
	12	#include <uapi/linux/io_uring.h>
	13
7aaff708 JA	14	#include "io_uring.h"
	15	#include "tctx.h"
	16	#include "poll.h"
	17	#include "timeout.h"
f31ecf67	18	#include "waitid.h"
194bb58c	19	#include "futex.h"
7aaff708 JA	20	#include "cancel.h"
	21
	22	struct io_cancel {
	23	struct file *file;
	24	u64 addr;
	25	u32 flags;
	26	s32 fd;
d7b8b079	27	u8 opcode;
7aaff708 JA	28	};
	29
	30	#define CANCEL_FLAGS (IORING_ASYNC_CANCEL_ALL \| IORING_ASYNC_CANCEL_FD \| \
8165b566	31	IORING_ASYNC_CANCEL_ANY \| IORING_ASYNC_CANCEL_FD_FIXED \| \
d7b8b079	32	IORING_ASYNC_CANCEL_USERDATA \| IORING_ASYNC_CANCEL_OP)
7aaff708	33
aa5cd116 JA	34	/*
	35	* Returns true if the request matches the criteria outlined by 'cd'.
	36	*/
	37	bool io_cancel_req_match(struct io_kiocb req, struct io_cancel_data cd)
7aaff708	38	{
8165b566 JA	39	bool match_user_data = cd->flags & IORING_ASYNC_CANCEL_USERDATA;
8165b566 JA	40
7aaff708 JA	41	if (req->ctx != cd->ctx)
7aaff708 JA	42	return false;
8165b566	43
d7b8b079	44	if (!(cd->flags & (IORING_ASYNC_CANCEL_FD \| IORING_ASYNC_CANCEL_OP)))
8165b566 JA	45	match_user_data = true;
	46
	47	if (cd->flags & IORING_ASYNC_CANCEL_ANY)
3a372b66	48	goto check_seq;
8165b566	49	if (cd->flags & IORING_ASYNC_CANCEL_FD) {
7aaff708 JA	50	if (req->file != cd->file)
7aaff708 JA	51	return false;
7aaff708	52	}
d7b8b079 JA	53	if (cd->flags & IORING_ASYNC_CANCEL_OP) {
	54	if (req->opcode != cd->opcode)
	55	return false;
	56	}
8165b566 JA	57	if (match_user_data && req->cqe.user_data != cd->data)
8165b566 JA	58	return false;
3a372b66 JA	59	if (cd->flags & IORING_ASYNC_CANCEL_ALL) {
3a372b66 JA	60	check_seq:
7aaff708 JA	61	if (cd->seq == req->work.cancel_seq)
	62	return false;
	63	req->work.cancel_seq = cd->seq;
	64	}
aa5cd116	65
7aaff708 JA	66	return true;
	67	}
	68
aa5cd116 JA	69	static bool io_cancel_cb(struct io_wq_work work, void data)
	70	{
	71	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
	72	struct io_cancel_data *cd = data;
	73
	74	return io_cancel_req_match(req, cd);
	75	}
	76
7aaff708 JA	77	static int io_async_cancel_one(struct io_uring_task *tctx,
	78	struct io_cancel_data *cd)
	79	{
	80	enum io_wq_cancel cancel_ret;
	81	int ret = 0;
	82	bool all;
	83
	84	if (!tctx \|\| !tctx->io_wq)
	85	return -ENOENT;
	86
	87	all = cd->flags & (IORING_ASYNC_CANCEL_ALL\|IORING_ASYNC_CANCEL_ANY);
	88	cancel_ret = io_wq_cancel_cb(tctx->io_wq, io_cancel_cb, cd, all);
	89	switch (cancel_ret) {
	90	case IO_WQ_CANCEL_OK:
	91	ret = 0;
	92	break;
	93	case IO_WQ_CANCEL_RUNNING:
	94	ret = -EALREADY;
	95	break;
	96	case IO_WQ_CANCEL_NOTFOUND:
	97	ret = -ENOENT;
	98	break;
	99	}
	100
	101	return ret;
	102	}
	103
88f52eaa	104	int io_try_cancel(struct io_uring_task tctx, struct io_cancel_data cd,
5d7943d9	105	unsigned issue_flags)
7aaff708	106	{
88f52eaa	107	struct io_ring_ctx *ctx = cd->ctx;
7aaff708 JA	108	int ret;
7aaff708 JA	109
88f52eaa	110	WARN_ON_ONCE(!io_wq_current_is_worker() && tctx != current->io_uring);
7aaff708	111
88f52eaa	112	ret = io_async_cancel_one(tctx, cd);
7aaff708 JA	113	/*
	114	* Fall-through even for -EALREADY, as we may have poll armed
	115	* that need unarming.
	116	*/
	117	if (!ret)
	118	return 0;
	119
5d7943d9	120	ret = io_poll_cancel(ctx, cd, issue_flags);
7aaff708	121	if (ret != -ENOENT)
4dfab8ab PB	122	return ret;
4dfab8ab PB	123
f31ecf67 JA	124	ret = io_waitid_cancel(ctx, cd, issue_flags);
	125	if (ret != -ENOENT)
	126	return ret;
	127
194bb58c JA	128	ret = io_futex_cancel(ctx, cd, issue_flags);
	129	if (ret != -ENOENT)
	130	return ret;
	131
38513c46	132	spin_lock(&ctx->completion_lock);
7aaff708 JA	133	if (!(cd->flags & IORING_ASYNC_CANCEL_FD))
7aaff708 JA	134	ret = io_timeout_cancel(ctx, cd);
7aaff708 JA	135	spin_unlock(&ctx->completion_lock);
	136	return ret;
	137	}
	138
7aaff708 JA	139	int io_async_cancel_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
7aaff708 JA	140	{
f2ccb5ae	141	struct io_cancel *cancel = io_kiocb_to_cmd(req, struct io_cancel);
7aaff708 JA	142
	143	if (unlikely(req->flags & REQ_F_BUFFER_SELECT))
	144	return -EINVAL;
d7b8b079	145	if (sqe->off \|\| sqe->splice_fd_in)
7aaff708 JA	146	return -EINVAL;
	147
	148	cancel->addr = READ_ONCE(sqe->addr);
	149	cancel->flags = READ_ONCE(sqe->cancel_flags);
	150	if (cancel->flags & ~CANCEL_FLAGS)
	151	return -EINVAL;
	152	if (cancel->flags & IORING_ASYNC_CANCEL_FD) {
	153	if (cancel->flags & IORING_ASYNC_CANCEL_ANY)
	154	return -EINVAL;
	155	cancel->fd = READ_ONCE(sqe->fd);
	156	}
d7b8b079 JA	157	if (cancel->flags & IORING_ASYNC_CANCEL_OP) {
	158	if (cancel->flags & IORING_ASYNC_CANCEL_ANY)
	159	return -EINVAL;
	160	cancel->opcode = READ_ONCE(sqe->len);
	161	}
7aaff708 JA	162
	163	return 0;
	164	}
	165
88f52eaa JA	166	static int __io_async_cancel(struct io_cancel_data *cd,
88f52eaa JA	167	struct io_uring_task *tctx,
7aaff708 JA	168	unsigned int issue_flags)
	169	{
	170	bool all = cd->flags & (IORING_ASYNC_CANCEL_ALL\|IORING_ASYNC_CANCEL_ANY);
	171	struct io_ring_ctx *ctx = cd->ctx;
	172	struct io_tctx_node *node;
	173	int ret, nr = 0;
	174
	175	do {
88f52eaa	176	ret = io_try_cancel(tctx, cd, issue_flags);
7aaff708 JA	177	if (ret == -ENOENT)
	178	break;
	179	if (!all)
	180	return ret;
	181	nr++;
	182	} while (1);
	183
	184	/* slow path, try all io-wq's */
	185	io_ring_submit_lock(ctx, issue_flags);
	186	ret = -ENOENT;
	187	list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
	188	struct io_uring_task *tctx = node->task->io_uring;
	189
	190	ret = io_async_cancel_one(tctx, cd);
	191	if (ret != -ENOENT) {
	192	if (!all)
	193	break;
	194	nr++;
	195	}
	196	}
	197	io_ring_submit_unlock(ctx, issue_flags);
	198	return all ? nr : ret;
	199	}
	200
	201	int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)
	202	{
f2ccb5ae	203	struct io_cancel *cancel = io_kiocb_to_cmd(req, struct io_cancel);
7aaff708 JA	204	struct io_cancel_data cd = {
	205	.ctx = req->ctx,
	206	.data = cancel->addr,
	207	.flags = cancel->flags,
d7b8b079	208	.opcode = cancel->opcode,
7aaff708 JA	209	.seq = atomic_inc_return(&req->ctx->cancel_seq),
7aaff708 JA	210	};
88f52eaa	211	struct io_uring_task *tctx = req->task->io_uring;
7aaff708 JA	212	int ret;
	213
	214	if (cd.flags & IORING_ASYNC_CANCEL_FD) {
7d8ca725 JA	215	if (req->flags & REQ_F_FIXED_FILE \|\|
	216	cd.flags & IORING_ASYNC_CANCEL_FD_FIXED) {
	217	req->flags \|= REQ_F_FIXED_FILE;
7aaff708 JA	218	req->file = io_file_get_fixed(req, cancel->fd,
7aaff708 JA	219	issue_flags);
7d8ca725	220	} else {
7aaff708	221	req->file = io_file_get_normal(req, cancel->fd);
7d8ca725	222	}
7aaff708 JA	223	if (!req->file) {
	224	ret = -EBADF;
	225	goto done;
	226	}
	227	cd.file = req->file;
	228	}
	229
88f52eaa	230	ret = __io_async_cancel(&cd, tctx, issue_flags);
7aaff708 JA	231	done:
	232	if (ret < 0)
	233	req_set_fail(req);
	234	io_req_set_res(req, ret, 0);
	235	return IOU_OK;
	236	}
38513c46	237
e6f89be6	238	void init_hash_table(struct io_hash_table *table, unsigned size)
38513c46 HX	239	{
	240	unsigned int i;
	241
	242	for (i = 0; i < size; i++) {
e6f89be6 PB	243	spin_lock_init(&table->hbs[i].lock);
e6f89be6 PB	244	INIT_HLIST_HEAD(&table->hbs[i].list);
38513c46 HX	245	}
38513c46 HX	246	}
78a861b9 JA	247
	248	static int __io_sync_cancel(struct io_uring_task *tctx,
	249	struct io_cancel_data *cd, int fd)
	250	{
	251	struct io_ring_ctx *ctx = cd->ctx;
	252
	253	/* fixed must be grabbed every time since we drop the uring_lock */
	254	if ((cd->flags & IORING_ASYNC_CANCEL_FD) &&
	255	(cd->flags & IORING_ASYNC_CANCEL_FD_FIXED)) {
47abea04	256	if (unlikely(fd >= ctx->nr_user_files))
78a861b9 JA	257	return -EBADF;
78a861b9 JA	258	fd = array_index_nospec(fd, ctx->nr_user_files);
60a666f0	259	cd->file = io_file_from_index(&ctx->file_table, fd);
78a861b9 JA	260	if (!cd->file)
	261	return -EBADF;
	262	}
	263
	264	return __io_async_cancel(cd, tctx, 0);
	265	}
	266
	267	int io_sync_cancel(struct io_ring_ctx ctx, void __user arg)
	268	__must_hold(&ctx->uring_lock)
	269	{
	270	struct io_cancel_data cd = {
	271	.ctx = ctx,
	272	.seq = atomic_inc_return(&ctx->cancel_seq),
	273	};
	274	ktime_t timeout = KTIME_MAX;
	275	struct io_uring_sync_cancel_reg sc;
73363c26	276	struct file *file = NULL;
78a861b9	277	DEFINE_WAIT(wait);
f77569d2	278	int ret, i;
78a861b9 JA	279
	280	if (copy_from_user(&sc, arg, sizeof(sc)))
	281	return -EFAULT;
	282	if (sc.flags & ~CANCEL_FLAGS)
	283	return -EINVAL;
f77569d2 JA	284	for (i = 0; i < ARRAY_SIZE(sc.pad); i++)
	285	if (sc.pad[i])
	286	return -EINVAL;
	287	for (i = 0; i < ARRAY_SIZE(sc.pad2); i++)
	288	if (sc.pad2[i])
	289	return -EINVAL;
78a861b9 JA	290
	291	cd.data = sc.addr;
	292	cd.flags = sc.flags;
f77569d2	293	cd.opcode = sc.opcode;
78a861b9 JA	294
	295	/* we can grab a normal file descriptor upfront */
	296	if ((cd.flags & IORING_ASYNC_CANCEL_FD) &&
	297	!(cd.flags & IORING_ASYNC_CANCEL_FD_FIXED)) {
73363c26 JA	298	file = fget(sc.fd);
73363c26 JA	299	if (!file)
78a861b9	300	return -EBADF;
73363c26	301	cd.file = file;
78a861b9 JA	302	}
	303
	304	ret = __io_sync_cancel(current->io_uring, &cd, sc.fd);
	305
	306	/* found something, done! */
	307	if (ret != -EALREADY)
	308	goto out;
	309
	310	if (sc.timeout.tv_sec != -1UL \|\| sc.timeout.tv_nsec != -1UL) {
	311	struct timespec64 ts = {
	312	.tv_sec = sc.timeout.tv_sec,
	313	.tv_nsec = sc.timeout.tv_nsec
	314	};
	315
	316	timeout = ktime_add_ns(timespec64_to_ktime(ts), ktime_get_ns());
	317	}
	318
	319	/*
	320	* Keep looking until we get -ENOENT. we'll get woken everytime
	321	* every time a request completes and will retry the cancelation.
	322	*/
	323	do {
	324	cd.seq = atomic_inc_return(&ctx->cancel_seq);
	325
	326	prepare_to_wait(&ctx->cq_wait, &wait, TASK_INTERRUPTIBLE);
	327
	328	ret = __io_sync_cancel(current->io_uring, &cd, sc.fd);
	329
23fffb2f	330	mutex_unlock(&ctx->uring_lock);
78a861b9 JA	331	if (ret != -EALREADY)
	332	break;
	333
c0e0d6ba	334	ret = io_run_task_work_sig(ctx);
23fffb2f	335	if (ret < 0)
78a861b9	336	break;
78a861b9	337	ret = schedule_hrtimeout(&timeout, HRTIMER_MODE_ABS);
78a861b9 JA	338	if (!ret) {
	339	ret = -ETIME;
	340	break;
	341	}
23fffb2f	342	mutex_lock(&ctx->uring_lock);
78a861b9 JA	343	} while (1);
	344
	345	finish_wait(&ctx->cq_wait, &wait);
23fffb2f	346	mutex_lock(&ctx->uring_lock);
78a861b9 JA	347
	348	if (ret == -ENOENT \|\| ret > 0)
	349	ret = 0;
	350	out:
73363c26 JA	351	if (file)
73363c26 JA	352	fput(file);
78a861b9 JA	353	return ret;
78a861b9 JA	354	}