[linux-block.git] / io_uring / opdef.c

// SPDX-License-Identifier: GPL-2.0
/*
 * io_uring opcode handling table
 */
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/io_uring.h>

#include "io_uring.h"
#include "opdef.h"
#include "refs.h"
#include "tctx.h"
#include "sqpoll.h"
#include "fdinfo.h"
#include "kbuf.h"
#include "rsrc.h"

#include "xattr.h"
#include "nop.h"
#include "fs.h"
#include "splice.h"
#include "sync.h"
#include "advise.h"
#include "openclose.h"
#include "uring_cmd.h"
#include "epoll.h"
#include "statx.h"
#include "net.h"
#include "msg_ring.h"
#include "timeout.h"
#include "poll.h"
#include "cancel.h"
#include "rw.h"

static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
{
	WARN_ON_ONCE(1);
	return -ECANCELED;
}

static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
					     const struct io_uring_sqe *sqe)
{
	return -EOPNOTSUPP;
}

const struct io_op_def io_op_defs[] = {
	[IORING_OP_NOP] = {
		.audit_skip		= 1,
		.iopoll			= 1,
		.name			= "NOP",
		.prep			= io_nop_prep,
		.issue			= io_nop,
	},
	[IORING_OP_READV] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.buffer_select		= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.async_size		= sizeof(struct io_async_rw),
		.name			= "READV",
		.prep			= io_prep_rw,
		.issue			= io_read,
		.prep_async		= io_readv_prep_async,
		.cleanup		= io_readv_writev_cleanup,
		.fail			= io_rw_fail,
	},
	[IORING_OP_WRITEV] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.async_size		= sizeof(struct io_async_rw),
		.name			= "WRITEV",
		.prep			= io_prep_rw,
		.issue			= io_write,
		.prep_async		= io_writev_prep_async,
		.cleanup		= io_readv_writev_cleanup,
		.fail			= io_rw_fail,
	},
	[IORING_OP_FSYNC] = {
		.needs_file		= 1,
		.audit_skip		= 1,
		.name			= "FSYNC",
		.prep			= io_fsync_prep,
		.issue			= io_fsync,
	},
	[IORING_OP_READ_FIXED] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.async_size		= sizeof(struct io_async_rw),
		.name			= "READ_FIXED",
		.prep			= io_prep_rw,
		.issue			= io_read,
		.fail			= io_rw_fail,
	},
	[IORING_OP_WRITE_FIXED] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.async_size		= sizeof(struct io_async_rw),
		.name			= "WRITE_FIXED",
		.prep			= io_prep_rw,
		.issue			= io_write,
		.fail			= io_rw_fail,
	},
	[IORING_OP_POLL_ADD] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.audit_skip		= 1,
		.name			= "POLL_ADD",
		.prep			= io_poll_add_prep,
		.issue			= io_poll_add,
	},
	[IORING_OP_POLL_REMOVE] = {
		.audit_skip		= 1,
		.name			= "POLL_REMOVE",
		.prep			= io_poll_remove_prep,
		.issue			= io_poll_remove,
	},
	[IORING_OP_SYNC_FILE_RANGE] = {
		.needs_file		= 1,
		.audit_skip		= 1,
		.name			= "SYNC_FILE_RANGE",
		.prep			= io_sfr_prep,
		.issue			= io_sync_file_range,
	},
	[IORING_OP_SENDMSG] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.ioprio			= 1,
		.manual_alloc		= 1,
		.name			= "SENDMSG",
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_msghdr),
		.prep			= io_sendmsg_prep,
		.issue			= io_sendmsg,
		.prep_async		= io_sendmsg_prep_async,
		.cleanup		= io_sendmsg_recvmsg_cleanup,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_RECVMSG] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.buffer_select		= 1,
		.ioprio			= 1,
		.manual_alloc		= 1,
		.name			= "RECVMSG",
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_msghdr),
		.prep			= io_recvmsg_prep,
		.issue			= io_recvmsg,
		.prep_async		= io_recvmsg_prep_async,
		.cleanup		= io_sendmsg_recvmsg_cleanup,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_TIMEOUT] = {
		.audit_skip		= 1,
		.async_size		= sizeof(struct io_timeout_data),
		.name			= "TIMEOUT",
		.prep			= io_timeout_prep,
		.issue			= io_timeout,
	},
	[IORING_OP_TIMEOUT_REMOVE] = {
		/* used by timeout updates' prep() */
		.audit_skip		= 1,
		.name			= "TIMEOUT_REMOVE",
		.prep			= io_timeout_remove_prep,
		.issue			= io_timeout_remove,
	},
	[IORING_OP_ACCEPT] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.poll_exclusive		= 1,
		.ioprio			= 1,	/* used for flags */
		.name			= "ACCEPT",
#if defined(CONFIG_NET)
		.prep			= io_accept_prep,
		.issue			= io_accept,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_ASYNC_CANCEL] = {
		.audit_skip		= 1,
		.name			= "ASYNC_CANCEL",
		.prep			= io_async_cancel_prep,
		.issue			= io_async_cancel,
	},
	[IORING_OP_LINK_TIMEOUT] = {
		.audit_skip		= 1,
		.async_size		= sizeof(struct io_timeout_data),
		.name			= "LINK_TIMEOUT",
		.prep			= io_link_timeout_prep,
		.issue			= io_no_issue,
	},
	[IORING_OP_CONNECT] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.name			= "CONNECT",
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_connect),
		.prep			= io_connect_prep,
		.issue			= io_connect,
		.prep_async		= io_connect_prep_async,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_FALLOCATE] = {
		.needs_file		= 1,
		.name			= "FALLOCATE",
		.prep			= io_fallocate_prep,
		.issue			= io_fallocate,
	},
	[IORING_OP_OPENAT] = {
		.name			= "OPENAT",
		.prep			= io_openat_prep,
		.issue			= io_openat,
		.cleanup		= io_open_cleanup,
	},
	[IORING_OP_CLOSE] = {
		.name			= "CLOSE",
		.prep			= io_close_prep,
		.issue			= io_close,
	},
	[IORING_OP_FILES_UPDATE] = {
		.audit_skip		= 1,
		.iopoll			= 1,
		.name			= "FILES_UPDATE",
		.prep			= io_files_update_prep,
		.issue			= io_files_update,
	},
	[IORING_OP_STATX] = {
		.audit_skip		= 1,
		.name			= "STATX",
		.prep			= io_statx_prep,
		.issue			= io_statx,
		.cleanup		= io_statx_cleanup,
	},
	[IORING_OP_READ] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.buffer_select		= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.async_size		= sizeof(struct io_async_rw),
		.name			= "READ",
		.prep			= io_prep_rw,
		.issue			= io_read,
		.fail			= io_rw_fail,
	},
	[IORING_OP_WRITE] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.async_size		= sizeof(struct io_async_rw),
		.name			= "WRITE",
		.prep			= io_prep_rw,
		.issue			= io_write,
		.fail			= io_rw_fail,
	},
	[IORING_OP_FADVISE] = {
		.needs_file		= 1,
		.audit_skip		= 1,
		.name			= "FADVISE",
		.prep			= io_fadvise_prep,
		.issue			= io_fadvise,
	},
	[IORING_OP_MADVISE] = {
		.name			= "MADVISE",
		.prep			= io_madvise_prep,
		.issue			= io_madvise,
	},
	[IORING_OP_SEND] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.name			= "SEND",
#if defined(CONFIG_NET)
		.prep			= io_sendmsg_prep,
		.issue			= io_send,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_RECV] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.buffer_select		= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.name			= "RECV",
#if defined(CONFIG_NET)
		.prep			= io_recvmsg_prep,
		.issue			= io_recv,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_OPENAT2] = {
		.name			= "OPENAT2",
		.prep			= io_openat2_prep,
		.issue			= io_openat2,
		.cleanup		= io_open_cleanup,
	},
	[IORING_OP_EPOLL_CTL] = {
		.unbound_nonreg_file	= 1,
		.audit_skip		= 1,
		.name			= "EPOLL",
#if defined(CONFIG_EPOLL)
		.prep			= io_epoll_ctl_prep,
		.issue			= io_epoll_ctl,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_SPLICE] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
		.audit_skip		= 1,
		.name			= "SPLICE",
		.prep			= io_splice_prep,
		.issue			= io_splice,
	},
	[IORING_OP_PROVIDE_BUFFERS] = {
		.audit_skip		= 1,
		.iopoll			= 1,
		.name			= "PROVIDE_BUFFERS",
		.prep			= io_provide_buffers_prep,
		.issue			= io_provide_buffers,
	},
	[IORING_OP_REMOVE_BUFFERS] = {
		.audit_skip		= 1,
		.iopoll			= 1,
		.name			= "REMOVE_BUFFERS",
		.prep			= io_remove_buffers_prep,
		.issue			= io_remove_buffers,
	},
	[IORING_OP_TEE] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
		.audit_skip		= 1,
		.name			= "TEE",
		.prep			= io_tee_prep,
		.issue			= io_tee,
	},
	[IORING_OP_SHUTDOWN] = {
		.needs_file		= 1,
		.name			= "SHUTDOWN",
#if defined(CONFIG_NET)
		.prep			= io_shutdown_prep,
		.issue			= io_shutdown,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_RENAMEAT] = {
		.name			= "RENAMEAT",
		.prep			= io_renameat_prep,
		.issue			= io_renameat,
		.cleanup		= io_renameat_cleanup,
	},
	[IORING_OP_UNLINKAT] = {
		.name			= "UNLINKAT",
		.prep			= io_unlinkat_prep,
		.issue			= io_unlinkat,
		.cleanup		= io_unlinkat_cleanup,
	},
	[IORING_OP_MKDIRAT] = {
		.name			= "MKDIRAT",
		.prep			= io_mkdirat_prep,
		.issue			= io_mkdirat,
		.cleanup		= io_mkdirat_cleanup,
	},
	[IORING_OP_SYMLINKAT] = {
		.name			= "SYMLINKAT",
		.prep			= io_symlinkat_prep,
		.issue			= io_symlinkat,
		.cleanup		= io_link_cleanup,
	},
	[IORING_OP_LINKAT] = {
		.name			= "LINKAT",
		.prep			= io_linkat_prep,
		.issue			= io_linkat,
		.cleanup		= io_link_cleanup,
	},
	[IORING_OP_MSG_RING] = {
		.needs_file		= 1,
		.iopoll			= 1,
		.name			= "MSG_RING",
		.prep			= io_msg_ring_prep,
		.issue			= io_msg_ring,
	},
	[IORING_OP_FSETXATTR] = {
		.needs_file = 1,
		.name			= "FSETXATTR",
		.prep			= io_fsetxattr_prep,
		.issue			= io_fsetxattr,
		.cleanup		= io_xattr_cleanup,
	},
	[IORING_OP_SETXATTR] = {
		.name			= "SETXATTR",
		.prep			= io_setxattr_prep,
		.issue			= io_setxattr,
		.cleanup		= io_xattr_cleanup,
	},
	[IORING_OP_FGETXATTR] = {
		.needs_file = 1,
		.name			= "FGETXATTR",
		.prep			= io_fgetxattr_prep,
		.issue			= io_fgetxattr,
		.cleanup		= io_xattr_cleanup,
	},
	[IORING_OP_GETXATTR] = {
		.name			= "GETXATTR",
		.prep			= io_getxattr_prep,
		.issue			= io_getxattr,
		.cleanup		= io_xattr_cleanup,
	},
	[IORING_OP_SOCKET] = {
		.audit_skip		= 1,
		.name			= "SOCKET",
#if defined(CONFIG_NET)
		.prep			= io_socket_prep,
		.issue			= io_socket,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_URING_CMD] = {
		.needs_file		= 1,
		.plug			= 1,
		.name			= "URING_CMD",
		.iopoll			= 1,
		.async_size		= uring_cmd_pdu_size(1),
		.prep			= io_uring_cmd_prep,
		.issue			= io_uring_cmd,
		.prep_async		= io_uring_cmd_prep_async,
	},
	[IORING_OP_SEND_ZC] = {
		.name			= "SEND_ZC",
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.manual_alloc		= 1,
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_msghdr),
		.prep			= io_sendzc_prep,
		.issue			= io_sendzc,
		.prep_async		= io_sendzc_prep_async,
		.cleanup		= io_sendzc_cleanup,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
};

const char *io_uring_get_opcode(u8 opcode)
{
	if (opcode < IORING_OP_LAST)
		return io_op_defs[opcode].name;
	return "INVALID";
}

void __init io_uring_optable_init(void)
{
	int i;

	BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);

	for (i = 0; i < ARRAY_SIZE(io_op_defs); i++) {
		BUG_ON(!io_op_defs[i].prep);
		if (io_op_defs[i].prep != io_eopnotsupp_prep)
			BUG_ON(!io_op_defs[i].issue);
		WARN_ON_ONCE(!io_op_defs[i].name);
	}
}
Commit	Line	Data
d9b57aa3 JA	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* io_uring opcode handling table
	4	*/
	5	#include <linux/kernel.h>
	6	#include <linux/errno.h>
	7	#include <linux/fs.h>
	8	#include <linux/file.h>
	9	#include <linux/io_uring.h>
	10
d9b57aa3 JA	11	#include "io_uring.h"
	12	#include "opdef.h"
	13	#include "refs.h"
	14	#include "tctx.h"
	15	#include "sqpoll.h"
	16	#include "fdinfo.h"
	17	#include "kbuf.h"
	18	#include "rsrc.h"
	19
	20	#include "xattr.h"
	21	#include "nop.h"
	22	#include "fs.h"
	23	#include "splice.h"
	24	#include "sync.h"
	25	#include "advise.h"
	26	#include "openclose.h"
	27	#include "uring_cmd.h"
	28	#include "epoll.h"
	29	#include "statx.h"
	30	#include "net.h"
	31	#include "msg_ring.h"
	32	#include "timeout.h"
	33	#include "poll.h"
	34	#include "cancel.h"
	35	#include "rw.h"
	36
	37	static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
	38	{
	39	WARN_ON_ONCE(1);
	40	return -ECANCELED;
	41	}
	42
	43	static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
	44	const struct io_uring_sqe *sqe)
	45	{
	46	return -EOPNOTSUPP;
	47	}
	48
	49	const struct io_op_def io_op_defs[] = {
	50	[IORING_OP_NOP] = {
	51	.audit_skip = 1,
	52	.iopoll = 1,
	53	.name = "NOP",
	54	.prep = io_nop_prep,
	55	.issue = io_nop,
	56	},
	57	[IORING_OP_READV] = {
	58	.needs_file = 1,
	59	.unbound_nonreg_file = 1,
	60	.pollin = 1,
	61	.buffer_select = 1,
	62	.plug = 1,
	63	.audit_skip = 1,
	64	.ioprio = 1,
	65	.iopoll = 1,
	66	.async_size = sizeof(struct io_async_rw),
	67	.name = "READV",
	68	.prep = io_prep_rw,
	69	.issue = io_read,
	70	.prep_async = io_readv_prep_async,
	71	.cleanup = io_readv_writev_cleanup,
47b4c686	72	.fail = io_rw_fail,
d9b57aa3 JA	73	},
	74	[IORING_OP_WRITEV] = {
	75	.needs_file = 1,
	76	.hash_reg_file = 1,
	77	.unbound_nonreg_file = 1,
	78	.pollout = 1,
	79	.plug = 1,
	80	.audit_skip = 1,
	81	.ioprio = 1,
	82	.iopoll = 1,
	83	.async_size = sizeof(struct io_async_rw),
	84	.name = "WRITEV",
	85	.prep = io_prep_rw,
	86	.issue = io_write,
	87	.prep_async = io_writev_prep_async,
	88	.cleanup = io_readv_writev_cleanup,
47b4c686	89	.fail = io_rw_fail,
d9b57aa3 JA	90	},
	91	[IORING_OP_FSYNC] = {
	92	.needs_file = 1,
	93	.audit_skip = 1,
	94	.name = "FSYNC",
	95	.prep = io_fsync_prep,
	96	.issue = io_fsync,
	97	},
	98	[IORING_OP_READ_FIXED] = {
	99	.needs_file = 1,
	100	.unbound_nonreg_file = 1,
	101	.pollin = 1,
	102	.plug = 1,
	103	.audit_skip = 1,
	104	.ioprio = 1,
	105	.iopoll = 1,
	106	.async_size = sizeof(struct io_async_rw),
	107	.name = "READ_FIXED",
	108	.prep = io_prep_rw,
	109	.issue = io_read,
47b4c686	110	.fail = io_rw_fail,
d9b57aa3 JA	111	},
	112	[IORING_OP_WRITE_FIXED] = {
	113	.needs_file = 1,
	114	.hash_reg_file = 1,
	115	.unbound_nonreg_file = 1,
	116	.pollout = 1,
	117	.plug = 1,
	118	.audit_skip = 1,
	119	.ioprio = 1,
	120	.iopoll = 1,
	121	.async_size = sizeof(struct io_async_rw),
	122	.name = "WRITE_FIXED",
	123	.prep = io_prep_rw,
	124	.issue = io_write,
47b4c686	125	.fail = io_rw_fail,
d9b57aa3 JA	126	},
	127	[IORING_OP_POLL_ADD] = {
	128	.needs_file = 1,
	129	.unbound_nonreg_file = 1,
	130	.audit_skip = 1,
	131	.name = "POLL_ADD",
	132	.prep = io_poll_add_prep,
	133	.issue = io_poll_add,
	134	},
	135	[IORING_OP_POLL_REMOVE] = {
	136	.audit_skip = 1,
	137	.name = "POLL_REMOVE",
	138	.prep = io_poll_remove_prep,
	139	.issue = io_poll_remove,
	140	},
	141	[IORING_OP_SYNC_FILE_RANGE] = {
	142	.needs_file = 1,
	143	.audit_skip = 1,
	144	.name = "SYNC_FILE_RANGE",
	145	.prep = io_sfr_prep,
	146	.issue = io_sync_file_range,
	147	},
	148	[IORING_OP_SENDMSG] = {
	149	.needs_file = 1,
	150	.unbound_nonreg_file = 1,
	151	.pollout = 1,
	152	.ioprio = 1,
858c293e	153	.manual_alloc = 1,
d9b57aa3 JA	154	.name = "SENDMSG",
	155	#if defined(CONFIG_NET)
	156	.async_size = sizeof(struct io_async_msghdr),
	157	.prep = io_sendmsg_prep,
	158	.issue = io_sendmsg,
	159	.prep_async = io_sendmsg_prep_async,
	160	.cleanup = io_sendmsg_recvmsg_cleanup,
	161	#else
	162	.prep = io_eopnotsupp_prep,
	163	#endif
	164	},
	165	[IORING_OP_RECVMSG] = {
	166	.needs_file = 1,
	167	.unbound_nonreg_file = 1,
	168	.pollin = 1,
	169	.buffer_select = 1,
	170	.ioprio = 1,
858c293e	171	.manual_alloc = 1,
d9b57aa3 JA	172	.name = "RECVMSG",
	173	#if defined(CONFIG_NET)
	174	.async_size = sizeof(struct io_async_msghdr),
	175	.prep = io_recvmsg_prep,
	176	.issue = io_recvmsg,
	177	.prep_async = io_recvmsg_prep_async,
	178	.cleanup = io_sendmsg_recvmsg_cleanup,
	179	#else
	180	.prep = io_eopnotsupp_prep,
	181	#endif
	182	},
	183	[IORING_OP_TIMEOUT] = {
	184	.audit_skip = 1,
	185	.async_size = sizeof(struct io_timeout_data),
	186	.name = "TIMEOUT",
	187	.prep = io_timeout_prep,
	188	.issue = io_timeout,
	189	},
	190	[IORING_OP_TIMEOUT_REMOVE] = {
	191	/* used by timeout updates' prep() */
	192	.audit_skip = 1,
	193	.name = "TIMEOUT_REMOVE",
	194	.prep = io_timeout_remove_prep,
	195	.issue = io_timeout_remove,
	196	},
	197	[IORING_OP_ACCEPT] = {
	198	.needs_file = 1,
	199	.unbound_nonreg_file = 1,
	200	.pollin = 1,
	201	.poll_exclusive = 1,
	202	.ioprio = 1, /* used for flags */
	203	.name = "ACCEPT",
	204	#if defined(CONFIG_NET)
	205	.prep = io_accept_prep,
	206	.issue = io_accept,
	207	#else
	208	.prep = io_eopnotsupp_prep,
	209	#endif
	210	},
	211	[IORING_OP_ASYNC_CANCEL] = {
	212	.audit_skip = 1,
	213	.name = "ASYNC_CANCEL",
	214	.prep = io_async_cancel_prep,
	215	.issue = io_async_cancel,
	216	},
	217	[IORING_OP_LINK_TIMEOUT] = {
	218	.audit_skip = 1,
	219	.async_size = sizeof(struct io_timeout_data),
	220	.name = "LINK_TIMEOUT",
	221	.prep = io_link_timeout_prep,
	222	.issue = io_no_issue,
	223	},
	224	[IORING_OP_CONNECT] = {
	225	.needs_file = 1,
	226	.unbound_nonreg_file = 1,
	227	.pollout = 1,
	228	.name = "CONNECT",
	229	#if defined(CONFIG_NET)
	230	.async_size = sizeof(struct io_async_connect),
	231	.prep = io_connect_prep,
	232	.issue = io_connect,
	233	.prep_async = io_connect_prep_async,
	234	#else
	235	.prep = io_eopnotsupp_prep,
236	#endif
237	},
238	[IORING_OP_FALLOCATE] = {
239	.needs_file = 1,
240	.name = "FALLOCATE",
241	.prep = io_fallocate_prep,
242	.issue = io_fallocate,
243	},
244	[IORING_OP_OPENAT] = {
245	.name = "OPENAT",
246	.prep = io_openat_prep,
247	.issue = io_openat,
248	.cleanup = io_open_cleanup,
249	},
250	[IORING_OP_CLOSE] = {
251	.name = "CLOSE",
252	.prep = io_close_prep,
253	.issue = io_close,
254	},
d9808ceb	255	[IORING_OP_FILES_UPDATE] = {
d9b57aa3 JA	256	.audit_skip = 1,
d9b57aa3 JA	257	.iopoll = 1,
d9808ceb PB	258	.name = "FILES_UPDATE",
	259	.prep = io_files_update_prep,
	260	.issue = io_files_update,
d9b57aa3 JA	261	},
	262	[IORING_OP_STATX] = {
	263	.audit_skip = 1,
	264	.name = "STATX",
	265	.prep = io_statx_prep,
	266	.issue = io_statx,
	267	.cleanup = io_statx_cleanup,
	268	},
	269	[IORING_OP_READ] = {
	270	.needs_file = 1,
	271	.unbound_nonreg_file = 1,
	272	.pollin = 1,
	273	.buffer_select = 1,
	274	.plug = 1,
	275	.audit_skip = 1,
	276	.ioprio = 1,
	277	.iopoll = 1,
	278	.async_size = sizeof(struct io_async_rw),
	279	.name = "READ",
	280	.prep = io_prep_rw,
	281	.issue = io_read,
47b4c686	282	.fail = io_rw_fail,
d9b57aa3 JA	283	},
	284	[IORING_OP_WRITE] = {
	285	.needs_file = 1,
	286	.hash_reg_file = 1,
	287	.unbound_nonreg_file = 1,
	288	.pollout = 1,
	289	.plug = 1,
	290	.audit_skip = 1,
	291	.ioprio = 1,
	292	.iopoll = 1,
	293	.async_size = sizeof(struct io_async_rw),
	294	.name = "WRITE",
	295	.prep = io_prep_rw,
	296	.issue = io_write,
47b4c686	297	.fail = io_rw_fail,
d9b57aa3 JA	298	},
	299	[IORING_OP_FADVISE] = {
	300	.needs_file = 1,
	301	.audit_skip = 1,
	302	.name = "FADVISE",
	303	.prep = io_fadvise_prep,
	304	.issue = io_fadvise,
	305	},
	306	[IORING_OP_MADVISE] = {
	307	.name = "MADVISE",
	308	.prep = io_madvise_prep,
	309	.issue = io_madvise,
	310	},
	311	[IORING_OP_SEND] = {
	312	.needs_file = 1,
	313	.unbound_nonreg_file = 1,
	314	.pollout = 1,
	315	.audit_skip = 1,
	316	.ioprio = 1,
	317	.name = "SEND",
	318	#if defined(CONFIG_NET)
	319	.prep = io_sendmsg_prep,
	320	.issue = io_send,
	321	#else
	322	.prep = io_eopnotsupp_prep,
	323	#endif
	324	},
	325	[IORING_OP_RECV] = {
	326	.needs_file = 1,
	327	.unbound_nonreg_file = 1,
	328	.pollin = 1,
	329	.buffer_select = 1,
	330	.audit_skip = 1,
	331	.ioprio = 1,
	332	.name = "RECV",
	333	#if defined(CONFIG_NET)
	334	.prep = io_recvmsg_prep,
	335	.issue = io_recv,
	336	#else
	337	.prep = io_eopnotsupp_prep,
	338	#endif
	339	},
	340	[IORING_OP_OPENAT2] = {
	341	.name = "OPENAT2",
	342	.prep = io_openat2_prep,
	343	.issue = io_openat2,
	344	.cleanup = io_open_cleanup,
	345	},
	346	[IORING_OP_EPOLL_CTL] = {
	347	.unbound_nonreg_file = 1,
	348	.audit_skip = 1,
	349	.name = "EPOLL",
	350	#if defined(CONFIG_EPOLL)
	351	.prep = io_epoll_ctl_prep,
	352	.issue = io_epoll_ctl,
	353	#else
	354	.prep = io_eopnotsupp_prep,
	355	#endif
	356	},
	357	[IORING_OP_SPLICE] = {
	358	.needs_file = 1,
	359	.hash_reg_file = 1,
	360	.unbound_nonreg_file = 1,
	361	.audit_skip = 1,
362	.name = "SPLICE",
363	.prep = io_splice_prep,
364	.issue = io_splice,
365	},
366	[IORING_OP_PROVIDE_BUFFERS] = {
367	.audit_skip = 1,
368	.iopoll = 1,
369	.name = "PROVIDE_BUFFERS",
370	.prep = io_provide_buffers_prep,
371	.issue = io_provide_buffers,
372	},
373	[IORING_OP_REMOVE_BUFFERS] = {
374	.audit_skip = 1,
375	.iopoll = 1,
376	.name = "REMOVE_BUFFERS",
377	.prep = io_remove_buffers_prep,
378	.issue = io_remove_buffers,
379	},
380	[IORING_OP_TEE] = {
381	.needs_file = 1,
382	.hash_reg_file = 1,
383	.unbound_nonreg_file = 1,
384	.audit_skip = 1,
385	.name = "TEE",
386	.prep = io_tee_prep,
387	.issue = io_tee,
388	},
389	[IORING_OP_SHUTDOWN] = {
390	.needs_file = 1,
391	.name = "SHUTDOWN",
392	#if defined(CONFIG_NET)
393	.prep = io_shutdown_prep,
394	.issue = io_shutdown,
395	#else
396	.prep = io_eopnotsupp_prep,
397	#endif
398	},
399	[IORING_OP_RENAMEAT] = {
400	.name = "RENAMEAT",
401	.prep = io_renameat_prep,
402	.issue = io_renameat,
403	.cleanup = io_renameat_cleanup,
404	},
405	[IORING_OP_UNLINKAT] = {
406	.name = "UNLINKAT",
407	.prep = io_unlinkat_prep,
408	.issue = io_unlinkat,
409	.cleanup = io_unlinkat_cleanup,
410	},
411	[IORING_OP_MKDIRAT] = {
412	.name = "MKDIRAT",
413	.prep = io_mkdirat_prep,
414	.issue = io_mkdirat,
415	.cleanup = io_mkdirat_cleanup,
416	},
417	[IORING_OP_SYMLINKAT] = {
418	.name = "SYMLINKAT",
419	.prep = io_symlinkat_prep,
420	.issue = io_symlinkat,
421	.cleanup = io_link_cleanup,
422	},
423	[IORING_OP_LINKAT] = {
424	.name = "LINKAT",
425	.prep = io_linkat_prep,
426	.issue = io_linkat,
427	.cleanup = io_link_cleanup,
428	},
429	[IORING_OP_MSG_RING] = {
430	.needs_file = 1,
431	.iopoll = 1,
432	.name = "MSG_RING",
433	.prep = io_msg_ring_prep,
434	.issue = io_msg_ring,
435	},
436	[IORING_OP_FSETXATTR] = {
437	.needs_file = 1,
438	.name = "FSETXATTR",
439	.prep = io_fsetxattr_prep,
440	.issue = io_fsetxattr,
441	.cleanup = io_xattr_cleanup,
442	},
443	[IORING_OP_SETXATTR] = {
444	.name = "SETXATTR",
445	.prep = io_setxattr_prep,
446	.issue = io_setxattr,
447	.cleanup = io_xattr_cleanup,
448	},
449	[IORING_OP_FGETXATTR] = {
450	.needs_file = 1,
451	.name = "FGETXATTR",
452	.prep = io_fgetxattr_prep,
453	.issue = io_fgetxattr,
454	.cleanup = io_xattr_cleanup,
455	},
456	[IORING_OP_GETXATTR] = {
457	.name = "GETXATTR",
458	.prep = io_getxattr_prep,
459	.issue = io_getxattr,
460	.cleanup = io_xattr_cleanup,
461	},
462	[IORING_OP_SOCKET] = {
463	.audit_skip = 1,
464	.name = "SOCKET",
465	#if defined(CONFIG_NET)
466	.prep = io_socket_prep,
467	.issue = io_socket,
468	#else
469	.prep = io_eopnotsupp_prep,
470	#endif
471	},
472	[IORING_OP_URING_CMD] = {
473	.needs_file = 1,
474	.plug = 1,
475	.name = "URING_CMD",
5756a3a7	476	.iopoll = 1,
d9b57aa3 JA	477	.async_size = uring_cmd_pdu_size(1),
	478	.prep = io_uring_cmd_prep,
	479	.issue = io_uring_cmd,
	480	.prep_async = io_uring_cmd_prep_async,
	481	},
b48c312b	482	[IORING_OP_SEND_ZC] = {
9bd3f728	483	.name = "SEND_ZC",
06a5464b PB	484	.needs_file = 1,
	485	.unbound_nonreg_file = 1,
	486	.pollout = 1,
	487	.audit_skip = 1,
	488	.ioprio = 1,
581711c4	489	.manual_alloc = 1,
06a5464b	490	#if defined(CONFIG_NET)
581711c4	491	.async_size = sizeof(struct io_async_msghdr),
06a5464b PB	492	.prep = io_sendzc_prep,
06a5464b PB	493	.issue = io_sendzc,
581711c4	494	.prep_async = io_sendzc_prep_async,
b48c312b	495	.cleanup = io_sendzc_cleanup,
06a5464b PB	496	#else
	497	.prep = io_eopnotsupp_prep,
	498	#endif
06a5464b	499	},
d9b57aa3 JA	500	};
	501
	502	const char *io_uring_get_opcode(u8 opcode)
	503	{
	504	if (opcode < IORING_OP_LAST)
	505	return io_op_defs[opcode].name;
	506	return "INVALID";
	507	}
	508
	509	void __init io_uring_optable_init(void)
	510	{
	511	int i;
	512
	513	BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
	514
	515	for (i = 0; i < ARRAY_SIZE(io_op_defs); i++) {
	516	BUG_ON(!io_op_defs[i].prep);
	517	if (io_op_defs[i].prep != io_eopnotsupp_prep)
	518	BUG_ON(!io_op_defs[i].issue);
	519	WARN_ON_ONCE(!io_op_defs[i].name);
	520	}
	521	}