[linux-block.git] / io_uring / opdef.c

// SPDX-License-Identifier: GPL-2.0
/*
 * io_uring opcode handling table
 */
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/io_uring.h>

#include "io_uring.h"
#include "opdef.h"
#include "refs.h"
#include "tctx.h"
#include "sqpoll.h"
#include "fdinfo.h"
#include "kbuf.h"
#include "rsrc.h"

#include "xattr.h"
#include "nop.h"
#include "fs.h"
#include "splice.h"
#include "sync.h"
#include "advise.h"
#include "openclose.h"
#include "uring_cmd.h"
#include "epoll.h"
#include "statx.h"
#include "net.h"
#include "msg_ring.h"
#include "timeout.h"
#include "poll.h"
#include "cancel.h"
#include "rw.h"

static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
{
	WARN_ON_ONCE(1);
	return -ECANCELED;
}

static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
					     const struct io_uring_sqe *sqe)
{
	return -EOPNOTSUPP;
}

const struct io_issue_def io_issue_defs[] = {
	[IORING_OP_NOP] = {
		.audit_skip		= 1,
		.iopoll			= 1,
		.name			= "NOP",
		.prep			= io_nop_prep,
		.issue			= io_nop,
	},
	[IORING_OP_READV] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.buffer_select		= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.async_size		= sizeof(struct io_async_rw),
		.name			= "READV",
		.prep			= io_prep_rw,
		.issue			= io_read,
		.prep_async		= io_readv_prep_async,
		.cleanup		= io_readv_writev_cleanup,
		.fail			= io_rw_fail,
	},
	[IORING_OP_WRITEV] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.async_size		= sizeof(struct io_async_rw),
		.name			= "WRITEV",
		.prep			= io_prep_rw,
		.issue			= io_write,
		.prep_async		= io_writev_prep_async,
		.cleanup		= io_readv_writev_cleanup,
		.fail			= io_rw_fail,
	},
	[IORING_OP_FSYNC] = {
		.needs_file		= 1,
		.audit_skip		= 1,
		.name			= "FSYNC",
		.prep			= io_fsync_prep,
		.issue			= io_fsync,
	},
	[IORING_OP_READ_FIXED] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.async_size		= sizeof(struct io_async_rw),
		.name			= "READ_FIXED",
		.prep			= io_prep_rw,
		.issue			= io_read,
		.fail			= io_rw_fail,
	},
	[IORING_OP_WRITE_FIXED] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.async_size		= sizeof(struct io_async_rw),
		.name			= "WRITE_FIXED",
		.prep			= io_prep_rw,
		.issue			= io_write,
		.fail			= io_rw_fail,
	},
	[IORING_OP_POLL_ADD] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.audit_skip		= 1,
		.name			= "POLL_ADD",
		.prep			= io_poll_add_prep,
		.issue			= io_poll_add,
	},
	[IORING_OP_POLL_REMOVE] = {
		.audit_skip		= 1,
		.name			= "POLL_REMOVE",
		.prep			= io_poll_remove_prep,
		.issue			= io_poll_remove,
	},
	[IORING_OP_SYNC_FILE_RANGE] = {
		.needs_file		= 1,
		.audit_skip		= 1,
		.name			= "SYNC_FILE_RANGE",
		.prep			= io_sfr_prep,
		.issue			= io_sync_file_range,
	},
	[IORING_OP_SENDMSG] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.ioprio			= 1,
		.manual_alloc		= 1,
		.name			= "SENDMSG",
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_msghdr),
		.prep			= io_sendmsg_prep,
		.issue			= io_sendmsg,
		.prep_async		= io_sendmsg_prep_async,
		.cleanup		= io_sendmsg_recvmsg_cleanup,
		.fail			= io_sendrecv_fail,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_RECVMSG] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.buffer_select		= 1,
		.ioprio			= 1,
		.manual_alloc		= 1,
		.name			= "RECVMSG",
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_msghdr),
		.prep			= io_recvmsg_prep,
		.issue			= io_recvmsg,
		.prep_async		= io_recvmsg_prep_async,
		.cleanup		= io_sendmsg_recvmsg_cleanup,
		.fail			= io_sendrecv_fail,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_TIMEOUT] = {
		.audit_skip		= 1,
		.async_size		= sizeof(struct io_timeout_data),
		.name			= "TIMEOUT",
		.prep			= io_timeout_prep,
		.issue			= io_timeout,
	},
	[IORING_OP_TIMEOUT_REMOVE] = {
		/* used by timeout updates' prep() */
		.audit_skip		= 1,
		.name			= "TIMEOUT_REMOVE",
		.prep			= io_timeout_remove_prep,
		.issue			= io_timeout_remove,
	},
	[IORING_OP_ACCEPT] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.poll_exclusive		= 1,
		.ioprio			= 1,	/* used for flags */
		.name			= "ACCEPT",
#if defined(CONFIG_NET)
		.prep			= io_accept_prep,
		.issue			= io_accept,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_ASYNC_CANCEL] = {
		.audit_skip		= 1,
		.name			= "ASYNC_CANCEL",
		.prep			= io_async_cancel_prep,
		.issue			= io_async_cancel,
	},
	[IORING_OP_LINK_TIMEOUT] = {
		.audit_skip		= 1,
		.async_size		= sizeof(struct io_timeout_data),
		.name			= "LINK_TIMEOUT",
		.prep			= io_link_timeout_prep,
		.issue			= io_no_issue,
	},
	[IORING_OP_CONNECT] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.name			= "CONNECT",
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_connect),
		.prep			= io_connect_prep,
		.issue			= io_connect,
		.prep_async		= io_connect_prep_async,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_FALLOCATE] = {
		.needs_file		= 1,
		.name			= "FALLOCATE",
		.prep			= io_fallocate_prep,
		.issue			= io_fallocate,
	},
	[IORING_OP_OPENAT] = {
		.name			= "OPENAT",
		.prep			= io_openat_prep,
		.issue			= io_openat,
		.cleanup		= io_open_cleanup,
	},
	[IORING_OP_CLOSE] = {
		.name			= "CLOSE",
		.prep			= io_close_prep,
		.issue			= io_close,
	},
	[IORING_OP_FILES_UPDATE] = {
		.audit_skip		= 1,
		.iopoll			= 1,
		.name			= "FILES_UPDATE",
		.prep			= io_files_update_prep,
		.issue			= io_files_update,
	},
	[IORING_OP_STATX] = {
		.audit_skip		= 1,
		.name			= "STATX",
		.prep			= io_statx_prep,
		.issue			= io_statx,
		.cleanup		= io_statx_cleanup,
	},
	[IORING_OP_READ] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.buffer_select		= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.async_size		= sizeof(struct io_async_rw),
		.name			= "READ",
		.prep			= io_prep_rw,
		.issue			= io_read,
		.fail			= io_rw_fail,
	},
	[IORING_OP_WRITE] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.async_size		= sizeof(struct io_async_rw),
		.name			= "WRITE",
		.prep			= io_prep_rw,
		.issue			= io_write,
		.fail			= io_rw_fail,
	},
	[IORING_OP_FADVISE] = {
		.needs_file		= 1,
		.audit_skip		= 1,
		.name			= "FADVISE",
		.prep			= io_fadvise_prep,
		.issue			= io_fadvise,
	},
	[IORING_OP_MADVISE] = {
		.name			= "MADVISE",
		.prep			= io_madvise_prep,
		.issue			= io_madvise,
	},
	[IORING_OP_SEND] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.manual_alloc		= 1,
		.name			= "SEND",
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_msghdr),
		.prep			= io_sendmsg_prep,
		.issue			= io_send,
		.fail			= io_sendrecv_fail,
		.prep_async		= io_send_prep_async,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_RECV] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.buffer_select		= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.name			= "RECV",
#if defined(CONFIG_NET)
		.prep			= io_recvmsg_prep,
		.issue			= io_recv,
		.fail			= io_sendrecv_fail,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_OPENAT2] = {
		.name			= "OPENAT2",
		.prep			= io_openat2_prep,
		.issue			= io_openat2,
		.cleanup		= io_open_cleanup,
	},
	[IORING_OP_EPOLL_CTL] = {
		.unbound_nonreg_file	= 1,
		.audit_skip		= 1,
		.name			= "EPOLL",
#if defined(CONFIG_EPOLL)
		.prep			= io_epoll_ctl_prep,
		.issue			= io_epoll_ctl,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_SPLICE] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
		.audit_skip		= 1,
		.name			= "SPLICE",
		.prep			= io_splice_prep,
		.issue			= io_splice,
	},
	[IORING_OP_PROVIDE_BUFFERS] = {
		.audit_skip		= 1,
		.iopoll			= 1,
		.name			= "PROVIDE_BUFFERS",
		.prep			= io_provide_buffers_prep,
		.issue			= io_provide_buffers,
	},
	[IORING_OP_REMOVE_BUFFERS] = {
		.audit_skip		= 1,
		.iopoll			= 1,
		.name			= "REMOVE_BUFFERS",
		.prep			= io_remove_buffers_prep,
		.issue			= io_remove_buffers,
	},
	[IORING_OP_TEE] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
		.audit_skip		= 1,
		.name			= "TEE",
		.prep			= io_tee_prep,
		.issue			= io_tee,
	},
	[IORING_OP_SHUTDOWN] = {
		.needs_file		= 1,
		.name			= "SHUTDOWN",
#if defined(CONFIG_NET)
		.prep			= io_shutdown_prep,
		.issue			= io_shutdown,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_RENAMEAT] = {
		.name			= "RENAMEAT",
		.prep			= io_renameat_prep,
		.issue			= io_renameat,
		.cleanup		= io_renameat_cleanup,
	},
	[IORING_OP_UNLINKAT] = {
		.name			= "UNLINKAT",
		.prep			= io_unlinkat_prep,
		.issue			= io_unlinkat,
		.cleanup		= io_unlinkat_cleanup,
	},
	[IORING_OP_MKDIRAT] = {
		.name			= "MKDIRAT",
		.prep			= io_mkdirat_prep,
		.issue			= io_mkdirat,
		.cleanup		= io_mkdirat_cleanup,
	},
	[IORING_OP_SYMLINKAT] = {
		.name			= "SYMLINKAT",
		.prep			= io_symlinkat_prep,
		.issue			= io_symlinkat,
		.cleanup		= io_link_cleanup,
	},
	[IORING_OP_LINKAT] = {
		.name			= "LINKAT",
		.prep			= io_linkat_prep,
		.issue			= io_linkat,
		.cleanup		= io_link_cleanup,
	},
	[IORING_OP_MSG_RING] = {
		.needs_file		= 1,
		.iopoll			= 1,
		.name			= "MSG_RING",
		.prep			= io_msg_ring_prep,
		.issue			= io_msg_ring,
		.cleanup		= io_msg_ring_cleanup,
	},
	[IORING_OP_FSETXATTR] = {
		.needs_file = 1,
		.name			= "FSETXATTR",
		.prep			= io_fsetxattr_prep,
		.issue			= io_fsetxattr,
		.cleanup		= io_xattr_cleanup,
	},
	[IORING_OP_SETXATTR] = {
		.name			= "SETXATTR",
		.prep			= io_setxattr_prep,
		.issue			= io_setxattr,
		.cleanup		= io_xattr_cleanup,
	},
	[IORING_OP_FGETXATTR] = {
		.needs_file = 1,
		.name			= "FGETXATTR",
		.prep			= io_fgetxattr_prep,
		.issue			= io_fgetxattr,
		.cleanup		= io_xattr_cleanup,
	},
	[IORING_OP_GETXATTR] = {
		.name			= "GETXATTR",
		.prep			= io_getxattr_prep,
		.issue			= io_getxattr,
		.cleanup		= io_xattr_cleanup,
	},
	[IORING_OP_SOCKET] = {
		.audit_skip		= 1,
		.name			= "SOCKET",
#if defined(CONFIG_NET)
		.prep			= io_socket_prep,
		.issue			= io_socket,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_URING_CMD] = {
		.needs_file		= 1,
		.plug			= 1,
		.name			= "URING_CMD",
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.async_size		= uring_cmd_pdu_size(1),
		.prep			= io_uring_cmd_prep,
		.issue			= io_uring_cmd,
		.prep_async		= io_uring_cmd_prep_async,
	},
	[IORING_OP_SEND_ZC] = {
		.name			= "SEND_ZC",
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.manual_alloc		= 1,
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_msghdr),
		.prep			= io_send_zc_prep,
		.issue			= io_send_zc,
		.prep_async		= io_send_prep_async,
		.cleanup		= io_send_zc_cleanup,
		.fail			= io_sendrecv_fail,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_SENDMSG_ZC] = {
		.name			= "SENDMSG_ZC",
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.ioprio			= 1,
		.manual_alloc		= 1,
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_msghdr),
		.prep			= io_send_zc_prep,
		.issue			= io_sendmsg_zc,
		.prep_async		= io_sendmsg_prep_async,
		.cleanup		= io_send_zc_cleanup,
		.fail			= io_sendrecv_fail,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
};

const char *io_uring_get_opcode(u8 opcode)
{
	if (opcode < IORING_OP_LAST)
		return io_issue_defs[opcode].name;
	return "INVALID";
}

void __init io_uring_optable_init(void)
{
	int i;

	BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);

	for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
		BUG_ON(!io_issue_defs[i].prep);
		if (io_issue_defs[i].prep != io_eopnotsupp_prep)
			BUG_ON(!io_issue_defs[i].issue);
		WARN_ON_ONCE(!io_issue_defs[i].name);
	}
}
Commit	Line	Data
d9b57aa3 JA	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* io_uring opcode handling table
	4	*/
	5	#include <linux/kernel.h>
	6	#include <linux/errno.h>
	7	#include <linux/fs.h>
	8	#include <linux/file.h>
	9	#include <linux/io_uring.h>
	10
d9b57aa3 JA	11	#include "io_uring.h"
	12	#include "opdef.h"
	13	#include "refs.h"
	14	#include "tctx.h"
	15	#include "sqpoll.h"
	16	#include "fdinfo.h"
	17	#include "kbuf.h"
	18	#include "rsrc.h"
	19
	20	#include "xattr.h"
	21	#include "nop.h"
	22	#include "fs.h"
	23	#include "splice.h"
	24	#include "sync.h"
	25	#include "advise.h"
	26	#include "openclose.h"
	27	#include "uring_cmd.h"
	28	#include "epoll.h"
	29	#include "statx.h"
	30	#include "net.h"
	31	#include "msg_ring.h"
	32	#include "timeout.h"
	33	#include "poll.h"
	34	#include "cancel.h"
	35	#include "rw.h"
	36
	37	static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
	38	{
	39	WARN_ON_ONCE(1);
	40	return -ECANCELED;
	41	}
	42
	43	static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
	44	const struct io_uring_sqe *sqe)
	45	{
	46	return -EOPNOTSUPP;
	47	}
	48
a7dd2782	49	const struct io_issue_def io_issue_defs[] = {
d9b57aa3 JA	50	[IORING_OP_NOP] = {
	51	.audit_skip = 1,
	52	.iopoll = 1,
	53	.name = "NOP",
	54	.prep = io_nop_prep,
	55	.issue = io_nop,
	56	},
	57	[IORING_OP_READV] = {
	58	.needs_file = 1,
	59	.unbound_nonreg_file = 1,
	60	.pollin = 1,
	61	.buffer_select = 1,
	62	.plug = 1,
	63	.audit_skip = 1,
	64	.ioprio = 1,
	65	.iopoll = 1,
ef0ec1ad	66	.iopoll_queue = 1,
d9b57aa3 JA	67	.async_size = sizeof(struct io_async_rw),
	68	.name = "READV",
	69	.prep = io_prep_rw,
	70	.issue = io_read,
	71	.prep_async = io_readv_prep_async,
	72	.cleanup = io_readv_writev_cleanup,
47b4c686	73	.fail = io_rw_fail,
d9b57aa3 JA	74	},
	75	[IORING_OP_WRITEV] = {
	76	.needs_file = 1,
	77	.hash_reg_file = 1,
	78	.unbound_nonreg_file = 1,
	79	.pollout = 1,
	80	.plug = 1,
	81	.audit_skip = 1,
	82	.ioprio = 1,
	83	.iopoll = 1,
ef0ec1ad	84	.iopoll_queue = 1,
d9b57aa3 JA	85	.async_size = sizeof(struct io_async_rw),
	86	.name = "WRITEV",
	87	.prep = io_prep_rw,
	88	.issue = io_write,
	89	.prep_async = io_writev_prep_async,
	90	.cleanup = io_readv_writev_cleanup,
47b4c686	91	.fail = io_rw_fail,
d9b57aa3 JA	92	},
	93	[IORING_OP_FSYNC] = {
	94	.needs_file = 1,
	95	.audit_skip = 1,
	96	.name = "FSYNC",
	97	.prep = io_fsync_prep,
	98	.issue = io_fsync,
	99	},
	100	[IORING_OP_READ_FIXED] = {
	101	.needs_file = 1,
	102	.unbound_nonreg_file = 1,
	103	.pollin = 1,
	104	.plug = 1,
	105	.audit_skip = 1,
	106	.ioprio = 1,
	107	.iopoll = 1,
ef0ec1ad	108	.iopoll_queue = 1,
d9b57aa3 JA	109	.async_size = sizeof(struct io_async_rw),
	110	.name = "READ_FIXED",
	111	.prep = io_prep_rw,
	112	.issue = io_read,
47b4c686	113	.fail = io_rw_fail,
d9b57aa3 JA	114	},
	115	[IORING_OP_WRITE_FIXED] = {
	116	.needs_file = 1,
	117	.hash_reg_file = 1,
	118	.unbound_nonreg_file = 1,
	119	.pollout = 1,
	120	.plug = 1,
	121	.audit_skip = 1,
	122	.ioprio = 1,
	123	.iopoll = 1,
ef0ec1ad	124	.iopoll_queue = 1,
d9b57aa3 JA	125	.async_size = sizeof(struct io_async_rw),
	126	.name = "WRITE_FIXED",
	127	.prep = io_prep_rw,
	128	.issue = io_write,
47b4c686	129	.fail = io_rw_fail,
d9b57aa3 JA	130	},
	131	[IORING_OP_POLL_ADD] = {
	132	.needs_file = 1,
	133	.unbound_nonreg_file = 1,
	134	.audit_skip = 1,
	135	.name = "POLL_ADD",
	136	.prep = io_poll_add_prep,
	137	.issue = io_poll_add,
	138	},
	139	[IORING_OP_POLL_REMOVE] = {
	140	.audit_skip = 1,
	141	.name = "POLL_REMOVE",
	142	.prep = io_poll_remove_prep,
	143	.issue = io_poll_remove,
	144	},
	145	[IORING_OP_SYNC_FILE_RANGE] = {
	146	.needs_file = 1,
	147	.audit_skip = 1,
	148	.name = "SYNC_FILE_RANGE",
	149	.prep = io_sfr_prep,
	150	.issue = io_sync_file_range,
	151	},
	152	[IORING_OP_SENDMSG] = {
	153	.needs_file = 1,
	154	.unbound_nonreg_file = 1,
	155	.pollout = 1,
	156	.ioprio = 1,
858c293e	157	.manual_alloc = 1,
d9b57aa3 JA	158	.name = "SENDMSG",
	159	#if defined(CONFIG_NET)
	160	.async_size = sizeof(struct io_async_msghdr),
	161	.prep = io_sendmsg_prep,
	162	.issue = io_sendmsg,
	163	.prep_async = io_sendmsg_prep_async,
	164	.cleanup = io_sendmsg_recvmsg_cleanup,
7e6b638e	165	.fail = io_sendrecv_fail,
d9b57aa3 JA	166	#else
	167	.prep = io_eopnotsupp_prep,
	168	#endif
	169	},
	170	[IORING_OP_RECVMSG] = {
	171	.needs_file = 1,
	172	.unbound_nonreg_file = 1,
	173	.pollin = 1,
	174	.buffer_select = 1,
	175	.ioprio = 1,
858c293e	176	.manual_alloc = 1,
d9b57aa3 JA	177	.name = "RECVMSG",
	178	#if defined(CONFIG_NET)
	179	.async_size = sizeof(struct io_async_msghdr),
	180	.prep = io_recvmsg_prep,
	181	.issue = io_recvmsg,
	182	.prep_async = io_recvmsg_prep_async,
	183	.cleanup = io_sendmsg_recvmsg_cleanup,
7e6b638e	184	.fail = io_sendrecv_fail,
d9b57aa3 JA	185	#else
	186	.prep = io_eopnotsupp_prep,
	187	#endif
	188	},
	189	[IORING_OP_TIMEOUT] = {
	190	.audit_skip = 1,
	191	.async_size = sizeof(struct io_timeout_data),
	192	.name = "TIMEOUT",
	193	.prep = io_timeout_prep,
	194	.issue = io_timeout,
	195	},
	196	[IORING_OP_TIMEOUT_REMOVE] = {
	197	/* used by timeout updates' prep() */
	198	.audit_skip = 1,
	199	.name = "TIMEOUT_REMOVE",
	200	.prep = io_timeout_remove_prep,
	201	.issue = io_timeout_remove,
	202	},
	203	[IORING_OP_ACCEPT] = {
	204	.needs_file = 1,
	205	.unbound_nonreg_file = 1,
	206	.pollin = 1,
	207	.poll_exclusive = 1,
	208	.ioprio = 1, /* used for flags */
	209	.name = "ACCEPT",
	210	#if defined(CONFIG_NET)
	211	.prep = io_accept_prep,
	212	.issue = io_accept,
	213	#else
	214	.prep = io_eopnotsupp_prep,
	215	#endif
	216	},
	217	[IORING_OP_ASYNC_CANCEL] = {
	218	.audit_skip = 1,
	219	.name = "ASYNC_CANCEL",
	220	.prep = io_async_cancel_prep,
	221	.issue = io_async_cancel,
	222	},
	223	[IORING_OP_LINK_TIMEOUT] = {
	224	.audit_skip = 1,
	225	.async_size = sizeof(struct io_timeout_data),
	226	.name = "LINK_TIMEOUT",
	227	.prep = io_link_timeout_prep,
	228	.issue = io_no_issue,
	229	},
	230	[IORING_OP_CONNECT] = {
	231	.needs_file = 1,
	232	.unbound_nonreg_file = 1,
	233	.pollout = 1,
	234	.name = "CONNECT",
	235	#if defined(CONFIG_NET)
	236	.async_size = sizeof(struct io_async_connect),
	237	.prep = io_connect_prep,
	238	.issue = io_connect,
	239	.prep_async = io_connect_prep_async,
	240	#else
	241	.prep = io_eopnotsupp_prep,
	242	#endif
	243	},
	244	[IORING_OP_FALLOCATE] = {
	245	.needs_file = 1,
	246	.name = "FALLOCATE",
	247	.prep = io_fallocate_prep,
	248	.issue = io_fallocate,
249	},
250	[IORING_OP_OPENAT] = {
251	.name = "OPENAT",
252	.prep = io_openat_prep,
253	.issue = io_openat,
254	.cleanup = io_open_cleanup,
255	},
256	[IORING_OP_CLOSE] = {
257	.name = "CLOSE",
258	.prep = io_close_prep,
259	.issue = io_close,
260	},
d9808ceb	261	[IORING_OP_FILES_UPDATE] = {
d9b57aa3 JA	262	.audit_skip = 1,
d9b57aa3 JA	263	.iopoll = 1,
d9808ceb PB	264	.name = "FILES_UPDATE",
	265	.prep = io_files_update_prep,
	266	.issue = io_files_update,
d9b57aa3 JA	267	},
	268	[IORING_OP_STATX] = {
	269	.audit_skip = 1,
	270	.name = "STATX",
	271	.prep = io_statx_prep,
	272	.issue = io_statx,
	273	.cleanup = io_statx_cleanup,
	274	},
	275	[IORING_OP_READ] = {
	276	.needs_file = 1,
	277	.unbound_nonreg_file = 1,
	278	.pollin = 1,
	279	.buffer_select = 1,
	280	.plug = 1,
	281	.audit_skip = 1,
	282	.ioprio = 1,
	283	.iopoll = 1,
ef0ec1ad	284	.iopoll_queue = 1,
d9b57aa3 JA	285	.async_size = sizeof(struct io_async_rw),
	286	.name = "READ",
	287	.prep = io_prep_rw,
	288	.issue = io_read,
47b4c686	289	.fail = io_rw_fail,
d9b57aa3 JA	290	},
	291	[IORING_OP_WRITE] = {
	292	.needs_file = 1,
	293	.hash_reg_file = 1,
	294	.unbound_nonreg_file = 1,
	295	.pollout = 1,
	296	.plug = 1,
	297	.audit_skip = 1,
	298	.ioprio = 1,
	299	.iopoll = 1,
ef0ec1ad	300	.iopoll_queue = 1,
d9b57aa3 JA	301	.async_size = sizeof(struct io_async_rw),
	302	.name = "WRITE",
	303	.prep = io_prep_rw,
	304	.issue = io_write,
47b4c686	305	.fail = io_rw_fail,
d9b57aa3 JA	306	},
	307	[IORING_OP_FADVISE] = {
	308	.needs_file = 1,
	309	.audit_skip = 1,
	310	.name = "FADVISE",
	311	.prep = io_fadvise_prep,
	312	.issue = io_fadvise,
	313	},
	314	[IORING_OP_MADVISE] = {
	315	.name = "MADVISE",
	316	.prep = io_madvise_prep,
	317	.issue = io_madvise,
	318	},
	319	[IORING_OP_SEND] = {
	320	.needs_file = 1,
	321	.unbound_nonreg_file = 1,
	322	.pollout = 1,
	323	.audit_skip = 1,
	324	.ioprio = 1,
516e82f0	325	.manual_alloc = 1,
d9b57aa3 JA	326	.name = "SEND",
d9b57aa3 JA	327	#if defined(CONFIG_NET)
516e82f0	328	.async_size = sizeof(struct io_async_msghdr),
d9b57aa3 JA	329	.prep = io_sendmsg_prep,
d9b57aa3 JA	330	.issue = io_send,
7e6b638e	331	.fail = io_sendrecv_fail,
516e82f0	332	.prep_async = io_send_prep_async,
d9b57aa3 JA	333	#else
	334	.prep = io_eopnotsupp_prep,
	335	#endif
	336	},
	337	[IORING_OP_RECV] = {
	338	.needs_file = 1,
	339	.unbound_nonreg_file = 1,
	340	.pollin = 1,
	341	.buffer_select = 1,
	342	.audit_skip = 1,
	343	.ioprio = 1,
	344	.name = "RECV",
	345	#if defined(CONFIG_NET)
	346	.prep = io_recvmsg_prep,
	347	.issue = io_recv,
7e6b638e	348	.fail = io_sendrecv_fail,
d9b57aa3 JA	349	#else
	350	.prep = io_eopnotsupp_prep,
	351	#endif
	352	},
	353	[IORING_OP_OPENAT2] = {
	354	.name = "OPENAT2",
	355	.prep = io_openat2_prep,
	356	.issue = io_openat2,
	357	.cleanup = io_open_cleanup,
	358	},
	359	[IORING_OP_EPOLL_CTL] = {
	360	.unbound_nonreg_file = 1,
	361	.audit_skip = 1,
	362	.name = "EPOLL",
	363	#if defined(CONFIG_EPOLL)
	364	.prep = io_epoll_ctl_prep,
	365	.issue = io_epoll_ctl,
	366	#else
	367	.prep = io_eopnotsupp_prep,
	368	#endif
	369	},
	370	[IORING_OP_SPLICE] = {
	371	.needs_file = 1,
	372	.hash_reg_file = 1,
	373	.unbound_nonreg_file = 1,
	374	.audit_skip = 1,
	375	.name = "SPLICE",
	376	.prep = io_splice_prep,
	377	.issue = io_splice,
	378	},
	379	[IORING_OP_PROVIDE_BUFFERS] = {
	380	.audit_skip = 1,
	381	.iopoll = 1,
	382	.name = "PROVIDE_BUFFERS",
	383	.prep = io_provide_buffers_prep,
	384	.issue = io_provide_buffers,
	385	},
	386	[IORING_OP_REMOVE_BUFFERS] = {
	387	.audit_skip = 1,
	388	.iopoll = 1,
	389	.name = "REMOVE_BUFFERS",
	390	.prep = io_remove_buffers_prep,
	391	.issue = io_remove_buffers,
	392	},
	393	[IORING_OP_TEE] = {
	394	.needs_file = 1,
	395	.hash_reg_file = 1,
	396	.unbound_nonreg_file = 1,
	397	.audit_skip = 1,
	398	.name = "TEE",
	399	.prep = io_tee_prep,
	400	.issue = io_tee,
	401	},
	402	[IORING_OP_SHUTDOWN] = {
	403	.needs_file = 1,
	404	.name = "SHUTDOWN",
	405	#if defined(CONFIG_NET)
	406	.prep = io_shutdown_prep,
	407	.issue = io_shutdown,
	408	#else
	409	.prep = io_eopnotsupp_prep,
	410	#endif
	411	},
	412	[IORING_OP_RENAMEAT] = {
413	.name = "RENAMEAT",
414	.prep = io_renameat_prep,
415	.issue = io_renameat,
416	.cleanup = io_renameat_cleanup,
417	},
418	[IORING_OP_UNLINKAT] = {
419	.name = "UNLINKAT",
420	.prep = io_unlinkat_prep,
421	.issue = io_unlinkat,
422	.cleanup = io_unlinkat_cleanup,
423	},
424	[IORING_OP_MKDIRAT] = {
425	.name = "MKDIRAT",
426	.prep = io_mkdirat_prep,
427	.issue = io_mkdirat,
428	.cleanup = io_mkdirat_cleanup,
429	},
430	[IORING_OP_SYMLINKAT] = {
431	.name = "SYMLINKAT",
432	.prep = io_symlinkat_prep,
433	.issue = io_symlinkat,
434	.cleanup = io_link_cleanup,
435	},
436	[IORING_OP_LINKAT] = {
437	.name = "LINKAT",
438	.prep = io_linkat_prep,
439	.issue = io_linkat,
440	.cleanup = io_link_cleanup,
441	},
442	[IORING_OP_MSG_RING] = {
443	.needs_file = 1,
444	.iopoll = 1,
445	.name = "MSG_RING",
446	.prep = io_msg_ring_prep,
447	.issue = io_msg_ring,
11373026	448	.cleanup = io_msg_ring_cleanup,
d9b57aa3 JA	449	},
	450	[IORING_OP_FSETXATTR] = {
	451	.needs_file = 1,
	452	.name = "FSETXATTR",
	453	.prep = io_fsetxattr_prep,
	454	.issue = io_fsetxattr,
	455	.cleanup = io_xattr_cleanup,
	456	},
	457	[IORING_OP_SETXATTR] = {
	458	.name = "SETXATTR",
	459	.prep = io_setxattr_prep,
	460	.issue = io_setxattr,
	461	.cleanup = io_xattr_cleanup,
	462	},
	463	[IORING_OP_FGETXATTR] = {
	464	.needs_file = 1,
	465	.name = "FGETXATTR",
	466	.prep = io_fgetxattr_prep,
	467	.issue = io_fgetxattr,
	468	.cleanup = io_xattr_cleanup,
	469	},
	470	[IORING_OP_GETXATTR] = {
	471	.name = "GETXATTR",
	472	.prep = io_getxattr_prep,
	473	.issue = io_getxattr,
	474	.cleanup = io_xattr_cleanup,
	475	},
	476	[IORING_OP_SOCKET] = {
	477	.audit_skip = 1,
	478	.name = "SOCKET",
	479	#if defined(CONFIG_NET)
	480	.prep = io_socket_prep,
	481	.issue = io_socket,
	482	#else
	483	.prep = io_eopnotsupp_prep,
	484	#endif
	485	},
	486	[IORING_OP_URING_CMD] = {
	487	.needs_file = 1,
	488	.plug = 1,
	489	.name = "URING_CMD",
5756a3a7	490	.iopoll = 1,
ef0ec1ad	491	.iopoll_queue = 1,
d9b57aa3 JA	492	.async_size = uring_cmd_pdu_size(1),
	493	.prep = io_uring_cmd_prep,
	494	.issue = io_uring_cmd,
	495	.prep_async = io_uring_cmd_prep_async,
	496	},
b48c312b	497	[IORING_OP_SEND_ZC] = {
9bd3f728	498	.name = "SEND_ZC",
06a5464b PB	499	.needs_file = 1,
	500	.unbound_nonreg_file = 1,
	501	.pollout = 1,
	502	.audit_skip = 1,
	503	.ioprio = 1,
581711c4	504	.manual_alloc = 1,
06a5464b	505	#if defined(CONFIG_NET)
581711c4	506	.async_size = sizeof(struct io_async_msghdr),
b0e9b551 PB	507	.prep = io_send_zc_prep,
b0e9b551 PB	508	.issue = io_send_zc,
516e82f0	509	.prep_async = io_send_prep_async,
b0e9b551	510	.cleanup = io_send_zc_cleanup,
c4c0009e	511	.fail = io_sendrecv_fail,
06a5464b PB	512	#else
06a5464b PB	513	.prep = io_eopnotsupp_prep,
493108d9 PB	514	#endif
	515	},
	516	[IORING_OP_SENDMSG_ZC] = {
	517	.name = "SENDMSG_ZC",
	518	.needs_file = 1,
	519	.unbound_nonreg_file = 1,
	520	.pollout = 1,
493108d9 PB	521	.ioprio = 1,
	522	.manual_alloc = 1,
	523	#if defined(CONFIG_NET)
	524	.async_size = sizeof(struct io_async_msghdr),
	525	.prep = io_send_zc_prep,
	526	.issue = io_sendmsg_zc,
	527	.prep_async = io_sendmsg_prep_async,
	528	.cleanup = io_send_zc_cleanup,
	529	.fail = io_sendrecv_fail,
	530	#else
	531	.prep = io_eopnotsupp_prep,
06a5464b	532	#endif
06a5464b	533	},
d9b57aa3 JA	534	};
	535
	536	const char *io_uring_get_opcode(u8 opcode)
	537	{
	538	if (opcode < IORING_OP_LAST)
a7dd2782	539	return io_issue_defs[opcode].name;
d9b57aa3 JA	540	return "INVALID";
	541	}
	542
	543	void __init io_uring_optable_init(void)
	544	{
	545	int i;
	546
a7dd2782	547	BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);
d9b57aa3	548
a7dd2782 BL	549	for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
	550	BUG_ON(!io_issue_defs[i].prep);
	551	if (io_issue_defs[i].prep != io_eopnotsupp_prep)
	552	BUG_ON(!io_issue_defs[i].issue);
	553	WARN_ON_ONCE(!io_issue_defs[i].name);
d9b57aa3 JA	554	}
d9b57aa3 JA	555	}