[linux-block.git] / io_uring / opdef.c

// SPDX-License-Identifier: GPL-2.0
/*
 * io_uring opcode handling table
 */
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/io_uring.h>

#include "io_uring.h"
#include "opdef.h"
#include "refs.h"
#include "tctx.h"
#include "sqpoll.h"
#include "fdinfo.h"
#include "kbuf.h"
#include "rsrc.h"

#include "xattr.h"
#include "nop.h"
#include "fs.h"
#include "splice.h"
#include "sync.h"
#include "advise.h"
#include "openclose.h"
#include "uring_cmd.h"
#include "epoll.h"
#include "statx.h"
#include "net.h"
#include "msg_ring.h"
#include "timeout.h"
#include "poll.h"
#include "cancel.h"
#include "rw.h"

static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
{
	WARN_ON_ONCE(1);
	return -ECANCELED;
}

static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
					     const struct io_uring_sqe *sqe)
{
	return -EOPNOTSUPP;
}

const struct io_issue_def io_issue_defs[] = {
	[IORING_OP_NOP] = {
		.audit_skip		= 1,
		.iopoll			= 1,
		.prep			= io_nop_prep,
		.issue			= io_nop,
	},
	[IORING_OP_READV] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.buffer_select		= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.prep			= io_prep_rw,
		.issue			= io_read,
	},
	[IORING_OP_WRITEV] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.prep			= io_prep_rw,
		.issue			= io_write,
	},
	[IORING_OP_FSYNC] = {
		.needs_file		= 1,
		.audit_skip		= 1,
		.prep			= io_fsync_prep,
		.issue			= io_fsync,
	},
	[IORING_OP_READ_FIXED] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.prep			= io_prep_rw,
		.issue			= io_read,
	},
	[IORING_OP_WRITE_FIXED] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.prep			= io_prep_rw,
		.issue			= io_write,
	},
	[IORING_OP_POLL_ADD] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.audit_skip		= 1,
		.prep			= io_poll_add_prep,
		.issue			= io_poll_add,
	},
	[IORING_OP_POLL_REMOVE] = {
		.audit_skip		= 1,
		.prep			= io_poll_remove_prep,
		.issue			= io_poll_remove,
	},
	[IORING_OP_SYNC_FILE_RANGE] = {
		.needs_file		= 1,
		.audit_skip		= 1,
		.prep			= io_sfr_prep,
		.issue			= io_sync_file_range,
	},
	[IORING_OP_SENDMSG] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.ioprio			= 1,
		.manual_alloc		= 1,
#if defined(CONFIG_NET)
		.prep			= io_sendmsg_prep,
		.issue			= io_sendmsg,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_RECVMSG] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.buffer_select		= 1,
		.ioprio			= 1,
		.manual_alloc		= 1,
#if defined(CONFIG_NET)
		.prep			= io_recvmsg_prep,
		.issue			= io_recvmsg,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_TIMEOUT] = {
		.audit_skip		= 1,
		.prep			= io_timeout_prep,
		.issue			= io_timeout,
	},
	[IORING_OP_TIMEOUT_REMOVE] = {
		/* used by timeout updates' prep() */
		.audit_skip		= 1,
		.prep			= io_timeout_remove_prep,
		.issue			= io_timeout_remove,
	},
	[IORING_OP_ACCEPT] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.poll_exclusive		= 1,
		.ioprio			= 1,	/* used for flags */
#if defined(CONFIG_NET)
		.prep			= io_accept_prep,
		.issue			= io_accept,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_ASYNC_CANCEL] = {
		.audit_skip		= 1,
		.prep			= io_async_cancel_prep,
		.issue			= io_async_cancel,
	},
	[IORING_OP_LINK_TIMEOUT] = {
		.audit_skip		= 1,
		.prep			= io_link_timeout_prep,
		.issue			= io_no_issue,
	},
	[IORING_OP_CONNECT] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
#if defined(CONFIG_NET)
		.prep			= io_connect_prep,
		.issue			= io_connect,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_FALLOCATE] = {
		.needs_file		= 1,
		.prep			= io_fallocate_prep,
		.issue			= io_fallocate,
	},
	[IORING_OP_OPENAT] = {
		.prep			= io_openat_prep,
		.issue			= io_openat,
	},
	[IORING_OP_CLOSE] = {
		.prep			= io_close_prep,
		.issue			= io_close,
	},
	[IORING_OP_FILES_UPDATE] = {
		.audit_skip		= 1,
		.iopoll			= 1,
		.prep			= io_files_update_prep,
		.issue			= io_files_update,
	},
	[IORING_OP_STATX] = {
		.audit_skip		= 1,
		.prep			= io_statx_prep,
		.issue			= io_statx,
	},
	[IORING_OP_READ] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.buffer_select		= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.prep			= io_prep_rw,
		.issue			= io_read,
	},
	[IORING_OP_WRITE] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.prep			= io_prep_rw,
		.issue			= io_write,
	},
	[IORING_OP_FADVISE] = {
		.needs_file		= 1,
		.audit_skip		= 1,
		.prep			= io_fadvise_prep,
		.issue			= io_fadvise,
	},
	[IORING_OP_MADVISE] = {
		.audit_skip		= 1,
		.prep			= io_madvise_prep,
		.issue			= io_madvise,
	},
	[IORING_OP_SEND] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.manual_alloc		= 1,
#if defined(CONFIG_NET)
		.prep			= io_sendmsg_prep,
		.issue			= io_send,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_RECV] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.buffer_select		= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
#if defined(CONFIG_NET)
		.prep			= io_recvmsg_prep,
		.issue			= io_recv,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_OPENAT2] = {
		.prep			= io_openat2_prep,
		.issue			= io_openat2,
	},
	[IORING_OP_EPOLL_CTL] = {
		.unbound_nonreg_file	= 1,
		.audit_skip		= 1,
#if defined(CONFIG_EPOLL)
		.prep			= io_epoll_ctl_prep,
		.issue			= io_epoll_ctl,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_SPLICE] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
		.audit_skip		= 1,
		.prep			= io_splice_prep,
		.issue			= io_splice,
	},
	[IORING_OP_PROVIDE_BUFFERS] = {
		.audit_skip		= 1,
		.iopoll			= 1,
		.prep			= io_provide_buffers_prep,
		.issue			= io_provide_buffers,
	},
	[IORING_OP_REMOVE_BUFFERS] = {
		.audit_skip		= 1,
		.iopoll			= 1,
		.prep			= io_remove_buffers_prep,
		.issue			= io_remove_buffers,
	},
	[IORING_OP_TEE] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
		.audit_skip		= 1,
		.prep			= io_tee_prep,
		.issue			= io_tee,
	},
	[IORING_OP_SHUTDOWN] = {
		.needs_file		= 1,
#if defined(CONFIG_NET)
		.prep			= io_shutdown_prep,
		.issue			= io_shutdown,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_RENAMEAT] = {
		.prep			= io_renameat_prep,
		.issue			= io_renameat,
	},
	[IORING_OP_UNLINKAT] = {
		.prep			= io_unlinkat_prep,
		.issue			= io_unlinkat,
	},
	[IORING_OP_MKDIRAT] = {
		.prep			= io_mkdirat_prep,
		.issue			= io_mkdirat,
	},
	[IORING_OP_SYMLINKAT] = {
		.prep			= io_symlinkat_prep,
		.issue			= io_symlinkat,
	},
	[IORING_OP_LINKAT] = {
		.prep			= io_linkat_prep,
		.issue			= io_linkat,
	},
	[IORING_OP_MSG_RING] = {
		.needs_file		= 1,
		.iopoll			= 1,
		.prep			= io_msg_ring_prep,
		.issue			= io_msg_ring,
	},
	[IORING_OP_FSETXATTR] = {
		.needs_file = 1,
		.prep			= io_fsetxattr_prep,
		.issue			= io_fsetxattr,
	},
	[IORING_OP_SETXATTR] = {
		.prep			= io_setxattr_prep,
		.issue			= io_setxattr,
	},
	[IORING_OP_FGETXATTR] = {
		.needs_file = 1,
		.prep			= io_fgetxattr_prep,
		.issue			= io_fgetxattr,
	},
	[IORING_OP_GETXATTR] = {
		.prep			= io_getxattr_prep,
		.issue			= io_getxattr,
	},
	[IORING_OP_SOCKET] = {
		.audit_skip		= 1,
#if defined(CONFIG_NET)
		.prep			= io_socket_prep,
		.issue			= io_socket,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_URING_CMD] = {
		.needs_file		= 1,
		.plug			= 1,
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.prep			= io_uring_cmd_prep,
		.issue			= io_uring_cmd,
	},
	[IORING_OP_SEND_ZC] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.manual_alloc		= 1,
#if defined(CONFIG_NET)
		.prep			= io_send_zc_prep,
		.issue			= io_send_zc,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_SENDMSG_ZC] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.ioprio			= 1,
		.manual_alloc		= 1,
#if defined(CONFIG_NET)
		.prep			= io_send_zc_prep,
		.issue			= io_sendmsg_zc,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
};


const struct io_cold_def io_cold_defs[] = {
	[IORING_OP_NOP] = {
		.name			= "NOP",
	},
	[IORING_OP_READV] = {
		.async_size		= sizeof(struct io_async_rw),
		.name			= "READV",
		.prep_async		= io_readv_prep_async,
		.cleanup		= io_readv_writev_cleanup,
		.fail			= io_rw_fail,
	},
	[IORING_OP_WRITEV] = {
		.async_size		= sizeof(struct io_async_rw),
		.name			= "WRITEV",
		.prep_async		= io_writev_prep_async,
		.cleanup		= io_readv_writev_cleanup,
		.fail			= io_rw_fail,
	},
	[IORING_OP_FSYNC] = {
		.name			= "FSYNC",
	},
	[IORING_OP_READ_FIXED] = {
		.async_size		= sizeof(struct io_async_rw),
		.name			= "READ_FIXED",
		.fail			= io_rw_fail,
	},
	[IORING_OP_WRITE_FIXED] = {
		.async_size		= sizeof(struct io_async_rw),
		.name			= "WRITE_FIXED",
		.fail			= io_rw_fail,
	},
	[IORING_OP_POLL_ADD] = {
		.name			= "POLL_ADD",
	},
	[IORING_OP_POLL_REMOVE] = {
		.name			= "POLL_REMOVE",
	},
	[IORING_OP_SYNC_FILE_RANGE] = {
		.name			= "SYNC_FILE_RANGE",
	},
	[IORING_OP_SENDMSG] = {
		.name			= "SENDMSG",
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_msghdr),
		.prep_async		= io_sendmsg_prep_async,
		.cleanup		= io_sendmsg_recvmsg_cleanup,
		.fail			= io_sendrecv_fail,
#endif
	},
	[IORING_OP_RECVMSG] = {
		.name			= "RECVMSG",
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_msghdr),
		.prep_async		= io_recvmsg_prep_async,
		.cleanup		= io_sendmsg_recvmsg_cleanup,
		.fail			= io_sendrecv_fail,
#endif
	},
	[IORING_OP_TIMEOUT] = {
		.async_size		= sizeof(struct io_timeout_data),
		.name			= "TIMEOUT",
	},
	[IORING_OP_TIMEOUT_REMOVE] = {
		.name			= "TIMEOUT_REMOVE",
	},
	[IORING_OP_ACCEPT] = {
		.name			= "ACCEPT",
	},
	[IORING_OP_ASYNC_CANCEL] = {
		.name			= "ASYNC_CANCEL",
	},
	[IORING_OP_LINK_TIMEOUT] = {
		.async_size		= sizeof(struct io_timeout_data),
		.name			= "LINK_TIMEOUT",
	},
	[IORING_OP_CONNECT] = {
		.name			= "CONNECT",
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_connect),
		.prep_async		= io_connect_prep_async,
#endif
	},
	[IORING_OP_FALLOCATE] = {
		.name			= "FALLOCATE",
	},
	[IORING_OP_OPENAT] = {
		.name			= "OPENAT",
		.cleanup		= io_open_cleanup,
	},
	[IORING_OP_CLOSE] = {
		.name			= "CLOSE",
	},
	[IORING_OP_FILES_UPDATE] = {
		.name			= "FILES_UPDATE",
	},
	[IORING_OP_STATX] = {
		.name			= "STATX",
		.cleanup		= io_statx_cleanup,
	},
	[IORING_OP_READ] = {
		.async_size		= sizeof(struct io_async_rw),
		.name			= "READ",
		.fail			= io_rw_fail,
	},
	[IORING_OP_WRITE] = {
		.async_size		= sizeof(struct io_async_rw),
		.name			= "WRITE",
		.fail			= io_rw_fail,
	},
	[IORING_OP_FADVISE] = {
		.name			= "FADVISE",
	},
	[IORING_OP_MADVISE] = {
		.name			= "MADVISE",
	},
	[IORING_OP_SEND] = {
		.name			= "SEND",
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_msghdr),
		.fail			= io_sendrecv_fail,
		.prep_async		= io_send_prep_async,
#endif
	},
	[IORING_OP_RECV] = {
		.name			= "RECV",
#if defined(CONFIG_NET)
		.fail			= io_sendrecv_fail,
#endif
	},
	[IORING_OP_OPENAT2] = {
		.name			= "OPENAT2",
		.cleanup		= io_open_cleanup,
	},
	[IORING_OP_EPOLL_CTL] = {
		.name			= "EPOLL",
	},
	[IORING_OP_SPLICE] = {
		.name			= "SPLICE",
	},
	[IORING_OP_PROVIDE_BUFFERS] = {
		.name			= "PROVIDE_BUFFERS",
	},
	[IORING_OP_REMOVE_BUFFERS] = {
		.name			= "REMOVE_BUFFERS",
	},
	[IORING_OP_TEE] = {
		.name			= "TEE",
	},
	[IORING_OP_SHUTDOWN] = {
		.name			= "SHUTDOWN",
	},
	[IORING_OP_RENAMEAT] = {
		.name			= "RENAMEAT",
		.cleanup		= io_renameat_cleanup,
	},
	[IORING_OP_UNLINKAT] = {
		.name			= "UNLINKAT",
		.cleanup		= io_unlinkat_cleanup,
	},
	[IORING_OP_MKDIRAT] = {
		.name			= "MKDIRAT",
		.cleanup		= io_mkdirat_cleanup,
	},
	[IORING_OP_SYMLINKAT] = {
		.name			= "SYMLINKAT",
		.cleanup		= io_link_cleanup,
	},
	[IORING_OP_LINKAT] = {
		.name			= "LINKAT",
		.cleanup		= io_link_cleanup,
	},
	[IORING_OP_MSG_RING] = {
		.name			= "MSG_RING",
		.cleanup		= io_msg_ring_cleanup,
	},
	[IORING_OP_FSETXATTR] = {
		.name			= "FSETXATTR",
		.cleanup		= io_xattr_cleanup,
	},
	[IORING_OP_SETXATTR] = {
		.name			= "SETXATTR",
		.cleanup		= io_xattr_cleanup,
	},
	[IORING_OP_FGETXATTR] = {
		.name			= "FGETXATTR",
		.cleanup		= io_xattr_cleanup,
	},
	[IORING_OP_GETXATTR] = {
		.name			= "GETXATTR",
		.cleanup		= io_xattr_cleanup,
	},
	[IORING_OP_SOCKET] = {
		.name			= "SOCKET",
	},
	[IORING_OP_URING_CMD] = {
		.name			= "URING_CMD",
		.async_size		= uring_cmd_pdu_size(1),
		.prep_async		= io_uring_cmd_prep_async,
	},
	[IORING_OP_SEND_ZC] = {
		.name			= "SEND_ZC",
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_msghdr),
		.prep_async		= io_send_prep_async,
		.cleanup		= io_send_zc_cleanup,
		.fail			= io_sendrecv_fail,
#endif
	},
	[IORING_OP_SENDMSG_ZC] = {
		.name			= "SENDMSG_ZC",
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_msghdr),
		.prep_async		= io_sendmsg_prep_async,
		.cleanup		= io_send_zc_cleanup,
		.fail			= io_sendrecv_fail,
#endif
	},
};

const char *io_uring_get_opcode(u8 opcode)
{
	if (opcode < IORING_OP_LAST)
		return io_cold_defs[opcode].name;
	return "INVALID";
}

void __init io_uring_optable_init(void)
{
	int i;

	BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);
	BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);

	for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
		BUG_ON(!io_issue_defs[i].prep);
		if (io_issue_defs[i].prep != io_eopnotsupp_prep)
			BUG_ON(!io_issue_defs[i].issue);
		WARN_ON_ONCE(!io_cold_defs[i].name);
	}
}
Commit	Line	Data
d9b57aa3 JA	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* io_uring opcode handling table
	4	*/
	5	#include <linux/kernel.h>
	6	#include <linux/errno.h>
	7	#include <linux/fs.h>
	8	#include <linux/file.h>
	9	#include <linux/io_uring.h>
	10
d9b57aa3 JA	11	#include "io_uring.h"
	12	#include "opdef.h"
	13	#include "refs.h"
	14	#include "tctx.h"
	15	#include "sqpoll.h"
	16	#include "fdinfo.h"
	17	#include "kbuf.h"
	18	#include "rsrc.h"
	19
	20	#include "xattr.h"
	21	#include "nop.h"
	22	#include "fs.h"
	23	#include "splice.h"
	24	#include "sync.h"
	25	#include "advise.h"
	26	#include "openclose.h"
	27	#include "uring_cmd.h"
	28	#include "epoll.h"
	29	#include "statx.h"
	30	#include "net.h"
	31	#include "msg_ring.h"
	32	#include "timeout.h"
	33	#include "poll.h"
	34	#include "cancel.h"
	35	#include "rw.h"
	36
	37	static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
	38	{
	39	WARN_ON_ONCE(1);
	40	return -ECANCELED;
	41	}
	42
	43	static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
	44	const struct io_uring_sqe *sqe)
	45	{
	46	return -EOPNOTSUPP;
	47	}
	48
a7dd2782	49	const struct io_issue_def io_issue_defs[] = {
d9b57aa3 JA	50	[IORING_OP_NOP] = {
	51	.audit_skip = 1,
	52	.iopoll = 1,
d9b57aa3 JA	53	.prep = io_nop_prep,
	54	.issue = io_nop,
	55	},
	56	[IORING_OP_READV] = {
	57	.needs_file = 1,
	58	.unbound_nonreg_file = 1,
	59	.pollin = 1,
	60	.buffer_select = 1,
	61	.plug = 1,
	62	.audit_skip = 1,
	63	.ioprio = 1,
	64	.iopoll = 1,
ef0ec1ad	65	.iopoll_queue = 1,
d9b57aa3 JA	66	.prep = io_prep_rw,
d9b57aa3 JA	67	.issue = io_read,
d9b57aa3 JA	68	},
	69	[IORING_OP_WRITEV] = {
	70	.needs_file = 1,
	71	.hash_reg_file = 1,
	72	.unbound_nonreg_file = 1,
	73	.pollout = 1,
	74	.plug = 1,
	75	.audit_skip = 1,
	76	.ioprio = 1,
	77	.iopoll = 1,
ef0ec1ad	78	.iopoll_queue = 1,
d9b57aa3 JA	79	.prep = io_prep_rw,
d9b57aa3 JA	80	.issue = io_write,
d9b57aa3 JA	81	},
	82	[IORING_OP_FSYNC] = {
	83	.needs_file = 1,
	84	.audit_skip = 1,
d9b57aa3 JA	85	.prep = io_fsync_prep,
	86	.issue = io_fsync,
	87	},
	88	[IORING_OP_READ_FIXED] = {
	89	.needs_file = 1,
	90	.unbound_nonreg_file = 1,
	91	.pollin = 1,
	92	.plug = 1,
	93	.audit_skip = 1,
	94	.ioprio = 1,
	95	.iopoll = 1,
ef0ec1ad	96	.iopoll_queue = 1,
d9b57aa3 JA	97	.prep = io_prep_rw,
	98	.issue = io_read,
	99	},
	100	[IORING_OP_WRITE_FIXED] = {
	101	.needs_file = 1,
	102	.hash_reg_file = 1,
	103	.unbound_nonreg_file = 1,
	104	.pollout = 1,
	105	.plug = 1,
	106	.audit_skip = 1,
	107	.ioprio = 1,
	108	.iopoll = 1,
ef0ec1ad	109	.iopoll_queue = 1,
d9b57aa3 JA	110	.prep = io_prep_rw,
	111	.issue = io_write,
	112	},
	113	[IORING_OP_POLL_ADD] = {
	114	.needs_file = 1,
	115	.unbound_nonreg_file = 1,
	116	.audit_skip = 1,
d9b57aa3 JA	117	.prep = io_poll_add_prep,
	118	.issue = io_poll_add,
	119	},
	120	[IORING_OP_POLL_REMOVE] = {
	121	.audit_skip = 1,
d9b57aa3 JA	122	.prep = io_poll_remove_prep,
	123	.issue = io_poll_remove,
	124	},
	125	[IORING_OP_SYNC_FILE_RANGE] = {
	126	.needs_file = 1,
	127	.audit_skip = 1,
d9b57aa3 JA	128	.prep = io_sfr_prep,
	129	.issue = io_sync_file_range,
	130	},
	131	[IORING_OP_SENDMSG] = {
	132	.needs_file = 1,
	133	.unbound_nonreg_file = 1,
	134	.pollout = 1,
	135	.ioprio = 1,
858c293e	136	.manual_alloc = 1,
d9b57aa3	137	#if defined(CONFIG_NET)
d9b57aa3 JA	138	.prep = io_sendmsg_prep,
d9b57aa3 JA	139	.issue = io_sendmsg,
d9b57aa3 JA	140	#else
	141	.prep = io_eopnotsupp_prep,
	142	#endif
	143	},
	144	[IORING_OP_RECVMSG] = {
	145	.needs_file = 1,
	146	.unbound_nonreg_file = 1,
	147	.pollin = 1,
	148	.buffer_select = 1,
	149	.ioprio = 1,
858c293e	150	.manual_alloc = 1,
d9b57aa3	151	#if defined(CONFIG_NET)
d9b57aa3 JA	152	.prep = io_recvmsg_prep,
d9b57aa3 JA	153	.issue = io_recvmsg,
d9b57aa3 JA	154	#else
	155	.prep = io_eopnotsupp_prep,
	156	#endif
	157	},
	158	[IORING_OP_TIMEOUT] = {
	159	.audit_skip = 1,
d9b57aa3 JA	160	.prep = io_timeout_prep,
	161	.issue = io_timeout,
	162	},
	163	[IORING_OP_TIMEOUT_REMOVE] = {
	164	/* used by timeout updates' prep() */
	165	.audit_skip = 1,
d9b57aa3 JA	166	.prep = io_timeout_remove_prep,
	167	.issue = io_timeout_remove,
	168	},
	169	[IORING_OP_ACCEPT] = {
	170	.needs_file = 1,
	171	.unbound_nonreg_file = 1,
	172	.pollin = 1,
	173	.poll_exclusive = 1,
	174	.ioprio = 1, /* used for flags */
d9b57aa3 JA	175	#if defined(CONFIG_NET)
	176	.prep = io_accept_prep,
	177	.issue = io_accept,
	178	#else
	179	.prep = io_eopnotsupp_prep,
	180	#endif
	181	},
	182	[IORING_OP_ASYNC_CANCEL] = {
	183	.audit_skip = 1,
d9b57aa3 JA	184	.prep = io_async_cancel_prep,
	185	.issue = io_async_cancel,
	186	},
	187	[IORING_OP_LINK_TIMEOUT] = {
	188	.audit_skip = 1,
d9b57aa3 JA	189	.prep = io_link_timeout_prep,
	190	.issue = io_no_issue,
	191	},
	192	[IORING_OP_CONNECT] = {
	193	.needs_file = 1,
	194	.unbound_nonreg_file = 1,
	195	.pollout = 1,
d9b57aa3	196	#if defined(CONFIG_NET)
d9b57aa3 JA	197	.prep = io_connect_prep,
d9b57aa3 JA	198	.issue = io_connect,
d9b57aa3 JA	199	#else
	200	.prep = io_eopnotsupp_prep,
	201	#endif
	202	},
	203	[IORING_OP_FALLOCATE] = {
	204	.needs_file = 1,
d9b57aa3 JA	205	.prep = io_fallocate_prep,
	206	.issue = io_fallocate,
	207	},
	208	[IORING_OP_OPENAT] = {
d9b57aa3 JA	209	.prep = io_openat_prep,
d9b57aa3 JA	210	.issue = io_openat,
d9b57aa3 JA	211	},
d9b57aa3 JA	212	[IORING_OP_CLOSE] = {
d9b57aa3 JA	213	.prep = io_close_prep,
	214	.issue = io_close,
	215	},
d9808ceb	216	[IORING_OP_FILES_UPDATE] = {
d9b57aa3 JA	217	.audit_skip = 1,
d9b57aa3 JA	218	.iopoll = 1,
d9808ceb PB	219	.prep = io_files_update_prep,
d9808ceb PB	220	.issue = io_files_update,
d9b57aa3 JA	221	},
	222	[IORING_OP_STATX] = {
	223	.audit_skip = 1,
d9b57aa3 JA	224	.prep = io_statx_prep,
d9b57aa3 JA	225	.issue = io_statx,
d9b57aa3 JA	226	},
	227	[IORING_OP_READ] = {
	228	.needs_file = 1,
	229	.unbound_nonreg_file = 1,
	230	.pollin = 1,
	231	.buffer_select = 1,
	232	.plug = 1,
	233	.audit_skip = 1,
	234	.ioprio = 1,
	235	.iopoll = 1,
ef0ec1ad	236	.iopoll_queue = 1,
d9b57aa3 JA	237	.prep = io_prep_rw,
	238	.issue = io_read,
	239	},
	240	[IORING_OP_WRITE] = {
	241	.needs_file = 1,
	242	.hash_reg_file = 1,
	243	.unbound_nonreg_file = 1,
	244	.pollout = 1,
	245	.plug = 1,
	246	.audit_skip = 1,
	247	.ioprio = 1,
	248	.iopoll = 1,
ef0ec1ad	249	.iopoll_queue = 1,
d9b57aa3 JA	250	.prep = io_prep_rw,
	251	.issue = io_write,
	252	},
	253	[IORING_OP_FADVISE] = {
	254	.needs_file = 1,
	255	.audit_skip = 1,
d9b57aa3 JA	256	.prep = io_fadvise_prep,
	257	.issue = io_fadvise,
	258	},
	259	[IORING_OP_MADVISE] = {
fbe870a7	260	.audit_skip = 1,
d9b57aa3 JA	261	.prep = io_madvise_prep,
	262	.issue = io_madvise,
	263	},
	264	[IORING_OP_SEND] = {
	265	.needs_file = 1,
	266	.unbound_nonreg_file = 1,
	267	.pollout = 1,
	268	.audit_skip = 1,
	269	.ioprio = 1,
516e82f0	270	.manual_alloc = 1,
d9b57aa3 JA	271	#if defined(CONFIG_NET)
	272	.prep = io_sendmsg_prep,
	273	.issue = io_send,
	274	#else
	275	.prep = io_eopnotsupp_prep,
	276	#endif
	277	},
	278	[IORING_OP_RECV] = {
	279	.needs_file = 1,
	280	.unbound_nonreg_file = 1,
	281	.pollin = 1,
	282	.buffer_select = 1,
	283	.audit_skip = 1,
	284	.ioprio = 1,
d9b57aa3 JA	285	#if defined(CONFIG_NET)
	286	.prep = io_recvmsg_prep,
	287	.issue = io_recv,
	288	#else
	289	.prep = io_eopnotsupp_prep,
	290	#endif
	291	},
	292	[IORING_OP_OPENAT2] = {
d9b57aa3 JA	293	.prep = io_openat2_prep,
d9b57aa3 JA	294	.issue = io_openat2,
d9b57aa3 JA	295	},
	296	[IORING_OP_EPOLL_CTL] = {
	297	.unbound_nonreg_file = 1,
	298	.audit_skip = 1,
d9b57aa3 JA	299	#if defined(CONFIG_EPOLL)
	300	.prep = io_epoll_ctl_prep,
	301	.issue = io_epoll_ctl,
	302	#else
	303	.prep = io_eopnotsupp_prep,
	304	#endif
	305	},
	306	[IORING_OP_SPLICE] = {
	307	.needs_file = 1,
	308	.hash_reg_file = 1,
	309	.unbound_nonreg_file = 1,
	310	.audit_skip = 1,
d9b57aa3 JA	311	.prep = io_splice_prep,
	312	.issue = io_splice,
	313	},
	314	[IORING_OP_PROVIDE_BUFFERS] = {
	315	.audit_skip = 1,
	316	.iopoll = 1,
d9b57aa3 JA	317	.prep = io_provide_buffers_prep,
	318	.issue = io_provide_buffers,
	319	},
	320	[IORING_OP_REMOVE_BUFFERS] = {
	321	.audit_skip = 1,
	322	.iopoll = 1,
d9b57aa3 JA	323	.prep = io_remove_buffers_prep,
	324	.issue = io_remove_buffers,
	325	},
	326	[IORING_OP_TEE] = {
	327	.needs_file = 1,
	328	.hash_reg_file = 1,
	329	.unbound_nonreg_file = 1,
	330	.audit_skip = 1,
d9b57aa3 JA	331	.prep = io_tee_prep,
	332	.issue = io_tee,
	333	},
	334	[IORING_OP_SHUTDOWN] = {
	335	.needs_file = 1,
d9b57aa3 JA	336	#if defined(CONFIG_NET)
	337	.prep = io_shutdown_prep,
	338	.issue = io_shutdown,
	339	#else
	340	.prep = io_eopnotsupp_prep,
	341	#endif
	342	},
	343	[IORING_OP_RENAMEAT] = {
d9b57aa3 JA	344	.prep = io_renameat_prep,
d9b57aa3 JA	345	.issue = io_renameat,
d9b57aa3 JA	346	},
d9b57aa3 JA	347	[IORING_OP_UNLINKAT] = {
d9b57aa3 JA	348	.prep = io_unlinkat_prep,
d9b57aa3 JA	349	.issue = io_unlinkat,
d9b57aa3 JA	350	},
d9b57aa3 JA	351	[IORING_OP_MKDIRAT] = {
d9b57aa3 JA	352	.prep = io_mkdirat_prep,
d9b57aa3 JA	353	.issue = io_mkdirat,
d9b57aa3 JA	354	},
d9b57aa3 JA	355	[IORING_OP_SYMLINKAT] = {
d9b57aa3 JA	356	.prep = io_symlinkat_prep,
d9b57aa3 JA	357	.issue = io_symlinkat,
d9b57aa3 JA	358	},
d9b57aa3 JA	359	[IORING_OP_LINKAT] = {
d9b57aa3 JA	360	.prep = io_linkat_prep,
d9b57aa3 JA	361	.issue = io_linkat,
d9b57aa3 JA	362	},
	363	[IORING_OP_MSG_RING] = {
	364	.needs_file = 1,
	365	.iopoll = 1,
d9b57aa3 JA	366	.prep = io_msg_ring_prep,
	367	.issue = io_msg_ring,
	368	},
	369	[IORING_OP_FSETXATTR] = {
	370	.needs_file = 1,
d9b57aa3 JA	371	.prep = io_fsetxattr_prep,
d9b57aa3 JA	372	.issue = io_fsetxattr,
d9b57aa3 JA	373	},
d9b57aa3 JA	374	[IORING_OP_SETXATTR] = {
d9b57aa3 JA	375	.prep = io_setxattr_prep,
d9b57aa3 JA	376	.issue = io_setxattr,
d9b57aa3 JA	377	},
	378	[IORING_OP_FGETXATTR] = {
	379	.needs_file = 1,
d9b57aa3 JA	380	.prep = io_fgetxattr_prep,
d9b57aa3 JA	381	.issue = io_fgetxattr,
d9b57aa3 JA	382	},
d9b57aa3 JA	383	[IORING_OP_GETXATTR] = {
d9b57aa3 JA	384	.prep = io_getxattr_prep,
d9b57aa3 JA	385	.issue = io_getxattr,
d9b57aa3 JA	386	},
	387	[IORING_OP_SOCKET] = {
	388	.audit_skip = 1,
d9b57aa3 JA	389	#if defined(CONFIG_NET)
	390	.prep = io_socket_prep,
	391	.issue = io_socket,
	392	#else
	393	.prep = io_eopnotsupp_prep,
	394	#endif
	395	},
	396	[IORING_OP_URING_CMD] = {
	397	.needs_file = 1,
	398	.plug = 1,
5756a3a7	399	.iopoll = 1,
ef0ec1ad	400	.iopoll_queue = 1,
d9b57aa3 JA	401	.prep = io_uring_cmd_prep,
d9b57aa3 JA	402	.issue = io_uring_cmd,
d9b57aa3	403	},
b48c312b	404	[IORING_OP_SEND_ZC] = {
06a5464b PB	405	.needs_file = 1,
	406	.unbound_nonreg_file = 1,
	407	.pollout = 1,
	408	.audit_skip = 1,
	409	.ioprio = 1,
581711c4	410	.manual_alloc = 1,
06a5464b	411	#if defined(CONFIG_NET)
b0e9b551 PB	412	.prep = io_send_zc_prep,
b0e9b551 PB	413	.issue = io_send_zc,
06a5464b PB	414	#else
06a5464b PB	415	.prep = io_eopnotsupp_prep,
493108d9 PB	416	#endif
	417	},
	418	[IORING_OP_SENDMSG_ZC] = {
493108d9 PB	419	.needs_file = 1,
	420	.unbound_nonreg_file = 1,
	421	.pollout = 1,
493108d9 PB	422	.ioprio = 1,
	423	.manual_alloc = 1,
	424	#if defined(CONFIG_NET)
493108d9 PB	425	.prep = io_send_zc_prep,
493108d9 PB	426	.issue = io_sendmsg_zc,
f30bd4d0 BL	427	#else
	428	.prep = io_eopnotsupp_prep,
	429	#endif
	430	},
	431	};
	432
	433
	434	const struct io_cold_def io_cold_defs[] = {
	435	[IORING_OP_NOP] = {
	436	.name = "NOP",
	437	},
	438	[IORING_OP_READV] = {
	439	.async_size = sizeof(struct io_async_rw),
	440	.name = "READV",
	441	.prep_async = io_readv_prep_async,
	442	.cleanup = io_readv_writev_cleanup,
	443	.fail = io_rw_fail,
	444	},
	445	[IORING_OP_WRITEV] = {
	446	.async_size = sizeof(struct io_async_rw),
	447	.name = "WRITEV",
	448	.prep_async = io_writev_prep_async,
	449	.cleanup = io_readv_writev_cleanup,
	450	.fail = io_rw_fail,
	451	},
	452	[IORING_OP_FSYNC] = {
	453	.name = "FSYNC",
	454	},
	455	[IORING_OP_READ_FIXED] = {
	456	.async_size = sizeof(struct io_async_rw),
	457	.name = "READ_FIXED",
	458	.fail = io_rw_fail,
	459	},
	460	[IORING_OP_WRITE_FIXED] = {
	461	.async_size = sizeof(struct io_async_rw),
	462	.name = "WRITE_FIXED",
	463	.fail = io_rw_fail,
	464	},
	465	[IORING_OP_POLL_ADD] = {
	466	.name = "POLL_ADD",
	467	},
	468	[IORING_OP_POLL_REMOVE] = {
	469	.name = "POLL_REMOVE",
	470	},
	471	[IORING_OP_SYNC_FILE_RANGE] = {
	472	.name = "SYNC_FILE_RANGE",
	473	},
	474	[IORING_OP_SENDMSG] = {
	475	.name = "SENDMSG",
	476	#if defined(CONFIG_NET)
	477	.async_size = sizeof(struct io_async_msghdr),
	478	.prep_async = io_sendmsg_prep_async,
	479	.cleanup = io_sendmsg_recvmsg_cleanup,
	480	.fail = io_sendrecv_fail,
	481	#endif
	482	},
	483	[IORING_OP_RECVMSG] = {
	484	.name = "RECVMSG",
	485	#if defined(CONFIG_NET)
	486	.async_size = sizeof(struct io_async_msghdr),
	487	.prep_async = io_recvmsg_prep_async,
	488	.cleanup = io_sendmsg_recvmsg_cleanup,
	489	.fail = io_sendrecv_fail,
	490	#endif
491	},
492	[IORING_OP_TIMEOUT] = {
493	.async_size = sizeof(struct io_timeout_data),
494	.name = "TIMEOUT",
495	},
496	[IORING_OP_TIMEOUT_REMOVE] = {
497	.name = "TIMEOUT_REMOVE",
498	},
499	[IORING_OP_ACCEPT] = {
500	.name = "ACCEPT",
501	},
502	[IORING_OP_ASYNC_CANCEL] = {
503	.name = "ASYNC_CANCEL",
504	},
505	[IORING_OP_LINK_TIMEOUT] = {
506	.async_size = sizeof(struct io_timeout_data),
507	.name = "LINK_TIMEOUT",
508	},
509	[IORING_OP_CONNECT] = {
510	.name = "CONNECT",
511	#if defined(CONFIG_NET)
512	.async_size = sizeof(struct io_async_connect),
513	.prep_async = io_connect_prep_async,
514	#endif
515	},
516	[IORING_OP_FALLOCATE] = {
517	.name = "FALLOCATE",
518	},
519	[IORING_OP_OPENAT] = {
520	.name = "OPENAT",
521	.cleanup = io_open_cleanup,
522	},
523	[IORING_OP_CLOSE] = {
524	.name = "CLOSE",
525	},
526	[IORING_OP_FILES_UPDATE] = {
527	.name = "FILES_UPDATE",
528	},
529	[IORING_OP_STATX] = {
530	.name = "STATX",
531	.cleanup = io_statx_cleanup,
532	},
533	[IORING_OP_READ] = {
534	.async_size = sizeof(struct io_async_rw),
535	.name = "READ",
536	.fail = io_rw_fail,
537	},
538	[IORING_OP_WRITE] = {
539	.async_size = sizeof(struct io_async_rw),
540	.name = "WRITE",
541	.fail = io_rw_fail,
542	},
543	[IORING_OP_FADVISE] = {
544	.name = "FADVISE",
545	},
546	[IORING_OP_MADVISE] = {
547	.name = "MADVISE",
548	},
549	[IORING_OP_SEND] = {
550	.name = "SEND",
551	#if defined(CONFIG_NET)
552	.async_size = sizeof(struct io_async_msghdr),
553	.fail = io_sendrecv_fail,
554	.prep_async = io_send_prep_async,
555	#endif
556	},
557	[IORING_OP_RECV] = {
558	.name = "RECV",
559	#if defined(CONFIG_NET)
560	.fail = io_sendrecv_fail,
561	#endif
562	},
563	[IORING_OP_OPENAT2] = {
564	.name = "OPENAT2",
565	.cleanup = io_open_cleanup,
566	},
567	[IORING_OP_EPOLL_CTL] = {
568	.name = "EPOLL",
569	},
570	[IORING_OP_SPLICE] = {
571	.name = "SPLICE",
572	},
573	[IORING_OP_PROVIDE_BUFFERS] = {
574	.name = "PROVIDE_BUFFERS",
575	},
576	[IORING_OP_REMOVE_BUFFERS] = {
577	.name = "REMOVE_BUFFERS",
578	},
579	[IORING_OP_TEE] = {
580	.name = "TEE",
581	},
582	[IORING_OP_SHUTDOWN] = {
583	.name = "SHUTDOWN",
584	},
585	[IORING_OP_RENAMEAT] = {
586	.name = "RENAMEAT",
587	.cleanup = io_renameat_cleanup,
588	},
589	[IORING_OP_UNLINKAT] = {
590	.name = "UNLINKAT",
591	.cleanup = io_unlinkat_cleanup,
592	},
593	[IORING_OP_MKDIRAT] = {
594	.name = "MKDIRAT",
595	.cleanup = io_mkdirat_cleanup,
596	},
597	[IORING_OP_SYMLINKAT] = {
598	.name = "SYMLINKAT",
599	.cleanup = io_link_cleanup,
600	},
601	[IORING_OP_LINKAT] = {
602	.name = "LINKAT",
603	.cleanup = io_link_cleanup,
604	},
605	[IORING_OP_MSG_RING] = {
606	.name = "MSG_RING",
607	.cleanup = io_msg_ring_cleanup,
608	},
609	[IORING_OP_FSETXATTR] = {
610	.name = "FSETXATTR",
611	.cleanup = io_xattr_cleanup,
612	},
613	[IORING_OP_SETXATTR] = {
614	.name = "SETXATTR",
615	.cleanup = io_xattr_cleanup,
616	},
617	[IORING_OP_FGETXATTR] = {
618	.name = "FGETXATTR",
619	.cleanup = io_xattr_cleanup,
620	},
621	[IORING_OP_GETXATTR] = {
622	.name = "GETXATTR",
623	.cleanup = io_xattr_cleanup,
624	},
625	[IORING_OP_SOCKET] = {
626	.name = "SOCKET",
627	},
628	[IORING_OP_URING_CMD] = {
629	.name = "URING_CMD",
630	.async_size = uring_cmd_pdu_size(1),
631	.prep_async = io_uring_cmd_prep_async,
632	},
633	[IORING_OP_SEND_ZC] = {
634	.name = "SEND_ZC",
635	#if defined(CONFIG_NET)
636	.async_size = sizeof(struct io_async_msghdr),
637	.prep_async = io_send_prep_async,
638	.cleanup = io_send_zc_cleanup,
639	.fail = io_sendrecv_fail,
640	#endif
641	},
642	[IORING_OP_SENDMSG_ZC] = {
643	.name = "SENDMSG_ZC",
644	#if defined(CONFIG_NET)
645	.async_size = sizeof(struct io_async_msghdr),
493108d9 PB	646	.prep_async = io_sendmsg_prep_async,
	647	.cleanup = io_send_zc_cleanup,
	648	.fail = io_sendrecv_fail,
06a5464b	649	#endif
06a5464b	650	},
d9b57aa3 JA	651	};
	652
	653	const char *io_uring_get_opcode(u8 opcode)
	654	{
	655	if (opcode < IORING_OP_LAST)
f30bd4d0	656	return io_cold_defs[opcode].name;
d9b57aa3 JA	657	return "INVALID";
	658	}
	659
	660	void __init io_uring_optable_init(void)
	661	{
	662	int i;
	663
f30bd4d0	664	BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);
a7dd2782	665	BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);
d9b57aa3	666
a7dd2782 BL	667	for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
	668	BUG_ON(!io_issue_defs[i].prep);
	669	if (io_issue_defs[i].prep != io_eopnotsupp_prep)
	670	BUG_ON(!io_issue_defs[i].issue);
f30bd4d0	671	WARN_ON_ONCE(!io_cold_defs[i].name);
d9b57aa3 JA	672	}
d9b57aa3 JA	673	}