[linux-block.git] / io_uring / opdef.c

// SPDX-License-Identifier: GPL-2.0
/*
 * io_uring opcode handling table
 */
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/io_uring.h>

#include "io_uring.h"
#include "opdef.h"
#include "refs.h"
#include "tctx.h"
#include "sqpoll.h"
#include "fdinfo.h"
#include "kbuf.h"
#include "rsrc.h"

#include "xattr.h"
#include "nop.h"
#include "fs.h"
#include "splice.h"
#include "sync.h"
#include "advise.h"
#include "openclose.h"
#include "uring_cmd.h"
#include "epoll.h"
#include "statx.h"
#include "net.h"
#include "msg_ring.h"
#include "timeout.h"
#include "poll.h"
#include "cancel.h"
#include "rw.h"

static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
{
	WARN_ON_ONCE(1);
	return -ECANCELED;
}

static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
					     const struct io_uring_sqe *sqe)
{
	return -EOPNOTSUPP;
}

const struct io_issue_def io_issue_defs[] = {
	[IORING_OP_NOP] = {
		.audit_skip		= 1,
		.iopoll			= 1,
		.prep			= io_nop_prep,
		.issue			= io_nop,
	},
	[IORING_OP_READV] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.buffer_select		= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.vectored		= 1,
		.prep			= io_prep_rw,
		.issue			= io_read,
	},
	[IORING_OP_WRITEV] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.vectored		= 1,
		.prep			= io_prep_rw,
		.issue			= io_write,
	},
	[IORING_OP_FSYNC] = {
		.needs_file		= 1,
		.audit_skip		= 1,
		.prep			= io_fsync_prep,
		.issue			= io_fsync,
	},
	[IORING_OP_READ_FIXED] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.prep			= io_prep_rw,
		.issue			= io_read,
	},
	[IORING_OP_WRITE_FIXED] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.prep			= io_prep_rw,
		.issue			= io_write,
	},
	[IORING_OP_POLL_ADD] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.audit_skip		= 1,
		.prep			= io_poll_add_prep,
		.issue			= io_poll_add,
	},
	[IORING_OP_POLL_REMOVE] = {
		.audit_skip		= 1,
		.prep			= io_poll_remove_prep,
		.issue			= io_poll_remove,
	},
	[IORING_OP_SYNC_FILE_RANGE] = {
		.needs_file		= 1,
		.audit_skip		= 1,
		.prep			= io_sfr_prep,
		.issue			= io_sync_file_range,
	},
	[IORING_OP_SENDMSG] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.ioprio			= 1,
		.manual_alloc		= 1,
#if defined(CONFIG_NET)
		.prep			= io_sendmsg_prep,
		.issue			= io_sendmsg,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_RECVMSG] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.buffer_select		= 1,
		.ioprio			= 1,
		.manual_alloc		= 1,
#if defined(CONFIG_NET)
		.prep			= io_recvmsg_prep,
		.issue			= io_recvmsg,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_TIMEOUT] = {
		.audit_skip		= 1,
		.prep			= io_timeout_prep,
		.issue			= io_timeout,
	},
	[IORING_OP_TIMEOUT_REMOVE] = {
		/* used by timeout updates' prep() */
		.audit_skip		= 1,
		.prep			= io_timeout_remove_prep,
		.issue			= io_timeout_remove,
	},
	[IORING_OP_ACCEPT] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.poll_exclusive		= 1,
		.ioprio			= 1,	/* used for flags */
#if defined(CONFIG_NET)
		.prep			= io_accept_prep,
		.issue			= io_accept,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_ASYNC_CANCEL] = {
		.audit_skip		= 1,
		.prep			= io_async_cancel_prep,
		.issue			= io_async_cancel,
	},
	[IORING_OP_LINK_TIMEOUT] = {
		.audit_skip		= 1,
		.prep			= io_link_timeout_prep,
		.issue			= io_no_issue,
	},
	[IORING_OP_CONNECT] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
#if defined(CONFIG_NET)
		.prep			= io_connect_prep,
		.issue			= io_connect,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_FALLOCATE] = {
		.needs_file		= 1,
		.prep			= io_fallocate_prep,
		.issue			= io_fallocate,
	},
	[IORING_OP_OPENAT] = {
		.prep			= io_openat_prep,
		.issue			= io_openat,
	},
	[IORING_OP_CLOSE] = {
		.prep			= io_close_prep,
		.issue			= io_close,
	},
	[IORING_OP_FILES_UPDATE] = {
		.audit_skip		= 1,
		.iopoll			= 1,
		.prep			= io_files_update_prep,
		.issue			= io_files_update,
	},
	[IORING_OP_STATX] = {
		.audit_skip		= 1,
		.prep			= io_statx_prep,
		.issue			= io_statx,
	},
	[IORING_OP_READ] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.buffer_select		= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.prep			= io_prep_rw,
		.issue			= io_read,
	},
	[IORING_OP_WRITE] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.prep			= io_prep_rw,
		.issue			= io_write,
	},
	[IORING_OP_FADVISE] = {
		.needs_file		= 1,
		.audit_skip		= 1,
		.prep			= io_fadvise_prep,
		.issue			= io_fadvise,
	},
	[IORING_OP_MADVISE] = {
		.audit_skip		= 1,
		.prep			= io_madvise_prep,
		.issue			= io_madvise,
	},
	[IORING_OP_SEND] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.manual_alloc		= 1,
#if defined(CONFIG_NET)
		.prep			= io_sendmsg_prep,
		.issue			= io_send,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_RECV] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.buffer_select		= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
#if defined(CONFIG_NET)
		.prep			= io_recvmsg_prep,
		.issue			= io_recv,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_OPENAT2] = {
		.prep			= io_openat2_prep,
		.issue			= io_openat2,
	},
	[IORING_OP_EPOLL_CTL] = {
		.unbound_nonreg_file	= 1,
		.audit_skip		= 1,
#if defined(CONFIG_EPOLL)
		.prep			= io_epoll_ctl_prep,
		.issue			= io_epoll_ctl,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_SPLICE] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
		.audit_skip		= 1,
		.prep			= io_splice_prep,
		.issue			= io_splice,
	},
	[IORING_OP_PROVIDE_BUFFERS] = {
		.audit_skip		= 1,
		.iopoll			= 1,
		.prep			= io_provide_buffers_prep,
		.issue			= io_provide_buffers,
	},
	[IORING_OP_REMOVE_BUFFERS] = {
		.audit_skip		= 1,
		.iopoll			= 1,
		.prep			= io_remove_buffers_prep,
		.issue			= io_remove_buffers,
	},
	[IORING_OP_TEE] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
		.audit_skip		= 1,
		.prep			= io_tee_prep,
		.issue			= io_tee,
	},
	[IORING_OP_SHUTDOWN] = {
		.needs_file		= 1,
#if defined(CONFIG_NET)
		.prep			= io_shutdown_prep,
		.issue			= io_shutdown,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_RENAMEAT] = {
		.prep			= io_renameat_prep,
		.issue			= io_renameat,
	},
	[IORING_OP_UNLINKAT] = {
		.prep			= io_unlinkat_prep,
		.issue			= io_unlinkat,
	},
	[IORING_OP_MKDIRAT] = {
		.prep			= io_mkdirat_prep,
		.issue			= io_mkdirat,
	},
	[IORING_OP_SYMLINKAT] = {
		.prep			= io_symlinkat_prep,
		.issue			= io_symlinkat,
	},
	[IORING_OP_LINKAT] = {
		.prep			= io_linkat_prep,
		.issue			= io_linkat,
	},
	[IORING_OP_MSG_RING] = {
		.needs_file		= 1,
		.iopoll			= 1,
		.prep			= io_msg_ring_prep,
		.issue			= io_msg_ring,
	},
	[IORING_OP_FSETXATTR] = {
		.needs_file = 1,
		.prep			= io_fsetxattr_prep,
		.issue			= io_fsetxattr,
	},
	[IORING_OP_SETXATTR] = {
		.prep			= io_setxattr_prep,
		.issue			= io_setxattr,
	},
	[IORING_OP_FGETXATTR] = {
		.needs_file = 1,
		.prep			= io_fgetxattr_prep,
		.issue			= io_fgetxattr,
	},
	[IORING_OP_GETXATTR] = {
		.prep			= io_getxattr_prep,
		.issue			= io_getxattr,
	},
	[IORING_OP_SOCKET] = {
		.audit_skip		= 1,
#if defined(CONFIG_NET)
		.prep			= io_socket_prep,
		.issue			= io_socket,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_URING_CMD] = {
		.needs_file		= 1,
		.plug			= 1,
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.prep			= io_uring_cmd_prep,
		.issue			= io_uring_cmd,
	},
	[IORING_OP_SEND_ZC] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.manual_alloc		= 1,
#if defined(CONFIG_NET)
		.prep			= io_send_zc_prep,
		.issue			= io_send_zc,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_SENDMSG_ZC] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.ioprio			= 1,
		.manual_alloc		= 1,
#if defined(CONFIG_NET)
		.prep			= io_send_zc_prep,
		.issue			= io_sendmsg_zc,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_READ_MULTISHOT] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.buffer_select		= 1,
		.audit_skip		= 1,
		.prep			= io_read_mshot_prep,
		.issue			= io_read_mshot,
	},
};

const struct io_cold_def io_cold_defs[] = {
	[IORING_OP_NOP] = {
		.name			= "NOP",
	},
	[IORING_OP_READV] = {
		.async_size		= sizeof(struct io_async_rw),
		.name			= "READV",
		.prep_async		= io_readv_prep_async,
		.cleanup		= io_readv_writev_cleanup,
		.fail			= io_rw_fail,
	},
	[IORING_OP_WRITEV] = {
		.async_size		= sizeof(struct io_async_rw),
		.name			= "WRITEV",
		.prep_async		= io_writev_prep_async,
		.cleanup		= io_readv_writev_cleanup,
		.fail			= io_rw_fail,
	},
	[IORING_OP_FSYNC] = {
		.name			= "FSYNC",
	},
	[IORING_OP_READ_FIXED] = {
		.async_size		= sizeof(struct io_async_rw),
		.name			= "READ_FIXED",
		.fail			= io_rw_fail,
	},
	[IORING_OP_WRITE_FIXED] = {
		.async_size		= sizeof(struct io_async_rw),
		.name			= "WRITE_FIXED",
		.fail			= io_rw_fail,
	},
	[IORING_OP_POLL_ADD] = {
		.name			= "POLL_ADD",
	},
	[IORING_OP_POLL_REMOVE] = {
		.name			= "POLL_REMOVE",
	},
	[IORING_OP_SYNC_FILE_RANGE] = {
		.name			= "SYNC_FILE_RANGE",
	},
	[IORING_OP_SENDMSG] = {
		.name			= "SENDMSG",
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_msghdr),
		.prep_async		= io_sendmsg_prep_async,
		.cleanup		= io_sendmsg_recvmsg_cleanup,
		.fail			= io_sendrecv_fail,
#endif
	},
	[IORING_OP_RECVMSG] = {
		.name			= "RECVMSG",
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_msghdr),
		.prep_async		= io_recvmsg_prep_async,
		.cleanup		= io_sendmsg_recvmsg_cleanup,
		.fail			= io_sendrecv_fail,
#endif
	},
	[IORING_OP_TIMEOUT] = {
		.async_size		= sizeof(struct io_timeout_data),
		.name			= "TIMEOUT",
	},
	[IORING_OP_TIMEOUT_REMOVE] = {
		.name			= "TIMEOUT_REMOVE",
	},
	[IORING_OP_ACCEPT] = {
		.name			= "ACCEPT",
	},
	[IORING_OP_ASYNC_CANCEL] = {
		.name			= "ASYNC_CANCEL",
	},
	[IORING_OP_LINK_TIMEOUT] = {
		.async_size		= sizeof(struct io_timeout_data),
		.name			= "LINK_TIMEOUT",
	},
	[IORING_OP_CONNECT] = {
		.name			= "CONNECT",
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_connect),
		.prep_async		= io_connect_prep_async,
#endif
	},
	[IORING_OP_FALLOCATE] = {
		.name			= "FALLOCATE",
	},
	[IORING_OP_OPENAT] = {
		.name			= "OPENAT",
		.cleanup		= io_open_cleanup,
	},
	[IORING_OP_CLOSE] = {
		.name			= "CLOSE",
	},
	[IORING_OP_FILES_UPDATE] = {
		.name			= "FILES_UPDATE",
	},
	[IORING_OP_STATX] = {
		.name			= "STATX",
		.cleanup		= io_statx_cleanup,
	},
	[IORING_OP_READ] = {
		.async_size		= sizeof(struct io_async_rw),
		.name			= "READ",
		.fail			= io_rw_fail,
	},
	[IORING_OP_WRITE] = {
		.async_size		= sizeof(struct io_async_rw),
		.name			= "WRITE",
		.fail			= io_rw_fail,
	},
	[IORING_OP_FADVISE] = {
		.name			= "FADVISE",
	},
	[IORING_OP_MADVISE] = {
		.name			= "MADVISE",
	},
	[IORING_OP_SEND] = {
		.name			= "SEND",
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_msghdr),
		.fail			= io_sendrecv_fail,
		.prep_async		= io_send_prep_async,
#endif
	},
	[IORING_OP_RECV] = {
		.name			= "RECV",
#if defined(CONFIG_NET)
		.fail			= io_sendrecv_fail,
#endif
	},
	[IORING_OP_OPENAT2] = {
		.name			= "OPENAT2",
		.cleanup		= io_open_cleanup,
	},
	[IORING_OP_EPOLL_CTL] = {
		.name			= "EPOLL",
	},
	[IORING_OP_SPLICE] = {
		.name			= "SPLICE",
	},
	[IORING_OP_PROVIDE_BUFFERS] = {
		.name			= "PROVIDE_BUFFERS",
	},
	[IORING_OP_REMOVE_BUFFERS] = {
		.name			= "REMOVE_BUFFERS",
	},
	[IORING_OP_TEE] = {
		.name			= "TEE",
	},
	[IORING_OP_SHUTDOWN] = {
		.name			= "SHUTDOWN",
	},
	[IORING_OP_RENAMEAT] = {
		.name			= "RENAMEAT",
		.cleanup		= io_renameat_cleanup,
	},
	[IORING_OP_UNLINKAT] = {
		.name			= "UNLINKAT",
		.cleanup		= io_unlinkat_cleanup,
	},
	[IORING_OP_MKDIRAT] = {
		.name			= "MKDIRAT",
		.cleanup		= io_mkdirat_cleanup,
	},
	[IORING_OP_SYMLINKAT] = {
		.name			= "SYMLINKAT",
		.cleanup		= io_link_cleanup,
	},
	[IORING_OP_LINKAT] = {
		.name			= "LINKAT",
		.cleanup		= io_link_cleanup,
	},
	[IORING_OP_MSG_RING] = {
		.name			= "MSG_RING",
		.cleanup		= io_msg_ring_cleanup,
	},
	[IORING_OP_FSETXATTR] = {
		.name			= "FSETXATTR",
		.cleanup		= io_xattr_cleanup,
	},
	[IORING_OP_SETXATTR] = {
		.name			= "SETXATTR",
		.cleanup		= io_xattr_cleanup,
	},
	[IORING_OP_FGETXATTR] = {
		.name			= "FGETXATTR",
		.cleanup		= io_xattr_cleanup,
	},
	[IORING_OP_GETXATTR] = {
		.name			= "GETXATTR",
		.cleanup		= io_xattr_cleanup,
	},
	[IORING_OP_SOCKET] = {
		.name			= "SOCKET",
	},
	[IORING_OP_URING_CMD] = {
		.name			= "URING_CMD",
		.async_size		= 2 * sizeof(struct io_uring_sqe),
		.prep_async		= io_uring_cmd_prep_async,
	},
	[IORING_OP_SEND_ZC] = {
		.name			= "SEND_ZC",
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_msghdr),
		.prep_async		= io_send_prep_async,
		.cleanup		= io_send_zc_cleanup,
		.fail			= io_sendrecv_fail,
#endif
	},
	[IORING_OP_SENDMSG_ZC] = {
		.name			= "SENDMSG_ZC",
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_msghdr),
		.prep_async		= io_sendmsg_prep_async,
		.cleanup		= io_send_zc_cleanup,
		.fail			= io_sendrecv_fail,
#endif
	},
	[IORING_OP_READ_MULTISHOT] = {
		.name			= "READ_MULTISHOT",
	},
};

const char *io_uring_get_opcode(u8 opcode)
{
	if (opcode < IORING_OP_LAST)
		return io_cold_defs[opcode].name;
	return "INVALID";
}

void __init io_uring_optable_init(void)
{
	int i;

	BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);
	BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);

	for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
		BUG_ON(!io_issue_defs[i].prep);
		if (io_issue_defs[i].prep != io_eopnotsupp_prep)
			BUG_ON(!io_issue_defs[i].issue);
		WARN_ON_ONCE(!io_cold_defs[i].name);
	}
}
Commit	Line	Data
d9b57aa3 JA	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* io_uring opcode handling table
	4	*/
	5	#include <linux/kernel.h>
	6	#include <linux/errno.h>
	7	#include <linux/fs.h>
	8	#include <linux/file.h>
	9	#include <linux/io_uring.h>
	10
d9b57aa3 JA	11	#include "io_uring.h"
	12	#include "opdef.h"
	13	#include "refs.h"
	14	#include "tctx.h"
	15	#include "sqpoll.h"
	16	#include "fdinfo.h"
	17	#include "kbuf.h"
	18	#include "rsrc.h"
	19
	20	#include "xattr.h"
	21	#include "nop.h"
	22	#include "fs.h"
	23	#include "splice.h"
	24	#include "sync.h"
	25	#include "advise.h"
	26	#include "openclose.h"
	27	#include "uring_cmd.h"
	28	#include "epoll.h"
	29	#include "statx.h"
	30	#include "net.h"
	31	#include "msg_ring.h"
	32	#include "timeout.h"
	33	#include "poll.h"
	34	#include "cancel.h"
	35	#include "rw.h"
	36
	37	static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
	38	{
	39	WARN_ON_ONCE(1);
	40	return -ECANCELED;
	41	}
	42
	43	static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
	44	const struct io_uring_sqe *sqe)
	45	{
	46	return -EOPNOTSUPP;
	47	}
	48
a7dd2782	49	const struct io_issue_def io_issue_defs[] = {
d9b57aa3 JA	50	[IORING_OP_NOP] = {
	51	.audit_skip = 1,
	52	.iopoll = 1,
d9b57aa3 JA	53	.prep = io_nop_prep,
	54	.issue = io_nop,
	55	},
	56	[IORING_OP_READV] = {
	57	.needs_file = 1,
	58	.unbound_nonreg_file = 1,
	59	.pollin = 1,
	60	.buffer_select = 1,
	61	.plug = 1,
	62	.audit_skip = 1,
	63	.ioprio = 1,
	64	.iopoll = 1,
ef0ec1ad	65	.iopoll_queue = 1,
d2d778fb	66	.vectored = 1,
d9b57aa3 JA	67	.prep = io_prep_rw,
d9b57aa3 JA	68	.issue = io_read,
d9b57aa3 JA	69	},
	70	[IORING_OP_WRITEV] = {
	71	.needs_file = 1,
	72	.hash_reg_file = 1,
	73	.unbound_nonreg_file = 1,
	74	.pollout = 1,
	75	.plug = 1,
	76	.audit_skip = 1,
	77	.ioprio = 1,
	78	.iopoll = 1,
ef0ec1ad	79	.iopoll_queue = 1,
d2d778fb	80	.vectored = 1,
d9b57aa3 JA	81	.prep = io_prep_rw,
d9b57aa3 JA	82	.issue = io_write,
d9b57aa3 JA	83	},
	84	[IORING_OP_FSYNC] = {
	85	.needs_file = 1,
	86	.audit_skip = 1,
d9b57aa3 JA	87	.prep = io_fsync_prep,
	88	.issue = io_fsync,
	89	},
	90	[IORING_OP_READ_FIXED] = {
	91	.needs_file = 1,
	92	.unbound_nonreg_file = 1,
	93	.pollin = 1,
	94	.plug = 1,
	95	.audit_skip = 1,
	96	.ioprio = 1,
	97	.iopoll = 1,
ef0ec1ad	98	.iopoll_queue = 1,
d9b57aa3 JA	99	.prep = io_prep_rw,
	100	.issue = io_read,
	101	},
	102	[IORING_OP_WRITE_FIXED] = {
	103	.needs_file = 1,
	104	.hash_reg_file = 1,
	105	.unbound_nonreg_file = 1,
	106	.pollout = 1,
	107	.plug = 1,
	108	.audit_skip = 1,
	109	.ioprio = 1,
	110	.iopoll = 1,
ef0ec1ad	111	.iopoll_queue = 1,
d9b57aa3 JA	112	.prep = io_prep_rw,
	113	.issue = io_write,
	114	},
	115	[IORING_OP_POLL_ADD] = {
	116	.needs_file = 1,
	117	.unbound_nonreg_file = 1,
	118	.audit_skip = 1,
d9b57aa3 JA	119	.prep = io_poll_add_prep,
	120	.issue = io_poll_add,
	121	},
	122	[IORING_OP_POLL_REMOVE] = {
	123	.audit_skip = 1,
d9b57aa3 JA	124	.prep = io_poll_remove_prep,
	125	.issue = io_poll_remove,
	126	},
	127	[IORING_OP_SYNC_FILE_RANGE] = {
	128	.needs_file = 1,
	129	.audit_skip = 1,
d9b57aa3 JA	130	.prep = io_sfr_prep,
	131	.issue = io_sync_file_range,
	132	},
	133	[IORING_OP_SENDMSG] = {
	134	.needs_file = 1,
	135	.unbound_nonreg_file = 1,
	136	.pollout = 1,
	137	.ioprio = 1,
858c293e	138	.manual_alloc = 1,
d9b57aa3	139	#if defined(CONFIG_NET)
d9b57aa3 JA	140	.prep = io_sendmsg_prep,
d9b57aa3 JA	141	.issue = io_sendmsg,
d9b57aa3 JA	142	#else
	143	.prep = io_eopnotsupp_prep,
	144	#endif
	145	},
	146	[IORING_OP_RECVMSG] = {
	147	.needs_file = 1,
	148	.unbound_nonreg_file = 1,
	149	.pollin = 1,
	150	.buffer_select = 1,
	151	.ioprio = 1,
858c293e	152	.manual_alloc = 1,
d9b57aa3	153	#if defined(CONFIG_NET)
d9b57aa3 JA	154	.prep = io_recvmsg_prep,
d9b57aa3 JA	155	.issue = io_recvmsg,
d9b57aa3 JA	156	#else
	157	.prep = io_eopnotsupp_prep,
	158	#endif
	159	},
	160	[IORING_OP_TIMEOUT] = {
	161	.audit_skip = 1,
d9b57aa3 JA	162	.prep = io_timeout_prep,
	163	.issue = io_timeout,
	164	},
	165	[IORING_OP_TIMEOUT_REMOVE] = {
	166	/* used by timeout updates' prep() */
	167	.audit_skip = 1,
d9b57aa3 JA	168	.prep = io_timeout_remove_prep,
	169	.issue = io_timeout_remove,
	170	},
	171	[IORING_OP_ACCEPT] = {
	172	.needs_file = 1,
	173	.unbound_nonreg_file = 1,
	174	.pollin = 1,
	175	.poll_exclusive = 1,
	176	.ioprio = 1, /* used for flags */
d9b57aa3 JA	177	#if defined(CONFIG_NET)
	178	.prep = io_accept_prep,
	179	.issue = io_accept,
	180	#else
	181	.prep = io_eopnotsupp_prep,
	182	#endif
	183	},
	184	[IORING_OP_ASYNC_CANCEL] = {
	185	.audit_skip = 1,
d9b57aa3 JA	186	.prep = io_async_cancel_prep,
	187	.issue = io_async_cancel,
	188	},
	189	[IORING_OP_LINK_TIMEOUT] = {
	190	.audit_skip = 1,
d9b57aa3 JA	191	.prep = io_link_timeout_prep,
	192	.issue = io_no_issue,
	193	},
	194	[IORING_OP_CONNECT] = {
	195	.needs_file = 1,
	196	.unbound_nonreg_file = 1,
	197	.pollout = 1,
d9b57aa3	198	#if defined(CONFIG_NET)
d9b57aa3 JA	199	.prep = io_connect_prep,
d9b57aa3 JA	200	.issue = io_connect,
d9b57aa3 JA	201	#else
	202	.prep = io_eopnotsupp_prep,
	203	#endif
	204	},
	205	[IORING_OP_FALLOCATE] = {
	206	.needs_file = 1,
d9b57aa3 JA	207	.prep = io_fallocate_prep,
	208	.issue = io_fallocate,
	209	},
	210	[IORING_OP_OPENAT] = {
d9b57aa3 JA	211	.prep = io_openat_prep,
d9b57aa3 JA	212	.issue = io_openat,
d9b57aa3 JA	213	},
d9b57aa3 JA	214	[IORING_OP_CLOSE] = {
d9b57aa3 JA	215	.prep = io_close_prep,
	216	.issue = io_close,
	217	},
d9808ceb	218	[IORING_OP_FILES_UPDATE] = {
d9b57aa3 JA	219	.audit_skip = 1,
d9b57aa3 JA	220	.iopoll = 1,
d9808ceb PB	221	.prep = io_files_update_prep,
d9808ceb PB	222	.issue = io_files_update,
d9b57aa3 JA	223	},
	224	[IORING_OP_STATX] = {
	225	.audit_skip = 1,
d9b57aa3 JA	226	.prep = io_statx_prep,
d9b57aa3 JA	227	.issue = io_statx,
d9b57aa3 JA	228	},
	229	[IORING_OP_READ] = {
	230	.needs_file = 1,
	231	.unbound_nonreg_file = 1,
	232	.pollin = 1,
	233	.buffer_select = 1,
	234	.plug = 1,
	235	.audit_skip = 1,
	236	.ioprio = 1,
	237	.iopoll = 1,
ef0ec1ad	238	.iopoll_queue = 1,
d9b57aa3 JA	239	.prep = io_prep_rw,
	240	.issue = io_read,
	241	},
	242	[IORING_OP_WRITE] = {
	243	.needs_file = 1,
	244	.hash_reg_file = 1,
	245	.unbound_nonreg_file = 1,
	246	.pollout = 1,
	247	.plug = 1,
	248	.audit_skip = 1,
	249	.ioprio = 1,
	250	.iopoll = 1,
ef0ec1ad	251	.iopoll_queue = 1,
d9b57aa3 JA	252	.prep = io_prep_rw,
	253	.issue = io_write,
	254	},
	255	[IORING_OP_FADVISE] = {
	256	.needs_file = 1,
	257	.audit_skip = 1,
d9b57aa3 JA	258	.prep = io_fadvise_prep,
	259	.issue = io_fadvise,
	260	},
	261	[IORING_OP_MADVISE] = {
fbe870a7	262	.audit_skip = 1,
d9b57aa3 JA	263	.prep = io_madvise_prep,
	264	.issue = io_madvise,
	265	},
	266	[IORING_OP_SEND] = {
	267	.needs_file = 1,
	268	.unbound_nonreg_file = 1,
	269	.pollout = 1,
	270	.audit_skip = 1,
	271	.ioprio = 1,
516e82f0	272	.manual_alloc = 1,
d9b57aa3 JA	273	#if defined(CONFIG_NET)
	274	.prep = io_sendmsg_prep,
	275	.issue = io_send,
	276	#else
	277	.prep = io_eopnotsupp_prep,
	278	#endif
	279	},
	280	[IORING_OP_RECV] = {
	281	.needs_file = 1,
	282	.unbound_nonreg_file = 1,
	283	.pollin = 1,
	284	.buffer_select = 1,
	285	.audit_skip = 1,
	286	.ioprio = 1,
d9b57aa3 JA	287	#if defined(CONFIG_NET)
	288	.prep = io_recvmsg_prep,
	289	.issue = io_recv,
	290	#else
	291	.prep = io_eopnotsupp_prep,
	292	#endif
	293	},
	294	[IORING_OP_OPENAT2] = {
d9b57aa3 JA	295	.prep = io_openat2_prep,
d9b57aa3 JA	296	.issue = io_openat2,
d9b57aa3 JA	297	},
	298	[IORING_OP_EPOLL_CTL] = {
	299	.unbound_nonreg_file = 1,
	300	.audit_skip = 1,
d9b57aa3 JA	301	#if defined(CONFIG_EPOLL)
	302	.prep = io_epoll_ctl_prep,
	303	.issue = io_epoll_ctl,
	304	#else
	305	.prep = io_eopnotsupp_prep,
	306	#endif
	307	},
	308	[IORING_OP_SPLICE] = {
	309	.needs_file = 1,
	310	.hash_reg_file = 1,
	311	.unbound_nonreg_file = 1,
	312	.audit_skip = 1,
d9b57aa3 JA	313	.prep = io_splice_prep,
	314	.issue = io_splice,
	315	},
	316	[IORING_OP_PROVIDE_BUFFERS] = {
	317	.audit_skip = 1,
	318	.iopoll = 1,
d9b57aa3 JA	319	.prep = io_provide_buffers_prep,
	320	.issue = io_provide_buffers,
	321	},
	322	[IORING_OP_REMOVE_BUFFERS] = {
	323	.audit_skip = 1,
	324	.iopoll = 1,
d9b57aa3 JA	325	.prep = io_remove_buffers_prep,
	326	.issue = io_remove_buffers,
	327	},
	328	[IORING_OP_TEE] = {
	329	.needs_file = 1,
	330	.hash_reg_file = 1,
	331	.unbound_nonreg_file = 1,
	332	.audit_skip = 1,
d9b57aa3 JA	333	.prep = io_tee_prep,
	334	.issue = io_tee,
	335	},
	336	[IORING_OP_SHUTDOWN] = {
	337	.needs_file = 1,
d9b57aa3 JA	338	#if defined(CONFIG_NET)
	339	.prep = io_shutdown_prep,
	340	.issue = io_shutdown,
	341	#else
	342	.prep = io_eopnotsupp_prep,
	343	#endif
	344	},
	345	[IORING_OP_RENAMEAT] = {
d9b57aa3 JA	346	.prep = io_renameat_prep,
d9b57aa3 JA	347	.issue = io_renameat,
d9b57aa3 JA	348	},
d9b57aa3 JA	349	[IORING_OP_UNLINKAT] = {
d9b57aa3 JA	350	.prep = io_unlinkat_prep,
d9b57aa3 JA	351	.issue = io_unlinkat,
d9b57aa3 JA	352	},
d9b57aa3 JA	353	[IORING_OP_MKDIRAT] = {
d9b57aa3 JA	354	.prep = io_mkdirat_prep,
d9b57aa3 JA	355	.issue = io_mkdirat,
d9b57aa3 JA	356	},
d9b57aa3 JA	357	[IORING_OP_SYMLINKAT] = {
d9b57aa3 JA	358	.prep = io_symlinkat_prep,
d9b57aa3 JA	359	.issue = io_symlinkat,
d9b57aa3 JA	360	},
d9b57aa3 JA	361	[IORING_OP_LINKAT] = {
d9b57aa3 JA	362	.prep = io_linkat_prep,
d9b57aa3 JA	363	.issue = io_linkat,
d9b57aa3 JA	364	},
	365	[IORING_OP_MSG_RING] = {
	366	.needs_file = 1,
	367	.iopoll = 1,
d9b57aa3 JA	368	.prep = io_msg_ring_prep,
	369	.issue = io_msg_ring,
	370	},
	371	[IORING_OP_FSETXATTR] = {
	372	.needs_file = 1,
d9b57aa3 JA	373	.prep = io_fsetxattr_prep,
d9b57aa3 JA	374	.issue = io_fsetxattr,
d9b57aa3 JA	375	},
d9b57aa3 JA	376	[IORING_OP_SETXATTR] = {
d9b57aa3 JA	377	.prep = io_setxattr_prep,
d9b57aa3 JA	378	.issue = io_setxattr,
d9b57aa3 JA	379	},
	380	[IORING_OP_FGETXATTR] = {
	381	.needs_file = 1,
d9b57aa3 JA	382	.prep = io_fgetxattr_prep,
d9b57aa3 JA	383	.issue = io_fgetxattr,
d9b57aa3 JA	384	},
d9b57aa3 JA	385	[IORING_OP_GETXATTR] = {
d9b57aa3 JA	386	.prep = io_getxattr_prep,
d9b57aa3 JA	387	.issue = io_getxattr,
d9b57aa3 JA	388	},
	389	[IORING_OP_SOCKET] = {
	390	.audit_skip = 1,
d9b57aa3 JA	391	#if defined(CONFIG_NET)
	392	.prep = io_socket_prep,
	393	.issue = io_socket,
	394	#else
	395	.prep = io_eopnotsupp_prep,
	396	#endif
	397	},
	398	[IORING_OP_URING_CMD] = {
	399	.needs_file = 1,
	400	.plug = 1,
5756a3a7	401	.iopoll = 1,
ef0ec1ad	402	.iopoll_queue = 1,
d9b57aa3 JA	403	.prep = io_uring_cmd_prep,
d9b57aa3 JA	404	.issue = io_uring_cmd,
d9b57aa3	405	},
b48c312b	406	[IORING_OP_SEND_ZC] = {
06a5464b PB	407	.needs_file = 1,
	408	.unbound_nonreg_file = 1,
	409	.pollout = 1,
	410	.audit_skip = 1,
	411	.ioprio = 1,
581711c4	412	.manual_alloc = 1,
06a5464b	413	#if defined(CONFIG_NET)
b0e9b551 PB	414	.prep = io_send_zc_prep,
b0e9b551 PB	415	.issue = io_send_zc,
06a5464b PB	416	#else
06a5464b PB	417	.prep = io_eopnotsupp_prep,
493108d9 PB	418	#endif
	419	},
	420	[IORING_OP_SENDMSG_ZC] = {
493108d9 PB	421	.needs_file = 1,
	422	.unbound_nonreg_file = 1,
	423	.pollout = 1,
493108d9 PB	424	.ioprio = 1,
	425	.manual_alloc = 1,
	426	#if defined(CONFIG_NET)
493108d9 PB	427	.prep = io_send_zc_prep,
493108d9 PB	428	.issue = io_sendmsg_zc,
f30bd4d0 BL	429	#else
	430	.prep = io_eopnotsupp_prep,
	431	#endif
	432	},
fc68fcda JA	433	[IORING_OP_READ_MULTISHOT] = {
	434	.needs_file = 1,
	435	.unbound_nonreg_file = 1,
	436	.pollin = 1,
	437	.buffer_select = 1,
	438	.audit_skip = 1,
	439	.prep = io_read_mshot_prep,
	440	.issue = io_read_mshot,
	441	},
f30bd4d0 BL	442	};
f30bd4d0 BL	443
f30bd4d0 BL	444	const struct io_cold_def io_cold_defs[] = {
	445	[IORING_OP_NOP] = {
	446	.name = "NOP",
	447	},
	448	[IORING_OP_READV] = {
	449	.async_size = sizeof(struct io_async_rw),
	450	.name = "READV",
	451	.prep_async = io_readv_prep_async,
	452	.cleanup = io_readv_writev_cleanup,
	453	.fail = io_rw_fail,
	454	},
	455	[IORING_OP_WRITEV] = {
	456	.async_size = sizeof(struct io_async_rw),
	457	.name = "WRITEV",
	458	.prep_async = io_writev_prep_async,
	459	.cleanup = io_readv_writev_cleanup,
	460	.fail = io_rw_fail,
	461	},
	462	[IORING_OP_FSYNC] = {
	463	.name = "FSYNC",
	464	},
	465	[IORING_OP_READ_FIXED] = {
	466	.async_size = sizeof(struct io_async_rw),
	467	.name = "READ_FIXED",
	468	.fail = io_rw_fail,
	469	},
	470	[IORING_OP_WRITE_FIXED] = {
	471	.async_size = sizeof(struct io_async_rw),
	472	.name = "WRITE_FIXED",
	473	.fail = io_rw_fail,
	474	},
	475	[IORING_OP_POLL_ADD] = {
	476	.name = "POLL_ADD",
	477	},
	478	[IORING_OP_POLL_REMOVE] = {
	479	.name = "POLL_REMOVE",
	480	},
	481	[IORING_OP_SYNC_FILE_RANGE] = {
	482	.name = "SYNC_FILE_RANGE",
	483	},
	484	[IORING_OP_SENDMSG] = {
	485	.name = "SENDMSG",
	486	#if defined(CONFIG_NET)
	487	.async_size = sizeof(struct io_async_msghdr),
	488	.prep_async = io_sendmsg_prep_async,
	489	.cleanup = io_sendmsg_recvmsg_cleanup,
	490	.fail = io_sendrecv_fail,
	491	#endif
	492	},
	493	[IORING_OP_RECVMSG] = {
	494	.name = "RECVMSG",
	495	#if defined(CONFIG_NET)
	496	.async_size = sizeof(struct io_async_msghdr),
	497	.prep_async = io_recvmsg_prep_async,
	498	.cleanup = io_sendmsg_recvmsg_cleanup,
	499	.fail = io_sendrecv_fail,
	500	#endif
	501	},
	502	[IORING_OP_TIMEOUT] = {
	503	.async_size = sizeof(struct io_timeout_data),
	504	.name = "TIMEOUT",
	505	},
	506	[IORING_OP_TIMEOUT_REMOVE] = {
	507	.name = "TIMEOUT_REMOVE",
508	},
509	[IORING_OP_ACCEPT] = {
510	.name = "ACCEPT",
511	},
512	[IORING_OP_ASYNC_CANCEL] = {
513	.name = "ASYNC_CANCEL",
514	},
515	[IORING_OP_LINK_TIMEOUT] = {
516	.async_size = sizeof(struct io_timeout_data),
517	.name = "LINK_TIMEOUT",
518	},
519	[IORING_OP_CONNECT] = {
520	.name = "CONNECT",
521	#if defined(CONFIG_NET)
522	.async_size = sizeof(struct io_async_connect),
523	.prep_async = io_connect_prep_async,
524	#endif
525	},
526	[IORING_OP_FALLOCATE] = {
527	.name = "FALLOCATE",
528	},
529	[IORING_OP_OPENAT] = {
530	.name = "OPENAT",
531	.cleanup = io_open_cleanup,
532	},
533	[IORING_OP_CLOSE] = {
534	.name = "CLOSE",
535	},
536	[IORING_OP_FILES_UPDATE] = {
537	.name = "FILES_UPDATE",
538	},
539	[IORING_OP_STATX] = {
540	.name = "STATX",
541	.cleanup = io_statx_cleanup,
542	},
543	[IORING_OP_READ] = {
544	.async_size = sizeof(struct io_async_rw),
545	.name = "READ",
546	.fail = io_rw_fail,
547	},
548	[IORING_OP_WRITE] = {
549	.async_size = sizeof(struct io_async_rw),
550	.name = "WRITE",
551	.fail = io_rw_fail,
552	},
553	[IORING_OP_FADVISE] = {
554	.name = "FADVISE",
555	},
556	[IORING_OP_MADVISE] = {
557	.name = "MADVISE",
558	},
559	[IORING_OP_SEND] = {
560	.name = "SEND",
561	#if defined(CONFIG_NET)
562	.async_size = sizeof(struct io_async_msghdr),
563	.fail = io_sendrecv_fail,
564	.prep_async = io_send_prep_async,
565	#endif
566	},
567	[IORING_OP_RECV] = {
568	.name = "RECV",
569	#if defined(CONFIG_NET)
570	.fail = io_sendrecv_fail,
571	#endif
572	},
573	[IORING_OP_OPENAT2] = {
574	.name = "OPENAT2",
575	.cleanup = io_open_cleanup,
576	},
577	[IORING_OP_EPOLL_CTL] = {
578	.name = "EPOLL",
579	},
580	[IORING_OP_SPLICE] = {
581	.name = "SPLICE",
582	},
583	[IORING_OP_PROVIDE_BUFFERS] = {
584	.name = "PROVIDE_BUFFERS",
585	},
586	[IORING_OP_REMOVE_BUFFERS] = {
587	.name = "REMOVE_BUFFERS",
588	},
589	[IORING_OP_TEE] = {
590	.name = "TEE",
591	},
592	[IORING_OP_SHUTDOWN] = {
593	.name = "SHUTDOWN",
594	},
595	[IORING_OP_RENAMEAT] = {
596	.name = "RENAMEAT",
597	.cleanup = io_renameat_cleanup,
598	},
599	[IORING_OP_UNLINKAT] = {
600	.name = "UNLINKAT",
601	.cleanup = io_unlinkat_cleanup,
602	},
603	[IORING_OP_MKDIRAT] = {
604	.name = "MKDIRAT",
605	.cleanup = io_mkdirat_cleanup,
606	},
607	[IORING_OP_SYMLINKAT] = {
608	.name = "SYMLINKAT",
609	.cleanup = io_link_cleanup,
610	},
611	[IORING_OP_LINKAT] = {
612	.name = "LINKAT",
613	.cleanup = io_link_cleanup,
614	},
615	[IORING_OP_MSG_RING] = {
616	.name = "MSG_RING",
617	.cleanup = io_msg_ring_cleanup,
618	},
619	[IORING_OP_FSETXATTR] = {
620	.name = "FSETXATTR",
621	.cleanup = io_xattr_cleanup,
622	},
623	[IORING_OP_SETXATTR] = {
624	.name = "SETXATTR",
625	.cleanup = io_xattr_cleanup,
626	},
627	[IORING_OP_FGETXATTR] = {
628	.name = "FGETXATTR",
629	.cleanup = io_xattr_cleanup,
630	},
631	[IORING_OP_GETXATTR] = {
632	.name = "GETXATTR",
633	.cleanup = io_xattr_cleanup,
634	},
635	[IORING_OP_SOCKET] = {
636	.name = "SOCKET",
637	},
638	[IORING_OP_URING_CMD] = {
639	.name = "URING_CMD",
fd9b8547	640	.async_size = 2 * sizeof(struct io_uring_sqe),
f30bd4d0 BL	641	.prep_async = io_uring_cmd_prep_async,
	642	},
	643	[IORING_OP_SEND_ZC] = {
	644	.name = "SEND_ZC",
	645	#if defined(CONFIG_NET)
	646	.async_size = sizeof(struct io_async_msghdr),
	647	.prep_async = io_send_prep_async,
	648	.cleanup = io_send_zc_cleanup,
	649	.fail = io_sendrecv_fail,
	650	#endif
	651	},
	652	[IORING_OP_SENDMSG_ZC] = {
	653	.name = "SENDMSG_ZC",
	654	#if defined(CONFIG_NET)
	655	.async_size = sizeof(struct io_async_msghdr),
493108d9 PB	656	.prep_async = io_sendmsg_prep_async,
	657	.cleanup = io_send_zc_cleanup,
	658	.fail = io_sendrecv_fail,
06a5464b	659	#endif
06a5464b	660	},
fc68fcda JA	661	[IORING_OP_READ_MULTISHOT] = {
	662	.name = "READ_MULTISHOT",
	663	},
d9b57aa3 JA	664	};
	665
	666	const char *io_uring_get_opcode(u8 opcode)
	667	{
	668	if (opcode < IORING_OP_LAST)
f30bd4d0	669	return io_cold_defs[opcode].name;
d9b57aa3 JA	670	return "INVALID";
	671	}
	672
	673	void __init io_uring_optable_init(void)
	674	{
	675	int i;
	676
f30bd4d0	677	BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);
a7dd2782	678	BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);
d9b57aa3	679
a7dd2782 BL	680	for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
	681	BUG_ON(!io_issue_defs[i].prep);
	682	if (io_issue_defs[i].prep != io_eopnotsupp_prep)
	683	BUG_ON(!io_issue_defs[i].issue);
f30bd4d0	684	WARN_ON_ONCE(!io_cold_defs[i].name);
d9b57aa3 JA	685	}
d9b57aa3 JA	686	}