[linux-block.git] / io_uring / opdef.c

// SPDX-License-Identifier: GPL-2.0
/*
 * io_uring opcode handling table
 */
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/io_uring.h>

#include "io_uring.h"
#include "opdef.h"
#include "refs.h"
#include "tctx.h"
#include "sqpoll.h"
#include "fdinfo.h"
#include "kbuf.h"
#include "rsrc.h"

#include "xattr.h"
#include "nop.h"
#include "fs.h"
#include "splice.h"
#include "sync.h"
#include "advise.h"
#include "openclose.h"
#include "uring_cmd.h"
#include "epoll.h"
#include "statx.h"
#include "net.h"
#include "msg_ring.h"
#include "timeout.h"
#include "poll.h"
#include "cancel.h"
#include "rw.h"
#include "waitid.h"
#include "futex.h"

static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
{
	WARN_ON_ONCE(1);
	return -ECANCELED;
}

static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
					     const struct io_uring_sqe *sqe)
{
	return -EOPNOTSUPP;
}

const struct io_issue_def io_issue_defs[] = {
	[IORING_OP_NOP] = {
		.audit_skip		= 1,
		.iopoll			= 1,
		.prep			= io_nop_prep,
		.issue			= io_nop,
	},
	[IORING_OP_READV] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.buffer_select		= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.vectored		= 1,
		.prep			= io_prep_rwv,
		.issue			= io_read,
	},
	[IORING_OP_WRITEV] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.vectored		= 1,
		.prep			= io_prep_rwv,
		.issue			= io_write,
	},
	[IORING_OP_FSYNC] = {
		.needs_file		= 1,
		.audit_skip		= 1,
		.prep			= io_fsync_prep,
		.issue			= io_fsync,
	},
	[IORING_OP_READ_FIXED] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.prep			= io_prep_rw_fixed,
		.issue			= io_read,
	},
	[IORING_OP_WRITE_FIXED] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.prep			= io_prep_rw_fixed,
		.issue			= io_write,
	},
	[IORING_OP_POLL_ADD] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.audit_skip		= 1,
		.prep			= io_poll_add_prep,
		.issue			= io_poll_add,
	},
	[IORING_OP_POLL_REMOVE] = {
		.audit_skip		= 1,
		.prep			= io_poll_remove_prep,
		.issue			= io_poll_remove,
	},
	[IORING_OP_SYNC_FILE_RANGE] = {
		.needs_file		= 1,
		.audit_skip		= 1,
		.prep			= io_sfr_prep,
		.issue			= io_sync_file_range,
	},
	[IORING_OP_SENDMSG] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.ioprio			= 1,
		.manual_alloc		= 1,
#if defined(CONFIG_NET)
		.prep			= io_sendmsg_prep,
		.issue			= io_sendmsg,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_RECVMSG] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.buffer_select		= 1,
		.ioprio			= 1,
		.manual_alloc		= 1,
#if defined(CONFIG_NET)
		.prep			= io_recvmsg_prep,
		.issue			= io_recvmsg,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_TIMEOUT] = {
		.audit_skip		= 1,
		.prep			= io_timeout_prep,
		.issue			= io_timeout,
	},
	[IORING_OP_TIMEOUT_REMOVE] = {
		/* used by timeout updates' prep() */
		.audit_skip		= 1,
		.prep			= io_timeout_remove_prep,
		.issue			= io_timeout_remove,
	},
	[IORING_OP_ACCEPT] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.poll_exclusive		= 1,
		.ioprio			= 1,	/* used for flags */
#if defined(CONFIG_NET)
		.prep			= io_accept_prep,
		.issue			= io_accept,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_ASYNC_CANCEL] = {
		.audit_skip		= 1,
		.prep			= io_async_cancel_prep,
		.issue			= io_async_cancel,
	},
	[IORING_OP_LINK_TIMEOUT] = {
		.audit_skip		= 1,
		.prep			= io_link_timeout_prep,
		.issue			= io_no_issue,
	},
	[IORING_OP_CONNECT] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
#if defined(CONFIG_NET)
		.prep			= io_connect_prep,
		.issue			= io_connect,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_FALLOCATE] = {
		.needs_file		= 1,
		.prep			= io_fallocate_prep,
		.issue			= io_fallocate,
	},
	[IORING_OP_OPENAT] = {
		.prep			= io_openat_prep,
		.issue			= io_openat,
	},
	[IORING_OP_CLOSE] = {
		.prep			= io_close_prep,
		.issue			= io_close,
	},
	[IORING_OP_FILES_UPDATE] = {
		.audit_skip		= 1,
		.iopoll			= 1,
		.prep			= io_files_update_prep,
		.issue			= io_files_update,
	},
	[IORING_OP_STATX] = {
		.audit_skip		= 1,
		.prep			= io_statx_prep,
		.issue			= io_statx,
	},
	[IORING_OP_READ] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.buffer_select		= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.prep			= io_prep_rw,
		.issue			= io_read,
	},
	[IORING_OP_WRITE] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.plug			= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.prep			= io_prep_rw,
		.issue			= io_write,
	},
	[IORING_OP_FADVISE] = {
		.needs_file		= 1,
		.audit_skip		= 1,
		.prep			= io_fadvise_prep,
		.issue			= io_fadvise,
	},
	[IORING_OP_MADVISE] = {
		.audit_skip		= 1,
		.prep			= io_madvise_prep,
		.issue			= io_madvise,
	},
	[IORING_OP_SEND] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.manual_alloc		= 1,
#if defined(CONFIG_NET)
		.prep			= io_sendmsg_prep,
		.issue			= io_send,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_RECV] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.buffer_select		= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
#if defined(CONFIG_NET)
		.prep			= io_recvmsg_prep,
		.issue			= io_recv,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_OPENAT2] = {
		.prep			= io_openat2_prep,
		.issue			= io_openat2,
	},
	[IORING_OP_EPOLL_CTL] = {
		.unbound_nonreg_file	= 1,
		.audit_skip		= 1,
#if defined(CONFIG_EPOLL)
		.prep			= io_epoll_ctl_prep,
		.issue			= io_epoll_ctl,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_SPLICE] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
		.audit_skip		= 1,
		.prep			= io_splice_prep,
		.issue			= io_splice,
	},
	[IORING_OP_PROVIDE_BUFFERS] = {
		.audit_skip		= 1,
		.iopoll			= 1,
		.prep			= io_provide_buffers_prep,
		.issue			= io_provide_buffers,
	},
	[IORING_OP_REMOVE_BUFFERS] = {
		.audit_skip		= 1,
		.iopoll			= 1,
		.prep			= io_remove_buffers_prep,
		.issue			= io_remove_buffers,
	},
	[IORING_OP_TEE] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
		.audit_skip		= 1,
		.prep			= io_tee_prep,
		.issue			= io_tee,
	},
	[IORING_OP_SHUTDOWN] = {
		.needs_file		= 1,
#if defined(CONFIG_NET)
		.prep			= io_shutdown_prep,
		.issue			= io_shutdown,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_RENAMEAT] = {
		.prep			= io_renameat_prep,
		.issue			= io_renameat,
	},
	[IORING_OP_UNLINKAT] = {
		.prep			= io_unlinkat_prep,
		.issue			= io_unlinkat,
	},
	[IORING_OP_MKDIRAT] = {
		.prep			= io_mkdirat_prep,
		.issue			= io_mkdirat,
	},
	[IORING_OP_SYMLINKAT] = {
		.prep			= io_symlinkat_prep,
		.issue			= io_symlinkat,
	},
	[IORING_OP_LINKAT] = {
		.prep			= io_linkat_prep,
		.issue			= io_linkat,
	},
	[IORING_OP_MSG_RING] = {
		.needs_file		= 1,
		.iopoll			= 1,
		.prep			= io_msg_ring_prep,
		.issue			= io_msg_ring,
	},
	[IORING_OP_FSETXATTR] = {
		.needs_file = 1,
		.prep			= io_fsetxattr_prep,
		.issue			= io_fsetxattr,
	},
	[IORING_OP_SETXATTR] = {
		.prep			= io_setxattr_prep,
		.issue			= io_setxattr,
	},
	[IORING_OP_FGETXATTR] = {
		.needs_file = 1,
		.prep			= io_fgetxattr_prep,
		.issue			= io_fgetxattr,
	},
	[IORING_OP_GETXATTR] = {
		.prep			= io_getxattr_prep,
		.issue			= io_getxattr,
	},
	[IORING_OP_SOCKET] = {
		.audit_skip		= 1,
#if defined(CONFIG_NET)
		.prep			= io_socket_prep,
		.issue			= io_socket,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_URING_CMD] = {
		.needs_file		= 1,
		.plug			= 1,
		.iopoll			= 1,
		.iopoll_queue		= 1,
		.prep			= io_uring_cmd_prep,
		.issue			= io_uring_cmd,
	},
	[IORING_OP_SEND_ZC] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.audit_skip		= 1,
		.ioprio			= 1,
		.manual_alloc		= 1,
#if defined(CONFIG_NET)
		.prep			= io_send_zc_prep,
		.issue			= io_send_zc,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_SENDMSG_ZC] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollout		= 1,
		.ioprio			= 1,
		.manual_alloc		= 1,
#if defined(CONFIG_NET)
		.prep			= io_send_zc_prep,
		.issue			= io_sendmsg_zc,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_READ_MULTISHOT] = {
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
		.pollin			= 1,
		.buffer_select		= 1,
		.audit_skip		= 1,
		.prep			= io_read_mshot_prep,
		.issue			= io_read_mshot,
	},
	[IORING_OP_WAITID] = {
		.prep			= io_waitid_prep,
		.issue			= io_waitid,
	},
	[IORING_OP_FUTEX_WAIT] = {
#if defined(CONFIG_FUTEX)
		.prep			= io_futex_prep,
		.issue			= io_futex_wait,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_FUTEX_WAKE] = {
#if defined(CONFIG_FUTEX)
		.prep			= io_futex_prep,
		.issue			= io_futex_wake,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_FUTEX_WAITV] = {
#if defined(CONFIG_FUTEX)
		.prep			= io_futexv_prep,
		.issue			= io_futexv_wait,
#else
		.prep			= io_eopnotsupp_prep,
#endif
	},
	[IORING_OP_FIXED_FD_INSTALL] = {
		.needs_file		= 1,
		.audit_skip		= 1,
		.prep			= io_install_fixed_fd_prep,
		.issue			= io_install_fixed_fd,
	},
};

const struct io_cold_def io_cold_defs[] = {
	[IORING_OP_NOP] = {
		.name			= "NOP",
	},
	[IORING_OP_READV] = {
		.async_size		= sizeof(struct io_async_rw),
		.name			= "READV",
		.prep_async		= io_readv_prep_async,
		.cleanup		= io_readv_writev_cleanup,
		.fail			= io_rw_fail,
	},
	[IORING_OP_WRITEV] = {
		.async_size		= sizeof(struct io_async_rw),
		.name			= "WRITEV",
		.prep_async		= io_writev_prep_async,
		.cleanup		= io_readv_writev_cleanup,
		.fail			= io_rw_fail,
	},
	[IORING_OP_FSYNC] = {
		.name			= "FSYNC",
	},
	[IORING_OP_READ_FIXED] = {
		.async_size		= sizeof(struct io_async_rw),
		.name			= "READ_FIXED",
		.fail			= io_rw_fail,
	},
	[IORING_OP_WRITE_FIXED] = {
		.async_size		= sizeof(struct io_async_rw),
		.name			= "WRITE_FIXED",
		.fail			= io_rw_fail,
	},
	[IORING_OP_POLL_ADD] = {
		.name			= "POLL_ADD",
	},
	[IORING_OP_POLL_REMOVE] = {
		.name			= "POLL_REMOVE",
	},
	[IORING_OP_SYNC_FILE_RANGE] = {
		.name			= "SYNC_FILE_RANGE",
	},
	[IORING_OP_SENDMSG] = {
		.name			= "SENDMSG",
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_msghdr),
		.prep_async		= io_sendmsg_prep_async,
		.cleanup		= io_sendmsg_recvmsg_cleanup,
		.fail			= io_sendrecv_fail,
#endif
	},
	[IORING_OP_RECVMSG] = {
		.name			= "RECVMSG",
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_msghdr),
		.prep_async		= io_recvmsg_prep_async,
		.cleanup		= io_sendmsg_recvmsg_cleanup,
		.fail			= io_sendrecv_fail,
#endif
	},
	[IORING_OP_TIMEOUT] = {
		.async_size		= sizeof(struct io_timeout_data),
		.name			= "TIMEOUT",
	},
	[IORING_OP_TIMEOUT_REMOVE] = {
		.name			= "TIMEOUT_REMOVE",
	},
	[IORING_OP_ACCEPT] = {
		.name			= "ACCEPT",
	},
	[IORING_OP_ASYNC_CANCEL] = {
		.name			= "ASYNC_CANCEL",
	},
	[IORING_OP_LINK_TIMEOUT] = {
		.async_size		= sizeof(struct io_timeout_data),
		.name			= "LINK_TIMEOUT",
	},
	[IORING_OP_CONNECT] = {
		.name			= "CONNECT",
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_connect),
		.prep_async		= io_connect_prep_async,
#endif
	},
	[IORING_OP_FALLOCATE] = {
		.name			= "FALLOCATE",
	},
	[IORING_OP_OPENAT] = {
		.name			= "OPENAT",
		.cleanup		= io_open_cleanup,
	},
	[IORING_OP_CLOSE] = {
		.name			= "CLOSE",
	},
	[IORING_OP_FILES_UPDATE] = {
		.name			= "FILES_UPDATE",
	},
	[IORING_OP_STATX] = {
		.name			= "STATX",
		.cleanup		= io_statx_cleanup,
	},
	[IORING_OP_READ] = {
		.async_size		= sizeof(struct io_async_rw),
		.name			= "READ",
		.fail			= io_rw_fail,
	},
	[IORING_OP_WRITE] = {
		.async_size		= sizeof(struct io_async_rw),
		.name			= "WRITE",
		.fail			= io_rw_fail,
	},
	[IORING_OP_FADVISE] = {
		.name			= "FADVISE",
	},
	[IORING_OP_MADVISE] = {
		.name			= "MADVISE",
	},
	[IORING_OP_SEND] = {
		.name			= "SEND",
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_msghdr),
		.fail			= io_sendrecv_fail,
		.prep_async		= io_send_prep_async,
#endif
	},
	[IORING_OP_RECV] = {
		.name			= "RECV",
#if defined(CONFIG_NET)
		.fail			= io_sendrecv_fail,
#endif
	},
	[IORING_OP_OPENAT2] = {
		.name			= "OPENAT2",
		.cleanup		= io_open_cleanup,
	},
	[IORING_OP_EPOLL_CTL] = {
		.name			= "EPOLL",
	},
	[IORING_OP_SPLICE] = {
		.name			= "SPLICE",
	},
	[IORING_OP_PROVIDE_BUFFERS] = {
		.name			= "PROVIDE_BUFFERS",
	},
	[IORING_OP_REMOVE_BUFFERS] = {
		.name			= "REMOVE_BUFFERS",
	},
	[IORING_OP_TEE] = {
		.name			= "TEE",
	},
	[IORING_OP_SHUTDOWN] = {
		.name			= "SHUTDOWN",
	},
	[IORING_OP_RENAMEAT] = {
		.name			= "RENAMEAT",
		.cleanup		= io_renameat_cleanup,
	},
	[IORING_OP_UNLINKAT] = {
		.name			= "UNLINKAT",
		.cleanup		= io_unlinkat_cleanup,
	},
	[IORING_OP_MKDIRAT] = {
		.name			= "MKDIRAT",
		.cleanup		= io_mkdirat_cleanup,
	},
	[IORING_OP_SYMLINKAT] = {
		.name			= "SYMLINKAT",
		.cleanup		= io_link_cleanup,
	},
	[IORING_OP_LINKAT] = {
		.name			= "LINKAT",
		.cleanup		= io_link_cleanup,
	},
	[IORING_OP_MSG_RING] = {
		.name			= "MSG_RING",
		.cleanup		= io_msg_ring_cleanup,
	},
	[IORING_OP_FSETXATTR] = {
		.name			= "FSETXATTR",
		.cleanup		= io_xattr_cleanup,
	},
	[IORING_OP_SETXATTR] = {
		.name			= "SETXATTR",
		.cleanup		= io_xattr_cleanup,
	},
	[IORING_OP_FGETXATTR] = {
		.name			= "FGETXATTR",
		.cleanup		= io_xattr_cleanup,
	},
	[IORING_OP_GETXATTR] = {
		.name			= "GETXATTR",
		.cleanup		= io_xattr_cleanup,
	},
	[IORING_OP_SOCKET] = {
		.name			= "SOCKET",
	},
	[IORING_OP_URING_CMD] = {
		.name			= "URING_CMD",
		.async_size		= 2 * sizeof(struct io_uring_sqe),
		.prep_async		= io_uring_cmd_prep_async,
	},
	[IORING_OP_SEND_ZC] = {
		.name			= "SEND_ZC",
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_msghdr),
		.prep_async		= io_send_prep_async,
		.cleanup		= io_send_zc_cleanup,
		.fail			= io_sendrecv_fail,
#endif
	},
	[IORING_OP_SENDMSG_ZC] = {
		.name			= "SENDMSG_ZC",
#if defined(CONFIG_NET)
		.async_size		= sizeof(struct io_async_msghdr),
		.prep_async		= io_sendmsg_prep_async,
		.cleanup		= io_send_zc_cleanup,
		.fail			= io_sendrecv_fail,
#endif
	},
	[IORING_OP_READ_MULTISHOT] = {
		.name			= "READ_MULTISHOT",
	},
	[IORING_OP_WAITID] = {
		.name			= "WAITID",
		.async_size		= sizeof(struct io_waitid_async),
	},
	[IORING_OP_FUTEX_WAIT] = {
		.name			= "FUTEX_WAIT",
	},
	[IORING_OP_FUTEX_WAKE] = {
		.name			= "FUTEX_WAKE",
	},
	[IORING_OP_FUTEX_WAITV] = {
		.name			= "FUTEX_WAITV",
	},
	[IORING_OP_FIXED_FD_INSTALL] = {
		.name			= "FIXED_FD_INSTALL",
	},
};

const char *io_uring_get_opcode(u8 opcode)
{
	if (opcode < IORING_OP_LAST)
		return io_cold_defs[opcode].name;
	return "INVALID";
}

void __init io_uring_optable_init(void)
{
	int i;

	BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);
	BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);

	for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
		BUG_ON(!io_issue_defs[i].prep);
		if (io_issue_defs[i].prep != io_eopnotsupp_prep)
			BUG_ON(!io_issue_defs[i].issue);
		WARN_ON_ONCE(!io_cold_defs[i].name);
	}
}
Commit	Line	Data
d9b57aa3 JA	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* io_uring opcode handling table
	4	*/
	5	#include <linux/kernel.h>
	6	#include <linux/errno.h>
	7	#include <linux/fs.h>
	8	#include <linux/file.h>
	9	#include <linux/io_uring.h>
	10
d9b57aa3 JA	11	#include "io_uring.h"
	12	#include "opdef.h"
	13	#include "refs.h"
	14	#include "tctx.h"
	15	#include "sqpoll.h"
	16	#include "fdinfo.h"
	17	#include "kbuf.h"
	18	#include "rsrc.h"
	19
	20	#include "xattr.h"
	21	#include "nop.h"
	22	#include "fs.h"
	23	#include "splice.h"
	24	#include "sync.h"
	25	#include "advise.h"
	26	#include "openclose.h"
	27	#include "uring_cmd.h"
	28	#include "epoll.h"
	29	#include "statx.h"
	30	#include "net.h"
	31	#include "msg_ring.h"
	32	#include "timeout.h"
	33	#include "poll.h"
	34	#include "cancel.h"
	35	#include "rw.h"
f31ecf67	36	#include "waitid.h"
194bb58c	37	#include "futex.h"
d9b57aa3 JA	38
	39	static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
	40	{
	41	WARN_ON_ONCE(1);
	42	return -ECANCELED;
	43	}
	44
	45	static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
	46	const struct io_uring_sqe *sqe)
	47	{
	48	return -EOPNOTSUPP;
	49	}
	50
a7dd2782	51	const struct io_issue_def io_issue_defs[] = {
d9b57aa3 JA	52	[IORING_OP_NOP] = {
	53	.audit_skip = 1,
	54	.iopoll = 1,
d9b57aa3 JA	55	.prep = io_nop_prep,
	56	.issue = io_nop,
	57	},
	58	[IORING_OP_READV] = {
	59	.needs_file = 1,
	60	.unbound_nonreg_file = 1,
	61	.pollin = 1,
	62	.buffer_select = 1,
	63	.plug = 1,
	64	.audit_skip = 1,
	65	.ioprio = 1,
	66	.iopoll = 1,
ef0ec1ad	67	.iopoll_queue = 1,
d2d778fb	68	.vectored = 1,
0e984ec8	69	.prep = io_prep_rwv,
d9b57aa3	70	.issue = io_read,
d9b57aa3 JA	71	},
	72	[IORING_OP_WRITEV] = {
	73	.needs_file = 1,
	74	.hash_reg_file = 1,
	75	.unbound_nonreg_file = 1,
	76	.pollout = 1,
	77	.plug = 1,
	78	.audit_skip = 1,
	79	.ioprio = 1,
	80	.iopoll = 1,
ef0ec1ad	81	.iopoll_queue = 1,
d2d778fb	82	.vectored = 1,
0e984ec8	83	.prep = io_prep_rwv,
d9b57aa3	84	.issue = io_write,
d9b57aa3 JA	85	},
	86	[IORING_OP_FSYNC] = {
	87	.needs_file = 1,
	88	.audit_skip = 1,
d9b57aa3 JA	89	.prep = io_fsync_prep,
	90	.issue = io_fsync,
	91	},
	92	[IORING_OP_READ_FIXED] = {
	93	.needs_file = 1,
	94	.unbound_nonreg_file = 1,
	95	.pollin = 1,
	96	.plug = 1,
	97	.audit_skip = 1,
	98	.ioprio = 1,
	99	.iopoll = 1,
ef0ec1ad	100	.iopoll_queue = 1,
f688944c	101	.prep = io_prep_rw_fixed,
d9b57aa3 JA	102	.issue = io_read,
	103	},
	104	[IORING_OP_WRITE_FIXED] = {
	105	.needs_file = 1,
	106	.hash_reg_file = 1,
	107	.unbound_nonreg_file = 1,
	108	.pollout = 1,
	109	.plug = 1,
	110	.audit_skip = 1,
	111	.ioprio = 1,
	112	.iopoll = 1,
ef0ec1ad	113	.iopoll_queue = 1,
f688944c	114	.prep = io_prep_rw_fixed,
d9b57aa3 JA	115	.issue = io_write,
	116	},
	117	[IORING_OP_POLL_ADD] = {
	118	.needs_file = 1,
	119	.unbound_nonreg_file = 1,
	120	.audit_skip = 1,
d9b57aa3 JA	121	.prep = io_poll_add_prep,
	122	.issue = io_poll_add,
	123	},
	124	[IORING_OP_POLL_REMOVE] = {
	125	.audit_skip = 1,
d9b57aa3 JA	126	.prep = io_poll_remove_prep,
	127	.issue = io_poll_remove,
	128	},
	129	[IORING_OP_SYNC_FILE_RANGE] = {
	130	.needs_file = 1,
	131	.audit_skip = 1,
d9b57aa3 JA	132	.prep = io_sfr_prep,
	133	.issue = io_sync_file_range,
	134	},
	135	[IORING_OP_SENDMSG] = {
	136	.needs_file = 1,
	137	.unbound_nonreg_file = 1,
	138	.pollout = 1,
	139	.ioprio = 1,
858c293e	140	.manual_alloc = 1,
d9b57aa3	141	#if defined(CONFIG_NET)
d9b57aa3 JA	142	.prep = io_sendmsg_prep,
d9b57aa3 JA	143	.issue = io_sendmsg,
d9b57aa3 JA	144	#else
	145	.prep = io_eopnotsupp_prep,
	146	#endif
	147	},
	148	[IORING_OP_RECVMSG] = {
	149	.needs_file = 1,
	150	.unbound_nonreg_file = 1,
	151	.pollin = 1,
	152	.buffer_select = 1,
	153	.ioprio = 1,
858c293e	154	.manual_alloc = 1,
d9b57aa3	155	#if defined(CONFIG_NET)
d9b57aa3 JA	156	.prep = io_recvmsg_prep,
d9b57aa3 JA	157	.issue = io_recvmsg,
d9b57aa3 JA	158	#else
	159	.prep = io_eopnotsupp_prep,
	160	#endif
	161	},
	162	[IORING_OP_TIMEOUT] = {
	163	.audit_skip = 1,
d9b57aa3 JA	164	.prep = io_timeout_prep,
	165	.issue = io_timeout,
	166	},
	167	[IORING_OP_TIMEOUT_REMOVE] = {
	168	/* used by timeout updates' prep() */
	169	.audit_skip = 1,
d9b57aa3 JA	170	.prep = io_timeout_remove_prep,
	171	.issue = io_timeout_remove,
	172	},
	173	[IORING_OP_ACCEPT] = {
	174	.needs_file = 1,
	175	.unbound_nonreg_file = 1,
	176	.pollin = 1,
	177	.poll_exclusive = 1,
	178	.ioprio = 1, /* used for flags */
d9b57aa3 JA	179	#if defined(CONFIG_NET)
	180	.prep = io_accept_prep,
	181	.issue = io_accept,
	182	#else
	183	.prep = io_eopnotsupp_prep,
	184	#endif
	185	},
	186	[IORING_OP_ASYNC_CANCEL] = {
	187	.audit_skip = 1,
d9b57aa3 JA	188	.prep = io_async_cancel_prep,
	189	.issue = io_async_cancel,
	190	},
	191	[IORING_OP_LINK_TIMEOUT] = {
	192	.audit_skip = 1,
d9b57aa3 JA	193	.prep = io_link_timeout_prep,
	194	.issue = io_no_issue,
	195	},
	196	[IORING_OP_CONNECT] = {
	197	.needs_file = 1,
	198	.unbound_nonreg_file = 1,
	199	.pollout = 1,
d9b57aa3	200	#if defined(CONFIG_NET)
d9b57aa3 JA	201	.prep = io_connect_prep,
d9b57aa3 JA	202	.issue = io_connect,
d9b57aa3 JA	203	#else
	204	.prep = io_eopnotsupp_prep,
	205	#endif
	206	},
	207	[IORING_OP_FALLOCATE] = {
	208	.needs_file = 1,
d9b57aa3 JA	209	.prep = io_fallocate_prep,
	210	.issue = io_fallocate,
	211	},
	212	[IORING_OP_OPENAT] = {
d9b57aa3 JA	213	.prep = io_openat_prep,
d9b57aa3 JA	214	.issue = io_openat,
d9b57aa3 JA	215	},
d9b57aa3 JA	216	[IORING_OP_CLOSE] = {
d9b57aa3 JA	217	.prep = io_close_prep,
	218	.issue = io_close,
	219	},
d9808ceb	220	[IORING_OP_FILES_UPDATE] = {
d9b57aa3 JA	221	.audit_skip = 1,
d9b57aa3 JA	222	.iopoll = 1,
d9808ceb PB	223	.prep = io_files_update_prep,
d9808ceb PB	224	.issue = io_files_update,
d9b57aa3 JA	225	},
	226	[IORING_OP_STATX] = {
	227	.audit_skip = 1,
d9b57aa3 JA	228	.prep = io_statx_prep,
d9b57aa3 JA	229	.issue = io_statx,
d9b57aa3 JA	230	},
	231	[IORING_OP_READ] = {
	232	.needs_file = 1,
	233	.unbound_nonreg_file = 1,
	234	.pollin = 1,
	235	.buffer_select = 1,
	236	.plug = 1,
	237	.audit_skip = 1,
	238	.ioprio = 1,
	239	.iopoll = 1,
ef0ec1ad	240	.iopoll_queue = 1,
d9b57aa3 JA	241	.prep = io_prep_rw,
	242	.issue = io_read,
	243	},
	244	[IORING_OP_WRITE] = {
	245	.needs_file = 1,
	246	.hash_reg_file = 1,
	247	.unbound_nonreg_file = 1,
	248	.pollout = 1,
	249	.plug = 1,
	250	.audit_skip = 1,
	251	.ioprio = 1,
	252	.iopoll = 1,
ef0ec1ad	253	.iopoll_queue = 1,
d9b57aa3 JA	254	.prep = io_prep_rw,
	255	.issue = io_write,
	256	},
	257	[IORING_OP_FADVISE] = {
	258	.needs_file = 1,
	259	.audit_skip = 1,
d9b57aa3 JA	260	.prep = io_fadvise_prep,
	261	.issue = io_fadvise,
	262	},
	263	[IORING_OP_MADVISE] = {
fbe870a7	264	.audit_skip = 1,
d9b57aa3 JA	265	.prep = io_madvise_prep,
	266	.issue = io_madvise,
	267	},
	268	[IORING_OP_SEND] = {
	269	.needs_file = 1,
	270	.unbound_nonreg_file = 1,
	271	.pollout = 1,
	272	.audit_skip = 1,
	273	.ioprio = 1,
516e82f0	274	.manual_alloc = 1,
d9b57aa3 JA	275	#if defined(CONFIG_NET)
	276	.prep = io_sendmsg_prep,
	277	.issue = io_send,
	278	#else
	279	.prep = io_eopnotsupp_prep,
	280	#endif
	281	},
	282	[IORING_OP_RECV] = {
	283	.needs_file = 1,
	284	.unbound_nonreg_file = 1,
	285	.pollin = 1,
	286	.buffer_select = 1,
	287	.audit_skip = 1,
	288	.ioprio = 1,
d9b57aa3 JA	289	#if defined(CONFIG_NET)
	290	.prep = io_recvmsg_prep,
	291	.issue = io_recv,
	292	#else
	293	.prep = io_eopnotsupp_prep,
	294	#endif
	295	},
	296	[IORING_OP_OPENAT2] = {
d9b57aa3 JA	297	.prep = io_openat2_prep,
d9b57aa3 JA	298	.issue = io_openat2,
d9b57aa3 JA	299	},
	300	[IORING_OP_EPOLL_CTL] = {
	301	.unbound_nonreg_file = 1,
	302	.audit_skip = 1,
d9b57aa3 JA	303	#if defined(CONFIG_EPOLL)
	304	.prep = io_epoll_ctl_prep,
	305	.issue = io_epoll_ctl,
	306	#else
	307	.prep = io_eopnotsupp_prep,
	308	#endif
	309	},
	310	[IORING_OP_SPLICE] = {
	311	.needs_file = 1,
	312	.hash_reg_file = 1,
	313	.unbound_nonreg_file = 1,
	314	.audit_skip = 1,
d9b57aa3 JA	315	.prep = io_splice_prep,
	316	.issue = io_splice,
	317	},
	318	[IORING_OP_PROVIDE_BUFFERS] = {
	319	.audit_skip = 1,
	320	.iopoll = 1,
d9b57aa3 JA	321	.prep = io_provide_buffers_prep,
	322	.issue = io_provide_buffers,
	323	},
	324	[IORING_OP_REMOVE_BUFFERS] = {
	325	.audit_skip = 1,
	326	.iopoll = 1,
d9b57aa3 JA	327	.prep = io_remove_buffers_prep,
	328	.issue = io_remove_buffers,
	329	},
	330	[IORING_OP_TEE] = {
	331	.needs_file = 1,
	332	.hash_reg_file = 1,
	333	.unbound_nonreg_file = 1,
	334	.audit_skip = 1,
d9b57aa3 JA	335	.prep = io_tee_prep,
	336	.issue = io_tee,
	337	},
	338	[IORING_OP_SHUTDOWN] = {
	339	.needs_file = 1,
d9b57aa3 JA	340	#if defined(CONFIG_NET)
	341	.prep = io_shutdown_prep,
	342	.issue = io_shutdown,
	343	#else
	344	.prep = io_eopnotsupp_prep,
	345	#endif
	346	},
	347	[IORING_OP_RENAMEAT] = {
d9b57aa3 JA	348	.prep = io_renameat_prep,
d9b57aa3 JA	349	.issue = io_renameat,
d9b57aa3 JA	350	},
d9b57aa3 JA	351	[IORING_OP_UNLINKAT] = {
d9b57aa3 JA	352	.prep = io_unlinkat_prep,
d9b57aa3 JA	353	.issue = io_unlinkat,
d9b57aa3 JA	354	},
d9b57aa3 JA	355	[IORING_OP_MKDIRAT] = {
d9b57aa3 JA	356	.prep = io_mkdirat_prep,
d9b57aa3 JA	357	.issue = io_mkdirat,
d9b57aa3 JA	358	},
d9b57aa3 JA	359	[IORING_OP_SYMLINKAT] = {
d9b57aa3 JA	360	.prep = io_symlinkat_prep,
d9b57aa3 JA	361	.issue = io_symlinkat,
d9b57aa3 JA	362	},
d9b57aa3 JA	363	[IORING_OP_LINKAT] = {
d9b57aa3 JA	364	.prep = io_linkat_prep,
d9b57aa3 JA	365	.issue = io_linkat,
d9b57aa3 JA	366	},
	367	[IORING_OP_MSG_RING] = {
	368	.needs_file = 1,
	369	.iopoll = 1,
d9b57aa3 JA	370	.prep = io_msg_ring_prep,
	371	.issue = io_msg_ring,
	372	},
	373	[IORING_OP_FSETXATTR] = {
	374	.needs_file = 1,
d9b57aa3 JA	375	.prep = io_fsetxattr_prep,
d9b57aa3 JA	376	.issue = io_fsetxattr,
d9b57aa3 JA	377	},
d9b57aa3 JA	378	[IORING_OP_SETXATTR] = {
d9b57aa3 JA	379	.prep = io_setxattr_prep,
d9b57aa3 JA	380	.issue = io_setxattr,
d9b57aa3 JA	381	},
	382	[IORING_OP_FGETXATTR] = {
	383	.needs_file = 1,
d9b57aa3 JA	384	.prep = io_fgetxattr_prep,
d9b57aa3 JA	385	.issue = io_fgetxattr,
d9b57aa3 JA	386	},
d9b57aa3 JA	387	[IORING_OP_GETXATTR] = {
d9b57aa3 JA	388	.prep = io_getxattr_prep,
d9b57aa3 JA	389	.issue = io_getxattr,
d9b57aa3 JA	390	},
	391	[IORING_OP_SOCKET] = {
	392	.audit_skip = 1,
d9b57aa3 JA	393	#if defined(CONFIG_NET)
	394	.prep = io_socket_prep,
	395	.issue = io_socket,
	396	#else
	397	.prep = io_eopnotsupp_prep,
	398	#endif
	399	},
	400	[IORING_OP_URING_CMD] = {
	401	.needs_file = 1,
	402	.plug = 1,
5756a3a7	403	.iopoll = 1,
ef0ec1ad	404	.iopoll_queue = 1,
d9b57aa3 JA	405	.prep = io_uring_cmd_prep,
d9b57aa3 JA	406	.issue = io_uring_cmd,
d9b57aa3	407	},
b48c312b	408	[IORING_OP_SEND_ZC] = {
06a5464b PB	409	.needs_file = 1,
	410	.unbound_nonreg_file = 1,
	411	.pollout = 1,
	412	.audit_skip = 1,
	413	.ioprio = 1,
581711c4	414	.manual_alloc = 1,
06a5464b	415	#if defined(CONFIG_NET)
b0e9b551 PB	416	.prep = io_send_zc_prep,
b0e9b551 PB	417	.issue = io_send_zc,
06a5464b PB	418	#else
06a5464b PB	419	.prep = io_eopnotsupp_prep,
493108d9 PB	420	#endif
	421	},
	422	[IORING_OP_SENDMSG_ZC] = {
493108d9 PB	423	.needs_file = 1,
	424	.unbound_nonreg_file = 1,
	425	.pollout = 1,
493108d9 PB	426	.ioprio = 1,
	427	.manual_alloc = 1,
	428	#if defined(CONFIG_NET)
493108d9 PB	429	.prep = io_send_zc_prep,
493108d9 PB	430	.issue = io_sendmsg_zc,
f30bd4d0 BL	431	#else
	432	.prep = io_eopnotsupp_prep,
	433	#endif
	434	},
fc68fcda JA	435	[IORING_OP_READ_MULTISHOT] = {
	436	.needs_file = 1,
	437	.unbound_nonreg_file = 1,
	438	.pollin = 1,
	439	.buffer_select = 1,
	440	.audit_skip = 1,
	441	.prep = io_read_mshot_prep,
	442	.issue = io_read_mshot,
	443	},
f31ecf67 JA	444	[IORING_OP_WAITID] = {
	445	.prep = io_waitid_prep,
	446	.issue = io_waitid,
	447	},
194bb58c JA	448	[IORING_OP_FUTEX_WAIT] = {
	449	#if defined(CONFIG_FUTEX)
	450	.prep = io_futex_prep,
	451	.issue = io_futex_wait,
	452	#else
	453	.prep = io_eopnotsupp_prep,
	454	#endif
	455	},
	456	[IORING_OP_FUTEX_WAKE] = {
	457	#if defined(CONFIG_FUTEX)
	458	.prep = io_futex_prep,
	459	.issue = io_futex_wake,
	460	#else
	461	.prep = io_eopnotsupp_prep,
8f350194 JA	462	#endif
	463	},
	464	[IORING_OP_FUTEX_WAITV] = {
	465	#if defined(CONFIG_FUTEX)
	466	.prep = io_futexv_prep,
	467	.issue = io_futexv_wait,
	468	#else
	469	.prep = io_eopnotsupp_prep,
194bb58c JA	470	#endif
194bb58c JA	471	},
dc18b89a JA	472	[IORING_OP_FIXED_FD_INSTALL] = {
	473	.needs_file = 1,
	474	.audit_skip = 1,
	475	.prep = io_install_fixed_fd_prep,
	476	.issue = io_install_fixed_fd,
	477	},
f30bd4d0 BL	478	};
f30bd4d0 BL	479
f30bd4d0 BL	480	const struct io_cold_def io_cold_defs[] = {
	481	[IORING_OP_NOP] = {
	482	.name = "NOP",
	483	},
	484	[IORING_OP_READV] = {
	485	.async_size = sizeof(struct io_async_rw),
	486	.name = "READV",
	487	.prep_async = io_readv_prep_async,
	488	.cleanup = io_readv_writev_cleanup,
	489	.fail = io_rw_fail,
	490	},
	491	[IORING_OP_WRITEV] = {
	492	.async_size = sizeof(struct io_async_rw),
	493	.name = "WRITEV",
	494	.prep_async = io_writev_prep_async,
	495	.cleanup = io_readv_writev_cleanup,
	496	.fail = io_rw_fail,
	497	},
	498	[IORING_OP_FSYNC] = {
	499	.name = "FSYNC",
	500	},
	501	[IORING_OP_READ_FIXED] = {
	502	.async_size = sizeof(struct io_async_rw),
	503	.name = "READ_FIXED",
	504	.fail = io_rw_fail,
	505	},
	506	[IORING_OP_WRITE_FIXED] = {
	507	.async_size = sizeof(struct io_async_rw),
	508	.name = "WRITE_FIXED",
	509	.fail = io_rw_fail,
	510	},
	511	[IORING_OP_POLL_ADD] = {
	512	.name = "POLL_ADD",
	513	},
	514	[IORING_OP_POLL_REMOVE] = {
	515	.name = "POLL_REMOVE",
	516	},
	517	[IORING_OP_SYNC_FILE_RANGE] = {
	518	.name = "SYNC_FILE_RANGE",
	519	},
	520	[IORING_OP_SENDMSG] = {
	521	.name = "SENDMSG",
	522	#if defined(CONFIG_NET)
	523	.async_size = sizeof(struct io_async_msghdr),
	524	.prep_async = io_sendmsg_prep_async,
	525	.cleanup = io_sendmsg_recvmsg_cleanup,
	526	.fail = io_sendrecv_fail,
	527	#endif
	528	},
	529	[IORING_OP_RECVMSG] = {
	530	.name = "RECVMSG",
	531	#if defined(CONFIG_NET)
	532	.async_size = sizeof(struct io_async_msghdr),
	533	.prep_async = io_recvmsg_prep_async,
	534	.cleanup = io_sendmsg_recvmsg_cleanup,
	535	.fail = io_sendrecv_fail,
	536	#endif
	537	},
	538	[IORING_OP_TIMEOUT] = {
	539	.async_size = sizeof(struct io_timeout_data),
	540	.name = "TIMEOUT",
	541	},
	542	[IORING_OP_TIMEOUT_REMOVE] = {
	543	.name = "TIMEOUT_REMOVE",
544	},
545	[IORING_OP_ACCEPT] = {
546	.name = "ACCEPT",
547	},
548	[IORING_OP_ASYNC_CANCEL] = {
549	.name = "ASYNC_CANCEL",
550	},
551	[IORING_OP_LINK_TIMEOUT] = {
552	.async_size = sizeof(struct io_timeout_data),
553	.name = "LINK_TIMEOUT",
554	},
555	[IORING_OP_CONNECT] = {
556	.name = "CONNECT",
557	#if defined(CONFIG_NET)
558	.async_size = sizeof(struct io_async_connect),
559	.prep_async = io_connect_prep_async,
560	#endif
561	},
562	[IORING_OP_FALLOCATE] = {
563	.name = "FALLOCATE",
564	},
565	[IORING_OP_OPENAT] = {
566	.name = "OPENAT",
567	.cleanup = io_open_cleanup,
568	},
569	[IORING_OP_CLOSE] = {
570	.name = "CLOSE",
571	},
572	[IORING_OP_FILES_UPDATE] = {
573	.name = "FILES_UPDATE",
574	},
575	[IORING_OP_STATX] = {
576	.name = "STATX",
577	.cleanup = io_statx_cleanup,
578	},
579	[IORING_OP_READ] = {
580	.async_size = sizeof(struct io_async_rw),
581	.name = "READ",
582	.fail = io_rw_fail,
583	},
584	[IORING_OP_WRITE] = {
585	.async_size = sizeof(struct io_async_rw),
586	.name = "WRITE",
587	.fail = io_rw_fail,
588	},
589	[IORING_OP_FADVISE] = {
590	.name = "FADVISE",
591	},
592	[IORING_OP_MADVISE] = {
593	.name = "MADVISE",
594	},
595	[IORING_OP_SEND] = {
596	.name = "SEND",
597	#if defined(CONFIG_NET)
598	.async_size = sizeof(struct io_async_msghdr),
599	.fail = io_sendrecv_fail,
600	.prep_async = io_send_prep_async,
601	#endif
602	},
603	[IORING_OP_RECV] = {
604	.name = "RECV",
605	#if defined(CONFIG_NET)
606	.fail = io_sendrecv_fail,
607	#endif
608	},
609	[IORING_OP_OPENAT2] = {
610	.name = "OPENAT2",
611	.cleanup = io_open_cleanup,
612	},
613	[IORING_OP_EPOLL_CTL] = {
614	.name = "EPOLL",
615	},
616	[IORING_OP_SPLICE] = {
617	.name = "SPLICE",
618	},
619	[IORING_OP_PROVIDE_BUFFERS] = {
620	.name = "PROVIDE_BUFFERS",
621	},
622	[IORING_OP_REMOVE_BUFFERS] = {
623	.name = "REMOVE_BUFFERS",
624	},
625	[IORING_OP_TEE] = {
626	.name = "TEE",
627	},
628	[IORING_OP_SHUTDOWN] = {
629	.name = "SHUTDOWN",
630	},
631	[IORING_OP_RENAMEAT] = {
632	.name = "RENAMEAT",
633	.cleanup = io_renameat_cleanup,
634	},
635	[IORING_OP_UNLINKAT] = {
636	.name = "UNLINKAT",
637	.cleanup = io_unlinkat_cleanup,
638	},
639	[IORING_OP_MKDIRAT] = {
640	.name = "MKDIRAT",
641	.cleanup = io_mkdirat_cleanup,
642	},
643	[IORING_OP_SYMLINKAT] = {
644	.name = "SYMLINKAT",
645	.cleanup = io_link_cleanup,
646	},
647	[IORING_OP_LINKAT] = {
648	.name = "LINKAT",
649	.cleanup = io_link_cleanup,
650	},
651	[IORING_OP_MSG_RING] = {
652	.name = "MSG_RING",
653	.cleanup = io_msg_ring_cleanup,
654	},
655	[IORING_OP_FSETXATTR] = {
656	.name = "FSETXATTR",
657	.cleanup = io_xattr_cleanup,
658	},
659	[IORING_OP_SETXATTR] = {
660	.name = "SETXATTR",
661	.cleanup = io_xattr_cleanup,
662	},
663	[IORING_OP_FGETXATTR] = {
664	.name = "FGETXATTR",
665	.cleanup = io_xattr_cleanup,
666	},
667	[IORING_OP_GETXATTR] = {
668	.name = "GETXATTR",
669	.cleanup = io_xattr_cleanup,
670	},
671	[IORING_OP_SOCKET] = {
672	.name = "SOCKET",
673	},
674	[IORING_OP_URING_CMD] = {
675	.name = "URING_CMD",
fd9b8547	676	.async_size = 2 * sizeof(struct io_uring_sqe),
f30bd4d0 BL	677	.prep_async = io_uring_cmd_prep_async,
	678	},
	679	[IORING_OP_SEND_ZC] = {
	680	.name = "SEND_ZC",
	681	#if defined(CONFIG_NET)
	682	.async_size = sizeof(struct io_async_msghdr),
	683	.prep_async = io_send_prep_async,
	684	.cleanup = io_send_zc_cleanup,
	685	.fail = io_sendrecv_fail,
	686	#endif
	687	},
	688	[IORING_OP_SENDMSG_ZC] = {
	689	.name = "SENDMSG_ZC",
	690	#if defined(CONFIG_NET)
	691	.async_size = sizeof(struct io_async_msghdr),
493108d9 PB	692	.prep_async = io_sendmsg_prep_async,
	693	.cleanup = io_send_zc_cleanup,
	694	.fail = io_sendrecv_fail,
06a5464b	695	#endif
06a5464b	696	},
fc68fcda JA	697	[IORING_OP_READ_MULTISHOT] = {
	698	.name = "READ_MULTISHOT",
	699	},
f31ecf67 JA	700	[IORING_OP_WAITID] = {
	701	.name = "WAITID",
	702	.async_size = sizeof(struct io_waitid_async),
	703	},
194bb58c JA	704	[IORING_OP_FUTEX_WAIT] = {
	705	.name = "FUTEX_WAIT",
	706	},
	707	[IORING_OP_FUTEX_WAKE] = {
	708	.name = "FUTEX_WAKE",
	709	},
8f350194 JA	710	[IORING_OP_FUTEX_WAITV] = {
	711	.name = "FUTEX_WAITV",
	712	},
dc18b89a JA	713	[IORING_OP_FIXED_FD_INSTALL] = {
	714	.name = "FIXED_FD_INSTALL",
	715	},
d9b57aa3 JA	716	};
	717
	718	const char *io_uring_get_opcode(u8 opcode)
	719	{
	720	if (opcode < IORING_OP_LAST)
f30bd4d0	721	return io_cold_defs[opcode].name;
d9b57aa3 JA	722	return "INVALID";
	723	}
	724
	725	void __init io_uring_optable_init(void)
	726	{
	727	int i;
	728
f30bd4d0	729	BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);
a7dd2782	730	BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);
d9b57aa3	731
a7dd2782 BL	732	for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
	733	BUG_ON(!io_issue_defs[i].prep);
	734	if (io_issue_defs[i].prep != io_eopnotsupp_prep)
	735	BUG_ON(!io_issue_defs[i].issue);
f30bd4d0	736	WARN_ON_ONCE(!io_cold_defs[i].name);
d9b57aa3 JA	737	}
d9b57aa3 JA	738	}