[linux-block.git] / include / linux / io_uring_types.h

#ifndef IO_URING_TYPES_H
#define IO_URING_TYPES_H

#include <linux/blkdev.h>
#include <linux/task_work.h>
#include <linux/bitmap.h>
#include <linux/llist.h>
#include <uapi/linux/io_uring.h>

enum io_uring_cmd_flags {
	IO_URING_F_COMPLETE_DEFER	= 1,
	IO_URING_F_UNLOCKED		= 2,
	/* the request is executed from poll, it should not be freed */
	IO_URING_F_MULTISHOT		= 4,
	/* executed by io-wq */
	IO_URING_F_IOWQ			= 8,
	/* int's last bit, sign checks are usually faster than a bit test */
	IO_URING_F_NONBLOCK		= INT_MIN,

	/* ctx state flags, for URING_CMD */
	IO_URING_F_SQE128		= (1 << 8),
	IO_URING_F_CQE32		= (1 << 9),
	IO_URING_F_IOPOLL		= (1 << 10),

	/* set when uring wants to cancel a previously issued command */
	IO_URING_F_CANCEL		= (1 << 11),
	IO_URING_F_COMPAT		= (1 << 12),
};

struct io_wq_work_node {
	struct io_wq_work_node *next;
};

struct io_wq_work_list {
	struct io_wq_work_node *first;
	struct io_wq_work_node *last;
};

struct io_wq_work {
	struct io_wq_work_node list;
	unsigned flags;
	/* place it here instead of io_kiocb as it fills padding and saves 4B */
	int cancel_seq;
};

struct io_fixed_file {
	/* file * with additional FFS_* flags */
	unsigned long file_ptr;
};

struct io_file_table {
	struct io_fixed_file *files;
	unsigned long *bitmap;
	unsigned int alloc_hint;
};

struct io_hash_bucket {
	spinlock_t		lock;
	struct hlist_head	list;
} ____cacheline_aligned_in_smp;

struct io_hash_table {
	struct io_hash_bucket	*hbs;
	unsigned		hash_bits;
};

/*
 * Arbitrary limit, can be raised if need be
 */
#define IO_RINGFD_REG_MAX 16

struct io_uring_task {
	/* submission side */
	int				cached_refs;
	const struct io_ring_ctx 	*last;
	struct io_wq			*io_wq;
	struct file			*registered_rings[IO_RINGFD_REG_MAX];

	struct xarray			xa;
	struct wait_queue_head		wait;
	atomic_t			in_cancel;
	atomic_t			inflight_tracked;
	struct percpu_counter		inflight;

	struct { /* task_work */
		struct llist_head	task_list;
		struct callback_head	task_work;
	} ____cacheline_aligned_in_smp;
};

struct io_uring {
	u32 head;
	u32 tail;
};

/*
 * This data is shared with the application through the mmap at offsets
 * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
 *
 * The offsets to the member fields are published through struct
 * io_sqring_offsets when calling io_uring_setup.
 */
struct io_rings {
	/*
	 * Head and tail offsets into the ring; the offsets need to be
	 * masked to get valid indices.
	 *
	 * The kernel controls head of the sq ring and the tail of the cq ring,
	 * and the application controls tail of the sq ring and the head of the
	 * cq ring.
	 */
	struct io_uring		sq, cq;
	/*
	 * Bitmasks to apply to head and tail offsets (constant, equals
	 * ring_entries - 1)
	 */
	u32			sq_ring_mask, cq_ring_mask;
	/* Ring sizes (constant, power of 2) */
	u32			sq_ring_entries, cq_ring_entries;
	/*
	 * Number of invalid entries dropped by the kernel due to
	 * invalid index stored in array
	 *
	 * Written by the kernel, shouldn't be modified by the
	 * application (i.e. get number of "new events" by comparing to
	 * cached value).
	 *
	 * After a new SQ head value was read by the application this
	 * counter includes all submissions that were dropped reaching
	 * the new SQ head (and possibly more).
	 */
	u32			sq_dropped;
	/*
	 * Runtime SQ flags
	 *
	 * Written by the kernel, shouldn't be modified by the
	 * application.
	 *
	 * The application needs a full memory barrier before checking
	 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
	 */
	atomic_t		sq_flags;
	/*
	 * Runtime CQ flags
	 *
	 * Written by the application, shouldn't be modified by the
	 * kernel.
	 */
	u32			cq_flags;
	/*
	 * Number of completion events lost because the queue was full;
	 * this should be avoided by the application by making sure
	 * there are not more requests pending than there is space in
	 * the completion queue.
	 *
	 * Written by the kernel, shouldn't be modified by the
	 * application (i.e. get number of "new events" by comparing to
	 * cached value).
	 *
	 * As completion events come in out of order this counter is not
	 * ordered with any other data.
	 */
	u32			cq_overflow;
	/*
	 * Ring buffer of completion events.
	 *
	 * The kernel writes completion events fresh every time they are
	 * produced, so the application is allowed to modify pending
	 * entries.
	 */
	struct io_uring_cqe	cqes[] ____cacheline_aligned_in_smp;
};

struct io_restriction {
	DECLARE_BITMAP(register_op, IORING_REGISTER_LAST);
	DECLARE_BITMAP(sqe_op, IORING_OP_LAST);
	u8 sqe_flags_allowed;
	u8 sqe_flags_required;
	bool registered;
};

struct io_submit_link {
	struct io_kiocb		*head;
	struct io_kiocb		*last;
};

struct io_submit_state {
	/* inline/task_work completion list, under ->uring_lock */
	struct io_wq_work_node	free_list;
	/* batch completion logic */
	struct io_wq_work_list	compl_reqs;
	struct io_submit_link	link;

	bool			plug_started;
	bool			need_plug;
	unsigned short		submit_nr;
	unsigned int		cqes_count;
	struct blk_plug		plug;
};

struct io_ev_fd {
	struct eventfd_ctx	*cq_ev_fd;
	unsigned int		eventfd_async: 1;
	struct rcu_head		rcu;
	atomic_t		refs;
	atomic_t		ops;
};

struct io_alloc_cache {
	struct io_wq_work_node	list;
	unsigned int		nr_cached;
	unsigned int		max_cached;
	size_t			elem_size;
};

struct io_ring_ctx {
	/* const or read-mostly hot data */
	struct {
		unsigned int		flags;
		unsigned int		drain_next: 1;
		unsigned int		restricted: 1;
		unsigned int		off_timeout_used: 1;
		unsigned int		drain_active: 1;
		unsigned int		has_evfd: 1;
		/* all CQEs should be posted only by the submitter task */
		unsigned int		task_complete: 1;
		unsigned int		lockless_cq: 1;
		unsigned int		syscall_iopoll: 1;
		unsigned int		poll_activated: 1;
		unsigned int		drain_disabled: 1;
		unsigned int		compat: 1;

		struct task_struct	*submitter_task;
		struct io_rings		*rings;
		struct percpu_ref	refs;

		enum task_work_notify_mode	notify_method;
	} ____cacheline_aligned_in_smp;

	/* submission data */
	struct {
		struct mutex		uring_lock;

		/*
		 * Ring buffer of indices into array of io_uring_sqe, which is
		 * mmapped by the application using the IORING_OFF_SQES offset.
		 *
		 * This indirection could e.g. be used to assign fixed
		 * io_uring_sqe entries to operations and only submit them to
		 * the queue when needed.
		 *
		 * The kernel modifies neither the indices array nor the entries
		 * array.
		 */
		u32			*sq_array;
		struct io_uring_sqe	*sq_sqes;
		unsigned		cached_sq_head;
		unsigned		sq_entries;

		/*
		 * Fixed resources fast path, should be accessed only under
		 * uring_lock, and updated through io_uring_register(2)
		 */
		struct io_rsrc_node	*rsrc_node;
		atomic_t		cancel_seq;
		struct io_file_table	file_table;
		unsigned		nr_user_files;
		unsigned		nr_user_bufs;
		struct io_mapped_ubuf	**user_bufs;

		struct io_submit_state	submit_state;

		struct io_buffer_list	*io_bl;
		struct xarray		io_bl_xa;

		struct io_hash_table	cancel_table_locked;
		struct io_alloc_cache	apoll_cache;
		struct io_alloc_cache	netmsg_cache;

		/*
		 * ->iopoll_list is protected by the ctx->uring_lock for
		 * io_uring instances that don't use IORING_SETUP_SQPOLL.
		 * For SQPOLL, only the single threaded io_sq_thread() will
		 * manipulate the list, hence no extra locking is needed there.
		 */
		struct io_wq_work_list	iopoll_list;
		bool			poll_multi_queue;

		/*
		 * Any cancelable uring_cmd is added to this list in
		 * ->uring_cmd() by io_uring_cmd_insert_cancelable()
		 */
		struct hlist_head	cancelable_uring_cmd;
	} ____cacheline_aligned_in_smp;

	struct {
		/*
		 * We cache a range of free CQEs we can use, once exhausted it
		 * should go through a slower range setup, see __io_get_cqe()
		 */
		struct io_uring_cqe	*cqe_cached;
		struct io_uring_cqe	*cqe_sentinel;

		unsigned		cached_cq_tail;
		unsigned		cq_entries;
		struct io_ev_fd	__rcu	*io_ev_fd;
		unsigned		cq_extra;
	} ____cacheline_aligned_in_smp;

	/*
	 * task_work and async notification delivery cacheline. Expected to
	 * regularly bounce b/w CPUs.
	 */
	struct {
		struct llist_head	work_llist;
		unsigned long		check_cq;
		atomic_t		cq_wait_nr;
		atomic_t		cq_timeouts;
		struct wait_queue_head	cq_wait;
	} ____cacheline_aligned_in_smp;

	/* timeouts */
	struct {
		spinlock_t		timeout_lock;
		struct list_head	timeout_list;
		struct list_head	ltimeout_list;
		unsigned		cq_last_tm_flush;
	} ____cacheline_aligned_in_smp;

	struct io_uring_cqe	completion_cqes[16];

	spinlock_t		completion_lock;

	/* IRQ completion list, under ->completion_lock */
	struct io_wq_work_list	locked_free_list;
	unsigned int		locked_free_nr;

	struct list_head	io_buffers_comp;
	struct list_head	cq_overflow_list;
	struct io_hash_table	cancel_table;

	struct hlist_head	waitid_list;

#ifdef CONFIG_FUTEX
	struct hlist_head	futex_list;
	struct io_alloc_cache	futex_cache;
#endif

	const struct cred	*sq_creds;	/* cred used for __io_sq_thread() */
	struct io_sq_data	*sq_data;	/* if using sq thread polling */

	struct wait_queue_head	sqo_sq_wait;
	struct list_head	sqd_list;

	unsigned int		file_alloc_start;
	unsigned int		file_alloc_end;

	struct xarray		personalities;
	u32			pers_next;

	struct list_head	io_buffers_cache;

	/* deferred free list, protected by ->uring_lock */
	struct hlist_head	io_buf_list;

	/* Keep this last, we don't need it for the fast path */
	struct wait_queue_head		poll_wq;
	struct io_restriction		restrictions;

	/* slow path rsrc auxilary data, used by update/register */
	struct io_mapped_ubuf		*dummy_ubuf;
	struct io_rsrc_data		*file_data;
	struct io_rsrc_data		*buf_data;

	/* protected by ->uring_lock */
	struct list_head		rsrc_ref_list;
	struct io_alloc_cache		rsrc_node_cache;
	struct wait_queue_head		rsrc_quiesce_wq;
	unsigned			rsrc_quiesce;

	#if defined(CONFIG_UNIX)
		struct socket		*ring_sock;
	#endif
	/* hashed buffered write serialization */
	struct io_wq_hash		*hash_map;

	/* Only used for accounting purposes */
	struct user_struct		*user;
	struct mm_struct		*mm_account;

	/* ctx exit and cancelation */
	struct llist_head		fallback_llist;
	struct delayed_work		fallback_work;
	struct work_struct		exit_work;
	struct list_head		tctx_list;
	struct completion		ref_comp;

	/* io-wq management, e.g. thread count */
	u32				iowq_limits[2];
	bool				iowq_limits_set;

	struct callback_head		poll_wq_task_work;
	struct list_head		defer_list;
	unsigned			sq_thread_idle;
	/* protected by ->completion_lock */
	unsigned			evfd_last_cq_tail;

	/*
	 * If IORING_SETUP_NO_MMAP is used, then the below holds
	 * the gup'ed pages for the two rings, and the sqes.
	 */
	unsigned short			n_ring_pages;
	unsigned short			n_sqe_pages;
	struct page			**ring_pages;
	struct page			**sqe_pages;
};

struct io_tw_state {
	/* ->uring_lock is taken, callbacks can use io_tw_lock to lock it */
	bool locked;
};

enum {
	REQ_F_FIXED_FILE_BIT	= IOSQE_FIXED_FILE_BIT,
	REQ_F_IO_DRAIN_BIT	= IOSQE_IO_DRAIN_BIT,
	REQ_F_LINK_BIT		= IOSQE_IO_LINK_BIT,
	REQ_F_HARDLINK_BIT	= IOSQE_IO_HARDLINK_BIT,
	REQ_F_FORCE_ASYNC_BIT	= IOSQE_ASYNC_BIT,
	REQ_F_BUFFER_SELECT_BIT	= IOSQE_BUFFER_SELECT_BIT,
	REQ_F_CQE_SKIP_BIT	= IOSQE_CQE_SKIP_SUCCESS_BIT,

	/* first byte is taken by user flags, shift it to not overlap */
	REQ_F_FAIL_BIT		= 8,
	REQ_F_INFLIGHT_BIT,
	REQ_F_CUR_POS_BIT,
	REQ_F_NOWAIT_BIT,
	REQ_F_LINK_TIMEOUT_BIT,
	REQ_F_NEED_CLEANUP_BIT,
	REQ_F_POLLED_BIT,
	REQ_F_BUFFER_SELECTED_BIT,
	REQ_F_BUFFER_RING_BIT,
	REQ_F_REISSUE_BIT,
	REQ_F_CREDS_BIT,
	REQ_F_REFCOUNT_BIT,
	REQ_F_ARM_LTIMEOUT_BIT,
	REQ_F_ASYNC_DATA_BIT,
	REQ_F_SKIP_LINK_CQES_BIT,
	REQ_F_SINGLE_POLL_BIT,
	REQ_F_DOUBLE_POLL_BIT,
	REQ_F_PARTIAL_IO_BIT,
	REQ_F_APOLL_MULTISHOT_BIT,
	REQ_F_CLEAR_POLLIN_BIT,
	REQ_F_HASH_LOCKED_BIT,
	/* keep async read/write and isreg together and in order */
	REQ_F_SUPPORT_NOWAIT_BIT,
	REQ_F_ISREG_BIT,

	/* not a real bit, just to check we're not overflowing the space */
	__REQ_F_LAST_BIT,
};

enum {
	/* ctx owns file */
	REQ_F_FIXED_FILE	= BIT(REQ_F_FIXED_FILE_BIT),
	/* drain existing IO first */
	REQ_F_IO_DRAIN		= BIT(REQ_F_IO_DRAIN_BIT),
	/* linked sqes */
	REQ_F_LINK		= BIT(REQ_F_LINK_BIT),
	/* doesn't sever on completion < 0 */
	REQ_F_HARDLINK		= BIT(REQ_F_HARDLINK_BIT),
	/* IOSQE_ASYNC */
	REQ_F_FORCE_ASYNC	= BIT(REQ_F_FORCE_ASYNC_BIT),
	/* IOSQE_BUFFER_SELECT */
	REQ_F_BUFFER_SELECT	= BIT(REQ_F_BUFFER_SELECT_BIT),
	/* IOSQE_CQE_SKIP_SUCCESS */
	REQ_F_CQE_SKIP		= BIT(REQ_F_CQE_SKIP_BIT),

	/* fail rest of links */
	REQ_F_FAIL		= BIT(REQ_F_FAIL_BIT),
	/* on inflight list, should be cancelled and waited on exit reliably */
	REQ_F_INFLIGHT		= BIT(REQ_F_INFLIGHT_BIT),
	/* read/write uses file position */
	REQ_F_CUR_POS		= BIT(REQ_F_CUR_POS_BIT),
	/* must not punt to workers */
	REQ_F_NOWAIT		= BIT(REQ_F_NOWAIT_BIT),
	/* has or had linked timeout */
	REQ_F_LINK_TIMEOUT	= BIT(REQ_F_LINK_TIMEOUT_BIT),
	/* needs cleanup */
	REQ_F_NEED_CLEANUP	= BIT(REQ_F_NEED_CLEANUP_BIT),
	/* already went through poll handler */
	REQ_F_POLLED		= BIT(REQ_F_POLLED_BIT),
	/* buffer already selected */
	REQ_F_BUFFER_SELECTED	= BIT(REQ_F_BUFFER_SELECTED_BIT),
	/* buffer selected from ring, needs commit */
	REQ_F_BUFFER_RING	= BIT(REQ_F_BUFFER_RING_BIT),
	/* caller should reissue async */
	REQ_F_REISSUE		= BIT(REQ_F_REISSUE_BIT),
	/* supports async reads/writes */
	REQ_F_SUPPORT_NOWAIT	= BIT(REQ_F_SUPPORT_NOWAIT_BIT),
	/* regular file */
	REQ_F_ISREG		= BIT(REQ_F_ISREG_BIT),
	/* has creds assigned */
	REQ_F_CREDS		= BIT(REQ_F_CREDS_BIT),
	/* skip refcounting if not set */
	REQ_F_REFCOUNT		= BIT(REQ_F_REFCOUNT_BIT),
	/* there is a linked timeout that has to be armed */
	REQ_F_ARM_LTIMEOUT	= BIT(REQ_F_ARM_LTIMEOUT_BIT),
	/* ->async_data allocated */
	REQ_F_ASYNC_DATA	= BIT(REQ_F_ASYNC_DATA_BIT),
	/* don't post CQEs while failing linked requests */
	REQ_F_SKIP_LINK_CQES	= BIT(REQ_F_SKIP_LINK_CQES_BIT),
	/* single poll may be active */
	REQ_F_SINGLE_POLL	= BIT(REQ_F_SINGLE_POLL_BIT),
	/* double poll may active */
	REQ_F_DOUBLE_POLL	= BIT(REQ_F_DOUBLE_POLL_BIT),
	/* request has already done partial IO */
	REQ_F_PARTIAL_IO	= BIT(REQ_F_PARTIAL_IO_BIT),
	/* fast poll multishot mode */
	REQ_F_APOLL_MULTISHOT	= BIT(REQ_F_APOLL_MULTISHOT_BIT),
	/* recvmsg special flag, clear EPOLLIN */
	REQ_F_CLEAR_POLLIN	= BIT(REQ_F_CLEAR_POLLIN_BIT),
	/* hashed into ->cancel_hash_locked, protected by ->uring_lock */
	REQ_F_HASH_LOCKED	= BIT(REQ_F_HASH_LOCKED_BIT),
};

typedef void (*io_req_tw_func_t)(struct io_kiocb *req, struct io_tw_state *ts);

struct io_task_work {
	struct llist_node		node;
	io_req_tw_func_t		func;
};

struct io_cqe {
	__u64	user_data;
	__s32	res;
	/* fd initially, then cflags for completion */
	union {
		__u32	flags;
		int	fd;
	};
};

/*
 * Each request type overlays its private data structure on top of this one.
 * They must not exceed this one in size.
 */
struct io_cmd_data {
	struct file		*file;
	/* each command gets 56 bytes of data */
	__u8			data[56];
};

static inline void io_kiocb_cmd_sz_check(size_t cmd_sz)
{
	BUILD_BUG_ON(cmd_sz > sizeof(struct io_cmd_data));
}
#define io_kiocb_to_cmd(req, cmd_type) ( \
	io_kiocb_cmd_sz_check(sizeof(cmd_type)) , \
	((cmd_type *)&(req)->cmd) \
)
#define cmd_to_io_kiocb(ptr)	((struct io_kiocb *) ptr)

struct io_kiocb {
	union {
		/*
		 * NOTE! Each of the io_kiocb union members has the file pointer
		 * as the first entry in their struct definition. So you can
		 * access the file pointer through any of the sub-structs,
		 * or directly as just 'file' in this struct.
		 */
		struct file		*file;
		struct io_cmd_data	cmd;
	};

	u8				opcode;
	/* polled IO has completed */
	u8				iopoll_completed;
	/*
	 * Can be either a fixed buffer index, or used with provided buffers.
	 * For the latter, before issue it points to the buffer group ID,
	 * and after selection it points to the buffer ID itself.
	 */
	u16				buf_index;
	unsigned int			flags;

	struct io_cqe			cqe;

	struct io_ring_ctx		*ctx;
	struct task_struct		*task;

	struct io_rsrc_node		*rsrc_node;

	union {
		/* store used ubuf, so we can prevent reloading */
		struct io_mapped_ubuf	*imu;

		/* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */
		struct io_buffer	*kbuf;

		/*
		 * stores buffer ID for ring provided buffers, valid IFF
		 * REQ_F_BUFFER_RING is set.
		 */
		struct io_buffer_list	*buf_list;
	};

	union {
		/* used by request caches, completion batching and iopoll */
		struct io_wq_work_node	comp_list;
		/* cache ->apoll->events */
		__poll_t apoll_events;
	};
	atomic_t			refs;
	atomic_t			poll_refs;
	struct io_task_work		io_task_work;
	unsigned			nr_tw;
	/* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */
	struct hlist_node		hash_node;
	/* internal polling, see IORING_FEAT_FAST_POLL */
	struct async_poll		*apoll;
	/* opcode allocated if it needs to store data for async defer */
	void				*async_data;
	/* linked requests, IFF REQ_F_HARDLINK or REQ_F_LINK are set */
	struct io_kiocb			*link;
	/* custom credentials, valid IFF REQ_F_CREDS is set */
	const struct cred		*creds;
	struct io_wq_work		work;

	struct {
		u64			extra1;
		u64			extra2;
	} big_cqe;
};

struct io_overflow_cqe {
	struct list_head list;
	struct io_uring_cqe cqe;
};

#endif
Commit	Line	Data
e27f928e JA	1	#ifndef IO_URING_TYPES_H
	2	#define IO_URING_TYPES_H
	3
	4	#include <linux/blkdev.h>
	5	#include <linux/task_work.h>
d9b57aa3	6	#include <linux/bitmap.h>
e70cb608	7	#include <linux/llist.h>
d9b57aa3	8	#include <uapi/linux/io_uring.h>
e27f928e	9
b66509b8 PB	10	enum io_uring_cmd_flags {
	11	IO_URING_F_COMPLETE_DEFER = 1,
	12	IO_URING_F_UNLOCKED = 2,
	13	/* the request is executed from poll, it should not be freed */
	14	IO_URING_F_MULTISHOT = 4,
	15	/* executed by io-wq */
	16	IO_URING_F_IOWQ = 8,
	17	/* int's last bit, sign checks are usually faster than a bit test */
	18	IO_URING_F_NONBLOCK = INT_MIN,
	19
	20	/* ctx state flags, for URING_CMD */
	21	IO_URING_F_SQE128 = (1 << 8),
	22	IO_URING_F_CQE32 = (1 << 9),
	23	IO_URING_F_IOPOLL = (1 << 10),
	24
	25	/* set when uring wants to cancel a previously issued command */
	26	IO_URING_F_CANCEL = (1 << 11),
	27	IO_URING_F_COMPAT = (1 << 12),
	28	};
	29
ab1c84d8 PB	30	struct io_wq_work_node {
	31	struct io_wq_work_node *next;
	32	};
	33
	34	struct io_wq_work_list {
	35	struct io_wq_work_node *first;
	36	struct io_wq_work_node *last;
	37	};
	38
	39	struct io_wq_work {
	40	struct io_wq_work_node list;
	41	unsigned flags;
	42	/* place it here instead of io_kiocb as it fills padding and saves 4B */
	43	int cancel_seq;
	44	};
	45
	46	struct io_fixed_file {
	47	/* file * with additional FFS_* flags */
	48	unsigned long file_ptr;
	49	};
	50
	51	struct io_file_table {
	52	struct io_fixed_file *files;
	53	unsigned long *bitmap;
	54	unsigned int alloc_hint;
	55	};
e27f928e	56
e6f89be6 PB	57	struct io_hash_bucket {
	58	spinlock_t lock;
	59	struct hlist_head list;
	60	} ____cacheline_aligned_in_smp;
	61
	62	struct io_hash_table {
	63	struct io_hash_bucket *hbs;
	64	unsigned hash_bits;
	65	};
	66
e70cb608 PB	67	/*
	68	* Arbitrary limit, can be raised if need be
	69	*/
	70	#define IO_RINGFD_REG_MAX 16
	71
	72	struct io_uring_task {
	73	/* submission side */
	74	int cached_refs;
	75	const struct io_ring_ctx *last;
	76	struct io_wq *io_wq;
	77	struct file *registered_rings[IO_RINGFD_REG_MAX];
	78
	79	struct xarray xa;
	80	struct wait_queue_head wait;
8d664282	81	atomic_t in_cancel;
e70cb608 PB	82	atomic_t inflight_tracked;
	83	struct percpu_counter inflight;
	84
	85	struct { /* task_work */
	86	struct llist_head task_list;
	87	struct callback_head task_work;
	88	} ____cacheline_aligned_in_smp;
	89	};
	90
e27f928e	91	struct io_uring {
e5598d6a PB	92	u32 head;
e5598d6a PB	93	u32 tail;
e27f928e JA	94	};
	95
	96	/*
	97	* This data is shared with the application through the mmap at offsets
	98	* IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
	99	*
	100	* The offsets to the member fields are published through struct
	101	* io_sqring_offsets when calling io_uring_setup.
	102	*/
	103	struct io_rings {
	104	/*
	105	* Head and tail offsets into the ring; the offsets need to be
	106	* masked to get valid indices.
	107	*
	108	* The kernel controls head of the sq ring and the tail of the cq ring,
	109	* and the application controls tail of the sq ring and the head of the
	110	* cq ring.
	111	*/
	112	struct io_uring sq, cq;
	113	/*
	114	* Bitmasks to apply to head and tail offsets (constant, equals
	115	* ring_entries - 1)
	116	*/
	117	u32 sq_ring_mask, cq_ring_mask;
	118	/* Ring sizes (constant, power of 2) */
	119	u32 sq_ring_entries, cq_ring_entries;
	120	/*
	121	* Number of invalid entries dropped by the kernel due to
	122	* invalid index stored in array
	123	*
	124	* Written by the kernel, shouldn't be modified by the
	125	* application (i.e. get number of "new events" by comparing to
	126	* cached value).
	127	*
	128	* After a new SQ head value was read by the application this
	129	* counter includes all submissions that were dropped reaching
	130	* the new SQ head (and possibly more).
	131	*/
	132	u32 sq_dropped;
	133	/*
	134	* Runtime SQ flags
	135	*
	136	* Written by the kernel, shouldn't be modified by the
	137	* application.
	138	*
	139	* The application needs a full memory barrier before checking
	140	* for IORING_SQ_NEED_WAKEUP after updating the sq tail.
	141	*/
	142	atomic_t sq_flags;
	143	/*
	144	* Runtime CQ flags
	145	*
	146	* Written by the application, shouldn't be modified by the
	147	* kernel.
	148	*/
	149	u32 cq_flags;
	150	/*
	151	* Number of completion events lost because the queue was full;
	152	* this should be avoided by the application by making sure
	153	* there are not more requests pending than there is space in
	154	* the completion queue.
	155	*
	156	* Written by the kernel, shouldn't be modified by the
	157	* application (i.e. get number of "new events" by comparing to
158	* cached value).
159	*
160	* As completion events come in out of order this counter is not
161	* ordered with any other data.
162	*/
163	u32 cq_overflow;
164	/*
165	* Ring buffer of completion events.
166	*
167	* The kernel writes completion events fresh every time they are
168	* produced, so the application is allowed to modify pending
169	* entries.
170	*/
171	struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp;
172	};
173
174	struct io_restriction {
175	DECLARE_BITMAP(register_op, IORING_REGISTER_LAST);
176	DECLARE_BITMAP(sqe_op, IORING_OP_LAST);
177	u8 sqe_flags_allowed;
178	u8 sqe_flags_required;
179	bool registered;
180	};
181
182	struct io_submit_link {
183	struct io_kiocb *head;
184	struct io_kiocb *last;
185	};
186
187	struct io_submit_state {
188	/* inline/task_work completion list, under ->uring_lock */
189	struct io_wq_work_node free_list;
190	/* batch completion logic */
191	struct io_wq_work_list compl_reqs;
192	struct io_submit_link link;
193
194	bool plug_started;
195	bool need_plug;
e27f928e	196	unsigned short submit_nr;
931147dd	197	unsigned int cqes_count;
e27f928e JA	198	struct blk_plug plug;
	199	};
	200
	201	struct io_ev_fd {
	202	struct eventfd_ctx *cq_ev_fd;
	203	unsigned int eventfd_async: 1;
	204	struct rcu_head rcu;
21a091b9 DY	205	atomic_t refs;
21a091b9 DY	206	atomic_t ops;
e27f928e JA	207	};
e27f928e JA	208
9b797a37	209	struct io_alloc_cache {
efba1a9e	210	struct io_wq_work_node list;
9731bc98	211	unsigned int nr_cached;
69bbc6ad	212	unsigned int max_cached;
e1fe7ee8	213	size_t elem_size;
9b797a37 JA	214	};
9b797a37 JA	215
e27f928e JA	216	struct io_ring_ctx {
	217	/* const or read-mostly hot data */
	218	struct {
e27f928e	219	unsigned int flags;
e27f928e JA	220	unsigned int drain_next: 1;
	221	unsigned int restricted: 1;
	222	unsigned int off_timeout_used: 1;
	223	unsigned int drain_active: 1;
e27f928e	224	unsigned int has_evfd: 1;
e6aeb272 PB	225	/* all CQEs should be posted only by the submitter task */
e6aeb272 PB	226	unsigned int task_complete: 1;
ec26c225	227	unsigned int lockless_cq: 1;
632ffe09	228	unsigned int syscall_iopoll: 1;
bca39f39	229	unsigned int poll_activated: 1;
632ffe09 PB	230	unsigned int drain_disabled: 1;
632ffe09 PB	231	unsigned int compat: 1;
dde40322	232
18df385f PB	233	struct task_struct *submitter_task;
	234	struct io_rings *rings;
	235	struct percpu_ref refs;
03d89a2d	236
18df385f	237	enum task_work_notify_mode notify_method;
e27f928e JA	238	} ____cacheline_aligned_in_smp;
	239
	240	/* submission data */
	241	struct {
	242	struct mutex uring_lock;
	243
	244	/*
	245	* Ring buffer of indices into array of io_uring_sqe, which is
	246	* mmapped by the application using the IORING_OFF_SQES offset.
	247	*
	248	* This indirection could e.g. be used to assign fixed
	249	* io_uring_sqe entries to operations and only submit them to
	250	* the queue when needed.
	251	*
	252	* The kernel modifies neither the indices array nor the entries
	253	* array.
	254	*/
	255	u32 *sq_array;
	256	struct io_uring_sqe *sq_sqes;
	257	unsigned cached_sq_head;
	258	unsigned sq_entries;
e27f928e JA	259
	260	/*
	261	* Fixed resources fast path, should be accessed only under
	262	* uring_lock, and updated through io_uring_register(2)
	263	*/
	264	struct io_rsrc_node *rsrc_node;
e27f928e JA	265	atomic_t cancel_seq;
	266	struct io_file_table file_table;
	267	unsigned nr_user_files;
	268	unsigned nr_user_bufs;
	269	struct io_mapped_ubuf **user_bufs;
	270
	271	struct io_submit_state submit_state;
	272
	273	struct io_buffer_list *io_bl;
	274	struct xarray io_bl_xa;
e27f928e	275
9ca9fb24	276	struct io_hash_table cancel_table_locked;
9b797a37	277	struct io_alloc_cache apoll_cache;
43e0bbbd	278	struct io_alloc_cache netmsg_cache;
644c4a7a PB	279
	280	/*
	281	* ->iopoll_list is protected by the ctx->uring_lock for
	282	* io_uring instances that don't use IORING_SETUP_SQPOLL.
	283	* For SQPOLL, only the single threaded io_sq_thread() will
	284	* manipulate the list, hence no extra locking is needed there.
	285	*/
	286	struct io_wq_work_list iopoll_list;
	287	bool poll_multi_queue;
93b8cc60 ML	288
	289	/*
	290	* Any cancelable uring_cmd is added to this list in
	291	* ->uring_cmd() by io_uring_cmd_insert_cancelable()
	292	*/
	293	struct hlist_head cancelable_uring_cmd;
e27f928e JA	294	} ____cacheline_aligned_in_smp;
e27f928e JA	295
e27f928e JA	296	struct {
	297	/*
	298	* We cache a range of free CQEs we can use, once exhausted it
	299	* should go through a slower range setup, see __io_get_cqe()
	300	*/
	301	struct io_uring_cqe *cqe_cached;
	302	struct io_uring_cqe *cqe_sentinel;
	303
	304	unsigned cached_cq_tail;
	305	unsigned cq_entries;
	306	struct io_ev_fd __rcu *io_ev_fd;
e27f928e	307	unsigned cq_extra;
e27f928e JA	308	} ____cacheline_aligned_in_smp;
e27f928e JA	309
c9def23d PB	310	/*
	311	* task_work and async notification delivery cacheline. Expected to
	312	* regularly bounce b/w CPUs.
	313	*/
	314	struct {
	315	struct llist_head work_llist;
	316	unsigned long check_cq;
	317	atomic_t cq_wait_nr;
	318	atomic_t cq_timeouts;
	319	struct wait_queue_head cq_wait;
	320	} ____cacheline_aligned_in_smp;
	321
aff5b2df PB	322	/* timeouts */
	323	struct {
	324	spinlock_t timeout_lock;
aff5b2df PB	325	struct list_head timeout_list;
	326	struct list_head ltimeout_list;
	327	unsigned cq_last_tm_flush;
	328	} ____cacheline_aligned_in_smp;
	329
0aa7aa5f PB	330	struct io_uring_cqe completion_cqes[16];
0aa7aa5f PB	331
644c4a7a PB	332	spinlock_t completion_lock;
644c4a7a PB	333
d7f06fea PB	334	/* IRQ completion list, under ->completion_lock */
	335	struct io_wq_work_list locked_free_list;
	336	unsigned int locked_free_nr;
	337
18df385f PB	338	struct list_head io_buffers_comp;
	339	struct list_head cq_overflow_list;
	340	struct io_hash_table cancel_table;
	341
f31ecf67 JA	342	struct hlist_head waitid_list;
f31ecf67 JA	343
194bb58c JA	344	#ifdef CONFIG_FUTEX
	345	struct hlist_head futex_list;
	346	struct io_alloc_cache futex_cache;
	347	#endif
	348
d7f06fea PB	349	const struct cred sq_creds; / cred used for __io_sq_thread() */
	350	struct io_sq_data sq_data; / if using sq thread polling */
	351
	352	struct wait_queue_head sqo_sq_wait;
	353	struct list_head sqd_list;
	354
d7f06fea PB	355	unsigned int file_alloc_start;
	356	unsigned int file_alloc_end;
	357
	358	struct xarray personalities;
	359	u32 pers_next;
	360
18df385f PB	361	struct list_head io_buffers_cache;
18df385f PB	362
c392cbec JA	363	/* deferred free list, protected by ->uring_lock */
	364	struct hlist_head io_buf_list;
	365
e27f928e	366	/* Keep this last, we don't need it for the fast path */
7b235dd8	367	struct wait_queue_head poll_wq;
22eb2a3f PB	368	struct io_restriction restrictions;
	369
	370	/* slow path rsrc auxilary data, used by update/register */
22eb2a3f PB	371	struct io_mapped_ubuf *dummy_ubuf;
	372	struct io_rsrc_data *file_data;
	373	struct io_rsrc_data *buf_data;
	374
0a4813b1	375	/* protected by ->uring_lock */
22eb2a3f	376	struct list_head rsrc_ref_list;
9eae8655	377	struct io_alloc_cache rsrc_node_cache;
4ea15b56	378	struct wait_queue_head rsrc_quiesce_wq;
0b222eeb	379	unsigned rsrc_quiesce;
22eb2a3f	380
22eb2a3f PB	381	#if defined(CONFIG_UNIX)
	382	struct socket *ring_sock;
	383	#endif
	384	/* hashed buffered write serialization */
	385	struct io_wq_hash *hash_map;
	386
	387	/* Only used for accounting purposes */
	388	struct user_struct *user;
	389	struct mm_struct *mm_account;
	390
	391	/* ctx exit and cancelation */
	392	struct llist_head fallback_llist;
	393	struct delayed_work fallback_work;
	394	struct work_struct exit_work;
	395	struct list_head tctx_list;
	396	struct completion ref_comp;
	397
	398	/* io-wq management, e.g. thread count */
	399	u32 iowq_limits[2];
	400	bool iowq_limits_set;
	401
bca39f39	402	struct callback_head poll_wq_task_work;
22eb2a3f PB	403	struct list_head defer_list;
22eb2a3f PB	404	unsigned sq_thread_idle;
305bef98 PB	405	/* protected by ->completion_lock */
305bef98 PB	406	unsigned evfd_last_cq_tail;
18df385f PB	407
	408	/*
	409	* If IORING_SETUP_NO_MMAP is used, then the below holds
	410	* the gup'ed pages for the two rings, and the sqes.
	411	*/
	412	unsigned short n_ring_pages;
	413	unsigned short n_sqe_pages;
	414	struct page **ring_pages;
	415	struct page **sqe_pages;
e27f928e JA	416	};
e27f928e JA	417
a282967c PB	418	struct io_tw_state {
	419	/* ->uring_lock is taken, callbacks can use io_tw_lock to lock it */
	420	bool locked;
	421	};
	422
e27f928e JA	423	enum {
	424	REQ_F_FIXED_FILE_BIT = IOSQE_FIXED_FILE_BIT,
	425	REQ_F_IO_DRAIN_BIT = IOSQE_IO_DRAIN_BIT,
	426	REQ_F_LINK_BIT = IOSQE_IO_LINK_BIT,
	427	REQ_F_HARDLINK_BIT = IOSQE_IO_HARDLINK_BIT,
	428	REQ_F_FORCE_ASYNC_BIT = IOSQE_ASYNC_BIT,
	429	REQ_F_BUFFER_SELECT_BIT = IOSQE_BUFFER_SELECT_BIT,
	430	REQ_F_CQE_SKIP_BIT = IOSQE_CQE_SKIP_SUCCESS_BIT,
	431
	432	/* first byte is taken by user flags, shift it to not overlap */
	433	REQ_F_FAIL_BIT = 8,
	434	REQ_F_INFLIGHT_BIT,
	435	REQ_F_CUR_POS_BIT,
	436	REQ_F_NOWAIT_BIT,
	437	REQ_F_LINK_TIMEOUT_BIT,
	438	REQ_F_NEED_CLEANUP_BIT,
	439	REQ_F_POLLED_BIT,
	440	REQ_F_BUFFER_SELECTED_BIT,
	441	REQ_F_BUFFER_RING_BIT,
e27f928e JA	442	REQ_F_REISSUE_BIT,
	443	REQ_F_CREDS_BIT,
	444	REQ_F_REFCOUNT_BIT,
	445	REQ_F_ARM_LTIMEOUT_BIT,
	446	REQ_F_ASYNC_DATA_BIT,
	447	REQ_F_SKIP_LINK_CQES_BIT,
	448	REQ_F_SINGLE_POLL_BIT,
	449	REQ_F_DOUBLE_POLL_BIT,
	450	REQ_F_PARTIAL_IO_BIT,
e27f928e JA	451	REQ_F_APOLL_MULTISHOT_BIT,
e27f928e JA	452	REQ_F_CLEAR_POLLIN_BIT,
9ca9fb24	453	REQ_F_HASH_LOCKED_BIT,
e27f928e JA	454	/* keep async read/write and isreg together and in order */
	455	REQ_F_SUPPORT_NOWAIT_BIT,
	456	REQ_F_ISREG_BIT,
	457
	458	/* not a real bit, just to check we're not overflowing the space */
	459	__REQ_F_LAST_BIT,
	460	};
	461
	462	enum {
	463	/* ctx owns file */
	464	REQ_F_FIXED_FILE = BIT(REQ_F_FIXED_FILE_BIT),
	465	/* drain existing IO first */
	466	REQ_F_IO_DRAIN = BIT(REQ_F_IO_DRAIN_BIT),
	467	/* linked sqes */
	468	REQ_F_LINK = BIT(REQ_F_LINK_BIT),
	469	/* doesn't sever on completion < 0 */
	470	REQ_F_HARDLINK = BIT(REQ_F_HARDLINK_BIT),
	471	/* IOSQE_ASYNC */
	472	REQ_F_FORCE_ASYNC = BIT(REQ_F_FORCE_ASYNC_BIT),
	473	/* IOSQE_BUFFER_SELECT */
	474	REQ_F_BUFFER_SELECT = BIT(REQ_F_BUFFER_SELECT_BIT),
	475	/* IOSQE_CQE_SKIP_SUCCESS */
	476	REQ_F_CQE_SKIP = BIT(REQ_F_CQE_SKIP_BIT),
	477
	478	/* fail rest of links */
	479	REQ_F_FAIL = BIT(REQ_F_FAIL_BIT),
	480	/* on inflight list, should be cancelled and waited on exit reliably */
	481	REQ_F_INFLIGHT = BIT(REQ_F_INFLIGHT_BIT),
	482	/* read/write uses file position */
	483	REQ_F_CUR_POS = BIT(REQ_F_CUR_POS_BIT),
	484	/* must not punt to workers */
	485	REQ_F_NOWAIT = BIT(REQ_F_NOWAIT_BIT),
	486	/* has or had linked timeout */
	487	REQ_F_LINK_TIMEOUT = BIT(REQ_F_LINK_TIMEOUT_BIT),
	488	/* needs cleanup */
	489	REQ_F_NEED_CLEANUP = BIT(REQ_F_NEED_CLEANUP_BIT),
	490	/* already went through poll handler */
	491	REQ_F_POLLED = BIT(REQ_F_POLLED_BIT),
	492	/* buffer already selected */
	493	REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT),
	494	/* buffer selected from ring, needs commit */
	495	REQ_F_BUFFER_RING = BIT(REQ_F_BUFFER_RING_BIT),
e27f928e JA	496	/* caller should reissue async */
	497	REQ_F_REISSUE = BIT(REQ_F_REISSUE_BIT),
	498	/* supports async reads/writes */
	499	REQ_F_SUPPORT_NOWAIT = BIT(REQ_F_SUPPORT_NOWAIT_BIT),
	500	/* regular file */
	501	REQ_F_ISREG = BIT(REQ_F_ISREG_BIT),
	502	/* has creds assigned */
	503	REQ_F_CREDS = BIT(REQ_F_CREDS_BIT),
	504	/* skip refcounting if not set */
	505	REQ_F_REFCOUNT = BIT(REQ_F_REFCOUNT_BIT),
	506	/* there is a linked timeout that has to be armed */
	507	REQ_F_ARM_LTIMEOUT = BIT(REQ_F_ARM_LTIMEOUT_BIT),
	508	/* ->async_data allocated */
	509	REQ_F_ASYNC_DATA = BIT(REQ_F_ASYNC_DATA_BIT),
	510	/* don't post CQEs while failing linked requests */
	511	REQ_F_SKIP_LINK_CQES = BIT(REQ_F_SKIP_LINK_CQES_BIT),
	512	/* single poll may be active */
	513	REQ_F_SINGLE_POLL = BIT(REQ_F_SINGLE_POLL_BIT),
	514	/* double poll may active */
	515	REQ_F_DOUBLE_POLL = BIT(REQ_F_DOUBLE_POLL_BIT),
	516	/* request has already done partial IO */
	517	REQ_F_PARTIAL_IO = BIT(REQ_F_PARTIAL_IO_BIT),
	518	/* fast poll multishot mode */
	519	REQ_F_APOLL_MULTISHOT = BIT(REQ_F_APOLL_MULTISHOT_BIT),
e27f928e JA	520	/* recvmsg special flag, clear EPOLLIN */
e27f928e JA	521	REQ_F_CLEAR_POLLIN = BIT(REQ_F_CLEAR_POLLIN_BIT),
9ca9fb24 PB	522	/* hashed into ->cancel_hash_locked, protected by ->uring_lock */
9ca9fb24 PB	523	REQ_F_HASH_LOCKED = BIT(REQ_F_HASH_LOCKED_BIT),
e27f928e JA	524	};
e27f928e JA	525
a282967c	526	typedef void (io_req_tw_func_t)(struct io_kiocb req, struct io_tw_state *ts);
e27f928e JA	527
e27f928e JA	528	struct io_task_work {
3218e5d3	529	struct llist_node node;
e27f928e JA	530	io_req_tw_func_t func;
	531	};
	532
	533	struct io_cqe {
	534	__u64 user_data;
	535	__s32 res;
	536	/* fd initially, then cflags for completion */
	537	union {
	538	__u32 flags;
	539	int fd;
	540	};
	541	};
	542
	543	/*
	544	* Each request type overlays its private data structure on top of this one.
	545	* They must not exceed this one in size.
	546	*/
	547	struct io_cmd_data {
	548	struct file *file;
	549	/* each command gets 56 bytes of data */
	550	__u8 data[56];
	551	};
	552
f2ccb5ae SM	553	static inline void io_kiocb_cmd_sz_check(size_t cmd_sz)
	554	{
	555	BUILD_BUG_ON(cmd_sz > sizeof(struct io_cmd_data));
	556	}
	557	#define io_kiocb_to_cmd(req, cmd_type) ( \
	558	io_kiocb_cmd_sz_check(sizeof(cmd_type)) , \
	559	((cmd_type *)&(req)->cmd) \
	560	)
e27f928e JA	561	#define cmd_to_io_kiocb(ptr) ((struct io_kiocb *) ptr)
	562
	563	struct io_kiocb {
	564	union {
	565	/*
	566	* NOTE! Each of the io_kiocb union members has the file pointer
	567	* as the first entry in their struct definition. So you can
	568	* access the file pointer through any of the sub-structs,
	569	* or directly as just 'file' in this struct.
	570	*/
	571	struct file *file;
	572	struct io_cmd_data cmd;
	573	};
	574
	575	u8 opcode;
	576	/* polled IO has completed */
	577	u8 iopoll_completed;
	578	/*
	579	* Can be either a fixed buffer index, or used with provided buffers.
	580	* For the latter, before issue it points to the buffer group ID,
	581	* and after selection it points to the buffer ID itself.
	582	*/
	583	u16 buf_index;
	584	unsigned int flags;
	585
	586	struct io_cqe cqe;
	587
	588	struct io_ring_ctx *ctx;
	589	struct task_struct *task;
	590
	591	struct io_rsrc_node *rsrc_node;
	592
	593	union {
	594	/* store used ubuf, so we can prevent reloading */
	595	struct io_mapped_ubuf *imu;
	596
	597	/* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */
	598	struct io_buffer *kbuf;
	599
	600	/*
	601	* stores buffer ID for ring provided buffers, valid IFF
	602	* REQ_F_BUFFER_RING is set.
	603	*/
	604	struct io_buffer_list *buf_list;
	605	};
	606
	607	union {
	608	/* used by request caches, completion batching and iopoll */
	609	struct io_wq_work_node comp_list;
	610	/* cache ->apoll->events */
	611	__poll_t apoll_events;
	612	};
	613	atomic_t refs;
	614	atomic_t poll_refs;
	615	struct io_task_work io_task_work;
8751d154	616	unsigned nr_tw;
e27f928e	617	/* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */
b24c5d75	618	struct hlist_node hash_node;
e27f928e JA	619	/* internal polling, see IORING_FEAT_FAST_POLL */
	620	struct async_poll *apoll;
	621	/* opcode allocated if it needs to store data for async defer */
	622	void *async_data;
	623	/* linked requests, IFF REQ_F_HARDLINK or REQ_F_LINK are set */
	624	struct io_kiocb *link;
	625	/* custom credentials, valid IFF REQ_F_CREDS is set */
	626	const struct cred *creds;
	627	struct io_wq_work work;
b24c5d75 PB	628
	629	struct {
	630	u64 extra1;
	631	u64 extra2;
	632	} big_cqe;
e27f928e JA	633	};
e27f928e JA	634
a4ad4f74 JA	635	struct io_overflow_cqe {
	636	struct list_head list;
	637	struct io_uring_cqe cqe;
	638	};
	639
e27f928e	640	#endif