[linux-2.6-block.git] / io_uring / io_uring_types.h

#ifndef IO_URING_TYPES_H
#define IO_URING_TYPES_H

#include <linux/blkdev.h>
#include <linux/task_work.h>
#include <linux/bitmap.h>
#include <uapi/linux/io_uring.h>

#include "io-wq.h"
#include "filetable.h"

struct io_hash_bucket {
	spinlock_t		lock;
	struct hlist_head	list;
} ____cacheline_aligned_in_smp;

struct io_hash_table {
	struct io_hash_bucket	*hbs;
	unsigned		hash_bits;
};

struct io_uring {
	u32 head ____cacheline_aligned_in_smp;
	u32 tail ____cacheline_aligned_in_smp;
};

/*
 * This data is shared with the application through the mmap at offsets
 * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
 *
 * The offsets to the member fields are published through struct
 * io_sqring_offsets when calling io_uring_setup.
 */
struct io_rings {
	/*
	 * Head and tail offsets into the ring; the offsets need to be
	 * masked to get valid indices.
	 *
	 * The kernel controls head of the sq ring and the tail of the cq ring,
	 * and the application controls tail of the sq ring and the head of the
	 * cq ring.
	 */
	struct io_uring		sq, cq;
	/*
	 * Bitmasks to apply to head and tail offsets (constant, equals
	 * ring_entries - 1)
	 */
	u32			sq_ring_mask, cq_ring_mask;
	/* Ring sizes (constant, power of 2) */
	u32			sq_ring_entries, cq_ring_entries;
	/*
	 * Number of invalid entries dropped by the kernel due to
	 * invalid index stored in array
	 *
	 * Written by the kernel, shouldn't be modified by the
	 * application (i.e. get number of "new events" by comparing to
	 * cached value).
	 *
	 * After a new SQ head value was read by the application this
	 * counter includes all submissions that were dropped reaching
	 * the new SQ head (and possibly more).
	 */
	u32			sq_dropped;
	/*
	 * Runtime SQ flags
	 *
	 * Written by the kernel, shouldn't be modified by the
	 * application.
	 *
	 * The application needs a full memory barrier before checking
	 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
	 */
	atomic_t		sq_flags;
	/*
	 * Runtime CQ flags
	 *
	 * Written by the application, shouldn't be modified by the
	 * kernel.
	 */
	u32			cq_flags;
	/*
	 * Number of completion events lost because the queue was full;
	 * this should be avoided by the application by making sure
	 * there are not more requests pending than there is space in
	 * the completion queue.
	 *
	 * Written by the kernel, shouldn't be modified by the
	 * application (i.e. get number of "new events" by comparing to
	 * cached value).
	 *
	 * As completion events come in out of order this counter is not
	 * ordered with any other data.
	 */
	u32			cq_overflow;
	/*
	 * Ring buffer of completion events.
	 *
	 * The kernel writes completion events fresh every time they are
	 * produced, so the application is allowed to modify pending
	 * entries.
	 */
	struct io_uring_cqe	cqes[] ____cacheline_aligned_in_smp;
};

struct io_restriction {
	DECLARE_BITMAP(register_op, IORING_REGISTER_LAST);
	DECLARE_BITMAP(sqe_op, IORING_OP_LAST);
	u8 sqe_flags_allowed;
	u8 sqe_flags_required;
	bool registered;
};

struct io_submit_link {
	struct io_kiocb		*head;
	struct io_kiocb		*last;
};

struct io_submit_state {
	/* inline/task_work completion list, under ->uring_lock */
	struct io_wq_work_node	free_list;
	/* batch completion logic */
	struct io_wq_work_list	compl_reqs;
	struct io_submit_link	link;

	bool			plug_started;
	bool			need_plug;
	bool			flush_cqes;
	unsigned short		submit_nr;
	struct blk_plug		plug;
};

struct io_ev_fd {
	struct eventfd_ctx	*cq_ev_fd;
	unsigned int		eventfd_async: 1;
	struct rcu_head		rcu;
};

struct io_ring_ctx {
	/* const or read-mostly hot data */
	struct {
		struct percpu_ref	refs;

		struct io_rings		*rings;
		unsigned int		flags;
		enum task_work_notify_mode	notify_method;
		unsigned int		compat: 1;
		unsigned int		drain_next: 1;
		unsigned int		restricted: 1;
		unsigned int		off_timeout_used: 1;
		unsigned int		drain_active: 1;
		unsigned int		drain_disabled: 1;
		unsigned int		has_evfd: 1;
		unsigned int		syscall_iopoll: 1;
	} ____cacheline_aligned_in_smp;

	/* submission data */
	struct {
		struct mutex		uring_lock;

		/*
		 * Ring buffer of indices into array of io_uring_sqe, which is
		 * mmapped by the application using the IORING_OFF_SQES offset.
		 *
		 * This indirection could e.g. be used to assign fixed
		 * io_uring_sqe entries to operations and only submit them to
		 * the queue when needed.
		 *
		 * The kernel modifies neither the indices array nor the entries
		 * array.
		 */
		u32			*sq_array;
		struct io_uring_sqe	*sq_sqes;
		unsigned		cached_sq_head;
		unsigned		sq_entries;

		/*
		 * Fixed resources fast path, should be accessed only under
		 * uring_lock, and updated through io_uring_register(2)
		 */
		struct io_rsrc_node	*rsrc_node;
		int			rsrc_cached_refs;
		atomic_t		cancel_seq;
		struct io_file_table	file_table;
		unsigned		nr_user_files;
		unsigned		nr_user_bufs;
		struct io_mapped_ubuf	**user_bufs;

		struct io_submit_state	submit_state;

		struct io_buffer_list	*io_bl;
		struct xarray		io_bl_xa;
		struct list_head	io_buffers_cache;

		struct list_head	cq_overflow_list;
		struct list_head	apoll_cache;
		struct xarray		personalities;
		u32			pers_next;
	} ____cacheline_aligned_in_smp;

	/* IRQ completion list, under ->completion_lock */
	struct io_wq_work_list	locked_free_list;
	unsigned int		locked_free_nr;

	const struct cred	*sq_creds;	/* cred used for __io_sq_thread() */
	struct io_sq_data	*sq_data;	/* if using sq thread polling */

	struct wait_queue_head	sqo_sq_wait;
	struct list_head	sqd_list;

	unsigned long		check_cq;

	struct {
		/*
		 * We cache a range of free CQEs we can use, once exhausted it
		 * should go through a slower range setup, see __io_get_cqe()
		 */
		struct io_uring_cqe	*cqe_cached;
		struct io_uring_cqe	*cqe_sentinel;

		unsigned		cached_cq_tail;
		unsigned		cq_entries;
		struct io_ev_fd	__rcu	*io_ev_fd;
		struct wait_queue_head	cq_wait;
		unsigned		cq_extra;
	} ____cacheline_aligned_in_smp;

	struct {
		spinlock_t		completion_lock;

		/*
		 * ->iopoll_list is protected by the ctx->uring_lock for
		 * io_uring instances that don't use IORING_SETUP_SQPOLL.
		 * For SQPOLL, only the single threaded io_sq_thread() will
		 * manipulate the list, hence no extra locking is needed there.
		 */
		struct io_wq_work_list	iopoll_list;
		struct io_hash_table	cancel_table;
		bool			poll_multi_queue;

		struct list_head	io_buffers_comp;
	} ____cacheline_aligned_in_smp;

	/* timeouts */
	struct {
		spinlock_t		timeout_lock;
		atomic_t		cq_timeouts;
		struct list_head	timeout_list;
		struct list_head	ltimeout_list;
		unsigned		cq_last_tm_flush;
	} ____cacheline_aligned_in_smp;

	/* Keep this last, we don't need it for the fast path */

	struct io_restriction		restrictions;
	struct task_struct		*submitter_task;

	/* slow path rsrc auxilary data, used by update/register */
	struct io_rsrc_node		*rsrc_backup_node;
	struct io_mapped_ubuf		*dummy_ubuf;
	struct io_rsrc_data		*file_data;
	struct io_rsrc_data		*buf_data;

	struct delayed_work		rsrc_put_work;
	struct llist_head		rsrc_put_llist;
	struct list_head		rsrc_ref_list;
	spinlock_t			rsrc_ref_lock;

	struct list_head		io_buffers_pages;

	#if defined(CONFIG_UNIX)
		struct socket		*ring_sock;
	#endif
	/* hashed buffered write serialization */
	struct io_wq_hash		*hash_map;

	/* Only used for accounting purposes */
	struct user_struct		*user;
	struct mm_struct		*mm_account;

	/* ctx exit and cancelation */
	struct llist_head		fallback_llist;
	struct delayed_work		fallback_work;
	struct work_struct		exit_work;
	struct list_head		tctx_list;
	struct completion		ref_comp;

	/* io-wq management, e.g. thread count */
	u32				iowq_limits[2];
	bool				iowq_limits_set;

	struct list_head		defer_list;
	unsigned			sq_thread_idle;
};

enum {
	REQ_F_FIXED_FILE_BIT	= IOSQE_FIXED_FILE_BIT,
	REQ_F_IO_DRAIN_BIT	= IOSQE_IO_DRAIN_BIT,
	REQ_F_LINK_BIT		= IOSQE_IO_LINK_BIT,
	REQ_F_HARDLINK_BIT	= IOSQE_IO_HARDLINK_BIT,
	REQ_F_FORCE_ASYNC_BIT	= IOSQE_ASYNC_BIT,
	REQ_F_BUFFER_SELECT_BIT	= IOSQE_BUFFER_SELECT_BIT,
	REQ_F_CQE_SKIP_BIT	= IOSQE_CQE_SKIP_SUCCESS_BIT,

	/* first byte is taken by user flags, shift it to not overlap */
	REQ_F_FAIL_BIT		= 8,
	REQ_F_INFLIGHT_BIT,
	REQ_F_CUR_POS_BIT,
	REQ_F_NOWAIT_BIT,
	REQ_F_LINK_TIMEOUT_BIT,
	REQ_F_NEED_CLEANUP_BIT,
	REQ_F_POLLED_BIT,
	REQ_F_BUFFER_SELECTED_BIT,
	REQ_F_BUFFER_RING_BIT,
	REQ_F_REISSUE_BIT,
	REQ_F_CREDS_BIT,
	REQ_F_REFCOUNT_BIT,
	REQ_F_ARM_LTIMEOUT_BIT,
	REQ_F_ASYNC_DATA_BIT,
	REQ_F_SKIP_LINK_CQES_BIT,
	REQ_F_SINGLE_POLL_BIT,
	REQ_F_DOUBLE_POLL_BIT,
	REQ_F_PARTIAL_IO_BIT,
	REQ_F_CQE32_INIT_BIT,
	REQ_F_APOLL_MULTISHOT_BIT,
	REQ_F_CLEAR_POLLIN_BIT,
	/* keep async read/write and isreg together and in order */
	REQ_F_SUPPORT_NOWAIT_BIT,
	REQ_F_ISREG_BIT,

	/* not a real bit, just to check we're not overflowing the space */
	__REQ_F_LAST_BIT,
};

enum {
	/* ctx owns file */
	REQ_F_FIXED_FILE	= BIT(REQ_F_FIXED_FILE_BIT),
	/* drain existing IO first */
	REQ_F_IO_DRAIN		= BIT(REQ_F_IO_DRAIN_BIT),
	/* linked sqes */
	REQ_F_LINK		= BIT(REQ_F_LINK_BIT),
	/* doesn't sever on completion < 0 */
	REQ_F_HARDLINK		= BIT(REQ_F_HARDLINK_BIT),
	/* IOSQE_ASYNC */
	REQ_F_FORCE_ASYNC	= BIT(REQ_F_FORCE_ASYNC_BIT),
	/* IOSQE_BUFFER_SELECT */
	REQ_F_BUFFER_SELECT	= BIT(REQ_F_BUFFER_SELECT_BIT),
	/* IOSQE_CQE_SKIP_SUCCESS */
	REQ_F_CQE_SKIP		= BIT(REQ_F_CQE_SKIP_BIT),

	/* fail rest of links */
	REQ_F_FAIL		= BIT(REQ_F_FAIL_BIT),
	/* on inflight list, should be cancelled and waited on exit reliably */
	REQ_F_INFLIGHT		= BIT(REQ_F_INFLIGHT_BIT),
	/* read/write uses file position */
	REQ_F_CUR_POS		= BIT(REQ_F_CUR_POS_BIT),
	/* must not punt to workers */
	REQ_F_NOWAIT		= BIT(REQ_F_NOWAIT_BIT),
	/* has or had linked timeout */
	REQ_F_LINK_TIMEOUT	= BIT(REQ_F_LINK_TIMEOUT_BIT),
	/* needs cleanup */
	REQ_F_NEED_CLEANUP	= BIT(REQ_F_NEED_CLEANUP_BIT),
	/* already went through poll handler */
	REQ_F_POLLED		= BIT(REQ_F_POLLED_BIT),
	/* buffer already selected */
	REQ_F_BUFFER_SELECTED	= BIT(REQ_F_BUFFER_SELECTED_BIT),
	/* buffer selected from ring, needs commit */
	REQ_F_BUFFER_RING	= BIT(REQ_F_BUFFER_RING_BIT),
	/* caller should reissue async */
	REQ_F_REISSUE		= BIT(REQ_F_REISSUE_BIT),
	/* supports async reads/writes */
	REQ_F_SUPPORT_NOWAIT	= BIT(REQ_F_SUPPORT_NOWAIT_BIT),
	/* regular file */
	REQ_F_ISREG		= BIT(REQ_F_ISREG_BIT),
	/* has creds assigned */
	REQ_F_CREDS		= BIT(REQ_F_CREDS_BIT),
	/* skip refcounting if not set */
	REQ_F_REFCOUNT		= BIT(REQ_F_REFCOUNT_BIT),
	/* there is a linked timeout that has to be armed */
	REQ_F_ARM_LTIMEOUT	= BIT(REQ_F_ARM_LTIMEOUT_BIT),
	/* ->async_data allocated */
	REQ_F_ASYNC_DATA	= BIT(REQ_F_ASYNC_DATA_BIT),
	/* don't post CQEs while failing linked requests */
	REQ_F_SKIP_LINK_CQES	= BIT(REQ_F_SKIP_LINK_CQES_BIT),
	/* single poll may be active */
	REQ_F_SINGLE_POLL	= BIT(REQ_F_SINGLE_POLL_BIT),
	/* double poll may active */
	REQ_F_DOUBLE_POLL	= BIT(REQ_F_DOUBLE_POLL_BIT),
	/* request has already done partial IO */
	REQ_F_PARTIAL_IO	= BIT(REQ_F_PARTIAL_IO_BIT),
	/* fast poll multishot mode */
	REQ_F_APOLL_MULTISHOT	= BIT(REQ_F_APOLL_MULTISHOT_BIT),
	/* ->extra1 and ->extra2 are initialised */
	REQ_F_CQE32_INIT	= BIT(REQ_F_CQE32_INIT_BIT),
	/* recvmsg special flag, clear EPOLLIN */
	REQ_F_CLEAR_POLLIN	= BIT(REQ_F_CLEAR_POLLIN_BIT),
};

typedef void (*io_req_tw_func_t)(struct io_kiocb *req, bool *locked);

struct io_task_work {
	union {
		struct io_wq_work_node	node;
		struct llist_node	fallback_node;
	};
	io_req_tw_func_t		func;
};

struct io_cqe {
	__u64	user_data;
	__s32	res;
	/* fd initially, then cflags for completion */
	union {
		__u32	flags;
		int	fd;
	};
};

/*
 * Each request type overlays its private data structure on top of this one.
 * They must not exceed this one in size.
 */
struct io_cmd_data {
	struct file		*file;
	/* each command gets 56 bytes of data */
	__u8			data[56];
};

#define io_kiocb_to_cmd(req)	((void *) &(req)->cmd)
#define cmd_to_io_kiocb(ptr)	((struct io_kiocb *) ptr)

struct io_kiocb {
	union {
		/*
		 * NOTE! Each of the io_kiocb union members has the file pointer
		 * as the first entry in their struct definition. So you can
		 * access the file pointer through any of the sub-structs,
		 * or directly as just 'file' in this struct.
		 */
		struct file		*file;
		struct io_cmd_data	cmd;
	};

	u8				opcode;
	/* polled IO has completed */
	u8				iopoll_completed;
	/*
	 * Can be either a fixed buffer index, or used with provided buffers.
	 * For the latter, before issue it points to the buffer group ID,
	 * and after selection it points to the buffer ID itself.
	 */
	u16				buf_index;
	unsigned int			flags;

	struct io_cqe			cqe;

	struct io_ring_ctx		*ctx;
	struct task_struct		*task;

	struct io_rsrc_node		*rsrc_node;

	union {
		/* store used ubuf, so we can prevent reloading */
		struct io_mapped_ubuf	*imu;

		/* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */
		struct io_buffer	*kbuf;

		/*
		 * stores buffer ID for ring provided buffers, valid IFF
		 * REQ_F_BUFFER_RING is set.
		 */
		struct io_buffer_list	*buf_list;
	};

	union {
		/* used by request caches, completion batching and iopoll */
		struct io_wq_work_node	comp_list;
		/* cache ->apoll->events */
		__poll_t apoll_events;
	};
	atomic_t			refs;
	atomic_t			poll_refs;
	struct io_task_work		io_task_work;
	/* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */
	union {
		struct hlist_node	hash_node;
		struct {
			u64		extra1;
			u64		extra2;
		};
	};
	/* internal polling, see IORING_FEAT_FAST_POLL */
	struct async_poll		*apoll;
	/* opcode allocated if it needs to store data for async defer */
	void				*async_data;
	/* linked requests, IFF REQ_F_HARDLINK or REQ_F_LINK are set */
	struct io_kiocb			*link;
	/* custom credentials, valid IFF REQ_F_CREDS is set */
	const struct cred		*creds;
	struct io_wq_work		work;
};

struct io_cancel_data {
	struct io_ring_ctx *ctx;
	union {
		u64 data;
		struct file *file;
	};
	u32 flags;
	int seq;
};

struct io_overflow_cqe {
	struct list_head list;
	struct io_uring_cqe cqe;
};

struct io_mapped_ubuf {
	u64		ubuf;
	u64		ubuf_end;
	unsigned int	nr_bvecs;
	unsigned long	acct_pages;
	struct bio_vec	bvec[];
};

#endif
Commit	Line	Data
e27f928e JA	1	#ifndef IO_URING_TYPES_H
	2	#define IO_URING_TYPES_H
	3
	4	#include <linux/blkdev.h>
	5	#include <linux/task_work.h>
d9b57aa3 JA	6	#include <linux/bitmap.h>
d9b57aa3 JA	7	#include <uapi/linux/io_uring.h>
e27f928e JA	8
e27f928e JA	9	#include "io-wq.h"
453b329b	10	#include "filetable.h"
e27f928e	11
e6f89be6 PB	12	struct io_hash_bucket {
	13	spinlock_t lock;
	14	struct hlist_head list;
	15	} ____cacheline_aligned_in_smp;
	16
	17	struct io_hash_table {
	18	struct io_hash_bucket *hbs;
	19	unsigned hash_bits;
	20	};
	21
e27f928e JA	22	struct io_uring {
	23	u32 head ____cacheline_aligned_in_smp;
	24	u32 tail ____cacheline_aligned_in_smp;
	25	};
	26
	27	/*
	28	* This data is shared with the application through the mmap at offsets
	29	* IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
	30	*
	31	* The offsets to the member fields are published through struct
	32	* io_sqring_offsets when calling io_uring_setup.
	33	*/
	34	struct io_rings {
	35	/*
	36	* Head and tail offsets into the ring; the offsets need to be
	37	* masked to get valid indices.
	38	*
	39	* The kernel controls head of the sq ring and the tail of the cq ring,
	40	* and the application controls tail of the sq ring and the head of the
	41	* cq ring.
	42	*/
	43	struct io_uring sq, cq;
	44	/*
	45	* Bitmasks to apply to head and tail offsets (constant, equals
	46	* ring_entries - 1)
	47	*/
	48	u32 sq_ring_mask, cq_ring_mask;
	49	/* Ring sizes (constant, power of 2) */
	50	u32 sq_ring_entries, cq_ring_entries;
	51	/*
	52	* Number of invalid entries dropped by the kernel due to
	53	* invalid index stored in array
	54	*
	55	* Written by the kernel, shouldn't be modified by the
	56	* application (i.e. get number of "new events" by comparing to
	57	* cached value).
	58	*
	59	* After a new SQ head value was read by the application this
	60	* counter includes all submissions that were dropped reaching
	61	* the new SQ head (and possibly more).
	62	*/
	63	u32 sq_dropped;
	64	/*
	65	* Runtime SQ flags
	66	*
	67	* Written by the kernel, shouldn't be modified by the
	68	* application.
	69	*
	70	* The application needs a full memory barrier before checking
	71	* for IORING_SQ_NEED_WAKEUP after updating the sq tail.
	72	*/
	73	atomic_t sq_flags;
	74	/*
	75	* Runtime CQ flags
	76	*
	77	* Written by the application, shouldn't be modified by the
	78	* kernel.
	79	*/
	80	u32 cq_flags;
	81	/*
	82	* Number of completion events lost because the queue was full;
	83	* this should be avoided by the application by making sure
	84	* there are not more requests pending than there is space in
	85	* the completion queue.
86	*
87	* Written by the kernel, shouldn't be modified by the
88	* application (i.e. get number of "new events" by comparing to
89	* cached value).
90	*
91	* As completion events come in out of order this counter is not
92	* ordered with any other data.
93	*/
94	u32 cq_overflow;
95	/*
96	* Ring buffer of completion events.
97	*
98	* The kernel writes completion events fresh every time they are
99	* produced, so the application is allowed to modify pending
100	* entries.
101	*/
102	struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp;
103	};
104
105	struct io_restriction {
106	DECLARE_BITMAP(register_op, IORING_REGISTER_LAST);
107	DECLARE_BITMAP(sqe_op, IORING_OP_LAST);
108	u8 sqe_flags_allowed;
109	u8 sqe_flags_required;
110	bool registered;
111	};
112
113	struct io_submit_link {
114	struct io_kiocb *head;
115	struct io_kiocb *last;
116	};
117
118	struct io_submit_state {
119	/* inline/task_work completion list, under ->uring_lock */
120	struct io_wq_work_node free_list;
121	/* batch completion logic */
122	struct io_wq_work_list compl_reqs;
123	struct io_submit_link link;
124
125	bool plug_started;
126	bool need_plug;
127	bool flush_cqes;
128	unsigned short submit_nr;
129	struct blk_plug plug;
130	};
131
132	struct io_ev_fd {
133	struct eventfd_ctx *cq_ev_fd;
134	unsigned int eventfd_async: 1;
135	struct rcu_head rcu;
136	};
137
e27f928e JA	138	struct io_ring_ctx {
	139	/* const or read-mostly hot data */
	140	struct {
	141	struct percpu_ref refs;
	142
	143	struct io_rings *rings;
	144	unsigned int flags;
	145	enum task_work_notify_mode notify_method;
	146	unsigned int compat: 1;
	147	unsigned int drain_next: 1;
	148	unsigned int restricted: 1;
	149	unsigned int off_timeout_used: 1;
	150	unsigned int drain_active: 1;
	151	unsigned int drain_disabled: 1;
	152	unsigned int has_evfd: 1;
	153	unsigned int syscall_iopoll: 1;
	154	} ____cacheline_aligned_in_smp;
	155
	156	/* submission data */
	157	struct {
	158	struct mutex uring_lock;
	159
	160	/*
	161	* Ring buffer of indices into array of io_uring_sqe, which is
	162	* mmapped by the application using the IORING_OFF_SQES offset.
	163	*
	164	* This indirection could e.g. be used to assign fixed
	165	* io_uring_sqe entries to operations and only submit them to
	166	* the queue when needed.
	167	*
	168	* The kernel modifies neither the indices array nor the entries
	169	* array.
	170	*/
	171	u32 *sq_array;
	172	struct io_uring_sqe *sq_sqes;
	173	unsigned cached_sq_head;
	174	unsigned sq_entries;
e27f928e JA	175
	176	/*
	177	* Fixed resources fast path, should be accessed only under
	178	* uring_lock, and updated through io_uring_register(2)
	179	*/
	180	struct io_rsrc_node *rsrc_node;
	181	int rsrc_cached_refs;
	182	atomic_t cancel_seq;
	183	struct io_file_table file_table;
	184	unsigned nr_user_files;
	185	unsigned nr_user_bufs;
	186	struct io_mapped_ubuf **user_bufs;
	187
	188	struct io_submit_state submit_state;
	189
	190	struct io_buffer_list *io_bl;
	191	struct xarray io_bl_xa;
	192	struct list_head io_buffers_cache;
	193
e27f928e JA	194	struct list_head cq_overflow_list;
	195	struct list_head apoll_cache;
	196	struct xarray personalities;
	197	u32 pers_next;
e27f928e JA	198	} ____cacheline_aligned_in_smp;
	199
	200	/* IRQ completion list, under ->completion_lock */
	201	struct io_wq_work_list locked_free_list;
	202	unsigned int locked_free_nr;
	203
	204	const struct cred sq_creds; / cred used for __io_sq_thread() */
	205	struct io_sq_data sq_data; / if using sq thread polling */
	206
	207	struct wait_queue_head sqo_sq_wait;
	208	struct list_head sqd_list;
	209
	210	unsigned long check_cq;
	211
	212	struct {
	213	/*
	214	* We cache a range of free CQEs we can use, once exhausted it
	215	* should go through a slower range setup, see __io_get_cqe()
	216	*/
	217	struct io_uring_cqe *cqe_cached;
	218	struct io_uring_cqe *cqe_sentinel;
	219
	220	unsigned cached_cq_tail;
	221	unsigned cq_entries;
	222	struct io_ev_fd __rcu *io_ev_fd;
	223	struct wait_queue_head cq_wait;
	224	unsigned cq_extra;
e27f928e JA	225	} ____cacheline_aligned_in_smp;
	226
	227	struct {
	228	spinlock_t completion_lock;
	229
e27f928e JA	230	/*
	231	* ->iopoll_list is protected by the ctx->uring_lock for
	232	* io_uring instances that don't use IORING_SETUP_SQPOLL.
	233	* For SQPOLL, only the single threaded io_sq_thread() will
	234	* manipulate the list, hence no extra locking is needed there.
	235	*/
	236	struct io_wq_work_list iopoll_list;
e6f89be6	237	struct io_hash_table cancel_table;
e27f928e JA	238	bool poll_multi_queue;
	239
	240	struct list_head io_buffers_comp;
	241	} ____cacheline_aligned_in_smp;
	242
aff5b2df PB	243	/* timeouts */
	244	struct {
	245	spinlock_t timeout_lock;
	246	atomic_t cq_timeouts;
	247	struct list_head timeout_list;
	248	struct list_head ltimeout_list;
	249	unsigned cq_last_tm_flush;
	250	} ____cacheline_aligned_in_smp;
	251
e27f928e	252	/* Keep this last, we don't need it for the fast path */
22eb2a3f PB	253
22eb2a3f PB	254	struct io_restriction restrictions;
97bbdc06	255	struct task_struct *submitter_task;
22eb2a3f PB	256
	257	/* slow path rsrc auxilary data, used by update/register */
	258	struct io_rsrc_node *rsrc_backup_node;
	259	struct io_mapped_ubuf *dummy_ubuf;
	260	struct io_rsrc_data *file_data;
	261	struct io_rsrc_data *buf_data;
	262
	263	struct delayed_work rsrc_put_work;
	264	struct llist_head rsrc_put_llist;
	265	struct list_head rsrc_ref_list;
	266	spinlock_t rsrc_ref_lock;
	267
	268	struct list_head io_buffers_pages;
	269
	270	#if defined(CONFIG_UNIX)
	271	struct socket *ring_sock;
	272	#endif
	273	/* hashed buffered write serialization */
	274	struct io_wq_hash *hash_map;
	275
	276	/* Only used for accounting purposes */
	277	struct user_struct *user;
	278	struct mm_struct *mm_account;
	279
	280	/* ctx exit and cancelation */
	281	struct llist_head fallback_llist;
	282	struct delayed_work fallback_work;
	283	struct work_struct exit_work;
	284	struct list_head tctx_list;
	285	struct completion ref_comp;
	286
	287	/* io-wq management, e.g. thread count */
	288	u32 iowq_limits[2];
	289	bool iowq_limits_set;
	290
	291	struct list_head defer_list;
	292	unsigned sq_thread_idle;
e27f928e JA	293	};
	294
	295	enum {
	296	REQ_F_FIXED_FILE_BIT = IOSQE_FIXED_FILE_BIT,
	297	REQ_F_IO_DRAIN_BIT = IOSQE_IO_DRAIN_BIT,
	298	REQ_F_LINK_BIT = IOSQE_IO_LINK_BIT,
	299	REQ_F_HARDLINK_BIT = IOSQE_IO_HARDLINK_BIT,
	300	REQ_F_FORCE_ASYNC_BIT = IOSQE_ASYNC_BIT,
	301	REQ_F_BUFFER_SELECT_BIT = IOSQE_BUFFER_SELECT_BIT,
	302	REQ_F_CQE_SKIP_BIT = IOSQE_CQE_SKIP_SUCCESS_BIT,
	303
	304	/* first byte is taken by user flags, shift it to not overlap */
	305	REQ_F_FAIL_BIT = 8,
	306	REQ_F_INFLIGHT_BIT,
	307	REQ_F_CUR_POS_BIT,
	308	REQ_F_NOWAIT_BIT,
	309	REQ_F_LINK_TIMEOUT_BIT,
	310	REQ_F_NEED_CLEANUP_BIT,
	311	REQ_F_POLLED_BIT,
	312	REQ_F_BUFFER_SELECTED_BIT,
	313	REQ_F_BUFFER_RING_BIT,
e27f928e JA	314	REQ_F_REISSUE_BIT,
	315	REQ_F_CREDS_BIT,
	316	REQ_F_REFCOUNT_BIT,
	317	REQ_F_ARM_LTIMEOUT_BIT,
	318	REQ_F_ASYNC_DATA_BIT,
	319	REQ_F_SKIP_LINK_CQES_BIT,
	320	REQ_F_SINGLE_POLL_BIT,
	321	REQ_F_DOUBLE_POLL_BIT,
	322	REQ_F_PARTIAL_IO_BIT,
	323	REQ_F_CQE32_INIT_BIT,
	324	REQ_F_APOLL_MULTISHOT_BIT,
	325	REQ_F_CLEAR_POLLIN_BIT,
	326	/* keep async read/write and isreg together and in order */
	327	REQ_F_SUPPORT_NOWAIT_BIT,
	328	REQ_F_ISREG_BIT,
	329
	330	/* not a real bit, just to check we're not overflowing the space */
	331	__REQ_F_LAST_BIT,
	332	};
	333
	334	enum {
	335	/* ctx owns file */
	336	REQ_F_FIXED_FILE = BIT(REQ_F_FIXED_FILE_BIT),
	337	/* drain existing IO first */
	338	REQ_F_IO_DRAIN = BIT(REQ_F_IO_DRAIN_BIT),
	339	/* linked sqes */
	340	REQ_F_LINK = BIT(REQ_F_LINK_BIT),
	341	/* doesn't sever on completion < 0 */
	342	REQ_F_HARDLINK = BIT(REQ_F_HARDLINK_BIT),
	343	/* IOSQE_ASYNC */
	344	REQ_F_FORCE_ASYNC = BIT(REQ_F_FORCE_ASYNC_BIT),
	345	/* IOSQE_BUFFER_SELECT */
	346	REQ_F_BUFFER_SELECT = BIT(REQ_F_BUFFER_SELECT_BIT),
	347	/* IOSQE_CQE_SKIP_SUCCESS */
	348	REQ_F_CQE_SKIP = BIT(REQ_F_CQE_SKIP_BIT),
	349
	350	/* fail rest of links */
	351	REQ_F_FAIL = BIT(REQ_F_FAIL_BIT),
	352	/* on inflight list, should be cancelled and waited on exit reliably */
	353	REQ_F_INFLIGHT = BIT(REQ_F_INFLIGHT_BIT),
	354	/* read/write uses file position */
	355	REQ_F_CUR_POS = BIT(REQ_F_CUR_POS_BIT),
	356	/* must not punt to workers */
	357	REQ_F_NOWAIT = BIT(REQ_F_NOWAIT_BIT),
	358	/* has or had linked timeout */
	359	REQ_F_LINK_TIMEOUT = BIT(REQ_F_LINK_TIMEOUT_BIT),
	360	/* needs cleanup */
	361	REQ_F_NEED_CLEANUP = BIT(REQ_F_NEED_CLEANUP_BIT),
	362	/* already went through poll handler */
	363	REQ_F_POLLED = BIT(REQ_F_POLLED_BIT),
	364	/* buffer already selected */
	365	REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT),
	366	/* buffer selected from ring, needs commit */
	367	REQ_F_BUFFER_RING = BIT(REQ_F_BUFFER_RING_BIT),
e27f928e JA	368	/* caller should reissue async */
	369	REQ_F_REISSUE = BIT(REQ_F_REISSUE_BIT),
	370	/* supports async reads/writes */
	371	REQ_F_SUPPORT_NOWAIT = BIT(REQ_F_SUPPORT_NOWAIT_BIT),
	372	/* regular file */
	373	REQ_F_ISREG = BIT(REQ_F_ISREG_BIT),
	374	/* has creds assigned */
	375	REQ_F_CREDS = BIT(REQ_F_CREDS_BIT),
	376	/* skip refcounting if not set */
	377	REQ_F_REFCOUNT = BIT(REQ_F_REFCOUNT_BIT),
	378	/* there is a linked timeout that has to be armed */
	379	REQ_F_ARM_LTIMEOUT = BIT(REQ_F_ARM_LTIMEOUT_BIT),
	380	/* ->async_data allocated */
	381	REQ_F_ASYNC_DATA = BIT(REQ_F_ASYNC_DATA_BIT),
	382	/* don't post CQEs while failing linked requests */
	383	REQ_F_SKIP_LINK_CQES = BIT(REQ_F_SKIP_LINK_CQES_BIT),
	384	/* single poll may be active */
	385	REQ_F_SINGLE_POLL = BIT(REQ_F_SINGLE_POLL_BIT),
	386	/* double poll may active */
	387	REQ_F_DOUBLE_POLL = BIT(REQ_F_DOUBLE_POLL_BIT),
	388	/* request has already done partial IO */
	389	REQ_F_PARTIAL_IO = BIT(REQ_F_PARTIAL_IO_BIT),
	390	/* fast poll multishot mode */
	391	REQ_F_APOLL_MULTISHOT = BIT(REQ_F_APOLL_MULTISHOT_BIT),
	392	/* ->extra1 and ->extra2 are initialised */
	393	REQ_F_CQE32_INIT = BIT(REQ_F_CQE32_INIT_BIT),
	394	/* recvmsg special flag, clear EPOLLIN */
	395	REQ_F_CLEAR_POLLIN = BIT(REQ_F_CLEAR_POLLIN_BIT),
	396	};
	397
	398	typedef void (io_req_tw_func_t)(struct io_kiocb req, bool *locked);
	399
	400	struct io_task_work {
	401	union {
	402	struct io_wq_work_node node;
	403	struct llist_node fallback_node;
	404	};
	405	io_req_tw_func_t func;
	406	};
	407
	408	struct io_cqe {
	409	__u64 user_data;
	410	__s32 res;
	411	/* fd initially, then cflags for completion */
	412	union {
	413	__u32 flags;
	414	int fd;
	415	};
	416	};
	417
	418	/*
	419	* Each request type overlays its private data structure on top of this one.
	420	* They must not exceed this one in size.
	421	*/
	422	struct io_cmd_data {
	423	struct file *file;
	424	/* each command gets 56 bytes of data */
	425	__u8 data[56];
	426	};
	427
	428	#define io_kiocb_to_cmd(req) ((void *) &(req)->cmd)
	429	#define cmd_to_io_kiocb(ptr) ((struct io_kiocb *) ptr)
	430
	431	struct io_kiocb {
432	union {
433	/*
434	* NOTE! Each of the io_kiocb union members has the file pointer
435	* as the first entry in their struct definition. So you can
436	* access the file pointer through any of the sub-structs,
437	* or directly as just 'file' in this struct.
438	*/
439	struct file *file;
440	struct io_cmd_data cmd;
441	};
442
443	u8 opcode;
444	/* polled IO has completed */
445	u8 iopoll_completed;
446	/*
447	* Can be either a fixed buffer index, or used with provided buffers.
448	* For the latter, before issue it points to the buffer group ID,
449	* and after selection it points to the buffer ID itself.
450	*/
451	u16 buf_index;
452	unsigned int flags;
453
454	struct io_cqe cqe;
455
456	struct io_ring_ctx *ctx;
457	struct task_struct *task;
458
459	struct io_rsrc_node *rsrc_node;
460
461	union {
462	/* store used ubuf, so we can prevent reloading */
463	struct io_mapped_ubuf *imu;
464
465	/* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */
466	struct io_buffer *kbuf;
467
468	/*
469	* stores buffer ID for ring provided buffers, valid IFF
470	* REQ_F_BUFFER_RING is set.
471	*/
472	struct io_buffer_list *buf_list;
473	};
474
475	union {
476	/* used by request caches, completion batching and iopoll */
477	struct io_wq_work_node comp_list;
478	/* cache ->apoll->events */
479	__poll_t apoll_events;
480	};
481	atomic_t refs;
482	atomic_t poll_refs;
483	struct io_task_work io_task_work;
484	/* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */
485	union {
486	struct hlist_node hash_node;
487	struct {
488	u64 extra1;
489	u64 extra2;
490	};
491	};
492	/* internal polling, see IORING_FEAT_FAST_POLL */
493	struct async_poll *apoll;
494	/* opcode allocated if it needs to store data for async defer */
495	void *async_data;
496	/* linked requests, IFF REQ_F_HARDLINK or REQ_F_LINK are set */
497	struct io_kiocb *link;
498	/* custom credentials, valid IFF REQ_F_CREDS is set */
499	const struct cred *creds;
500	struct io_wq_work work;
501	};
502
59915143 JA	503	struct io_cancel_data {
	504	struct io_ring_ctx *ctx;
	505	union {
	506	u64 data;
	507	struct file *file;
	508	};
	509	u32 flags;
	510	int seq;
	511	};
	512
a4ad4f74 JA	513	struct io_overflow_cqe {
	514	struct list_head list;
	515	struct io_uring_cqe cqe;
	516	};
	517
	518	struct io_mapped_ubuf {
	519	u64 ubuf;
	520	u64 ubuf_end;
	521	unsigned int nr_bvecs;
	522	unsigned long acct_pages;
	523	struct bio_vec bvec[];
	524	};
	525
e27f928e	526	#endif