[linux-block.git] / include / linux / pipe_fs_i.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_PIPE_FS_I_H
#define _LINUX_PIPE_FS_I_H

#define PIPE_DEF_BUFFERS	16

#define PIPE_BUF_FLAG_LRU	0x01	/* page is on the LRU */
#define PIPE_BUF_FLAG_ATOMIC	0x02	/* was atomically mapped */
#define PIPE_BUF_FLAG_GIFT	0x04	/* page is a gift */
#define PIPE_BUF_FLAG_PACKET	0x08	/* read() as a packet */
#define PIPE_BUF_FLAG_CAN_MERGE	0x10	/* can merge buffers */
#define PIPE_BUF_FLAG_WHOLE	0x20	/* read() must return entire buffer or error */
#ifdef CONFIG_WATCH_QUEUE
#define PIPE_BUF_FLAG_LOSS	0x40	/* Message loss happened after this buffer */
#endif

/**
 *	struct pipe_buffer - a linux kernel pipe buffer
 *	@page: the page containing the data for the pipe buffer
 *	@offset: offset of data inside the @page
 *	@len: length of data inside the @page
 *	@ops: operations associated with this buffer. See @pipe_buf_operations.
 *	@flags: pipe buffer flags. See above.
 *	@private: private data owned by the ops.
 **/
struct pipe_buffer {
	struct page *page;
	unsigned int offset, len;
	const struct pipe_buf_operations *ops;
	unsigned int flags;
	unsigned long private;
};

/**
 *	struct pipe_inode_info - a linux kernel pipe
 *	@mutex: mutex protecting the whole thing
 *	@rd_wait: reader wait point in case of empty pipe
 *	@wr_wait: writer wait point in case of full pipe
 *	@head: The point of buffer production
 *	@tail: The point of buffer consumption
 *	@note_loss: The next read() should insert a data-lost message
 *	@max_usage: The maximum number of slots that may be used in the ring
 *	@ring_size: total number of buffers (should be a power of 2)
 *	@nr_accounted: The amount this pipe accounts for in user->pipe_bufs
 *	@tmp_page: cached released page
 *	@readers: number of current readers of this pipe
 *	@writers: number of current writers of this pipe
 *	@files: number of struct file referring this pipe (protected by ->i_lock)
 *	@r_counter: reader counter
 *	@w_counter: writer counter
 *	@fasync_readers: reader side fasync
 *	@fasync_writers: writer side fasync
 *	@bufs: the circular array of pipe buffers
 *	@user: the user who created this pipe
 *	@watch_queue: If this pipe is a watch_queue, this is the stuff for that
 **/
struct pipe_inode_info {
	struct mutex mutex;
	wait_queue_head_t rd_wait, wr_wait;
	unsigned int head;
	unsigned int tail;
	unsigned int max_usage;
	unsigned int ring_size;
#ifdef CONFIG_WATCH_QUEUE
	bool note_loss;
#endif
	unsigned int nr_accounted;
	unsigned int readers;
	unsigned int writers;
	unsigned int files;
	unsigned int r_counter;
	unsigned int w_counter;
	struct page *tmp_page;
	struct fasync_struct *fasync_readers;
	struct fasync_struct *fasync_writers;
	struct pipe_buffer *bufs;
	struct user_struct *user;
#ifdef CONFIG_WATCH_QUEUE
	struct watch_queue *watch_queue;
#endif
};

/*
 * Note on the nesting of these functions:
 *
 * ->confirm()
 *	->try_steal()
 *
 * That is, ->try_steal() must be called on a confirmed buffer.  See below for
 * the meaning of each operation.  Also see the kerneldoc in fs/pipe.c for the
 * pipe and generic variants of these hooks.
 */
struct pipe_buf_operations {
	/*
	 * ->confirm() verifies that the data in the pipe buffer is there
	 * and that the contents are good. If the pages in the pipe belong
	 * to a file system, we may need to wait for IO completion in this
	 * hook. Returns 0 for good, or a negative error value in case of
	 * error.  If not present all pages are considered good.
	 */
	int (*confirm)(struct pipe_inode_info *, struct pipe_buffer *);

	/*
	 * When the contents of this pipe buffer has been completely
	 * consumed by a reader, ->release() is called.
	 */
	void (*release)(struct pipe_inode_info *, struct pipe_buffer *);

	/*
	 * Attempt to take ownership of the pipe buffer and its contents.
	 * ->try_steal() returns %true for success, in which case the contents
	 * of the pipe (the buf->page) is locked and now completely owned by the
	 * caller. The page may then be transferred to a different mapping, the
	 * most often used case is insertion into different file address space
	 * cache.
	 */
	bool (*try_steal)(struct pipe_inode_info *, struct pipe_buffer *);

	/*
	 * Get a reference to the pipe buffer.
	 */
	bool (*get)(struct pipe_inode_info *, struct pipe_buffer *);
};

/**
 * pipe_empty - Return true if the pipe is empty
 * @head: The pipe ring head pointer
 * @tail: The pipe ring tail pointer
 */
static inline bool pipe_empty(unsigned int head, unsigned int tail)
{
	return head == tail;
}

/**
 * pipe_occupancy - Return number of slots used in the pipe
 * @head: The pipe ring head pointer
 * @tail: The pipe ring tail pointer
 */
static inline unsigned int pipe_occupancy(unsigned int head, unsigned int tail)
{
	return head - tail;
}

/**
 * pipe_full - Return true if the pipe is full
 * @head: The pipe ring head pointer
 * @tail: The pipe ring tail pointer
 * @limit: The maximum amount of slots available.
 */
static inline bool pipe_full(unsigned int head, unsigned int tail,
			     unsigned int limit)
{
	return pipe_occupancy(head, tail) >= limit;
}

/**
 * pipe_space_for_user - Return number of slots available to userspace
 * @head: The pipe ring head pointer
 * @tail: The pipe ring tail pointer
 * @pipe: The pipe info structure
 */
static inline unsigned int pipe_space_for_user(unsigned int head, unsigned int tail,
					       struct pipe_inode_info *pipe)
{
	unsigned int p_occupancy, p_space;

	p_occupancy = pipe_occupancy(head, tail);
	if (p_occupancy >= pipe->max_usage)
		return 0;
	p_space = pipe->ring_size - p_occupancy;
	if (p_space > pipe->max_usage)
		p_space = pipe->max_usage;
	return p_space;
}

/**
 * pipe_buf_get - get a reference to a pipe_buffer
 * @pipe:	the pipe that the buffer belongs to
 * @buf:	the buffer to get a reference to
 *
 * Return: %true if the reference was successfully obtained.
 */
static inline __must_check bool pipe_buf_get(struct pipe_inode_info *pipe,
				struct pipe_buffer *buf)
{
	return buf->ops->get(pipe, buf);
}

/**
 * pipe_buf_release - put a reference to a pipe_buffer
 * @pipe:	the pipe that the buffer belongs to
 * @buf:	the buffer to put a reference to
 */
static inline void pipe_buf_release(struct pipe_inode_info *pipe,
				    struct pipe_buffer *buf)
{
	const struct pipe_buf_operations *ops = buf->ops;

	buf->ops = NULL;
	ops->release(pipe, buf);
}

/**
 * pipe_buf_confirm - verify contents of the pipe buffer
 * @pipe:	the pipe that the buffer belongs to
 * @buf:	the buffer to confirm
 */
static inline int pipe_buf_confirm(struct pipe_inode_info *pipe,
				   struct pipe_buffer *buf)
{
	if (!buf->ops->confirm)
		return 0;
	return buf->ops->confirm(pipe, buf);
}

/**
 * pipe_buf_try_steal - attempt to take ownership of a pipe_buffer
 * @pipe:	the pipe that the buffer belongs to
 * @buf:	the buffer to attempt to steal
 */
static inline bool pipe_buf_try_steal(struct pipe_inode_info *pipe,
		struct pipe_buffer *buf)
{
	if (!buf->ops->try_steal)
		return false;
	return buf->ops->try_steal(pipe, buf);
}

/* Differs from PIPE_BUF in that PIPE_SIZE is the length of the actual
   memory allocation, whereas PIPE_BUF makes atomicity guarantees.  */
#define PIPE_SIZE		PAGE_SIZE

/* Pipe lock and unlock operations */
void pipe_lock(struct pipe_inode_info *);
void pipe_unlock(struct pipe_inode_info *);
void pipe_double_lock(struct pipe_inode_info *, struct pipe_inode_info *);

extern unsigned int pipe_max_size;
extern unsigned long pipe_user_pages_hard;
extern unsigned long pipe_user_pages_soft;

/* Drop the inode semaphore and wait for a pipe event, atomically */
void pipe_wait(struct pipe_inode_info *pipe);

struct pipe_inode_info *alloc_pipe_info(void);
void free_pipe_info(struct pipe_inode_info *);

/* Generic pipe buffer ops functions */
bool generic_pipe_buf_get(struct pipe_inode_info *, struct pipe_buffer *);
bool generic_pipe_buf_try_steal(struct pipe_inode_info *, struct pipe_buffer *);
void generic_pipe_buf_release(struct pipe_inode_info *, struct pipe_buffer *);

extern const struct pipe_buf_operations nosteal_pipe_buf_ops;

#ifdef CONFIG_WATCH_QUEUE
unsigned long account_pipe_buffers(struct user_struct *user,
				   unsigned long old, unsigned long new);
bool too_many_pipe_buffers_soft(unsigned long user_bufs);
bool too_many_pipe_buffers_hard(unsigned long user_bufs);
bool pipe_is_unprivileged_user(void);
#endif

/* for F_SETPIPE_SZ and F_GETPIPE_SZ */
#ifdef CONFIG_WATCH_QUEUE
int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots);
#endif
long pipe_fcntl(struct file *, unsigned int, unsigned long arg);
struct pipe_inode_info *get_pipe_info(struct file *file, bool for_splice);

int create_pipe_files(struct file **, int);
unsigned int round_pipe_size(unsigned long size);

#endif
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
1da177e4 LT	2	#ifndef _LINUX_PIPE_FS_I_H
	3	#define _LINUX_PIPE_FS_I_H
	4
35f3d14d	5	#define PIPE_DEF_BUFFERS 16
1da177e4	6
1432873a JA	7	#define PIPE_BUF_FLAG_LRU 0x01 /* page is on the LRU */
	8	#define PIPE_BUF_FLAG_ATOMIC 0x02 /* was atomically mapped */
	9	#define PIPE_BUF_FLAG_GIFT 0x04 /* page is a gift */
9883035a	10	#define PIPE_BUF_FLAG_PACKET 0x08 /* read() as a packet */
f6dd9755	11	#define PIPE_BUF_FLAG_CAN_MERGE 0x10 /* can merge buffers */
6c329784	12	#define PIPE_BUF_FLAG_WHOLE 0x20 /* read() must return entire buffer or error */
e7d553d6	13	#ifdef CONFIG_WATCH_QUEUE
6c329784	14	#define PIPE_BUF_FLAG_LOSS 0x40 /* Message loss happened after this buffer */
e7d553d6	15	#endif
3e7ee3e7	16
0845718d JA	17	/**
	18	* struct pipe_buffer - a linux kernel pipe buffer
	19	* @page: the page containing the data for the pipe buffer
	20	* @offset: offset of data inside the @page
	21	* @len: length of data inside the @page
	22	* @ops: operations associated with this buffer. See @pipe_buf_operations.
	23	* @flags: pipe buffer flags. See above.
	24	* @private: private data owned by the ops.
	25	**/
1da177e4 LT	26	struct pipe_buffer {
	27	struct page *page;
	28	unsigned int offset, len;
d4c3cca9	29	const struct pipe_buf_operations *ops;
3e7ee3e7	30	unsigned int flags;
497f9625	31	unsigned long private;
1da177e4 LT	32	};
1da177e4 LT	33
0845718d JA	34	/**
0845718d JA	35	* struct pipe_inode_info - a linux kernel pipe
72b0d9aa	36	* @mutex: mutex protecting the whole thing
0bf999f9 RD	37	* @rd_wait: reader wait point in case of empty pipe
0bf999f9 RD	38	* @wr_wait: writer wait point in case of full pipe
8cefc107 DH	39	* @head: The point of buffer production
8cefc107 DH	40	* @tail: The point of buffer consumption
e7d553d6	41	* @note_loss: The next read() should insert a data-lost message
6718b6f8	42	* @max_usage: The maximum number of slots that may be used in the ring
8cefc107	43	* @ring_size: total number of buffers (should be a power of 2)
c73be61c	44	* @nr_accounted: The amount this pipe accounts for in user->pipe_bufs
0845718d JA	45	* @tmp_page: cached released page
	46	* @readers: number of current readers of this pipe
	47	* @writers: number of current writers of this pipe
e227867f	48	* @files: number of struct file referring this pipe (protected by ->i_lock)
0845718d JA	49	* @r_counter: reader counter
	50	* @w_counter: writer counter
	51	* @fasync_readers: reader side fasync
	52	* @fasync_writers: writer side fasync
0845718d	53	* @bufs: the circular array of pipe buffers
759c0114	54	* @user: the user who created this pipe
c73be61c	55	* @watch_queue: If this pipe is a watch_queue, this is the stuff for that
0845718d	56	**/
17374ff1	57	struct pipe_inode_info {
72b0d9aa	58	struct mutex mutex;
0ddad21d	59	wait_queue_head_t rd_wait, wr_wait;
8cefc107 DH	60	unsigned int head;
8cefc107 DH	61	unsigned int tail;
6718b6f8	62	unsigned int max_usage;
8cefc107	63	unsigned int ring_size;
e7d553d6 DH	64	#ifdef CONFIG_WATCH_QUEUE
	65	bool note_loss;
	66	#endif
c73be61c	67	unsigned int nr_accounted;
17374ff1 JA	68	unsigned int readers;
17374ff1 JA	69	unsigned int writers;
ba5bb147	70	unsigned int files;
17374ff1 JA	71	unsigned int r_counter;
17374ff1 JA	72	unsigned int w_counter;
35f3d14d	73	struct page *tmp_page;
17374ff1 JA	74	struct fasync_struct *fasync_readers;
17374ff1 JA	75	struct fasync_struct *fasync_writers;
35f3d14d	76	struct pipe_buffer *bufs;
759c0114	77	struct user_struct *user;
c73be61c DH	78	#ifdef CONFIG_WATCH_QUEUE
	79	struct watch_queue *watch_queue;
	80	#endif
17374ff1 JA	81	};
17374ff1 JA	82
f84d7519 JA	83	/*
	84	* Note on the nesting of these functions:
	85	*
cac36bb0	86	* ->confirm()
c928f642	87	* ->try_steal()
f84d7519	88	*
c928f642 CH	89	* That is, ->try_steal() must be called on a confirmed buffer. See below for
	90	* the meaning of each operation. Also see the kerneldoc in fs/pipe.c for the
	91	* pipe and generic variants of these hooks.
f84d7519	92	*/
1da177e4	93	struct pipe_buf_operations {
0845718d JA	94	/*
	95	* ->confirm() verifies that the data in the pipe buffer is there
	96	* and that the contents are good. If the pages in the pipe belong
	97	* to a file system, we may need to wait for IO completion in this
	98	* hook. Returns 0 for good, or a negative error value in case of
b8d9e7f2	99	* error. If not present all pages are considered good.
0845718d	100	*/
cac36bb0	101	int (confirm)(struct pipe_inode_info , struct pipe_buffer *);
0845718d JA	102
	103	/*
	104	* When the contents of this pipe buffer has been completely
	105	* consumed by a reader, ->release() is called.
	106	*/
1da177e4	107	void (release)(struct pipe_inode_info , struct pipe_buffer *);
0845718d JA	108
	109	/*
	110	* Attempt to take ownership of the pipe buffer and its contents.
c928f642 CH	111	* ->try_steal() returns %true for success, in which case the contents
	112	* of the pipe (the buf->page) is locked and now completely owned by the
	113	* caller. The page may then be transferred to a different mapping, the
	114	* most often used case is insertion into different file address space
	115	* cache.
0845718d	116	*/
c928f642	117	bool (try_steal)(struct pipe_inode_info , struct pipe_buffer *);
0845718d JA	118
	119	/*
	120	* Get a reference to the pipe buffer.
	121	*/
15fab63e	122	bool (get)(struct pipe_inode_info , struct pipe_buffer *);
1da177e4 LT	123	};
1da177e4 LT	124
8cefc107 DH	125	/**
	126	* pipe_empty - Return true if the pipe is empty
	127	* @head: The pipe ring head pointer
	128	* @tail: The pipe ring tail pointer
	129	*/
	130	static inline bool pipe_empty(unsigned int head, unsigned int tail)
	131	{
	132	return head == tail;
	133	}
	134
	135	/**
	136	* pipe_occupancy - Return number of slots used in the pipe
	137	* @head: The pipe ring head pointer
	138	* @tail: The pipe ring tail pointer
	139	*/
	140	static inline unsigned int pipe_occupancy(unsigned int head, unsigned int tail)
	141	{
	142	return head - tail;
	143	}
	144
	145	/**
	146	* pipe_full - Return true if the pipe is full
	147	* @head: The pipe ring head pointer
	148	* @tail: The pipe ring tail pointer
	149	* @limit: The maximum amount of slots available.
	150	*/
	151	static inline bool pipe_full(unsigned int head, unsigned int tail,
	152	unsigned int limit)
	153	{
	154	return pipe_occupancy(head, tail) >= limit;
	155	}
	156
	157	/**
	158	* pipe_space_for_user - Return number of slots available to userspace
	159	* @head: The pipe ring head pointer
	160	* @tail: The pipe ring tail pointer
	161	* @pipe: The pipe info structure
	162	*/
	163	static inline unsigned int pipe_space_for_user(unsigned int head, unsigned int tail,
	164	struct pipe_inode_info *pipe)
	165	{
	166	unsigned int p_occupancy, p_space;
	167
	168	p_occupancy = pipe_occupancy(head, tail);
6718b6f8	169	if (p_occupancy >= pipe->max_usage)
8cefc107 DH	170	return 0;
8cefc107 DH	171	p_space = pipe->ring_size - p_occupancy;
6718b6f8 DH	172	if (p_space > pipe->max_usage)
6718b6f8 DH	173	p_space = pipe->max_usage;
8cefc107 DH	174	return p_space;
	175	}
	176
7bf2d1df MS	177	/**
	178	* pipe_buf_get - get a reference to a pipe_buffer
	179	* @pipe: the pipe that the buffer belongs to
	180	* @buf: the buffer to get a reference to
15fab63e MW	181	*
15fab63e MW	182	* Return: %true if the reference was successfully obtained.
7bf2d1df	183	*/
15fab63e	184	static inline __must_check bool pipe_buf_get(struct pipe_inode_info *pipe,
7bf2d1df MS	185	struct pipe_buffer *buf)
7bf2d1df MS	186	{
15fab63e	187	return buf->ops->get(pipe, buf);
7bf2d1df MS	188	}
7bf2d1df MS	189
a779638c MS	190	/**
	191	* pipe_buf_release - put a reference to a pipe_buffer
	192	* @pipe: the pipe that the buffer belongs to
	193	* @buf: the buffer to put a reference to
	194	*/
	195	static inline void pipe_buf_release(struct pipe_inode_info *pipe,
	196	struct pipe_buffer *buf)
	197	{
	198	const struct pipe_buf_operations *ops = buf->ops;
	199
	200	buf->ops = NULL;
	201	ops->release(pipe, buf);
	202	}
	203
fba597db MS	204	/**
	205	* pipe_buf_confirm - verify contents of the pipe buffer
	206	* @pipe: the pipe that the buffer belongs to
	207	* @buf: the buffer to confirm
	208	*/
	209	static inline int pipe_buf_confirm(struct pipe_inode_info *pipe,
	210	struct pipe_buffer *buf)
	211	{
b8d9e7f2 CH	212	if (!buf->ops->confirm)
b8d9e7f2 CH	213	return 0;
fba597db MS	214	return buf->ops->confirm(pipe, buf);
	215	}
	216
ca76f5b6	217	/**
c928f642	218	* pipe_buf_try_steal - attempt to take ownership of a pipe_buffer
ca76f5b6 MS	219	* @pipe: the pipe that the buffer belongs to
	220	* @buf: the buffer to attempt to steal
	221	*/
c928f642 CH	222	static inline bool pipe_buf_try_steal(struct pipe_inode_info *pipe,
c928f642 CH	223	struct pipe_buffer *buf)
ca76f5b6	224	{
c928f642 CH	225	if (!buf->ops->try_steal)
	226	return false;
	227	return buf->ops->try_steal(pipe, buf);
ca76f5b6 MS	228	}
ca76f5b6 MS	229
1da177e4 LT	230	/* Differs from PIPE_BUF in that PIPE_SIZE is the length of the actual
	231	memory allocation, whereas PIPE_BUF makes atomicity guarantees. */
	232	#define PIPE_SIZE PAGE_SIZE
	233
61e0d47c MS	234	/* Pipe lock and unlock operations */
	235	void pipe_lock(struct pipe_inode_info *);
	236	void pipe_unlock(struct pipe_inode_info *);
	237	void pipe_double_lock(struct pipe_inode_info , struct pipe_inode_info );
	238
4c2e4bef	239	extern unsigned int pipe_max_size;
759c0114 WT	240	extern unsigned long pipe_user_pages_hard;
759c0114 WT	241	extern unsigned long pipe_user_pages_soft;
ff9da691	242
1da177e4	243	/* Drop the inode semaphore and wait for a pipe event, atomically */
3a326a2c	244	void pipe_wait(struct pipe_inode_info *pipe);
1da177e4	245
7bee130e	246	struct pipe_inode_info *alloc_pipe_info(void);
4b8a8f1e	247	void free_pipe_info(struct pipe_inode_info *);
1da177e4	248
f84d7519	249	/* Generic pipe buffer ops functions */
15fab63e	250	bool generic_pipe_buf_get(struct pipe_inode_info , struct pipe_buffer );
c928f642	251	bool generic_pipe_buf_try_steal(struct pipe_inode_info , struct pipe_buffer );
6818173b	252	void generic_pipe_buf_release(struct pipe_inode_info , struct pipe_buffer );
f84d7519	253
28a625cb MS	254	extern const struct pipe_buf_operations nosteal_pipe_buf_ops;
28a625cb MS	255
c73be61c DH	256	#ifdef CONFIG_WATCH_QUEUE
	257	unsigned long account_pipe_buffers(struct user_struct *user,
	258	unsigned long old, unsigned long new);
	259	bool too_many_pipe_buffers_soft(unsigned long user_bufs);
	260	bool too_many_pipe_buffers_hard(unsigned long user_bufs);
	261	bool pipe_is_unprivileged_user(void);
	262	#endif
	263
35f3d14d	264	/* for F_SETPIPE_SZ and F_GETPIPE_SZ */
c73be61c DH	265	#ifdef CONFIG_WATCH_QUEUE
	266	int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots);
	267	#endif
35f3d14d	268	long pipe_fcntl(struct file *, unsigned int, unsigned long arg);
c73be61c	269	struct pipe_inode_info get_pipe_info(struct file file, bool for_splice);
c66fb347	270
e4fad8e5	271	int create_pipe_files(struct file **, int);
96e99be4	272	unsigned int round_pipe_size(unsigned long size);
e4fad8e5	273
1da177e4	274	#endif