[linux-2.6-block.git] / io_uring / kbuf.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/namei.h>
#include <linux/poll.h>
#include <linux/io_uring.h>

#include <uapi/linux/io_uring.h>

#include "io_uring_types.h"
#include "io_uring.h"
#include "opdef.h"
#include "kbuf.h"

#define IO_BUFFER_LIST_BUF_PER_PAGE (PAGE_SIZE / sizeof(struct io_uring_buf))

#define BGID_ARRAY	64

struct io_provide_buf {
	struct file			*file;
	__u64				addr;
	__u32				len;
	__u32				bgid;
	__u16				nbufs;
	__u16				bid;
};

static inline struct io_buffer_list *io_buffer_get_list(struct io_ring_ctx *ctx,
							unsigned int bgid)
{
	if (ctx->io_bl && bgid < BGID_ARRAY)
		return &ctx->io_bl[bgid];

	return xa_load(&ctx->io_bl_xa, bgid);
}

void __io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags)
{
	struct io_ring_ctx *ctx = req->ctx;
	struct io_buffer_list *bl;
	struct io_buffer *buf;

	/*
	 * We don't need to recycle for REQ_F_BUFFER_RING, we can just clear
	 * the flag and hence ensure that bl->head doesn't get incremented.
	 * If the tail has already been incremented, hang on to it.
	 */
	if (req->flags & REQ_F_BUFFER_RING) {
		if (req->buf_list) {
			if (req->flags & REQ_F_PARTIAL_IO) {
				req->buf_list->head++;
				req->buf_list = NULL;
			} else {
				req->buf_index = req->buf_list->bgid;
				req->flags &= ~REQ_F_BUFFER_RING;
			}
		}
		return;
	}

	io_ring_submit_lock(ctx, issue_flags);

	buf = req->kbuf;
	bl = io_buffer_get_list(ctx, buf->bgid);
	list_add(&buf->list, &bl->buf_list);
	req->flags &= ~REQ_F_BUFFER_SELECTED;
	req->buf_index = buf->bgid;

	io_ring_submit_unlock(ctx, issue_flags);
}

static int io_buffer_add_list(struct io_ring_ctx *ctx,
			      struct io_buffer_list *bl, unsigned int bgid)
{
	bl->bgid = bgid;
	if (bgid < BGID_ARRAY)
		return 0;

	return xa_err(xa_store(&ctx->io_bl_xa, bgid, bl, GFP_KERNEL));
}

static void __user *io_provided_buffer_select(struct io_kiocb *req, size_t *len,
					      struct io_buffer_list *bl)
{
	if (!list_empty(&bl->buf_list)) {
		struct io_buffer *kbuf;

		kbuf = list_first_entry(&bl->buf_list, struct io_buffer, list);
		list_del(&kbuf->list);
		if (*len > kbuf->len)
			*len = kbuf->len;
		req->flags |= REQ_F_BUFFER_SELECTED;
		req->kbuf = kbuf;
		req->buf_index = kbuf->bid;
		return u64_to_user_ptr(kbuf->addr);
	}
	return NULL;
}

static void __user *io_ring_buffer_select(struct io_kiocb *req, size_t *len,
					  struct io_buffer_list *bl,
					  unsigned int issue_flags)
{
	struct io_uring_buf_ring *br = bl->buf_ring;
	struct io_uring_buf *buf;
	__u16 head = bl->head;

	if (unlikely(smp_load_acquire(&br->tail) == head))
		return NULL;

	head &= bl->mask;
	if (head < IO_BUFFER_LIST_BUF_PER_PAGE) {
		buf = &br->bufs[head];
	} else {
		int off = head & (IO_BUFFER_LIST_BUF_PER_PAGE - 1);
		int index = head / IO_BUFFER_LIST_BUF_PER_PAGE;
		buf = page_address(bl->buf_pages[index]);
		buf += off;
	}
	if (*len > buf->len)
		*len = buf->len;
	req->flags |= REQ_F_BUFFER_RING;
	req->buf_list = bl;
	req->buf_index = buf->bid;

	if (issue_flags & IO_URING_F_UNLOCKED || !file_can_poll(req->file)) {
		/*
		 * If we came in unlocked, we have no choice but to consume the
		 * buffer here. This does mean it'll be pinned until the IO
		 * completes. But coming in unlocked means we're in io-wq
		 * context, hence there should be no further retry. For the
		 * locked case, the caller must ensure to call the commit when
		 * the transfer completes (or if we get -EAGAIN and must poll
		 * or retry).
		 */
		req->buf_list = NULL;
		bl->head++;
	}
	return u64_to_user_ptr(buf->addr);
}

void __user *io_buffer_select(struct io_kiocb *req, size_t *len,
			      unsigned int issue_flags)
{
	struct io_ring_ctx *ctx = req->ctx;
	struct io_buffer_list *bl;
	void __user *ret = NULL;

	io_ring_submit_lock(req->ctx, issue_flags);

	bl = io_buffer_get_list(ctx, req->buf_index);
	if (likely(bl)) {
		if (bl->buf_nr_pages)
			ret = io_ring_buffer_select(req, len, bl, issue_flags);
		else
			ret = io_provided_buffer_select(req, len, bl);
	}
	io_ring_submit_unlock(req->ctx, issue_flags);
	return ret;
}

static __cold int io_init_bl_list(struct io_ring_ctx *ctx)
{
	int i;

	ctx->io_bl = kcalloc(BGID_ARRAY, sizeof(struct io_buffer_list),
				GFP_KERNEL);
	if (!ctx->io_bl)
		return -ENOMEM;

	for (i = 0; i < BGID_ARRAY; i++) {
		INIT_LIST_HEAD(&ctx->io_bl[i].buf_list);
		ctx->io_bl[i].bgid = i;
	}

	return 0;
}

static int __io_remove_buffers(struct io_ring_ctx *ctx,
			       struct io_buffer_list *bl, unsigned nbufs)
{
	unsigned i = 0;

	/* shouldn't happen */
	if (!nbufs)
		return 0;

	if (bl->buf_nr_pages) {
		int j;

		i = bl->buf_ring->tail - bl->head;
		for (j = 0; j < bl->buf_nr_pages; j++)
			unpin_user_page(bl->buf_pages[j]);
		kvfree(bl->buf_pages);
		bl->buf_pages = NULL;
		bl->buf_nr_pages = 0;
		/* make sure it's seen as empty */
		INIT_LIST_HEAD(&bl->buf_list);
		return i;
	}

	/* the head kbuf is the list itself */
	while (!list_empty(&bl->buf_list)) {
		struct io_buffer *nxt;

		nxt = list_first_entry(&bl->buf_list, struct io_buffer, list);
		list_del(&nxt->list);
		if (++i == nbufs)
			return i;
		cond_resched();
	}
	i++;

	return i;
}

void io_destroy_buffers(struct io_ring_ctx *ctx)
{
	struct io_buffer_list *bl;
	unsigned long index;
	int i;

	for (i = 0; i < BGID_ARRAY; i++) {
		if (!ctx->io_bl)
			break;
		__io_remove_buffers(ctx, &ctx->io_bl[i], -1U);
	}

	xa_for_each(&ctx->io_bl_xa, index, bl) {
		xa_erase(&ctx->io_bl_xa, bl->bgid);
		__io_remove_buffers(ctx, bl, -1U);
		kfree(bl);
	}

	while (!list_empty(&ctx->io_buffers_pages)) {
		struct page *page;

		page = list_first_entry(&ctx->io_buffers_pages, struct page, lru);
		list_del_init(&page->lru);
		__free_page(page);
	}
}

int io_remove_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	struct io_provide_buf *p = io_kiocb_to_cmd(req);
	u64 tmp;

	if (sqe->rw_flags || sqe->addr || sqe->len || sqe->off ||
	    sqe->splice_fd_in)
		return -EINVAL;

	tmp = READ_ONCE(sqe->fd);
	if (!tmp || tmp > USHRT_MAX)
		return -EINVAL;

	memset(p, 0, sizeof(*p));
	p->nbufs = tmp;
	p->bgid = READ_ONCE(sqe->buf_group);
	return 0;
}

int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags)
{
	struct io_provide_buf *p = io_kiocb_to_cmd(req);
	struct io_ring_ctx *ctx = req->ctx;
	struct io_buffer_list *bl;
	int ret = 0;

	io_ring_submit_lock(ctx, issue_flags);

	ret = -ENOENT;
	bl = io_buffer_get_list(ctx, p->bgid);
	if (bl) {
		ret = -EINVAL;
		/* can't use provide/remove buffers command on mapped buffers */
		if (!bl->buf_nr_pages)
			ret = __io_remove_buffers(ctx, bl, p->nbufs);
	}
	if (ret < 0)
		req_set_fail(req);

	/* complete before unlock, IOPOLL may need the lock */
	io_req_set_res(req, ret, 0);
	__io_req_complete(req, issue_flags);
	io_ring_submit_unlock(ctx, issue_flags);
	return IOU_ISSUE_SKIP_COMPLETE;
}

int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	unsigned long size, tmp_check;
	struct io_provide_buf *p = io_kiocb_to_cmd(req);
	u64 tmp;

	if (sqe->rw_flags || sqe->splice_fd_in)
		return -EINVAL;

	tmp = READ_ONCE(sqe->fd);
	if (!tmp || tmp > USHRT_MAX)
		return -E2BIG;
	p->nbufs = tmp;
	p->addr = READ_ONCE(sqe->addr);
	p->len = READ_ONCE(sqe->len);

	if (check_mul_overflow((unsigned long)p->len, (unsigned long)p->nbufs,
				&size))
		return -EOVERFLOW;
	if (check_add_overflow((unsigned long)p->addr, size, &tmp_check))
		return -EOVERFLOW;

	size = (unsigned long)p->len * p->nbufs;
	if (!access_ok(u64_to_user_ptr(p->addr), size))
		return -EFAULT;

	p->bgid = READ_ONCE(sqe->buf_group);
	tmp = READ_ONCE(sqe->off);
	if (tmp > USHRT_MAX)
		return -E2BIG;
	p->bid = tmp;
	return 0;
}

static int io_refill_buffer_cache(struct io_ring_ctx *ctx)
{
	struct io_buffer *buf;
	struct page *page;
	int bufs_in_page;

	/*
	 * Completions that don't happen inline (eg not under uring_lock) will
	 * add to ->io_buffers_comp. If we don't have any free buffers, check
	 * the completion list and splice those entries first.
	 */
	if (!list_empty_careful(&ctx->io_buffers_comp)) {
		spin_lock(&ctx->completion_lock);
		if (!list_empty(&ctx->io_buffers_comp)) {
			list_splice_init(&ctx->io_buffers_comp,
						&ctx->io_buffers_cache);
			spin_unlock(&ctx->completion_lock);
			return 0;
		}
		spin_unlock(&ctx->completion_lock);
	}

	/*
	 * No free buffers and no completion entries either. Allocate a new
	 * page worth of buffer entries and add those to our freelist.
	 */
	page = alloc_page(GFP_KERNEL_ACCOUNT);
	if (!page)
		return -ENOMEM;

	list_add(&page->lru, &ctx->io_buffers_pages);

	buf = page_address(page);
	bufs_in_page = PAGE_SIZE / sizeof(*buf);
	while (bufs_in_page) {
		list_add_tail(&buf->list, &ctx->io_buffers_cache);
		buf++;
		bufs_in_page--;
	}

	return 0;
}

static int io_add_buffers(struct io_ring_ctx *ctx, struct io_provide_buf *pbuf,
			  struct io_buffer_list *bl)
{
	struct io_buffer *buf;
	u64 addr = pbuf->addr;
	int i, bid = pbuf->bid;

	for (i = 0; i < pbuf->nbufs; i++) {
		if (list_empty(&ctx->io_buffers_cache) &&
		    io_refill_buffer_cache(ctx))
			break;
		buf = list_first_entry(&ctx->io_buffers_cache, struct io_buffer,
					list);
		list_move_tail(&buf->list, &bl->buf_list);
		buf->addr = addr;
		buf->len = min_t(__u32, pbuf->len, MAX_RW_COUNT);
		buf->bid = bid;
		buf->bgid = pbuf->bgid;
		addr += pbuf->len;
		bid++;
		cond_resched();
	}

	return i ? 0 : -ENOMEM;
}

int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)
{
	struct io_provide_buf *p = io_kiocb_to_cmd(req);
	struct io_ring_ctx *ctx = req->ctx;
	struct io_buffer_list *bl;
	int ret = 0;

	io_ring_submit_lock(ctx, issue_flags);

	if (unlikely(p->bgid < BGID_ARRAY && !ctx->io_bl)) {
		ret = io_init_bl_list(ctx);
		if (ret)
			goto err;
	}

	bl = io_buffer_get_list(ctx, p->bgid);
	if (unlikely(!bl)) {
		bl = kzalloc(sizeof(*bl), GFP_KERNEL);
		if (!bl) {
			ret = -ENOMEM;
			goto err;
		}
		INIT_LIST_HEAD(&bl->buf_list);
		ret = io_buffer_add_list(ctx, bl, p->bgid);
		if (ret) {
			kfree(bl);
			goto err;
		}
	}
	/* can't add buffers via this command for a mapped buffer ring */
	if (bl->buf_nr_pages) {
		ret = -EINVAL;
		goto err;
	}

	ret = io_add_buffers(ctx, p, bl);
err:
	if (ret < 0)
		req_set_fail(req);
	/* complete before unlock, IOPOLL may need the lock */
	io_req_set_res(req, ret, 0);
	__io_req_complete(req, issue_flags);
	io_ring_submit_unlock(ctx, issue_flags);
	return IOU_ISSUE_SKIP_COMPLETE;
}

int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg)
{
	struct io_uring_buf_ring *br;
	struct io_uring_buf_reg reg;
	struct io_buffer_list *bl, *free_bl = NULL;
	struct page **pages;
	int nr_pages;

	if (copy_from_user(&reg, arg, sizeof(reg)))
		return -EFAULT;

	if (reg.pad || reg.resv[0] || reg.resv[1] || reg.resv[2])
		return -EINVAL;
	if (!reg.ring_addr)
		return -EFAULT;
	if (reg.ring_addr & ~PAGE_MASK)
		return -EINVAL;
	if (!is_power_of_2(reg.ring_entries))
		return -EINVAL;

	/* cannot disambiguate full vs empty due to head/tail size */
	if (reg.ring_entries >= 65536)
		return -EINVAL;

	if (unlikely(reg.bgid < BGID_ARRAY && !ctx->io_bl)) {
		int ret = io_init_bl_list(ctx);
		if (ret)
			return ret;
	}

	bl = io_buffer_get_list(ctx, reg.bgid);
	if (bl) {
		/* if mapped buffer ring OR classic exists, don't allow */
		if (bl->buf_nr_pages || !list_empty(&bl->buf_list))
			return -EEXIST;
	} else {
		free_bl = bl = kzalloc(sizeof(*bl), GFP_KERNEL);
		if (!bl)
			return -ENOMEM;
	}

	pages = io_pin_pages(reg.ring_addr,
			     struct_size(br, bufs, reg.ring_entries),
			     &nr_pages);
	if (IS_ERR(pages)) {
		kfree(free_bl);
		return PTR_ERR(pages);
	}

	br = page_address(pages[0]);
	bl->buf_pages = pages;
	bl->buf_nr_pages = nr_pages;
	bl->nr_entries = reg.ring_entries;
	bl->buf_ring = br;
	bl->mask = reg.ring_entries - 1;
	io_buffer_add_list(ctx, bl, reg.bgid);
	return 0;
}

int io_unregister_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg)
{
	struct io_uring_buf_reg reg;
	struct io_buffer_list *bl;

	if (copy_from_user(&reg, arg, sizeof(reg)))
		return -EFAULT;
	if (reg.pad || reg.resv[0] || reg.resv[1] || reg.resv[2])
		return -EINVAL;

	bl = io_buffer_get_list(ctx, reg.bgid);
	if (!bl)
		return -ENOENT;
	if (!bl->buf_nr_pages)
		return -EINVAL;

	__io_remove_buffers(ctx, bl, -1U);
	if (bl->bgid >= BGID_ARRAY) {
		xa_erase(&ctx->io_bl_xa, bl->bgid);
		kfree(bl);
	}
	return 0;
}
Commit	Line	Data
3b77495a JA	1	// SPDX-License-Identifier: GPL-2.0
	2	#include <linux/kernel.h>
	3	#include <linux/errno.h>
	4	#include <linux/fs.h>
	5	#include <linux/file.h>
	6	#include <linux/mm.h>
	7	#include <linux/slab.h>
	8	#include <linux/namei.h>
	9	#include <linux/poll.h>
	10	#include <linux/io_uring.h>
	11
	12	#include <uapi/linux/io_uring.h>
	13
	14	#include "io_uring_types.h"
	15	#include "io_uring.h"
	16	#include "opdef.h"
	17	#include "kbuf.h"
	18
	19	#define IO_BUFFER_LIST_BUF_PER_PAGE (PAGE_SIZE / sizeof(struct io_uring_buf))
	20
	21	#define BGID_ARRAY 64
	22
	23	struct io_provide_buf {
	24	struct file *file;
	25	__u64 addr;
	26	__u32 len;
	27	__u32 bgid;
	28	__u16 nbufs;
	29	__u16 bid;
	30	};
	31
	32	static inline struct io_buffer_list io_buffer_get_list(struct io_ring_ctx ctx,
	33	unsigned int bgid)
	34	{
	35	if (ctx->io_bl && bgid < BGID_ARRAY)
	36	return &ctx->io_bl[bgid];
	37
	38	return xa_load(&ctx->io_bl_xa, bgid);
	39	}
	40
	41	void __io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags)
	42	{
	43	struct io_ring_ctx *ctx = req->ctx;
	44	struct io_buffer_list *bl;
	45	struct io_buffer *buf;
	46
	47	/*
	48	* We don't need to recycle for REQ_F_BUFFER_RING, we can just clear
	49	* the flag and hence ensure that bl->head doesn't get incremented.
	50	* If the tail has already been incremented, hang on to it.
	51	*/
	52	if (req->flags & REQ_F_BUFFER_RING) {
	53	if (req->buf_list) {
	54	if (req->flags & REQ_F_PARTIAL_IO) {
	55	req->buf_list->head++;
	56	req->buf_list = NULL;
	57	} else {
	58	req->buf_index = req->buf_list->bgid;
	59	req->flags &= ~REQ_F_BUFFER_RING;
	60	}
	61	}
	62	return;
	63	}
	64
65	io_ring_submit_lock(ctx, issue_flags);
66
67	buf = req->kbuf;
68	bl = io_buffer_get_list(ctx, buf->bgid);
69	list_add(&buf->list, &bl->buf_list);
70	req->flags &= ~REQ_F_BUFFER_SELECTED;
71	req->buf_index = buf->bgid;
72
73	io_ring_submit_unlock(ctx, issue_flags);
74	}
75
76	static int io_buffer_add_list(struct io_ring_ctx *ctx,
77	struct io_buffer_list *bl, unsigned int bgid)
78	{
79	bl->bgid = bgid;
80	if (bgid < BGID_ARRAY)
81	return 0;
82
83	return xa_err(xa_store(&ctx->io_bl_xa, bgid, bl, GFP_KERNEL));
84	}
85
86	static void __user io_provided_buffer_select(struct io_kiocb req, size_t *len,
87	struct io_buffer_list *bl)
88	{
89	if (!list_empty(&bl->buf_list)) {
90	struct io_buffer *kbuf;
91
92	kbuf = list_first_entry(&bl->buf_list, struct io_buffer, list);
93	list_del(&kbuf->list);
94	if (*len > kbuf->len)
95	*len = kbuf->len;
96	req->flags \|= REQ_F_BUFFER_SELECTED;
97	req->kbuf = kbuf;
98	req->buf_index = kbuf->bid;
99	return u64_to_user_ptr(kbuf->addr);
100	}
101	return NULL;
102	}
103
104	static void __user io_ring_buffer_select(struct io_kiocb req, size_t *len,
105	struct io_buffer_list *bl,
106	unsigned int issue_flags)
107	{
108	struct io_uring_buf_ring *br = bl->buf_ring;
109	struct io_uring_buf *buf;
110	__u16 head = bl->head;
111
112	if (unlikely(smp_load_acquire(&br->tail) == head))
113	return NULL;
114
115	head &= bl->mask;
116	if (head < IO_BUFFER_LIST_BUF_PER_PAGE) {
117	buf = &br->bufs[head];
118	} else {
119	int off = head & (IO_BUFFER_LIST_BUF_PER_PAGE - 1);
120	int index = head / IO_BUFFER_LIST_BUF_PER_PAGE;
121	buf = page_address(bl->buf_pages[index]);
122	buf += off;
123	}
124	if (*len > buf->len)
125	*len = buf->len;
126	req->flags \|= REQ_F_BUFFER_RING;
127	req->buf_list = bl;
128	req->buf_index = buf->bid;
129
130	if (issue_flags & IO_URING_F_UNLOCKED \|\| !file_can_poll(req->file)) {
131	/*
132	* If we came in unlocked, we have no choice but to consume the
133	* buffer here. This does mean it'll be pinned until the IO
134	* completes. But coming in unlocked means we're in io-wq
135	* context, hence there should be no further retry. For the
136	* locked case, the caller must ensure to call the commit when
137	* the transfer completes (or if we get -EAGAIN and must poll
138	* or retry).
139	*/
140	req->buf_list = NULL;
141	bl->head++;
142	}
143	return u64_to_user_ptr(buf->addr);
144	}
145
146	void __user io_buffer_select(struct io_kiocb req, size_t *len,
147	unsigned int issue_flags)
148	{
149	struct io_ring_ctx *ctx = req->ctx;
150	struct io_buffer_list *bl;
151	void __user *ret = NULL;
152
153	io_ring_submit_lock(req->ctx, issue_flags);
154
155	bl = io_buffer_get_list(ctx, req->buf_index);
156	if (likely(bl)) {
157	if (bl->buf_nr_pages)
158	ret = io_ring_buffer_select(req, len, bl, issue_flags);
159	else
160	ret = io_provided_buffer_select(req, len, bl);
161	}
162	io_ring_submit_unlock(req->ctx, issue_flags);
163	return ret;
164	}
165
166	static __cold int io_init_bl_list(struct io_ring_ctx *ctx)
167	{
168	int i;
169
170	ctx->io_bl = kcalloc(BGID_ARRAY, sizeof(struct io_buffer_list),
171	GFP_KERNEL);
172	if (!ctx->io_bl)
173	return -ENOMEM;
174
175	for (i = 0; i < BGID_ARRAY; i++) {
176	INIT_LIST_HEAD(&ctx->io_bl[i].buf_list);
177	ctx->io_bl[i].bgid = i;
178	}
179
180	return 0;
181	}
182
183	static int __io_remove_buffers(struct io_ring_ctx *ctx,
184	struct io_buffer_list *bl, unsigned nbufs)
185	{
186	unsigned i = 0;
187
188	/* shouldn't happen */
189	if (!nbufs)
190	return 0;
191
192	if (bl->buf_nr_pages) {
193	int j;
194
195	i = bl->buf_ring->tail - bl->head;
196	for (j = 0; j < bl->buf_nr_pages; j++)
197	unpin_user_page(bl->buf_pages[j]);
198	kvfree(bl->buf_pages);
199	bl->buf_pages = NULL;
200	bl->buf_nr_pages = 0;
201	/* make sure it's seen as empty */
202	INIT_LIST_HEAD(&bl->buf_list);
203	return i;
204	}
205
206	/* the head kbuf is the list itself */
207	while (!list_empty(&bl->buf_list)) {
208	struct io_buffer *nxt;
209
210	nxt = list_first_entry(&bl->buf_list, struct io_buffer, list);
211	list_del(&nxt->list);
212	if (++i == nbufs)
213	return i;
214	cond_resched();
215	}
216	i++;
217
218	return i;
219	}
220
221	void io_destroy_buffers(struct io_ring_ctx *ctx)
222	{
223	struct io_buffer_list *bl;
224	unsigned long index;
225	int i;
226
227	for (i = 0; i < BGID_ARRAY; i++) {
228	if (!ctx->io_bl)
229	break;
230	__io_remove_buffers(ctx, &ctx->io_bl[i], -1U);
231	}
232
233	xa_for_each(&ctx->io_bl_xa, index, bl) {
234	xa_erase(&ctx->io_bl_xa, bl->bgid);
235	__io_remove_buffers(ctx, bl, -1U);
236	kfree(bl);
237	}
238
239	while (!list_empty(&ctx->io_buffers_pages)) {
240	struct page *page;
241
242	page = list_first_entry(&ctx->io_buffers_pages, struct page, lru);
243	list_del_init(&page->lru);
244	__free_page(page);
245	}
246	}
247
248	int io_remove_buffers_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
249	{
250	struct io_provide_buf *p = io_kiocb_to_cmd(req);
251	u64 tmp;
252
253	if (sqe->rw_flags \|\| sqe->addr \|\| sqe->len \|\| sqe->off \|\|
254	sqe->splice_fd_in)
255	return -EINVAL;
256
257	tmp = READ_ONCE(sqe->fd);
258	if (!tmp \|\| tmp > USHRT_MAX)
259	return -EINVAL;
260
261	memset(p, 0, sizeof(*p));
262	p->nbufs = tmp;
263	p->bgid = READ_ONCE(sqe->buf_group);
264	return 0;
265	}
266
267	int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags)
268	{
269	struct io_provide_buf *p = io_kiocb_to_cmd(req);
270	struct io_ring_ctx *ctx = req->ctx;
271	struct io_buffer_list *bl;
272	int ret = 0;
273
274	io_ring_submit_lock(ctx, issue_flags);
275
276	ret = -ENOENT;
277	bl = io_buffer_get_list(ctx, p->bgid);
278	if (bl) {
279	ret = -EINVAL;
280	/* can't use provide/remove buffers command on mapped buffers */
281	if (!bl->buf_nr_pages)
282	ret = __io_remove_buffers(ctx, bl, p->nbufs);
283	}
284	if (ret < 0)
285	req_set_fail(req);
286
287	/* complete before unlock, IOPOLL may need the lock */
288	io_req_set_res(req, ret, 0);
289	__io_req_complete(req, issue_flags);
290	io_ring_submit_unlock(ctx, issue_flags);
291	return IOU_ISSUE_SKIP_COMPLETE;
292	}
293
294	int io_provide_buffers_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
295	{
296	unsigned long size, tmp_check;
297	struct io_provide_buf *p = io_kiocb_to_cmd(req);
298	u64 tmp;
299
300	if (sqe->rw_flags \|\| sqe->splice_fd_in)
301	return -EINVAL;
302
303	tmp = READ_ONCE(sqe->fd);
304	if (!tmp \|\| tmp > USHRT_MAX)
305	return -E2BIG;
306	p->nbufs = tmp;
307	p->addr = READ_ONCE(sqe->addr);
308	p->len = READ_ONCE(sqe->len);
309
310	if (check_mul_overflow((unsigned long)p->len, (unsigned long)p->nbufs,
311	&size))
312	return -EOVERFLOW;
313	if (check_add_overflow((unsigned long)p->addr, size, &tmp_check))
314	return -EOVERFLOW;
315
316	size = (unsigned long)p->len * p->nbufs;
317	if (!access_ok(u64_to_user_ptr(p->addr), size))
318	return -EFAULT;
319
320	p->bgid = READ_ONCE(sqe->buf_group);
321	tmp = READ_ONCE(sqe->off);
322	if (tmp > USHRT_MAX)
323	return -E2BIG;
324	p->bid = tmp;
325	return 0;
326	}
327
328	static int io_refill_buffer_cache(struct io_ring_ctx *ctx)
329	{
330	struct io_buffer *buf;
331	struct page *page;
332	int bufs_in_page;
333
334	/*
335	* Completions that don't happen inline (eg not under uring_lock) will
336	* add to ->io_buffers_comp. If we don't have any free buffers, check
337	* the completion list and splice those entries first.
338	*/
339	if (!list_empty_careful(&ctx->io_buffers_comp)) {
340	spin_lock(&ctx->completion_lock);
341	if (!list_empty(&ctx->io_buffers_comp)) {
342	list_splice_init(&ctx->io_buffers_comp,
343	&ctx->io_buffers_cache);
344	spin_unlock(&ctx->completion_lock);
345	return 0;
346	}
347	spin_unlock(&ctx->completion_lock);
348	}
349
350	/*
351	* No free buffers and no completion entries either. Allocate a new
352	* page worth of buffer entries and add those to our freelist.
353	*/
354	page = alloc_page(GFP_KERNEL_ACCOUNT);
355	if (!page)
356	return -ENOMEM;
357
358	list_add(&page->lru, &ctx->io_buffers_pages);
359
360	buf = page_address(page);
361	bufs_in_page = PAGE_SIZE / sizeof(*buf);
362	while (bufs_in_page) {
363	list_add_tail(&buf->list, &ctx->io_buffers_cache);
364	buf++;
365	bufs_in_page--;
366	}
367
368	return 0;
369	}
370
371	static int io_add_buffers(struct io_ring_ctx ctx, struct io_provide_buf pbuf,
372	struct io_buffer_list *bl)
373	{
374	struct io_buffer *buf;
375	u64 addr = pbuf->addr;
376	int i, bid = pbuf->bid;
377
378	for (i = 0; i < pbuf->nbufs; i++) {
379	if (list_empty(&ctx->io_buffers_cache) &&
380	io_refill_buffer_cache(ctx))
381	break;
382	buf = list_first_entry(&ctx->io_buffers_cache, struct io_buffer,
383	list);
384	list_move_tail(&buf->list, &bl->buf_list);
385	buf->addr = addr;
386	buf->len = min_t(__u32, pbuf->len, MAX_RW_COUNT);
387	buf->bid = bid;
388	buf->bgid = pbuf->bgid;
389	addr += pbuf->len;
390	bid++;
391	cond_resched();
392	}
393
394	return i ? 0 : -ENOMEM;
395	}
396
397	int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)
398	{
399	struct io_provide_buf *p = io_kiocb_to_cmd(req);
400	struct io_ring_ctx *ctx = req->ctx;
401	struct io_buffer_list *bl;
402	int ret = 0;
403
404	io_ring_submit_lock(ctx, issue_flags);
405
406	if (unlikely(p->bgid < BGID_ARRAY && !ctx->io_bl)) {
407	ret = io_init_bl_list(ctx);
408	if (ret)
409	goto err;
410	}
411
412	bl = io_buffer_get_list(ctx, p->bgid);
413	if (unlikely(!bl)) {
414	bl = kzalloc(sizeof(*bl), GFP_KERNEL);
415	if (!bl) {
416	ret = -ENOMEM;
417	goto err;
418	}
419	INIT_LIST_HEAD(&bl->buf_list);
420	ret = io_buffer_add_list(ctx, bl, p->bgid);
421	if (ret) {
422	kfree(bl);
423	goto err;
424	}
425	}
426	/* can't add buffers via this command for a mapped buffer ring */
427	if (bl->buf_nr_pages) {
428	ret = -EINVAL;
429	goto err;
430	}
431
432	ret = io_add_buffers(ctx, p, bl);
433	err:
434	if (ret < 0)
435	req_set_fail(req);
436	/* complete before unlock, IOPOLL may need the lock */
437	io_req_set_res(req, ret, 0);
438	__io_req_complete(req, issue_flags);
439	io_ring_submit_unlock(ctx, issue_flags);
440	return IOU_ISSUE_SKIP_COMPLETE;
441	}
442
443	int io_register_pbuf_ring(struct io_ring_ctx ctx, void __user arg)
444	{
445	struct io_uring_buf_ring *br;
446	struct io_uring_buf_reg reg;
447	struct io_buffer_list bl, free_bl = NULL;
448	struct page **pages;
449	int nr_pages;
450
451	if (copy_from_user(&reg, arg, sizeof(reg)))
452	return -EFAULT;
453
454	if (reg.pad \|\| reg.resv[0] \|\| reg.resv[1] \|\| reg.resv[2])
455	return -EINVAL;
456	if (!reg.ring_addr)
457	return -EFAULT;
458	if (reg.ring_addr & ~PAGE_MASK)
459	return -EINVAL;
460	if (!is_power_of_2(reg.ring_entries))
461	return -EINVAL;
462
463	/* cannot disambiguate full vs empty due to head/tail size */
464	if (reg.ring_entries >= 65536)
465	return -EINVAL;
466
467	if (unlikely(reg.bgid < BGID_ARRAY && !ctx->io_bl)) {
468	int ret = io_init_bl_list(ctx);
469	if (ret)
470	return ret;
471	}
472
473	bl = io_buffer_get_list(ctx, reg.bgid);
474	if (bl) {
475	/* if mapped buffer ring OR classic exists, don't allow */
476	if (bl->buf_nr_pages \|\| !list_empty(&bl->buf_list))
477	return -EEXIST;
478	} else {
479	free_bl = bl = kzalloc(sizeof(*bl), GFP_KERNEL);
480	if (!bl)
481	return -ENOMEM;
482	}
483
484	pages = io_pin_pages(reg.ring_addr,
485	struct_size(br, bufs, reg.ring_entries),
486	&nr_pages);
487	if (IS_ERR(pages)) {
488	kfree(free_bl);
489	return PTR_ERR(pages);
490	}
491
492	br = page_address(pages[0]);
493	bl->buf_pages = pages;
494	bl->buf_nr_pages = nr_pages;
495	bl->nr_entries = reg.ring_entries;
496	bl->buf_ring = br;
497	bl->mask = reg.ring_entries - 1;
498	io_buffer_add_list(ctx, bl, reg.bgid);
499	return 0;
500	}
501
502	int io_unregister_pbuf_ring(struct io_ring_ctx ctx, void __user arg)
503	{
504	struct io_uring_buf_reg reg;
505	struct io_buffer_list *bl;
506
507	if (copy_from_user(&reg, arg, sizeof(reg)))
508	return -EFAULT;
509	if (reg.pad \|\| reg.resv[0] \|\| reg.resv[1] \|\| reg.resv[2])
510	return -EINVAL;
511
512	bl = io_buffer_get_list(ctx, reg.bgid);
513	if (!bl)
514	return -ENOENT;
515	if (!bl->buf_nr_pages)
516	return -EINVAL;
517
518	__io_remove_buffers(ctx, bl, -1U);
519	if (bl->bgid >= BGID_ARRAY) {
520	xa_erase(&ctx->io_bl_xa, bl->bgid);
521	kfree(bl);
522	}
523	return 0;
524	}