[linux-block.git] / io_uring / kbuf.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/namei.h>
#include <linux/poll.h>
#include <linux/io_uring.h>

#include <uapi/linux/io_uring.h>

#include "io_uring.h"
#include "opdef.h"
#include "kbuf.h"

#define IO_BUFFER_LIST_BUF_PER_PAGE (PAGE_SIZE / sizeof(struct io_uring_buf))

#define BGID_ARRAY	64

struct io_provide_buf {
	struct file			*file;
	__u64				addr;
	__u32				len;
	__u32				bgid;
	__u16				nbufs;
	__u16				bid;
};

static inline struct io_buffer_list *io_buffer_get_list(struct io_ring_ctx *ctx,
							unsigned int bgid)
{
	if (ctx->io_bl && bgid < BGID_ARRAY)
		return &ctx->io_bl[bgid];

	return xa_load(&ctx->io_bl_xa, bgid);
}

static int io_buffer_add_list(struct io_ring_ctx *ctx,
			      struct io_buffer_list *bl, unsigned int bgid)
{
	bl->bgid = bgid;
	if (bgid < BGID_ARRAY)
		return 0;

	return xa_err(xa_store(&ctx->io_bl_xa, bgid, bl, GFP_KERNEL));
}

void io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags)
{
	struct io_ring_ctx *ctx = req->ctx;
	struct io_buffer_list *bl;
	struct io_buffer *buf;

	/*
	 * For legacy provided buffer mode, don't recycle if we already did
	 * IO to this buffer. For ring-mapped provided buffer mode, we should
	 * increment ring->head to explicitly monopolize the buffer to avoid
	 * multiple use.
	 */
	if (req->flags & REQ_F_PARTIAL_IO)
		return;

	io_ring_submit_lock(ctx, issue_flags);

	buf = req->kbuf;
	bl = io_buffer_get_list(ctx, buf->bgid);
	list_add(&buf->list, &bl->buf_list);
	req->flags &= ~REQ_F_BUFFER_SELECTED;
	req->buf_index = buf->bgid;

	io_ring_submit_unlock(ctx, issue_flags);
	return;
}

unsigned int __io_put_kbuf(struct io_kiocb *req, unsigned issue_flags)
{
	unsigned int cflags;

	/*
	 * We can add this buffer back to two lists:
	 *
	 * 1) The io_buffers_cache list. This one is protected by the
	 *    ctx->uring_lock. If we already hold this lock, add back to this
	 *    list as we can grab it from issue as well.
	 * 2) The io_buffers_comp list. This one is protected by the
	 *    ctx->completion_lock.
	 *
	 * We migrate buffers from the comp_list to the issue cache list
	 * when we need one.
	 */
	if (req->flags & REQ_F_BUFFER_RING) {
		/* no buffers to recycle for this case */
		cflags = __io_put_kbuf_list(req, NULL);
	} else if (issue_flags & IO_URING_F_UNLOCKED) {
		struct io_ring_ctx *ctx = req->ctx;

		spin_lock(&ctx->completion_lock);
		cflags = __io_put_kbuf_list(req, &ctx->io_buffers_comp);
		spin_unlock(&ctx->completion_lock);
	} else {
		lockdep_assert_held(&req->ctx->uring_lock);

		cflags = __io_put_kbuf_list(req, &req->ctx->io_buffers_cache);
	}
	return cflags;
}

static void __user *io_provided_buffer_select(struct io_kiocb *req, size_t *len,
					      struct io_buffer_list *bl)
{
	if (!list_empty(&bl->buf_list)) {
		struct io_buffer *kbuf;

		kbuf = list_first_entry(&bl->buf_list, struct io_buffer, list);
		list_del(&kbuf->list);
		if (*len == 0 || *len > kbuf->len)
			*len = kbuf->len;
		req->flags |= REQ_F_BUFFER_SELECTED;
		req->kbuf = kbuf;
		req->buf_index = kbuf->bid;
		return u64_to_user_ptr(kbuf->addr);
	}
	return NULL;
}

static void __user *io_ring_buffer_select(struct io_kiocb *req, size_t *len,
					  struct io_buffer_list *bl,
					  unsigned int issue_flags)
{
	struct io_uring_buf_ring *br = bl->buf_ring;
	struct io_uring_buf *buf;
	__u16 head = bl->head;

	if (unlikely(smp_load_acquire(&br->tail) == head))
		return NULL;

	head &= bl->mask;
	if (head < IO_BUFFER_LIST_BUF_PER_PAGE) {
		buf = &br->bufs[head];
	} else {
		int off = head & (IO_BUFFER_LIST_BUF_PER_PAGE - 1);
		int index = head / IO_BUFFER_LIST_BUF_PER_PAGE;
		buf = page_address(bl->buf_pages[index]);
		buf += off;
	}
	if (*len == 0 || *len > buf->len)
		*len = buf->len;
	req->flags |= REQ_F_BUFFER_RING;
	req->buf_list = bl;
	req->buf_index = buf->bid;

	if (issue_flags & IO_URING_F_UNLOCKED || !file_can_poll(req->file)) {
		/*
		 * If we came in unlocked, we have no choice but to consume the
		 * buffer here, otherwise nothing ensures that the buffer won't
		 * get used by others. This does mean it'll be pinned until the
		 * IO completes, coming in unlocked means we're being called from
		 * io-wq context and there may be further retries in async hybrid
		 * mode. For the locked case, the caller must call commit when
		 * the transfer completes (or if we get -EAGAIN and must poll of
		 * retry).
		 */
		req->buf_list = NULL;
		bl->head++;
	}
	return u64_to_user_ptr(buf->addr);
}

void __user *io_buffer_select(struct io_kiocb *req, size_t *len,
			      unsigned int issue_flags)
{
	struct io_ring_ctx *ctx = req->ctx;
	struct io_buffer_list *bl;
	void __user *ret = NULL;

	io_ring_submit_lock(req->ctx, issue_flags);

	bl = io_buffer_get_list(ctx, req->buf_index);
	if (likely(bl)) {
		if (bl->buf_nr_pages)
			ret = io_ring_buffer_select(req, len, bl, issue_flags);
		else
			ret = io_provided_buffer_select(req, len, bl);
	}
	io_ring_submit_unlock(req->ctx, issue_flags);
	return ret;
}

static __cold int io_init_bl_list(struct io_ring_ctx *ctx)
{
	int i;

	ctx->io_bl = kcalloc(BGID_ARRAY, sizeof(struct io_buffer_list),
				GFP_KERNEL);
	if (!ctx->io_bl)
		return -ENOMEM;

	for (i = 0; i < BGID_ARRAY; i++) {
		INIT_LIST_HEAD(&ctx->io_bl[i].buf_list);
		ctx->io_bl[i].bgid = i;
	}

	return 0;
}

static int __io_remove_buffers(struct io_ring_ctx *ctx,
			       struct io_buffer_list *bl, unsigned nbufs)
{
	unsigned i = 0;

	/* shouldn't happen */
	if (!nbufs)
		return 0;

	if (bl->buf_nr_pages) {
		int j;

		i = bl->buf_ring->tail - bl->head;
		for (j = 0; j < bl->buf_nr_pages; j++)
			unpin_user_page(bl->buf_pages[j]);
		kvfree(bl->buf_pages);
		bl->buf_pages = NULL;
		bl->buf_nr_pages = 0;
		/* make sure it's seen as empty */
		INIT_LIST_HEAD(&bl->buf_list);
		return i;
	}

	/* the head kbuf is the list itself */
	while (!list_empty(&bl->buf_list)) {
		struct io_buffer *nxt;

		nxt = list_first_entry(&bl->buf_list, struct io_buffer, list);
		list_del(&nxt->list);
		if (++i == nbufs)
			return i;
		cond_resched();
	}
	i++;

	return i;
}

void io_destroy_buffers(struct io_ring_ctx *ctx)
{
	struct io_buffer_list *bl;
	unsigned long index;
	int i;

	for (i = 0; i < BGID_ARRAY; i++) {
		if (!ctx->io_bl)
			break;
		__io_remove_buffers(ctx, &ctx->io_bl[i], -1U);
	}

	xa_for_each(&ctx->io_bl_xa, index, bl) {
		xa_erase(&ctx->io_bl_xa, bl->bgid);
		__io_remove_buffers(ctx, bl, -1U);
		kfree(bl);
	}

	while (!list_empty(&ctx->io_buffers_pages)) {
		struct page *page;

		page = list_first_entry(&ctx->io_buffers_pages, struct page, lru);
		list_del_init(&page->lru);
		__free_page(page);
	}
}

int io_remove_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf);
	u64 tmp;

	if (sqe->rw_flags || sqe->addr || sqe->len || sqe->off ||
	    sqe->splice_fd_in)
		return -EINVAL;

	tmp = READ_ONCE(sqe->fd);
	if (!tmp || tmp > USHRT_MAX)
		return -EINVAL;

	memset(p, 0, sizeof(*p));
	p->nbufs = tmp;
	p->bgid = READ_ONCE(sqe->buf_group);
	return 0;
}

int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags)
{
	struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf);
	struct io_ring_ctx *ctx = req->ctx;
	struct io_buffer_list *bl;
	int ret = 0;

	io_ring_submit_lock(ctx, issue_flags);

	ret = -ENOENT;
	bl = io_buffer_get_list(ctx, p->bgid);
	if (bl) {
		ret = -EINVAL;
		/* can't use provide/remove buffers command on mapped buffers */
		if (!bl->buf_nr_pages)
			ret = __io_remove_buffers(ctx, bl, p->nbufs);
	}
	io_ring_submit_unlock(ctx, issue_flags);
	if (ret < 0)
		req_set_fail(req);
	io_req_set_res(req, ret, 0);
	return IOU_OK;
}

int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	unsigned long size, tmp_check;
	struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf);
	u64 tmp;

	if (sqe->rw_flags || sqe->splice_fd_in)
		return -EINVAL;

	tmp = READ_ONCE(sqe->fd);
	if (!tmp || tmp > USHRT_MAX)
		return -E2BIG;
	p->nbufs = tmp;
	p->addr = READ_ONCE(sqe->addr);
	p->len = READ_ONCE(sqe->len);

	if (check_mul_overflow((unsigned long)p->len, (unsigned long)p->nbufs,
				&size))
		return -EOVERFLOW;
	if (check_add_overflow((unsigned long)p->addr, size, &tmp_check))
		return -EOVERFLOW;

	size = (unsigned long)p->len * p->nbufs;
	if (!access_ok(u64_to_user_ptr(p->addr), size))
		return -EFAULT;

	p->bgid = READ_ONCE(sqe->buf_group);
	tmp = READ_ONCE(sqe->off);
	if (tmp > USHRT_MAX)
		return -E2BIG;
	if (tmp + p->nbufs >= USHRT_MAX)
		return -EINVAL;
	p->bid = tmp;
	return 0;
}

static int io_refill_buffer_cache(struct io_ring_ctx *ctx)
{
	struct io_buffer *buf;
	struct page *page;
	int bufs_in_page;

	/*
	 * Completions that don't happen inline (eg not under uring_lock) will
	 * add to ->io_buffers_comp. If we don't have any free buffers, check
	 * the completion list and splice those entries first.
	 */
	if (!list_empty_careful(&ctx->io_buffers_comp)) {
		spin_lock(&ctx->completion_lock);
		if (!list_empty(&ctx->io_buffers_comp)) {
			list_splice_init(&ctx->io_buffers_comp,
						&ctx->io_buffers_cache);
			spin_unlock(&ctx->completion_lock);
			return 0;
		}
		spin_unlock(&ctx->completion_lock);
	}

	/*
	 * No free buffers and no completion entries either. Allocate a new
	 * page worth of buffer entries and add those to our freelist.
	 */
	page = alloc_page(GFP_KERNEL_ACCOUNT);
	if (!page)
		return -ENOMEM;

	list_add(&page->lru, &ctx->io_buffers_pages);

	buf = page_address(page);
	bufs_in_page = PAGE_SIZE / sizeof(*buf);
	while (bufs_in_page) {
		list_add_tail(&buf->list, &ctx->io_buffers_cache);
		buf++;
		bufs_in_page--;
	}

	return 0;
}

static int io_add_buffers(struct io_ring_ctx *ctx, struct io_provide_buf *pbuf,
			  struct io_buffer_list *bl)
{
	struct io_buffer *buf;
	u64 addr = pbuf->addr;
	int i, bid = pbuf->bid;

	for (i = 0; i < pbuf->nbufs; i++) {
		if (list_empty(&ctx->io_buffers_cache) &&
		    io_refill_buffer_cache(ctx))
			break;
		buf = list_first_entry(&ctx->io_buffers_cache, struct io_buffer,
					list);
		list_move_tail(&buf->list, &bl->buf_list);
		buf->addr = addr;
		buf->len = min_t(__u32, pbuf->len, MAX_RW_COUNT);
		buf->bid = bid;
		buf->bgid = pbuf->bgid;
		addr += pbuf->len;
		bid++;
		cond_resched();
	}

	return i ? 0 : -ENOMEM;
}

int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)
{
	struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf);
	struct io_ring_ctx *ctx = req->ctx;
	struct io_buffer_list *bl;
	int ret = 0;

	io_ring_submit_lock(ctx, issue_flags);

	if (unlikely(p->bgid < BGID_ARRAY && !ctx->io_bl)) {
		ret = io_init_bl_list(ctx);
		if (ret)
			goto err;
	}

	bl = io_buffer_get_list(ctx, p->bgid);
	if (unlikely(!bl)) {
		bl = kzalloc(sizeof(*bl), GFP_KERNEL_ACCOUNT);
		if (!bl) {
			ret = -ENOMEM;
			goto err;
		}
		INIT_LIST_HEAD(&bl->buf_list);
		ret = io_buffer_add_list(ctx, bl, p->bgid);
		if (ret) {
			kfree(bl);
			goto err;
		}
	}
	/* can't add buffers via this command for a mapped buffer ring */
	if (bl->buf_nr_pages) {
		ret = -EINVAL;
		goto err;
	}

	ret = io_add_buffers(ctx, p, bl);
err:
	io_ring_submit_unlock(ctx, issue_flags);

	if (ret < 0)
		req_set_fail(req);
	io_req_set_res(req, ret, 0);
	return IOU_OK;
}

int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg)
{
	struct io_uring_buf_ring *br;
	struct io_uring_buf_reg reg;
	struct io_buffer_list *bl, *free_bl = NULL;
	struct page **pages;
	int nr_pages;

	if (copy_from_user(&reg, arg, sizeof(reg)))
		return -EFAULT;

	if (reg.pad || reg.resv[0] || reg.resv[1] || reg.resv[2])
		return -EINVAL;
	if (!reg.ring_addr)
		return -EFAULT;
	if (reg.ring_addr & ~PAGE_MASK)
		return -EINVAL;
	if (!is_power_of_2(reg.ring_entries))
		return -EINVAL;

	/* cannot disambiguate full vs empty due to head/tail size */
	if (reg.ring_entries >= 65536)
		return -EINVAL;

	if (unlikely(reg.bgid < BGID_ARRAY && !ctx->io_bl)) {
		int ret = io_init_bl_list(ctx);
		if (ret)
			return ret;
	}

	bl = io_buffer_get_list(ctx, reg.bgid);
	if (bl) {
		/* if mapped buffer ring OR classic exists, don't allow */
		if (bl->buf_nr_pages || !list_empty(&bl->buf_list))
			return -EEXIST;
	} else {
		free_bl = bl = kzalloc(sizeof(*bl), GFP_KERNEL);
		if (!bl)
			return -ENOMEM;
	}

	pages = io_pin_pages(reg.ring_addr,
			     struct_size(br, bufs, reg.ring_entries),
			     &nr_pages);
	if (IS_ERR(pages)) {
		kfree(free_bl);
		return PTR_ERR(pages);
	}

	br = page_address(pages[0]);
	bl->buf_pages = pages;
	bl->buf_nr_pages = nr_pages;
	bl->nr_entries = reg.ring_entries;
	bl->buf_ring = br;
	bl->mask = reg.ring_entries - 1;
	io_buffer_add_list(ctx, bl, reg.bgid);
	return 0;
}

int io_unregister_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg)
{
	struct io_uring_buf_reg reg;
	struct io_buffer_list *bl;

	if (copy_from_user(&reg, arg, sizeof(reg)))
		return -EFAULT;
	if (reg.pad || reg.resv[0] || reg.resv[1] || reg.resv[2])
		return -EINVAL;

	bl = io_buffer_get_list(ctx, reg.bgid);
	if (!bl)
		return -ENOENT;
	if (!bl->buf_nr_pages)
		return -EINVAL;

	__io_remove_buffers(ctx, bl, -1U);
	if (bl->bgid >= BGID_ARRAY) {
		xa_erase(&ctx->io_bl_xa, bl->bgid);
		kfree(bl);
	}
	return 0;
}
Commit	Line	Data
3b77495a JA	1	// SPDX-License-Identifier: GPL-2.0
	2	#include <linux/kernel.h>
	3	#include <linux/errno.h>
	4	#include <linux/fs.h>
	5	#include <linux/file.h>
	6	#include <linux/mm.h>
	7	#include <linux/slab.h>
	8	#include <linux/namei.h>
	9	#include <linux/poll.h>
	10	#include <linux/io_uring.h>
	11
	12	#include <uapi/linux/io_uring.h>
	13
3b77495a JA	14	#include "io_uring.h"
	15	#include "opdef.h"
	16	#include "kbuf.h"
	17
	18	#define IO_BUFFER_LIST_BUF_PER_PAGE (PAGE_SIZE / sizeof(struct io_uring_buf))
	19
	20	#define BGID_ARRAY 64
	21
	22	struct io_provide_buf {
	23	struct file *file;
	24	__u64 addr;
	25	__u32 len;
	26	__u32 bgid;
	27	__u16 nbufs;
	28	__u16 bid;
	29	};
	30
	31	static inline struct io_buffer_list io_buffer_get_list(struct io_ring_ctx ctx,
	32	unsigned int bgid)
	33	{
	34	if (ctx->io_bl && bgid < BGID_ARRAY)
	35	return &ctx->io_bl[bgid];
	36
	37	return xa_load(&ctx->io_bl_xa, bgid);
	38	}
	39
024b8fde HX	40	static int io_buffer_add_list(struct io_ring_ctx *ctx,
	41	struct io_buffer_list *bl, unsigned int bgid)
	42	{
	43	bl->bgid = bgid;
	44	if (bgid < BGID_ARRAY)
	45	return 0;
	46
	47	return xa_err(xa_store(&ctx->io_bl_xa, bgid, bl, GFP_KERNEL));
	48	}
	49
	50	void io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags)
3b77495a JA	51	{
	52	struct io_ring_ctx *ctx = req->ctx;
	53	struct io_buffer_list *bl;
	54	struct io_buffer *buf;
	55
	56	/*
024b8fde HX	57	* For legacy provided buffer mode, don't recycle if we already did
	58	* IO to this buffer. For ring-mapped provided buffer mode, we should
	59	* increment ring->head to explicitly monopolize the buffer to avoid
	60	* multiple use.
3b77495a	61	*/
024b8fde	62	if (req->flags & REQ_F_PARTIAL_IO)
3b77495a	63	return;
3b77495a JA	64
	65	io_ring_submit_lock(ctx, issue_flags);
	66
	67	buf = req->kbuf;
	68	bl = io_buffer_get_list(ctx, buf->bgid);
	69	list_add(&buf->list, &bl->buf_list);
	70	req->flags &= ~REQ_F_BUFFER_SELECTED;
	71	req->buf_index = buf->bgid;
	72
	73	io_ring_submit_unlock(ctx, issue_flags);
024b8fde	74	return;
3b77495a JA	75	}
3b77495a JA	76
53ccf69b PB	77	unsigned int __io_put_kbuf(struct io_kiocb *req, unsigned issue_flags)
	78	{
	79	unsigned int cflags;
	80
	81	/*
	82	* We can add this buffer back to two lists:
	83	*
	84	* 1) The io_buffers_cache list. This one is protected by the
	85	* ctx->uring_lock. If we already hold this lock, add back to this
	86	* list as we can grab it from issue as well.
	87	* 2) The io_buffers_comp list. This one is protected by the
	88	* ctx->completion_lock.
	89	*
	90	* We migrate buffers from the comp_list to the issue cache list
	91	* when we need one.
	92	*/
	93	if (req->flags & REQ_F_BUFFER_RING) {
	94	/* no buffers to recycle for this case */
	95	cflags = __io_put_kbuf_list(req, NULL);
	96	} else if (issue_flags & IO_URING_F_UNLOCKED) {
	97	struct io_ring_ctx *ctx = req->ctx;
	98
	99	spin_lock(&ctx->completion_lock);
	100	cflags = __io_put_kbuf_list(req, &ctx->io_buffers_comp);
	101	spin_unlock(&ctx->completion_lock);
	102	} else {
	103	lockdep_assert_held(&req->ctx->uring_lock);
	104
	105	cflags = __io_put_kbuf_list(req, &req->ctx->io_buffers_cache);
	106	}
	107	return cflags;
	108	}
	109
3b77495a JA	110	static void __user io_provided_buffer_select(struct io_kiocb req, size_t *len,
	111	struct io_buffer_list *bl)
	112	{
	113	if (!list_empty(&bl->buf_list)) {
	114	struct io_buffer *kbuf;
	115
	116	kbuf = list_first_entry(&bl->buf_list, struct io_buffer, list);
	117	list_del(&kbuf->list);
b8c01559	118	if (len == 0 \|\| len > kbuf->len)
3b77495a JA	119	*len = kbuf->len;
	120	req->flags \|= REQ_F_BUFFER_SELECTED;
	121	req->kbuf = kbuf;
	122	req->buf_index = kbuf->bid;
	123	return u64_to_user_ptr(kbuf->addr);
	124	}
	125	return NULL;
	126	}
	127
	128	static void __user io_ring_buffer_select(struct io_kiocb req, size_t *len,
	129	struct io_buffer_list *bl,
	130	unsigned int issue_flags)
	131	{
	132	struct io_uring_buf_ring *br = bl->buf_ring;
	133	struct io_uring_buf *buf;
	134	__u16 head = bl->head;
	135
	136	if (unlikely(smp_load_acquire(&br->tail) == head))
	137	return NULL;
	138
	139	head &= bl->mask;
	140	if (head < IO_BUFFER_LIST_BUF_PER_PAGE) {
	141	buf = &br->bufs[head];
	142	} else {
	143	int off = head & (IO_BUFFER_LIST_BUF_PER_PAGE - 1);
	144	int index = head / IO_BUFFER_LIST_BUF_PER_PAGE;
	145	buf = page_address(bl->buf_pages[index]);
	146	buf += off;
	147	}
b8c01559	148	if (len == 0 \|\| len > buf->len)
3b77495a JA	149	*len = buf->len;
	150	req->flags \|= REQ_F_BUFFER_RING;
	151	req->buf_list = bl;
	152	req->buf_index = buf->bid;
	153
	154	if (issue_flags & IO_URING_F_UNLOCKED \|\| !file_can_poll(req->file)) {
	155	/*
	156	* If we came in unlocked, we have no choice but to consume the
f09c8643 HX	157	* buffer here, otherwise nothing ensures that the buffer won't
	158	* get used by others. This does mean it'll be pinned until the
	159	* IO completes, coming in unlocked means we're being called from
	160	* io-wq context and there may be further retries in async hybrid
	161	* mode. For the locked case, the caller must call commit when
	162	* the transfer completes (or if we get -EAGAIN and must poll of
	163	* retry).
3b77495a JA	164	*/
	165	req->buf_list = NULL;
	166	bl->head++;
	167	}
	168	return u64_to_user_ptr(buf->addr);
	169	}
	170
	171	void __user io_buffer_select(struct io_kiocb req, size_t *len,
	172	unsigned int issue_flags)
	173	{
	174	struct io_ring_ctx *ctx = req->ctx;
	175	struct io_buffer_list *bl;
	176	void __user *ret = NULL;
	177
	178	io_ring_submit_lock(req->ctx, issue_flags);
	179
	180	bl = io_buffer_get_list(ctx, req->buf_index);
	181	if (likely(bl)) {
	182	if (bl->buf_nr_pages)
	183	ret = io_ring_buffer_select(req, len, bl, issue_flags);
	184	else
	185	ret = io_provided_buffer_select(req, len, bl);
	186	}
	187	io_ring_submit_unlock(req->ctx, issue_flags);
	188	return ret;
	189	}
	190
	191	static __cold int io_init_bl_list(struct io_ring_ctx *ctx)
	192	{
	193	int i;
	194
	195	ctx->io_bl = kcalloc(BGID_ARRAY, sizeof(struct io_buffer_list),
	196	GFP_KERNEL);
	197	if (!ctx->io_bl)
	198	return -ENOMEM;
	199
	200	for (i = 0; i < BGID_ARRAY; i++) {
	201	INIT_LIST_HEAD(&ctx->io_bl[i].buf_list);
	202	ctx->io_bl[i].bgid = i;
	203	}
	204
	205	return 0;
	206	}
	207
	208	static int __io_remove_buffers(struct io_ring_ctx *ctx,
	209	struct io_buffer_list *bl, unsigned nbufs)
	210	{
	211	unsigned i = 0;
	212
	213	/* shouldn't happen */
	214	if (!nbufs)
	215	return 0;
	216
	217	if (bl->buf_nr_pages) {
	218	int j;
	219
	220	i = bl->buf_ring->tail - bl->head;
	221	for (j = 0; j < bl->buf_nr_pages; j++)
	222	unpin_user_page(bl->buf_pages[j]);
	223	kvfree(bl->buf_pages);
	224	bl->buf_pages = NULL;
	225	bl->buf_nr_pages = 0;
	226	/* make sure it's seen as empty */
	227	INIT_LIST_HEAD(&bl->buf_list);
228	return i;
229	}
230
231	/* the head kbuf is the list itself */
232	while (!list_empty(&bl->buf_list)) {
233	struct io_buffer *nxt;
234
235	nxt = list_first_entry(&bl->buf_list, struct io_buffer, list);
236	list_del(&nxt->list);
237	if (++i == nbufs)
238	return i;
239	cond_resched();
240	}
241	i++;
242
243	return i;
244	}
245
246	void io_destroy_buffers(struct io_ring_ctx *ctx)
247	{
248	struct io_buffer_list *bl;
249	unsigned long index;
250	int i;
251
252	for (i = 0; i < BGID_ARRAY; i++) {
253	if (!ctx->io_bl)
254	break;
255	__io_remove_buffers(ctx, &ctx->io_bl[i], -1U);
256	}
257
258	xa_for_each(&ctx->io_bl_xa, index, bl) {
259	xa_erase(&ctx->io_bl_xa, bl->bgid);
260	__io_remove_buffers(ctx, bl, -1U);
261	kfree(bl);
262	}
263
264	while (!list_empty(&ctx->io_buffers_pages)) {
265	struct page *page;
266
267	page = list_first_entry(&ctx->io_buffers_pages, struct page, lru);
268	list_del_init(&page->lru);
269	__free_page(page);
270	}
271	}
272
273	int io_remove_buffers_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
274	{
f2ccb5ae	275	struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf);
3b77495a JA	276	u64 tmp;
	277
	278	if (sqe->rw_flags \|\| sqe->addr \|\| sqe->len \|\| sqe->off \|\|
	279	sqe->splice_fd_in)
	280	return -EINVAL;
	281
	282	tmp = READ_ONCE(sqe->fd);
	283	if (!tmp \|\| tmp > USHRT_MAX)
	284	return -EINVAL;
	285
	286	memset(p, 0, sizeof(*p));
	287	p->nbufs = tmp;
	288	p->bgid = READ_ONCE(sqe->buf_group);
	289	return 0;
	290	}
	291
	292	int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags)
	293	{
f2ccb5ae	294	struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf);
3b77495a JA	295	struct io_ring_ctx *ctx = req->ctx;
	296	struct io_buffer_list *bl;
	297	int ret = 0;
	298
	299	io_ring_submit_lock(ctx, issue_flags);
	300
	301	ret = -ENOENT;
	302	bl = io_buffer_get_list(ctx, p->bgid);
	303	if (bl) {
	304	ret = -EINVAL;
	305	/* can't use provide/remove buffers command on mapped buffers */
	306	if (!bl->buf_nr_pages)
	307	ret = __io_remove_buffers(ctx, bl, p->nbufs);
	308	}
c3b49093	309	io_ring_submit_unlock(ctx, issue_flags);
3b77495a JA	310	if (ret < 0)
3b77495a JA	311	req_set_fail(req);
3b77495a	312	io_req_set_res(req, ret, 0);
c3b49093	313	return IOU_OK;
3b77495a JA	314	}
	315
	316	int io_provide_buffers_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
	317	{
	318	unsigned long size, tmp_check;
f2ccb5ae	319	struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf);
3b77495a JA	320	u64 tmp;
	321
	322	if (sqe->rw_flags \|\| sqe->splice_fd_in)
	323	return -EINVAL;
	324
	325	tmp = READ_ONCE(sqe->fd);
	326	if (!tmp \|\| tmp > USHRT_MAX)
	327	return -E2BIG;
	328	p->nbufs = tmp;
	329	p->addr = READ_ONCE(sqe->addr);
	330	p->len = READ_ONCE(sqe->len);
	331
	332	if (check_mul_overflow((unsigned long)p->len, (unsigned long)p->nbufs,
	333	&size))
	334	return -EOVERFLOW;
	335	if (check_add_overflow((unsigned long)p->addr, size, &tmp_check))
	336	return -EOVERFLOW;
	337
	338	size = (unsigned long)p->len * p->nbufs;
	339	if (!access_ok(u64_to_user_ptr(p->addr), size))
	340	return -EFAULT;
	341
	342	p->bgid = READ_ONCE(sqe->buf_group);
	343	tmp = READ_ONCE(sqe->off);
	344	if (tmp > USHRT_MAX)
	345	return -E2BIG;
3851d25c JA	346	if (tmp + p->nbufs >= USHRT_MAX)
3851d25c JA	347	return -EINVAL;
3b77495a JA	348	p->bid = tmp;
	349	return 0;
	350	}
	351
	352	static int io_refill_buffer_cache(struct io_ring_ctx *ctx)
	353	{
	354	struct io_buffer *buf;
	355	struct page *page;
	356	int bufs_in_page;
	357
	358	/*
	359	* Completions that don't happen inline (eg not under uring_lock) will
	360	* add to ->io_buffers_comp. If we don't have any free buffers, check
	361	* the completion list and splice those entries first.
	362	*/
	363	if (!list_empty_careful(&ctx->io_buffers_comp)) {
	364	spin_lock(&ctx->completion_lock);
	365	if (!list_empty(&ctx->io_buffers_comp)) {
	366	list_splice_init(&ctx->io_buffers_comp,
	367	&ctx->io_buffers_cache);
	368	spin_unlock(&ctx->completion_lock);
	369	return 0;
	370	}
	371	spin_unlock(&ctx->completion_lock);
	372	}
	373
	374	/*
	375	* No free buffers and no completion entries either. Allocate a new
	376	* page worth of buffer entries and add those to our freelist.
	377	*/
	378	page = alloc_page(GFP_KERNEL_ACCOUNT);
	379	if (!page)
	380	return -ENOMEM;
	381
	382	list_add(&page->lru, &ctx->io_buffers_pages);
	383
	384	buf = page_address(page);
	385	bufs_in_page = PAGE_SIZE / sizeof(*buf);
	386	while (bufs_in_page) {
	387	list_add_tail(&buf->list, &ctx->io_buffers_cache);
	388	buf++;
	389	bufs_in_page--;
	390	}
	391
	392	return 0;
	393	}
	394
	395	static int io_add_buffers(struct io_ring_ctx ctx, struct io_provide_buf pbuf,
	396	struct io_buffer_list *bl)
	397	{
	398	struct io_buffer *buf;
	399	u64 addr = pbuf->addr;
	400	int i, bid = pbuf->bid;
	401
	402	for (i = 0; i < pbuf->nbufs; i++) {
	403	if (list_empty(&ctx->io_buffers_cache) &&
	404	io_refill_buffer_cache(ctx))
	405	break;
	406	buf = list_first_entry(&ctx->io_buffers_cache, struct io_buffer,
	407	list);
	408	list_move_tail(&buf->list, &bl->buf_list);
	409	buf->addr = addr;
	410	buf->len = min_t(__u32, pbuf->len, MAX_RW_COUNT);
	411	buf->bid = bid;
412	buf->bgid = pbuf->bgid;
413	addr += pbuf->len;
414	bid++;
415	cond_resched();
416	}
417
418	return i ? 0 : -ENOMEM;
419	}
420
421	int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)
422	{
f2ccb5ae	423	struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf);
3b77495a JA	424	struct io_ring_ctx *ctx = req->ctx;
	425	struct io_buffer_list *bl;
	426	int ret = 0;
	427
	428	io_ring_submit_lock(ctx, issue_flags);
	429
	430	if (unlikely(p->bgid < BGID_ARRAY && !ctx->io_bl)) {
	431	ret = io_init_bl_list(ctx);
	432	if (ret)
	433	goto err;
	434	}
	435
	436	bl = io_buffer_get_list(ctx, p->bgid);
	437	if (unlikely(!bl)) {
cc18cc5e	438	bl = kzalloc(sizeof(*bl), GFP_KERNEL_ACCOUNT);
3b77495a JA	439	if (!bl) {
	440	ret = -ENOMEM;
	441	goto err;
	442	}
	443	INIT_LIST_HEAD(&bl->buf_list);
	444	ret = io_buffer_add_list(ctx, bl, p->bgid);
	445	if (ret) {
	446	kfree(bl);
	447	goto err;
	448	}
	449	}
	450	/* can't add buffers via this command for a mapped buffer ring */
	451	if (bl->buf_nr_pages) {
	452	ret = -EINVAL;
	453	goto err;
	454	}
	455
	456	ret = io_add_buffers(ctx, p, bl);
	457	err:
c3b49093 PB	458	io_ring_submit_unlock(ctx, issue_flags);
c3b49093 PB	459
3b77495a JA	460	if (ret < 0)
3b77495a JA	461	req_set_fail(req);
3b77495a	462	io_req_set_res(req, ret, 0);
c3b49093	463	return IOU_OK;
3b77495a JA	464	}
	465
	466	int io_register_pbuf_ring(struct io_ring_ctx ctx, void __user arg)
	467	{
	468	struct io_uring_buf_ring *br;
	469	struct io_uring_buf_reg reg;
	470	struct io_buffer_list bl, free_bl = NULL;
	471	struct page **pages;
	472	int nr_pages;
	473
	474	if (copy_from_user(&reg, arg, sizeof(reg)))
	475	return -EFAULT;
	476
	477	if (reg.pad \|\| reg.resv[0] \|\| reg.resv[1] \|\| reg.resv[2])
	478	return -EINVAL;
	479	if (!reg.ring_addr)
	480	return -EFAULT;
	481	if (reg.ring_addr & ~PAGE_MASK)
	482	return -EINVAL;
	483	if (!is_power_of_2(reg.ring_entries))
	484	return -EINVAL;
	485
	486	/* cannot disambiguate full vs empty due to head/tail size */
	487	if (reg.ring_entries >= 65536)
	488	return -EINVAL;
	489
	490	if (unlikely(reg.bgid < BGID_ARRAY && !ctx->io_bl)) {
	491	int ret = io_init_bl_list(ctx);
	492	if (ret)
	493	return ret;
	494	}
	495
	496	bl = io_buffer_get_list(ctx, reg.bgid);
	497	if (bl) {
	498	/* if mapped buffer ring OR classic exists, don't allow */
	499	if (bl->buf_nr_pages \|\| !list_empty(&bl->buf_list))
	500	return -EEXIST;
	501	} else {
	502	free_bl = bl = kzalloc(sizeof(*bl), GFP_KERNEL);
	503	if (!bl)
	504	return -ENOMEM;
	505	}
	506
	507	pages = io_pin_pages(reg.ring_addr,
	508	struct_size(br, bufs, reg.ring_entries),
	509	&nr_pages);
	510	if (IS_ERR(pages)) {
	511	kfree(free_bl);
	512	return PTR_ERR(pages);
	513	}
	514
	515	br = page_address(pages[0]);
	516	bl->buf_pages = pages;
	517	bl->buf_nr_pages = nr_pages;
	518	bl->nr_entries = reg.ring_entries;
	519	bl->buf_ring = br;
	520	bl->mask = reg.ring_entries - 1;
	521	io_buffer_add_list(ctx, bl, reg.bgid);
	522	return 0;
	523	}
	524
	525	int io_unregister_pbuf_ring(struct io_ring_ctx ctx, void __user arg)
	526	{
	527	struct io_uring_buf_reg reg;
528	struct io_buffer_list *bl;
529
530	if (copy_from_user(&reg, arg, sizeof(reg)))
531	return -EFAULT;
532	if (reg.pad \|\| reg.resv[0] \|\| reg.resv[1] \|\| reg.resv[2])
533	return -EINVAL;
534
535	bl = io_buffer_get_list(ctx, reg.bgid);
536	if (!bl)
537	return -ENOENT;
538	if (!bl->buf_nr_pages)
539	return -EINVAL;
540
541	__io_remove_buffers(ctx, bl, -1U);
542	if (bl->bgid >= BGID_ARRAY) {
543	xa_erase(&ctx->io_bl_xa, bl->bgid);
544	kfree(bl);
545	}
546	return 0;
547	}