[linux-block.git] / io_uring / filetable.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/file.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/nospec.h>
#include <linux/io_uring.h>

#include <uapi/linux/io_uring.h>

#include "io_uring.h"
#include "rsrc.h"
#include "filetable.h"

static int io_file_bitmap_get(struct io_ring_ctx *ctx)
{
	struct io_file_table *table = &ctx->file_table;
	unsigned long nr = ctx->file_alloc_end;
	int ret;

	if (!table->bitmap)
		return -ENFILE;

	do {
		ret = find_next_zero_bit(table->bitmap, nr, table->alloc_hint);
		if (ret != nr)
			return ret;

		if (table->alloc_hint == ctx->file_alloc_start)
			break;
		nr = table->alloc_hint;
		table->alloc_hint = ctx->file_alloc_start;
	} while (1);

	return -ENFILE;
}

bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files)
{
	table->files = kvcalloc(nr_files, sizeof(table->files[0]),
				GFP_KERNEL_ACCOUNT);
	if (unlikely(!table->files))
		return false;

	table->bitmap = bitmap_zalloc(nr_files, GFP_KERNEL_ACCOUNT);
	if (unlikely(!table->bitmap)) {
		kvfree(table->files);
		return false;
	}

	return true;
}

void io_free_file_tables(struct io_file_table *table)
{
	kvfree(table->files);
	bitmap_free(table->bitmap);
	table->files = NULL;
	table->bitmap = NULL;
}

static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file,
				 u32 slot_index)
	__must_hold(&req->ctx->uring_lock)
{
	struct io_fixed_file *file_slot;
	int ret;

	if (io_is_uring_fops(file))
		return -EBADF;
	if (!ctx->file_data)
		return -ENXIO;
	if (slot_index >= ctx->nr_user_files)
		return -EINVAL;

	slot_index = array_index_nospec(slot_index, ctx->nr_user_files);
	file_slot = io_fixed_file_slot(&ctx->file_table, slot_index);

	if (file_slot->file_ptr) {
		struct file *old_file;

		old_file = (struct file *)(file_slot->file_ptr & FFS_MASK);
		ret = io_queue_rsrc_removal(ctx->file_data, slot_index, old_file);
		if (ret)
			return ret;

		file_slot->file_ptr = 0;
		io_file_bitmap_clear(&ctx->file_table, slot_index);
	}

	ret = io_scm_file_account(ctx, file);
	if (!ret) {
		*io_get_tag_slot(ctx->file_data, slot_index) = 0;
		io_fixed_file_set(file_slot, file);
		io_file_bitmap_set(&ctx->file_table, slot_index);
	}
	return ret;
}

int __io_fixed_fd_install(struct io_ring_ctx *ctx, struct file *file,
			  unsigned int file_slot)
{
	bool alloc_slot = file_slot == IORING_FILE_INDEX_ALLOC;
	int ret;

	if (alloc_slot) {
		ret = io_file_bitmap_get(ctx);
		if (unlikely(ret < 0))
			return ret;
		file_slot = ret;
	} else {
		file_slot--;
	}

	ret = io_install_fixed_file(ctx, file, file_slot);
	if (!ret && alloc_slot)
		ret = file_slot;
	return ret;
}
/*
 * Note when io_fixed_fd_install() returns error value, it will ensure
 * fput() is called correspondingly.
 */
int io_fixed_fd_install(struct io_kiocb *req, unsigned int issue_flags,
			struct file *file, unsigned int file_slot)
{
	struct io_ring_ctx *ctx = req->ctx;
	int ret;

	io_ring_submit_lock(ctx, issue_flags);
	ret = __io_fixed_fd_install(ctx, file, file_slot);
	io_ring_submit_unlock(ctx, issue_flags);

	if (unlikely(ret < 0))
		fput(file);
	return ret;
}

int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset)
{
	struct io_fixed_file *file_slot;
	struct file *file;
	int ret;

	if (unlikely(!ctx->file_data))
		return -ENXIO;
	if (offset >= ctx->nr_user_files)
		return -EINVAL;

	offset = array_index_nospec(offset, ctx->nr_user_files);
	file_slot = io_fixed_file_slot(&ctx->file_table, offset);
	if (!file_slot->file_ptr)
		return -EBADF;

	file = (struct file *)(file_slot->file_ptr & FFS_MASK);
	ret = io_queue_rsrc_removal(ctx->file_data, offset, file);
	if (ret)
		return ret;

	file_slot->file_ptr = 0;
	io_file_bitmap_clear(&ctx->file_table, offset);
	return 0;
}

int io_register_file_alloc_range(struct io_ring_ctx *ctx,
				 struct io_uring_file_index_range __user *arg)
{
	struct io_uring_file_index_range range;
	u32 end;

	if (copy_from_user(&range, arg, sizeof(range)))
		return -EFAULT;
	if (check_add_overflow(range.off, range.len, &end))
		return -EOVERFLOW;
	if (range.resv || end > ctx->nr_user_files)
		return -EINVAL;

	io_file_table_set_alloc_range(ctx, range.off, range.len);
	return 0;
}
Commit	Line	Data
	1	// SPDX-License-Identifier: GPL-2.0
	2	#include <linux/kernel.h>
	3	#include <linux/errno.h>
	4	#include <linux/file.h>
	5	#include <linux/mm.h>
	6	#include <linux/slab.h>
	7	#include <linux/nospec.h>
	8	#include <linux/io_uring.h>
	9
	10	#include <uapi/linux/io_uring.h>
	11
	12	#include "io_uring.h"
	13	#include "rsrc.h"
	14	#include "filetable.h"
	15
	16	static int io_file_bitmap_get(struct io_ring_ctx *ctx)
	17	{
	18	struct io_file_table *table = &ctx->file_table;
	19	unsigned long nr = ctx->file_alloc_end;
	20	int ret;
	21
	22	if (!table->bitmap)
	23	return -ENFILE;
	24
	25	do {
	26	ret = find_next_zero_bit(table->bitmap, nr, table->alloc_hint);
	27	if (ret != nr)
	28	return ret;
	29
	30	if (table->alloc_hint == ctx->file_alloc_start)
	31	break;
	32	nr = table->alloc_hint;
	33	table->alloc_hint = ctx->file_alloc_start;
	34	} while (1);
	35
	36	return -ENFILE;
	37	}
	38
	39	bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files)
	40	{
	41	table->files = kvcalloc(nr_files, sizeof(table->files[0]),
	42	GFP_KERNEL_ACCOUNT);
	43	if (unlikely(!table->files))
	44	return false;
	45
	46	table->bitmap = bitmap_zalloc(nr_files, GFP_KERNEL_ACCOUNT);
	47	if (unlikely(!table->bitmap)) {
	48	kvfree(table->files);
	49	return false;
	50	}
	51
	52	return true;
	53	}
	54
	55	void io_free_file_tables(struct io_file_table *table)
	56	{
	57	kvfree(table->files);
	58	bitmap_free(table->bitmap);
	59	table->files = NULL;
	60	table->bitmap = NULL;
	61	}
	62
	63	static int io_install_fixed_file(struct io_ring_ctx ctx, struct file file,
	64	u32 slot_index)
	65	__must_hold(&req->ctx->uring_lock)
	66	{
	67	struct io_fixed_file *file_slot;
	68	int ret;
	69
	70	if (io_is_uring_fops(file))
	71	return -EBADF;
	72	if (!ctx->file_data)
	73	return -ENXIO;
	74	if (slot_index >= ctx->nr_user_files)
	75	return -EINVAL;
	76
	77	slot_index = array_index_nospec(slot_index, ctx->nr_user_files);
	78	file_slot = io_fixed_file_slot(&ctx->file_table, slot_index);
	79
	80	if (file_slot->file_ptr) {
	81	struct file *old_file;
	82
	83	old_file = (struct file *)(file_slot->file_ptr & FFS_MASK);
	84	ret = io_queue_rsrc_removal(ctx->file_data, slot_index, old_file);
	85	if (ret)
	86	return ret;
	87
	88	file_slot->file_ptr = 0;
	89	io_file_bitmap_clear(&ctx->file_table, slot_index);
	90	}
	91
	92	ret = io_scm_file_account(ctx, file);
	93	if (!ret) {
	94	*io_get_tag_slot(ctx->file_data, slot_index) = 0;
	95	io_fixed_file_set(file_slot, file);
	96	io_file_bitmap_set(&ctx->file_table, slot_index);
	97	}
	98	return ret;
	99	}
	100
	101	int __io_fixed_fd_install(struct io_ring_ctx ctx, struct file file,
	102	unsigned int file_slot)
	103	{
	104	bool alloc_slot = file_slot == IORING_FILE_INDEX_ALLOC;
	105	int ret;
	106
	107	if (alloc_slot) {
	108	ret = io_file_bitmap_get(ctx);
	109	if (unlikely(ret < 0))
	110	return ret;
	111	file_slot = ret;
	112	} else {
	113	file_slot--;
	114	}
	115
	116	ret = io_install_fixed_file(ctx, file, file_slot);
	117	if (!ret && alloc_slot)
	118	ret = file_slot;
	119	return ret;
	120	}
	121	/*
	122	* Note when io_fixed_fd_install() returns error value, it will ensure
	123	* fput() is called correspondingly.
	124	*/
	125	int io_fixed_fd_install(struct io_kiocb *req, unsigned int issue_flags,
	126	struct file *file, unsigned int file_slot)
	127	{
	128	struct io_ring_ctx *ctx = req->ctx;
	129	int ret;
	130
	131	io_ring_submit_lock(ctx, issue_flags);
	132	ret = __io_fixed_fd_install(ctx, file, file_slot);
	133	io_ring_submit_unlock(ctx, issue_flags);
	134
	135	if (unlikely(ret < 0))
	136	fput(file);
	137	return ret;
	138	}
	139
	140	int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset)
	141	{
	142	struct io_fixed_file *file_slot;
	143	struct file *file;
	144	int ret;
	145
	146	if (unlikely(!ctx->file_data))
	147	return -ENXIO;
	148	if (offset >= ctx->nr_user_files)
	149	return -EINVAL;
	150
	151	offset = array_index_nospec(offset, ctx->nr_user_files);
	152	file_slot = io_fixed_file_slot(&ctx->file_table, offset);
	153	if (!file_slot->file_ptr)
	154	return -EBADF;
	155
	156	file = (struct file *)(file_slot->file_ptr & FFS_MASK);
	157	ret = io_queue_rsrc_removal(ctx->file_data, offset, file);
	158	if (ret)
	159	return ret;
	160
	161	file_slot->file_ptr = 0;
	162	io_file_bitmap_clear(&ctx->file_table, offset);
	163	return 0;
	164	}
	165
	166	int io_register_file_alloc_range(struct io_ring_ctx *ctx,
	167	struct io_uring_file_index_range __user *arg)
	168	{
	169	struct io_uring_file_index_range range;
	170	u32 end;
	171
	172	if (copy_from_user(&range, arg, sizeof(range)))
	173	return -EFAULT;
	174	if (check_add_overflow(range.off, range.len, &end))
	175	return -EOVERFLOW;
	176	if (range.resv \|\| end > ctx->nr_user_files)
	177	return -EINVAL;
	178
	179	io_file_table_set_alloc_range(ctx, range.off, range.len);
	180	return 0;
	181	}