[linux-block.git] / kernel / bpf / bpf_inode_storage.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (c) 2019 Facebook
 * Copyright 2020 Google LLC.
 */

#include <linux/rculist.h>
#include <linux/list.h>
#include <linux/hash.h>
#include <linux/types.h>
#include <linux/spinlock.h>
#include <linux/bpf.h>
#include <linux/bpf_local_storage.h>
#include <net/sock.h>
#include <uapi/linux/sock_diag.h>
#include <uapi/linux/btf.h>
#include <linux/bpf_lsm.h>
#include <linux/btf_ids.h>
#include <linux/fdtable.h>
#include <linux/rcupdate_trace.h>

DEFINE_BPF_STORAGE_CACHE(inode_cache);

static struct bpf_local_storage __rcu **
inode_storage_ptr(void *owner)
{
	struct inode *inode = owner;
	struct bpf_storage_blob *bsb;

	bsb = bpf_inode(inode);
	if (!bsb)
		return NULL;
	return &bsb->storage;
}

static struct bpf_local_storage_data *inode_storage_lookup(struct inode *inode,
							   struct bpf_map *map,
							   bool cacheit_lockit)
{
	struct bpf_local_storage *inode_storage;
	struct bpf_local_storage_map *smap;
	struct bpf_storage_blob *bsb;

	bsb = bpf_inode(inode);
	if (!bsb)
		return NULL;

	inode_storage =
		rcu_dereference_check(bsb->storage, bpf_rcu_lock_held());
	if (!inode_storage)
		return NULL;

	smap = (struct bpf_local_storage_map *)map;
	return bpf_local_storage_lookup(inode_storage, smap, cacheit_lockit);
}

void bpf_inode_storage_free(struct inode *inode)
{
	struct bpf_local_storage *local_storage;
	bool free_inode_storage = false;
	struct bpf_storage_blob *bsb;

	bsb = bpf_inode(inode);
	if (!bsb)
		return;

	rcu_read_lock();

	local_storage = rcu_dereference(bsb->storage);
	if (!local_storage) {
		rcu_read_unlock();
		return;
	}

	raw_spin_lock_bh(&local_storage->lock);
	free_inode_storage = bpf_local_storage_unlink_nolock(local_storage);
	raw_spin_unlock_bh(&local_storage->lock);
	rcu_read_unlock();

	if (free_inode_storage)
		kfree_rcu(local_storage, rcu);
}

static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key)
{
	struct bpf_local_storage_data *sdata;
	struct file *f;
	int fd;

	fd = *(int *)key;
	f = fget_raw(fd);
	if (!f)
		return ERR_PTR(-EBADF);

	sdata = inode_storage_lookup(f->f_inode, map, true);
	fput(f);
	return sdata ? sdata->data : NULL;
}

static int bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key,
					 void *value, u64 map_flags)
{
	struct bpf_local_storage_data *sdata;
	struct file *f;
	int fd;

	fd = *(int *)key;
	f = fget_raw(fd);
	if (!f)
		return -EBADF;
	if (!inode_storage_ptr(f->f_inode)) {
		fput(f);
		return -EBADF;
	}

	sdata = bpf_local_storage_update(f->f_inode,
					 (struct bpf_local_storage_map *)map,
					 value, map_flags, GFP_ATOMIC);
	fput(f);
	return PTR_ERR_OR_ZERO(sdata);
}

static int inode_storage_delete(struct inode *inode, struct bpf_map *map)
{
	struct bpf_local_storage_data *sdata;

	sdata = inode_storage_lookup(inode, map, false);
	if (!sdata)
		return -ENOENT;

	bpf_selem_unlink(SELEM(sdata), true);

	return 0;
}

static int bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key)
{
	struct file *f;
	int fd, err;

	fd = *(int *)key;
	f = fget_raw(fd);
	if (!f)
		return -EBADF;

	err = inode_storage_delete(f->f_inode, map);
	fput(f);
	return err;
}

/* *gfp_flags* is a hidden argument provided by the verifier */
BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode,
	   void *, value, u64, flags, gfp_t, gfp_flags)
{
	struct bpf_local_storage_data *sdata;

	WARN_ON_ONCE(!bpf_rcu_lock_held());
	if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
		return (unsigned long)NULL;

	/* explicitly check that the inode_storage_ptr is not
	 * NULL as inode_storage_lookup returns NULL in this case and
	 * bpf_local_storage_update expects the owner to have a
	 * valid storage pointer.
	 */
	if (!inode || !inode_storage_ptr(inode))
		return (unsigned long)NULL;

	sdata = inode_storage_lookup(inode, map, true);
	if (sdata)
		return (unsigned long)sdata->data;

	/* This helper must only called from where the inode is guaranteed
	 * to have a refcount and cannot be freed.
	 */
	if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
		sdata = bpf_local_storage_update(
			inode, (struct bpf_local_storage_map *)map, value,
			BPF_NOEXIST, gfp_flags);
		return IS_ERR(sdata) ? (unsigned long)NULL :
					     (unsigned long)sdata->data;
	}

	return (unsigned long)NULL;
}

BPF_CALL_2(bpf_inode_storage_delete,
	   struct bpf_map *, map, struct inode *, inode)
{
	WARN_ON_ONCE(!bpf_rcu_lock_held());
	if (!inode)
		return -EINVAL;

	/* This helper must only called from where the inode is guaranteed
	 * to have a refcount and cannot be freed.
	 */
	return inode_storage_delete(inode, map);
}

static int notsupp_get_next_key(struct bpf_map *map, void *key,
				void *next_key)
{
	return -ENOTSUPP;
}

static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr)
{
	return bpf_local_storage_map_alloc(attr, &inode_cache);
}

static void inode_storage_map_free(struct bpf_map *map)
{
	bpf_local_storage_map_free(map, &inode_cache, NULL);
}

const struct bpf_map_ops inode_storage_map_ops = {
	.map_meta_equal = bpf_map_meta_equal,
	.map_alloc_check = bpf_local_storage_map_alloc_check,
	.map_alloc = inode_storage_map_alloc,
	.map_free = inode_storage_map_free,
	.map_get_next_key = notsupp_get_next_key,
	.map_lookup_elem = bpf_fd_inode_storage_lookup_elem,
	.map_update_elem = bpf_fd_inode_storage_update_elem,
	.map_delete_elem = bpf_fd_inode_storage_delete_elem,
	.map_check_btf = bpf_local_storage_map_check_btf,
	.map_btf_id = &bpf_local_storage_map_btf_id[0],
	.map_owner_storage_ptr = inode_storage_ptr,
};

BTF_ID_LIST_SINGLE(bpf_inode_storage_btf_ids, struct, inode)

const struct bpf_func_proto bpf_inode_storage_get_proto = {
	.func		= bpf_inode_storage_get,
	.gpl_only	= false,
	.ret_type	= RET_PTR_TO_MAP_VALUE_OR_NULL,
	.arg1_type	= ARG_CONST_MAP_PTR,
	.arg2_type	= ARG_PTR_TO_BTF_ID,
	.arg2_btf_id	= &bpf_inode_storage_btf_ids[0],
	.arg3_type	= ARG_PTR_TO_MAP_VALUE_OR_NULL,
	.arg4_type	= ARG_ANYTHING,
};

const struct bpf_func_proto bpf_inode_storage_delete_proto = {
	.func		= bpf_inode_storage_delete,
	.gpl_only	= false,
	.ret_type	= RET_INTEGER,
	.arg1_type	= ARG_CONST_MAP_PTR,
	.arg2_type	= ARG_PTR_TO_BTF_ID,
	.arg2_btf_id	= &bpf_inode_storage_btf_ids[0],
};
Commit	Line	Data
8ea63684 KS	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* Copyright (c) 2019 Facebook
	4	* Copyright 2020 Google LLC.
	5	*/
	6
	7	#include <linux/rculist.h>
	8	#include <linux/list.h>
	9	#include <linux/hash.h>
	10	#include <linux/types.h>
	11	#include <linux/spinlock.h>
	12	#include <linux/bpf.h>
	13	#include <linux/bpf_local_storage.h>
	14	#include <net/sock.h>
	15	#include <uapi/linux/sock_diag.h>
	16	#include <uapi/linux/btf.h>
	17	#include <linux/bpf_lsm.h>
	18	#include <linux/btf_ids.h>
	19	#include <linux/fdtable.h>
0fe4b381	20	#include <linux/rcupdate_trace.h>
8ea63684 KS	21
	22	DEFINE_BPF_STORAGE_CACHE(inode_cache);
	23
	24	static struct bpf_local_storage __rcu **
	25	inode_storage_ptr(void *owner)
	26	{
	27	struct inode *inode = owner;
	28	struct bpf_storage_blob *bsb;
	29
	30	bsb = bpf_inode(inode);
	31	if (!bsb)
	32	return NULL;
	33	return &bsb->storage;
	34	}
	35
	36	static struct bpf_local_storage_data inode_storage_lookup(struct inode inode,
	37	struct bpf_map *map,
	38	bool cacheit_lockit)
	39	{
	40	struct bpf_local_storage *inode_storage;
	41	struct bpf_local_storage_map *smap;
	42	struct bpf_storage_blob *bsb;
	43
	44	bsb = bpf_inode(inode);
	45	if (!bsb)
	46	return NULL;
	47
0fe4b381 KS	48	inode_storage =
0fe4b381 KS	49	rcu_dereference_check(bsb->storage, bpf_rcu_lock_held());
8ea63684 KS	50	if (!inode_storage)
	51	return NULL;
	52
	53	smap = (struct bpf_local_storage_map *)map;
	54	return bpf_local_storage_lookup(inode_storage, smap, cacheit_lockit);
	55	}
	56
	57	void bpf_inode_storage_free(struct inode *inode)
	58	{
8ea63684 KS	59	struct bpf_local_storage *local_storage;
	60	bool free_inode_storage = false;
	61	struct bpf_storage_blob *bsb;
8ea63684 KS	62
	63	bsb = bpf_inode(inode);
	64	if (!bsb)
	65	return;
	66
	67	rcu_read_lock();
	68
	69	local_storage = rcu_dereference(bsb->storage);
	70	if (!local_storage) {
	71	rcu_read_unlock();
	72	return;
	73	}
	74
8ea63684	75	raw_spin_lock_bh(&local_storage->lock);
c83597fa	76	free_inode_storage = bpf_local_storage_unlink_nolock(local_storage);
8ea63684 KS	77	raw_spin_unlock_bh(&local_storage->lock);
	78	rcu_read_unlock();
	79
8ea63684 KS	80	if (free_inode_storage)
	81	kfree_rcu(local_storage, rcu);
	82	}
	83
	84	static void bpf_fd_inode_storage_lookup_elem(struct bpf_map map, void *key)
	85	{
	86	struct bpf_local_storage_data *sdata;
	87	struct file *f;
	88	int fd;
	89
	90	fd = (int )key;
	91	f = fget_raw(fd);
	92	if (!f)
769c18b2	93	return ERR_PTR(-EBADF);
8ea63684 KS	94
	95	sdata = inode_storage_lookup(f->f_inode, map, true);
	96	fput(f);
	97	return sdata ? sdata->data : NULL;
	98	}
	99
	100	static int bpf_fd_inode_storage_update_elem(struct bpf_map map, void key,
	101	void *value, u64 map_flags)
	102	{
	103	struct bpf_local_storage_data *sdata;
	104	struct file *f;
	105	int fd;
	106
	107	fd = (int )key;
	108	f = fget_raw(fd);
b9557caa PB	109	if (!f)
	110	return -EBADF;
	111	if (!inode_storage_ptr(f->f_inode)) {
	112	fput(f);
8ea63684	113	return -EBADF;
b9557caa	114	}
8ea63684 KS	115
	116	sdata = bpf_local_storage_update(f->f_inode,
	117	(struct bpf_local_storage_map *)map,
b00fa38a	118	value, map_flags, GFP_ATOMIC);
8ea63684 KS	119	fput(f);
	120	return PTR_ERR_OR_ZERO(sdata);
	121	}
	122
	123	static int inode_storage_delete(struct inode inode, struct bpf_map map)
	124	{
	125	struct bpf_local_storage_data *sdata;
	126
	127	sdata = inode_storage_lookup(inode, map, false);
	128	if (!sdata)
	129	return -ENOENT;
	130
dcf456c9	131	bpf_selem_unlink(SELEM(sdata), true);
8ea63684 KS	132
	133	return 0;
	134	}
	135
	136	static int bpf_fd_inode_storage_delete_elem(struct bpf_map map, void key)
	137	{
	138	struct file *f;
	139	int fd, err;
	140
	141	fd = (int )key;
	142	f = fget_raw(fd);
	143	if (!f)
	144	return -EBADF;
	145
	146	err = inode_storage_delete(f->f_inode, map);
	147	fput(f);
	148	return err;
	149	}
	150
b00fa38a JK	151	/* gfp_flags is a hidden argument provided by the verifier */
	152	BPF_CALL_5(bpf_inode_storage_get, struct bpf_map , map, struct inode , inode,
	153	void *, value, u64, flags, gfp_t, gfp_flags)
8ea63684 KS	154	{
	155	struct bpf_local_storage_data *sdata;
	156
0fe4b381	157	WARN_ON_ONCE(!bpf_rcu_lock_held());
8ea63684 KS	158	if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
	159	return (unsigned long)NULL;
	160
	161	/* explicitly check that the inode_storage_ptr is not
	162	* NULL as inode_storage_lookup returns NULL in this case and
	163	* bpf_local_storage_update expects the owner to have a
	164	* valid storage pointer.
	165	*/
1a9c72ad	166	if (!inode \|\| !inode_storage_ptr(inode))
8ea63684 KS	167	return (unsigned long)NULL;
	168
	169	sdata = inode_storage_lookup(inode, map, true);
	170	if (sdata)
	171	return (unsigned long)sdata->data;
	172
84d571d4	173	/* This helper must only called from where the inode is guaranteed
8ea63684 KS	174	* to have a refcount and cannot be freed.
	175	*/
	176	if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
	177	sdata = bpf_local_storage_update(
	178	inode, (struct bpf_local_storage_map *)map, value,
b00fa38a	179	BPF_NOEXIST, gfp_flags);
8ea63684 KS	180	return IS_ERR(sdata) ? (unsigned long)NULL :
	181	(unsigned long)sdata->data;
	182	}
	183
	184	return (unsigned long)NULL;
	185	}
	186
	187	BPF_CALL_2(bpf_inode_storage_delete,
	188	struct bpf_map , map, struct inode , inode)
	189	{
0fe4b381	190	WARN_ON_ONCE(!bpf_rcu_lock_held());
1a9c72ad KS	191	if (!inode)
	192	return -EINVAL;
	193
84d571d4	194	/* This helper must only called from where the inode is guaranteed
8ea63684 KS	195	* to have a refcount and cannot be freed.
	196	*/
	197	return inode_storage_delete(inode, map);
	198	}
	199
	200	static int notsupp_get_next_key(struct bpf_map map, void key,
	201	void *next_key)
	202	{
	203	return -ENOTSUPP;
	204	}
	205
	206	static struct bpf_map inode_storage_map_alloc(union bpf_attr attr)
	207	{
c83597fa	208	return bpf_local_storage_map_alloc(attr, &inode_cache);
8ea63684 KS	209	}
	210
	211	static void inode_storage_map_free(struct bpf_map *map)
	212	{
c83597fa	213	bpf_local_storage_map_free(map, &inode_cache, NULL);
8ea63684 KS	214	}
8ea63684 KS	215
8ea63684	216	const struct bpf_map_ops inode_storage_map_ops = {
f4d05259	217	.map_meta_equal = bpf_map_meta_equal,
8ea63684 KS	218	.map_alloc_check = bpf_local_storage_map_alloc_check,
	219	.map_alloc = inode_storage_map_alloc,
	220	.map_free = inode_storage_map_free,
	221	.map_get_next_key = notsupp_get_next_key,
	222	.map_lookup_elem = bpf_fd_inode_storage_lookup_elem,
	223	.map_update_elem = bpf_fd_inode_storage_update_elem,
	224	.map_delete_elem = bpf_fd_inode_storage_delete_elem,
	225	.map_check_btf = bpf_local_storage_map_check_btf,
3144bfa5	226	.map_btf_id = &bpf_local_storage_map_btf_id[0],
8ea63684 KS	227	.map_owner_storage_ptr = inode_storage_ptr,
	228	};
	229
9436ef6e	230	BTF_ID_LIST_SINGLE(bpf_inode_storage_btf_ids, struct, inode)
8ea63684 KS	231
	232	const struct bpf_func_proto bpf_inode_storage_get_proto = {
	233	.func = bpf_inode_storage_get,
	234	.gpl_only = false,
	235	.ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
	236	.arg1_type = ARG_CONST_MAP_PTR,
	237	.arg2_type = ARG_PTR_TO_BTF_ID,
9436ef6e	238	.arg2_btf_id = &bpf_inode_storage_btf_ids[0],
8ea63684 KS	239	.arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
8ea63684 KS	240	.arg4_type = ARG_ANYTHING,
8ea63684 KS	241	};
	242
	243	const struct bpf_func_proto bpf_inode_storage_delete_proto = {
	244	.func = bpf_inode_storage_delete,
	245	.gpl_only = false,
	246	.ret_type = RET_INTEGER,
	247	.arg1_type = ARG_CONST_MAP_PTR,
	248	.arg2_type = ARG_PTR_TO_BTF_ID,
9436ef6e	249	.arg2_btf_id = &bpf_inode_storage_btf_ids[0],
8ea63684	250	};