[linux-block.git] / include / linux / bpf_local_storage.h

/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Copyright (c) 2019 Facebook
 * Copyright 2020 Google LLC.
 */

#ifndef _BPF_LOCAL_STORAGE_H
#define _BPF_LOCAL_STORAGE_H

#include <linux/bpf.h>
#include <linux/filter.h>
#include <linux/rculist.h>
#include <linux/list.h>
#include <linux/hash.h>
#include <linux/types.h>
#include <uapi/linux/btf.h>

#define BPF_LOCAL_STORAGE_CACHE_SIZE	16

#define bpf_rcu_lock_held()                                                    \
	(rcu_read_lock_held() || rcu_read_lock_trace_held() ||                 \
	 rcu_read_lock_bh_held())
struct bpf_local_storage_map_bucket {
	struct hlist_head list;
	raw_spinlock_t lock;
};

/* Thp map is not the primary owner of a bpf_local_storage_elem.
 * Instead, the container object (eg. sk->sk_bpf_storage) is.
 *
 * The map (bpf_local_storage_map) is for two purposes
 * 1. Define the size of the "local storage".  It is
 *    the map's value_size.
 *
 * 2. Maintain a list to keep track of all elems such
 *    that they can be cleaned up during the map destruction.
 *
 * When a bpf local storage is being looked up for a
 * particular object,  the "bpf_map" pointer is actually used
 * as the "key" to search in the list of elem in
 * the respective bpf_local_storage owned by the object.
 *
 * e.g. sk->sk_bpf_storage is the mini-map with the "bpf_map" pointer
 * as the searching key.
 */
struct bpf_local_storage_map {
	struct bpf_map map;
	/* Lookup elem does not require accessing the map.
	 *
	 * Updating/Deleting requires a bucket lock to
	 * link/unlink the elem from the map.  Having
	 * multiple buckets to improve contention.
	 */
	struct bpf_local_storage_map_bucket *buckets;
	u32 bucket_log;
	u16 elem_size;
	u16 cache_idx;
};

struct bpf_local_storage_data {
	/* smap is used as the searching key when looking up
	 * from the object's bpf_local_storage.
	 *
	 * Put it in the same cacheline as the data to minimize
	 * the number of cachelines accessed during the cache hit case.
	 */
	struct bpf_local_storage_map __rcu *smap;
	u8 data[] __aligned(8);
};

/* Linked to bpf_local_storage and bpf_local_storage_map */
struct bpf_local_storage_elem {
	struct hlist_node map_node;	/* Linked to bpf_local_storage_map */
	struct hlist_node snode;	/* Linked to bpf_local_storage */
	struct bpf_local_storage __rcu *local_storage;
	struct rcu_head rcu;
	/* 8 bytes hole */
	/* The data is stored in another cacheline to minimize
	 * the number of cachelines access during a cache hit.
	 */
	struct bpf_local_storage_data sdata ____cacheline_aligned;
};

struct bpf_local_storage {
	struct bpf_local_storage_data __rcu *cache[BPF_LOCAL_STORAGE_CACHE_SIZE];
	struct hlist_head list; /* List of bpf_local_storage_elem */
	void *owner;		/* The object that owns the above "list" of
				 * bpf_local_storage_elem.
				 */
	struct rcu_head rcu;
	raw_spinlock_t lock;	/* Protect adding/removing from the "list" */
};

/* U16_MAX is much more than enough for sk local storage
 * considering a tcp_sock is ~2k.
 */
#define BPF_LOCAL_STORAGE_MAX_VALUE_SIZE				       \
	min_t(u32,                                                             \
	      (KMALLOC_MAX_SIZE - MAX_BPF_STACK -                              \
	       sizeof(struct bpf_local_storage_elem)),                         \
	      (U16_MAX - sizeof(struct bpf_local_storage_elem)))

#define SELEM(_SDATA)                                                          \
	container_of((_SDATA), struct bpf_local_storage_elem, sdata)
#define SDATA(_SELEM) (&(_SELEM)->sdata)

#define BPF_LOCAL_STORAGE_CACHE_SIZE	16

struct bpf_local_storage_cache {
	spinlock_t idx_lock;
	u64 idx_usage_counts[BPF_LOCAL_STORAGE_CACHE_SIZE];
};

#define DEFINE_BPF_STORAGE_CACHE(name)				\
static struct bpf_local_storage_cache name = {			\
	.idx_lock = __SPIN_LOCK_UNLOCKED(name.idx_lock),	\
}

u16 bpf_local_storage_cache_idx_get(struct bpf_local_storage_cache *cache);
void bpf_local_storage_cache_idx_free(struct bpf_local_storage_cache *cache,
				      u16 idx);

/* Helper functions for bpf_local_storage */
int bpf_local_storage_map_alloc_check(union bpf_attr *attr);

struct bpf_local_storage_map *bpf_local_storage_map_alloc(union bpf_attr *attr);

struct bpf_local_storage_data *
bpf_local_storage_lookup(struct bpf_local_storage *local_storage,
			 struct bpf_local_storage_map *smap,
			 bool cacheit_lockit);

void bpf_local_storage_map_free(struct bpf_local_storage_map *smap,
				int __percpu *busy_counter);

int bpf_local_storage_map_check_btf(const struct bpf_map *map,
				    const struct btf *btf,
				    const struct btf_type *key_type,
				    const struct btf_type *value_type);

void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
				   struct bpf_local_storage_elem *selem);

bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,
				     struct bpf_local_storage_elem *selem,
				     bool uncharge_omem);

void bpf_selem_unlink(struct bpf_local_storage_elem *selem);

void bpf_selem_link_map(struct bpf_local_storage_map *smap,
			struct bpf_local_storage_elem *selem);

void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem);

struct bpf_local_storage_elem *
bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, void *value,
		bool charge_mem, gfp_t gfp_flags);

int
bpf_local_storage_alloc(void *owner,
			struct bpf_local_storage_map *smap,
			struct bpf_local_storage_elem *first_selem,
			gfp_t gfp_flags);

struct bpf_local_storage_data *
bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
			 void *value, u64 map_flags, gfp_t gfp_flags);

void bpf_local_storage_free_rcu(struct rcu_head *rcu);

#endif /* _BPF_LOCAL_STORAGE_H */
Commit	Line	Data
450af8d0 KS	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	/*
	3	* Copyright (c) 2019 Facebook
	4	* Copyright 2020 Google LLC.
	5	*/
	6
	7	#ifndef _BPF_LOCAL_STORAGE_H
	8	#define _BPF_LOCAL_STORAGE_H
	9
	10	#include <linux/bpf.h>
b6459415	11	#include <linux/filter.h>
450af8d0 KS	12	#include <linux/rculist.h>
	13	#include <linux/list.h>
	14	#include <linux/hash.h>
	15	#include <linux/types.h>
	16	#include <uapi/linux/btf.h>
	17
	18	#define BPF_LOCAL_STORAGE_CACHE_SIZE 16
	19
0fe4b381 KS	20	#define bpf_rcu_lock_held() \
	21	(rcu_read_lock_held() \|\| rcu_read_lock_trace_held() \|\| \
	22	rcu_read_lock_bh_held())
450af8d0 KS	23	struct bpf_local_storage_map_bucket {
	24	struct hlist_head list;
	25	raw_spinlock_t lock;
	26	};
	27
	28	/* Thp map is not the primary owner of a bpf_local_storage_elem.
	29	* Instead, the container object (eg. sk->sk_bpf_storage) is.
	30	*
	31	* The map (bpf_local_storage_map) is for two purposes
	32	* 1. Define the size of the "local storage". It is
	33	* the map's value_size.
	34	*
	35	* 2. Maintain a list to keep track of all elems such
	36	* that they can be cleaned up during the map destruction.
	37	*
	38	* When a bpf local storage is being looked up for a
	39	* particular object, the "bpf_map" pointer is actually used
	40	* as the "key" to search in the list of elem in
	41	* the respective bpf_local_storage owned by the object.
	42	*
	43	* e.g. sk->sk_bpf_storage is the mini-map with the "bpf_map" pointer
	44	* as the searching key.
	45	*/
	46	struct bpf_local_storage_map {
	47	struct bpf_map map;
	48	/* Lookup elem does not require accessing the map.
	49	*
	50	* Updating/Deleting requires a bucket lock to
	51	* link/unlink the elem from the map. Having
	52	* multiple buckets to improve contention.
	53	*/
	54	struct bpf_local_storage_map_bucket *buckets;
	55	u32 bucket_log;
	56	u16 elem_size;
	57	u16 cache_idx;
	58	};
	59
	60	struct bpf_local_storage_data {
	61	/* smap is used as the searching key when looking up
	62	* from the object's bpf_local_storage.
	63	*
	64	* Put it in the same cacheline as the data to minimize
8fb33b60	65	* the number of cachelines accessed during the cache hit case.
450af8d0 KS	66	*/
	67	struct bpf_local_storage_map __rcu *smap;
	68	u8 data[] __aligned(8);
	69	};
	70
	71	/* Linked to bpf_local_storage and bpf_local_storage_map */
	72	struct bpf_local_storage_elem {
	73	struct hlist_node map_node; /* Linked to bpf_local_storage_map */
	74	struct hlist_node snode; /* Linked to bpf_local_storage */
	75	struct bpf_local_storage __rcu *local_storage;
	76	struct rcu_head rcu;
	77	/* 8 bytes hole */
8fb33b60	78	/* The data is stored in another cacheline to minimize
450af8d0 KS	79	* the number of cachelines access during a cache hit.
	80	*/
	81	struct bpf_local_storage_data sdata ____cacheline_aligned;
	82	};
	83
	84	struct bpf_local_storage {
	85	struct bpf_local_storage_data __rcu *cache[BPF_LOCAL_STORAGE_CACHE_SIZE];
	86	struct hlist_head list; /* List of bpf_local_storage_elem */
	87	void owner; / The object that owns the above "list" of
	88	* bpf_local_storage_elem.
	89	*/
	90	struct rcu_head rcu;
	91	raw_spinlock_t lock; /* Protect adding/removing from the "list" */
	92	};
	93
	94	/* U16_MAX is much more than enough for sk local storage
	95	* considering a tcp_sock is ~2k.
	96	*/
	97	#define BPF_LOCAL_STORAGE_MAX_VALUE_SIZE \
	98	min_t(u32, \
	99	(KMALLOC_MAX_SIZE - MAX_BPF_STACK - \
	100	sizeof(struct bpf_local_storage_elem)), \
	101	(U16_MAX - sizeof(struct bpf_local_storage_elem)))
	102
	103	#define SELEM(_SDATA) \
	104	container_of((_SDATA), struct bpf_local_storage_elem, sdata)
	105	#define SDATA(_SELEM) (&(_SELEM)->sdata)
	106
	107	#define BPF_LOCAL_STORAGE_CACHE_SIZE 16
	108
	109	struct bpf_local_storage_cache {
	110	spinlock_t idx_lock;
	111	u64 idx_usage_counts[BPF_LOCAL_STORAGE_CACHE_SIZE];
	112	};
	113
	114	#define DEFINE_BPF_STORAGE_CACHE(name) \
	115	static struct bpf_local_storage_cache name = { \
	116	.idx_lock = __SPIN_LOCK_UNLOCKED(name.idx_lock), \
	117	}
	118
	119	u16 bpf_local_storage_cache_idx_get(struct bpf_local_storage_cache *cache);
	120	void bpf_local_storage_cache_idx_free(struct bpf_local_storage_cache *cache,
	121	u16 idx);
	122
	123	/* Helper functions for bpf_local_storage */
	124	int bpf_local_storage_map_alloc_check(union bpf_attr *attr);
	125
	126	struct bpf_local_storage_map bpf_local_storage_map_alloc(union bpf_attr attr);
	127
	128	struct bpf_local_storage_data *
	129	bpf_local_storage_lookup(struct bpf_local_storage *local_storage,
	130	struct bpf_local_storage_map *smap,
	131	bool cacheit_lockit);
	132
bc235cdb SL	133	void bpf_local_storage_map_free(struct bpf_local_storage_map *smap,
bc235cdb SL	134	int __percpu *busy_counter);
450af8d0 KS	135
	136	int bpf_local_storage_map_check_btf(const struct bpf_map *map,
	137	const struct btf *btf,
	138	const struct btf_type *key_type,
	139	const struct btf_type *value_type);
	140
	141	void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
	142	struct bpf_local_storage_elem *selem);
	143
	144	bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,
	145	struct bpf_local_storage_elem *selem,
	146	bool uncharge_omem);
	147
	148	void bpf_selem_unlink(struct bpf_local_storage_elem *selem);
	149
	150	void bpf_selem_link_map(struct bpf_local_storage_map *smap,
	151	struct bpf_local_storage_elem *selem);
	152
	153	void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem);
	154
	155	struct bpf_local_storage_elem *
	156	bpf_selem_alloc(struct bpf_local_storage_map smap, void owner, void *value,
b00fa38a	157	bool charge_mem, gfp_t gfp_flags);
450af8d0 KS	158
	159	int
	160	bpf_local_storage_alloc(void *owner,
	161	struct bpf_local_storage_map *smap,
b00fa38a JK	162	struct bpf_local_storage_elem *first_selem,
b00fa38a JK	163	gfp_t gfp_flags);
450af8d0 KS	164
	165	struct bpf_local_storage_data *
	166	bpf_local_storage_update(void owner, struct bpf_local_storage_map smap,
b00fa38a	167	void *value, u64 map_flags, gfp_t gfp_flags);
450af8d0	168
0fe4b381 KS	169	void bpf_local_storage_free_rcu(struct rcu_head *rcu);
0fe4b381 KS	170
450af8d0	171	#endif /* _BPF_LOCAL_STORAGE_H */