Commit | Line | Data |
---|---|---|
450af8d0 KS |
1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* | |
3 | * Copyright (c) 2019 Facebook | |
4 | * Copyright 2020 Google LLC. | |
5 | */ | |
6 | ||
7 | #ifndef _BPF_LOCAL_STORAGE_H | |
8 | #define _BPF_LOCAL_STORAGE_H | |
9 | ||
10 | #include <linux/bpf.h> | |
11 | #include <linux/rculist.h> | |
12 | #include <linux/list.h> | |
13 | #include <linux/hash.h> | |
14 | #include <linux/types.h> | |
15 | #include <uapi/linux/btf.h> | |
16 | ||
17 | #define BPF_LOCAL_STORAGE_CACHE_SIZE 16 | |
18 | ||
19 | struct bpf_local_storage_map_bucket { | |
20 | struct hlist_head list; | |
21 | raw_spinlock_t lock; | |
22 | }; | |
23 | ||
24 | /* Thp map is not the primary owner of a bpf_local_storage_elem. | |
25 | * Instead, the container object (eg. sk->sk_bpf_storage) is. | |
26 | * | |
27 | * The map (bpf_local_storage_map) is for two purposes | |
28 | * 1. Define the size of the "local storage". It is | |
29 | * the map's value_size. | |
30 | * | |
31 | * 2. Maintain a list to keep track of all elems such | |
32 | * that they can be cleaned up during the map destruction. | |
33 | * | |
34 | * When a bpf local storage is being looked up for a | |
35 | * particular object, the "bpf_map" pointer is actually used | |
36 | * as the "key" to search in the list of elem in | |
37 | * the respective bpf_local_storage owned by the object. | |
38 | * | |
39 | * e.g. sk->sk_bpf_storage is the mini-map with the "bpf_map" pointer | |
40 | * as the searching key. | |
41 | */ | |
42 | struct bpf_local_storage_map { | |
43 | struct bpf_map map; | |
44 | /* Lookup elem does not require accessing the map. | |
45 | * | |
46 | * Updating/Deleting requires a bucket lock to | |
47 | * link/unlink the elem from the map. Having | |
48 | * multiple buckets to improve contention. | |
49 | */ | |
50 | struct bpf_local_storage_map_bucket *buckets; | |
51 | u32 bucket_log; | |
52 | u16 elem_size; | |
53 | u16 cache_idx; | |
54 | }; | |
55 | ||
56 | struct bpf_local_storage_data { | |
57 | /* smap is used as the searching key when looking up | |
58 | * from the object's bpf_local_storage. | |
59 | * | |
60 | * Put it in the same cacheline as the data to minimize | |
8fb33b60 | 61 | * the number of cachelines accessed during the cache hit case. |
450af8d0 KS |
62 | */ |
63 | struct bpf_local_storage_map __rcu *smap; | |
64 | u8 data[] __aligned(8); | |
65 | }; | |
66 | ||
67 | /* Linked to bpf_local_storage and bpf_local_storage_map */ | |
68 | struct bpf_local_storage_elem { | |
69 | struct hlist_node map_node; /* Linked to bpf_local_storage_map */ | |
70 | struct hlist_node snode; /* Linked to bpf_local_storage */ | |
71 | struct bpf_local_storage __rcu *local_storage; | |
72 | struct rcu_head rcu; | |
73 | /* 8 bytes hole */ | |
8fb33b60 | 74 | /* The data is stored in another cacheline to minimize |
450af8d0 KS |
75 | * the number of cachelines access during a cache hit. |
76 | */ | |
77 | struct bpf_local_storage_data sdata ____cacheline_aligned; | |
78 | }; | |
79 | ||
80 | struct bpf_local_storage { | |
81 | struct bpf_local_storage_data __rcu *cache[BPF_LOCAL_STORAGE_CACHE_SIZE]; | |
82 | struct hlist_head list; /* List of bpf_local_storage_elem */ | |
83 | void *owner; /* The object that owns the above "list" of | |
84 | * bpf_local_storage_elem. | |
85 | */ | |
86 | struct rcu_head rcu; | |
87 | raw_spinlock_t lock; /* Protect adding/removing from the "list" */ | |
88 | }; | |
89 | ||
90 | /* U16_MAX is much more than enough for sk local storage | |
91 | * considering a tcp_sock is ~2k. | |
92 | */ | |
93 | #define BPF_LOCAL_STORAGE_MAX_VALUE_SIZE \ | |
94 | min_t(u32, \ | |
95 | (KMALLOC_MAX_SIZE - MAX_BPF_STACK - \ | |
96 | sizeof(struct bpf_local_storage_elem)), \ | |
97 | (U16_MAX - sizeof(struct bpf_local_storage_elem))) | |
98 | ||
99 | #define SELEM(_SDATA) \ | |
100 | container_of((_SDATA), struct bpf_local_storage_elem, sdata) | |
101 | #define SDATA(_SELEM) (&(_SELEM)->sdata) | |
102 | ||
103 | #define BPF_LOCAL_STORAGE_CACHE_SIZE 16 | |
104 | ||
105 | struct bpf_local_storage_cache { | |
106 | spinlock_t idx_lock; | |
107 | u64 idx_usage_counts[BPF_LOCAL_STORAGE_CACHE_SIZE]; | |
108 | }; | |
109 | ||
110 | #define DEFINE_BPF_STORAGE_CACHE(name) \ | |
111 | static struct bpf_local_storage_cache name = { \ | |
112 | .idx_lock = __SPIN_LOCK_UNLOCKED(name.idx_lock), \ | |
113 | } | |
114 | ||
115 | u16 bpf_local_storage_cache_idx_get(struct bpf_local_storage_cache *cache); | |
116 | void bpf_local_storage_cache_idx_free(struct bpf_local_storage_cache *cache, | |
117 | u16 idx); | |
118 | ||
119 | /* Helper functions for bpf_local_storage */ | |
120 | int bpf_local_storage_map_alloc_check(union bpf_attr *attr); | |
121 | ||
122 | struct bpf_local_storage_map *bpf_local_storage_map_alloc(union bpf_attr *attr); | |
123 | ||
124 | struct bpf_local_storage_data * | |
125 | bpf_local_storage_lookup(struct bpf_local_storage *local_storage, | |
126 | struct bpf_local_storage_map *smap, | |
127 | bool cacheit_lockit); | |
128 | ||
bc235cdb SL |
129 | void bpf_local_storage_map_free(struct bpf_local_storage_map *smap, |
130 | int __percpu *busy_counter); | |
450af8d0 KS |
131 | |
132 | int bpf_local_storage_map_check_btf(const struct bpf_map *map, | |
133 | const struct btf *btf, | |
134 | const struct btf_type *key_type, | |
135 | const struct btf_type *value_type); | |
136 | ||
137 | void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage, | |
138 | struct bpf_local_storage_elem *selem); | |
139 | ||
140 | bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage, | |
141 | struct bpf_local_storage_elem *selem, | |
142 | bool uncharge_omem); | |
143 | ||
144 | void bpf_selem_unlink(struct bpf_local_storage_elem *selem); | |
145 | ||
146 | void bpf_selem_link_map(struct bpf_local_storage_map *smap, | |
147 | struct bpf_local_storage_elem *selem); | |
148 | ||
149 | void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem); | |
150 | ||
151 | struct bpf_local_storage_elem * | |
152 | bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, void *value, | |
153 | bool charge_mem); | |
154 | ||
155 | int | |
156 | bpf_local_storage_alloc(void *owner, | |
157 | struct bpf_local_storage_map *smap, | |
158 | struct bpf_local_storage_elem *first_selem); | |
159 | ||
160 | struct bpf_local_storage_data * | |
161 | bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, | |
162 | void *value, u64 map_flags); | |
163 | ||
164 | #endif /* _BPF_LOCAL_STORAGE_H */ |