Commit | Line | Data |
---|---|---|
6ac99e8f MKL |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright (c) 2019 Facebook */ | |
3 | #include <linux/rculist.h> | |
4 | #include <linux/list.h> | |
5 | #include <linux/hash.h> | |
6 | #include <linux/types.h> | |
7 | #include <linux/spinlock.h> | |
8 | #include <linux/bpf.h> | |
5ce6e77c | 9 | #include <linux/btf_ids.h> |
450af8d0 | 10 | #include <linux/bpf_local_storage.h> |
6ac99e8f MKL |
11 | #include <net/bpf_sk_storage.h> |
12 | #include <net/sock.h> | |
1ed4d924 | 13 | #include <uapi/linux/sock_diag.h> |
6ac99e8f MKL |
14 | #include <uapi/linux/btf.h> |
15 | ||
4cc9ce4e KS |
16 | DEFINE_BPF_STORAGE_CACHE(sk_cache); |
17 | ||
1f00d375 | 18 | static struct bpf_local_storage_data * |
e794bfdd | 19 | bpf_sk_storage_lookup(struct sock *sk, struct bpf_map *map, bool cacheit_lockit) |
6ac99e8f | 20 | { |
1f00d375 KS |
21 | struct bpf_local_storage *sk_storage; |
22 | struct bpf_local_storage_map *smap; | |
6ac99e8f MKL |
23 | |
24 | sk_storage = rcu_dereference(sk->sk_bpf_storage); | |
25 | if (!sk_storage) | |
26 | return NULL; | |
27 | ||
1f00d375 KS |
28 | smap = (struct bpf_local_storage_map *)map; |
29 | return bpf_local_storage_lookup(sk_storage, smap, cacheit_lockit); | |
6ac99e8f MKL |
30 | } |
31 | ||
e794bfdd | 32 | static int bpf_sk_storage_del(struct sock *sk, struct bpf_map *map) |
6ac99e8f | 33 | { |
1f00d375 | 34 | struct bpf_local_storage_data *sdata; |
6ac99e8f | 35 | |
e794bfdd | 36 | sdata = bpf_sk_storage_lookup(sk, map, false); |
6ac99e8f MKL |
37 | if (!sdata) |
38 | return -ENOENT; | |
39 | ||
1f00d375 | 40 | bpf_selem_unlink(SELEM(sdata)); |
6ac99e8f MKL |
41 | |
42 | return 0; | |
43 | } | |
44 | ||
8f51dfc7 | 45 | /* Called by __sk_destruct() & bpf_sk_storage_clone() */ |
6ac99e8f MKL |
46 | void bpf_sk_storage_free(struct sock *sk) |
47 | { | |
1f00d375 KS |
48 | struct bpf_local_storage_elem *selem; |
49 | struct bpf_local_storage *sk_storage; | |
6ac99e8f MKL |
50 | bool free_sk_storage = false; |
51 | struct hlist_node *n; | |
52 | ||
53 | rcu_read_lock(); | |
54 | sk_storage = rcu_dereference(sk->sk_bpf_storage); | |
55 | if (!sk_storage) { | |
56 | rcu_read_unlock(); | |
57 | return; | |
58 | } | |
59 | ||
60 | /* Netiher the bpf_prog nor the bpf-map's syscall | |
61 | * could be modifying the sk_storage->list now. | |
62 | * Thus, no elem can be added-to or deleted-from the | |
63 | * sk_storage->list by the bpf_prog or by the bpf-map's syscall. | |
64 | * | |
1f00d375 | 65 | * It is racing with bpf_local_storage_map_free() alone |
6ac99e8f MKL |
66 | * when unlinking elem from the sk_storage->list and |
67 | * the map's bucket->list. | |
68 | */ | |
69 | raw_spin_lock_bh(&sk_storage->lock); | |
70 | hlist_for_each_entry_safe(selem, n, &sk_storage->list, snode) { | |
71 | /* Always unlink from map before unlinking from | |
72 | * sk_storage. | |
73 | */ | |
1f00d375 KS |
74 | bpf_selem_unlink_map(selem); |
75 | free_sk_storage = bpf_selem_unlink_storage_nolock(sk_storage, | |
76 | selem, true); | |
6ac99e8f MKL |
77 | } |
78 | raw_spin_unlock_bh(&sk_storage->lock); | |
79 | rcu_read_unlock(); | |
80 | ||
81 | if (free_sk_storage) | |
82 | kfree_rcu(sk_storage, rcu); | |
83 | } | |
84 | ||
e794bfdd | 85 | static void bpf_sk_storage_map_free(struct bpf_map *map) |
f836a56e KS |
86 | { |
87 | struct bpf_local_storage_map *smap; | |
88 | ||
89 | smap = (struct bpf_local_storage_map *)map; | |
90 | bpf_local_storage_cache_idx_free(&sk_cache, smap->cache_idx); | |
91 | bpf_local_storage_map_free(smap); | |
6ac99e8f MKL |
92 | } |
93 | ||
e794bfdd | 94 | static struct bpf_map *bpf_sk_storage_map_alloc(union bpf_attr *attr) |
f836a56e KS |
95 | { |
96 | struct bpf_local_storage_map *smap; | |
97 | ||
98 | smap = bpf_local_storage_map_alloc(attr); | |
99 | if (IS_ERR(smap)) | |
100 | return ERR_CAST(smap); | |
101 | ||
102 | smap->cache_idx = bpf_local_storage_cache_idx_get(&sk_cache); | |
6ac99e8f MKL |
103 | return &smap->map; |
104 | } | |
105 | ||
106 | static int notsupp_get_next_key(struct bpf_map *map, void *key, | |
107 | void *next_key) | |
108 | { | |
109 | return -ENOTSUPP; | |
110 | } | |
111 | ||
6ac99e8f MKL |
112 | static void *bpf_fd_sk_storage_lookup_elem(struct bpf_map *map, void *key) |
113 | { | |
1f00d375 | 114 | struct bpf_local_storage_data *sdata; |
6ac99e8f MKL |
115 | struct socket *sock; |
116 | int fd, err; | |
117 | ||
118 | fd = *(int *)key; | |
119 | sock = sockfd_lookup(fd, &err); | |
120 | if (sock) { | |
e794bfdd | 121 | sdata = bpf_sk_storage_lookup(sock->sk, map, true); |
6ac99e8f MKL |
122 | sockfd_put(sock); |
123 | return sdata ? sdata->data : NULL; | |
124 | } | |
125 | ||
126 | return ERR_PTR(err); | |
127 | } | |
128 | ||
129 | static int bpf_fd_sk_storage_update_elem(struct bpf_map *map, void *key, | |
130 | void *value, u64 map_flags) | |
131 | { | |
1f00d375 | 132 | struct bpf_local_storage_data *sdata; |
6ac99e8f MKL |
133 | struct socket *sock; |
134 | int fd, err; | |
135 | ||
136 | fd = *(int *)key; | |
137 | sock = sockfd_lookup(fd, &err); | |
138 | if (sock) { | |
f836a56e KS |
139 | sdata = bpf_local_storage_update( |
140 | sock->sk, (struct bpf_local_storage_map *)map, value, | |
141 | map_flags); | |
6ac99e8f | 142 | sockfd_put(sock); |
71f150f4 | 143 | return PTR_ERR_OR_ZERO(sdata); |
6ac99e8f MKL |
144 | } |
145 | ||
146 | return err; | |
147 | } | |
148 | ||
149 | static int bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key) | |
150 | { | |
151 | struct socket *sock; | |
152 | int fd, err; | |
153 | ||
154 | fd = *(int *)key; | |
155 | sock = sockfd_lookup(fd, &err); | |
156 | if (sock) { | |
e794bfdd | 157 | err = bpf_sk_storage_del(sock->sk, map); |
6ac99e8f MKL |
158 | sockfd_put(sock); |
159 | return err; | |
160 | } | |
161 | ||
162 | return err; | |
163 | } | |
164 | ||
1f00d375 | 165 | static struct bpf_local_storage_elem * |
8f51dfc7 | 166 | bpf_sk_storage_clone_elem(struct sock *newsk, |
1f00d375 KS |
167 | struct bpf_local_storage_map *smap, |
168 | struct bpf_local_storage_elem *selem) | |
8f51dfc7 | 169 | { |
1f00d375 | 170 | struct bpf_local_storage_elem *copy_selem; |
8f51dfc7 | 171 | |
1f00d375 | 172 | copy_selem = bpf_selem_alloc(smap, newsk, NULL, true); |
8f51dfc7 SF |
173 | if (!copy_selem) |
174 | return NULL; | |
175 | ||
176 | if (map_value_has_spin_lock(&smap->map)) | |
177 | copy_map_value_locked(&smap->map, SDATA(copy_selem)->data, | |
178 | SDATA(selem)->data, true); | |
179 | else | |
180 | copy_map_value(&smap->map, SDATA(copy_selem)->data, | |
181 | SDATA(selem)->data); | |
182 | ||
183 | return copy_selem; | |
184 | } | |
185 | ||
186 | int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk) | |
187 | { | |
1f00d375 KS |
188 | struct bpf_local_storage *new_sk_storage = NULL; |
189 | struct bpf_local_storage *sk_storage; | |
190 | struct bpf_local_storage_elem *selem; | |
8f51dfc7 SF |
191 | int ret = 0; |
192 | ||
193 | RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL); | |
194 | ||
195 | rcu_read_lock(); | |
196 | sk_storage = rcu_dereference(sk->sk_bpf_storage); | |
197 | ||
198 | if (!sk_storage || hlist_empty(&sk_storage->list)) | |
199 | goto out; | |
200 | ||
201 | hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) { | |
1f00d375 KS |
202 | struct bpf_local_storage_elem *copy_selem; |
203 | struct bpf_local_storage_map *smap; | |
8f51dfc7 SF |
204 | struct bpf_map *map; |
205 | ||
206 | smap = rcu_dereference(SDATA(selem)->smap); | |
207 | if (!(smap->map.map_flags & BPF_F_CLONE)) | |
208 | continue; | |
209 | ||
210 | /* Note that for lockless listeners adding new element | |
1f00d375 | 211 | * here can race with cleanup in bpf_local_storage_map_free. |
8f51dfc7 SF |
212 | * Try to grab map refcnt to make sure that it's still |
213 | * alive and prevent concurrent removal. | |
214 | */ | |
1e0bd5a0 | 215 | map = bpf_map_inc_not_zero(&smap->map); |
8f51dfc7 SF |
216 | if (IS_ERR(map)) |
217 | continue; | |
218 | ||
219 | copy_selem = bpf_sk_storage_clone_elem(newsk, smap, selem); | |
220 | if (!copy_selem) { | |
221 | ret = -ENOMEM; | |
222 | bpf_map_put(map); | |
223 | goto out; | |
224 | } | |
225 | ||
226 | if (new_sk_storage) { | |
1f00d375 KS |
227 | bpf_selem_link_map(smap, copy_selem); |
228 | bpf_selem_link_storage_nolock(new_sk_storage, copy_selem); | |
8f51dfc7 | 229 | } else { |
f836a56e | 230 | ret = bpf_local_storage_alloc(newsk, smap, copy_selem); |
8f51dfc7 SF |
231 | if (ret) { |
232 | kfree(copy_selem); | |
233 | atomic_sub(smap->elem_size, | |
234 | &newsk->sk_omem_alloc); | |
235 | bpf_map_put(map); | |
236 | goto out; | |
237 | } | |
238 | ||
1f00d375 KS |
239 | new_sk_storage = |
240 | rcu_dereference(copy_selem->local_storage); | |
8f51dfc7 SF |
241 | } |
242 | bpf_map_put(map); | |
243 | } | |
244 | ||
245 | out: | |
246 | rcu_read_unlock(); | |
247 | ||
248 | /* In case of an error, don't free anything explicitly here, the | |
249 | * caller is responsible to call bpf_sk_storage_free. | |
250 | */ | |
251 | ||
252 | return ret; | |
253 | } | |
254 | ||
6ac99e8f MKL |
255 | BPF_CALL_4(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk, |
256 | void *, value, u64, flags) | |
257 | { | |
1f00d375 | 258 | struct bpf_local_storage_data *sdata; |
6ac99e8f | 259 | |
592a3498 | 260 | if (!sk || !sk_fullsock(sk) || flags > BPF_SK_STORAGE_GET_F_CREATE) |
6ac99e8f MKL |
261 | return (unsigned long)NULL; |
262 | ||
e794bfdd | 263 | sdata = bpf_sk_storage_lookup(sk, map, true); |
6ac99e8f MKL |
264 | if (sdata) |
265 | return (unsigned long)sdata->data; | |
266 | ||
267 | if (flags == BPF_SK_STORAGE_GET_F_CREATE && | |
268 | /* Cannot add new elem to a going away sk. | |
269 | * Otherwise, the new elem may become a leak | |
270 | * (and also other memory issues during map | |
271 | * destruction). | |
272 | */ | |
273 | refcount_inc_not_zero(&sk->sk_refcnt)) { | |
f836a56e KS |
274 | sdata = bpf_local_storage_update( |
275 | sk, (struct bpf_local_storage_map *)map, value, | |
276 | BPF_NOEXIST); | |
6ac99e8f MKL |
277 | /* sk must be a fullsock (guaranteed by verifier), |
278 | * so sock_gen_put() is unnecessary. | |
279 | */ | |
280 | sock_put(sk); | |
281 | return IS_ERR(sdata) ? | |
282 | (unsigned long)NULL : (unsigned long)sdata->data; | |
283 | } | |
284 | ||
285 | return (unsigned long)NULL; | |
286 | } | |
287 | ||
288 | BPF_CALL_2(bpf_sk_storage_delete, struct bpf_map *, map, struct sock *, sk) | |
289 | { | |
592a3498 MKL |
290 | if (!sk || !sk_fullsock(sk)) |
291 | return -EINVAL; | |
292 | ||
6ac99e8f MKL |
293 | if (refcount_inc_not_zero(&sk->sk_refcnt)) { |
294 | int err; | |
295 | ||
e794bfdd | 296 | err = bpf_sk_storage_del(sk, map); |
6ac99e8f MKL |
297 | sock_put(sk); |
298 | return err; | |
299 | } | |
300 | ||
301 | return -ENOENT; | |
302 | } | |
303 | ||
e794bfdd MKL |
304 | static int bpf_sk_storage_charge(struct bpf_local_storage_map *smap, |
305 | void *owner, u32 size) | |
f836a56e | 306 | { |
9e838b02 MKL |
307 | struct sock *sk = (struct sock *)owner; |
308 | ||
309 | /* same check as in sock_kmalloc() */ | |
310 | if (size <= sysctl_optmem_max && | |
311 | atomic_read(&sk->sk_omem_alloc) + size < sysctl_optmem_max) { | |
312 | atomic_add(size, &sk->sk_omem_alloc); | |
313 | return 0; | |
314 | } | |
315 | ||
316 | return -ENOMEM; | |
f836a56e KS |
317 | } |
318 | ||
e794bfdd MKL |
319 | static void bpf_sk_storage_uncharge(struct bpf_local_storage_map *smap, |
320 | void *owner, u32 size) | |
f836a56e KS |
321 | { |
322 | struct sock *sk = owner; | |
323 | ||
324 | atomic_sub(size, &sk->sk_omem_alloc); | |
325 | } | |
326 | ||
327 | static struct bpf_local_storage __rcu ** | |
e794bfdd | 328 | bpf_sk_storage_ptr(void *owner) |
f836a56e KS |
329 | { |
330 | struct sock *sk = owner; | |
331 | ||
332 | return &sk->sk_bpf_storage; | |
333 | } | |
334 | ||
2872e9ac | 335 | static int sk_storage_map_btf_id; |
6ac99e8f | 336 | const struct bpf_map_ops sk_storage_map_ops = { |
f4d05259 | 337 | .map_meta_equal = bpf_map_meta_equal, |
1f00d375 | 338 | .map_alloc_check = bpf_local_storage_map_alloc_check, |
e794bfdd MKL |
339 | .map_alloc = bpf_sk_storage_map_alloc, |
340 | .map_free = bpf_sk_storage_map_free, | |
6ac99e8f MKL |
341 | .map_get_next_key = notsupp_get_next_key, |
342 | .map_lookup_elem = bpf_fd_sk_storage_lookup_elem, | |
343 | .map_update_elem = bpf_fd_sk_storage_update_elem, | |
344 | .map_delete_elem = bpf_fd_sk_storage_delete_elem, | |
1f00d375 KS |
345 | .map_check_btf = bpf_local_storage_map_check_btf, |
346 | .map_btf_name = "bpf_local_storage_map", | |
2872e9ac | 347 | .map_btf_id = &sk_storage_map_btf_id, |
e794bfdd MKL |
348 | .map_local_storage_charge = bpf_sk_storage_charge, |
349 | .map_local_storage_uncharge = bpf_sk_storage_uncharge, | |
350 | .map_owner_storage_ptr = bpf_sk_storage_ptr, | |
6ac99e8f MKL |
351 | }; |
352 | ||
353 | const struct bpf_func_proto bpf_sk_storage_get_proto = { | |
354 | .func = bpf_sk_storage_get, | |
355 | .gpl_only = false, | |
356 | .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, | |
357 | .arg1_type = ARG_CONST_MAP_PTR, | |
592a3498 | 358 | .arg2_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, |
6ac99e8f MKL |
359 | .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, |
360 | .arg4_type = ARG_ANYTHING, | |
361 | }; | |
362 | ||
f7c6cb1d SF |
363 | const struct bpf_func_proto bpf_sk_storage_get_cg_sock_proto = { |
364 | .func = bpf_sk_storage_get, | |
365 | .gpl_only = false, | |
366 | .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, | |
367 | .arg1_type = ARG_CONST_MAP_PTR, | |
368 | .arg2_type = ARG_PTR_TO_CTX, /* context is 'struct sock' */ | |
369 | .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, | |
370 | .arg4_type = ARG_ANYTHING, | |
371 | }; | |
372 | ||
6ac99e8f MKL |
373 | const struct bpf_func_proto bpf_sk_storage_delete_proto = { |
374 | .func = bpf_sk_storage_delete, | |
375 | .gpl_only = false, | |
376 | .ret_type = RET_INTEGER, | |
377 | .arg1_type = ARG_CONST_MAP_PTR, | |
592a3498 | 378 | .arg2_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, |
30897832 KS |
379 | }; |
380 | ||
1ed4d924 MKL |
381 | struct bpf_sk_storage_diag { |
382 | u32 nr_maps; | |
383 | struct bpf_map *maps[]; | |
384 | }; | |
385 | ||
386 | /* The reply will be like: | |
387 | * INET_DIAG_BPF_SK_STORAGES (nla_nest) | |
388 | * SK_DIAG_BPF_STORAGE (nla_nest) | |
389 | * SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32) | |
390 | * SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit) | |
391 | * SK_DIAG_BPF_STORAGE (nla_nest) | |
392 | * SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32) | |
393 | * SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit) | |
394 | * .... | |
395 | */ | |
396 | static int nla_value_size(u32 value_size) | |
397 | { | |
398 | /* SK_DIAG_BPF_STORAGE (nla_nest) | |
399 | * SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32) | |
400 | * SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit) | |
401 | */ | |
402 | return nla_total_size(0) + nla_total_size(sizeof(u32)) + | |
403 | nla_total_size_64bit(value_size); | |
404 | } | |
405 | ||
406 | void bpf_sk_storage_diag_free(struct bpf_sk_storage_diag *diag) | |
407 | { | |
408 | u32 i; | |
409 | ||
410 | if (!diag) | |
411 | return; | |
412 | ||
413 | for (i = 0; i < diag->nr_maps; i++) | |
414 | bpf_map_put(diag->maps[i]); | |
415 | ||
416 | kfree(diag); | |
417 | } | |
418 | EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_free); | |
419 | ||
420 | static bool diag_check_dup(const struct bpf_sk_storage_diag *diag, | |
421 | const struct bpf_map *map) | |
422 | { | |
423 | u32 i; | |
424 | ||
425 | for (i = 0; i < diag->nr_maps; i++) { | |
426 | if (diag->maps[i] == map) | |
427 | return true; | |
428 | } | |
429 | ||
430 | return false; | |
431 | } | |
432 | ||
433 | struct bpf_sk_storage_diag * | |
434 | bpf_sk_storage_diag_alloc(const struct nlattr *nla_stgs) | |
435 | { | |
436 | struct bpf_sk_storage_diag *diag; | |
437 | struct nlattr *nla; | |
438 | u32 nr_maps = 0; | |
439 | int rem, err; | |
440 | ||
1f00d375 | 441 | /* bpf_local_storage_map is currently limited to CAP_SYS_ADMIN as |
1ed4d924 MKL |
442 | * the map_alloc_check() side also does. |
443 | */ | |
2c78ee89 | 444 | if (!bpf_capable()) |
1ed4d924 MKL |
445 | return ERR_PTR(-EPERM); |
446 | ||
447 | nla_for_each_nested(nla, nla_stgs, rem) { | |
448 | if (nla_type(nla) == SK_DIAG_BPF_STORAGE_REQ_MAP_FD) | |
449 | nr_maps++; | |
450 | } | |
451 | ||
452 | diag = kzalloc(sizeof(*diag) + sizeof(diag->maps[0]) * nr_maps, | |
453 | GFP_KERNEL); | |
454 | if (!diag) | |
455 | return ERR_PTR(-ENOMEM); | |
456 | ||
457 | nla_for_each_nested(nla, nla_stgs, rem) { | |
458 | struct bpf_map *map; | |
459 | int map_fd; | |
460 | ||
461 | if (nla_type(nla) != SK_DIAG_BPF_STORAGE_REQ_MAP_FD) | |
462 | continue; | |
463 | ||
464 | map_fd = nla_get_u32(nla); | |
465 | map = bpf_map_get(map_fd); | |
466 | if (IS_ERR(map)) { | |
467 | err = PTR_ERR(map); | |
468 | goto err_free; | |
469 | } | |
470 | if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) { | |
471 | bpf_map_put(map); | |
472 | err = -EINVAL; | |
473 | goto err_free; | |
474 | } | |
475 | if (diag_check_dup(diag, map)) { | |
476 | bpf_map_put(map); | |
477 | err = -EEXIST; | |
478 | goto err_free; | |
479 | } | |
480 | diag->maps[diag->nr_maps++] = map; | |
481 | } | |
482 | ||
483 | return diag; | |
484 | ||
485 | err_free: | |
486 | bpf_sk_storage_diag_free(diag); | |
487 | return ERR_PTR(err); | |
488 | } | |
489 | EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_alloc); | |
490 | ||
1f00d375 | 491 | static int diag_get(struct bpf_local_storage_data *sdata, struct sk_buff *skb) |
1ed4d924 MKL |
492 | { |
493 | struct nlattr *nla_stg, *nla_value; | |
1f00d375 | 494 | struct bpf_local_storage_map *smap; |
1ed4d924 MKL |
495 | |
496 | /* It cannot exceed max nlattr's payload */ | |
1f00d375 | 497 | BUILD_BUG_ON(U16_MAX - NLA_HDRLEN < BPF_LOCAL_STORAGE_MAX_VALUE_SIZE); |
1ed4d924 MKL |
498 | |
499 | nla_stg = nla_nest_start(skb, SK_DIAG_BPF_STORAGE); | |
500 | if (!nla_stg) | |
501 | return -EMSGSIZE; | |
502 | ||
503 | smap = rcu_dereference(sdata->smap); | |
504 | if (nla_put_u32(skb, SK_DIAG_BPF_STORAGE_MAP_ID, smap->map.id)) | |
505 | goto errout; | |
506 | ||
507 | nla_value = nla_reserve_64bit(skb, SK_DIAG_BPF_STORAGE_MAP_VALUE, | |
508 | smap->map.value_size, | |
509 | SK_DIAG_BPF_STORAGE_PAD); | |
510 | if (!nla_value) | |
511 | goto errout; | |
512 | ||
513 | if (map_value_has_spin_lock(&smap->map)) | |
514 | copy_map_value_locked(&smap->map, nla_data(nla_value), | |
515 | sdata->data, true); | |
516 | else | |
517 | copy_map_value(&smap->map, nla_data(nla_value), sdata->data); | |
518 | ||
519 | nla_nest_end(skb, nla_stg); | |
520 | return 0; | |
521 | ||
522 | errout: | |
523 | nla_nest_cancel(skb, nla_stg); | |
524 | return -EMSGSIZE; | |
525 | } | |
526 | ||
527 | static int bpf_sk_storage_diag_put_all(struct sock *sk, struct sk_buff *skb, | |
528 | int stg_array_type, | |
529 | unsigned int *res_diag_size) | |
530 | { | |
531 | /* stg_array_type (e.g. INET_DIAG_BPF_SK_STORAGES) */ | |
532 | unsigned int diag_size = nla_total_size(0); | |
1f00d375 KS |
533 | struct bpf_local_storage *sk_storage; |
534 | struct bpf_local_storage_elem *selem; | |
535 | struct bpf_local_storage_map *smap; | |
1ed4d924 MKL |
536 | struct nlattr *nla_stgs; |
537 | unsigned int saved_len; | |
538 | int err = 0; | |
539 | ||
540 | rcu_read_lock(); | |
541 | ||
542 | sk_storage = rcu_dereference(sk->sk_bpf_storage); | |
543 | if (!sk_storage || hlist_empty(&sk_storage->list)) { | |
544 | rcu_read_unlock(); | |
545 | return 0; | |
546 | } | |
547 | ||
548 | nla_stgs = nla_nest_start(skb, stg_array_type); | |
549 | if (!nla_stgs) | |
550 | /* Continue to learn diag_size */ | |
551 | err = -EMSGSIZE; | |
552 | ||
553 | saved_len = skb->len; | |
554 | hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) { | |
555 | smap = rcu_dereference(SDATA(selem)->smap); | |
556 | diag_size += nla_value_size(smap->map.value_size); | |
557 | ||
558 | if (nla_stgs && diag_get(SDATA(selem), skb)) | |
559 | /* Continue to learn diag_size */ | |
560 | err = -EMSGSIZE; | |
561 | } | |
562 | ||
563 | rcu_read_unlock(); | |
564 | ||
565 | if (nla_stgs) { | |
566 | if (saved_len == skb->len) | |
567 | nla_nest_cancel(skb, nla_stgs); | |
568 | else | |
569 | nla_nest_end(skb, nla_stgs); | |
570 | } | |
571 | ||
572 | if (diag_size == nla_total_size(0)) { | |
573 | *res_diag_size = 0; | |
574 | return 0; | |
575 | } | |
576 | ||
577 | *res_diag_size = diag_size; | |
578 | return err; | |
579 | } | |
580 | ||
581 | int bpf_sk_storage_diag_put(struct bpf_sk_storage_diag *diag, | |
582 | struct sock *sk, struct sk_buff *skb, | |
583 | int stg_array_type, | |
584 | unsigned int *res_diag_size) | |
585 | { | |
586 | /* stg_array_type (e.g. INET_DIAG_BPF_SK_STORAGES) */ | |
587 | unsigned int diag_size = nla_total_size(0); | |
1f00d375 KS |
588 | struct bpf_local_storage *sk_storage; |
589 | struct bpf_local_storage_data *sdata; | |
1ed4d924 MKL |
590 | struct nlattr *nla_stgs; |
591 | unsigned int saved_len; | |
592 | int err = 0; | |
593 | u32 i; | |
594 | ||
595 | *res_diag_size = 0; | |
596 | ||
597 | /* No map has been specified. Dump all. */ | |
598 | if (!diag->nr_maps) | |
599 | return bpf_sk_storage_diag_put_all(sk, skb, stg_array_type, | |
600 | res_diag_size); | |
601 | ||
602 | rcu_read_lock(); | |
603 | sk_storage = rcu_dereference(sk->sk_bpf_storage); | |
604 | if (!sk_storage || hlist_empty(&sk_storage->list)) { | |
605 | rcu_read_unlock(); | |
606 | return 0; | |
607 | } | |
608 | ||
609 | nla_stgs = nla_nest_start(skb, stg_array_type); | |
610 | if (!nla_stgs) | |
611 | /* Continue to learn diag_size */ | |
612 | err = -EMSGSIZE; | |
613 | ||
614 | saved_len = skb->len; | |
615 | for (i = 0; i < diag->nr_maps; i++) { | |
1f00d375 KS |
616 | sdata = bpf_local_storage_lookup(sk_storage, |
617 | (struct bpf_local_storage_map *)diag->maps[i], | |
1ed4d924 MKL |
618 | false); |
619 | ||
620 | if (!sdata) | |
621 | continue; | |
622 | ||
623 | diag_size += nla_value_size(diag->maps[i]->value_size); | |
624 | ||
625 | if (nla_stgs && diag_get(sdata, skb)) | |
626 | /* Continue to learn diag_size */ | |
627 | err = -EMSGSIZE; | |
628 | } | |
629 | rcu_read_unlock(); | |
630 | ||
631 | if (nla_stgs) { | |
632 | if (saved_len == skb->len) | |
633 | nla_nest_cancel(skb, nla_stgs); | |
634 | else | |
635 | nla_nest_end(skb, nla_stgs); | |
636 | } | |
637 | ||
638 | if (diag_size == nla_total_size(0)) { | |
639 | *res_diag_size = 0; | |
640 | return 0; | |
641 | } | |
642 | ||
643 | *res_diag_size = diag_size; | |
644 | return err; | |
645 | } | |
646 | EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_put); | |
5ce6e77c YS |
647 | |
648 | struct bpf_iter_seq_sk_storage_map_info { | |
649 | struct bpf_map *map; | |
650 | unsigned int bucket_id; | |
651 | unsigned skip_elems; | |
652 | }; | |
653 | ||
1f00d375 | 654 | static struct bpf_local_storage_elem * |
5ce6e77c | 655 | bpf_sk_storage_map_seq_find_next(struct bpf_iter_seq_sk_storage_map_info *info, |
1f00d375 | 656 | struct bpf_local_storage_elem *prev_selem) |
c69d2ddb | 657 | __acquires(RCU) __releases(RCU) |
5ce6e77c | 658 | { |
1f00d375 KS |
659 | struct bpf_local_storage *sk_storage; |
660 | struct bpf_local_storage_elem *selem; | |
5ce6e77c | 661 | u32 skip_elems = info->skip_elems; |
1f00d375 | 662 | struct bpf_local_storage_map *smap; |
5ce6e77c YS |
663 | u32 bucket_id = info->bucket_id; |
664 | u32 i, count, n_buckets; | |
1f00d375 | 665 | struct bpf_local_storage_map_bucket *b; |
5ce6e77c | 666 | |
1f00d375 | 667 | smap = (struct bpf_local_storage_map *)info->map; |
5ce6e77c YS |
668 | n_buckets = 1U << smap->bucket_log; |
669 | if (bucket_id >= n_buckets) | |
670 | return NULL; | |
671 | ||
672 | /* try to find next selem in the same bucket */ | |
673 | selem = prev_selem; | |
674 | count = 0; | |
675 | while (selem) { | |
c69d2ddb | 676 | selem = hlist_entry_safe(rcu_dereference(hlist_next_rcu(&selem->map_node)), |
1f00d375 | 677 | struct bpf_local_storage_elem, map_node); |
5ce6e77c YS |
678 | if (!selem) { |
679 | /* not found, unlock and go to the next bucket */ | |
680 | b = &smap->buckets[bucket_id++]; | |
c69d2ddb | 681 | rcu_read_unlock(); |
5ce6e77c YS |
682 | skip_elems = 0; |
683 | break; | |
684 | } | |
c69d2ddb | 685 | sk_storage = rcu_dereference(selem->local_storage); |
5ce6e77c YS |
686 | if (sk_storage) { |
687 | info->skip_elems = skip_elems + count; | |
688 | return selem; | |
689 | } | |
690 | count++; | |
691 | } | |
692 | ||
693 | for (i = bucket_id; i < (1U << smap->bucket_log); i++) { | |
694 | b = &smap->buckets[i]; | |
c69d2ddb | 695 | rcu_read_lock(); |
5ce6e77c | 696 | count = 0; |
c69d2ddb YS |
697 | hlist_for_each_entry_rcu(selem, &b->list, map_node) { |
698 | sk_storage = rcu_dereference(selem->local_storage); | |
5ce6e77c YS |
699 | if (sk_storage && count >= skip_elems) { |
700 | info->bucket_id = i; | |
701 | info->skip_elems = count; | |
702 | return selem; | |
703 | } | |
704 | count++; | |
705 | } | |
c69d2ddb | 706 | rcu_read_unlock(); |
5ce6e77c YS |
707 | skip_elems = 0; |
708 | } | |
709 | ||
710 | info->bucket_id = i; | |
711 | info->skip_elems = 0; | |
712 | return NULL; | |
713 | } | |
714 | ||
715 | static void *bpf_sk_storage_map_seq_start(struct seq_file *seq, loff_t *pos) | |
716 | { | |
1f00d375 | 717 | struct bpf_local_storage_elem *selem; |
5ce6e77c YS |
718 | |
719 | selem = bpf_sk_storage_map_seq_find_next(seq->private, NULL); | |
720 | if (!selem) | |
721 | return NULL; | |
722 | ||
723 | if (*pos == 0) | |
724 | ++*pos; | |
725 | return selem; | |
726 | } | |
727 | ||
728 | static void *bpf_sk_storage_map_seq_next(struct seq_file *seq, void *v, | |
729 | loff_t *pos) | |
730 | { | |
731 | struct bpf_iter_seq_sk_storage_map_info *info = seq->private; | |
732 | ||
733 | ++*pos; | |
734 | ++info->skip_elems; | |
735 | return bpf_sk_storage_map_seq_find_next(seq->private, v); | |
736 | } | |
737 | ||
738 | struct bpf_iter__bpf_sk_storage_map { | |
739 | __bpf_md_ptr(struct bpf_iter_meta *, meta); | |
740 | __bpf_md_ptr(struct bpf_map *, map); | |
741 | __bpf_md_ptr(struct sock *, sk); | |
742 | __bpf_md_ptr(void *, value); | |
743 | }; | |
744 | ||
745 | DEFINE_BPF_ITER_FUNC(bpf_sk_storage_map, struct bpf_iter_meta *meta, | |
746 | struct bpf_map *map, struct sock *sk, | |
747 | void *value) | |
748 | ||
749 | static int __bpf_sk_storage_map_seq_show(struct seq_file *seq, | |
1f00d375 | 750 | struct bpf_local_storage_elem *selem) |
5ce6e77c YS |
751 | { |
752 | struct bpf_iter_seq_sk_storage_map_info *info = seq->private; | |
753 | struct bpf_iter__bpf_sk_storage_map ctx = {}; | |
1f00d375 | 754 | struct bpf_local_storage *sk_storage; |
5ce6e77c YS |
755 | struct bpf_iter_meta meta; |
756 | struct bpf_prog *prog; | |
757 | int ret = 0; | |
758 | ||
759 | meta.seq = seq; | |
760 | prog = bpf_iter_get_info(&meta, selem == NULL); | |
761 | if (prog) { | |
762 | ctx.meta = &meta; | |
763 | ctx.map = info->map; | |
764 | if (selem) { | |
c69d2ddb | 765 | sk_storage = rcu_dereference(selem->local_storage); |
1f00d375 | 766 | ctx.sk = sk_storage->owner; |
5ce6e77c YS |
767 | ctx.value = SDATA(selem)->data; |
768 | } | |
769 | ret = bpf_iter_run_prog(prog, &ctx); | |
770 | } | |
771 | ||
772 | return ret; | |
773 | } | |
774 | ||
775 | static int bpf_sk_storage_map_seq_show(struct seq_file *seq, void *v) | |
776 | { | |
777 | return __bpf_sk_storage_map_seq_show(seq, v); | |
778 | } | |
779 | ||
780 | static void bpf_sk_storage_map_seq_stop(struct seq_file *seq, void *v) | |
c69d2ddb | 781 | __releases(RCU) |
5ce6e77c | 782 | { |
c69d2ddb | 783 | if (!v) |
5ce6e77c | 784 | (void)__bpf_sk_storage_map_seq_show(seq, v); |
c69d2ddb YS |
785 | else |
786 | rcu_read_unlock(); | |
5ce6e77c YS |
787 | } |
788 | ||
789 | static int bpf_iter_init_sk_storage_map(void *priv_data, | |
790 | struct bpf_iter_aux_info *aux) | |
791 | { | |
792 | struct bpf_iter_seq_sk_storage_map_info *seq_info = priv_data; | |
793 | ||
794 | seq_info->map = aux->map; | |
795 | return 0; | |
796 | } | |
797 | ||
5e7b3020 YS |
798 | static int bpf_iter_attach_map(struct bpf_prog *prog, |
799 | union bpf_iter_link_info *linfo, | |
800 | struct bpf_iter_aux_info *aux) | |
5ce6e77c | 801 | { |
5e7b3020 YS |
802 | struct bpf_map *map; |
803 | int err = -EINVAL; | |
804 | ||
805 | if (!linfo->map.map_fd) | |
806 | return -EBADF; | |
807 | ||
808 | map = bpf_map_get_with_uref(linfo->map.map_fd); | |
809 | if (IS_ERR(map)) | |
810 | return PTR_ERR(map); | |
5ce6e77c YS |
811 | |
812 | if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) | |
5e7b3020 | 813 | goto put_map; |
5ce6e77c | 814 | |
5e7b3020 YS |
815 | if (prog->aux->max_rdonly_access > map->value_size) { |
816 | err = -EACCES; | |
817 | goto put_map; | |
818 | } | |
5ce6e77c | 819 | |
5e7b3020 | 820 | aux->map = map; |
5ce6e77c | 821 | return 0; |
5e7b3020 YS |
822 | |
823 | put_map: | |
824 | bpf_map_put_with_uref(map); | |
825 | return err; | |
826 | } | |
827 | ||
828 | static void bpf_iter_detach_map(struct bpf_iter_aux_info *aux) | |
829 | { | |
830 | bpf_map_put_with_uref(aux->map); | |
5ce6e77c YS |
831 | } |
832 | ||
833 | static const struct seq_operations bpf_sk_storage_map_seq_ops = { | |
834 | .start = bpf_sk_storage_map_seq_start, | |
835 | .next = bpf_sk_storage_map_seq_next, | |
836 | .stop = bpf_sk_storage_map_seq_stop, | |
837 | .show = bpf_sk_storage_map_seq_show, | |
838 | }; | |
839 | ||
840 | static const struct bpf_iter_seq_info iter_seq_info = { | |
841 | .seq_ops = &bpf_sk_storage_map_seq_ops, | |
842 | .init_seq_private = bpf_iter_init_sk_storage_map, | |
843 | .fini_seq_private = NULL, | |
844 | .seq_priv_size = sizeof(struct bpf_iter_seq_sk_storage_map_info), | |
845 | }; | |
846 | ||
847 | static struct bpf_iter_reg bpf_sk_storage_map_reg_info = { | |
848 | .target = "bpf_sk_storage_map", | |
5e7b3020 YS |
849 | .attach_target = bpf_iter_attach_map, |
850 | .detach_target = bpf_iter_detach_map, | |
b76f2226 YS |
851 | .show_fdinfo = bpf_iter_map_show_fdinfo, |
852 | .fill_link_info = bpf_iter_map_fill_link_info, | |
5ce6e77c YS |
853 | .ctx_arg_info_size = 2, |
854 | .ctx_arg_info = { | |
855 | { offsetof(struct bpf_iter__bpf_sk_storage_map, sk), | |
856 | PTR_TO_BTF_ID_OR_NULL }, | |
857 | { offsetof(struct bpf_iter__bpf_sk_storage_map, value), | |
858 | PTR_TO_RDWR_BUF_OR_NULL }, | |
859 | }, | |
860 | .seq_info = &iter_seq_info, | |
861 | }; | |
862 | ||
863 | static int __init bpf_sk_storage_map_iter_init(void) | |
864 | { | |
865 | bpf_sk_storage_map_reg_info.ctx_arg_info[0].btf_id = | |
866 | btf_sock_ids[BTF_SOCK_TYPE_SOCK]; | |
867 | return bpf_iter_reg_target(&bpf_sk_storage_map_reg_info); | |
868 | } | |
869 | late_initcall(bpf_sk_storage_map_iter_init); |