Commit | Line | Data |
---|---|---|
8ea63684 KS |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * Copyright (c) 2019 Facebook | |
4 | * Copyright 2020 Google LLC. | |
5 | */ | |
6 | ||
7 | #include <linux/rculist.h> | |
8 | #include <linux/list.h> | |
9 | #include <linux/hash.h> | |
10 | #include <linux/types.h> | |
11 | #include <linux/spinlock.h> | |
12 | #include <linux/bpf.h> | |
13 | #include <linux/bpf_local_storage.h> | |
14 | #include <net/sock.h> | |
15 | #include <uapi/linux/sock_diag.h> | |
16 | #include <uapi/linux/btf.h> | |
17 | #include <linux/bpf_lsm.h> | |
18 | #include <linux/btf_ids.h> | |
19 | #include <linux/fdtable.h> | |
0fe4b381 | 20 | #include <linux/rcupdate_trace.h> |
8ea63684 KS |
21 | |
22 | DEFINE_BPF_STORAGE_CACHE(inode_cache); | |
23 | ||
24 | static struct bpf_local_storage __rcu ** | |
25 | inode_storage_ptr(void *owner) | |
26 | { | |
27 | struct inode *inode = owner; | |
28 | struct bpf_storage_blob *bsb; | |
29 | ||
30 | bsb = bpf_inode(inode); | |
31 | if (!bsb) | |
32 | return NULL; | |
33 | return &bsb->storage; | |
34 | } | |
35 | ||
36 | static struct bpf_local_storage_data *inode_storage_lookup(struct inode *inode, | |
37 | struct bpf_map *map, | |
38 | bool cacheit_lockit) | |
39 | { | |
40 | struct bpf_local_storage *inode_storage; | |
41 | struct bpf_local_storage_map *smap; | |
42 | struct bpf_storage_blob *bsb; | |
43 | ||
44 | bsb = bpf_inode(inode); | |
45 | if (!bsb) | |
46 | return NULL; | |
47 | ||
0fe4b381 KS |
48 | inode_storage = |
49 | rcu_dereference_check(bsb->storage, bpf_rcu_lock_held()); | |
8ea63684 KS |
50 | if (!inode_storage) |
51 | return NULL; | |
52 | ||
53 | smap = (struct bpf_local_storage_map *)map; | |
54 | return bpf_local_storage_lookup(inode_storage, smap, cacheit_lockit); | |
55 | } | |
56 | ||
57 | void bpf_inode_storage_free(struct inode *inode) | |
58 | { | |
8ea63684 | 59 | struct bpf_local_storage *local_storage; |
8ea63684 | 60 | struct bpf_storage_blob *bsb; |
8ea63684 KS |
61 | |
62 | bsb = bpf_inode(inode); | |
63 | if (!bsb) | |
64 | return; | |
65 | ||
66 | rcu_read_lock(); | |
67 | ||
68 | local_storage = rcu_dereference(bsb->storage); | |
69 | if (!local_storage) { | |
70 | rcu_read_unlock(); | |
71 | return; | |
72 | } | |
73 | ||
2ffcb6fc | 74 | bpf_local_storage_destroy(local_storage); |
8ea63684 | 75 | rcu_read_unlock(); |
8ea63684 KS |
76 | } |
77 | ||
78 | static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key) | |
79 | { | |
80 | struct bpf_local_storage_data *sdata; | |
1d0027dc | 81 | struct fd f = fdget_raw(*(int *)key); |
8ea63684 | 82 | |
1d0027dc | 83 | if (!f.file) |
769c18b2 | 84 | return ERR_PTR(-EBADF); |
8ea63684 | 85 | |
1d0027dc AV |
86 | sdata = inode_storage_lookup(file_inode(f.file), map, true); |
87 | fdput(f); | |
8ea63684 KS |
88 | return sdata ? sdata->data : NULL; |
89 | } | |
90 | ||
d7ba4cc9 JK |
91 | static long bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key, |
92 | void *value, u64 map_flags) | |
8ea63684 KS |
93 | { |
94 | struct bpf_local_storage_data *sdata; | |
1d0027dc | 95 | struct fd f = fdget_raw(*(int *)key); |
8ea63684 | 96 | |
1d0027dc | 97 | if (!f.file) |
b9557caa | 98 | return -EBADF; |
1d0027dc AV |
99 | if (!inode_storage_ptr(file_inode(f.file))) { |
100 | fdput(f); | |
8ea63684 | 101 | return -EBADF; |
b9557caa | 102 | } |
8ea63684 | 103 | |
1d0027dc | 104 | sdata = bpf_local_storage_update(file_inode(f.file), |
8ea63684 | 105 | (struct bpf_local_storage_map *)map, |
b00fa38a | 106 | value, map_flags, GFP_ATOMIC); |
1d0027dc | 107 | fdput(f); |
8ea63684 KS |
108 | return PTR_ERR_OR_ZERO(sdata); |
109 | } | |
110 | ||
111 | static int inode_storage_delete(struct inode *inode, struct bpf_map *map) | |
112 | { | |
113 | struct bpf_local_storage_data *sdata; | |
114 | ||
115 | sdata = inode_storage_lookup(inode, map, false); | |
116 | if (!sdata) | |
117 | return -ENOENT; | |
118 | ||
a47eabf2 | 119 | bpf_selem_unlink(SELEM(sdata), false); |
8ea63684 KS |
120 | |
121 | return 0; | |
122 | } | |
123 | ||
d7ba4cc9 | 124 | static long bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key) |
8ea63684 | 125 | { |
1d0027dc AV |
126 | struct fd f = fdget_raw(*(int *)key); |
127 | int err; | |
8ea63684 | 128 | |
1d0027dc | 129 | if (!f.file) |
8ea63684 KS |
130 | return -EBADF; |
131 | ||
1d0027dc AV |
132 | err = inode_storage_delete(file_inode(f.file), map); |
133 | fdput(f); | |
8ea63684 KS |
134 | return err; |
135 | } | |
136 | ||
b00fa38a JK |
137 | /* *gfp_flags* is a hidden argument provided by the verifier */ |
138 | BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode, | |
139 | void *, value, u64, flags, gfp_t, gfp_flags) | |
8ea63684 KS |
140 | { |
141 | struct bpf_local_storage_data *sdata; | |
142 | ||
0fe4b381 | 143 | WARN_ON_ONCE(!bpf_rcu_lock_held()); |
8ea63684 KS |
144 | if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE)) |
145 | return (unsigned long)NULL; | |
146 | ||
147 | /* explicitly check that the inode_storage_ptr is not | |
148 | * NULL as inode_storage_lookup returns NULL in this case and | |
149 | * bpf_local_storage_update expects the owner to have a | |
150 | * valid storage pointer. | |
151 | */ | |
1a9c72ad | 152 | if (!inode || !inode_storage_ptr(inode)) |
8ea63684 KS |
153 | return (unsigned long)NULL; |
154 | ||
155 | sdata = inode_storage_lookup(inode, map, true); | |
156 | if (sdata) | |
157 | return (unsigned long)sdata->data; | |
158 | ||
84d571d4 | 159 | /* This helper must only called from where the inode is guaranteed |
8ea63684 KS |
160 | * to have a refcount and cannot be freed. |
161 | */ | |
162 | if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) { | |
163 | sdata = bpf_local_storage_update( | |
164 | inode, (struct bpf_local_storage_map *)map, value, | |
b00fa38a | 165 | BPF_NOEXIST, gfp_flags); |
8ea63684 KS |
166 | return IS_ERR(sdata) ? (unsigned long)NULL : |
167 | (unsigned long)sdata->data; | |
168 | } | |
169 | ||
170 | return (unsigned long)NULL; | |
171 | } | |
172 | ||
173 | BPF_CALL_2(bpf_inode_storage_delete, | |
174 | struct bpf_map *, map, struct inode *, inode) | |
175 | { | |
0fe4b381 | 176 | WARN_ON_ONCE(!bpf_rcu_lock_held()); |
1a9c72ad KS |
177 | if (!inode) |
178 | return -EINVAL; | |
179 | ||
84d571d4 | 180 | /* This helper must only called from where the inode is guaranteed |
8ea63684 KS |
181 | * to have a refcount and cannot be freed. |
182 | */ | |
183 | return inode_storage_delete(inode, map); | |
184 | } | |
185 | ||
186 | static int notsupp_get_next_key(struct bpf_map *map, void *key, | |
187 | void *next_key) | |
188 | { | |
189 | return -ENOTSUPP; | |
190 | } | |
191 | ||
192 | static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr) | |
193 | { | |
08a7ce38 | 194 | return bpf_local_storage_map_alloc(attr, &inode_cache, false); |
8ea63684 KS |
195 | } |
196 | ||
197 | static void inode_storage_map_free(struct bpf_map *map) | |
198 | { | |
c83597fa | 199 | bpf_local_storage_map_free(map, &inode_cache, NULL); |
8ea63684 KS |
200 | } |
201 | ||
8ea63684 | 202 | const struct bpf_map_ops inode_storage_map_ops = { |
f4d05259 | 203 | .map_meta_equal = bpf_map_meta_equal, |
8ea63684 KS |
204 | .map_alloc_check = bpf_local_storage_map_alloc_check, |
205 | .map_alloc = inode_storage_map_alloc, | |
206 | .map_free = inode_storage_map_free, | |
207 | .map_get_next_key = notsupp_get_next_key, | |
208 | .map_lookup_elem = bpf_fd_inode_storage_lookup_elem, | |
209 | .map_update_elem = bpf_fd_inode_storage_update_elem, | |
210 | .map_delete_elem = bpf_fd_inode_storage_delete_elem, | |
211 | .map_check_btf = bpf_local_storage_map_check_btf, | |
7490b7f1 | 212 | .map_mem_usage = bpf_local_storage_map_mem_usage, |
3144bfa5 | 213 | .map_btf_id = &bpf_local_storage_map_btf_id[0], |
8ea63684 KS |
214 | .map_owner_storage_ptr = inode_storage_ptr, |
215 | }; | |
216 | ||
9436ef6e | 217 | BTF_ID_LIST_SINGLE(bpf_inode_storage_btf_ids, struct, inode) |
8ea63684 KS |
218 | |
219 | const struct bpf_func_proto bpf_inode_storage_get_proto = { | |
220 | .func = bpf_inode_storage_get, | |
221 | .gpl_only = false, | |
222 | .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, | |
223 | .arg1_type = ARG_CONST_MAP_PTR, | |
91571a51 | 224 | .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL, |
9436ef6e | 225 | .arg2_btf_id = &bpf_inode_storage_btf_ids[0], |
8ea63684 KS |
226 | .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, |
227 | .arg4_type = ARG_ANYTHING, | |
8ea63684 KS |
228 | }; |
229 | ||
230 | const struct bpf_func_proto bpf_inode_storage_delete_proto = { | |
231 | .func = bpf_inode_storage_delete, | |
232 | .gpl_only = false, | |
233 | .ret_type = RET_INTEGER, | |
234 | .arg1_type = ARG_CONST_MAP_PTR, | |
91571a51 | 235 | .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL, |
9436ef6e | 236 | .arg2_btf_id = &bpf_inode_storage_btf_ids[0], |
8ea63684 | 237 | }; |