Commit | Line | Data |
---|---|---|
053c8e1f DB |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright (c) 2023 Isovalent */ | |
3 | ||
4 | #include <linux/bpf.h> | |
5 | #include <linux/bpf_mprog.h> | |
6 | ||
7 | static int bpf_mprog_link(struct bpf_tuple *tuple, | |
8 | u32 id_or_fd, u32 flags, | |
9 | enum bpf_prog_type type) | |
10 | { | |
11 | struct bpf_link *link = ERR_PTR(-EINVAL); | |
12 | bool id = flags & BPF_F_ID; | |
13 | ||
14 | if (id) | |
15 | link = bpf_link_by_id(id_or_fd); | |
16 | else if (id_or_fd) | |
17 | link = bpf_link_get_from_fd(id_or_fd); | |
18 | if (IS_ERR(link)) | |
19 | return PTR_ERR(link); | |
20 | if (type && link->prog->type != type) { | |
21 | bpf_link_put(link); | |
22 | return -EINVAL; | |
23 | } | |
24 | ||
25 | tuple->link = link; | |
26 | tuple->prog = link->prog; | |
27 | return 0; | |
28 | } | |
29 | ||
30 | static int bpf_mprog_prog(struct bpf_tuple *tuple, | |
31 | u32 id_or_fd, u32 flags, | |
32 | enum bpf_prog_type type) | |
33 | { | |
34 | struct bpf_prog *prog = ERR_PTR(-EINVAL); | |
35 | bool id = flags & BPF_F_ID; | |
36 | ||
37 | if (id) | |
38 | prog = bpf_prog_by_id(id_or_fd); | |
39 | else if (id_or_fd) | |
40 | prog = bpf_prog_get(id_or_fd); | |
41 | if (IS_ERR(prog)) | |
42 | return PTR_ERR(prog); | |
43 | if (type && prog->type != type) { | |
44 | bpf_prog_put(prog); | |
45 | return -EINVAL; | |
46 | } | |
47 | ||
48 | tuple->link = NULL; | |
49 | tuple->prog = prog; | |
50 | return 0; | |
51 | } | |
52 | ||
53 | static int bpf_mprog_tuple_relative(struct bpf_tuple *tuple, | |
54 | u32 id_or_fd, u32 flags, | |
55 | enum bpf_prog_type type) | |
56 | { | |
57 | bool link = flags & BPF_F_LINK; | |
58 | bool id = flags & BPF_F_ID; | |
59 | ||
60 | memset(tuple, 0, sizeof(*tuple)); | |
61 | if (link) | |
62 | return bpf_mprog_link(tuple, id_or_fd, flags, type); | |
63 | /* If no relevant flag is set and no id_or_fd was passed, then | |
64 | * tuple link/prog is just NULLed. This is the case when before/ | |
65 | * after selects first/last position without passing fd. | |
66 | */ | |
67 | if (!id && !id_or_fd) | |
68 | return 0; | |
69 | return bpf_mprog_prog(tuple, id_or_fd, flags, type); | |
70 | } | |
71 | ||
72 | static void bpf_mprog_tuple_put(struct bpf_tuple *tuple) | |
73 | { | |
74 | if (tuple->link) | |
75 | bpf_link_put(tuple->link); | |
76 | else if (tuple->prog) | |
77 | bpf_prog_put(tuple->prog); | |
78 | } | |
79 | ||
80 | /* The bpf_mprog_{replace,delete}() operate on exact idx position with the | |
81 | * one exception that for deletion we support delete from front/back. In | |
82 | * case of front idx is -1, in case of back idx is bpf_mprog_total(entry). | |
83 | * Adjustment to first and last entry is trivial. The bpf_mprog_insert() | |
84 | * we have to deal with the following cases: | |
85 | * | |
86 | * idx + before: | |
87 | * | |
88 | * Insert P4 before P3: idx for old array is 1, idx for new array is 2, | |
89 | * hence we adjust target idx for the new array, so that memmove copies | |
90 | * P1 and P2 to the new entry, and we insert P4 into idx 2. Inserting | |
91 | * before P1 would have old idx -1 and new idx 0. | |
92 | * | |
93 | * +--+--+--+ +--+--+--+--+ +--+--+--+--+ | |
94 | * |P1|P2|P3| ==> |P1|P2| |P3| ==> |P1|P2|P4|P3| | |
95 | * +--+--+--+ +--+--+--+--+ +--+--+--+--+ | |
96 | * | |
97 | * idx + after: | |
98 | * | |
99 | * Insert P4 after P2: idx for old array is 2, idx for new array is 2. | |
100 | * Again, memmove copies P1 and P2 to the new entry, and we insert P4 | |
101 | * into idx 2. Inserting after P3 would have both old/new idx at 4 aka | |
102 | * bpf_mprog_total(entry). | |
103 | * | |
104 | * +--+--+--+ +--+--+--+--+ +--+--+--+--+ | |
105 | * |P1|P2|P3| ==> |P1|P2| |P3| ==> |P1|P2|P4|P3| | |
106 | * +--+--+--+ +--+--+--+--+ +--+--+--+--+ | |
107 | */ | |
108 | static int bpf_mprog_replace(struct bpf_mprog_entry *entry, | |
109 | struct bpf_mprog_entry **entry_new, | |
110 | struct bpf_tuple *ntuple, int idx) | |
111 | { | |
112 | struct bpf_mprog_fp *fp; | |
113 | struct bpf_mprog_cp *cp; | |
114 | struct bpf_prog *oprog; | |
115 | ||
116 | bpf_mprog_read(entry, idx, &fp, &cp); | |
117 | oprog = READ_ONCE(fp->prog); | |
118 | bpf_mprog_write(fp, cp, ntuple); | |
119 | if (!ntuple->link) { | |
120 | WARN_ON_ONCE(cp->link); | |
121 | bpf_prog_put(oprog); | |
122 | } | |
123 | *entry_new = entry; | |
124 | return 0; | |
125 | } | |
126 | ||
127 | static int bpf_mprog_insert(struct bpf_mprog_entry *entry, | |
128 | struct bpf_mprog_entry **entry_new, | |
129 | struct bpf_tuple *ntuple, int idx, u32 flags) | |
130 | { | |
131 | int total = bpf_mprog_total(entry); | |
132 | struct bpf_mprog_entry *peer; | |
133 | struct bpf_mprog_fp *fp; | |
134 | struct bpf_mprog_cp *cp; | |
135 | ||
136 | peer = bpf_mprog_peer(entry); | |
137 | bpf_mprog_entry_copy(peer, entry); | |
138 | if (idx == total) | |
139 | goto insert; | |
140 | else if (flags & BPF_F_BEFORE) | |
141 | idx += 1; | |
142 | bpf_mprog_entry_grow(peer, idx); | |
143 | insert: | |
144 | bpf_mprog_read(peer, idx, &fp, &cp); | |
145 | bpf_mprog_write(fp, cp, ntuple); | |
146 | bpf_mprog_inc(peer); | |
147 | *entry_new = peer; | |
148 | return 0; | |
149 | } | |
150 | ||
151 | static int bpf_mprog_delete(struct bpf_mprog_entry *entry, | |
152 | struct bpf_mprog_entry **entry_new, | |
153 | struct bpf_tuple *dtuple, int idx) | |
154 | { | |
155 | int total = bpf_mprog_total(entry); | |
156 | struct bpf_mprog_entry *peer; | |
157 | ||
158 | peer = bpf_mprog_peer(entry); | |
159 | bpf_mprog_entry_copy(peer, entry); | |
160 | if (idx == -1) | |
161 | idx = 0; | |
162 | else if (idx == total) | |
163 | idx = total - 1; | |
164 | bpf_mprog_entry_shrink(peer, idx); | |
165 | bpf_mprog_dec(peer); | |
166 | bpf_mprog_mark_for_release(peer, dtuple); | |
167 | *entry_new = peer; | |
168 | return 0; | |
169 | } | |
170 | ||
171 | /* In bpf_mprog_pos_*() we evaluate the target position for the BPF | |
172 | * program/link that needs to be replaced, inserted or deleted for | |
173 | * each "rule" independently. If all rules agree on that position | |
174 | * or existing element, then enact replacement, addition or deletion. | |
175 | * If this is not the case, then the request cannot be satisfied and | |
176 | * we bail out with an error. | |
177 | */ | |
178 | static int bpf_mprog_pos_exact(struct bpf_mprog_entry *entry, | |
179 | struct bpf_tuple *tuple) | |
180 | { | |
181 | struct bpf_mprog_fp *fp; | |
182 | struct bpf_mprog_cp *cp; | |
183 | int i; | |
184 | ||
185 | for (i = 0; i < bpf_mprog_total(entry); i++) { | |
186 | bpf_mprog_read(entry, i, &fp, &cp); | |
187 | if (tuple->prog == READ_ONCE(fp->prog)) | |
188 | return tuple->link == cp->link ? i : -EBUSY; | |
189 | } | |
190 | return -ENOENT; | |
191 | } | |
192 | ||
193 | static int bpf_mprog_pos_before(struct bpf_mprog_entry *entry, | |
194 | struct bpf_tuple *tuple) | |
195 | { | |
196 | struct bpf_mprog_fp *fp; | |
197 | struct bpf_mprog_cp *cp; | |
198 | int i; | |
199 | ||
200 | for (i = 0; i < bpf_mprog_total(entry); i++) { | |
201 | bpf_mprog_read(entry, i, &fp, &cp); | |
202 | if (tuple->prog == READ_ONCE(fp->prog) && | |
203 | (!tuple->link || tuple->link == cp->link)) | |
204 | return i - 1; | |
205 | } | |
206 | return tuple->prog ? -ENOENT : -1; | |
207 | } | |
208 | ||
209 | static int bpf_mprog_pos_after(struct bpf_mprog_entry *entry, | |
210 | struct bpf_tuple *tuple) | |
211 | { | |
212 | struct bpf_mprog_fp *fp; | |
213 | struct bpf_mprog_cp *cp; | |
214 | int i; | |
215 | ||
216 | for (i = 0; i < bpf_mprog_total(entry); i++) { | |
217 | bpf_mprog_read(entry, i, &fp, &cp); | |
218 | if (tuple->prog == READ_ONCE(fp->prog) && | |
219 | (!tuple->link || tuple->link == cp->link)) | |
220 | return i + 1; | |
221 | } | |
222 | return tuple->prog ? -ENOENT : bpf_mprog_total(entry); | |
223 | } | |
224 | ||
225 | int bpf_mprog_attach(struct bpf_mprog_entry *entry, | |
226 | struct bpf_mprog_entry **entry_new, | |
227 | struct bpf_prog *prog_new, struct bpf_link *link, | |
228 | struct bpf_prog *prog_old, | |
229 | u32 flags, u32 id_or_fd, u64 revision) | |
230 | { | |
231 | struct bpf_tuple rtuple, ntuple = { | |
232 | .prog = prog_new, | |
233 | .link = link, | |
234 | }, otuple = { | |
235 | .prog = prog_old, | |
236 | .link = link, | |
237 | }; | |
238 | int ret, idx = -ERANGE, tidx; | |
239 | ||
240 | if (revision && revision != bpf_mprog_revision(entry)) | |
241 | return -ESTALE; | |
242 | if (bpf_mprog_exists(entry, prog_new)) | |
243 | return -EEXIST; | |
244 | ret = bpf_mprog_tuple_relative(&rtuple, id_or_fd, | |
245 | flags & ~BPF_F_REPLACE, | |
246 | prog_new->type); | |
247 | if (ret) | |
248 | return ret; | |
249 | if (flags & BPF_F_REPLACE) { | |
250 | tidx = bpf_mprog_pos_exact(entry, &otuple); | |
251 | if (tidx < 0) { | |
252 | ret = tidx; | |
253 | goto out; | |
254 | } | |
255 | idx = tidx; | |
f9b0e108 DB |
256 | } else if (bpf_mprog_total(entry) == bpf_mprog_max()) { |
257 | ret = -ERANGE; | |
258 | goto out; | |
053c8e1f DB |
259 | } |
260 | if (flags & BPF_F_BEFORE) { | |
261 | tidx = bpf_mprog_pos_before(entry, &rtuple); | |
262 | if (tidx < -1 || (idx >= -1 && tidx != idx)) { | |
263 | ret = tidx < -1 ? tidx : -ERANGE; | |
264 | goto out; | |
265 | } | |
266 | idx = tidx; | |
267 | } | |
268 | if (flags & BPF_F_AFTER) { | |
269 | tidx = bpf_mprog_pos_after(entry, &rtuple); | |
270 | if (tidx < -1 || (idx >= -1 && tidx != idx)) { | |
271 | ret = tidx < 0 ? tidx : -ERANGE; | |
272 | goto out; | |
273 | } | |
274 | idx = tidx; | |
275 | } | |
276 | if (idx < -1) { | |
277 | if (rtuple.prog || flags) { | |
278 | ret = -EINVAL; | |
279 | goto out; | |
280 | } | |
281 | idx = bpf_mprog_total(entry); | |
282 | flags = BPF_F_AFTER; | |
283 | } | |
284 | if (idx >= bpf_mprog_max()) { | |
285 | ret = -ERANGE; | |
286 | goto out; | |
287 | } | |
288 | if (flags & BPF_F_REPLACE) | |
289 | ret = bpf_mprog_replace(entry, entry_new, &ntuple, idx); | |
290 | else | |
291 | ret = bpf_mprog_insert(entry, entry_new, &ntuple, idx, flags); | |
292 | out: | |
293 | bpf_mprog_tuple_put(&rtuple); | |
294 | return ret; | |
295 | } | |
296 | ||
297 | static int bpf_mprog_fetch(struct bpf_mprog_entry *entry, | |
298 | struct bpf_tuple *tuple, int idx) | |
299 | { | |
300 | int total = bpf_mprog_total(entry); | |
301 | struct bpf_mprog_cp *cp; | |
302 | struct bpf_mprog_fp *fp; | |
303 | struct bpf_prog *prog; | |
304 | struct bpf_link *link; | |
305 | ||
306 | if (idx == -1) | |
307 | idx = 0; | |
308 | else if (idx == total) | |
309 | idx = total - 1; | |
310 | bpf_mprog_read(entry, idx, &fp, &cp); | |
311 | prog = READ_ONCE(fp->prog); | |
312 | link = cp->link; | |
313 | /* The deletion request can either be without filled tuple in which | |
314 | * case it gets populated here based on idx, or with filled tuple | |
315 | * where the only thing we end up doing is the WARN_ON_ONCE() assert. | |
316 | * If we hit a BPF link at the given index, it must not be removed | |
317 | * from opts path. | |
318 | */ | |
319 | if (link && !tuple->link) | |
320 | return -EBUSY; | |
321 | WARN_ON_ONCE(tuple->prog && tuple->prog != prog); | |
322 | WARN_ON_ONCE(tuple->link && tuple->link != link); | |
323 | tuple->prog = prog; | |
324 | tuple->link = link; | |
325 | return 0; | |
326 | } | |
327 | ||
328 | int bpf_mprog_detach(struct bpf_mprog_entry *entry, | |
329 | struct bpf_mprog_entry **entry_new, | |
330 | struct bpf_prog *prog, struct bpf_link *link, | |
331 | u32 flags, u32 id_or_fd, u64 revision) | |
332 | { | |
333 | struct bpf_tuple rtuple, dtuple = { | |
334 | .prog = prog, | |
335 | .link = link, | |
336 | }; | |
337 | int ret, idx = -ERANGE, tidx; | |
338 | ||
339 | if (flags & BPF_F_REPLACE) | |
340 | return -EINVAL; | |
341 | if (revision && revision != bpf_mprog_revision(entry)) | |
342 | return -ESTALE; | |
d210f973 DB |
343 | if (!bpf_mprog_total(entry)) |
344 | return -ENOENT; | |
053c8e1f DB |
345 | ret = bpf_mprog_tuple_relative(&rtuple, id_or_fd, flags, |
346 | prog ? prog->type : | |
347 | BPF_PROG_TYPE_UNSPEC); | |
348 | if (ret) | |
349 | return ret; | |
350 | if (dtuple.prog) { | |
351 | tidx = bpf_mprog_pos_exact(entry, &dtuple); | |
352 | if (tidx < 0) { | |
353 | ret = tidx; | |
354 | goto out; | |
355 | } | |
356 | idx = tidx; | |
357 | } | |
358 | if (flags & BPF_F_BEFORE) { | |
359 | tidx = bpf_mprog_pos_before(entry, &rtuple); | |
360 | if (tidx < -1 || (idx >= -1 && tidx != idx)) { | |
361 | ret = tidx < -1 ? tidx : -ERANGE; | |
362 | goto out; | |
363 | } | |
364 | idx = tidx; | |
365 | } | |
366 | if (flags & BPF_F_AFTER) { | |
367 | tidx = bpf_mprog_pos_after(entry, &rtuple); | |
368 | if (tidx < -1 || (idx >= -1 && tidx != idx)) { | |
369 | ret = tidx < 0 ? tidx : -ERANGE; | |
370 | goto out; | |
371 | } | |
372 | idx = tidx; | |
373 | } | |
374 | if (idx < -1) { | |
375 | if (rtuple.prog || flags) { | |
376 | ret = -EINVAL; | |
377 | goto out; | |
378 | } | |
379 | idx = bpf_mprog_total(entry); | |
380 | flags = BPF_F_AFTER; | |
381 | } | |
382 | if (idx >= bpf_mprog_max()) { | |
383 | ret = -ERANGE; | |
384 | goto out; | |
385 | } | |
386 | ret = bpf_mprog_fetch(entry, &dtuple, idx); | |
387 | if (ret) | |
388 | goto out; | |
389 | ret = bpf_mprog_delete(entry, entry_new, &dtuple, idx); | |
390 | out: | |
391 | bpf_mprog_tuple_put(&rtuple); | |
392 | return ret; | |
393 | } | |
394 | ||
395 | int bpf_mprog_query(const union bpf_attr *attr, union bpf_attr __user *uattr, | |
396 | struct bpf_mprog_entry *entry) | |
397 | { | |
398 | u32 __user *uprog_flags, *ulink_flags; | |
399 | u32 __user *uprog_id, *ulink_id; | |
400 | struct bpf_mprog_fp *fp; | |
401 | struct bpf_mprog_cp *cp; | |
402 | struct bpf_prog *prog; | |
403 | const u32 flags = 0; | |
edfa9af0 DB |
404 | u32 id, count = 0; |
405 | u64 revision = 1; | |
053c8e1f | 406 | int i, ret = 0; |
053c8e1f DB |
407 | |
408 | if (attr->query.query_flags || attr->query.attach_flags) | |
409 | return -EINVAL; | |
edfa9af0 DB |
410 | if (entry) { |
411 | revision = bpf_mprog_revision(entry); | |
412 | count = bpf_mprog_total(entry); | |
413 | } | |
053c8e1f DB |
414 | if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags))) |
415 | return -EFAULT; | |
416 | if (copy_to_user(&uattr->query.revision, &revision, sizeof(revision))) | |
417 | return -EFAULT; | |
418 | if (copy_to_user(&uattr->query.count, &count, sizeof(count))) | |
419 | return -EFAULT; | |
420 | uprog_id = u64_to_user_ptr(attr->query.prog_ids); | |
421 | uprog_flags = u64_to_user_ptr(attr->query.prog_attach_flags); | |
422 | ulink_id = u64_to_user_ptr(attr->query.link_ids); | |
423 | ulink_flags = u64_to_user_ptr(attr->query.link_attach_flags); | |
424 | if (attr->query.count == 0 || !uprog_id || !count) | |
425 | return 0; | |
426 | if (attr->query.count < count) { | |
427 | count = attr->query.count; | |
428 | ret = -ENOSPC; | |
429 | } | |
430 | for (i = 0; i < bpf_mprog_max(); i++) { | |
431 | bpf_mprog_read(entry, i, &fp, &cp); | |
432 | prog = READ_ONCE(fp->prog); | |
433 | if (!prog) | |
434 | break; | |
435 | id = prog->aux->id; | |
436 | if (copy_to_user(uprog_id + i, &id, sizeof(id))) | |
437 | return -EFAULT; | |
438 | if (uprog_flags && | |
439 | copy_to_user(uprog_flags + i, &flags, sizeof(flags))) | |
440 | return -EFAULT; | |
441 | id = cp->link ? cp->link->id : 0; | |
442 | if (ulink_id && | |
443 | copy_to_user(ulink_id + i, &id, sizeof(id))) | |
444 | return -EFAULT; | |
445 | if (ulink_flags && | |
446 | copy_to_user(ulink_flags + i, &flags, sizeof(flags))) | |
447 | return -EFAULT; | |
448 | if (i + 1 == count) | |
449 | break; | |
450 | } | |
451 | return ret; | |
452 | } |