Commit | Line | Data |
---|---|---|
25763b3c | 1 | // SPDX-License-Identifier: GPL-2.0-only |
1cf1cae9 | 2 | /* Copyright (c) 2017 Facebook |
1cf1cae9 AS |
3 | */ |
4 | #include <linux/bpf.h> | |
c48e51c8 | 5 | #include <linux/btf.h> |
7bd1590d | 6 | #include <linux/btf_ids.h> |
1cf1cae9 | 7 | #include <linux/slab.h> |
b202d844 | 8 | #include <linux/init.h> |
1cf1cae9 AS |
9 | #include <linux/vmalloc.h> |
10 | #include <linux/etherdevice.h> | |
11 | #include <linux/filter.h> | |
87b7b533 | 12 | #include <linux/rcupdate_trace.h> |
1cf1cae9 | 13 | #include <linux/sched/signal.h> |
6ac99e8f | 14 | #include <net/bpf_sk_storage.h> |
2cb494a3 SL |
15 | #include <net/sock.h> |
16 | #include <net/tcp.h> | |
7c32e8f8 | 17 | #include <net/net_namespace.h> |
3d08b6f2 | 18 | #include <linux/error-injection.h> |
1b4d60ec | 19 | #include <linux/smp.h> |
7c32e8f8 | 20 | #include <linux/sock_diag.h> |
47316f4a | 21 | #include <net/xdp.h> |
1cf1cae9 | 22 | |
e950e843 MM |
23 | #define CREATE_TRACE_POINTS |
24 | #include <trace/events/bpf_test_run.h> | |
25 | ||
607b9cc9 LB |
26 | struct bpf_test_timer { |
27 | enum { NO_PREEMPT, NO_MIGRATE } mode; | |
28 | u32 i; | |
29 | u64 time_start, time_spent; | |
30 | }; | |
31 | ||
32 | static void bpf_test_timer_enter(struct bpf_test_timer *t) | |
33 | __acquires(rcu) | |
34 | { | |
35 | rcu_read_lock(); | |
36 | if (t->mode == NO_PREEMPT) | |
37 | preempt_disable(); | |
38 | else | |
39 | migrate_disable(); | |
40 | ||
41 | t->time_start = ktime_get_ns(); | |
42 | } | |
43 | ||
44 | static void bpf_test_timer_leave(struct bpf_test_timer *t) | |
45 | __releases(rcu) | |
46 | { | |
47 | t->time_start = 0; | |
48 | ||
49 | if (t->mode == NO_PREEMPT) | |
50 | preempt_enable(); | |
51 | else | |
52 | migrate_enable(); | |
53 | rcu_read_unlock(); | |
54 | } | |
55 | ||
56 | static bool bpf_test_timer_continue(struct bpf_test_timer *t, u32 repeat, int *err, u32 *duration) | |
57 | __must_hold(rcu) | |
58 | { | |
59 | t->i++; | |
60 | if (t->i >= repeat) { | |
61 | /* We're done. */ | |
62 | t->time_spent += ktime_get_ns() - t->time_start; | |
63 | do_div(t->time_spent, t->i); | |
64 | *duration = t->time_spent > U32_MAX ? U32_MAX : (u32)t->time_spent; | |
65 | *err = 0; | |
66 | goto reset; | |
67 | } | |
68 | ||
69 | if (signal_pending(current)) { | |
70 | /* During iteration: we've been cancelled, abort. */ | |
71 | *err = -EINTR; | |
72 | goto reset; | |
73 | } | |
74 | ||
75 | if (need_resched()) { | |
76 | /* During iteration: we need to reschedule between runs. */ | |
77 | t->time_spent += ktime_get_ns() - t->time_start; | |
78 | bpf_test_timer_leave(t); | |
79 | cond_resched(); | |
80 | bpf_test_timer_enter(t); | |
81 | } | |
82 | ||
83 | /* Do another round. */ | |
84 | return true; | |
85 | ||
86 | reset: | |
87 | t->i = 0; | |
88 | return false; | |
89 | } | |
90 | ||
df1a2cb7 | 91 | static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, |
f23c4b39 | 92 | u32 *retval, u32 *time, bool xdp) |
1cf1cae9 | 93 | { |
c7603cfa AN |
94 | struct bpf_prog_array_item item = {.prog = prog}; |
95 | struct bpf_run_ctx *old_ctx; | |
96 | struct bpf_cg_run_ctx run_ctx; | |
607b9cc9 | 97 | struct bpf_test_timer t = { NO_MIGRATE }; |
8bad74f9 | 98 | enum bpf_cgroup_storage_type stype; |
607b9cc9 | 99 | int ret; |
1cf1cae9 | 100 | |
8bad74f9 | 101 | for_each_cgroup_storage_type(stype) { |
c7603cfa AN |
102 | item.cgroup_storage[stype] = bpf_cgroup_storage_alloc(prog, stype); |
103 | if (IS_ERR(item.cgroup_storage[stype])) { | |
104 | item.cgroup_storage[stype] = NULL; | |
8bad74f9 | 105 | for_each_cgroup_storage_type(stype) |
c7603cfa | 106 | bpf_cgroup_storage_free(item.cgroup_storage[stype]); |
8bad74f9 RG |
107 | return -ENOMEM; |
108 | } | |
109 | } | |
f42ee093 | 110 | |
1cf1cae9 AS |
111 | if (!repeat) |
112 | repeat = 1; | |
df1a2cb7 | 113 | |
607b9cc9 | 114 | bpf_test_timer_enter(&t); |
c7603cfa | 115 | old_ctx = bpf_set_run_ctx(&run_ctx.run_ctx); |
607b9cc9 | 116 | do { |
c7603cfa | 117 | run_ctx.prog_item = &item; |
f23c4b39 BT |
118 | if (xdp) |
119 | *retval = bpf_prog_run_xdp(prog, ctx); | |
120 | else | |
fb7dd8bc | 121 | *retval = bpf_prog_run(prog, ctx); |
607b9cc9 | 122 | } while (bpf_test_timer_continue(&t, repeat, &ret, time)); |
c7603cfa | 123 | bpf_reset_run_ctx(old_ctx); |
607b9cc9 | 124 | bpf_test_timer_leave(&t); |
1cf1cae9 | 125 | |
8bad74f9 | 126 | for_each_cgroup_storage_type(stype) |
c7603cfa | 127 | bpf_cgroup_storage_free(item.cgroup_storage[stype]); |
f42ee093 | 128 | |
df1a2cb7 | 129 | return ret; |
1cf1cae9 AS |
130 | } |
131 | ||
78e52272 DM |
132 | static int bpf_test_finish(const union bpf_attr *kattr, |
133 | union bpf_attr __user *uattr, const void *data, | |
7855e0db LB |
134 | struct skb_shared_info *sinfo, u32 size, |
135 | u32 retval, u32 duration) | |
1cf1cae9 | 136 | { |
78e52272 | 137 | void __user *data_out = u64_to_user_ptr(kattr->test.data_out); |
1cf1cae9 | 138 | int err = -EFAULT; |
b5a36b1e | 139 | u32 copy_size = size; |
1cf1cae9 | 140 | |
b5a36b1e LB |
141 | /* Clamp copy if the user has provided a size hint, but copy the full |
142 | * buffer if not to retain old behaviour. | |
143 | */ | |
144 | if (kattr->test.data_size_out && | |
145 | copy_size > kattr->test.data_size_out) { | |
146 | copy_size = kattr->test.data_size_out; | |
147 | err = -ENOSPC; | |
148 | } | |
149 | ||
7855e0db LB |
150 | if (data_out) { |
151 | int len = sinfo ? copy_size - sinfo->xdp_frags_size : copy_size; | |
152 | ||
530e214c SF |
153 | if (len < 0) { |
154 | err = -ENOSPC; | |
155 | goto out; | |
156 | } | |
157 | ||
7855e0db LB |
158 | if (copy_to_user(data_out, data, len)) |
159 | goto out; | |
160 | ||
161 | if (sinfo) { | |
5d1e9f43 SF |
162 | int i, offset = len; |
163 | u32 data_len; | |
7855e0db LB |
164 | |
165 | for (i = 0; i < sinfo->nr_frags; i++) { | |
166 | skb_frag_t *frag = &sinfo->frags[i]; | |
167 | ||
168 | if (offset >= copy_size) { | |
169 | err = -ENOSPC; | |
170 | break; | |
171 | } | |
172 | ||
5d1e9f43 | 173 | data_len = min_t(u32, copy_size - offset, |
7855e0db LB |
174 | skb_frag_size(frag)); |
175 | ||
176 | if (copy_to_user(data_out + offset, | |
177 | skb_frag_address(frag), | |
178 | data_len)) | |
179 | goto out; | |
180 | ||
181 | offset += data_len; | |
182 | } | |
183 | } | |
184 | } | |
185 | ||
1cf1cae9 AS |
186 | if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size))) |
187 | goto out; | |
188 | if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) | |
189 | goto out; | |
190 | if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration))) | |
191 | goto out; | |
b5a36b1e LB |
192 | if (err != -ENOSPC) |
193 | err = 0; | |
1cf1cae9 | 194 | out: |
e950e843 | 195 | trace_bpf_test_finish(&err); |
1cf1cae9 AS |
196 | return err; |
197 | } | |
198 | ||
faeb2dce AS |
199 | /* Integer types of various sizes and pointer combinations cover variety of |
200 | * architecture dependent calling conventions. 7+ can be supported in the | |
201 | * future. | |
202 | */ | |
e9ff9d52 | 203 | __diag_push(); |
0b206c6d KKD |
204 | __diag_ignore_all("-Wmissing-prototypes", |
205 | "Global functions as their definitions will be in vmlinux BTF"); | |
faeb2dce AS |
206 | int noinline bpf_fentry_test1(int a) |
207 | { | |
208 | return a + 1; | |
209 | } | |
46565696 KKD |
210 | EXPORT_SYMBOL_GPL(bpf_fentry_test1); |
211 | ALLOW_ERROR_INJECTION(bpf_fentry_test1, ERRNO); | |
faeb2dce AS |
212 | |
213 | int noinline bpf_fentry_test2(int a, u64 b) | |
214 | { | |
215 | return a + b; | |
216 | } | |
217 | ||
218 | int noinline bpf_fentry_test3(char a, int b, u64 c) | |
219 | { | |
220 | return a + b + c; | |
221 | } | |
222 | ||
223 | int noinline bpf_fentry_test4(void *a, char b, int c, u64 d) | |
224 | { | |
225 | return (long)a + b + c + d; | |
226 | } | |
227 | ||
228 | int noinline bpf_fentry_test5(u64 a, void *b, short c, int d, u64 e) | |
229 | { | |
230 | return a + (long)b + c + d + e; | |
231 | } | |
232 | ||
233 | int noinline bpf_fentry_test6(u64 a, void *b, short c, int d, void *e, u64 f) | |
234 | { | |
235 | return a + (long)b + c + d + (long)e + f; | |
236 | } | |
237 | ||
d923021c YS |
238 | struct bpf_fentry_test_t { |
239 | struct bpf_fentry_test_t *a; | |
240 | }; | |
241 | ||
242 | int noinline bpf_fentry_test7(struct bpf_fentry_test_t *arg) | |
243 | { | |
244 | return (long)arg; | |
245 | } | |
246 | ||
247 | int noinline bpf_fentry_test8(struct bpf_fentry_test_t *arg) | |
248 | { | |
249 | return (long)arg->a; | |
250 | } | |
251 | ||
3d08b6f2 KS |
252 | int noinline bpf_modify_return_test(int a, int *b) |
253 | { | |
254 | *b += 1; | |
255 | return a + *b; | |
256 | } | |
7bd1590d MKL |
257 | |
258 | u64 noinline bpf_kfunc_call_test1(struct sock *sk, u32 a, u64 b, u32 c, u64 d) | |
259 | { | |
260 | return a + b + c + d; | |
261 | } | |
262 | ||
263 | int noinline bpf_kfunc_call_test2(struct sock *sk, u32 a, u32 b) | |
264 | { | |
265 | return a + b; | |
266 | } | |
267 | ||
268 | struct sock * noinline bpf_kfunc_call_test3(struct sock *sk) | |
269 | { | |
270 | return sk; | |
271 | } | |
272 | ||
8218ccb5 KKD |
273 | struct prog_test_member { |
274 | u64 c; | |
275 | }; | |
276 | ||
c1ff181f KKD |
277 | struct prog_test_ref_kfunc { |
278 | int a; | |
279 | int b; | |
8218ccb5 | 280 | struct prog_test_member memb; |
c1ff181f KKD |
281 | struct prog_test_ref_kfunc *next; |
282 | }; | |
283 | ||
284 | static struct prog_test_ref_kfunc prog_test_struct = { | |
285 | .a = 42, | |
286 | .b = 108, | |
287 | .next = &prog_test_struct, | |
288 | }; | |
289 | ||
290 | noinline struct prog_test_ref_kfunc * | |
291 | bpf_kfunc_call_test_acquire(unsigned long *scalar_ptr) | |
292 | { | |
293 | /* randomly return NULL */ | |
294 | if (get_jiffies_64() % 2) | |
295 | return NULL; | |
296 | return &prog_test_struct; | |
297 | } | |
298 | ||
299 | noinline void bpf_kfunc_call_test_release(struct prog_test_ref_kfunc *p) | |
300 | { | |
301 | } | |
302 | ||
8218ccb5 KKD |
303 | noinline void bpf_kfunc_call_memb_release(struct prog_test_member *p) |
304 | { | |
305 | } | |
306 | ||
c1ff181f KKD |
307 | struct prog_test_pass1 { |
308 | int x0; | |
309 | struct { | |
310 | int x1; | |
311 | struct { | |
312 | int x2; | |
313 | struct { | |
314 | int x3; | |
315 | }; | |
316 | }; | |
317 | }; | |
318 | }; | |
319 | ||
320 | struct prog_test_pass2 { | |
321 | int len; | |
322 | short arr1[4]; | |
323 | struct { | |
324 | char arr2[4]; | |
325 | unsigned long arr3[8]; | |
326 | } x; | |
327 | }; | |
328 | ||
329 | struct prog_test_fail1 { | |
330 | void *p; | |
331 | int x; | |
332 | }; | |
333 | ||
334 | struct prog_test_fail2 { | |
335 | int x8; | |
336 | struct prog_test_pass1 x; | |
337 | }; | |
338 | ||
339 | struct prog_test_fail3 { | |
340 | int len; | |
341 | char arr1[2]; | |
ed8bb032 | 342 | char arr2[]; |
c1ff181f KKD |
343 | }; |
344 | ||
345 | noinline void bpf_kfunc_call_test_pass_ctx(struct __sk_buff *skb) | |
346 | { | |
347 | } | |
348 | ||
349 | noinline void bpf_kfunc_call_test_pass1(struct prog_test_pass1 *p) | |
350 | { | |
351 | } | |
352 | ||
353 | noinline void bpf_kfunc_call_test_pass2(struct prog_test_pass2 *p) | |
354 | { | |
355 | } | |
356 | ||
357 | noinline void bpf_kfunc_call_test_fail1(struct prog_test_fail1 *p) | |
358 | { | |
359 | } | |
360 | ||
361 | noinline void bpf_kfunc_call_test_fail2(struct prog_test_fail2 *p) | |
362 | { | |
363 | } | |
364 | ||
365 | noinline void bpf_kfunc_call_test_fail3(struct prog_test_fail3 *p) | |
366 | { | |
367 | } | |
368 | ||
369 | noinline void bpf_kfunc_call_test_mem_len_pass1(void *mem, int mem__sz) | |
370 | { | |
371 | } | |
372 | ||
373 | noinline void bpf_kfunc_call_test_mem_len_fail1(void *mem, int len) | |
374 | { | |
375 | } | |
376 | ||
377 | noinline void bpf_kfunc_call_test_mem_len_fail2(u64 *mem, int len) | |
378 | { | |
379 | } | |
380 | ||
e9ff9d52 | 381 | __diag_pop(); |
3d08b6f2 KS |
382 | |
383 | ALLOW_ERROR_INJECTION(bpf_modify_return_test, ERRNO); | |
384 | ||
b202d844 | 385 | BTF_SET_START(test_sk_check_kfunc_ids) |
7bd1590d MKL |
386 | BTF_ID(func, bpf_kfunc_call_test1) |
387 | BTF_ID(func, bpf_kfunc_call_test2) | |
388 | BTF_ID(func, bpf_kfunc_call_test3) | |
c1ff181f KKD |
389 | BTF_ID(func, bpf_kfunc_call_test_acquire) |
390 | BTF_ID(func, bpf_kfunc_call_test_release) | |
8218ccb5 | 391 | BTF_ID(func, bpf_kfunc_call_memb_release) |
c1ff181f KKD |
392 | BTF_ID(func, bpf_kfunc_call_test_pass_ctx) |
393 | BTF_ID(func, bpf_kfunc_call_test_pass1) | |
394 | BTF_ID(func, bpf_kfunc_call_test_pass2) | |
395 | BTF_ID(func, bpf_kfunc_call_test_fail1) | |
396 | BTF_ID(func, bpf_kfunc_call_test_fail2) | |
397 | BTF_ID(func, bpf_kfunc_call_test_fail3) | |
398 | BTF_ID(func, bpf_kfunc_call_test_mem_len_pass1) | |
399 | BTF_ID(func, bpf_kfunc_call_test_mem_len_fail1) | |
400 | BTF_ID(func, bpf_kfunc_call_test_mem_len_fail2) | |
b202d844 | 401 | BTF_SET_END(test_sk_check_kfunc_ids) |
7bd1590d | 402 | |
c1ff181f KKD |
403 | BTF_SET_START(test_sk_acquire_kfunc_ids) |
404 | BTF_ID(func, bpf_kfunc_call_test_acquire) | |
405 | BTF_SET_END(test_sk_acquire_kfunc_ids) | |
406 | ||
407 | BTF_SET_START(test_sk_release_kfunc_ids) | |
408 | BTF_ID(func, bpf_kfunc_call_test_release) | |
8218ccb5 | 409 | BTF_ID(func, bpf_kfunc_call_memb_release) |
c1ff181f KKD |
410 | BTF_SET_END(test_sk_release_kfunc_ids) |
411 | ||
412 | BTF_SET_START(test_sk_ret_null_kfunc_ids) | |
413 | BTF_ID(func, bpf_kfunc_call_test_acquire) | |
414 | BTF_SET_END(test_sk_ret_null_kfunc_ids) | |
415 | ||
be3d72a2 LB |
416 | static void *bpf_test_init(const union bpf_attr *kattr, u32 user_size, |
417 | u32 size, u32 headroom, u32 tailroom) | |
1cf1cae9 AS |
418 | { |
419 | void __user *data_in = u64_to_user_ptr(kattr->test.data_in); | |
420 | void *data; | |
421 | ||
422 | if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom) | |
423 | return ERR_PTR(-EINVAL); | |
424 | ||
d800bad6 JDB |
425 | if (user_size > size) |
426 | return ERR_PTR(-EMSGSIZE); | |
427 | ||
1cf1cae9 AS |
428 | data = kzalloc(size + headroom + tailroom, GFP_USER); |
429 | if (!data) | |
430 | return ERR_PTR(-ENOMEM); | |
431 | ||
d800bad6 | 432 | if (copy_from_user(data + headroom, data_in, user_size)) { |
1cf1cae9 AS |
433 | kfree(data); |
434 | return ERR_PTR(-EFAULT); | |
435 | } | |
da00d2f1 | 436 | |
1cf1cae9 AS |
437 | return data; |
438 | } | |
439 | ||
da00d2f1 KS |
440 | int bpf_prog_test_run_tracing(struct bpf_prog *prog, |
441 | const union bpf_attr *kattr, | |
442 | union bpf_attr __user *uattr) | |
443 | { | |
d923021c | 444 | struct bpf_fentry_test_t arg = {}; |
3d08b6f2 KS |
445 | u16 side_effect = 0, ret = 0; |
446 | int b = 2, err = -EFAULT; | |
447 | u32 retval = 0; | |
da00d2f1 | 448 | |
1b4d60ec SL |
449 | if (kattr->test.flags || kattr->test.cpu) |
450 | return -EINVAL; | |
451 | ||
da00d2f1 KS |
452 | switch (prog->expected_attach_type) { |
453 | case BPF_TRACE_FENTRY: | |
454 | case BPF_TRACE_FEXIT: | |
455 | if (bpf_fentry_test1(1) != 2 || | |
456 | bpf_fentry_test2(2, 3) != 5 || | |
457 | bpf_fentry_test3(4, 5, 6) != 15 || | |
458 | bpf_fentry_test4((void *)7, 8, 9, 10) != 34 || | |
459 | bpf_fentry_test5(11, (void *)12, 13, 14, 15) != 65 || | |
d923021c YS |
460 | bpf_fentry_test6(16, (void *)17, 18, 19, (void *)20, 21) != 111 || |
461 | bpf_fentry_test7((struct bpf_fentry_test_t *)0) != 0 || | |
462 | bpf_fentry_test8(&arg) != 0) | |
da00d2f1 KS |
463 | goto out; |
464 | break; | |
3d08b6f2 KS |
465 | case BPF_MODIFY_RETURN: |
466 | ret = bpf_modify_return_test(1, &b); | |
467 | if (b != 2) | |
468 | side_effect = 1; | |
469 | break; | |
da00d2f1 KS |
470 | default: |
471 | goto out; | |
472 | } | |
473 | ||
3d08b6f2 KS |
474 | retval = ((u32)side_effect << 16) | ret; |
475 | if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) | |
476 | goto out; | |
477 | ||
da00d2f1 KS |
478 | err = 0; |
479 | out: | |
480 | trace_bpf_test_finish(&err); | |
481 | return err; | |
482 | } | |
483 | ||
1b4d60ec SL |
484 | struct bpf_raw_tp_test_run_info { |
485 | struct bpf_prog *prog; | |
486 | void *ctx; | |
487 | u32 retval; | |
488 | }; | |
489 | ||
490 | static void | |
491 | __bpf_prog_test_run_raw_tp(void *data) | |
492 | { | |
493 | struct bpf_raw_tp_test_run_info *info = data; | |
494 | ||
495 | rcu_read_lock(); | |
fb7dd8bc | 496 | info->retval = bpf_prog_run(info->prog, info->ctx); |
1b4d60ec SL |
497 | rcu_read_unlock(); |
498 | } | |
499 | ||
500 | int bpf_prog_test_run_raw_tp(struct bpf_prog *prog, | |
501 | const union bpf_attr *kattr, | |
502 | union bpf_attr __user *uattr) | |
503 | { | |
504 | void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in); | |
505 | __u32 ctx_size_in = kattr->test.ctx_size_in; | |
506 | struct bpf_raw_tp_test_run_info info; | |
507 | int cpu = kattr->test.cpu, err = 0; | |
963ec27a | 508 | int current_cpu; |
1b4d60ec SL |
509 | |
510 | /* doesn't support data_in/out, ctx_out, duration, or repeat */ | |
511 | if (kattr->test.data_in || kattr->test.data_out || | |
512 | kattr->test.ctx_out || kattr->test.duration || | |
513 | kattr->test.repeat) | |
514 | return -EINVAL; | |
515 | ||
7ac6ad05 SL |
516 | if (ctx_size_in < prog->aux->max_ctx_offset || |
517 | ctx_size_in > MAX_BPF_FUNC_ARGS * sizeof(u64)) | |
1b4d60ec SL |
518 | return -EINVAL; |
519 | ||
520 | if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 && cpu != 0) | |
521 | return -EINVAL; | |
522 | ||
523 | if (ctx_size_in) { | |
db5b6a46 QW |
524 | info.ctx = memdup_user(ctx_in, ctx_size_in); |
525 | if (IS_ERR(info.ctx)) | |
526 | return PTR_ERR(info.ctx); | |
1b4d60ec SL |
527 | } else { |
528 | info.ctx = NULL; | |
529 | } | |
530 | ||
531 | info.prog = prog; | |
532 | ||
963ec27a | 533 | current_cpu = get_cpu(); |
1b4d60ec | 534 | if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 || |
963ec27a | 535 | cpu == current_cpu) { |
1b4d60ec | 536 | __bpf_prog_test_run_raw_tp(&info); |
963ec27a | 537 | } else if (cpu >= nr_cpu_ids || !cpu_online(cpu)) { |
1b4d60ec SL |
538 | /* smp_call_function_single() also checks cpu_online() |
539 | * after csd_lock(). However, since cpu is from user | |
540 | * space, let's do an extra quick check to filter out | |
541 | * invalid value before smp_call_function_single(). | |
542 | */ | |
963ec27a SL |
543 | err = -ENXIO; |
544 | } else { | |
1b4d60ec SL |
545 | err = smp_call_function_single(cpu, __bpf_prog_test_run_raw_tp, |
546 | &info, 1); | |
1b4d60ec | 547 | } |
963ec27a | 548 | put_cpu(); |
1b4d60ec | 549 | |
963ec27a SL |
550 | if (!err && |
551 | copy_to_user(&uattr->test.retval, &info.retval, sizeof(u32))) | |
1b4d60ec SL |
552 | err = -EFAULT; |
553 | ||
1b4d60ec SL |
554 | kfree(info.ctx); |
555 | return err; | |
556 | } | |
557 | ||
b0b9395d SF |
558 | static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size) |
559 | { | |
560 | void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in); | |
561 | void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); | |
562 | u32 size = kattr->test.ctx_size_in; | |
563 | void *data; | |
564 | int err; | |
565 | ||
566 | if (!data_in && !data_out) | |
567 | return NULL; | |
568 | ||
569 | data = kzalloc(max_size, GFP_USER); | |
570 | if (!data) | |
571 | return ERR_PTR(-ENOMEM); | |
572 | ||
573 | if (data_in) { | |
af2ac3e1 | 574 | err = bpf_check_uarg_tail_zero(USER_BPFPTR(data_in), max_size, size); |
b0b9395d SF |
575 | if (err) { |
576 | kfree(data); | |
577 | return ERR_PTR(err); | |
578 | } | |
579 | ||
580 | size = min_t(u32, max_size, size); | |
581 | if (copy_from_user(data, data_in, size)) { | |
582 | kfree(data); | |
583 | return ERR_PTR(-EFAULT); | |
584 | } | |
585 | } | |
586 | return data; | |
587 | } | |
588 | ||
589 | static int bpf_ctx_finish(const union bpf_attr *kattr, | |
590 | union bpf_attr __user *uattr, const void *data, | |
591 | u32 size) | |
592 | { | |
593 | void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); | |
594 | int err = -EFAULT; | |
595 | u32 copy_size = size; | |
596 | ||
597 | if (!data || !data_out) | |
598 | return 0; | |
599 | ||
600 | if (copy_size > kattr->test.ctx_size_out) { | |
601 | copy_size = kattr->test.ctx_size_out; | |
602 | err = -ENOSPC; | |
603 | } | |
604 | ||
605 | if (copy_to_user(data_out, data, copy_size)) | |
606 | goto out; | |
607 | if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size))) | |
608 | goto out; | |
609 | if (err != -ENOSPC) | |
610 | err = 0; | |
611 | out: | |
612 | return err; | |
613 | } | |
614 | ||
615 | /** | |
616 | * range_is_zero - test whether buffer is initialized | |
617 | * @buf: buffer to check | |
618 | * @from: check from this position | |
619 | * @to: check up until (excluding) this position | |
620 | * | |
621 | * This function returns true if the there is a non-zero byte | |
622 | * in the buf in the range [from,to). | |
623 | */ | |
624 | static inline bool range_is_zero(void *buf, size_t from, size_t to) | |
625 | { | |
626 | return !memchr_inv((u8 *)buf + from, 0, to - from); | |
627 | } | |
628 | ||
629 | static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb) | |
630 | { | |
631 | struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; | |
632 | ||
633 | if (!__skb) | |
634 | return 0; | |
635 | ||
636 | /* make sure the fields we don't use are zeroed */ | |
6de6c1f8 NS |
637 | if (!range_is_zero(__skb, 0, offsetof(struct __sk_buff, mark))) |
638 | return -EINVAL; | |
639 | ||
640 | /* mark is allowed */ | |
641 | ||
642 | if (!range_is_zero(__skb, offsetofend(struct __sk_buff, mark), | |
643 | offsetof(struct __sk_buff, priority))) | |
b0b9395d SF |
644 | return -EINVAL; |
645 | ||
646 | /* priority is allowed */ | |
b238290b | 647 | /* ingress_ifindex is allowed */ |
21594c44 DY |
648 | /* ifindex is allowed */ |
649 | ||
650 | if (!range_is_zero(__skb, offsetofend(struct __sk_buff, ifindex), | |
b0b9395d SF |
651 | offsetof(struct __sk_buff, cb))) |
652 | return -EINVAL; | |
653 | ||
654 | /* cb is allowed */ | |
655 | ||
b590cb5f | 656 | if (!range_is_zero(__skb, offsetofend(struct __sk_buff, cb), |
ba940948 SF |
657 | offsetof(struct __sk_buff, tstamp))) |
658 | return -EINVAL; | |
659 | ||
660 | /* tstamp is allowed */ | |
850a88cc SF |
661 | /* wire_len is allowed */ |
662 | /* gso_segs is allowed */ | |
ba940948 | 663 | |
850a88cc | 664 | if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_segs), |
cf62089b WB |
665 | offsetof(struct __sk_buff, gso_size))) |
666 | return -EINVAL; | |
667 | ||
668 | /* gso_size is allowed */ | |
669 | ||
670 | if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_size), | |
3384c7c7 VF |
671 | offsetof(struct __sk_buff, hwtstamp))) |
672 | return -EINVAL; | |
673 | ||
674 | /* hwtstamp is allowed */ | |
675 | ||
676 | if (!range_is_zero(__skb, offsetofend(struct __sk_buff, hwtstamp), | |
b0b9395d SF |
677 | sizeof(struct __sk_buff))) |
678 | return -EINVAL; | |
679 | ||
6de6c1f8 | 680 | skb->mark = __skb->mark; |
b0b9395d | 681 | skb->priority = __skb->priority; |
b238290b | 682 | skb->skb_iif = __skb->ingress_ifindex; |
ba940948 | 683 | skb->tstamp = __skb->tstamp; |
b0b9395d SF |
684 | memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN); |
685 | ||
850a88cc SF |
686 | if (__skb->wire_len == 0) { |
687 | cb->pkt_len = skb->len; | |
688 | } else { | |
689 | if (__skb->wire_len < skb->len || | |
690 | __skb->wire_len > GSO_MAX_SIZE) | |
691 | return -EINVAL; | |
692 | cb->pkt_len = __skb->wire_len; | |
693 | } | |
694 | ||
695 | if (__skb->gso_segs > GSO_MAX_SEGS) | |
696 | return -EINVAL; | |
697 | skb_shinfo(skb)->gso_segs = __skb->gso_segs; | |
cf62089b | 698 | skb_shinfo(skb)->gso_size = __skb->gso_size; |
3384c7c7 | 699 | skb_shinfo(skb)->hwtstamps.hwtstamp = __skb->hwtstamp; |
850a88cc | 700 | |
b0b9395d SF |
701 | return 0; |
702 | } | |
703 | ||
704 | static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb) | |
705 | { | |
706 | struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; | |
707 | ||
708 | if (!__skb) | |
709 | return; | |
710 | ||
6de6c1f8 | 711 | __skb->mark = skb->mark; |
b0b9395d | 712 | __skb->priority = skb->priority; |
b238290b | 713 | __skb->ingress_ifindex = skb->skb_iif; |
21594c44 | 714 | __skb->ifindex = skb->dev->ifindex; |
ba940948 | 715 | __skb->tstamp = skb->tstamp; |
b0b9395d | 716 | memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN); |
850a88cc SF |
717 | __skb->wire_len = cb->pkt_len; |
718 | __skb->gso_segs = skb_shinfo(skb)->gso_segs; | |
3384c7c7 | 719 | __skb->hwtstamp = skb_shinfo(skb)->hwtstamps.hwtstamp; |
b0b9395d SF |
720 | } |
721 | ||
435b08ec DB |
722 | static struct proto bpf_dummy_proto = { |
723 | .name = "bpf_dummy", | |
724 | .owner = THIS_MODULE, | |
725 | .obj_size = sizeof(struct sock), | |
726 | }; | |
727 | ||
1cf1cae9 AS |
728 | int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, |
729 | union bpf_attr __user *uattr) | |
730 | { | |
731 | bool is_l2 = false, is_direct_pkt_access = false; | |
21594c44 DY |
732 | struct net *net = current->nsproxy->net_ns; |
733 | struct net_device *dev = net->loopback_dev; | |
1cf1cae9 AS |
734 | u32 size = kattr->test.data_size_in; |
735 | u32 repeat = kattr->test.repeat; | |
b0b9395d | 736 | struct __sk_buff *ctx = NULL; |
1cf1cae9 | 737 | u32 retval, duration; |
6e6fddc7 | 738 | int hh_len = ETH_HLEN; |
1cf1cae9 | 739 | struct sk_buff *skb; |
2cb494a3 | 740 | struct sock *sk; |
1cf1cae9 AS |
741 | void *data; |
742 | int ret; | |
743 | ||
1b4d60ec SL |
744 | if (kattr->test.flags || kattr->test.cpu) |
745 | return -EINVAL; | |
746 | ||
be3d72a2 LB |
747 | data = bpf_test_init(kattr, kattr->test.data_size_in, |
748 | size, NET_SKB_PAD + NET_IP_ALIGN, | |
1cf1cae9 AS |
749 | SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); |
750 | if (IS_ERR(data)) | |
751 | return PTR_ERR(data); | |
752 | ||
b0b9395d SF |
753 | ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff)); |
754 | if (IS_ERR(ctx)) { | |
755 | kfree(data); | |
756 | return PTR_ERR(ctx); | |
757 | } | |
758 | ||
1cf1cae9 AS |
759 | switch (prog->type) { |
760 | case BPF_PROG_TYPE_SCHED_CLS: | |
761 | case BPF_PROG_TYPE_SCHED_ACT: | |
762 | is_l2 = true; | |
df561f66 | 763 | fallthrough; |
1cf1cae9 AS |
764 | case BPF_PROG_TYPE_LWT_IN: |
765 | case BPF_PROG_TYPE_LWT_OUT: | |
766 | case BPF_PROG_TYPE_LWT_XMIT: | |
767 | is_direct_pkt_access = true; | |
768 | break; | |
769 | default: | |
770 | break; | |
771 | } | |
772 | ||
435b08ec | 773 | sk = sk_alloc(net, AF_UNSPEC, GFP_USER, &bpf_dummy_proto, 1); |
2cb494a3 SL |
774 | if (!sk) { |
775 | kfree(data); | |
b0b9395d | 776 | kfree(ctx); |
2cb494a3 SL |
777 | return -ENOMEM; |
778 | } | |
2cb494a3 SL |
779 | sock_init_data(NULL, sk); |
780 | ||
1cf1cae9 AS |
781 | skb = build_skb(data, 0); |
782 | if (!skb) { | |
783 | kfree(data); | |
b0b9395d | 784 | kfree(ctx); |
435b08ec | 785 | sk_free(sk); |
1cf1cae9 AS |
786 | return -ENOMEM; |
787 | } | |
2cb494a3 | 788 | skb->sk = sk; |
1cf1cae9 | 789 | |
586f8525 | 790 | skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); |
1cf1cae9 | 791 | __skb_put(skb, size); |
21594c44 DY |
792 | if (ctx && ctx->ifindex > 1) { |
793 | dev = dev_get_by_index(net, ctx->ifindex); | |
794 | if (!dev) { | |
795 | ret = -ENODEV; | |
796 | goto out; | |
797 | } | |
798 | } | |
799 | skb->protocol = eth_type_trans(skb, dev); | |
1cf1cae9 AS |
800 | skb_reset_network_header(skb); |
801 | ||
fa5cb548 DY |
802 | switch (skb->protocol) { |
803 | case htons(ETH_P_IP): | |
804 | sk->sk_family = AF_INET; | |
805 | if (sizeof(struct iphdr) <= skb_headlen(skb)) { | |
806 | sk->sk_rcv_saddr = ip_hdr(skb)->saddr; | |
807 | sk->sk_daddr = ip_hdr(skb)->daddr; | |
808 | } | |
809 | break; | |
810 | #if IS_ENABLED(CONFIG_IPV6) | |
811 | case htons(ETH_P_IPV6): | |
812 | sk->sk_family = AF_INET6; | |
813 | if (sizeof(struct ipv6hdr) <= skb_headlen(skb)) { | |
814 | sk->sk_v6_rcv_saddr = ipv6_hdr(skb)->saddr; | |
815 | sk->sk_v6_daddr = ipv6_hdr(skb)->daddr; | |
816 | } | |
817 | break; | |
818 | #endif | |
819 | default: | |
820 | break; | |
821 | } | |
822 | ||
1cf1cae9 | 823 | if (is_l2) |
6e6fddc7 | 824 | __skb_push(skb, hh_len); |
1cf1cae9 | 825 | if (is_direct_pkt_access) |
6aaae2b6 | 826 | bpf_compute_data_pointers(skb); |
b0b9395d SF |
827 | ret = convert___skb_to_skb(skb, ctx); |
828 | if (ret) | |
829 | goto out; | |
f23c4b39 | 830 | ret = bpf_test_run(prog, skb, repeat, &retval, &duration, false); |
b0b9395d SF |
831 | if (ret) |
832 | goto out; | |
6e6fddc7 DB |
833 | if (!is_l2) { |
834 | if (skb_headroom(skb) < hh_len) { | |
835 | int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb)); | |
836 | ||
837 | if (pskb_expand_head(skb, nhead, 0, GFP_USER)) { | |
b0b9395d SF |
838 | ret = -ENOMEM; |
839 | goto out; | |
6e6fddc7 DB |
840 | } |
841 | } | |
842 | memset(__skb_push(skb, hh_len), 0, hh_len); | |
843 | } | |
b0b9395d | 844 | convert_skb_to___skb(skb, ctx); |
6e6fddc7 | 845 | |
1cf1cae9 AS |
846 | size = skb->len; |
847 | /* bpf program can never convert linear skb to non-linear */ | |
848 | if (WARN_ON_ONCE(skb_is_nonlinear(skb))) | |
849 | size = skb_headlen(skb); | |
7855e0db LB |
850 | ret = bpf_test_finish(kattr, uattr, skb->data, NULL, size, retval, |
851 | duration); | |
b0b9395d SF |
852 | if (!ret) |
853 | ret = bpf_ctx_finish(kattr, uattr, ctx, | |
854 | sizeof(struct __sk_buff)); | |
855 | out: | |
21594c44 DY |
856 | if (dev && dev != net->loopback_dev) |
857 | dev_put(dev); | |
1cf1cae9 | 858 | kfree_skb(skb); |
435b08ec | 859 | sk_free(sk); |
b0b9395d | 860 | kfree(ctx); |
1cf1cae9 AS |
861 | return ret; |
862 | } | |
863 | ||
47316f4a ZE |
864 | static int xdp_convert_md_to_buff(struct xdp_md *xdp_md, struct xdp_buff *xdp) |
865 | { | |
ec94670f ZE |
866 | unsigned int ingress_ifindex, rx_queue_index; |
867 | struct netdev_rx_queue *rxqueue; | |
868 | struct net_device *device; | |
869 | ||
47316f4a ZE |
870 | if (!xdp_md) |
871 | return 0; | |
872 | ||
873 | if (xdp_md->egress_ifindex != 0) | |
874 | return -EINVAL; | |
875 | ||
ec94670f ZE |
876 | ingress_ifindex = xdp_md->ingress_ifindex; |
877 | rx_queue_index = xdp_md->rx_queue_index; | |
878 | ||
879 | if (!ingress_ifindex && rx_queue_index) | |
47316f4a ZE |
880 | return -EINVAL; |
881 | ||
ec94670f ZE |
882 | if (ingress_ifindex) { |
883 | device = dev_get_by_index(current->nsproxy->net_ns, | |
884 | ingress_ifindex); | |
885 | if (!device) | |
886 | return -ENODEV; | |
887 | ||
888 | if (rx_queue_index >= device->real_num_rx_queues) | |
889 | goto free_dev; | |
890 | ||
891 | rxqueue = __netif_get_rx_queue(device, rx_queue_index); | |
47316f4a | 892 | |
ec94670f ZE |
893 | if (!xdp_rxq_info_is_reg(&rxqueue->xdp_rxq)) |
894 | goto free_dev; | |
895 | ||
896 | xdp->rxq = &rxqueue->xdp_rxq; | |
897 | /* The device is now tracked in the xdp->rxq for later | |
898 | * dev_put() | |
899 | */ | |
900 | } | |
901 | ||
902 | xdp->data = xdp->data_meta + xdp_md->data; | |
47316f4a | 903 | return 0; |
ec94670f ZE |
904 | |
905 | free_dev: | |
906 | dev_put(device); | |
907 | return -EINVAL; | |
908 | } | |
909 | ||
910 | static void xdp_convert_buff_to_md(struct xdp_buff *xdp, struct xdp_md *xdp_md) | |
911 | { | |
912 | if (!xdp_md) | |
913 | return; | |
914 | ||
915 | xdp_md->data = xdp->data - xdp->data_meta; | |
916 | xdp_md->data_end = xdp->data_end - xdp->data_meta; | |
917 | ||
918 | if (xdp_md->ingress_ifindex) | |
919 | dev_put(xdp->rxq->dev); | |
47316f4a ZE |
920 | } |
921 | ||
1cf1cae9 AS |
922 | int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, |
923 | union bpf_attr __user *uattr) | |
924 | { | |
bc56c919 | 925 | u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); |
1cf1cae9 | 926 | u32 size = kattr->test.data_size_in; |
1c194998 LB |
927 | u32 headroom = XDP_PACKET_HEADROOM; |
928 | u32 retval, duration, max_data_sz; | |
1cf1cae9 | 929 | u32 repeat = kattr->test.repeat; |
65073a67 | 930 | struct netdev_rx_queue *rxqueue; |
1c194998 | 931 | struct skb_shared_info *sinfo; |
1cf1cae9 | 932 | struct xdp_buff xdp = {}; |
1c194998 | 933 | int i, ret = -EINVAL; |
47316f4a | 934 | struct xdp_md *ctx; |
1cf1cae9 | 935 | void *data; |
1cf1cae9 | 936 | |
5e21bb4e XZ |
937 | if (prog->expected_attach_type == BPF_XDP_DEVMAP || |
938 | prog->expected_attach_type == BPF_XDP_CPUMAP) | |
939 | return -EINVAL; | |
6d4eb36d | 940 | |
47316f4a ZE |
941 | ctx = bpf_ctx_init(kattr, sizeof(struct xdp_md)); |
942 | if (IS_ERR(ctx)) | |
943 | return PTR_ERR(ctx); | |
944 | ||
945 | if (ctx) { | |
946 | /* There can't be user provided data before the meta data */ | |
947 | if (ctx->data_meta || ctx->data_end != size || | |
948 | ctx->data > ctx->data_end || | |
949 | unlikely(xdp_metalen_invalid(ctx->data))) | |
950 | goto free_ctx; | |
951 | /* Meta data is allocated from the headroom */ | |
952 | headroom -= ctx->data; | |
953 | } | |
947e8b59 | 954 | |
bc56c919 | 955 | max_data_sz = 4096 - headroom - tailroom; |
1c194998 | 956 | size = min_t(u32, size, max_data_sz); |
bc56c919 | 957 | |
1c194998 | 958 | data = bpf_test_init(kattr, size, max_data_sz, headroom, tailroom); |
47316f4a ZE |
959 | if (IS_ERR(data)) { |
960 | ret = PTR_ERR(data); | |
961 | goto free_ctx; | |
962 | } | |
1cf1cae9 | 963 | |
65073a67 | 964 | rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0); |
1c194998 LB |
965 | rxqueue->xdp_rxq.frag_size = headroom + max_data_sz + tailroom; |
966 | xdp_init_buff(&xdp, rxqueue->xdp_rxq.frag_size, &rxqueue->xdp_rxq); | |
be9df4af | 967 | xdp_prepare_buff(&xdp, data, headroom, size, true); |
1c194998 | 968 | sinfo = xdp_get_shared_info_from_buff(&xdp); |
be9df4af | 969 | |
47316f4a ZE |
970 | ret = xdp_convert_md_to_buff(ctx, &xdp); |
971 | if (ret) | |
972 | goto free_data; | |
973 | ||
1c194998 LB |
974 | if (unlikely(kattr->test.data_size_in > size)) { |
975 | void __user *data_in = u64_to_user_ptr(kattr->test.data_in); | |
976 | ||
977 | while (size < kattr->test.data_size_in) { | |
978 | struct page *page; | |
979 | skb_frag_t *frag; | |
9d63b59d | 980 | u32 data_len; |
1c194998 | 981 | |
a6763080 LB |
982 | if (sinfo->nr_frags == MAX_SKB_FRAGS) { |
983 | ret = -ENOMEM; | |
984 | goto out; | |
985 | } | |
986 | ||
1c194998 LB |
987 | page = alloc_page(GFP_KERNEL); |
988 | if (!page) { | |
989 | ret = -ENOMEM; | |
990 | goto out; | |
991 | } | |
992 | ||
993 | frag = &sinfo->frags[sinfo->nr_frags++]; | |
994 | __skb_frag_set_page(frag, page); | |
995 | ||
9d63b59d | 996 | data_len = min_t(u32, kattr->test.data_size_in - size, |
1c194998 LB |
997 | PAGE_SIZE); |
998 | skb_frag_size_set(frag, data_len); | |
999 | ||
1000 | if (copy_from_user(page_address(page), data_in + size, | |
1001 | data_len)) { | |
1002 | ret = -EFAULT; | |
1003 | goto out; | |
1004 | } | |
1005 | sinfo->xdp_frags_size += data_len; | |
1006 | size += data_len; | |
1007 | } | |
1008 | xdp_buff_set_frags_flag(&xdp); | |
1009 | } | |
1010 | ||
de21d8bf LB |
1011 | if (repeat > 1) |
1012 | bpf_prog_change_xdp(NULL, prog); | |
1c194998 | 1013 | |
f23c4b39 | 1014 | ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration, true); |
ec94670f ZE |
1015 | /* We convert the xdp_buff back to an xdp_md before checking the return |
1016 | * code so the reference count of any held netdevice will be decremented | |
1017 | * even if the test run failed. | |
1018 | */ | |
1019 | xdp_convert_buff_to_md(&xdp, ctx); | |
dcb40590 RG |
1020 | if (ret) |
1021 | goto out; | |
47316f4a | 1022 | |
1c194998 | 1023 | size = xdp.data_end - xdp.data_meta + sinfo->xdp_frags_size; |
7855e0db LB |
1024 | ret = bpf_test_finish(kattr, uattr, xdp.data_meta, sinfo, size, |
1025 | retval, duration); | |
47316f4a ZE |
1026 | if (!ret) |
1027 | ret = bpf_ctx_finish(kattr, uattr, ctx, | |
1028 | sizeof(struct xdp_md)); | |
1029 | ||
dcb40590 | 1030 | out: |
de21d8bf LB |
1031 | if (repeat > 1) |
1032 | bpf_prog_change_xdp(prog, NULL); | |
47316f4a | 1033 | free_data: |
1c194998 LB |
1034 | for (i = 0; i < sinfo->nr_frags; i++) |
1035 | __free_page(skb_frag_page(&sinfo->frags[i])); | |
1cf1cae9 | 1036 | kfree(data); |
47316f4a ZE |
1037 | free_ctx: |
1038 | kfree(ctx); | |
1cf1cae9 AS |
1039 | return ret; |
1040 | } | |
b7a1848e | 1041 | |
b2ca4e1c SF |
1042 | static int verify_user_bpf_flow_keys(struct bpf_flow_keys *ctx) |
1043 | { | |
1044 | /* make sure the fields we don't use are zeroed */ | |
1045 | if (!range_is_zero(ctx, 0, offsetof(struct bpf_flow_keys, flags))) | |
1046 | return -EINVAL; | |
1047 | ||
1048 | /* flags is allowed */ | |
1049 | ||
b590cb5f | 1050 | if (!range_is_zero(ctx, offsetofend(struct bpf_flow_keys, flags), |
b2ca4e1c SF |
1051 | sizeof(struct bpf_flow_keys))) |
1052 | return -EINVAL; | |
1053 | ||
1054 | return 0; | |
1055 | } | |
1056 | ||
b7a1848e SF |
1057 | int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, |
1058 | const union bpf_attr *kattr, | |
1059 | union bpf_attr __user *uattr) | |
1060 | { | |
607b9cc9 | 1061 | struct bpf_test_timer t = { NO_PREEMPT }; |
b7a1848e | 1062 | u32 size = kattr->test.data_size_in; |
7b8a1304 | 1063 | struct bpf_flow_dissector ctx = {}; |
b7a1848e | 1064 | u32 repeat = kattr->test.repeat; |
b2ca4e1c | 1065 | struct bpf_flow_keys *user_ctx; |
b7a1848e | 1066 | struct bpf_flow_keys flow_keys; |
7b8a1304 | 1067 | const struct ethhdr *eth; |
b2ca4e1c | 1068 | unsigned int flags = 0; |
b7a1848e | 1069 | u32 retval, duration; |
b7a1848e SF |
1070 | void *data; |
1071 | int ret; | |
b7a1848e SF |
1072 | |
1073 | if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR) | |
1074 | return -EINVAL; | |
1075 | ||
1b4d60ec SL |
1076 | if (kattr->test.flags || kattr->test.cpu) |
1077 | return -EINVAL; | |
1078 | ||
7b8a1304 SF |
1079 | if (size < ETH_HLEN) |
1080 | return -EINVAL; | |
1081 | ||
be3d72a2 | 1082 | data = bpf_test_init(kattr, kattr->test.data_size_in, size, 0, 0); |
b7a1848e SF |
1083 | if (IS_ERR(data)) |
1084 | return PTR_ERR(data); | |
1085 | ||
7b8a1304 | 1086 | eth = (struct ethhdr *)data; |
b7a1848e | 1087 | |
b7a1848e SF |
1088 | if (!repeat) |
1089 | repeat = 1; | |
1090 | ||
b2ca4e1c SF |
1091 | user_ctx = bpf_ctx_init(kattr, sizeof(struct bpf_flow_keys)); |
1092 | if (IS_ERR(user_ctx)) { | |
1093 | kfree(data); | |
1094 | return PTR_ERR(user_ctx); | |
1095 | } | |
1096 | if (user_ctx) { | |
1097 | ret = verify_user_bpf_flow_keys(user_ctx); | |
1098 | if (ret) | |
1099 | goto out; | |
1100 | flags = user_ctx->flags; | |
1101 | } | |
1102 | ||
7b8a1304 SF |
1103 | ctx.flow_keys = &flow_keys; |
1104 | ctx.data = data; | |
1105 | ctx.data_end = (__u8 *)data + size; | |
1106 | ||
607b9cc9 LB |
1107 | bpf_test_timer_enter(&t); |
1108 | do { | |
7b8a1304 | 1109 | retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN, |
b2ca4e1c | 1110 | size, flags); |
607b9cc9 LB |
1111 | } while (bpf_test_timer_continue(&t, repeat, &ret, &duration)); |
1112 | bpf_test_timer_leave(&t); | |
7b8a1304 | 1113 | |
607b9cc9 LB |
1114 | if (ret < 0) |
1115 | goto out; | |
b7a1848e | 1116 | |
7855e0db LB |
1117 | ret = bpf_test_finish(kattr, uattr, &flow_keys, NULL, |
1118 | sizeof(flow_keys), retval, duration); | |
b2ca4e1c SF |
1119 | if (!ret) |
1120 | ret = bpf_ctx_finish(kattr, uattr, user_ctx, | |
1121 | sizeof(struct bpf_flow_keys)); | |
b7a1848e | 1122 | |
a439184d | 1123 | out: |
b2ca4e1c | 1124 | kfree(user_ctx); |
7b8a1304 | 1125 | kfree(data); |
b7a1848e SF |
1126 | return ret; |
1127 | } | |
7c32e8f8 LB |
1128 | |
1129 | int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kattr, | |
1130 | union bpf_attr __user *uattr) | |
1131 | { | |
1132 | struct bpf_test_timer t = { NO_PREEMPT }; | |
1133 | struct bpf_prog_array *progs = NULL; | |
1134 | struct bpf_sk_lookup_kern ctx = {}; | |
1135 | u32 repeat = kattr->test.repeat; | |
1136 | struct bpf_sk_lookup *user_ctx; | |
1137 | u32 retval, duration; | |
1138 | int ret = -EINVAL; | |
1139 | ||
1140 | if (prog->type != BPF_PROG_TYPE_SK_LOOKUP) | |
1141 | return -EINVAL; | |
1142 | ||
1143 | if (kattr->test.flags || kattr->test.cpu) | |
1144 | return -EINVAL; | |
1145 | ||
1146 | if (kattr->test.data_in || kattr->test.data_size_in || kattr->test.data_out || | |
1147 | kattr->test.data_size_out) | |
1148 | return -EINVAL; | |
1149 | ||
1150 | if (!repeat) | |
1151 | repeat = 1; | |
1152 | ||
1153 | user_ctx = bpf_ctx_init(kattr, sizeof(*user_ctx)); | |
1154 | if (IS_ERR(user_ctx)) | |
1155 | return PTR_ERR(user_ctx); | |
1156 | ||
1157 | if (!user_ctx) | |
1158 | return -EINVAL; | |
1159 | ||
1160 | if (user_ctx->sk) | |
1161 | goto out; | |
1162 | ||
1163 | if (!range_is_zero(user_ctx, offsetofend(typeof(*user_ctx), local_port), sizeof(*user_ctx))) | |
1164 | goto out; | |
1165 | ||
9a69e2b3 | 1166 | if (user_ctx->local_port > U16_MAX) { |
7c32e8f8 LB |
1167 | ret = -ERANGE; |
1168 | goto out; | |
1169 | } | |
1170 | ||
1171 | ctx.family = (u16)user_ctx->family; | |
1172 | ctx.protocol = (u16)user_ctx->protocol; | |
1173 | ctx.dport = (u16)user_ctx->local_port; | |
9a69e2b3 | 1174 | ctx.sport = user_ctx->remote_port; |
7c32e8f8 LB |
1175 | |
1176 | switch (ctx.family) { | |
1177 | case AF_INET: | |
1178 | ctx.v4.daddr = (__force __be32)user_ctx->local_ip4; | |
1179 | ctx.v4.saddr = (__force __be32)user_ctx->remote_ip4; | |
1180 | break; | |
1181 | ||
1182 | #if IS_ENABLED(CONFIG_IPV6) | |
1183 | case AF_INET6: | |
1184 | ctx.v6.daddr = (struct in6_addr *)user_ctx->local_ip6; | |
1185 | ctx.v6.saddr = (struct in6_addr *)user_ctx->remote_ip6; | |
1186 | break; | |
1187 | #endif | |
1188 | ||
1189 | default: | |
1190 | ret = -EAFNOSUPPORT; | |
1191 | goto out; | |
1192 | } | |
1193 | ||
1194 | progs = bpf_prog_array_alloc(1, GFP_KERNEL); | |
1195 | if (!progs) { | |
1196 | ret = -ENOMEM; | |
1197 | goto out; | |
1198 | } | |
1199 | ||
1200 | progs->items[0].prog = prog; | |
1201 | ||
1202 | bpf_test_timer_enter(&t); | |
1203 | do { | |
1204 | ctx.selected_sk = NULL; | |
fb7dd8bc | 1205 | retval = BPF_PROG_SK_LOOKUP_RUN_ARRAY(progs, ctx, bpf_prog_run); |
7c32e8f8 LB |
1206 | } while (bpf_test_timer_continue(&t, repeat, &ret, &duration)); |
1207 | bpf_test_timer_leave(&t); | |
1208 | ||
1209 | if (ret < 0) | |
1210 | goto out; | |
1211 | ||
1212 | user_ctx->cookie = 0; | |
1213 | if (ctx.selected_sk) { | |
1214 | if (ctx.selected_sk->sk_reuseport && !ctx.no_reuseport) { | |
1215 | ret = -EOPNOTSUPP; | |
1216 | goto out; | |
1217 | } | |
1218 | ||
1219 | user_ctx->cookie = sock_gen_cookie(ctx.selected_sk); | |
1220 | } | |
1221 | ||
7855e0db | 1222 | ret = bpf_test_finish(kattr, uattr, NULL, NULL, 0, retval, duration); |
7c32e8f8 LB |
1223 | if (!ret) |
1224 | ret = bpf_ctx_finish(kattr, uattr, user_ctx, sizeof(*user_ctx)); | |
1225 | ||
1226 | out: | |
1227 | bpf_prog_array_free(progs); | |
1228 | kfree(user_ctx); | |
1229 | return ret; | |
1230 | } | |
79a7f8bd AS |
1231 | |
1232 | int bpf_prog_test_run_syscall(struct bpf_prog *prog, | |
1233 | const union bpf_attr *kattr, | |
1234 | union bpf_attr __user *uattr) | |
1235 | { | |
1236 | void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in); | |
1237 | __u32 ctx_size_in = kattr->test.ctx_size_in; | |
1238 | void *ctx = NULL; | |
1239 | u32 retval; | |
1240 | int err = 0; | |
1241 | ||
1242 | /* doesn't support data_in/out, ctx_out, duration, or repeat or flags */ | |
1243 | if (kattr->test.data_in || kattr->test.data_out || | |
1244 | kattr->test.ctx_out || kattr->test.duration || | |
1245 | kattr->test.repeat || kattr->test.flags) | |
1246 | return -EINVAL; | |
1247 | ||
1248 | if (ctx_size_in < prog->aux->max_ctx_offset || | |
1249 | ctx_size_in > U16_MAX) | |
1250 | return -EINVAL; | |
1251 | ||
1252 | if (ctx_size_in) { | |
db5b6a46 QW |
1253 | ctx = memdup_user(ctx_in, ctx_size_in); |
1254 | if (IS_ERR(ctx)) | |
1255 | return PTR_ERR(ctx); | |
79a7f8bd | 1256 | } |
87b7b533 YS |
1257 | |
1258 | rcu_read_lock_trace(); | |
79a7f8bd | 1259 | retval = bpf_prog_run_pin_on_cpu(prog, ctx); |
87b7b533 | 1260 | rcu_read_unlock_trace(); |
79a7f8bd AS |
1261 | |
1262 | if (copy_to_user(&uattr->test.retval, &retval, sizeof(u32))) { | |
1263 | err = -EFAULT; | |
1264 | goto out; | |
1265 | } | |
1266 | if (ctx_size_in) | |
1267 | if (copy_to_user(ctx_in, ctx, ctx_size_in)) | |
1268 | err = -EFAULT; | |
1269 | out: | |
1270 | kfree(ctx); | |
1271 | return err; | |
1272 | } | |
b202d844 KKD |
1273 | |
1274 | static const struct btf_kfunc_id_set bpf_prog_test_kfunc_set = { | |
c1ff181f KKD |
1275 | .owner = THIS_MODULE, |
1276 | .check_set = &test_sk_check_kfunc_ids, | |
1277 | .acquire_set = &test_sk_acquire_kfunc_ids, | |
1278 | .release_set = &test_sk_release_kfunc_ids, | |
1279 | .ret_null_set = &test_sk_ret_null_kfunc_ids, | |
b202d844 KKD |
1280 | }; |
1281 | ||
1282 | static int __init bpf_prog_test_run_init(void) | |
1283 | { | |
1284 | return register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_prog_test_kfunc_set); | |
1285 | } | |
1286 | late_initcall(bpf_prog_test_run_init); |