Commit | Line | Data |
---|---|---|
25763b3c | 1 | // SPDX-License-Identifier: GPL-2.0-only |
1cf1cae9 | 2 | /* Copyright (c) 2017 Facebook |
1cf1cae9 AS |
3 | */ |
4 | #include <linux/bpf.h> | |
5 | #include <linux/slab.h> | |
6 | #include <linux/vmalloc.h> | |
7 | #include <linux/etherdevice.h> | |
8 | #include <linux/filter.h> | |
9 | #include <linux/sched/signal.h> | |
6ac99e8f | 10 | #include <net/bpf_sk_storage.h> |
2cb494a3 SL |
11 | #include <net/sock.h> |
12 | #include <net/tcp.h> | |
1cf1cae9 | 13 | |
e950e843 MM |
14 | #define CREATE_TRACE_POINTS |
15 | #include <trace/events/bpf_test_run.h> | |
16 | ||
df1a2cb7 SF |
17 | static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, |
18 | u32 *retval, u32 *time) | |
1cf1cae9 | 19 | { |
71b91a50 | 20 | struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = { NULL }; |
8bad74f9 | 21 | enum bpf_cgroup_storage_type stype; |
1cf1cae9 | 22 | u64 time_start, time_spent = 0; |
df1a2cb7 | 23 | int ret = 0; |
dcb40590 | 24 | u32 i; |
1cf1cae9 | 25 | |
8bad74f9 RG |
26 | for_each_cgroup_storage_type(stype) { |
27 | storage[stype] = bpf_cgroup_storage_alloc(prog, stype); | |
28 | if (IS_ERR(storage[stype])) { | |
29 | storage[stype] = NULL; | |
30 | for_each_cgroup_storage_type(stype) | |
31 | bpf_cgroup_storage_free(storage[stype]); | |
32 | return -ENOMEM; | |
33 | } | |
34 | } | |
f42ee093 | 35 | |
1cf1cae9 AS |
36 | if (!repeat) |
37 | repeat = 1; | |
df1a2cb7 SF |
38 | |
39 | rcu_read_lock(); | |
40 | preempt_disable(); | |
1cf1cae9 AS |
41 | time_start = ktime_get_ns(); |
42 | for (i = 0; i < repeat; i++) { | |
df1a2cb7 SF |
43 | bpf_cgroup_storage_set(storage); |
44 | *retval = BPF_PROG_RUN(prog, ctx); | |
45 | ||
46 | if (signal_pending(current)) { | |
47 | ret = -EINTR; | |
48 | break; | |
49 | } | |
50 | ||
1cf1cae9 | 51 | if (need_resched()) { |
1cf1cae9 | 52 | time_spent += ktime_get_ns() - time_start; |
df1a2cb7 SF |
53 | preempt_enable(); |
54 | rcu_read_unlock(); | |
55 | ||
1cf1cae9 | 56 | cond_resched(); |
df1a2cb7 SF |
57 | |
58 | rcu_read_lock(); | |
59 | preempt_disable(); | |
1cf1cae9 AS |
60 | time_start = ktime_get_ns(); |
61 | } | |
62 | } | |
63 | time_spent += ktime_get_ns() - time_start; | |
df1a2cb7 SF |
64 | preempt_enable(); |
65 | rcu_read_unlock(); | |
66 | ||
1cf1cae9 AS |
67 | do_div(time_spent, repeat); |
68 | *time = time_spent > U32_MAX ? U32_MAX : (u32)time_spent; | |
69 | ||
8bad74f9 RG |
70 | for_each_cgroup_storage_type(stype) |
71 | bpf_cgroup_storage_free(storage[stype]); | |
f42ee093 | 72 | |
df1a2cb7 | 73 | return ret; |
1cf1cae9 AS |
74 | } |
75 | ||
78e52272 DM |
76 | static int bpf_test_finish(const union bpf_attr *kattr, |
77 | union bpf_attr __user *uattr, const void *data, | |
1cf1cae9 AS |
78 | u32 size, u32 retval, u32 duration) |
79 | { | |
78e52272 | 80 | void __user *data_out = u64_to_user_ptr(kattr->test.data_out); |
1cf1cae9 | 81 | int err = -EFAULT; |
b5a36b1e | 82 | u32 copy_size = size; |
1cf1cae9 | 83 | |
b5a36b1e LB |
84 | /* Clamp copy if the user has provided a size hint, but copy the full |
85 | * buffer if not to retain old behaviour. | |
86 | */ | |
87 | if (kattr->test.data_size_out && | |
88 | copy_size > kattr->test.data_size_out) { | |
89 | copy_size = kattr->test.data_size_out; | |
90 | err = -ENOSPC; | |
91 | } | |
92 | ||
93 | if (data_out && copy_to_user(data_out, data, copy_size)) | |
1cf1cae9 AS |
94 | goto out; |
95 | if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size))) | |
96 | goto out; | |
97 | if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) | |
98 | goto out; | |
99 | if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration))) | |
100 | goto out; | |
b5a36b1e LB |
101 | if (err != -ENOSPC) |
102 | err = 0; | |
1cf1cae9 | 103 | out: |
e950e843 | 104 | trace_bpf_test_finish(&err); |
1cf1cae9 AS |
105 | return err; |
106 | } | |
107 | ||
108 | static void *bpf_test_init(const union bpf_attr *kattr, u32 size, | |
109 | u32 headroom, u32 tailroom) | |
110 | { | |
111 | void __user *data_in = u64_to_user_ptr(kattr->test.data_in); | |
112 | void *data; | |
113 | ||
114 | if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom) | |
115 | return ERR_PTR(-EINVAL); | |
116 | ||
117 | data = kzalloc(size + headroom + tailroom, GFP_USER); | |
118 | if (!data) | |
119 | return ERR_PTR(-ENOMEM); | |
120 | ||
121 | if (copy_from_user(data + headroom, data_in, size)) { | |
122 | kfree(data); | |
123 | return ERR_PTR(-EFAULT); | |
124 | } | |
125 | return data; | |
126 | } | |
127 | ||
b0b9395d SF |
128 | static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size) |
129 | { | |
130 | void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in); | |
131 | void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); | |
132 | u32 size = kattr->test.ctx_size_in; | |
133 | void *data; | |
134 | int err; | |
135 | ||
136 | if (!data_in && !data_out) | |
137 | return NULL; | |
138 | ||
139 | data = kzalloc(max_size, GFP_USER); | |
140 | if (!data) | |
141 | return ERR_PTR(-ENOMEM); | |
142 | ||
143 | if (data_in) { | |
144 | err = bpf_check_uarg_tail_zero(data_in, max_size, size); | |
145 | if (err) { | |
146 | kfree(data); | |
147 | return ERR_PTR(err); | |
148 | } | |
149 | ||
150 | size = min_t(u32, max_size, size); | |
151 | if (copy_from_user(data, data_in, size)) { | |
152 | kfree(data); | |
153 | return ERR_PTR(-EFAULT); | |
154 | } | |
155 | } | |
156 | return data; | |
157 | } | |
158 | ||
159 | static int bpf_ctx_finish(const union bpf_attr *kattr, | |
160 | union bpf_attr __user *uattr, const void *data, | |
161 | u32 size) | |
162 | { | |
163 | void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); | |
164 | int err = -EFAULT; | |
165 | u32 copy_size = size; | |
166 | ||
167 | if (!data || !data_out) | |
168 | return 0; | |
169 | ||
170 | if (copy_size > kattr->test.ctx_size_out) { | |
171 | copy_size = kattr->test.ctx_size_out; | |
172 | err = -ENOSPC; | |
173 | } | |
174 | ||
175 | if (copy_to_user(data_out, data, copy_size)) | |
176 | goto out; | |
177 | if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size))) | |
178 | goto out; | |
179 | if (err != -ENOSPC) | |
180 | err = 0; | |
181 | out: | |
182 | return err; | |
183 | } | |
184 | ||
185 | /** | |
186 | * range_is_zero - test whether buffer is initialized | |
187 | * @buf: buffer to check | |
188 | * @from: check from this position | |
189 | * @to: check up until (excluding) this position | |
190 | * | |
191 | * This function returns true if the there is a non-zero byte | |
192 | * in the buf in the range [from,to). | |
193 | */ | |
194 | static inline bool range_is_zero(void *buf, size_t from, size_t to) | |
195 | { | |
196 | return !memchr_inv((u8 *)buf + from, 0, to - from); | |
197 | } | |
198 | ||
199 | static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb) | |
200 | { | |
201 | struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; | |
202 | ||
203 | if (!__skb) | |
204 | return 0; | |
205 | ||
206 | /* make sure the fields we don't use are zeroed */ | |
207 | if (!range_is_zero(__skb, 0, offsetof(struct __sk_buff, priority))) | |
208 | return -EINVAL; | |
209 | ||
210 | /* priority is allowed */ | |
211 | ||
212 | if (!range_is_zero(__skb, offsetof(struct __sk_buff, priority) + | |
213 | FIELD_SIZEOF(struct __sk_buff, priority), | |
214 | offsetof(struct __sk_buff, cb))) | |
215 | return -EINVAL; | |
216 | ||
217 | /* cb is allowed */ | |
218 | ||
219 | if (!range_is_zero(__skb, offsetof(struct __sk_buff, cb) + | |
220 | FIELD_SIZEOF(struct __sk_buff, cb), | |
221 | sizeof(struct __sk_buff))) | |
222 | return -EINVAL; | |
223 | ||
224 | skb->priority = __skb->priority; | |
225 | memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN); | |
226 | ||
227 | return 0; | |
228 | } | |
229 | ||
230 | static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb) | |
231 | { | |
232 | struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; | |
233 | ||
234 | if (!__skb) | |
235 | return; | |
236 | ||
237 | __skb->priority = skb->priority; | |
238 | memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN); | |
239 | } | |
240 | ||
1cf1cae9 AS |
241 | int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, |
242 | union bpf_attr __user *uattr) | |
243 | { | |
244 | bool is_l2 = false, is_direct_pkt_access = false; | |
245 | u32 size = kattr->test.data_size_in; | |
246 | u32 repeat = kattr->test.repeat; | |
b0b9395d | 247 | struct __sk_buff *ctx = NULL; |
1cf1cae9 | 248 | u32 retval, duration; |
6e6fddc7 | 249 | int hh_len = ETH_HLEN; |
1cf1cae9 | 250 | struct sk_buff *skb; |
2cb494a3 | 251 | struct sock *sk; |
1cf1cae9 AS |
252 | void *data; |
253 | int ret; | |
254 | ||
586f8525 | 255 | data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN, |
1cf1cae9 AS |
256 | SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); |
257 | if (IS_ERR(data)) | |
258 | return PTR_ERR(data); | |
259 | ||
b0b9395d SF |
260 | ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff)); |
261 | if (IS_ERR(ctx)) { | |
262 | kfree(data); | |
263 | return PTR_ERR(ctx); | |
264 | } | |
265 | ||
1cf1cae9 AS |
266 | switch (prog->type) { |
267 | case BPF_PROG_TYPE_SCHED_CLS: | |
268 | case BPF_PROG_TYPE_SCHED_ACT: | |
269 | is_l2 = true; | |
270 | /* fall through */ | |
271 | case BPF_PROG_TYPE_LWT_IN: | |
272 | case BPF_PROG_TYPE_LWT_OUT: | |
273 | case BPF_PROG_TYPE_LWT_XMIT: | |
274 | is_direct_pkt_access = true; | |
275 | break; | |
276 | default: | |
277 | break; | |
278 | } | |
279 | ||
2cb494a3 SL |
280 | sk = kzalloc(sizeof(struct sock), GFP_USER); |
281 | if (!sk) { | |
282 | kfree(data); | |
b0b9395d | 283 | kfree(ctx); |
2cb494a3 SL |
284 | return -ENOMEM; |
285 | } | |
286 | sock_net_set(sk, current->nsproxy->net_ns); | |
287 | sock_init_data(NULL, sk); | |
288 | ||
1cf1cae9 AS |
289 | skb = build_skb(data, 0); |
290 | if (!skb) { | |
291 | kfree(data); | |
b0b9395d | 292 | kfree(ctx); |
2cb494a3 | 293 | kfree(sk); |
1cf1cae9 AS |
294 | return -ENOMEM; |
295 | } | |
2cb494a3 | 296 | skb->sk = sk; |
1cf1cae9 | 297 | |
586f8525 | 298 | skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); |
1cf1cae9 AS |
299 | __skb_put(skb, size); |
300 | skb->protocol = eth_type_trans(skb, current->nsproxy->net_ns->loopback_dev); | |
301 | skb_reset_network_header(skb); | |
302 | ||
303 | if (is_l2) | |
6e6fddc7 | 304 | __skb_push(skb, hh_len); |
1cf1cae9 | 305 | if (is_direct_pkt_access) |
6aaae2b6 | 306 | bpf_compute_data_pointers(skb); |
b0b9395d SF |
307 | ret = convert___skb_to_skb(skb, ctx); |
308 | if (ret) | |
309 | goto out; | |
dcb40590 | 310 | ret = bpf_test_run(prog, skb, repeat, &retval, &duration); |
b0b9395d SF |
311 | if (ret) |
312 | goto out; | |
6e6fddc7 DB |
313 | if (!is_l2) { |
314 | if (skb_headroom(skb) < hh_len) { | |
315 | int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb)); | |
316 | ||
317 | if (pskb_expand_head(skb, nhead, 0, GFP_USER)) { | |
b0b9395d SF |
318 | ret = -ENOMEM; |
319 | goto out; | |
6e6fddc7 DB |
320 | } |
321 | } | |
322 | memset(__skb_push(skb, hh_len), 0, hh_len); | |
323 | } | |
b0b9395d | 324 | convert_skb_to___skb(skb, ctx); |
6e6fddc7 | 325 | |
1cf1cae9 AS |
326 | size = skb->len; |
327 | /* bpf program can never convert linear skb to non-linear */ | |
328 | if (WARN_ON_ONCE(skb_is_nonlinear(skb))) | |
329 | size = skb_headlen(skb); | |
78e52272 | 330 | ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration); |
b0b9395d SF |
331 | if (!ret) |
332 | ret = bpf_ctx_finish(kattr, uattr, ctx, | |
333 | sizeof(struct __sk_buff)); | |
334 | out: | |
1cf1cae9 | 335 | kfree_skb(skb); |
6ac99e8f | 336 | bpf_sk_storage_free(sk); |
2cb494a3 | 337 | kfree(sk); |
b0b9395d | 338 | kfree(ctx); |
1cf1cae9 AS |
339 | return ret; |
340 | } | |
341 | ||
342 | int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, | |
343 | union bpf_attr __user *uattr) | |
344 | { | |
345 | u32 size = kattr->test.data_size_in; | |
346 | u32 repeat = kattr->test.repeat; | |
65073a67 | 347 | struct netdev_rx_queue *rxqueue; |
1cf1cae9 AS |
348 | struct xdp_buff xdp = {}; |
349 | u32 retval, duration; | |
350 | void *data; | |
351 | int ret; | |
352 | ||
947e8b59 SF |
353 | if (kattr->test.ctx_in || kattr->test.ctx_out) |
354 | return -EINVAL; | |
355 | ||
586f8525 | 356 | data = bpf_test_init(kattr, size, XDP_PACKET_HEADROOM + NET_IP_ALIGN, 0); |
1cf1cae9 AS |
357 | if (IS_ERR(data)) |
358 | return PTR_ERR(data); | |
359 | ||
360 | xdp.data_hard_start = data; | |
586f8525 | 361 | xdp.data = data + XDP_PACKET_HEADROOM + NET_IP_ALIGN; |
de8f3a83 | 362 | xdp.data_meta = xdp.data; |
1cf1cae9 AS |
363 | xdp.data_end = xdp.data + size; |
364 | ||
65073a67 DB |
365 | rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0); |
366 | xdp.rxq = &rxqueue->xdp_rxq; | |
367 | ||
dcb40590 RG |
368 | ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration); |
369 | if (ret) | |
370 | goto out; | |
587b80cc NS |
371 | if (xdp.data != data + XDP_PACKET_HEADROOM + NET_IP_ALIGN || |
372 | xdp.data_end != xdp.data + size) | |
1cf1cae9 | 373 | size = xdp.data_end - xdp.data; |
78e52272 | 374 | ret = bpf_test_finish(kattr, uattr, xdp.data, size, retval, duration); |
dcb40590 | 375 | out: |
1cf1cae9 AS |
376 | kfree(data); |
377 | return ret; | |
378 | } | |
b7a1848e | 379 | |
b2ca4e1c SF |
380 | static int verify_user_bpf_flow_keys(struct bpf_flow_keys *ctx) |
381 | { | |
382 | /* make sure the fields we don't use are zeroed */ | |
383 | if (!range_is_zero(ctx, 0, offsetof(struct bpf_flow_keys, flags))) | |
384 | return -EINVAL; | |
385 | ||
386 | /* flags is allowed */ | |
387 | ||
388 | if (!range_is_zero(ctx, offsetof(struct bpf_flow_keys, flags) + | |
389 | FIELD_SIZEOF(struct bpf_flow_keys, flags), | |
390 | sizeof(struct bpf_flow_keys))) | |
391 | return -EINVAL; | |
392 | ||
393 | return 0; | |
394 | } | |
395 | ||
b7a1848e SF |
396 | int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, |
397 | const union bpf_attr *kattr, | |
398 | union bpf_attr __user *uattr) | |
399 | { | |
400 | u32 size = kattr->test.data_size_in; | |
7b8a1304 | 401 | struct bpf_flow_dissector ctx = {}; |
b7a1848e | 402 | u32 repeat = kattr->test.repeat; |
b2ca4e1c | 403 | struct bpf_flow_keys *user_ctx; |
b7a1848e SF |
404 | struct bpf_flow_keys flow_keys; |
405 | u64 time_start, time_spent = 0; | |
7b8a1304 | 406 | const struct ethhdr *eth; |
b2ca4e1c | 407 | unsigned int flags = 0; |
b7a1848e | 408 | u32 retval, duration; |
b7a1848e SF |
409 | void *data; |
410 | int ret; | |
411 | u32 i; | |
412 | ||
413 | if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR) | |
414 | return -EINVAL; | |
415 | ||
7b8a1304 SF |
416 | if (size < ETH_HLEN) |
417 | return -EINVAL; | |
418 | ||
419 | data = bpf_test_init(kattr, size, 0, 0); | |
b7a1848e SF |
420 | if (IS_ERR(data)) |
421 | return PTR_ERR(data); | |
422 | ||
7b8a1304 | 423 | eth = (struct ethhdr *)data; |
b7a1848e | 424 | |
b7a1848e SF |
425 | if (!repeat) |
426 | repeat = 1; | |
427 | ||
b2ca4e1c SF |
428 | user_ctx = bpf_ctx_init(kattr, sizeof(struct bpf_flow_keys)); |
429 | if (IS_ERR(user_ctx)) { | |
430 | kfree(data); | |
431 | return PTR_ERR(user_ctx); | |
432 | } | |
433 | if (user_ctx) { | |
434 | ret = verify_user_bpf_flow_keys(user_ctx); | |
435 | if (ret) | |
436 | goto out; | |
437 | flags = user_ctx->flags; | |
438 | } | |
439 | ||
7b8a1304 SF |
440 | ctx.flow_keys = &flow_keys; |
441 | ctx.data = data; | |
442 | ctx.data_end = (__u8 *)data + size; | |
443 | ||
a439184d SF |
444 | rcu_read_lock(); |
445 | preempt_disable(); | |
b7a1848e SF |
446 | time_start = ktime_get_ns(); |
447 | for (i = 0; i < repeat; i++) { | |
7b8a1304 | 448 | retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN, |
b2ca4e1c | 449 | size, flags); |
7b8a1304 | 450 | |
a439184d SF |
451 | if (signal_pending(current)) { |
452 | preempt_enable(); | |
453 | rcu_read_unlock(); | |
454 | ||
455 | ret = -EINTR; | |
456 | goto out; | |
457 | } | |
b7a1848e SF |
458 | |
459 | if (need_resched()) { | |
b7a1848e | 460 | time_spent += ktime_get_ns() - time_start; |
a439184d SF |
461 | preempt_enable(); |
462 | rcu_read_unlock(); | |
463 | ||
b7a1848e | 464 | cond_resched(); |
a439184d SF |
465 | |
466 | rcu_read_lock(); | |
467 | preempt_disable(); | |
b7a1848e SF |
468 | time_start = ktime_get_ns(); |
469 | } | |
470 | } | |
471 | time_spent += ktime_get_ns() - time_start; | |
a439184d SF |
472 | preempt_enable(); |
473 | rcu_read_unlock(); | |
474 | ||
b7a1848e SF |
475 | do_div(time_spent, repeat); |
476 | duration = time_spent > U32_MAX ? U32_MAX : (u32)time_spent; | |
477 | ||
478 | ret = bpf_test_finish(kattr, uattr, &flow_keys, sizeof(flow_keys), | |
479 | retval, duration); | |
b2ca4e1c SF |
480 | if (!ret) |
481 | ret = bpf_ctx_finish(kattr, uattr, user_ctx, | |
482 | sizeof(struct bpf_flow_keys)); | |
b7a1848e | 483 | |
a439184d | 484 | out: |
b2ca4e1c | 485 | kfree(user_ctx); |
7b8a1304 | 486 | kfree(data); |
b7a1848e SF |
487 | return ret; |
488 | } |