Commit | Line | Data |
---|---|---|
1cf1cae9 AS |
1 | /* Copyright (c) 2017 Facebook |
2 | * | |
3 | * This program is free software; you can redistribute it and/or | |
4 | * modify it under the terms of version 2 of the GNU General Public | |
5 | * License as published by the Free Software Foundation. | |
6 | */ | |
7 | #include <linux/bpf.h> | |
8 | #include <linux/slab.h> | |
9 | #include <linux/vmalloc.h> | |
10 | #include <linux/etherdevice.h> | |
11 | #include <linux/filter.h> | |
12 | #include <linux/sched/signal.h> | |
2cb494a3 SL |
13 | #include <net/sock.h> |
14 | #include <net/tcp.h> | |
1cf1cae9 | 15 | |
df1a2cb7 SF |
16 | static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, |
17 | u32 *retval, u32 *time) | |
1cf1cae9 | 18 | { |
71b91a50 | 19 | struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = { NULL }; |
8bad74f9 | 20 | enum bpf_cgroup_storage_type stype; |
1cf1cae9 | 21 | u64 time_start, time_spent = 0; |
df1a2cb7 | 22 | int ret = 0; |
dcb40590 | 23 | u32 i; |
1cf1cae9 | 24 | |
8bad74f9 RG |
25 | for_each_cgroup_storage_type(stype) { |
26 | storage[stype] = bpf_cgroup_storage_alloc(prog, stype); | |
27 | if (IS_ERR(storage[stype])) { | |
28 | storage[stype] = NULL; | |
29 | for_each_cgroup_storage_type(stype) | |
30 | bpf_cgroup_storage_free(storage[stype]); | |
31 | return -ENOMEM; | |
32 | } | |
33 | } | |
f42ee093 | 34 | |
1cf1cae9 AS |
35 | if (!repeat) |
36 | repeat = 1; | |
df1a2cb7 SF |
37 | |
38 | rcu_read_lock(); | |
39 | preempt_disable(); | |
1cf1cae9 AS |
40 | time_start = ktime_get_ns(); |
41 | for (i = 0; i < repeat; i++) { | |
df1a2cb7 SF |
42 | bpf_cgroup_storage_set(storage); |
43 | *retval = BPF_PROG_RUN(prog, ctx); | |
44 | ||
45 | if (signal_pending(current)) { | |
46 | ret = -EINTR; | |
47 | break; | |
48 | } | |
49 | ||
1cf1cae9 | 50 | if (need_resched()) { |
1cf1cae9 | 51 | time_spent += ktime_get_ns() - time_start; |
df1a2cb7 SF |
52 | preempt_enable(); |
53 | rcu_read_unlock(); | |
54 | ||
1cf1cae9 | 55 | cond_resched(); |
df1a2cb7 SF |
56 | |
57 | rcu_read_lock(); | |
58 | preempt_disable(); | |
1cf1cae9 AS |
59 | time_start = ktime_get_ns(); |
60 | } | |
61 | } | |
62 | time_spent += ktime_get_ns() - time_start; | |
df1a2cb7 SF |
63 | preempt_enable(); |
64 | rcu_read_unlock(); | |
65 | ||
1cf1cae9 AS |
66 | do_div(time_spent, repeat); |
67 | *time = time_spent > U32_MAX ? U32_MAX : (u32)time_spent; | |
68 | ||
8bad74f9 RG |
69 | for_each_cgroup_storage_type(stype) |
70 | bpf_cgroup_storage_free(storage[stype]); | |
f42ee093 | 71 | |
df1a2cb7 | 72 | return ret; |
1cf1cae9 AS |
73 | } |
74 | ||
78e52272 DM |
75 | static int bpf_test_finish(const union bpf_attr *kattr, |
76 | union bpf_attr __user *uattr, const void *data, | |
1cf1cae9 AS |
77 | u32 size, u32 retval, u32 duration) |
78 | { | |
78e52272 | 79 | void __user *data_out = u64_to_user_ptr(kattr->test.data_out); |
1cf1cae9 | 80 | int err = -EFAULT; |
b5a36b1e | 81 | u32 copy_size = size; |
1cf1cae9 | 82 | |
b5a36b1e LB |
83 | /* Clamp copy if the user has provided a size hint, but copy the full |
84 | * buffer if not to retain old behaviour. | |
85 | */ | |
86 | if (kattr->test.data_size_out && | |
87 | copy_size > kattr->test.data_size_out) { | |
88 | copy_size = kattr->test.data_size_out; | |
89 | err = -ENOSPC; | |
90 | } | |
91 | ||
92 | if (data_out && copy_to_user(data_out, data, copy_size)) | |
1cf1cae9 AS |
93 | goto out; |
94 | if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size))) | |
95 | goto out; | |
96 | if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) | |
97 | goto out; | |
98 | if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration))) | |
99 | goto out; | |
b5a36b1e LB |
100 | if (err != -ENOSPC) |
101 | err = 0; | |
1cf1cae9 AS |
102 | out: |
103 | return err; | |
104 | } | |
105 | ||
106 | static void *bpf_test_init(const union bpf_attr *kattr, u32 size, | |
107 | u32 headroom, u32 tailroom) | |
108 | { | |
109 | void __user *data_in = u64_to_user_ptr(kattr->test.data_in); | |
110 | void *data; | |
111 | ||
112 | if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom) | |
113 | return ERR_PTR(-EINVAL); | |
114 | ||
115 | data = kzalloc(size + headroom + tailroom, GFP_USER); | |
116 | if (!data) | |
117 | return ERR_PTR(-ENOMEM); | |
118 | ||
119 | if (copy_from_user(data + headroom, data_in, size)) { | |
120 | kfree(data); | |
121 | return ERR_PTR(-EFAULT); | |
122 | } | |
123 | return data; | |
124 | } | |
125 | ||
b0b9395d SF |
126 | static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size) |
127 | { | |
128 | void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in); | |
129 | void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); | |
130 | u32 size = kattr->test.ctx_size_in; | |
131 | void *data; | |
132 | int err; | |
133 | ||
134 | if (!data_in && !data_out) | |
135 | return NULL; | |
136 | ||
137 | data = kzalloc(max_size, GFP_USER); | |
138 | if (!data) | |
139 | return ERR_PTR(-ENOMEM); | |
140 | ||
141 | if (data_in) { | |
142 | err = bpf_check_uarg_tail_zero(data_in, max_size, size); | |
143 | if (err) { | |
144 | kfree(data); | |
145 | return ERR_PTR(err); | |
146 | } | |
147 | ||
148 | size = min_t(u32, max_size, size); | |
149 | if (copy_from_user(data, data_in, size)) { | |
150 | kfree(data); | |
151 | return ERR_PTR(-EFAULT); | |
152 | } | |
153 | } | |
154 | return data; | |
155 | } | |
156 | ||
157 | static int bpf_ctx_finish(const union bpf_attr *kattr, | |
158 | union bpf_attr __user *uattr, const void *data, | |
159 | u32 size) | |
160 | { | |
161 | void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); | |
162 | int err = -EFAULT; | |
163 | u32 copy_size = size; | |
164 | ||
165 | if (!data || !data_out) | |
166 | return 0; | |
167 | ||
168 | if (copy_size > kattr->test.ctx_size_out) { | |
169 | copy_size = kattr->test.ctx_size_out; | |
170 | err = -ENOSPC; | |
171 | } | |
172 | ||
173 | if (copy_to_user(data_out, data, copy_size)) | |
174 | goto out; | |
175 | if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size))) | |
176 | goto out; | |
177 | if (err != -ENOSPC) | |
178 | err = 0; | |
179 | out: | |
180 | return err; | |
181 | } | |
182 | ||
183 | /** | |
184 | * range_is_zero - test whether buffer is initialized | |
185 | * @buf: buffer to check | |
186 | * @from: check from this position | |
187 | * @to: check up until (excluding) this position | |
188 | * | |
189 | * This function returns true if the there is a non-zero byte | |
190 | * in the buf in the range [from,to). | |
191 | */ | |
192 | static inline bool range_is_zero(void *buf, size_t from, size_t to) | |
193 | { | |
194 | return !memchr_inv((u8 *)buf + from, 0, to - from); | |
195 | } | |
196 | ||
197 | static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb) | |
198 | { | |
199 | struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; | |
200 | ||
201 | if (!__skb) | |
202 | return 0; | |
203 | ||
204 | /* make sure the fields we don't use are zeroed */ | |
205 | if (!range_is_zero(__skb, 0, offsetof(struct __sk_buff, priority))) | |
206 | return -EINVAL; | |
207 | ||
208 | /* priority is allowed */ | |
209 | ||
210 | if (!range_is_zero(__skb, offsetof(struct __sk_buff, priority) + | |
211 | FIELD_SIZEOF(struct __sk_buff, priority), | |
212 | offsetof(struct __sk_buff, cb))) | |
213 | return -EINVAL; | |
214 | ||
215 | /* cb is allowed */ | |
216 | ||
217 | if (!range_is_zero(__skb, offsetof(struct __sk_buff, cb) + | |
218 | FIELD_SIZEOF(struct __sk_buff, cb), | |
219 | sizeof(struct __sk_buff))) | |
220 | return -EINVAL; | |
221 | ||
222 | skb->priority = __skb->priority; | |
223 | memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN); | |
224 | ||
225 | return 0; | |
226 | } | |
227 | ||
228 | static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb) | |
229 | { | |
230 | struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; | |
231 | ||
232 | if (!__skb) | |
233 | return; | |
234 | ||
235 | __skb->priority = skb->priority; | |
236 | memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN); | |
237 | } | |
238 | ||
1cf1cae9 AS |
239 | int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, |
240 | union bpf_attr __user *uattr) | |
241 | { | |
242 | bool is_l2 = false, is_direct_pkt_access = false; | |
243 | u32 size = kattr->test.data_size_in; | |
244 | u32 repeat = kattr->test.repeat; | |
b0b9395d | 245 | struct __sk_buff *ctx = NULL; |
1cf1cae9 | 246 | u32 retval, duration; |
6e6fddc7 | 247 | int hh_len = ETH_HLEN; |
1cf1cae9 | 248 | struct sk_buff *skb; |
2cb494a3 | 249 | struct sock *sk; |
1cf1cae9 AS |
250 | void *data; |
251 | int ret; | |
252 | ||
586f8525 | 253 | data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN, |
1cf1cae9 AS |
254 | SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); |
255 | if (IS_ERR(data)) | |
256 | return PTR_ERR(data); | |
257 | ||
b0b9395d SF |
258 | ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff)); |
259 | if (IS_ERR(ctx)) { | |
260 | kfree(data); | |
261 | return PTR_ERR(ctx); | |
262 | } | |
263 | ||
1cf1cae9 AS |
264 | switch (prog->type) { |
265 | case BPF_PROG_TYPE_SCHED_CLS: | |
266 | case BPF_PROG_TYPE_SCHED_ACT: | |
267 | is_l2 = true; | |
268 | /* fall through */ | |
269 | case BPF_PROG_TYPE_LWT_IN: | |
270 | case BPF_PROG_TYPE_LWT_OUT: | |
271 | case BPF_PROG_TYPE_LWT_XMIT: | |
272 | is_direct_pkt_access = true; | |
273 | break; | |
274 | default: | |
275 | break; | |
276 | } | |
277 | ||
2cb494a3 SL |
278 | sk = kzalloc(sizeof(struct sock), GFP_USER); |
279 | if (!sk) { | |
280 | kfree(data); | |
b0b9395d | 281 | kfree(ctx); |
2cb494a3 SL |
282 | return -ENOMEM; |
283 | } | |
284 | sock_net_set(sk, current->nsproxy->net_ns); | |
285 | sock_init_data(NULL, sk); | |
286 | ||
1cf1cae9 AS |
287 | skb = build_skb(data, 0); |
288 | if (!skb) { | |
289 | kfree(data); | |
b0b9395d | 290 | kfree(ctx); |
2cb494a3 | 291 | kfree(sk); |
1cf1cae9 AS |
292 | return -ENOMEM; |
293 | } | |
2cb494a3 | 294 | skb->sk = sk; |
1cf1cae9 | 295 | |
586f8525 | 296 | skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); |
1cf1cae9 AS |
297 | __skb_put(skb, size); |
298 | skb->protocol = eth_type_trans(skb, current->nsproxy->net_ns->loopback_dev); | |
299 | skb_reset_network_header(skb); | |
300 | ||
301 | if (is_l2) | |
6e6fddc7 | 302 | __skb_push(skb, hh_len); |
1cf1cae9 | 303 | if (is_direct_pkt_access) |
6aaae2b6 | 304 | bpf_compute_data_pointers(skb); |
b0b9395d SF |
305 | ret = convert___skb_to_skb(skb, ctx); |
306 | if (ret) | |
307 | goto out; | |
dcb40590 | 308 | ret = bpf_test_run(prog, skb, repeat, &retval, &duration); |
b0b9395d SF |
309 | if (ret) |
310 | goto out; | |
6e6fddc7 DB |
311 | if (!is_l2) { |
312 | if (skb_headroom(skb) < hh_len) { | |
313 | int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb)); | |
314 | ||
315 | if (pskb_expand_head(skb, nhead, 0, GFP_USER)) { | |
b0b9395d SF |
316 | ret = -ENOMEM; |
317 | goto out; | |
6e6fddc7 DB |
318 | } |
319 | } | |
320 | memset(__skb_push(skb, hh_len), 0, hh_len); | |
321 | } | |
b0b9395d | 322 | convert_skb_to___skb(skb, ctx); |
6e6fddc7 | 323 | |
1cf1cae9 AS |
324 | size = skb->len; |
325 | /* bpf program can never convert linear skb to non-linear */ | |
326 | if (WARN_ON_ONCE(skb_is_nonlinear(skb))) | |
327 | size = skb_headlen(skb); | |
78e52272 | 328 | ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration); |
b0b9395d SF |
329 | if (!ret) |
330 | ret = bpf_ctx_finish(kattr, uattr, ctx, | |
331 | sizeof(struct __sk_buff)); | |
332 | out: | |
1cf1cae9 | 333 | kfree_skb(skb); |
2cb494a3 | 334 | kfree(sk); |
b0b9395d | 335 | kfree(ctx); |
1cf1cae9 AS |
336 | return ret; |
337 | } | |
338 | ||
339 | int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, | |
340 | union bpf_attr __user *uattr) | |
341 | { | |
342 | u32 size = kattr->test.data_size_in; | |
343 | u32 repeat = kattr->test.repeat; | |
65073a67 | 344 | struct netdev_rx_queue *rxqueue; |
1cf1cae9 AS |
345 | struct xdp_buff xdp = {}; |
346 | u32 retval, duration; | |
347 | void *data; | |
348 | int ret; | |
349 | ||
947e8b59 SF |
350 | if (kattr->test.ctx_in || kattr->test.ctx_out) |
351 | return -EINVAL; | |
352 | ||
586f8525 | 353 | data = bpf_test_init(kattr, size, XDP_PACKET_HEADROOM + NET_IP_ALIGN, 0); |
1cf1cae9 AS |
354 | if (IS_ERR(data)) |
355 | return PTR_ERR(data); | |
356 | ||
357 | xdp.data_hard_start = data; | |
586f8525 | 358 | xdp.data = data + XDP_PACKET_HEADROOM + NET_IP_ALIGN; |
de8f3a83 | 359 | xdp.data_meta = xdp.data; |
1cf1cae9 AS |
360 | xdp.data_end = xdp.data + size; |
361 | ||
65073a67 DB |
362 | rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0); |
363 | xdp.rxq = &rxqueue->xdp_rxq; | |
364 | ||
dcb40590 RG |
365 | ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration); |
366 | if (ret) | |
367 | goto out; | |
587b80cc NS |
368 | if (xdp.data != data + XDP_PACKET_HEADROOM + NET_IP_ALIGN || |
369 | xdp.data_end != xdp.data + size) | |
1cf1cae9 | 370 | size = xdp.data_end - xdp.data; |
78e52272 | 371 | ret = bpf_test_finish(kattr, uattr, xdp.data, size, retval, duration); |
dcb40590 | 372 | out: |
1cf1cae9 AS |
373 | kfree(data); |
374 | return ret; | |
375 | } | |
b7a1848e SF |
376 | |
377 | int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, | |
378 | const union bpf_attr *kattr, | |
379 | union bpf_attr __user *uattr) | |
380 | { | |
381 | u32 size = kattr->test.data_size_in; | |
382 | u32 repeat = kattr->test.repeat; | |
383 | struct bpf_flow_keys flow_keys; | |
384 | u64 time_start, time_spent = 0; | |
b7a1848e SF |
385 | u32 retval, duration; |
386 | struct sk_buff *skb; | |
387 | struct sock *sk; | |
388 | void *data; | |
389 | int ret; | |
390 | u32 i; | |
391 | ||
392 | if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR) | |
393 | return -EINVAL; | |
394 | ||
947e8b59 SF |
395 | if (kattr->test.ctx_in || kattr->test.ctx_out) |
396 | return -EINVAL; | |
397 | ||
b7a1848e SF |
398 | data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN, |
399 | SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); | |
400 | if (IS_ERR(data)) | |
401 | return PTR_ERR(data); | |
402 | ||
403 | sk = kzalloc(sizeof(*sk), GFP_USER); | |
404 | if (!sk) { | |
405 | kfree(data); | |
406 | return -ENOMEM; | |
407 | } | |
408 | sock_net_set(sk, current->nsproxy->net_ns); | |
409 | sock_init_data(NULL, sk); | |
410 | ||
411 | skb = build_skb(data, 0); | |
412 | if (!skb) { | |
413 | kfree(data); | |
414 | kfree(sk); | |
415 | return -ENOMEM; | |
416 | } | |
417 | skb->sk = sk; | |
418 | ||
419 | skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); | |
420 | __skb_put(skb, size); | |
421 | skb->protocol = eth_type_trans(skb, | |
422 | current->nsproxy->net_ns->loopback_dev); | |
423 | skb_reset_network_header(skb); | |
424 | ||
b7a1848e SF |
425 | if (!repeat) |
426 | repeat = 1; | |
427 | ||
a439184d SF |
428 | rcu_read_lock(); |
429 | preempt_disable(); | |
b7a1848e SF |
430 | time_start = ktime_get_ns(); |
431 | for (i = 0; i < repeat; i++) { | |
b7a1848e SF |
432 | retval = __skb_flow_bpf_dissect(prog, skb, |
433 | &flow_keys_dissector, | |
434 | &flow_keys); | |
a439184d SF |
435 | |
436 | if (signal_pending(current)) { | |
437 | preempt_enable(); | |
438 | rcu_read_unlock(); | |
439 | ||
440 | ret = -EINTR; | |
441 | goto out; | |
442 | } | |
b7a1848e SF |
443 | |
444 | if (need_resched()) { | |
b7a1848e | 445 | time_spent += ktime_get_ns() - time_start; |
a439184d SF |
446 | preempt_enable(); |
447 | rcu_read_unlock(); | |
448 | ||
b7a1848e | 449 | cond_resched(); |
a439184d SF |
450 | |
451 | rcu_read_lock(); | |
452 | preempt_disable(); | |
b7a1848e SF |
453 | time_start = ktime_get_ns(); |
454 | } | |
455 | } | |
456 | time_spent += ktime_get_ns() - time_start; | |
a439184d SF |
457 | preempt_enable(); |
458 | rcu_read_unlock(); | |
459 | ||
b7a1848e SF |
460 | do_div(time_spent, repeat); |
461 | duration = time_spent > U32_MAX ? U32_MAX : (u32)time_spent; | |
462 | ||
463 | ret = bpf_test_finish(kattr, uattr, &flow_keys, sizeof(flow_keys), | |
464 | retval, duration); | |
465 | ||
a439184d | 466 | out: |
b7a1848e SF |
467 | kfree_skb(skb); |
468 | kfree(sk); | |
469 | return ret; | |
470 | } |