1 /* Copyright (c) 2017 Facebook
3 * This program is free software; you can redistribute it and/or
4 * modify it under the terms of version 2 of the GNU General Public
5 * License as published by the Free Software Foundation.
8 #include <linux/slab.h>
9 #include <linux/vmalloc.h>
10 #include <linux/etherdevice.h>
11 #include <linux/filter.h>
12 #include <linux/sched/signal.h>
16 static __always_inline u32 bpf_test_run_one(struct bpf_prog *prog, void *ctx,
17 struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE])
23 bpf_cgroup_storage_set(storage);
24 ret = BPF_PROG_RUN(prog, ctx);
31 static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, u32 *ret,
34 struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = { 0 };
35 enum bpf_cgroup_storage_type stype;
36 u64 time_start, time_spent = 0;
39 for_each_cgroup_storage_type(stype) {
40 storage[stype] = bpf_cgroup_storage_alloc(prog, stype);
41 if (IS_ERR(storage[stype])) {
42 storage[stype] = NULL;
43 for_each_cgroup_storage_type(stype)
44 bpf_cgroup_storage_free(storage[stype]);
51 time_start = ktime_get_ns();
52 for (i = 0; i < repeat; i++) {
53 *ret = bpf_test_run_one(prog, ctx, storage);
55 if (signal_pending(current))
57 time_spent += ktime_get_ns() - time_start;
59 time_start = ktime_get_ns();
62 time_spent += ktime_get_ns() - time_start;
63 do_div(time_spent, repeat);
64 *time = time_spent > U32_MAX ? U32_MAX : (u32)time_spent;
66 for_each_cgroup_storage_type(stype)
67 bpf_cgroup_storage_free(storage[stype]);
72 static int bpf_test_finish(const union bpf_attr *kattr,
73 union bpf_attr __user *uattr, const void *data,
74 u32 size, u32 retval, u32 duration)
76 void __user *data_out = u64_to_user_ptr(kattr->test.data_out);
80 /* Clamp copy if the user has provided a size hint, but copy the full
81 * buffer if not to retain old behaviour.
83 if (kattr->test.data_size_out &&
84 copy_size > kattr->test.data_size_out) {
85 copy_size = kattr->test.data_size_out;
89 if (data_out && copy_to_user(data_out, data, copy_size))
91 if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size)))
93 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))
95 if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration)))
103 static void *bpf_test_init(const union bpf_attr *kattr, u32 size,
104 u32 headroom, u32 tailroom)
106 void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
109 if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom)
110 return ERR_PTR(-EINVAL);
112 data = kzalloc(size + headroom + tailroom, GFP_USER);
114 return ERR_PTR(-ENOMEM);
116 if (copy_from_user(data + headroom, data_in, size)) {
118 return ERR_PTR(-EFAULT);
123 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
124 union bpf_attr __user *uattr)
126 bool is_l2 = false, is_direct_pkt_access = false;
127 u32 size = kattr->test.data_size_in;
128 u32 repeat = kattr->test.repeat;
129 u32 retval, duration;
130 int hh_len = ETH_HLEN;
136 data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN,
137 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
139 return PTR_ERR(data);
141 switch (prog->type) {
142 case BPF_PROG_TYPE_SCHED_CLS:
143 case BPF_PROG_TYPE_SCHED_ACT:
146 case BPF_PROG_TYPE_LWT_IN:
147 case BPF_PROG_TYPE_LWT_OUT:
148 case BPF_PROG_TYPE_LWT_XMIT:
149 is_direct_pkt_access = true;
155 sk = kzalloc(sizeof(struct sock), GFP_USER);
160 sock_net_set(sk, current->nsproxy->net_ns);
161 sock_init_data(NULL, sk);
163 skb = build_skb(data, 0);
171 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
172 __skb_put(skb, size);
173 skb->protocol = eth_type_trans(skb, current->nsproxy->net_ns->loopback_dev);
174 skb_reset_network_header(skb);
177 __skb_push(skb, hh_len);
178 if (is_direct_pkt_access)
179 bpf_compute_data_pointers(skb);
180 ret = bpf_test_run(prog, skb, repeat, &retval, &duration);
187 if (skb_headroom(skb) < hh_len) {
188 int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb));
190 if (pskb_expand_head(skb, nhead, 0, GFP_USER)) {
196 memset(__skb_push(skb, hh_len), 0, hh_len);
200 /* bpf program can never convert linear skb to non-linear */
201 if (WARN_ON_ONCE(skb_is_nonlinear(skb)))
202 size = skb_headlen(skb);
203 ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration);
209 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
210 union bpf_attr __user *uattr)
212 u32 size = kattr->test.data_size_in;
213 u32 repeat = kattr->test.repeat;
214 struct netdev_rx_queue *rxqueue;
215 struct xdp_buff xdp = {};
216 u32 retval, duration;
220 data = bpf_test_init(kattr, size, XDP_PACKET_HEADROOM + NET_IP_ALIGN, 0);
222 return PTR_ERR(data);
224 xdp.data_hard_start = data;
225 xdp.data = data + XDP_PACKET_HEADROOM + NET_IP_ALIGN;
226 xdp.data_meta = xdp.data;
227 xdp.data_end = xdp.data + size;
229 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0);
230 xdp.rxq = &rxqueue->xdp_rxq;
232 ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration);
235 if (xdp.data != data + XDP_PACKET_HEADROOM + NET_IP_ALIGN ||
236 xdp.data_end != xdp.data + size)
237 size = xdp.data_end - xdp.data;
238 ret = bpf_test_finish(kattr, uattr, xdp.data, size, retval, duration);