Commit | Line | Data |
---|---|---|
351e1581 HZ |
1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* Copyright (c) 2019, Microsoft Corporation. | |
3 | * | |
4 | * Author: | |
5 | * Haiyang Zhang <haiyangz@microsoft.com> | |
6 | */ | |
7 | ||
8 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
9 | ||
10 | #include <linux/netdevice.h> | |
11 | #include <linux/etherdevice.h> | |
12 | #include <linux/ethtool.h> | |
1cb9d3b6 | 13 | #include <linux/netpoll.h> |
351e1581 HZ |
14 | #include <linux/bpf.h> |
15 | #include <linux/bpf_trace.h> | |
16 | #include <linux/kernel.h> | |
17 | #include <net/xdp.h> | |
18 | ||
19 | #include <linux/mutex.h> | |
20 | #include <linux/rtnetlink.h> | |
21 | ||
22 | #include "hyperv_net.h" | |
23 | ||
24 | u32 netvsc_run_xdp(struct net_device *ndev, struct netvsc_channel *nvchan, | |
25 | struct xdp_buff *xdp) | |
26 | { | |
1cb9d3b6 | 27 | struct netvsc_stats_rx *rx_stats = &nvchan->rx_stats; |
351e1581 HZ |
28 | void *data = nvchan->rsc.data[0]; |
29 | u32 len = nvchan->rsc.len[0]; | |
30 | struct page *page = NULL; | |
31 | struct bpf_prog *prog; | |
32 | u32 act = XDP_PASS; | |
1cb9d3b6 | 33 | bool drop = true; |
351e1581 HZ |
34 | |
35 | xdp->data_hard_start = NULL; | |
36 | ||
37 | rcu_read_lock(); | |
38 | prog = rcu_dereference(nvchan->bpf_prog); | |
39 | ||
40 | if (!prog) | |
41 | goto out; | |
42 | ||
505e3f00 APM |
43 | /* Ensure that the below memcpy() won't overflow the page buffer. */ |
44 | if (len > ndev->mtu + ETH_HLEN) { | |
45 | act = XDP_DROP; | |
46 | goto out; | |
47 | } | |
48 | ||
351e1581 HZ |
49 | /* allocate page buffer for data */ |
50 | page = alloc_page(GFP_ATOMIC); | |
51 | if (!page) { | |
52 | act = XDP_DROP; | |
53 | goto out; | |
54 | } | |
55 | ||
43b5169d | 56 | xdp_init_buff(xdp, PAGE_SIZE, &nvchan->xdp_rxq); |
be9df4af | 57 | xdp_prepare_buff(xdp, page_address(page), NETVSC_XDP_HDRM, len, false); |
351e1581 HZ |
58 | |
59 | memcpy(xdp->data, data, len); | |
60 | ||
61 | act = bpf_prog_run_xdp(prog, xdp); | |
62 | ||
63 | switch (act) { | |
64 | case XDP_PASS: | |
65 | case XDP_TX: | |
1cb9d3b6 HZ |
66 | drop = false; |
67 | break; | |
68 | ||
351e1581 HZ |
69 | case XDP_DROP: |
70 | break; | |
71 | ||
1cb9d3b6 HZ |
72 | case XDP_REDIRECT: |
73 | if (!xdp_do_redirect(ndev, xdp, prog)) { | |
74 | nvchan->xdp_flush = true; | |
75 | drop = false; | |
76 | ||
77 | u64_stats_update_begin(&rx_stats->syncp); | |
78 | ||
79 | rx_stats->xdp_redirect++; | |
80 | rx_stats->packets++; | |
81 | rx_stats->bytes += nvchan->rsc.pktlen; | |
82 | ||
83 | u64_stats_update_end(&rx_stats->syncp); | |
84 | ||
85 | break; | |
86 | } else { | |
87 | u64_stats_update_begin(&rx_stats->syncp); | |
88 | rx_stats->xdp_drop++; | |
89 | u64_stats_update_end(&rx_stats->syncp); | |
90 | } | |
91 | ||
92 | fallthrough; | |
93 | ||
351e1581 HZ |
94 | case XDP_ABORTED: |
95 | trace_xdp_exception(ndev, prog, act); | |
96 | break; | |
97 | ||
98 | default: | |
c8064e5b | 99 | bpf_warn_invalid_xdp_action(ndev, prog, act); |
351e1581 HZ |
100 | } |
101 | ||
102 | out: | |
103 | rcu_read_unlock(); | |
104 | ||
1cb9d3b6 | 105 | if (page && drop) { |
351e1581 HZ |
106 | __free_page(page); |
107 | xdp->data_hard_start = NULL; | |
108 | } | |
109 | ||
110 | return act; | |
111 | } | |
112 | ||
113 | unsigned int netvsc_xdp_fraglen(unsigned int len) | |
114 | { | |
115 | return SKB_DATA_ALIGN(len) + | |
116 | SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); | |
117 | } | |
118 | ||
119 | struct bpf_prog *netvsc_xdp_get(struct netvsc_device *nvdev) | |
120 | { | |
121 | return rtnl_dereference(nvdev->chan_table[0].bpf_prog); | |
122 | } | |
123 | ||
124 | int netvsc_xdp_set(struct net_device *dev, struct bpf_prog *prog, | |
125 | struct netlink_ext_ack *extack, | |
126 | struct netvsc_device *nvdev) | |
127 | { | |
128 | struct bpf_prog *old_prog; | |
129 | int buf_max, i; | |
130 | ||
131 | old_prog = netvsc_xdp_get(nvdev); | |
132 | ||
133 | if (!old_prog && !prog) | |
134 | return 0; | |
135 | ||
136 | buf_max = NETVSC_XDP_HDRM + netvsc_xdp_fraglen(dev->mtu + ETH_HLEN); | |
137 | if (prog && buf_max > PAGE_SIZE) { | |
138 | netdev_err(dev, "XDP: mtu:%u too large, buf_max:%u\n", | |
139 | dev->mtu, buf_max); | |
140 | NL_SET_ERR_MSG_MOD(extack, "XDP: mtu too large"); | |
141 | ||
142 | return -EOPNOTSUPP; | |
143 | } | |
144 | ||
145 | if (prog && (dev->features & NETIF_F_LRO)) { | |
146 | netdev_err(dev, "XDP: not support LRO\n"); | |
147 | NL_SET_ERR_MSG_MOD(extack, "XDP: not support LRO"); | |
148 | ||
149 | return -EOPNOTSUPP; | |
150 | } | |
151 | ||
152 | if (prog) | |
184367dc | 153 | bpf_prog_add(prog, nvdev->num_chn - 1); |
351e1581 HZ |
154 | |
155 | for (i = 0; i < nvdev->num_chn; i++) | |
156 | rcu_assign_pointer(nvdev->chan_table[i].bpf_prog, prog); | |
157 | ||
158 | if (old_prog) | |
159 | for (i = 0; i < nvdev->num_chn; i++) | |
160 | bpf_prog_put(old_prog); | |
161 | ||
162 | return 0; | |
163 | } | |
164 | ||
165 | int netvsc_vf_setxdp(struct net_device *vf_netdev, struct bpf_prog *prog) | |
166 | { | |
167 | struct netdev_bpf xdp; | |
184367dc | 168 | int ret; |
351e1581 HZ |
169 | |
170 | ASSERT_RTNL(); | |
171 | ||
172 | if (!vf_netdev) | |
173 | return 0; | |
174 | ||
e416531f | 175 | if (!vf_netdev->netdev_ops->ndo_bpf) |
351e1581 HZ |
176 | return 0; |
177 | ||
178 | memset(&xdp, 0, sizeof(xdp)); | |
179 | ||
184367dc HZ |
180 | if (prog) |
181 | bpf_prog_inc(prog); | |
182 | ||
351e1581 HZ |
183 | xdp.command = XDP_SETUP_PROG; |
184 | xdp.prog = prog; | |
185 | ||
e416531f | 186 | ret = vf_netdev->netdev_ops->ndo_bpf(vf_netdev, &xdp); |
184367dc HZ |
187 | |
188 | if (ret && prog) | |
189 | bpf_prog_put(prog); | |
190 | ||
191 | return ret; | |
351e1581 HZ |
192 | } |
193 | ||
351e1581 HZ |
194 | int netvsc_bpf(struct net_device *dev, struct netdev_bpf *bpf) |
195 | { | |
196 | struct net_device_context *ndevctx = netdev_priv(dev); | |
197 | struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev); | |
198 | struct net_device *vf_netdev = rtnl_dereference(ndevctx->vf_netdev); | |
199 | struct netlink_ext_ack *extack = bpf->extack; | |
200 | int ret; | |
201 | ||
202 | if (!nvdev || nvdev->destroy) { | |
e8407fde | 203 | return -ENODEV; |
351e1581 HZ |
204 | } |
205 | ||
206 | switch (bpf->command) { | |
207 | case XDP_SETUP_PROG: | |
208 | ret = netvsc_xdp_set(dev, bpf->prog, extack, nvdev); | |
209 | ||
210 | if (ret) | |
211 | return ret; | |
212 | ||
213 | ret = netvsc_vf_setxdp(vf_netdev, bpf->prog); | |
214 | ||
215 | if (ret) { | |
216 | netdev_err(dev, "vf_setxdp failed:%d\n", ret); | |
217 | NL_SET_ERR_MSG_MOD(extack, "vf_setxdp failed"); | |
218 | ||
219 | netvsc_xdp_set(dev, NULL, extack, nvdev); | |
220 | } | |
221 | ||
222 | return ret; | |
223 | ||
351e1581 HZ |
224 | default: |
225 | return -EINVAL; | |
226 | } | |
227 | } | |
1cb9d3b6 HZ |
228 | |
229 | static int netvsc_ndoxdp_xmit_fm(struct net_device *ndev, | |
230 | struct xdp_frame *frame, u16 q_idx) | |
231 | { | |
232 | struct sk_buff *skb; | |
233 | ||
234 | skb = xdp_build_skb_from_frame(frame, ndev); | |
235 | if (unlikely(!skb)) | |
236 | return -ENOMEM; | |
237 | ||
238 | netvsc_get_hash(skb, netdev_priv(ndev)); | |
239 | ||
240 | skb_record_rx_queue(skb, q_idx); | |
241 | ||
242 | netvsc_xdp_xmit(skb, ndev); | |
243 | ||
244 | return 0; | |
245 | } | |
246 | ||
247 | int netvsc_ndoxdp_xmit(struct net_device *ndev, int n, | |
248 | struct xdp_frame **frames, u32 flags) | |
249 | { | |
250 | struct net_device_context *ndev_ctx = netdev_priv(ndev); | |
251 | const struct net_device_ops *vf_ops; | |
252 | struct netvsc_stats_tx *tx_stats; | |
253 | struct netvsc_device *nvsc_dev; | |
254 | struct net_device *vf_netdev; | |
255 | int i, count = 0; | |
256 | u16 q_idx; | |
257 | ||
258 | /* Don't transmit if netvsc_device is gone */ | |
259 | nvsc_dev = rcu_dereference_bh(ndev_ctx->nvdev); | |
260 | if (unlikely(!nvsc_dev || nvsc_dev->destroy)) | |
261 | return 0; | |
262 | ||
263 | /* If VF is present and up then redirect packets to it. | |
264 | * Skip the VF if it is marked down or has no carrier. | |
265 | * If netpoll is in uses, then VF can not be used either. | |
266 | */ | |
267 | vf_netdev = rcu_dereference_bh(ndev_ctx->vf_netdev); | |
268 | if (vf_netdev && netif_running(vf_netdev) && | |
269 | netif_carrier_ok(vf_netdev) && !netpoll_tx_running(ndev) && | |
270 | vf_netdev->netdev_ops->ndo_xdp_xmit && | |
271 | ndev_ctx->data_path_is_vf) { | |
272 | vf_ops = vf_netdev->netdev_ops; | |
273 | return vf_ops->ndo_xdp_xmit(vf_netdev, n, frames, flags); | |
274 | } | |
275 | ||
276 | q_idx = smp_processor_id() % ndev->real_num_tx_queues; | |
277 | ||
278 | for (i = 0; i < n; i++) { | |
279 | if (netvsc_ndoxdp_xmit_fm(ndev, frames[i], q_idx)) | |
280 | break; | |
281 | ||
282 | count++; | |
283 | } | |
284 | ||
285 | tx_stats = &nvsc_dev->chan_table[q_idx].tx_stats; | |
286 | ||
287 | u64_stats_update_begin(&tx_stats->syncp); | |
288 | tx_stats->xdp_xmit += count; | |
289 | u64_stats_update_end(&tx_stats->syncp); | |
290 | ||
291 | return count; | |
292 | } |