Commit | Line | Data |
---|---|---|
c0c77d8f BT |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* XDP user-space packet buffer | |
3 | * Copyright(c) 2018 Intel Corporation. | |
c0c77d8f BT |
4 | */ |
5 | ||
6 | #include <linux/init.h> | |
7 | #include <linux/sched/mm.h> | |
8 | #include <linux/sched/signal.h> | |
9 | #include <linux/sched/task.h> | |
10 | #include <linux/uaccess.h> | |
11 | #include <linux/slab.h> | |
12 | #include <linux/bpf.h> | |
13 | #include <linux/mm.h> | |
84c6b868 JK |
14 | #include <linux/netdevice.h> |
15 | #include <linux/rtnetlink.h> | |
50e74c01 | 16 | #include <linux/idr.h> |
c0c77d8f BT |
17 | |
18 | #include "xdp_umem.h" | |
e61e62b9 | 19 | #include "xsk_queue.h" |
c0c77d8f | 20 | |
bbff2f32 | 21 | #define XDP_UMEM_MIN_CHUNK_SIZE 2048 |
c0c77d8f | 22 | |
50e74c01 BT |
23 | static DEFINE_IDA(umem_ida); |
24 | ||
ac98d8aa MK |
25 | void xdp_add_sk_umem(struct xdp_umem *umem, struct xdp_sock *xs) |
26 | { | |
27 | unsigned long flags; | |
28 | ||
29 | spin_lock_irqsave(&umem->xsk_list_lock, flags); | |
30 | list_add_rcu(&xs->list, &umem->xsk_list); | |
31 | spin_unlock_irqrestore(&umem->xsk_list_lock, flags); | |
32 | } | |
33 | ||
34 | void xdp_del_sk_umem(struct xdp_umem *umem, struct xdp_sock *xs) | |
35 | { | |
36 | unsigned long flags; | |
37 | ||
541d7fdd BT |
38 | spin_lock_irqsave(&umem->xsk_list_lock, flags); |
39 | list_del_rcu(&xs->list); | |
40 | spin_unlock_irqrestore(&umem->xsk_list_lock, flags); | |
ac98d8aa MK |
41 | } |
42 | ||
c9b47cc1 MK |
43 | /* The umem is stored both in the _rx struct and the _tx struct as we do |
44 | * not know if the device has more tx queues than rx, or the opposite. | |
45 | * This might also change during run time. | |
46 | */ | |
cc5b5d35 KK |
47 | static int xdp_reg_umem_at_qid(struct net_device *dev, struct xdp_umem *umem, |
48 | u16 queue_id) | |
84c6b868 | 49 | { |
cc5b5d35 KK |
50 | if (queue_id >= max_t(unsigned int, |
51 | dev->real_num_rx_queues, | |
52 | dev->real_num_tx_queues)) | |
53 | return -EINVAL; | |
54 | ||
c9b47cc1 MK |
55 | if (queue_id < dev->real_num_rx_queues) |
56 | dev->_rx[queue_id].umem = umem; | |
57 | if (queue_id < dev->real_num_tx_queues) | |
58 | dev->_tx[queue_id].umem = umem; | |
cc5b5d35 KK |
59 | |
60 | return 0; | |
c9b47cc1 | 61 | } |
84c6b868 | 62 | |
1661d346 JK |
63 | struct xdp_umem *xdp_get_umem_from_qid(struct net_device *dev, |
64 | u16 queue_id) | |
c9b47cc1 MK |
65 | { |
66 | if (queue_id < dev->real_num_rx_queues) | |
67 | return dev->_rx[queue_id].umem; | |
68 | if (queue_id < dev->real_num_tx_queues) | |
69 | return dev->_tx[queue_id].umem; | |
84c6b868 | 70 | |
c9b47cc1 MK |
71 | return NULL; |
72 | } | |
5f4f3b2d | 73 | EXPORT_SYMBOL(xdp_get_umem_from_qid); |
84c6b868 | 74 | |
c9b47cc1 MK |
75 | static void xdp_clear_umem_at_qid(struct net_device *dev, u16 queue_id) |
76 | { | |
a41b4f3c | 77 | if (queue_id < dev->real_num_rx_queues) |
c9b47cc1 | 78 | dev->_rx[queue_id].umem = NULL; |
a41b4f3c | 79 | if (queue_id < dev->real_num_tx_queues) |
c9b47cc1 | 80 | dev->_tx[queue_id].umem = NULL; |
84c6b868 JK |
81 | } |
82 | ||
173d3adb | 83 | int xdp_umem_assign_dev(struct xdp_umem *umem, struct net_device *dev, |
c9b47cc1 | 84 | u16 queue_id, u16 flags) |
173d3adb BT |
85 | { |
86 | bool force_zc, force_copy; | |
87 | struct netdev_bpf bpf; | |
c9b47cc1 | 88 | int err = 0; |
173d3adb | 89 | |
5464c3a0 IM |
90 | ASSERT_RTNL(); |
91 | ||
173d3adb BT |
92 | force_zc = flags & XDP_ZEROCOPY; |
93 | force_copy = flags & XDP_COPY; | |
94 | ||
95 | if (force_zc && force_copy) | |
96 | return -EINVAL; | |
97 | ||
5464c3a0 IM |
98 | if (xdp_get_umem_from_qid(dev, queue_id)) |
99 | return -EBUSY; | |
173d3adb | 100 | |
cc5b5d35 KK |
101 | err = xdp_reg_umem_at_qid(dev, umem, queue_id); |
102 | if (err) | |
5464c3a0 | 103 | return err; |
cc5b5d35 | 104 | |
c9b47cc1 MK |
105 | umem->dev = dev; |
106 | umem->queue_id = queue_id; | |
162c820e IM |
107 | |
108 | dev_hold(dev); | |
109 | ||
c9b47cc1 MK |
110 | if (force_copy) |
111 | /* For copy-mode, we are done. */ | |
5464c3a0 | 112 | return 0; |
173d3adb | 113 | |
c9b47cc1 MK |
114 | if (!dev->netdev_ops->ndo_bpf || |
115 | !dev->netdev_ops->ndo_xsk_async_xmit) { | |
116 | err = -EOPNOTSUPP; | |
117 | goto err_unreg_umem; | |
84c6b868 | 118 | } |
173d3adb | 119 | |
f734607e JK |
120 | bpf.command = XDP_SETUP_XSK_UMEM; |
121 | bpf.xsk.umem = umem; | |
122 | bpf.xsk.queue_id = queue_id; | |
173d3adb | 123 | |
f734607e | 124 | err = dev->netdev_ops->ndo_bpf(dev, &bpf); |
f734607e | 125 | if (err) |
c9b47cc1 | 126 | goto err_unreg_umem; |
173d3adb | 127 | |
f734607e JK |
128 | umem->zc = true; |
129 | return 0; | |
84c6b868 | 130 | |
c9b47cc1 | 131 | err_unreg_umem: |
c9b47cc1 MK |
132 | if (!force_zc) |
133 | err = 0; /* fallback to copy mode */ | |
1e405c1a BT |
134 | if (err) |
135 | xdp_clear_umem_at_qid(dev, queue_id); | |
c9b47cc1 | 136 | return err; |
173d3adb BT |
137 | } |
138 | ||
455302d1 | 139 | void xdp_umem_clear_dev(struct xdp_umem *umem) |
173d3adb BT |
140 | { |
141 | struct netdev_bpf bpf; | |
142 | int err; | |
143 | ||
455302d1 IM |
144 | ASSERT_RTNL(); |
145 | ||
01d76b53 IM |
146 | if (!umem->dev) |
147 | return; | |
148 | ||
c9b47cc1 | 149 | if (umem->zc) { |
173d3adb BT |
150 | bpf.command = XDP_SETUP_XSK_UMEM; |
151 | bpf.xsk.umem = NULL; | |
152 | bpf.xsk.queue_id = umem->queue_id; | |
153 | ||
173d3adb | 154 | err = umem->dev->netdev_ops->ndo_bpf(umem->dev, &bpf); |
173d3adb BT |
155 | |
156 | if (err) | |
157 | WARN(1, "failed to disable umem!\n"); | |
c9b47cc1 MK |
158 | } |
159 | ||
01d76b53 | 160 | xdp_clear_umem_at_qid(umem->dev, umem->queue_id); |
173d3adb | 161 | |
162c820e IM |
162 | dev_put(umem->dev); |
163 | umem->dev = NULL; | |
164 | umem->zc = false; | |
173d3adb BT |
165 | } |
166 | ||
c0c77d8f BT |
167 | static void xdp_umem_unpin_pages(struct xdp_umem *umem) |
168 | { | |
169 | unsigned int i; | |
170 | ||
a49049ea BT |
171 | for (i = 0; i < umem->npgs; i++) { |
172 | struct page *page = umem->pgs[i]; | |
c0c77d8f | 173 | |
a49049ea BT |
174 | set_page_dirty_lock(page); |
175 | put_page(page); | |
c0c77d8f | 176 | } |
a49049ea BT |
177 | |
178 | kfree(umem->pgs); | |
179 | umem->pgs = NULL; | |
c0c77d8f BT |
180 | } |
181 | ||
182 | static void xdp_umem_unaccount_pages(struct xdp_umem *umem) | |
183 | { | |
c09290c5 DB |
184 | if (umem->user) { |
185 | atomic_long_sub(umem->npgs, &umem->user->locked_vm); | |
186 | free_uid(umem->user); | |
187 | } | |
c0c77d8f BT |
188 | } |
189 | ||
190 | static void xdp_umem_release(struct xdp_umem *umem) | |
191 | { | |
455302d1 | 192 | rtnl_lock(); |
173d3adb | 193 | xdp_umem_clear_dev(umem); |
455302d1 | 194 | rtnl_unlock(); |
173d3adb | 195 | |
50e74c01 BT |
196 | ida_simple_remove(&umem_ida, umem->id); |
197 | ||
423f3832 MK |
198 | if (umem->fq) { |
199 | xskq_destroy(umem->fq); | |
200 | umem->fq = NULL; | |
201 | } | |
202 | ||
fe230832 MK |
203 | if (umem->cq) { |
204 | xskq_destroy(umem->cq); | |
205 | umem->cq = NULL; | |
206 | } | |
207 | ||
f5bd9138 JK |
208 | xsk_reuseq_destroy(umem); |
209 | ||
a49049ea | 210 | xdp_umem_unpin_pages(umem); |
c0c77d8f | 211 | |
8aef7340 BT |
212 | kfree(umem->pages); |
213 | umem->pages = NULL; | |
214 | ||
c0c77d8f | 215 | xdp_umem_unaccount_pages(umem); |
c0c77d8f BT |
216 | kfree(umem); |
217 | } | |
218 | ||
219 | static void xdp_umem_release_deferred(struct work_struct *work) | |
220 | { | |
221 | struct xdp_umem *umem = container_of(work, struct xdp_umem, work); | |
222 | ||
223 | xdp_umem_release(umem); | |
224 | } | |
225 | ||
226 | void xdp_get_umem(struct xdp_umem *umem) | |
227 | { | |
d3b42f14 | 228 | refcount_inc(&umem->users); |
c0c77d8f BT |
229 | } |
230 | ||
231 | void xdp_put_umem(struct xdp_umem *umem) | |
232 | { | |
233 | if (!umem) | |
234 | return; | |
235 | ||
d3b42f14 | 236 | if (refcount_dec_and_test(&umem->users)) { |
c0c77d8f BT |
237 | INIT_WORK(&umem->work, xdp_umem_release_deferred); |
238 | schedule_work(&umem->work); | |
239 | } | |
240 | } | |
241 | ||
242 | static int xdp_umem_pin_pages(struct xdp_umem *umem) | |
243 | { | |
244 | unsigned int gup_flags = FOLL_WRITE; | |
245 | long npgs; | |
246 | int err; | |
247 | ||
a343993c BT |
248 | umem->pgs = kcalloc(umem->npgs, sizeof(*umem->pgs), |
249 | GFP_KERNEL | __GFP_NOWARN); | |
c0c77d8f BT |
250 | if (!umem->pgs) |
251 | return -ENOMEM; | |
252 | ||
e451eb51 | 253 | down_read(¤t->mm->mmap_sem); |
932f4a63 IW |
254 | npgs = get_user_pages(umem->address, umem->npgs, |
255 | gup_flags | FOLL_LONGTERM, &umem->pgs[0], NULL); | |
e451eb51 | 256 | up_read(¤t->mm->mmap_sem); |
c0c77d8f BT |
257 | |
258 | if (npgs != umem->npgs) { | |
259 | if (npgs >= 0) { | |
260 | umem->npgs = npgs; | |
261 | err = -ENOMEM; | |
262 | goto out_pin; | |
263 | } | |
264 | err = npgs; | |
265 | goto out_pgs; | |
266 | } | |
267 | return 0; | |
268 | ||
269 | out_pin: | |
270 | xdp_umem_unpin_pages(umem); | |
271 | out_pgs: | |
272 | kfree(umem->pgs); | |
273 | umem->pgs = NULL; | |
274 | return err; | |
275 | } | |
276 | ||
277 | static int xdp_umem_account_pages(struct xdp_umem *umem) | |
278 | { | |
279 | unsigned long lock_limit, new_npgs, old_npgs; | |
280 | ||
281 | if (capable(CAP_IPC_LOCK)) | |
282 | return 0; | |
283 | ||
284 | lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; | |
285 | umem->user = get_uid(current_user()); | |
286 | ||
287 | do { | |
288 | old_npgs = atomic_long_read(&umem->user->locked_vm); | |
289 | new_npgs = old_npgs + umem->npgs; | |
290 | if (new_npgs > lock_limit) { | |
291 | free_uid(umem->user); | |
292 | umem->user = NULL; | |
293 | return -ENOBUFS; | |
294 | } | |
295 | } while (atomic_long_cmpxchg(&umem->user->locked_vm, old_npgs, | |
296 | new_npgs) != old_npgs); | |
297 | return 0; | |
298 | } | |
299 | ||
a49049ea | 300 | static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr) |
c0c77d8f | 301 | { |
bbff2f32 BT |
302 | u32 chunk_size = mr->chunk_size, headroom = mr->headroom; |
303 | unsigned int chunks, chunks_per_page; | |
c0c77d8f | 304 | u64 addr = mr->addr, size = mr->len; |
8aef7340 | 305 | int size_chk, err, i; |
c0c77d8f | 306 | |
bbff2f32 | 307 | if (chunk_size < XDP_UMEM_MIN_CHUNK_SIZE || chunk_size > PAGE_SIZE) { |
c0c77d8f BT |
308 | /* Strictly speaking we could support this, if: |
309 | * - huge pages, or* | |
310 | * - using an IOMMU, or | |
311 | * - making sure the memory area is consecutive | |
312 | * but for now, we simply say "computer says no". | |
313 | */ | |
314 | return -EINVAL; | |
315 | } | |
316 | ||
bbff2f32 | 317 | if (!is_power_of_2(chunk_size)) |
c0c77d8f BT |
318 | return -EINVAL; |
319 | ||
320 | if (!PAGE_ALIGNED(addr)) { | |
321 | /* Memory area has to be page size aligned. For | |
322 | * simplicity, this might change. | |
323 | */ | |
324 | return -EINVAL; | |
325 | } | |
326 | ||
327 | if ((addr + size) < addr) | |
328 | return -EINVAL; | |
329 | ||
bbff2f32 BT |
330 | chunks = (unsigned int)div_u64(size, chunk_size); |
331 | if (chunks == 0) | |
c0c77d8f BT |
332 | return -EINVAL; |
333 | ||
bbff2f32 BT |
334 | chunks_per_page = PAGE_SIZE / chunk_size; |
335 | if (chunks < chunks_per_page || chunks % chunks_per_page) | |
c0c77d8f BT |
336 | return -EINVAL; |
337 | ||
bbff2f32 | 338 | headroom = ALIGN(headroom, 64); |
c0c77d8f | 339 | |
bbff2f32 | 340 | size_chk = chunk_size - headroom - XDP_PACKET_HEADROOM; |
c0c77d8f BT |
341 | if (size_chk < 0) |
342 | return -EINVAL; | |
343 | ||
c0c77d8f | 344 | umem->address = (unsigned long)addr; |
93ee30f3 MK |
345 | umem->chunk_mask = ~((u64)chunk_size - 1); |
346 | umem->size = size; | |
bbff2f32 BT |
347 | umem->headroom = headroom; |
348 | umem->chunk_size_nohr = chunk_size - headroom; | |
c0c77d8f BT |
349 | umem->npgs = size / PAGE_SIZE; |
350 | umem->pgs = NULL; | |
351 | umem->user = NULL; | |
ac98d8aa MK |
352 | INIT_LIST_HEAD(&umem->xsk_list); |
353 | spin_lock_init(&umem->xsk_list_lock); | |
c0c77d8f | 354 | |
d3b42f14 | 355 | refcount_set(&umem->users, 1); |
c0c77d8f BT |
356 | |
357 | err = xdp_umem_account_pages(umem); | |
358 | if (err) | |
044175a0 | 359 | return err; |
c0c77d8f BT |
360 | |
361 | err = xdp_umem_pin_pages(umem); | |
362 | if (err) | |
363 | goto out_account; | |
8aef7340 BT |
364 | |
365 | umem->pages = kcalloc(umem->npgs, sizeof(*umem->pages), GFP_KERNEL); | |
366 | if (!umem->pages) { | |
367 | err = -ENOMEM; | |
fb89c394 | 368 | goto out_pin; |
8aef7340 BT |
369 | } |
370 | ||
371 | for (i = 0; i < umem->npgs; i++) | |
372 | umem->pages[i].addr = page_address(umem->pgs[i]); | |
373 | ||
c0c77d8f BT |
374 | return 0; |
375 | ||
fb89c394 IK |
376 | out_pin: |
377 | xdp_umem_unpin_pages(umem); | |
c0c77d8f BT |
378 | out_account: |
379 | xdp_umem_unaccount_pages(umem); | |
c0c77d8f BT |
380 | return err; |
381 | } | |
965a9909 | 382 | |
a49049ea BT |
383 | struct xdp_umem *xdp_umem_create(struct xdp_umem_reg *mr) |
384 | { | |
385 | struct xdp_umem *umem; | |
386 | int err; | |
387 | ||
388 | umem = kzalloc(sizeof(*umem), GFP_KERNEL); | |
389 | if (!umem) | |
390 | return ERR_PTR(-ENOMEM); | |
391 | ||
50e74c01 BT |
392 | err = ida_simple_get(&umem_ida, 0, 0, GFP_KERNEL); |
393 | if (err < 0) { | |
394 | kfree(umem); | |
395 | return ERR_PTR(err); | |
396 | } | |
397 | umem->id = err; | |
398 | ||
a49049ea BT |
399 | err = xdp_umem_reg(umem, mr); |
400 | if (err) { | |
50e74c01 | 401 | ida_simple_remove(&umem_ida, umem->id); |
a49049ea BT |
402 | kfree(umem); |
403 | return ERR_PTR(err); | |
404 | } | |
405 | ||
406 | return umem; | |
407 | } | |
408 | ||
965a9909 MK |
409 | bool xdp_umem_validate_queues(struct xdp_umem *umem) |
410 | { | |
da60cf00 | 411 | return umem->fq && umem->cq; |
965a9909 | 412 | } |