Commit | Line | Data |
---|---|---|
aecd67b6 JDB |
1 | /* include/net/xdp.h |
2 | * | |
3 | * Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc. | |
4 | * Released under terms in GPL version 2. See COPYING. | |
5 | */ | |
6 | #ifndef __LINUX_NET_XDP_H__ | |
7 | #define __LINUX_NET_XDP_H__ | |
8 | ||
9 | /** | |
10 | * DOC: XDP RX-queue information | |
11 | * | |
12 | * The XDP RX-queue info (xdp_rxq_info) is associated with the driver | |
13 | * level RX-ring queues. It is information that is specific to how | |
14 | * the driver have configured a given RX-ring queue. | |
15 | * | |
16 | * Each xdp_buff frame received in the driver carry a (pointer) | |
17 | * reference to this xdp_rxq_info structure. This provides the XDP | |
18 | * data-path read-access to RX-info for both kernel and bpf-side | |
19 | * (limited subset). | |
20 | * | |
21 | * For now, direct access is only safe while running in NAPI/softirq | |
22 | * context. Contents is read-mostly and must not be updated during | |
23 | * driver NAPI/softirq poll. | |
24 | * | |
25 | * The driver usage API is a register and unregister API. | |
26 | * | |
27 | * The struct is not directly tied to the XDP prog. A new XDP prog | |
28 | * can be attached as long as it doesn't change the underlying | |
29 | * RX-ring. If the RX-ring does change significantly, the NIC driver | |
30 | * naturally need to stop the RX-ring before purging and reallocating | |
31 | * memory. In that process the driver MUST call unregistor (which | |
32 | * also apply for driver shutdown and unload). The register API is | |
33 | * also mandatory during RX-ring setup. | |
34 | */ | |
35 | ||
5ab073ff JDB |
36 | enum xdp_mem_type { |
37 | MEM_TYPE_PAGE_SHARED = 0, /* Split-page refcnt based model */ | |
38 | MEM_TYPE_PAGE_ORDER0, /* Orig XDP full page model */ | |
57d0a1c1 | 39 | MEM_TYPE_PAGE_POOL, |
02b55e56 | 40 | MEM_TYPE_ZERO_COPY, |
5ab073ff JDB |
41 | MEM_TYPE_MAX, |
42 | }; | |
43 | ||
42b33468 | 44 | /* XDP flags for ndo_xdp_xmit */ |
42b33468 JDB |
45 | #define XDP_XMIT_FLUSH (1U << 0) /* doorbell signal consumer */ |
46 | #define XDP_XMIT_FLAGS_MASK XDP_XMIT_FLUSH | |
47 | ||
5ab073ff JDB |
48 | struct xdp_mem_info { |
49 | u32 type; /* enum xdp_mem_type, but known size type */ | |
8d5d8852 | 50 | u32 id; |
5ab073ff JDB |
51 | }; |
52 | ||
57d0a1c1 JDB |
53 | struct page_pool; |
54 | ||
02b55e56 BT |
55 | struct zero_copy_allocator { |
56 | void (*free)(struct zero_copy_allocator *zca, unsigned long handle); | |
57 | }; | |
58 | ||
aecd67b6 JDB |
59 | struct xdp_rxq_info { |
60 | struct net_device *dev; | |
61 | u32 queue_index; | |
62 | u32 reg_state; | |
5ab073ff | 63 | struct xdp_mem_info mem; |
aecd67b6 JDB |
64 | } ____cacheline_aligned; /* perf critical, avoid false-sharing */ |
65 | ||
106ca27f JDB |
66 | struct xdp_buff { |
67 | void *data; | |
68 | void *data_end; | |
69 | void *data_meta; | |
70 | void *data_hard_start; | |
02b55e56 | 71 | unsigned long handle; |
106ca27f JDB |
72 | struct xdp_rxq_info *rxq; |
73 | }; | |
5ab073ff | 74 | |
c0048cff JDB |
75 | struct xdp_frame { |
76 | void *data; | |
77 | u16 len; | |
78 | u16 headroom; | |
79 | u16 metasize; | |
80 | /* Lifetime of xdp_rxq_info is limited to NAPI/enqueue time, | |
81 | * while mem info is valid on remote CPU. | |
82 | */ | |
83 | struct xdp_mem_info mem; | |
70280ed9 | 84 | struct net_device *dev_rx; /* used by cpumap */ |
c0048cff JDB |
85 | }; |
86 | ||
87 | /* Convert xdp_buff to xdp_frame */ | |
88 | static inline | |
89 | struct xdp_frame *convert_to_xdp_frame(struct xdp_buff *xdp) | |
90 | { | |
91 | struct xdp_frame *xdp_frame; | |
92 | int metasize; | |
93 | int headroom; | |
94 | ||
02b55e56 BT |
95 | /* TODO: implement clone, copy, use "native" MEM_TYPE */ |
96 | if (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY) | |
97 | return NULL; | |
98 | ||
c0048cff JDB |
99 | /* Assure headroom is available for storing info */ |
100 | headroom = xdp->data - xdp->data_hard_start; | |
101 | metasize = xdp->data - xdp->data_meta; | |
102 | metasize = metasize > 0 ? metasize : 0; | |
103 | if (unlikely((headroom - metasize) < sizeof(*xdp_frame))) | |
104 | return NULL; | |
105 | ||
106 | /* Store info in top of packet */ | |
107 | xdp_frame = xdp->data_hard_start; | |
108 | ||
109 | xdp_frame->data = xdp->data; | |
110 | xdp_frame->len = xdp->data_end - xdp->data; | |
111 | xdp_frame->headroom = headroom - sizeof(*xdp_frame); | |
112 | xdp_frame->metasize = metasize; | |
113 | ||
114 | /* rxq only valid until napi_schedule ends, convert to xdp_mem_info */ | |
115 | xdp_frame->mem = xdp->rxq->mem; | |
116 | ||
117 | return xdp_frame; | |
118 | } | |
119 | ||
03993094 | 120 | void xdp_return_frame(struct xdp_frame *xdpf); |
389ab7f0 | 121 | void xdp_return_frame_rx_napi(struct xdp_frame *xdpf); |
c497176c | 122 | void xdp_return_buff(struct xdp_buff *xdp); |
5ab073ff | 123 | |
aecd67b6 JDB |
124 | int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq, |
125 | struct net_device *dev, u32 queue_index); | |
126 | void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq); | |
127 | void xdp_rxq_info_unused(struct xdp_rxq_info *xdp_rxq); | |
c0124f32 | 128 | bool xdp_rxq_info_is_reg(struct xdp_rxq_info *xdp_rxq); |
5ab073ff JDB |
129 | int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq, |
130 | enum xdp_mem_type type, void *allocator); | |
aecd67b6 | 131 | |
106ca27f JDB |
132 | /* Drivers not supporting XDP metadata can use this helper, which |
133 | * rejects any room expansion for metadata as a result. | |
134 | */ | |
135 | static __always_inline void | |
136 | xdp_set_data_meta_invalid(struct xdp_buff *xdp) | |
137 | { | |
138 | xdp->data_meta = xdp->data + 1; | |
139 | } | |
140 | ||
141 | static __always_inline bool | |
142 | xdp_data_meta_unsupported(const struct xdp_buff *xdp) | |
143 | { | |
144 | return unlikely(xdp->data_meta > xdp->data); | |
145 | } | |
146 | ||
aecd67b6 | 147 | #endif /* __LINUX_NET_XDP_H__ */ |