Commit | Line | Data |
---|---|---|
a71506a4 MK |
1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* Interface for implementing AF_XDP zero-copy support in drivers. | |
3 | * Copyright(c) 2020 Intel Corporation. | |
4 | */ | |
5 | ||
6 | #ifndef _LINUX_XDP_SOCK_DRV_H | |
7 | #define _LINUX_XDP_SOCK_DRV_H | |
8 | ||
9 | #include <net/xdp_sock.h> | |
2b43470a | 10 | #include <net/xsk_buff_pool.h> |
a71506a4 | 11 | |
9ca66afe MM |
12 | #define XDP_UMEM_MIN_CHUNK_SHIFT 11 |
13 | #define XDP_UMEM_MIN_CHUNK_SIZE (1 << XDP_UMEM_MIN_CHUNK_SHIFT) | |
14 | ||
a71506a4 MK |
15 | #ifdef CONFIG_XDP_SOCKETS |
16 | ||
c4655761 MK |
17 | void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries); |
18 | bool xsk_tx_peek_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc); | |
d1bc532e | 19 | u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max); |
c4655761 MK |
20 | void xsk_tx_release(struct xsk_buff_pool *pool); |
21 | struct xsk_buff_pool *xsk_get_pool_from_qid(struct net_device *dev, | |
22 | u16 queue_id); | |
23 | void xsk_set_rx_need_wakeup(struct xsk_buff_pool *pool); | |
24 | void xsk_set_tx_need_wakeup(struct xsk_buff_pool *pool); | |
25 | void xsk_clear_rx_need_wakeup(struct xsk_buff_pool *pool); | |
26 | void xsk_clear_tx_need_wakeup(struct xsk_buff_pool *pool); | |
27 | bool xsk_uses_need_wakeup(struct xsk_buff_pool *pool); | |
a71506a4 | 28 | |
c4655761 | 29 | static inline u32 xsk_pool_get_headroom(struct xsk_buff_pool *pool) |
2b43470a | 30 | { |
c4655761 | 31 | return XDP_PACKET_HEADROOM + pool->headroom; |
2b43470a BT |
32 | } |
33 | ||
c4655761 | 34 | static inline u32 xsk_pool_get_chunk_size(struct xsk_buff_pool *pool) |
2b43470a | 35 | { |
c4655761 | 36 | return pool->chunk_size; |
2b43470a BT |
37 | } |
38 | ||
c4655761 | 39 | static inline u32 xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool) |
2b43470a | 40 | { |
c4655761 | 41 | return xsk_pool_get_chunk_size(pool) - xsk_pool_get_headroom(pool); |
2b43470a BT |
42 | } |
43 | ||
c4655761 | 44 | static inline void xsk_pool_set_rxq_info(struct xsk_buff_pool *pool, |
2b43470a BT |
45 | struct xdp_rxq_info *rxq) |
46 | { | |
c4655761 | 47 | xp_set_rxq_info(pool, rxq); |
2b43470a BT |
48 | } |
49 | ||
ca2e1a62 MF |
50 | static inline unsigned int xsk_pool_get_napi_id(struct xsk_buff_pool *pool) |
51 | { | |
52 | #ifdef CONFIG_NET_RX_BUSY_POLL | |
53 | return pool->heads[0].xdp.rxq->napi_id; | |
54 | #else | |
55 | return 0; | |
56 | #endif | |
57 | } | |
58 | ||
c4655761 | 59 | static inline void xsk_pool_dma_unmap(struct xsk_buff_pool *pool, |
2b43470a BT |
60 | unsigned long attrs) |
61 | { | |
c4655761 | 62 | xp_dma_unmap(pool, attrs); |
2b43470a BT |
63 | } |
64 | ||
c4655761 MK |
65 | static inline int xsk_pool_dma_map(struct xsk_buff_pool *pool, |
66 | struct device *dev, unsigned long attrs) | |
2b43470a | 67 | { |
c4655761 MK |
68 | struct xdp_umem *umem = pool->umem; |
69 | ||
70 | return xp_dma_map(pool, dev, attrs, umem->pgs, umem->npgs); | |
2b43470a BT |
71 | } |
72 | ||
73 | static inline dma_addr_t xsk_buff_xdp_get_dma(struct xdp_buff *xdp) | |
74 | { | |
75 | struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); | |
76 | ||
77 | return xp_get_dma(xskb); | |
78 | } | |
79 | ||
80 | static inline dma_addr_t xsk_buff_xdp_get_frame_dma(struct xdp_buff *xdp) | |
81 | { | |
82 | struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); | |
83 | ||
84 | return xp_get_frame_dma(xskb); | |
85 | } | |
86 | ||
c4655761 | 87 | static inline struct xdp_buff *xsk_buff_alloc(struct xsk_buff_pool *pool) |
2b43470a | 88 | { |
c4655761 | 89 | return xp_alloc(pool); |
2b43470a BT |
90 | } |
91 | ||
47e4075d MK |
92 | /* Returns as many entries as possible up to max. 0 <= N <= max. */ |
93 | static inline u32 xsk_buff_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max) | |
94 | { | |
95 | return xp_alloc_batch(pool, xdp, max); | |
96 | } | |
97 | ||
c4655761 | 98 | static inline bool xsk_buff_can_alloc(struct xsk_buff_pool *pool, u32 count) |
2b43470a | 99 | { |
c4655761 | 100 | return xp_can_alloc(pool, count); |
2b43470a BT |
101 | } |
102 | ||
103 | static inline void xsk_buff_free(struct xdp_buff *xdp) | |
104 | { | |
105 | struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); | |
106 | ||
107 | xp_free(xskb); | |
108 | } | |
109 | ||
47e4075d MK |
110 | static inline void xsk_buff_set_size(struct xdp_buff *xdp, u32 size) |
111 | { | |
112 | xdp->data = xdp->data_hard_start + XDP_PACKET_HEADROOM; | |
113 | xdp->data_meta = xdp->data; | |
114 | xdp->data_end = xdp->data + size; | |
115 | } | |
116 | ||
c4655761 MK |
117 | static inline dma_addr_t xsk_buff_raw_get_dma(struct xsk_buff_pool *pool, |
118 | u64 addr) | |
2b43470a | 119 | { |
c4655761 | 120 | return xp_raw_get_dma(pool, addr); |
2b43470a BT |
121 | } |
122 | ||
c4655761 | 123 | static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr) |
2b43470a | 124 | { |
c4655761 | 125 | return xp_raw_get_data(pool, addr); |
2b43470a BT |
126 | } |
127 | ||
9647c57b | 128 | static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp, struct xsk_buff_pool *pool) |
2b43470a BT |
129 | { |
130 | struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); | |
131 | ||
9647c57b MK |
132 | if (!pool->dma_need_sync) |
133 | return; | |
134 | ||
2b43470a BT |
135 | xp_dma_sync_for_cpu(xskb); |
136 | } | |
137 | ||
c4655761 | 138 | static inline void xsk_buff_raw_dma_sync_for_device(struct xsk_buff_pool *pool, |
2b43470a BT |
139 | dma_addr_t dma, |
140 | size_t size) | |
141 | { | |
c4655761 | 142 | xp_dma_sync_for_device(pool, dma, size); |
2b43470a BT |
143 | } |
144 | ||
a71506a4 MK |
145 | #else |
146 | ||
c4655761 | 147 | static inline void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries) |
a71506a4 MK |
148 | { |
149 | } | |
150 | ||
c4655761 MK |
151 | static inline bool xsk_tx_peek_desc(struct xsk_buff_pool *pool, |
152 | struct xdp_desc *desc) | |
a71506a4 MK |
153 | { |
154 | return false; | |
155 | } | |
156 | ||
d1bc532e | 157 | static inline u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max) |
9349eb3a MK |
158 | { |
159 | return 0; | |
160 | } | |
161 | ||
c4655761 | 162 | static inline void xsk_tx_release(struct xsk_buff_pool *pool) |
a71506a4 MK |
163 | { |
164 | } | |
165 | ||
1742b3d5 | 166 | static inline struct xsk_buff_pool * |
c4655761 | 167 | xsk_get_pool_from_qid(struct net_device *dev, u16 queue_id) |
a71506a4 MK |
168 | { |
169 | return NULL; | |
170 | } | |
171 | ||
c4655761 | 172 | static inline void xsk_set_rx_need_wakeup(struct xsk_buff_pool *pool) |
a71506a4 MK |
173 | { |
174 | } | |
175 | ||
c4655761 | 176 | static inline void xsk_set_tx_need_wakeup(struct xsk_buff_pool *pool) |
a71506a4 MK |
177 | { |
178 | } | |
179 | ||
c4655761 | 180 | static inline void xsk_clear_rx_need_wakeup(struct xsk_buff_pool *pool) |
a71506a4 MK |
181 | { |
182 | } | |
183 | ||
c4655761 | 184 | static inline void xsk_clear_tx_need_wakeup(struct xsk_buff_pool *pool) |
a71506a4 MK |
185 | { |
186 | } | |
187 | ||
c4655761 | 188 | static inline bool xsk_uses_need_wakeup(struct xsk_buff_pool *pool) |
a71506a4 MK |
189 | { |
190 | return false; | |
191 | } | |
192 | ||
c4655761 | 193 | static inline u32 xsk_pool_get_headroom(struct xsk_buff_pool *pool) |
2b43470a BT |
194 | { |
195 | return 0; | |
196 | } | |
197 | ||
c4655761 | 198 | static inline u32 xsk_pool_get_chunk_size(struct xsk_buff_pool *pool) |
2b43470a BT |
199 | { |
200 | return 0; | |
201 | } | |
202 | ||
c4655761 | 203 | static inline u32 xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool) |
2b43470a BT |
204 | { |
205 | return 0; | |
206 | } | |
207 | ||
c4655761 | 208 | static inline void xsk_pool_set_rxq_info(struct xsk_buff_pool *pool, |
2b43470a BT |
209 | struct xdp_rxq_info *rxq) |
210 | { | |
211 | } | |
212 | ||
ca2e1a62 MF |
213 | static inline unsigned int xsk_pool_get_napi_id(struct xsk_buff_pool *pool) |
214 | { | |
215 | return 0; | |
216 | } | |
217 | ||
c4655761 | 218 | static inline void xsk_pool_dma_unmap(struct xsk_buff_pool *pool, |
2b43470a BT |
219 | unsigned long attrs) |
220 | { | |
221 | } | |
222 | ||
c4655761 MK |
223 | static inline int xsk_pool_dma_map(struct xsk_buff_pool *pool, |
224 | struct device *dev, unsigned long attrs) | |
2b43470a BT |
225 | { |
226 | return 0; | |
227 | } | |
228 | ||
229 | static inline dma_addr_t xsk_buff_xdp_get_dma(struct xdp_buff *xdp) | |
230 | { | |
231 | return 0; | |
232 | } | |
233 | ||
234 | static inline dma_addr_t xsk_buff_xdp_get_frame_dma(struct xdp_buff *xdp) | |
235 | { | |
236 | return 0; | |
237 | } | |
238 | ||
c4655761 | 239 | static inline struct xdp_buff *xsk_buff_alloc(struct xsk_buff_pool *pool) |
2b43470a BT |
240 | { |
241 | return NULL; | |
242 | } | |
243 | ||
47e4075d MK |
244 | static inline u32 xsk_buff_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max) |
245 | { | |
246 | return 0; | |
247 | } | |
248 | ||
c4655761 | 249 | static inline bool xsk_buff_can_alloc(struct xsk_buff_pool *pool, u32 count) |
2b43470a BT |
250 | { |
251 | return false; | |
252 | } | |
253 | ||
254 | static inline void xsk_buff_free(struct xdp_buff *xdp) | |
255 | { | |
256 | } | |
257 | ||
8eaa1d11 MM |
258 | static inline void xsk_buff_discard(struct xdp_buff *xdp) |
259 | { | |
260 | } | |
261 | ||
47e4075d MK |
262 | static inline void xsk_buff_set_size(struct xdp_buff *xdp, u32 size) |
263 | { | |
264 | } | |
265 | ||
c4655761 MK |
266 | static inline dma_addr_t xsk_buff_raw_get_dma(struct xsk_buff_pool *pool, |
267 | u64 addr) | |
2b43470a BT |
268 | { |
269 | return 0; | |
270 | } | |
271 | ||
c4655761 | 272 | static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr) |
2b43470a BT |
273 | { |
274 | return NULL; | |
275 | } | |
276 | ||
9647c57b | 277 | static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp, struct xsk_buff_pool *pool) |
2b43470a BT |
278 | { |
279 | } | |
280 | ||
c4655761 | 281 | static inline void xsk_buff_raw_dma_sync_for_device(struct xsk_buff_pool *pool, |
2b43470a BT |
282 | dma_addr_t dma, |
283 | size_t size) | |
284 | { | |
285 | } | |
286 | ||
a71506a4 MK |
287 | #endif /* CONFIG_XDP_SOCKETS */ |
288 | ||
289 | #endif /* _LINUX_XDP_SOCK_DRV_H */ |