Commit | Line | Data |
---|---|---|
dac09149 BT |
1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* AF_XDP internal functions | |
c0c77d8f | 3 | * Copyright(c) 2018 Intel Corporation. |
c0c77d8f BT |
4 | */ |
5 | ||
6 | #ifndef _LINUX_XDP_SOCK_H | |
7 | #define _LINUX_XDP_SOCK_H | |
8 | ||
b6459415 | 9 | #include <linux/bpf.h> |
e61e62b9 BT |
10 | #include <linux/workqueue.h> |
11 | #include <linux/if_xdp.h> | |
c0c77d8f | 12 | #include <linux/mutex.h> |
ac98d8aa | 13 | #include <linux/spinlock.h> |
e61e62b9 | 14 | #include <linux/mm.h> |
c0c77d8f BT |
15 | #include <net/sock.h> |
16 | ||
d609f3d2 TS |
17 | #define XDP_UMEM_SG_FLAG (1 << 1) |
18 | ||
b9b6b68e BT |
19 | struct net_device; |
20 | struct xsk_queue; | |
a71506a4 | 21 | struct xdp_buff; |
e61e62b9 | 22 | |
e61e62b9 | 23 | struct xdp_umem { |
7f7ffa4e | 24 | void *addrs; |
93ee30f3 | 25 | u64 size; |
e61e62b9 | 26 | u32 headroom; |
2b43470a | 27 | u32 chunk_size; |
1c1efc2a | 28 | u32 chunks; |
8ef4e27e | 29 | u32 npgs; |
e61e62b9 | 30 | struct user_struct *user; |
e61e62b9 | 31 | refcount_t users; |
77cd0d7b | 32 | u8 flags; |
341ac980 | 33 | u8 tx_metadata_len; |
173d3adb | 34 | bool zc; |
8ef4e27e MK |
35 | struct page **pgs; |
36 | int id; | |
921b6869 | 37 | struct list_head xsk_dma_list; |
537cf4e3 | 38 | struct work_struct work; |
e61e62b9 | 39 | }; |
c0c77d8f | 40 | |
d817991c BT |
41 | struct xsk_map { |
42 | struct bpf_map map; | |
d817991c | 43 | spinlock_t lock; /* Synchronize map updates */ |
b4fd0d67 | 44 | atomic_t count; |
782347b6 | 45 | struct xdp_sock __rcu *xsk_map[]; |
d817991c BT |
46 | }; |
47 | ||
c0c77d8f BT |
48 | struct xdp_sock { |
49 | /* struct sock must be the first member of struct xdp_sock */ | |
50 | struct sock sk; | |
8ef4e27e | 51 | struct xsk_queue *rx ____cacheline_aligned_in_smp; |
b9b6b68e | 52 | struct net_device *dev; |
c0c77d8f | 53 | struct xdp_umem *umem; |
fbfc504a | 54 | struct list_head flush_node; |
c4655761 | 55 | struct xsk_buff_pool *pool; |
965a9909 | 56 | u16 queue_id; |
ac98d8aa | 57 | bool zc; |
81470b5c | 58 | bool sg; |
455302d1 IM |
59 | enum { |
60 | XSK_READY = 0, | |
61 | XSK_BOUND, | |
62 | XSK_UNBOUND, | |
63 | } state; | |
8ef4e27e | 64 | |
fada7fdc | 65 | struct xsk_queue *tx ____cacheline_aligned_in_smp; |
a5aa8e52 | 66 | struct list_head tx_list; |
99b29a49 AH |
67 | /* record the number of tx descriptors sent by this xsk and |
68 | * when it exceeds MAX_PER_SOCKET_BUDGET, an opportunity needs | |
69 | * to be given to other xsks for sending tx descriptors, thereby | |
70 | * preventing other XSKs from being starved. | |
71 | */ | |
72 | u32 tx_budget_spent; | |
73 | ||
bf0bdd13 IM |
74 | /* Protects generic receive. */ |
75 | spinlock_t rx_lock; | |
8aa5a335 CL |
76 | |
77 | /* Statistics */ | |
c497176c | 78 | u64 rx_dropped; |
8aa5a335 CL |
79 | u64 rx_queue_full; |
80 | ||
b7f72a30 TS |
81 | /* When __xsk_generic_xmit() must return before it sees the EOP descriptor for the current |
82 | * packet, the partially built skb is saved here so that packet building can resume in next | |
83 | * call of __xsk_generic_xmit(). | |
84 | */ | |
85 | struct sk_buff *skb; | |
86 | ||
0402acd6 BT |
87 | struct list_head map_list; |
88 | /* Protects map_list */ | |
89 | spinlock_t map_list_lock; | |
8ef4e27e MK |
90 | /* Protects multiple processes in the control path */ |
91 | struct mutex mutex; | |
7361f9c3 MK |
92 | struct xsk_queue *fq_tmp; /* Only as tmp storage before bind */ |
93 | struct xsk_queue *cq_tmp; /* Only as tmp storage before bind */ | |
c0c77d8f BT |
94 | }; |
95 | ||
48eb03dd SF |
96 | /* |
97 | * AF_XDP TX metadata hooks for network devices. | |
98 | * The following hooks can be defined; unless noted otherwise, they are | |
99 | * optional and can be filled with a null pointer. | |
100 | * | |
101 | * void (*tmo_request_timestamp)(void *priv) | |
102 | * Called when AF_XDP frame requested egress timestamp. | |
103 | * | |
104 | * u64 (*tmo_fill_timestamp)(void *priv) | |
105 | * Called when AF_XDP frame, that had requested egress timestamp, | |
106 | * received a completion. The hook needs to return the actual HW timestamp. | |
107 | * | |
108 | * void (*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv) | |
109 | * Called when AF_XDP frame requested HW checksum offload. csum_start | |
110 | * indicates position where checksumming should start. | |
111 | * csum_offset indicates position where checksum should be stored. | |
112 | * | |
113 | */ | |
114 | struct xsk_tx_metadata_ops { | |
115 | void (*tmo_request_timestamp)(void *priv); | |
116 | u64 (*tmo_fill_timestamp)(void *priv); | |
117 | void (*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv); | |
118 | }; | |
119 | ||
c497176c | 120 | #ifdef CONFIG_XDP_SOCKETS |
90254034 | 121 | |
a71506a4 | 122 | int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp); |
e312b9e7 BT |
123 | int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp); |
124 | void __xsk_map_flush(void); | |
d817991c | 125 | |
48eb03dd SF |
126 | /** |
127 | * xsk_tx_metadata_to_compl - Save enough relevant metadata information | |
128 | * to perform tx completion in the future. | |
129 | * @meta: pointer to AF_XDP metadata area | |
130 | * @compl: pointer to output struct xsk_tx_metadata_to_compl | |
131 | * | |
132 | * This function should be called by the networking device when | |
133 | * it prepares AF_XDP egress packet. The value of @compl should be stored | |
134 | * and passed to xsk_tx_metadata_complete upon TX completion. | |
135 | */ | |
136 | static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta, | |
137 | struct xsk_tx_metadata_compl *compl) | |
138 | { | |
139 | if (!meta) | |
140 | return; | |
141 | ||
142 | if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP) | |
143 | compl->tx_timestamp = &meta->completion.tx_timestamp; | |
144 | else | |
145 | compl->tx_timestamp = NULL; | |
146 | } | |
147 | ||
148 | /** | |
149 | * xsk_tx_metadata_request - Evaluate AF_XDP TX metadata at submission | |
150 | * and call appropriate xsk_tx_metadata_ops operation. | |
151 | * @meta: pointer to AF_XDP metadata area | |
152 | * @ops: pointer to struct xsk_tx_metadata_ops | |
153 | * @priv: pointer to driver-private aread | |
154 | * | |
155 | * This function should be called by the networking device when | |
156 | * it prepares AF_XDP egress packet. | |
157 | */ | |
158 | static inline void xsk_tx_metadata_request(const struct xsk_tx_metadata *meta, | |
159 | const struct xsk_tx_metadata_ops *ops, | |
160 | void *priv) | |
161 | { | |
162 | if (!meta) | |
163 | return; | |
164 | ||
165 | if (ops->tmo_request_timestamp) | |
166 | if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP) | |
167 | ops->tmo_request_timestamp(priv); | |
168 | ||
169 | if (ops->tmo_request_checksum) | |
170 | if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM) | |
171 | ops->tmo_request_checksum(meta->request.csum_start, | |
172 | meta->request.csum_offset, priv); | |
173 | } | |
174 | ||
175 | /** | |
176 | * xsk_tx_metadata_complete - Evaluate AF_XDP TX metadata at completion | |
177 | * and call appropriate xsk_tx_metadata_ops operation. | |
178 | * @compl: pointer to completion metadata produced from xsk_tx_metadata_to_compl | |
179 | * @ops: pointer to struct xsk_tx_metadata_ops | |
180 | * @priv: pointer to driver-private aread | |
181 | * | |
182 | * This function should be called by the networking device upon | |
183 | * AF_XDP egress completion. | |
184 | */ | |
185 | static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl, | |
186 | const struct xsk_tx_metadata_ops *ops, | |
187 | void *priv) | |
188 | { | |
189 | if (!compl) | |
190 | return; | |
191 | ||
192 | *compl->tx_timestamp = ops->tmo_fill_timestamp(priv); | |
193 | } | |
194 | ||
c497176c | 195 | #else |
a71506a4 | 196 | |
c497176c BT |
197 | static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) |
198 | { | |
199 | return -ENOTSUPP; | |
200 | } | |
201 | ||
a71506a4 | 202 | static inline int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp) |
f5bd9138 | 203 | { |
a71506a4 | 204 | return -EOPNOTSUPP; |
f5bd9138 JK |
205 | } |
206 | ||
a71506a4 | 207 | static inline void __xsk_map_flush(void) |
f5bd9138 JK |
208 | { |
209 | } | |
210 | ||
48eb03dd SF |
211 | static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta, |
212 | struct xsk_tx_metadata_compl *compl) | |
213 | { | |
214 | } | |
215 | ||
216 | static inline void xsk_tx_metadata_request(struct xsk_tx_metadata *meta, | |
217 | const struct xsk_tx_metadata_ops *ops, | |
218 | void *priv) | |
219 | { | |
220 | } | |
221 | ||
222 | static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl, | |
223 | const struct xsk_tx_metadata_ops *ops, | |
224 | void *priv) | |
225 | { | |
226 | } | |
227 | ||
c497176c BT |
228 | #endif /* CONFIG_XDP_SOCKETS */ |
229 | ||
9a675ba5 SAS |
230 | #if defined(CONFIG_XDP_SOCKETS) && defined(CONFIG_DEBUG_NET) |
231 | bool xsk_map_check_flush(void); | |
232 | #else | |
233 | static inline bool xsk_map_check_flush(void) | |
234 | { | |
235 | return false; | |
236 | } | |
237 | #endif | |
238 | ||
c0c77d8f | 239 | #endif /* _LINUX_XDP_SOCK_H */ |