1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* AF_XDP internal functions
3 * Copyright(c) 2018 Intel Corporation.
6 #ifndef _LINUX_XDP_SOCK_H
7 #define _LINUX_XDP_SOCK_H
10 #include <linux/workqueue.h>
11 #include <linux/if_xdp.h>
12 #include <linux/mutex.h>
13 #include <linux/spinlock.h>
17 #define XDP_UMEM_SG_FLAG (1 << 1)
30 struct user_struct *user;
37 struct list_head xsk_dma_list;
38 struct work_struct work;
43 spinlock_t lock; /* Synchronize map updates */
45 struct xdp_sock __rcu *xsk_map[];
49 /* struct sock must be the first member of struct xdp_sock */
51 struct xsk_queue *rx ____cacheline_aligned_in_smp;
52 struct net_device *dev;
53 struct xdp_umem *umem;
54 struct list_head flush_node;
55 struct xsk_buff_pool *pool;
65 struct xsk_queue *tx ____cacheline_aligned_in_smp;
66 struct list_head tx_list;
67 /* record the number of tx descriptors sent by this xsk and
68 * when it exceeds MAX_PER_SOCKET_BUDGET, an opportunity needs
69 * to be given to other xsks for sending tx descriptors, thereby
70 * preventing other XSKs from being starved.
74 /* Protects generic receive. */
81 /* When __xsk_generic_xmit() must return before it sees the EOP descriptor for the current
82 * packet, the partially built skb is saved here so that packet building can resume in next
83 * call of __xsk_generic_xmit().
87 struct list_head map_list;
88 /* Protects map_list */
89 spinlock_t map_list_lock;
90 /* Protects multiple processes in the control path */
92 struct xsk_queue *fq_tmp; /* Only as tmp storage before bind */
93 struct xsk_queue *cq_tmp; /* Only as tmp storage before bind */
97 * AF_XDP TX metadata hooks for network devices.
98 * The following hooks can be defined; unless noted otherwise, they are
99 * optional and can be filled with a null pointer.
101 * void (*tmo_request_timestamp)(void *priv)
102 * Called when AF_XDP frame requested egress timestamp.
104 * u64 (*tmo_fill_timestamp)(void *priv)
105 * Called when AF_XDP frame, that had requested egress timestamp,
106 * received a completion. The hook needs to return the actual HW timestamp.
108 * void (*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv)
109 * Called when AF_XDP frame requested HW checksum offload. csum_start
110 * indicates position where checksumming should start.
111 * csum_offset indicates position where checksum should be stored.
114 struct xsk_tx_metadata_ops {
115 void (*tmo_request_timestamp)(void *priv);
116 u64 (*tmo_fill_timestamp)(void *priv);
117 void (*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv);
120 #ifdef CONFIG_XDP_SOCKETS
122 int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);
123 int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp);
124 void __xsk_map_flush(void);
127 * xsk_tx_metadata_to_compl - Save enough relevant metadata information
128 * to perform tx completion in the future.
129 * @meta: pointer to AF_XDP metadata area
130 * @compl: pointer to output struct xsk_tx_metadata_to_compl
132 * This function should be called by the networking device when
133 * it prepares AF_XDP egress packet. The value of @compl should be stored
134 * and passed to xsk_tx_metadata_complete upon TX completion.
136 static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta,
137 struct xsk_tx_metadata_compl *compl)
142 if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP)
143 compl->tx_timestamp = &meta->completion.tx_timestamp;
145 compl->tx_timestamp = NULL;
149 * xsk_tx_metadata_request - Evaluate AF_XDP TX metadata at submission
150 * and call appropriate xsk_tx_metadata_ops operation.
151 * @meta: pointer to AF_XDP metadata area
152 * @ops: pointer to struct xsk_tx_metadata_ops
153 * @priv: pointer to driver-private aread
155 * This function should be called by the networking device when
156 * it prepares AF_XDP egress packet.
158 static inline void xsk_tx_metadata_request(const struct xsk_tx_metadata *meta,
159 const struct xsk_tx_metadata_ops *ops,
165 if (ops->tmo_request_timestamp)
166 if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP)
167 ops->tmo_request_timestamp(priv);
169 if (ops->tmo_request_checksum)
170 if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM)
171 ops->tmo_request_checksum(meta->request.csum_start,
172 meta->request.csum_offset, priv);
176 * xsk_tx_metadata_complete - Evaluate AF_XDP TX metadata at completion
177 * and call appropriate xsk_tx_metadata_ops operation.
178 * @compl: pointer to completion metadata produced from xsk_tx_metadata_to_compl
179 * @ops: pointer to struct xsk_tx_metadata_ops
180 * @priv: pointer to driver-private aread
182 * This function should be called by the networking device upon
183 * AF_XDP egress completion.
185 static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl,
186 const struct xsk_tx_metadata_ops *ops,
192 *compl->tx_timestamp = ops->tmo_fill_timestamp(priv);
197 static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
202 static inline int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
207 static inline void __xsk_map_flush(void)
211 static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta,
212 struct xsk_tx_metadata_compl *compl)
216 static inline void xsk_tx_metadata_request(struct xsk_tx_metadata *meta,
217 const struct xsk_tx_metadata_ops *ops,
222 static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl,
223 const struct xsk_tx_metadata_ops *ops,
228 #endif /* CONFIG_XDP_SOCKETS */
230 #if defined(CONFIG_XDP_SOCKETS) && defined(CONFIG_DEBUG_NET)
231 bool xsk_map_check_flush(void);
233 static inline bool xsk_map_check_flush(void)
239 #endif /* _LINUX_XDP_SOCK_H */