Commit | Line | Data |
---|---|---|
ae06c70b | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
51dce24b | 2 | /* Copyright(c) 1999 - 2018 Intel Corporation. */ |
9a799d71 AK |
3 | |
4 | #ifndef _IXGBE_H_ | |
5 | #define _IXGBE_H_ | |
6 | ||
f62bbb5e | 7 | #include <linux/bitops.h> |
9a799d71 AK |
8 | #include <linux/types.h> |
9 | #include <linux/pci.h> | |
10 | #include <linux/netdevice.h> | |
b25ebfd2 | 11 | #include <linux/cpumask.h> |
f62bbb5e | 12 | #include <linux/if_vlan.h> |
6cb562d6 | 13 | #include <linux/jiffies.h> |
8fa10ef0 | 14 | #include <linux/phy.h> |
9a799d71 | 15 | |
74d23cc7 | 16 | #include <linux/timecounter.h> |
3a6a4eda JK |
17 | #include <linux/net_tstamp.h> |
18 | #include <linux/ptp_clock_kernel.h> | |
3a6a4eda | 19 | |
9a799d71 AK |
20 | #include "ixgbe_type.h" |
21 | #include "ixgbe_common.h" | |
2f90b865 | 22 | #include "ixgbe_dcb.h" |
ee58c114 | 23 | #if IS_ENABLED(CONFIG_FCOE) |
eacd73f7 YZ |
24 | #define IXGBE_FCOE |
25 | #include "ixgbe_fcoe.h" | |
ee58c114 | 26 | #endif /* IS_ENABLED(CONFIG_FCOE) */ |
5dd2d332 | 27 | #ifdef CONFIG_IXGBE_DCA |
bd0362dd JC |
28 | #include <linux/dca.h> |
29 | #endif | |
8bbbc5e9 | 30 | #include "ixgbe_ipsec.h" |
9a799d71 | 31 | |
99ffc5ad | 32 | #include <net/xdp.h> |
5a85e737 | 33 | |
849c4542 ET |
34 | /* common prefix used by pr_<> macros */ |
35 | #undef pr_fmt | |
36 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
9a799d71 AK |
37 | |
38 | /* TX/RX descriptor defines */ | |
6bacb300 | 39 | #define IXGBE_DEFAULT_TXD 512 |
59224555 | 40 | #define IXGBE_DEFAULT_TX_WORK 256 |
864f8888 DW |
41 | #define IXGBE_MAX_TXD_82598 4096 |
42 | #define IXGBE_MAX_TXD_82599 8192 | |
43 | #define IXGBE_MAX_TXD_X540 8192 | |
44 | #define IXGBE_MAX_TXD_X550 32768 | |
9a799d71 AK |
45 | #define IXGBE_MIN_TXD 64 |
46 | ||
fb44519d | 47 | #if (PAGE_SIZE < 8192) |
6bacb300 | 48 | #define IXGBE_DEFAULT_RXD 512 |
fb44519d AB |
49 | #else |
50 | #define IXGBE_DEFAULT_RXD 128 | |
51 | #endif | |
864f8888 DW |
52 | #define IXGBE_MAX_RXD_82598 4096 |
53 | #define IXGBE_MAX_RXD_82599 8192 | |
54 | #define IXGBE_MAX_RXD_X540 8192 | |
55 | #define IXGBE_MAX_RXD_X550 32768 | |
9a799d71 AK |
56 | #define IXGBE_MIN_RXD 64 |
57 | ||
9a799d71 | 58 | /* flow control */ |
2b9ade93 | 59 | #define IXGBE_MIN_FCRTL 0x40 |
9a799d71 | 60 | #define IXGBE_MAX_FCRTL 0x7FF80 |
2b9ade93 | 61 | #define IXGBE_MIN_FCRTH 0x600 |
9a799d71 | 62 | #define IXGBE_MAX_FCRTH 0x7FFF0 |
2b9ade93 | 63 | #define IXGBE_DEFAULT_FCPAUSE 0xFFFF |
9a799d71 AK |
64 | #define IXGBE_MIN_FCPAUSE 0 |
65 | #define IXGBE_MAX_FCPAUSE 0xFFFF | |
66 | ||
67 | /* Supported Rx Buffer Sizes */ | |
252562c2 | 68 | #define IXGBE_RXBUFFER_256 256 /* Used for skb receive header */ |
541ea69a | 69 | #define IXGBE_RXBUFFER_1536 1536 |
09816fbe AD |
70 | #define IXGBE_RXBUFFER_2K 2048 |
71 | #define IXGBE_RXBUFFER_3K 3072 | |
72 | #define IXGBE_RXBUFFER_4K 4096 | |
919e78a6 | 73 | #define IXGBE_MAX_RXBUFFER 16384 /* largest size for a single descriptor */ |
9a799d71 | 74 | |
0967bf83 JX |
75 | #define IXGBE_PKT_HDR_PAD (ETH_HLEN + ETH_FCS_LEN + (VLAN_HLEN * 2)) |
76 | ||
541ea69a AD |
77 | /* Attempt to maximize the headroom available for incoming frames. We |
78 | * use a 2K buffer for receives and need 1536/1534 to store the data for | |
79 | * the frame. This leaves us with 512 bytes of room. From that we need | |
80 | * to deduct the space needed for the shared info and the padding needed | |
81 | * to IP align the frame. | |
82 | * | |
83 | * Note: For cache line sizes 256 or larger this value is going to end | |
84 | * up negative. In these cases we should fall back to the 3K | |
85 | * buffers. | |
86 | */ | |
2de6aa3a | 87 | #if (PAGE_SIZE < 8192) |
541ea69a AD |
88 | #define IXGBE_MAX_2K_FRAME_BUILD_SKB (IXGBE_RXBUFFER_1536 - NET_IP_ALIGN) |
89 | #define IXGBE_2K_TOO_SMALL_WITH_PADDING \ | |
90 | ((NET_SKB_PAD + IXGBE_RXBUFFER_1536) > SKB_WITH_OVERHEAD(IXGBE_RXBUFFER_2K)) | |
91 | ||
92 | static inline int ixgbe_compute_pad(int rx_buf_len) | |
93 | { | |
94 | int page_size, pad_size; | |
95 | ||
96 | page_size = ALIGN(rx_buf_len, PAGE_SIZE / 2); | |
97 | pad_size = SKB_WITH_OVERHEAD(page_size) - rx_buf_len; | |
98 | ||
99 | return pad_size; | |
100 | } | |
101 | ||
102 | static inline int ixgbe_skb_pad(void) | |
103 | { | |
104 | int rx_buf_len; | |
105 | ||
106 | /* If a 2K buffer cannot handle a standard Ethernet frame then | |
107 | * optimize padding for a 3K buffer instead of a 1.5K buffer. | |
108 | * | |
109 | * For a 3K buffer we need to add enough padding to allow for | |
110 | * tailroom due to NET_IP_ALIGN possibly shifting us out of | |
111 | * cache-line alignment. | |
112 | */ | |
113 | if (IXGBE_2K_TOO_SMALL_WITH_PADDING) | |
114 | rx_buf_len = IXGBE_RXBUFFER_3K + SKB_DATA_ALIGN(NET_IP_ALIGN); | |
115 | else | |
116 | rx_buf_len = IXGBE_RXBUFFER_1536; | |
117 | ||
118 | /* if needed make room for NET_IP_ALIGN */ | |
119 | rx_buf_len -= NET_IP_ALIGN; | |
120 | ||
121 | return ixgbe_compute_pad(rx_buf_len); | |
122 | } | |
123 | ||
124 | #define IXGBE_SKB_PAD ixgbe_skb_pad() | |
2de6aa3a | 125 | #else |
541ea69a | 126 | #define IXGBE_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN) |
2de6aa3a AD |
127 | #endif |
128 | ||
13958070 | 129 | /* |
252562c2 AD |
130 | * NOTE: netdev_alloc_skb reserves up to 64 bytes, NET_IP_ALIGN means we |
131 | * reserve 64 more, and skb_shared_info adds an additional 320 bytes more, | |
132 | * this adds up to 448 bytes of extra data. | |
133 | * | |
134 | * Since netdev_alloc_skb now allocates a page fragment we can use a value | |
135 | * of 256 and the resultant skb will have a truesize of 960 or less. | |
13958070 | 136 | */ |
252562c2 | 137 | #define IXGBE_RX_HDR_SIZE IXGBE_RXBUFFER_256 |
9a799d71 | 138 | |
9a799d71 AK |
139 | /* How many Rx Buffers do we bundle into one write to the hardware ? */ |
140 | #define IXGBE_RX_BUFFER_WRITE 16 /* Must be power of 2 */ | |
141 | ||
f3213d93 AD |
142 | #define IXGBE_RX_DMA_ATTR \ |
143 | (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING) | |
144 | ||
472148c3 AD |
145 | enum ixgbe_tx_flags { |
146 | /* cmd_type flags */ | |
147 | IXGBE_TX_FLAGS_HW_VLAN = 0x01, | |
148 | IXGBE_TX_FLAGS_TSO = 0x02, | |
149 | IXGBE_TX_FLAGS_TSTAMP = 0x04, | |
150 | ||
151 | /* olinfo flags */ | |
152 | IXGBE_TX_FLAGS_CC = 0x08, | |
153 | IXGBE_TX_FLAGS_IPV4 = 0x10, | |
154 | IXGBE_TX_FLAGS_CSUM = 0x20, | |
59259470 | 155 | IXGBE_TX_FLAGS_IPSEC = 0x40, |
472148c3 AD |
156 | |
157 | /* software defined flags */ | |
59259470 SN |
158 | IXGBE_TX_FLAGS_SW_VLAN = 0x80, |
159 | IXGBE_TX_FLAGS_FCOE = 0x100, | |
472148c3 AD |
160 | }; |
161 | ||
162 | /* VLAN info */ | |
9a799d71 | 163 | #define IXGBE_TX_FLAGS_VLAN_MASK 0xffff0000 |
66f32a8b AD |
164 | #define IXGBE_TX_FLAGS_VLAN_PRIO_MASK 0xe0000000 |
165 | #define IXGBE_TX_FLAGS_VLAN_PRIO_SHIFT 29 | |
9a799d71 AK |
166 | #define IXGBE_TX_FLAGS_VLAN_SHIFT 16 |
167 | ||
7f870475 GR |
168 | #define IXGBE_MAX_VF_MC_ENTRIES 30 |
169 | #define IXGBE_MAX_VF_FUNCTIONS 64 | |
170 | #define IXGBE_MAX_VFTA_ENTRIES 128 | |
171 | #define MAX_EMULATION_MAC_ADDRS 16 | |
a1cbb15c | 172 | #define IXGBE_MAX_PF_MACVLANS 15 |
1d9c0bfd | 173 | #define VMDQ_P(p) ((p) + adapter->ring_feature[RING_F_VMDQ].offset) |
83c61fa9 GR |
174 | #define IXGBE_82599_VF_DEVICE_ID 0x10ED |
175 | #define IXGBE_X540_VF_DEVICE_ID 0x1515 | |
7f870475 | 176 | |
37530030 MH |
177 | #define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \ |
178 | { \ | |
179 | u32 current_counter = IXGBE_READ_REG(hw, reg); \ | |
180 | if (current_counter < last_counter) \ | |
181 | counter += 0x100000000LL; \ | |
182 | last_counter = current_counter; \ | |
183 | counter &= 0xFFFFFFFF00000000LL; \ | |
184 | counter |= current_counter; \ | |
185 | } | |
186 | ||
187 | #define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \ | |
188 | { \ | |
189 | u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb); \ | |
190 | u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb); \ | |
191 | u64 current_counter = (current_counter_msb << 32) | \ | |
192 | current_counter_lsb; \ | |
193 | if (current_counter < last_counter) \ | |
194 | counter += 0x1000000000LL; \ | |
195 | last_counter = current_counter; \ | |
196 | counter &= 0xFFFFFFF000000000LL; \ | |
197 | counter |= current_counter; \ | |
198 | } | |
199 | ||
200 | struct vf_stats { | |
201 | u64 gprc; | |
202 | u64 gorc; | |
203 | u64 gptc; | |
204 | u64 gotc; | |
205 | u64 mprc; | |
206 | }; | |
207 | ||
7f870475 | 208 | struct vf_data_storage { |
988d1307 | 209 | struct pci_dev *vfdev; |
7f870475 GR |
210 | unsigned char vf_mac_addresses[ETH_ALEN]; |
211 | u16 vf_mc_hashes[IXGBE_MAX_VF_MC_ENTRIES]; | |
212 | u16 num_vf_mc_hashes; | |
7f870475 | 213 | bool clear_to_send; |
37530030 MH |
214 | struct vf_stats vfstats; |
215 | struct vf_stats last_vfstats; | |
216 | struct vf_stats saved_rst_vfstats; | |
7f01648a | 217 | bool pf_set_mac; |
7f01648a GR |
218 | u16 pf_vlan; /* When set, guest VLAN config not allowed. */ |
219 | u16 pf_qos; | |
ff4ab206 | 220 | u16 tx_rate; |
366fd100 SM |
221 | int link_enable; |
222 | int link_state; | |
de4c7f65 | 223 | u8 spoofchk_enabled; |
e65ce0d3 | 224 | bool rss_query_enabled; |
54011e4d | 225 | u8 trusted; |
8443c1a4 | 226 | int xcast_mode; |
374c65d6 | 227 | unsigned int vf_api; |
008ca35f | 228 | u8 primary_abort_count; |
7f870475 GR |
229 | }; |
230 | ||
8443c1a4 HS |
231 | enum ixgbevf_xcast_modes { |
232 | IXGBEVF_XCAST_MODE_NONE = 0, | |
233 | IXGBEVF_XCAST_MODE_MULTI, | |
234 | IXGBEVF_XCAST_MODE_ALLMULTI, | |
07eea570 | 235 | IXGBEVF_XCAST_MODE_PROMISC, |
8443c1a4 HS |
236 | }; |
237 | ||
a1cbb15c GR |
238 | struct vf_macvlans { |
239 | struct list_head l; | |
240 | int vf; | |
a1cbb15c GR |
241 | bool free; |
242 | bool is_macvlan; | |
243 | u8 vf_macvlan[ETH_ALEN]; | |
244 | }; | |
245 | ||
a535c30e | 246 | #define IXGBE_MAX_TXD_PWR 14 |
b4f47a48 | 247 | #define IXGBE_MAX_DATA_PER_TXD (1u << IXGBE_MAX_TXD_PWR) |
a535c30e AD |
248 | |
249 | /* Tx Descriptors needed, worst case */ | |
250 | #define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD) | |
990a3158 | 251 | #define DESC_NEEDED (MAX_SKB_FRAGS + 4) |
a535c30e | 252 | |
9a799d71 AK |
253 | /* wrapper around a pointer to a socket buffer, |
254 | * so a DMA handle can be stored along with the buffer */ | |
255 | struct ixgbe_tx_buffer { | |
d3d00239 | 256 | union ixgbe_adv_tx_desc *next_to_watch; |
9a799d71 | 257 | unsigned long time_stamp; |
33fdc82f JF |
258 | union { |
259 | struct sk_buff *skb; | |
03993094 | 260 | struct xdp_frame *xdpf; |
33fdc82f | 261 | }; |
fd0db0ed AD |
262 | unsigned int bytecount; |
263 | unsigned short gso_segs; | |
244e27ad | 264 | __be16 protocol; |
729739b7 AD |
265 | DEFINE_DMA_UNMAP_ADDR(dma); |
266 | DEFINE_DMA_UNMAP_LEN(len); | |
d3d00239 | 267 | u32 tx_flags; |
9a799d71 AK |
268 | }; |
269 | ||
270 | struct ixgbe_rx_buffer { | |
d0bcacd0 BT |
271 | union { |
272 | struct { | |
7117132b BT |
273 | struct sk_buff *skb; |
274 | dma_addr_t dma; | |
d0bcacd0 BT |
275 | struct page *page; |
276 | __u32 page_offset; | |
277 | __u16 pagecnt_bias; | |
278 | }; | |
279 | struct { | |
7117132b BT |
280 | bool discard; |
281 | struct xdp_buff *xdp; | |
d0bcacd0 BT |
282 | }; |
283 | }; | |
9a799d71 AK |
284 | }; |
285 | ||
286 | struct ixgbe_queue_stats { | |
287 | u64 packets; | |
288 | u64 bytes; | |
289 | }; | |
290 | ||
5b7da515 AD |
291 | struct ixgbe_tx_queue_stats { |
292 | u64 restart_queue; | |
293 | u64 tx_busy; | |
c84d324c | 294 | u64 tx_done_old; |
5b7da515 AD |
295 | }; |
296 | ||
297 | struct ixgbe_rx_queue_stats { | |
298 | u64 rsc_count; | |
299 | u64 rsc_flush; | |
300 | u64 non_eop_descs; | |
86e23494 | 301 | u64 alloc_rx_page; |
5b7da515 AD |
302 | u64 alloc_rx_page_failed; |
303 | u64 alloc_rx_buff_failed; | |
8a0da21b | 304 | u64 csum_err; |
5b7da515 AD |
305 | }; |
306 | ||
a9763f3c MR |
307 | #define IXGBE_TS_HDR_LEN 8 |
308 | ||
f800326d | 309 | enum ixgbe_ring_state_t { |
4f4542bf | 310 | __IXGBE_RX_3K_BUFFER, |
2de6aa3a | 311 | __IXGBE_RX_BUILD_SKB_ENABLED, |
4f4542bf AD |
312 | __IXGBE_RX_RSC_ENABLED, |
313 | __IXGBE_RX_CSUM_UDP_ZERO_ERR, | |
314 | __IXGBE_RX_FCOE, | |
7d637bcc | 315 | __IXGBE_TX_FDIR_INIT_DONE, |
fd786b7b | 316 | __IXGBE_TX_XPS_INIT_DONE, |
7d637bcc | 317 | __IXGBE_TX_DETECT_HANG, |
c84d324c | 318 | __IXGBE_HANG_CHECK_ARMED, |
33fdc82f | 319 | __IXGBE_TX_XDP_RING, |
024aa580 | 320 | __IXGBE_TX_DISABLED, |
7d637bcc AD |
321 | }; |
322 | ||
2de6aa3a AD |
323 | #define ring_uses_build_skb(ring) \ |
324 | test_bit(__IXGBE_RX_BUILD_SKB_ENABLED, &(ring)->state) | |
325 | ||
2a47fa45 JF |
326 | struct ixgbe_fwd_adapter { |
327 | unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)]; | |
328 | struct net_device *netdev; | |
2a47fa45 JF |
329 | unsigned int tx_base_queue; |
330 | unsigned int rx_base_queue; | |
331 | int pool; | |
332 | }; | |
333 | ||
7d637bcc AD |
334 | #define check_for_tx_hang(ring) \ |
335 | test_bit(__IXGBE_TX_DETECT_HANG, &(ring)->state) | |
336 | #define set_check_for_tx_hang(ring) \ | |
337 | set_bit(__IXGBE_TX_DETECT_HANG, &(ring)->state) | |
338 | #define clear_check_for_tx_hang(ring) \ | |
339 | clear_bit(__IXGBE_TX_DETECT_HANG, &(ring)->state) | |
340 | #define ring_is_rsc_enabled(ring) \ | |
341 | test_bit(__IXGBE_RX_RSC_ENABLED, &(ring)->state) | |
342 | #define set_ring_rsc_enabled(ring) \ | |
343 | set_bit(__IXGBE_RX_RSC_ENABLED, &(ring)->state) | |
344 | #define clear_ring_rsc_enabled(ring) \ | |
345 | clear_bit(__IXGBE_RX_RSC_ENABLED, &(ring)->state) | |
33fdc82f JF |
346 | #define ring_is_xdp(ring) \ |
347 | test_bit(__IXGBE_TX_XDP_RING, &(ring)->state) | |
348 | #define set_ring_xdp(ring) \ | |
349 | set_bit(__IXGBE_TX_XDP_RING, &(ring)->state) | |
350 | #define clear_ring_xdp(ring) \ | |
351 | clear_bit(__IXGBE_TX_XDP_RING, &(ring)->state) | |
9a799d71 | 352 | struct ixgbe_ring { |
efe3d3c8 | 353 | struct ixgbe_ring *next; /* pointer to next ring in q_vector */ |
d3ee4294 AD |
354 | struct ixgbe_q_vector *q_vector; /* backpointer to host q_vector */ |
355 | struct net_device *netdev; /* netdev ring belongs to */ | |
92470808 | 356 | struct bpf_prog *xdp_prog; |
d3ee4294 | 357 | struct device *dev; /* device for DMA mapping */ |
9a799d71 | 358 | void *desc; /* descriptor ring memory */ |
9a799d71 AK |
359 | union { |
360 | struct ixgbe_tx_buffer *tx_buffer_info; | |
361 | struct ixgbe_rx_buffer *rx_buffer_info; | |
362 | }; | |
7d637bcc | 363 | unsigned long state; |
bd198058 | 364 | u8 __iomem *tail; |
d3ee4294 AD |
365 | dma_addr_t dma; /* phys. address of descriptor ring */ |
366 | unsigned int size; /* length in bytes */ | |
bd198058 | 367 | |
ae540af1 | 368 | u16 count; /* amount of descriptors */ |
ae540af1 JB |
369 | |
370 | u8 queue_index; /* needed for multiqueue queue management */ | |
7d637bcc AD |
371 | u8 reg_idx; /* holds the special value that gets |
372 | * the hardware register offset | |
373 | * associated with this ring, which is | |
374 | * different for DCB and RSS modes | |
375 | */ | |
d3ee4294 AD |
376 | u16 next_to_use; |
377 | u16 next_to_clean; | |
378 | ||
a9763f3c MR |
379 | unsigned long last_rx_timestamp; |
380 | ||
f800326d | 381 | union { |
d3ee4294 | 382 | u16 next_to_alloc; |
f800326d AD |
383 | struct { |
384 | u8 atr_sample_rate; | |
385 | u8 atr_count; | |
386 | }; | |
f800326d | 387 | }; |
9a799d71 | 388 | |
bd198058 | 389 | u8 dcb_tc; |
9a799d71 | 390 | struct ixgbe_queue_stats stats; |
de1036b1 | 391 | struct u64_stats_sync syncp; |
5b7da515 AD |
392 | union { |
393 | struct ixgbe_tx_queue_stats tx_stats; | |
394 | struct ixgbe_rx_queue_stats rx_stats; | |
395 | }; | |
c0d4e9d2 | 396 | u16 rx_offset; |
99ffc5ad | 397 | struct xdp_rxq_info xdp_rxq; |
4fe81585 | 398 | spinlock_t tx_lock; /* used in XDP mode */ |
1742b3d5 | 399 | struct xsk_buff_pool *xsk_pool; |
d0bcacd0 BT |
400 | u16 ring_idx; /* {rx,tx,xdp}_ring back reference idx */ |
401 | u16 rx_buf_len; | |
7ca3bc58 | 402 | } ____cacheline_internodealigned_in_smp; |
9a799d71 | 403 | |
c7e4358a SN |
404 | enum ixgbe_ring_f_enum { |
405 | RING_F_NONE = 0, | |
7f870475 | 406 | RING_F_VMDQ, /* SR-IOV uses the same ring feature */ |
c7e4358a | 407 | RING_F_RSS, |
c4cf55e5 | 408 | RING_F_FDIR, |
0331a832 YZ |
409 | #ifdef IXGBE_FCOE |
410 | RING_F_FCOE, | |
411 | #endif /* IXGBE_FCOE */ | |
c7e4358a SN |
412 | |
413 | RING_F_ARRAY_SIZE /* must be last in enum set */ | |
414 | }; | |
415 | ||
0f9b232b | 416 | #define IXGBE_MAX_RSS_INDICES 16 |
e9ee3238 | 417 | #define IXGBE_MAX_RSS_INDICES_X550 63 |
0f9b232b DS |
418 | #define IXGBE_MAX_VMDQ_INDICES 64 |
419 | #define IXGBE_MAX_FDIR_INDICES 63 /* based on q_vector limit */ | |
420 | #define IXGBE_MAX_FCOE_INDICES 8 | |
421 | #define MAX_RX_QUEUES (IXGBE_MAX_FDIR_INDICES + 1) | |
422 | #define MAX_TX_QUEUES (IXGBE_MAX_FDIR_INDICES + 1) | |
4fe81585 | 423 | #define IXGBE_MAX_XDP_QS (IXGBE_MAX_FDIR_INDICES + 1) |
0f9b232b DS |
424 | #define IXGBE_MAX_L2A_QUEUES 4 |
425 | #define IXGBE_BAD_L2A_QUEUE 3 | |
4e039c16 | 426 | #define IXGBE_MAX_MACVLANS 63 |
2a47fa45 | 427 | |
4fe81585 JX |
428 | DECLARE_STATIC_KEY_FALSE(ixgbe_xdp_locking_key); |
429 | ||
021230d4 | 430 | struct ixgbe_ring_feature { |
c087663e AD |
431 | u16 limit; /* upper limit on feature indices */ |
432 | u16 indices; /* current value of indices */ | |
e4b317e9 AD |
433 | u16 mask; /* Mask used for feature to ring mapping */ |
434 | u16 offset; /* offset to start of feature */ | |
7ca3bc58 | 435 | } ____cacheline_internodealigned_in_smp; |
021230d4 | 436 | |
73079ea0 AD |
437 | #define IXGBE_82599_VMDQ_8Q_MASK 0x78 |
438 | #define IXGBE_82599_VMDQ_4Q_MASK 0x7C | |
439 | #define IXGBE_82599_VMDQ_2Q_MASK 0x7E | |
440 | ||
f800326d AD |
441 | /* |
442 | * FCoE requires that all Rx buffers be over 2200 bytes in length. Since | |
443 | * this is twice the size of a half page we need to double the page order | |
444 | * for FCoE enabled Rx queues. | |
445 | */ | |
09816fbe | 446 | static inline unsigned int ixgbe_rx_bufsz(struct ixgbe_ring *ring) |
f800326d | 447 | { |
4f4542bf AD |
448 | if (test_bit(__IXGBE_RX_3K_BUFFER, &ring->state)) |
449 | return IXGBE_RXBUFFER_3K; | |
2de6aa3a AD |
450 | #if (PAGE_SIZE < 8192) |
451 | if (ring_uses_build_skb(ring)) | |
541ea69a | 452 | return IXGBE_MAX_2K_FRAME_BUILD_SKB; |
2de6aa3a | 453 | #endif |
09816fbe | 454 | return IXGBE_RXBUFFER_2K; |
f800326d | 455 | } |
09816fbe AD |
456 | |
457 | static inline unsigned int ixgbe_rx_pg_order(struct ixgbe_ring *ring) | |
458 | { | |
4f4542bf AD |
459 | #if (PAGE_SIZE < 8192) |
460 | if (test_bit(__IXGBE_RX_3K_BUFFER, &ring->state)) | |
461 | return 1; | |
f800326d | 462 | #endif |
09816fbe AD |
463 | return 0; |
464 | } | |
f800326d | 465 | #define ixgbe_rx_pg_size(_ring) (PAGE_SIZE << ixgbe_rx_pg_order(_ring)) |
f800326d | 466 | |
b4ded832 AD |
467 | #define IXGBE_ITR_ADAPTIVE_MIN_INC 2 |
468 | #define IXGBE_ITR_ADAPTIVE_MIN_USECS 10 | |
469 | #define IXGBE_ITR_ADAPTIVE_MAX_USECS 126 | |
470 | #define IXGBE_ITR_ADAPTIVE_LATENCY 0x80 | |
471 | #define IXGBE_ITR_ADAPTIVE_BULK 0x00 | |
472 | ||
08c8833b | 473 | struct ixgbe_ring_container { |
efe3d3c8 | 474 | struct ixgbe_ring *ring; /* pointer to linked list of rings */ |
b4ded832 | 475 | unsigned long next_update; /* jiffies value of last update */ |
bd198058 AD |
476 | unsigned int total_bytes; /* total bytes processed this int */ |
477 | unsigned int total_packets; /* total packets processed this int */ | |
478 | u16 work_limit; /* total work allowed per interrupt */ | |
08c8833b AD |
479 | u8 count; /* total number of rings in vector */ |
480 | u8 itr; /* current ITR setting for ring */ | |
481 | }; | |
021230d4 | 482 | |
a557928e AD |
483 | /* iterator for handling rings in ring container */ |
484 | #define ixgbe_for_each_ring(pos, head) \ | |
485 | for (pos = (head).ring; pos != NULL; pos = pos->next) | |
486 | ||
2f90b865 | 487 | #define MAX_RX_PACKET_BUFFERS ((adapter->flags & IXGBE_FLAG_DCB_ENABLED) \ |
e7cf745b | 488 | ? 8 : 1) |
2f90b865 AD |
489 | #define MAX_TX_PACKET_BUFFERS MAX_RX_PACKET_BUFFERS |
490 | ||
49c7ffbe | 491 | /* MAX_Q_VECTORS of these are allocated, |
021230d4 AV |
492 | * but we only use one per queue-specific vector. |
493 | */ | |
494 | struct ixgbe_q_vector { | |
495 | struct ixgbe_adapter *adapter; | |
33cf09c9 AD |
496 | #ifdef CONFIG_IXGBE_DCA |
497 | int cpu; /* CPU for DCA */ | |
498 | #endif | |
d5bf4f67 ET |
499 | u16 v_idx; /* index of q_vector within array, also used for |
500 | * finding the bit in EICR and friends that | |
501 | * represents the vector for this ring */ | |
502 | u16 itr; /* Interrupt throttle rate written to EITR */ | |
08c8833b | 503 | struct ixgbe_ring_container rx, tx; |
d5bf4f67 ET |
504 | |
505 | struct napi_struct napi; | |
de88eeeb AD |
506 | cpumask_t affinity_mask; |
507 | int numa_node; | |
508 | struct rcu_head rcu; /* to avoid race with update stats on free */ | |
d0759ebb | 509 | char name[IFNAMSIZ + 9]; |
de88eeeb AD |
510 | |
511 | /* for dynamic allocation of rings associated with this q_vector */ | |
040efdb1 | 512 | struct ixgbe_ring ring[] ____cacheline_internodealigned_in_smp; |
021230d4 | 513 | }; |
adc81090 | 514 | |
3ca8bc6d DS |
515 | #ifdef CONFIG_IXGBE_HWMON |
516 | ||
517 | #define IXGBE_HWMON_TYPE_LOC 0 | |
518 | #define IXGBE_HWMON_TYPE_TEMP 1 | |
519 | #define IXGBE_HWMON_TYPE_CAUTION 2 | |
520 | #define IXGBE_HWMON_TYPE_MAX 3 | |
521 | ||
522 | struct hwmon_attr { | |
523 | struct device_attribute dev_attr; | |
524 | struct ixgbe_hw *hw; | |
525 | struct ixgbe_thermal_diode_data *sensor; | |
526 | char name[12]; | |
527 | }; | |
528 | ||
529 | struct hwmon_buff { | |
03b77d81 GR |
530 | struct attribute_group group; |
531 | const struct attribute_group *groups[2]; | |
532 | struct attribute *attrs[IXGBE_MAX_SENSORS * 4 + 1]; | |
533 | struct hwmon_attr hwmon_list[IXGBE_MAX_SENSORS * 4]; | |
3ca8bc6d DS |
534 | unsigned int n_hwmon; |
535 | }; | |
536 | #endif /* CONFIG_IXGBE_HWMON */ | |
021230d4 | 537 | |
d5bf4f67 ET |
538 | /* |
539 | * microsecond values for various ITR rates shifted by 2 to fit itr register | |
540 | * with the first 3 bits reserved 0 | |
9a799d71 | 541 | */ |
d5bf4f67 ET |
542 | #define IXGBE_MIN_RSC_ITR 24 |
543 | #define IXGBE_100K_ITR 40 | |
544 | #define IXGBE_20K_ITR 200 | |
8ac34f10 | 545 | #define IXGBE_12K_ITR 336 |
9a799d71 | 546 | |
f56e0cb1 AD |
547 | /* ixgbe_test_staterr - tests bits in Rx descriptor status and error fields */ |
548 | static inline __le32 ixgbe_test_staterr(union ixgbe_adv_rx_desc *rx_desc, | |
549 | const u32 stat_err_bits) | |
550 | { | |
551 | return rx_desc->wb.upper.status_error & cpu_to_le32(stat_err_bits); | |
552 | } | |
553 | ||
7d4987de AD |
554 | static inline u16 ixgbe_desc_unused(struct ixgbe_ring *ring) |
555 | { | |
556 | u16 ntc = ring->next_to_clean; | |
557 | u16 ntu = ring->next_to_use; | |
558 | ||
559 | return ((ntc > ntu) ? 0 : ring->count) + ntc - ntu - 1; | |
560 | } | |
9a799d71 | 561 | |
e4f74028 | 562 | #define IXGBE_RX_DESC(R, i) \ |
31f05a2d | 563 | (&(((union ixgbe_adv_rx_desc *)((R)->desc))[i])) |
e4f74028 | 564 | #define IXGBE_TX_DESC(R, i) \ |
31f05a2d | 565 | (&(((union ixgbe_adv_tx_desc *)((R)->desc))[i])) |
e4f74028 | 566 | #define IXGBE_TX_CTXTDESC(R, i) \ |
31f05a2d | 567 | (&(((struct ixgbe_adv_tx_context_desc *)((R)->desc))[i])) |
9a799d71 | 568 | |
c88887e0 | 569 | #define IXGBE_MAX_JUMBO_FRAME_SIZE 9728 /* Maximum Supported Size 9.5KB */ |
63f39bd1 YZ |
570 | #ifdef IXGBE_FCOE |
571 | /* Use 3K as the baby jumbo frame size for FCoE */ | |
572 | #define IXGBE_FCOE_JUMBO_FRAME_SIZE 3072 | |
573 | #endif /* IXGBE_FCOE */ | |
9a799d71 | 574 | |
021230d4 AV |
575 | #define OTHER_VECTOR 1 |
576 | #define NON_Q_VECTORS (OTHER_VECTOR) | |
577 | ||
e8e26350 | 578 | #define MAX_MSIX_VECTORS_82599 64 |
49c7ffbe | 579 | #define MAX_Q_VECTORS_82599 64 |
eb7f139c | 580 | #define MAX_MSIX_VECTORS_82598 18 |
49c7ffbe | 581 | #define MAX_Q_VECTORS_82598 16 |
eb7f139c | 582 | |
5d7daa35 JK |
583 | struct ixgbe_mac_addr { |
584 | u8 addr[ETH_ALEN]; | |
c9f53e63 | 585 | u16 pool; |
5d7daa35 JK |
586 | u16 state; /* bitmask */ |
587 | }; | |
c9f53e63 | 588 | |
5d7daa35 JK |
589 | #define IXGBE_MAC_STATE_DEFAULT 0x1 |
590 | #define IXGBE_MAC_STATE_MODIFIED 0x2 | |
591 | #define IXGBE_MAC_STATE_IN_USE 0x4 | |
592 | ||
49c7ffbe | 593 | #define MAX_Q_VECTORS MAX_Q_VECTORS_82599 |
e8e26350 | 594 | #define MAX_MSIX_COUNT MAX_MSIX_VECTORS_82599 |
eb7f139c | 595 | |
8f15486d | 596 | #define MIN_MSIX_Q_VECTORS 1 |
021230d4 AV |
597 | #define MIN_MSIX_COUNT (MIN_MSIX_Q_VECTORS + NON_Q_VECTORS) |
598 | ||
46646e61 AD |
599 | /* default to trying for four seconds */ |
600 | #define IXGBE_TRY_LINK_TIMEOUT (4 * HZ) | |
58e7cd24 | 601 | #define IXGBE_SFP_POLL_JIFFIES (2 * HZ) /* SFP poll every 2 seconds */ |
46646e61 | 602 | |
008ca35f SM |
603 | #define IXGBE_PRIMARY_ABORT_LIMIT 5 |
604 | ||
9a799d71 AK |
605 | /* board specific private data structure */ |
606 | struct ixgbe_adapter { | |
46646e61 AD |
607 | unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)]; |
608 | /* OS defined structs */ | |
609 | struct net_device *netdev; | |
92470808 | 610 | struct bpf_prog *xdp_prog; |
46646e61 | 611 | struct pci_dev *pdev; |
8fa10ef0 | 612 | struct mii_bus *mii_bus; |
46646e61 | 613 | |
e606bfe7 AD |
614 | unsigned long state; |
615 | ||
616 | /* Some features need tri-state capability, | |
617 | * thus the additional *_CAPABLE flags. | |
618 | */ | |
619 | u32 flags; | |
b4f47a48 JK |
620 | #define IXGBE_FLAG_MSI_ENABLED BIT(1) |
621 | #define IXGBE_FLAG_MSIX_ENABLED BIT(3) | |
622 | #define IXGBE_FLAG_RX_1BUF_CAPABLE BIT(4) | |
623 | #define IXGBE_FLAG_RX_PS_CAPABLE BIT(5) | |
624 | #define IXGBE_FLAG_RX_PS_ENABLED BIT(6) | |
625 | #define IXGBE_FLAG_DCA_ENABLED BIT(8) | |
626 | #define IXGBE_FLAG_DCA_CAPABLE BIT(9) | |
627 | #define IXGBE_FLAG_IMIR_ENABLED BIT(10) | |
628 | #define IXGBE_FLAG_MQ_CAPABLE BIT(11) | |
629 | #define IXGBE_FLAG_DCB_ENABLED BIT(12) | |
630 | #define IXGBE_FLAG_VMDQ_CAPABLE BIT(13) | |
631 | #define IXGBE_FLAG_VMDQ_ENABLED BIT(14) | |
632 | #define IXGBE_FLAG_FAN_FAIL_CAPABLE BIT(15) | |
633 | #define IXGBE_FLAG_NEED_LINK_UPDATE BIT(16) | |
634 | #define IXGBE_FLAG_NEED_LINK_CONFIG BIT(17) | |
635 | #define IXGBE_FLAG_FDIR_HASH_CAPABLE BIT(18) | |
636 | #define IXGBE_FLAG_FDIR_PERFECT_CAPABLE BIT(19) | |
637 | #define IXGBE_FLAG_FCOE_CAPABLE BIT(20) | |
638 | #define IXGBE_FLAG_FCOE_ENABLED BIT(21) | |
639 | #define IXGBE_FLAG_SRIOV_CAPABLE BIT(22) | |
640 | #define IXGBE_FLAG_SRIOV_ENABLED BIT(23) | |
a9763f3c MR |
641 | #define IXGBE_FLAG_RX_HWTSTAMP_ENABLED BIT(25) |
642 | #define IXGBE_FLAG_RX_HWTSTAMP_IN_REGISTER BIT(26) | |
8829009d | 643 | #define IXGBE_FLAG_DCB_CAPABLE BIT(27) |
e606bfe7 AD |
644 | |
645 | u32 flags2; | |
b4f47a48 JK |
646 | #define IXGBE_FLAG2_RSC_CAPABLE BIT(0) |
647 | #define IXGBE_FLAG2_RSC_ENABLED BIT(1) | |
648 | #define IXGBE_FLAG2_TEMP_SENSOR_CAPABLE BIT(2) | |
649 | #define IXGBE_FLAG2_TEMP_SENSOR_EVENT BIT(3) | |
650 | #define IXGBE_FLAG2_SEARCH_FOR_SFP BIT(4) | |
651 | #define IXGBE_FLAG2_SFP_NEEDS_RESET BIT(5) | |
b4f47a48 JK |
652 | #define IXGBE_FLAG2_FDIR_REQUIRES_REINIT BIT(7) |
653 | #define IXGBE_FLAG2_RSS_FIELD_IPV4_UDP BIT(8) | |
654 | #define IXGBE_FLAG2_RSS_FIELD_IPV6_UDP BIT(9) | |
655 | #define IXGBE_FLAG2_PTP_PPS_ENABLED BIT(10) | |
656 | #define IXGBE_FLAG2_PHY_INTERRUPT BIT(11) | |
16369564 | 657 | #define IXGBE_FLAG2_VLAN_PROMISC BIT(13) |
b3eb4e18 MR |
658 | #define IXGBE_FLAG2_EEE_CAPABLE BIT(14) |
659 | #define IXGBE_FLAG2_EEE_ENABLED BIT(15) | |
2de6aa3a | 660 | #define IXGBE_FLAG2_RX_LEGACY BIT(16) |
34c822e2 | 661 | #define IXGBE_FLAG2_IPSEC_ENABLED BIT(17) |
9e4e30cc | 662 | #define IXGBE_FLAG2_VF_IPSEC_ENABLED BIT(18) |
008ca35f | 663 | #define IXGBE_FLAG2_AUTO_DISABLE_VF BIT(19) |
d033d526 | 664 | |
46646e61 AD |
665 | /* Tx fast path data */ |
666 | int num_tx_queues; | |
667 | u16 tx_itr_setting; | |
bd198058 | 668 | u16 tx_work_limit; |
a8a43fda | 669 | u64 tx_ipsec; |
bd198058 | 670 | |
46646e61 AD |
671 | /* Rx fast path data */ |
672 | int num_rx_queues; | |
673 | u16 rx_itr_setting; | |
a8a43fda | 674 | u64 rx_ipsec; |
46646e61 | 675 | |
9f12df90 AD |
676 | /* Port number used to identify VXLAN traffic */ |
677 | __be16 vxlan_port; | |
a21d0822 | 678 | __be16 geneve_port; |
9f12df90 | 679 | |
33fdc82f JF |
680 | /* XDP */ |
681 | int num_xdp_queues; | |
4fe81585 | 682 | struct ixgbe_ring *xdp_ring[IXGBE_MAX_XDP_QS]; |
d49e286d | 683 | unsigned long *af_xdp_zc_qps; /* tracks AF_XDP ZC enabled rings */ |
33fdc82f | 684 | |
9a799d71 | 685 | /* TX */ |
4a0b9ca0 | 686 | struct ixgbe_ring *tx_ring[MAX_TX_QUEUES] ____cacheline_aligned_in_smp; |
9a799d71 | 687 | |
7ca3bc58 JB |
688 | u64 restart_queue; |
689 | u64 lsc_int; | |
46646e61 | 690 | u32 tx_timeout_count; |
7ca3bc58 | 691 | |
9a799d71 | 692 | /* RX */ |
46646e61 | 693 | struct ixgbe_ring *rx_ring[MAX_RX_QUEUES]; |
7f870475 GR |
694 | int num_rx_pools; /* == num_rx_queues in 82598 */ |
695 | int num_rx_queues_per_pool; /* 1 if 82598, can be many if 82599 */ | |
9a799d71 | 696 | u64 hw_csum_rx_error; |
e8e26350 | 697 | u64 hw_rx_no_dma_resources; |
46646e61 AD |
698 | u64 rsc_total_count; |
699 | u64 rsc_total_flush; | |
9a799d71 | 700 | u64 non_eop_descs; |
86e23494 | 701 | u32 alloc_rx_page; |
9a799d71 AK |
702 | u32 alloc_rx_page_failed; |
703 | u32 alloc_rx_buff_failed; | |
704 | ||
49c7ffbe | 705 | struct ixgbe_q_vector *q_vector[MAX_Q_VECTORS]; |
9a799d71 | 706 | |
46646e61 AD |
707 | /* DCB parameters */ |
708 | struct ieee_pfc *ixgbe_ieee_pfc; | |
709 | struct ieee_ets *ixgbe_ieee_ets; | |
710 | struct ixgbe_dcb_config dcb_cfg; | |
711 | struct ixgbe_dcb_config temp_dcb_cfg; | |
0efbf12b | 712 | u8 hw_tcs; |
46646e61 AD |
713 | u8 dcb_set_bitmap; |
714 | u8 dcbx_cap; | |
715 | enum ixgbe_fc_mode last_lfc_mode; | |
716 | ||
49c7ffbe AD |
717 | int num_q_vectors; /* current number of q_vectors for device */ |
718 | int max_q_vectors; /* true count of q_vectors for device */ | |
46646e61 AD |
719 | struct ixgbe_ring_feature ring_feature[RING_F_ARRAY_SIZE]; |
720 | struct msix_entry *msix_entries; | |
9a799d71 | 721 | |
da4dd0f7 PWJ |
722 | u32 test_icr; |
723 | struct ixgbe_ring test_tx_ring; | |
724 | struct ixgbe_ring test_rx_ring; | |
725 | ||
9a799d71 AK |
726 | /* structs defined in ixgbe_hw.h */ |
727 | struct ixgbe_hw hw; | |
728 | u16 msg_enable; | |
729 | struct ixgbe_hw_stats stats; | |
021230d4 | 730 | |
9a799d71 | 731 | u64 tx_busy; |
30efa5a3 | 732 | unsigned int tx_ring_count; |
33fdc82f | 733 | unsigned int xdp_ring_count; |
30efa5a3 | 734 | unsigned int rx_ring_count; |
cf8280ee JB |
735 | |
736 | u32 link_speed; | |
737 | bool link_up; | |
58e7cd24 | 738 | unsigned long sfp_poll_time; |
cf8280ee JB |
739 | unsigned long link_check_timeout; |
740 | ||
7086400d | 741 | struct timer_list service_timer; |
46646e61 AD |
742 | struct work_struct service_task; |
743 | ||
744 | struct hlist_head fdir_filter_list; | |
745 | unsigned long fdir_overflow; /* number of times ATR was backed off */ | |
746 | union ixgbe_atr_input fdir_mask; | |
747 | int fdir_filter_count; | |
c4cf55e5 PWJ |
748 | u32 fdir_pballoc; |
749 | u32 atr_sample_rate; | |
750 | spinlock_t fdir_perfect_lock; | |
46646e61 | 751 | |
d0ed8937 YZ |
752 | #ifdef IXGBE_FCOE |
753 | struct ixgbe_fcoe fcoe; | |
754 | #endif /* IXGBE_FCOE */ | |
2a1a091c | 755 | u8 __iomem *io_addr; /* Mainly for iounmap use */ |
e8e26350 | 756 | u32 wol; |
46646e61 | 757 | |
aa2bacb6 DS |
758 | u16 bridge_mode; |
759 | ||
73834aec | 760 | char eeprom_id[NVM_VER_SIZE]; |
c23f5b6b | 761 | u16 eeprom_cap; |
7f870475 | 762 | |
119fc60a | 763 | u32 interrupt_event; |
46646e61 | 764 | u32 led_reg; |
1a6c14a2 | 765 | |
3a6a4eda JK |
766 | struct ptp_clock *ptp_clock; |
767 | struct ptp_clock_info ptp_caps; | |
891dc082 JK |
768 | struct work_struct ptp_tx_work; |
769 | struct sk_buff *ptp_tx_skb; | |
93501d48 | 770 | struct hwtstamp_config tstamp_config; |
891dc082 | 771 | unsigned long ptp_tx_start; |
3a6a4eda | 772 | unsigned long last_overflow_check; |
6cb562d6 | 773 | unsigned long last_rx_ptp_check; |
eda183c2 | 774 | unsigned long last_rx_timestamp; |
3a6a4eda | 775 | spinlock_t tmreg_lock; |
a9763f3c MR |
776 | struct cyclecounter hw_cc; |
777 | struct timecounter hw_tc; | |
3a6a4eda | 778 | u32 base_incval; |
a9763f3c | 779 | u32 tx_hwtstamp_timeouts; |
4cc74c01 | 780 | u32 tx_hwtstamp_skipped; |
a9763f3c MR |
781 | u32 rx_hwtstamp_cleared; |
782 | void (*ptp_setup_sdp)(struct ixgbe_adapter *); | |
3a6a4eda | 783 | |
7f870475 GR |
784 | /* SR-IOV */ |
785 | DECLARE_BITMAP(active_vfs, IXGBE_MAX_VF_FUNCTIONS); | |
786 | unsigned int num_vfs; | |
787 | struct vf_data_storage *vfinfo; | |
ff4ab206 | 788 | int vf_rate_link_speed; |
a1cbb15c GR |
789 | struct vf_macvlans vf_mvs; |
790 | struct vf_macvlans *mv_list; | |
3e05334f | 791 | |
83c61fa9 GR |
792 | u32 timer_event_accumulator; |
793 | u32 vferr_refcount; | |
5d7daa35 | 794 | struct ixgbe_mac_addr *mac_table; |
3ca8bc6d DS |
795 | struct kobject *info_kobj; |
796 | #ifdef CONFIG_IXGBE_HWMON | |
03b77d81 | 797 | struct hwmon_buff *ixgbe_hwmon_buff; |
3ca8bc6d | 798 | #endif /* CONFIG_IXGBE_HWMON */ |
00949167 CS |
799 | #ifdef CONFIG_DEBUG_FS |
800 | struct dentry *ixgbe_dbg_adapter; | |
801 | #endif /*CONFIG_DEBUG_FS*/ | |
107d3018 AD |
802 | |
803 | u8 default_up; | |
4e039c16 AD |
804 | /* Bitmask indicating in use pools */ |
805 | DECLARE_BITMAP(fwd_bitmask, IXGBE_MAX_MACVLANS + 1); | |
dfaf891d | 806 | |
b82b17d9 | 807 | #define IXGBE_MAX_LINK_HANDLE 10 |
1cdaaf54 | 808 | struct ixgbe_jump_table *jump_tables[IXGBE_MAX_LINK_HANDLE]; |
db956ae8 | 809 | unsigned long tables; |
b82b17d9 | 810 | |
dfaf891d VZ |
811 | /* maximum number of RETA entries among all devices supported by ixgbe |
812 | * driver: currently it's x550 device in non-SRIOV mode | |
813 | */ | |
814 | #define IXGBE_MAX_RETA_ENTRIES 512 | |
815 | u8 rss_indir_tbl[IXGBE_MAX_RETA_ENTRIES]; | |
816 | ||
817 | #define IXGBE_RSS_KEY_SIZE 40 /* size of RSS Hash Key in bytes */ | |
3dfbfc7e | 818 | u32 *rss_key; |
34c822e2 | 819 | |
48e01e00 | 820 | #ifdef CONFIG_IXGBE_IPSEC |
34c822e2 | 821 | struct ixgbe_ipsec *ipsec; |
48e01e00 | 822 | #endif /* CONFIG_IXGBE_IPSEC */ |
1e53834c | 823 | spinlock_t vfs_lock; |
3e05334f AD |
824 | }; |
825 | ||
4fe81585 JX |
826 | static inline int ixgbe_determine_xdp_q_idx(int cpu) |
827 | { | |
828 | if (static_key_enabled(&ixgbe_xdp_locking_key)) | |
829 | return cpu % IXGBE_MAX_XDP_QS; | |
830 | else | |
831 | return cpu; | |
832 | } | |
833 | ||
834 | static inline | |
835 | struct ixgbe_ring *ixgbe_determine_xdp_ring(struct ixgbe_adapter *adapter) | |
836 | { | |
837 | int index = ixgbe_determine_xdp_q_idx(smp_processor_id()); | |
838 | ||
839 | return adapter->xdp_ring[index]; | |
840 | } | |
841 | ||
0f9b232b DS |
842 | static inline u8 ixgbe_max_rss_indices(struct ixgbe_adapter *adapter) |
843 | { | |
844 | switch (adapter->hw.mac.type) { | |
845 | case ixgbe_mac_82598EB: | |
846 | case ixgbe_mac_82599EB: | |
847 | case ixgbe_mac_X540: | |
848 | return IXGBE_MAX_RSS_INDICES; | |
849 | case ixgbe_mac_X550: | |
850 | case ixgbe_mac_X550EM_x: | |
49425dfc | 851 | case ixgbe_mac_x550em_a: |
0f9b232b DS |
852 | return IXGBE_MAX_RSS_INDICES_X550; |
853 | default: | |
854 | return 0; | |
855 | } | |
856 | } | |
857 | ||
3e05334f AD |
858 | struct ixgbe_fdir_filter { |
859 | struct hlist_node fdir_node; | |
860 | union ixgbe_atr_input filter; | |
861 | u16 sw_idx; | |
2a9ed5d1 | 862 | u64 action; |
9a799d71 AK |
863 | }; |
864 | ||
70e5576c | 865 | enum ixgbe_state_t { |
9a799d71 AK |
866 | __IXGBE_TESTING, |
867 | __IXGBE_RESETTING, | |
c4900be0 | 868 | __IXGBE_DOWN, |
41c62843 | 869 | __IXGBE_DISABLED, |
09f40aed | 870 | __IXGBE_REMOVING, |
7086400d | 871 | __IXGBE_SERVICE_SCHED, |
58cf663f | 872 | __IXGBE_SERVICE_INITED, |
7086400d | 873 | __IXGBE_IN_SFP_INIT, |
8fecf67c | 874 | __IXGBE_PTP_RUNNING, |
151b260c | 875 | __IXGBE_PTP_TX_IN_PROGRESS, |
57ca2a4f | 876 | __IXGBE_RESET_REQUESTED, |
9a799d71 AK |
877 | }; |
878 | ||
4c1975d7 AD |
879 | struct ixgbe_cb { |
880 | union { /* Union defining head/tail partner */ | |
881 | struct sk_buff *head; | |
882 | struct sk_buff *tail; | |
883 | }; | |
aa80175a | 884 | dma_addr_t dma; |
4c1975d7 | 885 | u16 append_cnt; |
f800326d | 886 | bool page_released; |
aa80175a | 887 | }; |
4c1975d7 | 888 | #define IXGBE_CB(skb) ((struct ixgbe_cb *)(skb)->cb) |
aa80175a | 889 | |
9a799d71 | 890 | enum ixgbe_boards { |
3957d63d | 891 | board_82598, |
e8e26350 | 892 | board_82599, |
fe15e8e1 | 893 | board_X540, |
6a14ee0c DS |
894 | board_X550, |
895 | board_X550EM_x, | |
8dc963e1 | 896 | board_x550em_x_fw, |
49425dfc | 897 | board_x550em_a, |
b3eb4e18 | 898 | board_x550em_a_fw, |
9a799d71 AK |
899 | }; |
900 | ||
37689010 MR |
901 | extern const struct ixgbe_info ixgbe_82598_info; |
902 | extern const struct ixgbe_info ixgbe_82599_info; | |
903 | extern const struct ixgbe_info ixgbe_X540_info; | |
904 | extern const struct ixgbe_info ixgbe_X550_info; | |
905 | extern const struct ixgbe_info ixgbe_X550EM_x_info; | |
8dc963e1 | 906 | extern const struct ixgbe_info ixgbe_x550em_x_fw_info; |
49425dfc | 907 | extern const struct ixgbe_info ixgbe_x550em_a_info; |
b3eb4e18 | 908 | extern const struct ixgbe_info ixgbe_x550em_a_fw_info; |
7a6b6f51 | 909 | #ifdef CONFIG_IXGBE_DCB |
3f40c74c | 910 | extern const struct dcbnl_rtnl_ops ixgbe_dcbnl_ops; |
2f90b865 | 911 | #endif |
9a799d71 AK |
912 | |
913 | extern char ixgbe_driver_name[]; | |
8af3c33f | 914 | #ifdef IXGBE_FCOE |
ea81875a | 915 | extern char ixgbe_default_device_descr[]; |
8af3c33f | 916 | #endif /* IXGBE_FCOE */ |
9a799d71 | 917 | |
6c211fe1 SA |
918 | int ixgbe_open(struct net_device *netdev); |
919 | int ixgbe_close(struct net_device *netdev); | |
5ccc921a JP |
920 | void ixgbe_up(struct ixgbe_adapter *adapter); |
921 | void ixgbe_down(struct ixgbe_adapter *adapter); | |
922 | void ixgbe_reinit_locked(struct ixgbe_adapter *adapter); | |
923 | void ixgbe_reset(struct ixgbe_adapter *adapter); | |
924 | void ixgbe_set_ethtool_ops(struct net_device *netdev); | |
92470808 | 925 | int ixgbe_setup_rx_resources(struct ixgbe_adapter *, struct ixgbe_ring *); |
5ccc921a JP |
926 | int ixgbe_setup_tx_resources(struct ixgbe_ring *); |
927 | void ixgbe_free_rx_resources(struct ixgbe_ring *); | |
928 | void ixgbe_free_tx_resources(struct ixgbe_ring *); | |
929 | void ixgbe_configure_rx_ring(struct ixgbe_adapter *, struct ixgbe_ring *); | |
930 | void ixgbe_configure_tx_ring(struct ixgbe_adapter *, struct ixgbe_ring *); | |
1918e937 AD |
931 | void ixgbe_disable_rx(struct ixgbe_adapter *adapter); |
932 | void ixgbe_disable_tx(struct ixgbe_adapter *adapter); | |
5ccc921a JP |
933 | void ixgbe_update_stats(struct ixgbe_adapter *adapter); |
934 | int ixgbe_init_interrupt_scheme(struct ixgbe_adapter *adapter); | |
740234f0 ET |
935 | bool ixgbe_wol_supported(struct ixgbe_adapter *adapter, u16 device_id, |
936 | u16 subdevice_id); | |
5d7daa35 JK |
937 | #ifdef CONFIG_PCI_IOV |
938 | void ixgbe_full_sync_mac_table(struct ixgbe_adapter *adapter); | |
939 | #endif | |
940 | int ixgbe_add_mac_filter(struct ixgbe_adapter *adapter, | |
c9f53e63 | 941 | const u8 *addr, u16 queue); |
5d7daa35 | 942 | int ixgbe_del_mac_filter(struct ixgbe_adapter *adapter, |
c9f53e63 | 943 | const u8 *addr, u16 queue); |
e1d0a2af | 944 | void ixgbe_update_pf_promisc_vlvf(struct ixgbe_adapter *adapter, u32 vid); |
5ccc921a JP |
945 | void ixgbe_clear_interrupt_scheme(struct ixgbe_adapter *adapter); |
946 | netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *, struct ixgbe_adapter *, | |
947 | struct ixgbe_ring *); | |
5ccc921a JP |
948 | void ixgbe_alloc_rx_buffers(struct ixgbe_ring *, u16); |
949 | void ixgbe_write_eitr(struct ixgbe_q_vector *); | |
950 | int ixgbe_poll(struct napi_struct *napi, int budget); | |
951 | int ethtool_ioctl(struct ifreq *ifr); | |
8f76c0f4 JJ |
952 | int ixgbe_reinit_fdir_tables_82599(struct ixgbe_hw *hw); |
953 | int ixgbe_init_fdir_signature_82599(struct ixgbe_hw *hw, u32 fdirctrl); | |
954 | int ixgbe_init_fdir_perfect_82599(struct ixgbe_hw *hw, u32 fdirctrl); | |
955 | int ixgbe_fdir_add_signature_filter_82599(struct ixgbe_hw *hw, | |
5ccc921a JP |
956 | union ixgbe_atr_hash_dword input, |
957 | union ixgbe_atr_hash_dword common, | |
958 | u8 queue); | |
8f76c0f4 | 959 | int ixgbe_fdir_set_input_mask_82599(struct ixgbe_hw *hw, |
5ccc921a | 960 | union ixgbe_atr_input *input_mask); |
8f76c0f4 | 961 | int ixgbe_fdir_write_perfect_filter_82599(struct ixgbe_hw *hw, |
5ccc921a JP |
962 | union ixgbe_atr_input *input, |
963 | u16 soft_id, u8 queue); | |
8f76c0f4 | 964 | int ixgbe_fdir_erase_perfect_filter_82599(struct ixgbe_hw *hw, |
5ccc921a JP |
965 | union ixgbe_atr_input *input, |
966 | u16 soft_id); | |
967 | void ixgbe_atr_compute_perfect_hash_82599(union ixgbe_atr_input *input, | |
968 | union ixgbe_atr_input *mask); | |
b82b17d9 JF |
969 | int ixgbe_update_ethtool_fdir_entry(struct ixgbe_adapter *adapter, |
970 | struct ixgbe_fdir_filter *input, | |
971 | u16 sw_idx); | |
5ccc921a | 972 | void ixgbe_set_rx_mode(struct net_device *netdev); |
8af3c33f | 973 | #ifdef CONFIG_IXGBE_DCB |
5ccc921a | 974 | void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter); |
8af3c33f | 975 | #endif |
5ccc921a JP |
976 | int ixgbe_setup_tc(struct net_device *dev, u8 tc); |
977 | void ixgbe_tx_ctxtdesc(struct ixgbe_ring *, u32, u32, u32, u32); | |
978 | void ixgbe_do_reset(struct net_device *netdev); | |
1210982b | 979 | #ifdef CONFIG_IXGBE_HWMON |
5ccc921a JP |
980 | void ixgbe_sysfs_exit(struct ixgbe_adapter *adapter); |
981 | int ixgbe_sysfs_init(struct ixgbe_adapter *adapter); | |
1210982b | 982 | #endif /* CONFIG_IXGBE_HWMON */ |
eacd73f7 | 983 | #ifdef IXGBE_FCOE |
5ccc921a JP |
984 | void ixgbe_configure_fcoe(struct ixgbe_adapter *adapter); |
985 | int ixgbe_fso(struct ixgbe_ring *tx_ring, struct ixgbe_tx_buffer *first, | |
986 | u8 *hdr_len); | |
987 | int ixgbe_fcoe_ddp(struct ixgbe_adapter *adapter, | |
988 | union ixgbe_adv_rx_desc *rx_desc, struct sk_buff *skb); | |
989 | int ixgbe_fcoe_ddp_get(struct net_device *netdev, u16 xid, | |
990 | struct scatterlist *sgl, unsigned int sgc); | |
991 | int ixgbe_fcoe_ddp_target(struct net_device *netdev, u16 xid, | |
992 | struct scatterlist *sgl, unsigned int sgc); | |
993 | int ixgbe_fcoe_ddp_put(struct net_device *netdev, u16 xid); | |
994 | int ixgbe_setup_fcoe_ddp_resources(struct ixgbe_adapter *adapter); | |
995 | void ixgbe_free_fcoe_ddp_resources(struct ixgbe_adapter *adapter); | |
996 | int ixgbe_fcoe_enable(struct net_device *netdev); | |
997 | int ixgbe_fcoe_disable(struct net_device *netdev); | |
5ccc921a JP |
998 | int ixgbe_fcoe_get_wwn(struct net_device *netdev, u64 *wwn, int type); |
999 | int ixgbe_fcoe_get_hbainfo(struct net_device *netdev, | |
1000 | struct netdev_fcoe_hbainfo *info); | |
1001 | u8 ixgbe_fcoe_get_tc(struct ixgbe_adapter *adapter); | |
eacd73f7 | 1002 | #endif /* IXGBE_FCOE */ |
00949167 | 1003 | #ifdef CONFIG_DEBUG_FS |
5ccc921a JP |
1004 | void ixgbe_dbg_adapter_init(struct ixgbe_adapter *adapter); |
1005 | void ixgbe_dbg_adapter_exit(struct ixgbe_adapter *adapter); | |
1006 | void ixgbe_dbg_init(void); | |
1007 | void ixgbe_dbg_exit(void); | |
33243fb0 JP |
1008 | #else |
1009 | static inline void ixgbe_dbg_adapter_init(struct ixgbe_adapter *adapter) {} | |
1010 | static inline void ixgbe_dbg_adapter_exit(struct ixgbe_adapter *adapter) {} | |
1011 | static inline void ixgbe_dbg_init(void) {} | |
1012 | static inline void ixgbe_dbg_exit(void) {} | |
00949167 | 1013 | #endif /* CONFIG_DEBUG_FS */ |
b2d96e0a AD |
1014 | static inline struct netdev_queue *txring_txq(const struct ixgbe_ring *ring) |
1015 | { | |
1016 | return netdev_get_tx_queue(ring->netdev, ring->queue_index); | |
1017 | } | |
1018 | ||
5ccc921a | 1019 | void ixgbe_ptp_init(struct ixgbe_adapter *adapter); |
9966d1ee | 1020 | void ixgbe_ptp_suspend(struct ixgbe_adapter *adapter); |
5ccc921a JP |
1021 | void ixgbe_ptp_stop(struct ixgbe_adapter *adapter); |
1022 | void ixgbe_ptp_overflow_check(struct ixgbe_adapter *adapter); | |
1023 | void ixgbe_ptp_rx_hang(struct ixgbe_adapter *adapter); | |
622a2ef5 | 1024 | void ixgbe_ptp_tx_hang(struct ixgbe_adapter *adapter); |
a9763f3c MR |
1025 | void ixgbe_ptp_rx_pktstamp(struct ixgbe_q_vector *, struct sk_buff *); |
1026 | void ixgbe_ptp_rx_rgtstamp(struct ixgbe_q_vector *, struct sk_buff *skb); | |
1027 | static inline void ixgbe_ptp_rx_hwtstamp(struct ixgbe_ring *rx_ring, | |
1028 | union ixgbe_adv_rx_desc *rx_desc, | |
1029 | struct sk_buff *skb) | |
1030 | { | |
1031 | if (unlikely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_TSIP))) { | |
1032 | ixgbe_ptp_rx_pktstamp(rx_ring->q_vector, skb); | |
1033 | return; | |
1034 | } | |
1035 | ||
1036 | if (unlikely(!ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_STAT_TS))) | |
1037 | return; | |
1038 | ||
1039 | ixgbe_ptp_rx_rgtstamp(rx_ring->q_vector, skb); | |
1040 | ||
1041 | /* Update the last_rx_timestamp timer in order to enable watchdog check | |
1042 | * for error case of latched timestamp on a dropped packet. | |
1043 | */ | |
1044 | rx_ring->last_rx_timestamp = jiffies; | |
1045 | } | |
1046 | ||
93501d48 JK |
1047 | int ixgbe_ptp_set_ts_config(struct ixgbe_adapter *adapter, struct ifreq *ifr); |
1048 | int ixgbe_ptp_get_ts_config(struct ixgbe_adapter *adapter, struct ifreq *ifr); | |
5ccc921a JP |
1049 | void ixgbe_ptp_start_cyclecounter(struct ixgbe_adapter *adapter); |
1050 | void ixgbe_ptp_reset(struct ixgbe_adapter *adapter); | |
a9763f3c | 1051 | void ixgbe_ptp_check_pps_event(struct ixgbe_adapter *adapter); |
da36b647 GR |
1052 | #ifdef CONFIG_PCI_IOV |
1053 | void ixgbe_sriov_reinit(struct ixgbe_adapter *adapter); | |
1054 | #endif | |
3a6a4eda | 1055 | |
2a47fa45 JF |
1056 | netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, |
1057 | struct ixgbe_adapter *adapter, | |
1058 | struct ixgbe_ring *tx_ring); | |
7f276efb | 1059 | u32 ixgbe_rss_indir_tbl_entries(struct ixgbe_adapter *adapter); |
d3aa9c9f | 1060 | void ixgbe_store_key(struct ixgbe_adapter *adapter); |
1c7cf078 | 1061 | void ixgbe_store_reta(struct ixgbe_adapter *adapter); |
8f76c0f4 | 1062 | int ixgbe_negotiate_fc(struct ixgbe_hw *hw, u32 adv_reg, u32 lp_reg, |
2916500d | 1063 | u32 adv_sym, u32 adv_asm, u32 lp_sym, u32 lp_asm); |
48e01e00 | 1064 | #ifdef CONFIG_IXGBE_IPSEC |
8bbbc5e9 | 1065 | void ixgbe_init_ipsec_offload(struct ixgbe_adapter *adapter); |
63a67fe2 | 1066 | void ixgbe_stop_ipsec_offload(struct ixgbe_adapter *adapter); |
6d73a154 | 1067 | void ixgbe_ipsec_restore(struct ixgbe_adapter *adapter); |
92103199 SN |
1068 | void ixgbe_ipsec_rx(struct ixgbe_ring *rx_ring, |
1069 | union ixgbe_adv_rx_desc *rx_desc, | |
1070 | struct sk_buff *skb); | |
59259470 SN |
1071 | int ixgbe_ipsec_tx(struct ixgbe_ring *tx_ring, struct ixgbe_tx_buffer *first, |
1072 | struct ixgbe_ipsec_tx_data *itd); | |
72698240 SN |
1073 | void ixgbe_ipsec_vf_clear(struct ixgbe_adapter *adapter, u32 vf); |
1074 | int ixgbe_ipsec_vf_add_sa(struct ixgbe_adapter *adapter, u32 *mbuf, u32 vf); | |
1075 | int ixgbe_ipsec_vf_del_sa(struct ixgbe_adapter *adapter, u32 *mbuf, u32 vf); | |
8bbbc5e9 | 1076 | #else |
72698240 SN |
1077 | static inline void ixgbe_init_ipsec_offload(struct ixgbe_adapter *adapter) { } |
1078 | static inline void ixgbe_stop_ipsec_offload(struct ixgbe_adapter *adapter) { } | |
1079 | static inline void ixgbe_ipsec_restore(struct ixgbe_adapter *adapter) { } | |
92103199 SN |
1080 | static inline void ixgbe_ipsec_rx(struct ixgbe_ring *rx_ring, |
1081 | union ixgbe_adv_rx_desc *rx_desc, | |
72698240 | 1082 | struct sk_buff *skb) { } |
59259470 SN |
1083 | static inline int ixgbe_ipsec_tx(struct ixgbe_ring *tx_ring, |
1084 | struct ixgbe_tx_buffer *first, | |
72698240 SN |
1085 | struct ixgbe_ipsec_tx_data *itd) { return 0; } |
1086 | static inline void ixgbe_ipsec_vf_clear(struct ixgbe_adapter *adapter, | |
1087 | u32 vf) { } | |
1088 | static inline int ixgbe_ipsec_vf_add_sa(struct ixgbe_adapter *adapter, | |
1089 | u32 *mbuf, u32 vf) { return -EACCES; } | |
1090 | static inline int ixgbe_ipsec_vf_del_sa(struct ixgbe_adapter *adapter, | |
1091 | u32 *mbuf, u32 vf) { return -EACCES; } | |
48e01e00 | 1092 | #endif /* CONFIG_IXGBE_IPSEC */ |
9ba095a6 JS |
1093 | |
1094 | static inline bool ixgbe_enabled_xdp_adapter(struct ixgbe_adapter *adapter) | |
1095 | { | |
1096 | return !!adapter->xdp_prog; | |
1097 | } | |
1098 | ||
9a799d71 | 1099 | #endif /* _IXGBE_H_ */ |