projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
i40e: Refactor force_wb and WB_ON_ITR functionality code
[linux-2.6-block.git] / drivers / net / ethernet / intel / i40e / i40e_txrx.c
CommitLineData
fd0a05ce
JB		 1/*******************************************************************************
2 *
3 * Intel Ethernet Controller XL710 Family Linux Driver
ecc6a239 4 * Copyright(c) 2013 - 2016 Intel Corporation.
fd0a05ce
JB		 5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * more details.
14 *
dc641b73
GR		 15 * You should have received a copy of the GNU General Public License along
16 * with this program. If not, see <http://www.gnu.org/licenses/>.
fd0a05ce
JB		 17 *
18 * The full GNU General Public License is included in this distribution in
19 * the file called "COPYING".
20 *
21 * Contact Information:
22 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
23 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
24 *
25 ******************************************************************************/
26
1c112a64 27#include <linux/prefetch.h>
a132af24 28#include <net/busy_poll.h>
fd0a05ce 29#include "i40e.h"
206812b5 30#include "i40e_prototype.h"
fd0a05ce
JB		 31
32static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size,
33 u32 td_tag)
34{
35 return cpu_to_le64(I40E_TX_DESC_DTYPE_DATA |
36 ((u64)td_cmd << I40E_TXD_QW1_CMD_SHIFT) |
37 ((u64)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) |
38 ((u64)size << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) |
39 ((u64)td_tag << I40E_TXD_QW1_L2TAG1_SHIFT));
40}
41
eaefbd06 42#define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS)
49d7d933 43#define I40E_FD_CLEAN_DELAY 10
fd0a05ce
JB		 44/**
45 * i40e_program_fdir_filter - Program a Flow Director filter
17a73f6b
JG		 46 * @fdir_data: Packet data that will be filter parameters
47 * @raw_packet: the pre-allocated packet buffer for FDir
b40c82e6 48 * @pf: The PF pointer
fd0a05ce
JB		 49 * @add: True for add/update, False for remove
50 **/
17a73f6b 51int i40e_program_fdir_filter(struct i40e_fdir_filter *fdir_data, u8 *raw_packet,
fd0a05ce
JB		 52 struct i40e_pf *pf, bool add)
53{
54 struct i40e_filter_program_desc *fdir_desc;
49d7d933 55 struct i40e_tx_buffer *tx_buf, *first;
fd0a05ce
JB		 56 struct i40e_tx_desc *tx_desc;
57 struct i40e_ring *tx_ring;
eaefbd06 58 unsigned int fpt, dcc;
fd0a05ce
JB		 59 struct i40e_vsi *vsi;
60 struct device *dev;
61 dma_addr_t dma;
62 u32 td_cmd = 0;
49d7d933 63 u16 delay = 0;
fd0a05ce
JB		 64 u16 i;
65
66 /* find existing FDIR VSI */
67 vsi = NULL;
505682cd 68 for (i = 0; i < pf->num_alloc_vsi; i++)
fd0a05ce
JB		 69 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR)
70 vsi = pf->vsi[i];
71 if (!vsi)
72 return -ENOENT;
73
9f65e15b 74 tx_ring = vsi->tx_rings[0];
fd0a05ce
JB		 75 dev = tx_ring->dev;
76
49d7d933
ASJ		 77 /* we need two descriptors to add/del a filter and we can wait */
78 do {
79 if (I40E_DESC_UNUSED(tx_ring) > 1)
80 break;
81 msleep_interruptible(1);
82 delay++;
83 } while (delay < I40E_FD_CLEAN_DELAY);
84
85 if (!(I40E_DESC_UNUSED(tx_ring) > 1))
86 return -EAGAIN;
87
17a73f6b
JG		 88 dma = dma_map_single(dev, raw_packet,
89 I40E_FDIR_MAX_RAW_PACKET_SIZE, DMA_TO_DEVICE);
fd0a05ce
JB		 90 if (dma_mapping_error(dev, dma))
91 goto dma_fail;
92
93 /* grab the next descriptor */
fc4ac67b
AD		 94 i = tx_ring->next_to_use;
95 fdir_desc = I40E_TX_FDIRDESC(tx_ring, i);
49d7d933
ASJ		 96 first = &tx_ring->tx_bi[i];
97 memset(first, 0, sizeof(struct i40e_tx_buffer));
fc4ac67b 98
49d7d933 99 tx_ring->next_to_use = ((i + 1) < tx_ring->count) ? i + 1 : 0;
fd0a05ce 100
eaefbd06
JB		 101 fpt = (fdir_data->q_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
102 I40E_TXD_FLTR_QW0_QINDEX_MASK;
fd0a05ce 103
eaefbd06
JB		 104 fpt |= (fdir_data->flex_off << I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT) &
105 I40E_TXD_FLTR_QW0_FLEXOFF_MASK;
fd0a05ce 106
eaefbd06
JB		 107 fpt |= (fdir_data->pctype << I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) &
108 I40E_TXD_FLTR_QW0_PCTYPE_MASK;
fd0a05ce
JB		 109
110 /* Use LAN VSI Id if not programmed by user */
111 if (fdir_data->dest_vsi == 0)
eaefbd06
JB		 112 fpt |= (pf->vsi[pf->lan_vsi]->id) <<
113 I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
fd0a05ce 114 else
eaefbd06
JB		 115 fpt |= ((u32)fdir_data->dest_vsi <<
116 I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT) &
117 I40E_TXD_FLTR_QW0_DEST_VSI_MASK;
118
eaefbd06 119 dcc = I40E_TX_DESC_DTYPE_FILTER_PROG;
fd0a05ce
JB		 120
121 if (add)
eaefbd06
JB		 122 dcc |= I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
123 I40E_TXD_FLTR_QW1_PCMD_SHIFT;
fd0a05ce 124 else
eaefbd06
JB		 125 dcc |= I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
126 I40E_TXD_FLTR_QW1_PCMD_SHIFT;
fd0a05ce 127
eaefbd06
JB		 128 dcc |= (fdir_data->dest_ctl << I40E_TXD_FLTR_QW1_DEST_SHIFT) &
129 I40E_TXD_FLTR_QW1_DEST_MASK;
fd0a05ce 130
eaefbd06
JB		 131 dcc |= (fdir_data->fd_status << I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT) &
132 I40E_TXD_FLTR_QW1_FD_STATUS_MASK;
fd0a05ce
JB		 133
134 if (fdir_data->cnt_index != 0) {
eaefbd06
JB		 135 dcc |= I40E_TXD_FLTR_QW1_CNT_ENA_MASK;
136 dcc |= ((u32)fdir_data->cnt_index <<
137 I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) &
433c47de 138 I40E_TXD_FLTR_QW1_CNTINDEX_MASK;
fd0a05ce
JB		 139 }
140
99753ea6
JB		 141 fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(fpt);
142 fdir_desc->rsvd = cpu_to_le32(0);
eaefbd06 143 fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dcc);
fd0a05ce
JB		 144 fdir_desc->fd_id = cpu_to_le32(fdir_data->fd_id);
145
146 /* Now program a dummy descriptor */
fc4ac67b
AD		 147 i = tx_ring->next_to_use;
148 tx_desc = I40E_TX_DESC(tx_ring, i);
298deef1 149 tx_buf = &tx_ring->tx_bi[i];
fc4ac67b 150
49d7d933
ASJ		 151 tx_ring->next_to_use = ((i + 1) < tx_ring->count) ? i + 1 : 0;
152
153 memset(tx_buf, 0, sizeof(struct i40e_tx_buffer));
fd0a05ce 154
298deef1 155 /* record length, and DMA address */
17a73f6b 156 dma_unmap_len_set(tx_buf, len, I40E_FDIR_MAX_RAW_PACKET_SIZE);
298deef1
ASJ		 157 dma_unmap_addr_set(tx_buf, dma, dma);
158
fd0a05ce 159 tx_desc->buffer_addr = cpu_to_le64(dma);
eaefbd06 160 td_cmd = I40E_TXD_CMD | I40E_TX_DESC_CMD_DUMMY;
fd0a05ce 161
49d7d933
ASJ		 162 tx_buf->tx_flags = I40E_TX_FLAGS_FD_SB;
163 tx_buf->raw_buf = (void *)raw_packet;
164
fd0a05ce 165 tx_desc->cmd_type_offset_bsz =
17a73f6b 166 build_ctob(td_cmd, 0, I40E_FDIR_MAX_RAW_PACKET_SIZE, 0);
fd0a05ce 167
fd0a05ce 168 /* Force memory writes to complete before letting h/w
49d7d933 169 * know there are new descriptors to fetch.
fd0a05ce
JB		 170 */
171 wmb();
172
fc4ac67b 173 /* Mark the data descriptor to be watched */
49d7d933 174 first->next_to_watch = tx_desc;
fc4ac67b 175
fd0a05ce
JB		 176 writel(tx_ring->next_to_use, tx_ring->tail);
177 return 0;
178
179dma_fail:
180 return -1;
181}
182
17a73f6b
JG		 183#define IP_HEADER_OFFSET 14
184#define I40E_UDPIP_DUMMY_PACKET_LEN 42
185/**
186 * i40e_add_del_fdir_udpv4 - Add/Remove UDPv4 filters
187 * @vsi: pointer to the targeted VSI
188 * @fd_data: the flow director data required for the FDir descriptor
17a73f6b
JG		 189 * @add: true adds a filter, false removes it
190 *
191 * Returns 0 if the filters were successfully added or removed
192 **/
193static int i40e_add_del_fdir_udpv4(struct i40e_vsi *vsi,
194 struct i40e_fdir_filter *fd_data,
49d7d933 195 bool add)
17a73f6b
JG		 196{
197 struct i40e_pf *pf = vsi->back;
198 struct udphdr *udp;
199 struct iphdr *ip;
200 bool err = false;
49d7d933 201 u8 *raw_packet;
17a73f6b 202 int ret;
17a73f6b
JG		 203 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0,
204 0x45, 0, 0, 0x1c, 0, 0, 0x40, 0, 0x40, 0x11, 0, 0, 0, 0, 0, 0,
205 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
206
49d7d933
ASJ		 207 raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL);
208 if (!raw_packet)
209 return -ENOMEM;
17a73f6b
JG		 210 memcpy(raw_packet, packet, I40E_UDPIP_DUMMY_PACKET_LEN);
211
212 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET);
213 udp = (struct udphdr *)(raw_packet + IP_HEADER_OFFSET
214 + sizeof(struct iphdr));
215
216 ip->daddr = fd_data->dst_ip[0];
217 udp->dest = fd_data->dst_port;
218 ip->saddr = fd_data->src_ip[0];
219 udp->source = fd_data->src_port;
220
b2d36c03
KS		 221 fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_UDP;
222 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
223 if (ret) {
224 dev_info(&pf->pdev->dev,
e99bdd39
CW		 225 "PCTYPE:%d, Filter command send failed for fd_id:%d (ret = %d)\n",
226 fd_data->pctype, fd_data->fd_id, ret);
b2d36c03 227 err = true;
4205d379 228 } else if (I40E_DEBUG_FD & pf->hw.debug_mask) {
f7233c54
ASJ		 229 if (add)
230 dev_info(&pf->pdev->dev,
231 "Filter OK for PCTYPE %d loc = %d\n",
232 fd_data->pctype, fd_data->fd_id);
233 else
234 dev_info(&pf->pdev->dev,
235 "Filter deleted for PCTYPE %d loc = %d\n",
236 fd_data->pctype, fd_data->fd_id);
17a73f6b 237 }
a42e7a36
KP		 238 if (err)
239 kfree(raw_packet);
240
17a73f6b
JG		 241 return err ? -EOPNOTSUPP : 0;
242}
243
244#define I40E_TCPIP_DUMMY_PACKET_LEN 54
245/**
246 * i40e_add_del_fdir_tcpv4 - Add/Remove TCPv4 filters
247 * @vsi: pointer to the targeted VSI
248 * @fd_data: the flow director data required for the FDir descriptor
17a73f6b
JG		 249 * @add: true adds a filter, false removes it
250 *
251 * Returns 0 if the filters were successfully added or removed
252 **/
253static int i40e_add_del_fdir_tcpv4(struct i40e_vsi *vsi,
254 struct i40e_fdir_filter *fd_data,
49d7d933 255 bool add)
17a73f6b
JG		 256{
257 struct i40e_pf *pf = vsi->back;
258 struct tcphdr *tcp;
259 struct iphdr *ip;
260 bool err = false;
49d7d933 261 u8 *raw_packet;
17a73f6b
JG		 262 int ret;
263 /* Dummy packet */
264 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0,
265 0x45, 0, 0, 0x28, 0, 0, 0x40, 0, 0x40, 0x6, 0, 0, 0, 0, 0, 0,
266 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x80, 0x11,
267 0x0, 0x72, 0, 0, 0, 0};
268
49d7d933
ASJ		 269 raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL);
270 if (!raw_packet)
271 return -ENOMEM;
17a73f6b
JG		 272 memcpy(raw_packet, packet, I40E_TCPIP_DUMMY_PACKET_LEN);
273
274 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET);
275 tcp = (struct tcphdr *)(raw_packet + IP_HEADER_OFFSET
276 + sizeof(struct iphdr));
277
278 ip->daddr = fd_data->dst_ip[0];
279 tcp->dest = fd_data->dst_port;
280 ip->saddr = fd_data->src_ip[0];
281 tcp->source = fd_data->src_port;
282
283 if (add) {
1e1be8f6 284 pf->fd_tcp_rule++;
17a73f6b 285 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) {
2e4875e3
ASJ		 286 if (I40E_DEBUG_FD & pf->hw.debug_mask)
287 dev_info(&pf->pdev->dev, "Forcing ATR off, sideband rules for TCP/IPv4 flow being applied\n");
17a73f6b
JG		 288 pf->flags &= ~I40E_FLAG_FD_ATR_ENABLED;
289 }
1e1be8f6
ASJ		 290 } else {
291 pf->fd_tcp_rule = (pf->fd_tcp_rule > 0) ?
292 (pf->fd_tcp_rule - 1) : 0;
293 if (pf->fd_tcp_rule == 0) {
294 pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
2e4875e3
ASJ		 295 if (I40E_DEBUG_FD & pf->hw.debug_mask)
296 dev_info(&pf->pdev->dev, "ATR re-enabled due to no sideband TCP/IPv4 rules\n");
1e1be8f6 297 }
17a73f6b
JG		 298 }
299
b2d36c03 300 fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_TCP;
17a73f6b
JG		 301 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
302
303 if (ret) {
304 dev_info(&pf->pdev->dev,
e99bdd39
CW		 305 "PCTYPE:%d, Filter command send failed for fd_id:%d (ret = %d)\n",
306 fd_data->pctype, fd_data->fd_id, ret);
17a73f6b 307 err = true;
4205d379 308 } else if (I40E_DEBUG_FD & pf->hw.debug_mask) {
f7233c54
ASJ		 309 if (add)
310 dev_info(&pf->pdev->dev, "Filter OK for PCTYPE %d loc = %d)\n",
311 fd_data->pctype, fd_data->fd_id);
312 else
313 dev_info(&pf->pdev->dev,
314 "Filter deleted for PCTYPE %d loc = %d\n",
315 fd_data->pctype, fd_data->fd_id);
17a73f6b
JG		 316 }
317
a42e7a36
KP		 318 if (err)
319 kfree(raw_packet);
320
17a73f6b
JG		 321 return err ? -EOPNOTSUPP : 0;
322}
323
324/**
325 * i40e_add_del_fdir_sctpv4 - Add/Remove SCTPv4 Flow Director filters for
326 * a specific flow spec
327 * @vsi: pointer to the targeted VSI
328 * @fd_data: the flow director data required for the FDir descriptor
17a73f6b
JG		 329 * @add: true adds a filter, false removes it
330 *
4eeb1fff 331 * Returns 0 if the filters were successfully added or removed
17a73f6b
JG		 332 **/
333static int i40e_add_del_fdir_sctpv4(struct i40e_vsi *vsi,
334 struct i40e_fdir_filter *fd_data,
49d7d933 335 bool add)
17a73f6b
JG		 336{
337 return -EOPNOTSUPP;
338}
339
340#define I40E_IP_DUMMY_PACKET_LEN 34
341/**
342 * i40e_add_del_fdir_ipv4 - Add/Remove IPv4 Flow Director filters for
343 * a specific flow spec
344 * @vsi: pointer to the targeted VSI
345 * @fd_data: the flow director data required for the FDir descriptor
17a73f6b
JG		 346 * @add: true adds a filter, false removes it
347 *
348 * Returns 0 if the filters were successfully added or removed
349 **/
350static int i40e_add_del_fdir_ipv4(struct i40e_vsi *vsi,
351 struct i40e_fdir_filter *fd_data,
49d7d933 352 bool add)
17a73f6b
JG		 353{
354 struct i40e_pf *pf = vsi->back;
355 struct iphdr *ip;
356 bool err = false;
49d7d933 357 u8 *raw_packet;
17a73f6b
JG		 358 int ret;
359 int i;
360 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0,
361 0x45, 0, 0, 0x14, 0, 0, 0x40, 0, 0x40, 0x10, 0, 0, 0, 0, 0, 0,
362 0, 0, 0, 0};
363
17a73f6b
JG		 364 for (i = I40E_FILTER_PCTYPE_NONF_IPV4_OTHER;
365 i <= I40E_FILTER_PCTYPE_FRAG_IPV4; i++) {
49d7d933
ASJ		 366 raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL);
367 if (!raw_packet)
368 return -ENOMEM;
369 memcpy(raw_packet, packet, I40E_IP_DUMMY_PACKET_LEN);
370 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET);
371
372 ip->saddr = fd_data->src_ip[0];
373 ip->daddr = fd_data->dst_ip[0];
374 ip->protocol = 0;
375
17a73f6b
JG		 376 fd_data->pctype = i;
377 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
378
379 if (ret) {
380 dev_info(&pf->pdev->dev,
e99bdd39
CW		 381 "PCTYPE:%d, Filter command send failed for fd_id:%d (ret = %d)\n",
382 fd_data->pctype, fd_data->fd_id, ret);
17a73f6b 383 err = true;
4205d379 384 } else if (I40E_DEBUG_FD & pf->hw.debug_mask) {
f7233c54
ASJ		 385 if (add)
386 dev_info(&pf->pdev->dev,
387 "Filter OK for PCTYPE %d loc = %d\n",
388 fd_data->pctype, fd_data->fd_id);
389 else
390 dev_info(&pf->pdev->dev,
391 "Filter deleted for PCTYPE %d loc = %d\n",
392 fd_data->pctype, fd_data->fd_id);
17a73f6b
JG		 393 }
394 }
395
a42e7a36
KP		 396 if (err)
397 kfree(raw_packet);
398
17a73f6b
JG		 399 return err ? -EOPNOTSUPP : 0;
400}
401
402/**
403 * i40e_add_del_fdir - Build raw packets to add/del fdir filter
404 * @vsi: pointer to the targeted VSI
405 * @cmd: command to get or set RX flow classification rules
406 * @add: true adds a filter, false removes it
407 *
408 **/
409int i40e_add_del_fdir(struct i40e_vsi *vsi,
410 struct i40e_fdir_filter *input, bool add)
411{
412 struct i40e_pf *pf = vsi->back;
17a73f6b
JG		 413 int ret;
414
17a73f6b
JG		 415 switch (input->flow_type & ~FLOW_EXT) {
416 case TCP_V4_FLOW:
49d7d933 417 ret = i40e_add_del_fdir_tcpv4(vsi, input, add);
17a73f6b
JG		 418 break;
419 case UDP_V4_FLOW:
49d7d933 420 ret = i40e_add_del_fdir_udpv4(vsi, input, add);
17a73f6b
JG		 421 break;
422 case SCTP_V4_FLOW:
49d7d933 423 ret = i40e_add_del_fdir_sctpv4(vsi, input, add);
17a73f6b
JG		 424 break;
425 case IPV4_FLOW:
49d7d933 426 ret = i40e_add_del_fdir_ipv4(vsi, input, add);
17a73f6b
JG		 427 break;
428 case IP_USER_FLOW:
429 switch (input->ip4_proto) {
430 case IPPROTO_TCP:
49d7d933 431 ret = i40e_add_del_fdir_tcpv4(vsi, input, add);
17a73f6b
JG		 432 break;
433 case IPPROTO_UDP:
49d7d933 434 ret = i40e_add_del_fdir_udpv4(vsi, input, add);
17a73f6b
JG		 435 break;
436 case IPPROTO_SCTP:
49d7d933 437 ret = i40e_add_del_fdir_sctpv4(vsi, input, add);
17a73f6b
JG		 438 break;
439 default:
49d7d933 440 ret = i40e_add_del_fdir_ipv4(vsi, input, add);
17a73f6b
JG		 441 break;
442 }
443 break;
444 default:
c5ffe7e1 445 dev_info(&pf->pdev->dev, "Could not specify spec type %d\n",
17a73f6b
JG		 446 input->flow_type);
447 ret = -EINVAL;
448 }
449
49d7d933 450 /* The buffer allocated here is freed by the i40e_clean_tx_ring() */
17a73f6b
JG		 451 return ret;
452}
453
fd0a05ce
JB		 454/**
455 * i40e_fd_handle_status - check the Programming Status for FD
456 * @rx_ring: the Rx ring for this descriptor
55a5e60b 457 * @rx_desc: the Rx descriptor for programming Status, not a packet descriptor.
fd0a05ce
JB		 458 * @prog_id: the id originally used for programming
459 *
460 * This is used to verify if the FD programming or invalidation
461 * requested by SW to the HW is successful or not and take actions accordingly.
462 **/
55a5e60b
ASJ		 463static void i40e_fd_handle_status(struct i40e_ring *rx_ring,
464 union i40e_rx_desc *rx_desc, u8 prog_id)
fd0a05ce 465{
55a5e60b
ASJ		 466 struct i40e_pf *pf = rx_ring->vsi->back;
467 struct pci_dev *pdev = pf->pdev;
468 u32 fcnt_prog, fcnt_avail;
fd0a05ce 469 u32 error;
55a5e60b 470 u64 qw;
fd0a05ce 471
55a5e60b 472 qw = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
fd0a05ce
JB		 473 error = (qw & I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >>
474 I40E_RX_PROG_STATUS_DESC_QW1_ERROR_SHIFT;
475
41a1d04b 476 if (error == BIT(I40E_RX_PROG_STATUS_DESC_FD_TBL_FULL_SHIFT)) {
3487b6c3 477 pf->fd_inv = le32_to_cpu(rx_desc->wb.qword0.hi_dword.fd_id);
f7233c54
ASJ		 478 if ((rx_desc->wb.qword0.hi_dword.fd_id != 0) ||
479 (I40E_DEBUG_FD & pf->hw.debug_mask))
480 dev_warn(&pdev->dev, "ntuple filter loc = %d, could not be added\n",
3487b6c3 481 pf->fd_inv);
55a5e60b 482
04294e38
ASJ		 483 /* Check if the programming error is for ATR.
484 * If so, auto disable ATR and set a state for
485 * flush in progress. Next time we come here if flush is in
486 * progress do nothing, once flush is complete the state will
487 * be cleared.
488 */
489 if (test_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state))
490 return;
491
1e1be8f6
ASJ		 492 pf->fd_add_err++;
493 /* store the current atr filter count */
494 pf->fd_atr_cnt = i40e_get_current_atr_cnt(pf);
495
04294e38
ASJ		 496 if ((rx_desc->wb.qword0.hi_dword.fd_id == 0) &&
497 (pf->auto_disable_flags & I40E_FLAG_FD_SB_ENABLED)) {
498 pf->auto_disable_flags |= I40E_FLAG_FD_ATR_ENABLED;
499 set_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state);
500 }
501
55a5e60b 502 /* filter programming failed most likely due to table full */
04294e38 503 fcnt_prog = i40e_get_global_fd_count(pf);
12957388 504 fcnt_avail = pf->fdir_pf_filter_count;
55a5e60b
ASJ		 505 /* If ATR is running fcnt_prog can quickly change,
506 * if we are very close to full, it makes sense to disable
507 * FD ATR/SB and then re-enable it when there is room.
508 */
509 if (fcnt_prog >= (fcnt_avail - I40E_FDIR_BUFFER_FULL_MARGIN)) {
1e1be8f6 510 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
b814ba65 511 !(pf->auto_disable_flags &
b814ba65 512 I40E_FLAG_FD_SB_ENABLED)) {
2e4875e3
ASJ		 513 if (I40E_DEBUG_FD & pf->hw.debug_mask)
514 dev_warn(&pdev->dev, "FD filter space full, new ntuple rules will not be added\n");
55a5e60b
ASJ		 515 pf->auto_disable_flags |=
516 I40E_FLAG_FD_SB_ENABLED;
55a5e60b 517 }
55a5e60b 518 }
41a1d04b 519 } else if (error == BIT(I40E_RX_PROG_STATUS_DESC_NO_FD_ENTRY_SHIFT)) {
13c2884f 520 if (I40E_DEBUG_FD & pf->hw.debug_mask)
e99bdd39 521 dev_info(&pdev->dev, "ntuple filter fd_id = %d, could not be removed\n",
13c2884f 522 rx_desc->wb.qword0.hi_dword.fd_id);
55a5e60b 523 }
fd0a05ce
JB		 524}
525
526/**
a5e9c572 527 * i40e_unmap_and_free_tx_resource - Release a Tx buffer
fd0a05ce
JB		 528 * @ring: the ring that owns the buffer
529 * @tx_buffer: the buffer to free
530 **/
a5e9c572
AD		 531static void i40e_unmap_and_free_tx_resource(struct i40e_ring *ring,
532 struct i40e_tx_buffer *tx_buffer)
fd0a05ce 533{
a5e9c572 534 if (tx_buffer->skb) {
a42e7a36 535 dev_kfree_skb_any(tx_buffer->skb);
a5e9c572 536 if (dma_unmap_len(tx_buffer, len))
fd0a05ce 537 dma_unmap_single(ring->dev,
35a1e2ad
AD		 538 dma_unmap_addr(tx_buffer, dma),
539 dma_unmap_len(tx_buffer, len),
fd0a05ce 540 DMA_TO_DEVICE);
a5e9c572
AD		 541 } else if (dma_unmap_len(tx_buffer, len)) {
542 dma_unmap_page(ring->dev,
543 dma_unmap_addr(tx_buffer, dma),
544 dma_unmap_len(tx_buffer, len),
545 DMA_TO_DEVICE);
fd0a05ce 546 }
a42e7a36
KP		 547
548 if (tx_buffer->tx_flags & I40E_TX_FLAGS_FD_SB)
549 kfree(tx_buffer->raw_buf);
550
a5e9c572
AD		 551 tx_buffer->next_to_watch = NULL;
552 tx_buffer->skb = NULL;
35a1e2ad 553 dma_unmap_len_set(tx_buffer, len, 0);
a5e9c572 554 /* tx_buffer must be completely set up in the transmit path */
fd0a05ce
JB		 555}
556
557/**
558 * i40e_clean_tx_ring - Free any empty Tx buffers
559 * @tx_ring: ring to be cleaned
560 **/
561void i40e_clean_tx_ring(struct i40e_ring *tx_ring)
562{
fd0a05ce
JB		 563 unsigned long bi_size;
564 u16 i;
565
566 /* ring already cleared, nothing to do */
567 if (!tx_ring->tx_bi)
568 return;
569
570 /* Free all the Tx ring sk_buffs */
a5e9c572
AD		 571 for (i = 0; i < tx_ring->count; i++)
572 i40e_unmap_and_free_tx_resource(tx_ring, &tx_ring->tx_bi[i]);
fd0a05ce
JB		 573
574 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
575 memset(tx_ring->tx_bi, 0, bi_size);
576
577 /* Zero out the descriptor ring */
578 memset(tx_ring->desc, 0, tx_ring->size);
579
580 tx_ring->next_to_use = 0;
581 tx_ring->next_to_clean = 0;
7070ce0a
AD		 582
583 if (!tx_ring->netdev)
584 return;
585
586 /* cleanup Tx queue statistics */
587 netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev,
588 tx_ring->queue_index));
fd0a05ce
JB		 589}
590
591/**
592 * i40e_free_tx_resources - Free Tx resources per queue
593 * @tx_ring: Tx descriptor ring for a specific queue
594 *
595 * Free all transmit software resources
596 **/
597void i40e_free_tx_resources(struct i40e_ring *tx_ring)
598{
599 i40e_clean_tx_ring(tx_ring);
600 kfree(tx_ring->tx_bi);
601 tx_ring->tx_bi = NULL;
602
603 if (tx_ring->desc) {
604 dma_free_coherent(tx_ring->dev, tx_ring->size,
605 tx_ring->desc, tx_ring->dma);
606 tx_ring->desc = NULL;
607 }
608}
609
610/**
611 * i40e_get_tx_pending - how many tx descriptors not processed
612 * @tx_ring: the ring of descriptors
613 *
614 * Since there is no access to the ring head register
615 * in XL710, we need to use our local copies
616 **/
b03a8c1f 617u32 i40e_get_tx_pending(struct i40e_ring *ring)
fd0a05ce 618{
a68de58d
JB		 619 u32 head, tail;
620
621 head = i40e_get_head(ring);
622 tail = readl(ring->tail);
623
624 if (head != tail)
625 return (head < tail) ?
626 tail - head : (tail + ring->count - head);
627
628 return 0;
fd0a05ce
JB		 629}
630
d91649f5
JB		 631#define WB_STRIDE 0x3
632
fd0a05ce
JB		 633/**
634 * i40e_clean_tx_irq - Reclaim resources after transmit completes
635 * @tx_ring: tx ring to clean
636 * @budget: how many cleans we're allowed
637 *
638 * Returns true if there's any budget left (e.g. the clean is finished)
639 **/
640static bool i40e_clean_tx_irq(struct i40e_ring *tx_ring, int budget)
641{
642 u16 i = tx_ring->next_to_clean;
643 struct i40e_tx_buffer *tx_buf;
1943d8ba 644 struct i40e_tx_desc *tx_head;
fd0a05ce
JB		 645 struct i40e_tx_desc *tx_desc;
646 unsigned int total_packets = 0;
647 unsigned int total_bytes = 0;
648
649 tx_buf = &tx_ring->tx_bi[i];
650 tx_desc = I40E_TX_DESC(tx_ring, i);
a5e9c572 651 i -= tx_ring->count;
fd0a05ce 652
1943d8ba
JB		 653 tx_head = I40E_TX_DESC(tx_ring, i40e_get_head(tx_ring));
654
a5e9c572
AD		 655 do {
656 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
fd0a05ce
JB		 657
658 /* if next_to_watch is not set then there is no work pending */
659 if (!eop_desc)
660 break;
661
a5e9c572
AD		 662 /* prevent any other reads prior to eop_desc */
663 read_barrier_depends();
664
1943d8ba
JB		 665 /* we have caught up to head, no work left to do */
666 if (tx_head == tx_desc)
fd0a05ce
JB		 667 break;
668
c304fdac 669 /* clear next_to_watch to prevent false hangs */
fd0a05ce 670 tx_buf->next_to_watch = NULL;
fd0a05ce 671
a5e9c572
AD		 672 /* update the statistics for this packet */
673 total_bytes += tx_buf->bytecount;
674 total_packets += tx_buf->gso_segs;
fd0a05ce 675
a5e9c572 676 /* free the skb */
a81fb049 677 dev_consume_skb_any(tx_buf->skb);
fd0a05ce 678
a5e9c572
AD		 679 /* unmap skb header data */
680 dma_unmap_single(tx_ring->dev,
681 dma_unmap_addr(tx_buf, dma),
682 dma_unmap_len(tx_buf, len),
683 DMA_TO_DEVICE);
fd0a05ce 684
a5e9c572
AD		 685 /* clear tx_buffer data */
686 tx_buf->skb = NULL;
687 dma_unmap_len_set(tx_buf, len, 0);
fd0a05ce 688
a5e9c572
AD		 689 /* unmap remaining buffers */
690 while (tx_desc != eop_desc) {
fd0a05ce
JB		 691
692 tx_buf++;
693 tx_desc++;
694 i++;
a5e9c572
AD		 695 if (unlikely(!i)) {
696 i -= tx_ring->count;
fd0a05ce
JB		 697 tx_buf = tx_ring->tx_bi;
698 tx_desc = I40E_TX_DESC(tx_ring, 0);
699 }
fd0a05ce 700
a5e9c572
AD		 701 /* unmap any remaining paged data */
702 if (dma_unmap_len(tx_buf, len)) {
703 dma_unmap_page(tx_ring->dev,
704 dma_unmap_addr(tx_buf, dma),
705 dma_unmap_len(tx_buf, len),
706 DMA_TO_DEVICE);
707 dma_unmap_len_set(tx_buf, len, 0);
708 }
709 }
710
711 /* move us one more past the eop_desc for start of next pkt */
712 tx_buf++;
713 tx_desc++;
714 i++;
715 if (unlikely(!i)) {
716 i -= tx_ring->count;
717 tx_buf = tx_ring->tx_bi;
718 tx_desc = I40E_TX_DESC(tx_ring, 0);
719 }
720
016890b9
JB		 721 prefetch(tx_desc);
722
a5e9c572
AD		 723 /* update budget accounting */
724 budget--;
725 } while (likely(budget));
726
727 i += tx_ring->count;
fd0a05ce 728 tx_ring->next_to_clean = i;
980e9b11 729 u64_stats_update_begin(&tx_ring->syncp);
a114d0a6
AD		 730 tx_ring->stats.bytes += total_bytes;
731 tx_ring->stats.packets += total_packets;
980e9b11 732 u64_stats_update_end(&tx_ring->syncp);
fd0a05ce
JB		 733 tx_ring->q_vector->tx.total_bytes += total_bytes;
734 tx_ring->q_vector->tx.total_packets += total_packets;
a5e9c572 735
58044743
AS		 736 if (tx_ring->flags & I40E_TXR_FLAGS_WB_ON_ITR) {
737 unsigned int j = 0;
738
739 /* check to see if there are < 4 descriptors
740 * waiting to be written back, then kick the hardware to force
741 * them to be written back in case we stay in NAPI.
742 * In this mode on X722 we do not enable Interrupt.
743 */
744 j = i40e_get_tx_pending(tx_ring);
745
746 if (budget &&
747 ((j / (WB_STRIDE + 1)) == 0) && (j != 0) &&
748 !test_bit(__I40E_DOWN, &tx_ring->vsi->state) &&
749 (I40E_DESC_UNUSED(tx_ring) != tx_ring->count))
750 tx_ring->arm_wb = true;
751 }
d91649f5 752
7070ce0a
AD		 753 netdev_tx_completed_queue(netdev_get_tx_queue(tx_ring->netdev,
754 tx_ring->queue_index),
755 total_packets, total_bytes);
756
fd0a05ce
JB		 757#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
758 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
759 (I40E_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
760 /* Make sure that anybody stopping the queue after this
761 * sees the new next_to_clean.
762 */
763 smp_mb();
764 if (__netif_subqueue_stopped(tx_ring->netdev,
765 tx_ring->queue_index) &&
766 !test_bit(__I40E_DOWN, &tx_ring->vsi->state)) {
767 netif_wake_subqueue(tx_ring->netdev,
768 tx_ring->queue_index);
769 ++tx_ring->tx_stats.restart_queue;
770 }
771 }
772
d91649f5
JB		 773 return !!budget;
774}
775
776/**
ecc6a239 777 * i40e_enable_wb_on_itr - Arm hardware to do a wb, interrupts are not enabled
d91649f5 778 * @vsi: the VSI we care about
ecc6a239 779 * @q_vector: the vector on which to enable writeback
d91649f5
JB		 780 *
781 **/
ecc6a239
ASJ		 782static void i40e_enable_wb_on_itr(struct i40e_vsi *vsi,
783 struct i40e_q_vector *q_vector)
d91649f5 784{
8e0764b4 785 u16 flags = q_vector->tx.ring[0].flags;
ecc6a239 786 u32 val;
8e0764b4 787
ecc6a239
ASJ		 788 if (!(flags & I40E_TXR_FLAGS_WB_ON_ITR))
789 return;
8e0764b4 790
ecc6a239
ASJ		 791 if (q_vector->arm_wb_state)
792 return;
8e0764b4 793
ecc6a239
ASJ		 794 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) {
795 val = I40E_PFINT_DYN_CTLN_WB_ON_ITR_MASK |
796 I40E_PFINT_DYN_CTLN_ITR_INDX_MASK; /* set noitr */
a3d772a3 797
ecc6a239
ASJ		 798 wr32(&vsi->back->hw,
799 I40E_PFINT_DYN_CTLN(q_vector->v_idx + vsi->base_vector - 1),
800 val);
801 } else {
802 val = I40E_PFINT_DYN_CTL0_WB_ON_ITR_MASK |
803 I40E_PFINT_DYN_CTL0_ITR_INDX_MASK; /* set noitr */
a3d772a3 804
ecc6a239
ASJ		 805 wr32(&vsi->back->hw, I40E_PFINT_DYN_CTL0, val);
806 }
807 q_vector->arm_wb_state = true;
808}
809
810/**
811 * i40e_force_wb - Issue SW Interrupt so HW does a wb
812 * @vsi: the VSI we care about
813 * @q_vector: the vector on which to force writeback
814 *
815 **/
816void i40e_force_wb(struct i40e_vsi *vsi, struct i40e_q_vector *q_vector)
817{
818 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) {
8e0764b4
ASJ		 819 u32 val = I40E_PFINT_DYN_CTLN_INTENA_MASK |
820 I40E_PFINT_DYN_CTLN_ITR_INDX_MASK | /* set noitr */
821 I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK |
822 I40E_PFINT_DYN_CTLN_SW_ITR_INDX_ENA_MASK;
823 /* allow 00 to be written to the index */
824
825 wr32(&vsi->back->hw,
826 I40E_PFINT_DYN_CTLN(q_vector->v_idx +
827 vsi->base_vector - 1), val);
828 } else {
829 u32 val = I40E_PFINT_DYN_CTL0_INTENA_MASK |
830 I40E_PFINT_DYN_CTL0_ITR_INDX_MASK | /* set noitr */
831 I40E_PFINT_DYN_CTL0_SWINT_TRIG_MASK |
832 I40E_PFINT_DYN_CTL0_SW_ITR_INDX_ENA_MASK;
833 /* allow 00 to be written to the index */
834
835 wr32(&vsi->back->hw, I40E_PFINT_DYN_CTL0, val);
836 }
fd0a05ce
JB		 837}
838
839/**
840 * i40e_set_new_dynamic_itr - Find new ITR level
841 * @rc: structure containing ring performance data
842 *
8f5e39ce
JB		 843 * Returns true if ITR changed, false if not
844 *
fd0a05ce
JB		 845 * Stores a new ITR value based on packets and byte counts during
846 * the last interrupt. The advantage of per interrupt computation
847 * is faster updates and more accurate ITR for the current traffic
848 * pattern. Constants in this function were computed based on
849 * theoretical maximum wire speed and thresholds were set based on
850 * testing data as well as attempting to minimize response time
851 * while increasing bulk throughput.
852 **/
8f5e39ce 853static bool i40e_set_new_dynamic_itr(struct i40e_ring_container *rc)
fd0a05ce
JB		 854{
855 enum i40e_latency_range new_latency_range = rc->latency_range;
c56625d5 856 struct i40e_q_vector *qv = rc->ring->q_vector;
fd0a05ce
JB		 857 u32 new_itr = rc->itr;
858 int bytes_per_int;
51cc6d9f 859 int usecs;
fd0a05ce
JB		 860
861 if (rc->total_packets == 0 || !rc->itr)
8f5e39ce 862 return false;
fd0a05ce
JB		 863
864 /* simple throttlerate management
c56625d5 865 * 0-10MB/s lowest (50000 ints/s)
fd0a05ce 866 * 10-20MB/s low (20000 ints/s)
c56625d5
JB		 867 * 20-1249MB/s bulk (18000 ints/s)
868 * > 40000 Rx packets per second (8000 ints/s)
51cc6d9f
JB		 869 *
870 * The math works out because the divisor is in 10^(-6) which
871 * turns the bytes/us input value into MB/s values, but
872 * make sure to use usecs, as the register values written
ee2319cf
JB		 873 * are in 2 usec increments in the ITR registers, and make sure
874 * to use the smoothed values that the countdown timer gives us.
fd0a05ce 875 */
ee2319cf 876 usecs = (rc->itr << 1) * ITR_COUNTDOWN_START;
51cc6d9f 877 bytes_per_int = rc->total_bytes / usecs;
ee2319cf 878
de32e3ef 879 switch (new_latency_range) {
fd0a05ce
JB		 880 case I40E_LOWEST_LATENCY:
881 if (bytes_per_int > 10)
882 new_latency_range = I40E_LOW_LATENCY;
883 break;
884 case I40E_LOW_LATENCY:
885 if (bytes_per_int > 20)
886 new_latency_range = I40E_BULK_LATENCY;
887 else if (bytes_per_int <= 10)
888 new_latency_range = I40E_LOWEST_LATENCY;
889 break;
890 case I40E_BULK_LATENCY:
c56625d5 891 case I40E_ULTRA_LATENCY:
de32e3ef
CW		 892 default:
893 if (bytes_per_int <= 20)
894 new_latency_range = I40E_LOW_LATENCY;
fd0a05ce
JB		 895 break;
896 }
c56625d5
JB		 897
898 /* this is to adjust RX more aggressively when streaming small
899 * packets. The value of 40000 was picked as it is just beyond
900 * what the hardware can receive per second if in low latency
901 * mode.
902 */
903#define RX_ULTRA_PACKET_RATE 40000
904
905 if ((((rc->total_packets * 1000000) / usecs) > RX_ULTRA_PACKET_RATE) &&
906 (&qv->rx == rc))
907 new_latency_range = I40E_ULTRA_LATENCY;
908
de32e3ef 909 rc->latency_range = new_latency_range;
fd0a05ce
JB		 910
911 switch (new_latency_range) {
912 case I40E_LOWEST_LATENCY:
c56625d5 913 new_itr = I40E_ITR_50K;
fd0a05ce
JB		 914 break;
915 case I40E_LOW_LATENCY:
916 new_itr = I40E_ITR_20K;
917 break;
918 case I40E_BULK_LATENCY:
c56625d5
JB		 919 new_itr = I40E_ITR_18K;
920 break;
921 case I40E_ULTRA_LATENCY:
fd0a05ce
JB		 922 new_itr = I40E_ITR_8K;
923 break;
924 default:
925 break;
926 }
927
fd0a05ce
JB		 928 rc->total_bytes = 0;
929 rc->total_packets = 0;
8f5e39ce
JB		 930
931 if (new_itr != rc->itr) {
932 rc->itr = new_itr;
933 return true;
934 }
935
936 return false;
fd0a05ce
JB		 937}
938
fd0a05ce
JB		 939/**
940 * i40e_clean_programming_status - clean the programming status descriptor
941 * @rx_ring: the rx ring that has this descriptor
942 * @rx_desc: the rx descriptor written back by HW
943 *
944 * Flow director should handle FD_FILTER_STATUS to check its filter programming
945 * status being successful or not and take actions accordingly. FCoE should
946 * handle its context/filter programming/invalidation status and take actions.
947 *
948 **/
949static void i40e_clean_programming_status(struct i40e_ring *rx_ring,
950 union i40e_rx_desc *rx_desc)
951{
952 u64 qw;
953 u8 id;
954
955 qw = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
956 id = (qw & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >>
957 I40E_RX_PROG_STATUS_DESC_QW1_PROGID_SHIFT;
958
959 if (id == I40E_RX_PROG_STATUS_DESC_FD_FILTER_STATUS)
55a5e60b 960 i40e_fd_handle_status(rx_ring, rx_desc, id);
38e00438
VD		 961#ifdef I40E_FCOE
962 else if ((id == I40E_RX_PROG_STATUS_DESC_FCOE_CTXT_PROG_STATUS) ||
963 (id == I40E_RX_PROG_STATUS_DESC_FCOE_CTXT_INVL_STATUS))
964 i40e_fcoe_handle_status(rx_ring, rx_desc, id);
965#endif
fd0a05ce
JB		 966}
967
968/**
969 * i40e_setup_tx_descriptors - Allocate the Tx descriptors
970 * @tx_ring: the tx ring to set up
971 *
972 * Return 0 on success, negative on error
973 **/
974int i40e_setup_tx_descriptors(struct i40e_ring *tx_ring)
975{
976 struct device *dev = tx_ring->dev;
977 int bi_size;
978
979 if (!dev)
980 return -ENOMEM;
981
e908f815
JB		 982 /* warn if we are about to overwrite the pointer */
983 WARN_ON(tx_ring->tx_bi);
fd0a05ce
JB		 984 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
985 tx_ring->tx_bi = kzalloc(bi_size, GFP_KERNEL);
986 if (!tx_ring->tx_bi)
987 goto err;
988
989 /* round up to nearest 4K */
990 tx_ring->size = tx_ring->count * sizeof(struct i40e_tx_desc);
1943d8ba
JB		 991 /* add u32 for head writeback, align after this takes care of
992 * guaranteeing this is at least one cache line in size
993 */
994 tx_ring->size += sizeof(u32);
fd0a05ce
JB		 995 tx_ring->size = ALIGN(tx_ring->size, 4096);
996 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
997 &tx_ring->dma, GFP_KERNEL);
998 if (!tx_ring->desc) {
999 dev_info(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n",
1000 tx_ring->size);
1001 goto err;
1002 }
1003
1004 tx_ring->next_to_use = 0;
1005 tx_ring->next_to_clean = 0;
1006 return 0;
1007
1008err:
1009 kfree(tx_ring->tx_bi);
1010 tx_ring->tx_bi = NULL;
1011 return -ENOMEM;
1012}
1013
1014/**
1015 * i40e_clean_rx_ring - Free Rx buffers
1016 * @rx_ring: ring to be cleaned
1017 **/
1018void i40e_clean_rx_ring(struct i40e_ring *rx_ring)
1019{
1020 struct device *dev = rx_ring->dev;
1021 struct i40e_rx_buffer *rx_bi;
1022 unsigned long bi_size;
1023 u16 i;
1024
1025 /* ring already cleared, nothing to do */
1026 if (!rx_ring->rx_bi)
1027 return;
1028
a132af24
MW		 1029 if (ring_is_ps_enabled(rx_ring)) {
1030 int bufsz = ALIGN(rx_ring->rx_hdr_len, 256) * rx_ring->count;
1031
1032 rx_bi = &rx_ring->rx_bi[0];
1033 if (rx_bi->hdr_buf) {
1034 dma_free_coherent(dev,
1035 bufsz,
1036 rx_bi->hdr_buf,
1037 rx_bi->dma);
1038 for (i = 0; i < rx_ring->count; i++) {
1039 rx_bi = &rx_ring->rx_bi[i];
1040 rx_bi->dma = 0;
37a2973a 1041 rx_bi->hdr_buf = NULL;
a132af24
MW		 1042 }
1043 }
1044 }
fd0a05ce
JB		 1045 /* Free all the Rx ring sk_buffs */
1046 for (i = 0; i < rx_ring->count; i++) {
1047 rx_bi = &rx_ring->rx_bi[i];
1048 if (rx_bi->dma) {
1049 dma_unmap_single(dev,
1050 rx_bi->dma,
1051 rx_ring->rx_buf_len,
1052 DMA_FROM_DEVICE);
1053 rx_bi->dma = 0;
1054 }
1055 if (rx_bi->skb) {
1056 dev_kfree_skb(rx_bi->skb);
1057 rx_bi->skb = NULL;
1058 }
1059 if (rx_bi->page) {
1060 if (rx_bi->page_dma) {
1061 dma_unmap_page(dev,
1062 rx_bi->page_dma,
1063 PAGE_SIZE / 2,
1064 DMA_FROM_DEVICE);
1065 rx_bi->page_dma = 0;
1066 }
1067 __free_page(rx_bi->page);
1068 rx_bi->page = NULL;
1069 rx_bi->page_offset = 0;
1070 }
1071 }
1072
1073 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
1074 memset(rx_ring->rx_bi, 0, bi_size);
1075
1076 /* Zero out the descriptor ring */
1077 memset(rx_ring->desc, 0, rx_ring->size);
1078
1079 rx_ring->next_to_clean = 0;
1080 rx_ring->next_to_use = 0;
1081}
1082
1083/**
1084 * i40e_free_rx_resources - Free Rx resources
1085 * @rx_ring: ring to clean the resources from
1086 *
1087 * Free all receive software resources
1088 **/
1089void i40e_free_rx_resources(struct i40e_ring *rx_ring)
1090{
1091 i40e_clean_rx_ring(rx_ring);
1092 kfree(rx_ring->rx_bi);
1093 rx_ring->rx_bi = NULL;
1094
1095 if (rx_ring->desc) {
1096 dma_free_coherent(rx_ring->dev, rx_ring->size,
1097 rx_ring->desc, rx_ring->dma);
1098 rx_ring->desc = NULL;
1099 }
1100}
1101
a132af24
MW		 1102/**
1103 * i40e_alloc_rx_headers - allocate rx header buffers
1104 * @rx_ring: ring to alloc buffers
1105 *
1106 * Allocate rx header buffers for the entire ring. As these are static,
1107 * this is only called when setting up a new ring.
1108 **/
1109void i40e_alloc_rx_headers(struct i40e_ring *rx_ring)
1110{
1111 struct device *dev = rx_ring->dev;
1112 struct i40e_rx_buffer *rx_bi;
1113 dma_addr_t dma;
1114 void *buffer;
1115 int buf_size;
1116 int i;
1117
1118 if (rx_ring->rx_bi[0].hdr_buf)
1119 return;
1120 /* Make sure the buffers don't cross cache line boundaries. */
1121 buf_size = ALIGN(rx_ring->rx_hdr_len, 256);
1122 buffer = dma_alloc_coherent(dev, buf_size * rx_ring->count,
1123 &dma, GFP_KERNEL);
1124 if (!buffer)
1125 return;
1126 for (i = 0; i < rx_ring->count; i++) {
1127 rx_bi = &rx_ring->rx_bi[i];
1128 rx_bi->dma = dma + (i * buf_size);
1129 rx_bi->hdr_buf = buffer + (i * buf_size);
1130 }
1131}
1132
fd0a05ce
JB		 1133/**
1134 * i40e_setup_rx_descriptors - Allocate Rx descriptors
1135 * @rx_ring: Rx descriptor ring (for a specific queue) to setup
1136 *
1137 * Returns 0 on success, negative on failure
1138 **/
1139int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring)
1140{
1141 struct device *dev = rx_ring->dev;
1142 int bi_size;
1143
e908f815
JB		 1144 /* warn if we are about to overwrite the pointer */
1145 WARN_ON(rx_ring->rx_bi);
fd0a05ce
JB		 1146 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
1147 rx_ring->rx_bi = kzalloc(bi_size, GFP_KERNEL);
1148 if (!rx_ring->rx_bi)
1149 goto err;
1150
f217d6ca 1151 u64_stats_init(&rx_ring->syncp);
638702bd 1152
fd0a05ce
JB		 1153 /* Round up to nearest 4K */
1154 rx_ring->size = ring_is_16byte_desc_enabled(rx_ring)
1155 ? rx_ring->count * sizeof(union i40e_16byte_rx_desc)
1156 : rx_ring->count * sizeof(union i40e_32byte_rx_desc);
1157 rx_ring->size = ALIGN(rx_ring->size, 4096);
1158 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
1159 &rx_ring->dma, GFP_KERNEL);
1160
1161 if (!rx_ring->desc) {
1162 dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n",
1163 rx_ring->size);
1164 goto err;
1165 }
1166
1167 rx_ring->next_to_clean = 0;
1168 rx_ring->next_to_use = 0;
1169
1170 return 0;
1171err:
1172 kfree(rx_ring->rx_bi);
1173 rx_ring->rx_bi = NULL;
1174 return -ENOMEM;
1175}
1176
1177/**
1178 * i40e_release_rx_desc - Store the new tail and head values
1179 * @rx_ring: ring to bump
1180 * @val: new head index
1181 **/
1182static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val)
1183{
1184 rx_ring->next_to_use = val;
1185 /* Force memory writes to complete before letting h/w
1186 * know there are new descriptors to fetch. (Only
1187 * applicable for weak-ordered memory model archs,
1188 * such as IA-64).
1189 */
1190 wmb();
1191 writel(val, rx_ring->tail);
1192}
1193
1194/**
a132af24 1195 * i40e_alloc_rx_buffers_ps - Replace used receive buffers; packet split
fd0a05ce
JB		 1196 * @rx_ring: ring to place buffers on
1197 * @cleaned_count: number of buffers to replace
1198 **/
a132af24
MW		 1199void i40e_alloc_rx_buffers_ps(struct i40e_ring *rx_ring, u16 cleaned_count)
1200{
1201 u16 i = rx_ring->next_to_use;
1202 union i40e_rx_desc *rx_desc;
1203 struct i40e_rx_buffer *bi;
1204
1205 /* do nothing if no valid netdev defined */
1206 if (!rx_ring->netdev || !cleaned_count)
1207 return;
1208
1209 while (cleaned_count--) {
1210 rx_desc = I40E_RX_DESC(rx_ring, i);
1211 bi = &rx_ring->rx_bi[i];
1212
1213 if (bi->skb) /* desc is in use */
1214 goto no_buffers;
1215 if (!bi->page) {
1216 bi->page = alloc_page(GFP_ATOMIC);
1217 if (!bi->page) {
1218 rx_ring->rx_stats.alloc_page_failed++;
1219 goto no_buffers;
1220 }
1221 }
1222
1223 if (!bi->page_dma) {
1224 /* use a half page if we're re-using */
1225 bi->page_offset ^= PAGE_SIZE / 2;
1226 bi->page_dma = dma_map_page(rx_ring->dev,
1227 bi->page,
1228 bi->page_offset,
1229 PAGE_SIZE / 2,
1230 DMA_FROM_DEVICE);
1231 if (dma_mapping_error(rx_ring->dev,
1232 bi->page_dma)) {
1233 rx_ring->rx_stats.alloc_page_failed++;
1234 bi->page_dma = 0;
1235 goto no_buffers;
1236 }
1237 }
1238
1239 dma_sync_single_range_for_device(rx_ring->dev,
3578fa0a
JB		 1240 rx_ring->rx_bi[0].dma,
1241 i * rx_ring->rx_hdr_len,
a132af24
MW		 1242 rx_ring->rx_hdr_len,
1243 DMA_FROM_DEVICE);
1244 /* Refresh the desc even if buffer_addrs didn't change
1245 * because each write-back erases this info.
1246 */
1247 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
1248 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
1249 i++;
1250 if (i == rx_ring->count)
1251 i = 0;
1252 }
1253
1254no_buffers:
1255 if (rx_ring->next_to_use != i)
1256 i40e_release_rx_desc(rx_ring, i);
1257}
1258
1259/**
1260 * i40e_alloc_rx_buffers_1buf - Replace used receive buffers; single buffer
1261 * @rx_ring: ring to place buffers on
1262 * @cleaned_count: number of buffers to replace
1263 **/
1264void i40e_alloc_rx_buffers_1buf(struct i40e_ring *rx_ring, u16 cleaned_count)
fd0a05ce
JB		 1265{
1266 u16 i = rx_ring->next_to_use;
1267 union i40e_rx_desc *rx_desc;
1268 struct i40e_rx_buffer *bi;
1269 struct sk_buff *skb;
1270
1271 /* do nothing if no valid netdev defined */
1272 if (!rx_ring->netdev || !cleaned_count)
1273 return;
1274
1275 while (cleaned_count--) {
1276 rx_desc = I40E_RX_DESC(rx_ring, i);
1277 bi = &rx_ring->rx_bi[i];
1278 skb = bi->skb;
1279
1280 if (!skb) {
1281 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
1282 rx_ring->rx_buf_len);
1283 if (!skb) {
420136cc 1284 rx_ring->rx_stats.alloc_buff_failed++;
fd0a05ce
JB		 1285 goto no_buffers;
1286 }
1287 /* initialize queue mapping */
1288 skb_record_rx_queue(skb, rx_ring->queue_index);
1289 bi->skb = skb;
1290 }
1291
1292 if (!bi->dma) {
1293 bi->dma = dma_map_single(rx_ring->dev,
1294 skb->data,
1295 rx_ring->rx_buf_len,
1296 DMA_FROM_DEVICE);
1297 if (dma_mapping_error(rx_ring->dev, bi->dma)) {
420136cc 1298 rx_ring->rx_stats.alloc_buff_failed++;
fd0a05ce
JB		 1299 bi->dma = 0;
1300 goto no_buffers;
1301 }
1302 }
1303
a132af24
MW		 1304 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
1305 rx_desc->read.hdr_addr = 0;
fd0a05ce
JB		 1306 i++;
1307 if (i == rx_ring->count)
1308 i = 0;
1309 }
1310
1311no_buffers:
1312 if (rx_ring->next_to_use != i)
1313 i40e_release_rx_desc(rx_ring, i);
1314}
1315
1316/**
1317 * i40e_receive_skb - Send a completed packet up the stack
1318 * @rx_ring: rx ring in play
1319 * @skb: packet to send up
1320 * @vlan_tag: vlan tag for packet
1321 **/
1322static void i40e_receive_skb(struct i40e_ring *rx_ring,
1323 struct sk_buff *skb, u16 vlan_tag)
1324{
1325 struct i40e_q_vector *q_vector = rx_ring->q_vector;
fd0a05ce
JB		 1326
1327 if (vlan_tag & VLAN_VID_MASK)
1328 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
1329
8b650359 1330 napi_gro_receive(&q_vector->napi, skb);
fd0a05ce
JB		 1331}
1332
1333/**
1334 * i40e_rx_checksum - Indicate in skb if hw indicated a good cksum
1335 * @vsi: the VSI we care about
1336 * @skb: skb currently being received and modified
1337 * @rx_status: status value of last descriptor in packet
1338 * @rx_error: error value of last descriptor in packet
8144f0f7 1339 * @rx_ptype: ptype value of last descriptor in packet
fd0a05ce
JB		 1340 **/
1341static inline void i40e_rx_checksum(struct i40e_vsi *vsi,
1342 struct sk_buff *skb,
1343 u32 rx_status,
8144f0f7
JG		 1344 u32 rx_error,
1345 u16 rx_ptype)
fd0a05ce 1346{
8a3c91cc
JB		 1347 struct i40e_rx_ptype_decoded decoded = decode_rx_desc_ptype(rx_ptype);
1348 bool ipv4 = false, ipv6 = false;
8144f0f7
JG		 1349 bool ipv4_tunnel, ipv6_tunnel;
1350 __wsum rx_udp_csum;
8144f0f7 1351 struct iphdr *iph;
8a3c91cc 1352 __sum16 csum;
8144f0f7 1353
f8faaa40
ASJ		 1354 ipv4_tunnel = (rx_ptype >= I40E_RX_PTYPE_GRENAT4_MAC_PAY3) &&
1355 (rx_ptype <= I40E_RX_PTYPE_GRENAT4_MACVLAN_IPV6_ICMP_PAY4);
1356 ipv6_tunnel = (rx_ptype >= I40E_RX_PTYPE_GRENAT6_MAC_PAY3) &&
1357 (rx_ptype <= I40E_RX_PTYPE_GRENAT6_MACVLAN_IPV6_ICMP_PAY4);
8144f0f7 1358
fd0a05ce
JB		 1359 skb->ip_summed = CHECKSUM_NONE;
1360
1361 /* Rx csum enabled and ip headers found? */
8a3c91cc
JB		 1362 if (!(vsi->netdev->features & NETIF_F_RXCSUM))
1363 return;
1364
1365 /* did the hardware decode the packet and checksum? */
41a1d04b 1366 if (!(rx_status & BIT(I40E_RX_DESC_STATUS_L3L4P_SHIFT)))
8a3c91cc
JB		 1367 return;
1368
1369 /* both known and outer_ip must be set for the below code to work */
1370 if (!(decoded.known && decoded.outer_ip))
fd0a05ce
JB		 1371 return;
1372
8a3c91cc
JB		 1373 if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1374 decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4)
1375 ipv4 = true;
1376 else if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1377 decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6)
1378 ipv6 = true;
1379
1380 if (ipv4 &&
41a1d04b
JB		 1381 (rx_error & (BIT(I40E_RX_DESC_ERROR_IPE_SHIFT) |
1382 BIT(I40E_RX_DESC_ERROR_EIPE_SHIFT))))
8a3c91cc
JB		 1383 goto checksum_fail;
1384
ddf1d0d7 1385 /* likely incorrect csum if alternate IP extension headers found */
8a3c91cc 1386 if (ipv6 &&
41a1d04b 1387 rx_status & BIT(I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT))
8a3c91cc 1388 /* don't increment checksum err here, non-fatal err */
8ee75a8e
SN		 1389 return;
1390
8a3c91cc 1391 /* there was some L4 error, count error and punt packet to the stack */
41a1d04b 1392 if (rx_error & BIT(I40E_RX_DESC_ERROR_L4E_SHIFT))
8a3c91cc
JB		 1393 goto checksum_fail;
1394
1395 /* handle packets that were not able to be checksummed due
1396 * to arrival speed, in this case the stack can compute
1397 * the csum.
1398 */
41a1d04b 1399 if (rx_error & BIT(I40E_RX_DESC_ERROR_PPRS_SHIFT))
fd0a05ce 1400 return;
fd0a05ce 1401
6a899024 1402 /* If VXLAN/GENEVE traffic has an outer UDPv4 checksum we need to check
8a3c91cc
JB		 1403 * it in the driver, hardware does not do it for us.
1404 * Since L3L4P bit was set we assume a valid IHL value (>=5)
1405 * so the total length of IPv4 header is IHL*4 bytes
1406 * The UDP_0 bit *may* bet set if the *inner* header is UDP
1407 */
527274c7
ASJ		 1408 if (!(vsi->back->flags & I40E_FLAG_OUTER_UDP_CSUM_CAPABLE) &&
1409 (ipv4_tunnel)) {
8144f0f7
JG		 1410 skb->transport_header = skb->mac_header +
1411 sizeof(struct ethhdr) +
1412 (ip_hdr(skb)->ihl * 4);
1413
1414 /* Add 4 bytes for VLAN tagged packets */
1415 skb->transport_header += (skb->protocol == htons(ETH_P_8021Q) ||
1416 skb->protocol == htons(ETH_P_8021AD))
1417 ? VLAN_HLEN : 0;
1418
f6385979
AS		 1419 if ((ip_hdr(skb)->protocol == IPPROTO_UDP) &&
1420 (udp_hdr(skb)->check != 0)) {
1421 rx_udp_csum = udp_csum(skb);
1422 iph = ip_hdr(skb);
1423 csum = csum_tcpudp_magic(
1424 iph->saddr, iph->daddr,
1425 (skb->len - skb_transport_offset(skb)),
1426 IPPROTO_UDP, rx_udp_csum);
8144f0f7 1427
f6385979
AS		 1428 if (udp_hdr(skb)->check != csum)
1429 goto checksum_fail;
1430
1431 } /* else its GRE and so no outer UDP header */
8144f0f7
JG		 1432 }
1433
fd0a05ce 1434 skb->ip_summed = CHECKSUM_UNNECESSARY;
fa4ba69b 1435 skb->csum_level = ipv4_tunnel || ipv6_tunnel;
8a3c91cc
JB		 1436
1437 return;
1438
1439checksum_fail:
1440 vsi->back->hw_csum_rx_error++;
fd0a05ce
JB		 1441}
1442
1443/**
857942fd 1444 * i40e_ptype_to_htype - get a hash type
206812b5
JB		 1445 * @ptype: the ptype value from the descriptor
1446 *
1447 * Returns a hash type to be used by skb_set_hash
1448 **/
857942fd 1449static inline enum pkt_hash_types i40e_ptype_to_htype(u8 ptype)
206812b5
JB		 1450{
1451 struct i40e_rx_ptype_decoded decoded = decode_rx_desc_ptype(ptype);
1452
1453 if (!decoded.known)
1454 return PKT_HASH_TYPE_NONE;
1455
1456 if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1457 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY4)
1458 return PKT_HASH_TYPE_L4;
1459 else if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1460 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY3)
1461 return PKT_HASH_TYPE_L3;
1462 else
1463 return PKT_HASH_TYPE_L2;
1464}
1465
857942fd
ASJ		 1466/**
1467 * i40e_rx_hash - set the hash value in the skb
1468 * @ring: descriptor ring
1469 * @rx_desc: specific descriptor
1470 **/
1471static inline void i40e_rx_hash(struct i40e_ring *ring,
1472 union i40e_rx_desc *rx_desc,
1473 struct sk_buff *skb,
1474 u8 rx_ptype)
1475{
1476 u32 hash;
1477 const __le64 rss_mask =
1478 cpu_to_le64((u64)I40E_RX_DESC_FLTSTAT_RSS_HASH <<
1479 I40E_RX_DESC_STATUS_FLTSTAT_SHIFT);
1480
1481 if (ring->netdev->features & NETIF_F_RXHASH)
1482 return;
1483
1484 if ((rx_desc->wb.qword1.status_error_len & rss_mask) == rss_mask) {
1485 hash = le32_to_cpu(rx_desc->wb.qword0.hi_dword.rss);
1486 skb_set_hash(skb, hash, i40e_ptype_to_htype(rx_ptype));
1487 }
1488}
1489
fd0a05ce 1490/**
a132af24 1491 * i40e_clean_rx_irq_ps - Reclaim resources after receive; packet split
fd0a05ce
JB		 1492 * @rx_ring: rx ring to clean
1493 * @budget: how many cleans we're allowed
1494 *
1495 * Returns true if there's any budget left (e.g. the clean is finished)
1496 **/
a132af24 1497static int i40e_clean_rx_irq_ps(struct i40e_ring *rx_ring, int budget)
fd0a05ce
JB		 1498{
1499 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1500 u16 rx_packet_len, rx_header_len, rx_sph, rx_hbo;
1501 u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);
8dc5562e 1502 const int current_node = numa_mem_id();
fd0a05ce
JB		 1503 struct i40e_vsi *vsi = rx_ring->vsi;
1504 u16 i = rx_ring->next_to_clean;
1505 union i40e_rx_desc *rx_desc;
1506 u32 rx_error, rx_status;
206812b5 1507 u8 rx_ptype;
fd0a05ce
JB		 1508 u64 qword;
1509
390f86df
EB		 1510 if (budget <= 0)
1511 return 0;
1512
a132af24 1513 do {
fd0a05ce
JB		 1514 struct i40e_rx_buffer *rx_bi;
1515 struct sk_buff *skb;
1516 u16 vlan_tag;
a132af24
MW		 1517 /* return some buffers to hardware, one at a time is too slow */
1518 if (cleaned_count >= I40E_RX_BUFFER_WRITE) {
1519 i40e_alloc_rx_buffers_ps(rx_ring, cleaned_count);
1520 cleaned_count = 0;
1521 }
1522
1523 i = rx_ring->next_to_clean;
1524 rx_desc = I40E_RX_DESC(rx_ring, i);
1525 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
1526 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >>
1527 I40E_RXD_QW1_STATUS_SHIFT;
1528
41a1d04b 1529 if (!(rx_status & BIT(I40E_RX_DESC_STATUS_DD_SHIFT)))
a132af24
MW		 1530 break;
1531
1532 /* This memory barrier is needed to keep us from reading
1533 * any other fields out of the rx_desc until we know the
1534 * DD bit is set.
1535 */
67317166 1536 dma_rmb();
fd0a05ce
JB		 1537 if (i40e_rx_is_programming_status(qword)) {
1538 i40e_clean_programming_status(rx_ring, rx_desc);
a132af24
MW		 1539 I40E_RX_INCREMENT(rx_ring, i);
1540 continue;
fd0a05ce
JB		 1541 }
1542 rx_bi = &rx_ring->rx_bi[i];
1543 skb = rx_bi->skb;
a132af24
MW		 1544 if (likely(!skb)) {
1545 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
1546 rx_ring->rx_hdr_len);
8b6ed9c2 1547 if (!skb) {
a132af24 1548 rx_ring->rx_stats.alloc_buff_failed++;
8b6ed9c2
JB		 1549 break;
1550 }
1551
a132af24
MW		 1552 /* initialize queue mapping */
1553 skb_record_rx_queue(skb, rx_ring->queue_index);
1554 /* we are reusing so sync this buffer for CPU use */
1555 dma_sync_single_range_for_cpu(rx_ring->dev,
3578fa0a
JB		 1556 rx_ring->rx_bi[0].dma,
1557 i * rx_ring->rx_hdr_len,
a132af24
MW		 1558 rx_ring->rx_hdr_len,
1559 DMA_FROM_DEVICE);
1560 }
829af3ac
MW		 1561 rx_packet_len = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
1562 I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
1563 rx_header_len = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK) >>
1564 I40E_RXD_QW1_LENGTH_HBUF_SHIFT;
1565 rx_sph = (qword & I40E_RXD_QW1_LENGTH_SPH_MASK) >>
1566 I40E_RXD_QW1_LENGTH_SPH_SHIFT;
1567
1568 rx_error = (qword & I40E_RXD_QW1_ERROR_MASK) >>
1569 I40E_RXD_QW1_ERROR_SHIFT;
41a1d04b
JB		 1570 rx_hbo = rx_error & BIT(I40E_RX_DESC_ERROR_HBO_SHIFT);
1571 rx_error &= ~BIT(I40E_RX_DESC_ERROR_HBO_SHIFT);
fd0a05ce 1572
8144f0f7
JG		 1573 rx_ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >>
1574 I40E_RXD_QW1_PTYPE_SHIFT;
a132af24 1575 prefetch(rx_bi->page);
fd0a05ce 1576 rx_bi->skb = NULL;
a132af24
MW		 1577 cleaned_count++;
1578 if (rx_hbo || rx_sph) {
1579 int len;
6995b36c 1580
fd0a05ce
JB		 1581 if (rx_hbo)
1582 len = I40E_RX_HDR_SIZE;
fd0a05ce 1583 else
a132af24
MW		 1584 len = rx_header_len;
1585 memcpy(__skb_put(skb, len), rx_bi->hdr_buf, len);
1586 } else if (skb->len == 0) {
1587 int len;
1588
1589 len = (rx_packet_len > skb_headlen(skb) ?
1590 skb_headlen(skb) : rx_packet_len);
1591 memcpy(__skb_put(skb, len),
1592 rx_bi->page + rx_bi->page_offset,
1593 len);
1594 rx_bi->page_offset += len;
1595 rx_packet_len -= len;
fd0a05ce
JB		 1596 }
1597
1598 /* Get the rest of the data if this was a header split */
a132af24 1599 if (rx_packet_len) {
fd0a05ce
JB		 1600 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
1601 rx_bi->page,
1602 rx_bi->page_offset,
1603 rx_packet_len);
1604
1605 skb->len += rx_packet_len;
1606 skb->data_len += rx_packet_len;
1607 skb->truesize += rx_packet_len;
1608
1609 if ((page_count(rx_bi->page) == 1) &&
1610 (page_to_nid(rx_bi->page) == current_node))
1611 get_page(rx_bi->page);
1612 else
1613 rx_bi->page = NULL;
1614
1615 dma_unmap_page(rx_ring->dev,
1616 rx_bi->page_dma,
1617 PAGE_SIZE / 2,
1618 DMA_FROM_DEVICE);
1619 rx_bi->page_dma = 0;
1620 }
a132af24 1621 I40E_RX_INCREMENT(rx_ring, i);
fd0a05ce
JB		 1622
1623 if (unlikely(
41a1d04b 1624 !(rx_status & BIT(I40E_RX_DESC_STATUS_EOF_SHIFT)))) {
fd0a05ce
JB		 1625 struct i40e_rx_buffer *next_buffer;
1626
1627 next_buffer = &rx_ring->rx_bi[i];
a132af24 1628 next_buffer->skb = skb;
fd0a05ce 1629 rx_ring->rx_stats.non_eop_descs++;
a132af24 1630 continue;
fd0a05ce
JB		 1631 }
1632
1633 /* ERR_MASK will only have valid bits if EOP set */
41a1d04b 1634 if (unlikely(rx_error & BIT(I40E_RX_DESC_ERROR_RXE_SHIFT))) {
fd0a05ce 1635 dev_kfree_skb_any(skb);
a132af24 1636 continue;
fd0a05ce
JB		 1637 }
1638
857942fd
ASJ		 1639 i40e_rx_hash(rx_ring, rx_desc, skb, rx_ptype);
1640
beb0dff1
JK		 1641 if (unlikely(rx_status & I40E_RXD_QW1_STATUS_TSYNVALID_MASK)) {
1642 i40e_ptp_rx_hwtstamp(vsi->back, skb, (rx_status &
1643 I40E_RXD_QW1_STATUS_TSYNINDX_MASK) >>
1644 I40E_RXD_QW1_STATUS_TSYNINDX_SHIFT);
1645 rx_ring->last_rx_timestamp = jiffies;
1646 }
1647
fd0a05ce
JB		 1648 /* probably a little skewed due to removing CRC */
1649 total_rx_bytes += skb->len;
1650 total_rx_packets++;
1651
1652 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
8144f0f7
JG		 1653
1654 i40e_rx_checksum(vsi, skb, rx_status, rx_error, rx_ptype);
1655
41a1d04b 1656 vlan_tag = rx_status & BIT(I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)
fd0a05ce
JB		 1657 ? le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1)
1658 : 0;
38e00438
VD		 1659#ifdef I40E_FCOE
1660 if (!i40e_fcoe_handle_offload(rx_ring, rx_desc, skb)) {
1661 dev_kfree_skb_any(skb);
a132af24 1662 continue;
38e00438
VD		 1663 }
1664#endif
fd0a05ce
JB		 1665 i40e_receive_skb(rx_ring, skb, vlan_tag);
1666
fd0a05ce 1667 rx_desc->wb.qword1.status_error_len = 0;
fd0a05ce 1668
a132af24
MW		 1669 } while (likely(total_rx_packets < budget));
1670
1671 u64_stats_update_begin(&rx_ring->syncp);
1672 rx_ring->stats.packets += total_rx_packets;
1673 rx_ring->stats.bytes += total_rx_bytes;
1674 u64_stats_update_end(&rx_ring->syncp);
1675 rx_ring->q_vector->rx.total_packets += total_rx_packets;
1676 rx_ring->q_vector->rx.total_bytes += total_rx_bytes;
1677
1678 return total_rx_packets;
1679}
1680
1681/**
1682 * i40e_clean_rx_irq_1buf - Reclaim resources after receive; single buffer
1683 * @rx_ring: rx ring to clean
1684 * @budget: how many cleans we're allowed
1685 *
1686 * Returns number of packets cleaned
1687 **/
1688static int i40e_clean_rx_irq_1buf(struct i40e_ring *rx_ring, int budget)
1689{
1690 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1691 u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);
1692 struct i40e_vsi *vsi = rx_ring->vsi;
1693 union i40e_rx_desc *rx_desc;
1694 u32 rx_error, rx_status;
1695 u16 rx_packet_len;
1696 u8 rx_ptype;
1697 u64 qword;
1698 u16 i;
1699
1700 do {
1701 struct i40e_rx_buffer *rx_bi;
1702 struct sk_buff *skb;
1703 u16 vlan_tag;
fd0a05ce
JB		 1704 /* return some buffers to hardware, one at a time is too slow */
1705 if (cleaned_count >= I40E_RX_BUFFER_WRITE) {
a132af24 1706 i40e_alloc_rx_buffers_1buf(rx_ring, cleaned_count);
fd0a05ce
JB		 1707 cleaned_count = 0;
1708 }
1709
a132af24
MW		 1710 i = rx_ring->next_to_clean;
1711 rx_desc = I40E_RX_DESC(rx_ring, i);
fd0a05ce 1712 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
829af3ac 1713 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >>
a132af24
MW		 1714 I40E_RXD_QW1_STATUS_SHIFT;
1715
41a1d04b 1716 if (!(rx_status & BIT(I40E_RX_DESC_STATUS_DD_SHIFT)))
a132af24
MW		 1717 break;
1718
1719 /* This memory barrier is needed to keep us from reading
1720 * any other fields out of the rx_desc until we know the
1721 * DD bit is set.
1722 */
67317166 1723 dma_rmb();
a132af24
MW		 1724
1725 if (i40e_rx_is_programming_status(qword)) {
1726 i40e_clean_programming_status(rx_ring, rx_desc);
1727 I40E_RX_INCREMENT(rx_ring, i);
1728 continue;
1729 }
1730 rx_bi = &rx_ring->rx_bi[i];
1731 skb = rx_bi->skb;
1732 prefetch(skb->data);
1733
1734 rx_packet_len = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
1735 I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
1736
1737 rx_error = (qword & I40E_RXD_QW1_ERROR_MASK) >>
1738 I40E_RXD_QW1_ERROR_SHIFT;
41a1d04b 1739 rx_error &= ~BIT(I40E_RX_DESC_ERROR_HBO_SHIFT);
a132af24
MW		 1740
1741 rx_ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >>
1742 I40E_RXD_QW1_PTYPE_SHIFT;
1743 rx_bi->skb = NULL;
1744 cleaned_count++;
1745
1746 /* Get the header and possibly the whole packet
1747 * If this is an skb from previous receive dma will be 0
1748 */
1749 skb_put(skb, rx_packet_len);
1750 dma_unmap_single(rx_ring->dev, rx_bi->dma, rx_ring->rx_buf_len,
1751 DMA_FROM_DEVICE);
1752 rx_bi->dma = 0;
1753
1754 I40E_RX_INCREMENT(rx_ring, i);
1755
1756 if (unlikely(
41a1d04b 1757 !(rx_status & BIT(I40E_RX_DESC_STATUS_EOF_SHIFT)))) {
a132af24
MW		 1758 rx_ring->rx_stats.non_eop_descs++;
1759 continue;
1760 }
1761
1762 /* ERR_MASK will only have valid bits if EOP set */
41a1d04b 1763 if (unlikely(rx_error & BIT(I40E_RX_DESC_ERROR_RXE_SHIFT))) {
a132af24 1764 dev_kfree_skb_any(skb);
a132af24
MW		 1765 continue;
1766 }
1767
857942fd 1768 i40e_rx_hash(rx_ring, rx_desc, skb, rx_ptype);
a132af24
MW		 1769 if (unlikely(rx_status & I40E_RXD_QW1_STATUS_TSYNVALID_MASK)) {
1770 i40e_ptp_rx_hwtstamp(vsi->back, skb, (rx_status &
1771 I40E_RXD_QW1_STATUS_TSYNINDX_MASK) >>
1772 I40E_RXD_QW1_STATUS_TSYNINDX_SHIFT);
1773 rx_ring->last_rx_timestamp = jiffies;
1774 }
1775
1776 /* probably a little skewed due to removing CRC */
1777 total_rx_bytes += skb->len;
1778 total_rx_packets++;
1779
1780 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1781
1782 i40e_rx_checksum(vsi, skb, rx_status, rx_error, rx_ptype);
1783
41a1d04b 1784 vlan_tag = rx_status & BIT(I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)
a132af24
MW		 1785 ? le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1)
1786 : 0;
1787#ifdef I40E_FCOE
1788 if (!i40e_fcoe_handle_offload(rx_ring, rx_desc, skb)) {
1789 dev_kfree_skb_any(skb);
1790 continue;
1791 }
1792#endif
1793 i40e_receive_skb(rx_ring, skb, vlan_tag);
1794
a132af24
MW		 1795 rx_desc->wb.qword1.status_error_len = 0;
1796 } while (likely(total_rx_packets < budget));
fd0a05ce 1797
980e9b11 1798 u64_stats_update_begin(&rx_ring->syncp);
a114d0a6
AD		 1799 rx_ring->stats.packets += total_rx_packets;
1800 rx_ring->stats.bytes += total_rx_bytes;
980e9b11 1801 u64_stats_update_end(&rx_ring->syncp);
fd0a05ce
JB		 1802 rx_ring->q_vector->rx.total_packets += total_rx_packets;
1803 rx_ring->q_vector->rx.total_bytes += total_rx_bytes;
1804
a132af24 1805 return total_rx_packets;
fd0a05ce
JB		 1806}
1807
8f5e39ce
JB		 1808static u32 i40e_buildreg_itr(const int type, const u16 itr)
1809{
1810 u32 val;
1811
1812 val = I40E_PFINT_DYN_CTLN_INTENA_MASK |
1813 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
1814 (type << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT) |
1815 (itr << I40E_PFINT_DYN_CTLN_INTERVAL_SHIFT);
1816
1817 return val;
1818}
1819
1820/* a small macro to shorten up some long lines */
1821#define INTREG I40E_PFINT_DYN_CTLN
1822
de32e3ef
CW		 1823/**
1824 * i40e_update_enable_itr - Update itr and re-enable MSIX interrupt
1825 * @vsi: the VSI we care about
1826 * @q_vector: q_vector for which itr is being updated and interrupt enabled
1827 *
1828 **/
1829static inline void i40e_update_enable_itr(struct i40e_vsi *vsi,
1830 struct i40e_q_vector *q_vector)
1831{
1832 struct i40e_hw *hw = &vsi->back->hw;
8f5e39ce
JB		 1833 bool rx = false, tx = false;
1834 u32 rxval, txval;
de32e3ef 1835 int vector;
de32e3ef
CW		 1836
1837 vector = (q_vector->v_idx + vsi->base_vector);
8f5e39ce 1838
ee2319cf
JB		 1839 /* avoid dynamic calculation if in countdown mode OR if
1840 * all dynamic is disabled
1841 */
8f5e39ce
JB		 1842 rxval = txval = i40e_buildreg_itr(I40E_ITR_NONE, 0);
1843
ee2319cf
JB		 1844 if (q_vector->itr_countdown > 0 ||
1845 (!ITR_IS_DYNAMIC(vsi->rx_itr_setting) &&
1846 !ITR_IS_DYNAMIC(vsi->tx_itr_setting))) {
1847 goto enable_int;
1848 }
1849
de32e3ef 1850 if (ITR_IS_DYNAMIC(vsi->rx_itr_setting)) {
8f5e39ce
JB		 1851 rx = i40e_set_new_dynamic_itr(&q_vector->rx);
1852 rxval = i40e_buildreg_itr(I40E_RX_ITR, q_vector->rx.itr);
de32e3ef 1853 }
8f5e39ce 1854
de32e3ef 1855 if (ITR_IS_DYNAMIC(vsi->tx_itr_setting)) {
8f5e39ce
JB		 1856 tx = i40e_set_new_dynamic_itr(&q_vector->tx);
1857 txval = i40e_buildreg_itr(I40E_TX_ITR, q_vector->tx.itr);
de32e3ef 1858 }
8f5e39ce
JB		 1859
1860 if (rx || tx) {
1861 /* get the higher of the two ITR adjustments and
1862 * use the same value for both ITR registers
1863 * when in adaptive mode (Rx and/or Tx)
1864 */
1865 u16 itr = max(q_vector->tx.itr, q_vector->rx.itr);
1866
1867 q_vector->tx.itr = q_vector->rx.itr = itr;
1868 txval = i40e_buildreg_itr(I40E_TX_ITR, itr);
1869 tx = true;
1870 rxval = i40e_buildreg_itr(I40E_RX_ITR, itr);
1871 rx = true;
1872 }
1873
1874 /* only need to enable the interrupt once, but need
1875 * to possibly update both ITR values
1876 */
1877 if (rx) {
1878 /* set the INTENA_MSK_MASK so that this first write
1879 * won't actually enable the interrupt, instead just
1880 * updating the ITR (it's bit 31 PF and VF)
1881 */
1882 rxval |= BIT(31);
1883 /* don't check _DOWN because interrupt isn't being enabled */
1884 wr32(hw, INTREG(vector - 1), rxval);
1885 }
1886
ee2319cf 1887enable_int:
8f5e39ce
JB		 1888 if (!test_bit(__I40E_DOWN, &vsi->state))
1889 wr32(hw, INTREG(vector - 1), txval);
ee2319cf
JB		 1890
1891 if (q_vector->itr_countdown)
1892 q_vector->itr_countdown--;
1893 else
1894 q_vector->itr_countdown = ITR_COUNTDOWN_START;
de32e3ef
CW		 1895}
1896
fd0a05ce
JB		 1897/**
1898 * i40e_napi_poll - NAPI polling Rx/Tx cleanup routine
1899 * @napi: napi struct with our devices info in it
1900 * @budget: amount of work driver is allowed to do this pass, in packets
1901 *
1902 * This function will clean all queues associated with a q_vector.
1903 *
1904 * Returns the amount of work done
1905 **/
1906int i40e_napi_poll(struct napi_struct *napi, int budget)
1907{
1908 struct i40e_q_vector *q_vector =
1909 container_of(napi, struct i40e_q_vector, napi);
1910 struct i40e_vsi *vsi = q_vector->vsi;
cd0b6fa6 1911 struct i40e_ring *ring;
fd0a05ce 1912 bool clean_complete = true;
d91649f5 1913 bool arm_wb = false;
fd0a05ce 1914 int budget_per_ring;
32b3e08f 1915 int work_done = 0;
fd0a05ce
JB		 1916
1917 if (test_bit(__I40E_DOWN, &vsi->state)) {
1918 napi_complete(napi);
1919 return 0;
1920 }
1921
9c6c1259
KP		 1922 /* Clear hung_detected bit */
1923 clear_bit(I40E_Q_VECTOR_HUNG_DETECT, &q_vector->hung_detected);
cd0b6fa6
AD		 1924 /* Since the actual Tx work is minimal, we can give the Tx a larger
1925 * budget and be more aggressive about cleaning up the Tx descriptors.
1926 */
d91649f5 1927 i40e_for_each_ring(ring, q_vector->tx) {
cd0b6fa6 1928 clean_complete &= i40e_clean_tx_irq(ring, vsi->work_limit);
44cdb791 1929 arm_wb = arm_wb || ring->arm_wb;
0deda868 1930 ring->arm_wb = false;
d91649f5 1931 }
cd0b6fa6 1932
c67caceb
AD		 1933 /* Handle case where we are called by netpoll with a budget of 0 */
1934 if (budget <= 0)
1935 goto tx_only;
1936
fd0a05ce
JB		 1937 /* We attempt to distribute budget to each Rx queue fairly, but don't
1938 * allow the budget to go below 1 because that would exit polling early.
fd0a05ce
JB		 1939 */
1940 budget_per_ring = max(budget/q_vector->num_ringpairs, 1);
cd0b6fa6 1941
a132af24 1942 i40e_for_each_ring(ring, q_vector->rx) {
32b3e08f
JB		 1943 int cleaned;
1944
a132af24
MW		 1945 if (ring_is_ps_enabled(ring))
1946 cleaned = i40e_clean_rx_irq_ps(ring, budget_per_ring);
1947 else
1948 cleaned = i40e_clean_rx_irq_1buf(ring, budget_per_ring);
32b3e08f
JB		 1949
1950 work_done += cleaned;
a132af24
MW		 1951 /* if we didn't clean as many as budgeted, we must be done */
1952 clean_complete &= (budget_per_ring != cleaned);
1953 }
fd0a05ce
JB		 1954
1955 /* If work not completed, return budget and polling will return */
d91649f5 1956 if (!clean_complete) {
c67caceb 1957tx_only:
164c9f54
ASJ		 1958 if (arm_wb) {
1959 q_vector->tx.ring[0].tx_stats.tx_force_wb++;
ecc6a239 1960 i40e_enable_wb_on_itr(vsi, q_vector);
164c9f54 1961 }
fd0a05ce 1962 return budget;
d91649f5 1963 }
fd0a05ce 1964
8e0764b4
ASJ		 1965 if (vsi->back->flags & I40E_TXR_FLAGS_WB_ON_ITR)
1966 q_vector->arm_wb_state = false;
1967
fd0a05ce 1968 /* Work is done so exit the polling mode and re-enable the interrupt */
32b3e08f 1969 napi_complete_done(napi, work_done);
de32e3ef
CW		 1970 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) {
1971 i40e_update_enable_itr(vsi, q_vector);
1972 } else { /* Legacy mode */
1973 struct i40e_hw *hw = &vsi->back->hw;
1974 /* We re-enable the queue 0 cause, but
1975 * don't worry about dynamic_enable
1976 * because we left it on for the other
1977 * possible interrupts during napi
1978 */
1979 u32 qval = rd32(hw, I40E_QINT_RQCTL(0)) |
1980 I40E_QINT_RQCTL_CAUSE_ENA_MASK;
1981
1982 wr32(hw, I40E_QINT_RQCTL(0), qval);
1983 qval = rd32(hw, I40E_QINT_TQCTL(0)) |
1984 I40E_QINT_TQCTL_CAUSE_ENA_MASK;
1985 wr32(hw, I40E_QINT_TQCTL(0), qval);
1986 i40e_irq_dynamic_enable_icr0(vsi->back);
fd0a05ce 1987 }
fd0a05ce
JB		 1988 return 0;
1989}
1990
1991/**
1992 * i40e_atr - Add a Flow Director ATR filter
1993 * @tx_ring: ring to add programming descriptor to
1994 * @skb: send buffer
89232c3b 1995 * @tx_flags: send tx flags
fd0a05ce
JB		 1996 * @protocol: wire protocol
1997 **/
1998static void i40e_atr(struct i40e_ring *tx_ring, struct sk_buff *skb,
89232c3b 1999 u32 tx_flags, __be16 protocol)
fd0a05ce
JB		 2000{
2001 struct i40e_filter_program_desc *fdir_desc;
2002 struct i40e_pf *pf = tx_ring->vsi->back;
2003 union {
2004 unsigned char *network;
2005 struct iphdr *ipv4;
2006 struct ipv6hdr *ipv6;
2007 } hdr;
2008 struct tcphdr *th;
2009 unsigned int hlen;
2010 u32 flex_ptype, dtype_cmd;
fc4ac67b 2011 u16 i;
fd0a05ce
JB		 2012
2013 /* make sure ATR is enabled */
60ea5f83 2014 if (!(pf->flags & I40E_FLAG_FD_ATR_ENABLED))
fd0a05ce
JB		 2015 return;
2016
04294e38
ASJ		 2017 if ((pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED))
2018 return;
2019
fd0a05ce
JB		 2020 /* if sampling is disabled do nothing */
2021 if (!tx_ring->atr_sample_rate)
2022 return;
2023
89232c3b
ASJ		 2024 if (!(tx_flags & (I40E_TX_FLAGS_IPV4 | I40E_TX_FLAGS_IPV6)))
2025 return;
fd0a05ce 2026
6a899024 2027 if (!(tx_flags & I40E_TX_FLAGS_UDP_TUNNEL)) {
89232c3b
ASJ		 2028 /* snag network header to get L4 type and address */
2029 hdr.network = skb_network_header(skb);
fd0a05ce 2030
89232c3b
ASJ		 2031 /* Currently only IPv4/IPv6 with TCP is supported
2032 * access ihl as u8 to avoid unaligned access on ia64
2033 */
2034 if (tx_flags & I40E_TX_FLAGS_IPV4)
2035 hlen = (hdr.network[0] & 0x0F) << 2;
2036 else if (protocol == htons(ETH_P_IPV6))
2037 hlen = sizeof(struct ipv6hdr);
2038 else
fd0a05ce 2039 return;
fd0a05ce 2040 } else {
89232c3b
ASJ		 2041 hdr.network = skb_inner_network_header(skb);
2042 hlen = skb_inner_network_header_len(skb);
fd0a05ce
JB		 2043 }
2044
89232c3b
ASJ		 2045 /* Currently only IPv4/IPv6 with TCP is supported
2046 * Note: tx_flags gets modified to reflect inner protocols in
2047 * tx_enable_csum function if encap is enabled.
2048 */
2049 if ((tx_flags & I40E_TX_FLAGS_IPV4) &&
2050 (hdr.ipv4->protocol != IPPROTO_TCP))
2051 return;
2052 else if ((tx_flags & I40E_TX_FLAGS_IPV6) &&
2053 (hdr.ipv6->nexthdr != IPPROTO_TCP))
2054 return;
2055
fd0a05ce
JB		 2056 th = (struct tcphdr *)(hdr.network + hlen);
2057
55a5e60b
ASJ		 2058 /* Due to lack of space, no more new filters can be programmed */
2059 if (th->syn && (pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED))
2060 return;
72b74869
ASJ		 2061 if ((pf->flags & I40E_FLAG_HW_ATR_EVICT_CAPABLE) &&
2062 (!(pf->auto_disable_flags & I40E_FLAG_HW_ATR_EVICT_CAPABLE))) {
52eb95ef
ASJ		 2063 /* HW ATR eviction will take care of removing filters on FIN
2064 * and RST packets.
2065 */
2066 if (th->fin || th->rst)
2067 return;
2068 }
55a5e60b
ASJ		 2069
2070 tx_ring->atr_count++;
2071
ce806783
ASJ		 2072 /* sample on all syn/fin/rst packets or once every atr sample rate */
2073 if (!th->fin &&
2074 !th->syn &&
2075 !th->rst &&
2076 (tx_ring->atr_count < tx_ring->atr_sample_rate))
fd0a05ce
JB		 2077 return;
2078
2079 tx_ring->atr_count = 0;
2080
2081 /* grab the next descriptor */
fc4ac67b
AD		 2082 i = tx_ring->next_to_use;
2083 fdir_desc = I40E_TX_FDIRDESC(tx_ring, i);
2084
2085 i++;
2086 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
fd0a05ce
JB		 2087
2088 flex_ptype = (tx_ring->queue_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
2089 I40E_TXD_FLTR_QW0_QINDEX_MASK;
2090 flex_ptype |= (protocol == htons(ETH_P_IP)) ?
2091 (I40E_FILTER_PCTYPE_NONF_IPV4_TCP <<
2092 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) :
2093 (I40E_FILTER_PCTYPE_NONF_IPV6_TCP <<
2094 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT);
2095
2096 flex_ptype |= tx_ring->vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
2097
2098 dtype_cmd = I40E_TX_DESC_DTYPE_FILTER_PROG;
2099
ce806783 2100 dtype_cmd |= (th->fin || th->rst) ?
fd0a05ce
JB		 2101 (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
2102 I40E_TXD_FLTR_QW1_PCMD_SHIFT) :
2103 (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
2104 I40E_TXD_FLTR_QW1_PCMD_SHIFT);
2105
2106 dtype_cmd |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX <<
2107 I40E_TXD_FLTR_QW1_DEST_SHIFT;
2108
2109 dtype_cmd |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID <<
2110 I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT;
2111
433c47de 2112 dtype_cmd |= I40E_TXD_FLTR_QW1_CNT_ENA_MASK;
6a899024 2113 if (!(tx_flags & I40E_TX_FLAGS_UDP_TUNNEL))
60ccd45c
ASJ		 2114 dtype_cmd |=
2115 ((u32)I40E_FD_ATR_STAT_IDX(pf->hw.pf_id) <<
2116 I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) &
2117 I40E_TXD_FLTR_QW1_CNTINDEX_MASK;
2118 else
2119 dtype_cmd |=
2120 ((u32)I40E_FD_ATR_TUNNEL_STAT_IDX(pf->hw.pf_id) <<
2121 I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) &
2122 I40E_TXD_FLTR_QW1_CNTINDEX_MASK;
433c47de 2123
72b74869
ASJ		 2124 if ((pf->flags & I40E_FLAG_HW_ATR_EVICT_CAPABLE) &&
2125 (!(pf->auto_disable_flags & I40E_FLAG_HW_ATR_EVICT_CAPABLE)))
52eb95ef
ASJ		 2126 dtype_cmd |= I40E_TXD_FLTR_QW1_ATR_MASK;
2127
fd0a05ce 2128 fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(flex_ptype);
99753ea6 2129 fdir_desc->rsvd = cpu_to_le32(0);
fd0a05ce 2130 fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dtype_cmd);
99753ea6 2131 fdir_desc->fd_id = cpu_to_le32(0);
fd0a05ce
JB		 2132}
2133
fd0a05ce
JB		 2134/**
2135 * i40e_tx_prepare_vlan_flags - prepare generic TX VLAN tagging flags for HW
2136 * @skb: send buffer
2137 * @tx_ring: ring to send buffer on
2138 * @flags: the tx flags to be set
2139 *
2140 * Checks the skb and set up correspondingly several generic transmit flags
2141 * related to VLAN tagging for the HW, such as VLAN, DCB, etc.
2142 *
2143 * Returns error code indicate the frame should be dropped upon error and the
2144 * otherwise returns 0 to indicate the flags has been set properly.
2145 **/
38e00438 2146#ifdef I40E_FCOE
3e587cf3 2147inline int i40e_tx_prepare_vlan_flags(struct sk_buff *skb,
fd0a05ce
JB		 2148 struct i40e_ring *tx_ring,
2149 u32 *flags)
3e587cf3
JB		 2150#else
2151static inline int i40e_tx_prepare_vlan_flags(struct sk_buff *skb,
2152 struct i40e_ring *tx_ring,
2153 u32 *flags)
38e00438 2154#endif
fd0a05ce
JB		 2155{
2156 __be16 protocol = skb->protocol;
2157 u32 tx_flags = 0;
2158
31eaaccf
GR		 2159 if (protocol == htons(ETH_P_8021Q) &&
2160 !(tx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) {
2161 /* When HW VLAN acceleration is turned off by the user the
2162 * stack sets the protocol to 8021q so that the driver
2163 * can take any steps required to support the SW only
2164 * VLAN handling. In our case the driver doesn't need
2165 * to take any further steps so just set the protocol
2166 * to the encapsulated ethertype.
2167 */
2168 skb->protocol = vlan_get_protocol(skb);
2169 goto out;
2170 }
2171
fd0a05ce 2172 /* if we have a HW VLAN tag being added, default to the HW one */
df8a39de
JP		 2173 if (skb_vlan_tag_present(skb)) {
2174 tx_flags |= skb_vlan_tag_get(skb) << I40E_TX_FLAGS_VLAN_SHIFT;
fd0a05ce
JB		 2175 tx_flags |= I40E_TX_FLAGS_HW_VLAN;
2176 /* else if it is a SW VLAN, check the next protocol and store the tag */
0e2fe46c 2177 } else if (protocol == htons(ETH_P_8021Q)) {
fd0a05ce 2178 struct vlan_hdr *vhdr, _vhdr;
6995b36c 2179
fd0a05ce
JB		 2180 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr);
2181 if (!vhdr)
2182 return -EINVAL;
2183
2184 protocol = vhdr->h_vlan_encapsulated_proto;
2185 tx_flags |= ntohs(vhdr->h_vlan_TCI) << I40E_TX_FLAGS_VLAN_SHIFT;
2186 tx_flags |= I40E_TX_FLAGS_SW_VLAN;
2187 }
2188
d40d00b1
NP		 2189 if (!(tx_ring->vsi->back->flags & I40E_FLAG_DCB_ENABLED))
2190 goto out;
2191
fd0a05ce 2192 /* Insert 802.1p priority into VLAN header */
38e00438
VD		 2193 if ((tx_flags & (I40E_TX_FLAGS_HW_VLAN | I40E_TX_FLAGS_SW_VLAN)) ||
2194 (skb->priority != TC_PRIO_CONTROL)) {
fd0a05ce
JB		 2195 tx_flags &= ~I40E_TX_FLAGS_VLAN_PRIO_MASK;
2196 tx_flags |= (skb->priority & 0x7) <<
2197 I40E_TX_FLAGS_VLAN_PRIO_SHIFT;
2198 if (tx_flags & I40E_TX_FLAGS_SW_VLAN) {
2199 struct vlan_ethhdr *vhdr;
dd225bc6
FR		 2200 int rc;
2201
2202 rc = skb_cow_head(skb, 0);
2203 if (rc < 0)
2204 return rc;
fd0a05ce
JB		 2205 vhdr = (struct vlan_ethhdr *)skb->data;
2206 vhdr->h_vlan_TCI = htons(tx_flags >>
2207 I40E_TX_FLAGS_VLAN_SHIFT);
2208 } else {
2209 tx_flags |= I40E_TX_FLAGS_HW_VLAN;
2210 }
2211 }
d40d00b1
NP		 2212
2213out:
fd0a05ce
JB		 2214 *flags = tx_flags;
2215 return 0;
2216}
2217
fd0a05ce
JB		 2218/**
2219 * i40e_tso - set up the tso context descriptor
2220 * @tx_ring: ptr to the ring to send
2221 * @skb: ptr to the skb we're sending
fd0a05ce 2222 * @hdr_len: ptr to the size of the packet header
9c883bd3 2223 * @cd_type_cmd_tso_mss: Quad Word 1
fd0a05ce
JB		 2224 *
2225 * Returns 0 if no TSO can happen, 1 if tso is going, or error
2226 **/
2227static int i40e_tso(struct i40e_ring *tx_ring, struct sk_buff *skb,
9c883bd3 2228 u8 *hdr_len, u64 *cd_type_cmd_tso_mss)
fd0a05ce
JB		 2229{
2230 u32 cd_cmd, cd_tso_len, cd_mss;
dd225bc6 2231 struct ipv6hdr *ipv6h;
fd0a05ce
JB		 2232 struct tcphdr *tcph;
2233 struct iphdr *iph;
2234 u32 l4len;
2235 int err;
fd0a05ce 2236
e9f6563d
SN		 2237 if (skb->ip_summed != CHECKSUM_PARTIAL)
2238 return 0;
2239
fd0a05ce
JB		 2240 if (!skb_is_gso(skb))
2241 return 0;
2242
dd225bc6
FR		 2243 err = skb_cow_head(skb, 0);
2244 if (err < 0)
2245 return err;
fd0a05ce 2246
df23075f
AS		 2247 iph = skb->encapsulation ? inner_ip_hdr(skb) : ip_hdr(skb);
2248 ipv6h = skb->encapsulation ? inner_ipv6_hdr(skb) : ipv6_hdr(skb);
2249
2250 if (iph->version == 4) {
fd0a05ce
JB		 2251 tcph = skb->encapsulation ? inner_tcp_hdr(skb) : tcp_hdr(skb);
2252 iph->tot_len = 0;
2253 iph->check = 0;
2254 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
2255 0, IPPROTO_TCP, 0);
df23075f 2256 } else if (ipv6h->version == 6) {
fd0a05ce
JB		 2257 tcph = skb->encapsulation ? inner_tcp_hdr(skb) : tcp_hdr(skb);
2258 ipv6h->payload_len = 0;
2259 tcph->check = ~csum_ipv6_magic(&ipv6h->saddr, &ipv6h->daddr,
2260 0, IPPROTO_TCP, 0);
2261 }
2262
2263 l4len = skb->encapsulation ? inner_tcp_hdrlen(skb) : tcp_hdrlen(skb);
2264 *hdr_len = (skb->encapsulation
2265 ? (skb_inner_transport_header(skb) - skb->data)
2266 : skb_transport_offset(skb)) + l4len;
2267
2268 /* find the field values */
2269 cd_cmd = I40E_TX_CTX_DESC_TSO;
2270 cd_tso_len = skb->len - *hdr_len;
2271 cd_mss = skb_shinfo(skb)->gso_size;
829af3ac
MW		 2272 *cd_type_cmd_tso_mss |= ((u64)cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) |
2273 ((u64)cd_tso_len <<
2274 I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) |
2275 ((u64)cd_mss << I40E_TXD_CTX_QW1_MSS_SHIFT);
fd0a05ce
JB		 2276 return 1;
2277}
2278
beb0dff1
JK		 2279/**
2280 * i40e_tsyn - set up the tsyn context descriptor
2281 * @tx_ring: ptr to the ring to send
2282 * @skb: ptr to the skb we're sending
2283 * @tx_flags: the collected send information
9c883bd3 2284 * @cd_type_cmd_tso_mss: Quad Word 1
beb0dff1
JK		 2285 *
2286 * Returns 0 if no Tx timestamp can happen and 1 if the timestamp will happen
2287 **/
2288static int i40e_tsyn(struct i40e_ring *tx_ring, struct sk_buff *skb,
2289 u32 tx_flags, u64 *cd_type_cmd_tso_mss)
2290{
2291 struct i40e_pf *pf;
2292
2293 if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)))
2294 return 0;
2295
2296 /* Tx timestamps cannot be sampled when doing TSO */
2297 if (tx_flags & I40E_TX_FLAGS_TSO)
2298 return 0;
2299
2300 /* only timestamp the outbound packet if the user has requested it and
2301 * we are not already transmitting a packet to be timestamped
2302 */
2303 pf = i40e_netdev_to_pf(tx_ring->netdev);
22b4777d
JK		 2304 if (!(pf->flags & I40E_FLAG_PTP))
2305 return 0;
2306
9ce34f02
JK		 2307 if (pf->ptp_tx &&
2308 !test_and_set_bit_lock(__I40E_PTP_TX_IN_PROGRESS, &pf->state)) {
beb0dff1
JK		 2309 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2310 pf->ptp_tx_skb = skb_get(skb);
2311 } else {
2312 return 0;
2313 }
2314
2315 *cd_type_cmd_tso_mss |= (u64)I40E_TX_CTX_DESC_TSYN <<
2316 I40E_TXD_CTX_QW1_CMD_SHIFT;
2317
beb0dff1
JK		 2318 return 1;
2319}
2320
fd0a05ce
JB		 2321/**
2322 * i40e_tx_enable_csum - Enable Tx checksum offloads
2323 * @skb: send buffer
89232c3b 2324 * @tx_flags: pointer to Tx flags currently set
fd0a05ce
JB		 2325 * @td_cmd: Tx descriptor command bits to set
2326 * @td_offset: Tx descriptor header offsets to set
554f4544 2327 * @tx_ring: Tx descriptor ring
fd0a05ce
JB		 2328 * @cd_tunneling: ptr to context desc bits
2329 **/
89232c3b 2330static void i40e_tx_enable_csum(struct sk_buff *skb, u32 *tx_flags,
fd0a05ce
JB		 2331 u32 *td_cmd, u32 *td_offset,
2332 struct i40e_ring *tx_ring,
2333 u32 *cd_tunneling)
2334{
2335 struct ipv6hdr *this_ipv6_hdr;
2336 unsigned int this_tcp_hdrlen;
2337 struct iphdr *this_ip_hdr;
2338 u32 network_hdr_len;
2339 u8 l4_hdr = 0;
79febbc1
AB		 2340 struct udphdr *oudph = NULL;
2341 struct iphdr *oiph = NULL;
45991204 2342 u32 l4_tunnel = 0;
fd0a05ce
JB		 2343
2344 if (skb->encapsulation) {
45991204
ASJ		 2345 switch (ip_hdr(skb)->protocol) {
2346 case IPPROTO_UDP:
527274c7
ASJ		 2347 oudph = udp_hdr(skb);
2348 oiph = ip_hdr(skb);
45991204 2349 l4_tunnel = I40E_TXD_CTX_UDP_TUNNELING;
6a899024 2350 *tx_flags |= I40E_TX_FLAGS_UDP_TUNNEL;
45991204 2351 break;
c1d1791d
SN		 2352 case IPPROTO_GRE:
2353 l4_tunnel = I40E_TXD_CTX_GRE_TUNNELING;
2354 break;
45991204
ASJ		 2355 default:
2356 return;
2357 }
fd0a05ce
JB		 2358 network_hdr_len = skb_inner_network_header_len(skb);
2359 this_ip_hdr = inner_ip_hdr(skb);
2360 this_ipv6_hdr = inner_ipv6_hdr(skb);
2361 this_tcp_hdrlen = inner_tcp_hdrlen(skb);
2362
89232c3b
ASJ		 2363 if (*tx_flags & I40E_TX_FLAGS_IPV4) {
2364 if (*tx_flags & I40E_TX_FLAGS_TSO) {
fd0a05ce
JB		 2365 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV4;
2366 ip_hdr(skb)->check = 0;
2367 } else {
2368 *cd_tunneling |=
2369 I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;
2370 }
89232c3b 2371 } else if (*tx_flags & I40E_TX_FLAGS_IPV6) {
df23075f 2372 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV6;
89232c3b 2373 if (*tx_flags & I40E_TX_FLAGS_TSO)
fd0a05ce 2374 ip_hdr(skb)->check = 0;
fd0a05ce
JB		 2375 }
2376
2377 /* Now set the ctx descriptor fields */
2378 *cd_tunneling |= (skb_network_header_len(skb) >> 2) <<
45991204
ASJ		 2379 I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT |
2380 l4_tunnel |
fd0a05ce
JB		 2381 ((skb_inner_network_offset(skb) -
2382 skb_transport_offset(skb)) >> 1) <<
2383 I40E_TXD_CTX_QW0_NATLEN_SHIFT;
df23075f 2384 if (this_ip_hdr->version == 6) {
89232c3b
ASJ		 2385 *tx_flags &= ~I40E_TX_FLAGS_IPV4;
2386 *tx_flags |= I40E_TX_FLAGS_IPV6;
df23075f 2387 }
527274c7
ASJ		 2388 if ((tx_ring->flags & I40E_TXR_FLAGS_OUTER_UDP_CSUM) &&
2389 (l4_tunnel == I40E_TXD_CTX_UDP_TUNNELING) &&
2390 (*cd_tunneling & I40E_TXD_CTX_QW0_EXT_IP_MASK)) {
2391 oudph->check = ~csum_tcpudp_magic(oiph->saddr,
2392 oiph->daddr,
2393 (skb->len - skb_transport_offset(skb)),
2394 IPPROTO_UDP, 0);
2395 *cd_tunneling |= I40E_TXD_CTX_QW0_L4T_CS_MASK;
2396 }
fd0a05ce
JB		 2397 } else {
2398 network_hdr_len = skb_network_header_len(skb);
2399 this_ip_hdr = ip_hdr(skb);
2400 this_ipv6_hdr = ipv6_hdr(skb);
2401 this_tcp_hdrlen = tcp_hdrlen(skb);
2402 }
2403
2404 /* Enable IP checksum offloads */
89232c3b 2405 if (*tx_flags & I40E_TX_FLAGS_IPV4) {
fd0a05ce
JB		 2406 l4_hdr = this_ip_hdr->protocol;
2407 /* the stack computes the IP header already, the only time we
2408 * need the hardware to recompute it is in the case of TSO.
2409 */
89232c3b 2410 if (*tx_flags & I40E_TX_FLAGS_TSO) {
fd0a05ce
JB		 2411 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM;
2412 this_ip_hdr->check = 0;
2413 } else {
2414 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4;
2415 }
2416 /* Now set the td_offset for IP header length */
2417 *td_offset = (network_hdr_len >> 2) <<
2418 I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
89232c3b 2419 } else if (*tx_flags & I40E_TX_FLAGS_IPV6) {
fd0a05ce
JB		 2420 l4_hdr = this_ipv6_hdr->nexthdr;
2421 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV6;
2422 /* Now set the td_offset for IP header length */
2423 *td_offset = (network_hdr_len >> 2) <<
2424 I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
2425 }
2426 /* words in MACLEN + dwords in IPLEN + dwords in L4Len */
2427 *td_offset |= (skb_network_offset(skb) >> 1) <<
2428 I40E_TX_DESC_LENGTH_MACLEN_SHIFT;
2429
2430 /* Enable L4 checksum offloads */
2431 switch (l4_hdr) {
2432 case IPPROTO_TCP:
2433 /* enable checksum offloads */
2434 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
2435 *td_offset |= (this_tcp_hdrlen >> 2) <<
2436 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
2437 break;
2438 case IPPROTO_SCTP:
2439 /* enable SCTP checksum offload */
2440 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP;
2441 *td_offset |= (sizeof(struct sctphdr) >> 2) <<
2442 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
2443 break;
2444 case IPPROTO_UDP:
2445 /* enable UDP checksum offload */
2446 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP;
2447 *td_offset |= (sizeof(struct udphdr) >> 2) <<
2448 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
2449 break;
2450 default:
2451 break;
2452 }
2453}
2454
2455/**
2456 * i40e_create_tx_ctx Build the Tx context descriptor
2457 * @tx_ring: ring to create the descriptor on
2458 * @cd_type_cmd_tso_mss: Quad Word 1
2459 * @cd_tunneling: Quad Word 0 - bits 0-31
2460 * @cd_l2tag2: Quad Word 0 - bits 32-63
2461 **/
2462static void i40e_create_tx_ctx(struct i40e_ring *tx_ring,
2463 const u64 cd_type_cmd_tso_mss,
2464 const u32 cd_tunneling, const u32 cd_l2tag2)
2465{
2466 struct i40e_tx_context_desc *context_desc;
fc4ac67b 2467 int i = tx_ring->next_to_use;
fd0a05ce 2468
ff40dd5d
JB		 2469 if ((cd_type_cmd_tso_mss == I40E_TX_DESC_DTYPE_CONTEXT) &&
2470 !cd_tunneling && !cd_l2tag2)
fd0a05ce
JB		 2471 return;
2472
2473 /* grab the next descriptor */
fc4ac67b
AD		 2474 context_desc = I40E_TX_CTXTDESC(tx_ring, i);
2475
2476 i++;
2477 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
fd0a05ce
JB		 2478
2479 /* cpu_to_le32 and assign to struct fields */
2480 context_desc->tunneling_params = cpu_to_le32(cd_tunneling);
2481 context_desc->l2tag2 = cpu_to_le16(cd_l2tag2);
3efbbb20 2482 context_desc->rsvd = cpu_to_le16(0);
fd0a05ce
JB		 2483 context_desc->type_cmd_tso_mss = cpu_to_le64(cd_type_cmd_tso_mss);
2484}
2485
4567dc10
ED		 2486/**
2487 * __i40e_maybe_stop_tx - 2nd level check for tx stop conditions
2488 * @tx_ring: the ring to be checked
2489 * @size: the size buffer we want to assure is available
2490 *
2491 * Returns -EBUSY if a stop is needed, else 0
2492 **/
2493static inline int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
2494{
2495 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
2496 /* Memory barrier before checking head and tail */
2497 smp_mb();
2498
2499 /* Check again in a case another CPU has just made room available. */
2500 if (likely(I40E_DESC_UNUSED(tx_ring) < size))
2501 return -EBUSY;
2502
2503 /* A reprieve! - use start_queue because it doesn't call schedule */
2504 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
2505 ++tx_ring->tx_stats.restart_queue;
2506 return 0;
2507}
2508
2509/**
2510 * i40e_maybe_stop_tx - 1st level check for tx stop conditions
2511 * @tx_ring: the ring to be checked
2512 * @size: the size buffer we want to assure is available
2513 *
2514 * Returns 0 if stop is not needed
2515 **/
2516#ifdef I40E_FCOE
3e587cf3 2517inline int i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
4567dc10 2518#else
3e587cf3 2519static inline int i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
4567dc10
ED		 2520#endif
2521{
2522 if (likely(I40E_DESC_UNUSED(tx_ring) >= size))
2523 return 0;
2524 return __i40e_maybe_stop_tx(tx_ring, size);
2525}
2526
71da6197
AS		 2527/**
2528 * i40e_chk_linearize - Check if there are more than 8 fragments per packet
2529 * @skb: send buffer
2530 * @tx_flags: collected send information
71da6197
AS		 2531 *
2532 * Note: Our HW can't scatter-gather more than 8 fragments to build
2533 * a packet on the wire and so we need to figure out the cases where we
2534 * need to linearize the skb.
2535 **/
30520831 2536static bool i40e_chk_linearize(struct sk_buff *skb, u32 tx_flags)
71da6197
AS		 2537{
2538 struct skb_frag_struct *frag;
2539 bool linearize = false;
2540 unsigned int size = 0;
2541 u16 num_frags;
2542 u16 gso_segs;
2543
2544 num_frags = skb_shinfo(skb)->nr_frags;
2545 gso_segs = skb_shinfo(skb)->gso_segs;
2546
2547 if (tx_flags & (I40E_TX_FLAGS_TSO | I40E_TX_FLAGS_FSO)) {
30520831 2548 u16 j = 0;
71da6197
AS		 2549
2550 if (num_frags < (I40E_MAX_BUFFER_TXD))
2551 goto linearize_chk_done;
2552 /* try the simple math, if we have too many frags per segment */
2553 if (DIV_ROUND_UP((num_frags + gso_segs), gso_segs) >
2554 I40E_MAX_BUFFER_TXD) {
2555 linearize = true;
2556 goto linearize_chk_done;
2557 }
2558 frag = &skb_shinfo(skb)->frags[0];
71da6197
AS		 2559 /* we might still have more fragments per segment */
2560 do {
2561 size += skb_frag_size(frag);
2562 frag++; j++;
30520831
ASJ		 2563 if ((size >= skb_shinfo(skb)->gso_size) &&
2564 (j < I40E_MAX_BUFFER_TXD)) {
2565 size = (size % skb_shinfo(skb)->gso_size);
2566 j = (size) ? 1 : 0;
2567 }
71da6197 2568 if (j == I40E_MAX_BUFFER_TXD) {
30520831
ASJ		 2569 linearize = true;
2570 break;
71da6197
AS		 2571 }
2572 num_frags--;
2573 } while (num_frags);
2574 } else {
2575 if (num_frags >= I40E_MAX_BUFFER_TXD)
2576 linearize = true;
2577 }
2578
2579linearize_chk_done:
2580 return linearize;
2581}
2582
fd0a05ce
JB		 2583/**
2584 * i40e_tx_map - Build the Tx descriptor
2585 * @tx_ring: ring to send buffer on
2586 * @skb: send buffer
2587 * @first: first buffer info buffer to use
2588 * @tx_flags: collected send information
2589 * @hdr_len: size of the packet header
2590 * @td_cmd: the command field in the descriptor
2591 * @td_offset: offset for checksum or crc
2592 **/
38e00438 2593#ifdef I40E_FCOE
3e587cf3 2594inline void i40e_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb,
fd0a05ce
JB		 2595 struct i40e_tx_buffer *first, u32 tx_flags,
2596 const u8 hdr_len, u32 td_cmd, u32 td_offset)
3e587cf3
JB		 2597#else
2598static inline void i40e_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb,
2599 struct i40e_tx_buffer *first, u32 tx_flags,
2600 const u8 hdr_len, u32 td_cmd, u32 td_offset)
38e00438 2601#endif
fd0a05ce 2602{
fd0a05ce
JB		 2603 unsigned int data_len = skb->data_len;
2604 unsigned int size = skb_headlen(skb);
a5e9c572 2605 struct skb_frag_struct *frag;
fd0a05ce
JB		 2606 struct i40e_tx_buffer *tx_bi;
2607 struct i40e_tx_desc *tx_desc;
a5e9c572 2608 u16 i = tx_ring->next_to_use;
fd0a05ce
JB		 2609 u32 td_tag = 0;
2610 dma_addr_t dma;
2611 u16 gso_segs;
58044743
AS		 2612 u16 desc_count = 0;
2613 bool tail_bump = true;
2614 bool do_rs = false;
fd0a05ce 2615
fd0a05ce
JB		 2616 if (tx_flags & I40E_TX_FLAGS_HW_VLAN) {
2617 td_cmd |= I40E_TX_DESC_CMD_IL2TAG1;
2618 td_tag = (tx_flags & I40E_TX_FLAGS_VLAN_MASK) >>
2619 I40E_TX_FLAGS_VLAN_SHIFT;
2620 }
2621
a5e9c572
AD		 2622 if (tx_flags & (I40E_TX_FLAGS_TSO | I40E_TX_FLAGS_FSO))
2623 gso_segs = skb_shinfo(skb)->gso_segs;
2624 else
2625 gso_segs = 1;
2626
2627 /* multiply data chunks by size of headers */
2628 first->bytecount = skb->len - hdr_len + (gso_segs * hdr_len);
2629 first->gso_segs = gso_segs;
2630 first->skb = skb;
2631 first->tx_flags = tx_flags;
2632
2633 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
2634
fd0a05ce 2635 tx_desc = I40E_TX_DESC(tx_ring, i);
a5e9c572
AD		 2636 tx_bi = first;
2637
2638 for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
2639 if (dma_mapping_error(tx_ring->dev, dma))
2640 goto dma_error;
2641
2642 /* record length, and DMA address */
2643 dma_unmap_len_set(tx_bi, len, size);
2644 dma_unmap_addr_set(tx_bi, dma, dma);
2645
2646 tx_desc->buffer_addr = cpu_to_le64(dma);
2647
2648 while (unlikely(size > I40E_MAX_DATA_PER_TXD)) {
fd0a05ce
JB		 2649 tx_desc->cmd_type_offset_bsz =
2650 build_ctob(td_cmd, td_offset,
2651 I40E_MAX_DATA_PER_TXD, td_tag);
2652
fd0a05ce
JB		 2653 tx_desc++;
2654 i++;
58044743
AS		 2655 desc_count++;
2656
fd0a05ce
JB		 2657 if (i == tx_ring->count) {
2658 tx_desc = I40E_TX_DESC(tx_ring, 0);
2659 i = 0;
2660 }
fd0a05ce 2661
a5e9c572
AD		 2662 dma += I40E_MAX_DATA_PER_TXD;
2663 size -= I40E_MAX_DATA_PER_TXD;
fd0a05ce 2664
a5e9c572
AD		 2665 tx_desc->buffer_addr = cpu_to_le64(dma);
2666 }
fd0a05ce
JB		 2667
2668 if (likely(!data_len))
2669 break;
2670
a5e9c572
AD		 2671 tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset,
2672 size, td_tag);
fd0a05ce
JB		 2673
2674 tx_desc++;
2675 i++;
58044743
AS		 2676 desc_count++;
2677
fd0a05ce
JB		 2678 if (i == tx_ring->count) {
2679 tx_desc = I40E_TX_DESC(tx_ring, 0);
2680 i = 0;
2681 }
2682
a5e9c572
AD		 2683 size = skb_frag_size(frag);
2684 data_len -= size;
fd0a05ce 2685
a5e9c572
AD		 2686 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
2687 DMA_TO_DEVICE);
fd0a05ce 2688
a5e9c572
AD		 2689 tx_bi = &tx_ring->tx_bi[i];
2690 }
fd0a05ce 2691
a5e9c572
AD		 2692 /* set next_to_watch value indicating a packet is present */
2693 first->next_to_watch = tx_desc;
2694
2695 i++;
2696 if (i == tx_ring->count)
2697 i = 0;
2698
2699 tx_ring->next_to_use = i;
2700
58044743
AS		 2701 netdev_tx_sent_queue(netdev_get_tx_queue(tx_ring->netdev,
2702 tx_ring->queue_index),
2703 first->bytecount);
4567dc10 2704 i40e_maybe_stop_tx(tx_ring, DESC_NEEDED);
58044743
AS		 2705
2706 /* Algorithm to optimize tail and RS bit setting:
2707 * if xmit_more is supported
2708 * if xmit_more is true
2709 * do not update tail and do not mark RS bit.
2710 * if xmit_more is false and last xmit_more was false
2711 * if every packet spanned less than 4 desc
2712 * then set RS bit on 4th packet and update tail
2713 * on every packet
2714 * else
2715 * update tail and set RS bit on every packet.
2716 * if xmit_more is false and last_xmit_more was true
2717 * update tail and set RS bit.
2718 *
2719 * Optimization: wmb to be issued only in case of tail update.
2720 * Also optimize the Descriptor WB path for RS bit with the same
2721 * algorithm.
2722 *
2723 * Note: If there are less than 4 packets
2724 * pending and interrupts were disabled the service task will
2725 * trigger a force WB.
2726 */
2727 if (skb->xmit_more &&
2728 !netif_xmit_stopped(netdev_get_tx_queue(tx_ring->netdev,
2729 tx_ring->queue_index))) {
2730 tx_ring->flags |= I40E_TXR_FLAGS_LAST_XMIT_MORE_SET;
2731 tail_bump = false;
2732 } else if (!skb->xmit_more &&
2733 !netif_xmit_stopped(netdev_get_tx_queue(tx_ring->netdev,
2734 tx_ring->queue_index)) &&
2735 (!(tx_ring->flags & I40E_TXR_FLAGS_LAST_XMIT_MORE_SET)) &&
2736 (tx_ring->packet_stride < WB_STRIDE) &&
2737 (desc_count < WB_STRIDE)) {
2738 tx_ring->packet_stride++;
2739 } else {
2740 tx_ring->packet_stride = 0;
2741 tx_ring->flags &= ~I40E_TXR_FLAGS_LAST_XMIT_MORE_SET;
2742 do_rs = true;
2743 }
2744 if (do_rs)
2745 tx_ring->packet_stride = 0;
2746
2747 tx_desc->cmd_type_offset_bsz =
2748 build_ctob(td_cmd, td_offset, size, td_tag) |
2749 cpu_to_le64((u64)(do_rs ? I40E_TXD_CMD :
2750 I40E_TX_DESC_CMD_EOP) <<
2751 I40E_TXD_QW1_CMD_SHIFT);
2752
a5e9c572 2753 /* notify HW of packet */
58044743 2754 if (!tail_bump)
489ce7a4 2755 prefetchw(tx_desc + 1);
a5e9c572 2756
58044743
AS		 2757 if (tail_bump) {
2758 /* Force memory writes to complete before letting h/w
2759 * know there are new descriptors to fetch. (Only
2760 * applicable for weak-ordered memory model archs,
2761 * such as IA-64).
2762 */
2763 wmb();
2764 writel(i, tx_ring->tail);
2765 }
2766
fd0a05ce
JB		 2767 return;
2768
2769dma_error:
a5e9c572 2770 dev_info(tx_ring->dev, "TX DMA map failed\n");
fd0a05ce
JB		 2771
2772 /* clear dma mappings for failed tx_bi map */
2773 for (;;) {
2774 tx_bi = &tx_ring->tx_bi[i];
a5e9c572 2775 i40e_unmap_and_free_tx_resource(tx_ring, tx_bi);
fd0a05ce
JB		 2776 if (tx_bi == first)
2777 break;
2778 if (i == 0)
2779 i = tx_ring->count;
2780 i--;
2781 }
2782
fd0a05ce
JB		 2783 tx_ring->next_to_use = i;
2784}
2785
fd0a05ce
JB		 2786/**
2787 * i40e_xmit_descriptor_count - calculate number of tx descriptors needed
2788 * @skb: send buffer
2789 * @tx_ring: ring to send buffer on
2790 *
2791 * Returns number of data descriptors needed for this skb. Returns 0 to indicate
2792 * there is not enough descriptors available in this ring since we need at least
2793 * one descriptor.
2794 **/
38e00438 2795#ifdef I40E_FCOE
3e587cf3 2796inline int i40e_xmit_descriptor_count(struct sk_buff *skb,
fd0a05ce 2797 struct i40e_ring *tx_ring)
3e587cf3
JB		 2798#else
2799static inline int i40e_xmit_descriptor_count(struct sk_buff *skb,
2800 struct i40e_ring *tx_ring)
38e00438 2801#endif
fd0a05ce 2802{
fd0a05ce 2803 unsigned int f;
fd0a05ce
JB		 2804 int count = 0;
2805
2806 /* need: 1 descriptor per page * PAGE_SIZE/I40E_MAX_DATA_PER_TXD,
2807 * + 1 desc for skb_head_len/I40E_MAX_DATA_PER_TXD,
be560521 2808 * + 4 desc gap to avoid the cache line where head is,
fd0a05ce
JB		 2809 * + 1 desc for context descriptor,
2810 * otherwise try next time
2811 */
fd0a05ce
JB		 2812 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
2813 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
980093eb 2814
fd0a05ce 2815 count += TXD_USE_COUNT(skb_headlen(skb));
be560521 2816 if (i40e_maybe_stop_tx(tx_ring, count + 4 + 1)) {
fd0a05ce
JB		 2817 tx_ring->tx_stats.tx_busy++;
2818 return 0;
2819 }
2820 return count;
2821}
2822
2823/**
2824 * i40e_xmit_frame_ring - Sends buffer on Tx ring
2825 * @skb: send buffer
2826 * @tx_ring: ring to send buffer on
2827 *
2828 * Returns NETDEV_TX_OK if sent, else an error code
2829 **/
2830static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb,
2831 struct i40e_ring *tx_ring)
2832{
2833 u64 cd_type_cmd_tso_mss = I40E_TX_DESC_DTYPE_CONTEXT;
2834 u32 cd_tunneling = 0, cd_l2tag2 = 0;
2835 struct i40e_tx_buffer *first;
2836 u32 td_offset = 0;
2837 u32 tx_flags = 0;
2838 __be16 protocol;
2839 u32 td_cmd = 0;
2840 u8 hdr_len = 0;
beb0dff1 2841 int tsyn;
fd0a05ce 2842 int tso;
6995b36c 2843
b74118f0
JB		 2844 /* prefetch the data, we'll need it later */
2845 prefetch(skb->data);
2846
fd0a05ce
JB		 2847 if (0 == i40e_xmit_descriptor_count(skb, tx_ring))
2848 return NETDEV_TX_BUSY;
2849
2850 /* prepare the xmit flags */
2851 if (i40e_tx_prepare_vlan_flags(skb, tx_ring, &tx_flags))
2852 goto out_drop;
2853
2854 /* obtain protocol of skb */
3d34dd03 2855 protocol = vlan_get_protocol(skb);
fd0a05ce
JB		 2856
2857 /* record the location of the first descriptor for this packet */
2858 first = &tx_ring->tx_bi[tx_ring->next_to_use];
2859
2860 /* setup IPv4/IPv6 offloads */
0e2fe46c 2861 if (protocol == htons(ETH_P_IP))
fd0a05ce 2862 tx_flags |= I40E_TX_FLAGS_IPV4;
0e2fe46c 2863 else if (protocol == htons(ETH_P_IPV6))
fd0a05ce
JB		 2864 tx_flags |= I40E_TX_FLAGS_IPV6;
2865
9c883bd3 2866 tso = i40e_tso(tx_ring, skb, &hdr_len, &cd_type_cmd_tso_mss);
fd0a05ce
JB		 2867
2868 if (tso < 0)
2869 goto out_drop;
2870 else if (tso)
2871 tx_flags |= I40E_TX_FLAGS_TSO;
2872
beb0dff1
JK		 2873 tsyn = i40e_tsyn(tx_ring, skb, tx_flags, &cd_type_cmd_tso_mss);
2874
2875 if (tsyn)
2876 tx_flags |= I40E_TX_FLAGS_TSYN;
2877
2fc3d715 2878 if (i40e_chk_linearize(skb, tx_flags)) {
71da6197
AS		 2879 if (skb_linearize(skb))
2880 goto out_drop;
2fc3d715
ASJ		 2881 tx_ring->tx_stats.tx_linearize++;
2882 }
259afec7
JK		 2883 skb_tx_timestamp(skb);
2884
b1941306
AD		 2885 /* always enable CRC insertion offload */
2886 td_cmd |= I40E_TX_DESC_CMD_ICRC;
2887
fd0a05ce 2888 /* Always offload the checksum, since it's in the data descriptor */
b1941306 2889 if (skb->ip_summed == CHECKSUM_PARTIAL) {
fd0a05ce
JB		 2890 tx_flags |= I40E_TX_FLAGS_CSUM;
2891
89232c3b 2892 i40e_tx_enable_csum(skb, &tx_flags, &td_cmd, &td_offset,
fd0a05ce 2893 tx_ring, &cd_tunneling);
b1941306 2894 }
fd0a05ce
JB		 2895
2896 i40e_create_tx_ctx(tx_ring, cd_type_cmd_tso_mss,
2897 cd_tunneling, cd_l2tag2);
2898
2899 /* Add Flow Director ATR if it's enabled.
2900 *
2901 * NOTE: this must always be directly before the data descriptor.
2902 */
2903 i40e_atr(tx_ring, skb, tx_flags, protocol);
2904
2905 i40e_tx_map(tx_ring, skb, first, tx_flags, hdr_len,
2906 td_cmd, td_offset);
2907
fd0a05ce
JB		 2908 return NETDEV_TX_OK;
2909
2910out_drop:
2911 dev_kfree_skb_any(skb);
2912 return NETDEV_TX_OK;
2913}
2914
2915/**
2916 * i40e_lan_xmit_frame - Selects the correct VSI and Tx queue to send buffer
2917 * @skb: send buffer
2918 * @netdev: network interface device structure
2919 *
2920 * Returns NETDEV_TX_OK if sent, else an error code
2921 **/
2922netdev_tx_t i40e_lan_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
2923{
2924 struct i40e_netdev_priv *np = netdev_priv(netdev);
2925 struct i40e_vsi *vsi = np->vsi;
9f65e15b 2926 struct i40e_ring *tx_ring = vsi->tx_rings[skb->queue_mapping];
fd0a05ce
JB		 2927
2928 /* hardware can't handle really short frames, hardware padding works
2929 * beyond this point
2930 */
a94d9e22
AD		 2931 if (skb_put_padto(skb, I40E_MIN_TX_LEN))
2932 return NETDEV_TX_OK;
fd0a05ce
JB		 2933
2934 return i40e_xmit_frame_ring(skb, tx_ring);
2935}
Linux 6.x block layer and io_uring tree(s)