projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
i40e: Flow Director sideband accounting
[linux-2.6-block.git] / drivers / net / ethernet / intel / i40e / i40e_txrx.c
CommitLineData
fd0a05ce
JB		 1/*******************************************************************************
2 *
3 * Intel Ethernet Controller XL710 Family Linux Driver
dc641b73 4 * Copyright(c) 2013 - 2014 Intel Corporation.
fd0a05ce
JB		 5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * more details.
14 *
dc641b73
GR		 15 * You should have received a copy of the GNU General Public License along
16 * with this program. If not, see <http://www.gnu.org/licenses/>.
fd0a05ce
JB		 17 *
18 * The full GNU General Public License is included in this distribution in
19 * the file called "COPYING".
20 *
21 * Contact Information:
22 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
23 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
24 *
25 ******************************************************************************/
26
27#include "i40e.h"
28
29static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size,
30 u32 td_tag)
31{
32 return cpu_to_le64(I40E_TX_DESC_DTYPE_DATA |
33 ((u64)td_cmd << I40E_TXD_QW1_CMD_SHIFT) |
34 ((u64)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) |
35 ((u64)size << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) |
36 ((u64)td_tag << I40E_TXD_QW1_L2TAG1_SHIFT));
37}
38
eaefbd06 39#define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS)
fd0a05ce
JB		 40/**
41 * i40e_program_fdir_filter - Program a Flow Director filter
17a73f6b
JG		 42 * @fdir_data: Packet data that will be filter parameters
43 * @raw_packet: the pre-allocated packet buffer for FDir
fd0a05ce
JB		 44 * @pf: The pf pointer
45 * @add: True for add/update, False for remove
46 **/
17a73f6b 47int i40e_program_fdir_filter(struct i40e_fdir_filter *fdir_data, u8 *raw_packet,
fd0a05ce
JB		 48 struct i40e_pf *pf, bool add)
49{
50 struct i40e_filter_program_desc *fdir_desc;
51 struct i40e_tx_buffer *tx_buf;
52 struct i40e_tx_desc *tx_desc;
53 struct i40e_ring *tx_ring;
eaefbd06 54 unsigned int fpt, dcc;
fd0a05ce
JB		 55 struct i40e_vsi *vsi;
56 struct device *dev;
57 dma_addr_t dma;
58 u32 td_cmd = 0;
59 u16 i;
60
61 /* find existing FDIR VSI */
62 vsi = NULL;
63 for (i = 0; i < pf->hw.func_caps.num_vsis; i++)
64 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR)
65 vsi = pf->vsi[i];
66 if (!vsi)
67 return -ENOENT;
68
9f65e15b 69 tx_ring = vsi->tx_rings[0];
fd0a05ce
JB		 70 dev = tx_ring->dev;
71
17a73f6b
JG		 72 dma = dma_map_single(dev, raw_packet,
73 I40E_FDIR_MAX_RAW_PACKET_SIZE, DMA_TO_DEVICE);
fd0a05ce
JB		 74 if (dma_mapping_error(dev, dma))
75 goto dma_fail;
76
77 /* grab the next descriptor */
fc4ac67b
AD		 78 i = tx_ring->next_to_use;
79 fdir_desc = I40E_TX_FDIRDESC(tx_ring, i);
fc4ac67b 80
eaefbd06 81 tx_ring->next_to_use = (i + 1 < tx_ring->count) ? i + 1 : 0;
fd0a05ce 82
eaefbd06
JB		 83 fpt = (fdir_data->q_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
84 I40E_TXD_FLTR_QW0_QINDEX_MASK;
fd0a05ce 85
eaefbd06
JB		 86 fpt |= (fdir_data->flex_off << I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT) &
87 I40E_TXD_FLTR_QW0_FLEXOFF_MASK;
fd0a05ce 88
eaefbd06
JB		 89 fpt |= (fdir_data->pctype << I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) &
90 I40E_TXD_FLTR_QW0_PCTYPE_MASK;
fd0a05ce
JB		 91
92 /* Use LAN VSI Id if not programmed by user */
93 if (fdir_data->dest_vsi == 0)
eaefbd06
JB		 94 fpt |= (pf->vsi[pf->lan_vsi]->id) <<
95 I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
fd0a05ce 96 else
eaefbd06
JB		 97 fpt |= ((u32)fdir_data->dest_vsi <<
98 I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT) &
99 I40E_TXD_FLTR_QW0_DEST_VSI_MASK;
100
101 fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(fpt);
fd0a05ce 102
eaefbd06 103 dcc = I40E_TX_DESC_DTYPE_FILTER_PROG;
fd0a05ce
JB		 104
105 if (add)
eaefbd06
JB		 106 dcc |= I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
107 I40E_TXD_FLTR_QW1_PCMD_SHIFT;
fd0a05ce 108 else
eaefbd06
JB		 109 dcc |= I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
110 I40E_TXD_FLTR_QW1_PCMD_SHIFT;
fd0a05ce 111
eaefbd06
JB		 112 dcc |= (fdir_data->dest_ctl << I40E_TXD_FLTR_QW1_DEST_SHIFT) &
113 I40E_TXD_FLTR_QW1_DEST_MASK;
fd0a05ce 114
eaefbd06
JB		 115 dcc |= (fdir_data->fd_status << I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT) &
116 I40E_TXD_FLTR_QW1_FD_STATUS_MASK;
fd0a05ce
JB		 117
118 if (fdir_data->cnt_index != 0) {
eaefbd06
JB		 119 dcc |= I40E_TXD_FLTR_QW1_CNT_ENA_MASK;
120 dcc |= ((u32)fdir_data->cnt_index <<
121 I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) &
122 I40E_TXD_FLTR_QW1_CNTINDEX_MASK;
fd0a05ce
JB		 123 }
124
eaefbd06 125 fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dcc);
fd0a05ce
JB		 126 fdir_desc->fd_id = cpu_to_le32(fdir_data->fd_id);
127
128 /* Now program a dummy descriptor */
fc4ac67b
AD		 129 i = tx_ring->next_to_use;
130 tx_desc = I40E_TX_DESC(tx_ring, i);
298deef1 131 tx_buf = &tx_ring->tx_bi[i];
fc4ac67b 132
eaefbd06 133 tx_ring->next_to_use = (i + 1 < tx_ring->count) ? i + 1 : 0;
fd0a05ce 134
298deef1 135 /* record length, and DMA address */
17a73f6b 136 dma_unmap_len_set(tx_buf, len, I40E_FDIR_MAX_RAW_PACKET_SIZE);
298deef1
ASJ		 137 dma_unmap_addr_set(tx_buf, dma, dma);
138
fd0a05ce 139 tx_desc->buffer_addr = cpu_to_le64(dma);
eaefbd06 140 td_cmd = I40E_TXD_CMD | I40E_TX_DESC_CMD_DUMMY;
fd0a05ce
JB		 141
142 tx_desc->cmd_type_offset_bsz =
17a73f6b 143 build_ctob(td_cmd, 0, I40E_FDIR_MAX_RAW_PACKET_SIZE, 0);
fd0a05ce 144
298deef1
ASJ		 145 /* set the timestamp */
146 tx_buf->time_stamp = jiffies;
147
fd0a05ce
JB		 148 /* Force memory writes to complete before letting h/w
149 * know there are new descriptors to fetch. (Only
150 * applicable for weak-ordered memory model archs,
151 * such as IA-64).
152 */
153 wmb();
154
fc4ac67b
AD		 155 /* Mark the data descriptor to be watched */
156 tx_buf->next_to_watch = tx_desc;
157
fd0a05ce
JB		 158 writel(tx_ring->next_to_use, tx_ring->tail);
159 return 0;
160
161dma_fail:
162 return -1;
163}
164
17a73f6b
JG		 165#define IP_HEADER_OFFSET 14
166#define I40E_UDPIP_DUMMY_PACKET_LEN 42
167/**
168 * i40e_add_del_fdir_udpv4 - Add/Remove UDPv4 filters
169 * @vsi: pointer to the targeted VSI
170 * @fd_data: the flow director data required for the FDir descriptor
171 * @raw_packet: the pre-allocated packet buffer for FDir
172 * @add: true adds a filter, false removes it
173 *
174 * Returns 0 if the filters were successfully added or removed
175 **/
176static int i40e_add_del_fdir_udpv4(struct i40e_vsi *vsi,
177 struct i40e_fdir_filter *fd_data,
178 u8 *raw_packet, bool add)
179{
180 struct i40e_pf *pf = vsi->back;
181 struct udphdr *udp;
182 struct iphdr *ip;
183 bool err = false;
184 int ret;
185 int i;
186 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0,
187 0x45, 0, 0, 0x1c, 0, 0, 0x40, 0, 0x40, 0x11, 0, 0, 0, 0, 0, 0,
188 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
189
190 memcpy(raw_packet, packet, I40E_UDPIP_DUMMY_PACKET_LEN);
191
192 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET);
193 udp = (struct udphdr *)(raw_packet + IP_HEADER_OFFSET
194 + sizeof(struct iphdr));
195
196 ip->daddr = fd_data->dst_ip[0];
197 udp->dest = fd_data->dst_port;
198 ip->saddr = fd_data->src_ip[0];
199 udp->source = fd_data->src_port;
200
201 for (i = I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP;
202 i <= I40E_FILTER_PCTYPE_NONF_IPV4_UDP; i++) {
203 fd_data->pctype = i;
204 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
205
206 if (ret) {
207 dev_info(&pf->pdev->dev,
208 "Filter command send failed for PCTYPE %d (ret = %d)\n",
209 fd_data->pctype, ret);
210 err = true;
211 } else {
212 dev_info(&pf->pdev->dev,
213 "Filter OK for PCTYPE %d (ret = %d)\n",
214 fd_data->pctype, ret);
215 }
216 }
217
218 return err ? -EOPNOTSUPP : 0;
219}
220
221#define I40E_TCPIP_DUMMY_PACKET_LEN 54
222/**
223 * i40e_add_del_fdir_tcpv4 - Add/Remove TCPv4 filters
224 * @vsi: pointer to the targeted VSI
225 * @fd_data: the flow director data required for the FDir descriptor
226 * @raw_packet: the pre-allocated packet buffer for FDir
227 * @add: true adds a filter, false removes it
228 *
229 * Returns 0 if the filters were successfully added or removed
230 **/
231static int i40e_add_del_fdir_tcpv4(struct i40e_vsi *vsi,
232 struct i40e_fdir_filter *fd_data,
233 u8 *raw_packet, bool add)
234{
235 struct i40e_pf *pf = vsi->back;
236 struct tcphdr *tcp;
237 struct iphdr *ip;
238 bool err = false;
239 int ret;
240 /* Dummy packet */
241 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0,
242 0x45, 0, 0, 0x28, 0, 0, 0x40, 0, 0x40, 0x6, 0, 0, 0, 0, 0, 0,
243 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x80, 0x11,
244 0x0, 0x72, 0, 0, 0, 0};
245
246 memcpy(raw_packet, packet, I40E_TCPIP_DUMMY_PACKET_LEN);
247
248 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET);
249 tcp = (struct tcphdr *)(raw_packet + IP_HEADER_OFFSET
250 + sizeof(struct iphdr));
251
252 ip->daddr = fd_data->dst_ip[0];
253 tcp->dest = fd_data->dst_port;
254 ip->saddr = fd_data->src_ip[0];
255 tcp->source = fd_data->src_port;
256
257 if (add) {
258 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) {
259 dev_info(&pf->pdev->dev, "Forcing ATR off, sideband rules for TCP/IPv4 flow being applied\n");
260 pf->flags &= ~I40E_FLAG_FD_ATR_ENABLED;
261 }
262 }
263
264 fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN;
265 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
266
267 if (ret) {
268 dev_info(&pf->pdev->dev,
269 "Filter command send failed for PCTYPE %d (ret = %d)\n",
270 fd_data->pctype, ret);
271 err = true;
272 } else {
273 dev_info(&pf->pdev->dev, "Filter OK for PCTYPE %d (ret = %d)\n",
274 fd_data->pctype, ret);
275 }
276
277 fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_TCP;
278
279 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
280 if (ret) {
281 dev_info(&pf->pdev->dev,
282 "Filter command send failed for PCTYPE %d (ret = %d)\n",
283 fd_data->pctype, ret);
284 err = true;
285 } else {
286 dev_info(&pf->pdev->dev, "Filter OK for PCTYPE %d (ret = %d)\n",
287 fd_data->pctype, ret);
288 }
289
290 return err ? -EOPNOTSUPP : 0;
291}
292
293/**
294 * i40e_add_del_fdir_sctpv4 - Add/Remove SCTPv4 Flow Director filters for
295 * a specific flow spec
296 * @vsi: pointer to the targeted VSI
297 * @fd_data: the flow director data required for the FDir descriptor
298 * @raw_packet: the pre-allocated packet buffer for FDir
299 * @add: true adds a filter, false removes it
300 *
301 * Returns 0 if the filters were successfully added or removed
302 **/
303static int i40e_add_del_fdir_sctpv4(struct i40e_vsi *vsi,
304 struct i40e_fdir_filter *fd_data,
305 u8 *raw_packet, bool add)
306{
307 return -EOPNOTSUPP;
308}
309
310#define I40E_IP_DUMMY_PACKET_LEN 34
311/**
312 * i40e_add_del_fdir_ipv4 - Add/Remove IPv4 Flow Director filters for
313 * a specific flow spec
314 * @vsi: pointer to the targeted VSI
315 * @fd_data: the flow director data required for the FDir descriptor
316 * @raw_packet: the pre-allocated packet buffer for FDir
317 * @add: true adds a filter, false removes it
318 *
319 * Returns 0 if the filters were successfully added or removed
320 **/
321static int i40e_add_del_fdir_ipv4(struct i40e_vsi *vsi,
322 struct i40e_fdir_filter *fd_data,
323 u8 *raw_packet, bool add)
324{
325 struct i40e_pf *pf = vsi->back;
326 struct iphdr *ip;
327 bool err = false;
328 int ret;
329 int i;
330 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0,
331 0x45, 0, 0, 0x14, 0, 0, 0x40, 0, 0x40, 0x10, 0, 0, 0, 0, 0, 0,
332 0, 0, 0, 0};
333
334 memcpy(raw_packet, packet, I40E_IP_DUMMY_PACKET_LEN);
335 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET);
336
337 ip->saddr = fd_data->src_ip[0];
338 ip->daddr = fd_data->dst_ip[0];
339 ip->protocol = 0;
340
341 for (i = I40E_FILTER_PCTYPE_NONF_IPV4_OTHER;
342 i <= I40E_FILTER_PCTYPE_FRAG_IPV4; i++) {
343 fd_data->pctype = i;
344 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
345
346 if (ret) {
347 dev_info(&pf->pdev->dev,
348 "Filter command send failed for PCTYPE %d (ret = %d)\n",
349 fd_data->pctype, ret);
350 err = true;
351 } else {
352 dev_info(&pf->pdev->dev,
353 "Filter OK for PCTYPE %d (ret = %d)\n",
354 fd_data->pctype, ret);
355 }
356 }
357
358 return err ? -EOPNOTSUPP : 0;
359}
360
361/**
362 * i40e_add_del_fdir - Build raw packets to add/del fdir filter
363 * @vsi: pointer to the targeted VSI
364 * @cmd: command to get or set RX flow classification rules
365 * @add: true adds a filter, false removes it
366 *
367 **/
368int i40e_add_del_fdir(struct i40e_vsi *vsi,
369 struct i40e_fdir_filter *input, bool add)
370{
371 struct i40e_pf *pf = vsi->back;
372 u8 *raw_packet;
373 int ret;
374
375 /* Populate the Flow Director that we have at the moment
376 * and allocate the raw packet buffer for the calling functions
377 */
378 raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL);
379 if (!raw_packet)
380 return -ENOMEM;
381
382 switch (input->flow_type & ~FLOW_EXT) {
383 case TCP_V4_FLOW:
384 ret = i40e_add_del_fdir_tcpv4(vsi, input, raw_packet,
385 add);
386 break;
387 case UDP_V4_FLOW:
388 ret = i40e_add_del_fdir_udpv4(vsi, input, raw_packet,
389 add);
390 break;
391 case SCTP_V4_FLOW:
392 ret = i40e_add_del_fdir_sctpv4(vsi, input, raw_packet,
393 add);
394 break;
395 case IPV4_FLOW:
396 ret = i40e_add_del_fdir_ipv4(vsi, input, raw_packet,
397 add);
398 break;
399 case IP_USER_FLOW:
400 switch (input->ip4_proto) {
401 case IPPROTO_TCP:
402 ret = i40e_add_del_fdir_tcpv4(vsi, input,
403 raw_packet, add);
404 break;
405 case IPPROTO_UDP:
406 ret = i40e_add_del_fdir_udpv4(vsi, input,
407 raw_packet, add);
408 break;
409 case IPPROTO_SCTP:
410 ret = i40e_add_del_fdir_sctpv4(vsi, input,
411 raw_packet, add);
412 break;
413 default:
414 ret = i40e_add_del_fdir_ipv4(vsi, input,
415 raw_packet, add);
416 break;
417 }
418 break;
419 default:
420 dev_info(&pf->pdev->dev, "Could not specify spec type %d",
421 input->flow_type);
422 ret = -EINVAL;
423 }
424
425 kfree(raw_packet);
426 return ret;
427}
428
fd0a05ce
JB		 429/**
430 * i40e_fd_handle_status - check the Programming Status for FD
431 * @rx_ring: the Rx ring for this descriptor
432 * @qw: the descriptor data
433 * @prog_id: the id originally used for programming
434 *
435 * This is used to verify if the FD programming or invalidation
436 * requested by SW to the HW is successful or not and take actions accordingly.
437 **/
438static void i40e_fd_handle_status(struct i40e_ring *rx_ring, u32 qw, u8 prog_id)
439{
440 struct pci_dev *pdev = rx_ring->vsi->back->pdev;
441 u32 error;
442
443 error = (qw & I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >>
444 I40E_RX_PROG_STATUS_DESC_QW1_ERROR_SHIFT;
445
446 /* for now just print the Status */
447 dev_info(&pdev->dev, "FD programming id %02x, Status %08x\n",
448 prog_id, error);
449}
450
451/**
a5e9c572 452 * i40e_unmap_and_free_tx_resource - Release a Tx buffer
fd0a05ce
JB		 453 * @ring: the ring that owns the buffer
454 * @tx_buffer: the buffer to free
455 **/
a5e9c572
AD		 456static void i40e_unmap_and_free_tx_resource(struct i40e_ring *ring,
457 struct i40e_tx_buffer *tx_buffer)
fd0a05ce 458{
a5e9c572
AD		 459 if (tx_buffer->skb) {
460 dev_kfree_skb_any(tx_buffer->skb);
461 if (dma_unmap_len(tx_buffer, len))
fd0a05ce 462 dma_unmap_single(ring->dev,
35a1e2ad
AD		 463 dma_unmap_addr(tx_buffer, dma),
464 dma_unmap_len(tx_buffer, len),
fd0a05ce 465 DMA_TO_DEVICE);
a5e9c572
AD		 466 } else if (dma_unmap_len(tx_buffer, len)) {
467 dma_unmap_page(ring->dev,
468 dma_unmap_addr(tx_buffer, dma),
469 dma_unmap_len(tx_buffer, len),
470 DMA_TO_DEVICE);
fd0a05ce 471 }
a5e9c572
AD		 472 tx_buffer->next_to_watch = NULL;
473 tx_buffer->skb = NULL;
35a1e2ad 474 dma_unmap_len_set(tx_buffer, len, 0);
a5e9c572 475 /* tx_buffer must be completely set up in the transmit path */
fd0a05ce
JB		 476}
477
478/**
479 * i40e_clean_tx_ring - Free any empty Tx buffers
480 * @tx_ring: ring to be cleaned
481 **/
482void i40e_clean_tx_ring(struct i40e_ring *tx_ring)
483{
fd0a05ce
JB		 484 unsigned long bi_size;
485 u16 i;
486
487 /* ring already cleared, nothing to do */
488 if (!tx_ring->tx_bi)
489 return;
490
491 /* Free all the Tx ring sk_buffs */
a5e9c572
AD		 492 for (i = 0; i < tx_ring->count; i++)
493 i40e_unmap_and_free_tx_resource(tx_ring, &tx_ring->tx_bi[i]);
fd0a05ce
JB		 494
495 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
496 memset(tx_ring->tx_bi, 0, bi_size);
497
498 /* Zero out the descriptor ring */
499 memset(tx_ring->desc, 0, tx_ring->size);
500
501 tx_ring->next_to_use = 0;
502 tx_ring->next_to_clean = 0;
7070ce0a
AD		 503
504 if (!tx_ring->netdev)
505 return;
506
507 /* cleanup Tx queue statistics */
508 netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev,
509 tx_ring->queue_index));
fd0a05ce
JB		 510}
511
512/**
513 * i40e_free_tx_resources - Free Tx resources per queue
514 * @tx_ring: Tx descriptor ring for a specific queue
515 *
516 * Free all transmit software resources
517 **/
518void i40e_free_tx_resources(struct i40e_ring *tx_ring)
519{
520 i40e_clean_tx_ring(tx_ring);
521 kfree(tx_ring->tx_bi);
522 tx_ring->tx_bi = NULL;
523
524 if (tx_ring->desc) {
525 dma_free_coherent(tx_ring->dev, tx_ring->size,
526 tx_ring->desc, tx_ring->dma);
527 tx_ring->desc = NULL;
528 }
529}
530
531/**
532 * i40e_get_tx_pending - how many tx descriptors not processed
533 * @tx_ring: the ring of descriptors
534 *
535 * Since there is no access to the ring head register
536 * in XL710, we need to use our local copies
537 **/
538static u32 i40e_get_tx_pending(struct i40e_ring *ring)
539{
540 u32 ntu = ((ring->next_to_clean <= ring->next_to_use)
541 ? ring->next_to_use
542 : ring->next_to_use + ring->count);
543 return ntu - ring->next_to_clean;
544}
545
546/**
547 * i40e_check_tx_hang - Is there a hang in the Tx queue
548 * @tx_ring: the ring of descriptors
549 **/
550static bool i40e_check_tx_hang(struct i40e_ring *tx_ring)
551{
552 u32 tx_pending = i40e_get_tx_pending(tx_ring);
553 bool ret = false;
554
555 clear_check_for_tx_hang(tx_ring);
556
557 /* Check for a hung queue, but be thorough. This verifies
558 * that a transmit has been completed since the previous
559 * check AND there is at least one packet pending. The
560 * ARMED bit is set to indicate a potential hang. The
561 * bit is cleared if a pause frame is received to remove
562 * false hang detection due to PFC or 802.3x frames. By
563 * requiring this to fail twice we avoid races with
564 * PFC clearing the ARMED bit and conditions where we
565 * run the check_tx_hang logic with a transmit completion
566 * pending but without time to complete it yet.
567 */
a114d0a6 568 if ((tx_ring->tx_stats.tx_done_old == tx_ring->stats.packets) &&
fd0a05ce
JB		 569 tx_pending) {
570 /* make sure it is true for two checks in a row */
571 ret = test_and_set_bit(__I40E_HANG_CHECK_ARMED,
572 &tx_ring->state);
573 } else {
574 /* update completed stats and disarm the hang check */
a114d0a6 575 tx_ring->tx_stats.tx_done_old = tx_ring->stats.packets;
fd0a05ce
JB		 576 clear_bit(__I40E_HANG_CHECK_ARMED, &tx_ring->state);
577 }
578
579 return ret;
580}
581
582/**
583 * i40e_clean_tx_irq - Reclaim resources after transmit completes
584 * @tx_ring: tx ring to clean
585 * @budget: how many cleans we're allowed
586 *
587 * Returns true if there's any budget left (e.g. the clean is finished)
588 **/
589static bool i40e_clean_tx_irq(struct i40e_ring *tx_ring, int budget)
590{
591 u16 i = tx_ring->next_to_clean;
592 struct i40e_tx_buffer *tx_buf;
593 struct i40e_tx_desc *tx_desc;
594 unsigned int total_packets = 0;
595 unsigned int total_bytes = 0;
596
597 tx_buf = &tx_ring->tx_bi[i];
598 tx_desc = I40E_TX_DESC(tx_ring, i);
a5e9c572 599 i -= tx_ring->count;
fd0a05ce 600
a5e9c572
AD		 601 do {
602 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
fd0a05ce
JB		 603
604 /* if next_to_watch is not set then there is no work pending */
605 if (!eop_desc)
606 break;
607
a5e9c572
AD		 608 /* prevent any other reads prior to eop_desc */
609 read_barrier_depends();
610
fd0a05ce
JB		 611 /* if the descriptor isn't done, no work yet to do */
612 if (!(eop_desc->cmd_type_offset_bsz &
613 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
614 break;
615
c304fdac 616 /* clear next_to_watch to prevent false hangs */
fd0a05ce 617 tx_buf->next_to_watch = NULL;
fd0a05ce 618
a5e9c572
AD		 619 /* update the statistics for this packet */
620 total_bytes += tx_buf->bytecount;
621 total_packets += tx_buf->gso_segs;
fd0a05ce 622
a5e9c572
AD		 623 /* free the skb */
624 dev_kfree_skb_any(tx_buf->skb);
fd0a05ce 625
a5e9c572
AD		 626 /* unmap skb header data */
627 dma_unmap_single(tx_ring->dev,
628 dma_unmap_addr(tx_buf, dma),
629 dma_unmap_len(tx_buf, len),
630 DMA_TO_DEVICE);
fd0a05ce 631
a5e9c572
AD		 632 /* clear tx_buffer data */
633 tx_buf->skb = NULL;
634 dma_unmap_len_set(tx_buf, len, 0);
fd0a05ce 635
a5e9c572
AD		 636 /* unmap remaining buffers */
637 while (tx_desc != eop_desc) {
fd0a05ce
JB		 638
639 tx_buf++;
640 tx_desc++;
641 i++;
a5e9c572
AD		 642 if (unlikely(!i)) {
643 i -= tx_ring->count;
fd0a05ce
JB		 644 tx_buf = tx_ring->tx_bi;
645 tx_desc = I40E_TX_DESC(tx_ring, 0);
646 }
fd0a05ce 647
a5e9c572
AD		 648 /* unmap any remaining paged data */
649 if (dma_unmap_len(tx_buf, len)) {
650 dma_unmap_page(tx_ring->dev,
651 dma_unmap_addr(tx_buf, dma),
652 dma_unmap_len(tx_buf, len),
653 DMA_TO_DEVICE);
654 dma_unmap_len_set(tx_buf, len, 0);
655 }
656 }
657
658 /* move us one more past the eop_desc for start of next pkt */
659 tx_buf++;
660 tx_desc++;
661 i++;
662 if (unlikely(!i)) {
663 i -= tx_ring->count;
664 tx_buf = tx_ring->tx_bi;
665 tx_desc = I40E_TX_DESC(tx_ring, 0);
666 }
667
668 /* update budget accounting */
669 budget--;
670 } while (likely(budget));
671
672 i += tx_ring->count;
fd0a05ce 673 tx_ring->next_to_clean = i;
980e9b11 674 u64_stats_update_begin(&tx_ring->syncp);
a114d0a6
AD		 675 tx_ring->stats.bytes += total_bytes;
676 tx_ring->stats.packets += total_packets;
980e9b11 677 u64_stats_update_end(&tx_ring->syncp);
fd0a05ce
JB		 678 tx_ring->q_vector->tx.total_bytes += total_bytes;
679 tx_ring->q_vector->tx.total_packets += total_packets;
a5e9c572 680
fd0a05ce
JB		 681 if (check_for_tx_hang(tx_ring) && i40e_check_tx_hang(tx_ring)) {
682 /* schedule immediate reset if we believe we hung */
683 dev_info(tx_ring->dev, "Detected Tx Unit Hang\n"
684 " VSI <%d>\n"
685 " Tx Queue <%d>\n"
686 " next_to_use <%x>\n"
687 " next_to_clean <%x>\n",
688 tx_ring->vsi->seid,
689 tx_ring->queue_index,
690 tx_ring->next_to_use, i);
691 dev_info(tx_ring->dev, "tx_bi[next_to_clean]\n"
692 " time_stamp <%lx>\n"
693 " jiffies <%lx>\n",
694 tx_ring->tx_bi[i].time_stamp, jiffies);
695
696 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
697
698 dev_info(tx_ring->dev,
699 "tx hang detected on queue %d, resetting adapter\n",
700 tx_ring->queue_index);
701
702 tx_ring->netdev->netdev_ops->ndo_tx_timeout(tx_ring->netdev);
703
704 /* the adapter is about to reset, no point in enabling stuff */
705 return true;
706 }
707
7070ce0a
AD		 708 netdev_tx_completed_queue(netdev_get_tx_queue(tx_ring->netdev,
709 tx_ring->queue_index),
710 total_packets, total_bytes);
711
fd0a05ce
JB		 712#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
713 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
714 (I40E_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
715 /* Make sure that anybody stopping the queue after this
716 * sees the new next_to_clean.
717 */
718 smp_mb();
719 if (__netif_subqueue_stopped(tx_ring->netdev,
720 tx_ring->queue_index) &&
721 !test_bit(__I40E_DOWN, &tx_ring->vsi->state)) {
722 netif_wake_subqueue(tx_ring->netdev,
723 tx_ring->queue_index);
724 ++tx_ring->tx_stats.restart_queue;
725 }
726 }
727
728 return budget > 0;
729}
730
731/**
732 * i40e_set_new_dynamic_itr - Find new ITR level
733 * @rc: structure containing ring performance data
734 *
735 * Stores a new ITR value based on packets and byte counts during
736 * the last interrupt. The advantage of per interrupt computation
737 * is faster updates and more accurate ITR for the current traffic
738 * pattern. Constants in this function were computed based on
739 * theoretical maximum wire speed and thresholds were set based on
740 * testing data as well as attempting to minimize response time
741 * while increasing bulk throughput.
742 **/
743static void i40e_set_new_dynamic_itr(struct i40e_ring_container *rc)
744{
745 enum i40e_latency_range new_latency_range = rc->latency_range;
746 u32 new_itr = rc->itr;
747 int bytes_per_int;
748
749 if (rc->total_packets == 0 || !rc->itr)
750 return;
751
752 /* simple throttlerate management
753 * 0-10MB/s lowest (100000 ints/s)
754 * 10-20MB/s low (20000 ints/s)
755 * 20-1249MB/s bulk (8000 ints/s)
756 */
757 bytes_per_int = rc->total_bytes / rc->itr;
758 switch (rc->itr) {
759 case I40E_LOWEST_LATENCY:
760 if (bytes_per_int > 10)
761 new_latency_range = I40E_LOW_LATENCY;
762 break;
763 case I40E_LOW_LATENCY:
764 if (bytes_per_int > 20)
765 new_latency_range = I40E_BULK_LATENCY;
766 else if (bytes_per_int <= 10)
767 new_latency_range = I40E_LOWEST_LATENCY;
768 break;
769 case I40E_BULK_LATENCY:
770 if (bytes_per_int <= 20)
771 rc->latency_range = I40E_LOW_LATENCY;
772 break;
773 }
774
775 switch (new_latency_range) {
776 case I40E_LOWEST_LATENCY:
777 new_itr = I40E_ITR_100K;
778 break;
779 case I40E_LOW_LATENCY:
780 new_itr = I40E_ITR_20K;
781 break;
782 case I40E_BULK_LATENCY:
783 new_itr = I40E_ITR_8K;
784 break;
785 default:
786 break;
787 }
788
789 if (new_itr != rc->itr) {
790 /* do an exponential smoothing */
791 new_itr = (10 * new_itr * rc->itr) /
792 ((9 * new_itr) + rc->itr);
793 rc->itr = new_itr & I40E_MAX_ITR;
794 }
795
796 rc->total_bytes = 0;
797 rc->total_packets = 0;
798}
799
800/**
801 * i40e_update_dynamic_itr - Adjust ITR based on bytes per int
802 * @q_vector: the vector to adjust
803 **/
804static void i40e_update_dynamic_itr(struct i40e_q_vector *q_vector)
805{
806 u16 vector = q_vector->vsi->base_vector + q_vector->v_idx;
807 struct i40e_hw *hw = &q_vector->vsi->back->hw;
808 u32 reg_addr;
809 u16 old_itr;
810
811 reg_addr = I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1);
812 old_itr = q_vector->rx.itr;
813 i40e_set_new_dynamic_itr(&q_vector->rx);
814 if (old_itr != q_vector->rx.itr)
815 wr32(hw, reg_addr, q_vector->rx.itr);
816
817 reg_addr = I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1);
818 old_itr = q_vector->tx.itr;
819 i40e_set_new_dynamic_itr(&q_vector->tx);
820 if (old_itr != q_vector->tx.itr)
821 wr32(hw, reg_addr, q_vector->tx.itr);
fd0a05ce
JB		 822}
823
824/**
825 * i40e_clean_programming_status - clean the programming status descriptor
826 * @rx_ring: the rx ring that has this descriptor
827 * @rx_desc: the rx descriptor written back by HW
828 *
829 * Flow director should handle FD_FILTER_STATUS to check its filter programming
830 * status being successful or not and take actions accordingly. FCoE should
831 * handle its context/filter programming/invalidation status and take actions.
832 *
833 **/
834static void i40e_clean_programming_status(struct i40e_ring *rx_ring,
835 union i40e_rx_desc *rx_desc)
836{
837 u64 qw;
838 u8 id;
839
840 qw = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
841 id = (qw & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >>
842 I40E_RX_PROG_STATUS_DESC_QW1_PROGID_SHIFT;
843
844 if (id == I40E_RX_PROG_STATUS_DESC_FD_FILTER_STATUS)
845 i40e_fd_handle_status(rx_ring, qw, id);
846}
847
848/**
849 * i40e_setup_tx_descriptors - Allocate the Tx descriptors
850 * @tx_ring: the tx ring to set up
851 *
852 * Return 0 on success, negative on error
853 **/
854int i40e_setup_tx_descriptors(struct i40e_ring *tx_ring)
855{
856 struct device *dev = tx_ring->dev;
857 int bi_size;
858
859 if (!dev)
860 return -ENOMEM;
861
862 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
863 tx_ring->tx_bi = kzalloc(bi_size, GFP_KERNEL);
864 if (!tx_ring->tx_bi)
865 goto err;
866
867 /* round up to nearest 4K */
868 tx_ring->size = tx_ring->count * sizeof(struct i40e_tx_desc);
869 tx_ring->size = ALIGN(tx_ring->size, 4096);
870 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
871 &tx_ring->dma, GFP_KERNEL);
872 if (!tx_ring->desc) {
873 dev_info(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n",
874 tx_ring->size);
875 goto err;
876 }
877
878 tx_ring->next_to_use = 0;
879 tx_ring->next_to_clean = 0;
880 return 0;
881
882err:
883 kfree(tx_ring->tx_bi);
884 tx_ring->tx_bi = NULL;
885 return -ENOMEM;
886}
887
888/**
889 * i40e_clean_rx_ring - Free Rx buffers
890 * @rx_ring: ring to be cleaned
891 **/
892void i40e_clean_rx_ring(struct i40e_ring *rx_ring)
893{
894 struct device *dev = rx_ring->dev;
895 struct i40e_rx_buffer *rx_bi;
896 unsigned long bi_size;
897 u16 i;
898
899 /* ring already cleared, nothing to do */
900 if (!rx_ring->rx_bi)
901 return;
902
903 /* Free all the Rx ring sk_buffs */
904 for (i = 0; i < rx_ring->count; i++) {
905 rx_bi = &rx_ring->rx_bi[i];
906 if (rx_bi->dma) {
907 dma_unmap_single(dev,
908 rx_bi->dma,
909 rx_ring->rx_buf_len,
910 DMA_FROM_DEVICE);
911 rx_bi->dma = 0;
912 }
913 if (rx_bi->skb) {
914 dev_kfree_skb(rx_bi->skb);
915 rx_bi->skb = NULL;
916 }
917 if (rx_bi->page) {
918 if (rx_bi->page_dma) {
919 dma_unmap_page(dev,
920 rx_bi->page_dma,
921 PAGE_SIZE / 2,
922 DMA_FROM_DEVICE);
923 rx_bi->page_dma = 0;
924 }
925 __free_page(rx_bi->page);
926 rx_bi->page = NULL;
927 rx_bi->page_offset = 0;
928 }
929 }
930
931 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
932 memset(rx_ring->rx_bi, 0, bi_size);
933
934 /* Zero out the descriptor ring */
935 memset(rx_ring->desc, 0, rx_ring->size);
936
937 rx_ring->next_to_clean = 0;
938 rx_ring->next_to_use = 0;
939}
940
941/**
942 * i40e_free_rx_resources - Free Rx resources
943 * @rx_ring: ring to clean the resources from
944 *
945 * Free all receive software resources
946 **/
947void i40e_free_rx_resources(struct i40e_ring *rx_ring)
948{
949 i40e_clean_rx_ring(rx_ring);
950 kfree(rx_ring->rx_bi);
951 rx_ring->rx_bi = NULL;
952
953 if (rx_ring->desc) {
954 dma_free_coherent(rx_ring->dev, rx_ring->size,
955 rx_ring->desc, rx_ring->dma);
956 rx_ring->desc = NULL;
957 }
958}
959
960/**
961 * i40e_setup_rx_descriptors - Allocate Rx descriptors
962 * @rx_ring: Rx descriptor ring (for a specific queue) to setup
963 *
964 * Returns 0 on success, negative on failure
965 **/
966int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring)
967{
968 struct device *dev = rx_ring->dev;
969 int bi_size;
970
971 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
972 rx_ring->rx_bi = kzalloc(bi_size, GFP_KERNEL);
973 if (!rx_ring->rx_bi)
974 goto err;
975
976 /* Round up to nearest 4K */
977 rx_ring->size = ring_is_16byte_desc_enabled(rx_ring)
978 ? rx_ring->count * sizeof(union i40e_16byte_rx_desc)
979 : rx_ring->count * sizeof(union i40e_32byte_rx_desc);
980 rx_ring->size = ALIGN(rx_ring->size, 4096);
981 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
982 &rx_ring->dma, GFP_KERNEL);
983
984 if (!rx_ring->desc) {
985 dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n",
986 rx_ring->size);
987 goto err;
988 }
989
990 rx_ring->next_to_clean = 0;
991 rx_ring->next_to_use = 0;
992
993 return 0;
994err:
995 kfree(rx_ring->rx_bi);
996 rx_ring->rx_bi = NULL;
997 return -ENOMEM;
998}
999
1000/**
1001 * i40e_release_rx_desc - Store the new tail and head values
1002 * @rx_ring: ring to bump
1003 * @val: new head index
1004 **/
1005static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val)
1006{
1007 rx_ring->next_to_use = val;
1008 /* Force memory writes to complete before letting h/w
1009 * know there are new descriptors to fetch. (Only
1010 * applicable for weak-ordered memory model archs,
1011 * such as IA-64).
1012 */
1013 wmb();
1014 writel(val, rx_ring->tail);
1015}
1016
1017/**
1018 * i40e_alloc_rx_buffers - Replace used receive buffers; packet split
1019 * @rx_ring: ring to place buffers on
1020 * @cleaned_count: number of buffers to replace
1021 **/
1022void i40e_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count)
1023{
1024 u16 i = rx_ring->next_to_use;
1025 union i40e_rx_desc *rx_desc;
1026 struct i40e_rx_buffer *bi;
1027 struct sk_buff *skb;
1028
1029 /* do nothing if no valid netdev defined */
1030 if (!rx_ring->netdev || !cleaned_count)
1031 return;
1032
1033 while (cleaned_count--) {
1034 rx_desc = I40E_RX_DESC(rx_ring, i);
1035 bi = &rx_ring->rx_bi[i];
1036 skb = bi->skb;
1037
1038 if (!skb) {
1039 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
1040 rx_ring->rx_buf_len);
1041 if (!skb) {
420136cc 1042 rx_ring->rx_stats.alloc_buff_failed++;
fd0a05ce
JB		 1043 goto no_buffers;
1044 }
1045 /* initialize queue mapping */
1046 skb_record_rx_queue(skb, rx_ring->queue_index);
1047 bi->skb = skb;
1048 }
1049
1050 if (!bi->dma) {
1051 bi->dma = dma_map_single(rx_ring->dev,
1052 skb->data,
1053 rx_ring->rx_buf_len,
1054 DMA_FROM_DEVICE);
1055 if (dma_mapping_error(rx_ring->dev, bi->dma)) {
420136cc 1056 rx_ring->rx_stats.alloc_buff_failed++;
fd0a05ce
JB		 1057 bi->dma = 0;
1058 goto no_buffers;
1059 }
1060 }
1061
1062 if (ring_is_ps_enabled(rx_ring)) {
1063 if (!bi->page) {
1064 bi->page = alloc_page(GFP_ATOMIC);
1065 if (!bi->page) {
420136cc 1066 rx_ring->rx_stats.alloc_page_failed++;
fd0a05ce
JB		 1067 goto no_buffers;
1068 }
1069 }
1070
1071 if (!bi->page_dma) {
1072 /* use a half page if we're re-using */
1073 bi->page_offset ^= PAGE_SIZE / 2;
1074 bi->page_dma = dma_map_page(rx_ring->dev,
1075 bi->page,
1076 bi->page_offset,
1077 PAGE_SIZE / 2,
1078 DMA_FROM_DEVICE);
1079 if (dma_mapping_error(rx_ring->dev,
1080 bi->page_dma)) {
420136cc 1081 rx_ring->rx_stats.alloc_page_failed++;
fd0a05ce
JB		 1082 bi->page_dma = 0;
1083 goto no_buffers;
1084 }
1085 }
1086
1087 /* Refresh the desc even if buffer_addrs didn't change
1088 * because each write-back erases this info.
1089 */
1090 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
1091 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
1092 } else {
1093 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
1094 rx_desc->read.hdr_addr = 0;
1095 }
1096 i++;
1097 if (i == rx_ring->count)
1098 i = 0;
1099 }
1100
1101no_buffers:
1102 if (rx_ring->next_to_use != i)
1103 i40e_release_rx_desc(rx_ring, i);
1104}
1105
1106/**
1107 * i40e_receive_skb - Send a completed packet up the stack
1108 * @rx_ring: rx ring in play
1109 * @skb: packet to send up
1110 * @vlan_tag: vlan tag for packet
1111 **/
1112static void i40e_receive_skb(struct i40e_ring *rx_ring,
1113 struct sk_buff *skb, u16 vlan_tag)
1114{
1115 struct i40e_q_vector *q_vector = rx_ring->q_vector;
1116 struct i40e_vsi *vsi = rx_ring->vsi;
1117 u64 flags = vsi->back->flags;
1118
1119 if (vlan_tag & VLAN_VID_MASK)
1120 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
1121
1122 if (flags & I40E_FLAG_IN_NETPOLL)
1123 netif_rx(skb);
1124 else
1125 napi_gro_receive(&q_vector->napi, skb);
1126}
1127
1128/**
1129 * i40e_rx_checksum - Indicate in skb if hw indicated a good cksum
1130 * @vsi: the VSI we care about
1131 * @skb: skb currently being received and modified
1132 * @rx_status: status value of last descriptor in packet
1133 * @rx_error: error value of last descriptor in packet
8144f0f7 1134 * @rx_ptype: ptype value of last descriptor in packet
fd0a05ce
JB		 1135 **/
1136static inline void i40e_rx_checksum(struct i40e_vsi *vsi,
1137 struct sk_buff *skb,
1138 u32 rx_status,
8144f0f7
JG		 1139 u32 rx_error,
1140 u16 rx_ptype)
fd0a05ce 1141{
8144f0f7
JG		 1142 bool ipv4_tunnel, ipv6_tunnel;
1143 __wsum rx_udp_csum;
1144 __sum16 csum;
1145 struct iphdr *iph;
1146
1147 ipv4_tunnel = (rx_ptype > I40E_RX_PTYPE_GRENAT4_MAC_PAY3) &&
1148 (rx_ptype < I40E_RX_PTYPE_GRENAT4_MACVLAN_IPV6_ICMP_PAY4);
1149 ipv6_tunnel = (rx_ptype > I40E_RX_PTYPE_GRENAT6_MAC_PAY3) &&
1150 (rx_ptype < I40E_RX_PTYPE_GRENAT6_MACVLAN_IPV6_ICMP_PAY4);
1151
1152 skb->encapsulation = ipv4_tunnel || ipv6_tunnel;
fd0a05ce
JB		 1153 skb->ip_summed = CHECKSUM_NONE;
1154
1155 /* Rx csum enabled and ip headers found? */
1156 if (!(vsi->netdev->features & NETIF_F_RXCSUM &&
1157 rx_status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT)))
1158 return;
1159
ddf1d0d7 1160 /* likely incorrect csum if alternate IP extension headers found */
8ee75a8e
SN		 1161 if (rx_status & (1 << I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT))
1162 return;
1163
8144f0f7 1164 /* IP or L4 or outmost IP checksum error */
fd0a05ce 1165 if (rx_error & ((1 << I40E_RX_DESC_ERROR_IPE_SHIFT) |
8144f0f7
JG		 1166 (1 << I40E_RX_DESC_ERROR_L4E_SHIFT) |
1167 (1 << I40E_RX_DESC_ERROR_EIPE_SHIFT))) {
fd0a05ce
JB		 1168 vsi->back->hw_csum_rx_error++;
1169 return;
1170 }
1171
8144f0f7
JG		 1172 if (ipv4_tunnel &&
1173 !(rx_status & (1 << I40E_RX_DESC_STATUS_UDP_0_SHIFT))) {
1174 /* If VXLAN traffic has an outer UDPv4 checksum we need to check
1175 * it in the driver, hardware does not do it for us.
1176 * Since L3L4P bit was set we assume a valid IHL value (>=5)
1177 * so the total length of IPv4 header is IHL*4 bytes
1178 */
1179 skb->transport_header = skb->mac_header +
1180 sizeof(struct ethhdr) +
1181 (ip_hdr(skb)->ihl * 4);
1182
1183 /* Add 4 bytes for VLAN tagged packets */
1184 skb->transport_header += (skb->protocol == htons(ETH_P_8021Q) ||
1185 skb->protocol == htons(ETH_P_8021AD))
1186 ? VLAN_HLEN : 0;
1187
1188 rx_udp_csum = udp_csum(skb);
1189 iph = ip_hdr(skb);
1190 csum = csum_tcpudp_magic(
1191 iph->saddr, iph->daddr,
1192 (skb->len - skb_transport_offset(skb)),
1193 IPPROTO_UDP, rx_udp_csum);
1194
1195 if (udp_hdr(skb)->check != csum) {
1196 vsi->back->hw_csum_rx_error++;
1197 return;
1198 }
1199 }
1200
fd0a05ce
JB		 1201 skb->ip_summed = CHECKSUM_UNNECESSARY;
1202}
1203
1204/**
1205 * i40e_rx_hash - returns the hash value from the Rx descriptor
1206 * @ring: descriptor ring
1207 * @rx_desc: specific descriptor
1208 **/
1209static inline u32 i40e_rx_hash(struct i40e_ring *ring,
1210 union i40e_rx_desc *rx_desc)
1211{
8a494920
JB		 1212 const __le64 rss_mask =
1213 cpu_to_le64((u64)I40E_RX_DESC_FLTSTAT_RSS_HASH <<
1214 I40E_RX_DESC_STATUS_FLTSTAT_SHIFT);
1215
1216 if ((ring->netdev->features & NETIF_F_RXHASH) &&
1217 (rx_desc->wb.qword1.status_error_len & rss_mask) == rss_mask)
1218 return le32_to_cpu(rx_desc->wb.qword0.hi_dword.rss);
1219 else
1220 return 0;
fd0a05ce
JB		 1221}
1222
1223/**
1224 * i40e_clean_rx_irq - Reclaim resources after receive completes
1225 * @rx_ring: rx ring to clean
1226 * @budget: how many cleans we're allowed
1227 *
1228 * Returns true if there's any budget left (e.g. the clean is finished)
1229 **/
1230static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
1231{
1232 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1233 u16 rx_packet_len, rx_header_len, rx_sph, rx_hbo;
1234 u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);
1235 const int current_node = numa_node_id();
1236 struct i40e_vsi *vsi = rx_ring->vsi;
1237 u16 i = rx_ring->next_to_clean;
1238 union i40e_rx_desc *rx_desc;
1239 u32 rx_error, rx_status;
1240 u64 qword;
8144f0f7 1241 u16 rx_ptype;
fd0a05ce
JB		 1242
1243 rx_desc = I40E_RX_DESC(rx_ring, i);
1244 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
6838b535
JB		 1245 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >>
1246 I40E_RXD_QW1_STATUS_SHIFT;
fd0a05ce
JB		 1247
1248 while (rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) {
1249 union i40e_rx_desc *next_rxd;
1250 struct i40e_rx_buffer *rx_bi;
1251 struct sk_buff *skb;
1252 u16 vlan_tag;
1253 if (i40e_rx_is_programming_status(qword)) {
1254 i40e_clean_programming_status(rx_ring, rx_desc);
1255 I40E_RX_NEXT_DESC_PREFETCH(rx_ring, i, next_rxd);
1256 goto next_desc;
1257 }
1258 rx_bi = &rx_ring->rx_bi[i];
1259 skb = rx_bi->skb;
1260 prefetch(skb->data);
1261
829af3ac
MW		 1262 rx_packet_len = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
1263 I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
1264 rx_header_len = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK) >>
1265 I40E_RXD_QW1_LENGTH_HBUF_SHIFT;
1266 rx_sph = (qword & I40E_RXD_QW1_LENGTH_SPH_MASK) >>
1267 I40E_RXD_QW1_LENGTH_SPH_SHIFT;
1268
1269 rx_error = (qword & I40E_RXD_QW1_ERROR_MASK) >>
1270 I40E_RXD_QW1_ERROR_SHIFT;
fd0a05ce
JB		 1271 rx_hbo = rx_error & (1 << I40E_RX_DESC_ERROR_HBO_SHIFT);
1272 rx_error &= ~(1 << I40E_RX_DESC_ERROR_HBO_SHIFT);
1273
8144f0f7
JG		 1274 rx_ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >>
1275 I40E_RXD_QW1_PTYPE_SHIFT;
fd0a05ce
JB		 1276 rx_bi->skb = NULL;
1277
1278 /* This memory barrier is needed to keep us from reading
1279 * any other fields out of the rx_desc until we know the
1280 * STATUS_DD bit is set
1281 */
1282 rmb();
1283
1284 /* Get the header and possibly the whole packet
1285 * If this is an skb from previous receive dma will be 0
1286 */
1287 if (rx_bi->dma) {
1288 u16 len;
1289
1290 if (rx_hbo)
1291 len = I40E_RX_HDR_SIZE;
1292 else if (rx_sph)
1293 len = rx_header_len;
1294 else if (rx_packet_len)
1295 len = rx_packet_len; /* 1buf/no split found */
1296 else
1297 len = rx_header_len; /* split always mode */
1298
1299 skb_put(skb, len);
1300 dma_unmap_single(rx_ring->dev,
1301 rx_bi->dma,
1302 rx_ring->rx_buf_len,
1303 DMA_FROM_DEVICE);
1304 rx_bi->dma = 0;
1305 }
1306
1307 /* Get the rest of the data if this was a header split */
1308 if (ring_is_ps_enabled(rx_ring) && rx_packet_len) {
1309
1310 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
1311 rx_bi->page,
1312 rx_bi->page_offset,
1313 rx_packet_len);
1314
1315 skb->len += rx_packet_len;
1316 skb->data_len += rx_packet_len;
1317 skb->truesize += rx_packet_len;
1318
1319 if ((page_count(rx_bi->page) == 1) &&
1320 (page_to_nid(rx_bi->page) == current_node))
1321 get_page(rx_bi->page);
1322 else
1323 rx_bi->page = NULL;
1324
1325 dma_unmap_page(rx_ring->dev,
1326 rx_bi->page_dma,
1327 PAGE_SIZE / 2,
1328 DMA_FROM_DEVICE);
1329 rx_bi->page_dma = 0;
1330 }
1331 I40E_RX_NEXT_DESC_PREFETCH(rx_ring, i, next_rxd);
1332
1333 if (unlikely(
1334 !(rx_status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)))) {
1335 struct i40e_rx_buffer *next_buffer;
1336
1337 next_buffer = &rx_ring->rx_bi[i];
1338
1339 if (ring_is_ps_enabled(rx_ring)) {
1340 rx_bi->skb = next_buffer->skb;
1341 rx_bi->dma = next_buffer->dma;
1342 next_buffer->skb = skb;
1343 next_buffer->dma = 0;
1344 }
1345 rx_ring->rx_stats.non_eop_descs++;
1346 goto next_desc;
1347 }
1348
1349 /* ERR_MASK will only have valid bits if EOP set */
1350 if (unlikely(rx_error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) {
1351 dev_kfree_skb_any(skb);
1352 goto next_desc;
1353 }
1354
1355 skb->rxhash = i40e_rx_hash(rx_ring, rx_desc);
beb0dff1
JK		 1356 if (unlikely(rx_status & I40E_RXD_QW1_STATUS_TSYNVALID_MASK)) {
1357 i40e_ptp_rx_hwtstamp(vsi->back, skb, (rx_status &
1358 I40E_RXD_QW1_STATUS_TSYNINDX_MASK) >>
1359 I40E_RXD_QW1_STATUS_TSYNINDX_SHIFT);
1360 rx_ring->last_rx_timestamp = jiffies;
1361 }
1362
fd0a05ce
JB		 1363 /* probably a little skewed due to removing CRC */
1364 total_rx_bytes += skb->len;
1365 total_rx_packets++;
1366
1367 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
8144f0f7
JG		 1368
1369 i40e_rx_checksum(vsi, skb, rx_status, rx_error, rx_ptype);
1370
fd0a05ce
JB		 1371 vlan_tag = rx_status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)
1372 ? le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1)
1373 : 0;
1374 i40e_receive_skb(rx_ring, skb, vlan_tag);
1375
1376 rx_ring->netdev->last_rx = jiffies;
1377 budget--;
1378next_desc:
1379 rx_desc->wb.qword1.status_error_len = 0;
1380 if (!budget)
1381 break;
1382
1383 cleaned_count++;
1384 /* return some buffers to hardware, one at a time is too slow */
1385 if (cleaned_count >= I40E_RX_BUFFER_WRITE) {
1386 i40e_alloc_rx_buffers(rx_ring, cleaned_count);
1387 cleaned_count = 0;
1388 }
1389
1390 /* use prefetched values */
1391 rx_desc = next_rxd;
1392 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
829af3ac
MW		 1393 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >>
1394 I40E_RXD_QW1_STATUS_SHIFT;
fd0a05ce
JB		 1395 }
1396
1397 rx_ring->next_to_clean = i;
980e9b11 1398 u64_stats_update_begin(&rx_ring->syncp);
a114d0a6
AD		 1399 rx_ring->stats.packets += total_rx_packets;
1400 rx_ring->stats.bytes += total_rx_bytes;
980e9b11 1401 u64_stats_update_end(&rx_ring->syncp);
fd0a05ce
JB		 1402 rx_ring->q_vector->rx.total_packets += total_rx_packets;
1403 rx_ring->q_vector->rx.total_bytes += total_rx_bytes;
1404
1405 if (cleaned_count)
1406 i40e_alloc_rx_buffers(rx_ring, cleaned_count);
1407
1408 return budget > 0;
1409}
1410
1411/**
1412 * i40e_napi_poll - NAPI polling Rx/Tx cleanup routine
1413 * @napi: napi struct with our devices info in it
1414 * @budget: amount of work driver is allowed to do this pass, in packets
1415 *
1416 * This function will clean all queues associated with a q_vector.
1417 *
1418 * Returns the amount of work done
1419 **/
1420int i40e_napi_poll(struct napi_struct *napi, int budget)
1421{
1422 struct i40e_q_vector *q_vector =
1423 container_of(napi, struct i40e_q_vector, napi);
1424 struct i40e_vsi *vsi = q_vector->vsi;
cd0b6fa6 1425 struct i40e_ring *ring;
fd0a05ce
JB		 1426 bool clean_complete = true;
1427 int budget_per_ring;
fd0a05ce
JB		 1428
1429 if (test_bit(__I40E_DOWN, &vsi->state)) {
1430 napi_complete(napi);
1431 return 0;
1432 }
1433
cd0b6fa6
AD		 1434 /* Since the actual Tx work is minimal, we can give the Tx a larger
1435 * budget and be more aggressive about cleaning up the Tx descriptors.
1436 */
1437 i40e_for_each_ring(ring, q_vector->tx)
1438 clean_complete &= i40e_clean_tx_irq(ring, vsi->work_limit);
1439
fd0a05ce
JB		 1440 /* We attempt to distribute budget to each Rx queue fairly, but don't
1441 * allow the budget to go below 1 because that would exit polling early.
fd0a05ce
JB		 1442 */
1443 budget_per_ring = max(budget/q_vector->num_ringpairs, 1);
cd0b6fa6
AD		 1444
1445 i40e_for_each_ring(ring, q_vector->rx)
1446 clean_complete &= i40e_clean_rx_irq(ring, budget_per_ring);
fd0a05ce
JB		 1447
1448 /* If work not completed, return budget and polling will return */
1449 if (!clean_complete)
1450 return budget;
1451
1452 /* Work is done so exit the polling mode and re-enable the interrupt */
1453 napi_complete(napi);
1454 if (ITR_IS_DYNAMIC(vsi->rx_itr_setting) ||
1455 ITR_IS_DYNAMIC(vsi->tx_itr_setting))
1456 i40e_update_dynamic_itr(q_vector);
1457
1458 if (!test_bit(__I40E_DOWN, &vsi->state)) {
1459 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) {
1460 i40e_irq_dynamic_enable(vsi,
1461 q_vector->v_idx + vsi->base_vector);
1462 } else {
1463 struct i40e_hw *hw = &vsi->back->hw;
1464 /* We re-enable the queue 0 cause, but
1465 * don't worry about dynamic_enable
1466 * because we left it on for the other
1467 * possible interrupts during napi
1468 */
1469 u32 qval = rd32(hw, I40E_QINT_RQCTL(0));
1470 qval |= I40E_QINT_RQCTL_CAUSE_ENA_MASK;
1471 wr32(hw, I40E_QINT_RQCTL(0), qval);
1472
1473 qval = rd32(hw, I40E_QINT_TQCTL(0));
1474 qval |= I40E_QINT_TQCTL_CAUSE_ENA_MASK;
1475 wr32(hw, I40E_QINT_TQCTL(0), qval);
116a57d4
SN		 1476
1477 i40e_irq_dynamic_enable_icr0(vsi->back);
fd0a05ce
JB		 1478 }
1479 }
1480
1481 return 0;
1482}
1483
1484/**
1485 * i40e_atr - Add a Flow Director ATR filter
1486 * @tx_ring: ring to add programming descriptor to
1487 * @skb: send buffer
1488 * @flags: send flags
1489 * @protocol: wire protocol
1490 **/
1491static void i40e_atr(struct i40e_ring *tx_ring, struct sk_buff *skb,
1492 u32 flags, __be16 protocol)
1493{
1494 struct i40e_filter_program_desc *fdir_desc;
1495 struct i40e_pf *pf = tx_ring->vsi->back;
1496 union {
1497 unsigned char *network;
1498 struct iphdr *ipv4;
1499 struct ipv6hdr *ipv6;
1500 } hdr;
1501 struct tcphdr *th;
1502 unsigned int hlen;
1503 u32 flex_ptype, dtype_cmd;
fc4ac67b 1504 u16 i;
fd0a05ce
JB		 1505
1506 /* make sure ATR is enabled */
60ea5f83 1507 if (!(pf->flags & I40E_FLAG_FD_ATR_ENABLED))
fd0a05ce
JB		 1508 return;
1509
1510 /* if sampling is disabled do nothing */
1511 if (!tx_ring->atr_sample_rate)
1512 return;
1513
1514 tx_ring->atr_count++;
1515
1516 /* snag network header to get L4 type and address */
1517 hdr.network = skb_network_header(skb);
1518
1519 /* Currently only IPv4/IPv6 with TCP is supported */
1520 if (protocol == htons(ETH_P_IP)) {
1521 if (hdr.ipv4->protocol != IPPROTO_TCP)
1522 return;
1523
1524 /* access ihl as a u8 to avoid unaligned access on ia64 */
1525 hlen = (hdr.network[0] & 0x0F) << 2;
1526 } else if (protocol == htons(ETH_P_IPV6)) {
1527 if (hdr.ipv6->nexthdr != IPPROTO_TCP)
1528 return;
1529
1530 hlen = sizeof(struct ipv6hdr);
1531 } else {
1532 return;
1533 }
1534
1535 th = (struct tcphdr *)(hdr.network + hlen);
1536
1537 /* sample on all syn/fin packets or once every atr sample rate */
1538 if (!th->fin && !th->syn && (tx_ring->atr_count < tx_ring->atr_sample_rate))
1539 return;
1540
1541 tx_ring->atr_count = 0;
1542
1543 /* grab the next descriptor */
fc4ac67b
AD		 1544 i = tx_ring->next_to_use;
1545 fdir_desc = I40E_TX_FDIRDESC(tx_ring, i);
1546
1547 i++;
1548 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
fd0a05ce
JB		 1549
1550 flex_ptype = (tx_ring->queue_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
1551 I40E_TXD_FLTR_QW0_QINDEX_MASK;
1552 flex_ptype |= (protocol == htons(ETH_P_IP)) ?
1553 (I40E_FILTER_PCTYPE_NONF_IPV4_TCP <<
1554 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) :
1555 (I40E_FILTER_PCTYPE_NONF_IPV6_TCP <<
1556 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT);
1557
1558 flex_ptype |= tx_ring->vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
1559
1560 dtype_cmd = I40E_TX_DESC_DTYPE_FILTER_PROG;
1561
1562 dtype_cmd |= th->fin ?
1563 (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
1564 I40E_TXD_FLTR_QW1_PCMD_SHIFT) :
1565 (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
1566 I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1567
1568 dtype_cmd |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX <<
1569 I40E_TXD_FLTR_QW1_DEST_SHIFT;
1570
1571 dtype_cmd |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID <<
1572 I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT;
1573
1574 fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(flex_ptype);
1575 fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dtype_cmd);
1576}
1577
fd0a05ce
JB		 1578/**
1579 * i40e_tx_prepare_vlan_flags - prepare generic TX VLAN tagging flags for HW
1580 * @skb: send buffer
1581 * @tx_ring: ring to send buffer on
1582 * @flags: the tx flags to be set
1583 *
1584 * Checks the skb and set up correspondingly several generic transmit flags
1585 * related to VLAN tagging for the HW, such as VLAN, DCB, etc.
1586 *
1587 * Returns error code indicate the frame should be dropped upon error and the
1588 * otherwise returns 0 to indicate the flags has been set properly.
1589 **/
1590static int i40e_tx_prepare_vlan_flags(struct sk_buff *skb,
1591 struct i40e_ring *tx_ring,
1592 u32 *flags)
1593{
1594 __be16 protocol = skb->protocol;
1595 u32 tx_flags = 0;
1596
1597 /* if we have a HW VLAN tag being added, default to the HW one */
1598 if (vlan_tx_tag_present(skb)) {
1599 tx_flags |= vlan_tx_tag_get(skb) << I40E_TX_FLAGS_VLAN_SHIFT;
1600 tx_flags |= I40E_TX_FLAGS_HW_VLAN;
1601 /* else if it is a SW VLAN, check the next protocol and store the tag */
0e2fe46c 1602 } else if (protocol == htons(ETH_P_8021Q)) {
fd0a05ce
JB		 1603 struct vlan_hdr *vhdr, _vhdr;
1604 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr);
1605 if (!vhdr)
1606 return -EINVAL;
1607
1608 protocol = vhdr->h_vlan_encapsulated_proto;
1609 tx_flags |= ntohs(vhdr->h_vlan_TCI) << I40E_TX_FLAGS_VLAN_SHIFT;
1610 tx_flags |= I40E_TX_FLAGS_SW_VLAN;
1611 }
1612
1613 /* Insert 802.1p priority into VLAN header */
1614 if ((tx_ring->vsi->back->flags & I40E_FLAG_DCB_ENABLED) &&
1615 ((tx_flags & (I40E_TX_FLAGS_HW_VLAN | I40E_TX_FLAGS_SW_VLAN)) ||
1616 (skb->priority != TC_PRIO_CONTROL))) {
1617 tx_flags &= ~I40E_TX_FLAGS_VLAN_PRIO_MASK;
1618 tx_flags |= (skb->priority & 0x7) <<
1619 I40E_TX_FLAGS_VLAN_PRIO_SHIFT;
1620 if (tx_flags & I40E_TX_FLAGS_SW_VLAN) {
1621 struct vlan_ethhdr *vhdr;
1622 if (skb_header_cloned(skb) &&
1623 pskb_expand_head(skb, 0, 0, GFP_ATOMIC))
1624 return -ENOMEM;
1625 vhdr = (struct vlan_ethhdr *)skb->data;
1626 vhdr->h_vlan_TCI = htons(tx_flags >>
1627 I40E_TX_FLAGS_VLAN_SHIFT);
1628 } else {
1629 tx_flags |= I40E_TX_FLAGS_HW_VLAN;
1630 }
1631 }
1632 *flags = tx_flags;
1633 return 0;
1634}
1635
fd0a05ce
JB		 1636/**
1637 * i40e_tso - set up the tso context descriptor
1638 * @tx_ring: ptr to the ring to send
1639 * @skb: ptr to the skb we're sending
1640 * @tx_flags: the collected send information
1641 * @protocol: the send protocol
1642 * @hdr_len: ptr to the size of the packet header
1643 * @cd_tunneling: ptr to context descriptor bits
1644 *
1645 * Returns 0 if no TSO can happen, 1 if tso is going, or error
1646 **/
1647static int i40e_tso(struct i40e_ring *tx_ring, struct sk_buff *skb,
1648 u32 tx_flags, __be16 protocol, u8 *hdr_len,
1649 u64 *cd_type_cmd_tso_mss, u32 *cd_tunneling)
1650{
1651 u32 cd_cmd, cd_tso_len, cd_mss;
1652 struct tcphdr *tcph;
1653 struct iphdr *iph;
1654 u32 l4len;
1655 int err;
1656 struct ipv6hdr *ipv6h;
1657
1658 if (!skb_is_gso(skb))
1659 return 0;
1660
1661 if (skb_header_cloned(skb)) {
1662 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
1663 if (err)
1664 return err;
1665 }
1666
0e2fe46c 1667 if (protocol == htons(ETH_P_IP)) {
fd0a05ce
JB		 1668 iph = skb->encapsulation ? inner_ip_hdr(skb) : ip_hdr(skb);
1669 tcph = skb->encapsulation ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1670 iph->tot_len = 0;
1671 iph->check = 0;
1672 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
1673 0, IPPROTO_TCP, 0);
1674 } else if (skb_is_gso_v6(skb)) {
1675
1676 ipv6h = skb->encapsulation ? inner_ipv6_hdr(skb)
1677 : ipv6_hdr(skb);
1678 tcph = skb->encapsulation ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1679 ipv6h->payload_len = 0;
1680 tcph->check = ~csum_ipv6_magic(&ipv6h->saddr, &ipv6h->daddr,
1681 0, IPPROTO_TCP, 0);
1682 }
1683
1684 l4len = skb->encapsulation ? inner_tcp_hdrlen(skb) : tcp_hdrlen(skb);
1685 *hdr_len = (skb->encapsulation
1686 ? (skb_inner_transport_header(skb) - skb->data)
1687 : skb_transport_offset(skb)) + l4len;
1688
1689 /* find the field values */
1690 cd_cmd = I40E_TX_CTX_DESC_TSO;
1691 cd_tso_len = skb->len - *hdr_len;
1692 cd_mss = skb_shinfo(skb)->gso_size;
829af3ac
MW		 1693 *cd_type_cmd_tso_mss |= ((u64)cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) |
1694 ((u64)cd_tso_len <<
1695 I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) |
1696 ((u64)cd_mss << I40E_TXD_CTX_QW1_MSS_SHIFT);
fd0a05ce
JB		 1697 return 1;
1698}
1699
beb0dff1
JK		 1700/**
1701 * i40e_tsyn - set up the tsyn context descriptor
1702 * @tx_ring: ptr to the ring to send
1703 * @skb: ptr to the skb we're sending
1704 * @tx_flags: the collected send information
1705 *
1706 * Returns 0 if no Tx timestamp can happen and 1 if the timestamp will happen
1707 **/
1708static int i40e_tsyn(struct i40e_ring *tx_ring, struct sk_buff *skb,
1709 u32 tx_flags, u64 *cd_type_cmd_tso_mss)
1710{
1711 struct i40e_pf *pf;
1712
1713 if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)))
1714 return 0;
1715
1716 /* Tx timestamps cannot be sampled when doing TSO */
1717 if (tx_flags & I40E_TX_FLAGS_TSO)
1718 return 0;
1719
1720 /* only timestamp the outbound packet if the user has requested it and
1721 * we are not already transmitting a packet to be timestamped
1722 */
1723 pf = i40e_netdev_to_pf(tx_ring->netdev);
1724 if (pf->ptp_tx && !pf->ptp_tx_skb) {
1725 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1726 pf->ptp_tx_skb = skb_get(skb);
1727 } else {
1728 return 0;
1729 }
1730
1731 *cd_type_cmd_tso_mss |= (u64)I40E_TX_CTX_DESC_TSYN <<
1732 I40E_TXD_CTX_QW1_CMD_SHIFT;
1733
1734 pf->ptp_tx_start = jiffies;
1735 schedule_work(&pf->ptp_tx_work);
1736
1737 return 1;
1738}
1739
fd0a05ce
JB		 1740/**
1741 * i40e_tx_enable_csum - Enable Tx checksum offloads
1742 * @skb: send buffer
1743 * @tx_flags: Tx flags currently set
1744 * @td_cmd: Tx descriptor command bits to set
1745 * @td_offset: Tx descriptor header offsets to set
1746 * @cd_tunneling: ptr to context desc bits
1747 **/
1748static void i40e_tx_enable_csum(struct sk_buff *skb, u32 tx_flags,
1749 u32 *td_cmd, u32 *td_offset,
1750 struct i40e_ring *tx_ring,
1751 u32 *cd_tunneling)
1752{
1753 struct ipv6hdr *this_ipv6_hdr;
1754 unsigned int this_tcp_hdrlen;
1755 struct iphdr *this_ip_hdr;
1756 u32 network_hdr_len;
1757 u8 l4_hdr = 0;
1758
1759 if (skb->encapsulation) {
1760 network_hdr_len = skb_inner_network_header_len(skb);
1761 this_ip_hdr = inner_ip_hdr(skb);
1762 this_ipv6_hdr = inner_ipv6_hdr(skb);
1763 this_tcp_hdrlen = inner_tcp_hdrlen(skb);
1764
1765 if (tx_flags & I40E_TX_FLAGS_IPV4) {
1766
1767 if (tx_flags & I40E_TX_FLAGS_TSO) {
1768 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV4;
1769 ip_hdr(skb)->check = 0;
1770 } else {
1771 *cd_tunneling |=
1772 I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;
1773 }
1774 } else if (tx_flags & I40E_TX_FLAGS_IPV6) {
1775 if (tx_flags & I40E_TX_FLAGS_TSO) {
1776 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV6;
1777 ip_hdr(skb)->check = 0;
1778 } else {
1779 *cd_tunneling |=
1780 I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;
1781 }
1782 }
1783
1784 /* Now set the ctx descriptor fields */
1785 *cd_tunneling |= (skb_network_header_len(skb) >> 2) <<
1786 I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT |
1787 I40E_TXD_CTX_UDP_TUNNELING |
1788 ((skb_inner_network_offset(skb) -
1789 skb_transport_offset(skb)) >> 1) <<
1790 I40E_TXD_CTX_QW0_NATLEN_SHIFT;
1791
1792 } else {
1793 network_hdr_len = skb_network_header_len(skb);
1794 this_ip_hdr = ip_hdr(skb);
1795 this_ipv6_hdr = ipv6_hdr(skb);
1796 this_tcp_hdrlen = tcp_hdrlen(skb);
1797 }
1798
1799 /* Enable IP checksum offloads */
1800 if (tx_flags & I40E_TX_FLAGS_IPV4) {
1801 l4_hdr = this_ip_hdr->protocol;
1802 /* the stack computes the IP header already, the only time we
1803 * need the hardware to recompute it is in the case of TSO.
1804 */
1805 if (tx_flags & I40E_TX_FLAGS_TSO) {
1806 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM;
1807 this_ip_hdr->check = 0;
1808 } else {
1809 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4;
1810 }
1811 /* Now set the td_offset for IP header length */
1812 *td_offset = (network_hdr_len >> 2) <<
1813 I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
1814 } else if (tx_flags & I40E_TX_FLAGS_IPV6) {
1815 l4_hdr = this_ipv6_hdr->nexthdr;
1816 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV6;
1817 /* Now set the td_offset for IP header length */
1818 *td_offset = (network_hdr_len >> 2) <<
1819 I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
1820 }
1821 /* words in MACLEN + dwords in IPLEN + dwords in L4Len */
1822 *td_offset |= (skb_network_offset(skb) >> 1) <<
1823 I40E_TX_DESC_LENGTH_MACLEN_SHIFT;
1824
1825 /* Enable L4 checksum offloads */
1826 switch (l4_hdr) {
1827 case IPPROTO_TCP:
1828 /* enable checksum offloads */
1829 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
1830 *td_offset |= (this_tcp_hdrlen >> 2) <<
1831 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1832 break;
1833 case IPPROTO_SCTP:
1834 /* enable SCTP checksum offload */
1835 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP;
1836 *td_offset |= (sizeof(struct sctphdr) >> 2) <<
1837 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1838 break;
1839 case IPPROTO_UDP:
1840 /* enable UDP checksum offload */
1841 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP;
1842 *td_offset |= (sizeof(struct udphdr) >> 2) <<
1843 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1844 break;
1845 default:
1846 break;
1847 }
1848}
1849
1850/**
1851 * i40e_create_tx_ctx Build the Tx context descriptor
1852 * @tx_ring: ring to create the descriptor on
1853 * @cd_type_cmd_tso_mss: Quad Word 1
1854 * @cd_tunneling: Quad Word 0 - bits 0-31
1855 * @cd_l2tag2: Quad Word 0 - bits 32-63
1856 **/
1857static void i40e_create_tx_ctx(struct i40e_ring *tx_ring,
1858 const u64 cd_type_cmd_tso_mss,
1859 const u32 cd_tunneling, const u32 cd_l2tag2)
1860{
1861 struct i40e_tx_context_desc *context_desc;
fc4ac67b 1862 int i = tx_ring->next_to_use;
fd0a05ce
JB		 1863
1864 if (!cd_type_cmd_tso_mss && !cd_tunneling && !cd_l2tag2)
1865 return;
1866
1867 /* grab the next descriptor */
fc4ac67b
AD		 1868 context_desc = I40E_TX_CTXTDESC(tx_ring, i);
1869
1870 i++;
1871 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
fd0a05ce
JB		 1872
1873 /* cpu_to_le32 and assign to struct fields */
1874 context_desc->tunneling_params = cpu_to_le32(cd_tunneling);
1875 context_desc->l2tag2 = cpu_to_le16(cd_l2tag2);
1876 context_desc->type_cmd_tso_mss = cpu_to_le64(cd_type_cmd_tso_mss);
1877}
1878
1879/**
1880 * i40e_tx_map - Build the Tx descriptor
1881 * @tx_ring: ring to send buffer on
1882 * @skb: send buffer
1883 * @first: first buffer info buffer to use
1884 * @tx_flags: collected send information
1885 * @hdr_len: size of the packet header
1886 * @td_cmd: the command field in the descriptor
1887 * @td_offset: offset for checksum or crc
1888 **/
1889static void i40e_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb,
1890 struct i40e_tx_buffer *first, u32 tx_flags,
1891 const u8 hdr_len, u32 td_cmd, u32 td_offset)
1892{
fd0a05ce
JB		 1893 unsigned int data_len = skb->data_len;
1894 unsigned int size = skb_headlen(skb);
a5e9c572 1895 struct skb_frag_struct *frag;
fd0a05ce
JB		 1896 struct i40e_tx_buffer *tx_bi;
1897 struct i40e_tx_desc *tx_desc;
a5e9c572 1898 u16 i = tx_ring->next_to_use;
fd0a05ce
JB		 1899 u32 td_tag = 0;
1900 dma_addr_t dma;
1901 u16 gso_segs;
1902
fd0a05ce
JB		 1903 if (tx_flags & I40E_TX_FLAGS_HW_VLAN) {
1904 td_cmd |= I40E_TX_DESC_CMD_IL2TAG1;
1905 td_tag = (tx_flags & I40E_TX_FLAGS_VLAN_MASK) >>
1906 I40E_TX_FLAGS_VLAN_SHIFT;
1907 }
1908
a5e9c572
AD		 1909 if (tx_flags & (I40E_TX_FLAGS_TSO | I40E_TX_FLAGS_FSO))
1910 gso_segs = skb_shinfo(skb)->gso_segs;
1911 else
1912 gso_segs = 1;
1913
1914 /* multiply data chunks by size of headers */
1915 first->bytecount = skb->len - hdr_len + (gso_segs * hdr_len);
1916 first->gso_segs = gso_segs;
1917 first->skb = skb;
1918 first->tx_flags = tx_flags;
1919
1920 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
1921
fd0a05ce 1922 tx_desc = I40E_TX_DESC(tx_ring, i);
a5e9c572
AD		 1923 tx_bi = first;
1924
1925 for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
1926 if (dma_mapping_error(tx_ring->dev, dma))
1927 goto dma_error;
1928
1929 /* record length, and DMA address */
1930 dma_unmap_len_set(tx_bi, len, size);
1931 dma_unmap_addr_set(tx_bi, dma, dma);
1932
1933 tx_desc->buffer_addr = cpu_to_le64(dma);
1934
1935 while (unlikely(size > I40E_MAX_DATA_PER_TXD)) {
fd0a05ce
JB		 1936 tx_desc->cmd_type_offset_bsz =
1937 build_ctob(td_cmd, td_offset,
1938 I40E_MAX_DATA_PER_TXD, td_tag);
1939
fd0a05ce
JB		 1940 tx_desc++;
1941 i++;
1942 if (i == tx_ring->count) {
1943 tx_desc = I40E_TX_DESC(tx_ring, 0);
1944 i = 0;
1945 }
fd0a05ce 1946
a5e9c572
AD		 1947 dma += I40E_MAX_DATA_PER_TXD;
1948 size -= I40E_MAX_DATA_PER_TXD;
fd0a05ce 1949
a5e9c572
AD		 1950 tx_desc->buffer_addr = cpu_to_le64(dma);
1951 }
fd0a05ce
JB		 1952
1953 if (likely(!data_len))
1954 break;
1955
a5e9c572
AD		 1956 tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset,
1957 size, td_tag);
fd0a05ce
JB		 1958
1959 tx_desc++;
1960 i++;
1961 if (i == tx_ring->count) {
1962 tx_desc = I40E_TX_DESC(tx_ring, 0);
1963 i = 0;
1964 }
1965
a5e9c572
AD		 1966 size = skb_frag_size(frag);
1967 data_len -= size;
fd0a05ce 1968
a5e9c572
AD		 1969 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
1970 DMA_TO_DEVICE);
fd0a05ce 1971
a5e9c572
AD		 1972 tx_bi = &tx_ring->tx_bi[i];
1973 }
fd0a05ce 1974
a5e9c572
AD		 1975 tx_desc->cmd_type_offset_bsz =
1976 build_ctob(td_cmd, td_offset, size, td_tag) |
1977 cpu_to_le64((u64)I40E_TXD_CMD << I40E_TXD_QW1_CMD_SHIFT);
fd0a05ce 1978
7070ce0a
AD		 1979 netdev_tx_sent_queue(netdev_get_tx_queue(tx_ring->netdev,
1980 tx_ring->queue_index),
1981 first->bytecount);
1982
a5e9c572 1983 /* set the timestamp */
fd0a05ce 1984 first->time_stamp = jiffies;
fd0a05ce
JB		 1985
1986 /* Force memory writes to complete before letting h/w
1987 * know there are new descriptors to fetch. (Only
1988 * applicable for weak-ordered memory model archs,
1989 * such as IA-64).
1990 */
1991 wmb();
1992
a5e9c572
AD		 1993 /* set next_to_watch value indicating a packet is present */
1994 first->next_to_watch = tx_desc;
1995
1996 i++;
1997 if (i == tx_ring->count)
1998 i = 0;
1999
2000 tx_ring->next_to_use = i;
2001
2002 /* notify HW of packet */
fd0a05ce 2003 writel(i, tx_ring->tail);
a5e9c572 2004
fd0a05ce
JB		 2005 return;
2006
2007dma_error:
a5e9c572 2008 dev_info(tx_ring->dev, "TX DMA map failed\n");
fd0a05ce
JB		 2009
2010 /* clear dma mappings for failed tx_bi map */
2011 for (;;) {
2012 tx_bi = &tx_ring->tx_bi[i];
a5e9c572 2013 i40e_unmap_and_free_tx_resource(tx_ring, tx_bi);
fd0a05ce
JB		 2014 if (tx_bi == first)
2015 break;
2016 if (i == 0)
2017 i = tx_ring->count;
2018 i--;
2019 }
2020
fd0a05ce
JB		 2021 tx_ring->next_to_use = i;
2022}
2023
2024/**
2025 * __i40e_maybe_stop_tx - 2nd level check for tx stop conditions
2026 * @tx_ring: the ring to be checked
2027 * @size: the size buffer we want to assure is available
2028 *
2029 * Returns -EBUSY if a stop is needed, else 0
2030 **/
2031static inline int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
2032{
2033 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
8e9dca53 2034 /* Memory barrier before checking head and tail */
fd0a05ce
JB		 2035 smp_mb();
2036
2037 /* Check again in a case another CPU has just made room available. */
2038 if (likely(I40E_DESC_UNUSED(tx_ring) < size))
2039 return -EBUSY;
2040
2041 /* A reprieve! - use start_queue because it doesn't call schedule */
2042 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
2043 ++tx_ring->tx_stats.restart_queue;
2044 return 0;
2045}
2046
2047/**
2048 * i40e_maybe_stop_tx - 1st level check for tx stop conditions
2049 * @tx_ring: the ring to be checked
2050 * @size: the size buffer we want to assure is available
2051 *
2052 * Returns 0 if stop is not needed
2053 **/
2054static int i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
2055{
2056 if (likely(I40E_DESC_UNUSED(tx_ring) >= size))
2057 return 0;
2058 return __i40e_maybe_stop_tx(tx_ring, size);
2059}
2060
2061/**
2062 * i40e_xmit_descriptor_count - calculate number of tx descriptors needed
2063 * @skb: send buffer
2064 * @tx_ring: ring to send buffer on
2065 *
2066 * Returns number of data descriptors needed for this skb. Returns 0 to indicate
2067 * there is not enough descriptors available in this ring since we need at least
2068 * one descriptor.
2069 **/
2070static int i40e_xmit_descriptor_count(struct sk_buff *skb,
2071 struct i40e_ring *tx_ring)
2072{
2073#if PAGE_SIZE > I40E_MAX_DATA_PER_TXD
2074 unsigned int f;
2075#endif
2076 int count = 0;
2077
2078 /* need: 1 descriptor per page * PAGE_SIZE/I40E_MAX_DATA_PER_TXD,
2079 * + 1 desc for skb_head_len/I40E_MAX_DATA_PER_TXD,
2080 * + 2 desc gap to keep tail from touching head,
2081 * + 1 desc for context descriptor,
2082 * otherwise try next time
2083 */
2084#if PAGE_SIZE > I40E_MAX_DATA_PER_TXD
2085 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
2086 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
2087#else
2088 count += skb_shinfo(skb)->nr_frags;
2089#endif
2090 count += TXD_USE_COUNT(skb_headlen(skb));
2091 if (i40e_maybe_stop_tx(tx_ring, count + 3)) {
2092 tx_ring->tx_stats.tx_busy++;
2093 return 0;
2094 }
2095 return count;
2096}
2097
2098/**
2099 * i40e_xmit_frame_ring - Sends buffer on Tx ring
2100 * @skb: send buffer
2101 * @tx_ring: ring to send buffer on
2102 *
2103 * Returns NETDEV_TX_OK if sent, else an error code
2104 **/
2105static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb,
2106 struct i40e_ring *tx_ring)
2107{
2108 u64 cd_type_cmd_tso_mss = I40E_TX_DESC_DTYPE_CONTEXT;
2109 u32 cd_tunneling = 0, cd_l2tag2 = 0;
2110 struct i40e_tx_buffer *first;
2111 u32 td_offset = 0;
2112 u32 tx_flags = 0;
2113 __be16 protocol;
2114 u32 td_cmd = 0;
2115 u8 hdr_len = 0;
beb0dff1 2116 int tsyn;
fd0a05ce
JB		 2117 int tso;
2118 if (0 == i40e_xmit_descriptor_count(skb, tx_ring))
2119 return NETDEV_TX_BUSY;
2120
2121 /* prepare the xmit flags */
2122 if (i40e_tx_prepare_vlan_flags(skb, tx_ring, &tx_flags))
2123 goto out_drop;
2124
2125 /* obtain protocol of skb */
2126 protocol = skb->protocol;
2127
2128 /* record the location of the first descriptor for this packet */
2129 first = &tx_ring->tx_bi[tx_ring->next_to_use];
2130
2131 /* setup IPv4/IPv6 offloads */
0e2fe46c 2132 if (protocol == htons(ETH_P_IP))
fd0a05ce 2133 tx_flags |= I40E_TX_FLAGS_IPV4;
0e2fe46c 2134 else if (protocol == htons(ETH_P_IPV6))
fd0a05ce
JB		 2135 tx_flags |= I40E_TX_FLAGS_IPV6;
2136
2137 tso = i40e_tso(tx_ring, skb, tx_flags, protocol, &hdr_len,
2138 &cd_type_cmd_tso_mss, &cd_tunneling);
2139
2140 if (tso < 0)
2141 goto out_drop;
2142 else if (tso)
2143 tx_flags |= I40E_TX_FLAGS_TSO;
2144
2145 skb_tx_timestamp(skb);
2146
beb0dff1
JK		 2147 tsyn = i40e_tsyn(tx_ring, skb, tx_flags, &cd_type_cmd_tso_mss);
2148
2149 if (tsyn)
2150 tx_flags |= I40E_TX_FLAGS_TSYN;
2151
b1941306
AD		 2152 /* always enable CRC insertion offload */
2153 td_cmd |= I40E_TX_DESC_CMD_ICRC;
2154
fd0a05ce 2155 /* Always offload the checksum, since it's in the data descriptor */
b1941306 2156 if (skb->ip_summed == CHECKSUM_PARTIAL) {
fd0a05ce
JB		 2157 tx_flags |= I40E_TX_FLAGS_CSUM;
2158
fd0a05ce
JB		 2159 i40e_tx_enable_csum(skb, tx_flags, &td_cmd, &td_offset,
2160 tx_ring, &cd_tunneling);
b1941306 2161 }
fd0a05ce
JB		 2162
2163 i40e_create_tx_ctx(tx_ring, cd_type_cmd_tso_mss,
2164 cd_tunneling, cd_l2tag2);
2165
2166 /* Add Flow Director ATR if it's enabled.
2167 *
2168 * NOTE: this must always be directly before the data descriptor.
2169 */
2170 i40e_atr(tx_ring, skb, tx_flags, protocol);
2171
2172 i40e_tx_map(tx_ring, skb, first, tx_flags, hdr_len,
2173 td_cmd, td_offset);
2174
2175 i40e_maybe_stop_tx(tx_ring, DESC_NEEDED);
2176
2177 return NETDEV_TX_OK;
2178
2179out_drop:
2180 dev_kfree_skb_any(skb);
2181 return NETDEV_TX_OK;
2182}
2183
2184/**
2185 * i40e_lan_xmit_frame - Selects the correct VSI and Tx queue to send buffer
2186 * @skb: send buffer
2187 * @netdev: network interface device structure
2188 *
2189 * Returns NETDEV_TX_OK if sent, else an error code
2190 **/
2191netdev_tx_t i40e_lan_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
2192{
2193 struct i40e_netdev_priv *np = netdev_priv(netdev);
2194 struct i40e_vsi *vsi = np->vsi;
9f65e15b 2195 struct i40e_ring *tx_ring = vsi->tx_rings[skb->queue_mapping];
fd0a05ce
JB		 2196
2197 /* hardware can't handle really short frames, hardware padding works
2198 * beyond this point
2199 */
2200 if (unlikely(skb->len < I40E_MIN_TX_LEN)) {
2201 if (skb_pad(skb, I40E_MIN_TX_LEN - skb->len))
2202 return NETDEV_TX_OK;
2203 skb->len = I40E_MIN_TX_LEN;
2204 skb_set_tail_pointer(skb, I40E_MIN_TX_LEN);
2205 }
2206
2207 return i40e_xmit_frame_ring(skb, tx_ring);
2208}
Linux 6.x block layer and io_uring tree(s)