projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
i40e: fix nvm version and remove firmware report
[linux-2.6-block.git] / drivers / net / ethernet / intel / i40e / i40e_txrx.c
CommitLineData
fd0a05ce
JB		 1/*******************************************************************************
2 *
3 * Intel Ethernet Controller XL710 Family Linux Driver
dc641b73 4 * Copyright(c) 2013 - 2014 Intel Corporation.
fd0a05ce
JB		 5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * more details.
14 *
dc641b73
GR		 15 * You should have received a copy of the GNU General Public License along
16 * with this program. If not, see <http://www.gnu.org/licenses/>.
fd0a05ce
JB		 17 *
18 * The full GNU General Public License is included in this distribution in
19 * the file called "COPYING".
20 *
21 * Contact Information:
22 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
23 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
24 *
25 ******************************************************************************/
26
27#include "i40e.h"
206812b5 28#include "i40e_prototype.h"
fd0a05ce
JB		 29
30static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size,
31 u32 td_tag)
32{
33 return cpu_to_le64(I40E_TX_DESC_DTYPE_DATA |
34 ((u64)td_cmd << I40E_TXD_QW1_CMD_SHIFT) |
35 ((u64)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) |
36 ((u64)size << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) |
37 ((u64)td_tag << I40E_TXD_QW1_L2TAG1_SHIFT));
38}
39
eaefbd06 40#define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS)
fd0a05ce
JB		 41/**
42 * i40e_program_fdir_filter - Program a Flow Director filter
17a73f6b
JG		 43 * @fdir_data: Packet data that will be filter parameters
44 * @raw_packet: the pre-allocated packet buffer for FDir
fd0a05ce
JB		 45 * @pf: The pf pointer
46 * @add: True for add/update, False for remove
47 **/
17a73f6b 48int i40e_program_fdir_filter(struct i40e_fdir_filter *fdir_data, u8 *raw_packet,
fd0a05ce
JB		 49 struct i40e_pf *pf, bool add)
50{
51 struct i40e_filter_program_desc *fdir_desc;
52 struct i40e_tx_buffer *tx_buf;
53 struct i40e_tx_desc *tx_desc;
54 struct i40e_ring *tx_ring;
eaefbd06 55 unsigned int fpt, dcc;
fd0a05ce
JB		 56 struct i40e_vsi *vsi;
57 struct device *dev;
58 dma_addr_t dma;
59 u32 td_cmd = 0;
60 u16 i;
61
62 /* find existing FDIR VSI */
63 vsi = NULL;
64 for (i = 0; i < pf->hw.func_caps.num_vsis; i++)
65 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR)
66 vsi = pf->vsi[i];
67 if (!vsi)
68 return -ENOENT;
69
9f65e15b 70 tx_ring = vsi->tx_rings[0];
fd0a05ce
JB		 71 dev = tx_ring->dev;
72
17a73f6b
JG		 73 dma = dma_map_single(dev, raw_packet,
74 I40E_FDIR_MAX_RAW_PACKET_SIZE, DMA_TO_DEVICE);
fd0a05ce
JB		 75 if (dma_mapping_error(dev, dma))
76 goto dma_fail;
77
78 /* grab the next descriptor */
fc4ac67b
AD		 79 i = tx_ring->next_to_use;
80 fdir_desc = I40E_TX_FDIRDESC(tx_ring, i);
fc4ac67b 81
eaefbd06 82 tx_ring->next_to_use = (i + 1 < tx_ring->count) ? i + 1 : 0;
fd0a05ce 83
eaefbd06
JB		 84 fpt = (fdir_data->q_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
85 I40E_TXD_FLTR_QW0_QINDEX_MASK;
fd0a05ce 86
eaefbd06
JB		 87 fpt |= (fdir_data->flex_off << I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT) &
88 I40E_TXD_FLTR_QW0_FLEXOFF_MASK;
fd0a05ce 89
eaefbd06
JB		 90 fpt |= (fdir_data->pctype << I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) &
91 I40E_TXD_FLTR_QW0_PCTYPE_MASK;
fd0a05ce
JB		 92
93 /* Use LAN VSI Id if not programmed by user */
94 if (fdir_data->dest_vsi == 0)
eaefbd06
JB		 95 fpt |= (pf->vsi[pf->lan_vsi]->id) <<
96 I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
fd0a05ce 97 else
eaefbd06
JB		 98 fpt |= ((u32)fdir_data->dest_vsi <<
99 I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT) &
100 I40E_TXD_FLTR_QW0_DEST_VSI_MASK;
101
102 fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(fpt);
fd0a05ce 103
eaefbd06 104 dcc = I40E_TX_DESC_DTYPE_FILTER_PROG;
fd0a05ce
JB		 105
106 if (add)
eaefbd06
JB		 107 dcc |= I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
108 I40E_TXD_FLTR_QW1_PCMD_SHIFT;
fd0a05ce 109 else
eaefbd06
JB		 110 dcc |= I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
111 I40E_TXD_FLTR_QW1_PCMD_SHIFT;
fd0a05ce 112
eaefbd06
JB		 113 dcc |= (fdir_data->dest_ctl << I40E_TXD_FLTR_QW1_DEST_SHIFT) &
114 I40E_TXD_FLTR_QW1_DEST_MASK;
fd0a05ce 115
eaefbd06
JB		 116 dcc |= (fdir_data->fd_status << I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT) &
117 I40E_TXD_FLTR_QW1_FD_STATUS_MASK;
fd0a05ce
JB		 118
119 if (fdir_data->cnt_index != 0) {
eaefbd06
JB		 120 dcc |= I40E_TXD_FLTR_QW1_CNT_ENA_MASK;
121 dcc |= ((u32)fdir_data->cnt_index <<
122 I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) &
123 I40E_TXD_FLTR_QW1_CNTINDEX_MASK;
fd0a05ce
JB		 124 }
125
eaefbd06 126 fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dcc);
fd0a05ce
JB		 127 fdir_desc->fd_id = cpu_to_le32(fdir_data->fd_id);
128
129 /* Now program a dummy descriptor */
fc4ac67b
AD		 130 i = tx_ring->next_to_use;
131 tx_desc = I40E_TX_DESC(tx_ring, i);
298deef1 132 tx_buf = &tx_ring->tx_bi[i];
fc4ac67b 133
eaefbd06 134 tx_ring->next_to_use = (i + 1 < tx_ring->count) ? i + 1 : 0;
fd0a05ce 135
298deef1 136 /* record length, and DMA address */
17a73f6b 137 dma_unmap_len_set(tx_buf, len, I40E_FDIR_MAX_RAW_PACKET_SIZE);
298deef1
ASJ		 138 dma_unmap_addr_set(tx_buf, dma, dma);
139
fd0a05ce 140 tx_desc->buffer_addr = cpu_to_le64(dma);
eaefbd06 141 td_cmd = I40E_TXD_CMD | I40E_TX_DESC_CMD_DUMMY;
fd0a05ce
JB		 142
143 tx_desc->cmd_type_offset_bsz =
17a73f6b 144 build_ctob(td_cmd, 0, I40E_FDIR_MAX_RAW_PACKET_SIZE, 0);
fd0a05ce 145
298deef1
ASJ		 146 /* set the timestamp */
147 tx_buf->time_stamp = jiffies;
148
fd0a05ce
JB		 149 /* Force memory writes to complete before letting h/w
150 * know there are new descriptors to fetch. (Only
151 * applicable for weak-ordered memory model archs,
152 * such as IA-64).
153 */
154 wmb();
155
fc4ac67b
AD		 156 /* Mark the data descriptor to be watched */
157 tx_buf->next_to_watch = tx_desc;
158
fd0a05ce
JB		 159 writel(tx_ring->next_to_use, tx_ring->tail);
160 return 0;
161
162dma_fail:
163 return -1;
164}
165
17a73f6b
JG		 166#define IP_HEADER_OFFSET 14
167#define I40E_UDPIP_DUMMY_PACKET_LEN 42
168/**
169 * i40e_add_del_fdir_udpv4 - Add/Remove UDPv4 filters
170 * @vsi: pointer to the targeted VSI
171 * @fd_data: the flow director data required for the FDir descriptor
172 * @raw_packet: the pre-allocated packet buffer for FDir
173 * @add: true adds a filter, false removes it
174 *
175 * Returns 0 if the filters were successfully added or removed
176 **/
177static int i40e_add_del_fdir_udpv4(struct i40e_vsi *vsi,
178 struct i40e_fdir_filter *fd_data,
179 u8 *raw_packet, bool add)
180{
181 struct i40e_pf *pf = vsi->back;
182 struct udphdr *udp;
183 struct iphdr *ip;
184 bool err = false;
185 int ret;
186 int i;
187 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0,
188 0x45, 0, 0, 0x1c, 0, 0, 0x40, 0, 0x40, 0x11, 0, 0, 0, 0, 0, 0,
189 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
190
191 memcpy(raw_packet, packet, I40E_UDPIP_DUMMY_PACKET_LEN);
192
193 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET);
194 udp = (struct udphdr *)(raw_packet + IP_HEADER_OFFSET
195 + sizeof(struct iphdr));
196
197 ip->daddr = fd_data->dst_ip[0];
198 udp->dest = fd_data->dst_port;
199 ip->saddr = fd_data->src_ip[0];
200 udp->source = fd_data->src_port;
201
202 for (i = I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP;
203 i <= I40E_FILTER_PCTYPE_NONF_IPV4_UDP; i++) {
204 fd_data->pctype = i;
205 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
206
207 if (ret) {
208 dev_info(&pf->pdev->dev,
209 "Filter command send failed for PCTYPE %d (ret = %d)\n",
210 fd_data->pctype, ret);
211 err = true;
212 } else {
213 dev_info(&pf->pdev->dev,
214 "Filter OK for PCTYPE %d (ret = %d)\n",
215 fd_data->pctype, ret);
216 }
217 }
218
219 return err ? -EOPNOTSUPP : 0;
220}
221
222#define I40E_TCPIP_DUMMY_PACKET_LEN 54
223/**
224 * i40e_add_del_fdir_tcpv4 - Add/Remove TCPv4 filters
225 * @vsi: pointer to the targeted VSI
226 * @fd_data: the flow director data required for the FDir descriptor
227 * @raw_packet: the pre-allocated packet buffer for FDir
228 * @add: true adds a filter, false removes it
229 *
230 * Returns 0 if the filters were successfully added or removed
231 **/
232static int i40e_add_del_fdir_tcpv4(struct i40e_vsi *vsi,
233 struct i40e_fdir_filter *fd_data,
234 u8 *raw_packet, bool add)
235{
236 struct i40e_pf *pf = vsi->back;
237 struct tcphdr *tcp;
238 struct iphdr *ip;
239 bool err = false;
240 int ret;
241 /* Dummy packet */
242 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0,
243 0x45, 0, 0, 0x28, 0, 0, 0x40, 0, 0x40, 0x6, 0, 0, 0, 0, 0, 0,
244 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x80, 0x11,
245 0x0, 0x72, 0, 0, 0, 0};
246
247 memcpy(raw_packet, packet, I40E_TCPIP_DUMMY_PACKET_LEN);
248
249 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET);
250 tcp = (struct tcphdr *)(raw_packet + IP_HEADER_OFFSET
251 + sizeof(struct iphdr));
252
253 ip->daddr = fd_data->dst_ip[0];
254 tcp->dest = fd_data->dst_port;
255 ip->saddr = fd_data->src_ip[0];
256 tcp->source = fd_data->src_port;
257
258 if (add) {
259 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) {
260 dev_info(&pf->pdev->dev, "Forcing ATR off, sideband rules for TCP/IPv4 flow being applied\n");
261 pf->flags &= ~I40E_FLAG_FD_ATR_ENABLED;
262 }
263 }
264
265 fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN;
266 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
267
268 if (ret) {
269 dev_info(&pf->pdev->dev,
270 "Filter command send failed for PCTYPE %d (ret = %d)\n",
271 fd_data->pctype, ret);
272 err = true;
273 } else {
274 dev_info(&pf->pdev->dev, "Filter OK for PCTYPE %d (ret = %d)\n",
275 fd_data->pctype, ret);
276 }
277
278 fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_TCP;
279
280 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
281 if (ret) {
282 dev_info(&pf->pdev->dev,
283 "Filter command send failed for PCTYPE %d (ret = %d)\n",
284 fd_data->pctype, ret);
285 err = true;
286 } else {
287 dev_info(&pf->pdev->dev, "Filter OK for PCTYPE %d (ret = %d)\n",
288 fd_data->pctype, ret);
289 }
290
291 return err ? -EOPNOTSUPP : 0;
292}
293
294/**
295 * i40e_add_del_fdir_sctpv4 - Add/Remove SCTPv4 Flow Director filters for
296 * a specific flow spec
297 * @vsi: pointer to the targeted VSI
298 * @fd_data: the flow director data required for the FDir descriptor
299 * @raw_packet: the pre-allocated packet buffer for FDir
300 * @add: true adds a filter, false removes it
301 *
302 * Returns 0 if the filters were successfully added or removed
303 **/
304static int i40e_add_del_fdir_sctpv4(struct i40e_vsi *vsi,
305 struct i40e_fdir_filter *fd_data,
306 u8 *raw_packet, bool add)
307{
308 return -EOPNOTSUPP;
309}
310
311#define I40E_IP_DUMMY_PACKET_LEN 34
312/**
313 * i40e_add_del_fdir_ipv4 - Add/Remove IPv4 Flow Director filters for
314 * a specific flow spec
315 * @vsi: pointer to the targeted VSI
316 * @fd_data: the flow director data required for the FDir descriptor
317 * @raw_packet: the pre-allocated packet buffer for FDir
318 * @add: true adds a filter, false removes it
319 *
320 * Returns 0 if the filters were successfully added or removed
321 **/
322static int i40e_add_del_fdir_ipv4(struct i40e_vsi *vsi,
323 struct i40e_fdir_filter *fd_data,
324 u8 *raw_packet, bool add)
325{
326 struct i40e_pf *pf = vsi->back;
327 struct iphdr *ip;
328 bool err = false;
329 int ret;
330 int i;
331 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0,
332 0x45, 0, 0, 0x14, 0, 0, 0x40, 0, 0x40, 0x10, 0, 0, 0, 0, 0, 0,
333 0, 0, 0, 0};
334
335 memcpy(raw_packet, packet, I40E_IP_DUMMY_PACKET_LEN);
336 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET);
337
338 ip->saddr = fd_data->src_ip[0];
339 ip->daddr = fd_data->dst_ip[0];
340 ip->protocol = 0;
341
342 for (i = I40E_FILTER_PCTYPE_NONF_IPV4_OTHER;
343 i <= I40E_FILTER_PCTYPE_FRAG_IPV4; i++) {
344 fd_data->pctype = i;
345 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
346
347 if (ret) {
348 dev_info(&pf->pdev->dev,
349 "Filter command send failed for PCTYPE %d (ret = %d)\n",
350 fd_data->pctype, ret);
351 err = true;
352 } else {
353 dev_info(&pf->pdev->dev,
354 "Filter OK for PCTYPE %d (ret = %d)\n",
355 fd_data->pctype, ret);
356 }
357 }
358
359 return err ? -EOPNOTSUPP : 0;
360}
361
362/**
363 * i40e_add_del_fdir - Build raw packets to add/del fdir filter
364 * @vsi: pointer to the targeted VSI
365 * @cmd: command to get or set RX flow classification rules
366 * @add: true adds a filter, false removes it
367 *
368 **/
369int i40e_add_del_fdir(struct i40e_vsi *vsi,
370 struct i40e_fdir_filter *input, bool add)
371{
372 struct i40e_pf *pf = vsi->back;
373 u8 *raw_packet;
374 int ret;
375
376 /* Populate the Flow Director that we have at the moment
377 * and allocate the raw packet buffer for the calling functions
378 */
379 raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL);
380 if (!raw_packet)
381 return -ENOMEM;
382
383 switch (input->flow_type & ~FLOW_EXT) {
384 case TCP_V4_FLOW:
385 ret = i40e_add_del_fdir_tcpv4(vsi, input, raw_packet,
386 add);
387 break;
388 case UDP_V4_FLOW:
389 ret = i40e_add_del_fdir_udpv4(vsi, input, raw_packet,
390 add);
391 break;
392 case SCTP_V4_FLOW:
393 ret = i40e_add_del_fdir_sctpv4(vsi, input, raw_packet,
394 add);
395 break;
396 case IPV4_FLOW:
397 ret = i40e_add_del_fdir_ipv4(vsi, input, raw_packet,
398 add);
399 break;
400 case IP_USER_FLOW:
401 switch (input->ip4_proto) {
402 case IPPROTO_TCP:
403 ret = i40e_add_del_fdir_tcpv4(vsi, input,
404 raw_packet, add);
405 break;
406 case IPPROTO_UDP:
407 ret = i40e_add_del_fdir_udpv4(vsi, input,
408 raw_packet, add);
409 break;
410 case IPPROTO_SCTP:
411 ret = i40e_add_del_fdir_sctpv4(vsi, input,
412 raw_packet, add);
413 break;
414 default:
415 ret = i40e_add_del_fdir_ipv4(vsi, input,
416 raw_packet, add);
417 break;
418 }
419 break;
420 default:
421 dev_info(&pf->pdev->dev, "Could not specify spec type %d",
422 input->flow_type);
423 ret = -EINVAL;
424 }
425
426 kfree(raw_packet);
427 return ret;
428}
429
fd0a05ce
JB		 430/**
431 * i40e_fd_handle_status - check the Programming Status for FD
432 * @rx_ring: the Rx ring for this descriptor
433 * @qw: the descriptor data
434 * @prog_id: the id originally used for programming
435 *
436 * This is used to verify if the FD programming or invalidation
437 * requested by SW to the HW is successful or not and take actions accordingly.
438 **/
439static void i40e_fd_handle_status(struct i40e_ring *rx_ring, u32 qw, u8 prog_id)
440{
441 struct pci_dev *pdev = rx_ring->vsi->back->pdev;
442 u32 error;
443
444 error = (qw & I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >>
445 I40E_RX_PROG_STATUS_DESC_QW1_ERROR_SHIFT;
446
447 /* for now just print the Status */
448 dev_info(&pdev->dev, "FD programming id %02x, Status %08x\n",
449 prog_id, error);
450}
451
452/**
a5e9c572 453 * i40e_unmap_and_free_tx_resource - Release a Tx buffer
fd0a05ce
JB		 454 * @ring: the ring that owns the buffer
455 * @tx_buffer: the buffer to free
456 **/
a5e9c572
AD		 457static void i40e_unmap_and_free_tx_resource(struct i40e_ring *ring,
458 struct i40e_tx_buffer *tx_buffer)
fd0a05ce 459{
a5e9c572
AD		 460 if (tx_buffer->skb) {
461 dev_kfree_skb_any(tx_buffer->skb);
462 if (dma_unmap_len(tx_buffer, len))
fd0a05ce 463 dma_unmap_single(ring->dev,
35a1e2ad
AD		 464 dma_unmap_addr(tx_buffer, dma),
465 dma_unmap_len(tx_buffer, len),
fd0a05ce 466 DMA_TO_DEVICE);
a5e9c572
AD		 467 } else if (dma_unmap_len(tx_buffer, len)) {
468 dma_unmap_page(ring->dev,
469 dma_unmap_addr(tx_buffer, dma),
470 dma_unmap_len(tx_buffer, len),
471 DMA_TO_DEVICE);
fd0a05ce 472 }
a5e9c572
AD		 473 tx_buffer->next_to_watch = NULL;
474 tx_buffer->skb = NULL;
35a1e2ad 475 dma_unmap_len_set(tx_buffer, len, 0);
a5e9c572 476 /* tx_buffer must be completely set up in the transmit path */
fd0a05ce
JB		 477}
478
479/**
480 * i40e_clean_tx_ring - Free any empty Tx buffers
481 * @tx_ring: ring to be cleaned
482 **/
483void i40e_clean_tx_ring(struct i40e_ring *tx_ring)
484{
fd0a05ce
JB		 485 unsigned long bi_size;
486 u16 i;
487
488 /* ring already cleared, nothing to do */
489 if (!tx_ring->tx_bi)
490 return;
491
492 /* Free all the Tx ring sk_buffs */
a5e9c572
AD		 493 for (i = 0; i < tx_ring->count; i++)
494 i40e_unmap_and_free_tx_resource(tx_ring, &tx_ring->tx_bi[i]);
fd0a05ce
JB		 495
496 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
497 memset(tx_ring->tx_bi, 0, bi_size);
498
499 /* Zero out the descriptor ring */
500 memset(tx_ring->desc, 0, tx_ring->size);
501
502 tx_ring->next_to_use = 0;
503 tx_ring->next_to_clean = 0;
7070ce0a
AD		 504
505 if (!tx_ring->netdev)
506 return;
507
508 /* cleanup Tx queue statistics */
509 netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev,
510 tx_ring->queue_index));
fd0a05ce
JB		 511}
512
513/**
514 * i40e_free_tx_resources - Free Tx resources per queue
515 * @tx_ring: Tx descriptor ring for a specific queue
516 *
517 * Free all transmit software resources
518 **/
519void i40e_free_tx_resources(struct i40e_ring *tx_ring)
520{
521 i40e_clean_tx_ring(tx_ring);
522 kfree(tx_ring->tx_bi);
523 tx_ring->tx_bi = NULL;
524
525 if (tx_ring->desc) {
526 dma_free_coherent(tx_ring->dev, tx_ring->size,
527 tx_ring->desc, tx_ring->dma);
528 tx_ring->desc = NULL;
529 }
530}
531
532/**
533 * i40e_get_tx_pending - how many tx descriptors not processed
534 * @tx_ring: the ring of descriptors
535 *
536 * Since there is no access to the ring head register
537 * in XL710, we need to use our local copies
538 **/
539static u32 i40e_get_tx_pending(struct i40e_ring *ring)
540{
541 u32 ntu = ((ring->next_to_clean <= ring->next_to_use)
542 ? ring->next_to_use
543 : ring->next_to_use + ring->count);
544 return ntu - ring->next_to_clean;
545}
546
547/**
548 * i40e_check_tx_hang - Is there a hang in the Tx queue
549 * @tx_ring: the ring of descriptors
550 **/
551static bool i40e_check_tx_hang(struct i40e_ring *tx_ring)
552{
553 u32 tx_pending = i40e_get_tx_pending(tx_ring);
554 bool ret = false;
555
556 clear_check_for_tx_hang(tx_ring);
557
558 /* Check for a hung queue, but be thorough. This verifies
559 * that a transmit has been completed since the previous
560 * check AND there is at least one packet pending. The
561 * ARMED bit is set to indicate a potential hang. The
562 * bit is cleared if a pause frame is received to remove
563 * false hang detection due to PFC or 802.3x frames. By
564 * requiring this to fail twice we avoid races with
565 * PFC clearing the ARMED bit and conditions where we
566 * run the check_tx_hang logic with a transmit completion
567 * pending but without time to complete it yet.
568 */
a114d0a6 569 if ((tx_ring->tx_stats.tx_done_old == tx_ring->stats.packets) &&
fd0a05ce
JB		 570 tx_pending) {
571 /* make sure it is true for two checks in a row */
572 ret = test_and_set_bit(__I40E_HANG_CHECK_ARMED,
573 &tx_ring->state);
574 } else {
575 /* update completed stats and disarm the hang check */
a114d0a6 576 tx_ring->tx_stats.tx_done_old = tx_ring->stats.packets;
fd0a05ce
JB		 577 clear_bit(__I40E_HANG_CHECK_ARMED, &tx_ring->state);
578 }
579
580 return ret;
581}
582
583/**
584 * i40e_clean_tx_irq - Reclaim resources after transmit completes
585 * @tx_ring: tx ring to clean
586 * @budget: how many cleans we're allowed
587 *
588 * Returns true if there's any budget left (e.g. the clean is finished)
589 **/
590static bool i40e_clean_tx_irq(struct i40e_ring *tx_ring, int budget)
591{
592 u16 i = tx_ring->next_to_clean;
593 struct i40e_tx_buffer *tx_buf;
594 struct i40e_tx_desc *tx_desc;
595 unsigned int total_packets = 0;
596 unsigned int total_bytes = 0;
597
598 tx_buf = &tx_ring->tx_bi[i];
599 tx_desc = I40E_TX_DESC(tx_ring, i);
a5e9c572 600 i -= tx_ring->count;
fd0a05ce 601
a5e9c572
AD		 602 do {
603 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
fd0a05ce
JB		 604
605 /* if next_to_watch is not set then there is no work pending */
606 if (!eop_desc)
607 break;
608
a5e9c572
AD		 609 /* prevent any other reads prior to eop_desc */
610 read_barrier_depends();
611
fd0a05ce
JB		 612 /* if the descriptor isn't done, no work yet to do */
613 if (!(eop_desc->cmd_type_offset_bsz &
614 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
615 break;
616
c304fdac 617 /* clear next_to_watch to prevent false hangs */
fd0a05ce 618 tx_buf->next_to_watch = NULL;
fd0a05ce 619
a5e9c572
AD		 620 /* update the statistics for this packet */
621 total_bytes += tx_buf->bytecount;
622 total_packets += tx_buf->gso_segs;
fd0a05ce 623
a5e9c572
AD		 624 /* free the skb */
625 dev_kfree_skb_any(tx_buf->skb);
fd0a05ce 626
a5e9c572
AD		 627 /* unmap skb header data */
628 dma_unmap_single(tx_ring->dev,
629 dma_unmap_addr(tx_buf, dma),
630 dma_unmap_len(tx_buf, len),
631 DMA_TO_DEVICE);
fd0a05ce 632
a5e9c572
AD		 633 /* clear tx_buffer data */
634 tx_buf->skb = NULL;
635 dma_unmap_len_set(tx_buf, len, 0);
fd0a05ce 636
a5e9c572
AD		 637 /* unmap remaining buffers */
638 while (tx_desc != eop_desc) {
fd0a05ce
JB		 639
640 tx_buf++;
641 tx_desc++;
642 i++;
a5e9c572
AD		 643 if (unlikely(!i)) {
644 i -= tx_ring->count;
fd0a05ce
JB		 645 tx_buf = tx_ring->tx_bi;
646 tx_desc = I40E_TX_DESC(tx_ring, 0);
647 }
fd0a05ce 648
a5e9c572
AD		 649 /* unmap any remaining paged data */
650 if (dma_unmap_len(tx_buf, len)) {
651 dma_unmap_page(tx_ring->dev,
652 dma_unmap_addr(tx_buf, dma),
653 dma_unmap_len(tx_buf, len),
654 DMA_TO_DEVICE);
655 dma_unmap_len_set(tx_buf, len, 0);
656 }
657 }
658
659 /* move us one more past the eop_desc for start of next pkt */
660 tx_buf++;
661 tx_desc++;
662 i++;
663 if (unlikely(!i)) {
664 i -= tx_ring->count;
665 tx_buf = tx_ring->tx_bi;
666 tx_desc = I40E_TX_DESC(tx_ring, 0);
667 }
668
669 /* update budget accounting */
670 budget--;
671 } while (likely(budget));
672
673 i += tx_ring->count;
fd0a05ce 674 tx_ring->next_to_clean = i;
980e9b11 675 u64_stats_update_begin(&tx_ring->syncp);
a114d0a6
AD		 676 tx_ring->stats.bytes += total_bytes;
677 tx_ring->stats.packets += total_packets;
980e9b11 678 u64_stats_update_end(&tx_ring->syncp);
fd0a05ce
JB		 679 tx_ring->q_vector->tx.total_bytes += total_bytes;
680 tx_ring->q_vector->tx.total_packets += total_packets;
a5e9c572 681
fd0a05ce
JB		 682 if (check_for_tx_hang(tx_ring) && i40e_check_tx_hang(tx_ring)) {
683 /* schedule immediate reset if we believe we hung */
684 dev_info(tx_ring->dev, "Detected Tx Unit Hang\n"
685 " VSI <%d>\n"
686 " Tx Queue <%d>\n"
687 " next_to_use <%x>\n"
688 " next_to_clean <%x>\n",
689 tx_ring->vsi->seid,
690 tx_ring->queue_index,
691 tx_ring->next_to_use, i);
692 dev_info(tx_ring->dev, "tx_bi[next_to_clean]\n"
693 " time_stamp <%lx>\n"
694 " jiffies <%lx>\n",
695 tx_ring->tx_bi[i].time_stamp, jiffies);
696
697 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
698
699 dev_info(tx_ring->dev,
700 "tx hang detected on queue %d, resetting adapter\n",
701 tx_ring->queue_index);
702
703 tx_ring->netdev->netdev_ops->ndo_tx_timeout(tx_ring->netdev);
704
705 /* the adapter is about to reset, no point in enabling stuff */
706 return true;
707 }
708
7070ce0a
AD		 709 netdev_tx_completed_queue(netdev_get_tx_queue(tx_ring->netdev,
710 tx_ring->queue_index),
711 total_packets, total_bytes);
712
fd0a05ce
JB		 713#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
714 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
715 (I40E_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
716 /* Make sure that anybody stopping the queue after this
717 * sees the new next_to_clean.
718 */
719 smp_mb();
720 if (__netif_subqueue_stopped(tx_ring->netdev,
721 tx_ring->queue_index) &&
722 !test_bit(__I40E_DOWN, &tx_ring->vsi->state)) {
723 netif_wake_subqueue(tx_ring->netdev,
724 tx_ring->queue_index);
725 ++tx_ring->tx_stats.restart_queue;
726 }
727 }
728
729 return budget > 0;
730}
731
732/**
733 * i40e_set_new_dynamic_itr - Find new ITR level
734 * @rc: structure containing ring performance data
735 *
736 * Stores a new ITR value based on packets and byte counts during
737 * the last interrupt. The advantage of per interrupt computation
738 * is faster updates and more accurate ITR for the current traffic
739 * pattern. Constants in this function were computed based on
740 * theoretical maximum wire speed and thresholds were set based on
741 * testing data as well as attempting to minimize response time
742 * while increasing bulk throughput.
743 **/
744static void i40e_set_new_dynamic_itr(struct i40e_ring_container *rc)
745{
746 enum i40e_latency_range new_latency_range = rc->latency_range;
747 u32 new_itr = rc->itr;
748 int bytes_per_int;
749
750 if (rc->total_packets == 0 || !rc->itr)
751 return;
752
753 /* simple throttlerate management
754 * 0-10MB/s lowest (100000 ints/s)
755 * 10-20MB/s low (20000 ints/s)
756 * 20-1249MB/s bulk (8000 ints/s)
757 */
758 bytes_per_int = rc->total_bytes / rc->itr;
759 switch (rc->itr) {
760 case I40E_LOWEST_LATENCY:
761 if (bytes_per_int > 10)
762 new_latency_range = I40E_LOW_LATENCY;
763 break;
764 case I40E_LOW_LATENCY:
765 if (bytes_per_int > 20)
766 new_latency_range = I40E_BULK_LATENCY;
767 else if (bytes_per_int <= 10)
768 new_latency_range = I40E_LOWEST_LATENCY;
769 break;
770 case I40E_BULK_LATENCY:
771 if (bytes_per_int <= 20)
772 rc->latency_range = I40E_LOW_LATENCY;
773 break;
774 }
775
776 switch (new_latency_range) {
777 case I40E_LOWEST_LATENCY:
778 new_itr = I40E_ITR_100K;
779 break;
780 case I40E_LOW_LATENCY:
781 new_itr = I40E_ITR_20K;
782 break;
783 case I40E_BULK_LATENCY:
784 new_itr = I40E_ITR_8K;
785 break;
786 default:
787 break;
788 }
789
790 if (new_itr != rc->itr) {
791 /* do an exponential smoothing */
792 new_itr = (10 * new_itr * rc->itr) /
793 ((9 * new_itr) + rc->itr);
794 rc->itr = new_itr & I40E_MAX_ITR;
795 }
796
797 rc->total_bytes = 0;
798 rc->total_packets = 0;
799}
800
801/**
802 * i40e_update_dynamic_itr - Adjust ITR based on bytes per int
803 * @q_vector: the vector to adjust
804 **/
805static void i40e_update_dynamic_itr(struct i40e_q_vector *q_vector)
806{
807 u16 vector = q_vector->vsi->base_vector + q_vector->v_idx;
808 struct i40e_hw *hw = &q_vector->vsi->back->hw;
809 u32 reg_addr;
810 u16 old_itr;
811
812 reg_addr = I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1);
813 old_itr = q_vector->rx.itr;
814 i40e_set_new_dynamic_itr(&q_vector->rx);
815 if (old_itr != q_vector->rx.itr)
816 wr32(hw, reg_addr, q_vector->rx.itr);
817
818 reg_addr = I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1);
819 old_itr = q_vector->tx.itr;
820 i40e_set_new_dynamic_itr(&q_vector->tx);
821 if (old_itr != q_vector->tx.itr)
822 wr32(hw, reg_addr, q_vector->tx.itr);
fd0a05ce
JB		 823}
824
825/**
826 * i40e_clean_programming_status - clean the programming status descriptor
827 * @rx_ring: the rx ring that has this descriptor
828 * @rx_desc: the rx descriptor written back by HW
829 *
830 * Flow director should handle FD_FILTER_STATUS to check its filter programming
831 * status being successful or not and take actions accordingly. FCoE should
832 * handle its context/filter programming/invalidation status and take actions.
833 *
834 **/
835static void i40e_clean_programming_status(struct i40e_ring *rx_ring,
836 union i40e_rx_desc *rx_desc)
837{
838 u64 qw;
839 u8 id;
840
841 qw = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
842 id = (qw & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >>
843 I40E_RX_PROG_STATUS_DESC_QW1_PROGID_SHIFT;
844
845 if (id == I40E_RX_PROG_STATUS_DESC_FD_FILTER_STATUS)
846 i40e_fd_handle_status(rx_ring, qw, id);
847}
848
849/**
850 * i40e_setup_tx_descriptors - Allocate the Tx descriptors
851 * @tx_ring: the tx ring to set up
852 *
853 * Return 0 on success, negative on error
854 **/
855int i40e_setup_tx_descriptors(struct i40e_ring *tx_ring)
856{
857 struct device *dev = tx_ring->dev;
858 int bi_size;
859
860 if (!dev)
861 return -ENOMEM;
862
863 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
864 tx_ring->tx_bi = kzalloc(bi_size, GFP_KERNEL);
865 if (!tx_ring->tx_bi)
866 goto err;
867
868 /* round up to nearest 4K */
869 tx_ring->size = tx_ring->count * sizeof(struct i40e_tx_desc);
870 tx_ring->size = ALIGN(tx_ring->size, 4096);
871 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
872 &tx_ring->dma, GFP_KERNEL);
873 if (!tx_ring->desc) {
874 dev_info(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n",
875 tx_ring->size);
876 goto err;
877 }
878
879 tx_ring->next_to_use = 0;
880 tx_ring->next_to_clean = 0;
881 return 0;
882
883err:
884 kfree(tx_ring->tx_bi);
885 tx_ring->tx_bi = NULL;
886 return -ENOMEM;
887}
888
889/**
890 * i40e_clean_rx_ring - Free Rx buffers
891 * @rx_ring: ring to be cleaned
892 **/
893void i40e_clean_rx_ring(struct i40e_ring *rx_ring)
894{
895 struct device *dev = rx_ring->dev;
896 struct i40e_rx_buffer *rx_bi;
897 unsigned long bi_size;
898 u16 i;
899
900 /* ring already cleared, nothing to do */
901 if (!rx_ring->rx_bi)
902 return;
903
904 /* Free all the Rx ring sk_buffs */
905 for (i = 0; i < rx_ring->count; i++) {
906 rx_bi = &rx_ring->rx_bi[i];
907 if (rx_bi->dma) {
908 dma_unmap_single(dev,
909 rx_bi->dma,
910 rx_ring->rx_buf_len,
911 DMA_FROM_DEVICE);
912 rx_bi->dma = 0;
913 }
914 if (rx_bi->skb) {
915 dev_kfree_skb(rx_bi->skb);
916 rx_bi->skb = NULL;
917 }
918 if (rx_bi->page) {
919 if (rx_bi->page_dma) {
920 dma_unmap_page(dev,
921 rx_bi->page_dma,
922 PAGE_SIZE / 2,
923 DMA_FROM_DEVICE);
924 rx_bi->page_dma = 0;
925 }
926 __free_page(rx_bi->page);
927 rx_bi->page = NULL;
928 rx_bi->page_offset = 0;
929 }
930 }
931
932 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
933 memset(rx_ring->rx_bi, 0, bi_size);
934
935 /* Zero out the descriptor ring */
936 memset(rx_ring->desc, 0, rx_ring->size);
937
938 rx_ring->next_to_clean = 0;
939 rx_ring->next_to_use = 0;
940}
941
942/**
943 * i40e_free_rx_resources - Free Rx resources
944 * @rx_ring: ring to clean the resources from
945 *
946 * Free all receive software resources
947 **/
948void i40e_free_rx_resources(struct i40e_ring *rx_ring)
949{
950 i40e_clean_rx_ring(rx_ring);
951 kfree(rx_ring->rx_bi);
952 rx_ring->rx_bi = NULL;
953
954 if (rx_ring->desc) {
955 dma_free_coherent(rx_ring->dev, rx_ring->size,
956 rx_ring->desc, rx_ring->dma);
957 rx_ring->desc = NULL;
958 }
959}
960
961/**
962 * i40e_setup_rx_descriptors - Allocate Rx descriptors
963 * @rx_ring: Rx descriptor ring (for a specific queue) to setup
964 *
965 * Returns 0 on success, negative on failure
966 **/
967int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring)
968{
969 struct device *dev = rx_ring->dev;
970 int bi_size;
971
972 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
973 rx_ring->rx_bi = kzalloc(bi_size, GFP_KERNEL);
974 if (!rx_ring->rx_bi)
975 goto err;
976
977 /* Round up to nearest 4K */
978 rx_ring->size = ring_is_16byte_desc_enabled(rx_ring)
979 ? rx_ring->count * sizeof(union i40e_16byte_rx_desc)
980 : rx_ring->count * sizeof(union i40e_32byte_rx_desc);
981 rx_ring->size = ALIGN(rx_ring->size, 4096);
982 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
983 &rx_ring->dma, GFP_KERNEL);
984
985 if (!rx_ring->desc) {
986 dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n",
987 rx_ring->size);
988 goto err;
989 }
990
991 rx_ring->next_to_clean = 0;
992 rx_ring->next_to_use = 0;
993
994 return 0;
995err:
996 kfree(rx_ring->rx_bi);
997 rx_ring->rx_bi = NULL;
998 return -ENOMEM;
999}
1000
1001/**
1002 * i40e_release_rx_desc - Store the new tail and head values
1003 * @rx_ring: ring to bump
1004 * @val: new head index
1005 **/
1006static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val)
1007{
1008 rx_ring->next_to_use = val;
1009 /* Force memory writes to complete before letting h/w
1010 * know there are new descriptors to fetch. (Only
1011 * applicable for weak-ordered memory model archs,
1012 * such as IA-64).
1013 */
1014 wmb();
1015 writel(val, rx_ring->tail);
1016}
1017
1018/**
1019 * i40e_alloc_rx_buffers - Replace used receive buffers; packet split
1020 * @rx_ring: ring to place buffers on
1021 * @cleaned_count: number of buffers to replace
1022 **/
1023void i40e_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count)
1024{
1025 u16 i = rx_ring->next_to_use;
1026 union i40e_rx_desc *rx_desc;
1027 struct i40e_rx_buffer *bi;
1028 struct sk_buff *skb;
1029
1030 /* do nothing if no valid netdev defined */
1031 if (!rx_ring->netdev || !cleaned_count)
1032 return;
1033
1034 while (cleaned_count--) {
1035 rx_desc = I40E_RX_DESC(rx_ring, i);
1036 bi = &rx_ring->rx_bi[i];
1037 skb = bi->skb;
1038
1039 if (!skb) {
1040 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
1041 rx_ring->rx_buf_len);
1042 if (!skb) {
420136cc 1043 rx_ring->rx_stats.alloc_buff_failed++;
fd0a05ce
JB		 1044 goto no_buffers;
1045 }
1046 /* initialize queue mapping */
1047 skb_record_rx_queue(skb, rx_ring->queue_index);
1048 bi->skb = skb;
1049 }
1050
1051 if (!bi->dma) {
1052 bi->dma = dma_map_single(rx_ring->dev,
1053 skb->data,
1054 rx_ring->rx_buf_len,
1055 DMA_FROM_DEVICE);
1056 if (dma_mapping_error(rx_ring->dev, bi->dma)) {
420136cc 1057 rx_ring->rx_stats.alloc_buff_failed++;
fd0a05ce
JB		 1058 bi->dma = 0;
1059 goto no_buffers;
1060 }
1061 }
1062
1063 if (ring_is_ps_enabled(rx_ring)) {
1064 if (!bi->page) {
1065 bi->page = alloc_page(GFP_ATOMIC);
1066 if (!bi->page) {
420136cc 1067 rx_ring->rx_stats.alloc_page_failed++;
fd0a05ce
JB		 1068 goto no_buffers;
1069 }
1070 }
1071
1072 if (!bi->page_dma) {
1073 /* use a half page if we're re-using */
1074 bi->page_offset ^= PAGE_SIZE / 2;
1075 bi->page_dma = dma_map_page(rx_ring->dev,
1076 bi->page,
1077 bi->page_offset,
1078 PAGE_SIZE / 2,
1079 DMA_FROM_DEVICE);
1080 if (dma_mapping_error(rx_ring->dev,
1081 bi->page_dma)) {
420136cc 1082 rx_ring->rx_stats.alloc_page_failed++;
fd0a05ce
JB		 1083 bi->page_dma = 0;
1084 goto no_buffers;
1085 }
1086 }
1087
1088 /* Refresh the desc even if buffer_addrs didn't change
1089 * because each write-back erases this info.
1090 */
1091 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
1092 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
1093 } else {
1094 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
1095 rx_desc->read.hdr_addr = 0;
1096 }
1097 i++;
1098 if (i == rx_ring->count)
1099 i = 0;
1100 }
1101
1102no_buffers:
1103 if (rx_ring->next_to_use != i)
1104 i40e_release_rx_desc(rx_ring, i);
1105}
1106
1107/**
1108 * i40e_receive_skb - Send a completed packet up the stack
1109 * @rx_ring: rx ring in play
1110 * @skb: packet to send up
1111 * @vlan_tag: vlan tag for packet
1112 **/
1113static void i40e_receive_skb(struct i40e_ring *rx_ring,
1114 struct sk_buff *skb, u16 vlan_tag)
1115{
1116 struct i40e_q_vector *q_vector = rx_ring->q_vector;
1117 struct i40e_vsi *vsi = rx_ring->vsi;
1118 u64 flags = vsi->back->flags;
1119
1120 if (vlan_tag & VLAN_VID_MASK)
1121 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
1122
1123 if (flags & I40E_FLAG_IN_NETPOLL)
1124 netif_rx(skb);
1125 else
1126 napi_gro_receive(&q_vector->napi, skb);
1127}
1128
1129/**
1130 * i40e_rx_checksum - Indicate in skb if hw indicated a good cksum
1131 * @vsi: the VSI we care about
1132 * @skb: skb currently being received and modified
1133 * @rx_status: status value of last descriptor in packet
1134 * @rx_error: error value of last descriptor in packet
8144f0f7 1135 * @rx_ptype: ptype value of last descriptor in packet
fd0a05ce
JB		 1136 **/
1137static inline void i40e_rx_checksum(struct i40e_vsi *vsi,
1138 struct sk_buff *skb,
1139 u32 rx_status,
8144f0f7
JG		 1140 u32 rx_error,
1141 u16 rx_ptype)
fd0a05ce 1142{
8144f0f7
JG		 1143 bool ipv4_tunnel, ipv6_tunnel;
1144 __wsum rx_udp_csum;
1145 __sum16 csum;
1146 struct iphdr *iph;
1147
1148 ipv4_tunnel = (rx_ptype > I40E_RX_PTYPE_GRENAT4_MAC_PAY3) &&
1149 (rx_ptype < I40E_RX_PTYPE_GRENAT4_MACVLAN_IPV6_ICMP_PAY4);
1150 ipv6_tunnel = (rx_ptype > I40E_RX_PTYPE_GRENAT6_MAC_PAY3) &&
1151 (rx_ptype < I40E_RX_PTYPE_GRENAT6_MACVLAN_IPV6_ICMP_PAY4);
1152
1153 skb->encapsulation = ipv4_tunnel || ipv6_tunnel;
fd0a05ce
JB		 1154 skb->ip_summed = CHECKSUM_NONE;
1155
1156 /* Rx csum enabled and ip headers found? */
1157 if (!(vsi->netdev->features & NETIF_F_RXCSUM &&
1158 rx_status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT)))
1159 return;
1160
ddf1d0d7 1161 /* likely incorrect csum if alternate IP extension headers found */
8ee75a8e
SN		 1162 if (rx_status & (1 << I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT))
1163 return;
1164
8144f0f7 1165 /* IP or L4 or outmost IP checksum error */
fd0a05ce 1166 if (rx_error & ((1 << I40E_RX_DESC_ERROR_IPE_SHIFT) |
8144f0f7
JG		 1167 (1 << I40E_RX_DESC_ERROR_L4E_SHIFT) |
1168 (1 << I40E_RX_DESC_ERROR_EIPE_SHIFT))) {
fd0a05ce
JB		 1169 vsi->back->hw_csum_rx_error++;
1170 return;
1171 }
1172
8144f0f7
JG		 1173 if (ipv4_tunnel &&
1174 !(rx_status & (1 << I40E_RX_DESC_STATUS_UDP_0_SHIFT))) {
1175 /* If VXLAN traffic has an outer UDPv4 checksum we need to check
1176 * it in the driver, hardware does not do it for us.
1177 * Since L3L4P bit was set we assume a valid IHL value (>=5)
1178 * so the total length of IPv4 header is IHL*4 bytes
1179 */
1180 skb->transport_header = skb->mac_header +
1181 sizeof(struct ethhdr) +
1182 (ip_hdr(skb)->ihl * 4);
1183
1184 /* Add 4 bytes for VLAN tagged packets */
1185 skb->transport_header += (skb->protocol == htons(ETH_P_8021Q) ||
1186 skb->protocol == htons(ETH_P_8021AD))
1187 ? VLAN_HLEN : 0;
1188
1189 rx_udp_csum = udp_csum(skb);
1190 iph = ip_hdr(skb);
1191 csum = csum_tcpudp_magic(
1192 iph->saddr, iph->daddr,
1193 (skb->len - skb_transport_offset(skb)),
1194 IPPROTO_UDP, rx_udp_csum);
1195
1196 if (udp_hdr(skb)->check != csum) {
1197 vsi->back->hw_csum_rx_error++;
1198 return;
1199 }
1200 }
1201
fd0a05ce
JB		 1202 skb->ip_summed = CHECKSUM_UNNECESSARY;
1203}
1204
1205/**
1206 * i40e_rx_hash - returns the hash value from the Rx descriptor
1207 * @ring: descriptor ring
1208 * @rx_desc: specific descriptor
1209 **/
1210static inline u32 i40e_rx_hash(struct i40e_ring *ring,
1211 union i40e_rx_desc *rx_desc)
1212{
8a494920
JB		 1213 const __le64 rss_mask =
1214 cpu_to_le64((u64)I40E_RX_DESC_FLTSTAT_RSS_HASH <<
1215 I40E_RX_DESC_STATUS_FLTSTAT_SHIFT);
1216
1217 if ((ring->netdev->features & NETIF_F_RXHASH) &&
1218 (rx_desc->wb.qword1.status_error_len & rss_mask) == rss_mask)
1219 return le32_to_cpu(rx_desc->wb.qword0.hi_dword.rss);
1220 else
1221 return 0;
fd0a05ce
JB		 1222}
1223
206812b5
JB		 1224/**
1225 * i40e_ptype_to_hash - get a hash type
1226 * @ptype: the ptype value from the descriptor
1227 *
1228 * Returns a hash type to be used by skb_set_hash
1229 **/
1230static inline enum pkt_hash_types i40e_ptype_to_hash(u8 ptype)
1231{
1232 struct i40e_rx_ptype_decoded decoded = decode_rx_desc_ptype(ptype);
1233
1234 if (!decoded.known)
1235 return PKT_HASH_TYPE_NONE;
1236
1237 if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1238 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY4)
1239 return PKT_HASH_TYPE_L4;
1240 else if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1241 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY3)
1242 return PKT_HASH_TYPE_L3;
1243 else
1244 return PKT_HASH_TYPE_L2;
1245}
1246
fd0a05ce
JB		 1247/**
1248 * i40e_clean_rx_irq - Reclaim resources after receive completes
1249 * @rx_ring: rx ring to clean
1250 * @budget: how many cleans we're allowed
1251 *
1252 * Returns true if there's any budget left (e.g. the clean is finished)
1253 **/
1254static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
1255{
1256 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1257 u16 rx_packet_len, rx_header_len, rx_sph, rx_hbo;
1258 u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);
1259 const int current_node = numa_node_id();
1260 struct i40e_vsi *vsi = rx_ring->vsi;
1261 u16 i = rx_ring->next_to_clean;
1262 union i40e_rx_desc *rx_desc;
1263 u32 rx_error, rx_status;
206812b5 1264 u8 rx_ptype;
fd0a05ce
JB		 1265 u64 qword;
1266
1267 rx_desc = I40E_RX_DESC(rx_ring, i);
1268 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
6838b535
JB		 1269 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >>
1270 I40E_RXD_QW1_STATUS_SHIFT;
fd0a05ce
JB		 1271
1272 while (rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) {
1273 union i40e_rx_desc *next_rxd;
1274 struct i40e_rx_buffer *rx_bi;
1275 struct sk_buff *skb;
1276 u16 vlan_tag;
1277 if (i40e_rx_is_programming_status(qword)) {
1278 i40e_clean_programming_status(rx_ring, rx_desc);
1279 I40E_RX_NEXT_DESC_PREFETCH(rx_ring, i, next_rxd);
1280 goto next_desc;
1281 }
1282 rx_bi = &rx_ring->rx_bi[i];
1283 skb = rx_bi->skb;
1284 prefetch(skb->data);
1285
829af3ac
MW		 1286 rx_packet_len = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
1287 I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
1288 rx_header_len = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK) >>
1289 I40E_RXD_QW1_LENGTH_HBUF_SHIFT;
1290 rx_sph = (qword & I40E_RXD_QW1_LENGTH_SPH_MASK) >>
1291 I40E_RXD_QW1_LENGTH_SPH_SHIFT;
1292
1293 rx_error = (qword & I40E_RXD_QW1_ERROR_MASK) >>
1294 I40E_RXD_QW1_ERROR_SHIFT;
fd0a05ce
JB		 1295 rx_hbo = rx_error & (1 << I40E_RX_DESC_ERROR_HBO_SHIFT);
1296 rx_error &= ~(1 << I40E_RX_DESC_ERROR_HBO_SHIFT);
1297
8144f0f7
JG		 1298 rx_ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >>
1299 I40E_RXD_QW1_PTYPE_SHIFT;
fd0a05ce
JB		 1300 rx_bi->skb = NULL;
1301
1302 /* This memory barrier is needed to keep us from reading
1303 * any other fields out of the rx_desc until we know the
1304 * STATUS_DD bit is set
1305 */
1306 rmb();
1307
1308 /* Get the header and possibly the whole packet
1309 * If this is an skb from previous receive dma will be 0
1310 */
1311 if (rx_bi->dma) {
1312 u16 len;
1313
1314 if (rx_hbo)
1315 len = I40E_RX_HDR_SIZE;
1316 else if (rx_sph)
1317 len = rx_header_len;
1318 else if (rx_packet_len)
1319 len = rx_packet_len; /* 1buf/no split found */
1320 else
1321 len = rx_header_len; /* split always mode */
1322
1323 skb_put(skb, len);
1324 dma_unmap_single(rx_ring->dev,
1325 rx_bi->dma,
1326 rx_ring->rx_buf_len,
1327 DMA_FROM_DEVICE);
1328 rx_bi->dma = 0;
1329 }
1330
1331 /* Get the rest of the data if this was a header split */
1332 if (ring_is_ps_enabled(rx_ring) && rx_packet_len) {
1333
1334 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
1335 rx_bi->page,
1336 rx_bi->page_offset,
1337 rx_packet_len);
1338
1339 skb->len += rx_packet_len;
1340 skb->data_len += rx_packet_len;
1341 skb->truesize += rx_packet_len;
1342
1343 if ((page_count(rx_bi->page) == 1) &&
1344 (page_to_nid(rx_bi->page) == current_node))
1345 get_page(rx_bi->page);
1346 else
1347 rx_bi->page = NULL;
1348
1349 dma_unmap_page(rx_ring->dev,
1350 rx_bi->page_dma,
1351 PAGE_SIZE / 2,
1352 DMA_FROM_DEVICE);
1353 rx_bi->page_dma = 0;
1354 }
1355 I40E_RX_NEXT_DESC_PREFETCH(rx_ring, i, next_rxd);
1356
1357 if (unlikely(
1358 !(rx_status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)))) {
1359 struct i40e_rx_buffer *next_buffer;
1360
1361 next_buffer = &rx_ring->rx_bi[i];
1362
1363 if (ring_is_ps_enabled(rx_ring)) {
1364 rx_bi->skb = next_buffer->skb;
1365 rx_bi->dma = next_buffer->dma;
1366 next_buffer->skb = skb;
1367 next_buffer->dma = 0;
1368 }
1369 rx_ring->rx_stats.non_eop_descs++;
1370 goto next_desc;
1371 }
1372
1373 /* ERR_MASK will only have valid bits if EOP set */
1374 if (unlikely(rx_error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) {
1375 dev_kfree_skb_any(skb);
1376 goto next_desc;
1377 }
1378
206812b5
JB		 1379 skb_set_hash(skb, i40e_rx_hash(rx_ring, rx_desc),
1380 i40e_ptype_to_hash(rx_ptype));
beb0dff1
JK		 1381 if (unlikely(rx_status & I40E_RXD_QW1_STATUS_TSYNVALID_MASK)) {
1382 i40e_ptp_rx_hwtstamp(vsi->back, skb, (rx_status &
1383 I40E_RXD_QW1_STATUS_TSYNINDX_MASK) >>
1384 I40E_RXD_QW1_STATUS_TSYNINDX_SHIFT);
1385 rx_ring->last_rx_timestamp = jiffies;
1386 }
1387
fd0a05ce
JB		 1388 /* probably a little skewed due to removing CRC */
1389 total_rx_bytes += skb->len;
1390 total_rx_packets++;
1391
1392 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
8144f0f7
JG		 1393
1394 i40e_rx_checksum(vsi, skb, rx_status, rx_error, rx_ptype);
1395
fd0a05ce
JB		 1396 vlan_tag = rx_status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)
1397 ? le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1)
1398 : 0;
1399 i40e_receive_skb(rx_ring, skb, vlan_tag);
1400
1401 rx_ring->netdev->last_rx = jiffies;
1402 budget--;
1403next_desc:
1404 rx_desc->wb.qword1.status_error_len = 0;
1405 if (!budget)
1406 break;
1407
1408 cleaned_count++;
1409 /* return some buffers to hardware, one at a time is too slow */
1410 if (cleaned_count >= I40E_RX_BUFFER_WRITE) {
1411 i40e_alloc_rx_buffers(rx_ring, cleaned_count);
1412 cleaned_count = 0;
1413 }
1414
1415 /* use prefetched values */
1416 rx_desc = next_rxd;
1417 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
829af3ac
MW		 1418 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >>
1419 I40E_RXD_QW1_STATUS_SHIFT;
fd0a05ce
JB		 1420 }
1421
1422 rx_ring->next_to_clean = i;
980e9b11 1423 u64_stats_update_begin(&rx_ring->syncp);
a114d0a6
AD		 1424 rx_ring->stats.packets += total_rx_packets;
1425 rx_ring->stats.bytes += total_rx_bytes;
980e9b11 1426 u64_stats_update_end(&rx_ring->syncp);
fd0a05ce
JB		 1427 rx_ring->q_vector->rx.total_packets += total_rx_packets;
1428 rx_ring->q_vector->rx.total_bytes += total_rx_bytes;
1429
1430 if (cleaned_count)
1431 i40e_alloc_rx_buffers(rx_ring, cleaned_count);
1432
1433 return budget > 0;
1434}
1435
1436/**
1437 * i40e_napi_poll - NAPI polling Rx/Tx cleanup routine
1438 * @napi: napi struct with our devices info in it
1439 * @budget: amount of work driver is allowed to do this pass, in packets
1440 *
1441 * This function will clean all queues associated with a q_vector.
1442 *
1443 * Returns the amount of work done
1444 **/
1445int i40e_napi_poll(struct napi_struct *napi, int budget)
1446{
1447 struct i40e_q_vector *q_vector =
1448 container_of(napi, struct i40e_q_vector, napi);
1449 struct i40e_vsi *vsi = q_vector->vsi;
cd0b6fa6 1450 struct i40e_ring *ring;
fd0a05ce
JB		 1451 bool clean_complete = true;
1452 int budget_per_ring;
fd0a05ce
JB		 1453
1454 if (test_bit(__I40E_DOWN, &vsi->state)) {
1455 napi_complete(napi);
1456 return 0;
1457 }
1458
cd0b6fa6
AD		 1459 /* Since the actual Tx work is minimal, we can give the Tx a larger
1460 * budget and be more aggressive about cleaning up the Tx descriptors.
1461 */
1462 i40e_for_each_ring(ring, q_vector->tx)
1463 clean_complete &= i40e_clean_tx_irq(ring, vsi->work_limit);
1464
fd0a05ce
JB		 1465 /* We attempt to distribute budget to each Rx queue fairly, but don't
1466 * allow the budget to go below 1 because that would exit polling early.
fd0a05ce
JB		 1467 */
1468 budget_per_ring = max(budget/q_vector->num_ringpairs, 1);
cd0b6fa6
AD		 1469
1470 i40e_for_each_ring(ring, q_vector->rx)
1471 clean_complete &= i40e_clean_rx_irq(ring, budget_per_ring);
fd0a05ce
JB		 1472
1473 /* If work not completed, return budget and polling will return */
1474 if (!clean_complete)
1475 return budget;
1476
1477 /* Work is done so exit the polling mode and re-enable the interrupt */
1478 napi_complete(napi);
1479 if (ITR_IS_DYNAMIC(vsi->rx_itr_setting) ||
1480 ITR_IS_DYNAMIC(vsi->tx_itr_setting))
1481 i40e_update_dynamic_itr(q_vector);
1482
1483 if (!test_bit(__I40E_DOWN, &vsi->state)) {
1484 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) {
1485 i40e_irq_dynamic_enable(vsi,
1486 q_vector->v_idx + vsi->base_vector);
1487 } else {
1488 struct i40e_hw *hw = &vsi->back->hw;
1489 /* We re-enable the queue 0 cause, but
1490 * don't worry about dynamic_enable
1491 * because we left it on for the other
1492 * possible interrupts during napi
1493 */
1494 u32 qval = rd32(hw, I40E_QINT_RQCTL(0));
1495 qval |= I40E_QINT_RQCTL_CAUSE_ENA_MASK;
1496 wr32(hw, I40E_QINT_RQCTL(0), qval);
1497
1498 qval = rd32(hw, I40E_QINT_TQCTL(0));
1499 qval |= I40E_QINT_TQCTL_CAUSE_ENA_MASK;
1500 wr32(hw, I40E_QINT_TQCTL(0), qval);
116a57d4
SN		 1501
1502 i40e_irq_dynamic_enable_icr0(vsi->back);
fd0a05ce
JB		 1503 }
1504 }
1505
1506 return 0;
1507}
1508
1509/**
1510 * i40e_atr - Add a Flow Director ATR filter
1511 * @tx_ring: ring to add programming descriptor to
1512 * @skb: send buffer
1513 * @flags: send flags
1514 * @protocol: wire protocol
1515 **/
1516static void i40e_atr(struct i40e_ring *tx_ring, struct sk_buff *skb,
1517 u32 flags, __be16 protocol)
1518{
1519 struct i40e_filter_program_desc *fdir_desc;
1520 struct i40e_pf *pf = tx_ring->vsi->back;
1521 union {
1522 unsigned char *network;
1523 struct iphdr *ipv4;
1524 struct ipv6hdr *ipv6;
1525 } hdr;
1526 struct tcphdr *th;
1527 unsigned int hlen;
1528 u32 flex_ptype, dtype_cmd;
fc4ac67b 1529 u16 i;
fd0a05ce
JB		 1530
1531 /* make sure ATR is enabled */
60ea5f83 1532 if (!(pf->flags & I40E_FLAG_FD_ATR_ENABLED))
fd0a05ce
JB		 1533 return;
1534
1535 /* if sampling is disabled do nothing */
1536 if (!tx_ring->atr_sample_rate)
1537 return;
1538
1539 tx_ring->atr_count++;
1540
1541 /* snag network header to get L4 type and address */
1542 hdr.network = skb_network_header(skb);
1543
1544 /* Currently only IPv4/IPv6 with TCP is supported */
1545 if (protocol == htons(ETH_P_IP)) {
1546 if (hdr.ipv4->protocol != IPPROTO_TCP)
1547 return;
1548
1549 /* access ihl as a u8 to avoid unaligned access on ia64 */
1550 hlen = (hdr.network[0] & 0x0F) << 2;
1551 } else if (protocol == htons(ETH_P_IPV6)) {
1552 if (hdr.ipv6->nexthdr != IPPROTO_TCP)
1553 return;
1554
1555 hlen = sizeof(struct ipv6hdr);
1556 } else {
1557 return;
1558 }
1559
1560 th = (struct tcphdr *)(hdr.network + hlen);
1561
1562 /* sample on all syn/fin packets or once every atr sample rate */
1563 if (!th->fin && !th->syn && (tx_ring->atr_count < tx_ring->atr_sample_rate))
1564 return;
1565
1566 tx_ring->atr_count = 0;
1567
1568 /* grab the next descriptor */
fc4ac67b
AD		 1569 i = tx_ring->next_to_use;
1570 fdir_desc = I40E_TX_FDIRDESC(tx_ring, i);
1571
1572 i++;
1573 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
fd0a05ce
JB		 1574
1575 flex_ptype = (tx_ring->queue_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
1576 I40E_TXD_FLTR_QW0_QINDEX_MASK;
1577 flex_ptype |= (protocol == htons(ETH_P_IP)) ?
1578 (I40E_FILTER_PCTYPE_NONF_IPV4_TCP <<
1579 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) :
1580 (I40E_FILTER_PCTYPE_NONF_IPV6_TCP <<
1581 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT);
1582
1583 flex_ptype |= tx_ring->vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
1584
1585 dtype_cmd = I40E_TX_DESC_DTYPE_FILTER_PROG;
1586
1587 dtype_cmd |= th->fin ?
1588 (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
1589 I40E_TXD_FLTR_QW1_PCMD_SHIFT) :
1590 (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
1591 I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1592
1593 dtype_cmd |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX <<
1594 I40E_TXD_FLTR_QW1_DEST_SHIFT;
1595
1596 dtype_cmd |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID <<
1597 I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT;
1598
1599 fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(flex_ptype);
1600 fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dtype_cmd);
1601}
1602
fd0a05ce
JB		 1603/**
1604 * i40e_tx_prepare_vlan_flags - prepare generic TX VLAN tagging flags for HW
1605 * @skb: send buffer
1606 * @tx_ring: ring to send buffer on
1607 * @flags: the tx flags to be set
1608 *
1609 * Checks the skb and set up correspondingly several generic transmit flags
1610 * related to VLAN tagging for the HW, such as VLAN, DCB, etc.
1611 *
1612 * Returns error code indicate the frame should be dropped upon error and the
1613 * otherwise returns 0 to indicate the flags has been set properly.
1614 **/
1615static int i40e_tx_prepare_vlan_flags(struct sk_buff *skb,
1616 struct i40e_ring *tx_ring,
1617 u32 *flags)
1618{
1619 __be16 protocol = skb->protocol;
1620 u32 tx_flags = 0;
1621
1622 /* if we have a HW VLAN tag being added, default to the HW one */
1623 if (vlan_tx_tag_present(skb)) {
1624 tx_flags |= vlan_tx_tag_get(skb) << I40E_TX_FLAGS_VLAN_SHIFT;
1625 tx_flags |= I40E_TX_FLAGS_HW_VLAN;
1626 /* else if it is a SW VLAN, check the next protocol and store the tag */
0e2fe46c 1627 } else if (protocol == htons(ETH_P_8021Q)) {
fd0a05ce
JB		 1628 struct vlan_hdr *vhdr, _vhdr;
1629 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr);
1630 if (!vhdr)
1631 return -EINVAL;
1632
1633 protocol = vhdr->h_vlan_encapsulated_proto;
1634 tx_flags |= ntohs(vhdr->h_vlan_TCI) << I40E_TX_FLAGS_VLAN_SHIFT;
1635 tx_flags |= I40E_TX_FLAGS_SW_VLAN;
1636 }
1637
1638 /* Insert 802.1p priority into VLAN header */
1639 if ((tx_ring->vsi->back->flags & I40E_FLAG_DCB_ENABLED) &&
1640 ((tx_flags & (I40E_TX_FLAGS_HW_VLAN | I40E_TX_FLAGS_SW_VLAN)) ||
1641 (skb->priority != TC_PRIO_CONTROL))) {
1642 tx_flags &= ~I40E_TX_FLAGS_VLAN_PRIO_MASK;
1643 tx_flags |= (skb->priority & 0x7) <<
1644 I40E_TX_FLAGS_VLAN_PRIO_SHIFT;
1645 if (tx_flags & I40E_TX_FLAGS_SW_VLAN) {
1646 struct vlan_ethhdr *vhdr;
1647 if (skb_header_cloned(skb) &&
1648 pskb_expand_head(skb, 0, 0, GFP_ATOMIC))
1649 return -ENOMEM;
1650 vhdr = (struct vlan_ethhdr *)skb->data;
1651 vhdr->h_vlan_TCI = htons(tx_flags >>
1652 I40E_TX_FLAGS_VLAN_SHIFT);
1653 } else {
1654 tx_flags |= I40E_TX_FLAGS_HW_VLAN;
1655 }
1656 }
1657 *flags = tx_flags;
1658 return 0;
1659}
1660
fd0a05ce
JB		 1661/**
1662 * i40e_tso - set up the tso context descriptor
1663 * @tx_ring: ptr to the ring to send
1664 * @skb: ptr to the skb we're sending
1665 * @tx_flags: the collected send information
1666 * @protocol: the send protocol
1667 * @hdr_len: ptr to the size of the packet header
1668 * @cd_tunneling: ptr to context descriptor bits
1669 *
1670 * Returns 0 if no TSO can happen, 1 if tso is going, or error
1671 **/
1672static int i40e_tso(struct i40e_ring *tx_ring, struct sk_buff *skb,
1673 u32 tx_flags, __be16 protocol, u8 *hdr_len,
1674 u64 *cd_type_cmd_tso_mss, u32 *cd_tunneling)
1675{
1676 u32 cd_cmd, cd_tso_len, cd_mss;
1677 struct tcphdr *tcph;
1678 struct iphdr *iph;
1679 u32 l4len;
1680 int err;
1681 struct ipv6hdr *ipv6h;
1682
1683 if (!skb_is_gso(skb))
1684 return 0;
1685
1686 if (skb_header_cloned(skb)) {
1687 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
1688 if (err)
1689 return err;
1690 }
1691
0e2fe46c 1692 if (protocol == htons(ETH_P_IP)) {
fd0a05ce
JB		 1693 iph = skb->encapsulation ? inner_ip_hdr(skb) : ip_hdr(skb);
1694 tcph = skb->encapsulation ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1695 iph->tot_len = 0;
1696 iph->check = 0;
1697 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
1698 0, IPPROTO_TCP, 0);
1699 } else if (skb_is_gso_v6(skb)) {
1700
1701 ipv6h = skb->encapsulation ? inner_ipv6_hdr(skb)
1702 : ipv6_hdr(skb);
1703 tcph = skb->encapsulation ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1704 ipv6h->payload_len = 0;
1705 tcph->check = ~csum_ipv6_magic(&ipv6h->saddr, &ipv6h->daddr,
1706 0, IPPROTO_TCP, 0);
1707 }
1708
1709 l4len = skb->encapsulation ? inner_tcp_hdrlen(skb) : tcp_hdrlen(skb);
1710 *hdr_len = (skb->encapsulation
1711 ? (skb_inner_transport_header(skb) - skb->data)
1712 : skb_transport_offset(skb)) + l4len;
1713
1714 /* find the field values */
1715 cd_cmd = I40E_TX_CTX_DESC_TSO;
1716 cd_tso_len = skb->len - *hdr_len;
1717 cd_mss = skb_shinfo(skb)->gso_size;
829af3ac
MW		 1718 *cd_type_cmd_tso_mss |= ((u64)cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) |
1719 ((u64)cd_tso_len <<
1720 I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) |
1721 ((u64)cd_mss << I40E_TXD_CTX_QW1_MSS_SHIFT);
fd0a05ce
JB		 1722 return 1;
1723}
1724
beb0dff1
JK		 1725/**
1726 * i40e_tsyn - set up the tsyn context descriptor
1727 * @tx_ring: ptr to the ring to send
1728 * @skb: ptr to the skb we're sending
1729 * @tx_flags: the collected send information
1730 *
1731 * Returns 0 if no Tx timestamp can happen and 1 if the timestamp will happen
1732 **/
1733static int i40e_tsyn(struct i40e_ring *tx_ring, struct sk_buff *skb,
1734 u32 tx_flags, u64 *cd_type_cmd_tso_mss)
1735{
1736 struct i40e_pf *pf;
1737
1738 if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)))
1739 return 0;
1740
1741 /* Tx timestamps cannot be sampled when doing TSO */
1742 if (tx_flags & I40E_TX_FLAGS_TSO)
1743 return 0;
1744
1745 /* only timestamp the outbound packet if the user has requested it and
1746 * we are not already transmitting a packet to be timestamped
1747 */
1748 pf = i40e_netdev_to_pf(tx_ring->netdev);
1749 if (pf->ptp_tx && !pf->ptp_tx_skb) {
1750 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1751 pf->ptp_tx_skb = skb_get(skb);
1752 } else {
1753 return 0;
1754 }
1755
1756 *cd_type_cmd_tso_mss |= (u64)I40E_TX_CTX_DESC_TSYN <<
1757 I40E_TXD_CTX_QW1_CMD_SHIFT;
1758
1759 pf->ptp_tx_start = jiffies;
1760 schedule_work(&pf->ptp_tx_work);
1761
1762 return 1;
1763}
1764
fd0a05ce
JB		 1765/**
1766 * i40e_tx_enable_csum - Enable Tx checksum offloads
1767 * @skb: send buffer
1768 * @tx_flags: Tx flags currently set
1769 * @td_cmd: Tx descriptor command bits to set
1770 * @td_offset: Tx descriptor header offsets to set
1771 * @cd_tunneling: ptr to context desc bits
1772 **/
1773static void i40e_tx_enable_csum(struct sk_buff *skb, u32 tx_flags,
1774 u32 *td_cmd, u32 *td_offset,
1775 struct i40e_ring *tx_ring,
1776 u32 *cd_tunneling)
1777{
1778 struct ipv6hdr *this_ipv6_hdr;
1779 unsigned int this_tcp_hdrlen;
1780 struct iphdr *this_ip_hdr;
1781 u32 network_hdr_len;
1782 u8 l4_hdr = 0;
1783
1784 if (skb->encapsulation) {
1785 network_hdr_len = skb_inner_network_header_len(skb);
1786 this_ip_hdr = inner_ip_hdr(skb);
1787 this_ipv6_hdr = inner_ipv6_hdr(skb);
1788 this_tcp_hdrlen = inner_tcp_hdrlen(skb);
1789
1790 if (tx_flags & I40E_TX_FLAGS_IPV4) {
1791
1792 if (tx_flags & I40E_TX_FLAGS_TSO) {
1793 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV4;
1794 ip_hdr(skb)->check = 0;
1795 } else {
1796 *cd_tunneling |=
1797 I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;
1798 }
1799 } else if (tx_flags & I40E_TX_FLAGS_IPV6) {
1800 if (tx_flags & I40E_TX_FLAGS_TSO) {
1801 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV6;
1802 ip_hdr(skb)->check = 0;
1803 } else {
1804 *cd_tunneling |=
1805 I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;
1806 }
1807 }
1808
1809 /* Now set the ctx descriptor fields */
1810 *cd_tunneling |= (skb_network_header_len(skb) >> 2) <<
1811 I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT |
1812 I40E_TXD_CTX_UDP_TUNNELING |
1813 ((skb_inner_network_offset(skb) -
1814 skb_transport_offset(skb)) >> 1) <<
1815 I40E_TXD_CTX_QW0_NATLEN_SHIFT;
1816
1817 } else {
1818 network_hdr_len = skb_network_header_len(skb);
1819 this_ip_hdr = ip_hdr(skb);
1820 this_ipv6_hdr = ipv6_hdr(skb);
1821 this_tcp_hdrlen = tcp_hdrlen(skb);
1822 }
1823
1824 /* Enable IP checksum offloads */
1825 if (tx_flags & I40E_TX_FLAGS_IPV4) {
1826 l4_hdr = this_ip_hdr->protocol;
1827 /* the stack computes the IP header already, the only time we
1828 * need the hardware to recompute it is in the case of TSO.
1829 */
1830 if (tx_flags & I40E_TX_FLAGS_TSO) {
1831 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM;
1832 this_ip_hdr->check = 0;
1833 } else {
1834 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4;
1835 }
1836 /* Now set the td_offset for IP header length */
1837 *td_offset = (network_hdr_len >> 2) <<
1838 I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
1839 } else if (tx_flags & I40E_TX_FLAGS_IPV6) {
1840 l4_hdr = this_ipv6_hdr->nexthdr;
1841 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV6;
1842 /* Now set the td_offset for IP header length */
1843 *td_offset = (network_hdr_len >> 2) <<
1844 I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
1845 }
1846 /* words in MACLEN + dwords in IPLEN + dwords in L4Len */
1847 *td_offset |= (skb_network_offset(skb) >> 1) <<
1848 I40E_TX_DESC_LENGTH_MACLEN_SHIFT;
1849
1850 /* Enable L4 checksum offloads */
1851 switch (l4_hdr) {
1852 case IPPROTO_TCP:
1853 /* enable checksum offloads */
1854 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
1855 *td_offset |= (this_tcp_hdrlen >> 2) <<
1856 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1857 break;
1858 case IPPROTO_SCTP:
1859 /* enable SCTP checksum offload */
1860 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP;
1861 *td_offset |= (sizeof(struct sctphdr) >> 2) <<
1862 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1863 break;
1864 case IPPROTO_UDP:
1865 /* enable UDP checksum offload */
1866 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP;
1867 *td_offset |= (sizeof(struct udphdr) >> 2) <<
1868 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1869 break;
1870 default:
1871 break;
1872 }
1873}
1874
1875/**
1876 * i40e_create_tx_ctx Build the Tx context descriptor
1877 * @tx_ring: ring to create the descriptor on
1878 * @cd_type_cmd_tso_mss: Quad Word 1
1879 * @cd_tunneling: Quad Word 0 - bits 0-31
1880 * @cd_l2tag2: Quad Word 0 - bits 32-63
1881 **/
1882static void i40e_create_tx_ctx(struct i40e_ring *tx_ring,
1883 const u64 cd_type_cmd_tso_mss,
1884 const u32 cd_tunneling, const u32 cd_l2tag2)
1885{
1886 struct i40e_tx_context_desc *context_desc;
fc4ac67b 1887 int i = tx_ring->next_to_use;
fd0a05ce
JB		 1888
1889 if (!cd_type_cmd_tso_mss && !cd_tunneling && !cd_l2tag2)
1890 return;
1891
1892 /* grab the next descriptor */
fc4ac67b
AD		 1893 context_desc = I40E_TX_CTXTDESC(tx_ring, i);
1894
1895 i++;
1896 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
fd0a05ce
JB		 1897
1898 /* cpu_to_le32 and assign to struct fields */
1899 context_desc->tunneling_params = cpu_to_le32(cd_tunneling);
1900 context_desc->l2tag2 = cpu_to_le16(cd_l2tag2);
1901 context_desc->type_cmd_tso_mss = cpu_to_le64(cd_type_cmd_tso_mss);
1902}
1903
1904/**
1905 * i40e_tx_map - Build the Tx descriptor
1906 * @tx_ring: ring to send buffer on
1907 * @skb: send buffer
1908 * @first: first buffer info buffer to use
1909 * @tx_flags: collected send information
1910 * @hdr_len: size of the packet header
1911 * @td_cmd: the command field in the descriptor
1912 * @td_offset: offset for checksum or crc
1913 **/
1914static void i40e_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb,
1915 struct i40e_tx_buffer *first, u32 tx_flags,
1916 const u8 hdr_len, u32 td_cmd, u32 td_offset)
1917{
fd0a05ce
JB		 1918 unsigned int data_len = skb->data_len;
1919 unsigned int size = skb_headlen(skb);
a5e9c572 1920 struct skb_frag_struct *frag;
fd0a05ce
JB		 1921 struct i40e_tx_buffer *tx_bi;
1922 struct i40e_tx_desc *tx_desc;
a5e9c572 1923 u16 i = tx_ring->next_to_use;
fd0a05ce
JB		 1924 u32 td_tag = 0;
1925 dma_addr_t dma;
1926 u16 gso_segs;
1927
fd0a05ce
JB		 1928 if (tx_flags & I40E_TX_FLAGS_HW_VLAN) {
1929 td_cmd |= I40E_TX_DESC_CMD_IL2TAG1;
1930 td_tag = (tx_flags & I40E_TX_FLAGS_VLAN_MASK) >>
1931 I40E_TX_FLAGS_VLAN_SHIFT;
1932 }
1933
a5e9c572
AD		 1934 if (tx_flags & (I40E_TX_FLAGS_TSO | I40E_TX_FLAGS_FSO))
1935 gso_segs = skb_shinfo(skb)->gso_segs;
1936 else
1937 gso_segs = 1;
1938
1939 /* multiply data chunks by size of headers */
1940 first->bytecount = skb->len - hdr_len + (gso_segs * hdr_len);
1941 first->gso_segs = gso_segs;
1942 first->skb = skb;
1943 first->tx_flags = tx_flags;
1944
1945 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
1946
fd0a05ce 1947 tx_desc = I40E_TX_DESC(tx_ring, i);
a5e9c572
AD		 1948 tx_bi = first;
1949
1950 for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
1951 if (dma_mapping_error(tx_ring->dev, dma))
1952 goto dma_error;
1953
1954 /* record length, and DMA address */
1955 dma_unmap_len_set(tx_bi, len, size);
1956 dma_unmap_addr_set(tx_bi, dma, dma);
1957
1958 tx_desc->buffer_addr = cpu_to_le64(dma);
1959
1960 while (unlikely(size > I40E_MAX_DATA_PER_TXD)) {
fd0a05ce
JB		 1961 tx_desc->cmd_type_offset_bsz =
1962 build_ctob(td_cmd, td_offset,
1963 I40E_MAX_DATA_PER_TXD, td_tag);
1964
fd0a05ce
JB		 1965 tx_desc++;
1966 i++;
1967 if (i == tx_ring->count) {
1968 tx_desc = I40E_TX_DESC(tx_ring, 0);
1969 i = 0;
1970 }
fd0a05ce 1971
a5e9c572
AD		 1972 dma += I40E_MAX_DATA_PER_TXD;
1973 size -= I40E_MAX_DATA_PER_TXD;
fd0a05ce 1974
a5e9c572
AD		 1975 tx_desc->buffer_addr = cpu_to_le64(dma);
1976 }
fd0a05ce
JB		 1977
1978 if (likely(!data_len))
1979 break;
1980
a5e9c572
AD		 1981 tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset,
1982 size, td_tag);
fd0a05ce
JB		 1983
1984 tx_desc++;
1985 i++;
1986 if (i == tx_ring->count) {
1987 tx_desc = I40E_TX_DESC(tx_ring, 0);
1988 i = 0;
1989 }
1990
a5e9c572
AD		 1991 size = skb_frag_size(frag);
1992 data_len -= size;
fd0a05ce 1993
a5e9c572
AD		 1994 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
1995 DMA_TO_DEVICE);
fd0a05ce 1996
a5e9c572
AD		 1997 tx_bi = &tx_ring->tx_bi[i];
1998 }
fd0a05ce 1999
a5e9c572
AD		 2000 tx_desc->cmd_type_offset_bsz =
2001 build_ctob(td_cmd, td_offset, size, td_tag) |
2002 cpu_to_le64((u64)I40E_TXD_CMD << I40E_TXD_QW1_CMD_SHIFT);
fd0a05ce 2003
7070ce0a
AD		 2004 netdev_tx_sent_queue(netdev_get_tx_queue(tx_ring->netdev,
2005 tx_ring->queue_index),
2006 first->bytecount);
2007
a5e9c572 2008 /* set the timestamp */
fd0a05ce 2009 first->time_stamp = jiffies;
fd0a05ce
JB		 2010
2011 /* Force memory writes to complete before letting h/w
2012 * know there are new descriptors to fetch. (Only
2013 * applicable for weak-ordered memory model archs,
2014 * such as IA-64).
2015 */
2016 wmb();
2017
a5e9c572
AD		 2018 /* set next_to_watch value indicating a packet is present */
2019 first->next_to_watch = tx_desc;
2020
2021 i++;
2022 if (i == tx_ring->count)
2023 i = 0;
2024
2025 tx_ring->next_to_use = i;
2026
2027 /* notify HW of packet */
fd0a05ce 2028 writel(i, tx_ring->tail);
a5e9c572 2029
fd0a05ce
JB		 2030 return;
2031
2032dma_error:
a5e9c572 2033 dev_info(tx_ring->dev, "TX DMA map failed\n");
fd0a05ce
JB		 2034
2035 /* clear dma mappings for failed tx_bi map */
2036 for (;;) {
2037 tx_bi = &tx_ring->tx_bi[i];
a5e9c572 2038 i40e_unmap_and_free_tx_resource(tx_ring, tx_bi);
fd0a05ce
JB		 2039 if (tx_bi == first)
2040 break;
2041 if (i == 0)
2042 i = tx_ring->count;
2043 i--;
2044 }
2045
fd0a05ce
JB		 2046 tx_ring->next_to_use = i;
2047}
2048
2049/**
2050 * __i40e_maybe_stop_tx - 2nd level check for tx stop conditions
2051 * @tx_ring: the ring to be checked
2052 * @size: the size buffer we want to assure is available
2053 *
2054 * Returns -EBUSY if a stop is needed, else 0
2055 **/
2056static inline int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
2057{
2058 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
8e9dca53 2059 /* Memory barrier before checking head and tail */
fd0a05ce
JB		 2060 smp_mb();
2061
2062 /* Check again in a case another CPU has just made room available. */
2063 if (likely(I40E_DESC_UNUSED(tx_ring) < size))
2064 return -EBUSY;
2065
2066 /* A reprieve! - use start_queue because it doesn't call schedule */
2067 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
2068 ++tx_ring->tx_stats.restart_queue;
2069 return 0;
2070}
2071
2072/**
2073 * i40e_maybe_stop_tx - 1st level check for tx stop conditions
2074 * @tx_ring: the ring to be checked
2075 * @size: the size buffer we want to assure is available
2076 *
2077 * Returns 0 if stop is not needed
2078 **/
2079static int i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
2080{
2081 if (likely(I40E_DESC_UNUSED(tx_ring) >= size))
2082 return 0;
2083 return __i40e_maybe_stop_tx(tx_ring, size);
2084}
2085
2086/**
2087 * i40e_xmit_descriptor_count - calculate number of tx descriptors needed
2088 * @skb: send buffer
2089 * @tx_ring: ring to send buffer on
2090 *
2091 * Returns number of data descriptors needed for this skb. Returns 0 to indicate
2092 * there is not enough descriptors available in this ring since we need at least
2093 * one descriptor.
2094 **/
2095static int i40e_xmit_descriptor_count(struct sk_buff *skb,
2096 struct i40e_ring *tx_ring)
2097{
2098#if PAGE_SIZE > I40E_MAX_DATA_PER_TXD
2099 unsigned int f;
2100#endif
2101 int count = 0;
2102
2103 /* need: 1 descriptor per page * PAGE_SIZE/I40E_MAX_DATA_PER_TXD,
2104 * + 1 desc for skb_head_len/I40E_MAX_DATA_PER_TXD,
2105 * + 2 desc gap to keep tail from touching head,
2106 * + 1 desc for context descriptor,
2107 * otherwise try next time
2108 */
2109#if PAGE_SIZE > I40E_MAX_DATA_PER_TXD
2110 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
2111 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
2112#else
2113 count += skb_shinfo(skb)->nr_frags;
2114#endif
2115 count += TXD_USE_COUNT(skb_headlen(skb));
2116 if (i40e_maybe_stop_tx(tx_ring, count + 3)) {
2117 tx_ring->tx_stats.tx_busy++;
2118 return 0;
2119 }
2120 return count;
2121}
2122
2123/**
2124 * i40e_xmit_frame_ring - Sends buffer on Tx ring
2125 * @skb: send buffer
2126 * @tx_ring: ring to send buffer on
2127 *
2128 * Returns NETDEV_TX_OK if sent, else an error code
2129 **/
2130static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb,
2131 struct i40e_ring *tx_ring)
2132{
2133 u64 cd_type_cmd_tso_mss = I40E_TX_DESC_DTYPE_CONTEXT;
2134 u32 cd_tunneling = 0, cd_l2tag2 = 0;
2135 struct i40e_tx_buffer *first;
2136 u32 td_offset = 0;
2137 u32 tx_flags = 0;
2138 __be16 protocol;
2139 u32 td_cmd = 0;
2140 u8 hdr_len = 0;
beb0dff1 2141 int tsyn;
fd0a05ce
JB		 2142 int tso;
2143 if (0 == i40e_xmit_descriptor_count(skb, tx_ring))
2144 return NETDEV_TX_BUSY;
2145
2146 /* prepare the xmit flags */
2147 if (i40e_tx_prepare_vlan_flags(skb, tx_ring, &tx_flags))
2148 goto out_drop;
2149
2150 /* obtain protocol of skb */
2151 protocol = skb->protocol;
2152
2153 /* record the location of the first descriptor for this packet */
2154 first = &tx_ring->tx_bi[tx_ring->next_to_use];
2155
2156 /* setup IPv4/IPv6 offloads */
0e2fe46c 2157 if (protocol == htons(ETH_P_IP))
fd0a05ce 2158 tx_flags |= I40E_TX_FLAGS_IPV4;
0e2fe46c 2159 else if (protocol == htons(ETH_P_IPV6))
fd0a05ce
JB		 2160 tx_flags |= I40E_TX_FLAGS_IPV6;
2161
2162 tso = i40e_tso(tx_ring, skb, tx_flags, protocol, &hdr_len,
2163 &cd_type_cmd_tso_mss, &cd_tunneling);
2164
2165 if (tso < 0)
2166 goto out_drop;
2167 else if (tso)
2168 tx_flags |= I40E_TX_FLAGS_TSO;
2169
2170 skb_tx_timestamp(skb);
2171
beb0dff1
JK		 2172 tsyn = i40e_tsyn(tx_ring, skb, tx_flags, &cd_type_cmd_tso_mss);
2173
2174 if (tsyn)
2175 tx_flags |= I40E_TX_FLAGS_TSYN;
2176
b1941306
AD		 2177 /* always enable CRC insertion offload */
2178 td_cmd |= I40E_TX_DESC_CMD_ICRC;
2179
fd0a05ce 2180 /* Always offload the checksum, since it's in the data descriptor */
b1941306 2181 if (skb->ip_summed == CHECKSUM_PARTIAL) {
fd0a05ce
JB		 2182 tx_flags |= I40E_TX_FLAGS_CSUM;
2183
fd0a05ce
JB		 2184 i40e_tx_enable_csum(skb, tx_flags, &td_cmd, &td_offset,
2185 tx_ring, &cd_tunneling);
b1941306 2186 }
fd0a05ce
JB		 2187
2188 i40e_create_tx_ctx(tx_ring, cd_type_cmd_tso_mss,
2189 cd_tunneling, cd_l2tag2);
2190
2191 /* Add Flow Director ATR if it's enabled.
2192 *
2193 * NOTE: this must always be directly before the data descriptor.
2194 */
2195 i40e_atr(tx_ring, skb, tx_flags, protocol);
2196
2197 i40e_tx_map(tx_ring, skb, first, tx_flags, hdr_len,
2198 td_cmd, td_offset);
2199
2200 i40e_maybe_stop_tx(tx_ring, DESC_NEEDED);
2201
2202 return NETDEV_TX_OK;
2203
2204out_drop:
2205 dev_kfree_skb_any(skb);
2206 return NETDEV_TX_OK;
2207}
2208
2209/**
2210 * i40e_lan_xmit_frame - Selects the correct VSI and Tx queue to send buffer
2211 * @skb: send buffer
2212 * @netdev: network interface device structure
2213 *
2214 * Returns NETDEV_TX_OK if sent, else an error code
2215 **/
2216netdev_tx_t i40e_lan_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
2217{
2218 struct i40e_netdev_priv *np = netdev_priv(netdev);
2219 struct i40e_vsi *vsi = np->vsi;
9f65e15b 2220 struct i40e_ring *tx_ring = vsi->tx_rings[skb->queue_mapping];
fd0a05ce
JB		 2221
2222 /* hardware can't handle really short frames, hardware padding works
2223 * beyond this point
2224 */
2225 if (unlikely(skb->len < I40E_MIN_TX_LEN)) {
2226 if (skb_pad(skb, I40E_MIN_TX_LEN - skb->len))
2227 return NETDEV_TX_OK;
2228 skb->len = I40E_MIN_TX_LEN;
2229 skb_set_tail_pointer(skb, I40E_MIN_TX_LEN);
2230 }
2231
2232 return i40e_xmit_frame_ring(skb, tx_ring);
2233}
Linux 6.x block layer and io_uring tree(s)