projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
i40e: Add ndo_get_phys_port_id() callback support
[linux-2.6-block.git] / drivers / net / ethernet / intel / i40e / i40e_txrx.c
CommitLineData
fd0a05ce
JB		 1/*******************************************************************************
2 *
3 * Intel Ethernet Controller XL710 Family Linux Driver
dc641b73 4 * Copyright(c) 2013 - 2014 Intel Corporation.
fd0a05ce
JB		 5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * more details.
14 *
dc641b73
GR		 15 * You should have received a copy of the GNU General Public License along
16 * with this program. If not, see <http://www.gnu.org/licenses/>.
fd0a05ce
JB		 17 *
18 * The full GNU General Public License is included in this distribution in
19 * the file called "COPYING".
20 *
21 * Contact Information:
22 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
23 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
24 *
25 ******************************************************************************/
26
1c112a64 27#include <linux/prefetch.h>
fd0a05ce 28#include "i40e.h"
206812b5 29#include "i40e_prototype.h"
fd0a05ce
JB		 30
31static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size,
32 u32 td_tag)
33{
34 return cpu_to_le64(I40E_TX_DESC_DTYPE_DATA |
35 ((u64)td_cmd << I40E_TXD_QW1_CMD_SHIFT) |
36 ((u64)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) |
37 ((u64)size << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) |
38 ((u64)td_tag << I40E_TXD_QW1_L2TAG1_SHIFT));
39}
40
eaefbd06 41#define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS)
49d7d933 42#define I40E_FD_CLEAN_DELAY 10
fd0a05ce
JB		 43/**
44 * i40e_program_fdir_filter - Program a Flow Director filter
17a73f6b
JG		 45 * @fdir_data: Packet data that will be filter parameters
46 * @raw_packet: the pre-allocated packet buffer for FDir
fd0a05ce
JB		 47 * @pf: The pf pointer
48 * @add: True for add/update, False for remove
49 **/
17a73f6b 50int i40e_program_fdir_filter(struct i40e_fdir_filter *fdir_data, u8 *raw_packet,
fd0a05ce
JB		 51 struct i40e_pf *pf, bool add)
52{
53 struct i40e_filter_program_desc *fdir_desc;
49d7d933 54 struct i40e_tx_buffer *tx_buf, *first;
fd0a05ce
JB		 55 struct i40e_tx_desc *tx_desc;
56 struct i40e_ring *tx_ring;
eaefbd06 57 unsigned int fpt, dcc;
fd0a05ce
JB		 58 struct i40e_vsi *vsi;
59 struct device *dev;
60 dma_addr_t dma;
61 u32 td_cmd = 0;
49d7d933 62 u16 delay = 0;
fd0a05ce
JB		 63 u16 i;
64
65 /* find existing FDIR VSI */
66 vsi = NULL;
505682cd 67 for (i = 0; i < pf->num_alloc_vsi; i++)
fd0a05ce
JB		 68 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR)
69 vsi = pf->vsi[i];
70 if (!vsi)
71 return -ENOENT;
72
9f65e15b 73 tx_ring = vsi->tx_rings[0];
fd0a05ce
JB		 74 dev = tx_ring->dev;
75
49d7d933
ASJ		 76 /* we need two descriptors to add/del a filter and we can wait */
77 do {
78 if (I40E_DESC_UNUSED(tx_ring) > 1)
79 break;
80 msleep_interruptible(1);
81 delay++;
82 } while (delay < I40E_FD_CLEAN_DELAY);
83
84 if (!(I40E_DESC_UNUSED(tx_ring) > 1))
85 return -EAGAIN;
86
17a73f6b
JG		 87 dma = dma_map_single(dev, raw_packet,
88 I40E_FDIR_MAX_RAW_PACKET_SIZE, DMA_TO_DEVICE);
fd0a05ce
JB		 89 if (dma_mapping_error(dev, dma))
90 goto dma_fail;
91
92 /* grab the next descriptor */
fc4ac67b
AD		 93 i = tx_ring->next_to_use;
94 fdir_desc = I40E_TX_FDIRDESC(tx_ring, i);
49d7d933
ASJ		 95 first = &tx_ring->tx_bi[i];
96 memset(first, 0, sizeof(struct i40e_tx_buffer));
fc4ac67b 97
49d7d933 98 tx_ring->next_to_use = ((i + 1) < tx_ring->count) ? i + 1 : 0;
fd0a05ce 99
eaefbd06
JB		 100 fpt = (fdir_data->q_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
101 I40E_TXD_FLTR_QW0_QINDEX_MASK;
fd0a05ce 102
eaefbd06
JB		 103 fpt |= (fdir_data->flex_off << I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT) &
104 I40E_TXD_FLTR_QW0_FLEXOFF_MASK;
fd0a05ce 105
eaefbd06
JB		 106 fpt |= (fdir_data->pctype << I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) &
107 I40E_TXD_FLTR_QW0_PCTYPE_MASK;
fd0a05ce
JB		 108
109 /* Use LAN VSI Id if not programmed by user */
110 if (fdir_data->dest_vsi == 0)
eaefbd06
JB		 111 fpt |= (pf->vsi[pf->lan_vsi]->id) <<
112 I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
fd0a05ce 113 else
eaefbd06
JB		 114 fpt |= ((u32)fdir_data->dest_vsi <<
115 I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT) &
116 I40E_TXD_FLTR_QW0_DEST_VSI_MASK;
117
eaefbd06 118 dcc = I40E_TX_DESC_DTYPE_FILTER_PROG;
fd0a05ce
JB		 119
120 if (add)
eaefbd06
JB		 121 dcc |= I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
122 I40E_TXD_FLTR_QW1_PCMD_SHIFT;
fd0a05ce 123 else
eaefbd06
JB		 124 dcc |= I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
125 I40E_TXD_FLTR_QW1_PCMD_SHIFT;
fd0a05ce 126
eaefbd06
JB		 127 dcc |= (fdir_data->dest_ctl << I40E_TXD_FLTR_QW1_DEST_SHIFT) &
128 I40E_TXD_FLTR_QW1_DEST_MASK;
fd0a05ce 129
eaefbd06
JB		 130 dcc |= (fdir_data->fd_status << I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT) &
131 I40E_TXD_FLTR_QW1_FD_STATUS_MASK;
fd0a05ce
JB		 132
133 if (fdir_data->cnt_index != 0) {
eaefbd06
JB		 134 dcc |= I40E_TXD_FLTR_QW1_CNT_ENA_MASK;
135 dcc |= ((u32)fdir_data->cnt_index <<
136 I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) &
433c47de 137 I40E_TXD_FLTR_QW1_CNTINDEX_MASK;
fd0a05ce
JB		 138 }
139
99753ea6
JB		 140 fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(fpt);
141 fdir_desc->rsvd = cpu_to_le32(0);
eaefbd06 142 fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dcc);
fd0a05ce
JB		 143 fdir_desc->fd_id = cpu_to_le32(fdir_data->fd_id);
144
145 /* Now program a dummy descriptor */
fc4ac67b
AD		 146 i = tx_ring->next_to_use;
147 tx_desc = I40E_TX_DESC(tx_ring, i);
298deef1 148 tx_buf = &tx_ring->tx_bi[i];
fc4ac67b 149
49d7d933
ASJ		 150 tx_ring->next_to_use = ((i + 1) < tx_ring->count) ? i + 1 : 0;
151
152 memset(tx_buf, 0, sizeof(struct i40e_tx_buffer));
fd0a05ce 153
298deef1 154 /* record length, and DMA address */
17a73f6b 155 dma_unmap_len_set(tx_buf, len, I40E_FDIR_MAX_RAW_PACKET_SIZE);
298deef1
ASJ		 156 dma_unmap_addr_set(tx_buf, dma, dma);
157
fd0a05ce 158 tx_desc->buffer_addr = cpu_to_le64(dma);
eaefbd06 159 td_cmd = I40E_TXD_CMD | I40E_TX_DESC_CMD_DUMMY;
fd0a05ce 160
49d7d933
ASJ		 161 tx_buf->tx_flags = I40E_TX_FLAGS_FD_SB;
162 tx_buf->raw_buf = (void *)raw_packet;
163
fd0a05ce 164 tx_desc->cmd_type_offset_bsz =
17a73f6b 165 build_ctob(td_cmd, 0, I40E_FDIR_MAX_RAW_PACKET_SIZE, 0);
fd0a05ce 166
298deef1
ASJ		 167 /* set the timestamp */
168 tx_buf->time_stamp = jiffies;
169
fd0a05ce 170 /* Force memory writes to complete before letting h/w
49d7d933 171 * know there are new descriptors to fetch.
fd0a05ce
JB		 172 */
173 wmb();
174
fc4ac67b 175 /* Mark the data descriptor to be watched */
49d7d933 176 first->next_to_watch = tx_desc;
fc4ac67b 177
fd0a05ce
JB		 178 writel(tx_ring->next_to_use, tx_ring->tail);
179 return 0;
180
181dma_fail:
182 return -1;
183}
184
17a73f6b
JG		 185#define IP_HEADER_OFFSET 14
186#define I40E_UDPIP_DUMMY_PACKET_LEN 42
187/**
188 * i40e_add_del_fdir_udpv4 - Add/Remove UDPv4 filters
189 * @vsi: pointer to the targeted VSI
190 * @fd_data: the flow director data required for the FDir descriptor
17a73f6b
JG		 191 * @add: true adds a filter, false removes it
192 *
193 * Returns 0 if the filters were successfully added or removed
194 **/
195static int i40e_add_del_fdir_udpv4(struct i40e_vsi *vsi,
196 struct i40e_fdir_filter *fd_data,
49d7d933 197 bool add)
17a73f6b
JG		 198{
199 struct i40e_pf *pf = vsi->back;
200 struct udphdr *udp;
201 struct iphdr *ip;
202 bool err = false;
49d7d933 203 u8 *raw_packet;
17a73f6b 204 int ret;
17a73f6b
JG		 205 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0,
206 0x45, 0, 0, 0x1c, 0, 0, 0x40, 0, 0x40, 0x11, 0, 0, 0, 0, 0, 0,
207 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
208
49d7d933
ASJ		 209 raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL);
210 if (!raw_packet)
211 return -ENOMEM;
17a73f6b
JG		 212 memcpy(raw_packet, packet, I40E_UDPIP_DUMMY_PACKET_LEN);
213
214 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET);
215 udp = (struct udphdr *)(raw_packet + IP_HEADER_OFFSET
216 + sizeof(struct iphdr));
217
218 ip->daddr = fd_data->dst_ip[0];
219 udp->dest = fd_data->dst_port;
220 ip->saddr = fd_data->src_ip[0];
221 udp->source = fd_data->src_port;
222
b2d36c03
KS		 223 fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_UDP;
224 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
225 if (ret) {
226 dev_info(&pf->pdev->dev,
227 "Filter command send failed for PCTYPE %d (ret = %d)\n",
228 fd_data->pctype, ret);
229 err = true;
230 } else {
231 dev_info(&pf->pdev->dev,
232 "Filter OK for PCTYPE %d (ret = %d)\n",
233 fd_data->pctype, ret);
17a73f6b
JG		 234 }
235
236 return err ? -EOPNOTSUPP : 0;
237}
238
239#define I40E_TCPIP_DUMMY_PACKET_LEN 54
240/**
241 * i40e_add_del_fdir_tcpv4 - Add/Remove TCPv4 filters
242 * @vsi: pointer to the targeted VSI
243 * @fd_data: the flow director data required for the FDir descriptor
17a73f6b
JG		 244 * @add: true adds a filter, false removes it
245 *
246 * Returns 0 if the filters were successfully added or removed
247 **/
248static int i40e_add_del_fdir_tcpv4(struct i40e_vsi *vsi,
249 struct i40e_fdir_filter *fd_data,
49d7d933 250 bool add)
17a73f6b
JG		 251{
252 struct i40e_pf *pf = vsi->back;
253 struct tcphdr *tcp;
254 struct iphdr *ip;
255 bool err = false;
49d7d933 256 u8 *raw_packet;
17a73f6b
JG		 257 int ret;
258 /* Dummy packet */
259 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0,
260 0x45, 0, 0, 0x28, 0, 0, 0x40, 0, 0x40, 0x6, 0, 0, 0, 0, 0, 0,
261 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x80, 0x11,
262 0x0, 0x72, 0, 0, 0, 0};
263
49d7d933
ASJ		 264 raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL);
265 if (!raw_packet)
266 return -ENOMEM;
17a73f6b
JG		 267 memcpy(raw_packet, packet, I40E_TCPIP_DUMMY_PACKET_LEN);
268
269 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET);
270 tcp = (struct tcphdr *)(raw_packet + IP_HEADER_OFFSET
271 + sizeof(struct iphdr));
272
273 ip->daddr = fd_data->dst_ip[0];
274 tcp->dest = fd_data->dst_port;
275 ip->saddr = fd_data->src_ip[0];
276 tcp->source = fd_data->src_port;
277
278 if (add) {
279 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) {
280 dev_info(&pf->pdev->dev, "Forcing ATR off, sideband rules for TCP/IPv4 flow being applied\n");
281 pf->flags &= ~I40E_FLAG_FD_ATR_ENABLED;
282 }
283 }
284
b2d36c03 285 fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_TCP;
17a73f6b
JG		 286 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
287
288 if (ret) {
289 dev_info(&pf->pdev->dev,
290 "Filter command send failed for PCTYPE %d (ret = %d)\n",
291 fd_data->pctype, ret);
292 err = true;
293 } else {
294 dev_info(&pf->pdev->dev, "Filter OK for PCTYPE %d (ret = %d)\n",
295 fd_data->pctype, ret);
296 }
297
17a73f6b
JG		 298 return err ? -EOPNOTSUPP : 0;
299}
300
301/**
302 * i40e_add_del_fdir_sctpv4 - Add/Remove SCTPv4 Flow Director filters for
303 * a specific flow spec
304 * @vsi: pointer to the targeted VSI
305 * @fd_data: the flow director data required for the FDir descriptor
306 * @raw_packet: the pre-allocated packet buffer for FDir
307 * @add: true adds a filter, false removes it
308 *
21d3efdc 309 * Always returns -EOPNOTSUPP
17a73f6b
JG		 310 **/
311static int i40e_add_del_fdir_sctpv4(struct i40e_vsi *vsi,
312 struct i40e_fdir_filter *fd_data,
49d7d933 313 bool add)
17a73f6b
JG		 314{
315 return -EOPNOTSUPP;
316}
317
318#define I40E_IP_DUMMY_PACKET_LEN 34
319/**
320 * i40e_add_del_fdir_ipv4 - Add/Remove IPv4 Flow Director filters for
321 * a specific flow spec
322 * @vsi: pointer to the targeted VSI
323 * @fd_data: the flow director data required for the FDir descriptor
17a73f6b
JG		 324 * @add: true adds a filter, false removes it
325 *
326 * Returns 0 if the filters were successfully added or removed
327 **/
328static int i40e_add_del_fdir_ipv4(struct i40e_vsi *vsi,
329 struct i40e_fdir_filter *fd_data,
49d7d933 330 bool add)
17a73f6b
JG		 331{
332 struct i40e_pf *pf = vsi->back;
333 struct iphdr *ip;
334 bool err = false;
49d7d933 335 u8 *raw_packet;
17a73f6b
JG		 336 int ret;
337 int i;
338 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0,
339 0x45, 0, 0, 0x14, 0, 0, 0x40, 0, 0x40, 0x10, 0, 0, 0, 0, 0, 0,
340 0, 0, 0, 0};
341
17a73f6b
JG		 342 for (i = I40E_FILTER_PCTYPE_NONF_IPV4_OTHER;
343 i <= I40E_FILTER_PCTYPE_FRAG_IPV4; i++) {
49d7d933
ASJ		 344 raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL);
345 if (!raw_packet)
346 return -ENOMEM;
347 memcpy(raw_packet, packet, I40E_IP_DUMMY_PACKET_LEN);
348 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET);
349
350 ip->saddr = fd_data->src_ip[0];
351 ip->daddr = fd_data->dst_ip[0];
352 ip->protocol = 0;
353
17a73f6b
JG		 354 fd_data->pctype = i;
355 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
356
357 if (ret) {
358 dev_info(&pf->pdev->dev,
359 "Filter command send failed for PCTYPE %d (ret = %d)\n",
360 fd_data->pctype, ret);
361 err = true;
362 } else {
363 dev_info(&pf->pdev->dev,
364 "Filter OK for PCTYPE %d (ret = %d)\n",
365 fd_data->pctype, ret);
366 }
367 }
368
369 return err ? -EOPNOTSUPP : 0;
370}
371
372/**
373 * i40e_add_del_fdir - Build raw packets to add/del fdir filter
374 * @vsi: pointer to the targeted VSI
375 * @cmd: command to get or set RX flow classification rules
376 * @add: true adds a filter, false removes it
377 *
378 **/
379int i40e_add_del_fdir(struct i40e_vsi *vsi,
380 struct i40e_fdir_filter *input, bool add)
381{
382 struct i40e_pf *pf = vsi->back;
17a73f6b
JG		 383 int ret;
384
17a73f6b
JG		 385 switch (input->flow_type & ~FLOW_EXT) {
386 case TCP_V4_FLOW:
49d7d933 387 ret = i40e_add_del_fdir_tcpv4(vsi, input, add);
17a73f6b
JG		 388 break;
389 case UDP_V4_FLOW:
49d7d933 390 ret = i40e_add_del_fdir_udpv4(vsi, input, add);
17a73f6b
JG		 391 break;
392 case SCTP_V4_FLOW:
49d7d933 393 ret = i40e_add_del_fdir_sctpv4(vsi, input, add);
17a73f6b
JG		 394 break;
395 case IPV4_FLOW:
49d7d933 396 ret = i40e_add_del_fdir_ipv4(vsi, input, add);
17a73f6b
JG		 397 break;
398 case IP_USER_FLOW:
399 switch (input->ip4_proto) {
400 case IPPROTO_TCP:
49d7d933 401 ret = i40e_add_del_fdir_tcpv4(vsi, input, add);
17a73f6b
JG		 402 break;
403 case IPPROTO_UDP:
49d7d933 404 ret = i40e_add_del_fdir_udpv4(vsi, input, add);
17a73f6b
JG		 405 break;
406 case IPPROTO_SCTP:
49d7d933 407 ret = i40e_add_del_fdir_sctpv4(vsi, input, add);
17a73f6b
JG		 408 break;
409 default:
49d7d933 410 ret = i40e_add_del_fdir_ipv4(vsi, input, add);
17a73f6b
JG		 411 break;
412 }
413 break;
414 default:
c5ffe7e1 415 dev_info(&pf->pdev->dev, "Could not specify spec type %d\n",
17a73f6b
JG		 416 input->flow_type);
417 ret = -EINVAL;
418 }
419
49d7d933 420 /* The buffer allocated here is freed by the i40e_clean_tx_ring() */
17a73f6b
JG		 421 return ret;
422}
423
fd0a05ce
JB		 424/**
425 * i40e_fd_handle_status - check the Programming Status for FD
426 * @rx_ring: the Rx ring for this descriptor
55a5e60b 427 * @rx_desc: the Rx descriptor for programming Status, not a packet descriptor.
fd0a05ce
JB		 428 * @prog_id: the id originally used for programming
429 *
430 * This is used to verify if the FD programming or invalidation
431 * requested by SW to the HW is successful or not and take actions accordingly.
432 **/
55a5e60b
ASJ		 433static void i40e_fd_handle_status(struct i40e_ring *rx_ring,
434 union i40e_rx_desc *rx_desc, u8 prog_id)
fd0a05ce 435{
55a5e60b
ASJ		 436 struct i40e_pf *pf = rx_ring->vsi->back;
437 struct pci_dev *pdev = pf->pdev;
438 u32 fcnt_prog, fcnt_avail;
fd0a05ce 439 u32 error;
55a5e60b 440 u64 qw;
fd0a05ce 441
55a5e60b 442 qw = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
fd0a05ce
JB		 443 error = (qw & I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >>
444 I40E_RX_PROG_STATUS_DESC_QW1_ERROR_SHIFT;
445
55a5e60b
ASJ		 446 if (error == (0x1 << I40E_RX_PROG_STATUS_DESC_FD_TBL_FULL_SHIFT)) {
447 dev_warn(&pdev->dev, "ntuple filter loc = %d, could not be added\n",
448 rx_desc->wb.qword0.hi_dword.fd_id);
449
450 /* filter programming failed most likely due to table full */
12957388
ASJ		 451 fcnt_prog = i40e_get_cur_guaranteed_fd_count(pf);
452 fcnt_avail = pf->fdir_pf_filter_count;
55a5e60b
ASJ		 453 /* If ATR is running fcnt_prog can quickly change,
454 * if we are very close to full, it makes sense to disable
455 * FD ATR/SB and then re-enable it when there is room.
456 */
457 if (fcnt_prog >= (fcnt_avail - I40E_FDIR_BUFFER_FULL_MARGIN)) {
458 /* Turn off ATR first */
b814ba65
ASJ		 459 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
460 !(pf->auto_disable_flags &
461 I40E_FLAG_FD_ATR_ENABLED)) {
55a5e60b
ASJ		 462 dev_warn(&pdev->dev, "FD filter space full, ATR for further flows will be turned off\n");
463 pf->auto_disable_flags |=
464 I40E_FLAG_FD_ATR_ENABLED;
465 pf->flags |= I40E_FLAG_FDIR_REQUIRES_REINIT;
b814ba65
ASJ		 466 } else if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
467 !(pf->auto_disable_flags &
468 I40E_FLAG_FD_SB_ENABLED)) {
55a5e60b
ASJ		 469 dev_warn(&pdev->dev, "FD filter space full, new ntuple rules will not be added\n");
470 pf->auto_disable_flags |=
471 I40E_FLAG_FD_SB_ENABLED;
472 pf->flags |= I40E_FLAG_FDIR_REQUIRES_REINIT;
473 }
474 } else {
c5ffe7e1 475 dev_info(&pdev->dev, "FD filter programming error\n");
55a5e60b
ASJ		 476 }
477 } else if (error ==
478 (0x1 << I40E_RX_PROG_STATUS_DESC_NO_FD_ENTRY_SHIFT)) {
13c2884f
ASJ		 479 if (I40E_DEBUG_FD & pf->hw.debug_mask)
480 dev_info(&pdev->dev, "ntuple filter loc = %d, could not be removed\n",
481 rx_desc->wb.qword0.hi_dword.fd_id);
55a5e60b 482 }
fd0a05ce
JB		 483}
484
485/**
a5e9c572 486 * i40e_unmap_and_free_tx_resource - Release a Tx buffer
fd0a05ce
JB		 487 * @ring: the ring that owns the buffer
488 * @tx_buffer: the buffer to free
489 **/
a5e9c572
AD		 490static void i40e_unmap_and_free_tx_resource(struct i40e_ring *ring,
491 struct i40e_tx_buffer *tx_buffer)
fd0a05ce 492{
a5e9c572 493 if (tx_buffer->skb) {
49d7d933
ASJ		 494 if (tx_buffer->tx_flags & I40E_TX_FLAGS_FD_SB)
495 kfree(tx_buffer->raw_buf);
496 else
497 dev_kfree_skb_any(tx_buffer->skb);
498
a5e9c572 499 if (dma_unmap_len(tx_buffer, len))
fd0a05ce 500 dma_unmap_single(ring->dev,
35a1e2ad
AD		 501 dma_unmap_addr(tx_buffer, dma),
502 dma_unmap_len(tx_buffer, len),
fd0a05ce 503 DMA_TO_DEVICE);
a5e9c572
AD		 504 } else if (dma_unmap_len(tx_buffer, len)) {
505 dma_unmap_page(ring->dev,
506 dma_unmap_addr(tx_buffer, dma),
507 dma_unmap_len(tx_buffer, len),
508 DMA_TO_DEVICE);
fd0a05ce 509 }
a5e9c572
AD		 510 tx_buffer->next_to_watch = NULL;
511 tx_buffer->skb = NULL;
35a1e2ad 512 dma_unmap_len_set(tx_buffer, len, 0);
a5e9c572 513 /* tx_buffer must be completely set up in the transmit path */
fd0a05ce
JB		 514}
515
516/**
517 * i40e_clean_tx_ring - Free any empty Tx buffers
518 * @tx_ring: ring to be cleaned
519 **/
520void i40e_clean_tx_ring(struct i40e_ring *tx_ring)
521{
fd0a05ce
JB		 522 unsigned long bi_size;
523 u16 i;
524
525 /* ring already cleared, nothing to do */
526 if (!tx_ring->tx_bi)
527 return;
528
529 /* Free all the Tx ring sk_buffs */
a5e9c572
AD		 530 for (i = 0; i < tx_ring->count; i++)
531 i40e_unmap_and_free_tx_resource(tx_ring, &tx_ring->tx_bi[i]);
fd0a05ce
JB		 532
533 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
534 memset(tx_ring->tx_bi, 0, bi_size);
535
536 /* Zero out the descriptor ring */
537 memset(tx_ring->desc, 0, tx_ring->size);
538
539 tx_ring->next_to_use = 0;
540 tx_ring->next_to_clean = 0;
7070ce0a
AD		 541
542 if (!tx_ring->netdev)
543 return;
544
545 /* cleanup Tx queue statistics */
546 netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev,
547 tx_ring->queue_index));
fd0a05ce
JB		 548}
549
550/**
551 * i40e_free_tx_resources - Free Tx resources per queue
552 * @tx_ring: Tx descriptor ring for a specific queue
553 *
554 * Free all transmit software resources
555 **/
556void i40e_free_tx_resources(struct i40e_ring *tx_ring)
557{
558 i40e_clean_tx_ring(tx_ring);
559 kfree(tx_ring->tx_bi);
560 tx_ring->tx_bi = NULL;
561
562 if (tx_ring->desc) {
563 dma_free_coherent(tx_ring->dev, tx_ring->size,
564 tx_ring->desc, tx_ring->dma);
565 tx_ring->desc = NULL;
566 }
567}
568
569/**
570 * i40e_get_tx_pending - how many tx descriptors not processed
571 * @tx_ring: the ring of descriptors
572 *
573 * Since there is no access to the ring head register
574 * in XL710, we need to use our local copies
575 **/
576static u32 i40e_get_tx_pending(struct i40e_ring *ring)
577{
578 u32 ntu = ((ring->next_to_clean <= ring->next_to_use)
579 ? ring->next_to_use
580 : ring->next_to_use + ring->count);
581 return ntu - ring->next_to_clean;
582}
583
584/**
585 * i40e_check_tx_hang - Is there a hang in the Tx queue
586 * @tx_ring: the ring of descriptors
587 **/
588static bool i40e_check_tx_hang(struct i40e_ring *tx_ring)
589{
590 u32 tx_pending = i40e_get_tx_pending(tx_ring);
591 bool ret = false;
592
593 clear_check_for_tx_hang(tx_ring);
594
595 /* Check for a hung queue, but be thorough. This verifies
596 * that a transmit has been completed since the previous
597 * check AND there is at least one packet pending. The
598 * ARMED bit is set to indicate a potential hang. The
599 * bit is cleared if a pause frame is received to remove
600 * false hang detection due to PFC or 802.3x frames. By
601 * requiring this to fail twice we avoid races with
602 * PFC clearing the ARMED bit and conditions where we
603 * run the check_tx_hang logic with a transmit completion
604 * pending but without time to complete it yet.
605 */
a114d0a6 606 if ((tx_ring->tx_stats.tx_done_old == tx_ring->stats.packets) &&
fd0a05ce
JB		 607 tx_pending) {
608 /* make sure it is true for two checks in a row */
609 ret = test_and_set_bit(__I40E_HANG_CHECK_ARMED,
610 &tx_ring->state);
611 } else {
612 /* update completed stats and disarm the hang check */
a114d0a6 613 tx_ring->tx_stats.tx_done_old = tx_ring->stats.packets;
fd0a05ce
JB		 614 clear_bit(__I40E_HANG_CHECK_ARMED, &tx_ring->state);
615 }
616
617 return ret;
618}
619
1943d8ba
JB		 620/**
621 * i40e_get_head - Retrieve head from head writeback
622 * @tx_ring: tx ring to fetch head of
623 *
624 * Returns value of Tx ring head based on value stored
625 * in head write-back location
626 **/
627static inline u32 i40e_get_head(struct i40e_ring *tx_ring)
628{
629 void *head = (struct i40e_tx_desc *)tx_ring->desc + tx_ring->count;
630
631 return le32_to_cpu(*(volatile __le32 *)head);
632}
633
fd0a05ce
JB		 634/**
635 * i40e_clean_tx_irq - Reclaim resources after transmit completes
636 * @tx_ring: tx ring to clean
637 * @budget: how many cleans we're allowed
638 *
639 * Returns true if there's any budget left (e.g. the clean is finished)
640 **/
641static bool i40e_clean_tx_irq(struct i40e_ring *tx_ring, int budget)
642{
643 u16 i = tx_ring->next_to_clean;
644 struct i40e_tx_buffer *tx_buf;
1943d8ba 645 struct i40e_tx_desc *tx_head;
fd0a05ce
JB		 646 struct i40e_tx_desc *tx_desc;
647 unsigned int total_packets = 0;
648 unsigned int total_bytes = 0;
649
650 tx_buf = &tx_ring->tx_bi[i];
651 tx_desc = I40E_TX_DESC(tx_ring, i);
a5e9c572 652 i -= tx_ring->count;
fd0a05ce 653
1943d8ba
JB		 654 tx_head = I40E_TX_DESC(tx_ring, i40e_get_head(tx_ring));
655
a5e9c572
AD		 656 do {
657 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
fd0a05ce
JB		 658
659 /* if next_to_watch is not set then there is no work pending */
660 if (!eop_desc)
661 break;
662
a5e9c572
AD		 663 /* prevent any other reads prior to eop_desc */
664 read_barrier_depends();
665
1943d8ba
JB		 666 /* we have caught up to head, no work left to do */
667 if (tx_head == tx_desc)
fd0a05ce
JB		 668 break;
669
c304fdac 670 /* clear next_to_watch to prevent false hangs */
fd0a05ce 671 tx_buf->next_to_watch = NULL;
fd0a05ce 672
a5e9c572
AD		 673 /* update the statistics for this packet */
674 total_bytes += tx_buf->bytecount;
675 total_packets += tx_buf->gso_segs;
fd0a05ce 676
a5e9c572
AD		 677 /* free the skb */
678 dev_kfree_skb_any(tx_buf->skb);
fd0a05ce 679
a5e9c572
AD		 680 /* unmap skb header data */
681 dma_unmap_single(tx_ring->dev,
682 dma_unmap_addr(tx_buf, dma),
683 dma_unmap_len(tx_buf, len),
684 DMA_TO_DEVICE);
fd0a05ce 685
a5e9c572
AD		 686 /* clear tx_buffer data */
687 tx_buf->skb = NULL;
688 dma_unmap_len_set(tx_buf, len, 0);
fd0a05ce 689
a5e9c572
AD		 690 /* unmap remaining buffers */
691 while (tx_desc != eop_desc) {
fd0a05ce
JB		 692
693 tx_buf++;
694 tx_desc++;
695 i++;
a5e9c572
AD		 696 if (unlikely(!i)) {
697 i -= tx_ring->count;
fd0a05ce
JB		 698 tx_buf = tx_ring->tx_bi;
699 tx_desc = I40E_TX_DESC(tx_ring, 0);
700 }
fd0a05ce 701
a5e9c572
AD		 702 /* unmap any remaining paged data */
703 if (dma_unmap_len(tx_buf, len)) {
704 dma_unmap_page(tx_ring->dev,
705 dma_unmap_addr(tx_buf, dma),
706 dma_unmap_len(tx_buf, len),
707 DMA_TO_DEVICE);
708 dma_unmap_len_set(tx_buf, len, 0);
709 }
710 }
711
712 /* move us one more past the eop_desc for start of next pkt */
713 tx_buf++;
714 tx_desc++;
715 i++;
716 if (unlikely(!i)) {
717 i -= tx_ring->count;
718 tx_buf = tx_ring->tx_bi;
719 tx_desc = I40E_TX_DESC(tx_ring, 0);
720 }
721
722 /* update budget accounting */
723 budget--;
724 } while (likely(budget));
725
726 i += tx_ring->count;
fd0a05ce 727 tx_ring->next_to_clean = i;
980e9b11 728 u64_stats_update_begin(&tx_ring->syncp);
a114d0a6
AD		 729 tx_ring->stats.bytes += total_bytes;
730 tx_ring->stats.packets += total_packets;
980e9b11 731 u64_stats_update_end(&tx_ring->syncp);
fd0a05ce
JB		 732 tx_ring->q_vector->tx.total_bytes += total_bytes;
733 tx_ring->q_vector->tx.total_packets += total_packets;
a5e9c572 734
fd0a05ce
JB		 735 if (check_for_tx_hang(tx_ring) && i40e_check_tx_hang(tx_ring)) {
736 /* schedule immediate reset if we believe we hung */
737 dev_info(tx_ring->dev, "Detected Tx Unit Hang\n"
738 " VSI <%d>\n"
739 " Tx Queue <%d>\n"
740 " next_to_use <%x>\n"
741 " next_to_clean <%x>\n",
742 tx_ring->vsi->seid,
743 tx_ring->queue_index,
744 tx_ring->next_to_use, i);
745 dev_info(tx_ring->dev, "tx_bi[next_to_clean]\n"
746 " time_stamp <%lx>\n"
747 " jiffies <%lx>\n",
748 tx_ring->tx_bi[i].time_stamp, jiffies);
749
750 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
751
752 dev_info(tx_ring->dev,
753 "tx hang detected on queue %d, resetting adapter\n",
754 tx_ring->queue_index);
755
756 tx_ring->netdev->netdev_ops->ndo_tx_timeout(tx_ring->netdev);
757
758 /* the adapter is about to reset, no point in enabling stuff */
759 return true;
760 }
761
7070ce0a
AD		 762 netdev_tx_completed_queue(netdev_get_tx_queue(tx_ring->netdev,
763 tx_ring->queue_index),
764 total_packets, total_bytes);
765
fd0a05ce
JB		 766#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
767 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
768 (I40E_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
769 /* Make sure that anybody stopping the queue after this
770 * sees the new next_to_clean.
771 */
772 smp_mb();
773 if (__netif_subqueue_stopped(tx_ring->netdev,
774 tx_ring->queue_index) &&
775 !test_bit(__I40E_DOWN, &tx_ring->vsi->state)) {
776 netif_wake_subqueue(tx_ring->netdev,
777 tx_ring->queue_index);
778 ++tx_ring->tx_stats.restart_queue;
779 }
780 }
781
782 return budget > 0;
783}
784
785/**
786 * i40e_set_new_dynamic_itr - Find new ITR level
787 * @rc: structure containing ring performance data
788 *
789 * Stores a new ITR value based on packets and byte counts during
790 * the last interrupt. The advantage of per interrupt computation
791 * is faster updates and more accurate ITR for the current traffic
792 * pattern. Constants in this function were computed based on
793 * theoretical maximum wire speed and thresholds were set based on
794 * testing data as well as attempting to minimize response time
795 * while increasing bulk throughput.
796 **/
797static void i40e_set_new_dynamic_itr(struct i40e_ring_container *rc)
798{
799 enum i40e_latency_range new_latency_range = rc->latency_range;
800 u32 new_itr = rc->itr;
801 int bytes_per_int;
802
803 if (rc->total_packets == 0 || !rc->itr)
804 return;
805
806 /* simple throttlerate management
807 * 0-10MB/s lowest (100000 ints/s)
808 * 10-20MB/s low (20000 ints/s)
809 * 20-1249MB/s bulk (8000 ints/s)
810 */
811 bytes_per_int = rc->total_bytes / rc->itr;
812 switch (rc->itr) {
813 case I40E_LOWEST_LATENCY:
814 if (bytes_per_int > 10)
815 new_latency_range = I40E_LOW_LATENCY;
816 break;
817 case I40E_LOW_LATENCY:
818 if (bytes_per_int > 20)
819 new_latency_range = I40E_BULK_LATENCY;
820 else if (bytes_per_int <= 10)
821 new_latency_range = I40E_LOWEST_LATENCY;
822 break;
823 case I40E_BULK_LATENCY:
824 if (bytes_per_int <= 20)
825 rc->latency_range = I40E_LOW_LATENCY;
826 break;
827 }
828
829 switch (new_latency_range) {
830 case I40E_LOWEST_LATENCY:
831 new_itr = I40E_ITR_100K;
832 break;
833 case I40E_LOW_LATENCY:
834 new_itr = I40E_ITR_20K;
835 break;
836 case I40E_BULK_LATENCY:
837 new_itr = I40E_ITR_8K;
838 break;
839 default:
840 break;
841 }
842
843 if (new_itr != rc->itr) {
844 /* do an exponential smoothing */
845 new_itr = (10 * new_itr * rc->itr) /
846 ((9 * new_itr) + rc->itr);
847 rc->itr = new_itr & I40E_MAX_ITR;
848 }
849
850 rc->total_bytes = 0;
851 rc->total_packets = 0;
852}
853
854/**
855 * i40e_update_dynamic_itr - Adjust ITR based on bytes per int
856 * @q_vector: the vector to adjust
857 **/
858static void i40e_update_dynamic_itr(struct i40e_q_vector *q_vector)
859{
860 u16 vector = q_vector->vsi->base_vector + q_vector->v_idx;
861 struct i40e_hw *hw = &q_vector->vsi->back->hw;
862 u32 reg_addr;
863 u16 old_itr;
864
865 reg_addr = I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1);
866 old_itr = q_vector->rx.itr;
867 i40e_set_new_dynamic_itr(&q_vector->rx);
868 if (old_itr != q_vector->rx.itr)
869 wr32(hw, reg_addr, q_vector->rx.itr);
870
871 reg_addr = I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1);
872 old_itr = q_vector->tx.itr;
873 i40e_set_new_dynamic_itr(&q_vector->tx);
874 if (old_itr != q_vector->tx.itr)
875 wr32(hw, reg_addr, q_vector->tx.itr);
fd0a05ce
JB		 876}
877
878/**
879 * i40e_clean_programming_status - clean the programming status descriptor
880 * @rx_ring: the rx ring that has this descriptor
881 * @rx_desc: the rx descriptor written back by HW
882 *
883 * Flow director should handle FD_FILTER_STATUS to check its filter programming
884 * status being successful or not and take actions accordingly. FCoE should
885 * handle its context/filter programming/invalidation status and take actions.
886 *
887 **/
888static void i40e_clean_programming_status(struct i40e_ring *rx_ring,
889 union i40e_rx_desc *rx_desc)
890{
891 u64 qw;
892 u8 id;
893
894 qw = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
895 id = (qw & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >>
896 I40E_RX_PROG_STATUS_DESC_QW1_PROGID_SHIFT;
897
898 if (id == I40E_RX_PROG_STATUS_DESC_FD_FILTER_STATUS)
55a5e60b 899 i40e_fd_handle_status(rx_ring, rx_desc, id);
fd0a05ce
JB		 900}
901
902/**
903 * i40e_setup_tx_descriptors - Allocate the Tx descriptors
904 * @tx_ring: the tx ring to set up
905 *
906 * Return 0 on success, negative on error
907 **/
908int i40e_setup_tx_descriptors(struct i40e_ring *tx_ring)
909{
910 struct device *dev = tx_ring->dev;
911 int bi_size;
912
913 if (!dev)
914 return -ENOMEM;
915
916 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
917 tx_ring->tx_bi = kzalloc(bi_size, GFP_KERNEL);
918 if (!tx_ring->tx_bi)
919 goto err;
920
921 /* round up to nearest 4K */
922 tx_ring->size = tx_ring->count * sizeof(struct i40e_tx_desc);
1943d8ba
JB		 923 /* add u32 for head writeback, align after this takes care of
924 * guaranteeing this is at least one cache line in size
925 */
926 tx_ring->size += sizeof(u32);
fd0a05ce
JB		 927 tx_ring->size = ALIGN(tx_ring->size, 4096);
928 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
929 &tx_ring->dma, GFP_KERNEL);
930 if (!tx_ring->desc) {
931 dev_info(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n",
932 tx_ring->size);
933 goto err;
934 }
935
936 tx_ring->next_to_use = 0;
937 tx_ring->next_to_clean = 0;
938 return 0;
939
940err:
941 kfree(tx_ring->tx_bi);
942 tx_ring->tx_bi = NULL;
943 return -ENOMEM;
944}
945
946/**
947 * i40e_clean_rx_ring - Free Rx buffers
948 * @rx_ring: ring to be cleaned
949 **/
950void i40e_clean_rx_ring(struct i40e_ring *rx_ring)
951{
952 struct device *dev = rx_ring->dev;
953 struct i40e_rx_buffer *rx_bi;
954 unsigned long bi_size;
955 u16 i;
956
957 /* ring already cleared, nothing to do */
958 if (!rx_ring->rx_bi)
959 return;
960
961 /* Free all the Rx ring sk_buffs */
962 for (i = 0; i < rx_ring->count; i++) {
963 rx_bi = &rx_ring->rx_bi[i];
964 if (rx_bi->dma) {
965 dma_unmap_single(dev,
966 rx_bi->dma,
967 rx_ring->rx_buf_len,
968 DMA_FROM_DEVICE);
969 rx_bi->dma = 0;
970 }
971 if (rx_bi->skb) {
972 dev_kfree_skb(rx_bi->skb);
973 rx_bi->skb = NULL;
974 }
975 if (rx_bi->page) {
976 if (rx_bi->page_dma) {
977 dma_unmap_page(dev,
978 rx_bi->page_dma,
979 PAGE_SIZE / 2,
980 DMA_FROM_DEVICE);
981 rx_bi->page_dma = 0;
982 }
983 __free_page(rx_bi->page);
984 rx_bi->page = NULL;
985 rx_bi->page_offset = 0;
986 }
987 }
988
989 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
990 memset(rx_ring->rx_bi, 0, bi_size);
991
992 /* Zero out the descriptor ring */
993 memset(rx_ring->desc, 0, rx_ring->size);
994
995 rx_ring->next_to_clean = 0;
996 rx_ring->next_to_use = 0;
997}
998
999/**
1000 * i40e_free_rx_resources - Free Rx resources
1001 * @rx_ring: ring to clean the resources from
1002 *
1003 * Free all receive software resources
1004 **/
1005void i40e_free_rx_resources(struct i40e_ring *rx_ring)
1006{
1007 i40e_clean_rx_ring(rx_ring);
1008 kfree(rx_ring->rx_bi);
1009 rx_ring->rx_bi = NULL;
1010
1011 if (rx_ring->desc) {
1012 dma_free_coherent(rx_ring->dev, rx_ring->size,
1013 rx_ring->desc, rx_ring->dma);
1014 rx_ring->desc = NULL;
1015 }
1016}
1017
1018/**
1019 * i40e_setup_rx_descriptors - Allocate Rx descriptors
1020 * @rx_ring: Rx descriptor ring (for a specific queue) to setup
1021 *
1022 * Returns 0 on success, negative on failure
1023 **/
1024int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring)
1025{
1026 struct device *dev = rx_ring->dev;
1027 int bi_size;
1028
1029 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
1030 rx_ring->rx_bi = kzalloc(bi_size, GFP_KERNEL);
1031 if (!rx_ring->rx_bi)
1032 goto err;
1033
1034 /* Round up to nearest 4K */
1035 rx_ring->size = ring_is_16byte_desc_enabled(rx_ring)
1036 ? rx_ring->count * sizeof(union i40e_16byte_rx_desc)
1037 : rx_ring->count * sizeof(union i40e_32byte_rx_desc);
1038 rx_ring->size = ALIGN(rx_ring->size, 4096);
1039 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
1040 &rx_ring->dma, GFP_KERNEL);
1041
1042 if (!rx_ring->desc) {
1043 dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n",
1044 rx_ring->size);
1045 goto err;
1046 }
1047
1048 rx_ring->next_to_clean = 0;
1049 rx_ring->next_to_use = 0;
1050
1051 return 0;
1052err:
1053 kfree(rx_ring->rx_bi);
1054 rx_ring->rx_bi = NULL;
1055 return -ENOMEM;
1056}
1057
1058/**
1059 * i40e_release_rx_desc - Store the new tail and head values
1060 * @rx_ring: ring to bump
1061 * @val: new head index
1062 **/
1063static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val)
1064{
1065 rx_ring->next_to_use = val;
1066 /* Force memory writes to complete before letting h/w
1067 * know there are new descriptors to fetch. (Only
1068 * applicable for weak-ordered memory model archs,
1069 * such as IA-64).
1070 */
1071 wmb();
1072 writel(val, rx_ring->tail);
1073}
1074
1075/**
1076 * i40e_alloc_rx_buffers - Replace used receive buffers; packet split
1077 * @rx_ring: ring to place buffers on
1078 * @cleaned_count: number of buffers to replace
1079 **/
1080void i40e_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count)
1081{
1082 u16 i = rx_ring->next_to_use;
1083 union i40e_rx_desc *rx_desc;
1084 struct i40e_rx_buffer *bi;
1085 struct sk_buff *skb;
1086
1087 /* do nothing if no valid netdev defined */
1088 if (!rx_ring->netdev || !cleaned_count)
1089 return;
1090
1091 while (cleaned_count--) {
1092 rx_desc = I40E_RX_DESC(rx_ring, i);
1093 bi = &rx_ring->rx_bi[i];
1094 skb = bi->skb;
1095
1096 if (!skb) {
1097 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
1098 rx_ring->rx_buf_len);
1099 if (!skb) {
420136cc 1100 rx_ring->rx_stats.alloc_buff_failed++;
fd0a05ce
JB		 1101 goto no_buffers;
1102 }
1103 /* initialize queue mapping */
1104 skb_record_rx_queue(skb, rx_ring->queue_index);
1105 bi->skb = skb;
1106 }
1107
1108 if (!bi->dma) {
1109 bi->dma = dma_map_single(rx_ring->dev,
1110 skb->data,
1111 rx_ring->rx_buf_len,
1112 DMA_FROM_DEVICE);
1113 if (dma_mapping_error(rx_ring->dev, bi->dma)) {
420136cc 1114 rx_ring->rx_stats.alloc_buff_failed++;
fd0a05ce
JB		 1115 bi->dma = 0;
1116 goto no_buffers;
1117 }
1118 }
1119
1120 if (ring_is_ps_enabled(rx_ring)) {
1121 if (!bi->page) {
1122 bi->page = alloc_page(GFP_ATOMIC);
1123 if (!bi->page) {
420136cc 1124 rx_ring->rx_stats.alloc_page_failed++;
fd0a05ce
JB		 1125 goto no_buffers;
1126 }
1127 }
1128
1129 if (!bi->page_dma) {
1130 /* use a half page if we're re-using */
1131 bi->page_offset ^= PAGE_SIZE / 2;
1132 bi->page_dma = dma_map_page(rx_ring->dev,
1133 bi->page,
1134 bi->page_offset,
1135 PAGE_SIZE / 2,
1136 DMA_FROM_DEVICE);
1137 if (dma_mapping_error(rx_ring->dev,
1138 bi->page_dma)) {
420136cc 1139 rx_ring->rx_stats.alloc_page_failed++;
fd0a05ce
JB		 1140 bi->page_dma = 0;
1141 goto no_buffers;
1142 }
1143 }
1144
1145 /* Refresh the desc even if buffer_addrs didn't change
1146 * because each write-back erases this info.
1147 */
1148 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
1149 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
1150 } else {
1151 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
1152 rx_desc->read.hdr_addr = 0;
1153 }
1154 i++;
1155 if (i == rx_ring->count)
1156 i = 0;
1157 }
1158
1159no_buffers:
1160 if (rx_ring->next_to_use != i)
1161 i40e_release_rx_desc(rx_ring, i);
1162}
1163
1164/**
1165 * i40e_receive_skb - Send a completed packet up the stack
1166 * @rx_ring: rx ring in play
1167 * @skb: packet to send up
1168 * @vlan_tag: vlan tag for packet
1169 **/
1170static void i40e_receive_skb(struct i40e_ring *rx_ring,
1171 struct sk_buff *skb, u16 vlan_tag)
1172{
1173 struct i40e_q_vector *q_vector = rx_ring->q_vector;
1174 struct i40e_vsi *vsi = rx_ring->vsi;
1175 u64 flags = vsi->back->flags;
1176
1177 if (vlan_tag & VLAN_VID_MASK)
1178 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
1179
1180 if (flags & I40E_FLAG_IN_NETPOLL)
1181 netif_rx(skb);
1182 else
1183 napi_gro_receive(&q_vector->napi, skb);
1184}
1185
1186/**
1187 * i40e_rx_checksum - Indicate in skb if hw indicated a good cksum
1188 * @vsi: the VSI we care about
1189 * @skb: skb currently being received and modified
1190 * @rx_status: status value of last descriptor in packet
1191 * @rx_error: error value of last descriptor in packet
8144f0f7 1192 * @rx_ptype: ptype value of last descriptor in packet
fd0a05ce
JB		 1193 **/
1194static inline void i40e_rx_checksum(struct i40e_vsi *vsi,
1195 struct sk_buff *skb,
1196 u32 rx_status,
8144f0f7
JG		 1197 u32 rx_error,
1198 u16 rx_ptype)
fd0a05ce 1199{
8a3c91cc
JB		 1200 struct i40e_rx_ptype_decoded decoded = decode_rx_desc_ptype(rx_ptype);
1201 bool ipv4 = false, ipv6 = false;
8144f0f7
JG		 1202 bool ipv4_tunnel, ipv6_tunnel;
1203 __wsum rx_udp_csum;
8144f0f7 1204 struct iphdr *iph;
8a3c91cc 1205 __sum16 csum;
8144f0f7
JG		 1206
1207 ipv4_tunnel = (rx_ptype > I40E_RX_PTYPE_GRENAT4_MAC_PAY3) &&
1208 (rx_ptype < I40E_RX_PTYPE_GRENAT4_MACVLAN_IPV6_ICMP_PAY4);
1209 ipv6_tunnel = (rx_ptype > I40E_RX_PTYPE_GRENAT6_MAC_PAY3) &&
1210 (rx_ptype < I40E_RX_PTYPE_GRENAT6_MACVLAN_IPV6_ICMP_PAY4);
1211
1212 skb->encapsulation = ipv4_tunnel || ipv6_tunnel;
fd0a05ce
JB		 1213 skb->ip_summed = CHECKSUM_NONE;
1214
1215 /* Rx csum enabled and ip headers found? */
8a3c91cc
JB		 1216 if (!(vsi->netdev->features & NETIF_F_RXCSUM))
1217 return;
1218
1219 /* did the hardware decode the packet and checksum? */
1220 if (!(rx_status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT)))
1221 return;
1222
1223 /* both known and outer_ip must be set for the below code to work */
1224 if (!(decoded.known && decoded.outer_ip))
fd0a05ce
JB		 1225 return;
1226
8a3c91cc
JB		 1227 if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1228 decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4)
1229 ipv4 = true;
1230 else if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1231 decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6)
1232 ipv6 = true;
1233
1234 if (ipv4 &&
1235 (rx_error & ((1 << I40E_RX_DESC_ERROR_IPE_SHIFT) |
1236 (1 << I40E_RX_DESC_ERROR_EIPE_SHIFT))))
1237 goto checksum_fail;
1238
ddf1d0d7 1239 /* likely incorrect csum if alternate IP extension headers found */
8a3c91cc
JB		 1240 if (ipv6 &&
1241 decoded.inner_prot == I40E_RX_PTYPE_INNER_PROT_TCP &&
1242 rx_error & (1 << I40E_RX_DESC_ERROR_L4E_SHIFT) &&
1243 rx_status & (1 << I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT))
1244 /* don't increment checksum err here, non-fatal err */
8ee75a8e
SN		 1245 return;
1246
8a3c91cc
JB		 1247 /* there was some L4 error, count error and punt packet to the stack */
1248 if (rx_error & (1 << I40E_RX_DESC_ERROR_L4E_SHIFT))
1249 goto checksum_fail;
1250
1251 /* handle packets that were not able to be checksummed due
1252 * to arrival speed, in this case the stack can compute
1253 * the csum.
1254 */
1255 if (rx_error & (1 << I40E_RX_DESC_ERROR_PPRS_SHIFT))
fd0a05ce 1256 return;
fd0a05ce 1257
8a3c91cc
JB		 1258 /* If VXLAN traffic has an outer UDPv4 checksum we need to check
1259 * it in the driver, hardware does not do it for us.
1260 * Since L3L4P bit was set we assume a valid IHL value (>=5)
1261 * so the total length of IPv4 header is IHL*4 bytes
1262 * The UDP_0 bit *may* bet set if the *inner* header is UDP
1263 */
8144f0f7 1264 if (ipv4_tunnel &&
8a3c91cc 1265 (decoded.inner_prot != I40E_RX_PTYPE_INNER_PROT_UDP) &&
8144f0f7 1266 !(rx_status & (1 << I40E_RX_DESC_STATUS_UDP_0_SHIFT))) {
8144f0f7
JG		 1267 skb->transport_header = skb->mac_header +
1268 sizeof(struct ethhdr) +
1269 (ip_hdr(skb)->ihl * 4);
1270
1271 /* Add 4 bytes for VLAN tagged packets */
1272 skb->transport_header += (skb->protocol == htons(ETH_P_8021Q) ||
1273 skb->protocol == htons(ETH_P_8021AD))
1274 ? VLAN_HLEN : 0;
1275
1276 rx_udp_csum = udp_csum(skb);
1277 iph = ip_hdr(skb);
1278 csum = csum_tcpudp_magic(
1279 iph->saddr, iph->daddr,
1280 (skb->len - skb_transport_offset(skb)),
1281 IPPROTO_UDP, rx_udp_csum);
1282
8a3c91cc
JB		 1283 if (udp_hdr(skb)->check != csum)
1284 goto checksum_fail;
8144f0f7
JG		 1285 }
1286
fd0a05ce 1287 skb->ip_summed = CHECKSUM_UNNECESSARY;
8a3c91cc
JB		 1288
1289 return;
1290
1291checksum_fail:
1292 vsi->back->hw_csum_rx_error++;
fd0a05ce
JB		 1293}
1294
1295/**
1296 * i40e_rx_hash - returns the hash value from the Rx descriptor
1297 * @ring: descriptor ring
1298 * @rx_desc: specific descriptor
1299 **/
1300static inline u32 i40e_rx_hash(struct i40e_ring *ring,
1301 union i40e_rx_desc *rx_desc)
1302{
8a494920
JB		 1303 const __le64 rss_mask =
1304 cpu_to_le64((u64)I40E_RX_DESC_FLTSTAT_RSS_HASH <<
1305 I40E_RX_DESC_STATUS_FLTSTAT_SHIFT);
1306
1307 if ((ring->netdev->features & NETIF_F_RXHASH) &&
1308 (rx_desc->wb.qword1.status_error_len & rss_mask) == rss_mask)
1309 return le32_to_cpu(rx_desc->wb.qword0.hi_dword.rss);
1310 else
1311 return 0;
fd0a05ce
JB		 1312}
1313
206812b5
JB		 1314/**
1315 * i40e_ptype_to_hash - get a hash type
1316 * @ptype: the ptype value from the descriptor
1317 *
1318 * Returns a hash type to be used by skb_set_hash
1319 **/
1320static inline enum pkt_hash_types i40e_ptype_to_hash(u8 ptype)
1321{
1322 struct i40e_rx_ptype_decoded decoded = decode_rx_desc_ptype(ptype);
1323
1324 if (!decoded.known)
1325 return PKT_HASH_TYPE_NONE;
1326
1327 if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1328 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY4)
1329 return PKT_HASH_TYPE_L4;
1330 else if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1331 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY3)
1332 return PKT_HASH_TYPE_L3;
1333 else
1334 return PKT_HASH_TYPE_L2;
1335}
1336
fd0a05ce
JB		 1337/**
1338 * i40e_clean_rx_irq - Reclaim resources after receive completes
1339 * @rx_ring: rx ring to clean
1340 * @budget: how many cleans we're allowed
1341 *
1342 * Returns true if there's any budget left (e.g. the clean is finished)
1343 **/
1344static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
1345{
1346 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1347 u16 rx_packet_len, rx_header_len, rx_sph, rx_hbo;
1348 u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);
1349 const int current_node = numa_node_id();
1350 struct i40e_vsi *vsi = rx_ring->vsi;
1351 u16 i = rx_ring->next_to_clean;
1352 union i40e_rx_desc *rx_desc;
1353 u32 rx_error, rx_status;
206812b5 1354 u8 rx_ptype;
fd0a05ce
JB		 1355 u64 qword;
1356
390f86df
EB		 1357 if (budget <= 0)
1358 return 0;
1359
fd0a05ce
JB		 1360 rx_desc = I40E_RX_DESC(rx_ring, i);
1361 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
6838b535
JB		 1362 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >>
1363 I40E_RXD_QW1_STATUS_SHIFT;
fd0a05ce
JB		 1364
1365 while (rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) {
1366 union i40e_rx_desc *next_rxd;
1367 struct i40e_rx_buffer *rx_bi;
1368 struct sk_buff *skb;
1369 u16 vlan_tag;
1370 if (i40e_rx_is_programming_status(qword)) {
1371 i40e_clean_programming_status(rx_ring, rx_desc);
1372 I40E_RX_NEXT_DESC_PREFETCH(rx_ring, i, next_rxd);
1373 goto next_desc;
1374 }
1375 rx_bi = &rx_ring->rx_bi[i];
1376 skb = rx_bi->skb;
1377 prefetch(skb->data);
1378
829af3ac
MW		 1379 rx_packet_len = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
1380 I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
1381 rx_header_len = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK) >>
1382 I40E_RXD_QW1_LENGTH_HBUF_SHIFT;
1383 rx_sph = (qword & I40E_RXD_QW1_LENGTH_SPH_MASK) >>
1384 I40E_RXD_QW1_LENGTH_SPH_SHIFT;
1385
1386 rx_error = (qword & I40E_RXD_QW1_ERROR_MASK) >>
1387 I40E_RXD_QW1_ERROR_SHIFT;
fd0a05ce
JB		 1388 rx_hbo = rx_error & (1 << I40E_RX_DESC_ERROR_HBO_SHIFT);
1389 rx_error &= ~(1 << I40E_RX_DESC_ERROR_HBO_SHIFT);
1390
8144f0f7
JG		 1391 rx_ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >>
1392 I40E_RXD_QW1_PTYPE_SHIFT;
fd0a05ce
JB		 1393 rx_bi->skb = NULL;
1394
1395 /* This memory barrier is needed to keep us from reading
1396 * any other fields out of the rx_desc until we know the
1397 * STATUS_DD bit is set
1398 */
1399 rmb();
1400
1401 /* Get the header and possibly the whole packet
1402 * If this is an skb from previous receive dma will be 0
1403 */
1404 if (rx_bi->dma) {
1405 u16 len;
1406
1407 if (rx_hbo)
1408 len = I40E_RX_HDR_SIZE;
1409 else if (rx_sph)
1410 len = rx_header_len;
1411 else if (rx_packet_len)
1412 len = rx_packet_len; /* 1buf/no split found */
1413 else
1414 len = rx_header_len; /* split always mode */
1415
1416 skb_put(skb, len);
1417 dma_unmap_single(rx_ring->dev,
1418 rx_bi->dma,
1419 rx_ring->rx_buf_len,
1420 DMA_FROM_DEVICE);
1421 rx_bi->dma = 0;
1422 }
1423
1424 /* Get the rest of the data if this was a header split */
1425 if (ring_is_ps_enabled(rx_ring) && rx_packet_len) {
1426
1427 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
1428 rx_bi->page,
1429 rx_bi->page_offset,
1430 rx_packet_len);
1431
1432 skb->len += rx_packet_len;
1433 skb->data_len += rx_packet_len;
1434 skb->truesize += rx_packet_len;
1435
1436 if ((page_count(rx_bi->page) == 1) &&
1437 (page_to_nid(rx_bi->page) == current_node))
1438 get_page(rx_bi->page);
1439 else
1440 rx_bi->page = NULL;
1441
1442 dma_unmap_page(rx_ring->dev,
1443 rx_bi->page_dma,
1444 PAGE_SIZE / 2,
1445 DMA_FROM_DEVICE);
1446 rx_bi->page_dma = 0;
1447 }
1448 I40E_RX_NEXT_DESC_PREFETCH(rx_ring, i, next_rxd);
1449
1450 if (unlikely(
1451 !(rx_status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)))) {
1452 struct i40e_rx_buffer *next_buffer;
1453
1454 next_buffer = &rx_ring->rx_bi[i];
1455
1456 if (ring_is_ps_enabled(rx_ring)) {
1457 rx_bi->skb = next_buffer->skb;
1458 rx_bi->dma = next_buffer->dma;
1459 next_buffer->skb = skb;
1460 next_buffer->dma = 0;
1461 }
1462 rx_ring->rx_stats.non_eop_descs++;
1463 goto next_desc;
1464 }
1465
1466 /* ERR_MASK will only have valid bits if EOP set */
1467 if (unlikely(rx_error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) {
1468 dev_kfree_skb_any(skb);
8a3c91cc
JB		 1469 /* TODO: shouldn't we increment a counter indicating the
1470 * drop?
1471 */
fd0a05ce
JB		 1472 goto next_desc;
1473 }
1474
206812b5
JB		 1475 skb_set_hash(skb, i40e_rx_hash(rx_ring, rx_desc),
1476 i40e_ptype_to_hash(rx_ptype));
beb0dff1
JK		 1477 if (unlikely(rx_status & I40E_RXD_QW1_STATUS_TSYNVALID_MASK)) {
1478 i40e_ptp_rx_hwtstamp(vsi->back, skb, (rx_status &
1479 I40E_RXD_QW1_STATUS_TSYNINDX_MASK) >>
1480 I40E_RXD_QW1_STATUS_TSYNINDX_SHIFT);
1481 rx_ring->last_rx_timestamp = jiffies;
1482 }
1483
fd0a05ce
JB		 1484 /* probably a little skewed due to removing CRC */
1485 total_rx_bytes += skb->len;
1486 total_rx_packets++;
1487
1488 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
8144f0f7
JG		 1489
1490 i40e_rx_checksum(vsi, skb, rx_status, rx_error, rx_ptype);
1491
fd0a05ce
JB		 1492 vlan_tag = rx_status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)
1493 ? le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1)
1494 : 0;
1495 i40e_receive_skb(rx_ring, skb, vlan_tag);
1496
1497 rx_ring->netdev->last_rx = jiffies;
1498 budget--;
1499next_desc:
1500 rx_desc->wb.qword1.status_error_len = 0;
1501 if (!budget)
1502 break;
1503
1504 cleaned_count++;
1505 /* return some buffers to hardware, one at a time is too slow */
1506 if (cleaned_count >= I40E_RX_BUFFER_WRITE) {
1507 i40e_alloc_rx_buffers(rx_ring, cleaned_count);
1508 cleaned_count = 0;
1509 }
1510
1511 /* use prefetched values */
1512 rx_desc = next_rxd;
1513 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
829af3ac
MW		 1514 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >>
1515 I40E_RXD_QW1_STATUS_SHIFT;
fd0a05ce
JB		 1516 }
1517
1518 rx_ring->next_to_clean = i;
980e9b11 1519 u64_stats_update_begin(&rx_ring->syncp);
a114d0a6
AD		 1520 rx_ring->stats.packets += total_rx_packets;
1521 rx_ring->stats.bytes += total_rx_bytes;
980e9b11 1522 u64_stats_update_end(&rx_ring->syncp);
fd0a05ce
JB		 1523 rx_ring->q_vector->rx.total_packets += total_rx_packets;
1524 rx_ring->q_vector->rx.total_bytes += total_rx_bytes;
1525
1526 if (cleaned_count)
1527 i40e_alloc_rx_buffers(rx_ring, cleaned_count);
1528
1529 return budget > 0;
1530}
1531
1532/**
1533 * i40e_napi_poll - NAPI polling Rx/Tx cleanup routine
1534 * @napi: napi struct with our devices info in it
1535 * @budget: amount of work driver is allowed to do this pass, in packets
1536 *
1537 * This function will clean all queues associated with a q_vector.
1538 *
1539 * Returns the amount of work done
1540 **/
1541int i40e_napi_poll(struct napi_struct *napi, int budget)
1542{
1543 struct i40e_q_vector *q_vector =
1544 container_of(napi, struct i40e_q_vector, napi);
1545 struct i40e_vsi *vsi = q_vector->vsi;
cd0b6fa6 1546 struct i40e_ring *ring;
fd0a05ce
JB		 1547 bool clean_complete = true;
1548 int budget_per_ring;
fd0a05ce
JB		 1549
1550 if (test_bit(__I40E_DOWN, &vsi->state)) {
1551 napi_complete(napi);
1552 return 0;
1553 }
1554
cd0b6fa6
AD		 1555 /* Since the actual Tx work is minimal, we can give the Tx a larger
1556 * budget and be more aggressive about cleaning up the Tx descriptors.
1557 */
1558 i40e_for_each_ring(ring, q_vector->tx)
1559 clean_complete &= i40e_clean_tx_irq(ring, vsi->work_limit);
1560
fd0a05ce
JB		 1561 /* We attempt to distribute budget to each Rx queue fairly, but don't
1562 * allow the budget to go below 1 because that would exit polling early.
fd0a05ce
JB		 1563 */
1564 budget_per_ring = max(budget/q_vector->num_ringpairs, 1);
cd0b6fa6
AD		 1565
1566 i40e_for_each_ring(ring, q_vector->rx)
1567 clean_complete &= i40e_clean_rx_irq(ring, budget_per_ring);
fd0a05ce
JB		 1568
1569 /* If work not completed, return budget and polling will return */
1570 if (!clean_complete)
1571 return budget;
1572
1573 /* Work is done so exit the polling mode and re-enable the interrupt */
1574 napi_complete(napi);
1575 if (ITR_IS_DYNAMIC(vsi->rx_itr_setting) ||
1576 ITR_IS_DYNAMIC(vsi->tx_itr_setting))
1577 i40e_update_dynamic_itr(q_vector);
1578
1579 if (!test_bit(__I40E_DOWN, &vsi->state)) {
1580 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) {
1581 i40e_irq_dynamic_enable(vsi,
1582 q_vector->v_idx + vsi->base_vector);
1583 } else {
1584 struct i40e_hw *hw = &vsi->back->hw;
1585 /* We re-enable the queue 0 cause, but
1586 * don't worry about dynamic_enable
1587 * because we left it on for the other
1588 * possible interrupts during napi
1589 */
1590 u32 qval = rd32(hw, I40E_QINT_RQCTL(0));
1591 qval |= I40E_QINT_RQCTL_CAUSE_ENA_MASK;
1592 wr32(hw, I40E_QINT_RQCTL(0), qval);
1593
1594 qval = rd32(hw, I40E_QINT_TQCTL(0));
1595 qval |= I40E_QINT_TQCTL_CAUSE_ENA_MASK;
1596 wr32(hw, I40E_QINT_TQCTL(0), qval);
116a57d4
SN		 1597
1598 i40e_irq_dynamic_enable_icr0(vsi->back);
fd0a05ce
JB		 1599 }
1600 }
1601
1602 return 0;
1603}
1604
1605/**
1606 * i40e_atr - Add a Flow Director ATR filter
1607 * @tx_ring: ring to add programming descriptor to
1608 * @skb: send buffer
1609 * @flags: send flags
1610 * @protocol: wire protocol
1611 **/
1612static void i40e_atr(struct i40e_ring *tx_ring, struct sk_buff *skb,
1613 u32 flags, __be16 protocol)
1614{
1615 struct i40e_filter_program_desc *fdir_desc;
1616 struct i40e_pf *pf = tx_ring->vsi->back;
1617 union {
1618 unsigned char *network;
1619 struct iphdr *ipv4;
1620 struct ipv6hdr *ipv6;
1621 } hdr;
1622 struct tcphdr *th;
1623 unsigned int hlen;
1624 u32 flex_ptype, dtype_cmd;
fc4ac67b 1625 u16 i;
fd0a05ce
JB		 1626
1627 /* make sure ATR is enabled */
60ea5f83 1628 if (!(pf->flags & I40E_FLAG_FD_ATR_ENABLED))
fd0a05ce
JB		 1629 return;
1630
1631 /* if sampling is disabled do nothing */
1632 if (!tx_ring->atr_sample_rate)
1633 return;
1634
fd0a05ce
JB		 1635 /* snag network header to get L4 type and address */
1636 hdr.network = skb_network_header(skb);
1637
1638 /* Currently only IPv4/IPv6 with TCP is supported */
1639 if (protocol == htons(ETH_P_IP)) {
1640 if (hdr.ipv4->protocol != IPPROTO_TCP)
1641 return;
1642
1643 /* access ihl as a u8 to avoid unaligned access on ia64 */
1644 hlen = (hdr.network[0] & 0x0F) << 2;
1645 } else if (protocol == htons(ETH_P_IPV6)) {
1646 if (hdr.ipv6->nexthdr != IPPROTO_TCP)
1647 return;
1648
1649 hlen = sizeof(struct ipv6hdr);
1650 } else {
1651 return;
1652 }
1653
1654 th = (struct tcphdr *)(hdr.network + hlen);
1655
55a5e60b
ASJ		 1656 /* Due to lack of space, no more new filters can be programmed */
1657 if (th->syn && (pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED))
1658 return;
1659
1660 tx_ring->atr_count++;
1661
ce806783
ASJ		 1662 /* sample on all syn/fin/rst packets or once every atr sample rate */
1663 if (!th->fin &&
1664 !th->syn &&
1665 !th->rst &&
1666 (tx_ring->atr_count < tx_ring->atr_sample_rate))
fd0a05ce
JB		 1667 return;
1668
1669 tx_ring->atr_count = 0;
1670
1671 /* grab the next descriptor */
fc4ac67b
AD		 1672 i = tx_ring->next_to_use;
1673 fdir_desc = I40E_TX_FDIRDESC(tx_ring, i);
1674
1675 i++;
1676 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
fd0a05ce
JB		 1677
1678 flex_ptype = (tx_ring->queue_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
1679 I40E_TXD_FLTR_QW0_QINDEX_MASK;
1680 flex_ptype |= (protocol == htons(ETH_P_IP)) ?
1681 (I40E_FILTER_PCTYPE_NONF_IPV4_TCP <<
1682 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) :
1683 (I40E_FILTER_PCTYPE_NONF_IPV6_TCP <<
1684 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT);
1685
1686 flex_ptype |= tx_ring->vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
1687
1688 dtype_cmd = I40E_TX_DESC_DTYPE_FILTER_PROG;
1689
ce806783 1690 dtype_cmd |= (th->fin || th->rst) ?
fd0a05ce
JB		 1691 (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
1692 I40E_TXD_FLTR_QW1_PCMD_SHIFT) :
1693 (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
1694 I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1695
1696 dtype_cmd |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX <<
1697 I40E_TXD_FLTR_QW1_DEST_SHIFT;
1698
1699 dtype_cmd |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID <<
1700 I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT;
1701
433c47de
ASJ		 1702 dtype_cmd |= I40E_TXD_FLTR_QW1_CNT_ENA_MASK;
1703 dtype_cmd |=
1704 ((u32)pf->fd_atr_cnt_idx << I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) &
1705 I40E_TXD_FLTR_QW1_CNTINDEX_MASK;
1706
fd0a05ce 1707 fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(flex_ptype);
99753ea6 1708 fdir_desc->rsvd = cpu_to_le32(0);
fd0a05ce 1709 fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dtype_cmd);
99753ea6 1710 fdir_desc->fd_id = cpu_to_le32(0);
fd0a05ce
JB		 1711}
1712
fd0a05ce
JB		 1713/**
1714 * i40e_tx_prepare_vlan_flags - prepare generic TX VLAN tagging flags for HW
1715 * @skb: send buffer
1716 * @tx_ring: ring to send buffer on
1717 * @flags: the tx flags to be set
1718 *
1719 * Checks the skb and set up correspondingly several generic transmit flags
1720 * related to VLAN tagging for the HW, such as VLAN, DCB, etc.
1721 *
1722 * Returns error code indicate the frame should be dropped upon error and the
1723 * otherwise returns 0 to indicate the flags has been set properly.
1724 **/
1725static int i40e_tx_prepare_vlan_flags(struct sk_buff *skb,
1726 struct i40e_ring *tx_ring,
1727 u32 *flags)
1728{
1729 __be16 protocol = skb->protocol;
1730 u32 tx_flags = 0;
1731
1732 /* if we have a HW VLAN tag being added, default to the HW one */
1733 if (vlan_tx_tag_present(skb)) {
1734 tx_flags |= vlan_tx_tag_get(skb) << I40E_TX_FLAGS_VLAN_SHIFT;
1735 tx_flags |= I40E_TX_FLAGS_HW_VLAN;
1736 /* else if it is a SW VLAN, check the next protocol and store the tag */
0e2fe46c 1737 } else if (protocol == htons(ETH_P_8021Q)) {
fd0a05ce
JB		 1738 struct vlan_hdr *vhdr, _vhdr;
1739 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr);
1740 if (!vhdr)
1741 return -EINVAL;
1742
1743 protocol = vhdr->h_vlan_encapsulated_proto;
1744 tx_flags |= ntohs(vhdr->h_vlan_TCI) << I40E_TX_FLAGS_VLAN_SHIFT;
1745 tx_flags |= I40E_TX_FLAGS_SW_VLAN;
1746 }
1747
1748 /* Insert 802.1p priority into VLAN header */
1749 if ((tx_ring->vsi->back->flags & I40E_FLAG_DCB_ENABLED) &&
1750 ((tx_flags & (I40E_TX_FLAGS_HW_VLAN | I40E_TX_FLAGS_SW_VLAN)) ||
1751 (skb->priority != TC_PRIO_CONTROL))) {
1752 tx_flags &= ~I40E_TX_FLAGS_VLAN_PRIO_MASK;
1753 tx_flags |= (skb->priority & 0x7) <<
1754 I40E_TX_FLAGS_VLAN_PRIO_SHIFT;
1755 if (tx_flags & I40E_TX_FLAGS_SW_VLAN) {
1756 struct vlan_ethhdr *vhdr;
dd225bc6
FR		 1757 int rc;
1758
1759 rc = skb_cow_head(skb, 0);
1760 if (rc < 0)
1761 return rc;
fd0a05ce
JB		 1762 vhdr = (struct vlan_ethhdr *)skb->data;
1763 vhdr->h_vlan_TCI = htons(tx_flags >>
1764 I40E_TX_FLAGS_VLAN_SHIFT);
1765 } else {
1766 tx_flags |= I40E_TX_FLAGS_HW_VLAN;
1767 }
1768 }
1769 *flags = tx_flags;
1770 return 0;
1771}
1772
fd0a05ce
JB		 1773/**
1774 * i40e_tso - set up the tso context descriptor
1775 * @tx_ring: ptr to the ring to send
1776 * @skb: ptr to the skb we're sending
1777 * @tx_flags: the collected send information
1778 * @protocol: the send protocol
1779 * @hdr_len: ptr to the size of the packet header
1780 * @cd_tunneling: ptr to context descriptor bits
1781 *
1782 * Returns 0 if no TSO can happen, 1 if tso is going, or error
1783 **/
1784static int i40e_tso(struct i40e_ring *tx_ring, struct sk_buff *skb,
1785 u32 tx_flags, __be16 protocol, u8 *hdr_len,
1786 u64 *cd_type_cmd_tso_mss, u32 *cd_tunneling)
1787{
1788 u32 cd_cmd, cd_tso_len, cd_mss;
dd225bc6 1789 struct ipv6hdr *ipv6h;
fd0a05ce
JB		 1790 struct tcphdr *tcph;
1791 struct iphdr *iph;
1792 u32 l4len;
1793 int err;
fd0a05ce
JB		 1794
1795 if (!skb_is_gso(skb))
1796 return 0;
1797
dd225bc6
FR		 1798 err = skb_cow_head(skb, 0);
1799 if (err < 0)
1800 return err;
fd0a05ce 1801
0e2fe46c 1802 if (protocol == htons(ETH_P_IP)) {
fd0a05ce
JB		 1803 iph = skb->encapsulation ? inner_ip_hdr(skb) : ip_hdr(skb);
1804 tcph = skb->encapsulation ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1805 iph->tot_len = 0;
1806 iph->check = 0;
1807 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
1808 0, IPPROTO_TCP, 0);
1809 } else if (skb_is_gso_v6(skb)) {
1810
1811 ipv6h = skb->encapsulation ? inner_ipv6_hdr(skb)
1812 : ipv6_hdr(skb);
1813 tcph = skb->encapsulation ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1814 ipv6h->payload_len = 0;
1815 tcph->check = ~csum_ipv6_magic(&ipv6h->saddr, &ipv6h->daddr,
1816 0, IPPROTO_TCP, 0);
1817 }
1818
1819 l4len = skb->encapsulation ? inner_tcp_hdrlen(skb) : tcp_hdrlen(skb);
1820 *hdr_len = (skb->encapsulation
1821 ? (skb_inner_transport_header(skb) - skb->data)
1822 : skb_transport_offset(skb)) + l4len;
1823
1824 /* find the field values */
1825 cd_cmd = I40E_TX_CTX_DESC_TSO;
1826 cd_tso_len = skb->len - *hdr_len;
1827 cd_mss = skb_shinfo(skb)->gso_size;
829af3ac
MW		 1828 *cd_type_cmd_tso_mss |= ((u64)cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) |
1829 ((u64)cd_tso_len <<
1830 I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) |
1831 ((u64)cd_mss << I40E_TXD_CTX_QW1_MSS_SHIFT);
fd0a05ce
JB		 1832 return 1;
1833}
1834
beb0dff1
JK		 1835/**
1836 * i40e_tsyn - set up the tsyn context descriptor
1837 * @tx_ring: ptr to the ring to send
1838 * @skb: ptr to the skb we're sending
1839 * @tx_flags: the collected send information
1840 *
1841 * Returns 0 if no Tx timestamp can happen and 1 if the timestamp will happen
1842 **/
1843static int i40e_tsyn(struct i40e_ring *tx_ring, struct sk_buff *skb,
1844 u32 tx_flags, u64 *cd_type_cmd_tso_mss)
1845{
1846 struct i40e_pf *pf;
1847
1848 if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)))
1849 return 0;
1850
1851 /* Tx timestamps cannot be sampled when doing TSO */
1852 if (tx_flags & I40E_TX_FLAGS_TSO)
1853 return 0;
1854
1855 /* only timestamp the outbound packet if the user has requested it and
1856 * we are not already transmitting a packet to be timestamped
1857 */
1858 pf = i40e_netdev_to_pf(tx_ring->netdev);
1859 if (pf->ptp_tx && !pf->ptp_tx_skb) {
1860 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1861 pf->ptp_tx_skb = skb_get(skb);
1862 } else {
1863 return 0;
1864 }
1865
1866 *cd_type_cmd_tso_mss |= (u64)I40E_TX_CTX_DESC_TSYN <<
1867 I40E_TXD_CTX_QW1_CMD_SHIFT;
1868
beb0dff1
JK		 1869 return 1;
1870}
1871
fd0a05ce
JB		 1872/**
1873 * i40e_tx_enable_csum - Enable Tx checksum offloads
1874 * @skb: send buffer
1875 * @tx_flags: Tx flags currently set
1876 * @td_cmd: Tx descriptor command bits to set
1877 * @td_offset: Tx descriptor header offsets to set
1878 * @cd_tunneling: ptr to context desc bits
1879 **/
1880static void i40e_tx_enable_csum(struct sk_buff *skb, u32 tx_flags,
1881 u32 *td_cmd, u32 *td_offset,
1882 struct i40e_ring *tx_ring,
1883 u32 *cd_tunneling)
1884{
1885 struct ipv6hdr *this_ipv6_hdr;
1886 unsigned int this_tcp_hdrlen;
1887 struct iphdr *this_ip_hdr;
1888 u32 network_hdr_len;
1889 u8 l4_hdr = 0;
1890
1891 if (skb->encapsulation) {
1892 network_hdr_len = skb_inner_network_header_len(skb);
1893 this_ip_hdr = inner_ip_hdr(skb);
1894 this_ipv6_hdr = inner_ipv6_hdr(skb);
1895 this_tcp_hdrlen = inner_tcp_hdrlen(skb);
1896
1897 if (tx_flags & I40E_TX_FLAGS_IPV4) {
1898
1899 if (tx_flags & I40E_TX_FLAGS_TSO) {
1900 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV4;
1901 ip_hdr(skb)->check = 0;
1902 } else {
1903 *cd_tunneling |=
1904 I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;
1905 }
1906 } else if (tx_flags & I40E_TX_FLAGS_IPV6) {
1907 if (tx_flags & I40E_TX_FLAGS_TSO) {
1908 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV6;
1909 ip_hdr(skb)->check = 0;
1910 } else {
1911 *cd_tunneling |=
1912 I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;
1913 }
1914 }
1915
1916 /* Now set the ctx descriptor fields */
1917 *cd_tunneling |= (skb_network_header_len(skb) >> 2) <<
1918 I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT |
1919 I40E_TXD_CTX_UDP_TUNNELING |
1920 ((skb_inner_network_offset(skb) -
1921 skb_transport_offset(skb)) >> 1) <<
1922 I40E_TXD_CTX_QW0_NATLEN_SHIFT;
1923
1924 } else {
1925 network_hdr_len = skb_network_header_len(skb);
1926 this_ip_hdr = ip_hdr(skb);
1927 this_ipv6_hdr = ipv6_hdr(skb);
1928 this_tcp_hdrlen = tcp_hdrlen(skb);
1929 }
1930
1931 /* Enable IP checksum offloads */
1932 if (tx_flags & I40E_TX_FLAGS_IPV4) {
1933 l4_hdr = this_ip_hdr->protocol;
1934 /* the stack computes the IP header already, the only time we
1935 * need the hardware to recompute it is in the case of TSO.
1936 */
1937 if (tx_flags & I40E_TX_FLAGS_TSO) {
1938 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM;
1939 this_ip_hdr->check = 0;
1940 } else {
1941 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4;
1942 }
1943 /* Now set the td_offset for IP header length */
1944 *td_offset = (network_hdr_len >> 2) <<
1945 I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
1946 } else if (tx_flags & I40E_TX_FLAGS_IPV6) {
1947 l4_hdr = this_ipv6_hdr->nexthdr;
1948 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV6;
1949 /* Now set the td_offset for IP header length */
1950 *td_offset = (network_hdr_len >> 2) <<
1951 I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
1952 }
1953 /* words in MACLEN + dwords in IPLEN + dwords in L4Len */
1954 *td_offset |= (skb_network_offset(skb) >> 1) <<
1955 I40E_TX_DESC_LENGTH_MACLEN_SHIFT;
1956
1957 /* Enable L4 checksum offloads */
1958 switch (l4_hdr) {
1959 case IPPROTO_TCP:
1960 /* enable checksum offloads */
1961 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
1962 *td_offset |= (this_tcp_hdrlen >> 2) <<
1963 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1964 break;
1965 case IPPROTO_SCTP:
1966 /* enable SCTP checksum offload */
1967 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP;
1968 *td_offset |= (sizeof(struct sctphdr) >> 2) <<
1969 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1970 break;
1971 case IPPROTO_UDP:
1972 /* enable UDP checksum offload */
1973 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP;
1974 *td_offset |= (sizeof(struct udphdr) >> 2) <<
1975 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1976 break;
1977 default:
1978 break;
1979 }
1980}
1981
1982/**
1983 * i40e_create_tx_ctx Build the Tx context descriptor
1984 * @tx_ring: ring to create the descriptor on
1985 * @cd_type_cmd_tso_mss: Quad Word 1
1986 * @cd_tunneling: Quad Word 0 - bits 0-31
1987 * @cd_l2tag2: Quad Word 0 - bits 32-63
1988 **/
1989static void i40e_create_tx_ctx(struct i40e_ring *tx_ring,
1990 const u64 cd_type_cmd_tso_mss,
1991 const u32 cd_tunneling, const u32 cd_l2tag2)
1992{
1993 struct i40e_tx_context_desc *context_desc;
fc4ac67b 1994 int i = tx_ring->next_to_use;
fd0a05ce 1995
ff40dd5d
JB		 1996 if ((cd_type_cmd_tso_mss == I40E_TX_DESC_DTYPE_CONTEXT) &&
1997 !cd_tunneling && !cd_l2tag2)
fd0a05ce
JB		 1998 return;
1999
2000 /* grab the next descriptor */
fc4ac67b
AD		 2001 context_desc = I40E_TX_CTXTDESC(tx_ring, i);
2002
2003 i++;
2004 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
fd0a05ce
JB		 2005
2006 /* cpu_to_le32 and assign to struct fields */
2007 context_desc->tunneling_params = cpu_to_le32(cd_tunneling);
2008 context_desc->l2tag2 = cpu_to_le16(cd_l2tag2);
3efbbb20 2009 context_desc->rsvd = cpu_to_le16(0);
fd0a05ce
JB		 2010 context_desc->type_cmd_tso_mss = cpu_to_le64(cd_type_cmd_tso_mss);
2011}
2012
2013/**
2014 * i40e_tx_map - Build the Tx descriptor
2015 * @tx_ring: ring to send buffer on
2016 * @skb: send buffer
2017 * @first: first buffer info buffer to use
2018 * @tx_flags: collected send information
2019 * @hdr_len: size of the packet header
2020 * @td_cmd: the command field in the descriptor
2021 * @td_offset: offset for checksum or crc
2022 **/
2023static void i40e_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb,
2024 struct i40e_tx_buffer *first, u32 tx_flags,
2025 const u8 hdr_len, u32 td_cmd, u32 td_offset)
2026{
fd0a05ce
JB		 2027 unsigned int data_len = skb->data_len;
2028 unsigned int size = skb_headlen(skb);
a5e9c572 2029 struct skb_frag_struct *frag;
fd0a05ce
JB		 2030 struct i40e_tx_buffer *tx_bi;
2031 struct i40e_tx_desc *tx_desc;
a5e9c572 2032 u16 i = tx_ring->next_to_use;
fd0a05ce
JB		 2033 u32 td_tag = 0;
2034 dma_addr_t dma;
2035 u16 gso_segs;
2036
fd0a05ce
JB		 2037 if (tx_flags & I40E_TX_FLAGS_HW_VLAN) {
2038 td_cmd |= I40E_TX_DESC_CMD_IL2TAG1;
2039 td_tag = (tx_flags & I40E_TX_FLAGS_VLAN_MASK) >>
2040 I40E_TX_FLAGS_VLAN_SHIFT;
2041 }
2042
a5e9c572
AD		 2043 if (tx_flags & (I40E_TX_FLAGS_TSO | I40E_TX_FLAGS_FSO))
2044 gso_segs = skb_shinfo(skb)->gso_segs;
2045 else
2046 gso_segs = 1;
2047
2048 /* multiply data chunks by size of headers */
2049 first->bytecount = skb->len - hdr_len + (gso_segs * hdr_len);
2050 first->gso_segs = gso_segs;
2051 first->skb = skb;
2052 first->tx_flags = tx_flags;
2053
2054 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
2055
fd0a05ce 2056 tx_desc = I40E_TX_DESC(tx_ring, i);
a5e9c572
AD		 2057 tx_bi = first;
2058
2059 for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
2060 if (dma_mapping_error(tx_ring->dev, dma))
2061 goto dma_error;
2062
2063 /* record length, and DMA address */
2064 dma_unmap_len_set(tx_bi, len, size);
2065 dma_unmap_addr_set(tx_bi, dma, dma);
2066
2067 tx_desc->buffer_addr = cpu_to_le64(dma);
2068
2069 while (unlikely(size > I40E_MAX_DATA_PER_TXD)) {
fd0a05ce
JB		 2070 tx_desc->cmd_type_offset_bsz =
2071 build_ctob(td_cmd, td_offset,
2072 I40E_MAX_DATA_PER_TXD, td_tag);
2073
fd0a05ce
JB		 2074 tx_desc++;
2075 i++;
2076 if (i == tx_ring->count) {
2077 tx_desc = I40E_TX_DESC(tx_ring, 0);
2078 i = 0;
2079 }
fd0a05ce 2080
a5e9c572
AD		 2081 dma += I40E_MAX_DATA_PER_TXD;
2082 size -= I40E_MAX_DATA_PER_TXD;
fd0a05ce 2083
a5e9c572
AD		 2084 tx_desc->buffer_addr = cpu_to_le64(dma);
2085 }
fd0a05ce
JB		 2086
2087 if (likely(!data_len))
2088 break;
2089
a5e9c572
AD		 2090 tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset,
2091 size, td_tag);
fd0a05ce
JB		 2092
2093 tx_desc++;
2094 i++;
2095 if (i == tx_ring->count) {
2096 tx_desc = I40E_TX_DESC(tx_ring, 0);
2097 i = 0;
2098 }
2099
a5e9c572
AD		 2100 size = skb_frag_size(frag);
2101 data_len -= size;
fd0a05ce 2102
a5e9c572
AD		 2103 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
2104 DMA_TO_DEVICE);
fd0a05ce 2105
a5e9c572
AD		 2106 tx_bi = &tx_ring->tx_bi[i];
2107 }
fd0a05ce 2108
1943d8ba
JB		 2109 /* Place RS bit on last descriptor of any packet that spans across the
2110 * 4th descriptor (WB_STRIDE aka 0x3) in a 64B cacheline.
2111 */
2112#define WB_STRIDE 0x3
2113 if (((i & WB_STRIDE) != WB_STRIDE) &&
2114 (first <= &tx_ring->tx_bi[i]) &&
2115 (first >= &tx_ring->tx_bi[i & ~WB_STRIDE])) {
2116 tx_desc->cmd_type_offset_bsz =
2117 build_ctob(td_cmd, td_offset, size, td_tag) |
2118 cpu_to_le64((u64)I40E_TX_DESC_CMD_EOP <<
2119 I40E_TXD_QW1_CMD_SHIFT);
2120 } else {
2121 tx_desc->cmd_type_offset_bsz =
2122 build_ctob(td_cmd, td_offset, size, td_tag) |
2123 cpu_to_le64((u64)I40E_TXD_CMD <<
2124 I40E_TXD_QW1_CMD_SHIFT);
2125 }
fd0a05ce 2126
7070ce0a
AD		 2127 netdev_tx_sent_queue(netdev_get_tx_queue(tx_ring->netdev,
2128 tx_ring->queue_index),
2129 first->bytecount);
2130
a5e9c572 2131 /* set the timestamp */
fd0a05ce 2132 first->time_stamp = jiffies;
fd0a05ce
JB		 2133
2134 /* Force memory writes to complete before letting h/w
2135 * know there are new descriptors to fetch. (Only
2136 * applicable for weak-ordered memory model archs,
2137 * such as IA-64).
2138 */
2139 wmb();
2140
a5e9c572
AD		 2141 /* set next_to_watch value indicating a packet is present */
2142 first->next_to_watch = tx_desc;
2143
2144 i++;
2145 if (i == tx_ring->count)
2146 i = 0;
2147
2148 tx_ring->next_to_use = i;
2149
2150 /* notify HW of packet */
fd0a05ce 2151 writel(i, tx_ring->tail);
a5e9c572 2152
fd0a05ce
JB		 2153 return;
2154
2155dma_error:
a5e9c572 2156 dev_info(tx_ring->dev, "TX DMA map failed\n");
fd0a05ce
JB		 2157
2158 /* clear dma mappings for failed tx_bi map */
2159 for (;;) {
2160 tx_bi = &tx_ring->tx_bi[i];
a5e9c572 2161 i40e_unmap_and_free_tx_resource(tx_ring, tx_bi);
fd0a05ce
JB		 2162 if (tx_bi == first)
2163 break;
2164 if (i == 0)
2165 i = tx_ring->count;
2166 i--;
2167 }
2168
fd0a05ce
JB		 2169 tx_ring->next_to_use = i;
2170}
2171
2172/**
2173 * __i40e_maybe_stop_tx - 2nd level check for tx stop conditions
2174 * @tx_ring: the ring to be checked
2175 * @size: the size buffer we want to assure is available
2176 *
2177 * Returns -EBUSY if a stop is needed, else 0
2178 **/
2179static inline int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
2180{
2181 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
8e9dca53 2182 /* Memory barrier before checking head and tail */
fd0a05ce
JB		 2183 smp_mb();
2184
2185 /* Check again in a case another CPU has just made room available. */
2186 if (likely(I40E_DESC_UNUSED(tx_ring) < size))
2187 return -EBUSY;
2188
2189 /* A reprieve! - use start_queue because it doesn't call schedule */
2190 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
2191 ++tx_ring->tx_stats.restart_queue;
2192 return 0;
2193}
2194
2195/**
2196 * i40e_maybe_stop_tx - 1st level check for tx stop conditions
2197 * @tx_ring: the ring to be checked
2198 * @size: the size buffer we want to assure is available
2199 *
2200 * Returns 0 if stop is not needed
2201 **/
2202static int i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
2203{
2204 if (likely(I40E_DESC_UNUSED(tx_ring) >= size))
2205 return 0;
2206 return __i40e_maybe_stop_tx(tx_ring, size);
2207}
2208
2209/**
2210 * i40e_xmit_descriptor_count - calculate number of tx descriptors needed
2211 * @skb: send buffer
2212 * @tx_ring: ring to send buffer on
2213 *
2214 * Returns number of data descriptors needed for this skb. Returns 0 to indicate
2215 * there is not enough descriptors available in this ring since we need at least
2216 * one descriptor.
2217 **/
2218static int i40e_xmit_descriptor_count(struct sk_buff *skb,
2219 struct i40e_ring *tx_ring)
2220{
fd0a05ce 2221 unsigned int f;
fd0a05ce
JB		 2222 int count = 0;
2223
2224 /* need: 1 descriptor per page * PAGE_SIZE/I40E_MAX_DATA_PER_TXD,
2225 * + 1 desc for skb_head_len/I40E_MAX_DATA_PER_TXD,
be560521 2226 * + 4 desc gap to avoid the cache line where head is,
fd0a05ce
JB		 2227 * + 1 desc for context descriptor,
2228 * otherwise try next time
2229 */
fd0a05ce
JB		 2230 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
2231 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
980093eb 2232
fd0a05ce 2233 count += TXD_USE_COUNT(skb_headlen(skb));
be560521 2234 if (i40e_maybe_stop_tx(tx_ring, count + 4 + 1)) {
fd0a05ce
JB		 2235 tx_ring->tx_stats.tx_busy++;
2236 return 0;
2237 }
2238 return count;
2239}
2240
2241/**
2242 * i40e_xmit_frame_ring - Sends buffer on Tx ring
2243 * @skb: send buffer
2244 * @tx_ring: ring to send buffer on
2245 *
2246 * Returns NETDEV_TX_OK if sent, else an error code
2247 **/
2248static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb,
2249 struct i40e_ring *tx_ring)
2250{
2251 u64 cd_type_cmd_tso_mss = I40E_TX_DESC_DTYPE_CONTEXT;
2252 u32 cd_tunneling = 0, cd_l2tag2 = 0;
2253 struct i40e_tx_buffer *first;
2254 u32 td_offset = 0;
2255 u32 tx_flags = 0;
2256 __be16 protocol;
2257 u32 td_cmd = 0;
2258 u8 hdr_len = 0;
beb0dff1 2259 int tsyn;
fd0a05ce
JB		 2260 int tso;
2261 if (0 == i40e_xmit_descriptor_count(skb, tx_ring))
2262 return NETDEV_TX_BUSY;
2263
2264 /* prepare the xmit flags */
2265 if (i40e_tx_prepare_vlan_flags(skb, tx_ring, &tx_flags))
2266 goto out_drop;
2267
2268 /* obtain protocol of skb */
2269 protocol = skb->protocol;
2270
2271 /* record the location of the first descriptor for this packet */
2272 first = &tx_ring->tx_bi[tx_ring->next_to_use];
2273
2274 /* setup IPv4/IPv6 offloads */
0e2fe46c 2275 if (protocol == htons(ETH_P_IP))
fd0a05ce 2276 tx_flags |= I40E_TX_FLAGS_IPV4;
0e2fe46c 2277 else if (protocol == htons(ETH_P_IPV6))
fd0a05ce
JB		 2278 tx_flags |= I40E_TX_FLAGS_IPV6;
2279
2280 tso = i40e_tso(tx_ring, skb, tx_flags, protocol, &hdr_len,
2281 &cd_type_cmd_tso_mss, &cd_tunneling);
2282
2283 if (tso < 0)
2284 goto out_drop;
2285 else if (tso)
2286 tx_flags |= I40E_TX_FLAGS_TSO;
2287
2288 skb_tx_timestamp(skb);
2289
beb0dff1
JK		 2290 tsyn = i40e_tsyn(tx_ring, skb, tx_flags, &cd_type_cmd_tso_mss);
2291
2292 if (tsyn)
2293 tx_flags |= I40E_TX_FLAGS_TSYN;
2294
b1941306
AD		 2295 /* always enable CRC insertion offload */
2296 td_cmd |= I40E_TX_DESC_CMD_ICRC;
2297
fd0a05ce 2298 /* Always offload the checksum, since it's in the data descriptor */
b1941306 2299 if (skb->ip_summed == CHECKSUM_PARTIAL) {
fd0a05ce
JB		 2300 tx_flags |= I40E_TX_FLAGS_CSUM;
2301
fd0a05ce
JB		 2302 i40e_tx_enable_csum(skb, tx_flags, &td_cmd, &td_offset,
2303 tx_ring, &cd_tunneling);
b1941306 2304 }
fd0a05ce
JB		 2305
2306 i40e_create_tx_ctx(tx_ring, cd_type_cmd_tso_mss,
2307 cd_tunneling, cd_l2tag2);
2308
2309 /* Add Flow Director ATR if it's enabled.
2310 *
2311 * NOTE: this must always be directly before the data descriptor.
2312 */
2313 i40e_atr(tx_ring, skb, tx_flags, protocol);
2314
2315 i40e_tx_map(tx_ring, skb, first, tx_flags, hdr_len,
2316 td_cmd, td_offset);
2317
2318 i40e_maybe_stop_tx(tx_ring, DESC_NEEDED);
2319
2320 return NETDEV_TX_OK;
2321
2322out_drop:
2323 dev_kfree_skb_any(skb);
2324 return NETDEV_TX_OK;
2325}
2326
2327/**
2328 * i40e_lan_xmit_frame - Selects the correct VSI and Tx queue to send buffer
2329 * @skb: send buffer
2330 * @netdev: network interface device structure
2331 *
2332 * Returns NETDEV_TX_OK if sent, else an error code
2333 **/
2334netdev_tx_t i40e_lan_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
2335{
2336 struct i40e_netdev_priv *np = netdev_priv(netdev);
2337 struct i40e_vsi *vsi = np->vsi;
9f65e15b 2338 struct i40e_ring *tx_ring = vsi->tx_rings[skb->queue_mapping];
fd0a05ce
JB		 2339
2340 /* hardware can't handle really short frames, hardware padding works
2341 * beyond this point
2342 */
2343 if (unlikely(skb->len < I40E_MIN_TX_LEN)) {
2344 if (skb_pad(skb, I40E_MIN_TX_LEN - skb->len))
2345 return NETDEV_TX_OK;
2346 skb->len = I40E_MIN_TX_LEN;
2347 skb_set_tail_pointer(skb, I40E_MIN_TX_LEN);
2348 }
2349
2350 return i40e_xmit_frame_ring(skb, tx_ring);
2351}
Linux 6.x block layer and io_uring tree(s)