projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
bnx2x: AER revised
[linux-2.6-block.git] / drivers / net / ethernet / broadcom / bnx2x / bnx2x_cmn.c
CommitLineData
9f6c9258
DK		 1/* bnx2x_cmn.c: Broadcom Everest network driver.
2 *
247fa82b 3 * Copyright (c) 2007-2013 Broadcom Corporation
9f6c9258
DK		 4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation.
8 *
9 * Maintained by: Eilon Greenstein <eilong@broadcom.com>
10 * Written by: Eliezer Tamir
11 * Based on code from Michael Chan's bnx2 driver
12 * UDP CSUM errata workaround by Arik Gendelman
13 * Slowpath and fastpath rework by Vladislav Zolotarov
14 * Statistics and Link management by Yitchak Gertner
15 *
16 */
17
f1deab50
JP		 18#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
19
9f6c9258 20#include <linux/etherdevice.h>
9bcc0893 21#include <linux/if_vlan.h>
a6b7a407 22#include <linux/interrupt.h>
9f6c9258 23#include <linux/ip.h>
9969085e 24#include <net/tcp.h>
f2e0899f 25#include <net/ipv6.h>
7f3e01fe 26#include <net/ip6_checksum.h>
c0cba59e 27#include <linux/prefetch.h>
9f6c9258 28#include "bnx2x_cmn.h"
523224a3 29#include "bnx2x_init.h"
042181f5 30#include "bnx2x_sp.h"
9f6c9258 31
b3b83c3f
DK		 32/**
33 * bnx2x_move_fp - move content of the fastpath structure.
34 *
35 * @bp: driver handle
36 * @from: source FP index
37 * @to: destination FP index
38 *
39 * Makes sure the contents of the bp->fp[to].napi is kept
72754080
AE		 40 * intact. This is done by first copying the napi struct from
41 * the target to the source, and then mem copying the entire
65565884
MS		 42 * source onto the target. Update txdata pointers and related
43 * content.
b3b83c3f
DK		 44 */
45static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
46{
47 struct bnx2x_fastpath *from_fp = &bp->fp[from];
48 struct bnx2x_fastpath *to_fp = &bp->fp[to];
15192a8c
BW		 49 struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
50 struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
51 struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
52 struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
65565884
MS		 53 int old_max_eth_txqs, new_max_eth_txqs;
54 int old_txdata_index = 0, new_txdata_index = 0;
72754080
AE		 55
56 /* Copy the NAPI object as it has been already initialized */
57 from_fp->napi = to_fp->napi;
58
b3b83c3f
DK		 59 /* Move bnx2x_fastpath contents */
60 memcpy(to_fp, from_fp, sizeof(*to_fp));
61 to_fp->index = to;
65565884 62
15192a8c
BW		 63 /* move sp_objs contents as well, as their indices match fp ones */
64 memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
65
66 /* move fp_stats contents as well, as their indices match fp ones */
67 memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
68
65565884
MS		 69 /* Update txdata pointers in fp and move txdata content accordingly:
70 * Each fp consumes 'max_cos' txdata structures, so the index should be
71 * decremented by max_cos x delta.
72 */
73
74 old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
75 new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
76 (bp)->max_cos;
77 if (from == FCOE_IDX(bp)) {
78 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
79 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
80 }
81
4864a16a
YM		 82 memcpy(&bp->bnx2x_txq[new_txdata_index],
83 &bp->bnx2x_txq[old_txdata_index],
65565884
MS		 84 sizeof(struct bnx2x_fp_txdata));
85 to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
b3b83c3f
DK		 86}
87
8ca5e17e
AE		 88/**
89 * bnx2x_fill_fw_str - Fill buffer with FW version string.
90 *
91 * @bp: driver handle
92 * @buf: character buffer to fill with the fw name
93 * @buf_len: length of the above buffer
94 *
95 */
96void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len)
97{
98 if (IS_PF(bp)) {
99 u8 phy_fw_ver[PHY_FW_VER_LEN];
100
101 phy_fw_ver[0] = '\0';
102 bnx2x_get_ext_phy_fw_version(&bp->link_params,
103 phy_fw_ver, PHY_FW_VER_LEN);
104 strlcpy(buf, bp->fw_ver, buf_len);
105 snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver),
106 "bc %d.%d.%d%s%s",
107 (bp->common.bc_ver & 0xff0000) >> 16,
108 (bp->common.bc_ver & 0xff00) >> 8,
109 (bp->common.bc_ver & 0xff),
110 ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver);
111 } else {
6411280a 112 bnx2x_vf_fill_fw_str(bp, buf, buf_len);
8ca5e17e
AE		 113 }
114}
115
4864a16a
YM		 116/**
117 * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
118 *
119 * @bp: driver handle
120 * @delta: number of eth queues which were not allocated
121 */
122static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
123{
124 int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
125
126 /* Queue pointer cannot be re-set on an fp-basis, as moving pointer
127 * backward along the array could cause memory to be overriden
128 */
129 for (cos = 1; cos < bp->max_cos; cos++) {
130 for (i = 0; i < old_eth_num - delta; i++) {
131 struct bnx2x_fastpath *fp = &bp->fp[i];
132 int new_idx = cos * (old_eth_num - delta) + i;
133
134 memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
135 sizeof(struct bnx2x_fp_txdata));
136 fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
137 }
138 }
139}
140
619c5cb6
VZ		 141int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
142
9f6c9258
DK		 143/* free skb in the packet ring at pos idx
144 * return idx of last bd freed
145 */
6383c0b3 146static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
2df1a70a
TH		 147 u16 idx, unsigned int *pkts_compl,
148 unsigned int *bytes_compl)
9f6c9258 149{
6383c0b3 150 struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
9f6c9258
DK		 151 struct eth_tx_start_bd *tx_start_bd;
152 struct eth_tx_bd *tx_data_bd;
153 struct sk_buff *skb = tx_buf->skb;
154 u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
155 int nbd;
156
157 /* prefetch skb end pointer to speedup dev_kfree_skb() */
158 prefetch(&skb->end);
159
51c1a580 160 DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d buff @(%p)->skb %p\n",
6383c0b3 161 txdata->txq_index, idx, tx_buf, skb);
9f6c9258
DK		 162
163 /* unmap first bd */
6383c0b3 164 tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
9f6c9258 165 dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
4bca60f4 166 BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
9f6c9258 167
619c5cb6 168
9f6c9258
DK		 169 nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
170#ifdef BNX2X_STOP_ON_ERROR
171 if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
172 BNX2X_ERR("BAD nbd!\n");
173 bnx2x_panic();
174 }
175#endif
176 new_cons = nbd + tx_buf->first_bd;
177
178 /* Get the next bd */
179 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
180
181 /* Skip a parse bd... */
182 --nbd;
183 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
184
185 /* ...and the TSO split header bd since they have no mapping */
186 if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
187 --nbd;
188 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
189 }
190
191 /* now free frags */
192 while (nbd > 0) {
193
6383c0b3 194 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
9f6c9258
DK		 195 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
196 BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
197 if (--nbd)
198 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
199 }
200
201 /* release skb */
202 WARN_ON(!skb);
d8290ae5 203 if (likely(skb)) {
2df1a70a
TH		 204 (*pkts_compl)++;
205 (*bytes_compl) += skb->len;
206 }
d8290ae5 207
40955532 208 dev_kfree_skb_any(skb);
9f6c9258
DK		 209 tx_buf->first_bd = 0;
210 tx_buf->skb = NULL;
211
212 return new_cons;
213}
214
6383c0b3 215int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
9f6c9258 216{
9f6c9258 217 struct netdev_queue *txq;
6383c0b3 218 u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
2df1a70a 219 unsigned int pkts_compl = 0, bytes_compl = 0;
9f6c9258
DK		 220
221#ifdef BNX2X_STOP_ON_ERROR
222 if (unlikely(bp->panic))
223 return -1;
224#endif
225
6383c0b3
AE		 226 txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
227 hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
228 sw_cons = txdata->tx_pkt_cons;
9f6c9258
DK		 229
230 while (sw_cons != hw_cons) {
231 u16 pkt_cons;
232
233 pkt_cons = TX_BD(sw_cons);
234
51c1a580
MS		 235 DP(NETIF_MSG_TX_DONE,
236 "queue[%d]: hw_cons %u sw_cons %u pkt_cons %u\n",
6383c0b3 237 txdata->txq_index, hw_cons, sw_cons, pkt_cons);
9f6c9258 238
2df1a70a 239 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
2de67439 240 &pkts_compl, &bytes_compl);
2df1a70a 241
9f6c9258
DK		 242 sw_cons++;
243 }
244
2df1a70a
TH		 245 netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
246
6383c0b3
AE		 247 txdata->tx_pkt_cons = sw_cons;
248 txdata->tx_bd_cons = bd_cons;
9f6c9258
DK		 249
250 /* Need to make the tx_bd_cons update visible to start_xmit()
251 * before checking for netif_tx_queue_stopped(). Without the
252 * memory barrier, there is a small possibility that
253 * start_xmit() will miss it and cause the queue to be stopped
254 * forever.
619c5cb6
VZ		 255 * On the other hand we need an rmb() here to ensure the proper
256 * ordering of bit testing in the following
257 * netif_tx_queue_stopped(txq) call.
9f6c9258
DK		 258 */
259 smp_mb();
260
9f6c9258
DK		 261 if (unlikely(netif_tx_queue_stopped(txq))) {
262 /* Taking tx_lock() is needed to prevent reenabling the queue
263 * while it's empty. This could have happen if rx_action() gets
264 * suspended in bnx2x_tx_int() after the condition before
265 * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
266 *
267 * stops the queue->sees fresh tx_bd_cons->releases the queue->
268 * sends some packets consuming the whole queue again->
269 * stops the queue
270 */
271
272 __netif_tx_lock(txq, smp_processor_id());
273
274 if ((netif_tx_queue_stopped(txq)) &&
275 (bp->state == BNX2X_STATE_OPEN) &&
7df2dc6b 276 (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
9f6c9258
DK		 277 netif_tx_wake_queue(txq);
278
279 __netif_tx_unlock(txq);
280 }
281 return 0;
282}
283
284static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
285 u16 idx)
286{
287 u16 last_max = fp->last_max_sge;
288
289 if (SUB_S16(idx, last_max) > 0)
290 fp->last_max_sge = idx;
291}
292
621b4d66
DK		 293static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
294 u16 sge_len,
295 struct eth_end_agg_rx_cqe *cqe)
9f6c9258
DK		 296{
297 struct bnx2x *bp = fp->bp;
9f6c9258
DK		 298 u16 last_max, last_elem, first_elem;
299 u16 delta = 0;
300 u16 i;
301
302 if (!sge_len)
303 return;
304
305 /* First mark all used pages */
306 for (i = 0; i < sge_len; i++)
619c5cb6 307 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
621b4d66 308 RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
9f6c9258
DK		 309
310 DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
621b4d66 311 sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
9f6c9258
DK		 312
313 /* Here we assume that the last SGE index is the biggest */
314 prefetch((void *)(fp->sge_mask));
523224a3 315 bnx2x_update_last_max_sge(fp,
621b4d66 316 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
9f6c9258
DK		 317
318 last_max = RX_SGE(fp->last_max_sge);
619c5cb6
VZ		 319 last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
320 first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
9f6c9258
DK		 321
322 /* If ring is not full */
323 if (last_elem + 1 != first_elem)
324 last_elem++;
325
326 /* Now update the prod */
327 for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
328 if (likely(fp->sge_mask[i]))
329 break;
330
619c5cb6
VZ		 331 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
332 delta += BIT_VEC64_ELEM_SZ;
9f6c9258
DK		 333 }
334
335 if (delta > 0) {
336 fp->rx_sge_prod += delta;
337 /* clear page-end entries */
338 bnx2x_clear_sge_mask_next_elems(fp);
339 }
340
341 DP(NETIF_MSG_RX_STATUS,
342 "fp->last_max_sge = %d fp->rx_sge_prod = %d\n",
343 fp->last_max_sge, fp->rx_sge_prod);
344}
345
2de67439 346/* Get Toeplitz hash value in the skb using the value from the
e52fcb24
ED		 347 * CQE (calculated by HW).
348 */
349static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
a334b5fb
ED		 350 const struct eth_fast_path_rx_cqe *cqe,
351 bool *l4_rxhash)
e52fcb24 352{
2de67439 353 /* Get Toeplitz hash from CQE */
e52fcb24 354 if ((bp->dev->features & NETIF_F_RXHASH) &&
a334b5fb
ED		 355 (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
356 enum eth_rss_hash_type htype;
357
358 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
359 *l4_rxhash = (htype == TCP_IPV4_HASH_TYPE) ||
360 (htype == TCP_IPV6_HASH_TYPE);
e52fcb24 361 return le32_to_cpu(cqe->rss_hash_result);
a334b5fb
ED		 362 }
363 *l4_rxhash = false;
e52fcb24
ED		 364 return 0;
365}
366
9f6c9258 367static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
e52fcb24 368 u16 cons, u16 prod,
619c5cb6 369 struct eth_fast_path_rx_cqe *cqe)
9f6c9258
DK		 370{
371 struct bnx2x *bp = fp->bp;
372 struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
373 struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
374 struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
375 dma_addr_t mapping;
619c5cb6
VZ		 376 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
377 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
9f6c9258 378
619c5cb6
VZ		 379 /* print error if current state != stop */
380 if (tpa_info->tpa_state != BNX2X_TPA_STOP)
9f6c9258
DK		 381 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
382
e52fcb24 383 /* Try to map an empty data buffer from the aggregation info */
619c5cb6 384 mapping = dma_map_single(&bp->pdev->dev,
e52fcb24 385 first_buf->data + NET_SKB_PAD,
619c5cb6
VZ		 386 fp->rx_buf_size, DMA_FROM_DEVICE);
387 /*
388 * ...if it fails - move the skb from the consumer to the producer
389 * and set the current aggregation state as ERROR to drop it
390 * when TPA_STOP arrives.
391 */
392
393 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
394 /* Move the BD from the consumer to the producer */
e52fcb24 395 bnx2x_reuse_rx_data(fp, cons, prod);
619c5cb6
VZ		 396 tpa_info->tpa_state = BNX2X_TPA_ERROR;
397 return;
398 }
9f6c9258 399
e52fcb24
ED		 400 /* move empty data from pool to prod */
401 prod_rx_buf->data = first_buf->data;
619c5cb6 402 dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
e52fcb24 403 /* point prod_bd to new data */
9f6c9258
DK		 404 prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
405 prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
406
619c5cb6
VZ		 407 /* move partial skb from cons to pool (don't unmap yet) */
408 *first_buf = *cons_rx_buf;
409
410 /* mark bin state as START */
411 tpa_info->parsing_flags =
412 le16_to_cpu(cqe->pars_flags.flags);
413 tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
414 tpa_info->tpa_state = BNX2X_TPA_START;
415 tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
416 tpa_info->placement_offset = cqe->placement_offset;
a334b5fb 417 tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->l4_rxhash);
621b4d66
DK		 418 if (fp->mode == TPA_MODE_GRO) {
419 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
924d75ab 420 tpa_info->full_page = SGE_PAGES / gro_size * gro_size;
621b4d66
DK		 421 tpa_info->gro_size = gro_size;
422 }
619c5cb6 423
9f6c9258
DK		 424#ifdef BNX2X_STOP_ON_ERROR
425 fp->tpa_queue_used |= (1 << queue);
426#ifdef _ASM_GENERIC_INT_L64_H
427 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
428#else
429 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
430#endif
431 fp->tpa_queue_used);
432#endif
433}
434
e4e3c02a
VZ		 435/* Timestamp option length allowed for TPA aggregation:
436 *
437 * nop nop kind length echo val
438 */
439#define TPA_TSTAMP_OPT_LEN 12
440/**
cbf1de72 441 * bnx2x_set_gro_params - compute GRO values
e4e3c02a 442 *
cbf1de72 443 * @skb: packet skb
e8920674
DK		 444 * @parsing_flags: parsing flags from the START CQE
445 * @len_on_bd: total length of the first packet for the
446 * aggregation.
cbf1de72 447 * @pkt_len: length of all segments
e8920674
DK		 448 *
449 * Approximate value of the MSS for this aggregation calculated using
450 * the first packet of it.
2de67439 451 * Compute number of aggregated segments, and gso_type.
e4e3c02a 452 */
cbf1de72 453static void bnx2x_set_gro_params(struct sk_buff *skb, u16 parsing_flags,
ab5777d7
YM		 454 u16 len_on_bd, unsigned int pkt_len,
455 u16 num_of_coalesced_segs)
e4e3c02a 456{
cbf1de72 457 /* TPA aggregation won't have either IP options or TCP options
619c5cb6 458 * other than timestamp or IPv6 extension headers.
e4e3c02a 459 */
619c5cb6
VZ		 460 u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
461
462 if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
cbf1de72 463 PRS_FLAG_OVERETH_IPV6) {
619c5cb6 464 hdrs_len += sizeof(struct ipv6hdr);
cbf1de72
YM		 465 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
466 } else {
619c5cb6 467 hdrs_len += sizeof(struct iphdr);
cbf1de72
YM		 468 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
469 }
e4e3c02a
VZ		 470
471 /* Check if there was a TCP timestamp, if there is it's will
472 * always be 12 bytes length: nop nop kind length echo val.
473 *
474 * Otherwise FW would close the aggregation.
475 */
476 if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
477 hdrs_len += TPA_TSTAMP_OPT_LEN;
478
cbf1de72
YM		 479 skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len;
480
481 /* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count
482 * to skb_shinfo(skb)->gso_segs
483 */
ab5777d7 484 NAPI_GRO_CB(skb)->count = num_of_coalesced_segs;
e4e3c02a
VZ		 485}
486
1191cb83
ED		 487static int bnx2x_alloc_rx_sge(struct bnx2x *bp,
488 struct bnx2x_fastpath *fp, u16 index)
489{
490 struct page *page = alloc_pages(GFP_ATOMIC, PAGES_PER_SGE_SHIFT);
491 struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
492 struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
493 dma_addr_t mapping;
494
495 if (unlikely(page == NULL)) {
496 BNX2X_ERR("Can't alloc sge\n");
497 return -ENOMEM;
498 }
499
500 mapping = dma_map_page(&bp->pdev->dev, page, 0,
924d75ab 501 SGE_PAGES, DMA_FROM_DEVICE);
1191cb83
ED		 502 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
503 __free_pages(page, PAGES_PER_SGE_SHIFT);
504 BNX2X_ERR("Can't map sge\n");
505 return -ENOMEM;
506 }
507
508 sw_buf->page = page;
509 dma_unmap_addr_set(sw_buf, mapping, mapping);
510
511 sge->addr_hi = cpu_to_le32(U64_HI(mapping));
512 sge->addr_lo = cpu_to_le32(U64_LO(mapping));
513
514 return 0;
515}
516
9f6c9258 517static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
621b4d66
DK		 518 struct bnx2x_agg_info *tpa_info,
519 u16 pages,
520 struct sk_buff *skb,
619c5cb6
VZ		 521 struct eth_end_agg_rx_cqe *cqe,
522 u16 cqe_idx)
9f6c9258
DK		 523{
524 struct sw_rx_page *rx_pg, old_rx_pg;
621b4d66
DK		 525 u32 i, frag_len, frag_size;
526 int err, j, frag_id = 0;
619c5cb6 527 u16 len_on_bd = tpa_info->len_on_bd;
621b4d66 528 u16 full_page = 0, gro_size = 0;
9f6c9258 529
619c5cb6 530 frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
621b4d66
DK		 531
532 if (fp->mode == TPA_MODE_GRO) {
533 gro_size = tpa_info->gro_size;
534 full_page = tpa_info->full_page;
535 }
9f6c9258
DK		 536
537 /* This is needed in order to enable forwarding support */
cbf1de72
YM		 538 if (frag_size)
539 bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd,
ab5777d7
YM		 540 le16_to_cpu(cqe->pkt_len),
541 le16_to_cpu(cqe->num_of_coalesced_segs));
621b4d66 542
9f6c9258 543#ifdef BNX2X_STOP_ON_ERROR
924d75ab 544 if (pages > min_t(u32, 8, MAX_SKB_FRAGS) * SGE_PAGES) {
9f6c9258
DK		 545 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
546 pages, cqe_idx);
619c5cb6 547 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
9f6c9258
DK		 548 bnx2x_panic();
549 return -EINVAL;
550 }
551#endif
552
553 /* Run through the SGL and compose the fragmented skb */
554 for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
619c5cb6 555 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
9f6c9258
DK		 556
557 /* FW gives the indices of the SGE as if the ring is an array
558 (meaning that "next" element will consume 2 indices) */
621b4d66
DK		 559 if (fp->mode == TPA_MODE_GRO)
560 frag_len = min_t(u32, frag_size, (u32)full_page);
561 else /* LRO */
924d75ab 562 frag_len = min_t(u32, frag_size, (u32)SGE_PAGES);
621b4d66 563
9f6c9258
DK		 564 rx_pg = &fp->rx_page_ring[sge_idx];
565 old_rx_pg = *rx_pg;
566
567 /* If we fail to allocate a substitute page, we simply stop
568 where we are and drop the whole packet */
569 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
570 if (unlikely(err)) {
15192a8c 571 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
9f6c9258
DK		 572 return err;
573 }
574
575 /* Unmap the page as we r going to pass it to the stack */
576 dma_unmap_page(&bp->pdev->dev,
577 dma_unmap_addr(&old_rx_pg, mapping),
924d75ab 578 SGE_PAGES, DMA_FROM_DEVICE);
9f6c9258 579 /* Add one frag and update the appropriate fields in the skb */
621b4d66
DK		 580 if (fp->mode == TPA_MODE_LRO)
581 skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
582 else { /* GRO */
583 int rem;
584 int offset = 0;
585 for (rem = frag_len; rem > 0; rem -= gro_size) {
586 int len = rem > gro_size ? gro_size : rem;
587 skb_fill_page_desc(skb, frag_id++,
588 old_rx_pg.page, offset, len);
589 if (offset)
590 get_page(old_rx_pg.page);
591 offset += len;
592 }
593 }
9f6c9258
DK		 594
595 skb->data_len += frag_len;
924d75ab 596 skb->truesize += SGE_PAGES;
9f6c9258
DK		 597 skb->len += frag_len;
598
599 frag_size -= frag_len;
600 }
601
602 return 0;
603}
604
d46d132c
ED		 605static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
606{
607 if (fp->rx_frag_size)
608 put_page(virt_to_head_page(data));
609 else
610 kfree(data);
611}
612
613static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp)
614{
615 if (fp->rx_frag_size)
616 return netdev_alloc_frag(fp->rx_frag_size);
617
618 return kmalloc(fp->rx_buf_size + NET_SKB_PAD, GFP_ATOMIC);
619}
620
9969085e
YM		 621#ifdef CONFIG_INET
622static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb)
623{
624 const struct iphdr *iph = ip_hdr(skb);
625 struct tcphdr *th;
626
627 skb_set_transport_header(skb, sizeof(struct iphdr));
628 th = tcp_hdr(skb);
629
630 th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb),
631 iph->saddr, iph->daddr, 0);
632}
633
634static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb)
635{
636 struct ipv6hdr *iph = ipv6_hdr(skb);
637 struct tcphdr *th;
638
639 skb_set_transport_header(skb, sizeof(struct ipv6hdr));
640 th = tcp_hdr(skb);
641
642 th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb),
643 &iph->saddr, &iph->daddr, 0);
644}
645#endif
646
647static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp,
648 struct sk_buff *skb)
649{
650#ifdef CONFIG_INET
cbf1de72 651 if (skb_shinfo(skb)->gso_size) {
9969085e
YM		 652 skb_set_network_header(skb, 0);
653 switch (be16_to_cpu(skb->protocol)) {
654 case ETH_P_IP:
655 bnx2x_gro_ip_csum(bp, skb);
656 break;
657 case ETH_P_IPV6:
658 bnx2x_gro_ipv6_csum(bp, skb);
659 break;
660 default:
661 BNX2X_ERR("FW GRO supports only IPv4/IPv6, not 0x%04x\n",
662 be16_to_cpu(skb->protocol));
663 }
664 tcp_gro_complete(skb);
665 }
666#endif
667 napi_gro_receive(&fp->napi, skb);
668}
669
1191cb83
ED		 670static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
671 struct bnx2x_agg_info *tpa_info,
672 u16 pages,
673 struct eth_end_agg_rx_cqe *cqe,
674 u16 cqe_idx)
9f6c9258 675{
619c5cb6 676 struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
621b4d66 677 u8 pad = tpa_info->placement_offset;
619c5cb6 678 u16 len = tpa_info->len_on_bd;
e52fcb24 679 struct sk_buff *skb = NULL;
621b4d66 680 u8 *new_data, *data = rx_buf->data;
619c5cb6
VZ		 681 u8 old_tpa_state = tpa_info->tpa_state;
682
683 tpa_info->tpa_state = BNX2X_TPA_STOP;
684
685 /* If we there was an error during the handling of the TPA_START -
686 * drop this aggregation.
687 */
688 if (old_tpa_state == BNX2X_TPA_ERROR)
689 goto drop;
690
e52fcb24 691 /* Try to allocate the new data */
d46d132c 692 new_data = bnx2x_frag_alloc(fp);
9f6c9258
DK		 693 /* Unmap skb in the pool anyway, as we are going to change
694 pool entry status to BNX2X_TPA_STOP even if new skb allocation
695 fails. */
696 dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
a8c94b91 697 fp->rx_buf_size, DMA_FROM_DEVICE);
e52fcb24 698 if (likely(new_data))
d46d132c 699 skb = build_skb(data, fp->rx_frag_size);
9f6c9258 700
e52fcb24 701 if (likely(skb)) {
9f6c9258 702#ifdef BNX2X_STOP_ON_ERROR
a8c94b91 703 if (pad + len > fp->rx_buf_size) {
51c1a580 704 BNX2X_ERR("skb_put is about to fail... pad %d len %d rx_buf_size %d\n",
a8c94b91 705 pad, len, fp->rx_buf_size);
9f6c9258
DK		 706 bnx2x_panic();
707 return;
708 }
709#endif
710
e52fcb24 711 skb_reserve(skb, pad + NET_SKB_PAD);
9f6c9258 712 skb_put(skb, len);
e52fcb24 713 skb->rxhash = tpa_info->rxhash;
a334b5fb 714 skb->l4_rxhash = tpa_info->l4_rxhash;
9f6c9258
DK		 715
716 skb->protocol = eth_type_trans(skb, bp->dev);
717 skb->ip_summed = CHECKSUM_UNNECESSARY;
718
621b4d66
DK		 719 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
720 skb, cqe, cqe_idx)) {
619c5cb6
VZ		 721 if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
722 __vlan_hwaccel_put_tag(skb, tpa_info->vlan_tag);
9969085e 723 bnx2x_gro_receive(bp, fp, skb);
9f6c9258 724 } else {
51c1a580
MS		 725 DP(NETIF_MSG_RX_STATUS,
726 "Failed to allocate new pages - dropping packet!\n");
40955532 727 dev_kfree_skb_any(skb);
9f6c9258
DK		 728 }
729
730
e52fcb24
ED		 731 /* put new data in bin */
732 rx_buf->data = new_data;
9f6c9258 733
619c5cb6 734 return;
9f6c9258 735 }
d46d132c 736 bnx2x_frag_free(fp, new_data);
619c5cb6
VZ		 737drop:
738 /* drop the packet and keep the buffer in the bin */
739 DP(NETIF_MSG_RX_STATUS,
740 "Failed to allocate or map a new skb - dropping packet!\n");
15192a8c 741 bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
9f6c9258
DK		 742}
743
1191cb83
ED		 744static int bnx2x_alloc_rx_data(struct bnx2x *bp,
745 struct bnx2x_fastpath *fp, u16 index)
746{
747 u8 *data;
748 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
749 struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
750 dma_addr_t mapping;
751
d46d132c 752 data = bnx2x_frag_alloc(fp);
1191cb83
ED		 753 if (unlikely(data == NULL))
754 return -ENOMEM;
755
756 mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
757 fp->rx_buf_size,
758 DMA_FROM_DEVICE);
759 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
d46d132c 760 bnx2x_frag_free(fp, data);
1191cb83
ED		 761 BNX2X_ERR("Can't map rx data\n");
762 return -ENOMEM;
763 }
764
765 rx_buf->data = data;
766 dma_unmap_addr_set(rx_buf, mapping, mapping);
767
768 rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
769 rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
770
771 return 0;
772}
773
15192a8c
BW		 774static
775void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
776 struct bnx2x_fastpath *fp,
777 struct bnx2x_eth_q_stats *qstats)
d6cb3e41 778{
e488921f
MS		 779 /* Do nothing if no L4 csum validation was done.
780 * We do not check whether IP csum was validated. For IPv4 we assume
781 * that if the card got as far as validating the L4 csum, it also
782 * validated the IP csum. IPv6 has no IP csum.
783 */
d6cb3e41 784 if (cqe->fast_path_cqe.status_flags &
e488921f 785 ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
d6cb3e41
ED		 786 return;
787
e488921f 788 /* If L4 validation was done, check if an error was found. */
d6cb3e41
ED		 789
790 if (cqe->fast_path_cqe.type_error_flags &
791 (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
792 ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
15192a8c 793 qstats->hw_csum_err++;
d6cb3e41
ED		 794 else
795 skb->ip_summed = CHECKSUM_UNNECESSARY;
796}
9f6c9258
DK		 797
798int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
799{
800 struct bnx2x *bp = fp->bp;
801 u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
802 u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
803 int rx_pkt = 0;
804
805#ifdef BNX2X_STOP_ON_ERROR
806 if (unlikely(bp->panic))
807 return 0;
808#endif
809
810 /* CQ "next element" is of the size of the regular element,
811 that's why it's ok here */
812 hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
813 if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
814 hw_comp_cons++;
815
816 bd_cons = fp->rx_bd_cons;
817 bd_prod = fp->rx_bd_prod;
818 bd_prod_fw = bd_prod;
819 sw_comp_cons = fp->rx_comp_cons;
820 sw_comp_prod = fp->rx_comp_prod;
821
822 /* Memory barrier necessary as speculative reads of the rx
823 * buffer can be ahead of the index in the status block
824 */
825 rmb();
826
827 DP(NETIF_MSG_RX_STATUS,
828 "queue[%d]: hw_comp_cons %u sw_comp_cons %u\n",
829 fp->index, hw_comp_cons, sw_comp_cons);
830
831 while (sw_comp_cons != hw_comp_cons) {
832 struct sw_rx_bd *rx_buf = NULL;
833 struct sk_buff *skb;
834 union eth_rx_cqe *cqe;
619c5cb6 835 struct eth_fast_path_rx_cqe *cqe_fp;
9f6c9258 836 u8 cqe_fp_flags;
619c5cb6 837 enum eth_rx_cqe_type cqe_fp_type;
621b4d66 838 u16 len, pad, queue;
e52fcb24 839 u8 *data;
a334b5fb 840 bool l4_rxhash;
9f6c9258 841
619c5cb6
VZ		 842#ifdef BNX2X_STOP_ON_ERROR
843 if (unlikely(bp->panic))
844 return 0;
845#endif
846
9f6c9258
DK		 847 comp_ring_cons = RCQ_BD(sw_comp_cons);
848 bd_prod = RX_BD(bd_prod);
849 bd_cons = RX_BD(bd_cons);
850
9f6c9258 851 cqe = &fp->rx_comp_ring[comp_ring_cons];
619c5cb6
VZ		 852 cqe_fp = &cqe->fast_path_cqe;
853 cqe_fp_flags = cqe_fp->type_error_flags;
854 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
9f6c9258 855
51c1a580
MS		 856 DP(NETIF_MSG_RX_STATUS,
857 "CQE type %x err %x status %x queue %x vlan %x len %u\n",
858 CQE_TYPE(cqe_fp_flags),
619c5cb6
VZ		 859 cqe_fp_flags, cqe_fp->status_flags,
860 le32_to_cpu(cqe_fp->rss_hash_result),
621b4d66
DK		 861 le16_to_cpu(cqe_fp->vlan_tag),
862 le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
9f6c9258
DK		 863
864 /* is this a slowpath msg? */
619c5cb6 865 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
9f6c9258
DK		 866 bnx2x_sp_event(fp, cqe);
867 goto next_cqe;
e52fcb24 868 }
621b4d66 869
e52fcb24
ED		 870 rx_buf = &fp->rx_buf_ring[bd_cons];
871 data = rx_buf->data;
9f6c9258 872
e52fcb24 873 if (!CQE_TYPE_FAST(cqe_fp_type)) {
621b4d66
DK		 874 struct bnx2x_agg_info *tpa_info;
875 u16 frag_size, pages;
619c5cb6 876#ifdef BNX2X_STOP_ON_ERROR
e52fcb24
ED		 877 /* sanity check */
878 if (fp->disable_tpa &&
879 (CQE_TYPE_START(cqe_fp_type) ||
880 CQE_TYPE_STOP(cqe_fp_type)))
51c1a580 881 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
e52fcb24 882 CQE_TYPE(cqe_fp_type));
619c5cb6 883#endif
9f6c9258 884
e52fcb24
ED		 885 if (CQE_TYPE_START(cqe_fp_type)) {
886 u16 queue = cqe_fp->queue_index;
887 DP(NETIF_MSG_RX_STATUS,
888 "calling tpa_start on queue %d\n",
889 queue);
9f6c9258 890
e52fcb24
ED		 891 bnx2x_tpa_start(fp, queue,
892 bd_cons, bd_prod,
893 cqe_fp);
621b4d66 894
e52fcb24 895 goto next_rx;
e52fcb24 896
621b4d66
DK		 897 }
898 queue = cqe->end_agg_cqe.queue_index;
899 tpa_info = &fp->tpa_info[queue];
900 DP(NETIF_MSG_RX_STATUS,
901 "calling tpa_stop on queue %d\n",
902 queue);
903
904 frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
905 tpa_info->len_on_bd;
906
907 if (fp->mode == TPA_MODE_GRO)
908 pages = (frag_size + tpa_info->full_page - 1) /
909 tpa_info->full_page;
910 else
911 pages = SGE_PAGE_ALIGN(frag_size) >>
912 SGE_PAGE_SHIFT;
913
914 bnx2x_tpa_stop(bp, fp, tpa_info, pages,
915 &cqe->end_agg_cqe, comp_ring_cons);
9f6c9258 916#ifdef BNX2X_STOP_ON_ERROR
621b4d66
DK		 917 if (bp->panic)
918 return 0;
9f6c9258
DK		 919#endif
920
621b4d66
DK		 921 bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
922 goto next_cqe;
e52fcb24
ED		 923 }
924 /* non TPA */
621b4d66 925 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
e52fcb24
ED		 926 pad = cqe_fp->placement_offset;
927 dma_sync_single_for_cpu(&bp->pdev->dev,
9f6c9258 928 dma_unmap_addr(rx_buf, mapping),
e52fcb24
ED		 929 pad + RX_COPY_THRESH,
930 DMA_FROM_DEVICE);
931 pad += NET_SKB_PAD;
932 prefetch(data + pad); /* speedup eth_type_trans() */
933 /* is this an error packet? */
934 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
51c1a580 935 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
e52fcb24
ED		 936 "ERROR flags %x rx packet %u\n",
937 cqe_fp_flags, sw_comp_cons);
15192a8c 938 bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
e52fcb24
ED		 939 goto reuse_rx;
940 }
9f6c9258 941
e52fcb24
ED		 942 /* Since we don't have a jumbo ring
943 * copy small packets if mtu > 1500
944 */
945 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
946 (len <= RX_COPY_THRESH)) {
947 skb = netdev_alloc_skb_ip_align(bp->dev, len);
948 if (skb == NULL) {
51c1a580 949 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
e52fcb24 950 "ERROR packet dropped because of alloc failure\n");
15192a8c 951 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
9f6c9258
DK		 952 goto reuse_rx;
953 }
e52fcb24
ED		 954 memcpy(skb->data, data + pad, len);
955 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
956 } else {
957 if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod) == 0)) {
9f6c9258 958 dma_unmap_single(&bp->pdev->dev,
e52fcb24 959 dma_unmap_addr(rx_buf, mapping),
a8c94b91 960 fp->rx_buf_size,
9f6c9258 961 DMA_FROM_DEVICE);
d46d132c 962 skb = build_skb(data, fp->rx_frag_size);
e52fcb24 963 if (unlikely(!skb)) {
d46d132c 964 bnx2x_frag_free(fp, data);
15192a8c
BW		 965 bnx2x_fp_qstats(bp, fp)->
966 rx_skb_alloc_failed++;
e52fcb24
ED		 967 goto next_rx;
968 }
9f6c9258 969 skb_reserve(skb, pad);
9f6c9258 970 } else {
51c1a580
MS		 971 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
972 "ERROR packet dropped because of alloc failure\n");
15192a8c 973 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
9f6c9258 974reuse_rx:
e52fcb24 975 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
9f6c9258
DK		 976 goto next_rx;
977 }
036d2df9 978 }
9f6c9258 979
036d2df9
DK		 980 skb_put(skb, len);
981 skb->protocol = eth_type_trans(skb, bp->dev);
9f6c9258 982
036d2df9 983 /* Set Toeplitz hash for a none-LRO skb */
a334b5fb
ED		 984 skb->rxhash = bnx2x_get_rxhash(bp, cqe_fp, &l4_rxhash);
985 skb->l4_rxhash = l4_rxhash;
9f6c9258 986
036d2df9 987 skb_checksum_none_assert(skb);
f85582f8 988
d6cb3e41 989 if (bp->dev->features & NETIF_F_RXCSUM)
15192a8c
BW		 990 bnx2x_csum_validate(skb, cqe, fp,
991 bnx2x_fp_qstats(bp, fp));
9f6c9258 992
f233cafe 993 skb_record_rx_queue(skb, fp->rx_queue);
9f6c9258 994
619c5cb6
VZ		 995 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
996 PARSING_FLAGS_VLAN)
9bcc0893 997 __vlan_hwaccel_put_tag(skb,
619c5cb6 998 le16_to_cpu(cqe_fp->vlan_tag));
9bcc0893 999 napi_gro_receive(&fp->napi, skb);
9f6c9258
DK		 1000
1001
1002next_rx:
e52fcb24 1003 rx_buf->data = NULL;
9f6c9258
DK		 1004
1005 bd_cons = NEXT_RX_IDX(bd_cons);
1006 bd_prod = NEXT_RX_IDX(bd_prod);
1007 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
1008 rx_pkt++;
1009next_cqe:
1010 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
1011 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
1012
1013 if (rx_pkt == budget)
1014 break;
1015 } /* while */
1016
1017 fp->rx_bd_cons = bd_cons;
1018 fp->rx_bd_prod = bd_prod_fw;
1019 fp->rx_comp_cons = sw_comp_cons;
1020 fp->rx_comp_prod = sw_comp_prod;
1021
1022 /* Update producers */
1023 bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
1024 fp->rx_sge_prod);
1025
1026 fp->rx_pkt += rx_pkt;
1027 fp->rx_calls++;
1028
1029 return rx_pkt;
1030}
1031
1032static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
1033{
1034 struct bnx2x_fastpath *fp = fp_cookie;
1035 struct bnx2x *bp = fp->bp;
6383c0b3 1036 u8 cos;
9f6c9258 1037
51c1a580
MS		 1038 DP(NETIF_MSG_INTR,
1039 "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
523224a3
DK		 1040 fp->index, fp->fw_sb_id, fp->igu_sb_id);
1041 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
9f6c9258
DK		 1042
1043#ifdef BNX2X_STOP_ON_ERROR
1044 if (unlikely(bp->panic))
1045 return IRQ_HANDLED;
1046#endif
1047
1048 /* Handle Rx and Tx according to MSI-X vector */
1049 prefetch(fp->rx_cons_sb);
6383c0b3
AE		 1050
1051 for_each_cos_in_tx_queue(fp, cos)
65565884 1052 prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
6383c0b3 1053
523224a3 1054 prefetch(&fp->sb_running_index[SM_RX_ID]);
9f6c9258
DK		 1055 napi_schedule(&bnx2x_fp(bp, fp->index, napi));
1056
1057 return IRQ_HANDLED;
1058}
1059
9f6c9258
DK		 1060/* HW Lock for shared dual port PHYs */
1061void bnx2x_acquire_phy_lock(struct bnx2x *bp)
1062{
1063 mutex_lock(&bp->port.phy_mutex);
1064
8203c4b6 1065 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
9f6c9258
DK		 1066}
1067
1068void bnx2x_release_phy_lock(struct bnx2x *bp)
1069{
8203c4b6 1070 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
9f6c9258
DK		 1071
1072 mutex_unlock(&bp->port.phy_mutex);
1073}
1074
0793f83f
DK		 1075/* calculates MF speed according to current linespeed and MF configuration */
1076u16 bnx2x_get_mf_speed(struct bnx2x *bp)
1077{
1078 u16 line_speed = bp->link_vars.line_speed;
1079 if (IS_MF(bp)) {
faa6fcbb
DK		 1080 u16 maxCfg = bnx2x_extract_max_cfg(bp,
1081 bp->mf_config[BP_VN(bp)]);
1082
1083 /* Calculate the current MAX line speed limit for the MF
1084 * devices
0793f83f 1085 */
faa6fcbb
DK		 1086 if (IS_MF_SI(bp))
1087 line_speed = (line_speed * maxCfg) / 100;
1088 else { /* SD mode */
0793f83f
DK		 1089 u16 vn_max_rate = maxCfg * 100;
1090
1091 if (vn_max_rate < line_speed)
1092 line_speed = vn_max_rate;
faa6fcbb 1093 }
0793f83f
DK		 1094 }
1095
1096 return line_speed;
1097}
1098
2ae17f66
VZ		 1099/**
1100 * bnx2x_fill_report_data - fill link report data to report
1101 *
1102 * @bp: driver handle
1103 * @data: link state to update
1104 *
1105 * It uses a none-atomic bit operations because is called under the mutex.
1106 */
1191cb83
ED		 1107static void bnx2x_fill_report_data(struct bnx2x *bp,
1108 struct bnx2x_link_report_data *data)
2ae17f66
VZ		 1109{
1110 u16 line_speed = bnx2x_get_mf_speed(bp);
1111
1112 memset(data, 0, sizeof(*data));
1113
1114 /* Fill the report data: efective line speed */
1115 data->line_speed = line_speed;
1116
1117 /* Link is down */
1118 if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1119 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1120 &data->link_report_flags);
1121
1122 /* Full DUPLEX */
1123 if (bp->link_vars.duplex == DUPLEX_FULL)
1124 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
1125
1126 /* Rx Flow Control is ON */
1127 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1128 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
1129
1130 /* Tx Flow Control is ON */
1131 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1132 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
1133}
1134
1135/**
1136 * bnx2x_link_report - report link status to OS.
1137 *
1138 * @bp: driver handle
1139 *
1140 * Calls the __bnx2x_link_report() under the same locking scheme
1141 * as a link/PHY state managing code to ensure a consistent link
1142 * reporting.
1143 */
1144
9f6c9258
DK		 1145void bnx2x_link_report(struct bnx2x *bp)
1146{
2ae17f66
VZ		 1147 bnx2x_acquire_phy_lock(bp);
1148 __bnx2x_link_report(bp);
1149 bnx2x_release_phy_lock(bp);
1150}
9f6c9258 1151
2ae17f66
VZ		 1152/**
1153 * __bnx2x_link_report - report link status to OS.
1154 *
1155 * @bp: driver handle
1156 *
1157 * None atomic inmlementation.
1158 * Should be called under the phy_lock.
1159 */
1160void __bnx2x_link_report(struct bnx2x *bp)
1161{
1162 struct bnx2x_link_report_data cur_data;
9f6c9258 1163
2ae17f66 1164 /* reread mf_cfg */
ad5afc89 1165 if (IS_PF(bp) && !CHIP_IS_E1(bp))
2ae17f66
VZ		 1166 bnx2x_read_mf_cfg(bp);
1167
1168 /* Read the current link report info */
1169 bnx2x_fill_report_data(bp, &cur_data);
1170
1171 /* Don't report link down or exactly the same link status twice */
1172 if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1173 (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1174 &bp->last_reported_link.link_report_flags) &&
1175 test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1176 &cur_data.link_report_flags)))
1177 return;
1178
1179 bp->link_cnt++;
9f6c9258 1180
2ae17f66
VZ		 1181 /* We are going to report a new link parameters now -
1182 * remember the current data for the next time.
1183 */
1184 memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
9f6c9258 1185
2ae17f66
VZ		 1186 if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1187 &cur_data.link_report_flags)) {
1188 netif_carrier_off(bp->dev);
1189 netdev_err(bp->dev, "NIC Link is Down\n");
1190 return;
1191 } else {
94f05b0f
JP		 1192 const char *duplex;
1193 const char *flow;
1194
2ae17f66 1195 netif_carrier_on(bp->dev);
9f6c9258 1196
2ae17f66
VZ		 1197 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1198 &cur_data.link_report_flags))
94f05b0f 1199 duplex = "full";
9f6c9258 1200 else
94f05b0f 1201 duplex = "half";
9f6c9258 1202
2ae17f66
VZ		 1203 /* Handle the FC at the end so that only these flags would be
1204 * possibly set. This way we may easily check if there is no FC
1205 * enabled.
1206 */
1207 if (cur_data.link_report_flags) {
1208 if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1209 &cur_data.link_report_flags)) {
2ae17f66
VZ		 1210 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1211 &cur_data.link_report_flags))
94f05b0f
JP		 1212 flow = "ON - receive & transmit";
1213 else
1214 flow = "ON - receive";
9f6c9258 1215 } else {
94f05b0f 1216 flow = "ON - transmit";
9f6c9258 1217 }
94f05b0f
JP		 1218 } else {
1219 flow = "none";
9f6c9258 1220 }
94f05b0f
JP		 1221 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1222 cur_data.line_speed, duplex, flow);
9f6c9258
DK		 1223 }
1224}
1225
1191cb83
ED		 1226static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1227{
1228 int i;
1229
1230 for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1231 struct eth_rx_sge *sge;
1232
1233 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1234 sge->addr_hi =
1235 cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1236 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1237
1238 sge->addr_lo =
1239 cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1240 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1241 }
1242}
1243
1244static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1245 struct bnx2x_fastpath *fp, int last)
1246{
1247 int i;
1248
1249 for (i = 0; i < last; i++) {
1250 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1251 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1252 u8 *data = first_buf->data;
1253
1254 if (data == NULL) {
1255 DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1256 continue;
1257 }
1258 if (tpa_info->tpa_state == BNX2X_TPA_START)
1259 dma_unmap_single(&bp->pdev->dev,
1260 dma_unmap_addr(first_buf, mapping),
1261 fp->rx_buf_size, DMA_FROM_DEVICE);
d46d132c 1262 bnx2x_frag_free(fp, data);
1191cb83
ED		 1263 first_buf->data = NULL;
1264 }
1265}
1266
55c11941
MS		 1267void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1268{
1269 int j;
1270
1271 for_each_rx_queue_cnic(bp, j) {
1272 struct bnx2x_fastpath *fp = &bp->fp[j];
1273
1274 fp->rx_bd_cons = 0;
1275
1276 /* Activate BD ring */
1277 /* Warning!
1278 * this will generate an interrupt (to the TSTORM)
1279 * must only be done after chip is initialized
1280 */
1281 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1282 fp->rx_sge_prod);
1283 }
1284}
1285
9f6c9258
DK		 1286void bnx2x_init_rx_rings(struct bnx2x *bp)
1287{
1288 int func = BP_FUNC(bp);
523224a3 1289 u16 ring_prod;
9f6c9258 1290 int i, j;
25141580 1291
b3b83c3f 1292 /* Allocate TPA resources */
55c11941 1293 for_each_eth_queue(bp, j) {
523224a3 1294 struct bnx2x_fastpath *fp = &bp->fp[j];
9f6c9258 1295
a8c94b91
VZ		 1296 DP(NETIF_MSG_IFUP,
1297 "mtu %d rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1298
523224a3 1299 if (!fp->disable_tpa) {
619c5cb6 1300 /* Fill the per-aggregtion pool */
dfacf138 1301 for (i = 0; i < MAX_AGG_QS(bp); i++) {
619c5cb6
VZ		 1302 struct bnx2x_agg_info *tpa_info =
1303 &fp->tpa_info[i];
1304 struct sw_rx_bd *first_buf =
1305 &tpa_info->first_buf;
1306
d46d132c 1307 first_buf->data = bnx2x_frag_alloc(fp);
e52fcb24 1308 if (!first_buf->data) {
51c1a580
MS		 1309 BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1310 j);
9f6c9258
DK		 1311 bnx2x_free_tpa_pool(bp, fp, i);
1312 fp->disable_tpa = 1;
1313 break;
1314 }
619c5cb6
VZ		 1315 dma_unmap_addr_set(first_buf, mapping, 0);
1316 tpa_info->tpa_state = BNX2X_TPA_STOP;
9f6c9258 1317 }
523224a3
DK		 1318
1319 /* "next page" elements initialization */
1320 bnx2x_set_next_page_sgl(fp);
1321
1322 /* set SGEs bit mask */
1323 bnx2x_init_sge_ring_bit_mask(fp);
1324
1325 /* Allocate SGEs and initialize the ring elements */
1326 for (i = 0, ring_prod = 0;
1327 i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1328
1329 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
51c1a580
MS		 1330 BNX2X_ERR("was only able to allocate %d rx sges\n",
1331 i);
1332 BNX2X_ERR("disabling TPA for queue[%d]\n",
1333 j);
523224a3 1334 /* Cleanup already allocated elements */
619c5cb6
VZ		 1335 bnx2x_free_rx_sge_range(bp, fp,
1336 ring_prod);
1337 bnx2x_free_tpa_pool(bp, fp,
dfacf138 1338 MAX_AGG_QS(bp));
523224a3
DK		 1339 fp->disable_tpa = 1;
1340 ring_prod = 0;
1341 break;
1342 }
1343 ring_prod = NEXT_SGE_IDX(ring_prod);
1344 }
1345
1346 fp->rx_sge_prod = ring_prod;
9f6c9258
DK		 1347 }
1348 }
1349
55c11941 1350 for_each_eth_queue(bp, j) {
9f6c9258
DK		 1351 struct bnx2x_fastpath *fp = &bp->fp[j];
1352
1353 fp->rx_bd_cons = 0;
9f6c9258 1354
b3b83c3f
DK		 1355 /* Activate BD ring */
1356 /* Warning!
1357 * this will generate an interrupt (to the TSTORM)
1358 * must only be done after chip is initialized
1359 */
1360 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1361 fp->rx_sge_prod);
9f6c9258 1362
9f6c9258
DK		 1363 if (j != 0)
1364 continue;
1365
619c5cb6 1366 if (CHIP_IS_E1(bp)) {
f2e0899f
DK		 1367 REG_WR(bp, BAR_USTRORM_INTMEM +
1368 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1369 U64_LO(fp->rx_comp_mapping));
1370 REG_WR(bp, BAR_USTRORM_INTMEM +
1371 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1372 U64_HI(fp->rx_comp_mapping));
1373 }
9f6c9258
DK		 1374 }
1375}
f85582f8 1376
55c11941 1377static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
9f6c9258 1378{
6383c0b3 1379 u8 cos;
55c11941 1380 struct bnx2x *bp = fp->bp;
9f6c9258 1381
55c11941
MS		 1382 for_each_cos_in_tx_queue(fp, cos) {
1383 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1384 unsigned pkts_compl = 0, bytes_compl = 0;
9f6c9258 1385
55c11941
MS		 1386 u16 sw_prod = txdata->tx_pkt_prod;
1387 u16 sw_cons = txdata->tx_pkt_cons;
9f6c9258 1388
55c11941
MS		 1389 while (sw_cons != sw_prod) {
1390 bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1391 &pkts_compl, &bytes_compl);
1392 sw_cons++;
9f6c9258 1393 }
55c11941
MS		 1394
1395 netdev_tx_reset_queue(
1396 netdev_get_tx_queue(bp->dev,
1397 txdata->txq_index));
1398 }
1399}
1400
1401static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1402{
1403 int i;
1404
1405 for_each_tx_queue_cnic(bp, i) {
1406 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1407 }
1408}
1409
1410static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1411{
1412 int i;
1413
1414 for_each_eth_queue(bp, i) {
1415 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
9f6c9258
DK		 1416 }
1417}
1418
b3b83c3f
DK		 1419static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1420{
1421 struct bnx2x *bp = fp->bp;
1422 int i;
1423
1424 /* ring wasn't allocated */
1425 if (fp->rx_buf_ring == NULL)
1426 return;
1427
1428 for (i = 0; i < NUM_RX_BD; i++) {
1429 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
e52fcb24 1430 u8 *data = rx_buf->data;
b3b83c3f 1431
e52fcb24 1432 if (data == NULL)
b3b83c3f 1433 continue;
b3b83c3f
DK		 1434 dma_unmap_single(&bp->pdev->dev,
1435 dma_unmap_addr(rx_buf, mapping),
1436 fp->rx_buf_size, DMA_FROM_DEVICE);
1437
e52fcb24 1438 rx_buf->data = NULL;
d46d132c 1439 bnx2x_frag_free(fp, data);
b3b83c3f
DK		 1440 }
1441}
1442
55c11941
MS		 1443static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1444{
1445 int j;
1446
1447 for_each_rx_queue_cnic(bp, j) {
1448 bnx2x_free_rx_bds(&bp->fp[j]);
1449 }
1450}
1451
9f6c9258
DK		 1452static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1453{
b3b83c3f 1454 int j;
9f6c9258 1455
55c11941 1456 for_each_eth_queue(bp, j) {
9f6c9258
DK		 1457 struct bnx2x_fastpath *fp = &bp->fp[j];
1458
b3b83c3f 1459 bnx2x_free_rx_bds(fp);
9f6c9258 1460
9f6c9258 1461 if (!fp->disable_tpa)
dfacf138 1462 bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
9f6c9258
DK		 1463 }
1464}
1465
55c11941
MS		 1466void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1467{
1468 bnx2x_free_tx_skbs_cnic(bp);
1469 bnx2x_free_rx_skbs_cnic(bp);
1470}
1471
9f6c9258
DK		 1472void bnx2x_free_skbs(struct bnx2x *bp)
1473{
1474 bnx2x_free_tx_skbs(bp);
1475 bnx2x_free_rx_skbs(bp);
1476}
1477
e3835b99
DK		 1478void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1479{
1480 /* load old values */
1481 u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1482
1483 if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1484 /* leave all but MAX value */
1485 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1486
1487 /* set new MAX value */
1488 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1489 & FUNC_MF_CFG_MAX_BW_MASK;
1490
1491 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1492 }
1493}
1494
ca92429f
DK		 1495/**
1496 * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1497 *
1498 * @bp: driver handle
1499 * @nvecs: number of vectors to be released
1500 */
1501static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
9f6c9258 1502{
ca92429f 1503 int i, offset = 0;
9f6c9258 1504
ca92429f
DK		 1505 if (nvecs == offset)
1506 return;
ad5afc89
AE		 1507
1508 /* VFs don't have a default SB */
1509 if (IS_PF(bp)) {
1510 free_irq(bp->msix_table[offset].vector, bp->dev);
1511 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1512 bp->msix_table[offset].vector);
1513 offset++;
1514 }
55c11941
MS		 1515
1516 if (CNIC_SUPPORT(bp)) {
1517 if (nvecs == offset)
1518 return;
1519 offset++;
1520 }
ca92429f 1521
ec6ba945 1522 for_each_eth_queue(bp, i) {
ca92429f
DK		 1523 if (nvecs == offset)
1524 return;
51c1a580
MS		 1525 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1526 i, bp->msix_table[offset].vector);
9f6c9258 1527
ca92429f 1528 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
9f6c9258
DK		 1529 }
1530}
1531
d6214d7a 1532void bnx2x_free_irq(struct bnx2x *bp)
9f6c9258 1533{
30a5de77 1534 if (bp->flags & USING_MSIX_FLAG &&
ad5afc89
AE		 1535 !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1536 int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp);
1537
1538 /* vfs don't have a default status block */
1539 if (IS_PF(bp))
1540 nvecs++;
1541
1542 bnx2x_free_msix_irqs(bp, nvecs);
1543 } else {
30a5de77 1544 free_irq(bp->dev->irq, bp->dev);
ad5afc89 1545 }
9f6c9258
DK		 1546}
1547
0e8d2ec5 1548int bnx2x_enable_msix(struct bnx2x *bp)
9f6c9258 1549{
1ab4434c 1550 int msix_vec = 0, i, rc;
9f6c9258 1551
1ab4434c
AE		 1552 /* VFs don't have a default status block */
1553 if (IS_PF(bp)) {
1554 bp->msix_table[msix_vec].entry = msix_vec;
1555 BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1556 bp->msix_table[0].entry);
1557 msix_vec++;
1558 }
9f6c9258 1559
55c11941
MS		 1560 /* Cnic requires an msix vector for itself */
1561 if (CNIC_SUPPORT(bp)) {
1562 bp->msix_table[msix_vec].entry = msix_vec;
1563 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1564 msix_vec, bp->msix_table[msix_vec].entry);
1565 msix_vec++;
1566 }
1567
6383c0b3 1568 /* We need separate vectors for ETH queues only (not FCoE) */
ec6ba945 1569 for_each_eth_queue(bp, i) {
d6214d7a 1570 bp->msix_table[msix_vec].entry = msix_vec;
51c1a580
MS		 1571 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1572 msix_vec, msix_vec, i);
d6214d7a 1573 msix_vec++;
9f6c9258
DK		 1574 }
1575
1ab4434c
AE		 1576 DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n",
1577 msix_vec);
d6214d7a 1578
1ab4434c 1579 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], msix_vec);
9f6c9258
DK		 1580
1581 /*
1582 * reconfigure number of tx/rx queues according to available
1583 * MSI-X vectors
1584 */
55c11941 1585 if (rc >= BNX2X_MIN_MSIX_VEC_CNT(bp)) {
d6214d7a 1586 /* how less vectors we will have? */
1ab4434c 1587 int diff = msix_vec - rc;
9f6c9258 1588
51c1a580 1589 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
9f6c9258
DK		 1590
1591 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1592
1593 if (rc) {
30a5de77
DK		 1594 BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1595 goto no_msix;
9f6c9258 1596 }
d6214d7a
DK		 1597 /*
1598 * decrease number of queues by number of unallocated entries
1599 */
55c11941
MS		 1600 bp->num_ethernet_queues -= diff;
1601 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
9f6c9258 1602
51c1a580 1603 BNX2X_DEV_INFO("New queue configuration set: %d\n",
30a5de77
DK		 1604 bp->num_queues);
1605 } else if (rc > 0) {
1606 /* Get by with single vector */
1607 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], 1);
1608 if (rc) {
1609 BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1610 rc);
1611 goto no_msix;
1612 }
1613
1614 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1615 bp->flags |= USING_SINGLE_MSIX_FLAG;
1616
55c11941
MS		 1617 BNX2X_DEV_INFO("set number of queues to 1\n");
1618 bp->num_ethernet_queues = 1;
1619 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
30a5de77 1620 } else if (rc < 0) {
51c1a580 1621 BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
30a5de77 1622 goto no_msix;
9f6c9258
DK		 1623 }
1624
1625 bp->flags |= USING_MSIX_FLAG;
1626
1627 return 0;
30a5de77
DK		 1628
1629no_msix:
1630 /* fall to INTx if not enough memory */
1631 if (rc == -ENOMEM)
1632 bp->flags |= DISABLE_MSI_FLAG;
1633
1634 return rc;
9f6c9258
DK		 1635}
1636
1637static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1638{
ca92429f 1639 int i, rc, offset = 0;
9f6c9258 1640
ad5afc89
AE		 1641 /* no default status block for vf */
1642 if (IS_PF(bp)) {
1643 rc = request_irq(bp->msix_table[offset++].vector,
1644 bnx2x_msix_sp_int, 0,
1645 bp->dev->name, bp->dev);
1646 if (rc) {
1647 BNX2X_ERR("request sp irq failed\n");
1648 return -EBUSY;
1649 }
9f6c9258
DK		 1650 }
1651
55c11941
MS		 1652 if (CNIC_SUPPORT(bp))
1653 offset++;
1654
ec6ba945 1655 for_each_eth_queue(bp, i) {
9f6c9258
DK		 1656 struct bnx2x_fastpath *fp = &bp->fp[i];
1657 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1658 bp->dev->name, i);
1659
d6214d7a 1660 rc = request_irq(bp->msix_table[offset].vector,
9f6c9258
DK		 1661 bnx2x_msix_fp_int, 0, fp->name, fp);
1662 if (rc) {
ca92429f
DK		 1663 BNX2X_ERR("request fp #%d irq (%d) failed rc %d\n", i,
1664 bp->msix_table[offset].vector, rc);
1665 bnx2x_free_msix_irqs(bp, offset);
9f6c9258
DK		 1666 return -EBUSY;
1667 }
1668
d6214d7a 1669 offset++;
9f6c9258
DK		 1670 }
1671
ec6ba945 1672 i = BNX2X_NUM_ETH_QUEUES(bp);
ad5afc89
AE		 1673 if (IS_PF(bp)) {
1674 offset = 1 + CNIC_SUPPORT(bp);
1675 netdev_info(bp->dev,
1676 "using MSI-X IRQs: sp %d fp[%d] %d ... fp[%d] %d\n",
1677 bp->msix_table[0].vector,
1678 0, bp->msix_table[offset].vector,
1679 i - 1, bp->msix_table[offset + i - 1].vector);
1680 } else {
1681 offset = CNIC_SUPPORT(bp);
1682 netdev_info(bp->dev,
1683 "using MSI-X IRQs: fp[%d] %d ... fp[%d] %d\n",
1684 0, bp->msix_table[offset].vector,
1685 i - 1, bp->msix_table[offset + i - 1].vector);
1686 }
9f6c9258
DK		 1687 return 0;
1688}
1689
d6214d7a 1690int bnx2x_enable_msi(struct bnx2x *bp)
9f6c9258
DK		 1691{
1692 int rc;
1693
1694 rc = pci_enable_msi(bp->pdev);
1695 if (rc) {
51c1a580 1696 BNX2X_DEV_INFO("MSI is not attainable\n");
9f6c9258
DK		 1697 return -1;
1698 }
1699 bp->flags |= USING_MSI_FLAG;
1700
1701 return 0;
1702}
1703
1704static int bnx2x_req_irq(struct bnx2x *bp)
1705{
1706 unsigned long flags;
30a5de77 1707 unsigned int irq;
9f6c9258 1708
30a5de77 1709 if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
9f6c9258
DK		 1710 flags = 0;
1711 else
1712 flags = IRQF_SHARED;
1713
30a5de77
DK		 1714 if (bp->flags & USING_MSIX_FLAG)
1715 irq = bp->msix_table[0].vector;
1716 else
1717 irq = bp->pdev->irq;
1718
1719 return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
9f6c9258
DK		 1720}
1721
1191cb83 1722static int bnx2x_setup_irqs(struct bnx2x *bp)
619c5cb6
VZ		 1723{
1724 int rc = 0;
30a5de77
DK		 1725 if (bp->flags & USING_MSIX_FLAG &&
1726 !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
619c5cb6
VZ		 1727 rc = bnx2x_req_msix_irqs(bp);
1728 if (rc)
1729 return rc;
1730 } else {
619c5cb6
VZ		 1731 rc = bnx2x_req_irq(bp);
1732 if (rc) {
1733 BNX2X_ERR("IRQ request failed rc %d, aborting\n", rc);
1734 return rc;
1735 }
1736 if (bp->flags & USING_MSI_FLAG) {
1737 bp->dev->irq = bp->pdev->irq;
30a5de77
DK		 1738 netdev_info(bp->dev, "using MSI IRQ %d\n",
1739 bp->dev->irq);
1740 }
1741 if (bp->flags & USING_MSIX_FLAG) {
1742 bp->dev->irq = bp->msix_table[0].vector;
1743 netdev_info(bp->dev, "using MSIX IRQ %d\n",
1744 bp->dev->irq);
619c5cb6
VZ		 1745 }
1746 }
1747
1748 return 0;
1749}
1750
55c11941
MS		 1751static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1752{
1753 int i;
1754
1755 for_each_rx_queue_cnic(bp, i)
1756 napi_enable(&bnx2x_fp(bp, i, napi));
1757}
1758
1191cb83 1759static void bnx2x_napi_enable(struct bnx2x *bp)
9f6c9258
DK		 1760{
1761 int i;
1762
55c11941 1763 for_each_eth_queue(bp, i)
9f6c9258
DK		 1764 napi_enable(&bnx2x_fp(bp, i, napi));
1765}
1766
55c11941
MS		 1767static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1768{
1769 int i;
1770
1771 for_each_rx_queue_cnic(bp, i)
1772 napi_disable(&bnx2x_fp(bp, i, napi));
1773}
1774
1191cb83 1775static void bnx2x_napi_disable(struct bnx2x *bp)
9f6c9258
DK		 1776{
1777 int i;
1778
55c11941 1779 for_each_eth_queue(bp, i)
9f6c9258
DK		 1780 napi_disable(&bnx2x_fp(bp, i, napi));
1781}
1782
1783void bnx2x_netif_start(struct bnx2x *bp)
1784{
4b7ed897
DK		 1785 if (netif_running(bp->dev)) {
1786 bnx2x_napi_enable(bp);
55c11941
MS		 1787 if (CNIC_LOADED(bp))
1788 bnx2x_napi_enable_cnic(bp);
4b7ed897
DK		 1789 bnx2x_int_enable(bp);
1790 if (bp->state == BNX2X_STATE_OPEN)
1791 netif_tx_wake_all_queues(bp->dev);
9f6c9258
DK		 1792 }
1793}
1794
1795void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1796{
1797 bnx2x_int_disable_sync(bp, disable_hw);
1798 bnx2x_napi_disable(bp);
55c11941
MS		 1799 if (CNIC_LOADED(bp))
1800 bnx2x_napi_disable_cnic(bp);
9f6c9258 1801}
9f6c9258 1802
8307fa3e
VZ		 1803u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1804{
8307fa3e 1805 struct bnx2x *bp = netdev_priv(dev);
cdb9d6ae 1806
55c11941 1807 if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
8307fa3e
VZ		 1808 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1809 u16 ether_type = ntohs(hdr->h_proto);
1810
1811 /* Skip VLAN tag if present */
1812 if (ether_type == ETH_P_8021Q) {
1813 struct vlan_ethhdr *vhdr =
1814 (struct vlan_ethhdr *)skb->data;
1815
1816 ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1817 }
1818
1819 /* If ethertype is FCoE or FIP - use FCoE ring */
1820 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
6383c0b3 1821 return bnx2x_fcoe_tx(bp, txq_index);
8307fa3e 1822 }
55c11941 1823
cdb9d6ae 1824 /* select a non-FCoE queue */
6383c0b3 1825 return __skb_tx_hash(dev, skb, BNX2X_NUM_ETH_QUEUES(bp));
8307fa3e
VZ		 1826}
1827
d6214d7a
DK		 1828void bnx2x_set_num_queues(struct bnx2x *bp)
1829{
96305234 1830 /* RSS queues */
55c11941 1831 bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
ec6ba945 1832
a3348722
BW		 1833 /* override in STORAGE SD modes */
1834 if (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))
55c11941
MS		 1835 bp->num_ethernet_queues = 1;
1836
ec6ba945 1837 /* Add special queues */
55c11941
MS		 1838 bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1839 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
65565884
MS		 1840
1841 BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
ec6ba945
VZ		 1842}
1843
cdb9d6ae
VZ		 1844/**
1845 * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1846 *
1847 * @bp: Driver handle
1848 *
1849 * We currently support for at most 16 Tx queues for each CoS thus we will
1850 * allocate a multiple of 16 for ETH L2 rings according to the value of the
1851 * bp->max_cos.
1852 *
1853 * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1854 * index after all ETH L2 indices.
1855 *
1856 * If the actual number of Tx queues (for each CoS) is less than 16 then there
1857 * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1858 * 16..31,...) with indicies that are not coupled with any real Tx queue.
1859 *
1860 * The proper configuration of skb->queue_mapping is handled by
1861 * bnx2x_select_queue() and __skb_tx_hash().
1862 *
1863 * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1864 * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1865 */
55c11941 1866static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
ec6ba945 1867{
6383c0b3 1868 int rc, tx, rx;
ec6ba945 1869
65565884 1870 tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
55c11941 1871 rx = BNX2X_NUM_ETH_QUEUES(bp);
ec6ba945 1872
6383c0b3 1873/* account for fcoe queue */
55c11941
MS		 1874 if (include_cnic && !NO_FCOE(bp)) {
1875 rx++;
1876 tx++;
6383c0b3 1877 }
6383c0b3
AE		 1878
1879 rc = netif_set_real_num_tx_queues(bp->dev, tx);
1880 if (rc) {
1881 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1882 return rc;
1883 }
1884 rc = netif_set_real_num_rx_queues(bp->dev, rx);
1885 if (rc) {
1886 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1887 return rc;
1888 }
1889
51c1a580 1890 DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
6383c0b3
AE		 1891 tx, rx);
1892
ec6ba945
VZ		 1893 return rc;
1894}
1895
1191cb83 1896static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
a8c94b91
VZ		 1897{
1898 int i;
1899
1900 for_each_queue(bp, i) {
1901 struct bnx2x_fastpath *fp = &bp->fp[i];
e52fcb24 1902 u32 mtu;
a8c94b91
VZ		 1903
1904 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1905 if (IS_FCOE_IDX(i))
1906 /*
1907 * Although there are no IP frames expected to arrive to
1908 * this ring we still want to add an
1909 * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1910 * overrun attack.
1911 */
e52fcb24 1912 mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
a8c94b91 1913 else
e52fcb24
ED		 1914 mtu = bp->dev->mtu;
1915 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
1916 IP_HEADER_ALIGNMENT_PADDING +
1917 ETH_OVREHEAD +
1918 mtu +
1919 BNX2X_FW_RX_ALIGN_END;
1920 /* Note : rx_buf_size doesnt take into account NET_SKB_PAD */
d46d132c
ED		 1921 if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
1922 fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
1923 else
1924 fp->rx_frag_size = 0;
a8c94b91
VZ		 1925 }
1926}
1927
1191cb83 1928static int bnx2x_init_rss_pf(struct bnx2x *bp)
619c5cb6
VZ		 1929{
1930 int i;
619c5cb6
VZ		 1931 u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
1932
96305234 1933 /* Prepare the initial contents fo the indirection table if RSS is
619c5cb6
VZ		 1934 * enabled
1935 */
5d317c6a
MS		 1936 for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
1937 bp->rss_conf_obj.ind_table[i] =
96305234
DK		 1938 bp->fp->cl_id +
1939 ethtool_rxfh_indir_default(i, num_eth_queues);
619c5cb6
VZ		 1940
1941 /*
1942 * For 57710 and 57711 SEARCHER configuration (rss_keys) is
1943 * per-port, so if explicit configuration is needed , do it only
1944 * for a PMF.
1945 *
1946 * For 57712 and newer on the other hand it's a per-function
1947 * configuration.
1948 */
5d317c6a 1949 return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
619c5cb6
VZ		 1950}
1951
96305234 1952int bnx2x_config_rss_pf(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
5d317c6a 1953 bool config_hash)
619c5cb6 1954{
3b603066 1955 struct bnx2x_config_rss_params params = {NULL};
619c5cb6
VZ		 1956
1957 /* Although RSS is meaningless when there is a single HW queue we
1958 * still need it enabled in order to have HW Rx hash generated.
1959 *
1960 * if (!is_eth_multi(bp))
1961 * bp->multi_mode = ETH_RSS_MODE_DISABLED;
1962 */
1963
96305234 1964 params.rss_obj = rss_obj;
619c5cb6
VZ		 1965
1966 __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
1967
96305234 1968 __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
619c5cb6 1969
96305234
DK		 1970 /* RSS configuration */
1971 __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
1972 __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
1973 __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
1974 __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
5d317c6a
MS		 1975 if (rss_obj->udp_rss_v4)
1976 __set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
1977 if (rss_obj->udp_rss_v6)
1978 __set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
619c5cb6 1979
96305234
DK		 1980 /* Hash bits */
1981 params.rss_result_mask = MULTI_MASK;
619c5cb6 1982
5d317c6a 1983 memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
619c5cb6 1984
96305234
DK		 1985 if (config_hash) {
1986 /* RSS keys */
8376d0bc 1987 prandom_bytes(params.rss_key, sizeof(params.rss_key));
96305234 1988 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
619c5cb6
VZ		 1989 }
1990
1991 return bnx2x_config_rss(bp, &params);
1992}
1993
1191cb83 1994static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
619c5cb6 1995{
3b603066 1996 struct bnx2x_func_state_params func_params = {NULL};
619c5cb6
VZ		 1997
1998 /* Prepare parameters for function state transitions */
1999 __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
2000
2001 func_params.f_obj = &bp->func_obj;
2002 func_params.cmd = BNX2X_F_CMD_HW_INIT;
2003
2004 func_params.params.hw_init.load_phase = load_code;
2005
2006 return bnx2x_func_state_change(bp, &func_params);
2007}
2008
2009/*
2010 * Cleans the object that have internal lists without sending
2011 * ramrods. Should be run when interrutps are disabled.
2012 */
7fa6f340 2013void bnx2x_squeeze_objects(struct bnx2x *bp)
619c5cb6
VZ		 2014{
2015 int rc;
2016 unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
3b603066 2017 struct bnx2x_mcast_ramrod_params rparam = {NULL};
15192a8c 2018 struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
619c5cb6
VZ		 2019
2020 /***************** Cleanup MACs' object first *************************/
2021
2022 /* Wait for completion of requested */
2023 __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
2024 /* Perform a dry cleanup */
2025 __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
2026
2027 /* Clean ETH primary MAC */
2028 __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
15192a8c 2029 rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
619c5cb6
VZ		 2030 &ramrod_flags);
2031 if (rc != 0)
2032 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
2033
2034 /* Cleanup UC list */
2035 vlan_mac_flags = 0;
2036 __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
2037 rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
2038 &ramrod_flags);
2039 if (rc != 0)
2040 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
2041
2042 /***************** Now clean mcast object *****************************/
2043 rparam.mcast_obj = &bp->mcast_obj;
2044 __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
2045
2046 /* Add a DEL command... */
2047 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
2048 if (rc < 0)
51c1a580
MS		 2049 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
2050 rc);
619c5cb6
VZ		 2051
2052 /* ...and wait until all pending commands are cleared */
2053 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2054 while (rc != 0) {
2055 if (rc < 0) {
2056 BNX2X_ERR("Failed to clean multi-cast object: %d\n",
2057 rc);
2058 return;
2059 }
2060
2061 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2062 }
2063}
2064
2065#ifndef BNX2X_STOP_ON_ERROR
2066#define LOAD_ERROR_EXIT(bp, label) \
2067 do { \
2068 (bp)->state = BNX2X_STATE_ERROR; \
2069 goto label; \
2070 } while (0)
55c11941
MS		 2071
2072#define LOAD_ERROR_EXIT_CNIC(bp, label) \
2073 do { \
2074 bp->cnic_loaded = false; \
2075 goto label; \
2076 } while (0)
2077#else /*BNX2X_STOP_ON_ERROR*/
619c5cb6
VZ		 2078#define LOAD_ERROR_EXIT(bp, label) \
2079 do { \
2080 (bp)->state = BNX2X_STATE_ERROR; \
2081 (bp)->panic = 1; \
2082 return -EBUSY; \
2083 } while (0)
55c11941
MS		 2084#define LOAD_ERROR_EXIT_CNIC(bp, label) \
2085 do { \
2086 bp->cnic_loaded = false; \
2087 (bp)->panic = 1; \
2088 return -EBUSY; \
2089 } while (0)
2090#endif /*BNX2X_STOP_ON_ERROR*/
619c5cb6 2091
ad5afc89
AE		 2092static void bnx2x_free_fw_stats_mem(struct bnx2x *bp)
2093{
2094 BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping,
2095 bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2096 return;
2097}
2098
2099static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp)
452427b0 2100{
8db573ba 2101 int num_groups, vf_headroom = 0;
ad5afc89 2102 int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1;
452427b0 2103
ad5afc89
AE		 2104 /* number of queues for statistics is number of eth queues + FCoE */
2105 u8 num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats;
452427b0 2106
ad5afc89
AE		 2107 /* Total number of FW statistics requests =
2108 * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper
2109 * and fcoe l2 queue) stats + num of queues (which includes another 1
2110 * for fcoe l2 queue if applicable)
2111 */
2112 bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats;
452427b0 2113
8db573ba
AE		 2114 /* vf stats appear in the request list, but their data is allocated by
2115 * the VFs themselves. We don't include them in the bp->fw_stats_num as
2116 * it is used to determine where to place the vf stats queries in the
2117 * request struct
2118 */
2119 if (IS_SRIOV(bp))
6411280a 2120 vf_headroom = bnx2x_vf_headroom(bp);
8db573ba 2121
ad5afc89
AE		 2122 /* Request is built from stats_query_header and an array of
2123 * stats_query_cmd_group each of which contains
2124 * STATS_QUERY_CMD_COUNT rules. The real number or requests is
2125 * configured in the stats_query_header.
2126 */
2127 num_groups =
8db573ba
AE		 2128 (((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) +
2129 (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ?
ad5afc89
AE		 2130 1 : 0));
2131
8db573ba
AE		 2132 DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n",
2133 bp->fw_stats_num, vf_headroom, num_groups);
ad5afc89
AE		 2134 bp->fw_stats_req_sz = sizeof(struct stats_query_header) +
2135 num_groups * sizeof(struct stats_query_cmd_group);
2136
2137 /* Data for statistics requests + stats_counter
2138 * stats_counter holds per-STORM counters that are incremented
2139 * when STORM has finished with the current request.
2140 * memory for FCoE offloaded statistics are counted anyway,
2141 * even if they will not be sent.
2142 * VF stats are not accounted for here as the data of VF stats is stored
2143 * in memory allocated by the VF, not here.
2144 */
2145 bp->fw_stats_data_sz = sizeof(struct per_port_stats) +
2146 sizeof(struct per_pf_stats) +
2147 sizeof(struct fcoe_statistics_params) +
2148 sizeof(struct per_queue_stats) * num_queue_stats +
2149 sizeof(struct stats_counter);
2150
2151 BNX2X_PCI_ALLOC(bp->fw_stats, &bp->fw_stats_mapping,
2152 bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2153
2154 /* Set shortcuts */
2155 bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats;
2156 bp->fw_stats_req_mapping = bp->fw_stats_mapping;
2157 bp->fw_stats_data = (struct bnx2x_fw_stats_data *)
2158 ((u8 *)bp->fw_stats + bp->fw_stats_req_sz);
2159 bp->fw_stats_data_mapping = bp->fw_stats_mapping +
2160 bp->fw_stats_req_sz;
2161
2162 DP(BNX2X_MSG_SP, "statistics request base address set to %x %x",
2163 U64_HI(bp->fw_stats_req_mapping),
2164 U64_LO(bp->fw_stats_req_mapping));
2165 DP(BNX2X_MSG_SP, "statistics data base address set to %x %x",
2166 U64_HI(bp->fw_stats_data_mapping),
2167 U64_LO(bp->fw_stats_data_mapping));
2168 return 0;
2169
2170alloc_mem_err:
2171 bnx2x_free_fw_stats_mem(bp);
2172 BNX2X_ERR("Can't allocate FW stats memory\n");
2173 return -ENOMEM;
2174}
2175
2176/* send load request to mcp and analyze response */
2177static int bnx2x_nic_load_request(struct bnx2x *bp, u32 *load_code)
2178{
2179 /* init fw_seq */
2180 bp->fw_seq =
2181 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2182 DRV_MSG_SEQ_NUMBER_MASK);
2183 BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2184
2185 /* Get current FW pulse sequence */
2186 bp->fw_drv_pulse_wr_seq =
2187 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2188 DRV_PULSE_SEQ_MASK);
2189 BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2190
2191 /* load request */
2192 (*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ,
2193 DRV_MSG_CODE_LOAD_REQ_WITH_LFA);
2194
2195 /* if mcp fails to respond we must abort */
2196 if (!(*load_code)) {
2197 BNX2X_ERR("MCP response failure, aborting\n");
2198 return -EBUSY;
2199 }
2200
2201 /* If mcp refused (e.g. other port is in diagnostic mode) we
2202 * must abort
2203 */
2204 if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2205 BNX2X_ERR("MCP refused load request, aborting\n");
2206 return -EBUSY;
2207 }
2208 return 0;
2209}
2210
2211/* check whether another PF has already loaded FW to chip. In
2212 * virtualized environments a pf from another VM may have already
2213 * initialized the device including loading FW
2214 */
2215int bnx2x_nic_load_analyze_req(struct bnx2x *bp, u32 load_code)
2216{
2217 /* is another pf loaded on this engine? */
2218 if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2219 load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2220 /* build my FW version dword */
2221 u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
2222 (BCM_5710_FW_MINOR_VERSION << 8) +
2223 (BCM_5710_FW_REVISION_VERSION << 16) +
2224 (BCM_5710_FW_ENGINEERING_VERSION << 24);
2225
2226 /* read loaded FW from chip */
2227 u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2228
2229 DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x\n",
2230 loaded_fw, my_fw);
2231
2232 /* abort nic load if version mismatch */
2233 if (my_fw != loaded_fw) {
2234 BNX2X_ERR("bnx2x with FW %x was already loaded which mismatches my %x FW. aborting\n",
452427b0 2235 loaded_fw, my_fw);
ad5afc89
AE		 2236 return -EBUSY;
2237 }
2238 }
2239 return 0;
2240}
2241
2242/* returns the "mcp load_code" according to global load_count array */
2243static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port)
2244{
2245 int path = BP_PATH(bp);
2246
2247 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d] %d, %d, %d\n",
2248 path, load_count[path][0], load_count[path][1],
2249 load_count[path][2]);
2250 load_count[path][0]++;
2251 load_count[path][1 + port]++;
2252 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d] %d, %d, %d\n",
2253 path, load_count[path][0], load_count[path][1],
2254 load_count[path][2]);
2255 if (load_count[path][0] == 1)
2256 return FW_MSG_CODE_DRV_LOAD_COMMON;
2257 else if (load_count[path][1 + port] == 1)
2258 return FW_MSG_CODE_DRV_LOAD_PORT;
2259 else
2260 return FW_MSG_CODE_DRV_LOAD_FUNCTION;
2261}
2262
2263/* mark PMF if applicable */
2264static void bnx2x_nic_load_pmf(struct bnx2x *bp, u32 load_code)
2265{
2266 if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2267 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2268 (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2269 bp->port.pmf = 1;
2270 /* We need the barrier to ensure the ordering between the
2271 * writing to bp->port.pmf here and reading it from the
2272 * bnx2x_periodic_task().
2273 */
2274 smp_mb();
2275 } else {
2276 bp->port.pmf = 0;
452427b0
YM		 2277 }
2278
ad5afc89
AE		 2279 DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
2280}
2281
2282static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code)
2283{
2284 if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2285 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2286 (bp->common.shmem2_base)) {
2287 if (SHMEM2_HAS(bp, dcc_support))
2288 SHMEM2_WR(bp, dcc_support,
2289 (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2290 SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2291 if (SHMEM2_HAS(bp, afex_driver_support))
2292 SHMEM2_WR(bp, afex_driver_support,
2293 SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2294 }
2295
2296 /* Set AFEX default VLAN tag to an invalid value */
2297 bp->afex_def_vlan_tag = -1;
452427b0
YM		 2298}
2299
1191cb83
ED		 2300/**
2301 * bnx2x_bz_fp - zero content of the fastpath structure.
2302 *
2303 * @bp: driver handle
2304 * @index: fastpath index to be zeroed
2305 *
2306 * Makes sure the contents of the bp->fp[index].napi is kept
2307 * intact.
2308 */
2309static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2310{
2311 struct bnx2x_fastpath *fp = &bp->fp[index];
15192a8c 2312
65565884 2313 int cos;
1191cb83 2314 struct napi_struct orig_napi = fp->napi;
15192a8c 2315 struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
1191cb83 2316 /* bzero bnx2x_fastpath contents */
c3146eb6
DK		 2317 if (fp->tpa_info)
2318 memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 *
2319 sizeof(struct bnx2x_agg_info));
2320 memset(fp, 0, sizeof(*fp));
1191cb83
ED		 2321
2322 /* Restore the NAPI object as it has been already initialized */
2323 fp->napi = orig_napi;
15192a8c 2324 fp->tpa_info = orig_tpa_info;
1191cb83
ED		 2325 fp->bp = bp;
2326 fp->index = index;
2327 if (IS_ETH_FP(fp))
2328 fp->max_cos = bp->max_cos;
2329 else
2330 /* Special queues support only one CoS */
2331 fp->max_cos = 1;
2332
65565884 2333 /* Init txdata pointers */
65565884
MS		 2334 if (IS_FCOE_FP(fp))
2335 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
65565884
MS		 2336 if (IS_ETH_FP(fp))
2337 for_each_cos_in_tx_queue(fp, cos)
2338 fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2339 BNX2X_NUM_ETH_QUEUES(bp) + index];
2340
1191cb83
ED		 2341 /*
2342 * set the tpa flag for each queue. The tpa flag determines the queue
2343 * minimal size so it must be set prior to queue memory allocation
2344 */
2345 fp->disable_tpa = !(bp->flags & TPA_ENABLE_FLAG ||
2346 (bp->flags & GRO_ENABLE_FLAG &&
2347 bnx2x_mtu_allows_gro(bp->dev->mtu)));
2348 if (bp->flags & TPA_ENABLE_FLAG)
2349 fp->mode = TPA_MODE_LRO;
2350 else if (bp->flags & GRO_ENABLE_FLAG)
2351 fp->mode = TPA_MODE_GRO;
2352
1191cb83
ED		 2353 /* We don't want TPA on an FCoE L2 ring */
2354 if (IS_FCOE_FP(fp))
2355 fp->disable_tpa = 1;
55c11941
MS		 2356}
2357
2358int bnx2x_load_cnic(struct bnx2x *bp)
2359{
2360 int i, rc, port = BP_PORT(bp);
2361
2362 DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2363
2364 mutex_init(&bp->cnic_mutex);
2365
ad5afc89
AE		 2366 if (IS_PF(bp)) {
2367 rc = bnx2x_alloc_mem_cnic(bp);
2368 if (rc) {
2369 BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2370 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2371 }
55c11941
MS		 2372 }
2373
2374 rc = bnx2x_alloc_fp_mem_cnic(bp);
2375 if (rc) {
2376 BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2377 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2378 }
2379
2380 /* Update the number of queues with the cnic queues */
2381 rc = bnx2x_set_real_num_queues(bp, 1);
2382 if (rc) {
2383 BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2384 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2385 }
2386
2387 /* Add all CNIC NAPI objects */
2388 bnx2x_add_all_napi_cnic(bp);
2389 DP(NETIF_MSG_IFUP, "cnic napi added\n");
2390 bnx2x_napi_enable_cnic(bp);
2391
2392 rc = bnx2x_init_hw_func_cnic(bp);
2393 if (rc)
2394 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2395
2396 bnx2x_nic_init_cnic(bp);
2397
ad5afc89
AE		 2398 if (IS_PF(bp)) {
2399 /* Enable Timer scan */
2400 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2401
2402 /* setup cnic queues */
2403 for_each_cnic_queue(bp, i) {
2404 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2405 if (rc) {
2406 BNX2X_ERR("Queue setup failed\n");
2407 LOAD_ERROR_EXIT(bp, load_error_cnic2);
2408 }
55c11941
MS		 2409 }
2410 }
2411
2412 /* Initialize Rx filter. */
2413 netif_addr_lock_bh(bp->dev);
2414 bnx2x_set_rx_mode(bp->dev);
2415 netif_addr_unlock_bh(bp->dev);
2416
2417 /* re-read iscsi info */
2418 bnx2x_get_iscsi_info(bp);
2419 bnx2x_setup_cnic_irq_info(bp);
2420 bnx2x_setup_cnic_info(bp);
2421 bp->cnic_loaded = true;
2422 if (bp->state == BNX2X_STATE_OPEN)
2423 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2424
2425
2426 DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2427
2428 return 0;
2429
2430#ifndef BNX2X_STOP_ON_ERROR
2431load_error_cnic2:
2432 /* Disable Timer scan */
2433 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2434
2435load_error_cnic1:
2436 bnx2x_napi_disable_cnic(bp);
2437 /* Update the number of queues without the cnic queues */
2438 rc = bnx2x_set_real_num_queues(bp, 0);
2439 if (rc)
2440 BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2441load_error_cnic0:
2442 BNX2X_ERR("CNIC-related load failed\n");
2443 bnx2x_free_fp_mem_cnic(bp);
2444 bnx2x_free_mem_cnic(bp);
2445 return rc;
2446#endif /* ! BNX2X_STOP_ON_ERROR */
1191cb83
ED		 2447}
2448
9f6c9258
DK		 2449/* must be called with rtnl_lock */
2450int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2451{
619c5cb6 2452 int port = BP_PORT(bp);
ad5afc89 2453 int i, rc = 0, load_code = 0;
9f6c9258 2454
55c11941
MS		 2455 DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2456 DP(NETIF_MSG_IFUP,
2457 "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2458
9f6c9258 2459#ifdef BNX2X_STOP_ON_ERROR
51c1a580
MS		 2460 if (unlikely(bp->panic)) {
2461 BNX2X_ERR("Can't load NIC when there is panic\n");
9f6c9258 2462 return -EPERM;
51c1a580 2463 }
9f6c9258
DK		 2464#endif
2465
2466 bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2467
2ae17f66
VZ		 2468 memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2469 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2470 &bp->last_reported_link.link_report_flags);
2ae17f66 2471
ad5afc89
AE		 2472 if (IS_PF(bp))
2473 /* must be called before memory allocation and HW init */
2474 bnx2x_ilt_set_info(bp);
523224a3 2475
6383c0b3
AE		 2476 /*
2477 * Zero fastpath structures preserving invariants like napi, which are
2478 * allocated only once, fp index, max_cos, bp pointer.
65565884 2479 * Also set fp->disable_tpa and txdata_ptr.
b3b83c3f 2480 */
51c1a580 2481 DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
b3b83c3f
DK		 2482 for_each_queue(bp, i)
2483 bnx2x_bz_fp(bp, i);
55c11941
MS		 2484 memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2485 bp->num_cnic_queues) *
2486 sizeof(struct bnx2x_fp_txdata));
b3b83c3f 2487
55c11941 2488 bp->fcoe_init = false;
6383c0b3 2489
a8c94b91
VZ		 2490 /* Set the receive queues buffer size */
2491 bnx2x_set_rx_buf_size(bp);
2492
ad5afc89
AE		 2493 if (IS_PF(bp)) {
2494 rc = bnx2x_alloc_mem(bp);
2495 if (rc) {
2496 BNX2X_ERR("Unable to allocate bp memory\n");
2497 return rc;
2498 }
2499 }
2500
2501 /* Allocated memory for FW statistics */
2502 if (bnx2x_alloc_fw_stats_mem(bp))
2503 LOAD_ERROR_EXIT(bp, load_error0);
2504
2505 /* need to be done after alloc mem, since it's self adjusting to amount
2506 * of memory available for RSS queues
2507 */
2508 rc = bnx2x_alloc_fp_mem(bp);
2509 if (rc) {
2510 BNX2X_ERR("Unable to allocate memory for fps\n");
2511 LOAD_ERROR_EXIT(bp, load_error0);
2512 }
d6214d7a 2513
8d9ac297
AE		 2514 /* request pf to initialize status blocks */
2515 if (IS_VF(bp)) {
2516 rc = bnx2x_vfpf_init(bp);
2517 if (rc)
2518 LOAD_ERROR_EXIT(bp, load_error0);
2519 }
2520
b3b83c3f
DK		 2521 /* As long as bnx2x_alloc_mem() may possibly update
2522 * bp->num_queues, bnx2x_set_real_num_queues() should always
55c11941 2523 * come after it. At this stage cnic queues are not counted.
b3b83c3f 2524 */
55c11941 2525 rc = bnx2x_set_real_num_queues(bp, 0);
d6214d7a 2526 if (rc) {
ec6ba945 2527 BNX2X_ERR("Unable to set real_num_queues\n");
619c5cb6 2528 LOAD_ERROR_EXIT(bp, load_error0);
9f6c9258
DK		 2529 }
2530
6383c0b3
AE		 2531 /* configure multi cos mappings in kernel.
2532 * this configuration may be overriden by a multi class queue discipline
2533 * or by a dcbx negotiation result.
2534 */
2535 bnx2x_setup_tc(bp->dev, bp->max_cos);
2536
26614ba5
MS		 2537 /* Add all NAPI objects */
2538 bnx2x_add_all_napi(bp);
55c11941 2539 DP(NETIF_MSG_IFUP, "napi added\n");
9f6c9258
DK		 2540 bnx2x_napi_enable(bp);
2541
ad5afc89
AE		 2542 if (IS_PF(bp)) {
2543 /* set pf load just before approaching the MCP */
2544 bnx2x_set_pf_load(bp);
2545
2546 /* if mcp exists send load request and analyze response */
2547 if (!BP_NOMCP(bp)) {
2548 /* attempt to load pf */
2549 rc = bnx2x_nic_load_request(bp, &load_code);
2550 if (rc)
2551 LOAD_ERROR_EXIT(bp, load_error1);
2552
2553 /* what did mcp say? */
2554 rc = bnx2x_nic_load_analyze_req(bp, load_code);
2555 if (rc) {
2556 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
d1e2d966
AE		 2557 LOAD_ERROR_EXIT(bp, load_error2);
2558 }
ad5afc89
AE		 2559 } else {
2560 load_code = bnx2x_nic_load_no_mcp(bp, port);
d1e2d966 2561 }
9f6c9258 2562
ad5afc89
AE		 2563 /* mark pmf if applicable */
2564 bnx2x_nic_load_pmf(bp, load_code);
9f6c9258 2565
ad5afc89
AE		 2566 /* Init Function state controlling object */
2567 bnx2x__init_func_obj(bp);
6383c0b3 2568
ad5afc89
AE		 2569 /* Initialize HW */
2570 rc = bnx2x_init_hw(bp, load_code);
2571 if (rc) {
2572 BNX2X_ERR("HW init failed, aborting\n");
2573 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2574 LOAD_ERROR_EXIT(bp, load_error2);
2575 }
9f6c9258
DK		 2576 }
2577
d6214d7a
DK		 2578 /* Connect to IRQs */
2579 rc = bnx2x_setup_irqs(bp);
523224a3 2580 if (rc) {
ad5afc89
AE		 2581 BNX2X_ERR("setup irqs failed\n");
2582 if (IS_PF(bp))
2583 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
619c5cb6 2584 LOAD_ERROR_EXIT(bp, load_error2);
523224a3
DK		 2585 }
2586
9f6c9258
DK		 2587 /* Setup NIC internals and enable interrupts */
2588 bnx2x_nic_init(bp, load_code);
2589
619c5cb6 2590 /* Init per-function objects */
ad5afc89
AE		 2591 if (IS_PF(bp)) {
2592 bnx2x_init_bp_objs(bp);
b56e9670 2593 bnx2x_iov_nic_init(bp);
a3348722 2594
ad5afc89
AE		 2595 /* Set AFEX default VLAN tag to an invalid value */
2596 bp->afex_def_vlan_tag = -1;
2597 bnx2x_nic_load_afex_dcc(bp, load_code);
2598 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2599 rc = bnx2x_func_start(bp);
2600 if (rc) {
2601 BNX2X_ERR("Function start failed!\n");
2602 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
9f6c9258 2603
619c5cb6 2604 LOAD_ERROR_EXIT(bp, load_error3);
9f6c9258 2605 }
9f6c9258 2606
ad5afc89
AE		 2607 /* Send LOAD_DONE command to MCP */
2608 if (!BP_NOMCP(bp)) {
2609 load_code = bnx2x_fw_command(bp,
2610 DRV_MSG_CODE_LOAD_DONE, 0);
2611 if (!load_code) {
2612 BNX2X_ERR("MCP response failure, aborting\n");
2613 rc = -EBUSY;
2614 LOAD_ERROR_EXIT(bp, load_error3);
2615 }
2616 }
9f6c9258 2617
ad5afc89
AE		 2618 /* setup the leading queue */
2619 rc = bnx2x_setup_leading(bp);
51c1a580 2620 if (rc) {
ad5afc89 2621 BNX2X_ERR("Setup leading failed!\n");
55c11941 2622 LOAD_ERROR_EXIT(bp, load_error3);
51c1a580 2623 }
523224a3 2624
ad5afc89
AE		 2625 /* set up the rest of the queues */
2626 for_each_nondefault_eth_queue(bp, i) {
2627 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2628 if (rc) {
2629 BNX2X_ERR("Queue setup failed\n");
2630 LOAD_ERROR_EXIT(bp, load_error3);
2631 }
2632 }
2633
2634 /* setup rss */
2635 rc = bnx2x_init_rss_pf(bp);
2636 if (rc) {
2637 BNX2X_ERR("PF RSS init failed\n");
2638 LOAD_ERROR_EXIT(bp, load_error3);
2639 }
8d9ac297
AE		 2640
2641 } else { /* vf */
2642 for_each_eth_queue(bp, i) {
2643 rc = bnx2x_vfpf_setup_q(bp, i);
2644 if (rc) {
2645 BNX2X_ERR("Queue setup failed\n");
2646 LOAD_ERROR_EXIT(bp, load_error3);
2647 }
2648 }
51c1a580 2649 }
619c5cb6 2650
523224a3
DK		 2651 /* Now when Clients are configured we are ready to work */
2652 bp->state = BNX2X_STATE_OPEN;
2653
619c5cb6 2654 /* Configure a ucast MAC */
ad5afc89
AE		 2655 if (IS_PF(bp))
2656 rc = bnx2x_set_eth_mac(bp, true);
8d9ac297
AE		 2657 else /* vf */
2658 rc = bnx2x_vfpf_set_mac(bp);
51c1a580
MS		 2659 if (rc) {
2660 BNX2X_ERR("Setting Ethernet MAC failed\n");
55c11941 2661 LOAD_ERROR_EXIT(bp, load_error3);
51c1a580 2662 }
6e30dd4e 2663
ad5afc89 2664 if (IS_PF(bp) && bp->pending_max) {
e3835b99
DK		 2665 bnx2x_update_max_mf_config(bp, bp->pending_max);
2666 bp->pending_max = 0;
2667 }
2668
ad5afc89
AE		 2669 if (bp->port.pmf) {
2670 rc = bnx2x_initial_phy_init(bp, load_mode);
2671 if (rc)
2672 LOAD_ERROR_EXIT(bp, load_error3);
2673 }
c63da990 2674 bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
9f6c9258 2675
619c5cb6
VZ		 2676 /* Start fast path */
2677
2678 /* Initialize Rx filter. */
2679 netif_addr_lock_bh(bp->dev);
6e30dd4e 2680 bnx2x_set_rx_mode(bp->dev);
619c5cb6 2681 netif_addr_unlock_bh(bp->dev);
6e30dd4e 2682
619c5cb6 2683 /* Start the Tx */
9f6c9258
DK		 2684 switch (load_mode) {
2685 case LOAD_NORMAL:
523224a3
DK		 2686 /* Tx queue should be only reenabled */
2687 netif_tx_wake_all_queues(bp->dev);
9f6c9258
DK		 2688 break;
2689
2690 case LOAD_OPEN:
2691 netif_tx_start_all_queues(bp->dev);
523224a3 2692 smp_mb__after_clear_bit();
9f6c9258
DK		 2693 break;
2694
2695 case LOAD_DIAG:
8970b2e4 2696 case LOAD_LOOPBACK_EXT:
9f6c9258
DK		 2697 bp->state = BNX2X_STATE_DIAG;
2698 break;
2699
2700 default:
2701 break;
2702 }
2703
00253a8c 2704 if (bp->port.pmf)
4c704899 2705 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
00253a8c 2706 else
9f6c9258
DK		 2707 bnx2x__link_status_update(bp);
2708
2709 /* start the timer */
2710 mod_timer(&bp->timer, jiffies + bp->current_interval);
2711
55c11941
MS		 2712 if (CNIC_ENABLED(bp))
2713 bnx2x_load_cnic(bp);
9f6c9258 2714
ad5afc89
AE		 2715 if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2716 /* mark driver is loaded in shmem2 */
9ce392d4
YM		 2717 u32 val;
2718 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2719 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2720 val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2721 DRV_FLAGS_CAPABILITIES_LOADED_L2);
2722 }
2723
619c5cb6 2724 /* Wait for all pending SP commands to complete */
ad5afc89 2725 if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) {
619c5cb6 2726 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
5d07d868 2727 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
619c5cb6
VZ		 2728 return -EBUSY;
2729 }
6891dd25 2730
9876879f
BW		 2731 /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2732 if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2733 bnx2x_dcbx_init(bp, false);
2734
55c11941
MS		 2735 DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2736
9f6c9258
DK		 2737 return 0;
2738
619c5cb6 2739#ifndef BNX2X_STOP_ON_ERROR
9f6c9258 2740load_error3:
ad5afc89
AE		 2741 if (IS_PF(bp)) {
2742 bnx2x_int_disable_sync(bp, 1);
d6214d7a 2743
ad5afc89
AE		 2744 /* Clean queueable objects */
2745 bnx2x_squeeze_objects(bp);
2746 }
619c5cb6 2747
9f6c9258
DK		 2748 /* Free SKBs, SGEs, TPA pool and driver internals */
2749 bnx2x_free_skbs(bp);
ec6ba945 2750 for_each_rx_queue(bp, i)
9f6c9258 2751 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
d6214d7a 2752
9f6c9258 2753 /* Release IRQs */
d6214d7a
DK		 2754 bnx2x_free_irq(bp);
2755load_error2:
ad5afc89 2756 if (IS_PF(bp) && !BP_NOMCP(bp)) {
d6214d7a
DK		 2757 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2758 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2759 }
2760
2761 bp->port.pmf = 0;
9f6c9258
DK		 2762load_error1:
2763 bnx2x_napi_disable(bp);
722c6f58 2764 bnx2x_del_all_napi(bp);
ad5afc89 2765
889b9af3 2766 /* clear pf_load status, as it was already set */
ad5afc89
AE		 2767 if (IS_PF(bp))
2768 bnx2x_clear_pf_load(bp);
d6214d7a 2769load_error0:
ad5afc89
AE		 2770 bnx2x_free_fp_mem(bp);
2771 bnx2x_free_fw_stats_mem(bp);
9f6c9258
DK		 2772 bnx2x_free_mem(bp);
2773
2774 return rc;
619c5cb6 2775#endif /* ! BNX2X_STOP_ON_ERROR */
9f6c9258
DK		 2776}
2777
7fa6f340 2778int bnx2x_drain_tx_queues(struct bnx2x *bp)
ad5afc89
AE		 2779{
2780 u8 rc = 0, cos, i;
2781
2782 /* Wait until tx fastpath tasks complete */
2783 for_each_tx_queue(bp, i) {
2784 struct bnx2x_fastpath *fp = &bp->fp[i];
2785
2786 for_each_cos_in_tx_queue(fp, cos)
2787 rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]);
2788 if (rc)
2789 return rc;
2790 }
2791 return 0;
2792}
2793
9f6c9258 2794/* must be called with rtnl_lock */
5d07d868 2795int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
9f6c9258
DK		 2796{
2797 int i;
c9ee9206
VZ		 2798 bool global = false;
2799
55c11941
MS		 2800 DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2801
9ce392d4 2802 /* mark driver is unloaded in shmem2 */
ad5afc89 2803 if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
9ce392d4
YM		 2804 u32 val;
2805 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2806 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2807 val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2808 }
2809
80bfe5cc 2810 if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE &&
ad5afc89
AE		 2811 (bp->state == BNX2X_STATE_CLOSED ||
2812 bp->state == BNX2X_STATE_ERROR)) {
c9ee9206
VZ		 2813 /* We can get here if the driver has been unloaded
2814 * during parity error recovery and is either waiting for a
2815 * leader to complete or for other functions to unload and
2816 * then ifdown has been issued. In this case we want to
2817 * unload and let other functions to complete a recovery
2818 * process.
2819 */
9f6c9258
DK		 2820 bp->recovery_state = BNX2X_RECOVERY_DONE;
2821 bp->is_leader = 0;
c9ee9206
VZ		 2822 bnx2x_release_leader_lock(bp);
2823 smp_mb();
2824
51c1a580
MS		 2825 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2826 BNX2X_ERR("Can't unload in closed or error state\n");
9f6c9258
DK		 2827 return -EINVAL;
2828 }
2829
80bfe5cc
YM		 2830 /* Nothing to do during unload if previous bnx2x_nic_load()
2831 * have not completed succesfully - all resourses are released.
2832 *
2833 * we can get here only after unsuccessful ndo_* callback, during which
2834 * dev->IFF_UP flag is still on.
2835 */
2836 if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR)
2837 return 0;
2838
2839 /* It's important to set the bp->state to the value different from
87b7ba3d
VZ		 2840 * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2841 * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2842 */
2843 bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2844 smp_mb();
2845
55c11941
MS		 2846 if (CNIC_LOADED(bp))
2847 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2848
9505ee37
VZ		 2849 /* Stop Tx */
2850 bnx2x_tx_disable(bp);
65565884 2851 netdev_reset_tc(bp->dev);
9505ee37 2852
9f6c9258 2853 bp->rx_mode = BNX2X_RX_MODE_NONE;
9f6c9258 2854
9f6c9258 2855 del_timer_sync(&bp->timer);
f85582f8 2856
ad5afc89
AE		 2857 if (IS_PF(bp)) {
2858 /* Set ALWAYS_ALIVE bit in shmem */
2859 bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2860 bnx2x_drv_pulse(bp);
2861 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2862 bnx2x_save_statistics(bp);
2863 }
9f6c9258 2864
ad5afc89
AE		 2865 /* wait till consumers catch up with producers in all queues */
2866 bnx2x_drain_tx_queues(bp);
9f6c9258 2867
9b176b6b
AE		 2868 /* if VF indicate to PF this function is going down (PF will delete sp
2869 * elements and clear initializations
2870 */
2871 if (IS_VF(bp))
2872 bnx2x_vfpf_close_vf(bp);
2873 else if (unload_mode != UNLOAD_RECOVERY)
2874 /* if this is a normal/close unload need to clean up chip*/
5d07d868 2875 bnx2x_chip_cleanup(bp, unload_mode, keep_link);
523224a3 2876 else {
c9ee9206
VZ		 2877 /* Send the UNLOAD_REQUEST to the MCP */
2878 bnx2x_send_unload_req(bp, unload_mode);
2879
2880 /*
2881 * Prevent transactions to host from the functions on the
2882 * engine that doesn't reset global blocks in case of global
2883 * attention once gloabl blocks are reset and gates are opened
2884 * (the engine which leader will perform the recovery
2885 * last).
2886 */
2887 if (!CHIP_IS_E1x(bp))
2888 bnx2x_pf_disable(bp);
2889
2890 /* Disable HW interrupts, NAPI */
523224a3 2891 bnx2x_netif_stop(bp, 1);
26614ba5
MS		 2892 /* Delete all NAPI objects */
2893 bnx2x_del_all_napi(bp);
55c11941
MS		 2894 if (CNIC_LOADED(bp))
2895 bnx2x_del_all_napi_cnic(bp);
523224a3 2896 /* Release IRQs */
d6214d7a 2897 bnx2x_free_irq(bp);
c9ee9206
VZ		 2898
2899 /* Report UNLOAD_DONE to MCP */
5d07d868 2900 bnx2x_send_unload_done(bp, false);
523224a3 2901 }
9f6c9258 2902
619c5cb6
VZ		 2903 /*
2904 * At this stage no more interrupts will arrive so we may safly clean
2905 * the queueable objects here in case they failed to get cleaned so far.
2906 */
ad5afc89
AE		 2907 if (IS_PF(bp))
2908 bnx2x_squeeze_objects(bp);
619c5cb6 2909
79616895
VZ		 2910 /* There should be no more pending SP commands at this stage */
2911 bp->sp_state = 0;
2912
9f6c9258
DK		 2913 bp->port.pmf = 0;
2914
2915 /* Free SKBs, SGEs, TPA pool and driver internals */
2916 bnx2x_free_skbs(bp);
55c11941
MS		 2917 if (CNIC_LOADED(bp))
2918 bnx2x_free_skbs_cnic(bp);
ec6ba945 2919 for_each_rx_queue(bp, i)
9f6c9258 2920 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
d6214d7a 2921
ad5afc89
AE		 2922 bnx2x_free_fp_mem(bp);
2923 if (CNIC_LOADED(bp))
55c11941 2924 bnx2x_free_fp_mem_cnic(bp);
9f6c9258 2925
ad5afc89
AE		 2926 if (IS_PF(bp)) {
2927 bnx2x_free_mem(bp);
2928 if (CNIC_LOADED(bp))
2929 bnx2x_free_mem_cnic(bp);
2930 }
9f6c9258 2931 bp->state = BNX2X_STATE_CLOSED;
55c11941 2932 bp->cnic_loaded = false;
9f6c9258 2933
c9ee9206
VZ		 2934 /* Check if there are pending parity attentions. If there are - set
2935 * RECOVERY_IN_PROGRESS.
2936 */
ad5afc89 2937 if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) {
c9ee9206
VZ		 2938 bnx2x_set_reset_in_progress(bp);
2939
2940 /* Set RESET_IS_GLOBAL if needed */
2941 if (global)
2942 bnx2x_set_reset_global(bp);
2943 }
2944
2945
9f6c9258
DK		 2946 /* The last driver must disable a "close the gate" if there is no
2947 * parity attention or "process kill" pending.
2948 */
ad5afc89
AE		 2949 if (IS_PF(bp) &&
2950 !bnx2x_clear_pf_load(bp) &&
2951 bnx2x_reset_is_done(bp, BP_PATH(bp)))
9f6c9258
DK		 2952 bnx2x_disable_close_the_gate(bp);
2953
55c11941
MS		 2954 DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
2955
9f6c9258
DK		 2956 return 0;
2957}
f85582f8 2958
9f6c9258
DK		 2959int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
2960{
2961 u16 pmcsr;
2962
adf5f6a1
DK		 2963 /* If there is no power capability, silently succeed */
2964 if (!bp->pm_cap) {
51c1a580 2965 BNX2X_DEV_INFO("No power capability. Breaking.\n");
adf5f6a1
DK		 2966 return 0;
2967 }
2968
9f6c9258
DK		 2969 pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
2970
2971 switch (state) {
2972 case PCI_D0:
2973 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2974 ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
2975 PCI_PM_CTRL_PME_STATUS));
2976
2977 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
2978 /* delay required during transition out of D3hot */
2979 msleep(20);
2980 break;
2981
2982 case PCI_D3hot:
2983 /* If there are other clients above don't
2984 shut down the power */
2985 if (atomic_read(&bp->pdev->enable_cnt) != 1)
2986 return 0;
2987 /* Don't shut down the power for emulation and FPGA */
2988 if (CHIP_REV_IS_SLOW(bp))
2989 return 0;
2990
2991 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
2992 pmcsr |= 3;
2993
2994 if (bp->wol)
2995 pmcsr |= PCI_PM_CTRL_PME_ENABLE;
2996
2997 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2998 pmcsr);
2999
3000 /* No more memory access after this point until
3001 * device is brought back to D0.
3002 */
3003 break;
3004
3005 default:
51c1a580 3006 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
9f6c9258
DK		 3007 return -EINVAL;
3008 }
3009 return 0;
3010}
3011
9f6c9258
DK		 3012/*
3013 * net_device service functions
3014 */
d6214d7a 3015int bnx2x_poll(struct napi_struct *napi, int budget)
9f6c9258
DK		 3016{
3017 int work_done = 0;
6383c0b3 3018 u8 cos;
9f6c9258
DK		 3019 struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3020 napi);
3021 struct bnx2x *bp = fp->bp;
3022
3023 while (1) {
3024#ifdef BNX2X_STOP_ON_ERROR
3025 if (unlikely(bp->panic)) {
3026 napi_complete(napi);
3027 return 0;
3028 }
3029#endif
3030
6383c0b3 3031 for_each_cos_in_tx_queue(fp, cos)
65565884
MS		 3032 if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
3033 bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
6383c0b3 3034
9f6c9258
DK		 3035 if (bnx2x_has_rx_work(fp)) {
3036 work_done += bnx2x_rx_int(fp, budget - work_done);
3037
3038 /* must not complete if we consumed full budget */
3039 if (work_done >= budget)
3040 break;
3041 }
3042
3043 /* Fall out from the NAPI loop if needed */
3044 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
55c11941 3045
ec6ba945
VZ		 3046 /* No need to update SB for FCoE L2 ring as long as
3047 * it's connected to the default SB and the SB
3048 * has been updated when NAPI was scheduled.
3049 */
3050 if (IS_FCOE_FP(fp)) {
3051 napi_complete(napi);
3052 break;
3053 }
9f6c9258 3054 bnx2x_update_fpsb_idx(fp);
f85582f8
DK		 3055 /* bnx2x_has_rx_work() reads the status block,
3056 * thus we need to ensure that status block indices
3057 * have been actually read (bnx2x_update_fpsb_idx)
3058 * prior to this check (bnx2x_has_rx_work) so that
3059 * we won't write the "newer" value of the status block
3060 * to IGU (if there was a DMA right after
3061 * bnx2x_has_rx_work and if there is no rmb, the memory
3062 * reading (bnx2x_update_fpsb_idx) may be postponed
3063 * to right before bnx2x_ack_sb). In this case there
3064 * will never be another interrupt until there is
3065 * another update of the status block, while there
3066 * is still unhandled work.
3067 */
9f6c9258
DK		 3068 rmb();
3069
3070 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3071 napi_complete(napi);
3072 /* Re-enable interrupts */
51c1a580 3073 DP(NETIF_MSG_RX_STATUS,
523224a3
DK		 3074 "Update index to %d\n", fp->fp_hc_idx);
3075 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
3076 le16_to_cpu(fp->fp_hc_idx),
9f6c9258
DK		 3077 IGU_INT_ENABLE, 1);
3078 break;
3079 }
3080 }
3081 }
3082
3083 return work_done;
3084}
3085
9f6c9258
DK		 3086/* we split the first BD into headers and data BDs
3087 * to ease the pain of our fellow microcode engineers
3088 * we use one mapping for both BDs
9f6c9258 3089 */
91226790
DK		 3090static u16 bnx2x_tx_split(struct bnx2x *bp,
3091 struct bnx2x_fp_txdata *txdata,
3092 struct sw_tx_bd *tx_buf,
3093 struct eth_tx_start_bd **tx_bd, u16 hlen,
3094 u16 bd_prod)
9f6c9258
DK		 3095{
3096 struct eth_tx_start_bd *h_tx_bd = *tx_bd;
3097 struct eth_tx_bd *d_tx_bd;
3098 dma_addr_t mapping;
3099 int old_len = le16_to_cpu(h_tx_bd->nbytes);
3100
3101 /* first fix first BD */
9f6c9258
DK		 3102 h_tx_bd->nbytes = cpu_to_le16(hlen);
3103
91226790
DK		 3104 DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x)\n",
3105 h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo);
9f6c9258
DK		 3106
3107 /* now get a new data BD
3108 * (after the pbd) and fill it */
3109 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
6383c0b3 3110 d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
9f6c9258
DK		 3111
3112 mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
3113 le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
3114
3115 d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3116 d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3117 d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
3118
3119 /* this marks the BD as one that has no individual mapping */
3120 tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
3121
3122 DP(NETIF_MSG_TX_QUEUED,
3123 "TSO split data size is %d (%x:%x)\n",
3124 d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
3125
3126 /* update tx_bd */
3127 *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
3128
3129 return bd_prod;
3130}
3131
86564c3f
YM		 3132#define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32)))
3133#define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16)))
91226790 3134static __le16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
9f6c9258 3135{
86564c3f
YM		 3136 __sum16 tsum = (__force __sum16) csum;
3137
9f6c9258 3138 if (fix > 0)
86564c3f
YM		 3139 tsum = ~csum_fold(csum_sub((__force __wsum) csum,
3140 csum_partial(t_header - fix, fix, 0)));
9f6c9258
DK		 3141
3142 else if (fix < 0)
86564c3f
YM		 3143 tsum = ~csum_fold(csum_add((__force __wsum) csum,
3144 csum_partial(t_header, -fix, 0)));
9f6c9258 3145
e2593fcd 3146 return bswab16(tsum);
9f6c9258
DK		 3147}
3148
91226790 3149static u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
9f6c9258
DK		 3150{
3151 u32 rc;
a848ade4
DK		 3152 __u8 prot = 0;
3153 __be16 protocol;
9f6c9258
DK		 3154
3155 if (skb->ip_summed != CHECKSUM_PARTIAL)
a848ade4 3156 return XMIT_PLAIN;
9f6c9258 3157
a848ade4
DK		 3158 protocol = vlan_get_protocol(skb);
3159 if (protocol == htons(ETH_P_IPV6)) {
3160 rc = XMIT_CSUM_V6;
3161 prot = ipv6_hdr(skb)->nexthdr;
3162 } else {
3163 rc = XMIT_CSUM_V4;
3164 prot = ip_hdr(skb)->protocol;
3165 }
9f6c9258 3166
a848ade4
DK		 3167 if (!CHIP_IS_E1x(bp) && skb->encapsulation) {
3168 if (inner_ip_hdr(skb)->version == 6) {
3169 rc |= XMIT_CSUM_ENC_V6;
3170 if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3171 rc |= XMIT_CSUM_TCP;
9f6c9258 3172 } else {
a848ade4
DK		 3173 rc |= XMIT_CSUM_ENC_V4;
3174 if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP)
9f6c9258
DK		 3175 rc |= XMIT_CSUM_TCP;
3176 }
3177 }
a848ade4
DK		 3178 if (prot == IPPROTO_TCP)
3179 rc |= XMIT_CSUM_TCP;
9f6c9258 3180
a848ade4
DK		 3181 if (skb_is_gso_v6(skb)) {
3182 rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6);
3183 if (rc & XMIT_CSUM_ENC)
3184 rc |= XMIT_GSO_ENC_V6;
3185 } else if (skb_is_gso(skb)) {
3186 rc |= (XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP);
3187 if (rc & XMIT_CSUM_ENC)
3188 rc |= XMIT_GSO_ENC_V4;
3189 }
9f6c9258
DK		 3190
3191 return rc;
3192}
3193
3194#if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
3195/* check if packet requires linearization (packet is too fragmented)
3196 no need to check fragmentation if page size > 8K (there will be no
3197 violation to FW restrictions) */
3198static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
3199 u32 xmit_type)
3200{
3201 int to_copy = 0;
3202 int hlen = 0;
3203 int first_bd_sz = 0;
3204
3205 /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
3206 if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
3207
3208 if (xmit_type & XMIT_GSO) {
3209 unsigned short lso_mss = skb_shinfo(skb)->gso_size;
3210 /* Check if LSO packet needs to be copied:
3211 3 = 1 (for headers BD) + 2 (for PBD and last BD) */
3212 int wnd_size = MAX_FETCH_BD - 3;
3213 /* Number of windows to check */
3214 int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
3215 int wnd_idx = 0;
3216 int frag_idx = 0;
3217 u32 wnd_sum = 0;
3218
3219 /* Headers length */
3220 hlen = (int)(skb_transport_header(skb) - skb->data) +
3221 tcp_hdrlen(skb);
3222
3223 /* Amount of data (w/o headers) on linear part of SKB*/
3224 first_bd_sz = skb_headlen(skb) - hlen;
3225
3226 wnd_sum = first_bd_sz;
3227
3228 /* Calculate the first sum - it's special */
3229 for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
3230 wnd_sum +=
9e903e08 3231 skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
9f6c9258
DK		 3232
3233 /* If there was data on linear skb data - check it */
3234 if (first_bd_sz > 0) {
3235 if (unlikely(wnd_sum < lso_mss)) {
3236 to_copy = 1;
3237 goto exit_lbl;
3238 }
3239
3240 wnd_sum -= first_bd_sz;
3241 }
3242
3243 /* Others are easier: run through the frag list and
3244 check all windows */
3245 for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
3246 wnd_sum +=
9e903e08 3247 skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
9f6c9258
DK		 3248
3249 if (unlikely(wnd_sum < lso_mss)) {
3250 to_copy = 1;
3251 break;
3252 }
3253 wnd_sum -=
9e903e08 3254 skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
9f6c9258
DK		 3255 }
3256 } else {
3257 /* in non-LSO too fragmented packet should always
3258 be linearized */
3259 to_copy = 1;
3260 }
3261 }
3262
3263exit_lbl:
3264 if (unlikely(to_copy))
3265 DP(NETIF_MSG_TX_QUEUED,
51c1a580 3266 "Linearization IS REQUIRED for %s packet. num_frags %d hlen %d first_bd_sz %d\n",
9f6c9258
DK		 3267 (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
3268 skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
3269
3270 return to_copy;
3271}
3272#endif
3273
91226790
DK		 3274static void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
3275 u32 xmit_type)
f2e0899f 3276{
a848ade4
DK		 3277 struct ipv6hdr *ipv6;
3278
2297a2da
VZ		 3279 *parsing_data |= (skb_shinfo(skb)->gso_size <<
3280 ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
3281 ETH_TX_PARSE_BD_E2_LSO_MSS;
a848ade4
DK		 3282
3283 if (xmit_type & XMIT_GSO_ENC_V6)
3284 ipv6 = inner_ipv6_hdr(skb);
3285 else if (xmit_type & XMIT_GSO_V6)
3286 ipv6 = ipv6_hdr(skb);
3287 else
3288 ipv6 = NULL;
3289
3290 if (ipv6 && ipv6->nexthdr == NEXTHDR_IPV6)
2297a2da 3291 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
f2e0899f
DK		 3292}
3293
3294/**
e8920674 3295 * bnx2x_set_pbd_gso - update PBD in GSO case.
f2e0899f 3296 *
e8920674
DK		 3297 * @skb: packet skb
3298 * @pbd: parse BD
3299 * @xmit_type: xmit flags
f2e0899f 3300 */
91226790
DK		 3301static void bnx2x_set_pbd_gso(struct sk_buff *skb,
3302 struct eth_tx_parse_bd_e1x *pbd,
3303 u32 xmit_type)
f2e0899f
DK		 3304{
3305 pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
86564c3f 3306 pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq);
91226790 3307 pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb));
f2e0899f
DK		 3308
3309 if (xmit_type & XMIT_GSO_V4) {
86564c3f 3310 pbd->ip_id = bswab16(ip_hdr(skb)->id);
f2e0899f 3311 pbd->tcp_pseudo_csum =
86564c3f
YM		 3312 bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3313 ip_hdr(skb)->daddr,
3314 0, IPPROTO_TCP, 0));
f2e0899f
DK		 3315
3316 } else
3317 pbd->tcp_pseudo_csum =
86564c3f
YM		 3318 bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3319 &ipv6_hdr(skb)->daddr,
3320 0, IPPROTO_TCP, 0));
f2e0899f 3321
86564c3f
YM		 3322 pbd->global_data |=
3323 cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN);
f2e0899f 3324}
f85582f8 3325
a848ade4
DK		 3326/**
3327 * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length
3328 *
3329 * @bp: driver handle
3330 * @skb: packet skb
3331 * @parsing_data: data to be updated
3332 * @xmit_type: xmit flags
3333 *
3334 * 57712/578xx related, when skb has encapsulation
3335 */
3336static u8 bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb,
3337 u32 *parsing_data, u32 xmit_type)
3338{
3339 *parsing_data |=
3340 ((((u8 *)skb_inner_transport_header(skb) - skb->data) >> 1) <<
3341 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3342 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3343
3344 if (xmit_type & XMIT_CSUM_TCP) {
3345 *parsing_data |= ((inner_tcp_hdrlen(skb) / 4) <<
3346 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3347 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3348
3349 return skb_inner_transport_header(skb) +
3350 inner_tcp_hdrlen(skb) - skb->data;
3351 }
3352
3353 /* We support checksum offload for TCP and UDP only.
3354 * No need to pass the UDP header length - it's a constant.
3355 */
3356 return skb_inner_transport_header(skb) +
3357 sizeof(struct udphdr) - skb->data;
3358}
3359
f2e0899f 3360/**
e8920674 3361 * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
f2e0899f 3362 *
e8920674
DK		 3363 * @bp: driver handle
3364 * @skb: packet skb
3365 * @parsing_data: data to be updated
3366 * @xmit_type: xmit flags
f2e0899f 3367 *
91226790 3368 * 57712/578xx related
f2e0899f 3369 */
91226790
DK		 3370static u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3371 u32 *parsing_data, u32 xmit_type)
f2e0899f 3372{
e39aece7 3373 *parsing_data |=
2de67439 3374 ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
91226790
DK		 3375 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3376 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
f2e0899f 3377
e39aece7
VZ		 3378 if (xmit_type & XMIT_CSUM_TCP) {
3379 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3380 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3381 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
f2e0899f 3382
e39aece7 3383 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
924d75ab
YM		 3384 }
3385 /* We support checksum offload for TCP and UDP only.
3386 * No need to pass the UDP header length - it's a constant.
3387 */
3388 return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data;
f2e0899f
DK		 3389}
3390
a848ade4 3391/* set FW indication according to inner or outer protocols if tunneled */
91226790
DK		 3392static void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3393 struct eth_tx_start_bd *tx_start_bd,
3394 u32 xmit_type)
93ef5c02 3395{
93ef5c02
DK		 3396 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3397
a848ade4 3398 if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6))
91226790 3399 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6;
93ef5c02
DK		 3400
3401 if (!(xmit_type & XMIT_CSUM_TCP))
3402 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
93ef5c02
DK		 3403}
3404
f2e0899f 3405/**
e8920674 3406 * bnx2x_set_pbd_csum - update PBD with checksum and return header length
f2e0899f 3407 *
e8920674
DK		 3408 * @bp: driver handle
3409 * @skb: packet skb
3410 * @pbd: parse BD to be updated
3411 * @xmit_type: xmit flags
f2e0899f 3412 */
91226790
DK		 3413static u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3414 struct eth_tx_parse_bd_e1x *pbd,
3415 u32 xmit_type)
f2e0899f 3416{
e39aece7 3417 u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
f2e0899f
DK		 3418
3419 /* for now NS flag is not used in Linux */
3420 pbd->global_data =
86564c3f
YM		 3421 cpu_to_le16(hlen |
3422 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3423 ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
f2e0899f
DK		 3424
3425 pbd->ip_hlen_w = (skb_transport_header(skb) -
e39aece7 3426 skb_network_header(skb)) >> 1;
f2e0899f 3427
e39aece7
VZ		 3428 hlen += pbd->ip_hlen_w;
3429
3430 /* We support checksum offload for TCP and UDP only */
3431 if (xmit_type & XMIT_CSUM_TCP)
3432 hlen += tcp_hdrlen(skb) / 2;
3433 else
3434 hlen += sizeof(struct udphdr) / 2;
f2e0899f
DK		 3435
3436 pbd->total_hlen_w = cpu_to_le16(hlen);
3437 hlen = hlen*2;
3438
3439 if (xmit_type & XMIT_CSUM_TCP) {
86564c3f 3440 pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check);
f2e0899f
DK		 3441
3442 } else {
3443 s8 fix = SKB_CS_OFF(skb); /* signed! */
3444
3445 DP(NETIF_MSG_TX_QUEUED,
3446 "hlen %d fix %d csum before fix %x\n",
3447 le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
3448
3449 /* HW bug: fixup the CSUM */
3450 pbd->tcp_pseudo_csum =
3451 bnx2x_csum_fix(skb_transport_header(skb),
3452 SKB_CS(skb), fix);
3453
3454 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
3455 pbd->tcp_pseudo_csum);
3456 }
3457
3458 return hlen;
3459}
f85582f8 3460
a848ade4
DK		 3461static void bnx2x_update_pbds_gso_enc(struct sk_buff *skb,
3462 struct eth_tx_parse_bd_e2 *pbd_e2,
3463 struct eth_tx_parse_2nd_bd *pbd2,
3464 u16 *global_data,
3465 u32 xmit_type)
3466{
3467 u16 inner_hlen_w = 0;
3468 u8 outerip_off, outerip_len = 0;
3469
3470 /* IP len */
3471 inner_hlen_w = (skb_inner_transport_header(skb) -
3472 skb_inner_network_header(skb)) >> 1;
3473
3474 /* transport len */
3475 if (xmit_type & XMIT_CSUM_TCP)
3476 inner_hlen_w += inner_tcp_hdrlen(skb) >> 1;
3477 else
3478 inner_hlen_w += sizeof(struct udphdr) >> 1;
3479
3480 pbd2->fw_ip_hdr_to_payload_w = inner_hlen_w;
3481
3482 if (xmit_type & XMIT_CSUM_ENC_V4) {
3483 struct iphdr *iph = inner_ip_hdr(skb);
3484
3485 pbd2->fw_ip_csum_wo_len_flags_frag =
3486 bswab16(csum_fold((~iph->check) -
3487 iph->tot_len - iph->frag_off));
3488 } else {
3489 pbd2->fw_ip_hdr_to_payload_w =
3490 inner_hlen_w - ((sizeof(struct ipv6hdr)) >> 1);
3491 }
3492
3493 pbd2->tcp_send_seq = bswab32(inner_tcp_hdr(skb)->seq);
3494
3495 pbd2->tcp_flags = pbd_tcp_flags(inner_tcp_hdr(skb));
3496
3497 if (xmit_type & XMIT_GSO_V4) {
3498 pbd2->hw_ip_id = bswab16(ip_hdr(skb)->id);
3499
3500 pbd_e2->data.tunnel_data.pseudo_csum =
3501 bswab16(~csum_tcpudp_magic(
3502 inner_ip_hdr(skb)->saddr,
3503 inner_ip_hdr(skb)->daddr,
3504 0, IPPROTO_TCP, 0));
3505
3506 outerip_len = ip_hdr(skb)->ihl << 1;
3507 } else {
3508 pbd_e2->data.tunnel_data.pseudo_csum =
3509 bswab16(~csum_ipv6_magic(
3510 &inner_ipv6_hdr(skb)->saddr,
3511 &inner_ipv6_hdr(skb)->daddr,
3512 0, IPPROTO_TCP, 0));
3513 }
3514
3515 outerip_off = (skb_network_header(skb) - skb->data) >> 1;
3516
3517 *global_data |=
3518 outerip_off |
3519 (!!(xmit_type & XMIT_CSUM_V6) <<
3520 ETH_TX_PARSE_2ND_BD_IP_HDR_TYPE_OUTER_SHIFT) |
3521 (outerip_len <<
3522 ETH_TX_PARSE_2ND_BD_IP_HDR_LEN_OUTER_W_SHIFT) |
3523 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3524 ETH_TX_PARSE_2ND_BD_LLC_SNAP_EN_SHIFT);
3525}
3526
9f6c9258
DK		 3527/* called with netif_tx_lock
3528 * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
3529 * netif_wake_queue()
3530 */
3531netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
3532{
3533 struct bnx2x *bp = netdev_priv(dev);
6383c0b3 3534
9f6c9258 3535 struct netdev_queue *txq;
6383c0b3 3536 struct bnx2x_fp_txdata *txdata;
9f6c9258 3537 struct sw_tx_bd *tx_buf;
619c5cb6 3538 struct eth_tx_start_bd *tx_start_bd, *first_bd;
9f6c9258 3539 struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
523224a3 3540 struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
f2e0899f 3541 struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
a848ade4 3542 struct eth_tx_parse_2nd_bd *pbd2 = NULL;
2297a2da 3543 u32 pbd_e2_parsing_data = 0;
9f6c9258 3544 u16 pkt_prod, bd_prod;
65565884 3545 int nbd, txq_index;
9f6c9258
DK		 3546 dma_addr_t mapping;
3547 u32 xmit_type = bnx2x_xmit_type(bp, skb);
3548 int i;
3549 u8 hlen = 0;
3550 __le16 pkt_size = 0;
3551 struct ethhdr *eth;
3552 u8 mac_type = UNICAST_ADDRESS;
3553
3554#ifdef BNX2X_STOP_ON_ERROR
3555 if (unlikely(bp->panic))
3556 return NETDEV_TX_BUSY;
3557#endif
3558
6383c0b3
AE		 3559 txq_index = skb_get_queue_mapping(skb);
3560 txq = netdev_get_tx_queue(dev, txq_index);
3561
55c11941 3562 BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0));
6383c0b3 3563
65565884 3564 txdata = &bp->bnx2x_txq[txq_index];
6383c0b3
AE		 3565
3566 /* enable this debug print to view the transmission queue being used
51c1a580 3567 DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
6383c0b3 3568 txq_index, fp_index, txdata_index); */
9f6c9258 3569
6383c0b3 3570 /* enable this debug print to view the tranmission details
51c1a580
MS		 3571 DP(NETIF_MSG_TX_QUEUED,
3572 "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
6383c0b3 3573 txdata->cid, fp_index, txdata_index, txdata, fp); */
9f6c9258 3574
6383c0b3 3575 if (unlikely(bnx2x_tx_avail(bp, txdata) <
7df2dc6b
DK		 3576 skb_shinfo(skb)->nr_frags +
3577 BDS_PER_TX_PKT +
3578 NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
2384d6aa 3579 /* Handle special storage cases separately */
c96bdc0c
DK		 3580 if (txdata->tx_ring_size == 0) {
3581 struct bnx2x_eth_q_stats *q_stats =
3582 bnx2x_fp_qstats(bp, txdata->parent_fp);
3583 q_stats->driver_filtered_tx_pkt++;
3584 dev_kfree_skb(skb);
3585 return NETDEV_TX_OK;
3586 }
2de67439
YM		 3587 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3588 netif_tx_stop_queue(txq);
c96bdc0c 3589 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
2384d6aa 3590
9f6c9258
DK		 3591 return NETDEV_TX_BUSY;
3592 }
3593
51c1a580 3594 DP(NETIF_MSG_TX_QUEUED,
04c46736 3595 "queue[%d]: SKB: summed %x protocol %x protocol(%x,%x) gso type %x xmit_type %x len %d\n",
6383c0b3 3596 txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
04c46736
YM		 3597 ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type,
3598 skb->len);
9f6c9258
DK		 3599
3600 eth = (struct ethhdr *)skb->data;
3601
3602 /* set flag according to packet type (UNICAST_ADDRESS is default)*/
3603 if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
3604 if (is_broadcast_ether_addr(eth->h_dest))
3605 mac_type = BROADCAST_ADDRESS;
3606 else
3607 mac_type = MULTICAST_ADDRESS;
3608 }
3609
91226790 3610#if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
9f6c9258
DK		 3611 /* First, check if we need to linearize the skb (due to FW
3612 restrictions). No need to check fragmentation if page size > 8K
3613 (there will be no violation to FW restrictions) */
3614 if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
3615 /* Statistics of linearization */
3616 bp->lin_cnt++;
3617 if (skb_linearize(skb) != 0) {
51c1a580
MS		 3618 DP(NETIF_MSG_TX_QUEUED,
3619 "SKB linearization failed - silently dropping this SKB\n");
9f6c9258
DK		 3620 dev_kfree_skb_any(skb);
3621 return NETDEV_TX_OK;
3622 }
3623 }
3624#endif
619c5cb6
VZ		 3625 /* Map skb linear data for DMA */
3626 mapping = dma_map_single(&bp->pdev->dev, skb->data,
3627 skb_headlen(skb), DMA_TO_DEVICE);
3628 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
51c1a580
MS		 3629 DP(NETIF_MSG_TX_QUEUED,
3630 "SKB mapping failed - silently dropping this SKB\n");
619c5cb6
VZ		 3631 dev_kfree_skb_any(skb);
3632 return NETDEV_TX_OK;
3633 }
9f6c9258
DK		 3634 /*
3635 Please read carefully. First we use one BD which we mark as start,
3636 then we have a parsing info BD (used for TSO or xsum),
3637 and only then we have the rest of the TSO BDs.
3638 (don't forget to mark the last one as last,
3639 and to unmap only AFTER you write to the BD ...)
3640 And above all, all pdb sizes are in words - NOT DWORDS!
3641 */
3642
619c5cb6
VZ		 3643 /* get current pkt produced now - advance it just before sending packet
3644 * since mapping of pages may fail and cause packet to be dropped
3645 */
6383c0b3
AE		 3646 pkt_prod = txdata->tx_pkt_prod;
3647 bd_prod = TX_BD(txdata->tx_bd_prod);
9f6c9258 3648
619c5cb6
VZ		 3649 /* get a tx_buf and first BD
3650 * tx_start_bd may be changed during SPLIT,
3651 * but first_bd will always stay first
3652 */
6383c0b3
AE		 3653 tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3654 tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
619c5cb6 3655 first_bd = tx_start_bd;
9f6c9258
DK		 3656
3657 tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
f85582f8 3658
91226790
DK		 3659 /* header nbd: indirectly zero other flags! */
3660 tx_start_bd->general_data = 1 << ETH_TX_START_BD_HDR_NBDS_SHIFT;
9f6c9258
DK		 3661
3662 /* remember the first BD of the packet */
6383c0b3 3663 tx_buf->first_bd = txdata->tx_bd_prod;
9f6c9258
DK		 3664 tx_buf->skb = skb;
3665 tx_buf->flags = 0;
3666
3667 DP(NETIF_MSG_TX_QUEUED,
3668 "sending pkt %u @%p next_idx %u bd %u @%p\n",
6383c0b3 3669 pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
9f6c9258 3670
eab6d18d 3671 if (vlan_tx_tag_present(skb)) {
523224a3
DK		 3672 tx_start_bd->vlan_or_ethertype =
3673 cpu_to_le16(vlan_tx_tag_get(skb));
3674 tx_start_bd->bd_flags.as_bitfield |=
3675 (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
dc1ba591
AE		 3676 } else {
3677 /* when transmitting in a vf, start bd must hold the ethertype
3678 * for fw to enforce it
3679 */
91226790 3680 if (IS_VF(bp))
dc1ba591
AE		 3681 tx_start_bd->vlan_or_ethertype =
3682 cpu_to_le16(ntohs(eth->h_proto));
91226790 3683 else
dc1ba591
AE		 3684 /* used by FW for packet accounting */
3685 tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
dc1ba591 3686 }
9f6c9258 3687
91226790
DK		 3688 nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3689
9f6c9258
DK		 3690 /* turn on parsing and get a BD */
3691 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
9f6c9258 3692
93ef5c02
DK		 3693 if (xmit_type & XMIT_CSUM)
3694 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
9f6c9258 3695
619c5cb6 3696 if (!CHIP_IS_E1x(bp)) {
6383c0b3 3697 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
f2e0899f 3698 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
a848ade4
DK		 3699
3700 if (xmit_type & XMIT_CSUM_ENC) {
3701 u16 global_data = 0;
3702
3703 /* Set PBD in enc checksum offload case */
3704 hlen = bnx2x_set_pbd_csum_enc(bp, skb,
3705 &pbd_e2_parsing_data,
3706 xmit_type);
3707
3708 /* turn on 2nd parsing and get a BD */
3709 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3710
3711 pbd2 = &txdata->tx_desc_ring[bd_prod].parse_2nd_bd;
3712
3713 memset(pbd2, 0, sizeof(*pbd2));
3714
3715 pbd_e2->data.tunnel_data.ip_hdr_start_inner_w =
3716 (skb_inner_network_header(skb) -
3717 skb->data) >> 1;
3718
3719 if (xmit_type & XMIT_GSO_ENC)
3720 bnx2x_update_pbds_gso_enc(skb, pbd_e2, pbd2,
3721 &global_data,
3722 xmit_type);
3723
3724 pbd2->global_data = cpu_to_le16(global_data);
3725
3726 /* add addition parse BD indication to start BD */
3727 SET_FLAG(tx_start_bd->general_data,
3728 ETH_TX_START_BD_PARSE_NBDS, 1);
3729 /* set encapsulation flag in start BD */
3730 SET_FLAG(tx_start_bd->general_data,
3731 ETH_TX_START_BD_TUNNEL_EXIST, 1);
3732 nbd++;
3733 } else if (xmit_type & XMIT_CSUM) {
91226790 3734 /* Set PBD in checksum offload case w/o encapsulation */
2297a2da
VZ		 3735 hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3736 &pbd_e2_parsing_data,
3737 xmit_type);
a848ade4 3738 }
dc1ba591 3739
91226790
DK		 3740 /* Add the macs to the parsing BD this is a vf */
3741 if (IS_VF(bp)) {
3742 /* override GRE parameters in BD */
3743 bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
3744 &pbd_e2->data.mac_addr.src_mid,
3745 &pbd_e2->data.mac_addr.src_lo,
619c5cb6 3746 eth->h_source);
91226790
DK		 3747
3748 bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi,
3749 &pbd_e2->data.mac_addr.dst_mid,
3750 &pbd_e2->data.mac_addr.dst_lo,
619c5cb6
VZ		 3751 eth->h_dest);
3752 }
96bed4b9
YM		 3753
3754 SET_FLAG(pbd_e2_parsing_data,
3755 ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type);
f2e0899f 3756 } else {
96bed4b9 3757 u16 global_data = 0;
6383c0b3 3758 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
f2e0899f
DK		 3759 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
3760 /* Set PBD in checksum offload case */
3761 if (xmit_type & XMIT_CSUM)
3762 hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
9f6c9258 3763
96bed4b9
YM		 3764 SET_FLAG(global_data,
3765 ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type);
3766 pbd_e1x->global_data |= cpu_to_le16(global_data);
9f6c9258
DK		 3767 }
3768
f85582f8 3769 /* Setup the data pointer of the first BD of the packet */
9f6c9258
DK		 3770 tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3771 tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
9f6c9258
DK		 3772 tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
3773 pkt_size = tx_start_bd->nbytes;
3774
51c1a580 3775 DP(NETIF_MSG_TX_QUEUED,
91226790 3776 "first bd @%p addr (%x:%x) nbytes %d flags %x vlan %x\n",
9f6c9258 3777 tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
91226790 3778 le16_to_cpu(tx_start_bd->nbytes),
523224a3
DK		 3779 tx_start_bd->bd_flags.as_bitfield,
3780 le16_to_cpu(tx_start_bd->vlan_or_ethertype));
9f6c9258
DK		 3781
3782 if (xmit_type & XMIT_GSO) {
3783
3784 DP(NETIF_MSG_TX_QUEUED,
3785 "TSO packet len %d hlen %d total len %d tso size %d\n",
3786 skb->len, hlen, skb_headlen(skb),
3787 skb_shinfo(skb)->gso_size);
3788
3789 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
3790
91226790
DK		 3791 if (unlikely(skb_headlen(skb) > hlen)) {
3792 nbd++;
6383c0b3
AE		 3793 bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
3794 &tx_start_bd, hlen,
91226790
DK		 3795 bd_prod);
3796 }
619c5cb6 3797 if (!CHIP_IS_E1x(bp))
2297a2da
VZ		 3798 bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
3799 xmit_type);
f2e0899f
DK		 3800 else
3801 bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
9f6c9258 3802 }
2297a2da
VZ		 3803
3804 /* Set the PBD's parsing_data field if not zero
3805 * (for the chips newer than 57711).
3806 */
3807 if (pbd_e2_parsing_data)
3808 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
3809
9f6c9258
DK		 3810 tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
3811
f85582f8 3812 /* Handle fragmented skb */
9f6c9258
DK		 3813 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3814 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3815
9e903e08
ED		 3816 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
3817 skb_frag_size(frag), DMA_TO_DEVICE);
619c5cb6 3818 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
2df1a70a 3819 unsigned int pkts_compl = 0, bytes_compl = 0;
619c5cb6 3820
51c1a580
MS		 3821 DP(NETIF_MSG_TX_QUEUED,
3822 "Unable to map page - dropping packet...\n");
619c5cb6
VZ		 3823
3824 /* we need unmap all buffers already mapped
3825 * for this SKB;
3826 * first_bd->nbd need to be properly updated
3827 * before call to bnx2x_free_tx_pkt
3828 */
3829 first_bd->nbd = cpu_to_le16(nbd);
6383c0b3 3830 bnx2x_free_tx_pkt(bp, txdata,
2df1a70a
TH		 3831 TX_BD(txdata->tx_pkt_prod),
3832 &pkts_compl, &bytes_compl);
619c5cb6
VZ		 3833 return NETDEV_TX_OK;
3834 }
3835
9f6c9258 3836 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
6383c0b3 3837 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
9f6c9258 3838 if (total_pkt_bd == NULL)
6383c0b3 3839 total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
9f6c9258 3840
9f6c9258
DK		 3841 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3842 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
9e903e08
ED		 3843 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
3844 le16_add_cpu(&pkt_size, skb_frag_size(frag));
619c5cb6 3845 nbd++;
9f6c9258
DK		 3846
3847 DP(NETIF_MSG_TX_QUEUED,
3848 "frag %d bd @%p addr (%x:%x) nbytes %d\n",
3849 i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
3850 le16_to_cpu(tx_data_bd->nbytes));
3851 }
3852
3853 DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
3854
619c5cb6
VZ		 3855 /* update with actual num BDs */
3856 first_bd->nbd = cpu_to_le16(nbd);
3857
9f6c9258
DK		 3858 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3859
3860 /* now send a tx doorbell, counting the next BD
3861 * if the packet contains or ends with it
3862 */
3863 if (TX_BD_POFF(bd_prod) < nbd)
3864 nbd++;
3865
619c5cb6
VZ		 3866 /* total_pkt_bytes should be set on the first data BD if
3867 * it's not an LSO packet and there is more than one
3868 * data BD. In this case pkt_size is limited by an MTU value.
3869 * However we prefer to set it for an LSO packet (while we don't
3870 * have to) in order to save some CPU cycles in a none-LSO
3871 * case, when we much more care about them.
3872 */
9f6c9258
DK		 3873 if (total_pkt_bd != NULL)
3874 total_pkt_bd->total_pkt_bytes = pkt_size;
3875
523224a3 3876 if (pbd_e1x)
9f6c9258 3877 DP(NETIF_MSG_TX_QUEUED,
51c1a580 3878 "PBD (E1X) @%p ip_data %x ip_hlen %u ip_id %u lso_mss %u tcp_flags %x xsum %x seq %u hlen %u\n",
523224a3
DK		 3879 pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
3880 pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
3881 pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
3882 le16_to_cpu(pbd_e1x->total_hlen_w));
f2e0899f
DK		 3883 if (pbd_e2)
3884 DP(NETIF_MSG_TX_QUEUED,
3885 "PBD (E2) @%p dst %x %x %x src %x %x %x parsing_data %x\n",
91226790
DK		 3886 pbd_e2,
3887 pbd_e2->data.mac_addr.dst_hi,
3888 pbd_e2->data.mac_addr.dst_mid,
3889 pbd_e2->data.mac_addr.dst_lo,
3890 pbd_e2->data.mac_addr.src_hi,
3891 pbd_e2->data.mac_addr.src_mid,
3892 pbd_e2->data.mac_addr.src_lo,
f2e0899f 3893 pbd_e2->parsing_data);
9f6c9258
DK		 3894 DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d bd %u\n", nbd, bd_prod);
3895
2df1a70a
TH		 3896 netdev_tx_sent_queue(txq, skb->len);
3897
8373c57d
WB		 3898 skb_tx_timestamp(skb);
3899
6383c0b3 3900 txdata->tx_pkt_prod++;
9f6c9258
DK		 3901 /*
3902 * Make sure that the BD data is updated before updating the producer
3903 * since FW might read the BD right after the producer is updated.
3904 * This is only applicable for weak-ordered memory model archs such
3905 * as IA-64. The following barrier is also mandatory since FW will
3906 * assumes packets must have BDs.
3907 */
3908 wmb();
3909
6383c0b3 3910 txdata->tx_db.data.prod += nbd;
9f6c9258 3911 barrier();
f85582f8 3912
6383c0b3 3913 DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
9f6c9258
DK		 3914
3915 mmiowb();
3916
6383c0b3 3917 txdata->tx_bd_prod += nbd;
9f6c9258 3918
7df2dc6b 3919 if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) {
9f6c9258
DK		 3920 netif_tx_stop_queue(txq);
3921
3922 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
3923 * ordering of set_bit() in netif_tx_stop_queue() and read of
3924 * fp->bd_tx_cons */
3925 smp_mb();
3926
15192a8c 3927 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
7df2dc6b 3928 if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)
9f6c9258
DK		 3929 netif_tx_wake_queue(txq);
3930 }
6383c0b3 3931 txdata->tx_pkt++;
9f6c9258
DK		 3932
3933 return NETDEV_TX_OK;
3934}
f85582f8 3935
6383c0b3
AE		 3936/**
3937 * bnx2x_setup_tc - routine to configure net_device for multi tc
3938 *
3939 * @netdev: net device to configure
3940 * @tc: number of traffic classes to enable
3941 *
3942 * callback connected to the ndo_setup_tc function pointer
3943 */
3944int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
3945{
3946 int cos, prio, count, offset;
3947 struct bnx2x *bp = netdev_priv(dev);
3948
3949 /* setup tc must be called under rtnl lock */
3950 ASSERT_RTNL();
3951
3952 /* no traffic classes requested. aborting */
3953 if (!num_tc) {
3954 netdev_reset_tc(dev);
3955 return 0;
3956 }
3957
3958 /* requested to support too many traffic classes */
3959 if (num_tc > bp->max_cos) {
51c1a580
MS		 3960 BNX2X_ERR("support for too many traffic classes requested: %d. max supported is %d\n",
3961 num_tc, bp->max_cos);
6383c0b3
AE		 3962 return -EINVAL;
3963 }
3964
3965 /* declare amount of supported traffic classes */
3966 if (netdev_set_num_tc(dev, num_tc)) {
51c1a580 3967 BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
6383c0b3
AE		 3968 return -EINVAL;
3969 }
3970
3971 /* configure priority to traffic class mapping */
3972 for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
3973 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
51c1a580
MS		 3974 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
3975 "mapping priority %d to tc %d\n",
6383c0b3
AE		 3976 prio, bp->prio_to_cos[prio]);
3977 }
3978
3979
3980 /* Use this configuration to diffrentiate tc0 from other COSes
3981 This can be used for ets or pfc, and save the effort of setting
3982 up a multio class queue disc or negotiating DCBX with a switch
3983 netdev_set_prio_tc_map(dev, 0, 0);
94f05b0f 3984 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
6383c0b3
AE		 3985 for (prio = 1; prio < 16; prio++) {
3986 netdev_set_prio_tc_map(dev, prio, 1);
94f05b0f 3987 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
6383c0b3
AE		 3988 } */
3989
3990 /* configure traffic class to transmission queue mapping */
3991 for (cos = 0; cos < bp->max_cos; cos++) {
3992 count = BNX2X_NUM_ETH_QUEUES(bp);
65565884 3993 offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp);
6383c0b3 3994 netdev_set_tc_queue(dev, cos, count, offset);
51c1a580
MS		 3995 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
3996 "mapping tc %d to offset %d count %d\n",
6383c0b3
AE		 3997 cos, offset, count);
3998 }
3999
4000 return 0;
4001}
4002
9f6c9258
DK		 4003/* called with rtnl_lock */
4004int bnx2x_change_mac_addr(struct net_device *dev, void *p)
4005{
4006 struct sockaddr *addr = p;
4007 struct bnx2x *bp = netdev_priv(dev);
619c5cb6 4008 int rc = 0;
9f6c9258 4009
51c1a580
MS		 4010 if (!bnx2x_is_valid_ether_addr(bp, addr->sa_data)) {
4011 BNX2X_ERR("Requested MAC address is not valid\n");
614c76df 4012 return -EINVAL;
51c1a580 4013 }
614c76df 4014
a3348722
BW		 4015 if ((IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp)) &&
4016 !is_zero_ether_addr(addr->sa_data)) {
51c1a580 4017 BNX2X_ERR("Can't configure non-zero address on iSCSI or FCoE functions in MF-SD mode\n");
9f6c9258 4018 return -EINVAL;
51c1a580 4019 }
9f6c9258 4020
619c5cb6
VZ		 4021 if (netif_running(dev)) {
4022 rc = bnx2x_set_eth_mac(bp, false);
4023 if (rc)
4024 return rc;
4025 }
4026
9f6c9258 4027 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
619c5cb6 4028
523224a3 4029 if (netif_running(dev))
619c5cb6 4030 rc = bnx2x_set_eth_mac(bp, true);
9f6c9258 4031
619c5cb6 4032 return rc;
9f6c9258
DK		 4033}
4034
b3b83c3f
DK		 4035static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
4036{
4037 union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
4038 struct bnx2x_fastpath *fp = &bp->fp[fp_index];
6383c0b3 4039 u8 cos;
b3b83c3f
DK		 4040
4041 /* Common */
55c11941 4042
b3b83c3f
DK		 4043 if (IS_FCOE_IDX(fp_index)) {
4044 memset(sb, 0, sizeof(union host_hc_status_block));
4045 fp->status_blk_mapping = 0;
b3b83c3f 4046 } else {
b3b83c3f 4047 /* status blocks */
619c5cb6 4048 if (!CHIP_IS_E1x(bp))
b3b83c3f
DK		 4049 BNX2X_PCI_FREE(sb->e2_sb,
4050 bnx2x_fp(bp, fp_index,
4051 status_blk_mapping),
4052 sizeof(struct host_hc_status_block_e2));
4053 else
4054 BNX2X_PCI_FREE(sb->e1x_sb,
4055 bnx2x_fp(bp, fp_index,
4056 status_blk_mapping),
4057 sizeof(struct host_hc_status_block_e1x));
b3b83c3f 4058 }
55c11941 4059
b3b83c3f
DK		 4060 /* Rx */
4061 if (!skip_rx_queue(bp, fp_index)) {
4062 bnx2x_free_rx_bds(fp);
4063
4064 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4065 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
4066 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
4067 bnx2x_fp(bp, fp_index, rx_desc_mapping),
4068 sizeof(struct eth_rx_bd) * NUM_RX_BD);
4069
4070 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
4071 bnx2x_fp(bp, fp_index, rx_comp_mapping),
4072 sizeof(struct eth_fast_path_rx_cqe) *
4073 NUM_RCQ_BD);
4074
4075 /* SGE ring */
4076 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
4077 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
4078 bnx2x_fp(bp, fp_index, rx_sge_mapping),
4079 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4080 }
4081
4082 /* Tx */
4083 if (!skip_tx_queue(bp, fp_index)) {
4084 /* fastpath tx rings: tx_buf tx_desc */
6383c0b3 4085 for_each_cos_in_tx_queue(fp, cos) {
65565884 4086 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
6383c0b3 4087
51c1a580 4088 DP(NETIF_MSG_IFDOWN,
94f05b0f 4089 "freeing tx memory of fp %d cos %d cid %d\n",
6383c0b3
AE		 4090 fp_index, cos, txdata->cid);
4091
4092 BNX2X_FREE(txdata->tx_buf_ring);
4093 BNX2X_PCI_FREE(txdata->tx_desc_ring,
4094 txdata->tx_desc_mapping,
4095 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4096 }
b3b83c3f
DK		 4097 }
4098 /* end of fastpath */
4099}
4100
55c11941
MS		 4101void bnx2x_free_fp_mem_cnic(struct bnx2x *bp)
4102{
4103 int i;
4104 for_each_cnic_queue(bp, i)
4105 bnx2x_free_fp_mem_at(bp, i);
4106}
4107
b3b83c3f
DK		 4108void bnx2x_free_fp_mem(struct bnx2x *bp)
4109{
4110 int i;
55c11941 4111 for_each_eth_queue(bp, i)
b3b83c3f
DK		 4112 bnx2x_free_fp_mem_at(bp, i);
4113}
4114
1191cb83 4115static void set_sb_shortcuts(struct bnx2x *bp, int index)
b3b83c3f
DK		 4116{
4117 union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
619c5cb6 4118 if (!CHIP_IS_E1x(bp)) {
b3b83c3f
DK		 4119 bnx2x_fp(bp, index, sb_index_values) =
4120 (__le16 *)status_blk.e2_sb->sb.index_values;
4121 bnx2x_fp(bp, index, sb_running_index) =
4122 (__le16 *)status_blk.e2_sb->sb.running_index;
4123 } else {
4124 bnx2x_fp(bp, index, sb_index_values) =
4125 (__le16 *)status_blk.e1x_sb->sb.index_values;
4126 bnx2x_fp(bp, index, sb_running_index) =
4127 (__le16 *)status_blk.e1x_sb->sb.running_index;
4128 }
4129}
4130
1191cb83
ED		 4131/* Returns the number of actually allocated BDs */
4132static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
4133 int rx_ring_size)
4134{
4135 struct bnx2x *bp = fp->bp;
4136 u16 ring_prod, cqe_ring_prod;
4137 int i, failure_cnt = 0;
4138
4139 fp->rx_comp_cons = 0;
4140 cqe_ring_prod = ring_prod = 0;
4141
4142 /* This routine is called only during fo init so
4143 * fp->eth_q_stats.rx_skb_alloc_failed = 0
4144 */
4145 for (i = 0; i < rx_ring_size; i++) {
4146 if (bnx2x_alloc_rx_data(bp, fp, ring_prod) < 0) {
4147 failure_cnt++;
4148 continue;
4149 }
4150 ring_prod = NEXT_RX_IDX(ring_prod);
4151 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
4152 WARN_ON(ring_prod <= (i - failure_cnt));
4153 }
4154
4155 if (failure_cnt)
4156 BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n",
4157 i - failure_cnt, fp->index);
4158
4159 fp->rx_bd_prod = ring_prod;
4160 /* Limit the CQE producer by the CQE ring size */
4161 fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
4162 cqe_ring_prod);
4163 fp->rx_pkt = fp->rx_calls = 0;
4164
15192a8c 4165 bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt;
1191cb83
ED		 4166
4167 return i - failure_cnt;
4168}
4169
4170static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp)
4171{
4172 int i;
4173
4174 for (i = 1; i <= NUM_RCQ_RINGS; i++) {
4175 struct eth_rx_cqe_next_page *nextpg;
4176
4177 nextpg = (struct eth_rx_cqe_next_page *)
4178 &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
4179 nextpg->addr_hi =
4180 cpu_to_le32(U64_HI(fp->rx_comp_mapping +
4181 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4182 nextpg->addr_lo =
4183 cpu_to_le32(U64_LO(fp->rx_comp_mapping +
4184 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4185 }
4186}
4187
b3b83c3f
DK		 4188static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
4189{
4190 union host_hc_status_block *sb;
4191 struct bnx2x_fastpath *fp = &bp->fp[index];
4192 int ring_size = 0;
6383c0b3 4193 u8 cos;
c2188952 4194 int rx_ring_size = 0;
b3b83c3f 4195
a3348722
BW		 4196 if (!bp->rx_ring_size &&
4197 (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))) {
614c76df
DK		 4198 rx_ring_size = MIN_RX_SIZE_NONTPA;
4199 bp->rx_ring_size = rx_ring_size;
55c11941 4200 } else if (!bp->rx_ring_size) {
c2188952
VZ		 4201 rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
4202
065f8b92
YM		 4203 if (CHIP_IS_E3(bp)) {
4204 u32 cfg = SHMEM_RD(bp,
4205 dev_info.port_hw_config[BP_PORT(bp)].
4206 default_cfg);
4207
4208 /* Decrease ring size for 1G functions */
4209 if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
4210 PORT_HW_CFG_NET_SERDES_IF_SGMII)
4211 rx_ring_size /= 10;
4212 }
d760fc37 4213
c2188952
VZ		 4214 /* allocate at least number of buffers required by FW */
4215 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
4216 MIN_RX_SIZE_TPA, rx_ring_size);
4217
4218 bp->rx_ring_size = rx_ring_size;
614c76df 4219 } else /* if rx_ring_size specified - use it */
c2188952 4220 rx_ring_size = bp->rx_ring_size;
b3b83c3f 4221
04c46736
YM		 4222 DP(BNX2X_MSG_SP, "calculated rx_ring_size %d\n", rx_ring_size);
4223
b3b83c3f
DK		 4224 /* Common */
4225 sb = &bnx2x_fp(bp, index, status_blk);
55c11941 4226
b3b83c3f 4227 if (!IS_FCOE_IDX(index)) {
b3b83c3f 4228 /* status blocks */
619c5cb6 4229 if (!CHIP_IS_E1x(bp))
b3b83c3f
DK		 4230 BNX2X_PCI_ALLOC(sb->e2_sb,
4231 &bnx2x_fp(bp, index, status_blk_mapping),
4232 sizeof(struct host_hc_status_block_e2));
4233 else
4234 BNX2X_PCI_ALLOC(sb->e1x_sb,
4235 &bnx2x_fp(bp, index, status_blk_mapping),
4236 sizeof(struct host_hc_status_block_e1x));
b3b83c3f 4237 }
8eef2af1
DK		 4238
4239 /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
4240 * set shortcuts for it.
4241 */
4242 if (!IS_FCOE_IDX(index))
4243 set_sb_shortcuts(bp, index);
b3b83c3f
DK		 4244
4245 /* Tx */
4246 if (!skip_tx_queue(bp, index)) {
4247 /* fastpath tx rings: tx_buf tx_desc */
6383c0b3 4248 for_each_cos_in_tx_queue(fp, cos) {
65565884 4249 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
6383c0b3 4250
51c1a580
MS		 4251 DP(NETIF_MSG_IFUP,
4252 "allocating tx memory of fp %d cos %d\n",
6383c0b3
AE		 4253 index, cos);
4254
4255 BNX2X_ALLOC(txdata->tx_buf_ring,
b3b83c3f 4256 sizeof(struct sw_tx_bd) * NUM_TX_BD);
6383c0b3
AE		 4257 BNX2X_PCI_ALLOC(txdata->tx_desc_ring,
4258 &txdata->tx_desc_mapping,
b3b83c3f 4259 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
6383c0b3 4260 }
b3b83c3f
DK		 4261 }
4262
4263 /* Rx */
4264 if (!skip_rx_queue(bp, index)) {
4265 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4266 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_buf_ring),
4267 sizeof(struct sw_rx_bd) * NUM_RX_BD);
4268 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_desc_ring),
4269 &bnx2x_fp(bp, index, rx_desc_mapping),
4270 sizeof(struct eth_rx_bd) * NUM_RX_BD);
4271
4272 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_comp_ring),
4273 &bnx2x_fp(bp, index, rx_comp_mapping),
4274 sizeof(struct eth_fast_path_rx_cqe) *
4275 NUM_RCQ_BD);
4276
4277 /* SGE ring */
4278 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_page_ring),
4279 sizeof(struct sw_rx_page) * NUM_RX_SGE);
4280 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_sge_ring),
4281 &bnx2x_fp(bp, index, rx_sge_mapping),
4282 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4283 /* RX BD ring */
4284 bnx2x_set_next_page_rx_bd(fp);
4285
4286 /* CQ ring */
4287 bnx2x_set_next_page_rx_cq(fp);
4288
4289 /* BDs */
4290 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
4291 if (ring_size < rx_ring_size)
4292 goto alloc_mem_err;
4293 }
4294
4295 return 0;
4296
4297/* handles low memory cases */
4298alloc_mem_err:
4299 BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
4300 index, ring_size);
4301 /* FW will drop all packets if queue is not big enough,
4302 * In these cases we disable the queue
6383c0b3 4303 * Min size is different for OOO, TPA and non-TPA queues
b3b83c3f
DK		 4304 */
4305 if (ring_size < (fp->disable_tpa ?
eb722d7a 4306 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
b3b83c3f
DK		 4307 /* release memory allocated for this queue */
4308 bnx2x_free_fp_mem_at(bp, index);
4309 return -ENOMEM;
4310 }
4311 return 0;
4312}
4313
55c11941
MS		 4314int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp)
4315{
4316 if (!NO_FCOE(bp))
4317 /* FCoE */
4318 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp)))
4319 /* we will fail load process instead of mark
4320 * NO_FCOE_FLAG
4321 */
4322 return -ENOMEM;
4323
4324 return 0;
4325}
4326
b3b83c3f
DK		 4327int bnx2x_alloc_fp_mem(struct bnx2x *bp)
4328{
4329 int i;
4330
55c11941
MS		 4331 /* 1. Allocate FP for leading - fatal if error
4332 * 2. Allocate RSS - fix number of queues if error
b3b83c3f
DK		 4333 */
4334
4335 /* leading */
4336 if (bnx2x_alloc_fp_mem_at(bp, 0))
4337 return -ENOMEM;
6383c0b3 4338
b3b83c3f
DK		 4339 /* RSS */
4340 for_each_nondefault_eth_queue(bp, i)
4341 if (bnx2x_alloc_fp_mem_at(bp, i))
4342 break;
4343
4344 /* handle memory failures */
4345 if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
4346 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
4347
4348 WARN_ON(delta < 0);
4864a16a 4349 bnx2x_shrink_eth_fp(bp, delta);
55c11941
MS		 4350 if (CNIC_SUPPORT(bp))
4351 /* move non eth FPs next to last eth FP
4352 * must be done in that order
4353 * FCOE_IDX < FWD_IDX < OOO_IDX
4354 */
b3b83c3f 4355
55c11941
MS		 4356 /* move FCoE fp even NO_FCOE_FLAG is on */
4357 bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta);
4358 bp->num_ethernet_queues -= delta;
4359 bp->num_queues = bp->num_ethernet_queues +
4360 bp->num_cnic_queues;
b3b83c3f
DK		 4361 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
4362 bp->num_queues + delta, bp->num_queues);
4363 }
4364
4365 return 0;
4366}
d6214d7a 4367
523224a3
DK		 4368void bnx2x_free_mem_bp(struct bnx2x *bp)
4369{
c3146eb6
DK		 4370 int i;
4371
4372 for (i = 0; i < bp->fp_array_size; i++)
4373 kfree(bp->fp[i].tpa_info);
523224a3 4374 kfree(bp->fp);
15192a8c
BW		 4375 kfree(bp->sp_objs);
4376 kfree(bp->fp_stats);
65565884 4377 kfree(bp->bnx2x_txq);
523224a3
DK		 4378 kfree(bp->msix_table);
4379 kfree(bp->ilt);
4380}
4381
0329aba1 4382int bnx2x_alloc_mem_bp(struct bnx2x *bp)
523224a3
DK		 4383{
4384 struct bnx2x_fastpath *fp;
4385 struct msix_entry *tbl;
4386 struct bnx2x_ilt *ilt;
6383c0b3 4387 int msix_table_size = 0;
55c11941 4388 int fp_array_size, txq_array_size;
15192a8c 4389 int i;
6383c0b3
AE		 4390
4391 /*
4392 * The biggest MSI-X table we might need is as a maximum number of fast
2de67439 4393 * path IGU SBs plus default SB (for PF only).
6383c0b3 4394 */
1ab4434c
AE		 4395 msix_table_size = bp->igu_sb_cnt;
4396 if (IS_PF(bp))
4397 msix_table_size++;
4398 BNX2X_DEV_INFO("msix_table_size %d\n", msix_table_size);
523224a3 4399
6383c0b3 4400 /* fp array: RSS plus CNIC related L2 queues */
55c11941 4401 fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp);
c3146eb6
DK		 4402 bp->fp_array_size = fp_array_size;
4403 BNX2X_DEV_INFO("fp_array_size %d\n", bp->fp_array_size);
15192a8c 4404
c3146eb6 4405 fp = kcalloc(bp->fp_array_size, sizeof(*fp), GFP_KERNEL);
523224a3
DK		 4406 if (!fp)
4407 goto alloc_err;
c3146eb6 4408 for (i = 0; i < bp->fp_array_size; i++) {
15192a8c
BW		 4409 fp[i].tpa_info =
4410 kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2,
4411 sizeof(struct bnx2x_agg_info), GFP_KERNEL);
4412 if (!(fp[i].tpa_info))
4413 goto alloc_err;
4414 }
4415
523224a3
DK		 4416 bp->fp = fp;
4417
15192a8c 4418 /* allocate sp objs */
c3146eb6 4419 bp->sp_objs = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_sp_objs),
15192a8c
BW		 4420 GFP_KERNEL);
4421 if (!bp->sp_objs)
4422 goto alloc_err;
4423
4424 /* allocate fp_stats */
c3146eb6 4425 bp->fp_stats = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_fp_stats),
15192a8c
BW		 4426 GFP_KERNEL);
4427 if (!bp->fp_stats)
4428 goto alloc_err;
4429
65565884 4430 /* Allocate memory for the transmission queues array */
55c11941
MS		 4431 txq_array_size =
4432 BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp);
4433 BNX2X_DEV_INFO("txq_array_size %d", txq_array_size);
4434
4435 bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata),
4436 GFP_KERNEL);
65565884
MS		 4437 if (!bp->bnx2x_txq)
4438 goto alloc_err;
4439
523224a3 4440 /* msix table */
01e23742 4441 tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
523224a3
DK		 4442 if (!tbl)
4443 goto alloc_err;
4444 bp->msix_table = tbl;
4445
4446 /* ilt */
4447 ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
4448 if (!ilt)
4449 goto alloc_err;
4450 bp->ilt = ilt;
4451
4452 return 0;
4453alloc_err:
4454 bnx2x_free_mem_bp(bp);
4455 return -ENOMEM;
4456
4457}
4458
a9fccec7 4459int bnx2x_reload_if_running(struct net_device *dev)
66371c44
MM		 4460{
4461 struct bnx2x *bp = netdev_priv(dev);
4462
4463 if (unlikely(!netif_running(dev)))
4464 return 0;
4465
5d07d868 4466 bnx2x_nic_unload(bp, UNLOAD_NORMAL, true);
66371c44
MM		 4467 return bnx2x_nic_load(bp, LOAD_NORMAL);
4468}
4469
1ac9e428
YR		 4470int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
4471{
4472 u32 sel_phy_idx = 0;
4473 if (bp->link_params.num_phys <= 1)
4474 return INT_PHY;
4475
4476 if (bp->link_vars.link_up) {
4477 sel_phy_idx = EXT_PHY1;
4478 /* In case link is SERDES, check if the EXT_PHY2 is the one */
4479 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
4480 (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
4481 sel_phy_idx = EXT_PHY2;
4482 } else {
4483
4484 switch (bnx2x_phy_selection(&bp->link_params)) {
4485 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
4486 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
4487 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
4488 sel_phy_idx = EXT_PHY1;
4489 break;
4490 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
4491 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
4492 sel_phy_idx = EXT_PHY2;
4493 break;
4494 }
4495 }
4496
4497 return sel_phy_idx;
4498
4499}
4500int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
4501{
4502 u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
4503 /*
2de67439 4504 * The selected activated PHY is always after swapping (in case PHY
1ac9e428
YR		 4505 * swapping is enabled). So when swapping is enabled, we need to reverse
4506 * the configuration
4507 */
4508
4509 if (bp->link_params.multi_phy_config &
4510 PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
4511 if (sel_phy_idx == EXT_PHY1)
4512 sel_phy_idx = EXT_PHY2;
4513 else if (sel_phy_idx == EXT_PHY2)
4514 sel_phy_idx = EXT_PHY1;
4515 }
4516 return LINK_CONFIG_IDX(sel_phy_idx);
4517}
4518
55c11941 4519#ifdef NETDEV_FCOE_WWNN
bf61ee14
VZ		 4520int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
4521{
4522 struct bnx2x *bp = netdev_priv(dev);
4523 struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
4524
4525 switch (type) {
4526 case NETDEV_FCOE_WWNN:
4527 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
4528 cp->fcoe_wwn_node_name_lo);
4529 break;
4530 case NETDEV_FCOE_WWPN:
4531 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
4532 cp->fcoe_wwn_port_name_lo);
4533 break;
4534 default:
51c1a580 4535 BNX2X_ERR("Wrong WWN type requested - %d\n", type);
bf61ee14
VZ		 4536 return -EINVAL;
4537 }
4538
4539 return 0;
4540}
4541#endif
4542
9f6c9258
DK		 4543/* called with rtnl_lock */
4544int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
4545{
4546 struct bnx2x *bp = netdev_priv(dev);
9f6c9258
DK		 4547
4548 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
51c1a580 4549 BNX2X_ERR("Can't perform change MTU during parity recovery\n");
9f6c9258
DK		 4550 return -EAGAIN;
4551 }
4552
4553 if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
51c1a580
MS		 4554 ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) {
4555 BNX2X_ERR("Can't support requested MTU size\n");
9f6c9258 4556 return -EINVAL;
51c1a580 4557 }
9f6c9258
DK		 4558
4559 /* This does not race with packet allocation
4560 * because the actual alloc size is
4561 * only updated as part of load
4562 */
4563 dev->mtu = new_mtu;
4564
66371c44
MM		 4565 return bnx2x_reload_if_running(dev);
4566}
4567
c8f44aff 4568netdev_features_t bnx2x_fix_features(struct net_device *dev,
621b4d66 4569 netdev_features_t features)
66371c44
MM		 4570{
4571 struct bnx2x *bp = netdev_priv(dev);
4572
4573 /* TPA requires Rx CSUM offloading */
621b4d66 4574 if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa) {
66371c44 4575 features &= ~NETIF_F_LRO;
621b4d66
DK		 4576 features &= ~NETIF_F_GRO;
4577 }
66371c44
MM		 4578
4579 return features;
4580}
4581
c8f44aff 4582int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
66371c44
MM		 4583{
4584 struct bnx2x *bp = netdev_priv(dev);
4585 u32 flags = bp->flags;
538dd2e3 4586 bool bnx2x_reload = false;
66371c44
MM		 4587
4588 if (features & NETIF_F_LRO)
4589 flags |= TPA_ENABLE_FLAG;
4590 else
4591 flags &= ~TPA_ENABLE_FLAG;
4592
621b4d66
DK		 4593 if (features & NETIF_F_GRO)
4594 flags |= GRO_ENABLE_FLAG;
4595 else
4596 flags &= ~GRO_ENABLE_FLAG;
4597
538dd2e3
MB		 4598 if (features & NETIF_F_LOOPBACK) {
4599 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
4600 bp->link_params.loopback_mode = LOOPBACK_BMAC;
4601 bnx2x_reload = true;
4602 }
4603 } else {
4604 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
4605 bp->link_params.loopback_mode = LOOPBACK_NONE;
4606 bnx2x_reload = true;
4607 }
4608 }
4609
66371c44
MM		 4610 if (flags ^ bp->flags) {
4611 bp->flags = flags;
538dd2e3
MB		 4612 bnx2x_reload = true;
4613 }
66371c44 4614
538dd2e3 4615 if (bnx2x_reload) {
66371c44
MM		 4616 if (bp->recovery_state == BNX2X_RECOVERY_DONE)
4617 return bnx2x_reload_if_running(dev);
4618 /* else: bnx2x_nic_load() will be called at end of recovery */
9f6c9258
DK		 4619 }
4620
66371c44 4621 return 0;
9f6c9258
DK		 4622}
4623
4624void bnx2x_tx_timeout(struct net_device *dev)
4625{
4626 struct bnx2x *bp = netdev_priv(dev);
4627
4628#ifdef BNX2X_STOP_ON_ERROR
4629 if (!bp->panic)
4630 bnx2x_panic();
4631#endif
7be08a72
AE		 4632
4633 smp_mb__before_clear_bit();
4634 set_bit(BNX2X_SP_RTNL_TX_TIMEOUT, &bp->sp_rtnl_state);
4635 smp_mb__after_clear_bit();
4636
9f6c9258 4637 /* This allows the netif to be shutdown gracefully before resetting */
7be08a72 4638 schedule_delayed_work(&bp->sp_rtnl_task, 0);
9f6c9258
DK		 4639}
4640
9f6c9258
DK		 4641int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
4642{
4643 struct net_device *dev = pci_get_drvdata(pdev);
4644 struct bnx2x *bp;
4645
4646 if (!dev) {
4647 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4648 return -ENODEV;
4649 }
4650 bp = netdev_priv(dev);
4651
4652 rtnl_lock();
4653
4654 pci_save_state(pdev);
4655
4656 if (!netif_running(dev)) {
4657 rtnl_unlock();
4658 return 0;
4659 }
4660
4661 netif_device_detach(dev);
4662
5d07d868 4663 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
9f6c9258
DK		 4664
4665 bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
4666
4667 rtnl_unlock();
4668
4669 return 0;
4670}
4671
4672int bnx2x_resume(struct pci_dev *pdev)
4673{
4674 struct net_device *dev = pci_get_drvdata(pdev);
4675 struct bnx2x *bp;
4676 int rc;
4677
4678 if (!dev) {
4679 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4680 return -ENODEV;
4681 }
4682 bp = netdev_priv(dev);
4683
4684 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
51c1a580 4685 BNX2X_ERR("Handling parity error recovery. Try again later\n");
9f6c9258
DK		 4686 return -EAGAIN;
4687 }
4688
4689 rtnl_lock();
4690
4691 pci_restore_state(pdev);
4692
4693 if (!netif_running(dev)) {
4694 rtnl_unlock();
4695 return 0;
4696 }
4697
4698 bnx2x_set_power_state(bp, PCI_D0);
4699 netif_device_attach(dev);
4700
4701 rc = bnx2x_nic_load(bp, LOAD_OPEN);
4702
4703 rtnl_unlock();
4704
4705 return rc;
4706}
619c5cb6
VZ		 4707
4708
4709void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
4710 u32 cid)
4711{
4712 /* ustorm cxt validation */
4713 cxt->ustorm_ag_context.cdu_usage =
4714 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4715 CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
4716 /* xcontext validation */
4717 cxt->xstorm_ag_context.cdu_reserved =
4718 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4719 CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
4720}
4721
1191cb83
ED		 4722static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
4723 u8 fw_sb_id, u8 sb_index,
4724 u8 ticks)
619c5cb6
VZ		 4725{
4726
4727 u32 addr = BAR_CSTRORM_INTMEM +
4728 CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
4729 REG_WR8(bp, addr, ticks);
51c1a580
MS		 4730 DP(NETIF_MSG_IFUP,
4731 "port %x fw_sb_id %d sb_index %d ticks %d\n",
4732 port, fw_sb_id, sb_index, ticks);
619c5cb6
VZ		 4733}
4734
1191cb83
ED		 4735static void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
4736 u16 fw_sb_id, u8 sb_index,
4737 u8 disable)
619c5cb6
VZ		 4738{
4739 u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
4740 u32 addr = BAR_CSTRORM_INTMEM +
4741 CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
4742 u16 flags = REG_RD16(bp, addr);
4743 /* clear and set */
4744 flags &= ~HC_INDEX_DATA_HC_ENABLED;
4745 flags |= enable_flag;
4746 REG_WR16(bp, addr, flags);
51c1a580
MS		 4747 DP(NETIF_MSG_IFUP,
4748 "port %x fw_sb_id %d sb_index %d disable %d\n",
4749 port, fw_sb_id, sb_index, disable);
619c5cb6
VZ		 4750}
4751
4752void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
4753 u8 sb_index, u8 disable, u16 usec)
4754{
4755 int port = BP_PORT(bp);
4756 u8 ticks = usec / BNX2X_BTR;
4757
4758 storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
4759
4760 disable = disable ? 1 : (usec ? 0 : 1);
4761 storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
4762}
Linux 6.x block layer and io_uring tree(s)