projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
bnx2x: add missing VF resource allocation during init
[linux-2.6-block.git] / drivers / net / ethernet / broadcom / bnx2x / bnx2x_cmn.c
CommitLineData
9f6c9258
DK		 1/* bnx2x_cmn.c: Broadcom Everest network driver.
2 *
247fa82b 3 * Copyright (c) 2007-2013 Broadcom Corporation
9f6c9258
DK		 4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation.
8 *
9 * Maintained by: Eilon Greenstein <eilong@broadcom.com>
10 * Written by: Eliezer Tamir
11 * Based on code from Michael Chan's bnx2 driver
12 * UDP CSUM errata workaround by Arik Gendelman
13 * Slowpath and fastpath rework by Vladislav Zolotarov
14 * Statistics and Link management by Yitchak Gertner
15 *
16 */
17
f1deab50
JP		 18#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
19
9f6c9258 20#include <linux/etherdevice.h>
9bcc0893 21#include <linux/if_vlan.h>
a6b7a407 22#include <linux/interrupt.h>
9f6c9258 23#include <linux/ip.h>
9969085e 24#include <net/tcp.h>
f2e0899f 25#include <net/ipv6.h>
7f3e01fe 26#include <net/ip6_checksum.h>
076bb0c8 27#include <net/busy_poll.h>
c0cba59e 28#include <linux/prefetch.h>
9f6c9258 29#include "bnx2x_cmn.h"
523224a3 30#include "bnx2x_init.h"
042181f5 31#include "bnx2x_sp.h"
9f6c9258 32
b3b83c3f
DK		 33/**
34 * bnx2x_move_fp - move content of the fastpath structure.
35 *
36 * @bp: driver handle
37 * @from: source FP index
38 * @to: destination FP index
39 *
40 * Makes sure the contents of the bp->fp[to].napi is kept
72754080
AE		 41 * intact. This is done by first copying the napi struct from
42 * the target to the source, and then mem copying the entire
65565884
MS		 43 * source onto the target. Update txdata pointers and related
44 * content.
b3b83c3f
DK		 45 */
46static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
47{
48 struct bnx2x_fastpath *from_fp = &bp->fp[from];
49 struct bnx2x_fastpath *to_fp = &bp->fp[to];
15192a8c
BW		 50 struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
51 struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
52 struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
53 struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
65565884
MS		 54 int old_max_eth_txqs, new_max_eth_txqs;
55 int old_txdata_index = 0, new_txdata_index = 0;
34d5626a 56 struct bnx2x_agg_info *old_tpa_info = to_fp->tpa_info;
72754080
AE		 57
58 /* Copy the NAPI object as it has been already initialized */
59 from_fp->napi = to_fp->napi;
60
b3b83c3f
DK		 61 /* Move bnx2x_fastpath contents */
62 memcpy(to_fp, from_fp, sizeof(*to_fp));
63 to_fp->index = to;
65565884 64
34d5626a
YM		 65 /* Retain the tpa_info of the original `to' version as we don't want
66 * 2 FPs to contain the same tpa_info pointer.
67 */
68 to_fp->tpa_info = old_tpa_info;
69
15192a8c
BW		 70 /* move sp_objs contents as well, as their indices match fp ones */
71 memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
72
73 /* move fp_stats contents as well, as their indices match fp ones */
74 memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
75
65565884
MS		 76 /* Update txdata pointers in fp and move txdata content accordingly:
77 * Each fp consumes 'max_cos' txdata structures, so the index should be
78 * decremented by max_cos x delta.
79 */
80
81 old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
82 new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
83 (bp)->max_cos;
84 if (from == FCOE_IDX(bp)) {
85 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
86 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
87 }
88
4864a16a
YM		 89 memcpy(&bp->bnx2x_txq[new_txdata_index],
90 &bp->bnx2x_txq[old_txdata_index],
65565884
MS		 91 sizeof(struct bnx2x_fp_txdata));
92 to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
b3b83c3f
DK		 93}
94
8ca5e17e
AE		 95/**
96 * bnx2x_fill_fw_str - Fill buffer with FW version string.
97 *
98 * @bp: driver handle
99 * @buf: character buffer to fill with the fw name
100 * @buf_len: length of the above buffer
101 *
102 */
103void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len)
104{
105 if (IS_PF(bp)) {
106 u8 phy_fw_ver[PHY_FW_VER_LEN];
107
108 phy_fw_ver[0] = '\0';
109 bnx2x_get_ext_phy_fw_version(&bp->link_params,
110 phy_fw_ver, PHY_FW_VER_LEN);
111 strlcpy(buf, bp->fw_ver, buf_len);
112 snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver),
113 "bc %d.%d.%d%s%s",
114 (bp->common.bc_ver & 0xff0000) >> 16,
115 (bp->common.bc_ver & 0xff00) >> 8,
116 (bp->common.bc_ver & 0xff),
117 ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver);
118 } else {
6411280a 119 bnx2x_vf_fill_fw_str(bp, buf, buf_len);
8ca5e17e
AE		 120 }
121}
122
4864a16a
YM		 123/**
124 * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
125 *
126 * @bp: driver handle
127 * @delta: number of eth queues which were not allocated
128 */
129static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
130{
131 int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
132
133 /* Queue pointer cannot be re-set on an fp-basis, as moving pointer
16a5fd92 134 * backward along the array could cause memory to be overridden
4864a16a
YM		 135 */
136 for (cos = 1; cos < bp->max_cos; cos++) {
137 for (i = 0; i < old_eth_num - delta; i++) {
138 struct bnx2x_fastpath *fp = &bp->fp[i];
139 int new_idx = cos * (old_eth_num - delta) + i;
140
141 memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
142 sizeof(struct bnx2x_fp_txdata));
143 fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
144 }
145 }
146}
147
619c5cb6
VZ		 148int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
149
9f6c9258
DK		 150/* free skb in the packet ring at pos idx
151 * return idx of last bd freed
152 */
6383c0b3 153static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
2df1a70a
TH		 154 u16 idx, unsigned int *pkts_compl,
155 unsigned int *bytes_compl)
9f6c9258 156{
6383c0b3 157 struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
9f6c9258
DK		 158 struct eth_tx_start_bd *tx_start_bd;
159 struct eth_tx_bd *tx_data_bd;
160 struct sk_buff *skb = tx_buf->skb;
161 u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
162 int nbd;
163
164 /* prefetch skb end pointer to speedup dev_kfree_skb() */
165 prefetch(&skb->end);
166
51c1a580 167 DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d buff @(%p)->skb %p\n",
6383c0b3 168 txdata->txq_index, idx, tx_buf, skb);
9f6c9258
DK		 169
170 /* unmap first bd */
6383c0b3 171 tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
9f6c9258 172 dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
4bca60f4 173 BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
9f6c9258
DK		 174
175 nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
176#ifdef BNX2X_STOP_ON_ERROR
177 if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
178 BNX2X_ERR("BAD nbd!\n");
179 bnx2x_panic();
180 }
181#endif
182 new_cons = nbd + tx_buf->first_bd;
183
184 /* Get the next bd */
185 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
186
187 /* Skip a parse bd... */
188 --nbd;
189 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
190
191 /* ...and the TSO split header bd since they have no mapping */
192 if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
193 --nbd;
194 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
195 }
196
197 /* now free frags */
198 while (nbd > 0) {
199
6383c0b3 200 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
9f6c9258
DK		 201 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
202 BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
203 if (--nbd)
204 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
205 }
206
207 /* release skb */
208 WARN_ON(!skb);
d8290ae5 209 if (likely(skb)) {
2df1a70a
TH		 210 (*pkts_compl)++;
211 (*bytes_compl) += skb->len;
212 }
d8290ae5 213
40955532 214 dev_kfree_skb_any(skb);
9f6c9258
DK		 215 tx_buf->first_bd = 0;
216 tx_buf->skb = NULL;
217
218 return new_cons;
219}
220
6383c0b3 221int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
9f6c9258 222{
9f6c9258 223 struct netdev_queue *txq;
6383c0b3 224 u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
2df1a70a 225 unsigned int pkts_compl = 0, bytes_compl = 0;
9f6c9258
DK		 226
227#ifdef BNX2X_STOP_ON_ERROR
228 if (unlikely(bp->panic))
229 return -1;
230#endif
231
6383c0b3
AE		 232 txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
233 hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
234 sw_cons = txdata->tx_pkt_cons;
9f6c9258
DK		 235
236 while (sw_cons != hw_cons) {
237 u16 pkt_cons;
238
239 pkt_cons = TX_BD(sw_cons);
240
51c1a580
MS		 241 DP(NETIF_MSG_TX_DONE,
242 "queue[%d]: hw_cons %u sw_cons %u pkt_cons %u\n",
6383c0b3 243 txdata->txq_index, hw_cons, sw_cons, pkt_cons);
9f6c9258 244
2df1a70a 245 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
2de67439 246 &pkts_compl, &bytes_compl);
2df1a70a 247
9f6c9258
DK		 248 sw_cons++;
249 }
250
2df1a70a
TH		 251 netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
252
6383c0b3
AE		 253 txdata->tx_pkt_cons = sw_cons;
254 txdata->tx_bd_cons = bd_cons;
9f6c9258
DK		 255
256 /* Need to make the tx_bd_cons update visible to start_xmit()
257 * before checking for netif_tx_queue_stopped(). Without the
258 * memory barrier, there is a small possibility that
259 * start_xmit() will miss it and cause the queue to be stopped
260 * forever.
619c5cb6
VZ		 261 * On the other hand we need an rmb() here to ensure the proper
262 * ordering of bit testing in the following
263 * netif_tx_queue_stopped(txq) call.
9f6c9258
DK		 264 */
265 smp_mb();
266
9f6c9258 267 if (unlikely(netif_tx_queue_stopped(txq))) {
16a5fd92 268 /* Taking tx_lock() is needed to prevent re-enabling the queue
9f6c9258
DK		 269 * while it's empty. This could have happen if rx_action() gets
270 * suspended in bnx2x_tx_int() after the condition before
271 * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
272 *
273 * stops the queue->sees fresh tx_bd_cons->releases the queue->
274 * sends some packets consuming the whole queue again->
275 * stops the queue
276 */
277
278 __netif_tx_lock(txq, smp_processor_id());
279
280 if ((netif_tx_queue_stopped(txq)) &&
281 (bp->state == BNX2X_STATE_OPEN) &&
7df2dc6b 282 (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
9f6c9258
DK		 283 netif_tx_wake_queue(txq);
284
285 __netif_tx_unlock(txq);
286 }
287 return 0;
288}
289
290static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
291 u16 idx)
292{
293 u16 last_max = fp->last_max_sge;
294
295 if (SUB_S16(idx, last_max) > 0)
296 fp->last_max_sge = idx;
297}
298
621b4d66
DK		 299static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
300 u16 sge_len,
301 struct eth_end_agg_rx_cqe *cqe)
9f6c9258
DK		 302{
303 struct bnx2x *bp = fp->bp;
9f6c9258
DK		 304 u16 last_max, last_elem, first_elem;
305 u16 delta = 0;
306 u16 i;
307
308 if (!sge_len)
309 return;
310
311 /* First mark all used pages */
312 for (i = 0; i < sge_len; i++)
619c5cb6 313 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
621b4d66 314 RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
9f6c9258
DK		 315
316 DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
621b4d66 317 sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
9f6c9258
DK		 318
319 /* Here we assume that the last SGE index is the biggest */
320 prefetch((void *)(fp->sge_mask));
523224a3 321 bnx2x_update_last_max_sge(fp,
621b4d66 322 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
9f6c9258
DK		 323
324 last_max = RX_SGE(fp->last_max_sge);
619c5cb6
VZ		 325 last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
326 first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
9f6c9258
DK		 327
328 /* If ring is not full */
329 if (last_elem + 1 != first_elem)
330 last_elem++;
331
332 /* Now update the prod */
333 for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
334 if (likely(fp->sge_mask[i]))
335 break;
336
619c5cb6
VZ		 337 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
338 delta += BIT_VEC64_ELEM_SZ;
9f6c9258
DK		 339 }
340
341 if (delta > 0) {
342 fp->rx_sge_prod += delta;
343 /* clear page-end entries */
344 bnx2x_clear_sge_mask_next_elems(fp);
345 }
346
347 DP(NETIF_MSG_RX_STATUS,
348 "fp->last_max_sge = %d fp->rx_sge_prod = %d\n",
349 fp->last_max_sge, fp->rx_sge_prod);
350}
351
2de67439 352/* Get Toeplitz hash value in the skb using the value from the
e52fcb24
ED		 353 * CQE (calculated by HW).
354 */
355static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
a334b5fb
ED		 356 const struct eth_fast_path_rx_cqe *cqe,
357 bool *l4_rxhash)
e52fcb24 358{
2de67439 359 /* Get Toeplitz hash from CQE */
e52fcb24 360 if ((bp->dev->features & NETIF_F_RXHASH) &&
a334b5fb
ED		 361 (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
362 enum eth_rss_hash_type htype;
363
364 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
365 *l4_rxhash = (htype == TCP_IPV4_HASH_TYPE) ||
366 (htype == TCP_IPV6_HASH_TYPE);
e52fcb24 367 return le32_to_cpu(cqe->rss_hash_result);
a334b5fb
ED		 368 }
369 *l4_rxhash = false;
e52fcb24
ED		 370 return 0;
371}
372
9f6c9258 373static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
e52fcb24 374 u16 cons, u16 prod,
619c5cb6 375 struct eth_fast_path_rx_cqe *cqe)
9f6c9258
DK		 376{
377 struct bnx2x *bp = fp->bp;
378 struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
379 struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
380 struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
381 dma_addr_t mapping;
619c5cb6
VZ		 382 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
383 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
9f6c9258 384
619c5cb6
VZ		 385 /* print error if current state != stop */
386 if (tpa_info->tpa_state != BNX2X_TPA_STOP)
9f6c9258
DK		 387 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
388
e52fcb24 389 /* Try to map an empty data buffer from the aggregation info */
619c5cb6 390 mapping = dma_map_single(&bp->pdev->dev,
e52fcb24 391 first_buf->data + NET_SKB_PAD,
619c5cb6
VZ		 392 fp->rx_buf_size, DMA_FROM_DEVICE);
393 /*
394 * ...if it fails - move the skb from the consumer to the producer
395 * and set the current aggregation state as ERROR to drop it
396 * when TPA_STOP arrives.
397 */
398
399 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
400 /* Move the BD from the consumer to the producer */
e52fcb24 401 bnx2x_reuse_rx_data(fp, cons, prod);
619c5cb6
VZ		 402 tpa_info->tpa_state = BNX2X_TPA_ERROR;
403 return;
404 }
9f6c9258 405
e52fcb24
ED		 406 /* move empty data from pool to prod */
407 prod_rx_buf->data = first_buf->data;
619c5cb6 408 dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
e52fcb24 409 /* point prod_bd to new data */
9f6c9258
DK		 410 prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
411 prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
412
619c5cb6
VZ		 413 /* move partial skb from cons to pool (don't unmap yet) */
414 *first_buf = *cons_rx_buf;
415
416 /* mark bin state as START */
417 tpa_info->parsing_flags =
418 le16_to_cpu(cqe->pars_flags.flags);
419 tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
420 tpa_info->tpa_state = BNX2X_TPA_START;
421 tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
422 tpa_info->placement_offset = cqe->placement_offset;
a334b5fb 423 tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->l4_rxhash);
621b4d66
DK		 424 if (fp->mode == TPA_MODE_GRO) {
425 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
924d75ab 426 tpa_info->full_page = SGE_PAGES / gro_size * gro_size;
621b4d66
DK		 427 tpa_info->gro_size = gro_size;
428 }
619c5cb6 429
9f6c9258
DK		 430#ifdef BNX2X_STOP_ON_ERROR
431 fp->tpa_queue_used |= (1 << queue);
432#ifdef _ASM_GENERIC_INT_L64_H
433 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
434#else
435 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
436#endif
437 fp->tpa_queue_used);
438#endif
439}
440
e4e3c02a
VZ		 441/* Timestamp option length allowed for TPA aggregation:
442 *
443 * nop nop kind length echo val
444 */
445#define TPA_TSTAMP_OPT_LEN 12
446/**
cbf1de72 447 * bnx2x_set_gro_params - compute GRO values
e4e3c02a 448 *
cbf1de72 449 * @skb: packet skb
e8920674
DK		 450 * @parsing_flags: parsing flags from the START CQE
451 * @len_on_bd: total length of the first packet for the
452 * aggregation.
cbf1de72 453 * @pkt_len: length of all segments
e8920674
DK		 454 *
455 * Approximate value of the MSS for this aggregation calculated using
456 * the first packet of it.
2de67439 457 * Compute number of aggregated segments, and gso_type.
e4e3c02a 458 */
cbf1de72 459static void bnx2x_set_gro_params(struct sk_buff *skb, u16 parsing_flags,
ab5777d7
YM		 460 u16 len_on_bd, unsigned int pkt_len,
461 u16 num_of_coalesced_segs)
e4e3c02a 462{
cbf1de72 463 /* TPA aggregation won't have either IP options or TCP options
619c5cb6 464 * other than timestamp or IPv6 extension headers.
e4e3c02a 465 */
619c5cb6
VZ		 466 u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
467
468 if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
cbf1de72 469 PRS_FLAG_OVERETH_IPV6) {
619c5cb6 470 hdrs_len += sizeof(struct ipv6hdr);
cbf1de72
YM		 471 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
472 } else {
619c5cb6 473 hdrs_len += sizeof(struct iphdr);
cbf1de72
YM		 474 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
475 }
e4e3c02a
VZ		 476
477 /* Check if there was a TCP timestamp, if there is it's will
478 * always be 12 bytes length: nop nop kind length echo val.
479 *
480 * Otherwise FW would close the aggregation.
481 */
482 if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
483 hdrs_len += TPA_TSTAMP_OPT_LEN;
484
cbf1de72
YM		 485 skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len;
486
487 /* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count
488 * to skb_shinfo(skb)->gso_segs
489 */
ab5777d7 490 NAPI_GRO_CB(skb)->count = num_of_coalesced_segs;
e4e3c02a
VZ		 491}
492
996dedba
MS		 493static int bnx2x_alloc_rx_sge(struct bnx2x *bp, struct bnx2x_fastpath *fp,
494 u16 index, gfp_t gfp_mask)
1191cb83 495{
996dedba 496 struct page *page = alloc_pages(gfp_mask, PAGES_PER_SGE_SHIFT);
1191cb83
ED		 497 struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
498 struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
499 dma_addr_t mapping;
500
501 if (unlikely(page == NULL)) {
502 BNX2X_ERR("Can't alloc sge\n");
503 return -ENOMEM;
504 }
505
506 mapping = dma_map_page(&bp->pdev->dev, page, 0,
924d75ab 507 SGE_PAGES, DMA_FROM_DEVICE);
1191cb83
ED		 508 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
509 __free_pages(page, PAGES_PER_SGE_SHIFT);
510 BNX2X_ERR("Can't map sge\n");
511 return -ENOMEM;
512 }
513
514 sw_buf->page = page;
515 dma_unmap_addr_set(sw_buf, mapping, mapping);
516
517 sge->addr_hi = cpu_to_le32(U64_HI(mapping));
518 sge->addr_lo = cpu_to_le32(U64_LO(mapping));
519
520 return 0;
521}
522
9f6c9258 523static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
621b4d66
DK		 524 struct bnx2x_agg_info *tpa_info,
525 u16 pages,
526 struct sk_buff *skb,
619c5cb6
VZ		 527 struct eth_end_agg_rx_cqe *cqe,
528 u16 cqe_idx)
9f6c9258
DK		 529{
530 struct sw_rx_page *rx_pg, old_rx_pg;
621b4d66
DK		 531 u32 i, frag_len, frag_size;
532 int err, j, frag_id = 0;
619c5cb6 533 u16 len_on_bd = tpa_info->len_on_bd;
621b4d66 534 u16 full_page = 0, gro_size = 0;
9f6c9258 535
619c5cb6 536 frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
621b4d66
DK		 537
538 if (fp->mode == TPA_MODE_GRO) {
539 gro_size = tpa_info->gro_size;
540 full_page = tpa_info->full_page;
541 }
9f6c9258
DK		 542
543 /* This is needed in order to enable forwarding support */
cbf1de72
YM		 544 if (frag_size)
545 bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd,
ab5777d7
YM		 546 le16_to_cpu(cqe->pkt_len),
547 le16_to_cpu(cqe->num_of_coalesced_segs));
621b4d66 548
9f6c9258 549#ifdef BNX2X_STOP_ON_ERROR
924d75ab 550 if (pages > min_t(u32, 8, MAX_SKB_FRAGS) * SGE_PAGES) {
9f6c9258
DK		 551 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
552 pages, cqe_idx);
619c5cb6 553 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
9f6c9258
DK		 554 bnx2x_panic();
555 return -EINVAL;
556 }
557#endif
558
559 /* Run through the SGL and compose the fragmented skb */
560 for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
619c5cb6 561 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
9f6c9258
DK		 562
563 /* FW gives the indices of the SGE as if the ring is an array
564 (meaning that "next" element will consume 2 indices) */
621b4d66
DK		 565 if (fp->mode == TPA_MODE_GRO)
566 frag_len = min_t(u32, frag_size, (u32)full_page);
567 else /* LRO */
924d75ab 568 frag_len = min_t(u32, frag_size, (u32)SGE_PAGES);
621b4d66 569
9f6c9258
DK		 570 rx_pg = &fp->rx_page_ring[sge_idx];
571 old_rx_pg = *rx_pg;
572
573 /* If we fail to allocate a substitute page, we simply stop
574 where we are and drop the whole packet */
996dedba 575 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx, GFP_ATOMIC);
9f6c9258 576 if (unlikely(err)) {
15192a8c 577 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
9f6c9258
DK		 578 return err;
579 }
580
16a5fd92 581 /* Unmap the page as we're going to pass it to the stack */
9f6c9258
DK		 582 dma_unmap_page(&bp->pdev->dev,
583 dma_unmap_addr(&old_rx_pg, mapping),
924d75ab 584 SGE_PAGES, DMA_FROM_DEVICE);
9f6c9258 585 /* Add one frag and update the appropriate fields in the skb */
621b4d66
DK		 586 if (fp->mode == TPA_MODE_LRO)
587 skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
588 else { /* GRO */
589 int rem;
590 int offset = 0;
591 for (rem = frag_len; rem > 0; rem -= gro_size) {
592 int len = rem > gro_size ? gro_size : rem;
593 skb_fill_page_desc(skb, frag_id++,
594 old_rx_pg.page, offset, len);
595 if (offset)
596 get_page(old_rx_pg.page);
597 offset += len;
598 }
599 }
9f6c9258
DK		 600
601 skb->data_len += frag_len;
924d75ab 602 skb->truesize += SGE_PAGES;
9f6c9258
DK		 603 skb->len += frag_len;
604
605 frag_size -= frag_len;
606 }
607
608 return 0;
609}
610
d46d132c
ED		 611static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
612{
613 if (fp->rx_frag_size)
614 put_page(virt_to_head_page(data));
615 else
616 kfree(data);
617}
618
996dedba 619static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp, gfp_t gfp_mask)
d46d132c 620{
996dedba
MS		 621 if (fp->rx_frag_size) {
622 /* GFP_KERNEL allocations are used only during initialization */
623 if (unlikely(gfp_mask & __GFP_WAIT))
624 return (void *)__get_free_page(gfp_mask);
625
d46d132c 626 return netdev_alloc_frag(fp->rx_frag_size);
996dedba 627 }
d46d132c 628
996dedba 629 return kmalloc(fp->rx_buf_size + NET_SKB_PAD, gfp_mask);
d46d132c
ED		 630}
631
9969085e
YM		 632#ifdef CONFIG_INET
633static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb)
634{
635 const struct iphdr *iph = ip_hdr(skb);
636 struct tcphdr *th;
637
638 skb_set_transport_header(skb, sizeof(struct iphdr));
639 th = tcp_hdr(skb);
640
641 th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb),
642 iph->saddr, iph->daddr, 0);
643}
644
645static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb)
646{
647 struct ipv6hdr *iph = ipv6_hdr(skb);
648 struct tcphdr *th;
649
650 skb_set_transport_header(skb, sizeof(struct ipv6hdr));
651 th = tcp_hdr(skb);
652
653 th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb),
654 &iph->saddr, &iph->daddr, 0);
655}
2c2d06d5
YM		 656
657static void bnx2x_gro_csum(struct bnx2x *bp, struct sk_buff *skb,
658 void (*gro_func)(struct bnx2x*, struct sk_buff*))
659{
660 skb_set_network_header(skb, 0);
661 gro_func(bp, skb);
662 tcp_gro_complete(skb);
663}
9969085e
YM		 664#endif
665
666static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp,
667 struct sk_buff *skb)
668{
669#ifdef CONFIG_INET
cbf1de72 670 if (skb_shinfo(skb)->gso_size) {
9969085e
YM		 671 switch (be16_to_cpu(skb->protocol)) {
672 case ETH_P_IP:
2c2d06d5 673 bnx2x_gro_csum(bp, skb, bnx2x_gro_ip_csum);
9969085e
YM		 674 break;
675 case ETH_P_IPV6:
2c2d06d5 676 bnx2x_gro_csum(bp, skb, bnx2x_gro_ipv6_csum);
9969085e
YM		 677 break;
678 default:
2c2d06d5 679 BNX2X_ERR("Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n",
9969085e
YM		 680 be16_to_cpu(skb->protocol));
681 }
9969085e
YM		 682 }
683#endif
684 napi_gro_receive(&fp->napi, skb);
685}
686
1191cb83
ED		 687static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
688 struct bnx2x_agg_info *tpa_info,
689 u16 pages,
690 struct eth_end_agg_rx_cqe *cqe,
691 u16 cqe_idx)
9f6c9258 692{
619c5cb6 693 struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
621b4d66 694 u8 pad = tpa_info->placement_offset;
619c5cb6 695 u16 len = tpa_info->len_on_bd;
e52fcb24 696 struct sk_buff *skb = NULL;
621b4d66 697 u8 *new_data, *data = rx_buf->data;
619c5cb6
VZ		 698 u8 old_tpa_state = tpa_info->tpa_state;
699
700 tpa_info->tpa_state = BNX2X_TPA_STOP;
701
702 /* If we there was an error during the handling of the TPA_START -
703 * drop this aggregation.
704 */
705 if (old_tpa_state == BNX2X_TPA_ERROR)
706 goto drop;
707
e52fcb24 708 /* Try to allocate the new data */
996dedba 709 new_data = bnx2x_frag_alloc(fp, GFP_ATOMIC);
9f6c9258
DK		 710 /* Unmap skb in the pool anyway, as we are going to change
711 pool entry status to BNX2X_TPA_STOP even if new skb allocation
712 fails. */
713 dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
a8c94b91 714 fp->rx_buf_size, DMA_FROM_DEVICE);
e52fcb24 715 if (likely(new_data))
d46d132c 716 skb = build_skb(data, fp->rx_frag_size);
9f6c9258 717
e52fcb24 718 if (likely(skb)) {
9f6c9258 719#ifdef BNX2X_STOP_ON_ERROR
a8c94b91 720 if (pad + len > fp->rx_buf_size) {
51c1a580 721 BNX2X_ERR("skb_put is about to fail... pad %d len %d rx_buf_size %d\n",
a8c94b91 722 pad, len, fp->rx_buf_size);
9f6c9258
DK		 723 bnx2x_panic();
724 return;
725 }
726#endif
727
e52fcb24 728 skb_reserve(skb, pad + NET_SKB_PAD);
9f6c9258 729 skb_put(skb, len);
e52fcb24 730 skb->rxhash = tpa_info->rxhash;
a334b5fb 731 skb->l4_rxhash = tpa_info->l4_rxhash;
9f6c9258
DK		 732
733 skb->protocol = eth_type_trans(skb, bp->dev);
734 skb->ip_summed = CHECKSUM_UNNECESSARY;
735
621b4d66
DK		 736 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
737 skb, cqe, cqe_idx)) {
619c5cb6 738 if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
86a9bad3 739 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tpa_info->vlan_tag);
9969085e 740 bnx2x_gro_receive(bp, fp, skb);
9f6c9258 741 } else {
51c1a580
MS		 742 DP(NETIF_MSG_RX_STATUS,
743 "Failed to allocate new pages - dropping packet!\n");
40955532 744 dev_kfree_skb_any(skb);
9f6c9258
DK		 745 }
746
e52fcb24
ED		 747 /* put new data in bin */
748 rx_buf->data = new_data;
9f6c9258 749
619c5cb6 750 return;
9f6c9258 751 }
d46d132c 752 bnx2x_frag_free(fp, new_data);
619c5cb6
VZ		 753drop:
754 /* drop the packet and keep the buffer in the bin */
755 DP(NETIF_MSG_RX_STATUS,
756 "Failed to allocate or map a new skb - dropping packet!\n");
15192a8c 757 bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
9f6c9258
DK		 758}
759
996dedba
MS		 760static int bnx2x_alloc_rx_data(struct bnx2x *bp, struct bnx2x_fastpath *fp,
761 u16 index, gfp_t gfp_mask)
1191cb83
ED		 762{
763 u8 *data;
764 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
765 struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
766 dma_addr_t mapping;
767
996dedba 768 data = bnx2x_frag_alloc(fp, gfp_mask);
1191cb83
ED		 769 if (unlikely(data == NULL))
770 return -ENOMEM;
771
772 mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
773 fp->rx_buf_size,
774 DMA_FROM_DEVICE);
775 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
d46d132c 776 bnx2x_frag_free(fp, data);
1191cb83
ED		 777 BNX2X_ERR("Can't map rx data\n");
778 return -ENOMEM;
779 }
780
781 rx_buf->data = data;
782 dma_unmap_addr_set(rx_buf, mapping, mapping);
783
784 rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
785 rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
786
787 return 0;
788}
789
15192a8c
BW		 790static
791void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
792 struct bnx2x_fastpath *fp,
793 struct bnx2x_eth_q_stats *qstats)
d6cb3e41 794{
e488921f
MS		 795 /* Do nothing if no L4 csum validation was done.
796 * We do not check whether IP csum was validated. For IPv4 we assume
797 * that if the card got as far as validating the L4 csum, it also
798 * validated the IP csum. IPv6 has no IP csum.
799 */
d6cb3e41 800 if (cqe->fast_path_cqe.status_flags &
e488921f 801 ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
d6cb3e41
ED		 802 return;
803
e488921f 804 /* If L4 validation was done, check if an error was found. */
d6cb3e41
ED		 805
806 if (cqe->fast_path_cqe.type_error_flags &
807 (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
808 ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
15192a8c 809 qstats->hw_csum_err++;
d6cb3e41
ED		 810 else
811 skb->ip_summed = CHECKSUM_UNNECESSARY;
812}
9f6c9258
DK		 813
814int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
815{
816 struct bnx2x *bp = fp->bp;
817 u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
75b29459 818 u16 sw_comp_cons, sw_comp_prod;
9f6c9258 819 int rx_pkt = 0;
75b29459
DK		 820 union eth_rx_cqe *cqe;
821 struct eth_fast_path_rx_cqe *cqe_fp;
9f6c9258
DK		 822
823#ifdef BNX2X_STOP_ON_ERROR
824 if (unlikely(bp->panic))
825 return 0;
826#endif
827
9f6c9258
DK		 828 bd_cons = fp->rx_bd_cons;
829 bd_prod = fp->rx_bd_prod;
830 bd_prod_fw = bd_prod;
831 sw_comp_cons = fp->rx_comp_cons;
832 sw_comp_prod = fp->rx_comp_prod;
833
75b29459
DK		 834 comp_ring_cons = RCQ_BD(sw_comp_cons);
835 cqe = &fp->rx_comp_ring[comp_ring_cons];
836 cqe_fp = &cqe->fast_path_cqe;
9f6c9258
DK		 837
838 DP(NETIF_MSG_RX_STATUS,
75b29459 839 "queue[%d]: sw_comp_cons %u\n", fp->index, sw_comp_cons);
9f6c9258 840
75b29459 841 while (BNX2X_IS_CQE_COMPLETED(cqe_fp)) {
9f6c9258
DK		 842 struct sw_rx_bd *rx_buf = NULL;
843 struct sk_buff *skb;
9f6c9258 844 u8 cqe_fp_flags;
619c5cb6 845 enum eth_rx_cqe_type cqe_fp_type;
621b4d66 846 u16 len, pad, queue;
e52fcb24 847 u8 *data;
a334b5fb 848 bool l4_rxhash;
9f6c9258 849
619c5cb6
VZ		 850#ifdef BNX2X_STOP_ON_ERROR
851 if (unlikely(bp->panic))
852 return 0;
853#endif
854
9f6c9258
DK		 855 bd_prod = RX_BD(bd_prod);
856 bd_cons = RX_BD(bd_cons);
857
619c5cb6
VZ		 858 cqe_fp_flags = cqe_fp->type_error_flags;
859 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
9f6c9258 860
51c1a580
MS		 861 DP(NETIF_MSG_RX_STATUS,
862 "CQE type %x err %x status %x queue %x vlan %x len %u\n",
863 CQE_TYPE(cqe_fp_flags),
619c5cb6
VZ		 864 cqe_fp_flags, cqe_fp->status_flags,
865 le32_to_cpu(cqe_fp->rss_hash_result),
621b4d66
DK		 866 le16_to_cpu(cqe_fp->vlan_tag),
867 le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
9f6c9258
DK		 868
869 /* is this a slowpath msg? */
619c5cb6 870 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
9f6c9258
DK		 871 bnx2x_sp_event(fp, cqe);
872 goto next_cqe;
e52fcb24 873 }
621b4d66 874
e52fcb24
ED		 875 rx_buf = &fp->rx_buf_ring[bd_cons];
876 data = rx_buf->data;
9f6c9258 877
e52fcb24 878 if (!CQE_TYPE_FAST(cqe_fp_type)) {
621b4d66
DK		 879 struct bnx2x_agg_info *tpa_info;
880 u16 frag_size, pages;
619c5cb6 881#ifdef BNX2X_STOP_ON_ERROR
e52fcb24
ED		 882 /* sanity check */
883 if (fp->disable_tpa &&
884 (CQE_TYPE_START(cqe_fp_type) ||
885 CQE_TYPE_STOP(cqe_fp_type)))
51c1a580 886 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
e52fcb24 887 CQE_TYPE(cqe_fp_type));
619c5cb6 888#endif
9f6c9258 889
e52fcb24
ED		 890 if (CQE_TYPE_START(cqe_fp_type)) {
891 u16 queue = cqe_fp->queue_index;
892 DP(NETIF_MSG_RX_STATUS,
893 "calling tpa_start on queue %d\n",
894 queue);
9f6c9258 895
e52fcb24
ED		 896 bnx2x_tpa_start(fp, queue,
897 bd_cons, bd_prod,
898 cqe_fp);
621b4d66 899
e52fcb24 900 goto next_rx;
621b4d66
DK		 901 }
902 queue = cqe->end_agg_cqe.queue_index;
903 tpa_info = &fp->tpa_info[queue];
904 DP(NETIF_MSG_RX_STATUS,
905 "calling tpa_stop on queue %d\n",
906 queue);
907
908 frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
909 tpa_info->len_on_bd;
910
911 if (fp->mode == TPA_MODE_GRO)
912 pages = (frag_size + tpa_info->full_page - 1) /
913 tpa_info->full_page;
914 else
915 pages = SGE_PAGE_ALIGN(frag_size) >>
916 SGE_PAGE_SHIFT;
917
918 bnx2x_tpa_stop(bp, fp, tpa_info, pages,
919 &cqe->end_agg_cqe, comp_ring_cons);
9f6c9258 920#ifdef BNX2X_STOP_ON_ERROR
621b4d66
DK		 921 if (bp->panic)
922 return 0;
9f6c9258
DK		 923#endif
924
621b4d66
DK		 925 bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
926 goto next_cqe;
e52fcb24
ED		 927 }
928 /* non TPA */
621b4d66 929 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
e52fcb24
ED		 930 pad = cqe_fp->placement_offset;
931 dma_sync_single_for_cpu(&bp->pdev->dev,
9f6c9258 932 dma_unmap_addr(rx_buf, mapping),
e52fcb24
ED		 933 pad + RX_COPY_THRESH,
934 DMA_FROM_DEVICE);
935 pad += NET_SKB_PAD;
936 prefetch(data + pad); /* speedup eth_type_trans() */
937 /* is this an error packet? */
938 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
51c1a580 939 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
e52fcb24
ED		 940 "ERROR flags %x rx packet %u\n",
941 cqe_fp_flags, sw_comp_cons);
15192a8c 942 bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
e52fcb24
ED		 943 goto reuse_rx;
944 }
9f6c9258 945
e52fcb24
ED		 946 /* Since we don't have a jumbo ring
947 * copy small packets if mtu > 1500
948 */
949 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
950 (len <= RX_COPY_THRESH)) {
951 skb = netdev_alloc_skb_ip_align(bp->dev, len);
952 if (skb == NULL) {
51c1a580 953 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
e52fcb24 954 "ERROR packet dropped because of alloc failure\n");
15192a8c 955 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
9f6c9258
DK		 956 goto reuse_rx;
957 }
e52fcb24
ED		 958 memcpy(skb->data, data + pad, len);
959 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
960 } else {
996dedba
MS		 961 if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod,
962 GFP_ATOMIC) == 0)) {
9f6c9258 963 dma_unmap_single(&bp->pdev->dev,
e52fcb24 964 dma_unmap_addr(rx_buf, mapping),
a8c94b91 965 fp->rx_buf_size,
9f6c9258 966 DMA_FROM_DEVICE);
d46d132c 967 skb = build_skb(data, fp->rx_frag_size);
e52fcb24 968 if (unlikely(!skb)) {
d46d132c 969 bnx2x_frag_free(fp, data);
15192a8c
BW		 970 bnx2x_fp_qstats(bp, fp)->
971 rx_skb_alloc_failed++;
e52fcb24
ED		 972 goto next_rx;
973 }
9f6c9258 974 skb_reserve(skb, pad);
9f6c9258 975 } else {
51c1a580
MS		 976 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
977 "ERROR packet dropped because of alloc failure\n");
15192a8c 978 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
9f6c9258 979reuse_rx:
e52fcb24 980 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
9f6c9258
DK		 981 goto next_rx;
982 }
036d2df9 983 }
9f6c9258 984
036d2df9
DK		 985 skb_put(skb, len);
986 skb->protocol = eth_type_trans(skb, bp->dev);
9f6c9258 987
036d2df9 988 /* Set Toeplitz hash for a none-LRO skb */
a334b5fb
ED		 989 skb->rxhash = bnx2x_get_rxhash(bp, cqe_fp, &l4_rxhash);
990 skb->l4_rxhash = l4_rxhash;
9f6c9258 991
036d2df9 992 skb_checksum_none_assert(skb);
f85582f8 993
d6cb3e41 994 if (bp->dev->features & NETIF_F_RXCSUM)
15192a8c
BW		 995 bnx2x_csum_validate(skb, cqe, fp,
996 bnx2x_fp_qstats(bp, fp));
9f6c9258 997
f233cafe 998 skb_record_rx_queue(skb, fp->rx_queue);
9f6c9258 999
619c5cb6
VZ		 1000 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
1001 PARSING_FLAGS_VLAN)
86a9bad3 1002 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
619c5cb6 1003 le16_to_cpu(cqe_fp->vlan_tag));
9f6c9258 1004
8b80cda5 1005 skb_mark_napi_id(skb, &fp->napi);
8f20aa57
DK		 1006
1007 if (bnx2x_fp_ll_polling(fp))
1008 netif_receive_skb(skb);
1009 else
1010 napi_gro_receive(&fp->napi, skb);
9f6c9258 1011next_rx:
e52fcb24 1012 rx_buf->data = NULL;
9f6c9258
DK		 1013
1014 bd_cons = NEXT_RX_IDX(bd_cons);
1015 bd_prod = NEXT_RX_IDX(bd_prod);
1016 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
1017 rx_pkt++;
1018next_cqe:
1019 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
1020 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
1021
75b29459
DK		 1022 /* mark CQE as free */
1023 BNX2X_SEED_CQE(cqe_fp);
1024
9f6c9258
DK		 1025 if (rx_pkt == budget)
1026 break;
75b29459
DK		 1027
1028 comp_ring_cons = RCQ_BD(sw_comp_cons);
1029 cqe = &fp->rx_comp_ring[comp_ring_cons];
1030 cqe_fp = &cqe->fast_path_cqe;
9f6c9258
DK		 1031 } /* while */
1032
1033 fp->rx_bd_cons = bd_cons;
1034 fp->rx_bd_prod = bd_prod_fw;
1035 fp->rx_comp_cons = sw_comp_cons;
1036 fp->rx_comp_prod = sw_comp_prod;
1037
1038 /* Update producers */
1039 bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
1040 fp->rx_sge_prod);
1041
1042 fp->rx_pkt += rx_pkt;
1043 fp->rx_calls++;
1044
1045 return rx_pkt;
1046}
1047
1048static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
1049{
1050 struct bnx2x_fastpath *fp = fp_cookie;
1051 struct bnx2x *bp = fp->bp;
6383c0b3 1052 u8 cos;
9f6c9258 1053
51c1a580
MS		 1054 DP(NETIF_MSG_INTR,
1055 "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
523224a3 1056 fp->index, fp->fw_sb_id, fp->igu_sb_id);
ecf01c22 1057
523224a3 1058 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
9f6c9258
DK		 1059
1060#ifdef BNX2X_STOP_ON_ERROR
1061 if (unlikely(bp->panic))
1062 return IRQ_HANDLED;
1063#endif
1064
1065 /* Handle Rx and Tx according to MSI-X vector */
6383c0b3 1066 for_each_cos_in_tx_queue(fp, cos)
65565884 1067 prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
6383c0b3 1068
523224a3 1069 prefetch(&fp->sb_running_index[SM_RX_ID]);
9f6c9258
DK		 1070 napi_schedule(&bnx2x_fp(bp, fp->index, napi));
1071
1072 return IRQ_HANDLED;
1073}
1074
9f6c9258
DK		 1075/* HW Lock for shared dual port PHYs */
1076void bnx2x_acquire_phy_lock(struct bnx2x *bp)
1077{
1078 mutex_lock(&bp->port.phy_mutex);
1079
8203c4b6 1080 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
9f6c9258
DK		 1081}
1082
1083void bnx2x_release_phy_lock(struct bnx2x *bp)
1084{
8203c4b6 1085 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
9f6c9258
DK		 1086
1087 mutex_unlock(&bp->port.phy_mutex);
1088}
1089
0793f83f
DK		 1090/* calculates MF speed according to current linespeed and MF configuration */
1091u16 bnx2x_get_mf_speed(struct bnx2x *bp)
1092{
1093 u16 line_speed = bp->link_vars.line_speed;
1094 if (IS_MF(bp)) {
faa6fcbb
DK		 1095 u16 maxCfg = bnx2x_extract_max_cfg(bp,
1096 bp->mf_config[BP_VN(bp)]);
1097
1098 /* Calculate the current MAX line speed limit for the MF
1099 * devices
0793f83f 1100 */
faa6fcbb
DK		 1101 if (IS_MF_SI(bp))
1102 line_speed = (line_speed * maxCfg) / 100;
1103 else { /* SD mode */
0793f83f
DK		 1104 u16 vn_max_rate = maxCfg * 100;
1105
1106 if (vn_max_rate < line_speed)
1107 line_speed = vn_max_rate;
faa6fcbb 1108 }
0793f83f
DK		 1109 }
1110
1111 return line_speed;
1112}
1113
2ae17f66
VZ		 1114/**
1115 * bnx2x_fill_report_data - fill link report data to report
1116 *
1117 * @bp: driver handle
1118 * @data: link state to update
1119 *
1120 * It uses a none-atomic bit operations because is called under the mutex.
1121 */
1191cb83
ED		 1122static void bnx2x_fill_report_data(struct bnx2x *bp,
1123 struct bnx2x_link_report_data *data)
2ae17f66
VZ		 1124{
1125 u16 line_speed = bnx2x_get_mf_speed(bp);
1126
1127 memset(data, 0, sizeof(*data));
1128
16a5fd92 1129 /* Fill the report data: effective line speed */
2ae17f66
VZ		 1130 data->line_speed = line_speed;
1131
1132 /* Link is down */
1133 if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1134 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1135 &data->link_report_flags);
1136
1137 /* Full DUPLEX */
1138 if (bp->link_vars.duplex == DUPLEX_FULL)
1139 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
1140
1141 /* Rx Flow Control is ON */
1142 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1143 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
1144
1145 /* Tx Flow Control is ON */
1146 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1147 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
1148}
1149
1150/**
1151 * bnx2x_link_report - report link status to OS.
1152 *
1153 * @bp: driver handle
1154 *
1155 * Calls the __bnx2x_link_report() under the same locking scheme
1156 * as a link/PHY state managing code to ensure a consistent link
1157 * reporting.
1158 */
1159
9f6c9258
DK		 1160void bnx2x_link_report(struct bnx2x *bp)
1161{
2ae17f66
VZ		 1162 bnx2x_acquire_phy_lock(bp);
1163 __bnx2x_link_report(bp);
1164 bnx2x_release_phy_lock(bp);
1165}
9f6c9258 1166
2ae17f66
VZ		 1167/**
1168 * __bnx2x_link_report - report link status to OS.
1169 *
1170 * @bp: driver handle
1171 *
16a5fd92 1172 * None atomic implementation.
2ae17f66
VZ		 1173 * Should be called under the phy_lock.
1174 */
1175void __bnx2x_link_report(struct bnx2x *bp)
1176{
1177 struct bnx2x_link_report_data cur_data;
9f6c9258 1178
2ae17f66 1179 /* reread mf_cfg */
ad5afc89 1180 if (IS_PF(bp) && !CHIP_IS_E1(bp))
2ae17f66
VZ		 1181 bnx2x_read_mf_cfg(bp);
1182
1183 /* Read the current link report info */
1184 bnx2x_fill_report_data(bp, &cur_data);
1185
1186 /* Don't report link down or exactly the same link status twice */
1187 if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1188 (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1189 &bp->last_reported_link.link_report_flags) &&
1190 test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1191 &cur_data.link_report_flags)))
1192 return;
1193
1194 bp->link_cnt++;
9f6c9258 1195
2ae17f66
VZ		 1196 /* We are going to report a new link parameters now -
1197 * remember the current data for the next time.
1198 */
1199 memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
9f6c9258 1200
2ae17f66
VZ		 1201 if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1202 &cur_data.link_report_flags)) {
1203 netif_carrier_off(bp->dev);
1204 netdev_err(bp->dev, "NIC Link is Down\n");
1205 return;
1206 } else {
94f05b0f
JP		 1207 const char *duplex;
1208 const char *flow;
1209
2ae17f66 1210 netif_carrier_on(bp->dev);
9f6c9258 1211
2ae17f66
VZ		 1212 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1213 &cur_data.link_report_flags))
94f05b0f 1214 duplex = "full";
9f6c9258 1215 else
94f05b0f 1216 duplex = "half";
9f6c9258 1217
2ae17f66
VZ		 1218 /* Handle the FC at the end so that only these flags would be
1219 * possibly set. This way we may easily check if there is no FC
1220 * enabled.
1221 */
1222 if (cur_data.link_report_flags) {
1223 if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1224 &cur_data.link_report_flags)) {
2ae17f66
VZ		 1225 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1226 &cur_data.link_report_flags))
94f05b0f
JP		 1227 flow = "ON - receive & transmit";
1228 else
1229 flow = "ON - receive";
9f6c9258 1230 } else {
94f05b0f 1231 flow = "ON - transmit";
9f6c9258 1232 }
94f05b0f
JP		 1233 } else {
1234 flow = "none";
9f6c9258 1235 }
94f05b0f
JP		 1236 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1237 cur_data.line_speed, duplex, flow);
9f6c9258
DK		 1238 }
1239}
1240
1191cb83
ED		 1241static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1242{
1243 int i;
1244
1245 for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1246 struct eth_rx_sge *sge;
1247
1248 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1249 sge->addr_hi =
1250 cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1251 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1252
1253 sge->addr_lo =
1254 cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1255 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1256 }
1257}
1258
1259static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1260 struct bnx2x_fastpath *fp, int last)
1261{
1262 int i;
1263
1264 for (i = 0; i < last; i++) {
1265 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1266 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1267 u8 *data = first_buf->data;
1268
1269 if (data == NULL) {
1270 DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1271 continue;
1272 }
1273 if (tpa_info->tpa_state == BNX2X_TPA_START)
1274 dma_unmap_single(&bp->pdev->dev,
1275 dma_unmap_addr(first_buf, mapping),
1276 fp->rx_buf_size, DMA_FROM_DEVICE);
d46d132c 1277 bnx2x_frag_free(fp, data);
1191cb83
ED		 1278 first_buf->data = NULL;
1279 }
1280}
1281
55c11941
MS		 1282void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1283{
1284 int j;
1285
1286 for_each_rx_queue_cnic(bp, j) {
1287 struct bnx2x_fastpath *fp = &bp->fp[j];
1288
1289 fp->rx_bd_cons = 0;
1290
1291 /* Activate BD ring */
1292 /* Warning!
1293 * this will generate an interrupt (to the TSTORM)
1294 * must only be done after chip is initialized
1295 */
1296 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1297 fp->rx_sge_prod);
1298 }
1299}
1300
9f6c9258
DK		 1301void bnx2x_init_rx_rings(struct bnx2x *bp)
1302{
1303 int func = BP_FUNC(bp);
523224a3 1304 u16 ring_prod;
9f6c9258 1305 int i, j;
25141580 1306
b3b83c3f 1307 /* Allocate TPA resources */
55c11941 1308 for_each_eth_queue(bp, j) {
523224a3 1309 struct bnx2x_fastpath *fp = &bp->fp[j];
9f6c9258 1310
a8c94b91
VZ		 1311 DP(NETIF_MSG_IFUP,
1312 "mtu %d rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1313
523224a3 1314 if (!fp->disable_tpa) {
16a5fd92 1315 /* Fill the per-aggregation pool */
dfacf138 1316 for (i = 0; i < MAX_AGG_QS(bp); i++) {
619c5cb6
VZ		 1317 struct bnx2x_agg_info *tpa_info =
1318 &fp->tpa_info[i];
1319 struct sw_rx_bd *first_buf =
1320 &tpa_info->first_buf;
1321
996dedba
MS		 1322 first_buf->data =
1323 bnx2x_frag_alloc(fp, GFP_KERNEL);
e52fcb24 1324 if (!first_buf->data) {
51c1a580
MS		 1325 BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1326 j);
9f6c9258
DK		 1327 bnx2x_free_tpa_pool(bp, fp, i);
1328 fp->disable_tpa = 1;
1329 break;
1330 }
619c5cb6
VZ		 1331 dma_unmap_addr_set(first_buf, mapping, 0);
1332 tpa_info->tpa_state = BNX2X_TPA_STOP;
9f6c9258 1333 }
523224a3
DK		 1334
1335 /* "next page" elements initialization */
1336 bnx2x_set_next_page_sgl(fp);
1337
1338 /* set SGEs bit mask */
1339 bnx2x_init_sge_ring_bit_mask(fp);
1340
1341 /* Allocate SGEs and initialize the ring elements */
1342 for (i = 0, ring_prod = 0;
1343 i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1344
996dedba
MS		 1345 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod,
1346 GFP_KERNEL) < 0) {
51c1a580
MS		 1347 BNX2X_ERR("was only able to allocate %d rx sges\n",
1348 i);
1349 BNX2X_ERR("disabling TPA for queue[%d]\n",
1350 j);
523224a3 1351 /* Cleanup already allocated elements */
619c5cb6
VZ		 1352 bnx2x_free_rx_sge_range(bp, fp,
1353 ring_prod);
1354 bnx2x_free_tpa_pool(bp, fp,
dfacf138 1355 MAX_AGG_QS(bp));
523224a3
DK		 1356 fp->disable_tpa = 1;
1357 ring_prod = 0;
1358 break;
1359 }
1360 ring_prod = NEXT_SGE_IDX(ring_prod);
1361 }
1362
1363 fp->rx_sge_prod = ring_prod;
9f6c9258
DK		 1364 }
1365 }
1366
55c11941 1367 for_each_eth_queue(bp, j) {
9f6c9258
DK		 1368 struct bnx2x_fastpath *fp = &bp->fp[j];
1369
1370 fp->rx_bd_cons = 0;
9f6c9258 1371
b3b83c3f
DK		 1372 /* Activate BD ring */
1373 /* Warning!
1374 * this will generate an interrupt (to the TSTORM)
1375 * must only be done after chip is initialized
1376 */
1377 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1378 fp->rx_sge_prod);
9f6c9258 1379
9f6c9258
DK		 1380 if (j != 0)
1381 continue;
1382
619c5cb6 1383 if (CHIP_IS_E1(bp)) {
f2e0899f
DK		 1384 REG_WR(bp, BAR_USTRORM_INTMEM +
1385 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1386 U64_LO(fp->rx_comp_mapping));
1387 REG_WR(bp, BAR_USTRORM_INTMEM +
1388 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1389 U64_HI(fp->rx_comp_mapping));
1390 }
9f6c9258
DK		 1391 }
1392}
f85582f8 1393
55c11941 1394static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
9f6c9258 1395{
6383c0b3 1396 u8 cos;
55c11941 1397 struct bnx2x *bp = fp->bp;
9f6c9258 1398
55c11941
MS		 1399 for_each_cos_in_tx_queue(fp, cos) {
1400 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1401 unsigned pkts_compl = 0, bytes_compl = 0;
9f6c9258 1402
55c11941
MS		 1403 u16 sw_prod = txdata->tx_pkt_prod;
1404 u16 sw_cons = txdata->tx_pkt_cons;
9f6c9258 1405
55c11941
MS		 1406 while (sw_cons != sw_prod) {
1407 bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1408 &pkts_compl, &bytes_compl);
1409 sw_cons++;
9f6c9258 1410 }
55c11941
MS		 1411
1412 netdev_tx_reset_queue(
1413 netdev_get_tx_queue(bp->dev,
1414 txdata->txq_index));
1415 }
1416}
1417
1418static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1419{
1420 int i;
1421
1422 for_each_tx_queue_cnic(bp, i) {
1423 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1424 }
1425}
1426
1427static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1428{
1429 int i;
1430
1431 for_each_eth_queue(bp, i) {
1432 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
9f6c9258
DK		 1433 }
1434}
1435
b3b83c3f
DK		 1436static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1437{
1438 struct bnx2x *bp = fp->bp;
1439 int i;
1440
1441 /* ring wasn't allocated */
1442 if (fp->rx_buf_ring == NULL)
1443 return;
1444
1445 for (i = 0; i < NUM_RX_BD; i++) {
1446 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
e52fcb24 1447 u8 *data = rx_buf->data;
b3b83c3f 1448
e52fcb24 1449 if (data == NULL)
b3b83c3f 1450 continue;
b3b83c3f
DK		 1451 dma_unmap_single(&bp->pdev->dev,
1452 dma_unmap_addr(rx_buf, mapping),
1453 fp->rx_buf_size, DMA_FROM_DEVICE);
1454
e52fcb24 1455 rx_buf->data = NULL;
d46d132c 1456 bnx2x_frag_free(fp, data);
b3b83c3f
DK		 1457 }
1458}
1459
55c11941
MS		 1460static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1461{
1462 int j;
1463
1464 for_each_rx_queue_cnic(bp, j) {
1465 bnx2x_free_rx_bds(&bp->fp[j]);
1466 }
1467}
1468
9f6c9258
DK		 1469static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1470{
b3b83c3f 1471 int j;
9f6c9258 1472
55c11941 1473 for_each_eth_queue(bp, j) {
9f6c9258
DK		 1474 struct bnx2x_fastpath *fp = &bp->fp[j];
1475
b3b83c3f 1476 bnx2x_free_rx_bds(fp);
9f6c9258 1477
9f6c9258 1478 if (!fp->disable_tpa)
dfacf138 1479 bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
9f6c9258
DK		 1480 }
1481}
1482
55c11941
MS		 1483void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1484{
1485 bnx2x_free_tx_skbs_cnic(bp);
1486 bnx2x_free_rx_skbs_cnic(bp);
1487}
1488
9f6c9258
DK		 1489void bnx2x_free_skbs(struct bnx2x *bp)
1490{
1491 bnx2x_free_tx_skbs(bp);
1492 bnx2x_free_rx_skbs(bp);
1493}
1494
e3835b99
DK		 1495void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1496{
1497 /* load old values */
1498 u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1499
1500 if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1501 /* leave all but MAX value */
1502 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1503
1504 /* set new MAX value */
1505 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1506 & FUNC_MF_CFG_MAX_BW_MASK;
1507
1508 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1509 }
1510}
1511
ca92429f
DK		 1512/**
1513 * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1514 *
1515 * @bp: driver handle
1516 * @nvecs: number of vectors to be released
1517 */
1518static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
9f6c9258 1519{
ca92429f 1520 int i, offset = 0;
9f6c9258 1521
ca92429f
DK		 1522 if (nvecs == offset)
1523 return;
ad5afc89
AE		 1524
1525 /* VFs don't have a default SB */
1526 if (IS_PF(bp)) {
1527 free_irq(bp->msix_table[offset].vector, bp->dev);
1528 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1529 bp->msix_table[offset].vector);
1530 offset++;
1531 }
55c11941
MS		 1532
1533 if (CNIC_SUPPORT(bp)) {
1534 if (nvecs == offset)
1535 return;
1536 offset++;
1537 }
ca92429f 1538
ec6ba945 1539 for_each_eth_queue(bp, i) {
ca92429f
DK		 1540 if (nvecs == offset)
1541 return;
51c1a580
MS		 1542 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1543 i, bp->msix_table[offset].vector);
9f6c9258 1544
ca92429f 1545 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
9f6c9258
DK		 1546 }
1547}
1548
d6214d7a 1549void bnx2x_free_irq(struct bnx2x *bp)
9f6c9258 1550{
30a5de77 1551 if (bp->flags & USING_MSIX_FLAG &&
ad5afc89
AE		 1552 !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1553 int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp);
1554
1555 /* vfs don't have a default status block */
1556 if (IS_PF(bp))
1557 nvecs++;
1558
1559 bnx2x_free_msix_irqs(bp, nvecs);
1560 } else {
30a5de77 1561 free_irq(bp->dev->irq, bp->dev);
ad5afc89 1562 }
9f6c9258
DK		 1563}
1564
0e8d2ec5 1565int bnx2x_enable_msix(struct bnx2x *bp)
9f6c9258 1566{
1ab4434c 1567 int msix_vec = 0, i, rc;
9f6c9258 1568
1ab4434c
AE		 1569 /* VFs don't have a default status block */
1570 if (IS_PF(bp)) {
1571 bp->msix_table[msix_vec].entry = msix_vec;
1572 BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1573 bp->msix_table[0].entry);
1574 msix_vec++;
1575 }
9f6c9258 1576
55c11941
MS		 1577 /* Cnic requires an msix vector for itself */
1578 if (CNIC_SUPPORT(bp)) {
1579 bp->msix_table[msix_vec].entry = msix_vec;
1580 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1581 msix_vec, bp->msix_table[msix_vec].entry);
1582 msix_vec++;
1583 }
1584
6383c0b3 1585 /* We need separate vectors for ETH queues only (not FCoE) */
ec6ba945 1586 for_each_eth_queue(bp, i) {
d6214d7a 1587 bp->msix_table[msix_vec].entry = msix_vec;
51c1a580
MS		 1588 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1589 msix_vec, msix_vec, i);
d6214d7a 1590 msix_vec++;
9f6c9258
DK		 1591 }
1592
1ab4434c
AE		 1593 DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n",
1594 msix_vec);
d6214d7a 1595
1ab4434c 1596 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], msix_vec);
9f6c9258
DK		 1597
1598 /*
1599 * reconfigure number of tx/rx queues according to available
1600 * MSI-X vectors
1601 */
55c11941 1602 if (rc >= BNX2X_MIN_MSIX_VEC_CNT(bp)) {
d6214d7a 1603 /* how less vectors we will have? */
1ab4434c 1604 int diff = msix_vec - rc;
9f6c9258 1605
51c1a580 1606 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
9f6c9258
DK		 1607
1608 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1609
1610 if (rc) {
30a5de77
DK		 1611 BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1612 goto no_msix;
9f6c9258 1613 }
d6214d7a
DK		 1614 /*
1615 * decrease number of queues by number of unallocated entries
1616 */
55c11941
MS		 1617 bp->num_ethernet_queues -= diff;
1618 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
9f6c9258 1619
51c1a580 1620 BNX2X_DEV_INFO("New queue configuration set: %d\n",
30a5de77
DK		 1621 bp->num_queues);
1622 } else if (rc > 0) {
1623 /* Get by with single vector */
1624 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], 1);
1625 if (rc) {
1626 BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1627 rc);
1628 goto no_msix;
1629 }
1630
1631 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1632 bp->flags |= USING_SINGLE_MSIX_FLAG;
1633
55c11941
MS		 1634 BNX2X_DEV_INFO("set number of queues to 1\n");
1635 bp->num_ethernet_queues = 1;
1636 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
30a5de77 1637 } else if (rc < 0) {
51c1a580 1638 BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
30a5de77 1639 goto no_msix;
9f6c9258
DK		 1640 }
1641
1642 bp->flags |= USING_MSIX_FLAG;
1643
1644 return 0;
30a5de77
DK		 1645
1646no_msix:
1647 /* fall to INTx if not enough memory */
1648 if (rc == -ENOMEM)
1649 bp->flags |= DISABLE_MSI_FLAG;
1650
1651 return rc;
9f6c9258
DK		 1652}
1653
1654static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1655{
ca92429f 1656 int i, rc, offset = 0;
9f6c9258 1657
ad5afc89
AE		 1658 /* no default status block for vf */
1659 if (IS_PF(bp)) {
1660 rc = request_irq(bp->msix_table[offset++].vector,
1661 bnx2x_msix_sp_int, 0,
1662 bp->dev->name, bp->dev);
1663 if (rc) {
1664 BNX2X_ERR("request sp irq failed\n");
1665 return -EBUSY;
1666 }
9f6c9258
DK		 1667 }
1668
55c11941
MS		 1669 if (CNIC_SUPPORT(bp))
1670 offset++;
1671
ec6ba945 1672 for_each_eth_queue(bp, i) {
9f6c9258
DK		 1673 struct bnx2x_fastpath *fp = &bp->fp[i];
1674 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1675 bp->dev->name, i);
1676
d6214d7a 1677 rc = request_irq(bp->msix_table[offset].vector,
9f6c9258
DK		 1678 bnx2x_msix_fp_int, 0, fp->name, fp);
1679 if (rc) {
ca92429f
DK		 1680 BNX2X_ERR("request fp #%d irq (%d) failed rc %d\n", i,
1681 bp->msix_table[offset].vector, rc);
1682 bnx2x_free_msix_irqs(bp, offset);
9f6c9258
DK		 1683 return -EBUSY;
1684 }
1685
d6214d7a 1686 offset++;
9f6c9258
DK		 1687 }
1688
ec6ba945 1689 i = BNX2X_NUM_ETH_QUEUES(bp);
ad5afc89
AE		 1690 if (IS_PF(bp)) {
1691 offset = 1 + CNIC_SUPPORT(bp);
1692 netdev_info(bp->dev,
1693 "using MSI-X IRQs: sp %d fp[%d] %d ... fp[%d] %d\n",
1694 bp->msix_table[0].vector,
1695 0, bp->msix_table[offset].vector,
1696 i - 1, bp->msix_table[offset + i - 1].vector);
1697 } else {
1698 offset = CNIC_SUPPORT(bp);
1699 netdev_info(bp->dev,
1700 "using MSI-X IRQs: fp[%d] %d ... fp[%d] %d\n",
1701 0, bp->msix_table[offset].vector,
1702 i - 1, bp->msix_table[offset + i - 1].vector);
1703 }
9f6c9258
DK		 1704 return 0;
1705}
1706
d6214d7a 1707int bnx2x_enable_msi(struct bnx2x *bp)
9f6c9258
DK		 1708{
1709 int rc;
1710
1711 rc = pci_enable_msi(bp->pdev);
1712 if (rc) {
51c1a580 1713 BNX2X_DEV_INFO("MSI is not attainable\n");
9f6c9258
DK		 1714 return -1;
1715 }
1716 bp->flags |= USING_MSI_FLAG;
1717
1718 return 0;
1719}
1720
1721static int bnx2x_req_irq(struct bnx2x *bp)
1722{
1723 unsigned long flags;
30a5de77 1724 unsigned int irq;
9f6c9258 1725
30a5de77 1726 if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
9f6c9258
DK		 1727 flags = 0;
1728 else
1729 flags = IRQF_SHARED;
1730
30a5de77
DK		 1731 if (bp->flags & USING_MSIX_FLAG)
1732 irq = bp->msix_table[0].vector;
1733 else
1734 irq = bp->pdev->irq;
1735
1736 return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
9f6c9258
DK		 1737}
1738
c957d09f 1739static int bnx2x_setup_irqs(struct bnx2x *bp)
619c5cb6
VZ		 1740{
1741 int rc = 0;
30a5de77
DK		 1742 if (bp->flags & USING_MSIX_FLAG &&
1743 !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
619c5cb6
VZ		 1744 rc = bnx2x_req_msix_irqs(bp);
1745 if (rc)
1746 return rc;
1747 } else {
619c5cb6
VZ		 1748 rc = bnx2x_req_irq(bp);
1749 if (rc) {
1750 BNX2X_ERR("IRQ request failed rc %d, aborting\n", rc);
1751 return rc;
1752 }
1753 if (bp->flags & USING_MSI_FLAG) {
1754 bp->dev->irq = bp->pdev->irq;
30a5de77
DK		 1755 netdev_info(bp->dev, "using MSI IRQ %d\n",
1756 bp->dev->irq);
1757 }
1758 if (bp->flags & USING_MSIX_FLAG) {
1759 bp->dev->irq = bp->msix_table[0].vector;
1760 netdev_info(bp->dev, "using MSIX IRQ %d\n",
1761 bp->dev->irq);
619c5cb6
VZ		 1762 }
1763 }
1764
1765 return 0;
1766}
1767
55c11941
MS		 1768static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1769{
1770 int i;
1771
8f20aa57
DK		 1772 for_each_rx_queue_cnic(bp, i) {
1773 bnx2x_fp_init_lock(&bp->fp[i]);
55c11941 1774 napi_enable(&bnx2x_fp(bp, i, napi));
8f20aa57 1775 }
55c11941
MS		 1776}
1777
1191cb83 1778static void bnx2x_napi_enable(struct bnx2x *bp)
9f6c9258
DK		 1779{
1780 int i;
1781
8f20aa57
DK		 1782 for_each_eth_queue(bp, i) {
1783 bnx2x_fp_init_lock(&bp->fp[i]);
9f6c9258 1784 napi_enable(&bnx2x_fp(bp, i, napi));
8f20aa57 1785 }
9f6c9258
DK		 1786}
1787
55c11941
MS		 1788static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1789{
1790 int i;
1791
8f20aa57
DK		 1792 local_bh_disable();
1793 for_each_rx_queue_cnic(bp, i) {
55c11941 1794 napi_disable(&bnx2x_fp(bp, i, napi));
8f20aa57
DK		 1795 while (!bnx2x_fp_lock_napi(&bp->fp[i]))
1796 mdelay(1);
1797 }
1798 local_bh_enable();
55c11941
MS		 1799}
1800
1191cb83 1801static void bnx2x_napi_disable(struct bnx2x *bp)
9f6c9258
DK		 1802{
1803 int i;
1804
8f20aa57
DK		 1805 local_bh_disable();
1806 for_each_eth_queue(bp, i) {
9f6c9258 1807 napi_disable(&bnx2x_fp(bp, i, napi));
8f20aa57
DK		 1808 while (!bnx2x_fp_lock_napi(&bp->fp[i]))
1809 mdelay(1);
1810 }
1811 local_bh_enable();
9f6c9258
DK		 1812}
1813
1814void bnx2x_netif_start(struct bnx2x *bp)
1815{
4b7ed897
DK		 1816 if (netif_running(bp->dev)) {
1817 bnx2x_napi_enable(bp);
55c11941
MS		 1818 if (CNIC_LOADED(bp))
1819 bnx2x_napi_enable_cnic(bp);
4b7ed897
DK		 1820 bnx2x_int_enable(bp);
1821 if (bp->state == BNX2X_STATE_OPEN)
1822 netif_tx_wake_all_queues(bp->dev);
9f6c9258
DK		 1823 }
1824}
1825
1826void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1827{
1828 bnx2x_int_disable_sync(bp, disable_hw);
1829 bnx2x_napi_disable(bp);
55c11941
MS		 1830 if (CNIC_LOADED(bp))
1831 bnx2x_napi_disable_cnic(bp);
9f6c9258 1832}
9f6c9258 1833
8307fa3e
VZ		 1834u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1835{
8307fa3e 1836 struct bnx2x *bp = netdev_priv(dev);
cdb9d6ae 1837
55c11941 1838 if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
8307fa3e
VZ		 1839 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1840 u16 ether_type = ntohs(hdr->h_proto);
1841
1842 /* Skip VLAN tag if present */
1843 if (ether_type == ETH_P_8021Q) {
1844 struct vlan_ethhdr *vhdr =
1845 (struct vlan_ethhdr *)skb->data;
1846
1847 ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1848 }
1849
1850 /* If ethertype is FCoE or FIP - use FCoE ring */
1851 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
6383c0b3 1852 return bnx2x_fcoe_tx(bp, txq_index);
8307fa3e 1853 }
55c11941 1854
cdb9d6ae 1855 /* select a non-FCoE queue */
ada7c19e 1856 return __netdev_pick_tx(dev, skb) % BNX2X_NUM_ETH_QUEUES(bp);
8307fa3e
VZ		 1857}
1858
d6214d7a
DK		 1859void bnx2x_set_num_queues(struct bnx2x *bp)
1860{
96305234 1861 /* RSS queues */
55c11941 1862 bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
ec6ba945 1863
a3348722
BW		 1864 /* override in STORAGE SD modes */
1865 if (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))
55c11941
MS		 1866 bp->num_ethernet_queues = 1;
1867
ec6ba945 1868 /* Add special queues */
55c11941
MS		 1869 bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1870 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
65565884
MS		 1871
1872 BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
ec6ba945
VZ		 1873}
1874
cdb9d6ae
VZ		 1875/**
1876 * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1877 *
1878 * @bp: Driver handle
1879 *
1880 * We currently support for at most 16 Tx queues for each CoS thus we will
1881 * allocate a multiple of 16 for ETH L2 rings according to the value of the
1882 * bp->max_cos.
1883 *
1884 * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1885 * index after all ETH L2 indices.
1886 *
1887 * If the actual number of Tx queues (for each CoS) is less than 16 then there
1888 * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
16a5fd92 1889 * 16..31,...) with indices that are not coupled with any real Tx queue.
cdb9d6ae
VZ		 1890 *
1891 * The proper configuration of skb->queue_mapping is handled by
1892 * bnx2x_select_queue() and __skb_tx_hash().
1893 *
1894 * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1895 * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1896 */
55c11941 1897static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
ec6ba945 1898{
6383c0b3 1899 int rc, tx, rx;
ec6ba945 1900
65565884 1901 tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
55c11941 1902 rx = BNX2X_NUM_ETH_QUEUES(bp);
ec6ba945 1903
6383c0b3 1904/* account for fcoe queue */
55c11941
MS		 1905 if (include_cnic && !NO_FCOE(bp)) {
1906 rx++;
1907 tx++;
6383c0b3 1908 }
6383c0b3
AE		 1909
1910 rc = netif_set_real_num_tx_queues(bp->dev, tx);
1911 if (rc) {
1912 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1913 return rc;
1914 }
1915 rc = netif_set_real_num_rx_queues(bp->dev, rx);
1916 if (rc) {
1917 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1918 return rc;
1919 }
1920
51c1a580 1921 DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
6383c0b3
AE		 1922 tx, rx);
1923
ec6ba945
VZ		 1924 return rc;
1925}
1926
1191cb83 1927static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
a8c94b91
VZ		 1928{
1929 int i;
1930
1931 for_each_queue(bp, i) {
1932 struct bnx2x_fastpath *fp = &bp->fp[i];
e52fcb24 1933 u32 mtu;
a8c94b91
VZ		 1934
1935 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1936 if (IS_FCOE_IDX(i))
1937 /*
1938 * Although there are no IP frames expected to arrive to
1939 * this ring we still want to add an
1940 * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1941 * overrun attack.
1942 */
e52fcb24 1943 mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
a8c94b91 1944 else
e52fcb24
ED		 1945 mtu = bp->dev->mtu;
1946 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
1947 IP_HEADER_ALIGNMENT_PADDING +
1948 ETH_OVREHEAD +
1949 mtu +
1950 BNX2X_FW_RX_ALIGN_END;
16a5fd92 1951 /* Note : rx_buf_size doesn't take into account NET_SKB_PAD */
d46d132c
ED		 1952 if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
1953 fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
1954 else
1955 fp->rx_frag_size = 0;
a8c94b91
VZ		 1956 }
1957}
1958
60cad4e6 1959static int bnx2x_init_rss(struct bnx2x *bp)
619c5cb6
VZ		 1960{
1961 int i;
619c5cb6
VZ		 1962 u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
1963
16a5fd92 1964 /* Prepare the initial contents for the indirection table if RSS is
619c5cb6
VZ		 1965 * enabled
1966 */
5d317c6a
MS		 1967 for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
1968 bp->rss_conf_obj.ind_table[i] =
96305234
DK		 1969 bp->fp->cl_id +
1970 ethtool_rxfh_indir_default(i, num_eth_queues);
619c5cb6
VZ		 1971
1972 /*
1973 * For 57710 and 57711 SEARCHER configuration (rss_keys) is
1974 * per-port, so if explicit configuration is needed , do it only
1975 * for a PMF.
1976 *
1977 * For 57712 and newer on the other hand it's a per-function
1978 * configuration.
1979 */
5d317c6a 1980 return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
619c5cb6
VZ		 1981}
1982
60cad4e6
AE		 1983int bnx2x_rss(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
1984 bool config_hash, bool enable)
619c5cb6 1985{
3b603066 1986 struct bnx2x_config_rss_params params = {NULL};
619c5cb6
VZ		 1987
1988 /* Although RSS is meaningless when there is a single HW queue we
1989 * still need it enabled in order to have HW Rx hash generated.
1990 *
1991 * if (!is_eth_multi(bp))
1992 * bp->multi_mode = ETH_RSS_MODE_DISABLED;
1993 */
1994
96305234 1995 params.rss_obj = rss_obj;
619c5cb6
VZ		 1996
1997 __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
1998
60cad4e6
AE		 1999 if (enable) {
2000 __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
2001
2002 /* RSS configuration */
2003 __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
2004 __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
2005 __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
2006 __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
2007 if (rss_obj->udp_rss_v4)
2008 __set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
2009 if (rss_obj->udp_rss_v6)
2010 __set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
2011 } else {
2012 __set_bit(BNX2X_RSS_MODE_DISABLED, &params.rss_flags);
2013 }
619c5cb6 2014
96305234
DK		 2015 /* Hash bits */
2016 params.rss_result_mask = MULTI_MASK;
619c5cb6 2017
5d317c6a 2018 memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
619c5cb6 2019
96305234
DK		 2020 if (config_hash) {
2021 /* RSS keys */
60cad4e6 2022 prandom_bytes(params.rss_key, T_ETH_RSS_KEY * 4);
96305234 2023 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
619c5cb6
VZ		 2024 }
2025
60cad4e6
AE		 2026 if (IS_PF(bp))
2027 return bnx2x_config_rss(bp, &params);
2028 else
2029 return bnx2x_vfpf_config_rss(bp, &params);
619c5cb6
VZ		 2030}
2031
1191cb83 2032static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
619c5cb6 2033{
3b603066 2034 struct bnx2x_func_state_params func_params = {NULL};
619c5cb6
VZ		 2035
2036 /* Prepare parameters for function state transitions */
2037 __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
2038
2039 func_params.f_obj = &bp->func_obj;
2040 func_params.cmd = BNX2X_F_CMD_HW_INIT;
2041
2042 func_params.params.hw_init.load_phase = load_code;
2043
2044 return bnx2x_func_state_change(bp, &func_params);
2045}
2046
2047/*
2048 * Cleans the object that have internal lists without sending
16a5fd92 2049 * ramrods. Should be run when interrupts are disabled.
619c5cb6 2050 */
7fa6f340 2051void bnx2x_squeeze_objects(struct bnx2x *bp)
619c5cb6
VZ		 2052{
2053 int rc;
2054 unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
3b603066 2055 struct bnx2x_mcast_ramrod_params rparam = {NULL};
15192a8c 2056 struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
619c5cb6
VZ		 2057
2058 /***************** Cleanup MACs' object first *************************/
2059
2060 /* Wait for completion of requested */
2061 __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
2062 /* Perform a dry cleanup */
2063 __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
2064
2065 /* Clean ETH primary MAC */
2066 __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
15192a8c 2067 rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
619c5cb6
VZ		 2068 &ramrod_flags);
2069 if (rc != 0)
2070 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
2071
2072 /* Cleanup UC list */
2073 vlan_mac_flags = 0;
2074 __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
2075 rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
2076 &ramrod_flags);
2077 if (rc != 0)
2078 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
2079
2080 /***************** Now clean mcast object *****************************/
2081 rparam.mcast_obj = &bp->mcast_obj;
2082 __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
2083
8b09be5f
YM		 2084 /* Add a DEL command... - Since we're doing a driver cleanup only,
2085 * we take a lock surrounding both the initial send and the CONTs,
2086 * as we don't want a true completion to disrupt us in the middle.
2087 */
2088 netif_addr_lock_bh(bp->dev);
619c5cb6
VZ		 2089 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
2090 if (rc < 0)
51c1a580
MS		 2091 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
2092 rc);
619c5cb6
VZ		 2093
2094 /* ...and wait until all pending commands are cleared */
2095 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2096 while (rc != 0) {
2097 if (rc < 0) {
2098 BNX2X_ERR("Failed to clean multi-cast object: %d\n",
2099 rc);
8b09be5f 2100 netif_addr_unlock_bh(bp->dev);
619c5cb6
VZ		 2101 return;
2102 }
2103
2104 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2105 }
8b09be5f 2106 netif_addr_unlock_bh(bp->dev);
619c5cb6
VZ		 2107}
2108
2109#ifndef BNX2X_STOP_ON_ERROR
2110#define LOAD_ERROR_EXIT(bp, label) \
2111 do { \
2112 (bp)->state = BNX2X_STATE_ERROR; \
2113 goto label; \
2114 } while (0)
55c11941
MS		 2115
2116#define LOAD_ERROR_EXIT_CNIC(bp, label) \
2117 do { \
2118 bp->cnic_loaded = false; \
2119 goto label; \
2120 } while (0)
2121#else /*BNX2X_STOP_ON_ERROR*/
619c5cb6
VZ		 2122#define LOAD_ERROR_EXIT(bp, label) \
2123 do { \
2124 (bp)->state = BNX2X_STATE_ERROR; \
2125 (bp)->panic = 1; \
2126 return -EBUSY; \
2127 } while (0)
55c11941
MS		 2128#define LOAD_ERROR_EXIT_CNIC(bp, label) \
2129 do { \
2130 bp->cnic_loaded = false; \
2131 (bp)->panic = 1; \
2132 return -EBUSY; \
2133 } while (0)
2134#endif /*BNX2X_STOP_ON_ERROR*/
619c5cb6 2135
ad5afc89
AE		 2136static void bnx2x_free_fw_stats_mem(struct bnx2x *bp)
2137{
2138 BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping,
2139 bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2140 return;
2141}
2142
2143static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp)
452427b0 2144{
8db573ba 2145 int num_groups, vf_headroom = 0;
ad5afc89 2146 int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1;
452427b0 2147
ad5afc89
AE		 2148 /* number of queues for statistics is number of eth queues + FCoE */
2149 u8 num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats;
452427b0 2150
ad5afc89
AE		 2151 /* Total number of FW statistics requests =
2152 * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper
2153 * and fcoe l2 queue) stats + num of queues (which includes another 1
2154 * for fcoe l2 queue if applicable)
2155 */
2156 bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats;
452427b0 2157
8db573ba
AE		 2158 /* vf stats appear in the request list, but their data is allocated by
2159 * the VFs themselves. We don't include them in the bp->fw_stats_num as
2160 * it is used to determine where to place the vf stats queries in the
2161 * request struct
2162 */
2163 if (IS_SRIOV(bp))
6411280a 2164 vf_headroom = bnx2x_vf_headroom(bp);
8db573ba 2165
ad5afc89
AE		 2166 /* Request is built from stats_query_header and an array of
2167 * stats_query_cmd_group each of which contains
2168 * STATS_QUERY_CMD_COUNT rules. The real number or requests is
2169 * configured in the stats_query_header.
2170 */
2171 num_groups =
8db573ba
AE		 2172 (((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) +
2173 (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ?
ad5afc89
AE		 2174 1 : 0));
2175
8db573ba
AE		 2176 DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n",
2177 bp->fw_stats_num, vf_headroom, num_groups);
ad5afc89
AE		 2178 bp->fw_stats_req_sz = sizeof(struct stats_query_header) +
2179 num_groups * sizeof(struct stats_query_cmd_group);
2180
2181 /* Data for statistics requests + stats_counter
2182 * stats_counter holds per-STORM counters that are incremented
2183 * when STORM has finished with the current request.
2184 * memory for FCoE offloaded statistics are counted anyway,
2185 * even if they will not be sent.
2186 * VF stats are not accounted for here as the data of VF stats is stored
2187 * in memory allocated by the VF, not here.
2188 */
2189 bp->fw_stats_data_sz = sizeof(struct per_port_stats) +
2190 sizeof(struct per_pf_stats) +
2191 sizeof(struct fcoe_statistics_params) +
2192 sizeof(struct per_queue_stats) * num_queue_stats +
2193 sizeof(struct stats_counter);
2194
2195 BNX2X_PCI_ALLOC(bp->fw_stats, &bp->fw_stats_mapping,
2196 bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2197
2198 /* Set shortcuts */
2199 bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats;
2200 bp->fw_stats_req_mapping = bp->fw_stats_mapping;
2201 bp->fw_stats_data = (struct bnx2x_fw_stats_data *)
2202 ((u8 *)bp->fw_stats + bp->fw_stats_req_sz);
2203 bp->fw_stats_data_mapping = bp->fw_stats_mapping +
2204 bp->fw_stats_req_sz;
2205
6bf07b8e 2206 DP(BNX2X_MSG_SP, "statistics request base address set to %x %x\n",
ad5afc89
AE		 2207 U64_HI(bp->fw_stats_req_mapping),
2208 U64_LO(bp->fw_stats_req_mapping));
6bf07b8e 2209 DP(BNX2X_MSG_SP, "statistics data base address set to %x %x\n",
ad5afc89
AE		 2210 U64_HI(bp->fw_stats_data_mapping),
2211 U64_LO(bp->fw_stats_data_mapping));
2212 return 0;
2213
2214alloc_mem_err:
2215 bnx2x_free_fw_stats_mem(bp);
2216 BNX2X_ERR("Can't allocate FW stats memory\n");
2217 return -ENOMEM;
2218}
2219
2220/* send load request to mcp and analyze response */
2221static int bnx2x_nic_load_request(struct bnx2x *bp, u32 *load_code)
2222{
178135c1
DK		 2223 u32 param;
2224
ad5afc89
AE		 2225 /* init fw_seq */
2226 bp->fw_seq =
2227 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2228 DRV_MSG_SEQ_NUMBER_MASK);
2229 BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2230
2231 /* Get current FW pulse sequence */
2232 bp->fw_drv_pulse_wr_seq =
2233 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2234 DRV_PULSE_SEQ_MASK);
2235 BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2236
178135c1
DK		 2237 param = DRV_MSG_CODE_LOAD_REQ_WITH_LFA;
2238
2239 if (IS_MF_SD(bp) && bnx2x_port_after_undi(bp))
2240 param |= DRV_MSG_CODE_LOAD_REQ_FORCE_LFA;
2241
ad5afc89 2242 /* load request */
178135c1 2243 (*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, param);
ad5afc89
AE		 2244
2245 /* if mcp fails to respond we must abort */
2246 if (!(*load_code)) {
2247 BNX2X_ERR("MCP response failure, aborting\n");
2248 return -EBUSY;
2249 }
2250
2251 /* If mcp refused (e.g. other port is in diagnostic mode) we
2252 * must abort
2253 */
2254 if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2255 BNX2X_ERR("MCP refused load request, aborting\n");
2256 return -EBUSY;
2257 }
2258 return 0;
2259}
2260
2261/* check whether another PF has already loaded FW to chip. In
2262 * virtualized environments a pf from another VM may have already
2263 * initialized the device including loading FW
2264 */
2265int bnx2x_nic_load_analyze_req(struct bnx2x *bp, u32 load_code)
2266{
2267 /* is another pf loaded on this engine? */
2268 if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2269 load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2270 /* build my FW version dword */
2271 u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
2272 (BCM_5710_FW_MINOR_VERSION << 8) +
2273 (BCM_5710_FW_REVISION_VERSION << 16) +
2274 (BCM_5710_FW_ENGINEERING_VERSION << 24);
2275
2276 /* read loaded FW from chip */
2277 u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2278
2279 DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x\n",
2280 loaded_fw, my_fw);
2281
2282 /* abort nic load if version mismatch */
2283 if (my_fw != loaded_fw) {
6bf07b8e 2284 BNX2X_ERR("bnx2x with FW %x was already loaded which mismatches my %x FW. Aborting\n",
452427b0 2285 loaded_fw, my_fw);
ad5afc89
AE		 2286 return -EBUSY;
2287 }
2288 }
2289 return 0;
2290}
2291
2292/* returns the "mcp load_code" according to global load_count array */
2293static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port)
2294{
2295 int path = BP_PATH(bp);
2296
2297 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d] %d, %d, %d\n",
2298 path, load_count[path][0], load_count[path][1],
2299 load_count[path][2]);
2300 load_count[path][0]++;
2301 load_count[path][1 + port]++;
2302 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d] %d, %d, %d\n",
2303 path, load_count[path][0], load_count[path][1],
2304 load_count[path][2]);
2305 if (load_count[path][0] == 1)
2306 return FW_MSG_CODE_DRV_LOAD_COMMON;
2307 else if (load_count[path][1 + port] == 1)
2308 return FW_MSG_CODE_DRV_LOAD_PORT;
2309 else
2310 return FW_MSG_CODE_DRV_LOAD_FUNCTION;
2311}
2312
2313/* mark PMF if applicable */
2314static void bnx2x_nic_load_pmf(struct bnx2x *bp, u32 load_code)
2315{
2316 if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2317 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2318 (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2319 bp->port.pmf = 1;
2320 /* We need the barrier to ensure the ordering between the
2321 * writing to bp->port.pmf here and reading it from the
2322 * bnx2x_periodic_task().
2323 */
2324 smp_mb();
2325 } else {
2326 bp->port.pmf = 0;
452427b0
YM		 2327 }
2328
ad5afc89
AE		 2329 DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
2330}
2331
2332static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code)
2333{
2334 if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2335 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2336 (bp->common.shmem2_base)) {
2337 if (SHMEM2_HAS(bp, dcc_support))
2338 SHMEM2_WR(bp, dcc_support,
2339 (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2340 SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2341 if (SHMEM2_HAS(bp, afex_driver_support))
2342 SHMEM2_WR(bp, afex_driver_support,
2343 SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2344 }
2345
2346 /* Set AFEX default VLAN tag to an invalid value */
2347 bp->afex_def_vlan_tag = -1;
452427b0
YM		 2348}
2349
1191cb83
ED		 2350/**
2351 * bnx2x_bz_fp - zero content of the fastpath structure.
2352 *
2353 * @bp: driver handle
2354 * @index: fastpath index to be zeroed
2355 *
2356 * Makes sure the contents of the bp->fp[index].napi is kept
2357 * intact.
2358 */
2359static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2360{
2361 struct bnx2x_fastpath *fp = &bp->fp[index];
65565884 2362 int cos;
1191cb83 2363 struct napi_struct orig_napi = fp->napi;
15192a8c 2364 struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
d76a6111 2365
1191cb83 2366 /* bzero bnx2x_fastpath contents */
c3146eb6
DK		 2367 if (fp->tpa_info)
2368 memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 *
2369 sizeof(struct bnx2x_agg_info));
2370 memset(fp, 0, sizeof(*fp));
1191cb83
ED		 2371
2372 /* Restore the NAPI object as it has been already initialized */
2373 fp->napi = orig_napi;
15192a8c 2374 fp->tpa_info = orig_tpa_info;
1191cb83
ED		 2375 fp->bp = bp;
2376 fp->index = index;
2377 if (IS_ETH_FP(fp))
2378 fp->max_cos = bp->max_cos;
2379 else
2380 /* Special queues support only one CoS */
2381 fp->max_cos = 1;
2382
65565884 2383 /* Init txdata pointers */
65565884
MS		 2384 if (IS_FCOE_FP(fp))
2385 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
65565884
MS		 2386 if (IS_ETH_FP(fp))
2387 for_each_cos_in_tx_queue(fp, cos)
2388 fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2389 BNX2X_NUM_ETH_QUEUES(bp) + index];
2390
16a5fd92 2391 /* set the tpa flag for each queue. The tpa flag determines the queue
1191cb83
ED		 2392 * minimal size so it must be set prior to queue memory allocation
2393 */
2394 fp->disable_tpa = !(bp->flags & TPA_ENABLE_FLAG ||
2395 (bp->flags & GRO_ENABLE_FLAG &&
2396 bnx2x_mtu_allows_gro(bp->dev->mtu)));
2397 if (bp->flags & TPA_ENABLE_FLAG)
2398 fp->mode = TPA_MODE_LRO;
2399 else if (bp->flags & GRO_ENABLE_FLAG)
2400 fp->mode = TPA_MODE_GRO;
2401
1191cb83
ED		 2402 /* We don't want TPA on an FCoE L2 ring */
2403 if (IS_FCOE_FP(fp))
2404 fp->disable_tpa = 1;
55c11941
MS		 2405}
2406
2407int bnx2x_load_cnic(struct bnx2x *bp)
2408{
2409 int i, rc, port = BP_PORT(bp);
2410
2411 DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2412
2413 mutex_init(&bp->cnic_mutex);
2414
ad5afc89
AE		 2415 if (IS_PF(bp)) {
2416 rc = bnx2x_alloc_mem_cnic(bp);
2417 if (rc) {
2418 BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2419 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2420 }
55c11941
MS		 2421 }
2422
2423 rc = bnx2x_alloc_fp_mem_cnic(bp);
2424 if (rc) {
2425 BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2426 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2427 }
2428
2429 /* Update the number of queues with the cnic queues */
2430 rc = bnx2x_set_real_num_queues(bp, 1);
2431 if (rc) {
2432 BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2433 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2434 }
2435
2436 /* Add all CNIC NAPI objects */
2437 bnx2x_add_all_napi_cnic(bp);
2438 DP(NETIF_MSG_IFUP, "cnic napi added\n");
2439 bnx2x_napi_enable_cnic(bp);
2440
2441 rc = bnx2x_init_hw_func_cnic(bp);
2442 if (rc)
2443 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2444
2445 bnx2x_nic_init_cnic(bp);
2446
ad5afc89
AE		 2447 if (IS_PF(bp)) {
2448 /* Enable Timer scan */
2449 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2450
2451 /* setup cnic queues */
2452 for_each_cnic_queue(bp, i) {
2453 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2454 if (rc) {
2455 BNX2X_ERR("Queue setup failed\n");
2456 LOAD_ERROR_EXIT(bp, load_error_cnic2);
2457 }
55c11941
MS		 2458 }
2459 }
2460
2461 /* Initialize Rx filter. */
8b09be5f 2462 bnx2x_set_rx_mode_inner(bp);
55c11941
MS		 2463
2464 /* re-read iscsi info */
2465 bnx2x_get_iscsi_info(bp);
2466 bnx2x_setup_cnic_irq_info(bp);
2467 bnx2x_setup_cnic_info(bp);
2468 bp->cnic_loaded = true;
2469 if (bp->state == BNX2X_STATE_OPEN)
2470 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2471
55c11941
MS		 2472 DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2473
2474 return 0;
2475
2476#ifndef BNX2X_STOP_ON_ERROR
2477load_error_cnic2:
2478 /* Disable Timer scan */
2479 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2480
2481load_error_cnic1:
2482 bnx2x_napi_disable_cnic(bp);
2483 /* Update the number of queues without the cnic queues */
2484 rc = bnx2x_set_real_num_queues(bp, 0);
2485 if (rc)
2486 BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2487load_error_cnic0:
2488 BNX2X_ERR("CNIC-related load failed\n");
2489 bnx2x_free_fp_mem_cnic(bp);
2490 bnx2x_free_mem_cnic(bp);
2491 return rc;
2492#endif /* ! BNX2X_STOP_ON_ERROR */
1191cb83
ED		 2493}
2494
9f6c9258
DK		 2495/* must be called with rtnl_lock */
2496int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2497{
619c5cb6 2498 int port = BP_PORT(bp);
ad5afc89 2499 int i, rc = 0, load_code = 0;
9f6c9258 2500
55c11941
MS		 2501 DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2502 DP(NETIF_MSG_IFUP,
2503 "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2504
9f6c9258 2505#ifdef BNX2X_STOP_ON_ERROR
51c1a580
MS		 2506 if (unlikely(bp->panic)) {
2507 BNX2X_ERR("Can't load NIC when there is panic\n");
9f6c9258 2508 return -EPERM;
51c1a580 2509 }
9f6c9258
DK		 2510#endif
2511
2512 bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2513
16a5fd92 2514 /* zero the structure w/o any lock, before SP handler is initialized */
2ae17f66
VZ		 2515 memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2516 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2517 &bp->last_reported_link.link_report_flags);
2ae17f66 2518
ad5afc89
AE		 2519 if (IS_PF(bp))
2520 /* must be called before memory allocation and HW init */
2521 bnx2x_ilt_set_info(bp);
523224a3 2522
6383c0b3
AE		 2523 /*
2524 * Zero fastpath structures preserving invariants like napi, which are
2525 * allocated only once, fp index, max_cos, bp pointer.
65565884 2526 * Also set fp->disable_tpa and txdata_ptr.
b3b83c3f 2527 */
51c1a580 2528 DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
b3b83c3f
DK		 2529 for_each_queue(bp, i)
2530 bnx2x_bz_fp(bp, i);
55c11941
MS		 2531 memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2532 bp->num_cnic_queues) *
2533 sizeof(struct bnx2x_fp_txdata));
b3b83c3f 2534
55c11941 2535 bp->fcoe_init = false;
6383c0b3 2536
a8c94b91
VZ		 2537 /* Set the receive queues buffer size */
2538 bnx2x_set_rx_buf_size(bp);
2539
ad5afc89
AE		 2540 if (IS_PF(bp)) {
2541 rc = bnx2x_alloc_mem(bp);
2542 if (rc) {
2543 BNX2X_ERR("Unable to allocate bp memory\n");
2544 return rc;
2545 }
2546 }
2547
2548 /* Allocated memory for FW statistics */
2549 if (bnx2x_alloc_fw_stats_mem(bp))
2550 LOAD_ERROR_EXIT(bp, load_error0);
2551
2552 /* need to be done after alloc mem, since it's self adjusting to amount
2553 * of memory available for RSS queues
2554 */
2555 rc = bnx2x_alloc_fp_mem(bp);
2556 if (rc) {
2557 BNX2X_ERR("Unable to allocate memory for fps\n");
2558 LOAD_ERROR_EXIT(bp, load_error0);
2559 }
d6214d7a 2560
8d9ac297
AE		 2561 /* request pf to initialize status blocks */
2562 if (IS_VF(bp)) {
2563 rc = bnx2x_vfpf_init(bp);
2564 if (rc)
2565 LOAD_ERROR_EXIT(bp, load_error0);
2566 }
2567
b3b83c3f
DK		 2568 /* As long as bnx2x_alloc_mem() may possibly update
2569 * bp->num_queues, bnx2x_set_real_num_queues() should always
55c11941 2570 * come after it. At this stage cnic queues are not counted.
b3b83c3f 2571 */
55c11941 2572 rc = bnx2x_set_real_num_queues(bp, 0);
d6214d7a 2573 if (rc) {
ec6ba945 2574 BNX2X_ERR("Unable to set real_num_queues\n");
619c5cb6 2575 LOAD_ERROR_EXIT(bp, load_error0);
9f6c9258
DK		 2576 }
2577
6383c0b3 2578 /* configure multi cos mappings in kernel.
16a5fd92
YM		 2579 * this configuration may be overridden by a multi class queue
2580 * discipline or by a dcbx negotiation result.
6383c0b3
AE		 2581 */
2582 bnx2x_setup_tc(bp->dev, bp->max_cos);
2583
26614ba5
MS		 2584 /* Add all NAPI objects */
2585 bnx2x_add_all_napi(bp);
55c11941 2586 DP(NETIF_MSG_IFUP, "napi added\n");
9f6c9258
DK		 2587 bnx2x_napi_enable(bp);
2588
ad5afc89
AE		 2589 if (IS_PF(bp)) {
2590 /* set pf load just before approaching the MCP */
2591 bnx2x_set_pf_load(bp);
2592
2593 /* if mcp exists send load request and analyze response */
2594 if (!BP_NOMCP(bp)) {
2595 /* attempt to load pf */
2596 rc = bnx2x_nic_load_request(bp, &load_code);
2597 if (rc)
2598 LOAD_ERROR_EXIT(bp, load_error1);
2599
2600 /* what did mcp say? */
2601 rc = bnx2x_nic_load_analyze_req(bp, load_code);
2602 if (rc) {
2603 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
d1e2d966
AE		 2604 LOAD_ERROR_EXIT(bp, load_error2);
2605 }
ad5afc89
AE		 2606 } else {
2607 load_code = bnx2x_nic_load_no_mcp(bp, port);
d1e2d966 2608 }
9f6c9258 2609
ad5afc89
AE		 2610 /* mark pmf if applicable */
2611 bnx2x_nic_load_pmf(bp, load_code);
9f6c9258 2612
ad5afc89
AE		 2613 /* Init Function state controlling object */
2614 bnx2x__init_func_obj(bp);
6383c0b3 2615
ad5afc89
AE		 2616 /* Initialize HW */
2617 rc = bnx2x_init_hw(bp, load_code);
2618 if (rc) {
2619 BNX2X_ERR("HW init failed, aborting\n");
2620 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2621 LOAD_ERROR_EXIT(bp, load_error2);
2622 }
9f6c9258
DK		 2623 }
2624
ecf01c22
YM		 2625 bnx2x_pre_irq_nic_init(bp);
2626
d6214d7a
DK		 2627 /* Connect to IRQs */
2628 rc = bnx2x_setup_irqs(bp);
523224a3 2629 if (rc) {
ad5afc89
AE		 2630 BNX2X_ERR("setup irqs failed\n");
2631 if (IS_PF(bp))
2632 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
619c5cb6 2633 LOAD_ERROR_EXIT(bp, load_error2);
523224a3
DK		 2634 }
2635
619c5cb6 2636 /* Init per-function objects */
ad5afc89 2637 if (IS_PF(bp)) {
ecf01c22
YM		 2638 /* Setup NIC internals and enable interrupts */
2639 bnx2x_post_irq_nic_init(bp, load_code);
2640
ad5afc89 2641 bnx2x_init_bp_objs(bp);
b56e9670 2642 bnx2x_iov_nic_init(bp);
a3348722 2643
ad5afc89
AE		 2644 /* Set AFEX default VLAN tag to an invalid value */
2645 bp->afex_def_vlan_tag = -1;
2646 bnx2x_nic_load_afex_dcc(bp, load_code);
2647 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2648 rc = bnx2x_func_start(bp);
2649 if (rc) {
2650 BNX2X_ERR("Function start failed!\n");
2651 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
9f6c9258 2652
619c5cb6 2653 LOAD_ERROR_EXIT(bp, load_error3);
9f6c9258 2654 }
9f6c9258 2655
ad5afc89
AE		 2656 /* Send LOAD_DONE command to MCP */
2657 if (!BP_NOMCP(bp)) {
2658 load_code = bnx2x_fw_command(bp,
2659 DRV_MSG_CODE_LOAD_DONE, 0);
2660 if (!load_code) {
2661 BNX2X_ERR("MCP response failure, aborting\n");
2662 rc = -EBUSY;
2663 LOAD_ERROR_EXIT(bp, load_error3);
2664 }
2665 }
9f6c9258 2666
0c14e5ce
AE		 2667 /* initialize FW coalescing state machines in RAM */
2668 bnx2x_update_coalesce(bp);
60cad4e6 2669 }
0c14e5ce 2670
60cad4e6
AE		 2671 /* setup the leading queue */
2672 rc = bnx2x_setup_leading(bp);
2673 if (rc) {
2674 BNX2X_ERR("Setup leading failed!\n");
2675 LOAD_ERROR_EXIT(bp, load_error3);
2676 }
ad5afc89 2677
60cad4e6
AE		 2678 /* set up the rest of the queues */
2679 for_each_nondefault_eth_queue(bp, i) {
2680 if (IS_PF(bp))
2681 rc = bnx2x_setup_queue(bp, &bp->fp[i], false);
2682 else /* VF */
2683 rc = bnx2x_vfpf_setup_q(bp, &bp->fp[i], false);
ad5afc89 2684 if (rc) {
60cad4e6 2685 BNX2X_ERR("Queue %d setup failed\n", i);
ad5afc89
AE		 2686 LOAD_ERROR_EXIT(bp, load_error3);
2687 }
60cad4e6 2688 }
8d9ac297 2689
60cad4e6
AE		 2690 /* setup rss */
2691 rc = bnx2x_init_rss(bp);
2692 if (rc) {
2693 BNX2X_ERR("PF RSS init failed\n");
2694 LOAD_ERROR_EXIT(bp, load_error3);
51c1a580 2695 }
619c5cb6 2696
523224a3
DK		 2697 /* Now when Clients are configured we are ready to work */
2698 bp->state = BNX2X_STATE_OPEN;
2699
619c5cb6 2700 /* Configure a ucast MAC */
ad5afc89
AE		 2701 if (IS_PF(bp))
2702 rc = bnx2x_set_eth_mac(bp, true);
8d9ac297 2703 else /* vf */
f8f4f61a
DK		 2704 rc = bnx2x_vfpf_config_mac(bp, bp->dev->dev_addr, bp->fp->index,
2705 true);
51c1a580
MS		 2706 if (rc) {
2707 BNX2X_ERR("Setting Ethernet MAC failed\n");
55c11941 2708 LOAD_ERROR_EXIT(bp, load_error3);
51c1a580 2709 }
6e30dd4e 2710
ad5afc89 2711 if (IS_PF(bp) && bp->pending_max) {
e3835b99
DK		 2712 bnx2x_update_max_mf_config(bp, bp->pending_max);
2713 bp->pending_max = 0;
2714 }
2715
ad5afc89
AE		 2716 if (bp->port.pmf) {
2717 rc = bnx2x_initial_phy_init(bp, load_mode);
2718 if (rc)
2719 LOAD_ERROR_EXIT(bp, load_error3);
2720 }
c63da990 2721 bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
9f6c9258 2722
619c5cb6
VZ		 2723 /* Start fast path */
2724
2725 /* Initialize Rx filter. */
8b09be5f 2726 bnx2x_set_rx_mode_inner(bp);
6e30dd4e 2727
619c5cb6 2728 /* Start the Tx */
9f6c9258
DK		 2729 switch (load_mode) {
2730 case LOAD_NORMAL:
16a5fd92 2731 /* Tx queue should be only re-enabled */
523224a3 2732 netif_tx_wake_all_queues(bp->dev);
9f6c9258
DK		 2733 break;
2734
2735 case LOAD_OPEN:
2736 netif_tx_start_all_queues(bp->dev);
523224a3 2737 smp_mb__after_clear_bit();
9f6c9258
DK		 2738 break;
2739
2740 case LOAD_DIAG:
8970b2e4 2741 case LOAD_LOOPBACK_EXT:
9f6c9258
DK		 2742 bp->state = BNX2X_STATE_DIAG;
2743 break;
2744
2745 default:
2746 break;
2747 }
2748
00253a8c 2749 if (bp->port.pmf)
4c704899 2750 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
00253a8c 2751 else
9f6c9258
DK		 2752 bnx2x__link_status_update(bp);
2753
2754 /* start the timer */
2755 mod_timer(&bp->timer, jiffies + bp->current_interval);
2756
55c11941
MS		 2757 if (CNIC_ENABLED(bp))
2758 bnx2x_load_cnic(bp);
9f6c9258 2759
ad5afc89
AE		 2760 if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2761 /* mark driver is loaded in shmem2 */
9ce392d4
YM		 2762 u32 val;
2763 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2764 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2765 val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2766 DRV_FLAGS_CAPABILITIES_LOADED_L2);
2767 }
2768
619c5cb6 2769 /* Wait for all pending SP commands to complete */
ad5afc89 2770 if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) {
619c5cb6 2771 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
5d07d868 2772 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
619c5cb6
VZ		 2773 return -EBUSY;
2774 }
6891dd25 2775
9876879f
BW		 2776 /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2777 if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2778 bnx2x_dcbx_init(bp, false);
2779
55c11941
MS		 2780 DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2781
9f6c9258
DK		 2782 return 0;
2783
619c5cb6 2784#ifndef BNX2X_STOP_ON_ERROR
9f6c9258 2785load_error3:
ad5afc89
AE		 2786 if (IS_PF(bp)) {
2787 bnx2x_int_disable_sync(bp, 1);
d6214d7a 2788
ad5afc89
AE		 2789 /* Clean queueable objects */
2790 bnx2x_squeeze_objects(bp);
2791 }
619c5cb6 2792
9f6c9258
DK		 2793 /* Free SKBs, SGEs, TPA pool and driver internals */
2794 bnx2x_free_skbs(bp);
ec6ba945 2795 for_each_rx_queue(bp, i)
9f6c9258 2796 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
d6214d7a 2797
9f6c9258 2798 /* Release IRQs */
d6214d7a
DK		 2799 bnx2x_free_irq(bp);
2800load_error2:
ad5afc89 2801 if (IS_PF(bp) && !BP_NOMCP(bp)) {
d6214d7a
DK		 2802 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2803 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2804 }
2805
2806 bp->port.pmf = 0;
9f6c9258
DK		 2807load_error1:
2808 bnx2x_napi_disable(bp);
722c6f58 2809 bnx2x_del_all_napi(bp);
ad5afc89 2810
889b9af3 2811 /* clear pf_load status, as it was already set */
ad5afc89
AE		 2812 if (IS_PF(bp))
2813 bnx2x_clear_pf_load(bp);
d6214d7a 2814load_error0:
ad5afc89
AE		 2815 bnx2x_free_fp_mem(bp);
2816 bnx2x_free_fw_stats_mem(bp);
9f6c9258
DK		 2817 bnx2x_free_mem(bp);
2818
2819 return rc;
619c5cb6 2820#endif /* ! BNX2X_STOP_ON_ERROR */
9f6c9258
DK		 2821}
2822
7fa6f340 2823int bnx2x_drain_tx_queues(struct bnx2x *bp)
ad5afc89
AE		 2824{
2825 u8 rc = 0, cos, i;
2826
2827 /* Wait until tx fastpath tasks complete */
2828 for_each_tx_queue(bp, i) {
2829 struct bnx2x_fastpath *fp = &bp->fp[i];
2830
2831 for_each_cos_in_tx_queue(fp, cos)
2832 rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]);
2833 if (rc)
2834 return rc;
2835 }
2836 return 0;
2837}
2838
9f6c9258 2839/* must be called with rtnl_lock */
5d07d868 2840int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
9f6c9258
DK		 2841{
2842 int i;
c9ee9206
VZ		 2843 bool global = false;
2844
55c11941
MS		 2845 DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2846
9ce392d4 2847 /* mark driver is unloaded in shmem2 */
ad5afc89 2848 if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
9ce392d4
YM		 2849 u32 val;
2850 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2851 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2852 val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2853 }
2854
80bfe5cc 2855 if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE &&
ad5afc89
AE		 2856 (bp->state == BNX2X_STATE_CLOSED ||
2857 bp->state == BNX2X_STATE_ERROR)) {
c9ee9206
VZ		 2858 /* We can get here if the driver has been unloaded
2859 * during parity error recovery and is either waiting for a
2860 * leader to complete or for other functions to unload and
2861 * then ifdown has been issued. In this case we want to
2862 * unload and let other functions to complete a recovery
2863 * process.
2864 */
9f6c9258
DK		 2865 bp->recovery_state = BNX2X_RECOVERY_DONE;
2866 bp->is_leader = 0;
c9ee9206
VZ		 2867 bnx2x_release_leader_lock(bp);
2868 smp_mb();
2869
51c1a580
MS		 2870 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2871 BNX2X_ERR("Can't unload in closed or error state\n");
9f6c9258
DK		 2872 return -EINVAL;
2873 }
2874
80bfe5cc 2875 /* Nothing to do during unload if previous bnx2x_nic_load()
16a5fd92 2876 * have not completed successfully - all resources are released.
80bfe5cc
YM		 2877 *
2878 * we can get here only after unsuccessful ndo_* callback, during which
2879 * dev->IFF_UP flag is still on.
2880 */
2881 if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR)
2882 return 0;
2883
2884 /* It's important to set the bp->state to the value different from
87b7ba3d
VZ		 2885 * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2886 * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2887 */
2888 bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2889 smp_mb();
2890
78c3bcc5
AE		 2891 /* indicate to VFs that the PF is going down */
2892 bnx2x_iov_channel_down(bp);
2893
55c11941
MS		 2894 if (CNIC_LOADED(bp))
2895 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2896
9505ee37
VZ		 2897 /* Stop Tx */
2898 bnx2x_tx_disable(bp);
65565884 2899 netdev_reset_tc(bp->dev);
9505ee37 2900
9f6c9258 2901 bp->rx_mode = BNX2X_RX_MODE_NONE;
9f6c9258 2902
9f6c9258 2903 del_timer_sync(&bp->timer);
f85582f8 2904
ad5afc89
AE		 2905 if (IS_PF(bp)) {
2906 /* Set ALWAYS_ALIVE bit in shmem */
2907 bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2908 bnx2x_drv_pulse(bp);
2909 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2910 bnx2x_save_statistics(bp);
2911 }
9f6c9258 2912
ad5afc89
AE		 2913 /* wait till consumers catch up with producers in all queues */
2914 bnx2x_drain_tx_queues(bp);
9f6c9258 2915
9b176b6b
AE		 2916 /* if VF indicate to PF this function is going down (PF will delete sp
2917 * elements and clear initializations
2918 */
2919 if (IS_VF(bp))
2920 bnx2x_vfpf_close_vf(bp);
2921 else if (unload_mode != UNLOAD_RECOVERY)
2922 /* if this is a normal/close unload need to clean up chip*/
5d07d868 2923 bnx2x_chip_cleanup(bp, unload_mode, keep_link);
523224a3 2924 else {
c9ee9206
VZ		 2925 /* Send the UNLOAD_REQUEST to the MCP */
2926 bnx2x_send_unload_req(bp, unload_mode);
2927
16a5fd92 2928 /* Prevent transactions to host from the functions on the
c9ee9206 2929 * engine that doesn't reset global blocks in case of global
16a5fd92 2930 * attention once global blocks are reset and gates are opened
c9ee9206
VZ		 2931 * (the engine which leader will perform the recovery
2932 * last).
2933 */
2934 if (!CHIP_IS_E1x(bp))
2935 bnx2x_pf_disable(bp);
2936
2937 /* Disable HW interrupts, NAPI */
523224a3 2938 bnx2x_netif_stop(bp, 1);
26614ba5
MS		 2939 /* Delete all NAPI objects */
2940 bnx2x_del_all_napi(bp);
55c11941
MS		 2941 if (CNIC_LOADED(bp))
2942 bnx2x_del_all_napi_cnic(bp);
523224a3 2943 /* Release IRQs */
d6214d7a 2944 bnx2x_free_irq(bp);
c9ee9206
VZ		 2945
2946 /* Report UNLOAD_DONE to MCP */
5d07d868 2947 bnx2x_send_unload_done(bp, false);
523224a3 2948 }
9f6c9258 2949
619c5cb6 2950 /*
16a5fd92 2951 * At this stage no more interrupts will arrive so we may safely clean
619c5cb6
VZ		 2952 * the queueable objects here in case they failed to get cleaned so far.
2953 */
ad5afc89
AE		 2954 if (IS_PF(bp))
2955 bnx2x_squeeze_objects(bp);
619c5cb6 2956
79616895
VZ		 2957 /* There should be no more pending SP commands at this stage */
2958 bp->sp_state = 0;
2959
9f6c9258
DK		 2960 bp->port.pmf = 0;
2961
2962 /* Free SKBs, SGEs, TPA pool and driver internals */
2963 bnx2x_free_skbs(bp);
55c11941
MS		 2964 if (CNIC_LOADED(bp))
2965 bnx2x_free_skbs_cnic(bp);
ec6ba945 2966 for_each_rx_queue(bp, i)
9f6c9258 2967 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
d6214d7a 2968
ad5afc89
AE		 2969 bnx2x_free_fp_mem(bp);
2970 if (CNIC_LOADED(bp))
55c11941 2971 bnx2x_free_fp_mem_cnic(bp);
9f6c9258 2972
ad5afc89 2973 if (IS_PF(bp)) {
ad5afc89
AE		 2974 if (CNIC_LOADED(bp))
2975 bnx2x_free_mem_cnic(bp);
2976 }
b4cddbd6
AE		 2977 bnx2x_free_mem(bp);
2978
9f6c9258 2979 bp->state = BNX2X_STATE_CLOSED;
55c11941 2980 bp->cnic_loaded = false;
9f6c9258 2981
c9ee9206
VZ		 2982 /* Check if there are pending parity attentions. If there are - set
2983 * RECOVERY_IN_PROGRESS.
2984 */
ad5afc89 2985 if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) {
c9ee9206
VZ		 2986 bnx2x_set_reset_in_progress(bp);
2987
2988 /* Set RESET_IS_GLOBAL if needed */
2989 if (global)
2990 bnx2x_set_reset_global(bp);
2991 }
2992
9f6c9258
DK		 2993 /* The last driver must disable a "close the gate" if there is no
2994 * parity attention or "process kill" pending.
2995 */
ad5afc89
AE		 2996 if (IS_PF(bp) &&
2997 !bnx2x_clear_pf_load(bp) &&
2998 bnx2x_reset_is_done(bp, BP_PATH(bp)))
9f6c9258
DK		 2999 bnx2x_disable_close_the_gate(bp);
3000
55c11941
MS		 3001 DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
3002
9f6c9258
DK		 3003 return 0;
3004}
f85582f8 3005
9f6c9258
DK		 3006int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
3007{
3008 u16 pmcsr;
3009
adf5f6a1 3010 /* If there is no power capability, silently succeed */
29ed74c3 3011 if (!bp->pdev->pm_cap) {
51c1a580 3012 BNX2X_DEV_INFO("No power capability. Breaking.\n");
adf5f6a1
DK		 3013 return 0;
3014 }
3015
29ed74c3 3016 pci_read_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL, &pmcsr);
9f6c9258
DK		 3017
3018 switch (state) {
3019 case PCI_D0:
29ed74c3 3020 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
9f6c9258
DK		 3021 ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
3022 PCI_PM_CTRL_PME_STATUS));
3023
3024 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
3025 /* delay required during transition out of D3hot */
3026 msleep(20);
3027 break;
3028
3029 case PCI_D3hot:
3030 /* If there are other clients above don't
3031 shut down the power */
3032 if (atomic_read(&bp->pdev->enable_cnt) != 1)
3033 return 0;
3034 /* Don't shut down the power for emulation and FPGA */
3035 if (CHIP_REV_IS_SLOW(bp))
3036 return 0;
3037
3038 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
3039 pmcsr |= 3;
3040
3041 if (bp->wol)
3042 pmcsr |= PCI_PM_CTRL_PME_ENABLE;
3043
29ed74c3 3044 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
9f6c9258
DK		 3045 pmcsr);
3046
3047 /* No more memory access after this point until
3048 * device is brought back to D0.
3049 */
3050 break;
3051
3052 default:
51c1a580 3053 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
9f6c9258
DK		 3054 return -EINVAL;
3055 }
3056 return 0;
3057}
3058
9f6c9258
DK		 3059/*
3060 * net_device service functions
3061 */
d6214d7a 3062int bnx2x_poll(struct napi_struct *napi, int budget)
9f6c9258
DK		 3063{
3064 int work_done = 0;
6383c0b3 3065 u8 cos;
9f6c9258
DK		 3066 struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3067 napi);
3068 struct bnx2x *bp = fp->bp;
3069
3070 while (1) {
3071#ifdef BNX2X_STOP_ON_ERROR
3072 if (unlikely(bp->panic)) {
3073 napi_complete(napi);
3074 return 0;
3075 }
3076#endif
8f20aa57
DK		 3077 if (!bnx2x_fp_lock_napi(fp))
3078 return work_done;
9f6c9258 3079
6383c0b3 3080 for_each_cos_in_tx_queue(fp, cos)
65565884
MS		 3081 if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
3082 bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
6383c0b3 3083
9f6c9258
DK		 3084 if (bnx2x_has_rx_work(fp)) {
3085 work_done += bnx2x_rx_int(fp, budget - work_done);
3086
3087 /* must not complete if we consumed full budget */
8f20aa57
DK		 3088 if (work_done >= budget) {
3089 bnx2x_fp_unlock_napi(fp);
9f6c9258 3090 break;
8f20aa57 3091 }
9f6c9258
DK		 3092 }
3093
3094 /* Fall out from the NAPI loop if needed */
8f20aa57
DK		 3095 if (!bnx2x_fp_unlock_napi(fp) &&
3096 !(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
55c11941 3097
ec6ba945
VZ		 3098 /* No need to update SB for FCoE L2 ring as long as
3099 * it's connected to the default SB and the SB
3100 * has been updated when NAPI was scheduled.
3101 */
3102 if (IS_FCOE_FP(fp)) {
3103 napi_complete(napi);
3104 break;
3105 }
9f6c9258 3106 bnx2x_update_fpsb_idx(fp);
f85582f8
DK		 3107 /* bnx2x_has_rx_work() reads the status block,
3108 * thus we need to ensure that status block indices
3109 * have been actually read (bnx2x_update_fpsb_idx)
3110 * prior to this check (bnx2x_has_rx_work) so that
3111 * we won't write the "newer" value of the status block
3112 * to IGU (if there was a DMA right after
3113 * bnx2x_has_rx_work and if there is no rmb, the memory
3114 * reading (bnx2x_update_fpsb_idx) may be postponed
3115 * to right before bnx2x_ack_sb). In this case there
3116 * will never be another interrupt until there is
3117 * another update of the status block, while there
3118 * is still unhandled work.
3119 */
9f6c9258
DK		 3120 rmb();
3121
3122 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3123 napi_complete(napi);
3124 /* Re-enable interrupts */
51c1a580 3125 DP(NETIF_MSG_RX_STATUS,
523224a3
DK		 3126 "Update index to %d\n", fp->fp_hc_idx);
3127 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
3128 le16_to_cpu(fp->fp_hc_idx),
9f6c9258
DK		 3129 IGU_INT_ENABLE, 1);
3130 break;
3131 }
3132 }
3133 }
3134
3135 return work_done;
3136}
3137
e0d1095a 3138#ifdef CONFIG_NET_RX_BUSY_POLL
8f20aa57
DK		 3139/* must be called with local_bh_disable()d */
3140int bnx2x_low_latency_recv(struct napi_struct *napi)
3141{
3142 struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3143 napi);
3144 struct bnx2x *bp = fp->bp;
3145 int found = 0;
3146
3147 if ((bp->state == BNX2X_STATE_CLOSED) ||
3148 (bp->state == BNX2X_STATE_ERROR) ||
3149 (bp->flags & (TPA_ENABLE_FLAG | GRO_ENABLE_FLAG)))
3150 return LL_FLUSH_FAILED;
3151
3152 if (!bnx2x_fp_lock_poll(fp))
3153 return LL_FLUSH_BUSY;
3154
75b29459 3155 if (bnx2x_has_rx_work(fp))
8f20aa57 3156 found = bnx2x_rx_int(fp, 4);
8f20aa57
DK		 3157
3158 bnx2x_fp_unlock_poll(fp);
3159
3160 return found;
3161}
3162#endif
3163
9f6c9258
DK		 3164/* we split the first BD into headers and data BDs
3165 * to ease the pain of our fellow microcode engineers
3166 * we use one mapping for both BDs
9f6c9258 3167 */
91226790
DK		 3168static u16 bnx2x_tx_split(struct bnx2x *bp,
3169 struct bnx2x_fp_txdata *txdata,
3170 struct sw_tx_bd *tx_buf,
3171 struct eth_tx_start_bd **tx_bd, u16 hlen,
3172 u16 bd_prod)
9f6c9258
DK		 3173{
3174 struct eth_tx_start_bd *h_tx_bd = *tx_bd;
3175 struct eth_tx_bd *d_tx_bd;
3176 dma_addr_t mapping;
3177 int old_len = le16_to_cpu(h_tx_bd->nbytes);
3178
3179 /* first fix first BD */
9f6c9258
DK		 3180 h_tx_bd->nbytes = cpu_to_le16(hlen);
3181
91226790
DK		 3182 DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x)\n",
3183 h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo);
9f6c9258
DK		 3184
3185 /* now get a new data BD
3186 * (after the pbd) and fill it */
3187 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
6383c0b3 3188 d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
9f6c9258
DK		 3189
3190 mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
3191 le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
3192
3193 d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3194 d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3195 d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
3196
3197 /* this marks the BD as one that has no individual mapping */
3198 tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
3199
3200 DP(NETIF_MSG_TX_QUEUED,
3201 "TSO split data size is %d (%x:%x)\n",
3202 d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
3203
3204 /* update tx_bd */
3205 *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
3206
3207 return bd_prod;
3208}
3209
86564c3f
YM		 3210#define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32)))
3211#define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16)))
91226790 3212static __le16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
9f6c9258 3213{
86564c3f
YM		 3214 __sum16 tsum = (__force __sum16) csum;
3215
9f6c9258 3216 if (fix > 0)
86564c3f
YM		 3217 tsum = ~csum_fold(csum_sub((__force __wsum) csum,
3218 csum_partial(t_header - fix, fix, 0)));
9f6c9258
DK		 3219
3220 else if (fix < 0)
86564c3f
YM		 3221 tsum = ~csum_fold(csum_add((__force __wsum) csum,
3222 csum_partial(t_header, -fix, 0)));
9f6c9258 3223
e2593fcd 3224 return bswab16(tsum);
9f6c9258
DK		 3225}
3226
91226790 3227static u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
9f6c9258
DK		 3228{
3229 u32 rc;
a848ade4
DK		 3230 __u8 prot = 0;
3231 __be16 protocol;
9f6c9258
DK		 3232
3233 if (skb->ip_summed != CHECKSUM_PARTIAL)
a848ade4 3234 return XMIT_PLAIN;
9f6c9258 3235
a848ade4
DK		 3236 protocol = vlan_get_protocol(skb);
3237 if (protocol == htons(ETH_P_IPV6)) {
3238 rc = XMIT_CSUM_V6;
3239 prot = ipv6_hdr(skb)->nexthdr;
3240 } else {
3241 rc = XMIT_CSUM_V4;
3242 prot = ip_hdr(skb)->protocol;
3243 }
9f6c9258 3244
a848ade4
DK		 3245 if (!CHIP_IS_E1x(bp) && skb->encapsulation) {
3246 if (inner_ip_hdr(skb)->version == 6) {
3247 rc |= XMIT_CSUM_ENC_V6;
3248 if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3249 rc |= XMIT_CSUM_TCP;
9f6c9258 3250 } else {
a848ade4
DK		 3251 rc |= XMIT_CSUM_ENC_V4;
3252 if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP)
9f6c9258
DK		 3253 rc |= XMIT_CSUM_TCP;
3254 }
3255 }
a848ade4
DK		 3256 if (prot == IPPROTO_TCP)
3257 rc |= XMIT_CSUM_TCP;
9f6c9258 3258
a848ade4 3259 if (skb_is_gso_v6(skb)) {
e768fb29 3260 rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP);
a848ade4
DK		 3261 if (rc & XMIT_CSUM_ENC)
3262 rc |= XMIT_GSO_ENC_V6;
3263 } else if (skb_is_gso(skb)) {
e768fb29 3264 rc |= (XMIT_GSO_V4 | XMIT_CSUM_TCP);
a848ade4
DK		 3265 if (rc & XMIT_CSUM_ENC)
3266 rc |= XMIT_GSO_ENC_V4;
3267 }
9f6c9258
DK		 3268
3269 return rc;
3270}
3271
3272#if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
3273/* check if packet requires linearization (packet is too fragmented)
3274 no need to check fragmentation if page size > 8K (there will be no
3275 violation to FW restrictions) */
3276static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
3277 u32 xmit_type)
3278{
3279 int to_copy = 0;
3280 int hlen = 0;
3281 int first_bd_sz = 0;
3282
3283 /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
3284 if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
3285
3286 if (xmit_type & XMIT_GSO) {
3287 unsigned short lso_mss = skb_shinfo(skb)->gso_size;
3288 /* Check if LSO packet needs to be copied:
3289 3 = 1 (for headers BD) + 2 (for PBD and last BD) */
3290 int wnd_size = MAX_FETCH_BD - 3;
3291 /* Number of windows to check */
3292 int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
3293 int wnd_idx = 0;
3294 int frag_idx = 0;
3295 u32 wnd_sum = 0;
3296
3297 /* Headers length */
3298 hlen = (int)(skb_transport_header(skb) - skb->data) +
3299 tcp_hdrlen(skb);
3300
3301 /* Amount of data (w/o headers) on linear part of SKB*/
3302 first_bd_sz = skb_headlen(skb) - hlen;
3303
3304 wnd_sum = first_bd_sz;
3305
3306 /* Calculate the first sum - it's special */
3307 for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
3308 wnd_sum +=
9e903e08 3309 skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
9f6c9258
DK		 3310
3311 /* If there was data on linear skb data - check it */
3312 if (first_bd_sz > 0) {
3313 if (unlikely(wnd_sum < lso_mss)) {
3314 to_copy = 1;
3315 goto exit_lbl;
3316 }
3317
3318 wnd_sum -= first_bd_sz;
3319 }
3320
3321 /* Others are easier: run through the frag list and
3322 check all windows */
3323 for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
3324 wnd_sum +=
9e903e08 3325 skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
9f6c9258
DK		 3326
3327 if (unlikely(wnd_sum < lso_mss)) {
3328 to_copy = 1;
3329 break;
3330 }
3331 wnd_sum -=
9e903e08 3332 skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
9f6c9258
DK		 3333 }
3334 } else {
3335 /* in non-LSO too fragmented packet should always
3336 be linearized */
3337 to_copy = 1;
3338 }
3339 }
3340
3341exit_lbl:
3342 if (unlikely(to_copy))
3343 DP(NETIF_MSG_TX_QUEUED,
51c1a580 3344 "Linearization IS REQUIRED for %s packet. num_frags %d hlen %d first_bd_sz %d\n",
9f6c9258
DK		 3345 (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
3346 skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
3347
3348 return to_copy;
3349}
3350#endif
3351
91226790
DK		 3352static void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
3353 u32 xmit_type)
f2e0899f 3354{
a848ade4
DK		 3355 struct ipv6hdr *ipv6;
3356
2297a2da
VZ		 3357 *parsing_data |= (skb_shinfo(skb)->gso_size <<
3358 ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
3359 ETH_TX_PARSE_BD_E2_LSO_MSS;
a848ade4
DK		 3360
3361 if (xmit_type & XMIT_GSO_ENC_V6)
3362 ipv6 = inner_ipv6_hdr(skb);
3363 else if (xmit_type & XMIT_GSO_V6)
3364 ipv6 = ipv6_hdr(skb);
3365 else
3366 ipv6 = NULL;
3367
3368 if (ipv6 && ipv6->nexthdr == NEXTHDR_IPV6)
2297a2da 3369 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
f2e0899f
DK		 3370}
3371
3372/**
e8920674 3373 * bnx2x_set_pbd_gso - update PBD in GSO case.
f2e0899f 3374 *
e8920674
DK		 3375 * @skb: packet skb
3376 * @pbd: parse BD
3377 * @xmit_type: xmit flags
f2e0899f 3378 */
91226790
DK		 3379static void bnx2x_set_pbd_gso(struct sk_buff *skb,
3380 struct eth_tx_parse_bd_e1x *pbd,
057cf65e 3381 struct eth_tx_start_bd *tx_start_bd,
91226790 3382 u32 xmit_type)
f2e0899f
DK		 3383{
3384 pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
86564c3f 3385 pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq);
91226790 3386 pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb));
f2e0899f
DK		 3387
3388 if (xmit_type & XMIT_GSO_V4) {
86564c3f 3389 pbd->ip_id = bswab16(ip_hdr(skb)->id);
f2e0899f 3390 pbd->tcp_pseudo_csum =
86564c3f
YM		 3391 bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3392 ip_hdr(skb)->daddr,
3393 0, IPPROTO_TCP, 0));
f2e0899f 3394
057cf65e
YM		 3395 /* GSO on 57710/57711 needs FW to calculate IP checksum */
3396 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IP_CSUM;
3397 } else {
f2e0899f 3398 pbd->tcp_pseudo_csum =
86564c3f
YM		 3399 bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3400 &ipv6_hdr(skb)->daddr,
3401 0, IPPROTO_TCP, 0));
057cf65e 3402 }
f2e0899f 3403
86564c3f
YM		 3404 pbd->global_data |=
3405 cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN);
f2e0899f 3406}
f85582f8 3407
a848ade4
DK		 3408/**
3409 * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length
3410 *
3411 * @bp: driver handle
3412 * @skb: packet skb
3413 * @parsing_data: data to be updated
3414 * @xmit_type: xmit flags
3415 *
3416 * 57712/578xx related, when skb has encapsulation
3417 */
3418static u8 bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb,
3419 u32 *parsing_data, u32 xmit_type)
3420{
3421 *parsing_data |=
3422 ((((u8 *)skb_inner_transport_header(skb) - skb->data) >> 1) <<
3423 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3424 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3425
3426 if (xmit_type & XMIT_CSUM_TCP) {
3427 *parsing_data |= ((inner_tcp_hdrlen(skb) / 4) <<
3428 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3429 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3430
3431 return skb_inner_transport_header(skb) +
3432 inner_tcp_hdrlen(skb) - skb->data;
3433 }
3434
3435 /* We support checksum offload for TCP and UDP only.
3436 * No need to pass the UDP header length - it's a constant.
3437 */
3438 return skb_inner_transport_header(skb) +
3439 sizeof(struct udphdr) - skb->data;
3440}
3441
f2e0899f 3442/**
e8920674 3443 * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
f2e0899f 3444 *
e8920674
DK		 3445 * @bp: driver handle
3446 * @skb: packet skb
3447 * @parsing_data: data to be updated
3448 * @xmit_type: xmit flags
f2e0899f 3449 *
91226790 3450 * 57712/578xx related
f2e0899f 3451 */
91226790
DK		 3452static u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3453 u32 *parsing_data, u32 xmit_type)
f2e0899f 3454{
e39aece7 3455 *parsing_data |=
2de67439 3456 ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
91226790
DK		 3457 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3458 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
f2e0899f 3459
e39aece7
VZ		 3460 if (xmit_type & XMIT_CSUM_TCP) {
3461 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3462 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3463 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
f2e0899f 3464
e39aece7 3465 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
924d75ab
YM		 3466 }
3467 /* We support checksum offload for TCP and UDP only.
3468 * No need to pass the UDP header length - it's a constant.
3469 */
3470 return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data;
f2e0899f
DK		 3471}
3472
a848ade4 3473/* set FW indication according to inner or outer protocols if tunneled */
91226790
DK		 3474static void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3475 struct eth_tx_start_bd *tx_start_bd,
3476 u32 xmit_type)
93ef5c02 3477{
93ef5c02
DK		 3478 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3479
a848ade4 3480 if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6))
91226790 3481 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6;
93ef5c02
DK		 3482
3483 if (!(xmit_type & XMIT_CSUM_TCP))
3484 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
93ef5c02
DK		 3485}
3486
f2e0899f 3487/**
e8920674 3488 * bnx2x_set_pbd_csum - update PBD with checksum and return header length
f2e0899f 3489 *
e8920674
DK		 3490 * @bp: driver handle
3491 * @skb: packet skb
3492 * @pbd: parse BD to be updated
3493 * @xmit_type: xmit flags
f2e0899f 3494 */
91226790
DK		 3495static u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3496 struct eth_tx_parse_bd_e1x *pbd,
3497 u32 xmit_type)
f2e0899f 3498{
e39aece7 3499 u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
f2e0899f
DK		 3500
3501 /* for now NS flag is not used in Linux */
3502 pbd->global_data =
86564c3f
YM		 3503 cpu_to_le16(hlen |
3504 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3505 ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
f2e0899f
DK		 3506
3507 pbd->ip_hlen_w = (skb_transport_header(skb) -
e39aece7 3508 skb_network_header(skb)) >> 1;
f2e0899f 3509
e39aece7
VZ		 3510 hlen += pbd->ip_hlen_w;
3511
3512 /* We support checksum offload for TCP and UDP only */
3513 if (xmit_type & XMIT_CSUM_TCP)
3514 hlen += tcp_hdrlen(skb) / 2;
3515 else
3516 hlen += sizeof(struct udphdr) / 2;
f2e0899f
DK		 3517
3518 pbd->total_hlen_w = cpu_to_le16(hlen);
3519 hlen = hlen*2;
3520
3521 if (xmit_type & XMIT_CSUM_TCP) {
86564c3f 3522 pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check);
f2e0899f
DK		 3523
3524 } else {
3525 s8 fix = SKB_CS_OFF(skb); /* signed! */
3526
3527 DP(NETIF_MSG_TX_QUEUED,
3528 "hlen %d fix %d csum before fix %x\n",
3529 le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
3530
3531 /* HW bug: fixup the CSUM */
3532 pbd->tcp_pseudo_csum =
3533 bnx2x_csum_fix(skb_transport_header(skb),
3534 SKB_CS(skb), fix);
3535
3536 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
3537 pbd->tcp_pseudo_csum);
3538 }
3539
3540 return hlen;
3541}
f85582f8 3542
a848ade4
DK		 3543static void bnx2x_update_pbds_gso_enc(struct sk_buff *skb,
3544 struct eth_tx_parse_bd_e2 *pbd_e2,
3545 struct eth_tx_parse_2nd_bd *pbd2,
3546 u16 *global_data,
3547 u32 xmit_type)
3548{
e287a75c 3549 u16 hlen_w = 0;
a848ade4 3550 u8 outerip_off, outerip_len = 0;
e768fb29 3551
e287a75c
DK		 3552 /* from outer IP to transport */
3553 hlen_w = (skb_inner_transport_header(skb) -
3554 skb_network_header(skb)) >> 1;
a848ade4
DK		 3555
3556 /* transport len */
e768fb29 3557 hlen_w += inner_tcp_hdrlen(skb) >> 1;
a848ade4 3558
e287a75c 3559 pbd2->fw_ip_hdr_to_payload_w = hlen_w;
a848ade4 3560
e768fb29
DK		 3561 /* outer IP header info */
3562 if (xmit_type & XMIT_CSUM_V4) {
e287a75c 3563 struct iphdr *iph = ip_hdr(skb);
1b4fc0e2
DK		 3564 u32 csum = (__force u32)(~iph->check) -
3565 (__force u32)iph->tot_len -
3566 (__force u32)iph->frag_off;
c957d09f 3567
a848ade4 3568 pbd2->fw_ip_csum_wo_len_flags_frag =
c957d09f 3569 bswab16(csum_fold((__force __wsum)csum));
a848ade4
DK		 3570 } else {
3571 pbd2->fw_ip_hdr_to_payload_w =
e287a75c 3572 hlen_w - ((sizeof(struct ipv6hdr)) >> 1);
a848ade4
DK		 3573 }
3574
3575 pbd2->tcp_send_seq = bswab32(inner_tcp_hdr(skb)->seq);
3576
3577 pbd2->tcp_flags = pbd_tcp_flags(inner_tcp_hdr(skb));
3578
3579 if (xmit_type & XMIT_GSO_V4) {
e287a75c 3580 pbd2->hw_ip_id = bswab16(inner_ip_hdr(skb)->id);
a848ade4
DK		 3581
3582 pbd_e2->data.tunnel_data.pseudo_csum =
3583 bswab16(~csum_tcpudp_magic(
3584 inner_ip_hdr(skb)->saddr,
3585 inner_ip_hdr(skb)->daddr,
3586 0, IPPROTO_TCP, 0));
3587
3588 outerip_len = ip_hdr(skb)->ihl << 1;
3589 } else {
3590 pbd_e2->data.tunnel_data.pseudo_csum =
3591 bswab16(~csum_ipv6_magic(
3592 &inner_ipv6_hdr(skb)->saddr,
3593 &inner_ipv6_hdr(skb)->daddr,
3594 0, IPPROTO_TCP, 0));
3595 }
3596
3597 outerip_off = (skb_network_header(skb) - skb->data) >> 1;
3598
3599 *global_data |=
3600 outerip_off |
3601 (!!(xmit_type & XMIT_CSUM_V6) <<
3602 ETH_TX_PARSE_2ND_BD_IP_HDR_TYPE_OUTER_SHIFT) |
3603 (outerip_len <<
3604 ETH_TX_PARSE_2ND_BD_IP_HDR_LEN_OUTER_W_SHIFT) |
3605 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3606 ETH_TX_PARSE_2ND_BD_LLC_SNAP_EN_SHIFT);
65bc0cfe
DK		 3607
3608 if (ip_hdr(skb)->protocol == IPPROTO_UDP) {
3609 SET_FLAG(*global_data, ETH_TX_PARSE_2ND_BD_TUNNEL_UDP_EXIST, 1);
3610 pbd2->tunnel_udp_hdr_start_w = skb_transport_offset(skb) >> 1;
3611 }
a848ade4
DK		 3612}
3613
9f6c9258
DK		 3614/* called with netif_tx_lock
3615 * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
3616 * netif_wake_queue()
3617 */
3618netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
3619{
3620 struct bnx2x *bp = netdev_priv(dev);
6383c0b3 3621
9f6c9258 3622 struct netdev_queue *txq;
6383c0b3 3623 struct bnx2x_fp_txdata *txdata;
9f6c9258 3624 struct sw_tx_bd *tx_buf;
619c5cb6 3625 struct eth_tx_start_bd *tx_start_bd, *first_bd;
9f6c9258 3626 struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
523224a3 3627 struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
f2e0899f 3628 struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
a848ade4 3629 struct eth_tx_parse_2nd_bd *pbd2 = NULL;
2297a2da 3630 u32 pbd_e2_parsing_data = 0;
9f6c9258 3631 u16 pkt_prod, bd_prod;
65565884 3632 int nbd, txq_index;
9f6c9258
DK		 3633 dma_addr_t mapping;
3634 u32 xmit_type = bnx2x_xmit_type(bp, skb);
3635 int i;
3636 u8 hlen = 0;
3637 __le16 pkt_size = 0;
3638 struct ethhdr *eth;
3639 u8 mac_type = UNICAST_ADDRESS;
3640
3641#ifdef BNX2X_STOP_ON_ERROR
3642 if (unlikely(bp->panic))
3643 return NETDEV_TX_BUSY;
3644#endif
3645
6383c0b3
AE		 3646 txq_index = skb_get_queue_mapping(skb);
3647 txq = netdev_get_tx_queue(dev, txq_index);
3648
55c11941 3649 BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0));
6383c0b3 3650
65565884 3651 txdata = &bp->bnx2x_txq[txq_index];
6383c0b3
AE		 3652
3653 /* enable this debug print to view the transmission queue being used
51c1a580 3654 DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
6383c0b3 3655 txq_index, fp_index, txdata_index); */
9f6c9258 3656
16a5fd92 3657 /* enable this debug print to view the transmission details
51c1a580
MS		 3658 DP(NETIF_MSG_TX_QUEUED,
3659 "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
6383c0b3 3660 txdata->cid, fp_index, txdata_index, txdata, fp); */
9f6c9258 3661
6383c0b3 3662 if (unlikely(bnx2x_tx_avail(bp, txdata) <
7df2dc6b
DK		 3663 skb_shinfo(skb)->nr_frags +
3664 BDS_PER_TX_PKT +
3665 NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
2384d6aa 3666 /* Handle special storage cases separately */
c96bdc0c
DK		 3667 if (txdata->tx_ring_size == 0) {
3668 struct bnx2x_eth_q_stats *q_stats =
3669 bnx2x_fp_qstats(bp, txdata->parent_fp);
3670 q_stats->driver_filtered_tx_pkt++;
3671 dev_kfree_skb(skb);
3672 return NETDEV_TX_OK;
3673 }
2de67439
YM		 3674 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3675 netif_tx_stop_queue(txq);
c96bdc0c 3676 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
2384d6aa 3677
9f6c9258
DK		 3678 return NETDEV_TX_BUSY;
3679 }
3680
51c1a580 3681 DP(NETIF_MSG_TX_QUEUED,
04c46736 3682 "queue[%d]: SKB: summed %x protocol %x protocol(%x,%x) gso type %x xmit_type %x len %d\n",
6383c0b3 3683 txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
04c46736
YM		 3684 ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type,
3685 skb->len);
9f6c9258
DK		 3686
3687 eth = (struct ethhdr *)skb->data;
3688
3689 /* set flag according to packet type (UNICAST_ADDRESS is default)*/
3690 if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
3691 if (is_broadcast_ether_addr(eth->h_dest))
3692 mac_type = BROADCAST_ADDRESS;
3693 else
3694 mac_type = MULTICAST_ADDRESS;
3695 }
3696
91226790 3697#if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
9f6c9258
DK		 3698 /* First, check if we need to linearize the skb (due to FW
3699 restrictions). No need to check fragmentation if page size > 8K
3700 (there will be no violation to FW restrictions) */
3701 if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
3702 /* Statistics of linearization */
3703 bp->lin_cnt++;
3704 if (skb_linearize(skb) != 0) {
51c1a580
MS		 3705 DP(NETIF_MSG_TX_QUEUED,
3706 "SKB linearization failed - silently dropping this SKB\n");
9f6c9258
DK		 3707 dev_kfree_skb_any(skb);
3708 return NETDEV_TX_OK;
3709 }
3710 }
3711#endif
619c5cb6
VZ		 3712 /* Map skb linear data for DMA */
3713 mapping = dma_map_single(&bp->pdev->dev, skb->data,
3714 skb_headlen(skb), DMA_TO_DEVICE);
3715 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
51c1a580
MS		 3716 DP(NETIF_MSG_TX_QUEUED,
3717 "SKB mapping failed - silently dropping this SKB\n");
619c5cb6
VZ		 3718 dev_kfree_skb_any(skb);
3719 return NETDEV_TX_OK;
3720 }
9f6c9258
DK		 3721 /*
3722 Please read carefully. First we use one BD which we mark as start,
3723 then we have a parsing info BD (used for TSO or xsum),
3724 and only then we have the rest of the TSO BDs.
3725 (don't forget to mark the last one as last,
3726 and to unmap only AFTER you write to the BD ...)
3727 And above all, all pdb sizes are in words - NOT DWORDS!
3728 */
3729
619c5cb6
VZ		 3730 /* get current pkt produced now - advance it just before sending packet
3731 * since mapping of pages may fail and cause packet to be dropped
3732 */
6383c0b3
AE		 3733 pkt_prod = txdata->tx_pkt_prod;
3734 bd_prod = TX_BD(txdata->tx_bd_prod);
9f6c9258 3735
619c5cb6
VZ		 3736 /* get a tx_buf and first BD
3737 * tx_start_bd may be changed during SPLIT,
3738 * but first_bd will always stay first
3739 */
6383c0b3
AE		 3740 tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3741 tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
619c5cb6 3742 first_bd = tx_start_bd;
9f6c9258
DK		 3743
3744 tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
f85582f8 3745
91226790
DK		 3746 /* header nbd: indirectly zero other flags! */
3747 tx_start_bd->general_data = 1 << ETH_TX_START_BD_HDR_NBDS_SHIFT;
9f6c9258
DK		 3748
3749 /* remember the first BD of the packet */
6383c0b3 3750 tx_buf->first_bd = txdata->tx_bd_prod;
9f6c9258
DK		 3751 tx_buf->skb = skb;
3752 tx_buf->flags = 0;
3753
3754 DP(NETIF_MSG_TX_QUEUED,
3755 "sending pkt %u @%p next_idx %u bd %u @%p\n",
6383c0b3 3756 pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
9f6c9258 3757
eab6d18d 3758 if (vlan_tx_tag_present(skb)) {
523224a3
DK		 3759 tx_start_bd->vlan_or_ethertype =
3760 cpu_to_le16(vlan_tx_tag_get(skb));
3761 tx_start_bd->bd_flags.as_bitfield |=
3762 (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
dc1ba591
AE		 3763 } else {
3764 /* when transmitting in a vf, start bd must hold the ethertype
3765 * for fw to enforce it
3766 */
91226790 3767 if (IS_VF(bp))
dc1ba591
AE		 3768 tx_start_bd->vlan_or_ethertype =
3769 cpu_to_le16(ntohs(eth->h_proto));
91226790 3770 else
dc1ba591
AE		 3771 /* used by FW for packet accounting */
3772 tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
dc1ba591 3773 }
9f6c9258 3774
91226790
DK		 3775 nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3776
9f6c9258
DK		 3777 /* turn on parsing and get a BD */
3778 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
9f6c9258 3779
93ef5c02
DK		 3780 if (xmit_type & XMIT_CSUM)
3781 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
9f6c9258 3782
619c5cb6 3783 if (!CHIP_IS_E1x(bp)) {
6383c0b3 3784 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
f2e0899f 3785 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
a848ade4
DK		 3786
3787 if (xmit_type & XMIT_CSUM_ENC) {
3788 u16 global_data = 0;
3789
3790 /* Set PBD in enc checksum offload case */
3791 hlen = bnx2x_set_pbd_csum_enc(bp, skb,
3792 &pbd_e2_parsing_data,
3793 xmit_type);
3794
3795 /* turn on 2nd parsing and get a BD */
3796 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3797
3798 pbd2 = &txdata->tx_desc_ring[bd_prod].parse_2nd_bd;
3799
3800 memset(pbd2, 0, sizeof(*pbd2));
3801
3802 pbd_e2->data.tunnel_data.ip_hdr_start_inner_w =
3803 (skb_inner_network_header(skb) -
3804 skb->data) >> 1;
3805
3806 if (xmit_type & XMIT_GSO_ENC)
3807 bnx2x_update_pbds_gso_enc(skb, pbd_e2, pbd2,
3808 &global_data,
3809 xmit_type);
3810
3811 pbd2->global_data = cpu_to_le16(global_data);
3812
3813 /* add addition parse BD indication to start BD */
3814 SET_FLAG(tx_start_bd->general_data,
3815 ETH_TX_START_BD_PARSE_NBDS, 1);
3816 /* set encapsulation flag in start BD */
3817 SET_FLAG(tx_start_bd->general_data,
3818 ETH_TX_START_BD_TUNNEL_EXIST, 1);
3819 nbd++;
3820 } else if (xmit_type & XMIT_CSUM) {
91226790 3821 /* Set PBD in checksum offload case w/o encapsulation */
2297a2da
VZ		 3822 hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3823 &pbd_e2_parsing_data,
3824 xmit_type);
a848ade4 3825 }
dc1ba591 3826
91226790
DK		 3827 /* Add the macs to the parsing BD this is a vf */
3828 if (IS_VF(bp)) {
3829 /* override GRE parameters in BD */
3830 bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
3831 &pbd_e2->data.mac_addr.src_mid,
3832 &pbd_e2->data.mac_addr.src_lo,
619c5cb6 3833 eth->h_source);
91226790
DK		 3834
3835 bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi,
3836 &pbd_e2->data.mac_addr.dst_mid,
3837 &pbd_e2->data.mac_addr.dst_lo,
619c5cb6
VZ		 3838 eth->h_dest);
3839 }
96bed4b9
YM		 3840
3841 SET_FLAG(pbd_e2_parsing_data,
3842 ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type);
f2e0899f 3843 } else {
96bed4b9 3844 u16 global_data = 0;
6383c0b3 3845 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
f2e0899f
DK		 3846 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
3847 /* Set PBD in checksum offload case */
3848 if (xmit_type & XMIT_CSUM)
3849 hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
9f6c9258 3850
96bed4b9
YM		 3851 SET_FLAG(global_data,
3852 ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type);
3853 pbd_e1x->global_data |= cpu_to_le16(global_data);
9f6c9258
DK		 3854 }
3855
f85582f8 3856 /* Setup the data pointer of the first BD of the packet */
9f6c9258
DK		 3857 tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3858 tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
9f6c9258
DK		 3859 tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
3860 pkt_size = tx_start_bd->nbytes;
3861
51c1a580 3862 DP(NETIF_MSG_TX_QUEUED,
91226790 3863 "first bd @%p addr (%x:%x) nbytes %d flags %x vlan %x\n",
9f6c9258 3864 tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
91226790 3865 le16_to_cpu(tx_start_bd->nbytes),
523224a3
DK		 3866 tx_start_bd->bd_flags.as_bitfield,
3867 le16_to_cpu(tx_start_bd->vlan_or_ethertype));
9f6c9258
DK		 3868
3869 if (xmit_type & XMIT_GSO) {
3870
3871 DP(NETIF_MSG_TX_QUEUED,
3872 "TSO packet len %d hlen %d total len %d tso size %d\n",
3873 skb->len, hlen, skb_headlen(skb),
3874 skb_shinfo(skb)->gso_size);
3875
3876 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
3877
91226790
DK		 3878 if (unlikely(skb_headlen(skb) > hlen)) {
3879 nbd++;
6383c0b3
AE		 3880 bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
3881 &tx_start_bd, hlen,
91226790
DK		 3882 bd_prod);
3883 }
619c5cb6 3884 if (!CHIP_IS_E1x(bp))
2297a2da
VZ		 3885 bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
3886 xmit_type);
f2e0899f 3887 else
44dbc78e 3888 bnx2x_set_pbd_gso(skb, pbd_e1x, first_bd, xmit_type);
9f6c9258 3889 }
2297a2da
VZ		 3890
3891 /* Set the PBD's parsing_data field if not zero
3892 * (for the chips newer than 57711).
3893 */
3894 if (pbd_e2_parsing_data)
3895 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
3896
9f6c9258
DK		 3897 tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
3898
f85582f8 3899 /* Handle fragmented skb */
9f6c9258
DK		 3900 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3901 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3902
9e903e08
ED		 3903 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
3904 skb_frag_size(frag), DMA_TO_DEVICE);
619c5cb6 3905 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
2df1a70a 3906 unsigned int pkts_compl = 0, bytes_compl = 0;
619c5cb6 3907
51c1a580
MS		 3908 DP(NETIF_MSG_TX_QUEUED,
3909 "Unable to map page - dropping packet...\n");
619c5cb6
VZ		 3910
3911 /* we need unmap all buffers already mapped
3912 * for this SKB;
3913 * first_bd->nbd need to be properly updated
3914 * before call to bnx2x_free_tx_pkt
3915 */
3916 first_bd->nbd = cpu_to_le16(nbd);
6383c0b3 3917 bnx2x_free_tx_pkt(bp, txdata,
2df1a70a
TH		 3918 TX_BD(txdata->tx_pkt_prod),
3919 &pkts_compl, &bytes_compl);
619c5cb6
VZ		 3920 return NETDEV_TX_OK;
3921 }
3922
9f6c9258 3923 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
6383c0b3 3924 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
9f6c9258 3925 if (total_pkt_bd == NULL)
6383c0b3 3926 total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
9f6c9258 3927
9f6c9258
DK		 3928 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3929 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
9e903e08
ED		 3930 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
3931 le16_add_cpu(&pkt_size, skb_frag_size(frag));
619c5cb6 3932 nbd++;
9f6c9258
DK		 3933
3934 DP(NETIF_MSG_TX_QUEUED,
3935 "frag %d bd @%p addr (%x:%x) nbytes %d\n",
3936 i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
3937 le16_to_cpu(tx_data_bd->nbytes));
3938 }
3939
3940 DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
3941
619c5cb6
VZ		 3942 /* update with actual num BDs */
3943 first_bd->nbd = cpu_to_le16(nbd);
3944
9f6c9258
DK		 3945 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3946
3947 /* now send a tx doorbell, counting the next BD
3948 * if the packet contains or ends with it
3949 */
3950 if (TX_BD_POFF(bd_prod) < nbd)
3951 nbd++;
3952
619c5cb6
VZ		 3953 /* total_pkt_bytes should be set on the first data BD if
3954 * it's not an LSO packet and there is more than one
3955 * data BD. In this case pkt_size is limited by an MTU value.
3956 * However we prefer to set it for an LSO packet (while we don't
3957 * have to) in order to save some CPU cycles in a none-LSO
3958 * case, when we much more care about them.
3959 */
9f6c9258
DK		 3960 if (total_pkt_bd != NULL)
3961 total_pkt_bd->total_pkt_bytes = pkt_size;
3962
523224a3 3963 if (pbd_e1x)
9f6c9258 3964 DP(NETIF_MSG_TX_QUEUED,
51c1a580 3965 "PBD (E1X) @%p ip_data %x ip_hlen %u ip_id %u lso_mss %u tcp_flags %x xsum %x seq %u hlen %u\n",
523224a3
DK		 3966 pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
3967 pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
3968 pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
3969 le16_to_cpu(pbd_e1x->total_hlen_w));
f2e0899f
DK		 3970 if (pbd_e2)
3971 DP(NETIF_MSG_TX_QUEUED,
3972 "PBD (E2) @%p dst %x %x %x src %x %x %x parsing_data %x\n",
91226790
DK		 3973 pbd_e2,
3974 pbd_e2->data.mac_addr.dst_hi,
3975 pbd_e2->data.mac_addr.dst_mid,
3976 pbd_e2->data.mac_addr.dst_lo,
3977 pbd_e2->data.mac_addr.src_hi,
3978 pbd_e2->data.mac_addr.src_mid,
3979 pbd_e2->data.mac_addr.src_lo,
f2e0899f 3980 pbd_e2->parsing_data);
9f6c9258
DK		 3981 DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d bd %u\n", nbd, bd_prod);
3982
2df1a70a
TH		 3983 netdev_tx_sent_queue(txq, skb->len);
3984
8373c57d
WB		 3985 skb_tx_timestamp(skb);
3986
6383c0b3 3987 txdata->tx_pkt_prod++;
9f6c9258
DK		 3988 /*
3989 * Make sure that the BD data is updated before updating the producer
3990 * since FW might read the BD right after the producer is updated.
3991 * This is only applicable for weak-ordered memory model archs such
3992 * as IA-64. The following barrier is also mandatory since FW will
3993 * assumes packets must have BDs.
3994 */
3995 wmb();
3996
6383c0b3 3997 txdata->tx_db.data.prod += nbd;
9f6c9258 3998 barrier();
f85582f8 3999
6383c0b3 4000 DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
9f6c9258
DK		 4001
4002 mmiowb();
4003
6383c0b3 4004 txdata->tx_bd_prod += nbd;
9f6c9258 4005
7df2dc6b 4006 if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) {
9f6c9258
DK		 4007 netif_tx_stop_queue(txq);
4008
4009 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
4010 * ordering of set_bit() in netif_tx_stop_queue() and read of
4011 * fp->bd_tx_cons */
4012 smp_mb();
4013
15192a8c 4014 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
7df2dc6b 4015 if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)
9f6c9258
DK		 4016 netif_tx_wake_queue(txq);
4017 }
6383c0b3 4018 txdata->tx_pkt++;
9f6c9258
DK		 4019
4020 return NETDEV_TX_OK;
4021}
f85582f8 4022
6383c0b3
AE		 4023/**
4024 * bnx2x_setup_tc - routine to configure net_device for multi tc
4025 *
4026 * @netdev: net device to configure
4027 * @tc: number of traffic classes to enable
4028 *
4029 * callback connected to the ndo_setup_tc function pointer
4030 */
4031int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
4032{
4033 int cos, prio, count, offset;
4034 struct bnx2x *bp = netdev_priv(dev);
4035
4036 /* setup tc must be called under rtnl lock */
4037 ASSERT_RTNL();
4038
16a5fd92 4039 /* no traffic classes requested. Aborting */
6383c0b3
AE		 4040 if (!num_tc) {
4041 netdev_reset_tc(dev);
4042 return 0;
4043 }
4044
4045 /* requested to support too many traffic classes */
4046 if (num_tc > bp->max_cos) {
6bf07b8e 4047 BNX2X_ERR("support for too many traffic classes requested: %d. Max supported is %d\n",
51c1a580 4048 num_tc, bp->max_cos);
6383c0b3
AE		 4049 return -EINVAL;
4050 }
4051
4052 /* declare amount of supported traffic classes */
4053 if (netdev_set_num_tc(dev, num_tc)) {
51c1a580 4054 BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
6383c0b3
AE		 4055 return -EINVAL;
4056 }
4057
4058 /* configure priority to traffic class mapping */
4059 for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
4060 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
51c1a580
MS		 4061 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4062 "mapping priority %d to tc %d\n",
6383c0b3
AE		 4063 prio, bp->prio_to_cos[prio]);
4064 }
4065
16a5fd92 4066 /* Use this configuration to differentiate tc0 from other COSes
6383c0b3
AE		 4067 This can be used for ets or pfc, and save the effort of setting
4068 up a multio class queue disc or negotiating DCBX with a switch
4069 netdev_set_prio_tc_map(dev, 0, 0);
94f05b0f 4070 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
6383c0b3
AE		 4071 for (prio = 1; prio < 16; prio++) {
4072 netdev_set_prio_tc_map(dev, prio, 1);
94f05b0f 4073 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
6383c0b3
AE		 4074 } */
4075
4076 /* configure traffic class to transmission queue mapping */
4077 for (cos = 0; cos < bp->max_cos; cos++) {
4078 count = BNX2X_NUM_ETH_QUEUES(bp);
65565884 4079 offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp);
6383c0b3 4080 netdev_set_tc_queue(dev, cos, count, offset);
51c1a580
MS		 4081 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4082 "mapping tc %d to offset %d count %d\n",
6383c0b3
AE		 4083 cos, offset, count);
4084 }
4085
4086 return 0;
4087}
4088
9f6c9258
DK		 4089/* called with rtnl_lock */
4090int bnx2x_change_mac_addr(struct net_device *dev, void *p)
4091{
4092 struct sockaddr *addr = p;
4093 struct bnx2x *bp = netdev_priv(dev);
619c5cb6 4094 int rc = 0;
9f6c9258 4095
51c1a580
MS		 4096 if (!bnx2x_is_valid_ether_addr(bp, addr->sa_data)) {
4097 BNX2X_ERR("Requested MAC address is not valid\n");
614c76df 4098 return -EINVAL;
51c1a580 4099 }
614c76df 4100
a3348722
BW		 4101 if ((IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp)) &&
4102 !is_zero_ether_addr(addr->sa_data)) {
51c1a580 4103 BNX2X_ERR("Can't configure non-zero address on iSCSI or FCoE functions in MF-SD mode\n");
9f6c9258 4104 return -EINVAL;
51c1a580 4105 }
9f6c9258 4106
619c5cb6
VZ		 4107 if (netif_running(dev)) {
4108 rc = bnx2x_set_eth_mac(bp, false);
4109 if (rc)
4110 return rc;
4111 }
4112
9f6c9258 4113 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
619c5cb6 4114
523224a3 4115 if (netif_running(dev))
619c5cb6 4116 rc = bnx2x_set_eth_mac(bp, true);
9f6c9258 4117
619c5cb6 4118 return rc;
9f6c9258
DK		 4119}
4120
b3b83c3f
DK		 4121static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
4122{
4123 union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
4124 struct bnx2x_fastpath *fp = &bp->fp[fp_index];
6383c0b3 4125 u8 cos;
b3b83c3f
DK		 4126
4127 /* Common */
55c11941 4128
b3b83c3f
DK		 4129 if (IS_FCOE_IDX(fp_index)) {
4130 memset(sb, 0, sizeof(union host_hc_status_block));
4131 fp->status_blk_mapping = 0;
b3b83c3f 4132 } else {
b3b83c3f 4133 /* status blocks */
619c5cb6 4134 if (!CHIP_IS_E1x(bp))
b3b83c3f
DK		 4135 BNX2X_PCI_FREE(sb->e2_sb,
4136 bnx2x_fp(bp, fp_index,
4137 status_blk_mapping),
4138 sizeof(struct host_hc_status_block_e2));
4139 else
4140 BNX2X_PCI_FREE(sb->e1x_sb,
4141 bnx2x_fp(bp, fp_index,
4142 status_blk_mapping),
4143 sizeof(struct host_hc_status_block_e1x));
b3b83c3f 4144 }
55c11941 4145
b3b83c3f
DK		 4146 /* Rx */
4147 if (!skip_rx_queue(bp, fp_index)) {
4148 bnx2x_free_rx_bds(fp);
4149
4150 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4151 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
4152 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
4153 bnx2x_fp(bp, fp_index, rx_desc_mapping),
4154 sizeof(struct eth_rx_bd) * NUM_RX_BD);
4155
4156 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
4157 bnx2x_fp(bp, fp_index, rx_comp_mapping),
4158 sizeof(struct eth_fast_path_rx_cqe) *
4159 NUM_RCQ_BD);
4160
4161 /* SGE ring */
4162 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
4163 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
4164 bnx2x_fp(bp, fp_index, rx_sge_mapping),
4165 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4166 }
4167
4168 /* Tx */
4169 if (!skip_tx_queue(bp, fp_index)) {
4170 /* fastpath tx rings: tx_buf tx_desc */
6383c0b3 4171 for_each_cos_in_tx_queue(fp, cos) {
65565884 4172 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
6383c0b3 4173
51c1a580 4174 DP(NETIF_MSG_IFDOWN,
94f05b0f 4175 "freeing tx memory of fp %d cos %d cid %d\n",
6383c0b3
AE		 4176 fp_index, cos, txdata->cid);
4177
4178 BNX2X_FREE(txdata->tx_buf_ring);
4179 BNX2X_PCI_FREE(txdata->tx_desc_ring,
4180 txdata->tx_desc_mapping,
4181 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4182 }
b3b83c3f
DK		 4183 }
4184 /* end of fastpath */
4185}
4186
55c11941
MS		 4187void bnx2x_free_fp_mem_cnic(struct bnx2x *bp)
4188{
4189 int i;
4190 for_each_cnic_queue(bp, i)
4191 bnx2x_free_fp_mem_at(bp, i);
4192}
4193
b3b83c3f
DK		 4194void bnx2x_free_fp_mem(struct bnx2x *bp)
4195{
4196 int i;
55c11941 4197 for_each_eth_queue(bp, i)
b3b83c3f
DK		 4198 bnx2x_free_fp_mem_at(bp, i);
4199}
4200
1191cb83 4201static void set_sb_shortcuts(struct bnx2x *bp, int index)
b3b83c3f
DK		 4202{
4203 union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
619c5cb6 4204 if (!CHIP_IS_E1x(bp)) {
b3b83c3f
DK		 4205 bnx2x_fp(bp, index, sb_index_values) =
4206 (__le16 *)status_blk.e2_sb->sb.index_values;
4207 bnx2x_fp(bp, index, sb_running_index) =
4208 (__le16 *)status_blk.e2_sb->sb.running_index;
4209 } else {
4210 bnx2x_fp(bp, index, sb_index_values) =
4211 (__le16 *)status_blk.e1x_sb->sb.index_values;
4212 bnx2x_fp(bp, index, sb_running_index) =
4213 (__le16 *)status_blk.e1x_sb->sb.running_index;
4214 }
4215}
4216
1191cb83
ED		 4217/* Returns the number of actually allocated BDs */
4218static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
4219 int rx_ring_size)
4220{
4221 struct bnx2x *bp = fp->bp;
4222 u16 ring_prod, cqe_ring_prod;
4223 int i, failure_cnt = 0;
4224
4225 fp->rx_comp_cons = 0;
4226 cqe_ring_prod = ring_prod = 0;
4227
4228 /* This routine is called only during fo init so
4229 * fp->eth_q_stats.rx_skb_alloc_failed = 0
4230 */
4231 for (i = 0; i < rx_ring_size; i++) {
996dedba 4232 if (bnx2x_alloc_rx_data(bp, fp, ring_prod, GFP_KERNEL) < 0) {
1191cb83
ED		 4233 failure_cnt++;
4234 continue;
4235 }
4236 ring_prod = NEXT_RX_IDX(ring_prod);
4237 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
4238 WARN_ON(ring_prod <= (i - failure_cnt));
4239 }
4240
4241 if (failure_cnt)
4242 BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n",
4243 i - failure_cnt, fp->index);
4244
4245 fp->rx_bd_prod = ring_prod;
4246 /* Limit the CQE producer by the CQE ring size */
4247 fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
4248 cqe_ring_prod);
4249 fp->rx_pkt = fp->rx_calls = 0;
4250
15192a8c 4251 bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt;
1191cb83
ED		 4252
4253 return i - failure_cnt;
4254}
4255
4256static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp)
4257{
4258 int i;
4259
4260 for (i = 1; i <= NUM_RCQ_RINGS; i++) {
4261 struct eth_rx_cqe_next_page *nextpg;
4262
4263 nextpg = (struct eth_rx_cqe_next_page *)
4264 &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
4265 nextpg->addr_hi =
4266 cpu_to_le32(U64_HI(fp->rx_comp_mapping +
4267 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4268 nextpg->addr_lo =
4269 cpu_to_le32(U64_LO(fp->rx_comp_mapping +
4270 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4271 }
4272}
4273
b3b83c3f
DK		 4274static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
4275{
4276 union host_hc_status_block *sb;
4277 struct bnx2x_fastpath *fp = &bp->fp[index];
4278 int ring_size = 0;
6383c0b3 4279 u8 cos;
c2188952 4280 int rx_ring_size = 0;
b3b83c3f 4281
a3348722
BW		 4282 if (!bp->rx_ring_size &&
4283 (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))) {
614c76df
DK		 4284 rx_ring_size = MIN_RX_SIZE_NONTPA;
4285 bp->rx_ring_size = rx_ring_size;
55c11941 4286 } else if (!bp->rx_ring_size) {
c2188952
VZ		 4287 rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
4288
065f8b92
YM		 4289 if (CHIP_IS_E3(bp)) {
4290 u32 cfg = SHMEM_RD(bp,
4291 dev_info.port_hw_config[BP_PORT(bp)].
4292 default_cfg);
4293
4294 /* Decrease ring size for 1G functions */
4295 if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
4296 PORT_HW_CFG_NET_SERDES_IF_SGMII)
4297 rx_ring_size /= 10;
4298 }
d760fc37 4299
c2188952
VZ		 4300 /* allocate at least number of buffers required by FW */
4301 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
4302 MIN_RX_SIZE_TPA, rx_ring_size);
4303
4304 bp->rx_ring_size = rx_ring_size;
614c76df 4305 } else /* if rx_ring_size specified - use it */
c2188952 4306 rx_ring_size = bp->rx_ring_size;
b3b83c3f 4307
04c46736
YM		 4308 DP(BNX2X_MSG_SP, "calculated rx_ring_size %d\n", rx_ring_size);
4309
b3b83c3f
DK		 4310 /* Common */
4311 sb = &bnx2x_fp(bp, index, status_blk);
55c11941 4312
b3b83c3f 4313 if (!IS_FCOE_IDX(index)) {
b3b83c3f 4314 /* status blocks */
619c5cb6 4315 if (!CHIP_IS_E1x(bp))
b3b83c3f
DK		 4316 BNX2X_PCI_ALLOC(sb->e2_sb,
4317 &bnx2x_fp(bp, index, status_blk_mapping),
4318 sizeof(struct host_hc_status_block_e2));
4319 else
4320 BNX2X_PCI_ALLOC(sb->e1x_sb,
4321 &bnx2x_fp(bp, index, status_blk_mapping),
4322 sizeof(struct host_hc_status_block_e1x));
b3b83c3f 4323 }
8eef2af1
DK		 4324
4325 /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
4326 * set shortcuts for it.
4327 */
4328 if (!IS_FCOE_IDX(index))
4329 set_sb_shortcuts(bp, index);
b3b83c3f
DK		 4330
4331 /* Tx */
4332 if (!skip_tx_queue(bp, index)) {
4333 /* fastpath tx rings: tx_buf tx_desc */
6383c0b3 4334 for_each_cos_in_tx_queue(fp, cos) {
65565884 4335 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
6383c0b3 4336
51c1a580
MS		 4337 DP(NETIF_MSG_IFUP,
4338 "allocating tx memory of fp %d cos %d\n",
6383c0b3
AE		 4339 index, cos);
4340
4341 BNX2X_ALLOC(txdata->tx_buf_ring,
b3b83c3f 4342 sizeof(struct sw_tx_bd) * NUM_TX_BD);
6383c0b3
AE		 4343 BNX2X_PCI_ALLOC(txdata->tx_desc_ring,
4344 &txdata->tx_desc_mapping,
b3b83c3f 4345 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
6383c0b3 4346 }
b3b83c3f
DK		 4347 }
4348
4349 /* Rx */
4350 if (!skip_rx_queue(bp, index)) {
4351 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4352 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_buf_ring),
4353 sizeof(struct sw_rx_bd) * NUM_RX_BD);
4354 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_desc_ring),
4355 &bnx2x_fp(bp, index, rx_desc_mapping),
4356 sizeof(struct eth_rx_bd) * NUM_RX_BD);
4357
75b29459
DK		 4358 /* Seed all CQEs by 1s */
4359 BNX2X_PCI_FALLOC(bnx2x_fp(bp, index, rx_comp_ring),
4360 &bnx2x_fp(bp, index, rx_comp_mapping),
4361 sizeof(struct eth_fast_path_rx_cqe) *
4362 NUM_RCQ_BD);
b3b83c3f
DK		 4363
4364 /* SGE ring */
4365 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_page_ring),
4366 sizeof(struct sw_rx_page) * NUM_RX_SGE);
4367 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_sge_ring),
4368 &bnx2x_fp(bp, index, rx_sge_mapping),
4369 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4370 /* RX BD ring */
4371 bnx2x_set_next_page_rx_bd(fp);
4372
4373 /* CQ ring */
4374 bnx2x_set_next_page_rx_cq(fp);
4375
4376 /* BDs */
4377 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
4378 if (ring_size < rx_ring_size)
4379 goto alloc_mem_err;
4380 }
4381
4382 return 0;
4383
4384/* handles low memory cases */
4385alloc_mem_err:
4386 BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
4387 index, ring_size);
4388 /* FW will drop all packets if queue is not big enough,
4389 * In these cases we disable the queue
6383c0b3 4390 * Min size is different for OOO, TPA and non-TPA queues
b3b83c3f
DK		 4391 */
4392 if (ring_size < (fp->disable_tpa ?
eb722d7a 4393 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
b3b83c3f
DK		 4394 /* release memory allocated for this queue */
4395 bnx2x_free_fp_mem_at(bp, index);
4396 return -ENOMEM;
4397 }
4398 return 0;
4399}
4400
55c11941
MS		 4401int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp)
4402{
4403 if (!NO_FCOE(bp))
4404 /* FCoE */
4405 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp)))
4406 /* we will fail load process instead of mark
4407 * NO_FCOE_FLAG
4408 */
4409 return -ENOMEM;
4410
4411 return 0;
4412}
4413
b3b83c3f
DK		 4414int bnx2x_alloc_fp_mem(struct bnx2x *bp)
4415{
4416 int i;
4417
55c11941
MS		 4418 /* 1. Allocate FP for leading - fatal if error
4419 * 2. Allocate RSS - fix number of queues if error
b3b83c3f
DK		 4420 */
4421
4422 /* leading */
4423 if (bnx2x_alloc_fp_mem_at(bp, 0))
4424 return -ENOMEM;
6383c0b3 4425
b3b83c3f
DK		 4426 /* RSS */
4427 for_each_nondefault_eth_queue(bp, i)
4428 if (bnx2x_alloc_fp_mem_at(bp, i))
4429 break;
4430
4431 /* handle memory failures */
4432 if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
4433 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
4434
4435 WARN_ON(delta < 0);
4864a16a 4436 bnx2x_shrink_eth_fp(bp, delta);
55c11941
MS		 4437 if (CNIC_SUPPORT(bp))
4438 /* move non eth FPs next to last eth FP
4439 * must be done in that order
4440 * FCOE_IDX < FWD_IDX < OOO_IDX
4441 */
b3b83c3f 4442
55c11941
MS		 4443 /* move FCoE fp even NO_FCOE_FLAG is on */
4444 bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta);
4445 bp->num_ethernet_queues -= delta;
4446 bp->num_queues = bp->num_ethernet_queues +
4447 bp->num_cnic_queues;
b3b83c3f
DK		 4448 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
4449 bp->num_queues + delta, bp->num_queues);
4450 }
4451
4452 return 0;
4453}
d6214d7a 4454
523224a3
DK		 4455void bnx2x_free_mem_bp(struct bnx2x *bp)
4456{
c3146eb6
DK		 4457 int i;
4458
4459 for (i = 0; i < bp->fp_array_size; i++)
4460 kfree(bp->fp[i].tpa_info);
523224a3 4461 kfree(bp->fp);
15192a8c
BW		 4462 kfree(bp->sp_objs);
4463 kfree(bp->fp_stats);
65565884 4464 kfree(bp->bnx2x_txq);
523224a3
DK		 4465 kfree(bp->msix_table);
4466 kfree(bp->ilt);
4467}
4468
0329aba1 4469int bnx2x_alloc_mem_bp(struct bnx2x *bp)
523224a3
DK		 4470{
4471 struct bnx2x_fastpath *fp;
4472 struct msix_entry *tbl;
4473 struct bnx2x_ilt *ilt;
6383c0b3 4474 int msix_table_size = 0;
55c11941 4475 int fp_array_size, txq_array_size;
15192a8c 4476 int i;
6383c0b3
AE		 4477
4478 /*
4479 * The biggest MSI-X table we might need is as a maximum number of fast
2de67439 4480 * path IGU SBs plus default SB (for PF only).
6383c0b3 4481 */
1ab4434c
AE		 4482 msix_table_size = bp->igu_sb_cnt;
4483 if (IS_PF(bp))
4484 msix_table_size++;
4485 BNX2X_DEV_INFO("msix_table_size %d\n", msix_table_size);
523224a3 4486
6383c0b3 4487 /* fp array: RSS plus CNIC related L2 queues */
55c11941 4488 fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp);
c3146eb6
DK		 4489 bp->fp_array_size = fp_array_size;
4490 BNX2X_DEV_INFO("fp_array_size %d\n", bp->fp_array_size);
15192a8c 4491
c3146eb6 4492 fp = kcalloc(bp->fp_array_size, sizeof(*fp), GFP_KERNEL);
523224a3
DK		 4493 if (!fp)
4494 goto alloc_err;
c3146eb6 4495 for (i = 0; i < bp->fp_array_size; i++) {
15192a8c
BW		 4496 fp[i].tpa_info =
4497 kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2,
4498 sizeof(struct bnx2x_agg_info), GFP_KERNEL);
4499 if (!(fp[i].tpa_info))
4500 goto alloc_err;
4501 }
4502
523224a3
DK		 4503 bp->fp = fp;
4504
15192a8c 4505 /* allocate sp objs */
c3146eb6 4506 bp->sp_objs = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_sp_objs),
15192a8c
BW		 4507 GFP_KERNEL);
4508 if (!bp->sp_objs)
4509 goto alloc_err;
4510
4511 /* allocate fp_stats */
c3146eb6 4512 bp->fp_stats = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_fp_stats),
15192a8c
BW		 4513 GFP_KERNEL);
4514 if (!bp->fp_stats)
4515 goto alloc_err;
4516
65565884 4517 /* Allocate memory for the transmission queues array */
55c11941
MS		 4518 txq_array_size =
4519 BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp);
4520 BNX2X_DEV_INFO("txq_array_size %d", txq_array_size);
4521
4522 bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata),
4523 GFP_KERNEL);
65565884
MS		 4524 if (!bp->bnx2x_txq)
4525 goto alloc_err;
4526
523224a3 4527 /* msix table */
01e23742 4528 tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
523224a3
DK		 4529 if (!tbl)
4530 goto alloc_err;
4531 bp->msix_table = tbl;
4532
4533 /* ilt */
4534 ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
4535 if (!ilt)
4536 goto alloc_err;
4537 bp->ilt = ilt;
4538
4539 return 0;
4540alloc_err:
4541 bnx2x_free_mem_bp(bp);
4542 return -ENOMEM;
523224a3
DK		 4543}
4544
a9fccec7 4545int bnx2x_reload_if_running(struct net_device *dev)
66371c44
MM		 4546{
4547 struct bnx2x *bp = netdev_priv(dev);
4548
4549 if (unlikely(!netif_running(dev)))
4550 return 0;
4551
5d07d868 4552 bnx2x_nic_unload(bp, UNLOAD_NORMAL, true);
66371c44
MM		 4553 return bnx2x_nic_load(bp, LOAD_NORMAL);
4554}
4555
1ac9e428
YR		 4556int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
4557{
4558 u32 sel_phy_idx = 0;
4559 if (bp->link_params.num_phys <= 1)
4560 return INT_PHY;
4561
4562 if (bp->link_vars.link_up) {
4563 sel_phy_idx = EXT_PHY1;
4564 /* In case link is SERDES, check if the EXT_PHY2 is the one */
4565 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
4566 (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
4567 sel_phy_idx = EXT_PHY2;
4568 } else {
4569
4570 switch (bnx2x_phy_selection(&bp->link_params)) {
4571 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
4572 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
4573 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
4574 sel_phy_idx = EXT_PHY1;
4575 break;
4576 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
4577 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
4578 sel_phy_idx = EXT_PHY2;
4579 break;
4580 }
4581 }
4582
4583 return sel_phy_idx;
1ac9e428
YR		 4584}
4585int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
4586{
4587 u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
4588 /*
2de67439 4589 * The selected activated PHY is always after swapping (in case PHY
1ac9e428
YR		 4590 * swapping is enabled). So when swapping is enabled, we need to reverse
4591 * the configuration
4592 */
4593
4594 if (bp->link_params.multi_phy_config &
4595 PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
4596 if (sel_phy_idx == EXT_PHY1)
4597 sel_phy_idx = EXT_PHY2;
4598 else if (sel_phy_idx == EXT_PHY2)
4599 sel_phy_idx = EXT_PHY1;
4600 }
4601 return LINK_CONFIG_IDX(sel_phy_idx);
4602}
4603
55c11941 4604#ifdef NETDEV_FCOE_WWNN
bf61ee14
VZ		 4605int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
4606{
4607 struct bnx2x *bp = netdev_priv(dev);
4608 struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
4609
4610 switch (type) {
4611 case NETDEV_FCOE_WWNN:
4612 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
4613 cp->fcoe_wwn_node_name_lo);
4614 break;
4615 case NETDEV_FCOE_WWPN:
4616 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
4617 cp->fcoe_wwn_port_name_lo);
4618 break;
4619 default:
51c1a580 4620 BNX2X_ERR("Wrong WWN type requested - %d\n", type);
bf61ee14
VZ		 4621 return -EINVAL;
4622 }
4623
4624 return 0;
4625}
4626#endif
4627
9f6c9258
DK		 4628/* called with rtnl_lock */
4629int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
4630{
4631 struct bnx2x *bp = netdev_priv(dev);
9f6c9258
DK		 4632
4633 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
51c1a580 4634 BNX2X_ERR("Can't perform change MTU during parity recovery\n");
9f6c9258
DK		 4635 return -EAGAIN;
4636 }
4637
4638 if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
51c1a580
MS		 4639 ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) {
4640 BNX2X_ERR("Can't support requested MTU size\n");
9f6c9258 4641 return -EINVAL;
51c1a580 4642 }
9f6c9258
DK		 4643
4644 /* This does not race with packet allocation
4645 * because the actual alloc size is
4646 * only updated as part of load
4647 */
4648 dev->mtu = new_mtu;
4649
66371c44
MM		 4650 return bnx2x_reload_if_running(dev);
4651}
4652
c8f44aff 4653netdev_features_t bnx2x_fix_features(struct net_device *dev,
621b4d66 4654 netdev_features_t features)
66371c44
MM		 4655{
4656 struct bnx2x *bp = netdev_priv(dev);
4657
4658 /* TPA requires Rx CSUM offloading */
621b4d66 4659 if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa) {
66371c44 4660 features &= ~NETIF_F_LRO;
621b4d66
DK		 4661 features &= ~NETIF_F_GRO;
4662 }
66371c44
MM		 4663
4664 return features;
4665}
4666
c8f44aff 4667int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
66371c44
MM		 4668{
4669 struct bnx2x *bp = netdev_priv(dev);
4670 u32 flags = bp->flags;
8802f579 4671 u32 changes;
538dd2e3 4672 bool bnx2x_reload = false;
66371c44
MM		 4673
4674 if (features & NETIF_F_LRO)
4675 flags |= TPA_ENABLE_FLAG;
4676 else
4677 flags &= ~TPA_ENABLE_FLAG;
4678
621b4d66
DK		 4679 if (features & NETIF_F_GRO)
4680 flags |= GRO_ENABLE_FLAG;
4681 else
4682 flags &= ~GRO_ENABLE_FLAG;
4683
538dd2e3
MB		 4684 if (features & NETIF_F_LOOPBACK) {
4685 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
4686 bp->link_params.loopback_mode = LOOPBACK_BMAC;
4687 bnx2x_reload = true;
4688 }
4689 } else {
4690 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
4691 bp->link_params.loopback_mode = LOOPBACK_NONE;
4692 bnx2x_reload = true;
4693 }
4694 }
4695
8802f579
ED		 4696 changes = flags ^ bp->flags;
4697
16a5fd92 4698 /* if GRO is changed while LRO is enabled, don't force a reload */
8802f579
ED		 4699 if ((changes & GRO_ENABLE_FLAG) && (flags & TPA_ENABLE_FLAG))
4700 changes &= ~GRO_ENABLE_FLAG;
4701
4702 if (changes)
538dd2e3 4703 bnx2x_reload = true;
8802f579
ED		 4704
4705 bp->flags = flags;
66371c44 4706
538dd2e3 4707 if (bnx2x_reload) {
66371c44
MM		 4708 if (bp->recovery_state == BNX2X_RECOVERY_DONE)
4709 return bnx2x_reload_if_running(dev);
4710 /* else: bnx2x_nic_load() will be called at end of recovery */
9f6c9258
DK		 4711 }
4712
66371c44 4713 return 0;
9f6c9258
DK		 4714}
4715
4716void bnx2x_tx_timeout(struct net_device *dev)
4717{
4718 struct bnx2x *bp = netdev_priv(dev);
4719
4720#ifdef BNX2X_STOP_ON_ERROR
4721 if (!bp->panic)
4722 bnx2x_panic();
4723#endif
7be08a72
AE		 4724
4725 smp_mb__before_clear_bit();
4726 set_bit(BNX2X_SP_RTNL_TX_TIMEOUT, &bp->sp_rtnl_state);
4727 smp_mb__after_clear_bit();
4728
9f6c9258 4729 /* This allows the netif to be shutdown gracefully before resetting */
7be08a72 4730 schedule_delayed_work(&bp->sp_rtnl_task, 0);
9f6c9258
DK		 4731}
4732
9f6c9258
DK		 4733int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
4734{
4735 struct net_device *dev = pci_get_drvdata(pdev);
4736 struct bnx2x *bp;
4737
4738 if (!dev) {
4739 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4740 return -ENODEV;
4741 }
4742 bp = netdev_priv(dev);
4743
4744 rtnl_lock();
4745
4746 pci_save_state(pdev);
4747
4748 if (!netif_running(dev)) {
4749 rtnl_unlock();
4750 return 0;
4751 }
4752
4753 netif_device_detach(dev);
4754
5d07d868 4755 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
9f6c9258
DK		 4756
4757 bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
4758
4759 rtnl_unlock();
4760
4761 return 0;
4762}
4763
4764int bnx2x_resume(struct pci_dev *pdev)
4765{
4766 struct net_device *dev = pci_get_drvdata(pdev);
4767 struct bnx2x *bp;
4768 int rc;
4769
4770 if (!dev) {
4771 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4772 return -ENODEV;
4773 }
4774 bp = netdev_priv(dev);
4775
4776 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
51c1a580 4777 BNX2X_ERR("Handling parity error recovery. Try again later\n");
9f6c9258
DK		 4778 return -EAGAIN;
4779 }
4780
4781 rtnl_lock();
4782
4783 pci_restore_state(pdev);
4784
4785 if (!netif_running(dev)) {
4786 rtnl_unlock();
4787 return 0;
4788 }
4789
4790 bnx2x_set_power_state(bp, PCI_D0);
4791 netif_device_attach(dev);
4792
4793 rc = bnx2x_nic_load(bp, LOAD_OPEN);
4794
4795 rtnl_unlock();
4796
4797 return rc;
4798}
619c5cb6 4799
619c5cb6
VZ		 4800void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
4801 u32 cid)
4802{
b9871bcf
AE		 4803 if (!cxt) {
4804 BNX2X_ERR("bad context pointer %p\n", cxt);
4805 return;
4806 }
4807
619c5cb6
VZ		 4808 /* ustorm cxt validation */
4809 cxt->ustorm_ag_context.cdu_usage =
4810 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4811 CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
4812 /* xcontext validation */
4813 cxt->xstorm_ag_context.cdu_reserved =
4814 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4815 CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
4816}
4817
1191cb83
ED		 4818static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
4819 u8 fw_sb_id, u8 sb_index,
4820 u8 ticks)
619c5cb6 4821{
619c5cb6
VZ		 4822 u32 addr = BAR_CSTRORM_INTMEM +
4823 CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
4824 REG_WR8(bp, addr, ticks);
51c1a580
MS		 4825 DP(NETIF_MSG_IFUP,
4826 "port %x fw_sb_id %d sb_index %d ticks %d\n",
4827 port, fw_sb_id, sb_index, ticks);
619c5cb6
VZ		 4828}
4829
1191cb83
ED		 4830static void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
4831 u16 fw_sb_id, u8 sb_index,
4832 u8 disable)
619c5cb6
VZ		 4833{
4834 u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
4835 u32 addr = BAR_CSTRORM_INTMEM +
4836 CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
0c14e5ce 4837 u8 flags = REG_RD8(bp, addr);
619c5cb6
VZ		 4838 /* clear and set */
4839 flags &= ~HC_INDEX_DATA_HC_ENABLED;
4840 flags |= enable_flag;
0c14e5ce 4841 REG_WR8(bp, addr, flags);
51c1a580
MS		 4842 DP(NETIF_MSG_IFUP,
4843 "port %x fw_sb_id %d sb_index %d disable %d\n",
4844 port, fw_sb_id, sb_index, disable);
619c5cb6
VZ		 4845}
4846
4847void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
4848 u8 sb_index, u8 disable, u16 usec)
4849{
4850 int port = BP_PORT(bp);
4851 u8 ticks = usec / BNX2X_BTR;
4852
4853 storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
4854
4855 disable = disable ? 1 : (usec ? 0 : 1);
4856 storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
4857}
Linux 6.x block layer and io_uring tree(s)