1 /* bnx2x_cmn.c: Broadcom Everest network driver.
3 * Copyright (c) 2007-2013 Broadcom Corporation
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation.
9 * Maintained by: Ariel Elior <ariel.elior@qlogic.com>
10 * Written by: Eliezer Tamir
11 * Based on code from Michael Chan's bnx2 driver
12 * UDP CSUM errata workaround by Arik Gendelman
13 * Slowpath and fastpath rework by Vladislav Zolotarov
14 * Statistics and Link management by Yitchak Gertner
18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
20 #include <linux/etherdevice.h>
21 #include <linux/if_vlan.h>
22 #include <linux/interrupt.h>
26 #include <net/ip6_checksum.h>
27 #include <net/busy_poll.h>
28 #include <linux/prefetch.h>
29 #include "bnx2x_cmn.h"
30 #include "bnx2x_init.h"
33 static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp);
34 static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp);
35 static int bnx2x_alloc_fp_mem(struct bnx2x *bp);
36 static int bnx2x_poll(struct napi_struct *napi, int budget);
38 static void bnx2x_add_all_napi_cnic(struct bnx2x *bp)
42 /* Add NAPI objects */
43 for_each_rx_queue_cnic(bp, i) {
44 netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
45 bnx2x_poll, NAPI_POLL_WEIGHT);
46 napi_hash_add(&bnx2x_fp(bp, i, napi));
50 static void bnx2x_add_all_napi(struct bnx2x *bp)
54 /* Add NAPI objects */
55 for_each_eth_queue(bp, i) {
56 netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
57 bnx2x_poll, NAPI_POLL_WEIGHT);
58 napi_hash_add(&bnx2x_fp(bp, i, napi));
62 static int bnx2x_calc_num_queues(struct bnx2x *bp)
64 int nq = bnx2x_num_queues ? : netif_get_num_default_rss_queues();
66 /* Reduce memory usage in kdump environment by using only one queue */
70 nq = clamp(nq, 1, BNX2X_MAX_QUEUES(bp));
75 * bnx2x_move_fp - move content of the fastpath structure.
78 * @from: source FP index
79 * @to: destination FP index
81 * Makes sure the contents of the bp->fp[to].napi is kept
82 * intact. This is done by first copying the napi struct from
83 * the target to the source, and then mem copying the entire
84 * source onto the target. Update txdata pointers and related
87 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
89 struct bnx2x_fastpath *from_fp = &bp->fp[from];
90 struct bnx2x_fastpath *to_fp = &bp->fp[to];
91 struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
92 struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
93 struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
94 struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
95 int old_max_eth_txqs, new_max_eth_txqs;
96 int old_txdata_index = 0, new_txdata_index = 0;
97 struct bnx2x_agg_info *old_tpa_info = to_fp->tpa_info;
99 /* Copy the NAPI object as it has been already initialized */
100 from_fp->napi = to_fp->napi;
102 /* Move bnx2x_fastpath contents */
103 memcpy(to_fp, from_fp, sizeof(*to_fp));
106 /* Retain the tpa_info of the original `to' version as we don't want
107 * 2 FPs to contain the same tpa_info pointer.
109 to_fp->tpa_info = old_tpa_info;
111 /* move sp_objs contents as well, as their indices match fp ones */
112 memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
114 /* move fp_stats contents as well, as their indices match fp ones */
115 memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
117 /* Update txdata pointers in fp and move txdata content accordingly:
118 * Each fp consumes 'max_cos' txdata structures, so the index should be
119 * decremented by max_cos x delta.
122 old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
123 new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
125 if (from == FCOE_IDX(bp)) {
126 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
127 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
130 memcpy(&bp->bnx2x_txq[new_txdata_index],
131 &bp->bnx2x_txq[old_txdata_index],
132 sizeof(struct bnx2x_fp_txdata));
133 to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
137 * bnx2x_fill_fw_str - Fill buffer with FW version string.
140 * @buf: character buffer to fill with the fw name
141 * @buf_len: length of the above buffer
144 void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len)
147 u8 phy_fw_ver[PHY_FW_VER_LEN];
149 phy_fw_ver[0] = '\0';
150 bnx2x_get_ext_phy_fw_version(&bp->link_params,
151 phy_fw_ver, PHY_FW_VER_LEN);
152 strlcpy(buf, bp->fw_ver, buf_len);
153 snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver),
155 (bp->common.bc_ver & 0xff0000) >> 16,
156 (bp->common.bc_ver & 0xff00) >> 8,
157 (bp->common.bc_ver & 0xff),
158 ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver);
160 bnx2x_vf_fill_fw_str(bp, buf, buf_len);
165 * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
168 * @delta: number of eth queues which were not allocated
170 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
172 int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
174 /* Queue pointer cannot be re-set on an fp-basis, as moving pointer
175 * backward along the array could cause memory to be overridden
177 for (cos = 1; cos < bp->max_cos; cos++) {
178 for (i = 0; i < old_eth_num - delta; i++) {
179 struct bnx2x_fastpath *fp = &bp->fp[i];
180 int new_idx = cos * (old_eth_num - delta) + i;
182 memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
183 sizeof(struct bnx2x_fp_txdata));
184 fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
189 int bnx2x_load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
191 /* free skb in the packet ring at pos idx
192 * return idx of last bd freed
194 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
195 u16 idx, unsigned int *pkts_compl,
196 unsigned int *bytes_compl)
198 struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
199 struct eth_tx_start_bd *tx_start_bd;
200 struct eth_tx_bd *tx_data_bd;
201 struct sk_buff *skb = tx_buf->skb;
202 u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
204 u16 split_bd_len = 0;
206 /* prefetch skb end pointer to speedup dev_kfree_skb() */
209 DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d buff @(%p)->skb %p\n",
210 txdata->txq_index, idx, tx_buf, skb);
212 tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
214 nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
215 #ifdef BNX2X_STOP_ON_ERROR
216 if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
217 BNX2X_ERR("BAD nbd!\n");
221 new_cons = nbd + tx_buf->first_bd;
223 /* Get the next bd */
224 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
226 /* Skip a parse bd... */
228 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
230 /* TSO headers+data bds share a common mapping. See bnx2x_tx_split() */
231 if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
232 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
233 split_bd_len = BD_UNMAP_LEN(tx_data_bd);
235 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
239 dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
240 BD_UNMAP_LEN(tx_start_bd) + split_bd_len,
246 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
247 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
248 BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
250 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
257 (*bytes_compl) += skb->len;
260 dev_kfree_skb_any(skb);
261 tx_buf->first_bd = 0;
267 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
269 struct netdev_queue *txq;
270 u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
271 unsigned int pkts_compl = 0, bytes_compl = 0;
273 #ifdef BNX2X_STOP_ON_ERROR
274 if (unlikely(bp->panic))
278 txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
279 hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
280 sw_cons = txdata->tx_pkt_cons;
282 while (sw_cons != hw_cons) {
285 pkt_cons = TX_BD(sw_cons);
287 DP(NETIF_MSG_TX_DONE,
288 "queue[%d]: hw_cons %u sw_cons %u pkt_cons %u\n",
289 txdata->txq_index, hw_cons, sw_cons, pkt_cons);
291 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
292 &pkts_compl, &bytes_compl);
297 netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
299 txdata->tx_pkt_cons = sw_cons;
300 txdata->tx_bd_cons = bd_cons;
302 /* Need to make the tx_bd_cons update visible to start_xmit()
303 * before checking for netif_tx_queue_stopped(). Without the
304 * memory barrier, there is a small possibility that
305 * start_xmit() will miss it and cause the queue to be stopped
307 * On the other hand we need an rmb() here to ensure the proper
308 * ordering of bit testing in the following
309 * netif_tx_queue_stopped(txq) call.
313 if (unlikely(netif_tx_queue_stopped(txq))) {
314 /* Taking tx_lock() is needed to prevent re-enabling the queue
315 * while it's empty. This could have happen if rx_action() gets
316 * suspended in bnx2x_tx_int() after the condition before
317 * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
319 * stops the queue->sees fresh tx_bd_cons->releases the queue->
320 * sends some packets consuming the whole queue again->
324 __netif_tx_lock(txq, smp_processor_id());
326 if ((netif_tx_queue_stopped(txq)) &&
327 (bp->state == BNX2X_STATE_OPEN) &&
328 (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
329 netif_tx_wake_queue(txq);
331 __netif_tx_unlock(txq);
336 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
339 u16 last_max = fp->last_max_sge;
341 if (SUB_S16(idx, last_max) > 0)
342 fp->last_max_sge = idx;
345 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
347 struct eth_end_agg_rx_cqe *cqe)
349 struct bnx2x *bp = fp->bp;
350 u16 last_max, last_elem, first_elem;
357 /* First mark all used pages */
358 for (i = 0; i < sge_len; i++)
359 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
360 RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
362 DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
363 sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
365 /* Here we assume that the last SGE index is the biggest */
366 prefetch((void *)(fp->sge_mask));
367 bnx2x_update_last_max_sge(fp,
368 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
370 last_max = RX_SGE(fp->last_max_sge);
371 last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
372 first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
374 /* If ring is not full */
375 if (last_elem + 1 != first_elem)
378 /* Now update the prod */
379 for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
380 if (likely(fp->sge_mask[i]))
383 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
384 delta += BIT_VEC64_ELEM_SZ;
388 fp->rx_sge_prod += delta;
389 /* clear page-end entries */
390 bnx2x_clear_sge_mask_next_elems(fp);
393 DP(NETIF_MSG_RX_STATUS,
394 "fp->last_max_sge = %d fp->rx_sge_prod = %d\n",
395 fp->last_max_sge, fp->rx_sge_prod);
398 /* Get Toeplitz hash value in the skb using the value from the
399 * CQE (calculated by HW).
401 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
402 const struct eth_fast_path_rx_cqe *cqe,
403 enum pkt_hash_types *rxhash_type)
405 /* Get Toeplitz hash from CQE */
406 if ((bp->dev->features & NETIF_F_RXHASH) &&
407 (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
408 enum eth_rss_hash_type htype;
410 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
411 *rxhash_type = ((htype == TCP_IPV4_HASH_TYPE) ||
412 (htype == TCP_IPV6_HASH_TYPE)) ?
413 PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3;
415 return le32_to_cpu(cqe->rss_hash_result);
417 *rxhash_type = PKT_HASH_TYPE_NONE;
421 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
423 struct eth_fast_path_rx_cqe *cqe)
425 struct bnx2x *bp = fp->bp;
426 struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
427 struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
428 struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
430 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
431 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
433 /* print error if current state != stop */
434 if (tpa_info->tpa_state != BNX2X_TPA_STOP)
435 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
437 /* Try to map an empty data buffer from the aggregation info */
438 mapping = dma_map_single(&bp->pdev->dev,
439 first_buf->data + NET_SKB_PAD,
440 fp->rx_buf_size, DMA_FROM_DEVICE);
442 * ...if it fails - move the skb from the consumer to the producer
443 * and set the current aggregation state as ERROR to drop it
444 * when TPA_STOP arrives.
447 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
448 /* Move the BD from the consumer to the producer */
449 bnx2x_reuse_rx_data(fp, cons, prod);
450 tpa_info->tpa_state = BNX2X_TPA_ERROR;
454 /* move empty data from pool to prod */
455 prod_rx_buf->data = first_buf->data;
456 dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
457 /* point prod_bd to new data */
458 prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
459 prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
461 /* move partial skb from cons to pool (don't unmap yet) */
462 *first_buf = *cons_rx_buf;
464 /* mark bin state as START */
465 tpa_info->parsing_flags =
466 le16_to_cpu(cqe->pars_flags.flags);
467 tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
468 tpa_info->tpa_state = BNX2X_TPA_START;
469 tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
470 tpa_info->placement_offset = cqe->placement_offset;
471 tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->rxhash_type);
472 if (fp->mode == TPA_MODE_GRO) {
473 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
474 tpa_info->full_page = SGE_PAGES / gro_size * gro_size;
475 tpa_info->gro_size = gro_size;
478 #ifdef BNX2X_STOP_ON_ERROR
479 fp->tpa_queue_used |= (1 << queue);
480 #ifdef _ASM_GENERIC_INT_L64_H
481 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
483 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
489 /* Timestamp option length allowed for TPA aggregation:
491 * nop nop kind length echo val
493 #define TPA_TSTAMP_OPT_LEN 12
495 * bnx2x_set_gro_params - compute GRO values
498 * @parsing_flags: parsing flags from the START CQE
499 * @len_on_bd: total length of the first packet for the
501 * @pkt_len: length of all segments
503 * Approximate value of the MSS for this aggregation calculated using
504 * the first packet of it.
505 * Compute number of aggregated segments, and gso_type.
507 static void bnx2x_set_gro_params(struct sk_buff *skb, u16 parsing_flags,
508 u16 len_on_bd, unsigned int pkt_len,
509 u16 num_of_coalesced_segs)
511 /* TPA aggregation won't have either IP options or TCP options
512 * other than timestamp or IPv6 extension headers.
514 u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
516 if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
517 PRS_FLAG_OVERETH_IPV6) {
518 hdrs_len += sizeof(struct ipv6hdr);
519 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
521 hdrs_len += sizeof(struct iphdr);
522 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
525 /* Check if there was a TCP timestamp, if there is it's will
526 * always be 12 bytes length: nop nop kind length echo val.
528 * Otherwise FW would close the aggregation.
530 if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
531 hdrs_len += TPA_TSTAMP_OPT_LEN;
533 skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len;
535 /* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count
536 * to skb_shinfo(skb)->gso_segs
538 NAPI_GRO_CB(skb)->count = num_of_coalesced_segs;
541 static int bnx2x_alloc_rx_sge(struct bnx2x *bp, struct bnx2x_fastpath *fp,
542 u16 index, gfp_t gfp_mask)
544 struct page *page = alloc_pages(gfp_mask, PAGES_PER_SGE_SHIFT);
545 struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
546 struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
549 if (unlikely(page == NULL)) {
550 BNX2X_ERR("Can't alloc sge\n");
554 mapping = dma_map_page(&bp->pdev->dev, page, 0,
555 SGE_PAGES, DMA_FROM_DEVICE);
556 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
557 __free_pages(page, PAGES_PER_SGE_SHIFT);
558 BNX2X_ERR("Can't map sge\n");
563 dma_unmap_addr_set(sw_buf, mapping, mapping);
565 sge->addr_hi = cpu_to_le32(U64_HI(mapping));
566 sge->addr_lo = cpu_to_le32(U64_LO(mapping));
571 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
572 struct bnx2x_agg_info *tpa_info,
575 struct eth_end_agg_rx_cqe *cqe,
578 struct sw_rx_page *rx_pg, old_rx_pg;
579 u32 i, frag_len, frag_size;
580 int err, j, frag_id = 0;
581 u16 len_on_bd = tpa_info->len_on_bd;
582 u16 full_page = 0, gro_size = 0;
584 frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
586 if (fp->mode == TPA_MODE_GRO) {
587 gro_size = tpa_info->gro_size;
588 full_page = tpa_info->full_page;
591 /* This is needed in order to enable forwarding support */
593 bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd,
594 le16_to_cpu(cqe->pkt_len),
595 le16_to_cpu(cqe->num_of_coalesced_segs));
597 #ifdef BNX2X_STOP_ON_ERROR
598 if (pages > min_t(u32, 8, MAX_SKB_FRAGS) * SGE_PAGES) {
599 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
601 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
607 /* Run through the SGL and compose the fragmented skb */
608 for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
609 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
611 /* FW gives the indices of the SGE as if the ring is an array
612 (meaning that "next" element will consume 2 indices) */
613 if (fp->mode == TPA_MODE_GRO)
614 frag_len = min_t(u32, frag_size, (u32)full_page);
616 frag_len = min_t(u32, frag_size, (u32)SGE_PAGES);
618 rx_pg = &fp->rx_page_ring[sge_idx];
621 /* If we fail to allocate a substitute page, we simply stop
622 where we are and drop the whole packet */
623 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx, GFP_ATOMIC);
625 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
629 /* Unmap the page as we're going to pass it to the stack */
630 dma_unmap_page(&bp->pdev->dev,
631 dma_unmap_addr(&old_rx_pg, mapping),
632 SGE_PAGES, DMA_FROM_DEVICE);
633 /* Add one frag and update the appropriate fields in the skb */
634 if (fp->mode == TPA_MODE_LRO)
635 skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
639 for (rem = frag_len; rem > 0; rem -= gro_size) {
640 int len = rem > gro_size ? gro_size : rem;
641 skb_fill_page_desc(skb, frag_id++,
642 old_rx_pg.page, offset, len);
644 get_page(old_rx_pg.page);
649 skb->data_len += frag_len;
650 skb->truesize += SGE_PAGES;
651 skb->len += frag_len;
653 frag_size -= frag_len;
659 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
661 if (fp->rx_frag_size)
662 put_page(virt_to_head_page(data));
667 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp, gfp_t gfp_mask)
669 if (fp->rx_frag_size) {
670 /* GFP_KERNEL allocations are used only during initialization */
671 if (unlikely(gfp_mask & __GFP_WAIT))
672 return (void *)__get_free_page(gfp_mask);
674 return netdev_alloc_frag(fp->rx_frag_size);
677 return kmalloc(fp->rx_buf_size + NET_SKB_PAD, gfp_mask);
681 static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb)
683 const struct iphdr *iph = ip_hdr(skb);
686 skb_set_transport_header(skb, sizeof(struct iphdr));
689 th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb),
690 iph->saddr, iph->daddr, 0);
693 static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb)
695 struct ipv6hdr *iph = ipv6_hdr(skb);
698 skb_set_transport_header(skb, sizeof(struct ipv6hdr));
701 th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb),
702 &iph->saddr, &iph->daddr, 0);
705 static void bnx2x_gro_csum(struct bnx2x *bp, struct sk_buff *skb,
706 void (*gro_func)(struct bnx2x*, struct sk_buff*))
708 skb_set_network_header(skb, 0);
710 tcp_gro_complete(skb);
714 static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp,
718 if (skb_shinfo(skb)->gso_size) {
719 switch (be16_to_cpu(skb->protocol)) {
721 bnx2x_gro_csum(bp, skb, bnx2x_gro_ip_csum);
724 bnx2x_gro_csum(bp, skb, bnx2x_gro_ipv6_csum);
727 BNX2X_ERR("Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n",
728 be16_to_cpu(skb->protocol));
732 skb_record_rx_queue(skb, fp->rx_queue);
733 napi_gro_receive(&fp->napi, skb);
736 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
737 struct bnx2x_agg_info *tpa_info,
739 struct eth_end_agg_rx_cqe *cqe,
742 struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
743 u8 pad = tpa_info->placement_offset;
744 u16 len = tpa_info->len_on_bd;
745 struct sk_buff *skb = NULL;
746 u8 *new_data, *data = rx_buf->data;
747 u8 old_tpa_state = tpa_info->tpa_state;
749 tpa_info->tpa_state = BNX2X_TPA_STOP;
751 /* If we there was an error during the handling of the TPA_START -
752 * drop this aggregation.
754 if (old_tpa_state == BNX2X_TPA_ERROR)
757 /* Try to allocate the new data */
758 new_data = bnx2x_frag_alloc(fp, GFP_ATOMIC);
759 /* Unmap skb in the pool anyway, as we are going to change
760 pool entry status to BNX2X_TPA_STOP even if new skb allocation
762 dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
763 fp->rx_buf_size, DMA_FROM_DEVICE);
764 if (likely(new_data))
765 skb = build_skb(data, fp->rx_frag_size);
768 #ifdef BNX2X_STOP_ON_ERROR
769 if (pad + len > fp->rx_buf_size) {
770 BNX2X_ERR("skb_put is about to fail... pad %d len %d rx_buf_size %d\n",
771 pad, len, fp->rx_buf_size);
777 skb_reserve(skb, pad + NET_SKB_PAD);
779 skb_set_hash(skb, tpa_info->rxhash, tpa_info->rxhash_type);
781 skb->protocol = eth_type_trans(skb, bp->dev);
782 skb->ip_summed = CHECKSUM_UNNECESSARY;
784 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
785 skb, cqe, cqe_idx)) {
786 if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
787 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tpa_info->vlan_tag);
788 bnx2x_gro_receive(bp, fp, skb);
790 DP(NETIF_MSG_RX_STATUS,
791 "Failed to allocate new pages - dropping packet!\n");
792 dev_kfree_skb_any(skb);
795 /* put new data in bin */
796 rx_buf->data = new_data;
800 bnx2x_frag_free(fp, new_data);
802 /* drop the packet and keep the buffer in the bin */
803 DP(NETIF_MSG_RX_STATUS,
804 "Failed to allocate or map a new skb - dropping packet!\n");
805 bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
808 static int bnx2x_alloc_rx_data(struct bnx2x *bp, struct bnx2x_fastpath *fp,
809 u16 index, gfp_t gfp_mask)
812 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
813 struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
816 data = bnx2x_frag_alloc(fp, gfp_mask);
817 if (unlikely(data == NULL))
820 mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
823 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
824 bnx2x_frag_free(fp, data);
825 BNX2X_ERR("Can't map rx data\n");
830 dma_unmap_addr_set(rx_buf, mapping, mapping);
832 rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
833 rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
839 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
840 struct bnx2x_fastpath *fp,
841 struct bnx2x_eth_q_stats *qstats)
843 /* Do nothing if no L4 csum validation was done.
844 * We do not check whether IP csum was validated. For IPv4 we assume
845 * that if the card got as far as validating the L4 csum, it also
846 * validated the IP csum. IPv6 has no IP csum.
848 if (cqe->fast_path_cqe.status_flags &
849 ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
852 /* If L4 validation was done, check if an error was found. */
854 if (cqe->fast_path_cqe.type_error_flags &
855 (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
856 ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
857 qstats->hw_csum_err++;
859 skb->ip_summed = CHECKSUM_UNNECESSARY;
862 static int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
864 struct bnx2x *bp = fp->bp;
865 u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
866 u16 sw_comp_cons, sw_comp_prod;
868 union eth_rx_cqe *cqe;
869 struct eth_fast_path_rx_cqe *cqe_fp;
871 #ifdef BNX2X_STOP_ON_ERROR
872 if (unlikely(bp->panic))
878 bd_cons = fp->rx_bd_cons;
879 bd_prod = fp->rx_bd_prod;
880 bd_prod_fw = bd_prod;
881 sw_comp_cons = fp->rx_comp_cons;
882 sw_comp_prod = fp->rx_comp_prod;
884 comp_ring_cons = RCQ_BD(sw_comp_cons);
885 cqe = &fp->rx_comp_ring[comp_ring_cons];
886 cqe_fp = &cqe->fast_path_cqe;
888 DP(NETIF_MSG_RX_STATUS,
889 "queue[%d]: sw_comp_cons %u\n", fp->index, sw_comp_cons);
891 while (BNX2X_IS_CQE_COMPLETED(cqe_fp)) {
892 struct sw_rx_bd *rx_buf = NULL;
895 enum eth_rx_cqe_type cqe_fp_type;
899 enum pkt_hash_types rxhash_type;
901 #ifdef BNX2X_STOP_ON_ERROR
902 if (unlikely(bp->panic))
906 bd_prod = RX_BD(bd_prod);
907 bd_cons = RX_BD(bd_cons);
909 /* A rmb() is required to ensure that the CQE is not read
910 * before it is written by the adapter DMA. PCI ordering
911 * rules will make sure the other fields are written before
912 * the marker at the end of struct eth_fast_path_rx_cqe
913 * but without rmb() a weakly ordered processor can process
914 * stale data. Without the barrier TPA state-machine might
915 * enter inconsistent state and kernel stack might be
916 * provided with incorrect packet description - these lead
917 * to various kernel crashed.
921 cqe_fp_flags = cqe_fp->type_error_flags;
922 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
924 DP(NETIF_MSG_RX_STATUS,
925 "CQE type %x err %x status %x queue %x vlan %x len %u\n",
926 CQE_TYPE(cqe_fp_flags),
927 cqe_fp_flags, cqe_fp->status_flags,
928 le32_to_cpu(cqe_fp->rss_hash_result),
929 le16_to_cpu(cqe_fp->vlan_tag),
930 le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
932 /* is this a slowpath msg? */
933 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
934 bnx2x_sp_event(fp, cqe);
938 rx_buf = &fp->rx_buf_ring[bd_cons];
941 if (!CQE_TYPE_FAST(cqe_fp_type)) {
942 struct bnx2x_agg_info *tpa_info;
943 u16 frag_size, pages;
944 #ifdef BNX2X_STOP_ON_ERROR
946 if (fp->disable_tpa &&
947 (CQE_TYPE_START(cqe_fp_type) ||
948 CQE_TYPE_STOP(cqe_fp_type)))
949 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
950 CQE_TYPE(cqe_fp_type));
953 if (CQE_TYPE_START(cqe_fp_type)) {
954 u16 queue = cqe_fp->queue_index;
955 DP(NETIF_MSG_RX_STATUS,
956 "calling tpa_start on queue %d\n",
959 bnx2x_tpa_start(fp, queue,
965 queue = cqe->end_agg_cqe.queue_index;
966 tpa_info = &fp->tpa_info[queue];
967 DP(NETIF_MSG_RX_STATUS,
968 "calling tpa_stop on queue %d\n",
971 frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
974 if (fp->mode == TPA_MODE_GRO)
975 pages = (frag_size + tpa_info->full_page - 1) /
978 pages = SGE_PAGE_ALIGN(frag_size) >>
981 bnx2x_tpa_stop(bp, fp, tpa_info, pages,
982 &cqe->end_agg_cqe, comp_ring_cons);
983 #ifdef BNX2X_STOP_ON_ERROR
988 bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
992 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
993 pad = cqe_fp->placement_offset;
994 dma_sync_single_for_cpu(&bp->pdev->dev,
995 dma_unmap_addr(rx_buf, mapping),
996 pad + RX_COPY_THRESH,
999 prefetch(data + pad); /* speedup eth_type_trans() */
1000 /* is this an error packet? */
1001 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
1002 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1003 "ERROR flags %x rx packet %u\n",
1004 cqe_fp_flags, sw_comp_cons);
1005 bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
1009 /* Since we don't have a jumbo ring
1010 * copy small packets if mtu > 1500
1012 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
1013 (len <= RX_COPY_THRESH)) {
1014 skb = netdev_alloc_skb_ip_align(bp->dev, len);
1016 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1017 "ERROR packet dropped because of alloc failure\n");
1018 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1021 memcpy(skb->data, data + pad, len);
1022 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1024 if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod,
1025 GFP_ATOMIC) == 0)) {
1026 dma_unmap_single(&bp->pdev->dev,
1027 dma_unmap_addr(rx_buf, mapping),
1030 skb = build_skb(data, fp->rx_frag_size);
1031 if (unlikely(!skb)) {
1032 bnx2x_frag_free(fp, data);
1033 bnx2x_fp_qstats(bp, fp)->
1034 rx_skb_alloc_failed++;
1037 skb_reserve(skb, pad);
1039 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1040 "ERROR packet dropped because of alloc failure\n");
1041 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1043 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1049 skb->protocol = eth_type_trans(skb, bp->dev);
1051 /* Set Toeplitz hash for a none-LRO skb */
1052 rxhash = bnx2x_get_rxhash(bp, cqe_fp, &rxhash_type);
1053 skb_set_hash(skb, rxhash, rxhash_type);
1055 skb_checksum_none_assert(skb);
1057 if (bp->dev->features & NETIF_F_RXCSUM)
1058 bnx2x_csum_validate(skb, cqe, fp,
1059 bnx2x_fp_qstats(bp, fp));
1061 skb_record_rx_queue(skb, fp->rx_queue);
1063 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
1065 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1066 le16_to_cpu(cqe_fp->vlan_tag));
1068 skb_mark_napi_id(skb, &fp->napi);
1070 if (bnx2x_fp_ll_polling(fp))
1071 netif_receive_skb(skb);
1073 napi_gro_receive(&fp->napi, skb);
1075 rx_buf->data = NULL;
1077 bd_cons = NEXT_RX_IDX(bd_cons);
1078 bd_prod = NEXT_RX_IDX(bd_prod);
1079 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
1082 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
1083 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
1085 /* mark CQE as free */
1086 BNX2X_SEED_CQE(cqe_fp);
1088 if (rx_pkt == budget)
1091 comp_ring_cons = RCQ_BD(sw_comp_cons);
1092 cqe = &fp->rx_comp_ring[comp_ring_cons];
1093 cqe_fp = &cqe->fast_path_cqe;
1096 fp->rx_bd_cons = bd_cons;
1097 fp->rx_bd_prod = bd_prod_fw;
1098 fp->rx_comp_cons = sw_comp_cons;
1099 fp->rx_comp_prod = sw_comp_prod;
1101 /* Update producers */
1102 bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
1105 fp->rx_pkt += rx_pkt;
1111 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
1113 struct bnx2x_fastpath *fp = fp_cookie;
1114 struct bnx2x *bp = fp->bp;
1118 "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
1119 fp->index, fp->fw_sb_id, fp->igu_sb_id);
1121 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
1123 #ifdef BNX2X_STOP_ON_ERROR
1124 if (unlikely(bp->panic))
1128 /* Handle Rx and Tx according to MSI-X vector */
1129 for_each_cos_in_tx_queue(fp, cos)
1130 prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
1132 prefetch(&fp->sb_running_index[SM_RX_ID]);
1133 napi_schedule(&bnx2x_fp(bp, fp->index, napi));
1138 /* HW Lock for shared dual port PHYs */
1139 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
1141 mutex_lock(&bp->port.phy_mutex);
1143 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1146 void bnx2x_release_phy_lock(struct bnx2x *bp)
1148 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1150 mutex_unlock(&bp->port.phy_mutex);
1153 /* calculates MF speed according to current linespeed and MF configuration */
1154 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
1156 u16 line_speed = bp->link_vars.line_speed;
1158 u16 maxCfg = bnx2x_extract_max_cfg(bp,
1159 bp->mf_config[BP_VN(bp)]);
1161 /* Calculate the current MAX line speed limit for the MF
1165 line_speed = (line_speed * maxCfg) / 100;
1166 else { /* SD mode */
1167 u16 vn_max_rate = maxCfg * 100;
1169 if (vn_max_rate < line_speed)
1170 line_speed = vn_max_rate;
1178 * bnx2x_fill_report_data - fill link report data to report
1180 * @bp: driver handle
1181 * @data: link state to update
1183 * It uses a none-atomic bit operations because is called under the mutex.
1185 static void bnx2x_fill_report_data(struct bnx2x *bp,
1186 struct bnx2x_link_report_data *data)
1188 u16 line_speed = bnx2x_get_mf_speed(bp);
1190 memset(data, 0, sizeof(*data));
1192 /* Fill the report data: effective line speed */
1193 data->line_speed = line_speed;
1196 if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1197 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1198 &data->link_report_flags);
1201 if (bp->link_vars.duplex == DUPLEX_FULL)
1202 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
1204 /* Rx Flow Control is ON */
1205 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1206 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
1208 /* Tx Flow Control is ON */
1209 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1210 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
1214 * bnx2x_link_report - report link status to OS.
1216 * @bp: driver handle
1218 * Calls the __bnx2x_link_report() under the same locking scheme
1219 * as a link/PHY state managing code to ensure a consistent link
1223 void bnx2x_link_report(struct bnx2x *bp)
1225 bnx2x_acquire_phy_lock(bp);
1226 __bnx2x_link_report(bp);
1227 bnx2x_release_phy_lock(bp);
1231 * __bnx2x_link_report - report link status to OS.
1233 * @bp: driver handle
1235 * None atomic implementation.
1236 * Should be called under the phy_lock.
1238 void __bnx2x_link_report(struct bnx2x *bp)
1240 struct bnx2x_link_report_data cur_data;
1243 if (IS_PF(bp) && !CHIP_IS_E1(bp))
1244 bnx2x_read_mf_cfg(bp);
1246 /* Read the current link report info */
1247 bnx2x_fill_report_data(bp, &cur_data);
1249 /* Don't report link down or exactly the same link status twice */
1250 if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1251 (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1252 &bp->last_reported_link.link_report_flags) &&
1253 test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1254 &cur_data.link_report_flags)))
1259 /* We are going to report a new link parameters now -
1260 * remember the current data for the next time.
1262 memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1264 if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1265 &cur_data.link_report_flags)) {
1266 netif_carrier_off(bp->dev);
1267 netdev_err(bp->dev, "NIC Link is Down\n");
1273 netif_carrier_on(bp->dev);
1275 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1276 &cur_data.link_report_flags))
1281 /* Handle the FC at the end so that only these flags would be
1282 * possibly set. This way we may easily check if there is no FC
1285 if (cur_data.link_report_flags) {
1286 if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1287 &cur_data.link_report_flags)) {
1288 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1289 &cur_data.link_report_flags))
1290 flow = "ON - receive & transmit";
1292 flow = "ON - receive";
1294 flow = "ON - transmit";
1299 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1300 cur_data.line_speed, duplex, flow);
1304 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1308 for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1309 struct eth_rx_sge *sge;
1311 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1313 cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1314 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1317 cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1318 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1322 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1323 struct bnx2x_fastpath *fp, int last)
1327 for (i = 0; i < last; i++) {
1328 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1329 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1330 u8 *data = first_buf->data;
1333 DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1336 if (tpa_info->tpa_state == BNX2X_TPA_START)
1337 dma_unmap_single(&bp->pdev->dev,
1338 dma_unmap_addr(first_buf, mapping),
1339 fp->rx_buf_size, DMA_FROM_DEVICE);
1340 bnx2x_frag_free(fp, data);
1341 first_buf->data = NULL;
1345 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1349 for_each_rx_queue_cnic(bp, j) {
1350 struct bnx2x_fastpath *fp = &bp->fp[j];
1354 /* Activate BD ring */
1356 * this will generate an interrupt (to the TSTORM)
1357 * must only be done after chip is initialized
1359 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1364 void bnx2x_init_rx_rings(struct bnx2x *bp)
1366 int func = BP_FUNC(bp);
1370 /* Allocate TPA resources */
1371 for_each_eth_queue(bp, j) {
1372 struct bnx2x_fastpath *fp = &bp->fp[j];
1375 "mtu %d rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1377 if (!fp->disable_tpa) {
1378 /* Fill the per-aggregation pool */
1379 for (i = 0; i < MAX_AGG_QS(bp); i++) {
1380 struct bnx2x_agg_info *tpa_info =
1382 struct sw_rx_bd *first_buf =
1383 &tpa_info->first_buf;
1386 bnx2x_frag_alloc(fp, GFP_KERNEL);
1387 if (!first_buf->data) {
1388 BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1390 bnx2x_free_tpa_pool(bp, fp, i);
1391 fp->disable_tpa = 1;
1394 dma_unmap_addr_set(first_buf, mapping, 0);
1395 tpa_info->tpa_state = BNX2X_TPA_STOP;
1398 /* "next page" elements initialization */
1399 bnx2x_set_next_page_sgl(fp);
1401 /* set SGEs bit mask */
1402 bnx2x_init_sge_ring_bit_mask(fp);
1404 /* Allocate SGEs and initialize the ring elements */
1405 for (i = 0, ring_prod = 0;
1406 i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1408 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod,
1410 BNX2X_ERR("was only able to allocate %d rx sges\n",
1412 BNX2X_ERR("disabling TPA for queue[%d]\n",
1414 /* Cleanup already allocated elements */
1415 bnx2x_free_rx_sge_range(bp, fp,
1417 bnx2x_free_tpa_pool(bp, fp,
1419 fp->disable_tpa = 1;
1423 ring_prod = NEXT_SGE_IDX(ring_prod);
1426 fp->rx_sge_prod = ring_prod;
1430 for_each_eth_queue(bp, j) {
1431 struct bnx2x_fastpath *fp = &bp->fp[j];
1435 /* Activate BD ring */
1437 * this will generate an interrupt (to the TSTORM)
1438 * must only be done after chip is initialized
1440 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1446 if (CHIP_IS_E1(bp)) {
1447 REG_WR(bp, BAR_USTRORM_INTMEM +
1448 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1449 U64_LO(fp->rx_comp_mapping));
1450 REG_WR(bp, BAR_USTRORM_INTMEM +
1451 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1452 U64_HI(fp->rx_comp_mapping));
1457 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1460 struct bnx2x *bp = fp->bp;
1462 for_each_cos_in_tx_queue(fp, cos) {
1463 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1464 unsigned pkts_compl = 0, bytes_compl = 0;
1466 u16 sw_prod = txdata->tx_pkt_prod;
1467 u16 sw_cons = txdata->tx_pkt_cons;
1469 while (sw_cons != sw_prod) {
1470 bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1471 &pkts_compl, &bytes_compl);
1475 netdev_tx_reset_queue(
1476 netdev_get_tx_queue(bp->dev,
1477 txdata->txq_index));
1481 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1485 for_each_tx_queue_cnic(bp, i) {
1486 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1490 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1494 for_each_eth_queue(bp, i) {
1495 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1499 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1501 struct bnx2x *bp = fp->bp;
1504 /* ring wasn't allocated */
1505 if (fp->rx_buf_ring == NULL)
1508 for (i = 0; i < NUM_RX_BD; i++) {
1509 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1510 u8 *data = rx_buf->data;
1514 dma_unmap_single(&bp->pdev->dev,
1515 dma_unmap_addr(rx_buf, mapping),
1516 fp->rx_buf_size, DMA_FROM_DEVICE);
1518 rx_buf->data = NULL;
1519 bnx2x_frag_free(fp, data);
1523 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1527 for_each_rx_queue_cnic(bp, j) {
1528 bnx2x_free_rx_bds(&bp->fp[j]);
1532 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1536 for_each_eth_queue(bp, j) {
1537 struct bnx2x_fastpath *fp = &bp->fp[j];
1539 bnx2x_free_rx_bds(fp);
1541 if (!fp->disable_tpa)
1542 bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1546 static void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1548 bnx2x_free_tx_skbs_cnic(bp);
1549 bnx2x_free_rx_skbs_cnic(bp);
1552 void bnx2x_free_skbs(struct bnx2x *bp)
1554 bnx2x_free_tx_skbs(bp);
1555 bnx2x_free_rx_skbs(bp);
1558 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1560 /* load old values */
1561 u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1563 if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1564 /* leave all but MAX value */
1565 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1567 /* set new MAX value */
1568 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1569 & FUNC_MF_CFG_MAX_BW_MASK;
1571 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1576 * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1578 * @bp: driver handle
1579 * @nvecs: number of vectors to be released
1581 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1585 if (nvecs == offset)
1588 /* VFs don't have a default SB */
1590 free_irq(bp->msix_table[offset].vector, bp->dev);
1591 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1592 bp->msix_table[offset].vector);
1596 if (CNIC_SUPPORT(bp)) {
1597 if (nvecs == offset)
1602 for_each_eth_queue(bp, i) {
1603 if (nvecs == offset)
1605 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1606 i, bp->msix_table[offset].vector);
1608 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1612 void bnx2x_free_irq(struct bnx2x *bp)
1614 if (bp->flags & USING_MSIX_FLAG &&
1615 !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1616 int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp);
1618 /* vfs don't have a default status block */
1622 bnx2x_free_msix_irqs(bp, nvecs);
1624 free_irq(bp->dev->irq, bp->dev);
1628 int bnx2x_enable_msix(struct bnx2x *bp)
1630 int msix_vec = 0, i, rc;
1632 /* VFs don't have a default status block */
1634 bp->msix_table[msix_vec].entry = msix_vec;
1635 BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1636 bp->msix_table[0].entry);
1640 /* Cnic requires an msix vector for itself */
1641 if (CNIC_SUPPORT(bp)) {
1642 bp->msix_table[msix_vec].entry = msix_vec;
1643 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1644 msix_vec, bp->msix_table[msix_vec].entry);
1648 /* We need separate vectors for ETH queues only (not FCoE) */
1649 for_each_eth_queue(bp, i) {
1650 bp->msix_table[msix_vec].entry = msix_vec;
1651 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1652 msix_vec, msix_vec, i);
1656 DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n",
1659 rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0],
1660 BNX2X_MIN_MSIX_VEC_CNT(bp), msix_vec);
1662 * reconfigure number of tx/rx queues according to available
1665 if (rc == -ENOSPC) {
1666 /* Get by with single vector */
1667 rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0], 1, 1);
1669 BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1674 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1675 bp->flags |= USING_SINGLE_MSIX_FLAG;
1677 BNX2X_DEV_INFO("set number of queues to 1\n");
1678 bp->num_ethernet_queues = 1;
1679 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1680 } else if (rc < 0) {
1681 BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1683 } else if (rc < msix_vec) {
1684 /* how less vectors we will have? */
1685 int diff = msix_vec - rc;
1687 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1690 * decrease number of queues by number of unallocated entries
1692 bp->num_ethernet_queues -= diff;
1693 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1695 BNX2X_DEV_INFO("New queue configuration set: %d\n",
1699 bp->flags |= USING_MSIX_FLAG;
1704 /* fall to INTx if not enough memory */
1706 bp->flags |= DISABLE_MSI_FLAG;
1711 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1713 int i, rc, offset = 0;
1715 /* no default status block for vf */
1717 rc = request_irq(bp->msix_table[offset++].vector,
1718 bnx2x_msix_sp_int, 0,
1719 bp->dev->name, bp->dev);
1721 BNX2X_ERR("request sp irq failed\n");
1726 if (CNIC_SUPPORT(bp))
1729 for_each_eth_queue(bp, i) {
1730 struct bnx2x_fastpath *fp = &bp->fp[i];
1731 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1734 rc = request_irq(bp->msix_table[offset].vector,
1735 bnx2x_msix_fp_int, 0, fp->name, fp);
1737 BNX2X_ERR("request fp #%d irq (%d) failed rc %d\n", i,
1738 bp->msix_table[offset].vector, rc);
1739 bnx2x_free_msix_irqs(bp, offset);
1746 i = BNX2X_NUM_ETH_QUEUES(bp);
1748 offset = 1 + CNIC_SUPPORT(bp);
1749 netdev_info(bp->dev,
1750 "using MSI-X IRQs: sp %d fp[%d] %d ... fp[%d] %d\n",
1751 bp->msix_table[0].vector,
1752 0, bp->msix_table[offset].vector,
1753 i - 1, bp->msix_table[offset + i - 1].vector);
1755 offset = CNIC_SUPPORT(bp);
1756 netdev_info(bp->dev,
1757 "using MSI-X IRQs: fp[%d] %d ... fp[%d] %d\n",
1758 0, bp->msix_table[offset].vector,
1759 i - 1, bp->msix_table[offset + i - 1].vector);
1764 int bnx2x_enable_msi(struct bnx2x *bp)
1768 rc = pci_enable_msi(bp->pdev);
1770 BNX2X_DEV_INFO("MSI is not attainable\n");
1773 bp->flags |= USING_MSI_FLAG;
1778 static int bnx2x_req_irq(struct bnx2x *bp)
1780 unsigned long flags;
1783 if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1786 flags = IRQF_SHARED;
1788 if (bp->flags & USING_MSIX_FLAG)
1789 irq = bp->msix_table[0].vector;
1791 irq = bp->pdev->irq;
1793 return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1796 static int bnx2x_setup_irqs(struct bnx2x *bp)
1799 if (bp->flags & USING_MSIX_FLAG &&
1800 !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1801 rc = bnx2x_req_msix_irqs(bp);
1805 rc = bnx2x_req_irq(bp);
1807 BNX2X_ERR("IRQ request failed rc %d, aborting\n", rc);
1810 if (bp->flags & USING_MSI_FLAG) {
1811 bp->dev->irq = bp->pdev->irq;
1812 netdev_info(bp->dev, "using MSI IRQ %d\n",
1815 if (bp->flags & USING_MSIX_FLAG) {
1816 bp->dev->irq = bp->msix_table[0].vector;
1817 netdev_info(bp->dev, "using MSIX IRQ %d\n",
1825 static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1829 for_each_rx_queue_cnic(bp, i) {
1830 bnx2x_fp_init_lock(&bp->fp[i]);
1831 napi_enable(&bnx2x_fp(bp, i, napi));
1835 static void bnx2x_napi_enable(struct bnx2x *bp)
1839 for_each_eth_queue(bp, i) {
1840 bnx2x_fp_init_lock(&bp->fp[i]);
1841 napi_enable(&bnx2x_fp(bp, i, napi));
1845 static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1849 for_each_rx_queue_cnic(bp, i) {
1850 napi_disable(&bnx2x_fp(bp, i, napi));
1851 while (!bnx2x_fp_ll_disable(&bp->fp[i]))
1852 usleep_range(1000, 2000);
1856 static void bnx2x_napi_disable(struct bnx2x *bp)
1860 for_each_eth_queue(bp, i) {
1861 napi_disable(&bnx2x_fp(bp, i, napi));
1862 while (!bnx2x_fp_ll_disable(&bp->fp[i]))
1863 usleep_range(1000, 2000);
1867 void bnx2x_netif_start(struct bnx2x *bp)
1869 if (netif_running(bp->dev)) {
1870 bnx2x_napi_enable(bp);
1871 if (CNIC_LOADED(bp))
1872 bnx2x_napi_enable_cnic(bp);
1873 bnx2x_int_enable(bp);
1874 if (bp->state == BNX2X_STATE_OPEN)
1875 netif_tx_wake_all_queues(bp->dev);
1879 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1881 bnx2x_int_disable_sync(bp, disable_hw);
1882 bnx2x_napi_disable(bp);
1883 if (CNIC_LOADED(bp))
1884 bnx2x_napi_disable_cnic(bp);
1887 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb,
1888 void *accel_priv, select_queue_fallback_t fallback)
1890 struct bnx2x *bp = netdev_priv(dev);
1892 if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1893 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1894 u16 ether_type = ntohs(hdr->h_proto);
1896 /* Skip VLAN tag if present */
1897 if (ether_type == ETH_P_8021Q) {
1898 struct vlan_ethhdr *vhdr =
1899 (struct vlan_ethhdr *)skb->data;
1901 ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1904 /* If ethertype is FCoE or FIP - use FCoE ring */
1905 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1906 return bnx2x_fcoe_tx(bp, txq_index);
1909 /* select a non-FCoE queue */
1910 return fallback(dev, skb) % BNX2X_NUM_ETH_QUEUES(bp);
1913 void bnx2x_set_num_queues(struct bnx2x *bp)
1916 bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1918 /* override in STORAGE SD modes */
1919 if (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))
1920 bp->num_ethernet_queues = 1;
1922 /* Add special queues */
1923 bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1924 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1926 BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1930 * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1932 * @bp: Driver handle
1934 * We currently support for at most 16 Tx queues for each CoS thus we will
1935 * allocate a multiple of 16 for ETH L2 rings according to the value of the
1938 * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1939 * index after all ETH L2 indices.
1941 * If the actual number of Tx queues (for each CoS) is less than 16 then there
1942 * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1943 * 16..31,...) with indices that are not coupled with any real Tx queue.
1945 * The proper configuration of skb->queue_mapping is handled by
1946 * bnx2x_select_queue() and __skb_tx_hash().
1948 * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1949 * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1951 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
1955 tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1956 rx = BNX2X_NUM_ETH_QUEUES(bp);
1958 /* account for fcoe queue */
1959 if (include_cnic && !NO_FCOE(bp)) {
1964 rc = netif_set_real_num_tx_queues(bp->dev, tx);
1966 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1969 rc = netif_set_real_num_rx_queues(bp->dev, rx);
1971 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1975 DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
1981 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1985 for_each_queue(bp, i) {
1986 struct bnx2x_fastpath *fp = &bp->fp[i];
1989 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1992 * Although there are no IP frames expected to arrive to
1993 * this ring we still want to add an
1994 * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1997 mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
2000 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
2001 IP_HEADER_ALIGNMENT_PADDING +
2004 BNX2X_FW_RX_ALIGN_END;
2005 /* Note : rx_buf_size doesn't take into account NET_SKB_PAD */
2006 if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
2007 fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
2009 fp->rx_frag_size = 0;
2013 static int bnx2x_init_rss(struct bnx2x *bp)
2016 u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
2018 /* Prepare the initial contents for the indirection table if RSS is
2021 for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
2022 bp->rss_conf_obj.ind_table[i] =
2024 ethtool_rxfh_indir_default(i, num_eth_queues);
2027 * For 57710 and 57711 SEARCHER configuration (rss_keys) is
2028 * per-port, so if explicit configuration is needed , do it only
2031 * For 57712 and newer on the other hand it's a per-function
2034 return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
2037 int bnx2x_rss(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
2038 bool config_hash, bool enable)
2040 struct bnx2x_config_rss_params params = {NULL};
2042 /* Although RSS is meaningless when there is a single HW queue we
2043 * still need it enabled in order to have HW Rx hash generated.
2045 * if (!is_eth_multi(bp))
2046 * bp->multi_mode = ETH_RSS_MODE_DISABLED;
2049 params.rss_obj = rss_obj;
2051 __set_bit(RAMROD_COMP_WAIT, ¶ms.ramrod_flags);
2054 __set_bit(BNX2X_RSS_MODE_REGULAR, ¶ms.rss_flags);
2056 /* RSS configuration */
2057 __set_bit(BNX2X_RSS_IPV4, ¶ms.rss_flags);
2058 __set_bit(BNX2X_RSS_IPV4_TCP, ¶ms.rss_flags);
2059 __set_bit(BNX2X_RSS_IPV6, ¶ms.rss_flags);
2060 __set_bit(BNX2X_RSS_IPV6_TCP, ¶ms.rss_flags);
2061 if (rss_obj->udp_rss_v4)
2062 __set_bit(BNX2X_RSS_IPV4_UDP, ¶ms.rss_flags);
2063 if (rss_obj->udp_rss_v6)
2064 __set_bit(BNX2X_RSS_IPV6_UDP, ¶ms.rss_flags);
2066 __set_bit(BNX2X_RSS_MODE_DISABLED, ¶ms.rss_flags);
2070 params.rss_result_mask = MULTI_MASK;
2072 memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
2076 prandom_bytes(params.rss_key, T_ETH_RSS_KEY * 4);
2077 __set_bit(BNX2X_RSS_SET_SRCH, ¶ms.rss_flags);
2081 return bnx2x_config_rss(bp, ¶ms);
2083 return bnx2x_vfpf_config_rss(bp, ¶ms);
2086 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
2088 struct bnx2x_func_state_params func_params = {NULL};
2090 /* Prepare parameters for function state transitions */
2091 __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
2093 func_params.f_obj = &bp->func_obj;
2094 func_params.cmd = BNX2X_F_CMD_HW_INIT;
2096 func_params.params.hw_init.load_phase = load_code;
2098 return bnx2x_func_state_change(bp, &func_params);
2102 * Cleans the object that have internal lists without sending
2103 * ramrods. Should be run when interrupts are disabled.
2105 void bnx2x_squeeze_objects(struct bnx2x *bp)
2108 unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
2109 struct bnx2x_mcast_ramrod_params rparam = {NULL};
2110 struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
2112 /***************** Cleanup MACs' object first *************************/
2114 /* Wait for completion of requested */
2115 __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
2116 /* Perform a dry cleanup */
2117 __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
2119 /* Clean ETH primary MAC */
2120 __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
2121 rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
2124 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
2126 /* Cleanup UC list */
2128 __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
2129 rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
2132 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
2134 /***************** Now clean mcast object *****************************/
2135 rparam.mcast_obj = &bp->mcast_obj;
2136 __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
2138 /* Add a DEL command... - Since we're doing a driver cleanup only,
2139 * we take a lock surrounding both the initial send and the CONTs,
2140 * as we don't want a true completion to disrupt us in the middle.
2142 netif_addr_lock_bh(bp->dev);
2143 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
2145 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
2148 /* ...and wait until all pending commands are cleared */
2149 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2152 BNX2X_ERR("Failed to clean multi-cast object: %d\n",
2154 netif_addr_unlock_bh(bp->dev);
2158 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2160 netif_addr_unlock_bh(bp->dev);
2163 #ifndef BNX2X_STOP_ON_ERROR
2164 #define LOAD_ERROR_EXIT(bp, label) \
2166 (bp)->state = BNX2X_STATE_ERROR; \
2170 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2172 bp->cnic_loaded = false; \
2175 #else /*BNX2X_STOP_ON_ERROR*/
2176 #define LOAD_ERROR_EXIT(bp, label) \
2178 (bp)->state = BNX2X_STATE_ERROR; \
2182 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2184 bp->cnic_loaded = false; \
2188 #endif /*BNX2X_STOP_ON_ERROR*/
2190 static void bnx2x_free_fw_stats_mem(struct bnx2x *bp)
2192 BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping,
2193 bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2197 static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp)
2199 int num_groups, vf_headroom = 0;
2200 int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1;
2202 /* number of queues for statistics is number of eth queues + FCoE */
2203 u8 num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats;
2205 /* Total number of FW statistics requests =
2206 * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper
2207 * and fcoe l2 queue) stats + num of queues (which includes another 1
2208 * for fcoe l2 queue if applicable)
2210 bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats;
2212 /* vf stats appear in the request list, but their data is allocated by
2213 * the VFs themselves. We don't include them in the bp->fw_stats_num as
2214 * it is used to determine where to place the vf stats queries in the
2218 vf_headroom = bnx2x_vf_headroom(bp);
2220 /* Request is built from stats_query_header and an array of
2221 * stats_query_cmd_group each of which contains
2222 * STATS_QUERY_CMD_COUNT rules. The real number or requests is
2223 * configured in the stats_query_header.
2226 (((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) +
2227 (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ?
2230 DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n",
2231 bp->fw_stats_num, vf_headroom, num_groups);
2232 bp->fw_stats_req_sz = sizeof(struct stats_query_header) +
2233 num_groups * sizeof(struct stats_query_cmd_group);
2235 /* Data for statistics requests + stats_counter
2236 * stats_counter holds per-STORM counters that are incremented
2237 * when STORM has finished with the current request.
2238 * memory for FCoE offloaded statistics are counted anyway,
2239 * even if they will not be sent.
2240 * VF stats are not accounted for here as the data of VF stats is stored
2241 * in memory allocated by the VF, not here.
2243 bp->fw_stats_data_sz = sizeof(struct per_port_stats) +
2244 sizeof(struct per_pf_stats) +
2245 sizeof(struct fcoe_statistics_params) +
2246 sizeof(struct per_queue_stats) * num_queue_stats +
2247 sizeof(struct stats_counter);
2249 bp->fw_stats = BNX2X_PCI_ALLOC(&bp->fw_stats_mapping,
2250 bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2255 bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats;
2256 bp->fw_stats_req_mapping = bp->fw_stats_mapping;
2257 bp->fw_stats_data = (struct bnx2x_fw_stats_data *)
2258 ((u8 *)bp->fw_stats + bp->fw_stats_req_sz);
2259 bp->fw_stats_data_mapping = bp->fw_stats_mapping +
2260 bp->fw_stats_req_sz;
2262 DP(BNX2X_MSG_SP, "statistics request base address set to %x %x\n",
2263 U64_HI(bp->fw_stats_req_mapping),
2264 U64_LO(bp->fw_stats_req_mapping));
2265 DP(BNX2X_MSG_SP, "statistics data base address set to %x %x\n",
2266 U64_HI(bp->fw_stats_data_mapping),
2267 U64_LO(bp->fw_stats_data_mapping));
2271 bnx2x_free_fw_stats_mem(bp);
2272 BNX2X_ERR("Can't allocate FW stats memory\n");
2276 /* send load request to mcp and analyze response */
2277 static int bnx2x_nic_load_request(struct bnx2x *bp, u32 *load_code)
2283 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2284 DRV_MSG_SEQ_NUMBER_MASK);
2285 BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2287 /* Get current FW pulse sequence */
2288 bp->fw_drv_pulse_wr_seq =
2289 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2290 DRV_PULSE_SEQ_MASK);
2291 BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2293 param = DRV_MSG_CODE_LOAD_REQ_WITH_LFA;
2295 if (IS_MF_SD(bp) && bnx2x_port_after_undi(bp))
2296 param |= DRV_MSG_CODE_LOAD_REQ_FORCE_LFA;
2299 (*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, param);
2301 /* if mcp fails to respond we must abort */
2302 if (!(*load_code)) {
2303 BNX2X_ERR("MCP response failure, aborting\n");
2307 /* If mcp refused (e.g. other port is in diagnostic mode) we
2310 if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2311 BNX2X_ERR("MCP refused load request, aborting\n");
2317 /* check whether another PF has already loaded FW to chip. In
2318 * virtualized environments a pf from another VM may have already
2319 * initialized the device including loading FW
2321 int bnx2x_compare_fw_ver(struct bnx2x *bp, u32 load_code, bool print_err)
2323 /* is another pf loaded on this engine? */
2324 if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2325 load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2326 /* build my FW version dword */
2327 u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
2328 (BCM_5710_FW_MINOR_VERSION << 8) +
2329 (BCM_5710_FW_REVISION_VERSION << 16) +
2330 (BCM_5710_FW_ENGINEERING_VERSION << 24);
2332 /* read loaded FW from chip */
2333 u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2335 DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x\n",
2338 /* abort nic load if version mismatch */
2339 if (my_fw != loaded_fw) {
2341 BNX2X_ERR("bnx2x with FW %x was already loaded which mismatches my %x FW. Aborting\n",
2344 BNX2X_DEV_INFO("bnx2x with FW %x was already loaded which mismatches my %x FW, possibly due to MF UNDI\n",
2352 /* returns the "mcp load_code" according to global load_count array */
2353 static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port)
2355 int path = BP_PATH(bp);
2357 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d] %d, %d, %d\n",
2358 path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2359 bnx2x_load_count[path][2]);
2360 bnx2x_load_count[path][0]++;
2361 bnx2x_load_count[path][1 + port]++;
2362 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d] %d, %d, %d\n",
2363 path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2364 bnx2x_load_count[path][2]);
2365 if (bnx2x_load_count[path][0] == 1)
2366 return FW_MSG_CODE_DRV_LOAD_COMMON;
2367 else if (bnx2x_load_count[path][1 + port] == 1)
2368 return FW_MSG_CODE_DRV_LOAD_PORT;
2370 return FW_MSG_CODE_DRV_LOAD_FUNCTION;
2373 /* mark PMF if applicable */
2374 static void bnx2x_nic_load_pmf(struct bnx2x *bp, u32 load_code)
2376 if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2377 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2378 (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2380 /* We need the barrier to ensure the ordering between the
2381 * writing to bp->port.pmf here and reading it from the
2382 * bnx2x_periodic_task().
2389 DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
2392 static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code)
2394 if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2395 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2396 (bp->common.shmem2_base)) {
2397 if (SHMEM2_HAS(bp, dcc_support))
2398 SHMEM2_WR(bp, dcc_support,
2399 (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2400 SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2401 if (SHMEM2_HAS(bp, afex_driver_support))
2402 SHMEM2_WR(bp, afex_driver_support,
2403 SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2406 /* Set AFEX default VLAN tag to an invalid value */
2407 bp->afex_def_vlan_tag = -1;
2411 * bnx2x_bz_fp - zero content of the fastpath structure.
2413 * @bp: driver handle
2414 * @index: fastpath index to be zeroed
2416 * Makes sure the contents of the bp->fp[index].napi is kept
2419 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2421 struct bnx2x_fastpath *fp = &bp->fp[index];
2423 struct napi_struct orig_napi = fp->napi;
2424 struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2426 /* bzero bnx2x_fastpath contents */
2428 memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 *
2429 sizeof(struct bnx2x_agg_info));
2430 memset(fp, 0, sizeof(*fp));
2432 /* Restore the NAPI object as it has been already initialized */
2433 fp->napi = orig_napi;
2434 fp->tpa_info = orig_tpa_info;
2438 fp->max_cos = bp->max_cos;
2440 /* Special queues support only one CoS */
2443 /* Init txdata pointers */
2445 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2447 for_each_cos_in_tx_queue(fp, cos)
2448 fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2449 BNX2X_NUM_ETH_QUEUES(bp) + index];
2451 /* set the tpa flag for each queue. The tpa flag determines the queue
2452 * minimal size so it must be set prior to queue memory allocation
2454 fp->disable_tpa = !(bp->flags & TPA_ENABLE_FLAG ||
2455 (bp->flags & GRO_ENABLE_FLAG &&
2456 bnx2x_mtu_allows_gro(bp->dev->mtu)));
2457 if (bp->flags & TPA_ENABLE_FLAG)
2458 fp->mode = TPA_MODE_LRO;
2459 else if (bp->flags & GRO_ENABLE_FLAG)
2460 fp->mode = TPA_MODE_GRO;
2462 /* We don't want TPA on an FCoE L2 ring */
2464 fp->disable_tpa = 1;
2467 int bnx2x_load_cnic(struct bnx2x *bp)
2469 int i, rc, port = BP_PORT(bp);
2471 DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2473 mutex_init(&bp->cnic_mutex);
2476 rc = bnx2x_alloc_mem_cnic(bp);
2478 BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2479 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2483 rc = bnx2x_alloc_fp_mem_cnic(bp);
2485 BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2486 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2489 /* Update the number of queues with the cnic queues */
2490 rc = bnx2x_set_real_num_queues(bp, 1);
2492 BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2493 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2496 /* Add all CNIC NAPI objects */
2497 bnx2x_add_all_napi_cnic(bp);
2498 DP(NETIF_MSG_IFUP, "cnic napi added\n");
2499 bnx2x_napi_enable_cnic(bp);
2501 rc = bnx2x_init_hw_func_cnic(bp);
2503 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2505 bnx2x_nic_init_cnic(bp);
2508 /* Enable Timer scan */
2509 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2511 /* setup cnic queues */
2512 for_each_cnic_queue(bp, i) {
2513 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2515 BNX2X_ERR("Queue setup failed\n");
2516 LOAD_ERROR_EXIT(bp, load_error_cnic2);
2521 /* Initialize Rx filter. */
2522 bnx2x_set_rx_mode_inner(bp);
2524 /* re-read iscsi info */
2525 bnx2x_get_iscsi_info(bp);
2526 bnx2x_setup_cnic_irq_info(bp);
2527 bnx2x_setup_cnic_info(bp);
2528 bp->cnic_loaded = true;
2529 if (bp->state == BNX2X_STATE_OPEN)
2530 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2532 DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2536 #ifndef BNX2X_STOP_ON_ERROR
2538 /* Disable Timer scan */
2539 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2542 bnx2x_napi_disable_cnic(bp);
2543 /* Update the number of queues without the cnic queues */
2544 if (bnx2x_set_real_num_queues(bp, 0))
2545 BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2547 BNX2X_ERR("CNIC-related load failed\n");
2548 bnx2x_free_fp_mem_cnic(bp);
2549 bnx2x_free_mem_cnic(bp);
2551 #endif /* ! BNX2X_STOP_ON_ERROR */
2554 /* must be called with rtnl_lock */
2555 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2557 int port = BP_PORT(bp);
2558 int i, rc = 0, load_code = 0;
2560 DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2562 "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2564 #ifdef BNX2X_STOP_ON_ERROR
2565 if (unlikely(bp->panic)) {
2566 BNX2X_ERR("Can't load NIC when there is panic\n");
2571 bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2573 /* zero the structure w/o any lock, before SP handler is initialized */
2574 memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2575 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2576 &bp->last_reported_link.link_report_flags);
2579 /* must be called before memory allocation and HW init */
2580 bnx2x_ilt_set_info(bp);
2583 * Zero fastpath structures preserving invariants like napi, which are
2584 * allocated only once, fp index, max_cos, bp pointer.
2585 * Also set fp->disable_tpa and txdata_ptr.
2587 DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2588 for_each_queue(bp, i)
2590 memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2591 bp->num_cnic_queues) *
2592 sizeof(struct bnx2x_fp_txdata));
2594 bp->fcoe_init = false;
2596 /* Set the receive queues buffer size */
2597 bnx2x_set_rx_buf_size(bp);
2600 rc = bnx2x_alloc_mem(bp);
2602 BNX2X_ERR("Unable to allocate bp memory\n");
2607 /* need to be done after alloc mem, since it's self adjusting to amount
2608 * of memory available for RSS queues
2610 rc = bnx2x_alloc_fp_mem(bp);
2612 BNX2X_ERR("Unable to allocate memory for fps\n");
2613 LOAD_ERROR_EXIT(bp, load_error0);
2616 /* Allocated memory for FW statistics */
2617 if (bnx2x_alloc_fw_stats_mem(bp))
2618 LOAD_ERROR_EXIT(bp, load_error0);
2620 /* request pf to initialize status blocks */
2622 rc = bnx2x_vfpf_init(bp);
2624 LOAD_ERROR_EXIT(bp, load_error0);
2627 /* As long as bnx2x_alloc_mem() may possibly update
2628 * bp->num_queues, bnx2x_set_real_num_queues() should always
2629 * come after it. At this stage cnic queues are not counted.
2631 rc = bnx2x_set_real_num_queues(bp, 0);
2633 BNX2X_ERR("Unable to set real_num_queues\n");
2634 LOAD_ERROR_EXIT(bp, load_error0);
2637 /* configure multi cos mappings in kernel.
2638 * this configuration may be overridden by a multi class queue
2639 * discipline or by a dcbx negotiation result.
2641 bnx2x_setup_tc(bp->dev, bp->max_cos);
2643 /* Add all NAPI objects */
2644 bnx2x_add_all_napi(bp);
2645 DP(NETIF_MSG_IFUP, "napi added\n");
2646 bnx2x_napi_enable(bp);
2649 /* set pf load just before approaching the MCP */
2650 bnx2x_set_pf_load(bp);
2652 /* if mcp exists send load request and analyze response */
2653 if (!BP_NOMCP(bp)) {
2654 /* attempt to load pf */
2655 rc = bnx2x_nic_load_request(bp, &load_code);
2657 LOAD_ERROR_EXIT(bp, load_error1);
2659 /* what did mcp say? */
2660 rc = bnx2x_compare_fw_ver(bp, load_code, true);
2662 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2663 LOAD_ERROR_EXIT(bp, load_error2);
2666 load_code = bnx2x_nic_load_no_mcp(bp, port);
2669 /* mark pmf if applicable */
2670 bnx2x_nic_load_pmf(bp, load_code);
2672 /* Init Function state controlling object */
2673 bnx2x__init_func_obj(bp);
2676 rc = bnx2x_init_hw(bp, load_code);
2678 BNX2X_ERR("HW init failed, aborting\n");
2679 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2680 LOAD_ERROR_EXIT(bp, load_error2);
2684 bnx2x_pre_irq_nic_init(bp);
2686 /* Connect to IRQs */
2687 rc = bnx2x_setup_irqs(bp);
2689 BNX2X_ERR("setup irqs failed\n");
2691 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2692 LOAD_ERROR_EXIT(bp, load_error2);
2695 /* Init per-function objects */
2697 /* Setup NIC internals and enable interrupts */
2698 bnx2x_post_irq_nic_init(bp, load_code);
2700 bnx2x_init_bp_objs(bp);
2701 bnx2x_iov_nic_init(bp);
2703 /* Set AFEX default VLAN tag to an invalid value */
2704 bp->afex_def_vlan_tag = -1;
2705 bnx2x_nic_load_afex_dcc(bp, load_code);
2706 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2707 rc = bnx2x_func_start(bp);
2709 BNX2X_ERR("Function start failed!\n");
2710 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2712 LOAD_ERROR_EXIT(bp, load_error3);
2715 /* Send LOAD_DONE command to MCP */
2716 if (!BP_NOMCP(bp)) {
2717 load_code = bnx2x_fw_command(bp,
2718 DRV_MSG_CODE_LOAD_DONE, 0);
2720 BNX2X_ERR("MCP response failure, aborting\n");
2722 LOAD_ERROR_EXIT(bp, load_error3);
2726 /* initialize FW coalescing state machines in RAM */
2727 bnx2x_update_coalesce(bp);
2730 /* setup the leading queue */
2731 rc = bnx2x_setup_leading(bp);
2733 BNX2X_ERR("Setup leading failed!\n");
2734 LOAD_ERROR_EXIT(bp, load_error3);
2737 /* set up the rest of the queues */
2738 for_each_nondefault_eth_queue(bp, i) {
2740 rc = bnx2x_setup_queue(bp, &bp->fp[i], false);
2742 rc = bnx2x_vfpf_setup_q(bp, &bp->fp[i], false);
2744 BNX2X_ERR("Queue %d setup failed\n", i);
2745 LOAD_ERROR_EXIT(bp, load_error3);
2750 rc = bnx2x_init_rss(bp);
2752 BNX2X_ERR("PF RSS init failed\n");
2753 LOAD_ERROR_EXIT(bp, load_error3);
2756 /* Now when Clients are configured we are ready to work */
2757 bp->state = BNX2X_STATE_OPEN;
2759 /* Configure a ucast MAC */
2761 rc = bnx2x_set_eth_mac(bp, true);
2763 rc = bnx2x_vfpf_config_mac(bp, bp->dev->dev_addr, bp->fp->index,
2766 BNX2X_ERR("Setting Ethernet MAC failed\n");
2767 LOAD_ERROR_EXIT(bp, load_error3);
2770 if (IS_PF(bp) && bp->pending_max) {
2771 bnx2x_update_max_mf_config(bp, bp->pending_max);
2772 bp->pending_max = 0;
2776 rc = bnx2x_initial_phy_init(bp, load_mode);
2778 LOAD_ERROR_EXIT(bp, load_error3);
2780 bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2782 /* Start fast path */
2784 /* Initialize Rx filter. */
2785 bnx2x_set_rx_mode_inner(bp);
2788 switch (load_mode) {
2790 /* Tx queue should be only re-enabled */
2791 netif_tx_wake_all_queues(bp->dev);
2795 netif_tx_start_all_queues(bp->dev);
2796 smp_mb__after_atomic();
2800 case LOAD_LOOPBACK_EXT:
2801 bp->state = BNX2X_STATE_DIAG;
2809 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2811 bnx2x__link_status_update(bp);
2813 /* start the timer */
2814 mod_timer(&bp->timer, jiffies + bp->current_interval);
2816 if (CNIC_ENABLED(bp))
2817 bnx2x_load_cnic(bp);
2820 bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_GET_DRV_VERSION, 0);
2822 if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2823 /* mark driver is loaded in shmem2 */
2825 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2826 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2827 val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2828 DRV_FLAGS_CAPABILITIES_LOADED_L2);
2831 /* Wait for all pending SP commands to complete */
2832 if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2833 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2834 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2838 /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2839 if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2840 bnx2x_dcbx_init(bp, false);
2842 DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2846 #ifndef BNX2X_STOP_ON_ERROR
2849 bnx2x_int_disable_sync(bp, 1);
2851 /* Clean queueable objects */
2852 bnx2x_squeeze_objects(bp);
2855 /* Free SKBs, SGEs, TPA pool and driver internals */
2856 bnx2x_free_skbs(bp);
2857 for_each_rx_queue(bp, i)
2858 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2863 if (IS_PF(bp) && !BP_NOMCP(bp)) {
2864 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2865 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2870 bnx2x_napi_disable(bp);
2871 bnx2x_del_all_napi(bp);
2873 /* clear pf_load status, as it was already set */
2875 bnx2x_clear_pf_load(bp);
2877 bnx2x_free_fw_stats_mem(bp);
2878 bnx2x_free_fp_mem(bp);
2882 #endif /* ! BNX2X_STOP_ON_ERROR */
2885 int bnx2x_drain_tx_queues(struct bnx2x *bp)
2889 /* Wait until tx fastpath tasks complete */
2890 for_each_tx_queue(bp, i) {
2891 struct bnx2x_fastpath *fp = &bp->fp[i];
2893 for_each_cos_in_tx_queue(fp, cos)
2894 rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]);
2901 /* must be called with rtnl_lock */
2902 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2905 bool global = false;
2907 DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2909 /* mark driver is unloaded in shmem2 */
2910 if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2912 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2913 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2914 val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2917 if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE &&
2918 (bp->state == BNX2X_STATE_CLOSED ||
2919 bp->state == BNX2X_STATE_ERROR)) {
2920 /* We can get here if the driver has been unloaded
2921 * during parity error recovery and is either waiting for a
2922 * leader to complete or for other functions to unload and
2923 * then ifdown has been issued. In this case we want to
2924 * unload and let other functions to complete a recovery
2927 bp->recovery_state = BNX2X_RECOVERY_DONE;
2929 bnx2x_release_leader_lock(bp);
2932 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2933 BNX2X_ERR("Can't unload in closed or error state\n");
2937 /* Nothing to do during unload if previous bnx2x_nic_load()
2938 * have not completed successfully - all resources are released.
2940 * we can get here only after unsuccessful ndo_* callback, during which
2941 * dev->IFF_UP flag is still on.
2943 if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR)
2946 /* It's important to set the bp->state to the value different from
2947 * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2948 * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2950 bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2953 /* indicate to VFs that the PF is going down */
2954 bnx2x_iov_channel_down(bp);
2956 if (CNIC_LOADED(bp))
2957 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2960 bnx2x_tx_disable(bp);
2961 netdev_reset_tc(bp->dev);
2963 bp->rx_mode = BNX2X_RX_MODE_NONE;
2965 del_timer_sync(&bp->timer);
2968 /* Set ALWAYS_ALIVE bit in shmem */
2969 bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2970 bnx2x_drv_pulse(bp);
2971 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2972 bnx2x_save_statistics(bp);
2975 /* wait till consumers catch up with producers in all queues */
2976 bnx2x_drain_tx_queues(bp);
2978 /* if VF indicate to PF this function is going down (PF will delete sp
2979 * elements and clear initializations
2982 bnx2x_vfpf_close_vf(bp);
2983 else if (unload_mode != UNLOAD_RECOVERY)
2984 /* if this is a normal/close unload need to clean up chip*/
2985 bnx2x_chip_cleanup(bp, unload_mode, keep_link);
2987 /* Send the UNLOAD_REQUEST to the MCP */
2988 bnx2x_send_unload_req(bp, unload_mode);
2990 /* Prevent transactions to host from the functions on the
2991 * engine that doesn't reset global blocks in case of global
2992 * attention once global blocks are reset and gates are opened
2993 * (the engine which leader will perform the recovery
2996 if (!CHIP_IS_E1x(bp))
2997 bnx2x_pf_disable(bp);
2999 /* Disable HW interrupts, NAPI */
3000 bnx2x_netif_stop(bp, 1);
3001 /* Delete all NAPI objects */
3002 bnx2x_del_all_napi(bp);
3003 if (CNIC_LOADED(bp))
3004 bnx2x_del_all_napi_cnic(bp);
3008 /* Report UNLOAD_DONE to MCP */
3009 bnx2x_send_unload_done(bp, false);
3013 * At this stage no more interrupts will arrive so we may safely clean
3014 * the queueable objects here in case they failed to get cleaned so far.
3017 bnx2x_squeeze_objects(bp);
3019 /* There should be no more pending SP commands at this stage */
3024 /* clear pending work in rtnl task */
3025 bp->sp_rtnl_state = 0;
3028 /* Free SKBs, SGEs, TPA pool and driver internals */
3029 bnx2x_free_skbs(bp);
3030 if (CNIC_LOADED(bp))
3031 bnx2x_free_skbs_cnic(bp);
3032 for_each_rx_queue(bp, i)
3033 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
3035 bnx2x_free_fp_mem(bp);
3036 if (CNIC_LOADED(bp))
3037 bnx2x_free_fp_mem_cnic(bp);
3040 if (CNIC_LOADED(bp))
3041 bnx2x_free_mem_cnic(bp);
3045 bp->state = BNX2X_STATE_CLOSED;
3046 bp->cnic_loaded = false;
3048 /* Clear driver version indication in shmem */
3050 bnx2x_update_mng_version(bp);
3052 /* Check if there are pending parity attentions. If there are - set
3053 * RECOVERY_IN_PROGRESS.
3055 if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) {
3056 bnx2x_set_reset_in_progress(bp);
3058 /* Set RESET_IS_GLOBAL if needed */
3060 bnx2x_set_reset_global(bp);
3063 /* The last driver must disable a "close the gate" if there is no
3064 * parity attention or "process kill" pending.
3067 !bnx2x_clear_pf_load(bp) &&
3068 bnx2x_reset_is_done(bp, BP_PATH(bp)))
3069 bnx2x_disable_close_the_gate(bp);
3071 DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
3076 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
3080 /* If there is no power capability, silently succeed */
3081 if (!bp->pdev->pm_cap) {
3082 BNX2X_DEV_INFO("No power capability. Breaking.\n");
3086 pci_read_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL, &pmcsr);
3090 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3091 ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
3092 PCI_PM_CTRL_PME_STATUS));
3094 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
3095 /* delay required during transition out of D3hot */
3100 /* If there are other clients above don't
3101 shut down the power */
3102 if (atomic_read(&bp->pdev->enable_cnt) != 1)
3104 /* Don't shut down the power for emulation and FPGA */
3105 if (CHIP_REV_IS_SLOW(bp))
3108 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
3112 pmcsr |= PCI_PM_CTRL_PME_ENABLE;
3114 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3117 /* No more memory access after this point until
3118 * device is brought back to D0.
3123 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
3130 * net_device service functions
3132 static int bnx2x_poll(struct napi_struct *napi, int budget)
3136 struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3138 struct bnx2x *bp = fp->bp;
3141 #ifdef BNX2X_STOP_ON_ERROR
3142 if (unlikely(bp->panic)) {
3143 napi_complete(napi);
3147 if (!bnx2x_fp_lock_napi(fp))
3150 for_each_cos_in_tx_queue(fp, cos)
3151 if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
3152 bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
3154 if (bnx2x_has_rx_work(fp)) {
3155 work_done += bnx2x_rx_int(fp, budget - work_done);
3157 /* must not complete if we consumed full budget */
3158 if (work_done >= budget) {
3159 bnx2x_fp_unlock_napi(fp);
3164 /* Fall out from the NAPI loop if needed */
3165 if (!bnx2x_fp_unlock_napi(fp) &&
3166 !(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3168 /* No need to update SB for FCoE L2 ring as long as
3169 * it's connected to the default SB and the SB
3170 * has been updated when NAPI was scheduled.
3172 if (IS_FCOE_FP(fp)) {
3173 napi_complete(napi);
3176 bnx2x_update_fpsb_idx(fp);
3177 /* bnx2x_has_rx_work() reads the status block,
3178 * thus we need to ensure that status block indices
3179 * have been actually read (bnx2x_update_fpsb_idx)
3180 * prior to this check (bnx2x_has_rx_work) so that
3181 * we won't write the "newer" value of the status block
3182 * to IGU (if there was a DMA right after
3183 * bnx2x_has_rx_work and if there is no rmb, the memory
3184 * reading (bnx2x_update_fpsb_idx) may be postponed
3185 * to right before bnx2x_ack_sb). In this case there
3186 * will never be another interrupt until there is
3187 * another update of the status block, while there
3188 * is still unhandled work.
3192 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3193 napi_complete(napi);
3194 /* Re-enable interrupts */
3195 DP(NETIF_MSG_RX_STATUS,
3196 "Update index to %d\n", fp->fp_hc_idx);
3197 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
3198 le16_to_cpu(fp->fp_hc_idx),
3208 #ifdef CONFIG_NET_RX_BUSY_POLL
3209 /* must be called with local_bh_disable()d */
3210 int bnx2x_low_latency_recv(struct napi_struct *napi)
3212 struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3214 struct bnx2x *bp = fp->bp;
3217 if ((bp->state == BNX2X_STATE_CLOSED) ||
3218 (bp->state == BNX2X_STATE_ERROR) ||
3219 (bp->flags & (TPA_ENABLE_FLAG | GRO_ENABLE_FLAG)))
3220 return LL_FLUSH_FAILED;
3222 if (!bnx2x_fp_lock_poll(fp))
3223 return LL_FLUSH_BUSY;
3225 if (bnx2x_has_rx_work(fp))
3226 found = bnx2x_rx_int(fp, 4);
3228 bnx2x_fp_unlock_poll(fp);
3234 /* we split the first BD into headers and data BDs
3235 * to ease the pain of our fellow microcode engineers
3236 * we use one mapping for both BDs
3238 static u16 bnx2x_tx_split(struct bnx2x *bp,
3239 struct bnx2x_fp_txdata *txdata,
3240 struct sw_tx_bd *tx_buf,
3241 struct eth_tx_start_bd **tx_bd, u16 hlen,
3244 struct eth_tx_start_bd *h_tx_bd = *tx_bd;
3245 struct eth_tx_bd *d_tx_bd;
3247 int old_len = le16_to_cpu(h_tx_bd->nbytes);
3249 /* first fix first BD */
3250 h_tx_bd->nbytes = cpu_to_le16(hlen);
3252 DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x)\n",
3253 h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo);
3255 /* now get a new data BD
3256 * (after the pbd) and fill it */
3257 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3258 d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3260 mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
3261 le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
3263 d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3264 d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3265 d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
3267 /* this marks the BD as one that has no individual mapping */
3268 tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
3270 DP(NETIF_MSG_TX_QUEUED,
3271 "TSO split data size is %d (%x:%x)\n",
3272 d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
3275 *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
3280 #define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32)))
3281 #define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16)))
3282 static __le16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
3284 __sum16 tsum = (__force __sum16) csum;
3287 tsum = ~csum_fold(csum_sub((__force __wsum) csum,
3288 csum_partial(t_header - fix, fix, 0)));
3291 tsum = ~csum_fold(csum_add((__force __wsum) csum,
3292 csum_partial(t_header, -fix, 0)));
3294 return bswab16(tsum);
3297 static u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
3303 if (skb->ip_summed != CHECKSUM_PARTIAL)
3306 protocol = vlan_get_protocol(skb);
3307 if (protocol == htons(ETH_P_IPV6)) {
3309 prot = ipv6_hdr(skb)->nexthdr;
3312 prot = ip_hdr(skb)->protocol;
3315 if (!CHIP_IS_E1x(bp) && skb->encapsulation) {
3316 if (inner_ip_hdr(skb)->version == 6) {
3317 rc |= XMIT_CSUM_ENC_V6;
3318 if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3319 rc |= XMIT_CSUM_TCP;
3321 rc |= XMIT_CSUM_ENC_V4;
3322 if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP)
3323 rc |= XMIT_CSUM_TCP;
3326 if (prot == IPPROTO_TCP)
3327 rc |= XMIT_CSUM_TCP;
3329 if (skb_is_gso(skb)) {
3330 if (skb_is_gso_v6(skb)) {
3331 rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP);
3332 if (rc & XMIT_CSUM_ENC)
3333 rc |= XMIT_GSO_ENC_V6;
3335 rc |= (XMIT_GSO_V4 | XMIT_CSUM_TCP);
3336 if (rc & XMIT_CSUM_ENC)
3337 rc |= XMIT_GSO_ENC_V4;
3344 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
3345 /* check if packet requires linearization (packet is too fragmented)
3346 no need to check fragmentation if page size > 8K (there will be no
3347 violation to FW restrictions) */
3348 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
3353 int first_bd_sz = 0;
3355 /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
3356 if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
3358 if (xmit_type & XMIT_GSO) {
3359 unsigned short lso_mss = skb_shinfo(skb)->gso_size;
3360 /* Check if LSO packet needs to be copied:
3361 3 = 1 (for headers BD) + 2 (for PBD and last BD) */
3362 int wnd_size = MAX_FETCH_BD - 3;
3363 /* Number of windows to check */
3364 int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
3369 /* Headers length */
3370 hlen = (int)(skb_transport_header(skb) - skb->data) +
3373 /* Amount of data (w/o headers) on linear part of SKB*/
3374 first_bd_sz = skb_headlen(skb) - hlen;
3376 wnd_sum = first_bd_sz;
3378 /* Calculate the first sum - it's special */
3379 for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
3381 skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
3383 /* If there was data on linear skb data - check it */
3384 if (first_bd_sz > 0) {
3385 if (unlikely(wnd_sum < lso_mss)) {
3390 wnd_sum -= first_bd_sz;
3393 /* Others are easier: run through the frag list and
3394 check all windows */
3395 for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
3397 skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
3399 if (unlikely(wnd_sum < lso_mss)) {
3404 skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
3407 /* in non-LSO too fragmented packet should always
3414 if (unlikely(to_copy))
3415 DP(NETIF_MSG_TX_QUEUED,
3416 "Linearization IS REQUIRED for %s packet. num_frags %d hlen %d first_bd_sz %d\n",
3417 (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
3418 skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
3424 static void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
3427 struct ipv6hdr *ipv6;
3429 *parsing_data |= (skb_shinfo(skb)->gso_size <<
3430 ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
3431 ETH_TX_PARSE_BD_E2_LSO_MSS;
3433 if (xmit_type & XMIT_GSO_ENC_V6)
3434 ipv6 = inner_ipv6_hdr(skb);
3435 else if (xmit_type & XMIT_GSO_V6)
3436 ipv6 = ipv6_hdr(skb);
3440 if (ipv6 && ipv6->nexthdr == NEXTHDR_IPV6)
3441 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
3445 * bnx2x_set_pbd_gso - update PBD in GSO case.
3449 * @xmit_type: xmit flags
3451 static void bnx2x_set_pbd_gso(struct sk_buff *skb,
3452 struct eth_tx_parse_bd_e1x *pbd,
3453 struct eth_tx_start_bd *tx_start_bd,
3456 pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
3457 pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq);
3458 pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb));
3460 if (xmit_type & XMIT_GSO_V4) {
3461 pbd->ip_id = bswab16(ip_hdr(skb)->id);
3462 pbd->tcp_pseudo_csum =
3463 bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3465 0, IPPROTO_TCP, 0));
3467 /* GSO on 57710/57711 needs FW to calculate IP checksum */
3468 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IP_CSUM;
3470 pbd->tcp_pseudo_csum =
3471 bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3472 &ipv6_hdr(skb)->daddr,
3473 0, IPPROTO_TCP, 0));
3477 cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN);
3481 * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length
3483 * @bp: driver handle
3485 * @parsing_data: data to be updated
3486 * @xmit_type: xmit flags
3488 * 57712/578xx related, when skb has encapsulation
3490 static u8 bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb,
3491 u32 *parsing_data, u32 xmit_type)
3494 ((((u8 *)skb_inner_transport_header(skb) - skb->data) >> 1) <<
3495 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3496 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3498 if (xmit_type & XMIT_CSUM_TCP) {
3499 *parsing_data |= ((inner_tcp_hdrlen(skb) / 4) <<
3500 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3501 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3503 return skb_inner_transport_header(skb) +
3504 inner_tcp_hdrlen(skb) - skb->data;
3507 /* We support checksum offload for TCP and UDP only.
3508 * No need to pass the UDP header length - it's a constant.
3510 return skb_inner_transport_header(skb) +
3511 sizeof(struct udphdr) - skb->data;
3515 * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
3517 * @bp: driver handle
3519 * @parsing_data: data to be updated
3520 * @xmit_type: xmit flags
3522 * 57712/578xx related
3524 static u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3525 u32 *parsing_data, u32 xmit_type)
3528 ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
3529 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3530 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3532 if (xmit_type & XMIT_CSUM_TCP) {
3533 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3534 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3535 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3537 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
3539 /* We support checksum offload for TCP and UDP only.
3540 * No need to pass the UDP header length - it's a constant.
3542 return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data;
3545 /* set FW indication according to inner or outer protocols if tunneled */
3546 static void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3547 struct eth_tx_start_bd *tx_start_bd,
3550 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3552 if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6))
3553 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6;
3555 if (!(xmit_type & XMIT_CSUM_TCP))
3556 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
3560 * bnx2x_set_pbd_csum - update PBD with checksum and return header length
3562 * @bp: driver handle
3564 * @pbd: parse BD to be updated
3565 * @xmit_type: xmit flags
3567 static u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3568 struct eth_tx_parse_bd_e1x *pbd,
3571 u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
3573 /* for now NS flag is not used in Linux */
3576 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3577 ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
3579 pbd->ip_hlen_w = (skb_transport_header(skb) -
3580 skb_network_header(skb)) >> 1;
3582 hlen += pbd->ip_hlen_w;
3584 /* We support checksum offload for TCP and UDP only */
3585 if (xmit_type & XMIT_CSUM_TCP)
3586 hlen += tcp_hdrlen(skb) / 2;
3588 hlen += sizeof(struct udphdr) / 2;
3590 pbd->total_hlen_w = cpu_to_le16(hlen);
3593 if (xmit_type & XMIT_CSUM_TCP) {
3594 pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check);
3597 s8 fix = SKB_CS_OFF(skb); /* signed! */
3599 DP(NETIF_MSG_TX_QUEUED,
3600 "hlen %d fix %d csum before fix %x\n",
3601 le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
3603 /* HW bug: fixup the CSUM */
3604 pbd->tcp_pseudo_csum =
3605 bnx2x_csum_fix(skb_transport_header(skb),
3608 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
3609 pbd->tcp_pseudo_csum);
3615 static void bnx2x_update_pbds_gso_enc(struct sk_buff *skb,
3616 struct eth_tx_parse_bd_e2 *pbd_e2,
3617 struct eth_tx_parse_2nd_bd *pbd2,
3622 u8 outerip_off, outerip_len = 0;
3624 /* from outer IP to transport */
3625 hlen_w = (skb_inner_transport_header(skb) -
3626 skb_network_header(skb)) >> 1;
3629 hlen_w += inner_tcp_hdrlen(skb) >> 1;
3631 pbd2->fw_ip_hdr_to_payload_w = hlen_w;
3633 /* outer IP header info */
3634 if (xmit_type & XMIT_CSUM_V4) {
3635 struct iphdr *iph = ip_hdr(skb);
3636 u32 csum = (__force u32)(~iph->check) -
3637 (__force u32)iph->tot_len -
3638 (__force u32)iph->frag_off;
3640 pbd2->fw_ip_csum_wo_len_flags_frag =
3641 bswab16(csum_fold((__force __wsum)csum));
3643 pbd2->fw_ip_hdr_to_payload_w =
3644 hlen_w - ((sizeof(struct ipv6hdr)) >> 1);
3647 pbd2->tcp_send_seq = bswab32(inner_tcp_hdr(skb)->seq);
3649 pbd2->tcp_flags = pbd_tcp_flags(inner_tcp_hdr(skb));
3651 if (xmit_type & XMIT_GSO_V4) {
3652 pbd2->hw_ip_id = bswab16(inner_ip_hdr(skb)->id);
3654 pbd_e2->data.tunnel_data.pseudo_csum =
3655 bswab16(~csum_tcpudp_magic(
3656 inner_ip_hdr(skb)->saddr,
3657 inner_ip_hdr(skb)->daddr,
3658 0, IPPROTO_TCP, 0));
3660 outerip_len = ip_hdr(skb)->ihl << 1;
3662 pbd_e2->data.tunnel_data.pseudo_csum =
3663 bswab16(~csum_ipv6_magic(
3664 &inner_ipv6_hdr(skb)->saddr,
3665 &inner_ipv6_hdr(skb)->daddr,
3666 0, IPPROTO_TCP, 0));
3669 outerip_off = (skb_network_header(skb) - skb->data) >> 1;
3673 (!!(xmit_type & XMIT_CSUM_V6) <<
3674 ETH_TX_PARSE_2ND_BD_IP_HDR_TYPE_OUTER_SHIFT) |
3676 ETH_TX_PARSE_2ND_BD_IP_HDR_LEN_OUTER_W_SHIFT) |
3677 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3678 ETH_TX_PARSE_2ND_BD_LLC_SNAP_EN_SHIFT);
3680 if (ip_hdr(skb)->protocol == IPPROTO_UDP) {
3681 SET_FLAG(*global_data, ETH_TX_PARSE_2ND_BD_TUNNEL_UDP_EXIST, 1);
3682 pbd2->tunnel_udp_hdr_start_w = skb_transport_offset(skb) >> 1;
3686 /* called with netif_tx_lock
3687 * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
3688 * netif_wake_queue()
3690 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
3692 struct bnx2x *bp = netdev_priv(dev);
3694 struct netdev_queue *txq;
3695 struct bnx2x_fp_txdata *txdata;
3696 struct sw_tx_bd *tx_buf;
3697 struct eth_tx_start_bd *tx_start_bd, *first_bd;
3698 struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
3699 struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
3700 struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
3701 struct eth_tx_parse_2nd_bd *pbd2 = NULL;
3702 u32 pbd_e2_parsing_data = 0;
3703 u16 pkt_prod, bd_prod;
3706 u32 xmit_type = bnx2x_xmit_type(bp, skb);
3709 __le16 pkt_size = 0;
3711 u8 mac_type = UNICAST_ADDRESS;
3713 #ifdef BNX2X_STOP_ON_ERROR
3714 if (unlikely(bp->panic))
3715 return NETDEV_TX_BUSY;
3718 txq_index = skb_get_queue_mapping(skb);
3719 txq = netdev_get_tx_queue(dev, txq_index);
3721 BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0));
3723 txdata = &bp->bnx2x_txq[txq_index];
3725 /* enable this debug print to view the transmission queue being used
3726 DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
3727 txq_index, fp_index, txdata_index); */
3729 /* enable this debug print to view the transmission details
3730 DP(NETIF_MSG_TX_QUEUED,
3731 "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
3732 txdata->cid, fp_index, txdata_index, txdata, fp); */
3734 if (unlikely(bnx2x_tx_avail(bp, txdata) <
3735 skb_shinfo(skb)->nr_frags +
3737 NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
3738 /* Handle special storage cases separately */
3739 if (txdata->tx_ring_size == 0) {
3740 struct bnx2x_eth_q_stats *q_stats =
3741 bnx2x_fp_qstats(bp, txdata->parent_fp);
3742 q_stats->driver_filtered_tx_pkt++;
3744 return NETDEV_TX_OK;
3746 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3747 netif_tx_stop_queue(txq);
3748 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
3750 return NETDEV_TX_BUSY;
3753 DP(NETIF_MSG_TX_QUEUED,
3754 "queue[%d]: SKB: summed %x protocol %x protocol(%x,%x) gso type %x xmit_type %x len %d\n",
3755 txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
3756 ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type,
3759 eth = (struct ethhdr *)skb->data;
3761 /* set flag according to packet type (UNICAST_ADDRESS is default)*/
3762 if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
3763 if (is_broadcast_ether_addr(eth->h_dest))
3764 mac_type = BROADCAST_ADDRESS;
3766 mac_type = MULTICAST_ADDRESS;
3769 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
3770 /* First, check if we need to linearize the skb (due to FW
3771 restrictions). No need to check fragmentation if page size > 8K
3772 (there will be no violation to FW restrictions) */
3773 if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
3774 /* Statistics of linearization */
3776 if (skb_linearize(skb) != 0) {
3777 DP(NETIF_MSG_TX_QUEUED,
3778 "SKB linearization failed - silently dropping this SKB\n");
3779 dev_kfree_skb_any(skb);
3780 return NETDEV_TX_OK;
3784 /* Map skb linear data for DMA */
3785 mapping = dma_map_single(&bp->pdev->dev, skb->data,
3786 skb_headlen(skb), DMA_TO_DEVICE);
3787 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3788 DP(NETIF_MSG_TX_QUEUED,
3789 "SKB mapping failed - silently dropping this SKB\n");
3790 dev_kfree_skb_any(skb);
3791 return NETDEV_TX_OK;
3794 Please read carefully. First we use one BD which we mark as start,
3795 then we have a parsing info BD (used for TSO or xsum),
3796 and only then we have the rest of the TSO BDs.
3797 (don't forget to mark the last one as last,
3798 and to unmap only AFTER you write to the BD ...)
3799 And above all, all pdb sizes are in words - NOT DWORDS!
3802 /* get current pkt produced now - advance it just before sending packet
3803 * since mapping of pages may fail and cause packet to be dropped
3805 pkt_prod = txdata->tx_pkt_prod;
3806 bd_prod = TX_BD(txdata->tx_bd_prod);
3808 /* get a tx_buf and first BD
3809 * tx_start_bd may be changed during SPLIT,
3810 * but first_bd will always stay first
3812 tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3813 tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
3814 first_bd = tx_start_bd;
3816 tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
3818 /* header nbd: indirectly zero other flags! */
3819 tx_start_bd->general_data = 1 << ETH_TX_START_BD_HDR_NBDS_SHIFT;
3821 /* remember the first BD of the packet */
3822 tx_buf->first_bd = txdata->tx_bd_prod;
3826 DP(NETIF_MSG_TX_QUEUED,
3827 "sending pkt %u @%p next_idx %u bd %u @%p\n",
3828 pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
3830 if (vlan_tx_tag_present(skb)) {
3831 tx_start_bd->vlan_or_ethertype =
3832 cpu_to_le16(vlan_tx_tag_get(skb));
3833 tx_start_bd->bd_flags.as_bitfield |=
3834 (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3836 /* when transmitting in a vf, start bd must hold the ethertype
3837 * for fw to enforce it
3840 tx_start_bd->vlan_or_ethertype =
3841 cpu_to_le16(ntohs(eth->h_proto));
3843 /* used by FW for packet accounting */
3844 tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
3847 nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3849 /* turn on parsing and get a BD */
3850 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3852 if (xmit_type & XMIT_CSUM)
3853 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
3855 if (!CHIP_IS_E1x(bp)) {
3856 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
3857 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
3859 if (xmit_type & XMIT_CSUM_ENC) {
3860 u16 global_data = 0;
3862 /* Set PBD in enc checksum offload case */
3863 hlen = bnx2x_set_pbd_csum_enc(bp, skb,
3864 &pbd_e2_parsing_data,
3867 /* turn on 2nd parsing and get a BD */
3868 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3870 pbd2 = &txdata->tx_desc_ring[bd_prod].parse_2nd_bd;
3872 memset(pbd2, 0, sizeof(*pbd2));
3874 pbd_e2->data.tunnel_data.ip_hdr_start_inner_w =
3875 (skb_inner_network_header(skb) -
3878 if (xmit_type & XMIT_GSO_ENC)
3879 bnx2x_update_pbds_gso_enc(skb, pbd_e2, pbd2,
3883 pbd2->global_data = cpu_to_le16(global_data);
3885 /* add addition parse BD indication to start BD */
3886 SET_FLAG(tx_start_bd->general_data,
3887 ETH_TX_START_BD_PARSE_NBDS, 1);
3888 /* set encapsulation flag in start BD */
3889 SET_FLAG(tx_start_bd->general_data,
3890 ETH_TX_START_BD_TUNNEL_EXIST, 1);
3892 } else if (xmit_type & XMIT_CSUM) {
3893 /* Set PBD in checksum offload case w/o encapsulation */
3894 hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3895 &pbd_e2_parsing_data,
3899 /* Add the macs to the parsing BD if this is a vf or if
3900 * Tx Switching is enabled.
3903 /* override GRE parameters in BD */
3904 bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
3905 &pbd_e2->data.mac_addr.src_mid,
3906 &pbd_e2->data.mac_addr.src_lo,
3909 bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi,
3910 &pbd_e2->data.mac_addr.dst_mid,
3911 &pbd_e2->data.mac_addr.dst_lo,
3913 } else if (bp->flags & TX_SWITCHING) {
3914 bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi,
3915 &pbd_e2->data.mac_addr.dst_mid,
3916 &pbd_e2->data.mac_addr.dst_lo,
3920 SET_FLAG(pbd_e2_parsing_data,
3921 ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type);
3923 u16 global_data = 0;
3924 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
3925 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
3926 /* Set PBD in checksum offload case */
3927 if (xmit_type & XMIT_CSUM)
3928 hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
3930 SET_FLAG(global_data,
3931 ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type);
3932 pbd_e1x->global_data |= cpu_to_le16(global_data);
3935 /* Setup the data pointer of the first BD of the packet */
3936 tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3937 tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3938 tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
3939 pkt_size = tx_start_bd->nbytes;
3941 DP(NETIF_MSG_TX_QUEUED,
3942 "first bd @%p addr (%x:%x) nbytes %d flags %x vlan %x\n",
3943 tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
3944 le16_to_cpu(tx_start_bd->nbytes),
3945 tx_start_bd->bd_flags.as_bitfield,
3946 le16_to_cpu(tx_start_bd->vlan_or_ethertype));
3948 if (xmit_type & XMIT_GSO) {
3950 DP(NETIF_MSG_TX_QUEUED,
3951 "TSO packet len %d hlen %d total len %d tso size %d\n",
3952 skb->len, hlen, skb_headlen(skb),
3953 skb_shinfo(skb)->gso_size);
3955 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
3957 if (unlikely(skb_headlen(skb) > hlen)) {
3959 bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
3963 if (!CHIP_IS_E1x(bp))
3964 bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
3967 bnx2x_set_pbd_gso(skb, pbd_e1x, first_bd, xmit_type);
3970 /* Set the PBD's parsing_data field if not zero
3971 * (for the chips newer than 57711).
3973 if (pbd_e2_parsing_data)
3974 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
3976 tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
3978 /* Handle fragmented skb */
3979 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3980 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3982 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
3983 skb_frag_size(frag), DMA_TO_DEVICE);
3984 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3985 unsigned int pkts_compl = 0, bytes_compl = 0;
3987 DP(NETIF_MSG_TX_QUEUED,
3988 "Unable to map page - dropping packet...\n");
3990 /* we need unmap all buffers already mapped
3992 * first_bd->nbd need to be properly updated
3993 * before call to bnx2x_free_tx_pkt
3995 first_bd->nbd = cpu_to_le16(nbd);
3996 bnx2x_free_tx_pkt(bp, txdata,
3997 TX_BD(txdata->tx_pkt_prod),
3998 &pkts_compl, &bytes_compl);
3999 return NETDEV_TX_OK;
4002 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
4003 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
4004 if (total_pkt_bd == NULL)
4005 total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
4007 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
4008 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
4009 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
4010 le16_add_cpu(&pkt_size, skb_frag_size(frag));
4013 DP(NETIF_MSG_TX_QUEUED,
4014 "frag %d bd @%p addr (%x:%x) nbytes %d\n",
4015 i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
4016 le16_to_cpu(tx_data_bd->nbytes));
4019 DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
4021 /* update with actual num BDs */
4022 first_bd->nbd = cpu_to_le16(nbd);
4024 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
4026 /* now send a tx doorbell, counting the next BD
4027 * if the packet contains or ends with it
4029 if (TX_BD_POFF(bd_prod) < nbd)
4032 /* total_pkt_bytes should be set on the first data BD if
4033 * it's not an LSO packet and there is more than one
4034 * data BD. In this case pkt_size is limited by an MTU value.
4035 * However we prefer to set it for an LSO packet (while we don't
4036 * have to) in order to save some CPU cycles in a none-LSO
4037 * case, when we much more care about them.
4039 if (total_pkt_bd != NULL)
4040 total_pkt_bd->total_pkt_bytes = pkt_size;
4043 DP(NETIF_MSG_TX_QUEUED,
4044 "PBD (E1X) @%p ip_data %x ip_hlen %u ip_id %u lso_mss %u tcp_flags %x xsum %x seq %u hlen %u\n",
4045 pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
4046 pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
4047 pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
4048 le16_to_cpu(pbd_e1x->total_hlen_w));
4050 DP(NETIF_MSG_TX_QUEUED,
4051 "PBD (E2) @%p dst %x %x %x src %x %x %x parsing_data %x\n",
4053 pbd_e2->data.mac_addr.dst_hi,
4054 pbd_e2->data.mac_addr.dst_mid,
4055 pbd_e2->data.mac_addr.dst_lo,
4056 pbd_e2->data.mac_addr.src_hi,
4057 pbd_e2->data.mac_addr.src_mid,
4058 pbd_e2->data.mac_addr.src_lo,
4059 pbd_e2->parsing_data);
4060 DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d bd %u\n", nbd, bd_prod);
4062 netdev_tx_sent_queue(txq, skb->len);
4064 skb_tx_timestamp(skb);
4066 txdata->tx_pkt_prod++;
4068 * Make sure that the BD data is updated before updating the producer
4069 * since FW might read the BD right after the producer is updated.
4070 * This is only applicable for weak-ordered memory model archs such
4071 * as IA-64. The following barrier is also mandatory since FW will
4072 * assumes packets must have BDs.
4076 txdata->tx_db.data.prod += nbd;
4079 DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
4083 txdata->tx_bd_prod += nbd;
4085 if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) {
4086 netif_tx_stop_queue(txq);
4088 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
4089 * ordering of set_bit() in netif_tx_stop_queue() and read of
4093 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
4094 if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)
4095 netif_tx_wake_queue(txq);
4099 return NETDEV_TX_OK;
4103 * bnx2x_setup_tc - routine to configure net_device for multi tc
4105 * @netdev: net device to configure
4106 * @tc: number of traffic classes to enable
4108 * callback connected to the ndo_setup_tc function pointer
4110 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
4112 int cos, prio, count, offset;
4113 struct bnx2x *bp = netdev_priv(dev);
4115 /* setup tc must be called under rtnl lock */
4118 /* no traffic classes requested. Aborting */
4120 netdev_reset_tc(dev);
4124 /* requested to support too many traffic classes */
4125 if (num_tc > bp->max_cos) {
4126 BNX2X_ERR("support for too many traffic classes requested: %d. Max supported is %d\n",
4127 num_tc, bp->max_cos);
4131 /* declare amount of supported traffic classes */
4132 if (netdev_set_num_tc(dev, num_tc)) {
4133 BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
4137 /* configure priority to traffic class mapping */
4138 for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
4139 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
4140 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4141 "mapping priority %d to tc %d\n",
4142 prio, bp->prio_to_cos[prio]);
4145 /* Use this configuration to differentiate tc0 from other COSes
4146 This can be used for ets or pfc, and save the effort of setting
4147 up a multio class queue disc or negotiating DCBX with a switch
4148 netdev_set_prio_tc_map(dev, 0, 0);
4149 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
4150 for (prio = 1; prio < 16; prio++) {
4151 netdev_set_prio_tc_map(dev, prio, 1);
4152 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
4155 /* configure traffic class to transmission queue mapping */
4156 for (cos = 0; cos < bp->max_cos; cos++) {
4157 count = BNX2X_NUM_ETH_QUEUES(bp);
4158 offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp);
4159 netdev_set_tc_queue(dev, cos, count, offset);
4160 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4161 "mapping tc %d to offset %d count %d\n",
4162 cos, offset, count);
4168 /* called with rtnl_lock */
4169 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
4171 struct sockaddr *addr = p;
4172 struct bnx2x *bp = netdev_priv(dev);
4175 if (!bnx2x_is_valid_ether_addr(bp, addr->sa_data)) {
4176 BNX2X_ERR("Requested MAC address is not valid\n");
4180 if ((IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp)) &&
4181 !is_zero_ether_addr(addr->sa_data)) {
4182 BNX2X_ERR("Can't configure non-zero address on iSCSI or FCoE functions in MF-SD mode\n");
4186 if (netif_running(dev)) {
4187 rc = bnx2x_set_eth_mac(bp, false);
4192 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4194 if (netif_running(dev))
4195 rc = bnx2x_set_eth_mac(bp, true);
4200 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
4202 union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
4203 struct bnx2x_fastpath *fp = &bp->fp[fp_index];
4208 if (IS_FCOE_IDX(fp_index)) {
4209 memset(sb, 0, sizeof(union host_hc_status_block));
4210 fp->status_blk_mapping = 0;
4213 if (!CHIP_IS_E1x(bp))
4214 BNX2X_PCI_FREE(sb->e2_sb,
4215 bnx2x_fp(bp, fp_index,
4216 status_blk_mapping),
4217 sizeof(struct host_hc_status_block_e2));
4219 BNX2X_PCI_FREE(sb->e1x_sb,
4220 bnx2x_fp(bp, fp_index,
4221 status_blk_mapping),
4222 sizeof(struct host_hc_status_block_e1x));
4226 if (!skip_rx_queue(bp, fp_index)) {
4227 bnx2x_free_rx_bds(fp);
4229 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4230 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
4231 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
4232 bnx2x_fp(bp, fp_index, rx_desc_mapping),
4233 sizeof(struct eth_rx_bd) * NUM_RX_BD);
4235 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
4236 bnx2x_fp(bp, fp_index, rx_comp_mapping),
4237 sizeof(struct eth_fast_path_rx_cqe) *
4241 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
4242 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
4243 bnx2x_fp(bp, fp_index, rx_sge_mapping),
4244 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4248 if (!skip_tx_queue(bp, fp_index)) {
4249 /* fastpath tx rings: tx_buf tx_desc */
4250 for_each_cos_in_tx_queue(fp, cos) {
4251 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4253 DP(NETIF_MSG_IFDOWN,
4254 "freeing tx memory of fp %d cos %d cid %d\n",
4255 fp_index, cos, txdata->cid);
4257 BNX2X_FREE(txdata->tx_buf_ring);
4258 BNX2X_PCI_FREE(txdata->tx_desc_ring,
4259 txdata->tx_desc_mapping,
4260 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4263 /* end of fastpath */
4266 static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp)
4269 for_each_cnic_queue(bp, i)
4270 bnx2x_free_fp_mem_at(bp, i);
4273 void bnx2x_free_fp_mem(struct bnx2x *bp)
4276 for_each_eth_queue(bp, i)
4277 bnx2x_free_fp_mem_at(bp, i);
4280 static void set_sb_shortcuts(struct bnx2x *bp, int index)
4282 union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
4283 if (!CHIP_IS_E1x(bp)) {
4284 bnx2x_fp(bp, index, sb_index_values) =
4285 (__le16 *)status_blk.e2_sb->sb.index_values;
4286 bnx2x_fp(bp, index, sb_running_index) =
4287 (__le16 *)status_blk.e2_sb->sb.running_index;
4289 bnx2x_fp(bp, index, sb_index_values) =
4290 (__le16 *)status_blk.e1x_sb->sb.index_values;
4291 bnx2x_fp(bp, index, sb_running_index) =
4292 (__le16 *)status_blk.e1x_sb->sb.running_index;
4296 /* Returns the number of actually allocated BDs */
4297 static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
4300 struct bnx2x *bp = fp->bp;
4301 u16 ring_prod, cqe_ring_prod;
4302 int i, failure_cnt = 0;
4304 fp->rx_comp_cons = 0;
4305 cqe_ring_prod = ring_prod = 0;
4307 /* This routine is called only during fo init so
4308 * fp->eth_q_stats.rx_skb_alloc_failed = 0
4310 for (i = 0; i < rx_ring_size; i++) {
4311 if (bnx2x_alloc_rx_data(bp, fp, ring_prod, GFP_KERNEL) < 0) {
4315 ring_prod = NEXT_RX_IDX(ring_prod);
4316 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
4317 WARN_ON(ring_prod <= (i - failure_cnt));
4321 BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n",
4322 i - failure_cnt, fp->index);
4324 fp->rx_bd_prod = ring_prod;
4325 /* Limit the CQE producer by the CQE ring size */
4326 fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
4328 fp->rx_pkt = fp->rx_calls = 0;
4330 bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt;
4332 return i - failure_cnt;
4335 static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp)
4339 for (i = 1; i <= NUM_RCQ_RINGS; i++) {
4340 struct eth_rx_cqe_next_page *nextpg;
4342 nextpg = (struct eth_rx_cqe_next_page *)
4343 &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
4345 cpu_to_le32(U64_HI(fp->rx_comp_mapping +
4346 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4348 cpu_to_le32(U64_LO(fp->rx_comp_mapping +
4349 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4353 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
4355 union host_hc_status_block *sb;
4356 struct bnx2x_fastpath *fp = &bp->fp[index];
4359 int rx_ring_size = 0;
4361 if (!bp->rx_ring_size &&
4362 (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))) {
4363 rx_ring_size = MIN_RX_SIZE_NONTPA;
4364 bp->rx_ring_size = rx_ring_size;
4365 } else if (!bp->rx_ring_size) {
4366 rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
4368 if (CHIP_IS_E3(bp)) {
4369 u32 cfg = SHMEM_RD(bp,
4370 dev_info.port_hw_config[BP_PORT(bp)].
4373 /* Decrease ring size for 1G functions */
4374 if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
4375 PORT_HW_CFG_NET_SERDES_IF_SGMII)
4379 /* allocate at least number of buffers required by FW */
4380 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
4381 MIN_RX_SIZE_TPA, rx_ring_size);
4383 bp->rx_ring_size = rx_ring_size;
4384 } else /* if rx_ring_size specified - use it */
4385 rx_ring_size = bp->rx_ring_size;
4387 DP(BNX2X_MSG_SP, "calculated rx_ring_size %d\n", rx_ring_size);
4390 sb = &bnx2x_fp(bp, index, status_blk);
4392 if (!IS_FCOE_IDX(index)) {
4394 if (!CHIP_IS_E1x(bp)) {
4395 sb->e2_sb = BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, status_blk_mapping),
4396 sizeof(struct host_hc_status_block_e2));
4400 sb->e1x_sb = BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, status_blk_mapping),
4401 sizeof(struct host_hc_status_block_e1x));
4407 /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
4408 * set shortcuts for it.
4410 if (!IS_FCOE_IDX(index))
4411 set_sb_shortcuts(bp, index);
4414 if (!skip_tx_queue(bp, index)) {
4415 /* fastpath tx rings: tx_buf tx_desc */
4416 for_each_cos_in_tx_queue(fp, cos) {
4417 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4420 "allocating tx memory of fp %d cos %d\n",
4423 txdata->tx_buf_ring = kcalloc(NUM_TX_BD,
4424 sizeof(struct sw_tx_bd),
4426 if (!txdata->tx_buf_ring)
4428 txdata->tx_desc_ring = BNX2X_PCI_ALLOC(&txdata->tx_desc_mapping,
4429 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4430 if (!txdata->tx_desc_ring)
4436 if (!skip_rx_queue(bp, index)) {
4437 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4438 bnx2x_fp(bp, index, rx_buf_ring) =
4439 kcalloc(NUM_RX_BD, sizeof(struct sw_rx_bd), GFP_KERNEL);
4440 if (!bnx2x_fp(bp, index, rx_buf_ring))
4442 bnx2x_fp(bp, index, rx_desc_ring) =
4443 BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, rx_desc_mapping),
4444 sizeof(struct eth_rx_bd) * NUM_RX_BD);
4445 if (!bnx2x_fp(bp, index, rx_desc_ring))
4448 /* Seed all CQEs by 1s */
4449 bnx2x_fp(bp, index, rx_comp_ring) =
4450 BNX2X_PCI_FALLOC(&bnx2x_fp(bp, index, rx_comp_mapping),
4451 sizeof(struct eth_fast_path_rx_cqe) * NUM_RCQ_BD);
4452 if (!bnx2x_fp(bp, index, rx_comp_ring))
4456 bnx2x_fp(bp, index, rx_page_ring) =
4457 kcalloc(NUM_RX_SGE, sizeof(struct sw_rx_page),
4459 if (!bnx2x_fp(bp, index, rx_page_ring))
4461 bnx2x_fp(bp, index, rx_sge_ring) =
4462 BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, rx_sge_mapping),
4463 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4464 if (!bnx2x_fp(bp, index, rx_sge_ring))
4467 bnx2x_set_next_page_rx_bd(fp);
4470 bnx2x_set_next_page_rx_cq(fp);
4473 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
4474 if (ring_size < rx_ring_size)
4480 /* handles low memory cases */
4482 BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
4484 /* FW will drop all packets if queue is not big enough,
4485 * In these cases we disable the queue
4486 * Min size is different for OOO, TPA and non-TPA queues
4488 if (ring_size < (fp->disable_tpa ?
4489 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
4490 /* release memory allocated for this queue */
4491 bnx2x_free_fp_mem_at(bp, index);
4497 static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp)
4501 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp)))
4502 /* we will fail load process instead of mark
4510 static int bnx2x_alloc_fp_mem(struct bnx2x *bp)
4514 /* 1. Allocate FP for leading - fatal if error
4515 * 2. Allocate RSS - fix number of queues if error
4519 if (bnx2x_alloc_fp_mem_at(bp, 0))
4523 for_each_nondefault_eth_queue(bp, i)
4524 if (bnx2x_alloc_fp_mem_at(bp, i))
4527 /* handle memory failures */
4528 if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
4529 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
4532 bnx2x_shrink_eth_fp(bp, delta);
4533 if (CNIC_SUPPORT(bp))
4534 /* move non eth FPs next to last eth FP
4535 * must be done in that order
4536 * FCOE_IDX < FWD_IDX < OOO_IDX
4539 /* move FCoE fp even NO_FCOE_FLAG is on */
4540 bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta);
4541 bp->num_ethernet_queues -= delta;
4542 bp->num_queues = bp->num_ethernet_queues +
4543 bp->num_cnic_queues;
4544 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
4545 bp->num_queues + delta, bp->num_queues);
4551 void bnx2x_free_mem_bp(struct bnx2x *bp)
4555 for (i = 0; i < bp->fp_array_size; i++)
4556 kfree(bp->fp[i].tpa_info);
4559 kfree(bp->fp_stats);
4560 kfree(bp->bnx2x_txq);
4561 kfree(bp->msix_table);
4565 int bnx2x_alloc_mem_bp(struct bnx2x *bp)
4567 struct bnx2x_fastpath *fp;
4568 struct msix_entry *tbl;
4569 struct bnx2x_ilt *ilt;
4570 int msix_table_size = 0;
4571 int fp_array_size, txq_array_size;
4575 * The biggest MSI-X table we might need is as a maximum number of fast
4576 * path IGU SBs plus default SB (for PF only).
4578 msix_table_size = bp->igu_sb_cnt;
4581 BNX2X_DEV_INFO("msix_table_size %d\n", msix_table_size);
4583 /* fp array: RSS plus CNIC related L2 queues */
4584 fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp);
4585 bp->fp_array_size = fp_array_size;
4586 BNX2X_DEV_INFO("fp_array_size %d\n", bp->fp_array_size);
4588 fp = kcalloc(bp->fp_array_size, sizeof(*fp), GFP_KERNEL);
4591 for (i = 0; i < bp->fp_array_size; i++) {
4593 kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2,
4594 sizeof(struct bnx2x_agg_info), GFP_KERNEL);
4595 if (!(fp[i].tpa_info))
4601 /* allocate sp objs */
4602 bp->sp_objs = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_sp_objs),
4607 /* allocate fp_stats */
4608 bp->fp_stats = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_fp_stats),
4613 /* Allocate memory for the transmission queues array */
4615 BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp);
4616 BNX2X_DEV_INFO("txq_array_size %d", txq_array_size);
4618 bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata),
4624 tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
4627 bp->msix_table = tbl;
4630 ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
4637 bnx2x_free_mem_bp(bp);
4641 int bnx2x_reload_if_running(struct net_device *dev)
4643 struct bnx2x *bp = netdev_priv(dev);
4645 if (unlikely(!netif_running(dev)))
4648 bnx2x_nic_unload(bp, UNLOAD_NORMAL, true);
4649 return bnx2x_nic_load(bp, LOAD_NORMAL);
4652 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
4654 u32 sel_phy_idx = 0;
4655 if (bp->link_params.num_phys <= 1)
4658 if (bp->link_vars.link_up) {
4659 sel_phy_idx = EXT_PHY1;
4660 /* In case link is SERDES, check if the EXT_PHY2 is the one */
4661 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
4662 (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
4663 sel_phy_idx = EXT_PHY2;
4666 switch (bnx2x_phy_selection(&bp->link_params)) {
4667 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
4668 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
4669 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
4670 sel_phy_idx = EXT_PHY1;
4672 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
4673 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
4674 sel_phy_idx = EXT_PHY2;
4681 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
4683 u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
4685 * The selected activated PHY is always after swapping (in case PHY
4686 * swapping is enabled). So when swapping is enabled, we need to reverse
4690 if (bp->link_params.multi_phy_config &
4691 PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
4692 if (sel_phy_idx == EXT_PHY1)
4693 sel_phy_idx = EXT_PHY2;
4694 else if (sel_phy_idx == EXT_PHY2)
4695 sel_phy_idx = EXT_PHY1;
4697 return LINK_CONFIG_IDX(sel_phy_idx);
4700 #ifdef NETDEV_FCOE_WWNN
4701 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
4703 struct bnx2x *bp = netdev_priv(dev);
4704 struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
4707 case NETDEV_FCOE_WWNN:
4708 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
4709 cp->fcoe_wwn_node_name_lo);
4711 case NETDEV_FCOE_WWPN:
4712 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
4713 cp->fcoe_wwn_port_name_lo);
4716 BNX2X_ERR("Wrong WWN type requested - %d\n", type);
4724 /* called with rtnl_lock */
4725 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
4727 struct bnx2x *bp = netdev_priv(dev);
4729 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4730 BNX2X_ERR("Can't perform change MTU during parity recovery\n");
4734 if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
4735 ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) {
4736 BNX2X_ERR("Can't support requested MTU size\n");
4740 /* This does not race with packet allocation
4741 * because the actual alloc size is
4742 * only updated as part of load
4746 return bnx2x_reload_if_running(dev);
4749 netdev_features_t bnx2x_fix_features(struct net_device *dev,
4750 netdev_features_t features)
4752 struct bnx2x *bp = netdev_priv(dev);
4754 /* TPA requires Rx CSUM offloading */
4755 if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa) {
4756 features &= ~NETIF_F_LRO;
4757 features &= ~NETIF_F_GRO;
4763 int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
4765 struct bnx2x *bp = netdev_priv(dev);
4766 u32 flags = bp->flags;
4768 bool bnx2x_reload = false;
4770 if (features & NETIF_F_LRO)
4771 flags |= TPA_ENABLE_FLAG;
4773 flags &= ~TPA_ENABLE_FLAG;
4775 if (features & NETIF_F_GRO)
4776 flags |= GRO_ENABLE_FLAG;
4778 flags &= ~GRO_ENABLE_FLAG;
4780 if (features & NETIF_F_LOOPBACK) {
4781 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
4782 bp->link_params.loopback_mode = LOOPBACK_BMAC;
4783 bnx2x_reload = true;
4786 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
4787 bp->link_params.loopback_mode = LOOPBACK_NONE;
4788 bnx2x_reload = true;
4792 changes = flags ^ bp->flags;
4794 /* if GRO is changed while LRO is enabled, don't force a reload */
4795 if ((changes & GRO_ENABLE_FLAG) && (flags & TPA_ENABLE_FLAG))
4796 changes &= ~GRO_ENABLE_FLAG;
4799 bnx2x_reload = true;
4804 if (bp->recovery_state == BNX2X_RECOVERY_DONE)
4805 return bnx2x_reload_if_running(dev);
4806 /* else: bnx2x_nic_load() will be called at end of recovery */
4812 void bnx2x_tx_timeout(struct net_device *dev)
4814 struct bnx2x *bp = netdev_priv(dev);
4816 #ifdef BNX2X_STOP_ON_ERROR
4821 /* This allows the netif to be shutdown gracefully before resetting */
4822 bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_TX_TIMEOUT, 0);
4825 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
4827 struct net_device *dev = pci_get_drvdata(pdev);
4831 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4834 bp = netdev_priv(dev);
4838 pci_save_state(pdev);
4840 if (!netif_running(dev)) {
4845 netif_device_detach(dev);
4847 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
4849 bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
4856 int bnx2x_resume(struct pci_dev *pdev)
4858 struct net_device *dev = pci_get_drvdata(pdev);
4863 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4866 bp = netdev_priv(dev);
4868 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4869 BNX2X_ERR("Handling parity error recovery. Try again later\n");
4875 pci_restore_state(pdev);
4877 if (!netif_running(dev)) {
4882 bnx2x_set_power_state(bp, PCI_D0);
4883 netif_device_attach(dev);
4885 rc = bnx2x_nic_load(bp, LOAD_OPEN);
4892 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
4896 BNX2X_ERR("bad context pointer %p\n", cxt);
4900 /* ustorm cxt validation */
4901 cxt->ustorm_ag_context.cdu_usage =
4902 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4903 CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
4904 /* xcontext validation */
4905 cxt->xstorm_ag_context.cdu_reserved =
4906 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4907 CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
4910 static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
4911 u8 fw_sb_id, u8 sb_index,
4914 u32 addr = BAR_CSTRORM_INTMEM +
4915 CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
4916 REG_WR8(bp, addr, ticks);
4918 "port %x fw_sb_id %d sb_index %d ticks %d\n",
4919 port, fw_sb_id, sb_index, ticks);
4922 static void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
4923 u16 fw_sb_id, u8 sb_index,
4926 u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
4927 u32 addr = BAR_CSTRORM_INTMEM +
4928 CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
4929 u8 flags = REG_RD8(bp, addr);
4931 flags &= ~HC_INDEX_DATA_HC_ENABLED;
4932 flags |= enable_flag;
4933 REG_WR8(bp, addr, flags);
4935 "port %x fw_sb_id %d sb_index %d disable %d\n",
4936 port, fw_sb_id, sb_index, disable);
4939 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
4940 u8 sb_index, u8 disable, u16 usec)
4942 int port = BP_PORT(bp);
4943 u8 ticks = usec / BNX2X_BTR;
4945 storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
4947 disable = disable ? 1 : (usec ? 0 : 1);
4948 storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
4951 void bnx2x_schedule_sp_rtnl(struct bnx2x *bp, enum sp_rtnl_flag flag,
4954 smp_mb__before_atomic();
4955 set_bit(flag, &bp->sp_rtnl_state);
4956 smp_mb__after_atomic();
4957 DP((BNX2X_MSG_SP | verbose), "Scheduling sp_rtnl task [Flag: %d]\n",
4959 schedule_delayed_work(&bp->sp_rtnl_task, 0);
4961 EXPORT_SYMBOL(bnx2x_schedule_sp_rtnl);