projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
bnx2x: Remove local defines for %pM and mac address
[linux-2.6-block.git] / drivers / net / ethernet / broadcom / bnx2x / bnx2x_cmn.c
CommitLineData
9f6c9258
DK		 1/* bnx2x_cmn.c: Broadcom Everest network driver.
2 *
5de92408 3 * Copyright (c) 2007-2011 Broadcom Corporation
9f6c9258
DK		 4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation.
8 *
9 * Maintained by: Eilon Greenstein <eilong@broadcom.com>
10 * Written by: Eliezer Tamir
11 * Based on code from Michael Chan's bnx2 driver
12 * UDP CSUM errata workaround by Arik Gendelman
13 * Slowpath and fastpath rework by Vladislav Zolotarov
14 * Statistics and Link management by Yitchak Gertner
15 *
16 */
17
9f6c9258 18#include <linux/etherdevice.h>
9bcc0893 19#include <linux/if_vlan.h>
a6b7a407 20#include <linux/interrupt.h>
9f6c9258 21#include <linux/ip.h>
f2e0899f 22#include <net/ipv6.h>
7f3e01fe 23#include <net/ip6_checksum.h>
6891dd25 24#include <linux/firmware.h>
c0cba59e 25#include <linux/prefetch.h>
9f6c9258 26#include "bnx2x_cmn.h"
523224a3 27#include "bnx2x_init.h"
042181f5 28#include "bnx2x_sp.h"
523224a3 29
619c5cb6 30
9f6c9258 31
b3b83c3f
DK		 32/**
33 * bnx2x_bz_fp - zero content of the fastpath structure.
34 *
35 * @bp: driver handle
36 * @index: fastpath index to be zeroed
37 *
38 * Makes sure the contents of the bp->fp[index].napi is kept
39 * intact.
40 */
41static inline void bnx2x_bz_fp(struct bnx2x *bp, int index)
42{
43 struct bnx2x_fastpath *fp = &bp->fp[index];
44 struct napi_struct orig_napi = fp->napi;
45 /* bzero bnx2x_fastpath contents */
46 memset(fp, 0, sizeof(*fp));
47
48 /* Restore the NAPI object as it has been already initialized */
49 fp->napi = orig_napi;
6383c0b3
AE		 50
51 fp->bp = bp;
52 fp->index = index;
53 if (IS_ETH_FP(fp))
54 fp->max_cos = bp->max_cos;
55 else
56 /* Special queues support only one CoS */
57 fp->max_cos = 1;
58
59 /*
60 * set the tpa flag for each queue. The tpa flag determines the queue
61 * minimal size so it must be set prior to queue memory allocation
62 */
63 fp->disable_tpa = ((bp->flags & TPA_ENABLE_FLAG) == 0);
64
65#ifdef BCM_CNIC
66 /* We don't want TPA on FCoE, FWD and OOO L2 rings */
67 bnx2x_fcoe(bp, disable_tpa) = 1;
68#endif
b3b83c3f
DK		 69}
70
71/**
72 * bnx2x_move_fp - move content of the fastpath structure.
73 *
74 * @bp: driver handle
75 * @from: source FP index
76 * @to: destination FP index
77 *
78 * Makes sure the contents of the bp->fp[to].napi is kept
79 * intact.
80 */
81static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
82{
83 struct bnx2x_fastpath *from_fp = &bp->fp[from];
84 struct bnx2x_fastpath *to_fp = &bp->fp[to];
85 struct napi_struct orig_napi = to_fp->napi;
86 /* Move bnx2x_fastpath contents */
87 memcpy(to_fp, from_fp, sizeof(*to_fp));
88 to_fp->index = to;
89
90 /* Restore the NAPI object as it has been already initialized */
91 to_fp->napi = orig_napi;
92}
93
619c5cb6
VZ		 94int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
95
9f6c9258
DK		 96/* free skb in the packet ring at pos idx
97 * return idx of last bd freed
98 */
6383c0b3 99static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
9f6c9258
DK		 100 u16 idx)
101{
6383c0b3 102 struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
9f6c9258
DK		 103 struct eth_tx_start_bd *tx_start_bd;
104 struct eth_tx_bd *tx_data_bd;
105 struct sk_buff *skb = tx_buf->skb;
106 u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
107 int nbd;
108
109 /* prefetch skb end pointer to speedup dev_kfree_skb() */
110 prefetch(&skb->end);
111
619c5cb6 112 DP(BNX2X_MSG_FP, "fp[%d]: pkt_idx %d buff @(%p)->skb %p\n",
6383c0b3 113 txdata->txq_index, idx, tx_buf, skb);
9f6c9258
DK		 114
115 /* unmap first bd */
116 DP(BNX2X_MSG_OFF, "free bd_idx %d\n", bd_idx);
6383c0b3 117 tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
9f6c9258 118 dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
4bca60f4 119 BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
9f6c9258 120
619c5cb6 121
9f6c9258
DK		 122 nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
123#ifdef BNX2X_STOP_ON_ERROR
124 if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
125 BNX2X_ERR("BAD nbd!\n");
126 bnx2x_panic();
127 }
128#endif
129 new_cons = nbd + tx_buf->first_bd;
130
131 /* Get the next bd */
132 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
133
134 /* Skip a parse bd... */
135 --nbd;
136 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
137
138 /* ...and the TSO split header bd since they have no mapping */
139 if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
140 --nbd;
141 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
142 }
143
144 /* now free frags */
145 while (nbd > 0) {
146
147 DP(BNX2X_MSG_OFF, "free frag bd_idx %d\n", bd_idx);
6383c0b3 148 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
9f6c9258
DK		 149 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
150 BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
151 if (--nbd)
152 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
153 }
154
155 /* release skb */
156 WARN_ON(!skb);
40955532 157 dev_kfree_skb_any(skb);
9f6c9258
DK		 158 tx_buf->first_bd = 0;
159 tx_buf->skb = NULL;
160
161 return new_cons;
162}
163
6383c0b3 164int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
9f6c9258 165{
9f6c9258 166 struct netdev_queue *txq;
6383c0b3 167 u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
9f6c9258
DK		 168
169#ifdef BNX2X_STOP_ON_ERROR
170 if (unlikely(bp->panic))
171 return -1;
172#endif
173
6383c0b3
AE		 174 txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
175 hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
176 sw_cons = txdata->tx_pkt_cons;
9f6c9258
DK		 177
178 while (sw_cons != hw_cons) {
179 u16 pkt_cons;
180
181 pkt_cons = TX_BD(sw_cons);
182
f2e0899f
DK		 183 DP(NETIF_MSG_TX_DONE, "queue[%d]: hw_cons %u sw_cons %u "
184 " pkt_cons %u\n",
6383c0b3 185 txdata->txq_index, hw_cons, sw_cons, pkt_cons);
9f6c9258 186
6383c0b3 187 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons);
9f6c9258
DK		 188 sw_cons++;
189 }
190
6383c0b3
AE		 191 txdata->tx_pkt_cons = sw_cons;
192 txdata->tx_bd_cons = bd_cons;
9f6c9258
DK		 193
194 /* Need to make the tx_bd_cons update visible to start_xmit()
195 * before checking for netif_tx_queue_stopped(). Without the
196 * memory barrier, there is a small possibility that
197 * start_xmit() will miss it and cause the queue to be stopped
198 * forever.
619c5cb6
VZ		 199 * On the other hand we need an rmb() here to ensure the proper
200 * ordering of bit testing in the following
201 * netif_tx_queue_stopped(txq) call.
9f6c9258
DK		 202 */
203 smp_mb();
204
9f6c9258
DK		 205 if (unlikely(netif_tx_queue_stopped(txq))) {
206 /* Taking tx_lock() is needed to prevent reenabling the queue
207 * while it's empty. This could have happen if rx_action() gets
208 * suspended in bnx2x_tx_int() after the condition before
209 * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
210 *
211 * stops the queue->sees fresh tx_bd_cons->releases the queue->
212 * sends some packets consuming the whole queue again->
213 * stops the queue
214 */
215
216 __netif_tx_lock(txq, smp_processor_id());
217
218 if ((netif_tx_queue_stopped(txq)) &&
219 (bp->state == BNX2X_STATE_OPEN) &&
6383c0b3 220 (bnx2x_tx_avail(bp, txdata) >= MAX_SKB_FRAGS + 3))
9f6c9258
DK		 221 netif_tx_wake_queue(txq);
222
223 __netif_tx_unlock(txq);
224 }
225 return 0;
226}
227
228static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
229 u16 idx)
230{
231 u16 last_max = fp->last_max_sge;
232
233 if (SUB_S16(idx, last_max) > 0)
234 fp->last_max_sge = idx;
235}
236
237static void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
238 struct eth_fast_path_rx_cqe *fp_cqe)
239{
240 struct bnx2x *bp = fp->bp;
241 u16 sge_len = SGE_PAGE_ALIGN(le16_to_cpu(fp_cqe->pkt_len) -
242 le16_to_cpu(fp_cqe->len_on_bd)) >>
243 SGE_PAGE_SHIFT;
244 u16 last_max, last_elem, first_elem;
245 u16 delta = 0;
246 u16 i;
247
248 if (!sge_len)
249 return;
250
251 /* First mark all used pages */
252 for (i = 0; i < sge_len; i++)
619c5cb6 253 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
523224a3 254 RX_SGE(le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[i])));
9f6c9258
DK		 255
256 DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
523224a3 257 sge_len - 1, le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[sge_len - 1]));
9f6c9258
DK		 258
259 /* Here we assume that the last SGE index is the biggest */
260 prefetch((void *)(fp->sge_mask));
523224a3
DK		 261 bnx2x_update_last_max_sge(fp,
262 le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[sge_len - 1]));
9f6c9258
DK		 263
264 last_max = RX_SGE(fp->last_max_sge);
619c5cb6
VZ		 265 last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
266 first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
9f6c9258
DK		 267
268 /* If ring is not full */
269 if (last_elem + 1 != first_elem)
270 last_elem++;
271
272 /* Now update the prod */
273 for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
274 if (likely(fp->sge_mask[i]))
275 break;
276
619c5cb6
VZ		 277 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
278 delta += BIT_VEC64_ELEM_SZ;
9f6c9258
DK		 279 }
280
281 if (delta > 0) {
282 fp->rx_sge_prod += delta;
283 /* clear page-end entries */
284 bnx2x_clear_sge_mask_next_elems(fp);
285 }
286
287 DP(NETIF_MSG_RX_STATUS,
288 "fp->last_max_sge = %d fp->rx_sge_prod = %d\n",
289 fp->last_max_sge, fp->rx_sge_prod);
290}
291
292static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
619c5cb6
VZ		 293 struct sk_buff *skb, u16 cons, u16 prod,
294 struct eth_fast_path_rx_cqe *cqe)
9f6c9258
DK		 295{
296 struct bnx2x *bp = fp->bp;
297 struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
298 struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
299 struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
300 dma_addr_t mapping;
619c5cb6
VZ		 301 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
302 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
9f6c9258 303
619c5cb6
VZ		 304 /* print error if current state != stop */
305 if (tpa_info->tpa_state != BNX2X_TPA_STOP)
9f6c9258
DK		 306 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
307
619c5cb6
VZ		 308 /* Try to map an empty skb from the aggregation info */
309 mapping = dma_map_single(&bp->pdev->dev,
310 first_buf->skb->data,
311 fp->rx_buf_size, DMA_FROM_DEVICE);
312 /*
313 * ...if it fails - move the skb from the consumer to the producer
314 * and set the current aggregation state as ERROR to drop it
315 * when TPA_STOP arrives.
316 */
317
318 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
319 /* Move the BD from the consumer to the producer */
320 bnx2x_reuse_rx_skb(fp, cons, prod);
321 tpa_info->tpa_state = BNX2X_TPA_ERROR;
322 return;
323 }
9f6c9258 324
619c5cb6
VZ		 325 /* move empty skb from pool to prod */
326 prod_rx_buf->skb = first_buf->skb;
327 dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
9f6c9258
DK		 328 /* point prod_bd to new skb */
329 prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
330 prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
331
619c5cb6
VZ		 332 /* move partial skb from cons to pool (don't unmap yet) */
333 *first_buf = *cons_rx_buf;
334
335 /* mark bin state as START */
336 tpa_info->parsing_flags =
337 le16_to_cpu(cqe->pars_flags.flags);
338 tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
339 tpa_info->tpa_state = BNX2X_TPA_START;
340 tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
341 tpa_info->placement_offset = cqe->placement_offset;
342
9f6c9258
DK		 343#ifdef BNX2X_STOP_ON_ERROR
344 fp->tpa_queue_used |= (1 << queue);
345#ifdef _ASM_GENERIC_INT_L64_H
346 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
347#else
348 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
349#endif
350 fp->tpa_queue_used);
351#endif
352}
353
e4e3c02a
VZ		 354/* Timestamp option length allowed for TPA aggregation:
355 *
356 * nop nop kind length echo val
357 */
358#define TPA_TSTAMP_OPT_LEN 12
359/**
e8920674 360 * bnx2x_set_lro_mss - calculate the approximate value of the MSS
e4e3c02a 361 *
e8920674
DK		 362 * @bp: driver handle
363 * @parsing_flags: parsing flags from the START CQE
364 * @len_on_bd: total length of the first packet for the
365 * aggregation.
366 *
367 * Approximate value of the MSS for this aggregation calculated using
368 * the first packet of it.
e4e3c02a
VZ		 369 */
370static inline u16 bnx2x_set_lro_mss(struct bnx2x *bp, u16 parsing_flags,
371 u16 len_on_bd)
372{
619c5cb6
VZ		 373 /*
374 * TPA arrgregation won't have either IP options or TCP options
375 * other than timestamp or IPv6 extension headers.
e4e3c02a 376 */
619c5cb6
VZ		 377 u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
378
379 if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
380 PRS_FLAG_OVERETH_IPV6)
381 hdrs_len += sizeof(struct ipv6hdr);
382 else /* IPv4 */
383 hdrs_len += sizeof(struct iphdr);
e4e3c02a
VZ		 384
385
386 /* Check if there was a TCP timestamp, if there is it's will
387 * always be 12 bytes length: nop nop kind length echo val.
388 *
389 * Otherwise FW would close the aggregation.
390 */
391 if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
392 hdrs_len += TPA_TSTAMP_OPT_LEN;
393
394 return len_on_bd - hdrs_len;
395}
396
9f6c9258 397static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
619c5cb6
VZ		 398 u16 queue, struct sk_buff *skb,
399 struct eth_end_agg_rx_cqe *cqe,
400 u16 cqe_idx)
9f6c9258
DK		 401{
402 struct sw_rx_page *rx_pg, old_rx_pg;
9f6c9258
DK		 403 u32 i, frag_len, frag_size, pages;
404 int err;
405 int j;
619c5cb6
VZ		 406 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
407 u16 len_on_bd = tpa_info->len_on_bd;
9f6c9258 408
619c5cb6 409 frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
9f6c9258
DK		 410 pages = SGE_PAGE_ALIGN(frag_size) >> SGE_PAGE_SHIFT;
411
412 /* This is needed in order to enable forwarding support */
413 if (frag_size)
619c5cb6
VZ		 414 skb_shinfo(skb)->gso_size = bnx2x_set_lro_mss(bp,
415 tpa_info->parsing_flags, len_on_bd);
9f6c9258
DK		 416
417#ifdef BNX2X_STOP_ON_ERROR
418 if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
419 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
420 pages, cqe_idx);
619c5cb6 421 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
9f6c9258
DK		 422 bnx2x_panic();
423 return -EINVAL;
424 }
425#endif
426
427 /* Run through the SGL and compose the fragmented skb */
428 for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
619c5cb6 429 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
9f6c9258
DK		 430
431 /* FW gives the indices of the SGE as if the ring is an array
432 (meaning that "next" element will consume 2 indices) */
433 frag_len = min(frag_size, (u32)(SGE_PAGE_SIZE*PAGES_PER_SGE));
434 rx_pg = &fp->rx_page_ring[sge_idx];
435 old_rx_pg = *rx_pg;
436
437 /* If we fail to allocate a substitute page, we simply stop
438 where we are and drop the whole packet */
439 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
440 if (unlikely(err)) {
441 fp->eth_q_stats.rx_skb_alloc_failed++;
442 return err;
443 }
444
445 /* Unmap the page as we r going to pass it to the stack */
446 dma_unmap_page(&bp->pdev->dev,
447 dma_unmap_addr(&old_rx_pg, mapping),
448 SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
449
450 /* Add one frag and update the appropriate fields in the skb */
451 skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
452
453 skb->data_len += frag_len;
454 skb->truesize += frag_len;
455 skb->len += frag_len;
456
457 frag_size -= frag_len;
458 }
459
460 return 0;
461}
462
463static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
619c5cb6 464 u16 queue, struct eth_end_agg_rx_cqe *cqe,
9f6c9258
DK		 465 u16 cqe_idx)
466{
619c5cb6
VZ		 467 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
468 struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
469 u8 pad = tpa_info->placement_offset;
470 u16 len = tpa_info->len_on_bd;
9f6c9258
DK		 471 struct sk_buff *skb = rx_buf->skb;
472 /* alloc new skb */
619c5cb6
VZ		 473 struct sk_buff *new_skb;
474 u8 old_tpa_state = tpa_info->tpa_state;
475
476 tpa_info->tpa_state = BNX2X_TPA_STOP;
477
478 /* If we there was an error during the handling of the TPA_START -
479 * drop this aggregation.
480 */
481 if (old_tpa_state == BNX2X_TPA_ERROR)
482 goto drop;
483
484 /* Try to allocate the new skb */
485 new_skb = netdev_alloc_skb(bp->dev, fp->rx_buf_size);
9f6c9258
DK		 486
487 /* Unmap skb in the pool anyway, as we are going to change
488 pool entry status to BNX2X_TPA_STOP even if new skb allocation
489 fails. */
490 dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
a8c94b91 491 fp->rx_buf_size, DMA_FROM_DEVICE);
9f6c9258
DK		 492
493 if (likely(new_skb)) {
9f6c9258 494 prefetch(skb);
217de5aa 495 prefetch(((char *)(skb)) + L1_CACHE_BYTES);
9f6c9258
DK		 496
497#ifdef BNX2X_STOP_ON_ERROR
a8c94b91 498 if (pad + len > fp->rx_buf_size) {
9f6c9258
DK		 499 BNX2X_ERR("skb_put is about to fail... "
500 "pad %d len %d rx_buf_size %d\n",
a8c94b91 501 pad, len, fp->rx_buf_size);
9f6c9258
DK		 502 bnx2x_panic();
503 return;
504 }
505#endif
506
507 skb_reserve(skb, pad);
508 skb_put(skb, len);
509
510 skb->protocol = eth_type_trans(skb, bp->dev);
511 skb->ip_summed = CHECKSUM_UNNECESSARY;
512
619c5cb6
VZ		 513 if (!bnx2x_fill_frag_skb(bp, fp, queue, skb, cqe, cqe_idx)) {
514 if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
515 __vlan_hwaccel_put_tag(skb, tpa_info->vlan_tag);
9bcc0893 516 napi_gro_receive(&fp->napi, skb);
9f6c9258
DK		 517 } else {
518 DP(NETIF_MSG_RX_STATUS, "Failed to allocate new pages"
519 " - dropping packet!\n");
40955532 520 dev_kfree_skb_any(skb);
9f6c9258
DK		 521 }
522
523
524 /* put new skb in bin */
619c5cb6 525 rx_buf->skb = new_skb;
9f6c9258 526
619c5cb6 527 return;
9f6c9258
DK		 528 }
529
619c5cb6
VZ		 530drop:
531 /* drop the packet and keep the buffer in the bin */
532 DP(NETIF_MSG_RX_STATUS,
533 "Failed to allocate or map a new skb - dropping packet!\n");
534 fp->eth_q_stats.rx_skb_alloc_failed++;
9f6c9258
DK		 535}
536
537/* Set Toeplitz hash value in the skb using the value from the
538 * CQE (calculated by HW).
539 */
540static inline void bnx2x_set_skb_rxhash(struct bnx2x *bp, union eth_rx_cqe *cqe,
541 struct sk_buff *skb)
542{
543 /* Set Toeplitz hash from CQE */
544 if ((bp->dev->features & NETIF_F_RXHASH) &&
545 (cqe->fast_path_cqe.status_flags &
546 ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG))
547 skb->rxhash =
548 le32_to_cpu(cqe->fast_path_cqe.rss_hash_result);
549}
550
551int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
552{
553 struct bnx2x *bp = fp->bp;
554 u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
555 u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
556 int rx_pkt = 0;
557
558#ifdef BNX2X_STOP_ON_ERROR
559 if (unlikely(bp->panic))
560 return 0;
561#endif
562
563 /* CQ "next element" is of the size of the regular element,
564 that's why it's ok here */
565 hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
566 if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
567 hw_comp_cons++;
568
569 bd_cons = fp->rx_bd_cons;
570 bd_prod = fp->rx_bd_prod;
571 bd_prod_fw = bd_prod;
572 sw_comp_cons = fp->rx_comp_cons;
573 sw_comp_prod = fp->rx_comp_prod;
574
575 /* Memory barrier necessary as speculative reads of the rx
576 * buffer can be ahead of the index in the status block
577 */
578 rmb();
579
580 DP(NETIF_MSG_RX_STATUS,
581 "queue[%d]: hw_comp_cons %u sw_comp_cons %u\n",
582 fp->index, hw_comp_cons, sw_comp_cons);
583
584 while (sw_comp_cons != hw_comp_cons) {
585 struct sw_rx_bd *rx_buf = NULL;
586 struct sk_buff *skb;
587 union eth_rx_cqe *cqe;
619c5cb6 588 struct eth_fast_path_rx_cqe *cqe_fp;
9f6c9258 589 u8 cqe_fp_flags;
619c5cb6 590 enum eth_rx_cqe_type cqe_fp_type;
9f6c9258
DK		 591 u16 len, pad;
592
619c5cb6
VZ		 593#ifdef BNX2X_STOP_ON_ERROR
594 if (unlikely(bp->panic))
595 return 0;
596#endif
597
9f6c9258
DK		 598 comp_ring_cons = RCQ_BD(sw_comp_cons);
599 bd_prod = RX_BD(bd_prod);
600 bd_cons = RX_BD(bd_cons);
601
602 /* Prefetch the page containing the BD descriptor
603 at producer's index. It will be needed when new skb is
604 allocated */
605 prefetch((void *)(PAGE_ALIGN((unsigned long)
606 (&fp->rx_desc_ring[bd_prod])) -
607 PAGE_SIZE + 1));
608
609 cqe = &fp->rx_comp_ring[comp_ring_cons];
619c5cb6
VZ		 610 cqe_fp = &cqe->fast_path_cqe;
611 cqe_fp_flags = cqe_fp->type_error_flags;
612 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
9f6c9258
DK		 613
614 DP(NETIF_MSG_RX_STATUS, "CQE type %x err %x status %x"
615 " queue %x vlan %x len %u\n", CQE_TYPE(cqe_fp_flags),
619c5cb6
VZ		 616 cqe_fp_flags, cqe_fp->status_flags,
617 le32_to_cpu(cqe_fp->rss_hash_result),
618 le16_to_cpu(cqe_fp->vlan_tag), le16_to_cpu(cqe_fp->pkt_len));
9f6c9258
DK		 619
620 /* is this a slowpath msg? */
619c5cb6 621 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
9f6c9258
DK		 622 bnx2x_sp_event(fp, cqe);
623 goto next_cqe;
624
625 /* this is an rx packet */
626 } else {
627 rx_buf = &fp->rx_buf_ring[bd_cons];
628 skb = rx_buf->skb;
629 prefetch(skb);
9f6c9258 630
619c5cb6
VZ		 631 if (!CQE_TYPE_FAST(cqe_fp_type)) {
632#ifdef BNX2X_STOP_ON_ERROR
633 /* sanity check */
634 if (fp->disable_tpa &&
635 (CQE_TYPE_START(cqe_fp_type) ||
636 CQE_TYPE_STOP(cqe_fp_type)))
637 BNX2X_ERR("START/STOP packet while "
638 "disable_tpa type %x\n",
639 CQE_TYPE(cqe_fp_type));
640#endif
9f6c9258 641
619c5cb6
VZ		 642 if (CQE_TYPE_START(cqe_fp_type)) {
643 u16 queue = cqe_fp->queue_index;
9f6c9258
DK		 644 DP(NETIF_MSG_RX_STATUS,
645 "calling tpa_start on queue %d\n",
646 queue);
647
648 bnx2x_tpa_start(fp, queue, skb,
619c5cb6
VZ		 649 bd_cons, bd_prod,
650 cqe_fp);
9f6c9258 651
619c5cb6 652 /* Set Toeplitz hash for LRO skb */
9f6c9258
DK		 653 bnx2x_set_skb_rxhash(bp, cqe, skb);
654
655 goto next_rx;
619c5cb6
VZ		 656
657 } else {
658 u16 queue =
659 cqe->end_agg_cqe.queue_index;
9f6c9258
DK		 660 DP(NETIF_MSG_RX_STATUS,
661 "calling tpa_stop on queue %d\n",
662 queue);
663
619c5cb6
VZ		 664 bnx2x_tpa_stop(bp, fp, queue,
665 &cqe->end_agg_cqe,
666 comp_ring_cons);
9f6c9258
DK		 667#ifdef BNX2X_STOP_ON_ERROR
668 if (bp->panic)
669 return 0;
670#endif
671
619c5cb6 672 bnx2x_update_sge_prod(fp, cqe_fp);
9f6c9258
DK		 673 goto next_cqe;
674 }
675 }
619c5cb6
VZ		 676 /* non TPA */
677 len = le16_to_cpu(cqe_fp->pkt_len);
678 pad = cqe_fp->placement_offset;
9924cafc 679 dma_sync_single_for_cpu(&bp->pdev->dev,
9f6c9258 680 dma_unmap_addr(rx_buf, mapping),
619c5cb6
VZ		 681 pad + RX_COPY_THRESH,
682 DMA_FROM_DEVICE);
217de5aa 683 prefetch(((char *)(skb)) + L1_CACHE_BYTES);
9f6c9258
DK		 684
685 /* is this an error packet? */
686 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
687 DP(NETIF_MSG_RX_ERR,
688 "ERROR flags %x rx packet %u\n",
689 cqe_fp_flags, sw_comp_cons);
690 fp->eth_q_stats.rx_err_discard_pkt++;
691 goto reuse_rx;
692 }
693
694 /* Since we don't have a jumbo ring
695 * copy small packets if mtu > 1500
696 */
697 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
698 (len <= RX_COPY_THRESH)) {
699 struct sk_buff *new_skb;
700
619c5cb6 701 new_skb = netdev_alloc_skb(bp->dev, len + pad);
9f6c9258
DK		 702 if (new_skb == NULL) {
703 DP(NETIF_MSG_RX_ERR,
704 "ERROR packet dropped "
705 "because of alloc failure\n");
706 fp->eth_q_stats.rx_skb_alloc_failed++;
707 goto reuse_rx;
708 }
709
710 /* aligned copy */
711 skb_copy_from_linear_data_offset(skb, pad,
712 new_skb->data + pad, len);
713 skb_reserve(new_skb, pad);
714 skb_put(new_skb, len);
715
749a8503 716 bnx2x_reuse_rx_skb(fp, bd_cons, bd_prod);
9f6c9258
DK		 717
718 skb = new_skb;
719
720 } else
721 if (likely(bnx2x_alloc_rx_skb(bp, fp, bd_prod) == 0)) {
722 dma_unmap_single(&bp->pdev->dev,
723 dma_unmap_addr(rx_buf, mapping),
a8c94b91 724 fp->rx_buf_size,
9f6c9258
DK		 725 DMA_FROM_DEVICE);
726 skb_reserve(skb, pad);
727 skb_put(skb, len);
728
729 } else {
730 DP(NETIF_MSG_RX_ERR,
731 "ERROR packet dropped because "
732 "of alloc failure\n");
733 fp->eth_q_stats.rx_skb_alloc_failed++;
734reuse_rx:
749a8503 735 bnx2x_reuse_rx_skb(fp, bd_cons, bd_prod);
9f6c9258
DK		 736 goto next_rx;
737 }
738
739 skb->protocol = eth_type_trans(skb, bp->dev);
740
741 /* Set Toeplitz hash for a none-LRO skb */
742 bnx2x_set_skb_rxhash(bp, cqe, skb);
743
bc8acf2c 744 skb_checksum_none_assert(skb);
f85582f8 745
66371c44 746 if (bp->dev->features & NETIF_F_RXCSUM) {
619c5cb6 747
9f6c9258
DK		 748 if (likely(BNX2X_RX_CSUM_OK(cqe)))
749 skb->ip_summed = CHECKSUM_UNNECESSARY;
750 else
751 fp->eth_q_stats.hw_csum_err++;
752 }
753 }
754
755 skb_record_rx_queue(skb, fp->index);
756
619c5cb6
VZ		 757 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
758 PARSING_FLAGS_VLAN)
9bcc0893 759 __vlan_hwaccel_put_tag(skb,
619c5cb6 760 le16_to_cpu(cqe_fp->vlan_tag));
9bcc0893 761 napi_gro_receive(&fp->napi, skb);
9f6c9258
DK		 762
763
764next_rx:
765 rx_buf->skb = NULL;
766
767 bd_cons = NEXT_RX_IDX(bd_cons);
768 bd_prod = NEXT_RX_IDX(bd_prod);
769 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
770 rx_pkt++;
771next_cqe:
772 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
773 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
774
775 if (rx_pkt == budget)
776 break;
777 } /* while */
778
779 fp->rx_bd_cons = bd_cons;
780 fp->rx_bd_prod = bd_prod_fw;
781 fp->rx_comp_cons = sw_comp_cons;
782 fp->rx_comp_prod = sw_comp_prod;
783
784 /* Update producers */
785 bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
786 fp->rx_sge_prod);
787
788 fp->rx_pkt += rx_pkt;
789 fp->rx_calls++;
790
791 return rx_pkt;
792}
793
794static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
795{
796 struct bnx2x_fastpath *fp = fp_cookie;
797 struct bnx2x *bp = fp->bp;
6383c0b3 798 u8 cos;
9f6c9258 799
523224a3
DK		 800 DP(BNX2X_MSG_FP, "got an MSI-X interrupt on IDX:SB "
801 "[fp %d fw_sd %d igusb %d]\n",
802 fp->index, fp->fw_sb_id, fp->igu_sb_id);
803 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
9f6c9258
DK		 804
805#ifdef BNX2X_STOP_ON_ERROR
806 if (unlikely(bp->panic))
807 return IRQ_HANDLED;
808#endif
809
810 /* Handle Rx and Tx according to MSI-X vector */
811 prefetch(fp->rx_cons_sb);
6383c0b3
AE		 812
813 for_each_cos_in_tx_queue(fp, cos)
814 prefetch(fp->txdata[cos].tx_cons_sb);
815
523224a3 816 prefetch(&fp->sb_running_index[SM_RX_ID]);
9f6c9258
DK		 817 napi_schedule(&bnx2x_fp(bp, fp->index, napi));
818
819 return IRQ_HANDLED;
820}
821
9f6c9258
DK		 822/* HW Lock for shared dual port PHYs */
823void bnx2x_acquire_phy_lock(struct bnx2x *bp)
824{
825 mutex_lock(&bp->port.phy_mutex);
826
827 if (bp->port.need_hw_lock)
828 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
829}
830
831void bnx2x_release_phy_lock(struct bnx2x *bp)
832{
833 if (bp->port.need_hw_lock)
834 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
835
836 mutex_unlock(&bp->port.phy_mutex);
837}
838
0793f83f
DK		 839/* calculates MF speed according to current linespeed and MF configuration */
840u16 bnx2x_get_mf_speed(struct bnx2x *bp)
841{
842 u16 line_speed = bp->link_vars.line_speed;
843 if (IS_MF(bp)) {
faa6fcbb
DK		 844 u16 maxCfg = bnx2x_extract_max_cfg(bp,
845 bp->mf_config[BP_VN(bp)]);
846
847 /* Calculate the current MAX line speed limit for the MF
848 * devices
0793f83f 849 */
faa6fcbb
DK		 850 if (IS_MF_SI(bp))
851 line_speed = (line_speed * maxCfg) / 100;
852 else { /* SD mode */
0793f83f
DK		 853 u16 vn_max_rate = maxCfg * 100;
854
855 if (vn_max_rate < line_speed)
856 line_speed = vn_max_rate;
faa6fcbb 857 }
0793f83f
DK		 858 }
859
860 return line_speed;
861}
862
2ae17f66
VZ		 863/**
864 * bnx2x_fill_report_data - fill link report data to report
865 *
866 * @bp: driver handle
867 * @data: link state to update
868 *
869 * It uses a none-atomic bit operations because is called under the mutex.
870 */
871static inline void bnx2x_fill_report_data(struct bnx2x *bp,
872 struct bnx2x_link_report_data *data)
873{
874 u16 line_speed = bnx2x_get_mf_speed(bp);
875
876 memset(data, 0, sizeof(*data));
877
878 /* Fill the report data: efective line speed */
879 data->line_speed = line_speed;
880
881 /* Link is down */
882 if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
883 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
884 &data->link_report_flags);
885
886 /* Full DUPLEX */
887 if (bp->link_vars.duplex == DUPLEX_FULL)
888 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
889
890 /* Rx Flow Control is ON */
891 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
892 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
893
894 /* Tx Flow Control is ON */
895 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
896 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
897}
898
899/**
900 * bnx2x_link_report - report link status to OS.
901 *
902 * @bp: driver handle
903 *
904 * Calls the __bnx2x_link_report() under the same locking scheme
905 * as a link/PHY state managing code to ensure a consistent link
906 * reporting.
907 */
908
9f6c9258
DK		 909void bnx2x_link_report(struct bnx2x *bp)
910{
2ae17f66
VZ		 911 bnx2x_acquire_phy_lock(bp);
912 __bnx2x_link_report(bp);
913 bnx2x_release_phy_lock(bp);
914}
9f6c9258 915
2ae17f66
VZ		 916/**
917 * __bnx2x_link_report - report link status to OS.
918 *
919 * @bp: driver handle
920 *
921 * None atomic inmlementation.
922 * Should be called under the phy_lock.
923 */
924void __bnx2x_link_report(struct bnx2x *bp)
925{
926 struct bnx2x_link_report_data cur_data;
9f6c9258 927
2ae17f66
VZ		 928 /* reread mf_cfg */
929 if (!CHIP_IS_E1(bp))
930 bnx2x_read_mf_cfg(bp);
931
932 /* Read the current link report info */
933 bnx2x_fill_report_data(bp, &cur_data);
934
935 /* Don't report link down or exactly the same link status twice */
936 if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
937 (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
938 &bp->last_reported_link.link_report_flags) &&
939 test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
940 &cur_data.link_report_flags)))
941 return;
942
943 bp->link_cnt++;
9f6c9258 944
2ae17f66
VZ		 945 /* We are going to report a new link parameters now -
946 * remember the current data for the next time.
947 */
948 memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
9f6c9258 949
2ae17f66
VZ		 950 if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
951 &cur_data.link_report_flags)) {
952 netif_carrier_off(bp->dev);
953 netdev_err(bp->dev, "NIC Link is Down\n");
954 return;
955 } else {
956 netif_carrier_on(bp->dev);
957 netdev_info(bp->dev, "NIC Link is Up, ");
958 pr_cont("%d Mbps ", cur_data.line_speed);
9f6c9258 959
2ae17f66
VZ		 960 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
961 &cur_data.link_report_flags))
9f6c9258
DK		 962 pr_cont("full duplex");
963 else
964 pr_cont("half duplex");
965
2ae17f66
VZ		 966 /* Handle the FC at the end so that only these flags would be
967 * possibly set. This way we may easily check if there is no FC
968 * enabled.
969 */
970 if (cur_data.link_report_flags) {
971 if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
972 &cur_data.link_report_flags)) {
9f6c9258 973 pr_cont(", receive ");
2ae17f66
VZ		 974 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
975 &cur_data.link_report_flags))
9f6c9258
DK		 976 pr_cont("& transmit ");
977 } else {
978 pr_cont(", transmit ");
979 }
980 pr_cont("flow control ON");
981 }
982 pr_cont("\n");
9f6c9258
DK		 983 }
984}
985
986void bnx2x_init_rx_rings(struct bnx2x *bp)
987{
988 int func = BP_FUNC(bp);
989 int max_agg_queues = CHIP_IS_E1(bp) ? ETH_MAX_AGGREGATION_QUEUES_E1 :
619c5cb6 990 ETH_MAX_AGGREGATION_QUEUES_E1H_E2;
523224a3 991 u16 ring_prod;
9f6c9258 992 int i, j;
25141580 993
b3b83c3f 994 /* Allocate TPA resources */
ec6ba945 995 for_each_rx_queue(bp, j) {
523224a3 996 struct bnx2x_fastpath *fp = &bp->fp[j];
9f6c9258 997
a8c94b91
VZ		 998 DP(NETIF_MSG_IFUP,
999 "mtu %d rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1000
523224a3 1001 if (!fp->disable_tpa) {
619c5cb6 1002 /* Fill the per-aggregtion pool */
9f6c9258 1003 for (i = 0; i < max_agg_queues; i++) {
619c5cb6
VZ		 1004 struct bnx2x_agg_info *tpa_info =
1005 &fp->tpa_info[i];
1006 struct sw_rx_bd *first_buf =
1007 &tpa_info->first_buf;
1008
1009 first_buf->skb = netdev_alloc_skb(bp->dev,
1010 fp->rx_buf_size);
1011 if (!first_buf->skb) {
9f6c9258
DK		 1012 BNX2X_ERR("Failed to allocate TPA "
1013 "skb pool for queue[%d] - "
1014 "disabling TPA on this "
1015 "queue!\n", j);
1016 bnx2x_free_tpa_pool(bp, fp, i);
1017 fp->disable_tpa = 1;
1018 break;
1019 }
619c5cb6
VZ		 1020 dma_unmap_addr_set(first_buf, mapping, 0);
1021 tpa_info->tpa_state = BNX2X_TPA_STOP;
9f6c9258 1022 }
523224a3
DK		 1023
1024 /* "next page" elements initialization */
1025 bnx2x_set_next_page_sgl(fp);
1026
1027 /* set SGEs bit mask */
1028 bnx2x_init_sge_ring_bit_mask(fp);
1029
1030 /* Allocate SGEs and initialize the ring elements */
1031 for (i = 0, ring_prod = 0;
1032 i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1033
1034 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
1035 BNX2X_ERR("was only able to allocate "
1036 "%d rx sges\n", i);
619c5cb6
VZ		 1037 BNX2X_ERR("disabling TPA for "
1038 "queue[%d]\n", j);
523224a3 1039 /* Cleanup already allocated elements */
619c5cb6
VZ		 1040 bnx2x_free_rx_sge_range(bp, fp,
1041 ring_prod);
1042 bnx2x_free_tpa_pool(bp, fp,
1043 max_agg_queues);
523224a3
DK		 1044 fp->disable_tpa = 1;
1045 ring_prod = 0;
1046 break;
1047 }
1048 ring_prod = NEXT_SGE_IDX(ring_prod);
1049 }
1050
1051 fp->rx_sge_prod = ring_prod;
9f6c9258
DK		 1052 }
1053 }
1054
ec6ba945 1055 for_each_rx_queue(bp, j) {
9f6c9258
DK		 1056 struct bnx2x_fastpath *fp = &bp->fp[j];
1057
1058 fp->rx_bd_cons = 0;
9f6c9258 1059
b3b83c3f
DK		 1060 /* Activate BD ring */
1061 /* Warning!
1062 * this will generate an interrupt (to the TSTORM)
1063 * must only be done after chip is initialized
1064 */
1065 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1066 fp->rx_sge_prod);
9f6c9258 1067
9f6c9258
DK		 1068 if (j != 0)
1069 continue;
1070
619c5cb6 1071 if (CHIP_IS_E1(bp)) {
f2e0899f
DK		 1072 REG_WR(bp, BAR_USTRORM_INTMEM +
1073 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1074 U64_LO(fp->rx_comp_mapping));
1075 REG_WR(bp, BAR_USTRORM_INTMEM +
1076 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1077 U64_HI(fp->rx_comp_mapping));
1078 }
9f6c9258
DK		 1079 }
1080}
f85582f8 1081
9f6c9258
DK		 1082static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1083{
1084 int i;
6383c0b3 1085 u8 cos;
9f6c9258 1086
ec6ba945 1087 for_each_tx_queue(bp, i) {
9f6c9258 1088 struct bnx2x_fastpath *fp = &bp->fp[i];
6383c0b3
AE		 1089 for_each_cos_in_tx_queue(fp, cos) {
1090 struct bnx2x_fp_txdata *txdata = &fp->txdata[cos];
9f6c9258 1091
6383c0b3
AE		 1092 u16 bd_cons = txdata->tx_bd_cons;
1093 u16 sw_prod = txdata->tx_pkt_prod;
1094 u16 sw_cons = txdata->tx_pkt_cons;
9f6c9258 1095
6383c0b3
AE		 1096 while (sw_cons != sw_prod) {
1097 bd_cons = bnx2x_free_tx_pkt(bp, txdata,
1098 TX_BD(sw_cons));
1099 sw_cons++;
1100 }
9f6c9258
DK		 1101 }
1102 }
1103}
1104
b3b83c3f
DK		 1105static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1106{
1107 struct bnx2x *bp = fp->bp;
1108 int i;
1109
1110 /* ring wasn't allocated */
1111 if (fp->rx_buf_ring == NULL)
1112 return;
1113
1114 for (i = 0; i < NUM_RX_BD; i++) {
1115 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1116 struct sk_buff *skb = rx_buf->skb;
1117
1118 if (skb == NULL)
1119 continue;
b3b83c3f
DK		 1120 dma_unmap_single(&bp->pdev->dev,
1121 dma_unmap_addr(rx_buf, mapping),
1122 fp->rx_buf_size, DMA_FROM_DEVICE);
1123
1124 rx_buf->skb = NULL;
1125 dev_kfree_skb(skb);
1126 }
1127}
1128
9f6c9258
DK		 1129static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1130{
b3b83c3f 1131 int j;
9f6c9258 1132
ec6ba945 1133 for_each_rx_queue(bp, j) {
9f6c9258
DK		 1134 struct bnx2x_fastpath *fp = &bp->fp[j];
1135
b3b83c3f 1136 bnx2x_free_rx_bds(fp);
9f6c9258 1137
9f6c9258
DK		 1138 if (!fp->disable_tpa)
1139 bnx2x_free_tpa_pool(bp, fp, CHIP_IS_E1(bp) ?
1140 ETH_MAX_AGGREGATION_QUEUES_E1 :
619c5cb6 1141 ETH_MAX_AGGREGATION_QUEUES_E1H_E2);
9f6c9258
DK		 1142 }
1143}
1144
1145void bnx2x_free_skbs(struct bnx2x *bp)
1146{
1147 bnx2x_free_tx_skbs(bp);
1148 bnx2x_free_rx_skbs(bp);
1149}
1150
e3835b99
DK		 1151void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1152{
1153 /* load old values */
1154 u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1155
1156 if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1157 /* leave all but MAX value */
1158 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1159
1160 /* set new MAX value */
1161 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1162 & FUNC_MF_CFG_MAX_BW_MASK;
1163
1164 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1165 }
1166}
1167
ca92429f
DK		 1168/**
1169 * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1170 *
1171 * @bp: driver handle
1172 * @nvecs: number of vectors to be released
1173 */
1174static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
9f6c9258 1175{
ca92429f 1176 int i, offset = 0;
9f6c9258 1177
ca92429f
DK		 1178 if (nvecs == offset)
1179 return;
1180 free_irq(bp->msix_table[offset].vector, bp->dev);
9f6c9258 1181 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
ca92429f
DK		 1182 bp->msix_table[offset].vector);
1183 offset++;
9f6c9258 1184#ifdef BCM_CNIC
ca92429f
DK		 1185 if (nvecs == offset)
1186 return;
9f6c9258
DK		 1187 offset++;
1188#endif
ca92429f 1189
ec6ba945 1190 for_each_eth_queue(bp, i) {
ca92429f
DK		 1191 if (nvecs == offset)
1192 return;
1193 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d "
1194 "irq\n", i, bp->msix_table[offset].vector);
9f6c9258 1195
ca92429f 1196 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
9f6c9258
DK		 1197 }
1198}
1199
d6214d7a 1200void bnx2x_free_irq(struct bnx2x *bp)
9f6c9258 1201{
d6214d7a 1202 if (bp->flags & USING_MSIX_FLAG)
ca92429f 1203 bnx2x_free_msix_irqs(bp, BNX2X_NUM_ETH_QUEUES(bp) +
6383c0b3 1204 CNIC_PRESENT + 1);
d6214d7a
DK		 1205 else if (bp->flags & USING_MSI_FLAG)
1206 free_irq(bp->pdev->irq, bp->dev);
1207 else
9f6c9258
DK		 1208 free_irq(bp->pdev->irq, bp->dev);
1209}
1210
d6214d7a 1211int bnx2x_enable_msix(struct bnx2x *bp)
9f6c9258 1212{
d6214d7a 1213 int msix_vec = 0, i, rc, req_cnt;
9f6c9258 1214
d6214d7a
DK		 1215 bp->msix_table[msix_vec].entry = msix_vec;
1216 DP(NETIF_MSG_IFUP, "msix_table[0].entry = %d (slowpath)\n",
1217 bp->msix_table[0].entry);
1218 msix_vec++;
9f6c9258
DK		 1219
1220#ifdef BCM_CNIC
d6214d7a
DK		 1221 bp->msix_table[msix_vec].entry = msix_vec;
1222 DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d (CNIC)\n",
1223 bp->msix_table[msix_vec].entry, bp->msix_table[msix_vec].entry);
1224 msix_vec++;
9f6c9258 1225#endif
6383c0b3 1226 /* We need separate vectors for ETH queues only (not FCoE) */
ec6ba945 1227 for_each_eth_queue(bp, i) {
d6214d7a 1228 bp->msix_table[msix_vec].entry = msix_vec;
9f6c9258 1229 DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d "
d6214d7a
DK		 1230 "(fastpath #%u)\n", msix_vec, msix_vec, i);
1231 msix_vec++;
9f6c9258
DK		 1232 }
1233
6383c0b3 1234 req_cnt = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_PRESENT + 1;
d6214d7a
DK		 1235
1236 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], req_cnt);
9f6c9258
DK		 1237
1238 /*
1239 * reconfigure number of tx/rx queues according to available
1240 * MSI-X vectors
1241 */
1242 if (rc >= BNX2X_MIN_MSIX_VEC_CNT) {
d6214d7a
DK		 1243 /* how less vectors we will have? */
1244 int diff = req_cnt - rc;
9f6c9258
DK		 1245
1246 DP(NETIF_MSG_IFUP,
1247 "Trying to use less MSI-X vectors: %d\n", rc);
1248
1249 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1250
1251 if (rc) {
1252 DP(NETIF_MSG_IFUP,
1253 "MSI-X is not attainable rc %d\n", rc);
1254 return rc;
1255 }
d6214d7a
DK		 1256 /*
1257 * decrease number of queues by number of unallocated entries
1258 */
1259 bp->num_queues -= diff;
9f6c9258
DK		 1260
1261 DP(NETIF_MSG_IFUP, "New queue configuration set: %d\n",
1262 bp->num_queues);
1263 } else if (rc) {
d6214d7a
DK		 1264 /* fall to INTx if not enough memory */
1265 if (rc == -ENOMEM)
1266 bp->flags |= DISABLE_MSI_FLAG;
9f6c9258
DK		 1267 DP(NETIF_MSG_IFUP, "MSI-X is not attainable rc %d\n", rc);
1268 return rc;
1269 }
1270
1271 bp->flags |= USING_MSIX_FLAG;
1272
1273 return 0;
1274}
1275
1276static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1277{
ca92429f 1278 int i, rc, offset = 0;
9f6c9258 1279
ca92429f
DK		 1280 rc = request_irq(bp->msix_table[offset++].vector,
1281 bnx2x_msix_sp_int, 0,
9f6c9258
DK		 1282 bp->dev->name, bp->dev);
1283 if (rc) {
1284 BNX2X_ERR("request sp irq failed\n");
1285 return -EBUSY;
1286 }
1287
1288#ifdef BCM_CNIC
1289 offset++;
1290#endif
ec6ba945 1291 for_each_eth_queue(bp, i) {
9f6c9258
DK		 1292 struct bnx2x_fastpath *fp = &bp->fp[i];
1293 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1294 bp->dev->name, i);
1295
d6214d7a 1296 rc = request_irq(bp->msix_table[offset].vector,
9f6c9258
DK		 1297 bnx2x_msix_fp_int, 0, fp->name, fp);
1298 if (rc) {
ca92429f
DK		 1299 BNX2X_ERR("request fp #%d irq (%d) failed rc %d\n", i,
1300 bp->msix_table[offset].vector, rc);
1301 bnx2x_free_msix_irqs(bp, offset);
9f6c9258
DK		 1302 return -EBUSY;
1303 }
1304
d6214d7a 1305 offset++;
9f6c9258
DK		 1306 }
1307
ec6ba945 1308 i = BNX2X_NUM_ETH_QUEUES(bp);
6383c0b3 1309 offset = 1 + CNIC_PRESENT;
9f6c9258
DK		 1310 netdev_info(bp->dev, "using MSI-X IRQs: sp %d fp[%d] %d"
1311 " ... fp[%d] %d\n",
1312 bp->msix_table[0].vector,
1313 0, bp->msix_table[offset].vector,
1314 i - 1, bp->msix_table[offset + i - 1].vector);
1315
1316 return 0;
1317}
1318
d6214d7a 1319int bnx2x_enable_msi(struct bnx2x *bp)
9f6c9258
DK		 1320{
1321 int rc;
1322
1323 rc = pci_enable_msi(bp->pdev);
1324 if (rc) {
1325 DP(NETIF_MSG_IFUP, "MSI is not attainable\n");
1326 return -1;
1327 }
1328 bp->flags |= USING_MSI_FLAG;
1329
1330 return 0;
1331}
1332
1333static int bnx2x_req_irq(struct bnx2x *bp)
1334{
1335 unsigned long flags;
1336 int rc;
1337
1338 if (bp->flags & USING_MSI_FLAG)
1339 flags = 0;
1340 else
1341 flags = IRQF_SHARED;
1342
1343 rc = request_irq(bp->pdev->irq, bnx2x_interrupt, flags,
1344 bp->dev->name, bp->dev);
9f6c9258
DK		 1345 return rc;
1346}
1347
619c5cb6
VZ		 1348static inline int bnx2x_setup_irqs(struct bnx2x *bp)
1349{
1350 int rc = 0;
1351 if (bp->flags & USING_MSIX_FLAG) {
1352 rc = bnx2x_req_msix_irqs(bp);
1353 if (rc)
1354 return rc;
1355 } else {
1356 bnx2x_ack_int(bp);
1357 rc = bnx2x_req_irq(bp);
1358 if (rc) {
1359 BNX2X_ERR("IRQ request failed rc %d, aborting\n", rc);
1360 return rc;
1361 }
1362 if (bp->flags & USING_MSI_FLAG) {
1363 bp->dev->irq = bp->pdev->irq;
1364 netdev_info(bp->dev, "using MSI IRQ %d\n",
1365 bp->pdev->irq);
1366 }
1367 }
1368
1369 return 0;
1370}
1371
1372static inline void bnx2x_napi_enable(struct bnx2x *bp)
9f6c9258
DK		 1373{
1374 int i;
1375
619c5cb6 1376 for_each_rx_queue(bp, i)
9f6c9258
DK		 1377 napi_enable(&bnx2x_fp(bp, i, napi));
1378}
1379
619c5cb6 1380static inline void bnx2x_napi_disable(struct bnx2x *bp)
9f6c9258
DK		 1381{
1382 int i;
1383
619c5cb6 1384 for_each_rx_queue(bp, i)
9f6c9258
DK		 1385 napi_disable(&bnx2x_fp(bp, i, napi));
1386}
1387
1388void bnx2x_netif_start(struct bnx2x *bp)
1389{
4b7ed897
DK		 1390 if (netif_running(bp->dev)) {
1391 bnx2x_napi_enable(bp);
1392 bnx2x_int_enable(bp);
1393 if (bp->state == BNX2X_STATE_OPEN)
1394 netif_tx_wake_all_queues(bp->dev);
9f6c9258
DK		 1395 }
1396}
1397
1398void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1399{
1400 bnx2x_int_disable_sync(bp, disable_hw);
1401 bnx2x_napi_disable(bp);
9f6c9258 1402}
9f6c9258 1403
8307fa3e
VZ		 1404u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1405{
8307fa3e 1406 struct bnx2x *bp = netdev_priv(dev);
faa28314 1407#ifdef BCM_CNIC
8307fa3e
VZ		 1408 if (NO_FCOE(bp))
1409 return skb_tx_hash(dev, skb);
1410 else {
1411 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1412 u16 ether_type = ntohs(hdr->h_proto);
1413
1414 /* Skip VLAN tag if present */
1415 if (ether_type == ETH_P_8021Q) {
1416 struct vlan_ethhdr *vhdr =
1417 (struct vlan_ethhdr *)skb->data;
1418
1419 ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1420 }
1421
1422 /* If ethertype is FCoE or FIP - use FCoE ring */
1423 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
6383c0b3 1424 return bnx2x_fcoe_tx(bp, txq_index);
8307fa3e
VZ		 1425 }
1426#endif
1427 /* Select a none-FCoE queue: if FCoE is enabled, exclude FCoE L2 ring
1428 */
6383c0b3 1429 return __skb_tx_hash(dev, skb, BNX2X_NUM_ETH_QUEUES(bp));
8307fa3e
VZ		 1430}
1431
d6214d7a
DK		 1432void bnx2x_set_num_queues(struct bnx2x *bp)
1433{
1434 switch (bp->multi_mode) {
1435 case ETH_RSS_MODE_DISABLED:
9f6c9258 1436 bp->num_queues = 1;
d6214d7a
DK		 1437 break;
1438 case ETH_RSS_MODE_REGULAR:
1439 bp->num_queues = bnx2x_calc_num_queues(bp);
9f6c9258 1440 break;
f85582f8 1441
9f6c9258 1442 default:
d6214d7a 1443 bp->num_queues = 1;
9f6c9258
DK		 1444 break;
1445 }
ec6ba945
VZ		 1446
1447 /* Add special queues */
6383c0b3 1448 bp->num_queues += NON_ETH_CONTEXT_USE;
ec6ba945
VZ		 1449}
1450
ec6ba945
VZ		 1451static inline int bnx2x_set_real_num_queues(struct bnx2x *bp)
1452{
6383c0b3 1453 int rc, tx, rx;
ec6ba945 1454
6383c0b3
AE		 1455 tx = MAX_TXQS_PER_COS * bp->max_cos;
1456 rx = BNX2X_NUM_ETH_QUEUES(bp);
ec6ba945 1457
6383c0b3
AE		 1458/* account for fcoe queue */
1459#ifdef BCM_CNIC
1460 if (!NO_FCOE(bp)) {
1461 rx += FCOE_PRESENT;
1462 tx += FCOE_PRESENT;
1463 }
ec6ba945 1464#endif
6383c0b3
AE		 1465
1466 rc = netif_set_real_num_tx_queues(bp->dev, tx);
1467 if (rc) {
1468 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1469 return rc;
1470 }
1471 rc = netif_set_real_num_rx_queues(bp->dev, rx);
1472 if (rc) {
1473 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1474 return rc;
1475 }
1476
1477 DP(NETIF_MSG_DRV, "Setting real num queues to (tx, rx) (%d, %d)\n",
1478 tx, rx);
1479
ec6ba945
VZ		 1480 return rc;
1481}
1482
a8c94b91
VZ		 1483static inline void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1484{
1485 int i;
1486
1487 for_each_queue(bp, i) {
1488 struct bnx2x_fastpath *fp = &bp->fp[i];
1489
1490 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1491 if (IS_FCOE_IDX(i))
1492 /*
1493 * Although there are no IP frames expected to arrive to
1494 * this ring we still want to add an
1495 * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1496 * overrun attack.
1497 */
1498 fp->rx_buf_size =
1499 BNX2X_FCOE_MINI_JUMBO_MTU + ETH_OVREHEAD +
619c5cb6 1500 BNX2X_FW_RX_ALIGN + IP_HEADER_ALIGNMENT_PADDING;
a8c94b91
VZ		 1501 else
1502 fp->rx_buf_size =
619c5cb6
VZ		 1503 bp->dev->mtu + ETH_OVREHEAD +
1504 BNX2X_FW_RX_ALIGN + IP_HEADER_ALIGNMENT_PADDING;
a8c94b91
VZ		 1505 }
1506}
1507
619c5cb6
VZ		 1508static inline int bnx2x_init_rss_pf(struct bnx2x *bp)
1509{
1510 int i;
1511 u8 ind_table[T_ETH_INDIRECTION_TABLE_SIZE] = {0};
1512 u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
1513
1514 /*
1515 * Prepare the inital contents fo the indirection table if RSS is
1516 * enabled
1517 */
1518 if (bp->multi_mode != ETH_RSS_MODE_DISABLED) {
1519 for (i = 0; i < sizeof(ind_table); i++)
1520 ind_table[i] =
1521 bp->fp->cl_id + (i % num_eth_queues);
1522 }
1523
1524 /*
1525 * For 57710 and 57711 SEARCHER configuration (rss_keys) is
1526 * per-port, so if explicit configuration is needed , do it only
1527 * for a PMF.
1528 *
1529 * For 57712 and newer on the other hand it's a per-function
1530 * configuration.
1531 */
1532 return bnx2x_config_rss_pf(bp, ind_table,
1533 bp->port.pmf || !CHIP_IS_E1x(bp));
1534}
1535
1536int bnx2x_config_rss_pf(struct bnx2x *bp, u8 *ind_table, bool config_hash)
1537{
1538 struct bnx2x_config_rss_params params = {0};
1539 int i;
1540
1541 /* Although RSS is meaningless when there is a single HW queue we
1542 * still need it enabled in order to have HW Rx hash generated.
1543 *
1544 * if (!is_eth_multi(bp))
1545 * bp->multi_mode = ETH_RSS_MODE_DISABLED;
1546 */
1547
1548 params.rss_obj = &bp->rss_conf_obj;
1549
1550 __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
1551
1552 /* RSS mode */
1553 switch (bp->multi_mode) {
1554 case ETH_RSS_MODE_DISABLED:
1555 __set_bit(BNX2X_RSS_MODE_DISABLED, &params.rss_flags);
1556 break;
1557 case ETH_RSS_MODE_REGULAR:
1558 __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
1559 break;
1560 case ETH_RSS_MODE_VLAN_PRI:
1561 __set_bit(BNX2X_RSS_MODE_VLAN_PRI, &params.rss_flags);
1562 break;
1563 case ETH_RSS_MODE_E1HOV_PRI:
1564 __set_bit(BNX2X_RSS_MODE_E1HOV_PRI, &params.rss_flags);
1565 break;
1566 case ETH_RSS_MODE_IP_DSCP:
1567 __set_bit(BNX2X_RSS_MODE_IP_DSCP, &params.rss_flags);
1568 break;
1569 default:
1570 BNX2X_ERR("Unknown multi_mode: %d\n", bp->multi_mode);
1571 return -EINVAL;
1572 }
1573
1574 /* If RSS is enabled */
1575 if (bp->multi_mode != ETH_RSS_MODE_DISABLED) {
1576 /* RSS configuration */
1577 __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
1578 __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
1579 __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
1580 __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
1581
1582 /* Hash bits */
1583 params.rss_result_mask = MULTI_MASK;
1584
1585 memcpy(params.ind_table, ind_table, sizeof(params.ind_table));
1586
1587 if (config_hash) {
1588 /* RSS keys */
1589 for (i = 0; i < sizeof(params.rss_key) / 4; i++)
1590 params.rss_key[i] = random32();
1591
1592 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
1593 }
1594 }
1595
1596 return bnx2x_config_rss(bp, &params);
1597}
1598
1599static inline int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
1600{
1601 struct bnx2x_func_state_params func_params = {0};
1602
1603 /* Prepare parameters for function state transitions */
1604 __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
1605
1606 func_params.f_obj = &bp->func_obj;
1607 func_params.cmd = BNX2X_F_CMD_HW_INIT;
1608
1609 func_params.params.hw_init.load_phase = load_code;
1610
1611 return bnx2x_func_state_change(bp, &func_params);
1612}
1613
1614/*
1615 * Cleans the object that have internal lists without sending
1616 * ramrods. Should be run when interrutps are disabled.
1617 */
1618static void bnx2x_squeeze_objects(struct bnx2x *bp)
1619{
1620 int rc;
1621 unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
1622 struct bnx2x_mcast_ramrod_params rparam = {0};
1623 struct bnx2x_vlan_mac_obj *mac_obj = &bp->fp->mac_obj;
1624
1625 /***************** Cleanup MACs' object first *************************/
1626
1627 /* Wait for completion of requested */
1628 __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
1629 /* Perform a dry cleanup */
1630 __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
1631
1632 /* Clean ETH primary MAC */
1633 __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
1634 rc = mac_obj->delete_all(bp, &bp->fp->mac_obj, &vlan_mac_flags,
1635 &ramrod_flags);
1636 if (rc != 0)
1637 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
1638
1639 /* Cleanup UC list */
1640 vlan_mac_flags = 0;
1641 __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
1642 rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
1643 &ramrod_flags);
1644 if (rc != 0)
1645 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
1646
1647 /***************** Now clean mcast object *****************************/
1648 rparam.mcast_obj = &bp->mcast_obj;
1649 __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
1650
1651 /* Add a DEL command... */
1652 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
1653 if (rc < 0)
1654 BNX2X_ERR("Failed to add a new DEL command to a multi-cast "
1655 "object: %d\n", rc);
1656
1657 /* ...and wait until all pending commands are cleared */
1658 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1659 while (rc != 0) {
1660 if (rc < 0) {
1661 BNX2X_ERR("Failed to clean multi-cast object: %d\n",
1662 rc);
1663 return;
1664 }
1665
1666 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1667 }
1668}
1669
1670#ifndef BNX2X_STOP_ON_ERROR
1671#define LOAD_ERROR_EXIT(bp, label) \
1672 do { \
1673 (bp)->state = BNX2X_STATE_ERROR; \
1674 goto label; \
1675 } while (0)
1676#else
1677#define LOAD_ERROR_EXIT(bp, label) \
1678 do { \
1679 (bp)->state = BNX2X_STATE_ERROR; \
1680 (bp)->panic = 1; \
1681 return -EBUSY; \
1682 } while (0)
1683#endif
1684
9f6c9258
DK		 1685/* must be called with rtnl_lock */
1686int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
1687{
619c5cb6 1688 int port = BP_PORT(bp);
9f6c9258
DK		 1689 u32 load_code;
1690 int i, rc;
1691
1692#ifdef BNX2X_STOP_ON_ERROR
1693 if (unlikely(bp->panic))
1694 return -EPERM;
1695#endif
1696
1697 bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
1698
2ae17f66
VZ		 1699 /* Set the initial link reported state to link down */
1700 bnx2x_acquire_phy_lock(bp);
1701 memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
1702 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1703 &bp->last_reported_link.link_report_flags);
1704 bnx2x_release_phy_lock(bp);
1705
523224a3
DK		 1706 /* must be called before memory allocation and HW init */
1707 bnx2x_ilt_set_info(bp);
1708
6383c0b3
AE		 1709 /*
1710 * Zero fastpath structures preserving invariants like napi, which are
1711 * allocated only once, fp index, max_cos, bp pointer.
1712 * Also set fp->disable_tpa.
b3b83c3f
DK		 1713 */
1714 for_each_queue(bp, i)
1715 bnx2x_bz_fp(bp, i);
1716
6383c0b3 1717
a8c94b91
VZ		 1718 /* Set the receive queues buffer size */
1719 bnx2x_set_rx_buf_size(bp);
1720
d6214d7a 1721 if (bnx2x_alloc_mem(bp))
9f6c9258 1722 return -ENOMEM;
d6214d7a 1723
b3b83c3f
DK		 1724 /* As long as bnx2x_alloc_mem() may possibly update
1725 * bp->num_queues, bnx2x_set_real_num_queues() should always
1726 * come after it.
1727 */
ec6ba945 1728 rc = bnx2x_set_real_num_queues(bp);
d6214d7a 1729 if (rc) {
ec6ba945 1730 BNX2X_ERR("Unable to set real_num_queues\n");
619c5cb6 1731 LOAD_ERROR_EXIT(bp, load_error0);
9f6c9258
DK		 1732 }
1733
6383c0b3
AE		 1734 /* configure multi cos mappings in kernel.
1735 * this configuration may be overriden by a multi class queue discipline
1736 * or by a dcbx negotiation result.
1737 */
1738 bnx2x_setup_tc(bp->dev, bp->max_cos);
1739
9f6c9258
DK		 1740 bnx2x_napi_enable(bp);
1741
9f6c9258 1742 /* Send LOAD_REQUEST command to MCP
619c5cb6
VZ		 1743 * Returns the type of LOAD command:
1744 * if it is the first port to be initialized
1745 * common blocks should be initialized, otherwise - not
1746 */
9f6c9258 1747 if (!BP_NOMCP(bp)) {
a22f0788 1748 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, 0);
9f6c9258
DK		 1749 if (!load_code) {
1750 BNX2X_ERR("MCP response failure, aborting\n");
1751 rc = -EBUSY;
619c5cb6 1752 LOAD_ERROR_EXIT(bp, load_error1);
9f6c9258
DK		 1753 }
1754 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
1755 rc = -EBUSY; /* other port in diagnostic mode */
619c5cb6 1756 LOAD_ERROR_EXIT(bp, load_error1);
9f6c9258
DK		 1757 }
1758
1759 } else {
f2e0899f 1760 int path = BP_PATH(bp);
9f6c9258 1761
f2e0899f
DK		 1762 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d] %d, %d, %d\n",
1763 path, load_count[path][0], load_count[path][1],
1764 load_count[path][2]);
1765 load_count[path][0]++;
1766 load_count[path][1 + port]++;
1767 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d] %d, %d, %d\n",
1768 path, load_count[path][0], load_count[path][1],
1769 load_count[path][2]);
1770 if (load_count[path][0] == 1)
9f6c9258 1771 load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
f2e0899f 1772 else if (load_count[path][1 + port] == 1)
9f6c9258
DK		 1773 load_code = FW_MSG_CODE_DRV_LOAD_PORT;
1774 else
1775 load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
1776 }
1777
1778 if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
f2e0899f 1779 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
3deb8167 1780 (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
9f6c9258 1781 bp->port.pmf = 1;
3deb8167
YR		 1782 /*
1783 * We need the barrier to ensure the ordering between the
1784 * writing to bp->port.pmf here and reading it from the
1785 * bnx2x_periodic_task().
1786 */
1787 smp_mb();
1788 queue_delayed_work(bnx2x_wq, &bp->period_task, 0);
1789 } else
9f6c9258 1790 bp->port.pmf = 0;
6383c0b3 1791
9f6c9258
DK		 1792 DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
1793
619c5cb6
VZ		 1794 /* Init Function state controlling object */
1795 bnx2x__init_func_obj(bp);
1796
9f6c9258
DK		 1797 /* Initialize HW */
1798 rc = bnx2x_init_hw(bp, load_code);
1799 if (rc) {
1800 BNX2X_ERR("HW init failed, aborting\n");
a22f0788 1801 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
619c5cb6 1802 LOAD_ERROR_EXIT(bp, load_error2);
9f6c9258
DK		 1803 }
1804
d6214d7a
DK		 1805 /* Connect to IRQs */
1806 rc = bnx2x_setup_irqs(bp);
523224a3
DK		 1807 if (rc) {
1808 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
619c5cb6 1809 LOAD_ERROR_EXIT(bp, load_error2);
523224a3
DK		 1810 }
1811
9f6c9258
DK		 1812 /* Setup NIC internals and enable interrupts */
1813 bnx2x_nic_init(bp, load_code);
1814
619c5cb6
VZ		 1815 /* Init per-function objects */
1816 bnx2x_init_bp_objs(bp);
1817
f2e0899f
DK		 1818 if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1819 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
619c5cb6
VZ		 1820 (bp->common.shmem2_base)) {
1821 if (SHMEM2_HAS(bp, dcc_support))
1822 SHMEM2_WR(bp, dcc_support,
1823 (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
1824 SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
1825 }
1826
1827 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
1828 rc = bnx2x_func_start(bp);
1829 if (rc) {
1830 BNX2X_ERR("Function start failed!\n");
c636322b 1831 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
619c5cb6
VZ		 1832 LOAD_ERROR_EXIT(bp, load_error3);
1833 }
9f6c9258
DK		 1834
1835 /* Send LOAD_DONE command to MCP */
1836 if (!BP_NOMCP(bp)) {
a22f0788 1837 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
9f6c9258
DK		 1838 if (!load_code) {
1839 BNX2X_ERR("MCP response failure, aborting\n");
1840 rc = -EBUSY;
619c5cb6 1841 LOAD_ERROR_EXIT(bp, load_error3);
9f6c9258
DK		 1842 }
1843 }
1844
619c5cb6 1845 rc = bnx2x_setup_leading(bp);
9f6c9258
DK		 1846 if (rc) {
1847 BNX2X_ERR("Setup leading failed!\n");
619c5cb6 1848 LOAD_ERROR_EXIT(bp, load_error3);
f2e0899f 1849 }
9f6c9258 1850
9f6c9258 1851#ifdef BCM_CNIC
523224a3 1852 /* Enable Timer scan */
619c5cb6 1853 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
9f6c9258 1854#endif
f85582f8 1855
523224a3 1856 for_each_nondefault_queue(bp, i) {
619c5cb6 1857 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
523224a3 1858 if (rc)
619c5cb6 1859 LOAD_ERROR_EXIT(bp, load_error4);
523224a3
DK		 1860 }
1861
619c5cb6
VZ		 1862 rc = bnx2x_init_rss_pf(bp);
1863 if (rc)
1864 LOAD_ERROR_EXIT(bp, load_error4);
1865
523224a3
DK		 1866 /* Now when Clients are configured we are ready to work */
1867 bp->state = BNX2X_STATE_OPEN;
1868
619c5cb6
VZ		 1869 /* Configure a ucast MAC */
1870 rc = bnx2x_set_eth_mac(bp, true);
1871 if (rc)
1872 LOAD_ERROR_EXIT(bp, load_error4);
6e30dd4e 1873
e3835b99
DK		 1874 if (bp->pending_max) {
1875 bnx2x_update_max_mf_config(bp, bp->pending_max);
1876 bp->pending_max = 0;
1877 }
1878
9f6c9258
DK		 1879 if (bp->port.pmf)
1880 bnx2x_initial_phy_init(bp, load_mode);
1881
619c5cb6
VZ		 1882 /* Start fast path */
1883
1884 /* Initialize Rx filter. */
1885 netif_addr_lock_bh(bp->dev);
6e30dd4e 1886 bnx2x_set_rx_mode(bp->dev);
619c5cb6 1887 netif_addr_unlock_bh(bp->dev);
6e30dd4e 1888
619c5cb6 1889 /* Start the Tx */
9f6c9258
DK		 1890 switch (load_mode) {
1891 case LOAD_NORMAL:
523224a3
DK		 1892 /* Tx queue should be only reenabled */
1893 netif_tx_wake_all_queues(bp->dev);
9f6c9258
DK		 1894 break;
1895
1896 case LOAD_OPEN:
1897 netif_tx_start_all_queues(bp->dev);
523224a3 1898 smp_mb__after_clear_bit();
9f6c9258
DK		 1899 break;
1900
1901 case LOAD_DIAG:
9f6c9258
DK		 1902 bp->state = BNX2X_STATE_DIAG;
1903 break;
1904
1905 default:
1906 break;
1907 }
1908
1909 if (!bp->port.pmf)
1910 bnx2x__link_status_update(bp);
1911
1912 /* start the timer */
1913 mod_timer(&bp->timer, jiffies + bp->current_interval);
1914
1915#ifdef BCM_CNIC
1916 bnx2x_setup_cnic_irq_info(bp);
1917 if (bp->state == BNX2X_STATE_OPEN)
1918 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
1919#endif
1920 bnx2x_inc_load_cnt(bp);
1921
619c5cb6
VZ		 1922 /* Wait for all pending SP commands to complete */
1923 if (!bnx2x_wait_sp_comp(bp, ~0x0UL)) {
1924 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
1925 bnx2x_nic_unload(bp, UNLOAD_CLOSE);
1926 return -EBUSY;
1927 }
6891dd25 1928
619c5cb6 1929 bnx2x_dcbx_init(bp);
9f6c9258
DK		 1930 return 0;
1931
619c5cb6 1932#ifndef BNX2X_STOP_ON_ERROR
9f6c9258 1933load_error4:
619c5cb6 1934#ifdef BCM_CNIC
9f6c9258 1935 /* Disable Timer scan */
619c5cb6 1936 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
9f6c9258
DK		 1937#endif
1938load_error3:
1939 bnx2x_int_disable_sync(bp, 1);
d6214d7a 1940
619c5cb6
VZ		 1941 /* Clean queueable objects */
1942 bnx2x_squeeze_objects(bp);
1943
9f6c9258
DK		 1944 /* Free SKBs, SGEs, TPA pool and driver internals */
1945 bnx2x_free_skbs(bp);
ec6ba945 1946 for_each_rx_queue(bp, i)
9f6c9258 1947 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
d6214d7a 1948
9f6c9258 1949 /* Release IRQs */
d6214d7a
DK		 1950 bnx2x_free_irq(bp);
1951load_error2:
1952 if (!BP_NOMCP(bp)) {
1953 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
1954 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
1955 }
1956
1957 bp->port.pmf = 0;
9f6c9258
DK		 1958load_error1:
1959 bnx2x_napi_disable(bp);
d6214d7a 1960load_error0:
9f6c9258
DK		 1961 bnx2x_free_mem(bp);
1962
1963 return rc;
619c5cb6 1964#endif /* ! BNX2X_STOP_ON_ERROR */
9f6c9258
DK		 1965}
1966
1967/* must be called with rtnl_lock */
1968int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode)
1969{
1970 int i;
c9ee9206
VZ		 1971 bool global = false;
1972
1973 if ((bp->state == BNX2X_STATE_CLOSED) ||
1974 (bp->state == BNX2X_STATE_ERROR)) {
1975 /* We can get here if the driver has been unloaded
1976 * during parity error recovery and is either waiting for a
1977 * leader to complete or for other functions to unload and
1978 * then ifdown has been issued. In this case we want to
1979 * unload and let other functions to complete a recovery
1980 * process.
1981 */
9f6c9258
DK		 1982 bp->recovery_state = BNX2X_RECOVERY_DONE;
1983 bp->is_leader = 0;
c9ee9206
VZ		 1984 bnx2x_release_leader_lock(bp);
1985 smp_mb();
1986
1987 DP(NETIF_MSG_HW, "Releasing a leadership...\n");
9f6c9258
DK		 1988
1989 return -EINVAL;
1990 }
1991
87b7ba3d
VZ		 1992 /*
1993 * It's important to set the bp->state to the value different from
1994 * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
1995 * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
1996 */
1997 bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
1998 smp_mb();
1999
9505ee37
VZ		 2000 /* Stop Tx */
2001 bnx2x_tx_disable(bp);
2002
9f6c9258
DK		 2003#ifdef BCM_CNIC
2004 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2005#endif
9f6c9258 2006
9f6c9258 2007 bp->rx_mode = BNX2X_RX_MODE_NONE;
9f6c9258 2008
9f6c9258 2009 del_timer_sync(&bp->timer);
f85582f8 2010
619c5cb6
VZ		 2011 /* Set ALWAYS_ALIVE bit in shmem */
2012 bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2013
2014 bnx2x_drv_pulse(bp);
9f6c9258 2015
f85582f8 2016 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
9f6c9258
DK		 2017
2018 /* Cleanup the chip if needed */
2019 if (unload_mode != UNLOAD_RECOVERY)
2020 bnx2x_chip_cleanup(bp, unload_mode);
523224a3 2021 else {
c9ee9206
VZ		 2022 /* Send the UNLOAD_REQUEST to the MCP */
2023 bnx2x_send_unload_req(bp, unload_mode);
2024
2025 /*
2026 * Prevent transactions to host from the functions on the
2027 * engine that doesn't reset global blocks in case of global
2028 * attention once gloabl blocks are reset and gates are opened
2029 * (the engine which leader will perform the recovery
2030 * last).
2031 */
2032 if (!CHIP_IS_E1x(bp))
2033 bnx2x_pf_disable(bp);
2034
2035 /* Disable HW interrupts, NAPI */
523224a3
DK		 2036 bnx2x_netif_stop(bp, 1);
2037
2038 /* Release IRQs */
d6214d7a 2039 bnx2x_free_irq(bp);
c9ee9206
VZ		 2040
2041 /* Report UNLOAD_DONE to MCP */
2042 bnx2x_send_unload_done(bp);
523224a3 2043 }
9f6c9258 2044
619c5cb6
VZ		 2045 /*
2046 * At this stage no more interrupts will arrive so we may safly clean
2047 * the queueable objects here in case they failed to get cleaned so far.
2048 */
2049 bnx2x_squeeze_objects(bp);
2050
79616895
VZ		 2051 /* There should be no more pending SP commands at this stage */
2052 bp->sp_state = 0;
2053
9f6c9258
DK		 2054 bp->port.pmf = 0;
2055
2056 /* Free SKBs, SGEs, TPA pool and driver internals */
2057 bnx2x_free_skbs(bp);
ec6ba945 2058 for_each_rx_queue(bp, i)
9f6c9258 2059 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
d6214d7a 2060
9f6c9258
DK		 2061 bnx2x_free_mem(bp);
2062
2063 bp->state = BNX2X_STATE_CLOSED;
2064
c9ee9206
VZ		 2065 /* Check if there are pending parity attentions. If there are - set
2066 * RECOVERY_IN_PROGRESS.
2067 */
2068 if (bnx2x_chk_parity_attn(bp, &global, false)) {
2069 bnx2x_set_reset_in_progress(bp);
2070
2071 /* Set RESET_IS_GLOBAL if needed */
2072 if (global)
2073 bnx2x_set_reset_global(bp);
2074 }
2075
2076
9f6c9258
DK		 2077 /* The last driver must disable a "close the gate" if there is no
2078 * parity attention or "process kill" pending.
2079 */
c9ee9206 2080 if (!bnx2x_dec_load_cnt(bp) && bnx2x_reset_is_done(bp, BP_PATH(bp)))
9f6c9258
DK		 2081 bnx2x_disable_close_the_gate(bp);
2082
9f6c9258
DK		 2083 return 0;
2084}
f85582f8 2085
9f6c9258
DK		 2086int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
2087{
2088 u16 pmcsr;
2089
adf5f6a1
DK		 2090 /* If there is no power capability, silently succeed */
2091 if (!bp->pm_cap) {
2092 DP(NETIF_MSG_HW, "No power capability. Breaking.\n");
2093 return 0;
2094 }
2095
9f6c9258
DK		 2096 pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
2097
2098 switch (state) {
2099 case PCI_D0:
2100 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2101 ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
2102 PCI_PM_CTRL_PME_STATUS));
2103
2104 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
2105 /* delay required during transition out of D3hot */
2106 msleep(20);
2107 break;
2108
2109 case PCI_D3hot:
2110 /* If there are other clients above don't
2111 shut down the power */
2112 if (atomic_read(&bp->pdev->enable_cnt) != 1)
2113 return 0;
2114 /* Don't shut down the power for emulation and FPGA */
2115 if (CHIP_REV_IS_SLOW(bp))
2116 return 0;
2117
2118 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
2119 pmcsr |= 3;
2120
2121 if (bp->wol)
2122 pmcsr |= PCI_PM_CTRL_PME_ENABLE;
2123
2124 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2125 pmcsr);
2126
2127 /* No more memory access after this point until
2128 * device is brought back to D0.
2129 */
2130 break;
2131
2132 default:
2133 return -EINVAL;
2134 }
2135 return 0;
2136}
2137
9f6c9258
DK		 2138/*
2139 * net_device service functions
2140 */
d6214d7a 2141int bnx2x_poll(struct napi_struct *napi, int budget)
9f6c9258
DK		 2142{
2143 int work_done = 0;
6383c0b3 2144 u8 cos;
9f6c9258
DK		 2145 struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
2146 napi);
2147 struct bnx2x *bp = fp->bp;
2148
2149 while (1) {
2150#ifdef BNX2X_STOP_ON_ERROR
2151 if (unlikely(bp->panic)) {
2152 napi_complete(napi);
2153 return 0;
2154 }
2155#endif
2156
6383c0b3
AE		 2157 for_each_cos_in_tx_queue(fp, cos)
2158 if (bnx2x_tx_queue_has_work(&fp->txdata[cos]))
2159 bnx2x_tx_int(bp, &fp->txdata[cos]);
2160
9f6c9258
DK		 2161
2162 if (bnx2x_has_rx_work(fp)) {
2163 work_done += bnx2x_rx_int(fp, budget - work_done);
2164
2165 /* must not complete if we consumed full budget */
2166 if (work_done >= budget)
2167 break;
2168 }
2169
2170 /* Fall out from the NAPI loop if needed */
2171 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
ec6ba945
VZ		 2172#ifdef BCM_CNIC
2173 /* No need to update SB for FCoE L2 ring as long as
2174 * it's connected to the default SB and the SB
2175 * has been updated when NAPI was scheduled.
2176 */
2177 if (IS_FCOE_FP(fp)) {
2178 napi_complete(napi);
2179 break;
2180 }
2181#endif
2182
9f6c9258 2183 bnx2x_update_fpsb_idx(fp);
f85582f8
DK		 2184 /* bnx2x_has_rx_work() reads the status block,
2185 * thus we need to ensure that status block indices
2186 * have been actually read (bnx2x_update_fpsb_idx)
2187 * prior to this check (bnx2x_has_rx_work) so that
2188 * we won't write the "newer" value of the status block
2189 * to IGU (if there was a DMA right after
2190 * bnx2x_has_rx_work and if there is no rmb, the memory
2191 * reading (bnx2x_update_fpsb_idx) may be postponed
2192 * to right before bnx2x_ack_sb). In this case there
2193 * will never be another interrupt until there is
2194 * another update of the status block, while there
2195 * is still unhandled work.
2196 */
9f6c9258
DK		 2197 rmb();
2198
2199 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
2200 napi_complete(napi);
2201 /* Re-enable interrupts */
523224a3
DK		 2202 DP(NETIF_MSG_HW,
2203 "Update index to %d\n", fp->fp_hc_idx);
2204 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
2205 le16_to_cpu(fp->fp_hc_idx),
9f6c9258
DK		 2206 IGU_INT_ENABLE, 1);
2207 break;
2208 }
2209 }
2210 }
2211
2212 return work_done;
2213}
2214
9f6c9258
DK		 2215/* we split the first BD into headers and data BDs
2216 * to ease the pain of our fellow microcode engineers
2217 * we use one mapping for both BDs
2218 * So far this has only been observed to happen
2219 * in Other Operating Systems(TM)
2220 */
2221static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
6383c0b3 2222 struct bnx2x_fp_txdata *txdata,
9f6c9258
DK		 2223 struct sw_tx_bd *tx_buf,
2224 struct eth_tx_start_bd **tx_bd, u16 hlen,
2225 u16 bd_prod, int nbd)
2226{
2227 struct eth_tx_start_bd *h_tx_bd = *tx_bd;
2228 struct eth_tx_bd *d_tx_bd;
2229 dma_addr_t mapping;
2230 int old_len = le16_to_cpu(h_tx_bd->nbytes);
2231
2232 /* first fix first BD */
2233 h_tx_bd->nbd = cpu_to_le16(nbd);
2234 h_tx_bd->nbytes = cpu_to_le16(hlen);
2235
2236 DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d "
2237 "(%x:%x) nbd %d\n", h_tx_bd->nbytes, h_tx_bd->addr_hi,
2238 h_tx_bd->addr_lo, h_tx_bd->nbd);
2239
2240 /* now get a new data BD
2241 * (after the pbd) and fill it */
2242 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
6383c0b3 2243 d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
9f6c9258
DK		 2244
2245 mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
2246 le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
2247
2248 d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2249 d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2250 d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
2251
2252 /* this marks the BD as one that has no individual mapping */
2253 tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
2254
2255 DP(NETIF_MSG_TX_QUEUED,
2256 "TSO split data size is %d (%x:%x)\n",
2257 d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
2258
2259 /* update tx_bd */
2260 *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
2261
2262 return bd_prod;
2263}
2264
2265static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
2266{
2267 if (fix > 0)
2268 csum = (u16) ~csum_fold(csum_sub(csum,
2269 csum_partial(t_header - fix, fix, 0)));
2270
2271 else if (fix < 0)
2272 csum = (u16) ~csum_fold(csum_add(csum,
2273 csum_partial(t_header, -fix, 0)));
2274
2275 return swab16(csum);
2276}
2277
2278static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
2279{
2280 u32 rc;
2281
2282 if (skb->ip_summed != CHECKSUM_PARTIAL)
2283 rc = XMIT_PLAIN;
2284
2285 else {
d0d9d8ef 2286 if (vlan_get_protocol(skb) == htons(ETH_P_IPV6)) {
9f6c9258
DK		 2287 rc = XMIT_CSUM_V6;
2288 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
2289 rc |= XMIT_CSUM_TCP;
2290
2291 } else {
2292 rc = XMIT_CSUM_V4;
2293 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2294 rc |= XMIT_CSUM_TCP;
2295 }
2296 }
2297
5892b9e9
VZ		 2298 if (skb_is_gso_v6(skb))
2299 rc |= XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6;
2300 else if (skb_is_gso(skb))
2301 rc |= XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP;
9f6c9258
DK		 2302
2303 return rc;
2304}
2305
2306#if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2307/* check if packet requires linearization (packet is too fragmented)
2308 no need to check fragmentation if page size > 8K (there will be no
2309 violation to FW restrictions) */
2310static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
2311 u32 xmit_type)
2312{
2313 int to_copy = 0;
2314 int hlen = 0;
2315 int first_bd_sz = 0;
2316
2317 /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
2318 if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
2319
2320 if (xmit_type & XMIT_GSO) {
2321 unsigned short lso_mss = skb_shinfo(skb)->gso_size;
2322 /* Check if LSO packet needs to be copied:
2323 3 = 1 (for headers BD) + 2 (for PBD and last BD) */
2324 int wnd_size = MAX_FETCH_BD - 3;
2325 /* Number of windows to check */
2326 int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
2327 int wnd_idx = 0;
2328 int frag_idx = 0;
2329 u32 wnd_sum = 0;
2330
2331 /* Headers length */
2332 hlen = (int)(skb_transport_header(skb) - skb->data) +
2333 tcp_hdrlen(skb);
2334
2335 /* Amount of data (w/o headers) on linear part of SKB*/
2336 first_bd_sz = skb_headlen(skb) - hlen;
2337
2338 wnd_sum = first_bd_sz;
2339
2340 /* Calculate the first sum - it's special */
2341 for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
2342 wnd_sum +=
2343 skb_shinfo(skb)->frags[frag_idx].size;
2344
2345 /* If there was data on linear skb data - check it */
2346 if (first_bd_sz > 0) {
2347 if (unlikely(wnd_sum < lso_mss)) {
2348 to_copy = 1;
2349 goto exit_lbl;
2350 }
2351
2352 wnd_sum -= first_bd_sz;
2353 }
2354
2355 /* Others are easier: run through the frag list and
2356 check all windows */
2357 for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
2358 wnd_sum +=
2359 skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1].size;
2360
2361 if (unlikely(wnd_sum < lso_mss)) {
2362 to_copy = 1;
2363 break;
2364 }
2365 wnd_sum -=
2366 skb_shinfo(skb)->frags[wnd_idx].size;
2367 }
2368 } else {
2369 /* in non-LSO too fragmented packet should always
2370 be linearized */
2371 to_copy = 1;
2372 }
2373 }
2374
2375exit_lbl:
2376 if (unlikely(to_copy))
2377 DP(NETIF_MSG_TX_QUEUED,
2378 "Linearization IS REQUIRED for %s packet. "
2379 "num_frags %d hlen %d first_bd_sz %d\n",
2380 (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
2381 skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
2382
2383 return to_copy;
2384}
2385#endif
2386
2297a2da
VZ		 2387static inline void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
2388 u32 xmit_type)
f2e0899f 2389{
2297a2da
VZ		 2390 *parsing_data |= (skb_shinfo(skb)->gso_size <<
2391 ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
2392 ETH_TX_PARSE_BD_E2_LSO_MSS;
f2e0899f
DK		 2393 if ((xmit_type & XMIT_GSO_V6) &&
2394 (ipv6_hdr(skb)->nexthdr == NEXTHDR_IPV6))
2297a2da 2395 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
f2e0899f
DK		 2396}
2397
2398/**
e8920674 2399 * bnx2x_set_pbd_gso - update PBD in GSO case.
f2e0899f 2400 *
e8920674
DK		 2401 * @skb: packet skb
2402 * @pbd: parse BD
2403 * @xmit_type: xmit flags
f2e0899f
DK		 2404 */
2405static inline void bnx2x_set_pbd_gso(struct sk_buff *skb,
2406 struct eth_tx_parse_bd_e1x *pbd,
2407 u32 xmit_type)
2408{
2409 pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
2410 pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
2411 pbd->tcp_flags = pbd_tcp_flags(skb);
2412
2413 if (xmit_type & XMIT_GSO_V4) {
2414 pbd->ip_id = swab16(ip_hdr(skb)->id);
2415 pbd->tcp_pseudo_csum =
2416 swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
2417 ip_hdr(skb)->daddr,
2418 0, IPPROTO_TCP, 0));
2419
2420 } else
2421 pbd->tcp_pseudo_csum =
2422 swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2423 &ipv6_hdr(skb)->daddr,
2424 0, IPPROTO_TCP, 0));
2425
2426 pbd->global_data |= ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN;
2427}
f85582f8 2428
f2e0899f 2429/**
e8920674 2430 * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
f2e0899f 2431 *
e8920674
DK		 2432 * @bp: driver handle
2433 * @skb: packet skb
2434 * @parsing_data: data to be updated
2435 * @xmit_type: xmit flags
f2e0899f 2436 *
e8920674 2437 * 57712 related
f2e0899f
DK		 2438 */
2439static inline u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
2297a2da 2440 u32 *parsing_data, u32 xmit_type)
f2e0899f 2441{
e39aece7
VZ		 2442 *parsing_data |=
2443 ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
2444 ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W_SHIFT) &
2445 ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W;
f2e0899f 2446
e39aece7
VZ		 2447 if (xmit_type & XMIT_CSUM_TCP) {
2448 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
2449 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
2450 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
f2e0899f 2451
e39aece7
VZ		 2452 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
2453 } else
2454 /* We support checksum offload for TCP and UDP only.
2455 * No need to pass the UDP header length - it's a constant.
2456 */
2457 return skb_transport_header(skb) +
2458 sizeof(struct udphdr) - skb->data;
f2e0899f
DK		 2459}
2460
93ef5c02
DK		 2461static inline void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
2462 struct eth_tx_start_bd *tx_start_bd, u32 xmit_type)
2463{
93ef5c02
DK		 2464 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
2465
2466 if (xmit_type & XMIT_CSUM_V4)
2467 tx_start_bd->bd_flags.as_bitfield |=
2468 ETH_TX_BD_FLAGS_IP_CSUM;
2469 else
2470 tx_start_bd->bd_flags.as_bitfield |=
2471 ETH_TX_BD_FLAGS_IPV6;
2472
2473 if (!(xmit_type & XMIT_CSUM_TCP))
2474 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
93ef5c02
DK		 2475}
2476
f2e0899f 2477/**
e8920674 2478 * bnx2x_set_pbd_csum - update PBD with checksum and return header length
f2e0899f 2479 *
e8920674
DK		 2480 * @bp: driver handle
2481 * @skb: packet skb
2482 * @pbd: parse BD to be updated
2483 * @xmit_type: xmit flags
f2e0899f
DK		 2484 */
2485static inline u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
2486 struct eth_tx_parse_bd_e1x *pbd,
2487 u32 xmit_type)
2488{
e39aece7 2489 u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
f2e0899f
DK		 2490
2491 /* for now NS flag is not used in Linux */
2492 pbd->global_data =
2493 (hlen | ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
2494 ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
2495
2496 pbd->ip_hlen_w = (skb_transport_header(skb) -
e39aece7 2497 skb_network_header(skb)) >> 1;
f2e0899f 2498
e39aece7
VZ		 2499 hlen += pbd->ip_hlen_w;
2500
2501 /* We support checksum offload for TCP and UDP only */
2502 if (xmit_type & XMIT_CSUM_TCP)
2503 hlen += tcp_hdrlen(skb) / 2;
2504 else
2505 hlen += sizeof(struct udphdr) / 2;
f2e0899f
DK		 2506
2507 pbd->total_hlen_w = cpu_to_le16(hlen);
2508 hlen = hlen*2;
2509
2510 if (xmit_type & XMIT_CSUM_TCP) {
2511 pbd->tcp_pseudo_csum = swab16(tcp_hdr(skb)->check);
2512
2513 } else {
2514 s8 fix = SKB_CS_OFF(skb); /* signed! */
2515
2516 DP(NETIF_MSG_TX_QUEUED,
2517 "hlen %d fix %d csum before fix %x\n",
2518 le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
2519
2520 /* HW bug: fixup the CSUM */
2521 pbd->tcp_pseudo_csum =
2522 bnx2x_csum_fix(skb_transport_header(skb),
2523 SKB_CS(skb), fix);
2524
2525 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
2526 pbd->tcp_pseudo_csum);
2527 }
2528
2529 return hlen;
2530}
f85582f8 2531
9f6c9258
DK		 2532/* called with netif_tx_lock
2533 * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
2534 * netif_wake_queue()
2535 */
2536netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
2537{
2538 struct bnx2x *bp = netdev_priv(dev);
6383c0b3 2539
9f6c9258
DK		 2540 struct bnx2x_fastpath *fp;
2541 struct netdev_queue *txq;
6383c0b3 2542 struct bnx2x_fp_txdata *txdata;
9f6c9258 2543 struct sw_tx_bd *tx_buf;
619c5cb6 2544 struct eth_tx_start_bd *tx_start_bd, *first_bd;
9f6c9258 2545 struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
523224a3 2546 struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
f2e0899f 2547 struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
2297a2da 2548 u32 pbd_e2_parsing_data = 0;
9f6c9258 2549 u16 pkt_prod, bd_prod;
6383c0b3 2550 int nbd, txq_index, fp_index, txdata_index;
9f6c9258
DK		 2551 dma_addr_t mapping;
2552 u32 xmit_type = bnx2x_xmit_type(bp, skb);
2553 int i;
2554 u8 hlen = 0;
2555 __le16 pkt_size = 0;
2556 struct ethhdr *eth;
2557 u8 mac_type = UNICAST_ADDRESS;
2558
2559#ifdef BNX2X_STOP_ON_ERROR
2560 if (unlikely(bp->panic))
2561 return NETDEV_TX_BUSY;
2562#endif
2563
6383c0b3
AE		 2564 txq_index = skb_get_queue_mapping(skb);
2565 txq = netdev_get_tx_queue(dev, txq_index);
2566
2567 BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + FCOE_PRESENT);
2568
2569 /* decode the fastpath index and the cos index from the txq */
2570 fp_index = TXQ_TO_FP(txq_index);
2571 txdata_index = TXQ_TO_COS(txq_index);
2572
2573#ifdef BCM_CNIC
2574 /*
2575 * Override the above for the FCoE queue:
2576 * - FCoE fp entry is right after the ETH entries.
2577 * - FCoE L2 queue uses bp->txdata[0] only.
2578 */
2579 if (unlikely(!NO_FCOE(bp) && (txq_index ==
2580 bnx2x_fcoe_tx(bp, txq_index)))) {
2581 fp_index = FCOE_IDX;
2582 txdata_index = 0;
2583 }
2584#endif
2585
2586 /* enable this debug print to view the transmission queue being used
2587 DP(BNX2X_MSG_FP, "indices: txq %d, fp %d, txdata %d",
2588 txq_index, fp_index, txdata_index); */
9f6c9258 2589
6383c0b3 2590 /* locate the fastpath and the txdata */
9f6c9258 2591 fp = &bp->fp[fp_index];
6383c0b3
AE		 2592 txdata = &fp->txdata[txdata_index];
2593
2594 /* enable this debug print to view the tranmission details
2595 DP(BNX2X_MSG_FP,"transmitting packet cid %d fp index %d txdata_index %d"
2596 " tx_data ptr %p fp pointer %p",
2597 txdata->cid, fp_index, txdata_index, txdata, fp); */
9f6c9258 2598
6383c0b3
AE		 2599 if (unlikely(bnx2x_tx_avail(bp, txdata) <
2600 (skb_shinfo(skb)->nr_frags + 3))) {
9f6c9258
DK		 2601 fp->eth_q_stats.driver_xoff++;
2602 netif_tx_stop_queue(txq);
2603 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
2604 return NETDEV_TX_BUSY;
2605 }
2606
f2e0899f
DK		 2607 DP(NETIF_MSG_TX_QUEUED, "queue[%d]: SKB: summed %x protocol %x "
2608 "protocol(%x,%x) gso type %x xmit_type %x\n",
6383c0b3 2609 txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
9f6c9258
DK		 2610 ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type);
2611
2612 eth = (struct ethhdr *)skb->data;
2613
2614 /* set flag according to packet type (UNICAST_ADDRESS is default)*/
2615 if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
2616 if (is_broadcast_ether_addr(eth->h_dest))
2617 mac_type = BROADCAST_ADDRESS;
2618 else
2619 mac_type = MULTICAST_ADDRESS;
2620 }
2621
2622#if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2623 /* First, check if we need to linearize the skb (due to FW
2624 restrictions). No need to check fragmentation if page size > 8K
2625 (there will be no violation to FW restrictions) */
2626 if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
2627 /* Statistics of linearization */
2628 bp->lin_cnt++;
2629 if (skb_linearize(skb) != 0) {
2630 DP(NETIF_MSG_TX_QUEUED, "SKB linearization failed - "
2631 "silently dropping this SKB\n");
2632 dev_kfree_skb_any(skb);
2633 return NETDEV_TX_OK;
2634 }
2635 }
2636#endif
619c5cb6
VZ		 2637 /* Map skb linear data for DMA */
2638 mapping = dma_map_single(&bp->pdev->dev, skb->data,
2639 skb_headlen(skb), DMA_TO_DEVICE);
2640 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
2641 DP(NETIF_MSG_TX_QUEUED, "SKB mapping failed - "
2642 "silently dropping this SKB\n");
2643 dev_kfree_skb_any(skb);
2644 return NETDEV_TX_OK;
2645 }
9f6c9258
DK		 2646 /*
2647 Please read carefully. First we use one BD which we mark as start,
2648 then we have a parsing info BD (used for TSO or xsum),
2649 and only then we have the rest of the TSO BDs.
2650 (don't forget to mark the last one as last,
2651 and to unmap only AFTER you write to the BD ...)
2652 And above all, all pdb sizes are in words - NOT DWORDS!
2653 */
2654
619c5cb6
VZ		 2655 /* get current pkt produced now - advance it just before sending packet
2656 * since mapping of pages may fail and cause packet to be dropped
2657 */
6383c0b3
AE		 2658 pkt_prod = txdata->tx_pkt_prod;
2659 bd_prod = TX_BD(txdata->tx_bd_prod);
9f6c9258 2660
619c5cb6
VZ		 2661 /* get a tx_buf and first BD
2662 * tx_start_bd may be changed during SPLIT,
2663 * but first_bd will always stay first
2664 */
6383c0b3
AE		 2665 tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
2666 tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
619c5cb6 2667 first_bd = tx_start_bd;
9f6c9258
DK		 2668
2669 tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
f85582f8
DK		 2670 SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_ETH_ADDR_TYPE,
2671 mac_type);
2672
9f6c9258 2673 /* header nbd */
f85582f8 2674 SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_HDR_NBDS, 1);
9f6c9258
DK		 2675
2676 /* remember the first BD of the packet */
6383c0b3 2677 tx_buf->first_bd = txdata->tx_bd_prod;
9f6c9258
DK		 2678 tx_buf->skb = skb;
2679 tx_buf->flags = 0;
2680
2681 DP(NETIF_MSG_TX_QUEUED,
2682 "sending pkt %u @%p next_idx %u bd %u @%p\n",
6383c0b3 2683 pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
9f6c9258 2684
eab6d18d 2685 if (vlan_tx_tag_present(skb)) {
523224a3
DK		 2686 tx_start_bd->vlan_or_ethertype =
2687 cpu_to_le16(vlan_tx_tag_get(skb));
2688 tx_start_bd->bd_flags.as_bitfield |=
2689 (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
9f6c9258 2690 } else
523224a3 2691 tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
9f6c9258
DK		 2692
2693 /* turn on parsing and get a BD */
2694 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
9f6c9258 2695
93ef5c02
DK		 2696 if (xmit_type & XMIT_CSUM)
2697 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
9f6c9258 2698
619c5cb6 2699 if (!CHIP_IS_E1x(bp)) {
6383c0b3 2700 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
f2e0899f
DK		 2701 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
2702 /* Set PBD in checksum offload case */
2703 if (xmit_type & XMIT_CSUM)
2297a2da
VZ		 2704 hlen = bnx2x_set_pbd_csum_e2(bp, skb,
2705 &pbd_e2_parsing_data,
2706 xmit_type);
619c5cb6
VZ		 2707 if (IS_MF_SI(bp)) {
2708 /*
2709 * fill in the MAC addresses in the PBD - for local
2710 * switching
2711 */
2712 bnx2x_set_fw_mac_addr(&pbd_e2->src_mac_addr_hi,
2713 &pbd_e2->src_mac_addr_mid,
2714 &pbd_e2->src_mac_addr_lo,
2715 eth->h_source);
2716 bnx2x_set_fw_mac_addr(&pbd_e2->dst_mac_addr_hi,
2717 &pbd_e2->dst_mac_addr_mid,
2718 &pbd_e2->dst_mac_addr_lo,
2719 eth->h_dest);
2720 }
f2e0899f 2721 } else {
6383c0b3 2722 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
f2e0899f
DK		 2723 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
2724 /* Set PBD in checksum offload case */
2725 if (xmit_type & XMIT_CSUM)
2726 hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
9f6c9258 2727
9f6c9258
DK		 2728 }
2729
f85582f8 2730 /* Setup the data pointer of the first BD of the packet */
9f6c9258
DK		 2731 tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2732 tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
619c5cb6 2733 nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
9f6c9258
DK		 2734 tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
2735 pkt_size = tx_start_bd->nbytes;
2736
2737 DP(NETIF_MSG_TX_QUEUED, "first bd @%p addr (%x:%x) nbd %d"
2738 " nbytes %d flags %x vlan %x\n",
2739 tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
2740 le16_to_cpu(tx_start_bd->nbd), le16_to_cpu(tx_start_bd->nbytes),
523224a3
DK		 2741 tx_start_bd->bd_flags.as_bitfield,
2742 le16_to_cpu(tx_start_bd->vlan_or_ethertype));
9f6c9258
DK		 2743
2744 if (xmit_type & XMIT_GSO) {
2745
2746 DP(NETIF_MSG_TX_QUEUED,
2747 "TSO packet len %d hlen %d total len %d tso size %d\n",
2748 skb->len, hlen, skb_headlen(skb),
2749 skb_shinfo(skb)->gso_size);
2750
2751 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
2752
2753 if (unlikely(skb_headlen(skb) > hlen))
6383c0b3
AE		 2754 bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
2755 &tx_start_bd, hlen,
2756 bd_prod, ++nbd);
619c5cb6 2757 if (!CHIP_IS_E1x(bp))
2297a2da
VZ		 2758 bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
2759 xmit_type);
f2e0899f
DK		 2760 else
2761 bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
9f6c9258 2762 }
2297a2da
VZ		 2763
2764 /* Set the PBD's parsing_data field if not zero
2765 * (for the chips newer than 57711).
2766 */
2767 if (pbd_e2_parsing_data)
2768 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
2769
9f6c9258
DK		 2770 tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
2771
f85582f8 2772 /* Handle fragmented skb */
9f6c9258
DK		 2773 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2774 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2775
619c5cb6
VZ		 2776 mapping = dma_map_page(&bp->pdev->dev, frag->page,
2777 frag->page_offset, frag->size,
2778 DMA_TO_DEVICE);
2779 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
2780
2781 DP(NETIF_MSG_TX_QUEUED, "Unable to map page - "
2782 "dropping packet...\n");
2783
2784 /* we need unmap all buffers already mapped
2785 * for this SKB;
2786 * first_bd->nbd need to be properly updated
2787 * before call to bnx2x_free_tx_pkt
2788 */
2789 first_bd->nbd = cpu_to_le16(nbd);
6383c0b3
AE		 2790 bnx2x_free_tx_pkt(bp, txdata,
2791 TX_BD(txdata->tx_pkt_prod));
619c5cb6
VZ		 2792 return NETDEV_TX_OK;
2793 }
2794
9f6c9258 2795 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
6383c0b3 2796 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
9f6c9258 2797 if (total_pkt_bd == NULL)
6383c0b3 2798 total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
9f6c9258 2799
9f6c9258
DK		 2800 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2801 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2802 tx_data_bd->nbytes = cpu_to_le16(frag->size);
2803 le16_add_cpu(&pkt_size, frag->size);
619c5cb6 2804 nbd++;
9f6c9258
DK		 2805
2806 DP(NETIF_MSG_TX_QUEUED,
2807 "frag %d bd @%p addr (%x:%x) nbytes %d\n",
2808 i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
2809 le16_to_cpu(tx_data_bd->nbytes));
2810 }
2811
2812 DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
2813
619c5cb6
VZ		 2814 /* update with actual num BDs */
2815 first_bd->nbd = cpu_to_le16(nbd);
2816
9f6c9258
DK		 2817 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2818
2819 /* now send a tx doorbell, counting the next BD
2820 * if the packet contains or ends with it
2821 */
2822 if (TX_BD_POFF(bd_prod) < nbd)
2823 nbd++;
2824
619c5cb6
VZ		 2825 /* total_pkt_bytes should be set on the first data BD if
2826 * it's not an LSO packet and there is more than one
2827 * data BD. In this case pkt_size is limited by an MTU value.
2828 * However we prefer to set it for an LSO packet (while we don't
2829 * have to) in order to save some CPU cycles in a none-LSO
2830 * case, when we much more care about them.
2831 */
9f6c9258
DK		 2832 if (total_pkt_bd != NULL)
2833 total_pkt_bd->total_pkt_bytes = pkt_size;
2834
523224a3 2835 if (pbd_e1x)
9f6c9258 2836 DP(NETIF_MSG_TX_QUEUED,
523224a3 2837 "PBD (E1X) @%p ip_data %x ip_hlen %u ip_id %u lso_mss %u"
9f6c9258 2838 " tcp_flags %x xsum %x seq %u hlen %u\n",
523224a3
DK		 2839 pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
2840 pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
2841 pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
2842 le16_to_cpu(pbd_e1x->total_hlen_w));
f2e0899f
DK		 2843 if (pbd_e2)
2844 DP(NETIF_MSG_TX_QUEUED,
2845 "PBD (E2) @%p dst %x %x %x src %x %x %x parsing_data %x\n",
2846 pbd_e2, pbd_e2->dst_mac_addr_hi, pbd_e2->dst_mac_addr_mid,
2847 pbd_e2->dst_mac_addr_lo, pbd_e2->src_mac_addr_hi,
2848 pbd_e2->src_mac_addr_mid, pbd_e2->src_mac_addr_lo,
2849 pbd_e2->parsing_data);
9f6c9258
DK		 2850 DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d bd %u\n", nbd, bd_prod);
2851
6383c0b3 2852 txdata->tx_pkt_prod++;
9f6c9258
DK		 2853 /*
2854 * Make sure that the BD data is updated before updating the producer
2855 * since FW might read the BD right after the producer is updated.
2856 * This is only applicable for weak-ordered memory model archs such
2857 * as IA-64. The following barrier is also mandatory since FW will
2858 * assumes packets must have BDs.
2859 */
2860 wmb();
2861
6383c0b3 2862 txdata->tx_db.data.prod += nbd;
9f6c9258 2863 barrier();
f85582f8 2864
6383c0b3 2865 DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
9f6c9258
DK		 2866
2867 mmiowb();
2868
6383c0b3 2869 txdata->tx_bd_prod += nbd;
9f6c9258 2870
6383c0b3 2871 if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_SKB_FRAGS + 3)) {
9f6c9258
DK		 2872 netif_tx_stop_queue(txq);
2873
2874 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
2875 * ordering of set_bit() in netif_tx_stop_queue() and read of
2876 * fp->bd_tx_cons */
2877 smp_mb();
2878
2879 fp->eth_q_stats.driver_xoff++;
6383c0b3 2880 if (bnx2x_tx_avail(bp, txdata) >= MAX_SKB_FRAGS + 3)
9f6c9258
DK		 2881 netif_tx_wake_queue(txq);
2882 }
6383c0b3 2883 txdata->tx_pkt++;
9f6c9258
DK		 2884
2885 return NETDEV_TX_OK;
2886}
f85582f8 2887
6383c0b3
AE		 2888/**
2889 * bnx2x_setup_tc - routine to configure net_device for multi tc
2890 *
2891 * @netdev: net device to configure
2892 * @tc: number of traffic classes to enable
2893 *
2894 * callback connected to the ndo_setup_tc function pointer
2895 */
2896int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
2897{
2898 int cos, prio, count, offset;
2899 struct bnx2x *bp = netdev_priv(dev);
2900
2901 /* setup tc must be called under rtnl lock */
2902 ASSERT_RTNL();
2903
2904 /* no traffic classes requested. aborting */
2905 if (!num_tc) {
2906 netdev_reset_tc(dev);
2907 return 0;
2908 }
2909
2910 /* requested to support too many traffic classes */
2911 if (num_tc > bp->max_cos) {
2912 DP(NETIF_MSG_TX_ERR, "support for too many traffic classes"
2913 " requested: %d. max supported is %d",
2914 num_tc, bp->max_cos);
2915 return -EINVAL;
2916 }
2917
2918 /* declare amount of supported traffic classes */
2919 if (netdev_set_num_tc(dev, num_tc)) {
2920 DP(NETIF_MSG_TX_ERR, "failed to declare %d traffic classes",
2921 num_tc);
2922 return -EINVAL;
2923 }
2924
2925 /* configure priority to traffic class mapping */
2926 for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
2927 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
2928 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d",
2929 prio, bp->prio_to_cos[prio]);
2930 }
2931
2932
2933 /* Use this configuration to diffrentiate tc0 from other COSes
2934 This can be used for ets or pfc, and save the effort of setting
2935 up a multio class queue disc or negotiating DCBX with a switch
2936 netdev_set_prio_tc_map(dev, 0, 0);
2937 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d", 0, 0);
2938 for (prio = 1; prio < 16; prio++) {
2939 netdev_set_prio_tc_map(dev, prio, 1);
2940 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d", prio, 1);
2941 } */
2942
2943 /* configure traffic class to transmission queue mapping */
2944 for (cos = 0; cos < bp->max_cos; cos++) {
2945 count = BNX2X_NUM_ETH_QUEUES(bp);
2946 offset = cos * MAX_TXQS_PER_COS;
2947 netdev_set_tc_queue(dev, cos, count, offset);
2948 DP(BNX2X_MSG_SP, "mapping tc %d to offset %d count %d",
2949 cos, offset, count);
2950 }
2951
2952 return 0;
2953}
2954
9f6c9258
DK		 2955/* called with rtnl_lock */
2956int bnx2x_change_mac_addr(struct net_device *dev, void *p)
2957{
2958 struct sockaddr *addr = p;
2959 struct bnx2x *bp = netdev_priv(dev);
619c5cb6 2960 int rc = 0;
9f6c9258
DK		 2961
2962 if (!is_valid_ether_addr((u8 *)(addr->sa_data)))
2963 return -EINVAL;
2964
619c5cb6
VZ		 2965 if (netif_running(dev)) {
2966 rc = bnx2x_set_eth_mac(bp, false);
2967 if (rc)
2968 return rc;
2969 }
2970
9f6c9258 2971 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
619c5cb6 2972
523224a3 2973 if (netif_running(dev))
619c5cb6 2974 rc = bnx2x_set_eth_mac(bp, true);
9f6c9258 2975
619c5cb6 2976 return rc;
9f6c9258
DK		 2977}
2978
b3b83c3f
DK		 2979static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
2980{
2981 union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
2982 struct bnx2x_fastpath *fp = &bp->fp[fp_index];
6383c0b3 2983 u8 cos;
b3b83c3f
DK		 2984
2985 /* Common */
2986#ifdef BCM_CNIC
2987 if (IS_FCOE_IDX(fp_index)) {
2988 memset(sb, 0, sizeof(union host_hc_status_block));
2989 fp->status_blk_mapping = 0;
2990
2991 } else {
2992#endif
2993 /* status blocks */
619c5cb6 2994 if (!CHIP_IS_E1x(bp))
b3b83c3f
DK		 2995 BNX2X_PCI_FREE(sb->e2_sb,
2996 bnx2x_fp(bp, fp_index,
2997 status_blk_mapping),
2998 sizeof(struct host_hc_status_block_e2));
2999 else
3000 BNX2X_PCI_FREE(sb->e1x_sb,
3001 bnx2x_fp(bp, fp_index,
3002 status_blk_mapping),
3003 sizeof(struct host_hc_status_block_e1x));
3004#ifdef BCM_CNIC
3005 }
3006#endif
3007 /* Rx */
3008 if (!skip_rx_queue(bp, fp_index)) {
3009 bnx2x_free_rx_bds(fp);
3010
3011 /* fastpath rx rings: rx_buf rx_desc rx_comp */
3012 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
3013 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
3014 bnx2x_fp(bp, fp_index, rx_desc_mapping),
3015 sizeof(struct eth_rx_bd) * NUM_RX_BD);
3016
3017 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
3018 bnx2x_fp(bp, fp_index, rx_comp_mapping),
3019 sizeof(struct eth_fast_path_rx_cqe) *
3020 NUM_RCQ_BD);
3021
3022 /* SGE ring */
3023 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
3024 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
3025 bnx2x_fp(bp, fp_index, rx_sge_mapping),
3026 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
3027 }
3028
3029 /* Tx */
3030 if (!skip_tx_queue(bp, fp_index)) {
3031 /* fastpath tx rings: tx_buf tx_desc */
6383c0b3
AE		 3032 for_each_cos_in_tx_queue(fp, cos) {
3033 struct bnx2x_fp_txdata *txdata = &fp->txdata[cos];
3034
3035 DP(BNX2X_MSG_SP,
3036 "freeing tx memory of fp %d cos %d cid %d",
3037 fp_index, cos, txdata->cid);
3038
3039 BNX2X_FREE(txdata->tx_buf_ring);
3040 BNX2X_PCI_FREE(txdata->tx_desc_ring,
3041 txdata->tx_desc_mapping,
3042 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
3043 }
b3b83c3f
DK		 3044 }
3045 /* end of fastpath */
3046}
3047
3048void bnx2x_free_fp_mem(struct bnx2x *bp)
3049{
3050 int i;
3051 for_each_queue(bp, i)
3052 bnx2x_free_fp_mem_at(bp, i);
3053}
3054
3055static inline void set_sb_shortcuts(struct bnx2x *bp, int index)
3056{
3057 union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
619c5cb6 3058 if (!CHIP_IS_E1x(bp)) {
b3b83c3f
DK		 3059 bnx2x_fp(bp, index, sb_index_values) =
3060 (__le16 *)status_blk.e2_sb->sb.index_values;
3061 bnx2x_fp(bp, index, sb_running_index) =
3062 (__le16 *)status_blk.e2_sb->sb.running_index;
3063 } else {
3064 bnx2x_fp(bp, index, sb_index_values) =
3065 (__le16 *)status_blk.e1x_sb->sb.index_values;
3066 bnx2x_fp(bp, index, sb_running_index) =
3067 (__le16 *)status_blk.e1x_sb->sb.running_index;
3068 }
3069}
3070
3071static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
3072{
3073 union host_hc_status_block *sb;
3074 struct bnx2x_fastpath *fp = &bp->fp[index];
3075 int ring_size = 0;
6383c0b3 3076 u8 cos;
b3b83c3f
DK		 3077
3078 /* if rx_ring_size specified - use it */
3079 int rx_ring_size = bp->rx_ring_size ? bp->rx_ring_size :
6383c0b3 3080 MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
b3b83c3f
DK		 3081
3082 /* allocate at least number of buffers required by FW */
6383c0b3 3083 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
b3b83c3f
DK		 3084 MIN_RX_SIZE_TPA,
3085 rx_ring_size);
3086
b3b83c3f
DK		 3087 /* Common */
3088 sb = &bnx2x_fp(bp, index, status_blk);
3089#ifdef BCM_CNIC
3090 if (!IS_FCOE_IDX(index)) {
3091#endif
3092 /* status blocks */
619c5cb6 3093 if (!CHIP_IS_E1x(bp))
b3b83c3f
DK		 3094 BNX2X_PCI_ALLOC(sb->e2_sb,
3095 &bnx2x_fp(bp, index, status_blk_mapping),
3096 sizeof(struct host_hc_status_block_e2));
3097 else
3098 BNX2X_PCI_ALLOC(sb->e1x_sb,
3099 &bnx2x_fp(bp, index, status_blk_mapping),
3100 sizeof(struct host_hc_status_block_e1x));
3101#ifdef BCM_CNIC
3102 }
3103#endif
8eef2af1
DK		 3104
3105 /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
3106 * set shortcuts for it.
3107 */
3108 if (!IS_FCOE_IDX(index))
3109 set_sb_shortcuts(bp, index);
b3b83c3f
DK		 3110
3111 /* Tx */
3112 if (!skip_tx_queue(bp, index)) {
3113 /* fastpath tx rings: tx_buf tx_desc */
6383c0b3
AE		 3114 for_each_cos_in_tx_queue(fp, cos) {
3115 struct bnx2x_fp_txdata *txdata = &fp->txdata[cos];
3116
3117 DP(BNX2X_MSG_SP, "allocating tx memory of "
3118 "fp %d cos %d",
3119 index, cos);
3120
3121 BNX2X_ALLOC(txdata->tx_buf_ring,
b3b83c3f 3122 sizeof(struct sw_tx_bd) * NUM_TX_BD);
6383c0b3
AE		 3123 BNX2X_PCI_ALLOC(txdata->tx_desc_ring,
3124 &txdata->tx_desc_mapping,
b3b83c3f 3125 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
6383c0b3 3126 }
b3b83c3f
DK		 3127 }
3128
3129 /* Rx */
3130 if (!skip_rx_queue(bp, index)) {
3131 /* fastpath rx rings: rx_buf rx_desc rx_comp */
3132 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_buf_ring),
3133 sizeof(struct sw_rx_bd) * NUM_RX_BD);
3134 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_desc_ring),
3135 &bnx2x_fp(bp, index, rx_desc_mapping),
3136 sizeof(struct eth_rx_bd) * NUM_RX_BD);
3137
3138 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_comp_ring),
3139 &bnx2x_fp(bp, index, rx_comp_mapping),
3140 sizeof(struct eth_fast_path_rx_cqe) *
3141 NUM_RCQ_BD);
3142
3143 /* SGE ring */
3144 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_page_ring),
3145 sizeof(struct sw_rx_page) * NUM_RX_SGE);
3146 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_sge_ring),
3147 &bnx2x_fp(bp, index, rx_sge_mapping),
3148 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
3149 /* RX BD ring */
3150 bnx2x_set_next_page_rx_bd(fp);
3151
3152 /* CQ ring */
3153 bnx2x_set_next_page_rx_cq(fp);
3154
3155 /* BDs */
3156 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
3157 if (ring_size < rx_ring_size)
3158 goto alloc_mem_err;
3159 }
3160
3161 return 0;
3162
3163/* handles low memory cases */
3164alloc_mem_err:
3165 BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
3166 index, ring_size);
3167 /* FW will drop all packets if queue is not big enough,
3168 * In these cases we disable the queue
6383c0b3 3169 * Min size is different for OOO, TPA and non-TPA queues
b3b83c3f
DK		 3170 */
3171 if (ring_size < (fp->disable_tpa ?
eb722d7a 3172 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
b3b83c3f
DK		 3173 /* release memory allocated for this queue */
3174 bnx2x_free_fp_mem_at(bp, index);
3175 return -ENOMEM;
3176 }
3177 return 0;
3178}
3179
3180int bnx2x_alloc_fp_mem(struct bnx2x *bp)
3181{
3182 int i;
3183
3184 /**
3185 * 1. Allocate FP for leading - fatal if error
3186 * 2. {CNIC} Allocate FCoE FP - fatal if error
6383c0b3
AE		 3187 * 3. {CNIC} Allocate OOO + FWD - disable OOO if error
3188 * 4. Allocate RSS - fix number of queues if error
b3b83c3f
DK		 3189 */
3190
3191 /* leading */
3192 if (bnx2x_alloc_fp_mem_at(bp, 0))
3193 return -ENOMEM;
6383c0b3 3194
b3b83c3f 3195#ifdef BCM_CNIC
8eef2af1
DK		 3196 if (!NO_FCOE(bp))
3197 /* FCoE */
3198 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX))
3199 /* we will fail load process instead of mark
3200 * NO_FCOE_FLAG
3201 */
3202 return -ENOMEM;
b3b83c3f 3203#endif
6383c0b3 3204
b3b83c3f
DK		 3205 /* RSS */
3206 for_each_nondefault_eth_queue(bp, i)
3207 if (bnx2x_alloc_fp_mem_at(bp, i))
3208 break;
3209
3210 /* handle memory failures */
3211 if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
3212 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
3213
3214 WARN_ON(delta < 0);
3215#ifdef BCM_CNIC
3216 /**
3217 * move non eth FPs next to last eth FP
3218 * must be done in that order
3219 * FCOE_IDX < FWD_IDX < OOO_IDX
3220 */
3221
6383c0b3 3222 /* move FCoE fp even NO_FCOE_FLAG is on */
b3b83c3f
DK		 3223 bnx2x_move_fp(bp, FCOE_IDX, FCOE_IDX - delta);
3224#endif
3225 bp->num_queues -= delta;
3226 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
3227 bp->num_queues + delta, bp->num_queues);
3228 }
3229
3230 return 0;
3231}
d6214d7a 3232
523224a3
DK		 3233void bnx2x_free_mem_bp(struct bnx2x *bp)
3234{
3235 kfree(bp->fp);
3236 kfree(bp->msix_table);
3237 kfree(bp->ilt);
3238}
3239
3240int __devinit bnx2x_alloc_mem_bp(struct bnx2x *bp)
3241{
3242 struct bnx2x_fastpath *fp;
3243 struct msix_entry *tbl;
3244 struct bnx2x_ilt *ilt;
6383c0b3
AE		 3245 int msix_table_size = 0;
3246
3247 /*
3248 * The biggest MSI-X table we might need is as a maximum number of fast
3249 * path IGU SBs plus default SB (for PF).
3250 */
3251 msix_table_size = bp->igu_sb_cnt + 1;
523224a3 3252
6383c0b3
AE		 3253 /* fp array: RSS plus CNIC related L2 queues */
3254 fp = kzalloc((BNX2X_MAX_RSS_COUNT(bp) + NON_ETH_CONTEXT_USE) *
3255 sizeof(*fp), GFP_KERNEL);
523224a3
DK		 3256 if (!fp)
3257 goto alloc_err;
3258 bp->fp = fp;
3259
3260 /* msix table */
6383c0b3 3261 tbl = kzalloc(msix_table_size * sizeof(*tbl), GFP_KERNEL);
523224a3
DK		 3262 if (!tbl)
3263 goto alloc_err;
3264 bp->msix_table = tbl;
3265
3266 /* ilt */
3267 ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
3268 if (!ilt)
3269 goto alloc_err;
3270 bp->ilt = ilt;
3271
3272 return 0;
3273alloc_err:
3274 bnx2x_free_mem_bp(bp);
3275 return -ENOMEM;
3276
3277}
3278
a9fccec7 3279int bnx2x_reload_if_running(struct net_device *dev)
66371c44
MM		 3280{
3281 struct bnx2x *bp = netdev_priv(dev);
3282
3283 if (unlikely(!netif_running(dev)))
3284 return 0;
3285
3286 bnx2x_nic_unload(bp, UNLOAD_NORMAL);
3287 return bnx2x_nic_load(bp, LOAD_NORMAL);
3288}
3289
1ac9e428
YR		 3290int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
3291{
3292 u32 sel_phy_idx = 0;
3293 if (bp->link_params.num_phys <= 1)
3294 return INT_PHY;
3295
3296 if (bp->link_vars.link_up) {
3297 sel_phy_idx = EXT_PHY1;
3298 /* In case link is SERDES, check if the EXT_PHY2 is the one */
3299 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
3300 (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
3301 sel_phy_idx = EXT_PHY2;
3302 } else {
3303
3304 switch (bnx2x_phy_selection(&bp->link_params)) {
3305 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
3306 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
3307 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
3308 sel_phy_idx = EXT_PHY1;
3309 break;
3310 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
3311 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
3312 sel_phy_idx = EXT_PHY2;
3313 break;
3314 }
3315 }
3316
3317 return sel_phy_idx;
3318
3319}
3320int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
3321{
3322 u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
3323 /*
3324 * The selected actived PHY is always after swapping (in case PHY
3325 * swapping is enabled). So when swapping is enabled, we need to reverse
3326 * the configuration
3327 */
3328
3329 if (bp->link_params.multi_phy_config &
3330 PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
3331 if (sel_phy_idx == EXT_PHY1)
3332 sel_phy_idx = EXT_PHY2;
3333 else if (sel_phy_idx == EXT_PHY2)
3334 sel_phy_idx = EXT_PHY1;
3335 }
3336 return LINK_CONFIG_IDX(sel_phy_idx);
3337}
3338
bf61ee14
VZ		 3339#if defined(NETDEV_FCOE_WWNN) && defined(BCM_CNIC)
3340int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
3341{
3342 struct bnx2x *bp = netdev_priv(dev);
3343 struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
3344
3345 switch (type) {
3346 case NETDEV_FCOE_WWNN:
3347 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
3348 cp->fcoe_wwn_node_name_lo);
3349 break;
3350 case NETDEV_FCOE_WWPN:
3351 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
3352 cp->fcoe_wwn_port_name_lo);
3353 break;
3354 default:
3355 return -EINVAL;
3356 }
3357
3358 return 0;
3359}
3360#endif
3361
9f6c9258
DK		 3362/* called with rtnl_lock */
3363int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
3364{
3365 struct bnx2x *bp = netdev_priv(dev);
9f6c9258
DK		 3366
3367 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
3368 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
3369 return -EAGAIN;
3370 }
3371
3372 if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
3373 ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE))
3374 return -EINVAL;
3375
3376 /* This does not race with packet allocation
3377 * because the actual alloc size is
3378 * only updated as part of load
3379 */
3380 dev->mtu = new_mtu;
3381
66371c44
MM		 3382 return bnx2x_reload_if_running(dev);
3383}
3384
3385u32 bnx2x_fix_features(struct net_device *dev, u32 features)
3386{
3387 struct bnx2x *bp = netdev_priv(dev);
3388
3389 /* TPA requires Rx CSUM offloading */
3390 if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa)
3391 features &= ~NETIF_F_LRO;
3392
3393 return features;
3394}
3395
3396int bnx2x_set_features(struct net_device *dev, u32 features)
3397{
3398 struct bnx2x *bp = netdev_priv(dev);
3399 u32 flags = bp->flags;
538dd2e3 3400 bool bnx2x_reload = false;
66371c44
MM		 3401
3402 if (features & NETIF_F_LRO)
3403 flags |= TPA_ENABLE_FLAG;
3404 else
3405 flags &= ~TPA_ENABLE_FLAG;
3406
538dd2e3
MB		 3407 if (features & NETIF_F_LOOPBACK) {
3408 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
3409 bp->link_params.loopback_mode = LOOPBACK_BMAC;
3410 bnx2x_reload = true;
3411 }
3412 } else {
3413 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
3414 bp->link_params.loopback_mode = LOOPBACK_NONE;
3415 bnx2x_reload = true;
3416 }
3417 }
3418
66371c44
MM		 3419 if (flags ^ bp->flags) {
3420 bp->flags = flags;
538dd2e3
MB		 3421 bnx2x_reload = true;
3422 }
66371c44 3423
538dd2e3 3424 if (bnx2x_reload) {
66371c44
MM		 3425 if (bp->recovery_state == BNX2X_RECOVERY_DONE)
3426 return bnx2x_reload_if_running(dev);
3427 /* else: bnx2x_nic_load() will be called at end of recovery */
9f6c9258
DK		 3428 }
3429
66371c44 3430 return 0;
9f6c9258
DK		 3431}
3432
3433void bnx2x_tx_timeout(struct net_device *dev)
3434{
3435 struct bnx2x *bp = netdev_priv(dev);
3436
3437#ifdef BNX2X_STOP_ON_ERROR
3438 if (!bp->panic)
3439 bnx2x_panic();
3440#endif
7be08a72
AE		 3441
3442 smp_mb__before_clear_bit();
3443 set_bit(BNX2X_SP_RTNL_TX_TIMEOUT, &bp->sp_rtnl_state);
3444 smp_mb__after_clear_bit();
3445
9f6c9258 3446 /* This allows the netif to be shutdown gracefully before resetting */
7be08a72 3447 schedule_delayed_work(&bp->sp_rtnl_task, 0);
9f6c9258
DK		 3448}
3449
9f6c9258
DK		 3450int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
3451{
3452 struct net_device *dev = pci_get_drvdata(pdev);
3453 struct bnx2x *bp;
3454
3455 if (!dev) {
3456 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
3457 return -ENODEV;
3458 }
3459 bp = netdev_priv(dev);
3460
3461 rtnl_lock();
3462
3463 pci_save_state(pdev);
3464
3465 if (!netif_running(dev)) {
3466 rtnl_unlock();
3467 return 0;
3468 }
3469
3470 netif_device_detach(dev);
3471
3472 bnx2x_nic_unload(bp, UNLOAD_CLOSE);
3473
3474 bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
3475
3476 rtnl_unlock();
3477
3478 return 0;
3479}
3480
3481int bnx2x_resume(struct pci_dev *pdev)
3482{
3483 struct net_device *dev = pci_get_drvdata(pdev);
3484 struct bnx2x *bp;
3485 int rc;
3486
3487 if (!dev) {
3488 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
3489 return -ENODEV;
3490 }
3491 bp = netdev_priv(dev);
3492
3493 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
3494 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
3495 return -EAGAIN;
3496 }
3497
3498 rtnl_lock();
3499
3500 pci_restore_state(pdev);
3501
3502 if (!netif_running(dev)) {
3503 rtnl_unlock();
3504 return 0;
3505 }
3506
3507 bnx2x_set_power_state(bp, PCI_D0);
3508 netif_device_attach(dev);
3509
f2e0899f
DK		 3510 /* Since the chip was reset, clear the FW sequence number */
3511 bp->fw_seq = 0;
9f6c9258
DK		 3512 rc = bnx2x_nic_load(bp, LOAD_OPEN);
3513
3514 rtnl_unlock();
3515
3516 return rc;
3517}
619c5cb6
VZ		 3518
3519
3520void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
3521 u32 cid)
3522{
3523 /* ustorm cxt validation */
3524 cxt->ustorm_ag_context.cdu_usage =
3525 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
3526 CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
3527 /* xcontext validation */
3528 cxt->xstorm_ag_context.cdu_reserved =
3529 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
3530 CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
3531}
3532
3533static inline void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
3534 u8 fw_sb_id, u8 sb_index,
3535 u8 ticks)
3536{
3537
3538 u32 addr = BAR_CSTRORM_INTMEM +
3539 CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
3540 REG_WR8(bp, addr, ticks);
3541 DP(NETIF_MSG_HW, "port %x fw_sb_id %d sb_index %d ticks %d\n",
3542 port, fw_sb_id, sb_index, ticks);
3543}
3544
3545static inline void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
3546 u16 fw_sb_id, u8 sb_index,
3547 u8 disable)
3548{
3549 u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
3550 u32 addr = BAR_CSTRORM_INTMEM +
3551 CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
3552 u16 flags = REG_RD16(bp, addr);
3553 /* clear and set */
3554 flags &= ~HC_INDEX_DATA_HC_ENABLED;
3555 flags |= enable_flag;
3556 REG_WR16(bp, addr, flags);
3557 DP(NETIF_MSG_HW, "port %x fw_sb_id %d sb_index %d disable %d\n",
3558 port, fw_sb_id, sb_index, disable);
3559}
3560
3561void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
3562 u8 sb_index, u8 disable, u16 usec)
3563{
3564 int port = BP_PORT(bp);
3565 u8 ticks = usec / BNX2X_BTR;
3566
3567 storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
3568
3569 disable = disable ? 1 : (usec ? 0 : 1);
3570 storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
3571}
Linux 6.x block layer and io_uring tree(s)