projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
bnx2x: add 6.0.34 fw files
[linux-2.6-block.git] / drivers / net / bnx2x / bnx2x_cmn.c
CommitLineData
9f6c9258
DK		 1/* bnx2x_cmn.c: Broadcom Everest network driver.
2 *
3 * Copyright (c) 2007-2010 Broadcom Corporation
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation.
8 *
9 * Maintained by: Eilon Greenstein <eilong@broadcom.com>
10 * Written by: Eliezer Tamir
11 * Based on code from Michael Chan's bnx2 driver
12 * UDP CSUM errata workaround by Arik Gendelman
13 * Slowpath and fastpath rework by Vladislav Zolotarov
14 * Statistics and Link management by Yitchak Gertner
15 *
16 */
17
18
19#include <linux/etherdevice.h>
20#include <linux/ip.h>
21#include <linux/ipv6.h>
7f3e01fe 22#include <net/ip6_checksum.h>
6891dd25 23#include <linux/firmware.h>
9f6c9258
DK		 24#include "bnx2x_cmn.h"
25
26#ifdef BCM_VLAN
27#include <linux/if_vlan.h>
28#endif
29
30static int bnx2x_poll(struct napi_struct *napi, int budget);
31
32/* free skb in the packet ring at pos idx
33 * return idx of last bd freed
34 */
35static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fastpath *fp,
36 u16 idx)
37{
38 struct sw_tx_bd *tx_buf = &fp->tx_buf_ring[idx];
39 struct eth_tx_start_bd *tx_start_bd;
40 struct eth_tx_bd *tx_data_bd;
41 struct sk_buff *skb = tx_buf->skb;
42 u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
43 int nbd;
44
45 /* prefetch skb end pointer to speedup dev_kfree_skb() */
46 prefetch(&skb->end);
47
48 DP(BNX2X_MSG_OFF, "pkt_idx %d buff @(%p)->skb %p\n",
49 idx, tx_buf, skb);
50
51 /* unmap first bd */
52 DP(BNX2X_MSG_OFF, "free bd_idx %d\n", bd_idx);
53 tx_start_bd = &fp->tx_desc_ring[bd_idx].start_bd;
54 dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
55 BD_UNMAP_LEN(tx_start_bd), PCI_DMA_TODEVICE);
56
57 nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
58#ifdef BNX2X_STOP_ON_ERROR
59 if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
60 BNX2X_ERR("BAD nbd!\n");
61 bnx2x_panic();
62 }
63#endif
64 new_cons = nbd + tx_buf->first_bd;
65
66 /* Get the next bd */
67 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
68
69 /* Skip a parse bd... */
70 --nbd;
71 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
72
73 /* ...and the TSO split header bd since they have no mapping */
74 if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
75 --nbd;
76 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
77 }
78
79 /* now free frags */
80 while (nbd > 0) {
81
82 DP(BNX2X_MSG_OFF, "free frag bd_idx %d\n", bd_idx);
83 tx_data_bd = &fp->tx_desc_ring[bd_idx].reg_bd;
84 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
85 BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
86 if (--nbd)
87 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
88 }
89
90 /* release skb */
91 WARN_ON(!skb);
92 dev_kfree_skb(skb);
93 tx_buf->first_bd = 0;
94 tx_buf->skb = NULL;
95
96 return new_cons;
97}
98
99int bnx2x_tx_int(struct bnx2x_fastpath *fp)
100{
101 struct bnx2x *bp = fp->bp;
102 struct netdev_queue *txq;
103 u16 hw_cons, sw_cons, bd_cons = fp->tx_bd_cons;
104
105#ifdef BNX2X_STOP_ON_ERROR
106 if (unlikely(bp->panic))
107 return -1;
108#endif
109
110 txq = netdev_get_tx_queue(bp->dev, fp->index);
111 hw_cons = le16_to_cpu(*fp->tx_cons_sb);
112 sw_cons = fp->tx_pkt_cons;
113
114 while (sw_cons != hw_cons) {
115 u16 pkt_cons;
116
117 pkt_cons = TX_BD(sw_cons);
118
119 /* prefetch(bp->tx_buf_ring[pkt_cons].skb); */
120
121 DP(NETIF_MSG_TX_DONE, "hw_cons %u sw_cons %u pkt_cons %u\n",
122 hw_cons, sw_cons, pkt_cons);
123
124/* if (NEXT_TX_IDX(sw_cons) != hw_cons) {
125 rmb();
126 prefetch(fp->tx_buf_ring[NEXT_TX_IDX(sw_cons)].skb);
127 }
128*/
129 bd_cons = bnx2x_free_tx_pkt(bp, fp, pkt_cons);
130 sw_cons++;
131 }
132
133 fp->tx_pkt_cons = sw_cons;
134 fp->tx_bd_cons = bd_cons;
135
136 /* Need to make the tx_bd_cons update visible to start_xmit()
137 * before checking for netif_tx_queue_stopped(). Without the
138 * memory barrier, there is a small possibility that
139 * start_xmit() will miss it and cause the queue to be stopped
140 * forever.
141 */
142 smp_mb();
143
144 /* TBD need a thresh? */
145 if (unlikely(netif_tx_queue_stopped(txq))) {
146 /* Taking tx_lock() is needed to prevent reenabling the queue
147 * while it's empty. This could have happen if rx_action() gets
148 * suspended in bnx2x_tx_int() after the condition before
149 * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
150 *
151 * stops the queue->sees fresh tx_bd_cons->releases the queue->
152 * sends some packets consuming the whole queue again->
153 * stops the queue
154 */
155
156 __netif_tx_lock(txq, smp_processor_id());
157
158 if ((netif_tx_queue_stopped(txq)) &&
159 (bp->state == BNX2X_STATE_OPEN) &&
160 (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3))
161 netif_tx_wake_queue(txq);
162
163 __netif_tx_unlock(txq);
164 }
165 return 0;
166}
167
168static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
169 u16 idx)
170{
171 u16 last_max = fp->last_max_sge;
172
173 if (SUB_S16(idx, last_max) > 0)
174 fp->last_max_sge = idx;
175}
176
177static void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
178 struct eth_fast_path_rx_cqe *fp_cqe)
179{
180 struct bnx2x *bp = fp->bp;
181 u16 sge_len = SGE_PAGE_ALIGN(le16_to_cpu(fp_cqe->pkt_len) -
182 le16_to_cpu(fp_cqe->len_on_bd)) >>
183 SGE_PAGE_SHIFT;
184 u16 last_max, last_elem, first_elem;
185 u16 delta = 0;
186 u16 i;
187
188 if (!sge_len)
189 return;
190
191 /* First mark all used pages */
192 for (i = 0; i < sge_len; i++)
193 SGE_MASK_CLEAR_BIT(fp, RX_SGE(le16_to_cpu(fp_cqe->sgl[i])));
194
195 DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
196 sge_len - 1, le16_to_cpu(fp_cqe->sgl[sge_len - 1]));
197
198 /* Here we assume that the last SGE index is the biggest */
199 prefetch((void *)(fp->sge_mask));
200 bnx2x_update_last_max_sge(fp, le16_to_cpu(fp_cqe->sgl[sge_len - 1]));
201
202 last_max = RX_SGE(fp->last_max_sge);
203 last_elem = last_max >> RX_SGE_MASK_ELEM_SHIFT;
204 first_elem = RX_SGE(fp->rx_sge_prod) >> RX_SGE_MASK_ELEM_SHIFT;
205
206 /* If ring is not full */
207 if (last_elem + 1 != first_elem)
208 last_elem++;
209
210 /* Now update the prod */
211 for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
212 if (likely(fp->sge_mask[i]))
213 break;
214
215 fp->sge_mask[i] = RX_SGE_MASK_ELEM_ONE_MASK;
216 delta += RX_SGE_MASK_ELEM_SZ;
217 }
218
219 if (delta > 0) {
220 fp->rx_sge_prod += delta;
221 /* clear page-end entries */
222 bnx2x_clear_sge_mask_next_elems(fp);
223 }
224
225 DP(NETIF_MSG_RX_STATUS,
226 "fp->last_max_sge = %d fp->rx_sge_prod = %d\n",
227 fp->last_max_sge, fp->rx_sge_prod);
228}
229
230static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
231 struct sk_buff *skb, u16 cons, u16 prod)
232{
233 struct bnx2x *bp = fp->bp;
234 struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
235 struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
236 struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
237 dma_addr_t mapping;
238
239 /* move empty skb from pool to prod and map it */
240 prod_rx_buf->skb = fp->tpa_pool[queue].skb;
241 mapping = dma_map_single(&bp->pdev->dev, fp->tpa_pool[queue].skb->data,
242 bp->rx_buf_size, DMA_FROM_DEVICE);
243 dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
244
245 /* move partial skb from cons to pool (don't unmap yet) */
246 fp->tpa_pool[queue] = *cons_rx_buf;
247
248 /* mark bin state as start - print error if current state != stop */
249 if (fp->tpa_state[queue] != BNX2X_TPA_STOP)
250 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
251
252 fp->tpa_state[queue] = BNX2X_TPA_START;
253
254 /* point prod_bd to new skb */
255 prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
256 prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
257
258#ifdef BNX2X_STOP_ON_ERROR
259 fp->tpa_queue_used |= (1 << queue);
260#ifdef _ASM_GENERIC_INT_L64_H
261 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
262#else
263 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
264#endif
265 fp->tpa_queue_used);
266#endif
267}
268
269static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
270 struct sk_buff *skb,
271 struct eth_fast_path_rx_cqe *fp_cqe,
272 u16 cqe_idx)
273{
274 struct sw_rx_page *rx_pg, old_rx_pg;
275 u16 len_on_bd = le16_to_cpu(fp_cqe->len_on_bd);
276 u32 i, frag_len, frag_size, pages;
277 int err;
278 int j;
279
280 frag_size = le16_to_cpu(fp_cqe->pkt_len) - len_on_bd;
281 pages = SGE_PAGE_ALIGN(frag_size) >> SGE_PAGE_SHIFT;
282
283 /* This is needed in order to enable forwarding support */
284 if (frag_size)
285 skb_shinfo(skb)->gso_size = min((u32)SGE_PAGE_SIZE,
286 max(frag_size, (u32)len_on_bd));
287
288#ifdef BNX2X_STOP_ON_ERROR
289 if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
290 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
291 pages, cqe_idx);
292 BNX2X_ERR("fp_cqe->pkt_len = %d fp_cqe->len_on_bd = %d\n",
293 fp_cqe->pkt_len, len_on_bd);
294 bnx2x_panic();
295 return -EINVAL;
296 }
297#endif
298
299 /* Run through the SGL and compose the fragmented skb */
300 for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
301 u16 sge_idx = RX_SGE(le16_to_cpu(fp_cqe->sgl[j]));
302
303 /* FW gives the indices of the SGE as if the ring is an array
304 (meaning that "next" element will consume 2 indices) */
305 frag_len = min(frag_size, (u32)(SGE_PAGE_SIZE*PAGES_PER_SGE));
306 rx_pg = &fp->rx_page_ring[sge_idx];
307 old_rx_pg = *rx_pg;
308
309 /* If we fail to allocate a substitute page, we simply stop
310 where we are and drop the whole packet */
311 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
312 if (unlikely(err)) {
313 fp->eth_q_stats.rx_skb_alloc_failed++;
314 return err;
315 }
316
317 /* Unmap the page as we r going to pass it to the stack */
318 dma_unmap_page(&bp->pdev->dev,
319 dma_unmap_addr(&old_rx_pg, mapping),
320 SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
321
322 /* Add one frag and update the appropriate fields in the skb */
323 skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
324
325 skb->data_len += frag_len;
326 skb->truesize += frag_len;
327 skb->len += frag_len;
328
329 frag_size -= frag_len;
330 }
331
332 return 0;
333}
334
335static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
336 u16 queue, int pad, int len, union eth_rx_cqe *cqe,
337 u16 cqe_idx)
338{
339 struct sw_rx_bd *rx_buf = &fp->tpa_pool[queue];
340 struct sk_buff *skb = rx_buf->skb;
341 /* alloc new skb */
342 struct sk_buff *new_skb = netdev_alloc_skb(bp->dev, bp->rx_buf_size);
343
344 /* Unmap skb in the pool anyway, as we are going to change
345 pool entry status to BNX2X_TPA_STOP even if new skb allocation
346 fails. */
347 dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
348 bp->rx_buf_size, DMA_FROM_DEVICE);
349
350 if (likely(new_skb)) {
351 /* fix ip xsum and give it to the stack */
352 /* (no need to map the new skb) */
353#ifdef BCM_VLAN
354 int is_vlan_cqe =
355 (le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags) &
356 PARSING_FLAGS_VLAN);
357 int is_not_hwaccel_vlan_cqe =
358 (is_vlan_cqe && (!(bp->flags & HW_VLAN_RX_FLAG)));
359#endif
360
361 prefetch(skb);
362 prefetch(((char *)(skb)) + 128);
363
364#ifdef BNX2X_STOP_ON_ERROR
365 if (pad + len > bp->rx_buf_size) {
366 BNX2X_ERR("skb_put is about to fail... "
367 "pad %d len %d rx_buf_size %d\n",
368 pad, len, bp->rx_buf_size);
369 bnx2x_panic();
370 return;
371 }
372#endif
373
374 skb_reserve(skb, pad);
375 skb_put(skb, len);
376
377 skb->protocol = eth_type_trans(skb, bp->dev);
378 skb->ip_summed = CHECKSUM_UNNECESSARY;
379
380 {
381 struct iphdr *iph;
382
383 iph = (struct iphdr *)skb->data;
384#ifdef BCM_VLAN
385 /* If there is no Rx VLAN offloading -
386 take VLAN tag into an account */
387 if (unlikely(is_not_hwaccel_vlan_cqe))
388 iph = (struct iphdr *)((u8 *)iph + VLAN_HLEN);
389#endif
390 iph->check = 0;
391 iph->check = ip_fast_csum((u8 *)iph, iph->ihl);
392 }
393
394 if (!bnx2x_fill_frag_skb(bp, fp, skb,
395 &cqe->fast_path_cqe, cqe_idx)) {
396#ifdef BCM_VLAN
397 if ((bp->vlgrp != NULL) && is_vlan_cqe &&
398 (!is_not_hwaccel_vlan_cqe))
399 vlan_gro_receive(&fp->napi, bp->vlgrp,
400 le16_to_cpu(cqe->fast_path_cqe.
401 vlan_tag), skb);
402 else
403#endif
404 napi_gro_receive(&fp->napi, skb);
405 } else {
406 DP(NETIF_MSG_RX_STATUS, "Failed to allocate new pages"
407 " - dropping packet!\n");
408 dev_kfree_skb(skb);
409 }
410
411
412 /* put new skb in bin */
413 fp->tpa_pool[queue].skb = new_skb;
414
415 } else {
416 /* else drop the packet and keep the buffer in the bin */
417 DP(NETIF_MSG_RX_STATUS,
418 "Failed to allocate new skb - dropping packet!\n");
419 fp->eth_q_stats.rx_skb_alloc_failed++;
420 }
421
422 fp->tpa_state[queue] = BNX2X_TPA_STOP;
423}
424
425/* Set Toeplitz hash value in the skb using the value from the
426 * CQE (calculated by HW).
427 */
428static inline void bnx2x_set_skb_rxhash(struct bnx2x *bp, union eth_rx_cqe *cqe,
429 struct sk_buff *skb)
430{
431 /* Set Toeplitz hash from CQE */
432 if ((bp->dev->features & NETIF_F_RXHASH) &&
433 (cqe->fast_path_cqe.status_flags &
434 ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG))
435 skb->rxhash =
436 le32_to_cpu(cqe->fast_path_cqe.rss_hash_result);
437}
438
439int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
440{
441 struct bnx2x *bp = fp->bp;
442 u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
443 u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
444 int rx_pkt = 0;
445
446#ifdef BNX2X_STOP_ON_ERROR
447 if (unlikely(bp->panic))
448 return 0;
449#endif
450
451 /* CQ "next element" is of the size of the regular element,
452 that's why it's ok here */
453 hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
454 if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
455 hw_comp_cons++;
456
457 bd_cons = fp->rx_bd_cons;
458 bd_prod = fp->rx_bd_prod;
459 bd_prod_fw = bd_prod;
460 sw_comp_cons = fp->rx_comp_cons;
461 sw_comp_prod = fp->rx_comp_prod;
462
463 /* Memory barrier necessary as speculative reads of the rx
464 * buffer can be ahead of the index in the status block
465 */
466 rmb();
467
468 DP(NETIF_MSG_RX_STATUS,
469 "queue[%d]: hw_comp_cons %u sw_comp_cons %u\n",
470 fp->index, hw_comp_cons, sw_comp_cons);
471
472 while (sw_comp_cons != hw_comp_cons) {
473 struct sw_rx_bd *rx_buf = NULL;
474 struct sk_buff *skb;
475 union eth_rx_cqe *cqe;
476 u8 cqe_fp_flags;
477 u16 len, pad;
478
479 comp_ring_cons = RCQ_BD(sw_comp_cons);
480 bd_prod = RX_BD(bd_prod);
481 bd_cons = RX_BD(bd_cons);
482
483 /* Prefetch the page containing the BD descriptor
484 at producer's index. It will be needed when new skb is
485 allocated */
486 prefetch((void *)(PAGE_ALIGN((unsigned long)
487 (&fp->rx_desc_ring[bd_prod])) -
488 PAGE_SIZE + 1));
489
490 cqe = &fp->rx_comp_ring[comp_ring_cons];
491 cqe_fp_flags = cqe->fast_path_cqe.type_error_flags;
492
493 DP(NETIF_MSG_RX_STATUS, "CQE type %x err %x status %x"
494 " queue %x vlan %x len %u\n", CQE_TYPE(cqe_fp_flags),
495 cqe_fp_flags, cqe->fast_path_cqe.status_flags,
496 le32_to_cpu(cqe->fast_path_cqe.rss_hash_result),
497 le16_to_cpu(cqe->fast_path_cqe.vlan_tag),
498 le16_to_cpu(cqe->fast_path_cqe.pkt_len));
499
500 /* is this a slowpath msg? */
501 if (unlikely(CQE_TYPE(cqe_fp_flags))) {
502 bnx2x_sp_event(fp, cqe);
503 goto next_cqe;
504
505 /* this is an rx packet */
506 } else {
507 rx_buf = &fp->rx_buf_ring[bd_cons];
508 skb = rx_buf->skb;
509 prefetch(skb);
510 len = le16_to_cpu(cqe->fast_path_cqe.pkt_len);
511 pad = cqe->fast_path_cqe.placement_offset;
512
513 /* If CQE is marked both TPA_START and TPA_END
514 it is a non-TPA CQE */
515 if ((!fp->disable_tpa) &&
516 (TPA_TYPE(cqe_fp_flags) !=
517 (TPA_TYPE_START | TPA_TYPE_END))) {
518 u16 queue = cqe->fast_path_cqe.queue_index;
519
520 if (TPA_TYPE(cqe_fp_flags) == TPA_TYPE_START) {
521 DP(NETIF_MSG_RX_STATUS,
522 "calling tpa_start on queue %d\n",
523 queue);
524
525 bnx2x_tpa_start(fp, queue, skb,
526 bd_cons, bd_prod);
527
528 /* Set Toeplitz hash for an LRO skb */
529 bnx2x_set_skb_rxhash(bp, cqe, skb);
530
531 goto next_rx;
532 }
533
534 if (TPA_TYPE(cqe_fp_flags) == TPA_TYPE_END) {
535 DP(NETIF_MSG_RX_STATUS,
536 "calling tpa_stop on queue %d\n",
537 queue);
538
539 if (!BNX2X_RX_SUM_FIX(cqe))
540 BNX2X_ERR("STOP on none TCP "
541 "data\n");
542
543 /* This is a size of the linear data
544 on this skb */
545 len = le16_to_cpu(cqe->fast_path_cqe.
546 len_on_bd);
547 bnx2x_tpa_stop(bp, fp, queue, pad,
548 len, cqe, comp_ring_cons);
549#ifdef BNX2X_STOP_ON_ERROR
550 if (bp->panic)
551 return 0;
552#endif
553
554 bnx2x_update_sge_prod(fp,
555 &cqe->fast_path_cqe);
556 goto next_cqe;
557 }
558 }
559
560 dma_sync_single_for_device(&bp->pdev->dev,
561 dma_unmap_addr(rx_buf, mapping),
562 pad + RX_COPY_THRESH,
563 DMA_FROM_DEVICE);
564 prefetch(((char *)(skb)) + 128);
565
566 /* is this an error packet? */
567 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
568 DP(NETIF_MSG_RX_ERR,
569 "ERROR flags %x rx packet %u\n",
570 cqe_fp_flags, sw_comp_cons);
571 fp->eth_q_stats.rx_err_discard_pkt++;
572 goto reuse_rx;
573 }
574
575 /* Since we don't have a jumbo ring
576 * copy small packets if mtu > 1500
577 */
578 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
579 (len <= RX_COPY_THRESH)) {
580 struct sk_buff *new_skb;
581
582 new_skb = netdev_alloc_skb(bp->dev,
583 len + pad);
584 if (new_skb == NULL) {
585 DP(NETIF_MSG_RX_ERR,
586 "ERROR packet dropped "
587 "because of alloc failure\n");
588 fp->eth_q_stats.rx_skb_alloc_failed++;
589 goto reuse_rx;
590 }
591
592 /* aligned copy */
593 skb_copy_from_linear_data_offset(skb, pad,
594 new_skb->data + pad, len);
595 skb_reserve(new_skb, pad);
596 skb_put(new_skb, len);
597
598 bnx2x_reuse_rx_skb(fp, skb, bd_cons, bd_prod);
599
600 skb = new_skb;
601
602 } else
603 if (likely(bnx2x_alloc_rx_skb(bp, fp, bd_prod) == 0)) {
604 dma_unmap_single(&bp->pdev->dev,
605 dma_unmap_addr(rx_buf, mapping),
606 bp->rx_buf_size,
607 DMA_FROM_DEVICE);
608 skb_reserve(skb, pad);
609 skb_put(skb, len);
610
611 } else {
612 DP(NETIF_MSG_RX_ERR,
613 "ERROR packet dropped because "
614 "of alloc failure\n");
615 fp->eth_q_stats.rx_skb_alloc_failed++;
616reuse_rx:
617 bnx2x_reuse_rx_skb(fp, skb, bd_cons, bd_prod);
618 goto next_rx;
619 }
620
621 skb->protocol = eth_type_trans(skb, bp->dev);
622
623 /* Set Toeplitz hash for a none-LRO skb */
624 bnx2x_set_skb_rxhash(bp, cqe, skb);
625
bc8acf2c 626 skb_checksum_none_assert(skb);
9f6c9258
DK		 627 if (bp->rx_csum) {
628 if (likely(BNX2X_RX_CSUM_OK(cqe)))
629 skb->ip_summed = CHECKSUM_UNNECESSARY;
630 else
631 fp->eth_q_stats.hw_csum_err++;
632 }
633 }
634
635 skb_record_rx_queue(skb, fp->index);
636
637#ifdef BCM_VLAN
638 if ((bp->vlgrp != NULL) && (bp->flags & HW_VLAN_RX_FLAG) &&
639 (le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags) &
640 PARSING_FLAGS_VLAN))
641 vlan_gro_receive(&fp->napi, bp->vlgrp,
642 le16_to_cpu(cqe->fast_path_cqe.vlan_tag), skb);
643 else
644#endif
645 napi_gro_receive(&fp->napi, skb);
646
647
648next_rx:
649 rx_buf->skb = NULL;
650
651 bd_cons = NEXT_RX_IDX(bd_cons);
652 bd_prod = NEXT_RX_IDX(bd_prod);
653 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
654 rx_pkt++;
655next_cqe:
656 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
657 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
658
659 if (rx_pkt == budget)
660 break;
661 } /* while */
662
663 fp->rx_bd_cons = bd_cons;
664 fp->rx_bd_prod = bd_prod_fw;
665 fp->rx_comp_cons = sw_comp_cons;
666 fp->rx_comp_prod = sw_comp_prod;
667
668 /* Update producers */
669 bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
670 fp->rx_sge_prod);
671
672 fp->rx_pkt += rx_pkt;
673 fp->rx_calls++;
674
675 return rx_pkt;
676}
677
678static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
679{
680 struct bnx2x_fastpath *fp = fp_cookie;
681 struct bnx2x *bp = fp->bp;
682
683 /* Return here if interrupt is disabled */
684 if (unlikely(atomic_read(&bp->intr_sem) != 0)) {
685 DP(NETIF_MSG_INTR, "called but intr_sem not 0, returning\n");
686 return IRQ_HANDLED;
687 }
688
689 DP(BNX2X_MSG_FP, "got an MSI-X interrupt on IDX:SB [%d:%d]\n",
690 fp->index, fp->sb_id);
691 bnx2x_ack_sb(bp, fp->sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
692
693#ifdef BNX2X_STOP_ON_ERROR
694 if (unlikely(bp->panic))
695 return IRQ_HANDLED;
696#endif
697
698 /* Handle Rx and Tx according to MSI-X vector */
699 prefetch(fp->rx_cons_sb);
700 prefetch(fp->tx_cons_sb);
701 prefetch(&fp->status_blk->u_status_block.status_block_index);
702 prefetch(&fp->status_blk->c_status_block.status_block_index);
703 napi_schedule(&bnx2x_fp(bp, fp->index, napi));
704
705 return IRQ_HANDLED;
706}
707
708
709/* HW Lock for shared dual port PHYs */
710void bnx2x_acquire_phy_lock(struct bnx2x *bp)
711{
712 mutex_lock(&bp->port.phy_mutex);
713
714 if (bp->port.need_hw_lock)
715 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
716}
717
718void bnx2x_release_phy_lock(struct bnx2x *bp)
719{
720 if (bp->port.need_hw_lock)
721 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
722
723 mutex_unlock(&bp->port.phy_mutex);
724}
725
726void bnx2x_link_report(struct bnx2x *bp)
727{
728 if (bp->flags & MF_FUNC_DIS) {
729 netif_carrier_off(bp->dev);
730 netdev_err(bp->dev, "NIC Link is Down\n");
731 return;
732 }
733
734 if (bp->link_vars.link_up) {
735 u16 line_speed;
736
737 if (bp->state == BNX2X_STATE_OPEN)
738 netif_carrier_on(bp->dev);
739 netdev_info(bp->dev, "NIC Link is Up, ");
740
741 line_speed = bp->link_vars.line_speed;
742 if (IS_E1HMF(bp)) {
743 u16 vn_max_rate;
744
745 vn_max_rate =
746 ((bp->mf_config & FUNC_MF_CFG_MAX_BW_MASK) >>
747 FUNC_MF_CFG_MAX_BW_SHIFT) * 100;
748 if (vn_max_rate < line_speed)
749 line_speed = vn_max_rate;
750 }
751 pr_cont("%d Mbps ", line_speed);
752
753 if (bp->link_vars.duplex == DUPLEX_FULL)
754 pr_cont("full duplex");
755 else
756 pr_cont("half duplex");
757
758 if (bp->link_vars.flow_ctrl != BNX2X_FLOW_CTRL_NONE) {
759 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX) {
760 pr_cont(", receive ");
761 if (bp->link_vars.flow_ctrl &
762 BNX2X_FLOW_CTRL_TX)
763 pr_cont("& transmit ");
764 } else {
765 pr_cont(", transmit ");
766 }
767 pr_cont("flow control ON");
768 }
769 pr_cont("\n");
770
771 } else { /* link_down */
772 netif_carrier_off(bp->dev);
773 netdev_err(bp->dev, "NIC Link is Down\n");
774 }
775}
776
777void bnx2x_init_rx_rings(struct bnx2x *bp)
778{
779 int func = BP_FUNC(bp);
780 int max_agg_queues = CHIP_IS_E1(bp) ? ETH_MAX_AGGREGATION_QUEUES_E1 :
781 ETH_MAX_AGGREGATION_QUEUES_E1H;
782 u16 ring_prod, cqe_ring_prod;
783 int i, j;
25141580
DK		 784 int rx_ring_size = bp->rx_ring_size ? bp->rx_ring_size :
785 MAX_RX_AVAIL/bp->num_queues;
786
787 rx_ring_size = max_t(int, MIN_RX_AVAIL, rx_ring_size);
9f6c9258
DK		 788
789 bp->rx_buf_size = bp->dev->mtu + ETH_OVREHEAD + BNX2X_RX_ALIGN;
790 DP(NETIF_MSG_IFUP,
791 "mtu %d rx_buf_size %d\n", bp->dev->mtu, bp->rx_buf_size);
792
793 if (bp->flags & TPA_ENABLE_FLAG) {
794
795 for_each_queue(bp, j) {
796 struct bnx2x_fastpath *fp = &bp->fp[j];
797
798 for (i = 0; i < max_agg_queues; i++) {
799 fp->tpa_pool[i].skb =
800 netdev_alloc_skb(bp->dev, bp->rx_buf_size);
801 if (!fp->tpa_pool[i].skb) {
802 BNX2X_ERR("Failed to allocate TPA "
803 "skb pool for queue[%d] - "
804 "disabling TPA on this "
805 "queue!\n", j);
806 bnx2x_free_tpa_pool(bp, fp, i);
807 fp->disable_tpa = 1;
808 break;
809 }
810 dma_unmap_addr_set((struct sw_rx_bd *)
811 &bp->fp->tpa_pool[i],
812 mapping, 0);
813 fp->tpa_state[i] = BNX2X_TPA_STOP;
814 }
815 }
816 }
817
818 for_each_queue(bp, j) {
819 struct bnx2x_fastpath *fp = &bp->fp[j];
820
821 fp->rx_bd_cons = 0;
822 fp->rx_cons_sb = BNX2X_RX_SB_INDEX;
823 fp->rx_bd_cons_sb = BNX2X_RX_SB_BD_INDEX;
824
825 /* "next page" elements initialization */
826 /* SGE ring */
827 for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
828 struct eth_rx_sge *sge;
829
830 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
831 sge->addr_hi =
832 cpu_to_le32(U64_HI(fp->rx_sge_mapping +
833 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
834 sge->addr_lo =
835 cpu_to_le32(U64_LO(fp->rx_sge_mapping +
836 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
837 }
838
839 bnx2x_init_sge_ring_bit_mask(fp);
840
841 /* RX BD ring */
842 for (i = 1; i <= NUM_RX_RINGS; i++) {
843 struct eth_rx_bd *rx_bd;
844
845 rx_bd = &fp->rx_desc_ring[RX_DESC_CNT * i - 2];
846 rx_bd->addr_hi =
847 cpu_to_le32(U64_HI(fp->rx_desc_mapping +
848 BCM_PAGE_SIZE*(i % NUM_RX_RINGS)));
849 rx_bd->addr_lo =
850 cpu_to_le32(U64_LO(fp->rx_desc_mapping +
851 BCM_PAGE_SIZE*(i % NUM_RX_RINGS)));
852 }
853
854 /* CQ ring */
855 for (i = 1; i <= NUM_RCQ_RINGS; i++) {
856 struct eth_rx_cqe_next_page *nextpg;
857
858 nextpg = (struct eth_rx_cqe_next_page *)
859 &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
860 nextpg->addr_hi =
861 cpu_to_le32(U64_HI(fp->rx_comp_mapping +
862 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
863 nextpg->addr_lo =
864 cpu_to_le32(U64_LO(fp->rx_comp_mapping +
865 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
866 }
867
868 /* Allocate SGEs and initialize the ring elements */
869 for (i = 0, ring_prod = 0;
870 i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
871
872 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
873 BNX2X_ERR("was only able to allocate "
874 "%d rx sges\n", i);
875 BNX2X_ERR("disabling TPA for queue[%d]\n", j);
876 /* Cleanup already allocated elements */
877 bnx2x_free_rx_sge_range(bp, fp, ring_prod);
878 bnx2x_free_tpa_pool(bp, fp, max_agg_queues);
879 fp->disable_tpa = 1;
880 ring_prod = 0;
881 break;
882 }
883 ring_prod = NEXT_SGE_IDX(ring_prod);
884 }
885 fp->rx_sge_prod = ring_prod;
886
887 /* Allocate BDs and initialize BD ring */
888 fp->rx_comp_cons = 0;
889 cqe_ring_prod = ring_prod = 0;
25141580 890 for (i = 0; i < rx_ring_size; i++) {
9f6c9258
DK		 891 if (bnx2x_alloc_rx_skb(bp, fp, ring_prod) < 0) {
892 BNX2X_ERR("was only able to allocate "
893 "%d rx skbs on queue[%d]\n", i, j);
894 fp->eth_q_stats.rx_skb_alloc_failed++;
895 break;
896 }
897 ring_prod = NEXT_RX_IDX(ring_prod);
898 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
899 WARN_ON(ring_prod <= i);
900 }
901
902 fp->rx_bd_prod = ring_prod;
903 /* must not have more available CQEs than BDs */
904 fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
905 cqe_ring_prod);
906 fp->rx_pkt = fp->rx_calls = 0;
907
908 /* Warning!
909 * this will generate an interrupt (to the TSTORM)
910 * must only be done after chip is initialized
911 */
912 bnx2x_update_rx_prod(bp, fp, ring_prod, fp->rx_comp_prod,
913 fp->rx_sge_prod);
914 if (j != 0)
915 continue;
916
917 REG_WR(bp, BAR_USTRORM_INTMEM +
918 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
919 U64_LO(fp->rx_comp_mapping));
920 REG_WR(bp, BAR_USTRORM_INTMEM +
921 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
922 U64_HI(fp->rx_comp_mapping));
923 }
924}
925static void bnx2x_free_tx_skbs(struct bnx2x *bp)
926{
927 int i;
928
929 for_each_queue(bp, i) {
930 struct bnx2x_fastpath *fp = &bp->fp[i];
931
932 u16 bd_cons = fp->tx_bd_cons;
933 u16 sw_prod = fp->tx_pkt_prod;
934 u16 sw_cons = fp->tx_pkt_cons;
935
936 while (sw_cons != sw_prod) {
937 bd_cons = bnx2x_free_tx_pkt(bp, fp, TX_BD(sw_cons));
938 sw_cons++;
939 }
940 }
941}
942
943static void bnx2x_free_rx_skbs(struct bnx2x *bp)
944{
945 int i, j;
946
947 for_each_queue(bp, j) {
948 struct bnx2x_fastpath *fp = &bp->fp[j];
949
950 for (i = 0; i < NUM_RX_BD; i++) {
951 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
952 struct sk_buff *skb = rx_buf->skb;
953
954 if (skb == NULL)
955 continue;
956
957 dma_unmap_single(&bp->pdev->dev,
958 dma_unmap_addr(rx_buf, mapping),
959 bp->rx_buf_size, DMA_FROM_DEVICE);
960
961 rx_buf->skb = NULL;
962 dev_kfree_skb(skb);
963 }
964 if (!fp->disable_tpa)
965 bnx2x_free_tpa_pool(bp, fp, CHIP_IS_E1(bp) ?
966 ETH_MAX_AGGREGATION_QUEUES_E1 :
967 ETH_MAX_AGGREGATION_QUEUES_E1H);
968 }
969}
970
971void bnx2x_free_skbs(struct bnx2x *bp)
972{
973 bnx2x_free_tx_skbs(bp);
974 bnx2x_free_rx_skbs(bp);
975}
976
977static void bnx2x_free_msix_irqs(struct bnx2x *bp)
978{
979 int i, offset = 1;
980
981 free_irq(bp->msix_table[0].vector, bp->dev);
982 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
983 bp->msix_table[0].vector);
984
985#ifdef BCM_CNIC
986 offset++;
987#endif
988 for_each_queue(bp, i) {
989 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq "
990 "state %x\n", i, bp->msix_table[i + offset].vector,
991 bnx2x_fp(bp, i, state));
992
993 free_irq(bp->msix_table[i + offset].vector, &bp->fp[i]);
994 }
995}
996
997void bnx2x_free_irq(struct bnx2x *bp, bool disable_only)
998{
999 if (bp->flags & USING_MSIX_FLAG) {
1000 if (!disable_only)
1001 bnx2x_free_msix_irqs(bp);
1002 pci_disable_msix(bp->pdev);
1003 bp->flags &= ~USING_MSIX_FLAG;
1004
1005 } else if (bp->flags & USING_MSI_FLAG) {
1006 if (!disable_only)
1007 free_irq(bp->pdev->irq, bp->dev);
1008 pci_disable_msi(bp->pdev);
1009 bp->flags &= ~USING_MSI_FLAG;
1010
1011 } else if (!disable_only)
1012 free_irq(bp->pdev->irq, bp->dev);
1013}
1014
1015static int bnx2x_enable_msix(struct bnx2x *bp)
1016{
1017 int i, rc, offset = 1;
1018 int igu_vec = 0;
1019
1020 bp->msix_table[0].entry = igu_vec;
1021 DP(NETIF_MSG_IFUP, "msix_table[0].entry = %d (slowpath)\n", igu_vec);
1022
1023#ifdef BCM_CNIC
1024 igu_vec = BP_L_ID(bp) + offset;
1025 bp->msix_table[1].entry = igu_vec;
1026 DP(NETIF_MSG_IFUP, "msix_table[1].entry = %d (CNIC)\n", igu_vec);
1027 offset++;
1028#endif
1029 for_each_queue(bp, i) {
1030 igu_vec = BP_L_ID(bp) + offset + i;
1031 bp->msix_table[i + offset].entry = igu_vec;
1032 DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d "
1033 "(fastpath #%u)\n", i + offset, igu_vec, i);
1034 }
1035
1036 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0],
1037 BNX2X_NUM_QUEUES(bp) + offset);
1038
1039 /*
1040 * reconfigure number of tx/rx queues according to available
1041 * MSI-X vectors
1042 */
1043 if (rc >= BNX2X_MIN_MSIX_VEC_CNT) {
1044 /* vectors available for FP */
1045 int fp_vec = rc - BNX2X_MSIX_VEC_FP_START;
1046
1047 DP(NETIF_MSG_IFUP,
1048 "Trying to use less MSI-X vectors: %d\n", rc);
1049
1050 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1051
1052 if (rc) {
1053 DP(NETIF_MSG_IFUP,
1054 "MSI-X is not attainable rc %d\n", rc);
1055 return rc;
1056 }
1057
1058 bp->num_queues = min(bp->num_queues, fp_vec);
1059
1060 DP(NETIF_MSG_IFUP, "New queue configuration set: %d\n",
1061 bp->num_queues);
1062 } else if (rc) {
1063 DP(NETIF_MSG_IFUP, "MSI-X is not attainable rc %d\n", rc);
1064 return rc;
1065 }
1066
1067 bp->flags |= USING_MSIX_FLAG;
1068
1069 return 0;
1070}
1071
1072static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1073{
1074 int i, rc, offset = 1;
1075
1076 rc = request_irq(bp->msix_table[0].vector, bnx2x_msix_sp_int, 0,
1077 bp->dev->name, bp->dev);
1078 if (rc) {
1079 BNX2X_ERR("request sp irq failed\n");
1080 return -EBUSY;
1081 }
1082
1083#ifdef BCM_CNIC
1084 offset++;
1085#endif
1086 for_each_queue(bp, i) {
1087 struct bnx2x_fastpath *fp = &bp->fp[i];
1088 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1089 bp->dev->name, i);
1090
1091 rc = request_irq(bp->msix_table[i + offset].vector,
1092 bnx2x_msix_fp_int, 0, fp->name, fp);
1093 if (rc) {
1094 BNX2X_ERR("request fp #%d irq failed rc %d\n", i, rc);
1095 bnx2x_free_msix_irqs(bp);
1096 return -EBUSY;
1097 }
1098
1099 fp->state = BNX2X_FP_STATE_IRQ;
1100 }
1101
1102 i = BNX2X_NUM_QUEUES(bp);
1103 netdev_info(bp->dev, "using MSI-X IRQs: sp %d fp[%d] %d"
1104 " ... fp[%d] %d\n",
1105 bp->msix_table[0].vector,
1106 0, bp->msix_table[offset].vector,
1107 i - 1, bp->msix_table[offset + i - 1].vector);
1108
1109 return 0;
1110}
1111
1112static int bnx2x_enable_msi(struct bnx2x *bp)
1113{
1114 int rc;
1115
1116 rc = pci_enable_msi(bp->pdev);
1117 if (rc) {
1118 DP(NETIF_MSG_IFUP, "MSI is not attainable\n");
1119 return -1;
1120 }
1121 bp->flags |= USING_MSI_FLAG;
1122
1123 return 0;
1124}
1125
1126static int bnx2x_req_irq(struct bnx2x *bp)
1127{
1128 unsigned long flags;
1129 int rc;
1130
1131 if (bp->flags & USING_MSI_FLAG)
1132 flags = 0;
1133 else
1134 flags = IRQF_SHARED;
1135
1136 rc = request_irq(bp->pdev->irq, bnx2x_interrupt, flags,
1137 bp->dev->name, bp->dev);
1138 if (!rc)
1139 bnx2x_fp(bp, 0, state) = BNX2X_FP_STATE_IRQ;
1140
1141 return rc;
1142}
1143
1144static void bnx2x_napi_enable(struct bnx2x *bp)
1145{
1146 int i;
1147
1148 for_each_queue(bp, i)
1149 napi_enable(&bnx2x_fp(bp, i, napi));
1150}
1151
1152static void bnx2x_napi_disable(struct bnx2x *bp)
1153{
1154 int i;
1155
1156 for_each_queue(bp, i)
1157 napi_disable(&bnx2x_fp(bp, i, napi));
1158}
1159
1160void bnx2x_netif_start(struct bnx2x *bp)
1161{
1162 int intr_sem;
1163
1164 intr_sem = atomic_dec_and_test(&bp->intr_sem);
1165 smp_wmb(); /* Ensure that bp->intr_sem update is SMP-safe */
1166
1167 if (intr_sem) {
1168 if (netif_running(bp->dev)) {
1169 bnx2x_napi_enable(bp);
1170 bnx2x_int_enable(bp);
1171 if (bp->state == BNX2X_STATE_OPEN)
1172 netif_tx_wake_all_queues(bp->dev);
1173 }
1174 }
1175}
1176
1177void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1178{
1179 bnx2x_int_disable_sync(bp, disable_hw);
1180 bnx2x_napi_disable(bp);
1181 netif_tx_disable(bp->dev);
1182}
1183static int bnx2x_set_num_queues(struct bnx2x *bp)
1184{
1185 int rc = 0;
1186
1187 switch (bp->int_mode) {
9f6c9258 1188 case INT_MODE_MSI:
8681dc3a
DK		 1189 bnx2x_enable_msi(bp);
1190 /* falling through... */
1191 case INT_MODE_INTx:
9f6c9258
DK		 1192 bp->num_queues = 1;
1193 DP(NETIF_MSG_IFUP, "set number of queues to 1\n");
1194 break;
1195 default:
1196 /* Set number of queues according to bp->multi_mode value */
1197 bnx2x_set_num_queues_msix(bp);
1198
1199 DP(NETIF_MSG_IFUP, "set number of queues to %d\n",
1200 bp->num_queues);
1201
1202 /* if we can't use MSI-X we only need one fp,
1203 * so try to enable MSI-X with the requested number of fp's
1204 * and fallback to MSI or legacy INTx with one fp
1205 */
1206 rc = bnx2x_enable_msix(bp);
8681dc3a 1207 if (rc) {
9f6c9258
DK		 1208 /* failed to enable MSI-X */
1209 bp->num_queues = 1;
8681dc3a
DK		 1210
1211 /* Fall to INTx if failed to enable MSI-X due to lack of
1212 * memory (in bnx2x_set_num_queues()) */
1213 if ((rc != -ENOMEM) && (bp->int_mode != INT_MODE_INTx))
1214 bnx2x_enable_msi(bp);
1215 }
1216
9f6c9258
DK		 1217 break;
1218 }
31b600b5
BH		 1219 netif_set_real_num_tx_queues(bp->dev, bp->num_queues);
1220 return netif_set_real_num_rx_queues(bp->dev, bp->num_queues);
9f6c9258
DK		 1221}
1222
6891dd25
DK		 1223static void bnx2x_release_firmware(struct bnx2x *bp)
1224{
1225 kfree(bp->init_ops_offsets);
1226 kfree(bp->init_ops);
1227 kfree(bp->init_data);
1228 release_firmware(bp->firmware);
1229}
1230
9f6c9258
DK		 1231/* must be called with rtnl_lock */
1232int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
1233{
1234 u32 load_code;
1235 int i, rc;
1236
6891dd25
DK		 1237 /* Set init arrays */
1238 rc = bnx2x_init_firmware(bp);
1239 if (rc) {
1240 BNX2X_ERR("Error loading firmware\n");
1241 return rc;
1242 }
1243
9f6c9258
DK		 1244#ifdef BNX2X_STOP_ON_ERROR
1245 if (unlikely(bp->panic))
1246 return -EPERM;
1247#endif
1248
1249 bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
1250
1251 rc = bnx2x_set_num_queues(bp);
31b600b5
BH		 1252 if (rc)
1253 return rc;
9f6c9258
DK		 1254
1255 if (bnx2x_alloc_mem(bp)) {
1256 bnx2x_free_irq(bp, true);
1257 return -ENOMEM;
1258 }
1259
1260 for_each_queue(bp, i)
1261 bnx2x_fp(bp, i, disable_tpa) =
1262 ((bp->flags & TPA_ENABLE_FLAG) == 0);
1263
1264 for_each_queue(bp, i)
1265 netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
1266 bnx2x_poll, 128);
1267
1268 bnx2x_napi_enable(bp);
1269
1270 if (bp->flags & USING_MSIX_FLAG) {
1271 rc = bnx2x_req_msix_irqs(bp);
1272 if (rc) {
1273 bnx2x_free_irq(bp, true);
1274 goto load_error1;
1275 }
1276 } else {
9f6c9258
DK		 1277 bnx2x_ack_int(bp);
1278 rc = bnx2x_req_irq(bp);
1279 if (rc) {
1280 BNX2X_ERR("IRQ request failed rc %d, aborting\n", rc);
1281 bnx2x_free_irq(bp, true);
1282 goto load_error1;
1283 }
1284 if (bp->flags & USING_MSI_FLAG) {
1285 bp->dev->irq = bp->pdev->irq;
1286 netdev_info(bp->dev, "using MSI IRQ %d\n",
1287 bp->pdev->irq);
1288 }
1289 }
1290
1291 /* Send LOAD_REQUEST command to MCP
1292 Returns the type of LOAD command:
1293 if it is the first port to be initialized
1294 common blocks should be initialized, otherwise - not
1295 */
1296 if (!BP_NOMCP(bp)) {
a22f0788 1297 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, 0);
9f6c9258
DK		 1298 if (!load_code) {
1299 BNX2X_ERR("MCP response failure, aborting\n");
1300 rc = -EBUSY;
1301 goto load_error2;
1302 }
1303 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
1304 rc = -EBUSY; /* other port in diagnostic mode */
1305 goto load_error2;
1306 }
1307
1308 } else {
1309 int port = BP_PORT(bp);
1310
1311 DP(NETIF_MSG_IFUP, "NO MCP - load counts %d, %d, %d\n",
1312 load_count[0], load_count[1], load_count[2]);
1313 load_count[0]++;
1314 load_count[1 + port]++;
1315 DP(NETIF_MSG_IFUP, "NO MCP - new load counts %d, %d, %d\n",
1316 load_count[0], load_count[1], load_count[2]);
1317 if (load_count[0] == 1)
1318 load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
1319 else if (load_count[1 + port] == 1)
1320 load_code = FW_MSG_CODE_DRV_LOAD_PORT;
1321 else
1322 load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
1323 }
1324
1325 if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1326 (load_code == FW_MSG_CODE_DRV_LOAD_PORT))
1327 bp->port.pmf = 1;
1328 else
1329 bp->port.pmf = 0;
1330 DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
1331
1332 /* Initialize HW */
1333 rc = bnx2x_init_hw(bp, load_code);
1334 if (rc) {
1335 BNX2X_ERR("HW init failed, aborting\n");
a22f0788
YR		 1336 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1337 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
1338 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
9f6c9258
DK		 1339 goto load_error2;
1340 }
1341
1342 /* Setup NIC internals and enable interrupts */
1343 bnx2x_nic_init(bp, load_code);
1344
1345 if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) &&
1346 (bp->common.shmem2_base))
1347 SHMEM2_WR(bp, dcc_support,
1348 (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
1349 SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
1350
1351 /* Send LOAD_DONE command to MCP */
1352 if (!BP_NOMCP(bp)) {
a22f0788 1353 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
9f6c9258
DK		 1354 if (!load_code) {
1355 BNX2X_ERR("MCP response failure, aborting\n");
1356 rc = -EBUSY;
1357 goto load_error3;
1358 }
1359 }
1360
1361 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
1362
1363 rc = bnx2x_setup_leading(bp);
1364 if (rc) {
1365 BNX2X_ERR("Setup leading failed!\n");
1366#ifndef BNX2X_STOP_ON_ERROR
1367 goto load_error3;
1368#else
1369 bp->panic = 1;
1370 return -EBUSY;
1371#endif
1372 }
1373
1374 if (CHIP_IS_E1H(bp))
1375 if (bp->mf_config & FUNC_MF_CFG_FUNC_DISABLED) {
1376 DP(NETIF_MSG_IFUP, "mf_cfg function disabled\n");
1377 bp->flags |= MF_FUNC_DIS;
1378 }
1379
1380 if (bp->state == BNX2X_STATE_OPEN) {
1381#ifdef BCM_CNIC
1382 /* Enable Timer scan */
1383 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 1);
1384#endif
1385 for_each_nondefault_queue(bp, i) {
1386 rc = bnx2x_setup_multi(bp, i);
1387 if (rc)
1388#ifdef BCM_CNIC
1389 goto load_error4;
1390#else
1391 goto load_error3;
1392#endif
1393 }
1394
1395 if (CHIP_IS_E1(bp))
1396 bnx2x_set_eth_mac_addr_e1(bp, 1);
1397 else
1398 bnx2x_set_eth_mac_addr_e1h(bp, 1);
1399#ifdef BCM_CNIC
1400 /* Set iSCSI L2 MAC */
1401 mutex_lock(&bp->cnic_mutex);
1402 if (bp->cnic_eth_dev.drv_state & CNIC_DRV_STATE_REGD) {
1403 bnx2x_set_iscsi_eth_mac_addr(bp, 1);
1404 bp->cnic_flags |= BNX2X_CNIC_FLAG_MAC_SET;
1405 bnx2x_init_sb(bp, bp->cnic_sb, bp->cnic_sb_mapping,
1406 CNIC_SB_ID(bp));
1407 }
1408 mutex_unlock(&bp->cnic_mutex);
1409#endif
1410 }
1411
1412 if (bp->port.pmf)
1413 bnx2x_initial_phy_init(bp, load_mode);
1414
1415 /* Start fast path */
1416 switch (load_mode) {
1417 case LOAD_NORMAL:
1418 if (bp->state == BNX2X_STATE_OPEN) {
1419 /* Tx queue should be only reenabled */
1420 netif_tx_wake_all_queues(bp->dev);
1421 }
1422 /* Initialize the receive filter. */
1423 bnx2x_set_rx_mode(bp->dev);
1424 break;
1425
1426 case LOAD_OPEN:
1427 netif_tx_start_all_queues(bp->dev);
1428 if (bp->state != BNX2X_STATE_OPEN)
1429 netif_tx_disable(bp->dev);
1430 /* Initialize the receive filter. */
1431 bnx2x_set_rx_mode(bp->dev);
1432 break;
1433
1434 case LOAD_DIAG:
1435 /* Initialize the receive filter. */
1436 bnx2x_set_rx_mode(bp->dev);
1437 bp->state = BNX2X_STATE_DIAG;
1438 break;
1439
1440 default:
1441 break;
1442 }
1443
1444 if (!bp->port.pmf)
1445 bnx2x__link_status_update(bp);
1446
1447 /* start the timer */
1448 mod_timer(&bp->timer, jiffies + bp->current_interval);
1449
1450#ifdef BCM_CNIC
1451 bnx2x_setup_cnic_irq_info(bp);
1452 if (bp->state == BNX2X_STATE_OPEN)
1453 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
1454#endif
1455 bnx2x_inc_load_cnt(bp);
1456
6891dd25
DK		 1457 bnx2x_release_firmware(bp);
1458
9f6c9258
DK		 1459 return 0;
1460
1461#ifdef BCM_CNIC
1462load_error4:
1463 /* Disable Timer scan */
1464 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 0);
1465#endif
1466load_error3:
1467 bnx2x_int_disable_sync(bp, 1);
1468 if (!BP_NOMCP(bp)) {
a22f0788
YR		 1469 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
1470 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
9f6c9258
DK		 1471 }
1472 bp->port.pmf = 0;
1473 /* Free SKBs, SGEs, TPA pool and driver internals */
1474 bnx2x_free_skbs(bp);
1475 for_each_queue(bp, i)
1476 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
1477load_error2:
1478 /* Release IRQs */
1479 bnx2x_free_irq(bp, false);
1480load_error1:
1481 bnx2x_napi_disable(bp);
1482 for_each_queue(bp, i)
1483 netif_napi_del(&bnx2x_fp(bp, i, napi));
1484 bnx2x_free_mem(bp);
1485
6891dd25
DK		 1486 bnx2x_release_firmware(bp);
1487
9f6c9258
DK		 1488 return rc;
1489}
1490
1491/* must be called with rtnl_lock */
1492int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode)
1493{
1494 int i;
1495
1496 if (bp->state == BNX2X_STATE_CLOSED) {
1497 /* Interface has been removed - nothing to recover */
1498 bp->recovery_state = BNX2X_RECOVERY_DONE;
1499 bp->is_leader = 0;
1500 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_RESERVED_08);
1501 smp_wmb();
1502
1503 return -EINVAL;
1504 }
1505
1506#ifdef BCM_CNIC
1507 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
1508#endif
1509 bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
1510
1511 /* Set "drop all" */
1512 bp->rx_mode = BNX2X_RX_MODE_NONE;
1513 bnx2x_set_storm_rx_mode(bp);
1514
1515 /* Disable HW interrupts, NAPI and Tx */
1516 bnx2x_netif_stop(bp, 1);
1517 netif_carrier_off(bp->dev);
1518
1519 del_timer_sync(&bp->timer);
1520 SHMEM_WR(bp, func_mb[BP_FUNC(bp)].drv_pulse_mb,
1521 (DRV_PULSE_ALWAYS_ALIVE | bp->fw_drv_pulse_wr_seq));
1522 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
1523
1524 /* Release IRQs */
1525 bnx2x_free_irq(bp, false);
1526
1527 /* Cleanup the chip if needed */
1528 if (unload_mode != UNLOAD_RECOVERY)
1529 bnx2x_chip_cleanup(bp, unload_mode);
1530
1531 bp->port.pmf = 0;
1532
1533 /* Free SKBs, SGEs, TPA pool and driver internals */
1534 bnx2x_free_skbs(bp);
1535 for_each_queue(bp, i)
1536 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
1537 for_each_queue(bp, i)
1538 netif_napi_del(&bnx2x_fp(bp, i, napi));
1539 bnx2x_free_mem(bp);
1540
1541 bp->state = BNX2X_STATE_CLOSED;
1542
1543 /* The last driver must disable a "close the gate" if there is no
1544 * parity attention or "process kill" pending.
1545 */
1546 if ((!bnx2x_dec_load_cnt(bp)) && (!bnx2x_chk_parity_attn(bp)) &&
1547 bnx2x_reset_is_done(bp))
1548 bnx2x_disable_close_the_gate(bp);
1549
1550 /* Reset MCP mail box sequence if there is on going recovery */
1551 if (unload_mode == UNLOAD_RECOVERY)
1552 bp->fw_seq = 0;
1553
1554 return 0;
1555}
1556int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
1557{
1558 u16 pmcsr;
1559
1560 pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
1561
1562 switch (state) {
1563 case PCI_D0:
1564 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
1565 ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
1566 PCI_PM_CTRL_PME_STATUS));
1567
1568 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
1569 /* delay required during transition out of D3hot */
1570 msleep(20);
1571 break;
1572
1573 case PCI_D3hot:
1574 /* If there are other clients above don't
1575 shut down the power */
1576 if (atomic_read(&bp->pdev->enable_cnt) != 1)
1577 return 0;
1578 /* Don't shut down the power for emulation and FPGA */
1579 if (CHIP_REV_IS_SLOW(bp))
1580 return 0;
1581
1582 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
1583 pmcsr |= 3;
1584
1585 if (bp->wol)
1586 pmcsr |= PCI_PM_CTRL_PME_ENABLE;
1587
1588 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
1589 pmcsr);
1590
1591 /* No more memory access after this point until
1592 * device is brought back to D0.
1593 */
1594 break;
1595
1596 default:
1597 return -EINVAL;
1598 }
1599 return 0;
1600}
1601
1602
1603
1604/*
1605 * net_device service functions
1606 */
1607
1608static int bnx2x_poll(struct napi_struct *napi, int budget)
1609{
1610 int work_done = 0;
1611 struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
1612 napi);
1613 struct bnx2x *bp = fp->bp;
1614
1615 while (1) {
1616#ifdef BNX2X_STOP_ON_ERROR
1617 if (unlikely(bp->panic)) {
1618 napi_complete(napi);
1619 return 0;
1620 }
1621#endif
1622
1623 if (bnx2x_has_tx_work(fp))
1624 bnx2x_tx_int(fp);
1625
1626 if (bnx2x_has_rx_work(fp)) {
1627 work_done += bnx2x_rx_int(fp, budget - work_done);
1628
1629 /* must not complete if we consumed full budget */
1630 if (work_done >= budget)
1631 break;
1632 }
1633
1634 /* Fall out from the NAPI loop if needed */
1635 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
1636 bnx2x_update_fpsb_idx(fp);
1637 /* bnx2x_has_rx_work() reads the status block, thus we need
1638 * to ensure that status block indices have been actually read
1639 * (bnx2x_update_fpsb_idx) prior to this check
1640 * (bnx2x_has_rx_work) so that we won't write the "newer"
1641 * value of the status block to IGU (if there was a DMA right
1642 * after bnx2x_has_rx_work and if there is no rmb, the memory
1643 * reading (bnx2x_update_fpsb_idx) may be postponed to right
1644 * before bnx2x_ack_sb). In this case there will never be
1645 * another interrupt until there is another update of the
1646 * status block, while there is still unhandled work.
1647 */
1648 rmb();
1649
1650 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
1651 napi_complete(napi);
1652 /* Re-enable interrupts */
1653 bnx2x_ack_sb(bp, fp->sb_id, CSTORM_ID,
1654 le16_to_cpu(fp->fp_c_idx),
1655 IGU_INT_NOP, 1);
1656 bnx2x_ack_sb(bp, fp->sb_id, USTORM_ID,
1657 le16_to_cpu(fp->fp_u_idx),
1658 IGU_INT_ENABLE, 1);
1659 break;
1660 }
1661 }
1662 }
1663
1664 return work_done;
1665}
1666
1667
1668/* we split the first BD into headers and data BDs
1669 * to ease the pain of our fellow microcode engineers
1670 * we use one mapping for both BDs
1671 * So far this has only been observed to happen
1672 * in Other Operating Systems(TM)
1673 */
1674static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
1675 struct bnx2x_fastpath *fp,
1676 struct sw_tx_bd *tx_buf,
1677 struct eth_tx_start_bd **tx_bd, u16 hlen,
1678 u16 bd_prod, int nbd)
1679{
1680 struct eth_tx_start_bd *h_tx_bd = *tx_bd;
1681 struct eth_tx_bd *d_tx_bd;
1682 dma_addr_t mapping;
1683 int old_len = le16_to_cpu(h_tx_bd->nbytes);
1684
1685 /* first fix first BD */
1686 h_tx_bd->nbd = cpu_to_le16(nbd);
1687 h_tx_bd->nbytes = cpu_to_le16(hlen);
1688
1689 DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d "
1690 "(%x:%x) nbd %d\n", h_tx_bd->nbytes, h_tx_bd->addr_hi,
1691 h_tx_bd->addr_lo, h_tx_bd->nbd);
1692
1693 /* now get a new data BD
1694 * (after the pbd) and fill it */
1695 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
1696 d_tx_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
1697
1698 mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
1699 le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
1700
1701 d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
1702 d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
1703 d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
1704
1705 /* this marks the BD as one that has no individual mapping */
1706 tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
1707
1708 DP(NETIF_MSG_TX_QUEUED,
1709 "TSO split data size is %d (%x:%x)\n",
1710 d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
1711
1712 /* update tx_bd */
1713 *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
1714
1715 return bd_prod;
1716}
1717
1718static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
1719{
1720 if (fix > 0)
1721 csum = (u16) ~csum_fold(csum_sub(csum,
1722 csum_partial(t_header - fix, fix, 0)));
1723
1724 else if (fix < 0)
1725 csum = (u16) ~csum_fold(csum_add(csum,
1726 csum_partial(t_header, -fix, 0)));
1727
1728 return swab16(csum);
1729}
1730
1731static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
1732{
1733 u32 rc;
1734
1735 if (skb->ip_summed != CHECKSUM_PARTIAL)
1736 rc = XMIT_PLAIN;
1737
1738 else {
1739 if (skb->protocol == htons(ETH_P_IPV6)) {
1740 rc = XMIT_CSUM_V6;
1741 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
1742 rc |= XMIT_CSUM_TCP;
1743
1744 } else {
1745 rc = XMIT_CSUM_V4;
1746 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
1747 rc |= XMIT_CSUM_TCP;
1748 }
1749 }
1750
1751 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
1752 rc |= (XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP);
1753
1754 else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
1755 rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6);
1756
1757 return rc;
1758}
1759
1760#if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
1761/* check if packet requires linearization (packet is too fragmented)
1762 no need to check fragmentation if page size > 8K (there will be no
1763 violation to FW restrictions) */
1764static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
1765 u32 xmit_type)
1766{
1767 int to_copy = 0;
1768 int hlen = 0;
1769 int first_bd_sz = 0;
1770
1771 /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
1772 if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
1773
1774 if (xmit_type & XMIT_GSO) {
1775 unsigned short lso_mss = skb_shinfo(skb)->gso_size;
1776 /* Check if LSO packet needs to be copied:
1777 3 = 1 (for headers BD) + 2 (for PBD and last BD) */
1778 int wnd_size = MAX_FETCH_BD - 3;
1779 /* Number of windows to check */
1780 int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
1781 int wnd_idx = 0;
1782 int frag_idx = 0;
1783 u32 wnd_sum = 0;
1784
1785 /* Headers length */
1786 hlen = (int)(skb_transport_header(skb) - skb->data) +
1787 tcp_hdrlen(skb);
1788
1789 /* Amount of data (w/o headers) on linear part of SKB*/
1790 first_bd_sz = skb_headlen(skb) - hlen;
1791
1792 wnd_sum = first_bd_sz;
1793
1794 /* Calculate the first sum - it's special */
1795 for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
1796 wnd_sum +=
1797 skb_shinfo(skb)->frags[frag_idx].size;
1798
1799 /* If there was data on linear skb data - check it */
1800 if (first_bd_sz > 0) {
1801 if (unlikely(wnd_sum < lso_mss)) {
1802 to_copy = 1;
1803 goto exit_lbl;
1804 }
1805
1806 wnd_sum -= first_bd_sz;
1807 }
1808
1809 /* Others are easier: run through the frag list and
1810 check all windows */
1811 for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
1812 wnd_sum +=
1813 skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1].size;
1814
1815 if (unlikely(wnd_sum < lso_mss)) {
1816 to_copy = 1;
1817 break;
1818 }
1819 wnd_sum -=
1820 skb_shinfo(skb)->frags[wnd_idx].size;
1821 }
1822 } else {
1823 /* in non-LSO too fragmented packet should always
1824 be linearized */
1825 to_copy = 1;
1826 }
1827 }
1828
1829exit_lbl:
1830 if (unlikely(to_copy))
1831 DP(NETIF_MSG_TX_QUEUED,
1832 "Linearization IS REQUIRED for %s packet. "
1833 "num_frags %d hlen %d first_bd_sz %d\n",
1834 (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
1835 skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
1836
1837 return to_copy;
1838}
1839#endif
1840
1841/* called with netif_tx_lock
1842 * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
1843 * netif_wake_queue()
1844 */
1845netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
1846{
1847 struct bnx2x *bp = netdev_priv(dev);
1848 struct bnx2x_fastpath *fp;
1849 struct netdev_queue *txq;
1850 struct sw_tx_bd *tx_buf;
1851 struct eth_tx_start_bd *tx_start_bd;
1852 struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
1853 struct eth_tx_parse_bd *pbd = NULL;
1854 u16 pkt_prod, bd_prod;
1855 int nbd, fp_index;
1856 dma_addr_t mapping;
1857 u32 xmit_type = bnx2x_xmit_type(bp, skb);
1858 int i;
1859 u8 hlen = 0;
1860 __le16 pkt_size = 0;
1861 struct ethhdr *eth;
1862 u8 mac_type = UNICAST_ADDRESS;
1863
1864#ifdef BNX2X_STOP_ON_ERROR
1865 if (unlikely(bp->panic))
1866 return NETDEV_TX_BUSY;
1867#endif
1868
1869 fp_index = skb_get_queue_mapping(skb);
1870 txq = netdev_get_tx_queue(dev, fp_index);
1871
1872 fp = &bp->fp[fp_index];
1873
1874 if (unlikely(bnx2x_tx_avail(fp) < (skb_shinfo(skb)->nr_frags + 3))) {
1875 fp->eth_q_stats.driver_xoff++;
1876 netif_tx_stop_queue(txq);
1877 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
1878 return NETDEV_TX_BUSY;
1879 }
1880
1881 DP(NETIF_MSG_TX_QUEUED, "SKB: summed %x protocol %x protocol(%x,%x)"
1882 " gso type %x xmit_type %x\n",
1883 skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
1884 ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type);
1885
1886 eth = (struct ethhdr *)skb->data;
1887
1888 /* set flag according to packet type (UNICAST_ADDRESS is default)*/
1889 if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
1890 if (is_broadcast_ether_addr(eth->h_dest))
1891 mac_type = BROADCAST_ADDRESS;
1892 else
1893 mac_type = MULTICAST_ADDRESS;
1894 }
1895
1896#if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
1897 /* First, check if we need to linearize the skb (due to FW
1898 restrictions). No need to check fragmentation if page size > 8K
1899 (there will be no violation to FW restrictions) */
1900 if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
1901 /* Statistics of linearization */
1902 bp->lin_cnt++;
1903 if (skb_linearize(skb) != 0) {
1904 DP(NETIF_MSG_TX_QUEUED, "SKB linearization failed - "
1905 "silently dropping this SKB\n");
1906 dev_kfree_skb_any(skb);
1907 return NETDEV_TX_OK;
1908 }
1909 }
1910#endif
1911
1912 /*
1913 Please read carefully. First we use one BD which we mark as start,
1914 then we have a parsing info BD (used for TSO or xsum),
1915 and only then we have the rest of the TSO BDs.
1916 (don't forget to mark the last one as last,
1917 and to unmap only AFTER you write to the BD ...)
1918 And above all, all pdb sizes are in words - NOT DWORDS!
1919 */
1920
1921 pkt_prod = fp->tx_pkt_prod++;
1922 bd_prod = TX_BD(fp->tx_bd_prod);
1923
1924 /* get a tx_buf and first BD */
1925 tx_buf = &fp->tx_buf_ring[TX_BD(pkt_prod)];
1926 tx_start_bd = &fp->tx_desc_ring[bd_prod].start_bd;
1927
1928 tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
1929 tx_start_bd->general_data = (mac_type <<
1930 ETH_TX_START_BD_ETH_ADDR_TYPE_SHIFT);
1931 /* header nbd */
1932 tx_start_bd->general_data |= (1 << ETH_TX_START_BD_HDR_NBDS_SHIFT);
1933
1934 /* remember the first BD of the packet */
1935 tx_buf->first_bd = fp->tx_bd_prod;
1936 tx_buf->skb = skb;
1937 tx_buf->flags = 0;
1938
1939 DP(NETIF_MSG_TX_QUEUED,
1940 "sending pkt %u @%p next_idx %u bd %u @%p\n",
1941 pkt_prod, tx_buf, fp->tx_pkt_prod, bd_prod, tx_start_bd);
1942
1943#ifdef BCM_VLAN
1944 if ((bp->vlgrp != NULL) && vlan_tx_tag_present(skb) &&
1945 (bp->flags & HW_VLAN_TX_FLAG)) {
1946 tx_start_bd->vlan = cpu_to_le16(vlan_tx_tag_get(skb));
1947 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_VLAN_TAG;
1948 } else
1949#endif
1950 tx_start_bd->vlan = cpu_to_le16(pkt_prod);
1951
1952 /* turn on parsing and get a BD */
1953 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
1954 pbd = &fp->tx_desc_ring[bd_prod].parse_bd;
1955
1956 memset(pbd, 0, sizeof(struct eth_tx_parse_bd));
1957
1958 if (xmit_type & XMIT_CSUM) {
1959 hlen = (skb_network_header(skb) - skb->data) / 2;
1960
1961 /* for now NS flag is not used in Linux */
1962 pbd->global_data =
1963 (hlen | ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
1964 ETH_TX_PARSE_BD_LLC_SNAP_EN_SHIFT));
1965
1966 pbd->ip_hlen = (skb_transport_header(skb) -
1967 skb_network_header(skb)) / 2;
1968
1969 hlen += pbd->ip_hlen + tcp_hdrlen(skb) / 2;
1970
1971 pbd->total_hlen = cpu_to_le16(hlen);
1972 hlen = hlen*2;
1973
1974 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
1975
1976 if (xmit_type & XMIT_CSUM_V4)
1977 tx_start_bd->bd_flags.as_bitfield |=
1978 ETH_TX_BD_FLAGS_IP_CSUM;
1979 else
1980 tx_start_bd->bd_flags.as_bitfield |=
1981 ETH_TX_BD_FLAGS_IPV6;
1982
1983 if (xmit_type & XMIT_CSUM_TCP) {
1984 pbd->tcp_pseudo_csum = swab16(tcp_hdr(skb)->check);
1985
1986 } else {
1987 s8 fix = SKB_CS_OFF(skb); /* signed! */
1988
1989 pbd->global_data |= ETH_TX_PARSE_BD_UDP_CS_FLG;
1990
1991 DP(NETIF_MSG_TX_QUEUED,
1992 "hlen %d fix %d csum before fix %x\n",
1993 le16_to_cpu(pbd->total_hlen), fix, SKB_CS(skb));
1994
1995 /* HW bug: fixup the CSUM */
1996 pbd->tcp_pseudo_csum =
1997 bnx2x_csum_fix(skb_transport_header(skb),
1998 SKB_CS(skb), fix);
1999
2000 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
2001 pbd->tcp_pseudo_csum);
2002 }
2003 }
2004
2005 mapping = dma_map_single(&bp->pdev->dev, skb->data,
2006 skb_headlen(skb), DMA_TO_DEVICE);
2007
2008 tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2009 tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2010 nbd = skb_shinfo(skb)->nr_frags + 2; /* start_bd + pbd + frags */
2011 tx_start_bd->nbd = cpu_to_le16(nbd);
2012 tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
2013 pkt_size = tx_start_bd->nbytes;
2014
2015 DP(NETIF_MSG_TX_QUEUED, "first bd @%p addr (%x:%x) nbd %d"
2016 " nbytes %d flags %x vlan %x\n",
2017 tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
2018 le16_to_cpu(tx_start_bd->nbd), le16_to_cpu(tx_start_bd->nbytes),
2019 tx_start_bd->bd_flags.as_bitfield, le16_to_cpu(tx_start_bd->vlan));
2020
2021 if (xmit_type & XMIT_GSO) {
2022
2023 DP(NETIF_MSG_TX_QUEUED,
2024 "TSO packet len %d hlen %d total len %d tso size %d\n",
2025 skb->len, hlen, skb_headlen(skb),
2026 skb_shinfo(skb)->gso_size);
2027
2028 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
2029
2030 if (unlikely(skb_headlen(skb) > hlen))
2031 bd_prod = bnx2x_tx_split(bp, fp, tx_buf, &tx_start_bd,
2032 hlen, bd_prod, ++nbd);
2033
2034 pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
2035 pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
2036 pbd->tcp_flags = pbd_tcp_flags(skb);
2037
2038 if (xmit_type & XMIT_GSO_V4) {
2039 pbd->ip_id = swab16(ip_hdr(skb)->id);
2040 pbd->tcp_pseudo_csum =
2041 swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
2042 ip_hdr(skb)->daddr,
2043 0, IPPROTO_TCP, 0));
2044
2045 } else
2046 pbd->tcp_pseudo_csum =
2047 swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2048 &ipv6_hdr(skb)->daddr,
2049 0, IPPROTO_TCP, 0));
2050
2051 pbd->global_data |= ETH_TX_PARSE_BD_PSEUDO_CS_WITHOUT_LEN;
2052 }
2053 tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
2054
2055 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2056 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2057
2058 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2059 tx_data_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
2060 if (total_pkt_bd == NULL)
2061 total_pkt_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
2062
2063 mapping = dma_map_page(&bp->pdev->dev, frag->page,
2064 frag->page_offset,
2065 frag->size, DMA_TO_DEVICE);
2066
2067 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2068 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2069 tx_data_bd->nbytes = cpu_to_le16(frag->size);
2070 le16_add_cpu(&pkt_size, frag->size);
2071
2072 DP(NETIF_MSG_TX_QUEUED,
2073 "frag %d bd @%p addr (%x:%x) nbytes %d\n",
2074 i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
2075 le16_to_cpu(tx_data_bd->nbytes));
2076 }
2077
2078 DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
2079
2080 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2081
2082 /* now send a tx doorbell, counting the next BD
2083 * if the packet contains or ends with it
2084 */
2085 if (TX_BD_POFF(bd_prod) < nbd)
2086 nbd++;
2087
2088 if (total_pkt_bd != NULL)
2089 total_pkt_bd->total_pkt_bytes = pkt_size;
2090
2091 if (pbd)
2092 DP(NETIF_MSG_TX_QUEUED,
2093 "PBD @%p ip_data %x ip_hlen %u ip_id %u lso_mss %u"
2094 " tcp_flags %x xsum %x seq %u hlen %u\n",
2095 pbd, pbd->global_data, pbd->ip_hlen, pbd->ip_id,
2096 pbd->lso_mss, pbd->tcp_flags, pbd->tcp_pseudo_csum,
2097 pbd->tcp_send_seq, le16_to_cpu(pbd->total_hlen));
2098
2099 DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d bd %u\n", nbd, bd_prod);
2100
2101 /*
2102 * Make sure that the BD data is updated before updating the producer
2103 * since FW might read the BD right after the producer is updated.
2104 * This is only applicable for weak-ordered memory model archs such
2105 * as IA-64. The following barrier is also mandatory since FW will
2106 * assumes packets must have BDs.
2107 */
2108 wmb();
2109
2110 fp->tx_db.data.prod += nbd;
2111 barrier();
2112 DOORBELL(bp, fp->index, fp->tx_db.raw);
2113
2114 mmiowb();
2115
2116 fp->tx_bd_prod += nbd;
2117
2118 if (unlikely(bnx2x_tx_avail(fp) < MAX_SKB_FRAGS + 3)) {
2119 netif_tx_stop_queue(txq);
2120
2121 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
2122 * ordering of set_bit() in netif_tx_stop_queue() and read of
2123 * fp->bd_tx_cons */
2124 smp_mb();
2125
2126 fp->eth_q_stats.driver_xoff++;
2127 if (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3)
2128 netif_tx_wake_queue(txq);
2129 }
2130 fp->tx_pkt++;
2131
2132 return NETDEV_TX_OK;
2133}
2134/* called with rtnl_lock */
2135int bnx2x_change_mac_addr(struct net_device *dev, void *p)
2136{
2137 struct sockaddr *addr = p;
2138 struct bnx2x *bp = netdev_priv(dev);
2139
2140 if (!is_valid_ether_addr((u8 *)(addr->sa_data)))
2141 return -EINVAL;
2142
2143 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2144 if (netif_running(dev)) {
2145 if (CHIP_IS_E1(bp))
2146 bnx2x_set_eth_mac_addr_e1(bp, 1);
2147 else
2148 bnx2x_set_eth_mac_addr_e1h(bp, 1);
2149 }
2150
2151 return 0;
2152}
2153
2154/* called with rtnl_lock */
2155int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
2156{
2157 struct bnx2x *bp = netdev_priv(dev);
2158 int rc = 0;
2159
2160 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
2161 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
2162 return -EAGAIN;
2163 }
2164
2165 if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
2166 ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE))
2167 return -EINVAL;
2168
2169 /* This does not race with packet allocation
2170 * because the actual alloc size is
2171 * only updated as part of load
2172 */
2173 dev->mtu = new_mtu;
2174
2175 if (netif_running(dev)) {
2176 bnx2x_nic_unload(bp, UNLOAD_NORMAL);
2177 rc = bnx2x_nic_load(bp, LOAD_NORMAL);
2178 }
2179
2180 return rc;
2181}
2182
2183void bnx2x_tx_timeout(struct net_device *dev)
2184{
2185 struct bnx2x *bp = netdev_priv(dev);
2186
2187#ifdef BNX2X_STOP_ON_ERROR
2188 if (!bp->panic)
2189 bnx2x_panic();
2190#endif
2191 /* This allows the netif to be shutdown gracefully before resetting */
2192 schedule_delayed_work(&bp->reset_task, 0);
2193}
2194
2195#ifdef BCM_VLAN
2196/* called with rtnl_lock */
2197void bnx2x_vlan_rx_register(struct net_device *dev,
2198 struct vlan_group *vlgrp)
2199{
2200 struct bnx2x *bp = netdev_priv(dev);
2201
2202 bp->vlgrp = vlgrp;
2203
2204 /* Set flags according to the required capabilities */
2205 bp->flags &= ~(HW_VLAN_RX_FLAG | HW_VLAN_TX_FLAG);
2206
2207 if (dev->features & NETIF_F_HW_VLAN_TX)
2208 bp->flags |= HW_VLAN_TX_FLAG;
2209
2210 if (dev->features & NETIF_F_HW_VLAN_RX)
2211 bp->flags |= HW_VLAN_RX_FLAG;
2212
2213 if (netif_running(dev))
2214 bnx2x_set_client_config(bp);
2215}
2216
2217#endif
2218int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
2219{
2220 struct net_device *dev = pci_get_drvdata(pdev);
2221 struct bnx2x *bp;
2222
2223 if (!dev) {
2224 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
2225 return -ENODEV;
2226 }
2227 bp = netdev_priv(dev);
2228
2229 rtnl_lock();
2230
2231 pci_save_state(pdev);
2232
2233 if (!netif_running(dev)) {
2234 rtnl_unlock();
2235 return 0;
2236 }
2237
2238 netif_device_detach(dev);
2239
2240 bnx2x_nic_unload(bp, UNLOAD_CLOSE);
2241
2242 bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
2243
2244 rtnl_unlock();
2245
2246 return 0;
2247}
2248
2249int bnx2x_resume(struct pci_dev *pdev)
2250{
2251 struct net_device *dev = pci_get_drvdata(pdev);
2252 struct bnx2x *bp;
2253 int rc;
2254
2255 if (!dev) {
2256 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
2257 return -ENODEV;
2258 }
2259 bp = netdev_priv(dev);
2260
2261 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
2262 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
2263 return -EAGAIN;
2264 }
2265
2266 rtnl_lock();
2267
2268 pci_restore_state(pdev);
2269
2270 if (!netif_running(dev)) {
2271 rtnl_unlock();
2272 return 0;
2273 }
2274
2275 bnx2x_set_power_state(bp, PCI_D0);
2276 netif_device_attach(dev);
2277
2278 rc = bnx2x_nic_load(bp, LOAD_OPEN);
2279
2280 rtnl_unlock();
2281
2282 return rc;
2283}
Linux 6.x block layer and io_uring tree(s)