projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ks8842: Fix warnings on 64-bit.
[linux-2.6-block.git] / drivers / net / bnx2x / bnx2x_cmn.c
CommitLineData
9f6c9258
DK		 1/* bnx2x_cmn.c: Broadcom Everest network driver.
2 *
3 * Copyright (c) 2007-2010 Broadcom Corporation
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation.
8 *
9 * Maintained by: Eilon Greenstein <eilong@broadcom.com>
10 * Written by: Eliezer Tamir
11 * Based on code from Michael Chan's bnx2 driver
12 * UDP CSUM errata workaround by Arik Gendelman
13 * Slowpath and fastpath rework by Vladislav Zolotarov
14 * Statistics and Link management by Yitchak Gertner
15 *
16 */
17
18
19#include <linux/etherdevice.h>
20#include <linux/ip.h>
21#include <linux/ipv6.h>
22#include "bnx2x_cmn.h"
23
24#ifdef BCM_VLAN
25#include <linux/if_vlan.h>
26#endif
27
28static int bnx2x_poll(struct napi_struct *napi, int budget);
29
30/* free skb in the packet ring at pos idx
31 * return idx of last bd freed
32 */
33static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fastpath *fp,
34 u16 idx)
35{
36 struct sw_tx_bd *tx_buf = &fp->tx_buf_ring[idx];
37 struct eth_tx_start_bd *tx_start_bd;
38 struct eth_tx_bd *tx_data_bd;
39 struct sk_buff *skb = tx_buf->skb;
40 u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
41 int nbd;
42
43 /* prefetch skb end pointer to speedup dev_kfree_skb() */
44 prefetch(&skb->end);
45
46 DP(BNX2X_MSG_OFF, "pkt_idx %d buff @(%p)->skb %p\n",
47 idx, tx_buf, skb);
48
49 /* unmap first bd */
50 DP(BNX2X_MSG_OFF, "free bd_idx %d\n", bd_idx);
51 tx_start_bd = &fp->tx_desc_ring[bd_idx].start_bd;
52 dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
53 BD_UNMAP_LEN(tx_start_bd), PCI_DMA_TODEVICE);
54
55 nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
56#ifdef BNX2X_STOP_ON_ERROR
57 if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
58 BNX2X_ERR("BAD nbd!\n");
59 bnx2x_panic();
60 }
61#endif
62 new_cons = nbd + tx_buf->first_bd;
63
64 /* Get the next bd */
65 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
66
67 /* Skip a parse bd... */
68 --nbd;
69 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
70
71 /* ...and the TSO split header bd since they have no mapping */
72 if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
73 --nbd;
74 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
75 }
76
77 /* now free frags */
78 while (nbd > 0) {
79
80 DP(BNX2X_MSG_OFF, "free frag bd_idx %d\n", bd_idx);
81 tx_data_bd = &fp->tx_desc_ring[bd_idx].reg_bd;
82 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
83 BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
84 if (--nbd)
85 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
86 }
87
88 /* release skb */
89 WARN_ON(!skb);
90 dev_kfree_skb(skb);
91 tx_buf->first_bd = 0;
92 tx_buf->skb = NULL;
93
94 return new_cons;
95}
96
97int bnx2x_tx_int(struct bnx2x_fastpath *fp)
98{
99 struct bnx2x *bp = fp->bp;
100 struct netdev_queue *txq;
101 u16 hw_cons, sw_cons, bd_cons = fp->tx_bd_cons;
102
103#ifdef BNX2X_STOP_ON_ERROR
104 if (unlikely(bp->panic))
105 return -1;
106#endif
107
108 txq = netdev_get_tx_queue(bp->dev, fp->index);
109 hw_cons = le16_to_cpu(*fp->tx_cons_sb);
110 sw_cons = fp->tx_pkt_cons;
111
112 while (sw_cons != hw_cons) {
113 u16 pkt_cons;
114
115 pkt_cons = TX_BD(sw_cons);
116
117 /* prefetch(bp->tx_buf_ring[pkt_cons].skb); */
118
119 DP(NETIF_MSG_TX_DONE, "hw_cons %u sw_cons %u pkt_cons %u\n",
120 hw_cons, sw_cons, pkt_cons);
121
122/* if (NEXT_TX_IDX(sw_cons) != hw_cons) {
123 rmb();
124 prefetch(fp->tx_buf_ring[NEXT_TX_IDX(sw_cons)].skb);
125 }
126*/
127 bd_cons = bnx2x_free_tx_pkt(bp, fp, pkt_cons);
128 sw_cons++;
129 }
130
131 fp->tx_pkt_cons = sw_cons;
132 fp->tx_bd_cons = bd_cons;
133
134 /* Need to make the tx_bd_cons update visible to start_xmit()
135 * before checking for netif_tx_queue_stopped(). Without the
136 * memory barrier, there is a small possibility that
137 * start_xmit() will miss it and cause the queue to be stopped
138 * forever.
139 */
140 smp_mb();
141
142 /* TBD need a thresh? */
143 if (unlikely(netif_tx_queue_stopped(txq))) {
144 /* Taking tx_lock() is needed to prevent reenabling the queue
145 * while it's empty. This could have happen if rx_action() gets
146 * suspended in bnx2x_tx_int() after the condition before
147 * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
148 *
149 * stops the queue->sees fresh tx_bd_cons->releases the queue->
150 * sends some packets consuming the whole queue again->
151 * stops the queue
152 */
153
154 __netif_tx_lock(txq, smp_processor_id());
155
156 if ((netif_tx_queue_stopped(txq)) &&
157 (bp->state == BNX2X_STATE_OPEN) &&
158 (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3))
159 netif_tx_wake_queue(txq);
160
161 __netif_tx_unlock(txq);
162 }
163 return 0;
164}
165
166static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
167 u16 idx)
168{
169 u16 last_max = fp->last_max_sge;
170
171 if (SUB_S16(idx, last_max) > 0)
172 fp->last_max_sge = idx;
173}
174
175static void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
176 struct eth_fast_path_rx_cqe *fp_cqe)
177{
178 struct bnx2x *bp = fp->bp;
179 u16 sge_len = SGE_PAGE_ALIGN(le16_to_cpu(fp_cqe->pkt_len) -
180 le16_to_cpu(fp_cqe->len_on_bd)) >>
181 SGE_PAGE_SHIFT;
182 u16 last_max, last_elem, first_elem;
183 u16 delta = 0;
184 u16 i;
185
186 if (!sge_len)
187 return;
188
189 /* First mark all used pages */
190 for (i = 0; i < sge_len; i++)
191 SGE_MASK_CLEAR_BIT(fp, RX_SGE(le16_to_cpu(fp_cqe->sgl[i])));
192
193 DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
194 sge_len - 1, le16_to_cpu(fp_cqe->sgl[sge_len - 1]));
195
196 /* Here we assume that the last SGE index is the biggest */
197 prefetch((void *)(fp->sge_mask));
198 bnx2x_update_last_max_sge(fp, le16_to_cpu(fp_cqe->sgl[sge_len - 1]));
199
200 last_max = RX_SGE(fp->last_max_sge);
201 last_elem = last_max >> RX_SGE_MASK_ELEM_SHIFT;
202 first_elem = RX_SGE(fp->rx_sge_prod) >> RX_SGE_MASK_ELEM_SHIFT;
203
204 /* If ring is not full */
205 if (last_elem + 1 != first_elem)
206 last_elem++;
207
208 /* Now update the prod */
209 for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
210 if (likely(fp->sge_mask[i]))
211 break;
212
213 fp->sge_mask[i] = RX_SGE_MASK_ELEM_ONE_MASK;
214 delta += RX_SGE_MASK_ELEM_SZ;
215 }
216
217 if (delta > 0) {
218 fp->rx_sge_prod += delta;
219 /* clear page-end entries */
220 bnx2x_clear_sge_mask_next_elems(fp);
221 }
222
223 DP(NETIF_MSG_RX_STATUS,
224 "fp->last_max_sge = %d fp->rx_sge_prod = %d\n",
225 fp->last_max_sge, fp->rx_sge_prod);
226}
227
228static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
229 struct sk_buff *skb, u16 cons, u16 prod)
230{
231 struct bnx2x *bp = fp->bp;
232 struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
233 struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
234 struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
235 dma_addr_t mapping;
236
237 /* move empty skb from pool to prod and map it */
238 prod_rx_buf->skb = fp->tpa_pool[queue].skb;
239 mapping = dma_map_single(&bp->pdev->dev, fp->tpa_pool[queue].skb->data,
240 bp->rx_buf_size, DMA_FROM_DEVICE);
241 dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
242
243 /* move partial skb from cons to pool (don't unmap yet) */
244 fp->tpa_pool[queue] = *cons_rx_buf;
245
246 /* mark bin state as start - print error if current state != stop */
247 if (fp->tpa_state[queue] != BNX2X_TPA_STOP)
248 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
249
250 fp->tpa_state[queue] = BNX2X_TPA_START;
251
252 /* point prod_bd to new skb */
253 prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
254 prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
255
256#ifdef BNX2X_STOP_ON_ERROR
257 fp->tpa_queue_used |= (1 << queue);
258#ifdef _ASM_GENERIC_INT_L64_H
259 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
260#else
261 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
262#endif
263 fp->tpa_queue_used);
264#endif
265}
266
267static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
268 struct sk_buff *skb,
269 struct eth_fast_path_rx_cqe *fp_cqe,
270 u16 cqe_idx)
271{
272 struct sw_rx_page *rx_pg, old_rx_pg;
273 u16 len_on_bd = le16_to_cpu(fp_cqe->len_on_bd);
274 u32 i, frag_len, frag_size, pages;
275 int err;
276 int j;
277
278 frag_size = le16_to_cpu(fp_cqe->pkt_len) - len_on_bd;
279 pages = SGE_PAGE_ALIGN(frag_size) >> SGE_PAGE_SHIFT;
280
281 /* This is needed in order to enable forwarding support */
282 if (frag_size)
283 skb_shinfo(skb)->gso_size = min((u32)SGE_PAGE_SIZE,
284 max(frag_size, (u32)len_on_bd));
285
286#ifdef BNX2X_STOP_ON_ERROR
287 if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
288 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
289 pages, cqe_idx);
290 BNX2X_ERR("fp_cqe->pkt_len = %d fp_cqe->len_on_bd = %d\n",
291 fp_cqe->pkt_len, len_on_bd);
292 bnx2x_panic();
293 return -EINVAL;
294 }
295#endif
296
297 /* Run through the SGL and compose the fragmented skb */
298 for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
299 u16 sge_idx = RX_SGE(le16_to_cpu(fp_cqe->sgl[j]));
300
301 /* FW gives the indices of the SGE as if the ring is an array
302 (meaning that "next" element will consume 2 indices) */
303 frag_len = min(frag_size, (u32)(SGE_PAGE_SIZE*PAGES_PER_SGE));
304 rx_pg = &fp->rx_page_ring[sge_idx];
305 old_rx_pg = *rx_pg;
306
307 /* If we fail to allocate a substitute page, we simply stop
308 where we are and drop the whole packet */
309 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
310 if (unlikely(err)) {
311 fp->eth_q_stats.rx_skb_alloc_failed++;
312 return err;
313 }
314
315 /* Unmap the page as we r going to pass it to the stack */
316 dma_unmap_page(&bp->pdev->dev,
317 dma_unmap_addr(&old_rx_pg, mapping),
318 SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
319
320 /* Add one frag and update the appropriate fields in the skb */
321 skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
322
323 skb->data_len += frag_len;
324 skb->truesize += frag_len;
325 skb->len += frag_len;
326
327 frag_size -= frag_len;
328 }
329
330 return 0;
331}
332
333static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
334 u16 queue, int pad, int len, union eth_rx_cqe *cqe,
335 u16 cqe_idx)
336{
337 struct sw_rx_bd *rx_buf = &fp->tpa_pool[queue];
338 struct sk_buff *skb = rx_buf->skb;
339 /* alloc new skb */
340 struct sk_buff *new_skb = netdev_alloc_skb(bp->dev, bp->rx_buf_size);
341
342 /* Unmap skb in the pool anyway, as we are going to change
343 pool entry status to BNX2X_TPA_STOP even if new skb allocation
344 fails. */
345 dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
346 bp->rx_buf_size, DMA_FROM_DEVICE);
347
348 if (likely(new_skb)) {
349 /* fix ip xsum and give it to the stack */
350 /* (no need to map the new skb) */
351#ifdef BCM_VLAN
352 int is_vlan_cqe =
353 (le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags) &
354 PARSING_FLAGS_VLAN);
355 int is_not_hwaccel_vlan_cqe =
356 (is_vlan_cqe && (!(bp->flags & HW_VLAN_RX_FLAG)));
357#endif
358
359 prefetch(skb);
360 prefetch(((char *)(skb)) + 128);
361
362#ifdef BNX2X_STOP_ON_ERROR
363 if (pad + len > bp->rx_buf_size) {
364 BNX2X_ERR("skb_put is about to fail... "
365 "pad %d len %d rx_buf_size %d\n",
366 pad, len, bp->rx_buf_size);
367 bnx2x_panic();
368 return;
369 }
370#endif
371
372 skb_reserve(skb, pad);
373 skb_put(skb, len);
374
375 skb->protocol = eth_type_trans(skb, bp->dev);
376 skb->ip_summed = CHECKSUM_UNNECESSARY;
377
378 {
379 struct iphdr *iph;
380
381 iph = (struct iphdr *)skb->data;
382#ifdef BCM_VLAN
383 /* If there is no Rx VLAN offloading -
384 take VLAN tag into an account */
385 if (unlikely(is_not_hwaccel_vlan_cqe))
386 iph = (struct iphdr *)((u8 *)iph + VLAN_HLEN);
387#endif
388 iph->check = 0;
389 iph->check = ip_fast_csum((u8 *)iph, iph->ihl);
390 }
391
392 if (!bnx2x_fill_frag_skb(bp, fp, skb,
393 &cqe->fast_path_cqe, cqe_idx)) {
394#ifdef BCM_VLAN
395 if ((bp->vlgrp != NULL) && is_vlan_cqe &&
396 (!is_not_hwaccel_vlan_cqe))
397 vlan_gro_receive(&fp->napi, bp->vlgrp,
398 le16_to_cpu(cqe->fast_path_cqe.
399 vlan_tag), skb);
400 else
401#endif
402 napi_gro_receive(&fp->napi, skb);
403 } else {
404 DP(NETIF_MSG_RX_STATUS, "Failed to allocate new pages"
405 " - dropping packet!\n");
406 dev_kfree_skb(skb);
407 }
408
409
410 /* put new skb in bin */
411 fp->tpa_pool[queue].skb = new_skb;
412
413 } else {
414 /* else drop the packet and keep the buffer in the bin */
415 DP(NETIF_MSG_RX_STATUS,
416 "Failed to allocate new skb - dropping packet!\n");
417 fp->eth_q_stats.rx_skb_alloc_failed++;
418 }
419
420 fp->tpa_state[queue] = BNX2X_TPA_STOP;
421}
422
423/* Set Toeplitz hash value in the skb using the value from the
424 * CQE (calculated by HW).
425 */
426static inline void bnx2x_set_skb_rxhash(struct bnx2x *bp, union eth_rx_cqe *cqe,
427 struct sk_buff *skb)
428{
429 /* Set Toeplitz hash from CQE */
430 if ((bp->dev->features & NETIF_F_RXHASH) &&
431 (cqe->fast_path_cqe.status_flags &
432 ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG))
433 skb->rxhash =
434 le32_to_cpu(cqe->fast_path_cqe.rss_hash_result);
435}
436
437int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
438{
439 struct bnx2x *bp = fp->bp;
440 u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
441 u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
442 int rx_pkt = 0;
443
444#ifdef BNX2X_STOP_ON_ERROR
445 if (unlikely(bp->panic))
446 return 0;
447#endif
448
449 /* CQ "next element" is of the size of the regular element,
450 that's why it's ok here */
451 hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
452 if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
453 hw_comp_cons++;
454
455 bd_cons = fp->rx_bd_cons;
456 bd_prod = fp->rx_bd_prod;
457 bd_prod_fw = bd_prod;
458 sw_comp_cons = fp->rx_comp_cons;
459 sw_comp_prod = fp->rx_comp_prod;
460
461 /* Memory barrier necessary as speculative reads of the rx
462 * buffer can be ahead of the index in the status block
463 */
464 rmb();
465
466 DP(NETIF_MSG_RX_STATUS,
467 "queue[%d]: hw_comp_cons %u sw_comp_cons %u\n",
468 fp->index, hw_comp_cons, sw_comp_cons);
469
470 while (sw_comp_cons != hw_comp_cons) {
471 struct sw_rx_bd *rx_buf = NULL;
472 struct sk_buff *skb;
473 union eth_rx_cqe *cqe;
474 u8 cqe_fp_flags;
475 u16 len, pad;
476
477 comp_ring_cons = RCQ_BD(sw_comp_cons);
478 bd_prod = RX_BD(bd_prod);
479 bd_cons = RX_BD(bd_cons);
480
481 /* Prefetch the page containing the BD descriptor
482 at producer's index. It will be needed when new skb is
483 allocated */
484 prefetch((void *)(PAGE_ALIGN((unsigned long)
485 (&fp->rx_desc_ring[bd_prod])) -
486 PAGE_SIZE + 1));
487
488 cqe = &fp->rx_comp_ring[comp_ring_cons];
489 cqe_fp_flags = cqe->fast_path_cqe.type_error_flags;
490
491 DP(NETIF_MSG_RX_STATUS, "CQE type %x err %x status %x"
492 " queue %x vlan %x len %u\n", CQE_TYPE(cqe_fp_flags),
493 cqe_fp_flags, cqe->fast_path_cqe.status_flags,
494 le32_to_cpu(cqe->fast_path_cqe.rss_hash_result),
495 le16_to_cpu(cqe->fast_path_cqe.vlan_tag),
496 le16_to_cpu(cqe->fast_path_cqe.pkt_len));
497
498 /* is this a slowpath msg? */
499 if (unlikely(CQE_TYPE(cqe_fp_flags))) {
500 bnx2x_sp_event(fp, cqe);
501 goto next_cqe;
502
503 /* this is an rx packet */
504 } else {
505 rx_buf = &fp->rx_buf_ring[bd_cons];
506 skb = rx_buf->skb;
507 prefetch(skb);
508 len = le16_to_cpu(cqe->fast_path_cqe.pkt_len);
509 pad = cqe->fast_path_cqe.placement_offset;
510
511 /* If CQE is marked both TPA_START and TPA_END
512 it is a non-TPA CQE */
513 if ((!fp->disable_tpa) &&
514 (TPA_TYPE(cqe_fp_flags) !=
515 (TPA_TYPE_START | TPA_TYPE_END))) {
516 u16 queue = cqe->fast_path_cqe.queue_index;
517
518 if (TPA_TYPE(cqe_fp_flags) == TPA_TYPE_START) {
519 DP(NETIF_MSG_RX_STATUS,
520 "calling tpa_start on queue %d\n",
521 queue);
522
523 bnx2x_tpa_start(fp, queue, skb,
524 bd_cons, bd_prod);
525
526 /* Set Toeplitz hash for an LRO skb */
527 bnx2x_set_skb_rxhash(bp, cqe, skb);
528
529 goto next_rx;
530 }
531
532 if (TPA_TYPE(cqe_fp_flags) == TPA_TYPE_END) {
533 DP(NETIF_MSG_RX_STATUS,
534 "calling tpa_stop on queue %d\n",
535 queue);
536
537 if (!BNX2X_RX_SUM_FIX(cqe))
538 BNX2X_ERR("STOP on none TCP "
539 "data\n");
540
541 /* This is a size of the linear data
542 on this skb */
543 len = le16_to_cpu(cqe->fast_path_cqe.
544 len_on_bd);
545 bnx2x_tpa_stop(bp, fp, queue, pad,
546 len, cqe, comp_ring_cons);
547#ifdef BNX2X_STOP_ON_ERROR
548 if (bp->panic)
549 return 0;
550#endif
551
552 bnx2x_update_sge_prod(fp,
553 &cqe->fast_path_cqe);
554 goto next_cqe;
555 }
556 }
557
558 dma_sync_single_for_device(&bp->pdev->dev,
559 dma_unmap_addr(rx_buf, mapping),
560 pad + RX_COPY_THRESH,
561 DMA_FROM_DEVICE);
562 prefetch(((char *)(skb)) + 128);
563
564 /* is this an error packet? */
565 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
566 DP(NETIF_MSG_RX_ERR,
567 "ERROR flags %x rx packet %u\n",
568 cqe_fp_flags, sw_comp_cons);
569 fp->eth_q_stats.rx_err_discard_pkt++;
570 goto reuse_rx;
571 }
572
573 /* Since we don't have a jumbo ring
574 * copy small packets if mtu > 1500
575 */
576 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
577 (len <= RX_COPY_THRESH)) {
578 struct sk_buff *new_skb;
579
580 new_skb = netdev_alloc_skb(bp->dev,
581 len + pad);
582 if (new_skb == NULL) {
583 DP(NETIF_MSG_RX_ERR,
584 "ERROR packet dropped "
585 "because of alloc failure\n");
586 fp->eth_q_stats.rx_skb_alloc_failed++;
587 goto reuse_rx;
588 }
589
590 /* aligned copy */
591 skb_copy_from_linear_data_offset(skb, pad,
592 new_skb->data + pad, len);
593 skb_reserve(new_skb, pad);
594 skb_put(new_skb, len);
595
596 bnx2x_reuse_rx_skb(fp, skb, bd_cons, bd_prod);
597
598 skb = new_skb;
599
600 } else
601 if (likely(bnx2x_alloc_rx_skb(bp, fp, bd_prod) == 0)) {
602 dma_unmap_single(&bp->pdev->dev,
603 dma_unmap_addr(rx_buf, mapping),
604 bp->rx_buf_size,
605 DMA_FROM_DEVICE);
606 skb_reserve(skb, pad);
607 skb_put(skb, len);
608
609 } else {
610 DP(NETIF_MSG_RX_ERR,
611 "ERROR packet dropped because "
612 "of alloc failure\n");
613 fp->eth_q_stats.rx_skb_alloc_failed++;
614reuse_rx:
615 bnx2x_reuse_rx_skb(fp, skb, bd_cons, bd_prod);
616 goto next_rx;
617 }
618
619 skb->protocol = eth_type_trans(skb, bp->dev);
620
621 /* Set Toeplitz hash for a none-LRO skb */
622 bnx2x_set_skb_rxhash(bp, cqe, skb);
623
624 skb->ip_summed = CHECKSUM_NONE;
625 if (bp->rx_csum) {
626 if (likely(BNX2X_RX_CSUM_OK(cqe)))
627 skb->ip_summed = CHECKSUM_UNNECESSARY;
628 else
629 fp->eth_q_stats.hw_csum_err++;
630 }
631 }
632
633 skb_record_rx_queue(skb, fp->index);
634
635#ifdef BCM_VLAN
636 if ((bp->vlgrp != NULL) && (bp->flags & HW_VLAN_RX_FLAG) &&
637 (le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags) &
638 PARSING_FLAGS_VLAN))
639 vlan_gro_receive(&fp->napi, bp->vlgrp,
640 le16_to_cpu(cqe->fast_path_cqe.vlan_tag), skb);
641 else
642#endif
643 napi_gro_receive(&fp->napi, skb);
644
645
646next_rx:
647 rx_buf->skb = NULL;
648
649 bd_cons = NEXT_RX_IDX(bd_cons);
650 bd_prod = NEXT_RX_IDX(bd_prod);
651 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
652 rx_pkt++;
653next_cqe:
654 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
655 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
656
657 if (rx_pkt == budget)
658 break;
659 } /* while */
660
661 fp->rx_bd_cons = bd_cons;
662 fp->rx_bd_prod = bd_prod_fw;
663 fp->rx_comp_cons = sw_comp_cons;
664 fp->rx_comp_prod = sw_comp_prod;
665
666 /* Update producers */
667 bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
668 fp->rx_sge_prod);
669
670 fp->rx_pkt += rx_pkt;
671 fp->rx_calls++;
672
673 return rx_pkt;
674}
675
676static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
677{
678 struct bnx2x_fastpath *fp = fp_cookie;
679 struct bnx2x *bp = fp->bp;
680
681 /* Return here if interrupt is disabled */
682 if (unlikely(atomic_read(&bp->intr_sem) != 0)) {
683 DP(NETIF_MSG_INTR, "called but intr_sem not 0, returning\n");
684 return IRQ_HANDLED;
685 }
686
687 DP(BNX2X_MSG_FP, "got an MSI-X interrupt on IDX:SB [%d:%d]\n",
688 fp->index, fp->sb_id);
689 bnx2x_ack_sb(bp, fp->sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
690
691#ifdef BNX2X_STOP_ON_ERROR
692 if (unlikely(bp->panic))
693 return IRQ_HANDLED;
694#endif
695
696 /* Handle Rx and Tx according to MSI-X vector */
697 prefetch(fp->rx_cons_sb);
698 prefetch(fp->tx_cons_sb);
699 prefetch(&fp->status_blk->u_status_block.status_block_index);
700 prefetch(&fp->status_blk->c_status_block.status_block_index);
701 napi_schedule(&bnx2x_fp(bp, fp->index, napi));
702
703 return IRQ_HANDLED;
704}
705
706
707/* HW Lock for shared dual port PHYs */
708void bnx2x_acquire_phy_lock(struct bnx2x *bp)
709{
710 mutex_lock(&bp->port.phy_mutex);
711
712 if (bp->port.need_hw_lock)
713 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
714}
715
716void bnx2x_release_phy_lock(struct bnx2x *bp)
717{
718 if (bp->port.need_hw_lock)
719 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
720
721 mutex_unlock(&bp->port.phy_mutex);
722}
723
724void bnx2x_link_report(struct bnx2x *bp)
725{
726 if (bp->flags & MF_FUNC_DIS) {
727 netif_carrier_off(bp->dev);
728 netdev_err(bp->dev, "NIC Link is Down\n");
729 return;
730 }
731
732 if (bp->link_vars.link_up) {
733 u16 line_speed;
734
735 if (bp->state == BNX2X_STATE_OPEN)
736 netif_carrier_on(bp->dev);
737 netdev_info(bp->dev, "NIC Link is Up, ");
738
739 line_speed = bp->link_vars.line_speed;
740 if (IS_E1HMF(bp)) {
741 u16 vn_max_rate;
742
743 vn_max_rate =
744 ((bp->mf_config & FUNC_MF_CFG_MAX_BW_MASK) >>
745 FUNC_MF_CFG_MAX_BW_SHIFT) * 100;
746 if (vn_max_rate < line_speed)
747 line_speed = vn_max_rate;
748 }
749 pr_cont("%d Mbps ", line_speed);
750
751 if (bp->link_vars.duplex == DUPLEX_FULL)
752 pr_cont("full duplex");
753 else
754 pr_cont("half duplex");
755
756 if (bp->link_vars.flow_ctrl != BNX2X_FLOW_CTRL_NONE) {
757 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX) {
758 pr_cont(", receive ");
759 if (bp->link_vars.flow_ctrl &
760 BNX2X_FLOW_CTRL_TX)
761 pr_cont("& transmit ");
762 } else {
763 pr_cont(", transmit ");
764 }
765 pr_cont("flow control ON");
766 }
767 pr_cont("\n");
768
769 } else { /* link_down */
770 netif_carrier_off(bp->dev);
771 netdev_err(bp->dev, "NIC Link is Down\n");
772 }
773}
774
775void bnx2x_init_rx_rings(struct bnx2x *bp)
776{
777 int func = BP_FUNC(bp);
778 int max_agg_queues = CHIP_IS_E1(bp) ? ETH_MAX_AGGREGATION_QUEUES_E1 :
779 ETH_MAX_AGGREGATION_QUEUES_E1H;
780 u16 ring_prod, cqe_ring_prod;
781 int i, j;
782
783 bp->rx_buf_size = bp->dev->mtu + ETH_OVREHEAD + BNX2X_RX_ALIGN;
784 DP(NETIF_MSG_IFUP,
785 "mtu %d rx_buf_size %d\n", bp->dev->mtu, bp->rx_buf_size);
786
787 if (bp->flags & TPA_ENABLE_FLAG) {
788
789 for_each_queue(bp, j) {
790 struct bnx2x_fastpath *fp = &bp->fp[j];
791
792 for (i = 0; i < max_agg_queues; i++) {
793 fp->tpa_pool[i].skb =
794 netdev_alloc_skb(bp->dev, bp->rx_buf_size);
795 if (!fp->tpa_pool[i].skb) {
796 BNX2X_ERR("Failed to allocate TPA "
797 "skb pool for queue[%d] - "
798 "disabling TPA on this "
799 "queue!\n", j);
800 bnx2x_free_tpa_pool(bp, fp, i);
801 fp->disable_tpa = 1;
802 break;
803 }
804 dma_unmap_addr_set((struct sw_rx_bd *)
805 &bp->fp->tpa_pool[i],
806 mapping, 0);
807 fp->tpa_state[i] = BNX2X_TPA_STOP;
808 }
809 }
810 }
811
812 for_each_queue(bp, j) {
813 struct bnx2x_fastpath *fp = &bp->fp[j];
814
815 fp->rx_bd_cons = 0;
816 fp->rx_cons_sb = BNX2X_RX_SB_INDEX;
817 fp->rx_bd_cons_sb = BNX2X_RX_SB_BD_INDEX;
818
819 /* "next page" elements initialization */
820 /* SGE ring */
821 for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
822 struct eth_rx_sge *sge;
823
824 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
825 sge->addr_hi =
826 cpu_to_le32(U64_HI(fp->rx_sge_mapping +
827 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
828 sge->addr_lo =
829 cpu_to_le32(U64_LO(fp->rx_sge_mapping +
830 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
831 }
832
833 bnx2x_init_sge_ring_bit_mask(fp);
834
835 /* RX BD ring */
836 for (i = 1; i <= NUM_RX_RINGS; i++) {
837 struct eth_rx_bd *rx_bd;
838
839 rx_bd = &fp->rx_desc_ring[RX_DESC_CNT * i - 2];
840 rx_bd->addr_hi =
841 cpu_to_le32(U64_HI(fp->rx_desc_mapping +
842 BCM_PAGE_SIZE*(i % NUM_RX_RINGS)));
843 rx_bd->addr_lo =
844 cpu_to_le32(U64_LO(fp->rx_desc_mapping +
845 BCM_PAGE_SIZE*(i % NUM_RX_RINGS)));
846 }
847
848 /* CQ ring */
849 for (i = 1; i <= NUM_RCQ_RINGS; i++) {
850 struct eth_rx_cqe_next_page *nextpg;
851
852 nextpg = (struct eth_rx_cqe_next_page *)
853 &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
854 nextpg->addr_hi =
855 cpu_to_le32(U64_HI(fp->rx_comp_mapping +
856 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
857 nextpg->addr_lo =
858 cpu_to_le32(U64_LO(fp->rx_comp_mapping +
859 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
860 }
861
862 /* Allocate SGEs and initialize the ring elements */
863 for (i = 0, ring_prod = 0;
864 i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
865
866 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
867 BNX2X_ERR("was only able to allocate "
868 "%d rx sges\n", i);
869 BNX2X_ERR("disabling TPA for queue[%d]\n", j);
870 /* Cleanup already allocated elements */
871 bnx2x_free_rx_sge_range(bp, fp, ring_prod);
872 bnx2x_free_tpa_pool(bp, fp, max_agg_queues);
873 fp->disable_tpa = 1;
874 ring_prod = 0;
875 break;
876 }
877 ring_prod = NEXT_SGE_IDX(ring_prod);
878 }
879 fp->rx_sge_prod = ring_prod;
880
881 /* Allocate BDs and initialize BD ring */
882 fp->rx_comp_cons = 0;
883 cqe_ring_prod = ring_prod = 0;
884 for (i = 0; i < bp->rx_ring_size; i++) {
885 if (bnx2x_alloc_rx_skb(bp, fp, ring_prod) < 0) {
886 BNX2X_ERR("was only able to allocate "
887 "%d rx skbs on queue[%d]\n", i, j);
888 fp->eth_q_stats.rx_skb_alloc_failed++;
889 break;
890 }
891 ring_prod = NEXT_RX_IDX(ring_prod);
892 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
893 WARN_ON(ring_prod <= i);
894 }
895
896 fp->rx_bd_prod = ring_prod;
897 /* must not have more available CQEs than BDs */
898 fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
899 cqe_ring_prod);
900 fp->rx_pkt = fp->rx_calls = 0;
901
902 /* Warning!
903 * this will generate an interrupt (to the TSTORM)
904 * must only be done after chip is initialized
905 */
906 bnx2x_update_rx_prod(bp, fp, ring_prod, fp->rx_comp_prod,
907 fp->rx_sge_prod);
908 if (j != 0)
909 continue;
910
911 REG_WR(bp, BAR_USTRORM_INTMEM +
912 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
913 U64_LO(fp->rx_comp_mapping));
914 REG_WR(bp, BAR_USTRORM_INTMEM +
915 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
916 U64_HI(fp->rx_comp_mapping));
917 }
918}
919static void bnx2x_free_tx_skbs(struct bnx2x *bp)
920{
921 int i;
922
923 for_each_queue(bp, i) {
924 struct bnx2x_fastpath *fp = &bp->fp[i];
925
926 u16 bd_cons = fp->tx_bd_cons;
927 u16 sw_prod = fp->tx_pkt_prod;
928 u16 sw_cons = fp->tx_pkt_cons;
929
930 while (sw_cons != sw_prod) {
931 bd_cons = bnx2x_free_tx_pkt(bp, fp, TX_BD(sw_cons));
932 sw_cons++;
933 }
934 }
935}
936
937static void bnx2x_free_rx_skbs(struct bnx2x *bp)
938{
939 int i, j;
940
941 for_each_queue(bp, j) {
942 struct bnx2x_fastpath *fp = &bp->fp[j];
943
944 for (i = 0; i < NUM_RX_BD; i++) {
945 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
946 struct sk_buff *skb = rx_buf->skb;
947
948 if (skb == NULL)
949 continue;
950
951 dma_unmap_single(&bp->pdev->dev,
952 dma_unmap_addr(rx_buf, mapping),
953 bp->rx_buf_size, DMA_FROM_DEVICE);
954
955 rx_buf->skb = NULL;
956 dev_kfree_skb(skb);
957 }
958 if (!fp->disable_tpa)
959 bnx2x_free_tpa_pool(bp, fp, CHIP_IS_E1(bp) ?
960 ETH_MAX_AGGREGATION_QUEUES_E1 :
961 ETH_MAX_AGGREGATION_QUEUES_E1H);
962 }
963}
964
965void bnx2x_free_skbs(struct bnx2x *bp)
966{
967 bnx2x_free_tx_skbs(bp);
968 bnx2x_free_rx_skbs(bp);
969}
970
971static void bnx2x_free_msix_irqs(struct bnx2x *bp)
972{
973 int i, offset = 1;
974
975 free_irq(bp->msix_table[0].vector, bp->dev);
976 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
977 bp->msix_table[0].vector);
978
979#ifdef BCM_CNIC
980 offset++;
981#endif
982 for_each_queue(bp, i) {
983 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq "
984 "state %x\n", i, bp->msix_table[i + offset].vector,
985 bnx2x_fp(bp, i, state));
986
987 free_irq(bp->msix_table[i + offset].vector, &bp->fp[i]);
988 }
989}
990
991void bnx2x_free_irq(struct bnx2x *bp, bool disable_only)
992{
993 if (bp->flags & USING_MSIX_FLAG) {
994 if (!disable_only)
995 bnx2x_free_msix_irqs(bp);
996 pci_disable_msix(bp->pdev);
997 bp->flags &= ~USING_MSIX_FLAG;
998
999 } else if (bp->flags & USING_MSI_FLAG) {
1000 if (!disable_only)
1001 free_irq(bp->pdev->irq, bp->dev);
1002 pci_disable_msi(bp->pdev);
1003 bp->flags &= ~USING_MSI_FLAG;
1004
1005 } else if (!disable_only)
1006 free_irq(bp->pdev->irq, bp->dev);
1007}
1008
1009static int bnx2x_enable_msix(struct bnx2x *bp)
1010{
1011 int i, rc, offset = 1;
1012 int igu_vec = 0;
1013
1014 bp->msix_table[0].entry = igu_vec;
1015 DP(NETIF_MSG_IFUP, "msix_table[0].entry = %d (slowpath)\n", igu_vec);
1016
1017#ifdef BCM_CNIC
1018 igu_vec = BP_L_ID(bp) + offset;
1019 bp->msix_table[1].entry = igu_vec;
1020 DP(NETIF_MSG_IFUP, "msix_table[1].entry = %d (CNIC)\n", igu_vec);
1021 offset++;
1022#endif
1023 for_each_queue(bp, i) {
1024 igu_vec = BP_L_ID(bp) + offset + i;
1025 bp->msix_table[i + offset].entry = igu_vec;
1026 DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d "
1027 "(fastpath #%u)\n", i + offset, igu_vec, i);
1028 }
1029
1030 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0],
1031 BNX2X_NUM_QUEUES(bp) + offset);
1032
1033 /*
1034 * reconfigure number of tx/rx queues according to available
1035 * MSI-X vectors
1036 */
1037 if (rc >= BNX2X_MIN_MSIX_VEC_CNT) {
1038 /* vectors available for FP */
1039 int fp_vec = rc - BNX2X_MSIX_VEC_FP_START;
1040
1041 DP(NETIF_MSG_IFUP,
1042 "Trying to use less MSI-X vectors: %d\n", rc);
1043
1044 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1045
1046 if (rc) {
1047 DP(NETIF_MSG_IFUP,
1048 "MSI-X is not attainable rc %d\n", rc);
1049 return rc;
1050 }
1051
1052 bp->num_queues = min(bp->num_queues, fp_vec);
1053
1054 DP(NETIF_MSG_IFUP, "New queue configuration set: %d\n",
1055 bp->num_queues);
1056 } else if (rc) {
1057 DP(NETIF_MSG_IFUP, "MSI-X is not attainable rc %d\n", rc);
1058 return rc;
1059 }
1060
1061 bp->flags |= USING_MSIX_FLAG;
1062
1063 return 0;
1064}
1065
1066static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1067{
1068 int i, rc, offset = 1;
1069
1070 rc = request_irq(bp->msix_table[0].vector, bnx2x_msix_sp_int, 0,
1071 bp->dev->name, bp->dev);
1072 if (rc) {
1073 BNX2X_ERR("request sp irq failed\n");
1074 return -EBUSY;
1075 }
1076
1077#ifdef BCM_CNIC
1078 offset++;
1079#endif
1080 for_each_queue(bp, i) {
1081 struct bnx2x_fastpath *fp = &bp->fp[i];
1082 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1083 bp->dev->name, i);
1084
1085 rc = request_irq(bp->msix_table[i + offset].vector,
1086 bnx2x_msix_fp_int, 0, fp->name, fp);
1087 if (rc) {
1088 BNX2X_ERR("request fp #%d irq failed rc %d\n", i, rc);
1089 bnx2x_free_msix_irqs(bp);
1090 return -EBUSY;
1091 }
1092
1093 fp->state = BNX2X_FP_STATE_IRQ;
1094 }
1095
1096 i = BNX2X_NUM_QUEUES(bp);
1097 netdev_info(bp->dev, "using MSI-X IRQs: sp %d fp[%d] %d"
1098 " ... fp[%d] %d\n",
1099 bp->msix_table[0].vector,
1100 0, bp->msix_table[offset].vector,
1101 i - 1, bp->msix_table[offset + i - 1].vector);
1102
1103 return 0;
1104}
1105
1106static int bnx2x_enable_msi(struct bnx2x *bp)
1107{
1108 int rc;
1109
1110 rc = pci_enable_msi(bp->pdev);
1111 if (rc) {
1112 DP(NETIF_MSG_IFUP, "MSI is not attainable\n");
1113 return -1;
1114 }
1115 bp->flags |= USING_MSI_FLAG;
1116
1117 return 0;
1118}
1119
1120static int bnx2x_req_irq(struct bnx2x *bp)
1121{
1122 unsigned long flags;
1123 int rc;
1124
1125 if (bp->flags & USING_MSI_FLAG)
1126 flags = 0;
1127 else
1128 flags = IRQF_SHARED;
1129
1130 rc = request_irq(bp->pdev->irq, bnx2x_interrupt, flags,
1131 bp->dev->name, bp->dev);
1132 if (!rc)
1133 bnx2x_fp(bp, 0, state) = BNX2X_FP_STATE_IRQ;
1134
1135 return rc;
1136}
1137
1138static void bnx2x_napi_enable(struct bnx2x *bp)
1139{
1140 int i;
1141
1142 for_each_queue(bp, i)
1143 napi_enable(&bnx2x_fp(bp, i, napi));
1144}
1145
1146static void bnx2x_napi_disable(struct bnx2x *bp)
1147{
1148 int i;
1149
1150 for_each_queue(bp, i)
1151 napi_disable(&bnx2x_fp(bp, i, napi));
1152}
1153
1154void bnx2x_netif_start(struct bnx2x *bp)
1155{
1156 int intr_sem;
1157
1158 intr_sem = atomic_dec_and_test(&bp->intr_sem);
1159 smp_wmb(); /* Ensure that bp->intr_sem update is SMP-safe */
1160
1161 if (intr_sem) {
1162 if (netif_running(bp->dev)) {
1163 bnx2x_napi_enable(bp);
1164 bnx2x_int_enable(bp);
1165 if (bp->state == BNX2X_STATE_OPEN)
1166 netif_tx_wake_all_queues(bp->dev);
1167 }
1168 }
1169}
1170
1171void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1172{
1173 bnx2x_int_disable_sync(bp, disable_hw);
1174 bnx2x_napi_disable(bp);
1175 netif_tx_disable(bp->dev);
1176}
1177static int bnx2x_set_num_queues(struct bnx2x *bp)
1178{
1179 int rc = 0;
1180
1181 switch (bp->int_mode) {
1182 case INT_MODE_INTx:
1183 case INT_MODE_MSI:
1184 bp->num_queues = 1;
1185 DP(NETIF_MSG_IFUP, "set number of queues to 1\n");
1186 break;
1187 default:
1188 /* Set number of queues according to bp->multi_mode value */
1189 bnx2x_set_num_queues_msix(bp);
1190
1191 DP(NETIF_MSG_IFUP, "set number of queues to %d\n",
1192 bp->num_queues);
1193
1194 /* if we can't use MSI-X we only need one fp,
1195 * so try to enable MSI-X with the requested number of fp's
1196 * and fallback to MSI or legacy INTx with one fp
1197 */
1198 rc = bnx2x_enable_msix(bp);
1199 if (rc)
1200 /* failed to enable MSI-X */
1201 bp->num_queues = 1;
1202 break;
1203 }
1204 bp->dev->real_num_tx_queues = bp->num_queues;
1205 return rc;
1206}
1207
1208/* must be called with rtnl_lock */
1209int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
1210{
1211 u32 load_code;
1212 int i, rc;
1213
1214#ifdef BNX2X_STOP_ON_ERROR
1215 if (unlikely(bp->panic))
1216 return -EPERM;
1217#endif
1218
1219 bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
1220
1221 rc = bnx2x_set_num_queues(bp);
1222
1223 if (bnx2x_alloc_mem(bp)) {
1224 bnx2x_free_irq(bp, true);
1225 return -ENOMEM;
1226 }
1227
1228 for_each_queue(bp, i)
1229 bnx2x_fp(bp, i, disable_tpa) =
1230 ((bp->flags & TPA_ENABLE_FLAG) == 0);
1231
1232 for_each_queue(bp, i)
1233 netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
1234 bnx2x_poll, 128);
1235
1236 bnx2x_napi_enable(bp);
1237
1238 if (bp->flags & USING_MSIX_FLAG) {
1239 rc = bnx2x_req_msix_irqs(bp);
1240 if (rc) {
1241 bnx2x_free_irq(bp, true);
1242 goto load_error1;
1243 }
1244 } else {
1245 /* Fall to INTx if failed to enable MSI-X due to lack of
1246 memory (in bnx2x_set_num_queues()) */
1247 if ((rc != -ENOMEM) && (bp->int_mode != INT_MODE_INTx))
1248 bnx2x_enable_msi(bp);
1249 bnx2x_ack_int(bp);
1250 rc = bnx2x_req_irq(bp);
1251 if (rc) {
1252 BNX2X_ERR("IRQ request failed rc %d, aborting\n", rc);
1253 bnx2x_free_irq(bp, true);
1254 goto load_error1;
1255 }
1256 if (bp->flags & USING_MSI_FLAG) {
1257 bp->dev->irq = bp->pdev->irq;
1258 netdev_info(bp->dev, "using MSI IRQ %d\n",
1259 bp->pdev->irq);
1260 }
1261 }
1262
1263 /* Send LOAD_REQUEST command to MCP
1264 Returns the type of LOAD command:
1265 if it is the first port to be initialized
1266 common blocks should be initialized, otherwise - not
1267 */
1268 if (!BP_NOMCP(bp)) {
1269 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ);
1270 if (!load_code) {
1271 BNX2X_ERR("MCP response failure, aborting\n");
1272 rc = -EBUSY;
1273 goto load_error2;
1274 }
1275 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
1276 rc = -EBUSY; /* other port in diagnostic mode */
1277 goto load_error2;
1278 }
1279
1280 } else {
1281 int port = BP_PORT(bp);
1282
1283 DP(NETIF_MSG_IFUP, "NO MCP - load counts %d, %d, %d\n",
1284 load_count[0], load_count[1], load_count[2]);
1285 load_count[0]++;
1286 load_count[1 + port]++;
1287 DP(NETIF_MSG_IFUP, "NO MCP - new load counts %d, %d, %d\n",
1288 load_count[0], load_count[1], load_count[2]);
1289 if (load_count[0] == 1)
1290 load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
1291 else if (load_count[1 + port] == 1)
1292 load_code = FW_MSG_CODE_DRV_LOAD_PORT;
1293 else
1294 load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
1295 }
1296
1297 if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1298 (load_code == FW_MSG_CODE_DRV_LOAD_PORT))
1299 bp->port.pmf = 1;
1300 else
1301 bp->port.pmf = 0;
1302 DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
1303
1304 /* Initialize HW */
1305 rc = bnx2x_init_hw(bp, load_code);
1306 if (rc) {
1307 BNX2X_ERR("HW init failed, aborting\n");
1308 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE);
1309 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP);
1310 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE);
1311 goto load_error2;
1312 }
1313
1314 /* Setup NIC internals and enable interrupts */
1315 bnx2x_nic_init(bp, load_code);
1316
1317 if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) &&
1318 (bp->common.shmem2_base))
1319 SHMEM2_WR(bp, dcc_support,
1320 (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
1321 SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
1322
1323 /* Send LOAD_DONE command to MCP */
1324 if (!BP_NOMCP(bp)) {
1325 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE);
1326 if (!load_code) {
1327 BNX2X_ERR("MCP response failure, aborting\n");
1328 rc = -EBUSY;
1329 goto load_error3;
1330 }
1331 }
1332
1333 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
1334
1335 rc = bnx2x_setup_leading(bp);
1336 if (rc) {
1337 BNX2X_ERR("Setup leading failed!\n");
1338#ifndef BNX2X_STOP_ON_ERROR
1339 goto load_error3;
1340#else
1341 bp->panic = 1;
1342 return -EBUSY;
1343#endif
1344 }
1345
1346 if (CHIP_IS_E1H(bp))
1347 if (bp->mf_config & FUNC_MF_CFG_FUNC_DISABLED) {
1348 DP(NETIF_MSG_IFUP, "mf_cfg function disabled\n");
1349 bp->flags |= MF_FUNC_DIS;
1350 }
1351
1352 if (bp->state == BNX2X_STATE_OPEN) {
1353#ifdef BCM_CNIC
1354 /* Enable Timer scan */
1355 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 1);
1356#endif
1357 for_each_nondefault_queue(bp, i) {
1358 rc = bnx2x_setup_multi(bp, i);
1359 if (rc)
1360#ifdef BCM_CNIC
1361 goto load_error4;
1362#else
1363 goto load_error3;
1364#endif
1365 }
1366
1367 if (CHIP_IS_E1(bp))
1368 bnx2x_set_eth_mac_addr_e1(bp, 1);
1369 else
1370 bnx2x_set_eth_mac_addr_e1h(bp, 1);
1371#ifdef BCM_CNIC
1372 /* Set iSCSI L2 MAC */
1373 mutex_lock(&bp->cnic_mutex);
1374 if (bp->cnic_eth_dev.drv_state & CNIC_DRV_STATE_REGD) {
1375 bnx2x_set_iscsi_eth_mac_addr(bp, 1);
1376 bp->cnic_flags |= BNX2X_CNIC_FLAG_MAC_SET;
1377 bnx2x_init_sb(bp, bp->cnic_sb, bp->cnic_sb_mapping,
1378 CNIC_SB_ID(bp));
1379 }
1380 mutex_unlock(&bp->cnic_mutex);
1381#endif
1382 }
1383
1384 if (bp->port.pmf)
1385 bnx2x_initial_phy_init(bp, load_mode);
1386
1387 /* Start fast path */
1388 switch (load_mode) {
1389 case LOAD_NORMAL:
1390 if (bp->state == BNX2X_STATE_OPEN) {
1391 /* Tx queue should be only reenabled */
1392 netif_tx_wake_all_queues(bp->dev);
1393 }
1394 /* Initialize the receive filter. */
1395 bnx2x_set_rx_mode(bp->dev);
1396 break;
1397
1398 case LOAD_OPEN:
1399 netif_tx_start_all_queues(bp->dev);
1400 if (bp->state != BNX2X_STATE_OPEN)
1401 netif_tx_disable(bp->dev);
1402 /* Initialize the receive filter. */
1403 bnx2x_set_rx_mode(bp->dev);
1404 break;
1405
1406 case LOAD_DIAG:
1407 /* Initialize the receive filter. */
1408 bnx2x_set_rx_mode(bp->dev);
1409 bp->state = BNX2X_STATE_DIAG;
1410 break;
1411
1412 default:
1413 break;
1414 }
1415
1416 if (!bp->port.pmf)
1417 bnx2x__link_status_update(bp);
1418
1419 /* start the timer */
1420 mod_timer(&bp->timer, jiffies + bp->current_interval);
1421
1422#ifdef BCM_CNIC
1423 bnx2x_setup_cnic_irq_info(bp);
1424 if (bp->state == BNX2X_STATE_OPEN)
1425 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
1426#endif
1427 bnx2x_inc_load_cnt(bp);
1428
1429 return 0;
1430
1431#ifdef BCM_CNIC
1432load_error4:
1433 /* Disable Timer scan */
1434 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 0);
1435#endif
1436load_error3:
1437 bnx2x_int_disable_sync(bp, 1);
1438 if (!BP_NOMCP(bp)) {
1439 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP);
1440 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE);
1441 }
1442 bp->port.pmf = 0;
1443 /* Free SKBs, SGEs, TPA pool and driver internals */
1444 bnx2x_free_skbs(bp);
1445 for_each_queue(bp, i)
1446 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
1447load_error2:
1448 /* Release IRQs */
1449 bnx2x_free_irq(bp, false);
1450load_error1:
1451 bnx2x_napi_disable(bp);
1452 for_each_queue(bp, i)
1453 netif_napi_del(&bnx2x_fp(bp, i, napi));
1454 bnx2x_free_mem(bp);
1455
1456 return rc;
1457}
1458
1459/* must be called with rtnl_lock */
1460int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode)
1461{
1462 int i;
1463
1464 if (bp->state == BNX2X_STATE_CLOSED) {
1465 /* Interface has been removed - nothing to recover */
1466 bp->recovery_state = BNX2X_RECOVERY_DONE;
1467 bp->is_leader = 0;
1468 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_RESERVED_08);
1469 smp_wmb();
1470
1471 return -EINVAL;
1472 }
1473
1474#ifdef BCM_CNIC
1475 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
1476#endif
1477 bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
1478
1479 /* Set "drop all" */
1480 bp->rx_mode = BNX2X_RX_MODE_NONE;
1481 bnx2x_set_storm_rx_mode(bp);
1482
1483 /* Disable HW interrupts, NAPI and Tx */
1484 bnx2x_netif_stop(bp, 1);
1485 netif_carrier_off(bp->dev);
1486
1487 del_timer_sync(&bp->timer);
1488 SHMEM_WR(bp, func_mb[BP_FUNC(bp)].drv_pulse_mb,
1489 (DRV_PULSE_ALWAYS_ALIVE | bp->fw_drv_pulse_wr_seq));
1490 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
1491
1492 /* Release IRQs */
1493 bnx2x_free_irq(bp, false);
1494
1495 /* Cleanup the chip if needed */
1496 if (unload_mode != UNLOAD_RECOVERY)
1497 bnx2x_chip_cleanup(bp, unload_mode);
1498
1499 bp->port.pmf = 0;
1500
1501 /* Free SKBs, SGEs, TPA pool and driver internals */
1502 bnx2x_free_skbs(bp);
1503 for_each_queue(bp, i)
1504 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
1505 for_each_queue(bp, i)
1506 netif_napi_del(&bnx2x_fp(bp, i, napi));
1507 bnx2x_free_mem(bp);
1508
1509 bp->state = BNX2X_STATE_CLOSED;
1510
1511 /* The last driver must disable a "close the gate" if there is no
1512 * parity attention or "process kill" pending.
1513 */
1514 if ((!bnx2x_dec_load_cnt(bp)) && (!bnx2x_chk_parity_attn(bp)) &&
1515 bnx2x_reset_is_done(bp))
1516 bnx2x_disable_close_the_gate(bp);
1517
1518 /* Reset MCP mail box sequence if there is on going recovery */
1519 if (unload_mode == UNLOAD_RECOVERY)
1520 bp->fw_seq = 0;
1521
1522 return 0;
1523}
1524int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
1525{
1526 u16 pmcsr;
1527
1528 pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
1529
1530 switch (state) {
1531 case PCI_D0:
1532 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
1533 ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
1534 PCI_PM_CTRL_PME_STATUS));
1535
1536 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
1537 /* delay required during transition out of D3hot */
1538 msleep(20);
1539 break;
1540
1541 case PCI_D3hot:
1542 /* If there are other clients above don't
1543 shut down the power */
1544 if (atomic_read(&bp->pdev->enable_cnt) != 1)
1545 return 0;
1546 /* Don't shut down the power for emulation and FPGA */
1547 if (CHIP_REV_IS_SLOW(bp))
1548 return 0;
1549
1550 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
1551 pmcsr |= 3;
1552
1553 if (bp->wol)
1554 pmcsr |= PCI_PM_CTRL_PME_ENABLE;
1555
1556 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
1557 pmcsr);
1558
1559 /* No more memory access after this point until
1560 * device is brought back to D0.
1561 */
1562 break;
1563
1564 default:
1565 return -EINVAL;
1566 }
1567 return 0;
1568}
1569
1570
1571
1572/*
1573 * net_device service functions
1574 */
1575
1576static int bnx2x_poll(struct napi_struct *napi, int budget)
1577{
1578 int work_done = 0;
1579 struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
1580 napi);
1581 struct bnx2x *bp = fp->bp;
1582
1583 while (1) {
1584#ifdef BNX2X_STOP_ON_ERROR
1585 if (unlikely(bp->panic)) {
1586 napi_complete(napi);
1587 return 0;
1588 }
1589#endif
1590
1591 if (bnx2x_has_tx_work(fp))
1592 bnx2x_tx_int(fp);
1593
1594 if (bnx2x_has_rx_work(fp)) {
1595 work_done += bnx2x_rx_int(fp, budget - work_done);
1596
1597 /* must not complete if we consumed full budget */
1598 if (work_done >= budget)
1599 break;
1600 }
1601
1602 /* Fall out from the NAPI loop if needed */
1603 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
1604 bnx2x_update_fpsb_idx(fp);
1605 /* bnx2x_has_rx_work() reads the status block, thus we need
1606 * to ensure that status block indices have been actually read
1607 * (bnx2x_update_fpsb_idx) prior to this check
1608 * (bnx2x_has_rx_work) so that we won't write the "newer"
1609 * value of the status block to IGU (if there was a DMA right
1610 * after bnx2x_has_rx_work and if there is no rmb, the memory
1611 * reading (bnx2x_update_fpsb_idx) may be postponed to right
1612 * before bnx2x_ack_sb). In this case there will never be
1613 * another interrupt until there is another update of the
1614 * status block, while there is still unhandled work.
1615 */
1616 rmb();
1617
1618 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
1619 napi_complete(napi);
1620 /* Re-enable interrupts */
1621 bnx2x_ack_sb(bp, fp->sb_id, CSTORM_ID,
1622 le16_to_cpu(fp->fp_c_idx),
1623 IGU_INT_NOP, 1);
1624 bnx2x_ack_sb(bp, fp->sb_id, USTORM_ID,
1625 le16_to_cpu(fp->fp_u_idx),
1626 IGU_INT_ENABLE, 1);
1627 break;
1628 }
1629 }
1630 }
1631
1632 return work_done;
1633}
1634
1635
1636/* we split the first BD into headers and data BDs
1637 * to ease the pain of our fellow microcode engineers
1638 * we use one mapping for both BDs
1639 * So far this has only been observed to happen
1640 * in Other Operating Systems(TM)
1641 */
1642static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
1643 struct bnx2x_fastpath *fp,
1644 struct sw_tx_bd *tx_buf,
1645 struct eth_tx_start_bd **tx_bd, u16 hlen,
1646 u16 bd_prod, int nbd)
1647{
1648 struct eth_tx_start_bd *h_tx_bd = *tx_bd;
1649 struct eth_tx_bd *d_tx_bd;
1650 dma_addr_t mapping;
1651 int old_len = le16_to_cpu(h_tx_bd->nbytes);
1652
1653 /* first fix first BD */
1654 h_tx_bd->nbd = cpu_to_le16(nbd);
1655 h_tx_bd->nbytes = cpu_to_le16(hlen);
1656
1657 DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d "
1658 "(%x:%x) nbd %d\n", h_tx_bd->nbytes, h_tx_bd->addr_hi,
1659 h_tx_bd->addr_lo, h_tx_bd->nbd);
1660
1661 /* now get a new data BD
1662 * (after the pbd) and fill it */
1663 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
1664 d_tx_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
1665
1666 mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
1667 le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
1668
1669 d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
1670 d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
1671 d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
1672
1673 /* this marks the BD as one that has no individual mapping */
1674 tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
1675
1676 DP(NETIF_MSG_TX_QUEUED,
1677 "TSO split data size is %d (%x:%x)\n",
1678 d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
1679
1680 /* update tx_bd */
1681 *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
1682
1683 return bd_prod;
1684}
1685
1686static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
1687{
1688 if (fix > 0)
1689 csum = (u16) ~csum_fold(csum_sub(csum,
1690 csum_partial(t_header - fix, fix, 0)));
1691
1692 else if (fix < 0)
1693 csum = (u16) ~csum_fold(csum_add(csum,
1694 csum_partial(t_header, -fix, 0)));
1695
1696 return swab16(csum);
1697}
1698
1699static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
1700{
1701 u32 rc;
1702
1703 if (skb->ip_summed != CHECKSUM_PARTIAL)
1704 rc = XMIT_PLAIN;
1705
1706 else {
1707 if (skb->protocol == htons(ETH_P_IPV6)) {
1708 rc = XMIT_CSUM_V6;
1709 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
1710 rc |= XMIT_CSUM_TCP;
1711
1712 } else {
1713 rc = XMIT_CSUM_V4;
1714 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
1715 rc |= XMIT_CSUM_TCP;
1716 }
1717 }
1718
1719 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
1720 rc |= (XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP);
1721
1722 else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
1723 rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6);
1724
1725 return rc;
1726}
1727
1728#if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
1729/* check if packet requires linearization (packet is too fragmented)
1730 no need to check fragmentation if page size > 8K (there will be no
1731 violation to FW restrictions) */
1732static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
1733 u32 xmit_type)
1734{
1735 int to_copy = 0;
1736 int hlen = 0;
1737 int first_bd_sz = 0;
1738
1739 /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
1740 if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
1741
1742 if (xmit_type & XMIT_GSO) {
1743 unsigned short lso_mss = skb_shinfo(skb)->gso_size;
1744 /* Check if LSO packet needs to be copied:
1745 3 = 1 (for headers BD) + 2 (for PBD and last BD) */
1746 int wnd_size = MAX_FETCH_BD - 3;
1747 /* Number of windows to check */
1748 int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
1749 int wnd_idx = 0;
1750 int frag_idx = 0;
1751 u32 wnd_sum = 0;
1752
1753 /* Headers length */
1754 hlen = (int)(skb_transport_header(skb) - skb->data) +
1755 tcp_hdrlen(skb);
1756
1757 /* Amount of data (w/o headers) on linear part of SKB*/
1758 first_bd_sz = skb_headlen(skb) - hlen;
1759
1760 wnd_sum = first_bd_sz;
1761
1762 /* Calculate the first sum - it's special */
1763 for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
1764 wnd_sum +=
1765 skb_shinfo(skb)->frags[frag_idx].size;
1766
1767 /* If there was data on linear skb data - check it */
1768 if (first_bd_sz > 0) {
1769 if (unlikely(wnd_sum < lso_mss)) {
1770 to_copy = 1;
1771 goto exit_lbl;
1772 }
1773
1774 wnd_sum -= first_bd_sz;
1775 }
1776
1777 /* Others are easier: run through the frag list and
1778 check all windows */
1779 for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
1780 wnd_sum +=
1781 skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1].size;
1782
1783 if (unlikely(wnd_sum < lso_mss)) {
1784 to_copy = 1;
1785 break;
1786 }
1787 wnd_sum -=
1788 skb_shinfo(skb)->frags[wnd_idx].size;
1789 }
1790 } else {
1791 /* in non-LSO too fragmented packet should always
1792 be linearized */
1793 to_copy = 1;
1794 }
1795 }
1796
1797exit_lbl:
1798 if (unlikely(to_copy))
1799 DP(NETIF_MSG_TX_QUEUED,
1800 "Linearization IS REQUIRED for %s packet. "
1801 "num_frags %d hlen %d first_bd_sz %d\n",
1802 (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
1803 skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
1804
1805 return to_copy;
1806}
1807#endif
1808
1809/* called with netif_tx_lock
1810 * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
1811 * netif_wake_queue()
1812 */
1813netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
1814{
1815 struct bnx2x *bp = netdev_priv(dev);
1816 struct bnx2x_fastpath *fp;
1817 struct netdev_queue *txq;
1818 struct sw_tx_bd *tx_buf;
1819 struct eth_tx_start_bd *tx_start_bd;
1820 struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
1821 struct eth_tx_parse_bd *pbd = NULL;
1822 u16 pkt_prod, bd_prod;
1823 int nbd, fp_index;
1824 dma_addr_t mapping;
1825 u32 xmit_type = bnx2x_xmit_type(bp, skb);
1826 int i;
1827 u8 hlen = 0;
1828 __le16 pkt_size = 0;
1829 struct ethhdr *eth;
1830 u8 mac_type = UNICAST_ADDRESS;
1831
1832#ifdef BNX2X_STOP_ON_ERROR
1833 if (unlikely(bp->panic))
1834 return NETDEV_TX_BUSY;
1835#endif
1836
1837 fp_index = skb_get_queue_mapping(skb);
1838 txq = netdev_get_tx_queue(dev, fp_index);
1839
1840 fp = &bp->fp[fp_index];
1841
1842 if (unlikely(bnx2x_tx_avail(fp) < (skb_shinfo(skb)->nr_frags + 3))) {
1843 fp->eth_q_stats.driver_xoff++;
1844 netif_tx_stop_queue(txq);
1845 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
1846 return NETDEV_TX_BUSY;
1847 }
1848
1849 DP(NETIF_MSG_TX_QUEUED, "SKB: summed %x protocol %x protocol(%x,%x)"
1850 " gso type %x xmit_type %x\n",
1851 skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
1852 ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type);
1853
1854 eth = (struct ethhdr *)skb->data;
1855
1856 /* set flag according to packet type (UNICAST_ADDRESS is default)*/
1857 if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
1858 if (is_broadcast_ether_addr(eth->h_dest))
1859 mac_type = BROADCAST_ADDRESS;
1860 else
1861 mac_type = MULTICAST_ADDRESS;
1862 }
1863
1864#if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
1865 /* First, check if we need to linearize the skb (due to FW
1866 restrictions). No need to check fragmentation if page size > 8K
1867 (there will be no violation to FW restrictions) */
1868 if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
1869 /* Statistics of linearization */
1870 bp->lin_cnt++;
1871 if (skb_linearize(skb) != 0) {
1872 DP(NETIF_MSG_TX_QUEUED, "SKB linearization failed - "
1873 "silently dropping this SKB\n");
1874 dev_kfree_skb_any(skb);
1875 return NETDEV_TX_OK;
1876 }
1877 }
1878#endif
1879
1880 /*
1881 Please read carefully. First we use one BD which we mark as start,
1882 then we have a parsing info BD (used for TSO or xsum),
1883 and only then we have the rest of the TSO BDs.
1884 (don't forget to mark the last one as last,
1885 and to unmap only AFTER you write to the BD ...)
1886 And above all, all pdb sizes are in words - NOT DWORDS!
1887 */
1888
1889 pkt_prod = fp->tx_pkt_prod++;
1890 bd_prod = TX_BD(fp->tx_bd_prod);
1891
1892 /* get a tx_buf and first BD */
1893 tx_buf = &fp->tx_buf_ring[TX_BD(pkt_prod)];
1894 tx_start_bd = &fp->tx_desc_ring[bd_prod].start_bd;
1895
1896 tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
1897 tx_start_bd->general_data = (mac_type <<
1898 ETH_TX_START_BD_ETH_ADDR_TYPE_SHIFT);
1899 /* header nbd */
1900 tx_start_bd->general_data |= (1 << ETH_TX_START_BD_HDR_NBDS_SHIFT);
1901
1902 /* remember the first BD of the packet */
1903 tx_buf->first_bd = fp->tx_bd_prod;
1904 tx_buf->skb = skb;
1905 tx_buf->flags = 0;
1906
1907 DP(NETIF_MSG_TX_QUEUED,
1908 "sending pkt %u @%p next_idx %u bd %u @%p\n",
1909 pkt_prod, tx_buf, fp->tx_pkt_prod, bd_prod, tx_start_bd);
1910
1911#ifdef BCM_VLAN
1912 if ((bp->vlgrp != NULL) && vlan_tx_tag_present(skb) &&
1913 (bp->flags & HW_VLAN_TX_FLAG)) {
1914 tx_start_bd->vlan = cpu_to_le16(vlan_tx_tag_get(skb));
1915 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_VLAN_TAG;
1916 } else
1917#endif
1918 tx_start_bd->vlan = cpu_to_le16(pkt_prod);
1919
1920 /* turn on parsing and get a BD */
1921 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
1922 pbd = &fp->tx_desc_ring[bd_prod].parse_bd;
1923
1924 memset(pbd, 0, sizeof(struct eth_tx_parse_bd));
1925
1926 if (xmit_type & XMIT_CSUM) {
1927 hlen = (skb_network_header(skb) - skb->data) / 2;
1928
1929 /* for now NS flag is not used in Linux */
1930 pbd->global_data =
1931 (hlen | ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
1932 ETH_TX_PARSE_BD_LLC_SNAP_EN_SHIFT));
1933
1934 pbd->ip_hlen = (skb_transport_header(skb) -
1935 skb_network_header(skb)) / 2;
1936
1937 hlen += pbd->ip_hlen + tcp_hdrlen(skb) / 2;
1938
1939 pbd->total_hlen = cpu_to_le16(hlen);
1940 hlen = hlen*2;
1941
1942 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
1943
1944 if (xmit_type & XMIT_CSUM_V4)
1945 tx_start_bd->bd_flags.as_bitfield |=
1946 ETH_TX_BD_FLAGS_IP_CSUM;
1947 else
1948 tx_start_bd->bd_flags.as_bitfield |=
1949 ETH_TX_BD_FLAGS_IPV6;
1950
1951 if (xmit_type & XMIT_CSUM_TCP) {
1952 pbd->tcp_pseudo_csum = swab16(tcp_hdr(skb)->check);
1953
1954 } else {
1955 s8 fix = SKB_CS_OFF(skb); /* signed! */
1956
1957 pbd->global_data |= ETH_TX_PARSE_BD_UDP_CS_FLG;
1958
1959 DP(NETIF_MSG_TX_QUEUED,
1960 "hlen %d fix %d csum before fix %x\n",
1961 le16_to_cpu(pbd->total_hlen), fix, SKB_CS(skb));
1962
1963 /* HW bug: fixup the CSUM */
1964 pbd->tcp_pseudo_csum =
1965 bnx2x_csum_fix(skb_transport_header(skb),
1966 SKB_CS(skb), fix);
1967
1968 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
1969 pbd->tcp_pseudo_csum);
1970 }
1971 }
1972
1973 mapping = dma_map_single(&bp->pdev->dev, skb->data,
1974 skb_headlen(skb), DMA_TO_DEVICE);
1975
1976 tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
1977 tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
1978 nbd = skb_shinfo(skb)->nr_frags + 2; /* start_bd + pbd + frags */
1979 tx_start_bd->nbd = cpu_to_le16(nbd);
1980 tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
1981 pkt_size = tx_start_bd->nbytes;
1982
1983 DP(NETIF_MSG_TX_QUEUED, "first bd @%p addr (%x:%x) nbd %d"
1984 " nbytes %d flags %x vlan %x\n",
1985 tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
1986 le16_to_cpu(tx_start_bd->nbd), le16_to_cpu(tx_start_bd->nbytes),
1987 tx_start_bd->bd_flags.as_bitfield, le16_to_cpu(tx_start_bd->vlan));
1988
1989 if (xmit_type & XMIT_GSO) {
1990
1991 DP(NETIF_MSG_TX_QUEUED,
1992 "TSO packet len %d hlen %d total len %d tso size %d\n",
1993 skb->len, hlen, skb_headlen(skb),
1994 skb_shinfo(skb)->gso_size);
1995
1996 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
1997
1998 if (unlikely(skb_headlen(skb) > hlen))
1999 bd_prod = bnx2x_tx_split(bp, fp, tx_buf, &tx_start_bd,
2000 hlen, bd_prod, ++nbd);
2001
2002 pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
2003 pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
2004 pbd->tcp_flags = pbd_tcp_flags(skb);
2005
2006 if (xmit_type & XMIT_GSO_V4) {
2007 pbd->ip_id = swab16(ip_hdr(skb)->id);
2008 pbd->tcp_pseudo_csum =
2009 swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
2010 ip_hdr(skb)->daddr,
2011 0, IPPROTO_TCP, 0));
2012
2013 } else
2014 pbd->tcp_pseudo_csum =
2015 swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2016 &ipv6_hdr(skb)->daddr,
2017 0, IPPROTO_TCP, 0));
2018
2019 pbd->global_data |= ETH_TX_PARSE_BD_PSEUDO_CS_WITHOUT_LEN;
2020 }
2021 tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
2022
2023 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2024 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2025
2026 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2027 tx_data_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
2028 if (total_pkt_bd == NULL)
2029 total_pkt_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
2030
2031 mapping = dma_map_page(&bp->pdev->dev, frag->page,
2032 frag->page_offset,
2033 frag->size, DMA_TO_DEVICE);
2034
2035 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2036 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2037 tx_data_bd->nbytes = cpu_to_le16(frag->size);
2038 le16_add_cpu(&pkt_size, frag->size);
2039
2040 DP(NETIF_MSG_TX_QUEUED,
2041 "frag %d bd @%p addr (%x:%x) nbytes %d\n",
2042 i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
2043 le16_to_cpu(tx_data_bd->nbytes));
2044 }
2045
2046 DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
2047
2048 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2049
2050 /* now send a tx doorbell, counting the next BD
2051 * if the packet contains or ends with it
2052 */
2053 if (TX_BD_POFF(bd_prod) < nbd)
2054 nbd++;
2055
2056 if (total_pkt_bd != NULL)
2057 total_pkt_bd->total_pkt_bytes = pkt_size;
2058
2059 if (pbd)
2060 DP(NETIF_MSG_TX_QUEUED,
2061 "PBD @%p ip_data %x ip_hlen %u ip_id %u lso_mss %u"
2062 " tcp_flags %x xsum %x seq %u hlen %u\n",
2063 pbd, pbd->global_data, pbd->ip_hlen, pbd->ip_id,
2064 pbd->lso_mss, pbd->tcp_flags, pbd->tcp_pseudo_csum,
2065 pbd->tcp_send_seq, le16_to_cpu(pbd->total_hlen));
2066
2067 DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d bd %u\n", nbd, bd_prod);
2068
2069 /*
2070 * Make sure that the BD data is updated before updating the producer
2071 * since FW might read the BD right after the producer is updated.
2072 * This is only applicable for weak-ordered memory model archs such
2073 * as IA-64. The following barrier is also mandatory since FW will
2074 * assumes packets must have BDs.
2075 */
2076 wmb();
2077
2078 fp->tx_db.data.prod += nbd;
2079 barrier();
2080 DOORBELL(bp, fp->index, fp->tx_db.raw);
2081
2082 mmiowb();
2083
2084 fp->tx_bd_prod += nbd;
2085
2086 if (unlikely(bnx2x_tx_avail(fp) < MAX_SKB_FRAGS + 3)) {
2087 netif_tx_stop_queue(txq);
2088
2089 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
2090 * ordering of set_bit() in netif_tx_stop_queue() and read of
2091 * fp->bd_tx_cons */
2092 smp_mb();
2093
2094 fp->eth_q_stats.driver_xoff++;
2095 if (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3)
2096 netif_tx_wake_queue(txq);
2097 }
2098 fp->tx_pkt++;
2099
2100 return NETDEV_TX_OK;
2101}
2102/* called with rtnl_lock */
2103int bnx2x_change_mac_addr(struct net_device *dev, void *p)
2104{
2105 struct sockaddr *addr = p;
2106 struct bnx2x *bp = netdev_priv(dev);
2107
2108 if (!is_valid_ether_addr((u8 *)(addr->sa_data)))
2109 return -EINVAL;
2110
2111 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2112 if (netif_running(dev)) {
2113 if (CHIP_IS_E1(bp))
2114 bnx2x_set_eth_mac_addr_e1(bp, 1);
2115 else
2116 bnx2x_set_eth_mac_addr_e1h(bp, 1);
2117 }
2118
2119 return 0;
2120}
2121
2122/* called with rtnl_lock */
2123int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
2124{
2125 struct bnx2x *bp = netdev_priv(dev);
2126 int rc = 0;
2127
2128 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
2129 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
2130 return -EAGAIN;
2131 }
2132
2133 if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
2134 ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE))
2135 return -EINVAL;
2136
2137 /* This does not race with packet allocation
2138 * because the actual alloc size is
2139 * only updated as part of load
2140 */
2141 dev->mtu = new_mtu;
2142
2143 if (netif_running(dev)) {
2144 bnx2x_nic_unload(bp, UNLOAD_NORMAL);
2145 rc = bnx2x_nic_load(bp, LOAD_NORMAL);
2146 }
2147
2148 return rc;
2149}
2150
2151void bnx2x_tx_timeout(struct net_device *dev)
2152{
2153 struct bnx2x *bp = netdev_priv(dev);
2154
2155#ifdef BNX2X_STOP_ON_ERROR
2156 if (!bp->panic)
2157 bnx2x_panic();
2158#endif
2159 /* This allows the netif to be shutdown gracefully before resetting */
2160 schedule_delayed_work(&bp->reset_task, 0);
2161}
2162
2163#ifdef BCM_VLAN
2164/* called with rtnl_lock */
2165void bnx2x_vlan_rx_register(struct net_device *dev,
2166 struct vlan_group *vlgrp)
2167{
2168 struct bnx2x *bp = netdev_priv(dev);
2169
2170 bp->vlgrp = vlgrp;
2171
2172 /* Set flags according to the required capabilities */
2173 bp->flags &= ~(HW_VLAN_RX_FLAG | HW_VLAN_TX_FLAG);
2174
2175 if (dev->features & NETIF_F_HW_VLAN_TX)
2176 bp->flags |= HW_VLAN_TX_FLAG;
2177
2178 if (dev->features & NETIF_F_HW_VLAN_RX)
2179 bp->flags |= HW_VLAN_RX_FLAG;
2180
2181 if (netif_running(dev))
2182 bnx2x_set_client_config(bp);
2183}
2184
2185#endif
2186int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
2187{
2188 struct net_device *dev = pci_get_drvdata(pdev);
2189 struct bnx2x *bp;
2190
2191 if (!dev) {
2192 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
2193 return -ENODEV;
2194 }
2195 bp = netdev_priv(dev);
2196
2197 rtnl_lock();
2198
2199 pci_save_state(pdev);
2200
2201 if (!netif_running(dev)) {
2202 rtnl_unlock();
2203 return 0;
2204 }
2205
2206 netif_device_detach(dev);
2207
2208 bnx2x_nic_unload(bp, UNLOAD_CLOSE);
2209
2210 bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
2211
2212 rtnl_unlock();
2213
2214 return 0;
2215}
2216
2217int bnx2x_resume(struct pci_dev *pdev)
2218{
2219 struct net_device *dev = pci_get_drvdata(pdev);
2220 struct bnx2x *bp;
2221 int rc;
2222
2223 if (!dev) {
2224 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
2225 return -ENODEV;
2226 }
2227 bp = netdev_priv(dev);
2228
2229 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
2230 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
2231 return -EAGAIN;
2232 }
2233
2234 rtnl_lock();
2235
2236 pci_restore_state(pdev);
2237
2238 if (!netif_running(dev)) {
2239 rtnl_unlock();
2240 return 0;
2241 }
2242
2243 bnx2x_set_power_state(bp, PCI_D0);
2244 netif_device_attach(dev);
2245
2246 rc = bnx2x_nic_load(bp, LOAD_OPEN);
2247
2248 rtnl_unlock();
2249
2250 return rc;
2251}
Linux 6.x block layer and io_uring tree(s)