2 * Linux driver for VMware's vmxnet3 ethernet NIC.
4 * Copyright (C) 2008-2009, VMware, Inc. All Rights Reserved.
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License as published by the
8 * Free Software Foundation; version 2 of the License and no later version.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
13 * NON INFRINGEMENT. See the GNU General Public License for more
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 * The full GNU General Public License is included in this distribution in
21 * the file called "COPYING".
23 * Maintained by: Shreyas Bhatewara <pv-drivers@vmware.com>
27 #include <linux/module.h>
28 #include <net/ip6_checksum.h>
30 #include "vmxnet3_int.h"
32 char vmxnet3_driver_name[] = "vmxnet3";
33 #define VMXNET3_DRIVER_DESC "VMware vmxnet3 virtual NIC driver"
37 * Last entry must be all 0s
39 static DEFINE_PCI_DEVICE_TABLE(vmxnet3_pciid_table) = {
40 {PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_VMXNET3)},
44 MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table);
46 static int enable_mq = 1;
49 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac);
52 * Enable/Disable the given intr
55 vmxnet3_enable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
57 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 0);
62 vmxnet3_disable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
64 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 1);
69 * Enable/Disable all intrs used by the device
72 vmxnet3_enable_all_intrs(struct vmxnet3_adapter *adapter)
76 for (i = 0; i < adapter->intr.num_intrs; i++)
77 vmxnet3_enable_intr(adapter, i);
78 adapter->shared->devRead.intrConf.intrCtrl &=
79 cpu_to_le32(~VMXNET3_IC_DISABLE_ALL);
84 vmxnet3_disable_all_intrs(struct vmxnet3_adapter *adapter)
88 adapter->shared->devRead.intrConf.intrCtrl |=
89 cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
90 for (i = 0; i < adapter->intr.num_intrs; i++)
91 vmxnet3_disable_intr(adapter, i);
96 vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events)
98 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_ECR, events);
103 vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
110 vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
113 netif_start_subqueue(adapter->netdev, tq - adapter->tx_queue);
118 vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
121 netif_wake_subqueue(adapter->netdev, (tq - adapter->tx_queue));
126 vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
130 netif_stop_subqueue(adapter->netdev, (tq - adapter->tx_queue));
135 * Check the link state. This may start or stop the tx queue.
138 vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue)
144 spin_lock_irqsave(&adapter->cmd_lock, flags);
145 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
146 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
147 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
149 adapter->link_speed = ret >> 16;
150 if (ret & 1) { /* Link is up. */
151 netdev_info(adapter->netdev, "NIC Link is Up %d Mbps\n",
152 adapter->link_speed);
153 netif_carrier_on(adapter->netdev);
156 for (i = 0; i < adapter->num_tx_queues; i++)
157 vmxnet3_tq_start(&adapter->tx_queue[i],
161 netdev_info(adapter->netdev, "NIC Link is Down\n");
162 netif_carrier_off(adapter->netdev);
165 for (i = 0; i < adapter->num_tx_queues; i++)
166 vmxnet3_tq_stop(&adapter->tx_queue[i], adapter);
172 vmxnet3_process_events(struct vmxnet3_adapter *adapter)
176 u32 events = le32_to_cpu(adapter->shared->ecr);
180 vmxnet3_ack_events(adapter, events);
182 /* Check if link state has changed */
183 if (events & VMXNET3_ECR_LINK)
184 vmxnet3_check_link(adapter, true);
186 /* Check if there is an error on xmit/recv queues */
187 if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) {
188 spin_lock_irqsave(&adapter->cmd_lock, flags);
189 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
190 VMXNET3_CMD_GET_QUEUE_STATUS);
191 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
193 for (i = 0; i < adapter->num_tx_queues; i++)
194 if (adapter->tqd_start[i].status.stopped)
195 dev_err(&adapter->netdev->dev,
196 "%s: tq[%d] error 0x%x\n",
197 adapter->netdev->name, i, le32_to_cpu(
198 adapter->tqd_start[i].status.error));
199 for (i = 0; i < adapter->num_rx_queues; i++)
200 if (adapter->rqd_start[i].status.stopped)
201 dev_err(&adapter->netdev->dev,
202 "%s: rq[%d] error 0x%x\n",
203 adapter->netdev->name, i,
204 adapter->rqd_start[i].status.error);
206 schedule_work(&adapter->work);
210 #ifdef __BIG_ENDIAN_BITFIELD
212 * The device expects the bitfields in shared structures to be written in
213 * little endian. When CPU is big endian, the following routines are used to
214 * correctly read and write into ABI.
215 * The general technique used here is : double word bitfields are defined in
216 * opposite order for big endian architecture. Then before reading them in
217 * driver the complete double word is translated using le32_to_cpu. Similarly
218 * After the driver writes into bitfields, cpu_to_le32 is used to translate the
219 * double words into required format.
220 * In order to avoid touching bits in shared structure more than once, temporary
221 * descriptors are used. These are passed as srcDesc to following functions.
223 static void vmxnet3_RxDescToCPU(const struct Vmxnet3_RxDesc *srcDesc,
224 struct Vmxnet3_RxDesc *dstDesc)
226 u32 *src = (u32 *)srcDesc + 2;
227 u32 *dst = (u32 *)dstDesc + 2;
228 dstDesc->addr = le64_to_cpu(srcDesc->addr);
229 *dst = le32_to_cpu(*src);
230 dstDesc->ext1 = le32_to_cpu(srcDesc->ext1);
233 static void vmxnet3_TxDescToLe(const struct Vmxnet3_TxDesc *srcDesc,
234 struct Vmxnet3_TxDesc *dstDesc)
237 u32 *src = (u32 *)(srcDesc + 1);
238 u32 *dst = (u32 *)(dstDesc + 1);
240 /* Working backwards so that the gen bit is set at the end. */
241 for (i = 2; i > 0; i--) {
244 *dst = cpu_to_le32(*src);
249 static void vmxnet3_RxCompToCPU(const struct Vmxnet3_RxCompDesc *srcDesc,
250 struct Vmxnet3_RxCompDesc *dstDesc)
253 u32 *src = (u32 *)srcDesc;
254 u32 *dst = (u32 *)dstDesc;
255 for (i = 0; i < sizeof(struct Vmxnet3_RxCompDesc) / sizeof(u32); i++) {
256 *dst = le32_to_cpu(*src);
263 /* Used to read bitfield values from double words. */
264 static u32 get_bitfield32(const __le32 *bitfield, u32 pos, u32 size)
266 u32 temp = le32_to_cpu(*bitfield);
267 u32 mask = ((1 << size) - 1) << pos;
275 #endif /* __BIG_ENDIAN_BITFIELD */
277 #ifdef __BIG_ENDIAN_BITFIELD
279 # define VMXNET3_TXDESC_GET_GEN(txdesc) get_bitfield32(((const __le32 *) \
280 txdesc) + VMXNET3_TXD_GEN_DWORD_SHIFT, \
281 VMXNET3_TXD_GEN_SHIFT, VMXNET3_TXD_GEN_SIZE)
282 # define VMXNET3_TXDESC_GET_EOP(txdesc) get_bitfield32(((const __le32 *) \
283 txdesc) + VMXNET3_TXD_EOP_DWORD_SHIFT, \
284 VMXNET3_TXD_EOP_SHIFT, VMXNET3_TXD_EOP_SIZE)
285 # define VMXNET3_TCD_GET_GEN(tcd) get_bitfield32(((const __le32 *)tcd) + \
286 VMXNET3_TCD_GEN_DWORD_SHIFT, VMXNET3_TCD_GEN_SHIFT, \
287 VMXNET3_TCD_GEN_SIZE)
288 # define VMXNET3_TCD_GET_TXIDX(tcd) get_bitfield32((const __le32 *)tcd, \
289 VMXNET3_TCD_TXIDX_SHIFT, VMXNET3_TCD_TXIDX_SIZE)
290 # define vmxnet3_getRxComp(dstrcd, rcd, tmp) do { \
292 vmxnet3_RxCompToCPU((rcd), (tmp)); \
294 # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) do { \
296 vmxnet3_RxDescToCPU((rxd), (tmp)); \
301 # define VMXNET3_TXDESC_GET_GEN(txdesc) ((txdesc)->gen)
302 # define VMXNET3_TXDESC_GET_EOP(txdesc) ((txdesc)->eop)
303 # define VMXNET3_TCD_GET_GEN(tcd) ((tcd)->gen)
304 # define VMXNET3_TCD_GET_TXIDX(tcd) ((tcd)->txdIdx)
305 # define vmxnet3_getRxComp(dstrcd, rcd, tmp) (dstrcd) = (rcd)
306 # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) (dstrxd) = (rxd)
308 #endif /* __BIG_ENDIAN_BITFIELD */
312 vmxnet3_unmap_tx_buf(struct vmxnet3_tx_buf_info *tbi,
313 struct pci_dev *pdev)
315 if (tbi->map_type == VMXNET3_MAP_SINGLE)
316 dma_unmap_single(&pdev->dev, tbi->dma_addr, tbi->len,
318 else if (tbi->map_type == VMXNET3_MAP_PAGE)
319 dma_unmap_page(&pdev->dev, tbi->dma_addr, tbi->len,
322 BUG_ON(tbi->map_type != VMXNET3_MAP_NONE);
324 tbi->map_type = VMXNET3_MAP_NONE; /* to help debugging */
329 vmxnet3_unmap_pkt(u32 eop_idx, struct vmxnet3_tx_queue *tq,
330 struct pci_dev *pdev, struct vmxnet3_adapter *adapter)
335 /* no out of order completion */
336 BUG_ON(tq->buf_info[eop_idx].sop_idx != tq->tx_ring.next2comp);
337 BUG_ON(VMXNET3_TXDESC_GET_EOP(&(tq->tx_ring.base[eop_idx].txd)) != 1);
339 skb = tq->buf_info[eop_idx].skb;
341 tq->buf_info[eop_idx].skb = NULL;
343 VMXNET3_INC_RING_IDX_ONLY(eop_idx, tq->tx_ring.size);
345 while (tq->tx_ring.next2comp != eop_idx) {
346 vmxnet3_unmap_tx_buf(tq->buf_info + tq->tx_ring.next2comp,
349 /* update next2comp w/o tx_lock. Since we are marking more,
350 * instead of less, tx ring entries avail, the worst case is
351 * that the tx routine incorrectly re-queues a pkt due to
352 * insufficient tx ring entries.
354 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
358 dev_kfree_skb_any(skb);
364 vmxnet3_tq_tx_complete(struct vmxnet3_tx_queue *tq,
365 struct vmxnet3_adapter *adapter)
368 union Vmxnet3_GenericDesc *gdesc;
370 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
371 while (VMXNET3_TCD_GET_GEN(&gdesc->tcd) == tq->comp_ring.gen) {
372 completed += vmxnet3_unmap_pkt(VMXNET3_TCD_GET_TXIDX(
373 &gdesc->tcd), tq, adapter->pdev,
376 vmxnet3_comp_ring_adv_next2proc(&tq->comp_ring);
377 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
381 spin_lock(&tq->tx_lock);
382 if (unlikely(vmxnet3_tq_stopped(tq, adapter) &&
383 vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) >
384 VMXNET3_WAKE_QUEUE_THRESHOLD(tq) &&
385 netif_carrier_ok(adapter->netdev))) {
386 vmxnet3_tq_wake(tq, adapter);
388 spin_unlock(&tq->tx_lock);
395 vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq,
396 struct vmxnet3_adapter *adapter)
400 while (tq->tx_ring.next2comp != tq->tx_ring.next2fill) {
401 struct vmxnet3_tx_buf_info *tbi;
403 tbi = tq->buf_info + tq->tx_ring.next2comp;
405 vmxnet3_unmap_tx_buf(tbi, adapter->pdev);
407 dev_kfree_skb_any(tbi->skb);
410 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
413 /* sanity check, verify all buffers are indeed unmapped and freed */
414 for (i = 0; i < tq->tx_ring.size; i++) {
415 BUG_ON(tq->buf_info[i].skb != NULL ||
416 tq->buf_info[i].map_type != VMXNET3_MAP_NONE);
419 tq->tx_ring.gen = VMXNET3_INIT_GEN;
420 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
422 tq->comp_ring.gen = VMXNET3_INIT_GEN;
423 tq->comp_ring.next2proc = 0;
428 vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq,
429 struct vmxnet3_adapter *adapter)
431 if (tq->tx_ring.base) {
432 dma_free_coherent(&adapter->pdev->dev, tq->tx_ring.size *
433 sizeof(struct Vmxnet3_TxDesc),
434 tq->tx_ring.base, tq->tx_ring.basePA);
435 tq->tx_ring.base = NULL;
437 if (tq->data_ring.base) {
438 dma_free_coherent(&adapter->pdev->dev, tq->data_ring.size *
439 sizeof(struct Vmxnet3_TxDataDesc),
440 tq->data_ring.base, tq->data_ring.basePA);
441 tq->data_ring.base = NULL;
443 if (tq->comp_ring.base) {
444 dma_free_coherent(&adapter->pdev->dev, tq->comp_ring.size *
445 sizeof(struct Vmxnet3_TxCompDesc),
446 tq->comp_ring.base, tq->comp_ring.basePA);
447 tq->comp_ring.base = NULL;
450 dma_free_coherent(&adapter->pdev->dev,
451 tq->tx_ring.size * sizeof(tq->buf_info[0]),
452 tq->buf_info, tq->buf_info_pa);
458 /* Destroy all tx queues */
460 vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter)
464 for (i = 0; i < adapter->num_tx_queues; i++)
465 vmxnet3_tq_destroy(&adapter->tx_queue[i], adapter);
470 vmxnet3_tq_init(struct vmxnet3_tx_queue *tq,
471 struct vmxnet3_adapter *adapter)
475 /* reset the tx ring contents to 0 and reset the tx ring states */
476 memset(tq->tx_ring.base, 0, tq->tx_ring.size *
477 sizeof(struct Vmxnet3_TxDesc));
478 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
479 tq->tx_ring.gen = VMXNET3_INIT_GEN;
481 memset(tq->data_ring.base, 0, tq->data_ring.size *
482 sizeof(struct Vmxnet3_TxDataDesc));
484 /* reset the tx comp ring contents to 0 and reset comp ring states */
485 memset(tq->comp_ring.base, 0, tq->comp_ring.size *
486 sizeof(struct Vmxnet3_TxCompDesc));
487 tq->comp_ring.next2proc = 0;
488 tq->comp_ring.gen = VMXNET3_INIT_GEN;
490 /* reset the bookkeeping data */
491 memset(tq->buf_info, 0, sizeof(tq->buf_info[0]) * tq->tx_ring.size);
492 for (i = 0; i < tq->tx_ring.size; i++)
493 tq->buf_info[i].map_type = VMXNET3_MAP_NONE;
495 /* stats are not reset */
500 vmxnet3_tq_create(struct vmxnet3_tx_queue *tq,
501 struct vmxnet3_adapter *adapter)
505 BUG_ON(tq->tx_ring.base || tq->data_ring.base ||
506 tq->comp_ring.base || tq->buf_info);
508 tq->tx_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
509 tq->tx_ring.size * sizeof(struct Vmxnet3_TxDesc),
510 &tq->tx_ring.basePA, GFP_KERNEL);
511 if (!tq->tx_ring.base) {
512 netdev_err(adapter->netdev, "failed to allocate tx ring\n");
516 tq->data_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
517 tq->data_ring.size * sizeof(struct Vmxnet3_TxDataDesc),
518 &tq->data_ring.basePA, GFP_KERNEL);
519 if (!tq->data_ring.base) {
520 netdev_err(adapter->netdev, "failed to allocate data ring\n");
524 tq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
525 tq->comp_ring.size * sizeof(struct Vmxnet3_TxCompDesc),
526 &tq->comp_ring.basePA, GFP_KERNEL);
527 if (!tq->comp_ring.base) {
528 netdev_err(adapter->netdev, "failed to allocate tx comp ring\n");
532 sz = tq->tx_ring.size * sizeof(tq->buf_info[0]);
533 tq->buf_info = dma_zalloc_coherent(&adapter->pdev->dev, sz,
534 &tq->buf_info_pa, GFP_KERNEL);
541 vmxnet3_tq_destroy(tq, adapter);
546 vmxnet3_tq_cleanup_all(struct vmxnet3_adapter *adapter)
550 for (i = 0; i < adapter->num_tx_queues; i++)
551 vmxnet3_tq_cleanup(&adapter->tx_queue[i], adapter);
555 * starting from ring->next2fill, allocate rx buffers for the given ring
556 * of the rx queue and update the rx desc. stop after @num_to_alloc buffers
557 * are allocated or allocation fails
561 vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx,
562 int num_to_alloc, struct vmxnet3_adapter *adapter)
564 int num_allocated = 0;
565 struct vmxnet3_rx_buf_info *rbi_base = rq->buf_info[ring_idx];
566 struct vmxnet3_cmd_ring *ring = &rq->rx_ring[ring_idx];
569 while (num_allocated <= num_to_alloc) {
570 struct vmxnet3_rx_buf_info *rbi;
571 union Vmxnet3_GenericDesc *gd;
573 rbi = rbi_base + ring->next2fill;
574 gd = ring->base + ring->next2fill;
576 if (rbi->buf_type == VMXNET3_RX_BUF_SKB) {
577 if (rbi->skb == NULL) {
578 rbi->skb = __netdev_alloc_skb_ip_align(adapter->netdev,
581 if (unlikely(rbi->skb == NULL)) {
582 rq->stats.rx_buf_alloc_failure++;
586 rbi->dma_addr = dma_map_single(
588 rbi->skb->data, rbi->len,
591 /* rx buffer skipped by the device */
593 val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT;
595 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE ||
596 rbi->len != PAGE_SIZE);
598 if (rbi->page == NULL) {
599 rbi->page = alloc_page(GFP_ATOMIC);
600 if (unlikely(rbi->page == NULL)) {
601 rq->stats.rx_buf_alloc_failure++;
604 rbi->dma_addr = dma_map_page(
606 rbi->page, 0, PAGE_SIZE,
609 /* rx buffers skipped by the device */
611 val = VMXNET3_RXD_BTYPE_BODY << VMXNET3_RXD_BTYPE_SHIFT;
614 BUG_ON(rbi->dma_addr == 0);
615 gd->rxd.addr = cpu_to_le64(rbi->dma_addr);
616 gd->dword[2] = cpu_to_le32((!ring->gen << VMXNET3_RXD_GEN_SHIFT)
619 /* Fill the last buffer but dont mark it ready, or else the
620 * device will think that the queue is full */
621 if (num_allocated == num_to_alloc)
624 gd->dword[2] |= cpu_to_le32(ring->gen << VMXNET3_RXD_GEN_SHIFT);
626 vmxnet3_cmd_ring_adv_next2fill(ring);
629 netdev_dbg(adapter->netdev,
630 "alloc_rx_buf: %d allocated, next2fill %u, next2comp %u\n",
631 num_allocated, ring->next2fill, ring->next2comp);
633 /* so that the device can distinguish a full ring and an empty ring */
634 BUG_ON(num_allocated != 0 && ring->next2fill == ring->next2comp);
636 return num_allocated;
641 vmxnet3_append_frag(struct sk_buff *skb, struct Vmxnet3_RxCompDesc *rcd,
642 struct vmxnet3_rx_buf_info *rbi)
644 struct skb_frag_struct *frag = skb_shinfo(skb)->frags +
645 skb_shinfo(skb)->nr_frags;
647 BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS);
649 __skb_frag_set_page(frag, rbi->page);
650 frag->page_offset = 0;
651 skb_frag_size_set(frag, rcd->len);
652 skb->data_len += rcd->len;
653 skb->truesize += PAGE_SIZE;
654 skb_shinfo(skb)->nr_frags++;
659 vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx,
660 struct vmxnet3_tx_queue *tq, struct pci_dev *pdev,
661 struct vmxnet3_adapter *adapter)
664 unsigned long buf_offset;
666 union Vmxnet3_GenericDesc *gdesc;
667 struct vmxnet3_tx_buf_info *tbi = NULL;
669 BUG_ON(ctx->copy_size > skb_headlen(skb));
671 /* use the previous gen bit for the SOP desc */
672 dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT;
674 ctx->sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill;
675 gdesc = ctx->sop_txd; /* both loops below can be skipped */
677 /* no need to map the buffer if headers are copied */
678 if (ctx->copy_size) {
679 ctx->sop_txd->txd.addr = cpu_to_le64(tq->data_ring.basePA +
680 tq->tx_ring.next2fill *
681 sizeof(struct Vmxnet3_TxDataDesc));
682 ctx->sop_txd->dword[2] = cpu_to_le32(dw2 | ctx->copy_size);
683 ctx->sop_txd->dword[3] = 0;
685 tbi = tq->buf_info + tq->tx_ring.next2fill;
686 tbi->map_type = VMXNET3_MAP_NONE;
688 netdev_dbg(adapter->netdev,
689 "txd[%u]: 0x%Lx 0x%x 0x%x\n",
690 tq->tx_ring.next2fill,
691 le64_to_cpu(ctx->sop_txd->txd.addr),
692 ctx->sop_txd->dword[2], ctx->sop_txd->dword[3]);
693 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
695 /* use the right gen for non-SOP desc */
696 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
699 /* linear part can use multiple tx desc if it's big */
700 len = skb_headlen(skb) - ctx->copy_size;
701 buf_offset = ctx->copy_size;
705 if (len < VMXNET3_MAX_TX_BUF_SIZE) {
709 buf_size = VMXNET3_MAX_TX_BUF_SIZE;
710 /* spec says that for TxDesc.len, 0 == 2^14 */
713 tbi = tq->buf_info + tq->tx_ring.next2fill;
714 tbi->map_type = VMXNET3_MAP_SINGLE;
715 tbi->dma_addr = dma_map_single(&adapter->pdev->dev,
716 skb->data + buf_offset, buf_size,
721 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
722 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
724 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);
725 gdesc->dword[2] = cpu_to_le32(dw2);
728 netdev_dbg(adapter->netdev,
729 "txd[%u]: 0x%Lx 0x%x 0x%x\n",
730 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),
731 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);
732 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
733 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
736 buf_offset += buf_size;
739 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
740 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
744 len = skb_frag_size(frag);
746 tbi = tq->buf_info + tq->tx_ring.next2fill;
747 if (len < VMXNET3_MAX_TX_BUF_SIZE) {
751 buf_size = VMXNET3_MAX_TX_BUF_SIZE;
752 /* spec says that for TxDesc.len, 0 == 2^14 */
754 tbi->map_type = VMXNET3_MAP_PAGE;
755 tbi->dma_addr = skb_frag_dma_map(&adapter->pdev->dev, frag,
756 buf_offset, buf_size,
761 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
762 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
764 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);
765 gdesc->dword[2] = cpu_to_le32(dw2);
768 netdev_dbg(adapter->netdev,
769 "txd[%u]: 0x%llu %u %u\n",
770 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),
771 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);
772 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
773 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
776 buf_offset += buf_size;
780 ctx->eop_txd = gdesc;
782 /* set the last buf_info for the pkt */
784 tbi->sop_idx = ctx->sop_txd - tq->tx_ring.base;
788 /* Init all tx queues */
790 vmxnet3_tq_init_all(struct vmxnet3_adapter *adapter)
794 for (i = 0; i < adapter->num_tx_queues; i++)
795 vmxnet3_tq_init(&adapter->tx_queue[i], adapter);
800 * parse and copy relevant protocol headers:
801 * For a tso pkt, relevant headers are L2/3/4 including options
802 * For a pkt requesting csum offloading, they are L2/3 and may include L4
803 * if it's a TCP/UDP pkt
806 * -1: error happens during parsing
807 * 0: protocol headers parsed, but too big to be copied
808 * 1: protocol headers parsed and copied
811 * 1. related *ctx fields are updated.
812 * 2. ctx->copy_size is # of bytes copied
813 * 3. the portion copied is guaranteed to be in the linear part
817 vmxnet3_parse_and_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
818 struct vmxnet3_tx_ctx *ctx,
819 struct vmxnet3_adapter *adapter)
821 struct Vmxnet3_TxDataDesc *tdd;
823 if (ctx->mss) { /* TSO */
824 ctx->eth_ip_hdr_size = skb_transport_offset(skb);
825 ctx->l4_hdr_size = tcp_hdrlen(skb);
826 ctx->copy_size = ctx->eth_ip_hdr_size + ctx->l4_hdr_size;
828 if (skb->ip_summed == CHECKSUM_PARTIAL) {
829 ctx->eth_ip_hdr_size = skb_checksum_start_offset(skb);
832 const struct iphdr *iph = ip_hdr(skb);
834 if (iph->protocol == IPPROTO_TCP)
835 ctx->l4_hdr_size = tcp_hdrlen(skb);
836 else if (iph->protocol == IPPROTO_UDP)
837 ctx->l4_hdr_size = sizeof(struct udphdr);
839 ctx->l4_hdr_size = 0;
841 /* for simplicity, don't copy L4 headers */
842 ctx->l4_hdr_size = 0;
844 ctx->copy_size = min(ctx->eth_ip_hdr_size +
845 ctx->l4_hdr_size, skb->len);
847 ctx->eth_ip_hdr_size = 0;
848 ctx->l4_hdr_size = 0;
849 /* copy as much as allowed */
850 ctx->copy_size = min((unsigned int)VMXNET3_HDR_COPY_SIZE
854 /* make sure headers are accessible directly */
855 if (unlikely(!pskb_may_pull(skb, ctx->copy_size)))
859 if (unlikely(ctx->copy_size > VMXNET3_HDR_COPY_SIZE)) {
860 tq->stats.oversized_hdr++;
865 tdd = tq->data_ring.base + tq->tx_ring.next2fill;
867 memcpy(tdd->data, skb->data, ctx->copy_size);
868 netdev_dbg(adapter->netdev,
869 "copy %u bytes to dataRing[%u]\n",
870 ctx->copy_size, tq->tx_ring.next2fill);
879 vmxnet3_prepare_tso(struct sk_buff *skb,
880 struct vmxnet3_tx_ctx *ctx)
882 struct tcphdr *tcph = tcp_hdr(skb);
885 struct iphdr *iph = ip_hdr(skb);
888 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0,
891 struct ipv6hdr *iph = ipv6_hdr(skb);
893 tcph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, 0,
898 static int txd_estimate(const struct sk_buff *skb)
900 int count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
903 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
904 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
906 count += VMXNET3_TXD_NEEDED(skb_frag_size(frag));
912 * Transmits a pkt thru a given tq
914 * NETDEV_TX_OK: descriptors are setup successfully
915 * NETDEV_TX_OK: error occurred, the pkt is dropped
916 * NETDEV_TX_BUSY: tx ring is full, queue is stopped
919 * 1. tx ring may be changed
920 * 2. tq stats may be updated accordingly
921 * 3. shared->txNumDeferred may be updated
925 vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
926 struct vmxnet3_adapter *adapter, struct net_device *netdev)
931 struct vmxnet3_tx_ctx ctx;
932 union Vmxnet3_GenericDesc *gdesc;
933 #ifdef __BIG_ENDIAN_BITFIELD
934 /* Use temporary descriptor to avoid touching bits multiple times */
935 union Vmxnet3_GenericDesc tempTxDesc;
938 count = txd_estimate(skb);
940 ctx.ipv4 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IP));
942 ctx.mss = skb_shinfo(skb)->gso_size;
944 if (skb_header_cloned(skb)) {
945 if (unlikely(pskb_expand_head(skb, 0, 0,
947 tq->stats.drop_tso++;
950 tq->stats.copy_skb_header++;
952 vmxnet3_prepare_tso(skb, &ctx);
954 if (unlikely(count > VMXNET3_MAX_TXD_PER_PKT)) {
956 /* non-tso pkts must not use more than
957 * VMXNET3_MAX_TXD_PER_PKT entries
959 if (skb_linearize(skb) != 0) {
960 tq->stats.drop_too_many_frags++;
963 tq->stats.linearized++;
965 /* recalculate the # of descriptors to use */
966 count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
970 spin_lock_irqsave(&tq->tx_lock, flags);
972 if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) {
973 tq->stats.tx_ring_full++;
974 netdev_dbg(adapter->netdev,
975 "tx queue stopped on %s, next2comp %u"
976 " next2fill %u\n", adapter->netdev->name,
977 tq->tx_ring.next2comp, tq->tx_ring.next2fill);
979 vmxnet3_tq_stop(tq, adapter);
980 spin_unlock_irqrestore(&tq->tx_lock, flags);
981 return NETDEV_TX_BUSY;
985 ret = vmxnet3_parse_and_copy_hdr(skb, tq, &ctx, adapter);
987 BUG_ON(ret <= 0 && ctx.copy_size != 0);
988 /* hdrs parsed, check against other limits */
990 if (unlikely(ctx.eth_ip_hdr_size + ctx.l4_hdr_size >
991 VMXNET3_MAX_TX_BUF_SIZE)) {
995 if (skb->ip_summed == CHECKSUM_PARTIAL) {
996 if (unlikely(ctx.eth_ip_hdr_size +
998 VMXNET3_MAX_CSUM_OFFSET)) {
1004 tq->stats.drop_hdr_inspect_err++;
1005 goto unlock_drop_pkt;
1008 /* fill tx descs related to addr & len */
1009 vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter);
1011 /* setup the EOP desc */
1012 ctx.eop_txd->dword[3] = cpu_to_le32(VMXNET3_TXD_CQ | VMXNET3_TXD_EOP);
1014 /* setup the SOP desc */
1015 #ifdef __BIG_ENDIAN_BITFIELD
1016 gdesc = &tempTxDesc;
1017 gdesc->dword[2] = ctx.sop_txd->dword[2];
1018 gdesc->dword[3] = ctx.sop_txd->dword[3];
1020 gdesc = ctx.sop_txd;
1023 gdesc->txd.hlen = ctx.eth_ip_hdr_size + ctx.l4_hdr_size;
1024 gdesc->txd.om = VMXNET3_OM_TSO;
1025 gdesc->txd.msscof = ctx.mss;
1026 le32_add_cpu(&tq->shared->txNumDeferred, (skb->len -
1027 gdesc->txd.hlen + ctx.mss - 1) / ctx.mss);
1029 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1030 gdesc->txd.hlen = ctx.eth_ip_hdr_size;
1031 gdesc->txd.om = VMXNET3_OM_CSUM;
1032 gdesc->txd.msscof = ctx.eth_ip_hdr_size +
1036 gdesc->txd.msscof = 0;
1038 le32_add_cpu(&tq->shared->txNumDeferred, 1);
1041 if (vlan_tx_tag_present(skb)) {
1043 gdesc->txd.tci = vlan_tx_tag_get(skb);
1046 /* finally flips the GEN bit of the SOP desc. */
1047 gdesc->dword[2] = cpu_to_le32(le32_to_cpu(gdesc->dword[2]) ^
1049 #ifdef __BIG_ENDIAN_BITFIELD
1050 /* Finished updating in bitfields of Tx Desc, so write them in original
1053 vmxnet3_TxDescToLe((struct Vmxnet3_TxDesc *)gdesc,
1054 (struct Vmxnet3_TxDesc *)ctx.sop_txd);
1055 gdesc = ctx.sop_txd;
1057 netdev_dbg(adapter->netdev,
1058 "txd[%u]: SOP 0x%Lx 0x%x 0x%x\n",
1060 tq->tx_ring.base), le64_to_cpu(gdesc->txd.addr),
1061 le32_to_cpu(gdesc->dword[2]), le32_to_cpu(gdesc->dword[3]));
1063 spin_unlock_irqrestore(&tq->tx_lock, flags);
1065 if (le32_to_cpu(tq->shared->txNumDeferred) >=
1066 le32_to_cpu(tq->shared->txThreshold)) {
1067 tq->shared->txNumDeferred = 0;
1068 VMXNET3_WRITE_BAR0_REG(adapter,
1069 VMXNET3_REG_TXPROD + tq->qid * 8,
1070 tq->tx_ring.next2fill);
1073 return NETDEV_TX_OK;
1076 tq->stats.drop_oversized_hdr++;
1078 spin_unlock_irqrestore(&tq->tx_lock, flags);
1080 tq->stats.drop_total++;
1081 dev_kfree_skb_any(skb);
1082 return NETDEV_TX_OK;
1087 vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
1089 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1091 BUG_ON(skb->queue_mapping > adapter->num_tx_queues);
1092 return vmxnet3_tq_xmit(skb,
1093 &adapter->tx_queue[skb->queue_mapping],
1099 vmxnet3_rx_csum(struct vmxnet3_adapter *adapter,
1100 struct sk_buff *skb,
1101 union Vmxnet3_GenericDesc *gdesc)
1103 if (!gdesc->rcd.cnc && adapter->netdev->features & NETIF_F_RXCSUM) {
1104 /* typical case: TCP/UDP over IP and both csums are correct */
1105 if ((le32_to_cpu(gdesc->dword[3]) & VMXNET3_RCD_CSUM_OK) ==
1106 VMXNET3_RCD_CSUM_OK) {
1107 skb->ip_summed = CHECKSUM_UNNECESSARY;
1108 BUG_ON(!(gdesc->rcd.tcp || gdesc->rcd.udp));
1109 BUG_ON(!(gdesc->rcd.v4 || gdesc->rcd.v6));
1110 BUG_ON(gdesc->rcd.frg);
1112 if (gdesc->rcd.csum) {
1113 skb->csum = htons(gdesc->rcd.csum);
1114 skb->ip_summed = CHECKSUM_PARTIAL;
1116 skb_checksum_none_assert(skb);
1120 skb_checksum_none_assert(skb);
1126 vmxnet3_rx_error(struct vmxnet3_rx_queue *rq, struct Vmxnet3_RxCompDesc *rcd,
1127 struct vmxnet3_rx_ctx *ctx, struct vmxnet3_adapter *adapter)
1129 rq->stats.drop_err++;
1131 rq->stats.drop_fcs++;
1133 rq->stats.drop_total++;
1136 * We do not unmap and chain the rx buffer to the skb.
1137 * We basically pretend this buffer is not used and will be recycled
1138 * by vmxnet3_rq_alloc_rx_buf()
1142 * ctx->skb may be NULL if this is the first and the only one
1146 dev_kfree_skb_irq(ctx->skb);
1153 vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,
1154 struct vmxnet3_adapter *adapter, int quota)
1156 static const u32 rxprod_reg[2] = {
1157 VMXNET3_REG_RXPROD, VMXNET3_REG_RXPROD2
1160 bool skip_page_frags = false;
1161 struct Vmxnet3_RxCompDesc *rcd;
1162 struct vmxnet3_rx_ctx *ctx = &rq->rx_ctx;
1163 #ifdef __BIG_ENDIAN_BITFIELD
1164 struct Vmxnet3_RxDesc rxCmdDesc;
1165 struct Vmxnet3_RxCompDesc rxComp;
1167 vmxnet3_getRxComp(rcd, &rq->comp_ring.base[rq->comp_ring.next2proc].rcd,
1169 while (rcd->gen == rq->comp_ring.gen) {
1170 struct vmxnet3_rx_buf_info *rbi;
1171 struct sk_buff *skb, *new_skb = NULL;
1172 struct page *new_page = NULL;
1174 struct Vmxnet3_RxDesc *rxd;
1176 struct vmxnet3_cmd_ring *ring = NULL;
1177 if (num_rxd >= quota) {
1178 /* we may stop even before we see the EOP desc of
1184 BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2);
1186 ring_idx = rcd->rqID < adapter->num_rx_queues ? 0 : 1;
1187 ring = rq->rx_ring + ring_idx;
1188 vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd,
1190 rbi = rq->buf_info[ring_idx] + idx;
1192 BUG_ON(rxd->addr != rbi->dma_addr ||
1193 rxd->len != rbi->len);
1195 if (unlikely(rcd->eop && rcd->err)) {
1196 vmxnet3_rx_error(rq, rcd, ctx, adapter);
1200 if (rcd->sop) { /* first buf of the pkt */
1201 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_HEAD ||
1202 rcd->rqID != rq->qid);
1204 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_SKB);
1205 BUG_ON(ctx->skb != NULL || rbi->skb == NULL);
1207 if (unlikely(rcd->len == 0)) {
1208 /* Pretend the rx buffer is skipped. */
1209 BUG_ON(!(rcd->sop && rcd->eop));
1210 netdev_dbg(adapter->netdev,
1211 "rxRing[%u][%u] 0 length\n",
1216 skip_page_frags = false;
1217 ctx->skb = rbi->skb;
1218 new_skb = netdev_alloc_skb_ip_align(adapter->netdev,
1220 if (new_skb == NULL) {
1221 /* Skb allocation failed, do not handover this
1222 * skb to stack. Reuse it. Drop the existing pkt
1224 rq->stats.rx_buf_alloc_failure++;
1226 rq->stats.drop_total++;
1227 skip_page_frags = true;
1231 dma_unmap_single(&adapter->pdev->dev, rbi->dma_addr,
1233 PCI_DMA_FROMDEVICE);
1236 if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE &&
1237 (adapter->netdev->features & NETIF_F_RXHASH))
1238 skb_set_hash(ctx->skb,
1239 le32_to_cpu(rcd->rssHash),
1242 skb_put(ctx->skb, rcd->len);
1244 /* Immediate refill */
1246 rbi->dma_addr = dma_map_single(&adapter->pdev->dev,
1247 rbi->skb->data, rbi->len,
1248 PCI_DMA_FROMDEVICE);
1249 rxd->addr = cpu_to_le64(rbi->dma_addr);
1250 rxd->len = rbi->len;
1253 BUG_ON(ctx->skb == NULL && !skip_page_frags);
1255 /* non SOP buffer must be type 1 in most cases */
1256 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE);
1257 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_BODY);
1259 /* If an sop buffer was dropped, skip all
1260 * following non-sop fragments. They will be reused.
1262 if (skip_page_frags)
1265 new_page = alloc_page(GFP_ATOMIC);
1266 if (unlikely(new_page == NULL)) {
1267 /* Replacement page frag could not be allocated.
1268 * Reuse this page. Drop the pkt and free the
1269 * skb which contained this page as a frag. Skip
1270 * processing all the following non-sop frags.
1272 rq->stats.rx_buf_alloc_failure++;
1273 dev_kfree_skb(ctx->skb);
1275 skip_page_frags = true;
1280 dma_unmap_page(&adapter->pdev->dev,
1281 rbi->dma_addr, rbi->len,
1282 PCI_DMA_FROMDEVICE);
1284 vmxnet3_append_frag(ctx->skb, rcd, rbi);
1287 /* Immediate refill */
1288 rbi->page = new_page;
1289 rbi->dma_addr = dma_map_page(&adapter->pdev->dev,
1292 PCI_DMA_FROMDEVICE);
1293 rxd->addr = cpu_to_le64(rbi->dma_addr);
1294 rxd->len = rbi->len;
1300 skb->len += skb->data_len;
1302 vmxnet3_rx_csum(adapter, skb,
1303 (union Vmxnet3_GenericDesc *)rcd);
1304 skb->protocol = eth_type_trans(skb, adapter->netdev);
1306 if (unlikely(rcd->ts))
1307 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rcd->tci);
1309 if (adapter->netdev->features & NETIF_F_LRO)
1310 netif_receive_skb(skb);
1312 napi_gro_receive(&rq->napi, skb);
1318 /* device may have skipped some rx descs */
1319 ring->next2comp = idx;
1320 num_to_alloc = vmxnet3_cmd_ring_desc_avail(ring);
1321 ring = rq->rx_ring + ring_idx;
1322 while (num_to_alloc) {
1323 vmxnet3_getRxDesc(rxd, &ring->base[ring->next2fill].rxd,
1327 /* Recv desc is ready to be used by the device */
1328 rxd->gen = ring->gen;
1329 vmxnet3_cmd_ring_adv_next2fill(ring);
1333 /* if needed, update the register */
1334 if (unlikely(rq->shared->updateRxProd)) {
1335 VMXNET3_WRITE_BAR0_REG(adapter,
1336 rxprod_reg[ring_idx] + rq->qid * 8,
1340 vmxnet3_comp_ring_adv_next2proc(&rq->comp_ring);
1341 vmxnet3_getRxComp(rcd,
1342 &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, &rxComp);
1350 vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq,
1351 struct vmxnet3_adapter *adapter)
1354 struct Vmxnet3_RxDesc *rxd;
1356 for (ring_idx = 0; ring_idx < 2; ring_idx++) {
1357 for (i = 0; i < rq->rx_ring[ring_idx].size; i++) {
1358 #ifdef __BIG_ENDIAN_BITFIELD
1359 struct Vmxnet3_RxDesc rxDesc;
1361 vmxnet3_getRxDesc(rxd,
1362 &rq->rx_ring[ring_idx].base[i].rxd, &rxDesc);
1364 if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD &&
1365 rq->buf_info[ring_idx][i].skb) {
1366 dma_unmap_single(&adapter->pdev->dev, rxd->addr,
1367 rxd->len, PCI_DMA_FROMDEVICE);
1368 dev_kfree_skb(rq->buf_info[ring_idx][i].skb);
1369 rq->buf_info[ring_idx][i].skb = NULL;
1370 } else if (rxd->btype == VMXNET3_RXD_BTYPE_BODY &&
1371 rq->buf_info[ring_idx][i].page) {
1372 dma_unmap_page(&adapter->pdev->dev, rxd->addr,
1373 rxd->len, PCI_DMA_FROMDEVICE);
1374 put_page(rq->buf_info[ring_idx][i].page);
1375 rq->buf_info[ring_idx][i].page = NULL;
1379 rq->rx_ring[ring_idx].gen = VMXNET3_INIT_GEN;
1380 rq->rx_ring[ring_idx].next2fill =
1381 rq->rx_ring[ring_idx].next2comp = 0;
1384 rq->comp_ring.gen = VMXNET3_INIT_GEN;
1385 rq->comp_ring.next2proc = 0;
1390 vmxnet3_rq_cleanup_all(struct vmxnet3_adapter *adapter)
1394 for (i = 0; i < adapter->num_rx_queues; i++)
1395 vmxnet3_rq_cleanup(&adapter->rx_queue[i], adapter);
1399 static void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq,
1400 struct vmxnet3_adapter *adapter)
1405 /* all rx buffers must have already been freed */
1406 for (i = 0; i < 2; i++) {
1407 if (rq->buf_info[i]) {
1408 for (j = 0; j < rq->rx_ring[i].size; j++)
1409 BUG_ON(rq->buf_info[i][j].page != NULL);
1414 for (i = 0; i < 2; i++) {
1415 if (rq->rx_ring[i].base) {
1416 dma_free_coherent(&adapter->pdev->dev,
1418 * sizeof(struct Vmxnet3_RxDesc),
1419 rq->rx_ring[i].base,
1420 rq->rx_ring[i].basePA);
1421 rq->rx_ring[i].base = NULL;
1423 rq->buf_info[i] = NULL;
1426 if (rq->comp_ring.base) {
1427 dma_free_coherent(&adapter->pdev->dev, rq->comp_ring.size
1428 * sizeof(struct Vmxnet3_RxCompDesc),
1429 rq->comp_ring.base, rq->comp_ring.basePA);
1430 rq->comp_ring.base = NULL;
1433 if (rq->buf_info[0]) {
1434 size_t sz = sizeof(struct vmxnet3_rx_buf_info) *
1435 (rq->rx_ring[0].size + rq->rx_ring[1].size);
1436 dma_free_coherent(&adapter->pdev->dev, sz, rq->buf_info[0],
1443 vmxnet3_rq_init(struct vmxnet3_rx_queue *rq,
1444 struct vmxnet3_adapter *adapter)
1448 /* initialize buf_info */
1449 for (i = 0; i < rq->rx_ring[0].size; i++) {
1451 /* 1st buf for a pkt is skbuff */
1452 if (i % adapter->rx_buf_per_pkt == 0) {
1453 rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_SKB;
1454 rq->buf_info[0][i].len = adapter->skb_buf_size;
1455 } else { /* subsequent bufs for a pkt is frag */
1456 rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_PAGE;
1457 rq->buf_info[0][i].len = PAGE_SIZE;
1460 for (i = 0; i < rq->rx_ring[1].size; i++) {
1461 rq->buf_info[1][i].buf_type = VMXNET3_RX_BUF_PAGE;
1462 rq->buf_info[1][i].len = PAGE_SIZE;
1465 /* reset internal state and allocate buffers for both rings */
1466 for (i = 0; i < 2; i++) {
1467 rq->rx_ring[i].next2fill = rq->rx_ring[i].next2comp = 0;
1469 memset(rq->rx_ring[i].base, 0, rq->rx_ring[i].size *
1470 sizeof(struct Vmxnet3_RxDesc));
1471 rq->rx_ring[i].gen = VMXNET3_INIT_GEN;
1473 if (vmxnet3_rq_alloc_rx_buf(rq, 0, rq->rx_ring[0].size - 1,
1475 /* at least has 1 rx buffer for the 1st ring */
1478 vmxnet3_rq_alloc_rx_buf(rq, 1, rq->rx_ring[1].size - 1, adapter);
1480 /* reset the comp ring */
1481 rq->comp_ring.next2proc = 0;
1482 memset(rq->comp_ring.base, 0, rq->comp_ring.size *
1483 sizeof(struct Vmxnet3_RxCompDesc));
1484 rq->comp_ring.gen = VMXNET3_INIT_GEN;
1487 rq->rx_ctx.skb = NULL;
1489 /* stats are not reset */
1495 vmxnet3_rq_init_all(struct vmxnet3_adapter *adapter)
1499 for (i = 0; i < adapter->num_rx_queues; i++) {
1500 err = vmxnet3_rq_init(&adapter->rx_queue[i], adapter);
1501 if (unlikely(err)) {
1502 dev_err(&adapter->netdev->dev, "%s: failed to "
1503 "initialize rx queue%i\n",
1504 adapter->netdev->name, i);
1514 vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter)
1518 struct vmxnet3_rx_buf_info *bi;
1520 for (i = 0; i < 2; i++) {
1522 sz = rq->rx_ring[i].size * sizeof(struct Vmxnet3_RxDesc);
1523 rq->rx_ring[i].base = dma_alloc_coherent(
1524 &adapter->pdev->dev, sz,
1525 &rq->rx_ring[i].basePA,
1527 if (!rq->rx_ring[i].base) {
1528 netdev_err(adapter->netdev,
1529 "failed to allocate rx ring %d\n", i);
1534 sz = rq->comp_ring.size * sizeof(struct Vmxnet3_RxCompDesc);
1535 rq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, sz,
1536 &rq->comp_ring.basePA,
1538 if (!rq->comp_ring.base) {
1539 netdev_err(adapter->netdev, "failed to allocate rx comp ring\n");
1543 sz = sizeof(struct vmxnet3_rx_buf_info) * (rq->rx_ring[0].size +
1544 rq->rx_ring[1].size);
1545 bi = dma_zalloc_coherent(&adapter->pdev->dev, sz, &rq->buf_info_pa,
1550 rq->buf_info[0] = bi;
1551 rq->buf_info[1] = bi + rq->rx_ring[0].size;
1556 vmxnet3_rq_destroy(rq, adapter);
1562 vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter)
1566 for (i = 0; i < adapter->num_rx_queues; i++) {
1567 err = vmxnet3_rq_create(&adapter->rx_queue[i], adapter);
1568 if (unlikely(err)) {
1569 dev_err(&adapter->netdev->dev,
1570 "%s: failed to create rx queue%i\n",
1571 adapter->netdev->name, i);
1577 vmxnet3_rq_destroy_all(adapter);
1582 /* Multiple queue aware polling function for tx and rx */
1585 vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget)
1587 int rcd_done = 0, i;
1588 if (unlikely(adapter->shared->ecr))
1589 vmxnet3_process_events(adapter);
1590 for (i = 0; i < adapter->num_tx_queues; i++)
1591 vmxnet3_tq_tx_complete(&adapter->tx_queue[i], adapter);
1593 for (i = 0; i < adapter->num_rx_queues; i++)
1594 rcd_done += vmxnet3_rq_rx_complete(&adapter->rx_queue[i],
1601 vmxnet3_poll(struct napi_struct *napi, int budget)
1603 struct vmxnet3_rx_queue *rx_queue = container_of(napi,
1604 struct vmxnet3_rx_queue, napi);
1607 rxd_done = vmxnet3_do_poll(rx_queue->adapter, budget);
1609 if (rxd_done < budget) {
1610 napi_complete(napi);
1611 vmxnet3_enable_all_intrs(rx_queue->adapter);
1617 * NAPI polling function for MSI-X mode with multiple Rx queues
1618 * Returns the # of the NAPI credit consumed (# of rx descriptors processed)
1622 vmxnet3_poll_rx_only(struct napi_struct *napi, int budget)
1624 struct vmxnet3_rx_queue *rq = container_of(napi,
1625 struct vmxnet3_rx_queue, napi);
1626 struct vmxnet3_adapter *adapter = rq->adapter;
1629 /* When sharing interrupt with corresponding tx queue, process
1630 * tx completions in that queue as well
1632 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) {
1633 struct vmxnet3_tx_queue *tq =
1634 &adapter->tx_queue[rq - adapter->rx_queue];
1635 vmxnet3_tq_tx_complete(tq, adapter);
1638 rxd_done = vmxnet3_rq_rx_complete(rq, adapter, budget);
1640 if (rxd_done < budget) {
1641 napi_complete(napi);
1642 vmxnet3_enable_intr(adapter, rq->comp_ring.intr_idx);
1648 #ifdef CONFIG_PCI_MSI
1651 * Handle completion interrupts on tx queues
1652 * Returns whether or not the intr is handled
1656 vmxnet3_msix_tx(int irq, void *data)
1658 struct vmxnet3_tx_queue *tq = data;
1659 struct vmxnet3_adapter *adapter = tq->adapter;
1661 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1662 vmxnet3_disable_intr(adapter, tq->comp_ring.intr_idx);
1664 /* Handle the case where only one irq is allocate for all tx queues */
1665 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
1667 for (i = 0; i < adapter->num_tx_queues; i++) {
1668 struct vmxnet3_tx_queue *txq = &adapter->tx_queue[i];
1669 vmxnet3_tq_tx_complete(txq, adapter);
1672 vmxnet3_tq_tx_complete(tq, adapter);
1674 vmxnet3_enable_intr(adapter, tq->comp_ring.intr_idx);
1681 * Handle completion interrupts on rx queues. Returns whether or not the
1686 vmxnet3_msix_rx(int irq, void *data)
1688 struct vmxnet3_rx_queue *rq = data;
1689 struct vmxnet3_adapter *adapter = rq->adapter;
1691 /* disable intr if needed */
1692 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1693 vmxnet3_disable_intr(adapter, rq->comp_ring.intr_idx);
1694 napi_schedule(&rq->napi);
1700 *----------------------------------------------------------------------------
1702 * vmxnet3_msix_event --
1704 * vmxnet3 msix event intr handler
1707 * whether or not the intr is handled
1709 *----------------------------------------------------------------------------
1713 vmxnet3_msix_event(int irq, void *data)
1715 struct net_device *dev = data;
1716 struct vmxnet3_adapter *adapter = netdev_priv(dev);
1718 /* disable intr if needed */
1719 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1720 vmxnet3_disable_intr(adapter, adapter->intr.event_intr_idx);
1722 if (adapter->shared->ecr)
1723 vmxnet3_process_events(adapter);
1725 vmxnet3_enable_intr(adapter, adapter->intr.event_intr_idx);
1730 #endif /* CONFIG_PCI_MSI */
1733 /* Interrupt handler for vmxnet3 */
1735 vmxnet3_intr(int irq, void *dev_id)
1737 struct net_device *dev = dev_id;
1738 struct vmxnet3_adapter *adapter = netdev_priv(dev);
1740 if (adapter->intr.type == VMXNET3_IT_INTX) {
1741 u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR);
1742 if (unlikely(icr == 0))
1748 /* disable intr if needed */
1749 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1750 vmxnet3_disable_all_intrs(adapter);
1752 napi_schedule(&adapter->rx_queue[0].napi);
1757 #ifdef CONFIG_NET_POLL_CONTROLLER
1759 /* netpoll callback. */
1761 vmxnet3_netpoll(struct net_device *netdev)
1763 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1765 switch (adapter->intr.type) {
1766 #ifdef CONFIG_PCI_MSI
1767 case VMXNET3_IT_MSIX: {
1769 for (i = 0; i < adapter->num_rx_queues; i++)
1770 vmxnet3_msix_rx(0, &adapter->rx_queue[i]);
1774 case VMXNET3_IT_MSI:
1776 vmxnet3_intr(0, adapter->netdev);
1781 #endif /* CONFIG_NET_POLL_CONTROLLER */
1784 vmxnet3_request_irqs(struct vmxnet3_adapter *adapter)
1786 struct vmxnet3_intr *intr = &adapter->intr;
1790 #ifdef CONFIG_PCI_MSI
1791 if (adapter->intr.type == VMXNET3_IT_MSIX) {
1792 for (i = 0; i < adapter->num_tx_queues; i++) {
1793 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
1794 sprintf(adapter->tx_queue[i].name, "%s-tx-%d",
1795 adapter->netdev->name, vector);
1797 intr->msix_entries[vector].vector,
1799 adapter->tx_queue[i].name,
1800 &adapter->tx_queue[i]);
1802 sprintf(adapter->tx_queue[i].name, "%s-rxtx-%d",
1803 adapter->netdev->name, vector);
1806 dev_err(&adapter->netdev->dev,
1807 "Failed to request irq for MSIX, %s, "
1809 adapter->tx_queue[i].name, err);
1813 /* Handle the case where only 1 MSIx was allocated for
1815 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
1816 for (; i < adapter->num_tx_queues; i++)
1817 adapter->tx_queue[i].comp_ring.intr_idx
1822 adapter->tx_queue[i].comp_ring.intr_idx
1826 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE)
1829 for (i = 0; i < adapter->num_rx_queues; i++) {
1830 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE)
1831 sprintf(adapter->rx_queue[i].name, "%s-rx-%d",
1832 adapter->netdev->name, vector);
1834 sprintf(adapter->rx_queue[i].name, "%s-rxtx-%d",
1835 adapter->netdev->name, vector);
1836 err = request_irq(intr->msix_entries[vector].vector,
1838 adapter->rx_queue[i].name,
1839 &(adapter->rx_queue[i]));
1841 netdev_err(adapter->netdev,
1842 "Failed to request irq for MSIX, "
1844 adapter->rx_queue[i].name, err);
1848 adapter->rx_queue[i].comp_ring.intr_idx = vector++;
1851 sprintf(intr->event_msi_vector_name, "%s-event-%d",
1852 adapter->netdev->name, vector);
1853 err = request_irq(intr->msix_entries[vector].vector,
1854 vmxnet3_msix_event, 0,
1855 intr->event_msi_vector_name, adapter->netdev);
1856 intr->event_intr_idx = vector;
1858 } else if (intr->type == VMXNET3_IT_MSI) {
1859 adapter->num_rx_queues = 1;
1860 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0,
1861 adapter->netdev->name, adapter->netdev);
1864 adapter->num_rx_queues = 1;
1865 err = request_irq(adapter->pdev->irq, vmxnet3_intr,
1866 IRQF_SHARED, adapter->netdev->name,
1868 #ifdef CONFIG_PCI_MSI
1871 intr->num_intrs = vector + 1;
1873 netdev_err(adapter->netdev,
1874 "Failed to request irq (intr type:%d), error %d\n",
1877 /* Number of rx queues will not change after this */
1878 for (i = 0; i < adapter->num_rx_queues; i++) {
1879 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
1881 rq->qid2 = i + adapter->num_rx_queues;
1886 /* init our intr settings */
1887 for (i = 0; i < intr->num_intrs; i++)
1888 intr->mod_levels[i] = UPT1_IML_ADAPTIVE;
1889 if (adapter->intr.type != VMXNET3_IT_MSIX) {
1890 adapter->intr.event_intr_idx = 0;
1891 for (i = 0; i < adapter->num_tx_queues; i++)
1892 adapter->tx_queue[i].comp_ring.intr_idx = 0;
1893 adapter->rx_queue[0].comp_ring.intr_idx = 0;
1896 netdev_info(adapter->netdev,
1897 "intr type %u, mode %u, %u vectors allocated\n",
1898 intr->type, intr->mask_mode, intr->num_intrs);
1906 vmxnet3_free_irqs(struct vmxnet3_adapter *adapter)
1908 struct vmxnet3_intr *intr = &adapter->intr;
1909 BUG_ON(intr->type == VMXNET3_IT_AUTO || intr->num_intrs <= 0);
1911 switch (intr->type) {
1912 #ifdef CONFIG_PCI_MSI
1913 case VMXNET3_IT_MSIX:
1917 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
1918 for (i = 0; i < adapter->num_tx_queues; i++) {
1919 free_irq(intr->msix_entries[vector++].vector,
1920 &(adapter->tx_queue[i]));
1921 if (adapter->share_intr == VMXNET3_INTR_TXSHARE)
1926 for (i = 0; i < adapter->num_rx_queues; i++) {
1927 free_irq(intr->msix_entries[vector++].vector,
1928 &(adapter->rx_queue[i]));
1931 free_irq(intr->msix_entries[vector].vector,
1933 BUG_ON(vector >= intr->num_intrs);
1937 case VMXNET3_IT_MSI:
1938 free_irq(adapter->pdev->irq, adapter->netdev);
1940 case VMXNET3_IT_INTX:
1941 free_irq(adapter->pdev->irq, adapter->netdev);
1950 vmxnet3_restore_vlan(struct vmxnet3_adapter *adapter)
1952 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1955 /* allow untagged pkts */
1956 VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0);
1958 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
1959 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
1964 vmxnet3_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
1966 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1968 if (!(netdev->flags & IFF_PROMISC)) {
1969 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1970 unsigned long flags;
1972 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
1973 spin_lock_irqsave(&adapter->cmd_lock, flags);
1974 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1975 VMXNET3_CMD_UPDATE_VLAN_FILTERS);
1976 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
1979 set_bit(vid, adapter->active_vlans);
1986 vmxnet3_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)
1988 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1990 if (!(netdev->flags & IFF_PROMISC)) {
1991 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1992 unsigned long flags;
1994 VMXNET3_CLEAR_VFTABLE_ENTRY(vfTable, vid);
1995 spin_lock_irqsave(&adapter->cmd_lock, flags);
1996 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1997 VMXNET3_CMD_UPDATE_VLAN_FILTERS);
1998 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2001 clear_bit(vid, adapter->active_vlans);
2008 vmxnet3_copy_mc(struct net_device *netdev)
2011 u32 sz = netdev_mc_count(netdev) * ETH_ALEN;
2013 /* struct Vmxnet3_RxFilterConf.mfTableLen is u16. */
2015 /* We may be called with BH disabled */
2016 buf = kmalloc(sz, GFP_ATOMIC);
2018 struct netdev_hw_addr *ha;
2021 netdev_for_each_mc_addr(ha, netdev)
2022 memcpy(buf + i++ * ETH_ALEN, ha->addr,
2031 vmxnet3_set_mc(struct net_device *netdev)
2033 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2034 unsigned long flags;
2035 struct Vmxnet3_RxFilterConf *rxConf =
2036 &adapter->shared->devRead.rxFilterConf;
2037 u8 *new_table = NULL;
2038 dma_addr_t new_table_pa = 0;
2039 u32 new_mode = VMXNET3_RXM_UCAST;
2041 if (netdev->flags & IFF_PROMISC) {
2042 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
2043 memset(vfTable, 0, VMXNET3_VFT_SIZE * sizeof(*vfTable));
2045 new_mode |= VMXNET3_RXM_PROMISC;
2047 vmxnet3_restore_vlan(adapter);
2050 if (netdev->flags & IFF_BROADCAST)
2051 new_mode |= VMXNET3_RXM_BCAST;
2053 if (netdev->flags & IFF_ALLMULTI)
2054 new_mode |= VMXNET3_RXM_ALL_MULTI;
2056 if (!netdev_mc_empty(netdev)) {
2057 new_table = vmxnet3_copy_mc(netdev);
2059 new_mode |= VMXNET3_RXM_MCAST;
2060 rxConf->mfTableLen = cpu_to_le16(
2061 netdev_mc_count(netdev) * ETH_ALEN);
2062 new_table_pa = dma_map_single(
2063 &adapter->pdev->dev,
2067 rxConf->mfTablePA = cpu_to_le64(new_table_pa);
2069 netdev_info(netdev, "failed to copy mcast list"
2070 ", setting ALL_MULTI\n");
2071 new_mode |= VMXNET3_RXM_ALL_MULTI;
2076 if (!(new_mode & VMXNET3_RXM_MCAST)) {
2077 rxConf->mfTableLen = 0;
2078 rxConf->mfTablePA = 0;
2081 spin_lock_irqsave(&adapter->cmd_lock, flags);
2082 if (new_mode != rxConf->rxMode) {
2083 rxConf->rxMode = cpu_to_le32(new_mode);
2084 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2085 VMXNET3_CMD_UPDATE_RX_MODE);
2086 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2087 VMXNET3_CMD_UPDATE_VLAN_FILTERS);
2090 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2091 VMXNET3_CMD_UPDATE_MAC_FILTERS);
2092 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2095 dma_unmap_single(&adapter->pdev->dev, new_table_pa,
2096 rxConf->mfTableLen, PCI_DMA_TODEVICE);
2102 vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter)
2106 for (i = 0; i < adapter->num_rx_queues; i++)
2107 vmxnet3_rq_destroy(&adapter->rx_queue[i], adapter);
2112 * Set up driver_shared based on settings in adapter.
2116 vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter)
2118 struct Vmxnet3_DriverShared *shared = adapter->shared;
2119 struct Vmxnet3_DSDevRead *devRead = &shared->devRead;
2120 struct Vmxnet3_TxQueueConf *tqc;
2121 struct Vmxnet3_RxQueueConf *rqc;
2124 memset(shared, 0, sizeof(*shared));
2126 /* driver settings */
2127 shared->magic = cpu_to_le32(VMXNET3_REV1_MAGIC);
2128 devRead->misc.driverInfo.version = cpu_to_le32(
2129 VMXNET3_DRIVER_VERSION_NUM);
2130 devRead->misc.driverInfo.gos.gosBits = (sizeof(void *) == 4 ?
2131 VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64);
2132 devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX;
2133 *((u32 *)&devRead->misc.driverInfo.gos) = cpu_to_le32(
2134 *((u32 *)&devRead->misc.driverInfo.gos));
2135 devRead->misc.driverInfo.vmxnet3RevSpt = cpu_to_le32(1);
2136 devRead->misc.driverInfo.uptVerSpt = cpu_to_le32(1);
2138 devRead->misc.ddPA = cpu_to_le64(adapter->adapter_pa);
2139 devRead->misc.ddLen = cpu_to_le32(sizeof(struct vmxnet3_adapter));
2141 /* set up feature flags */
2142 if (adapter->netdev->features & NETIF_F_RXCSUM)
2143 devRead->misc.uptFeatures |= UPT1_F_RXCSUM;
2145 if (adapter->netdev->features & NETIF_F_LRO) {
2146 devRead->misc.uptFeatures |= UPT1_F_LRO;
2147 devRead->misc.maxNumRxSG = cpu_to_le16(1 + MAX_SKB_FRAGS);
2149 if (adapter->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
2150 devRead->misc.uptFeatures |= UPT1_F_RXVLAN;
2152 devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu);
2153 devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa);
2154 devRead->misc.queueDescLen = cpu_to_le32(
2155 adapter->num_tx_queues * sizeof(struct Vmxnet3_TxQueueDesc) +
2156 adapter->num_rx_queues * sizeof(struct Vmxnet3_RxQueueDesc));
2158 /* tx queue settings */
2159 devRead->misc.numTxQueues = adapter->num_tx_queues;
2160 for (i = 0; i < adapter->num_tx_queues; i++) {
2161 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i];
2162 BUG_ON(adapter->tx_queue[i].tx_ring.base == NULL);
2163 tqc = &adapter->tqd_start[i].conf;
2164 tqc->txRingBasePA = cpu_to_le64(tq->tx_ring.basePA);
2165 tqc->dataRingBasePA = cpu_to_le64(tq->data_ring.basePA);
2166 tqc->compRingBasePA = cpu_to_le64(tq->comp_ring.basePA);
2167 tqc->ddPA = cpu_to_le64(tq->buf_info_pa);
2168 tqc->txRingSize = cpu_to_le32(tq->tx_ring.size);
2169 tqc->dataRingSize = cpu_to_le32(tq->data_ring.size);
2170 tqc->compRingSize = cpu_to_le32(tq->comp_ring.size);
2171 tqc->ddLen = cpu_to_le32(
2172 sizeof(struct vmxnet3_tx_buf_info) *
2174 tqc->intrIdx = tq->comp_ring.intr_idx;
2177 /* rx queue settings */
2178 devRead->misc.numRxQueues = adapter->num_rx_queues;
2179 for (i = 0; i < adapter->num_rx_queues; i++) {
2180 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
2181 rqc = &adapter->rqd_start[i].conf;
2182 rqc->rxRingBasePA[0] = cpu_to_le64(rq->rx_ring[0].basePA);
2183 rqc->rxRingBasePA[1] = cpu_to_le64(rq->rx_ring[1].basePA);
2184 rqc->compRingBasePA = cpu_to_le64(rq->comp_ring.basePA);
2185 rqc->ddPA = cpu_to_le64(rq->buf_info_pa);
2186 rqc->rxRingSize[0] = cpu_to_le32(rq->rx_ring[0].size);
2187 rqc->rxRingSize[1] = cpu_to_le32(rq->rx_ring[1].size);
2188 rqc->compRingSize = cpu_to_le32(rq->comp_ring.size);
2189 rqc->ddLen = cpu_to_le32(
2190 sizeof(struct vmxnet3_rx_buf_info) *
2191 (rqc->rxRingSize[0] +
2192 rqc->rxRingSize[1]));
2193 rqc->intrIdx = rq->comp_ring.intr_idx;
2197 memset(adapter->rss_conf, 0, sizeof(*adapter->rss_conf));
2200 struct UPT1_RSSConf *rssConf = adapter->rss_conf;
2201 static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = {
2202 0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac,
2203 0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28,
2204 0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70,
2205 0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3,
2206 0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9,
2209 devRead->misc.uptFeatures |= UPT1_F_RSS;
2210 devRead->misc.numRxQueues = adapter->num_rx_queues;
2211 rssConf->hashType = UPT1_RSS_HASH_TYPE_TCP_IPV4 |
2212 UPT1_RSS_HASH_TYPE_IPV4 |
2213 UPT1_RSS_HASH_TYPE_TCP_IPV6 |
2214 UPT1_RSS_HASH_TYPE_IPV6;
2215 rssConf->hashFunc = UPT1_RSS_HASH_FUNC_TOEPLITZ;
2216 rssConf->hashKeySize = UPT1_RSS_MAX_KEY_SIZE;
2217 rssConf->indTableSize = VMXNET3_RSS_IND_TABLE_SIZE;
2218 memcpy(rssConf->hashKey, rss_key, sizeof(rss_key));
2220 for (i = 0; i < rssConf->indTableSize; i++)
2221 rssConf->indTable[i] = ethtool_rxfh_indir_default(
2222 i, adapter->num_rx_queues);
2224 devRead->rssConfDesc.confVer = 1;
2225 devRead->rssConfDesc.confLen = cpu_to_le32(sizeof(*rssConf));
2226 devRead->rssConfDesc.confPA =
2227 cpu_to_le64(adapter->rss_conf_pa);
2230 #endif /* VMXNET3_RSS */
2233 devRead->intrConf.autoMask = adapter->intr.mask_mode ==
2235 devRead->intrConf.numIntrs = adapter->intr.num_intrs;
2236 for (i = 0; i < adapter->intr.num_intrs; i++)
2237 devRead->intrConf.modLevels[i] = adapter->intr.mod_levels[i];
2239 devRead->intrConf.eventIntrIdx = adapter->intr.event_intr_idx;
2240 devRead->intrConf.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
2242 /* rx filter settings */
2243 devRead->rxFilterConf.rxMode = 0;
2244 vmxnet3_restore_vlan(adapter);
2245 vmxnet3_write_mac_addr(adapter, adapter->netdev->dev_addr);
2247 /* the rest are already zeroed */
2252 vmxnet3_activate_dev(struct vmxnet3_adapter *adapter)
2256 unsigned long flags;
2258 netdev_dbg(adapter->netdev, "%s: skb_buf_size %d, rx_buf_per_pkt %d,"
2259 " ring sizes %u %u %u\n", adapter->netdev->name,
2260 adapter->skb_buf_size, adapter->rx_buf_per_pkt,
2261 adapter->tx_queue[0].tx_ring.size,
2262 adapter->rx_queue[0].rx_ring[0].size,
2263 adapter->rx_queue[0].rx_ring[1].size);
2265 vmxnet3_tq_init_all(adapter);
2266 err = vmxnet3_rq_init_all(adapter);
2268 netdev_err(adapter->netdev,
2269 "Failed to init rx queue error %d\n", err);
2273 err = vmxnet3_request_irqs(adapter);
2275 netdev_err(adapter->netdev,
2276 "Failed to setup irq for error %d\n", err);
2280 vmxnet3_setup_driver_shared(adapter);
2282 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, VMXNET3_GET_ADDR_LO(
2283 adapter->shared_pa));
2284 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, VMXNET3_GET_ADDR_HI(
2285 adapter->shared_pa));
2286 spin_lock_irqsave(&adapter->cmd_lock, flags);
2287 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2288 VMXNET3_CMD_ACTIVATE_DEV);
2289 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2290 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2293 netdev_err(adapter->netdev,
2294 "Failed to activate dev: error %u\n", ret);
2299 for (i = 0; i < adapter->num_rx_queues; i++) {
2300 VMXNET3_WRITE_BAR0_REG(adapter,
2301 VMXNET3_REG_RXPROD + i * VMXNET3_REG_ALIGN,
2302 adapter->rx_queue[i].rx_ring[0].next2fill);
2303 VMXNET3_WRITE_BAR0_REG(adapter, (VMXNET3_REG_RXPROD2 +
2304 (i * VMXNET3_REG_ALIGN)),
2305 adapter->rx_queue[i].rx_ring[1].next2fill);
2308 /* Apply the rx filter settins last. */
2309 vmxnet3_set_mc(adapter->netdev);
2312 * Check link state when first activating device. It will start the
2313 * tx queue if the link is up.
2315 vmxnet3_check_link(adapter, true);
2316 for (i = 0; i < adapter->num_rx_queues; i++)
2317 napi_enable(&adapter->rx_queue[i].napi);
2318 vmxnet3_enable_all_intrs(adapter);
2319 clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
2323 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, 0);
2324 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, 0);
2325 vmxnet3_free_irqs(adapter);
2328 /* free up buffers we allocated */
2329 vmxnet3_rq_cleanup_all(adapter);
2335 vmxnet3_reset_dev(struct vmxnet3_adapter *adapter)
2337 unsigned long flags;
2338 spin_lock_irqsave(&adapter->cmd_lock, flags);
2339 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
2340 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2345 vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter)
2348 unsigned long flags;
2349 if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state))
2353 spin_lock_irqsave(&adapter->cmd_lock, flags);
2354 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2355 VMXNET3_CMD_QUIESCE_DEV);
2356 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2357 vmxnet3_disable_all_intrs(adapter);
2359 for (i = 0; i < adapter->num_rx_queues; i++)
2360 napi_disable(&adapter->rx_queue[i].napi);
2361 netif_tx_disable(adapter->netdev);
2362 adapter->link_speed = 0;
2363 netif_carrier_off(adapter->netdev);
2365 vmxnet3_tq_cleanup_all(adapter);
2366 vmxnet3_rq_cleanup_all(adapter);
2367 vmxnet3_free_irqs(adapter);
2373 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
2378 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACL, tmp);
2380 tmp = (mac[5] << 8) | mac[4];
2381 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACH, tmp);
2386 vmxnet3_set_mac_addr(struct net_device *netdev, void *p)
2388 struct sockaddr *addr = p;
2389 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2391 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2392 vmxnet3_write_mac_addr(adapter, addr->sa_data);
2398 /* ==================== initialization and cleanup routines ============ */
2401 vmxnet3_alloc_pci_resources(struct vmxnet3_adapter *adapter, bool *dma64)
2404 unsigned long mmio_start, mmio_len;
2405 struct pci_dev *pdev = adapter->pdev;
2407 err = pci_enable_device(pdev);
2409 dev_err(&pdev->dev, "Failed to enable adapter: error %d\n", err);
2413 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) == 0) {
2414 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)) != 0) {
2416 "pci_set_consistent_dma_mask failed\n");
2422 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(32)) != 0) {
2424 "pci_set_dma_mask failed\n");
2431 err = pci_request_selected_regions(pdev, (1 << 2) - 1,
2432 vmxnet3_driver_name);
2435 "Failed to request region for adapter: error %d\n", err);
2439 pci_set_master(pdev);
2441 mmio_start = pci_resource_start(pdev, 0);
2442 mmio_len = pci_resource_len(pdev, 0);
2443 adapter->hw_addr0 = ioremap(mmio_start, mmio_len);
2444 if (!adapter->hw_addr0) {
2445 dev_err(&pdev->dev, "Failed to map bar0\n");
2450 mmio_start = pci_resource_start(pdev, 1);
2451 mmio_len = pci_resource_len(pdev, 1);
2452 adapter->hw_addr1 = ioremap(mmio_start, mmio_len);
2453 if (!adapter->hw_addr1) {
2454 dev_err(&pdev->dev, "Failed to map bar1\n");
2461 iounmap(adapter->hw_addr0);
2463 pci_release_selected_regions(pdev, (1 << 2) - 1);
2465 pci_disable_device(pdev);
2471 vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter)
2473 BUG_ON(!adapter->pdev);
2475 iounmap(adapter->hw_addr0);
2476 iounmap(adapter->hw_addr1);
2477 pci_release_selected_regions(adapter->pdev, (1 << 2) - 1);
2478 pci_disable_device(adapter->pdev);
2483 vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter)
2485 size_t sz, i, ring0_size, ring1_size, comp_size;
2486 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[0];
2489 if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE -
2490 VMXNET3_MAX_ETH_HDR_SIZE) {
2491 adapter->skb_buf_size = adapter->netdev->mtu +
2492 VMXNET3_MAX_ETH_HDR_SIZE;
2493 if (adapter->skb_buf_size < VMXNET3_MIN_T0_BUF_SIZE)
2494 adapter->skb_buf_size = VMXNET3_MIN_T0_BUF_SIZE;
2496 adapter->rx_buf_per_pkt = 1;
2498 adapter->skb_buf_size = VMXNET3_MAX_SKB_BUF_SIZE;
2499 sz = adapter->netdev->mtu - VMXNET3_MAX_SKB_BUF_SIZE +
2500 VMXNET3_MAX_ETH_HDR_SIZE;
2501 adapter->rx_buf_per_pkt = 1 + (sz + PAGE_SIZE - 1) / PAGE_SIZE;
2505 * for simplicity, force the ring0 size to be a multiple of
2506 * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN
2508 sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN;
2509 ring0_size = adapter->rx_queue[0].rx_ring[0].size;
2510 ring0_size = (ring0_size + sz - 1) / sz * sz;
2511 ring0_size = min_t(u32, ring0_size, VMXNET3_RX_RING_MAX_SIZE /
2513 ring1_size = adapter->rx_queue[0].rx_ring[1].size;
2514 comp_size = ring0_size + ring1_size;
2516 for (i = 0; i < adapter->num_rx_queues; i++) {
2517 rq = &adapter->rx_queue[i];
2518 rq->rx_ring[0].size = ring0_size;
2519 rq->rx_ring[1].size = ring1_size;
2520 rq->comp_ring.size = comp_size;
2526 vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size,
2527 u32 rx_ring_size, u32 rx_ring2_size)
2531 for (i = 0; i < adapter->num_tx_queues; i++) {
2532 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i];
2533 tq->tx_ring.size = tx_ring_size;
2534 tq->data_ring.size = tx_ring_size;
2535 tq->comp_ring.size = tx_ring_size;
2536 tq->shared = &adapter->tqd_start[i].ctrl;
2538 tq->adapter = adapter;
2540 err = vmxnet3_tq_create(tq, adapter);
2542 * Too late to change num_tx_queues. We cannot do away with
2543 * lesser number of queues than what we asked for
2549 adapter->rx_queue[0].rx_ring[0].size = rx_ring_size;
2550 adapter->rx_queue[0].rx_ring[1].size = rx_ring2_size;
2551 vmxnet3_adjust_rx_ring_size(adapter);
2552 for (i = 0; i < adapter->num_rx_queues; i++) {
2553 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
2554 /* qid and qid2 for rx queues will be assigned later when num
2555 * of rx queues is finalized after allocating intrs */
2556 rq->shared = &adapter->rqd_start[i].ctrl;
2557 rq->adapter = adapter;
2558 err = vmxnet3_rq_create(rq, adapter);
2561 netdev_err(adapter->netdev,
2562 "Could not allocate any rx queues. "
2566 netdev_info(adapter->netdev,
2567 "Number of rx queues changed "
2569 adapter->num_rx_queues = i;
2577 vmxnet3_tq_destroy_all(adapter);
2582 vmxnet3_open(struct net_device *netdev)
2584 struct vmxnet3_adapter *adapter;
2587 adapter = netdev_priv(netdev);
2589 for (i = 0; i < adapter->num_tx_queues; i++)
2590 spin_lock_init(&adapter->tx_queue[i].tx_lock);
2592 err = vmxnet3_create_queues(adapter, VMXNET3_DEF_TX_RING_SIZE,
2593 VMXNET3_DEF_RX_RING_SIZE,
2594 VMXNET3_DEF_RX_RING_SIZE);
2598 err = vmxnet3_activate_dev(adapter);
2605 vmxnet3_rq_destroy_all(adapter);
2606 vmxnet3_tq_destroy_all(adapter);
2613 vmxnet3_close(struct net_device *netdev)
2615 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2618 * Reset_work may be in the middle of resetting the device, wait for its
2621 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
2624 vmxnet3_quiesce_dev(adapter);
2626 vmxnet3_rq_destroy_all(adapter);
2627 vmxnet3_tq_destroy_all(adapter);
2629 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2637 vmxnet3_force_close(struct vmxnet3_adapter *adapter)
2642 * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise
2643 * vmxnet3_close() will deadlock.
2645 BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state));
2647 /* we need to enable NAPI, otherwise dev_close will deadlock */
2648 for (i = 0; i < adapter->num_rx_queues; i++)
2649 napi_enable(&adapter->rx_queue[i].napi);
2650 dev_close(adapter->netdev);
2655 vmxnet3_change_mtu(struct net_device *netdev, int new_mtu)
2657 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2660 if (new_mtu < VMXNET3_MIN_MTU || new_mtu > VMXNET3_MAX_MTU)
2663 netdev->mtu = new_mtu;
2666 * Reset_work may be in the middle of resetting the device, wait for its
2669 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
2672 if (netif_running(netdev)) {
2673 vmxnet3_quiesce_dev(adapter);
2674 vmxnet3_reset_dev(adapter);
2676 /* we need to re-create the rx queue based on the new mtu */
2677 vmxnet3_rq_destroy_all(adapter);
2678 vmxnet3_adjust_rx_ring_size(adapter);
2679 err = vmxnet3_rq_create_all(adapter);
2682 "failed to re-create rx queues, "
2683 " error %d. Closing it.\n", err);
2687 err = vmxnet3_activate_dev(adapter);
2690 "failed to re-activate, error %d. "
2691 "Closing it\n", err);
2697 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2699 vmxnet3_force_close(adapter);
2706 vmxnet3_declare_features(struct vmxnet3_adapter *adapter, bool dma64)
2708 struct net_device *netdev = adapter->netdev;
2710 netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM |
2711 NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX |
2712 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 |
2715 netdev->hw_features |= NETIF_F_HIGHDMA;
2716 netdev->vlan_features = netdev->hw_features &
2717 ~(NETIF_F_HW_VLAN_CTAG_TX |
2718 NETIF_F_HW_VLAN_CTAG_RX);
2719 netdev->features = netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
2724 vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
2728 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACL);
2731 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACH);
2732 mac[4] = tmp & 0xff;
2733 mac[5] = (tmp >> 8) & 0xff;
2736 #ifdef CONFIG_PCI_MSI
2739 * Enable MSIx vectors.
2741 * VMXNET3_LINUX_MIN_MSIX_VECT when only minimum number of vectors required
2743 * number of vectors which were enabled otherwise (this number is greater
2744 * than VMXNET3_LINUX_MIN_MSIX_VECT)
2748 vmxnet3_acquire_msix_vectors(struct vmxnet3_adapter *adapter, int nvec)
2750 int ret = pci_enable_msix_range(adapter->pdev,
2751 adapter->intr.msix_entries, nvec, nvec);
2753 if (ret == -ENOSPC && nvec > VMXNET3_LINUX_MIN_MSIX_VECT) {
2754 dev_err(&adapter->netdev->dev,
2755 "Failed to enable %d MSI-X, trying %d\n",
2756 nvec, VMXNET3_LINUX_MIN_MSIX_VECT);
2758 ret = pci_enable_msix_range(adapter->pdev,
2759 adapter->intr.msix_entries,
2760 VMXNET3_LINUX_MIN_MSIX_VECT,
2761 VMXNET3_LINUX_MIN_MSIX_VECT);
2765 dev_err(&adapter->netdev->dev,
2766 "Failed to enable MSI-X, error: %d\n", ret);
2773 #endif /* CONFIG_PCI_MSI */
2776 vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter)
2779 unsigned long flags;
2782 spin_lock_irqsave(&adapter->cmd_lock, flags);
2783 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2784 VMXNET3_CMD_GET_CONF_INTR);
2785 cfg = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2786 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2787 adapter->intr.type = cfg & 0x3;
2788 adapter->intr.mask_mode = (cfg >> 2) & 0x3;
2790 if (adapter->intr.type == VMXNET3_IT_AUTO) {
2791 adapter->intr.type = VMXNET3_IT_MSIX;
2794 #ifdef CONFIG_PCI_MSI
2795 if (adapter->intr.type == VMXNET3_IT_MSIX) {
2798 nvec = adapter->share_intr == VMXNET3_INTR_TXSHARE ?
2799 1 : adapter->num_tx_queues;
2800 nvec += adapter->share_intr == VMXNET3_INTR_BUDDYSHARE ?
2801 0 : adapter->num_rx_queues;
2802 nvec += 1; /* for link event */
2803 nvec = nvec > VMXNET3_LINUX_MIN_MSIX_VECT ?
2804 nvec : VMXNET3_LINUX_MIN_MSIX_VECT;
2806 for (i = 0; i < nvec; i++)
2807 adapter->intr.msix_entries[i].entry = i;
2809 nvec = vmxnet3_acquire_msix_vectors(adapter, nvec);
2813 /* If we cannot allocate one MSIx vector per queue
2814 * then limit the number of rx queues to 1
2816 if (nvec == VMXNET3_LINUX_MIN_MSIX_VECT) {
2817 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE
2818 || adapter->num_rx_queues != 1) {
2819 adapter->share_intr = VMXNET3_INTR_TXSHARE;
2820 netdev_err(adapter->netdev,
2821 "Number of rx queues : 1\n");
2822 adapter->num_rx_queues = 1;
2826 adapter->intr.num_intrs = nvec;
2830 /* If we cannot allocate MSIx vectors use only one rx queue */
2831 dev_info(&adapter->pdev->dev,
2832 "Failed to enable MSI-X, error %d. "
2833 "Limiting #rx queues to 1, try MSI.\n", nvec);
2835 adapter->intr.type = VMXNET3_IT_MSI;
2838 if (adapter->intr.type == VMXNET3_IT_MSI) {
2839 if (!pci_enable_msi(adapter->pdev)) {
2840 adapter->num_rx_queues = 1;
2841 adapter->intr.num_intrs = 1;
2845 #endif /* CONFIG_PCI_MSI */
2847 adapter->num_rx_queues = 1;
2848 dev_info(&adapter->netdev->dev,
2849 "Using INTx interrupt, #Rx queues: 1.\n");
2850 adapter->intr.type = VMXNET3_IT_INTX;
2852 /* INT-X related setting */
2853 adapter->intr.num_intrs = 1;
2858 vmxnet3_free_intr_resources(struct vmxnet3_adapter *adapter)
2860 if (adapter->intr.type == VMXNET3_IT_MSIX)
2861 pci_disable_msix(adapter->pdev);
2862 else if (adapter->intr.type == VMXNET3_IT_MSI)
2863 pci_disable_msi(adapter->pdev);
2865 BUG_ON(adapter->intr.type != VMXNET3_IT_INTX);
2870 vmxnet3_tx_timeout(struct net_device *netdev)
2872 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2873 adapter->tx_timeout_count++;
2875 netdev_err(adapter->netdev, "tx hang\n");
2876 schedule_work(&adapter->work);
2877 netif_wake_queue(adapter->netdev);
2882 vmxnet3_reset_work(struct work_struct *data)
2884 struct vmxnet3_adapter *adapter;
2886 adapter = container_of(data, struct vmxnet3_adapter, work);
2888 /* if another thread is resetting the device, no need to proceed */
2889 if (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
2892 /* if the device is closed, we must leave it alone */
2894 if (netif_running(adapter->netdev)) {
2895 netdev_notice(adapter->netdev, "resetting\n");
2896 vmxnet3_quiesce_dev(adapter);
2897 vmxnet3_reset_dev(adapter);
2898 vmxnet3_activate_dev(adapter);
2900 netdev_info(adapter->netdev, "already closed\n");
2904 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2909 vmxnet3_probe_device(struct pci_dev *pdev,
2910 const struct pci_device_id *id)
2912 static const struct net_device_ops vmxnet3_netdev_ops = {
2913 .ndo_open = vmxnet3_open,
2914 .ndo_stop = vmxnet3_close,
2915 .ndo_start_xmit = vmxnet3_xmit_frame,
2916 .ndo_set_mac_address = vmxnet3_set_mac_addr,
2917 .ndo_change_mtu = vmxnet3_change_mtu,
2918 .ndo_set_features = vmxnet3_set_features,
2919 .ndo_get_stats64 = vmxnet3_get_stats64,
2920 .ndo_tx_timeout = vmxnet3_tx_timeout,
2921 .ndo_set_rx_mode = vmxnet3_set_mc,
2922 .ndo_vlan_rx_add_vid = vmxnet3_vlan_rx_add_vid,
2923 .ndo_vlan_rx_kill_vid = vmxnet3_vlan_rx_kill_vid,
2924 #ifdef CONFIG_NET_POLL_CONTROLLER
2925 .ndo_poll_controller = vmxnet3_netpoll,
2929 bool dma64 = false; /* stupid gcc */
2931 struct net_device *netdev;
2932 struct vmxnet3_adapter *adapter;
2938 if (!pci_msi_enabled())
2943 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
2944 (int)num_online_cpus());
2948 num_rx_queues = rounddown_pow_of_two(num_rx_queues);
2951 num_tx_queues = min(VMXNET3_DEVICE_MAX_TX_QUEUES,
2952 (int)num_online_cpus());
2956 num_tx_queues = rounddown_pow_of_two(num_tx_queues);
2957 netdev = alloc_etherdev_mq(sizeof(struct vmxnet3_adapter),
2958 max(num_tx_queues, num_rx_queues));
2959 dev_info(&pdev->dev,
2960 "# of Tx queues : %d, # of Rx queues : %d\n",
2961 num_tx_queues, num_rx_queues);
2966 pci_set_drvdata(pdev, netdev);
2967 adapter = netdev_priv(netdev);
2968 adapter->netdev = netdev;
2969 adapter->pdev = pdev;
2971 spin_lock_init(&adapter->cmd_lock);
2972 adapter->adapter_pa = dma_map_single(&adapter->pdev->dev, adapter,
2973 sizeof(struct vmxnet3_adapter),
2975 adapter->shared = dma_alloc_coherent(
2976 &adapter->pdev->dev,
2977 sizeof(struct Vmxnet3_DriverShared),
2978 &adapter->shared_pa, GFP_KERNEL);
2979 if (!adapter->shared) {
2980 dev_err(&pdev->dev, "Failed to allocate memory\n");
2982 goto err_alloc_shared;
2985 adapter->num_rx_queues = num_rx_queues;
2986 adapter->num_tx_queues = num_tx_queues;
2987 adapter->rx_buf_per_pkt = 1;
2989 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
2990 size += sizeof(struct Vmxnet3_RxQueueDesc) * adapter->num_rx_queues;
2991 adapter->tqd_start = dma_alloc_coherent(&adapter->pdev->dev, size,
2992 &adapter->queue_desc_pa,
2995 if (!adapter->tqd_start) {
2996 dev_err(&pdev->dev, "Failed to allocate memory\n");
2998 goto err_alloc_queue_desc;
3000 adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start +
3001 adapter->num_tx_queues);
3003 adapter->pm_conf = dma_alloc_coherent(&adapter->pdev->dev,
3004 sizeof(struct Vmxnet3_PMConf),
3005 &adapter->pm_conf_pa,
3007 if (adapter->pm_conf == NULL) {
3014 adapter->rss_conf = dma_alloc_coherent(&adapter->pdev->dev,
3015 sizeof(struct UPT1_RSSConf),
3016 &adapter->rss_conf_pa,
3018 if (adapter->rss_conf == NULL) {
3022 #endif /* VMXNET3_RSS */
3024 err = vmxnet3_alloc_pci_resources(adapter, &dma64);
3028 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS);
3030 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_VRRS, 1);
3033 "Incompatible h/w version (0x%x) for adapter\n", ver);
3038 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_UVRS);
3040 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_UVRS, 1);
3043 "Incompatible upt version (0x%x) for adapter\n", ver);
3048 SET_NETDEV_DEV(netdev, &pdev->dev);
3049 vmxnet3_declare_features(adapter, dma64);
3051 if (adapter->num_tx_queues == adapter->num_rx_queues)
3052 adapter->share_intr = VMXNET3_INTR_BUDDYSHARE;
3054 adapter->share_intr = VMXNET3_INTR_DONTSHARE;
3056 vmxnet3_alloc_intr_resources(adapter);
3059 if (adapter->num_rx_queues > 1 &&
3060 adapter->intr.type == VMXNET3_IT_MSIX) {
3061 adapter->rss = true;
3062 netdev->hw_features |= NETIF_F_RXHASH;
3063 netdev->features |= NETIF_F_RXHASH;
3064 dev_dbg(&pdev->dev, "RSS is enabled.\n");
3066 adapter->rss = false;
3070 vmxnet3_read_mac_addr(adapter, mac);
3071 memcpy(netdev->dev_addr, mac, netdev->addr_len);
3073 netdev->netdev_ops = &vmxnet3_netdev_ops;
3074 vmxnet3_set_ethtool_ops(netdev);
3075 netdev->watchdog_timeo = 5 * HZ;
3077 INIT_WORK(&adapter->work, vmxnet3_reset_work);
3078 set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
3080 if (adapter->intr.type == VMXNET3_IT_MSIX) {
3082 for (i = 0; i < adapter->num_rx_queues; i++) {
3083 netif_napi_add(adapter->netdev,
3084 &adapter->rx_queue[i].napi,
3085 vmxnet3_poll_rx_only, 64);
3088 netif_napi_add(adapter->netdev, &adapter->rx_queue[0].napi,
3092 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
3093 netif_set_real_num_rx_queues(adapter->netdev, adapter->num_rx_queues);
3095 netif_carrier_off(netdev);
3096 err = register_netdev(netdev);
3099 dev_err(&pdev->dev, "Failed to register adapter\n");
3103 vmxnet3_check_link(adapter, false);
3107 vmxnet3_free_intr_resources(adapter);
3109 vmxnet3_free_pci_resources(adapter);
3112 dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf),
3113 adapter->rss_conf, adapter->rss_conf_pa);
3116 dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf),
3117 adapter->pm_conf, adapter->pm_conf_pa);
3119 dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start,
3120 adapter->queue_desc_pa);
3121 err_alloc_queue_desc:
3122 dma_free_coherent(&adapter->pdev->dev,
3123 sizeof(struct Vmxnet3_DriverShared),
3124 adapter->shared, adapter->shared_pa);
3126 dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa,
3127 sizeof(struct vmxnet3_adapter), PCI_DMA_TODEVICE);
3128 free_netdev(netdev);
3134 vmxnet3_remove_device(struct pci_dev *pdev)
3136 struct net_device *netdev = pci_get_drvdata(pdev);
3137 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3143 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
3144 (int)num_online_cpus());
3148 num_rx_queues = rounddown_pow_of_two(num_rx_queues);
3150 cancel_work_sync(&adapter->work);
3152 unregister_netdev(netdev);
3154 vmxnet3_free_intr_resources(adapter);
3155 vmxnet3_free_pci_resources(adapter);
3157 dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf),
3158 adapter->rss_conf, adapter->rss_conf_pa);
3160 dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf),
3161 adapter->pm_conf, adapter->pm_conf_pa);
3163 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
3164 size += sizeof(struct Vmxnet3_RxQueueDesc) * num_rx_queues;
3165 dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start,
3166 adapter->queue_desc_pa);
3167 dma_free_coherent(&adapter->pdev->dev,
3168 sizeof(struct Vmxnet3_DriverShared),
3169 adapter->shared, adapter->shared_pa);
3170 dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa,
3171 sizeof(struct vmxnet3_adapter), PCI_DMA_TODEVICE);
3172 free_netdev(netdev);
3179 vmxnet3_suspend(struct device *device)
3181 struct pci_dev *pdev = to_pci_dev(device);
3182 struct net_device *netdev = pci_get_drvdata(pdev);
3183 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3184 struct Vmxnet3_PMConf *pmConf;
3185 struct ethhdr *ehdr;
3186 struct arphdr *ahdr;
3188 struct in_device *in_dev;
3189 struct in_ifaddr *ifa;
3190 unsigned long flags;
3193 if (!netif_running(netdev))
3196 for (i = 0; i < adapter->num_rx_queues; i++)
3197 napi_disable(&adapter->rx_queue[i].napi);
3199 vmxnet3_disable_all_intrs(adapter);
3200 vmxnet3_free_irqs(adapter);
3201 vmxnet3_free_intr_resources(adapter);
3203 netif_device_detach(netdev);
3204 netif_tx_stop_all_queues(netdev);
3206 /* Create wake-up filters. */
3207 pmConf = adapter->pm_conf;
3208 memset(pmConf, 0, sizeof(*pmConf));
3210 if (adapter->wol & WAKE_UCAST) {
3211 pmConf->filters[i].patternSize = ETH_ALEN;
3212 pmConf->filters[i].maskSize = 1;
3213 memcpy(pmConf->filters[i].pattern, netdev->dev_addr, ETH_ALEN);
3214 pmConf->filters[i].mask[0] = 0x3F; /* LSB ETH_ALEN bits */
3216 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
3220 if (adapter->wol & WAKE_ARP) {
3221 in_dev = in_dev_get(netdev);
3225 ifa = (struct in_ifaddr *)in_dev->ifa_list;
3229 pmConf->filters[i].patternSize = ETH_HLEN + /* Ethernet header*/
3230 sizeof(struct arphdr) + /* ARP header */
3231 2 * ETH_ALEN + /* 2 Ethernet addresses*/
3232 2 * sizeof(u32); /*2 IPv4 addresses */
3233 pmConf->filters[i].maskSize =
3234 (pmConf->filters[i].patternSize - 1) / 8 + 1;
3236 /* ETH_P_ARP in Ethernet header. */
3237 ehdr = (struct ethhdr *)pmConf->filters[i].pattern;
3238 ehdr->h_proto = htons(ETH_P_ARP);
3240 /* ARPOP_REQUEST in ARP header. */
3241 ahdr = (struct arphdr *)&pmConf->filters[i].pattern[ETH_HLEN];
3242 ahdr->ar_op = htons(ARPOP_REQUEST);
3243 arpreq = (u8 *)(ahdr + 1);
3245 /* The Unicast IPv4 address in 'tip' field. */
3246 arpreq += 2 * ETH_ALEN + sizeof(u32);
3247 *(u32 *)arpreq = ifa->ifa_address;
3249 /* The mask for the relevant bits. */
3250 pmConf->filters[i].mask[0] = 0x00;
3251 pmConf->filters[i].mask[1] = 0x30; /* ETH_P_ARP */
3252 pmConf->filters[i].mask[2] = 0x30; /* ARPOP_REQUEST */
3253 pmConf->filters[i].mask[3] = 0x00;
3254 pmConf->filters[i].mask[4] = 0xC0; /* IPv4 TIP */
3255 pmConf->filters[i].mask[5] = 0x03; /* IPv4 TIP */
3258 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
3263 if (adapter->wol & WAKE_MAGIC)
3264 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_MAGIC;
3266 pmConf->numFilters = i;
3268 adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1);
3269 adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof(
3271 adapter->shared->devRead.pmConfDesc.confPA =
3272 cpu_to_le64(adapter->pm_conf_pa);
3274 spin_lock_irqsave(&adapter->cmd_lock, flags);
3275 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3276 VMXNET3_CMD_UPDATE_PMCFG);
3277 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3279 pci_save_state(pdev);
3280 pci_enable_wake(pdev, pci_choose_state(pdev, PMSG_SUSPEND),
3282 pci_disable_device(pdev);
3283 pci_set_power_state(pdev, pci_choose_state(pdev, PMSG_SUSPEND));
3290 vmxnet3_resume(struct device *device)
3293 unsigned long flags;
3294 struct pci_dev *pdev = to_pci_dev(device);
3295 struct net_device *netdev = pci_get_drvdata(pdev);
3296 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3297 struct Vmxnet3_PMConf *pmConf;
3299 if (!netif_running(netdev))
3302 /* Destroy wake-up filters. */
3303 pmConf = adapter->pm_conf;
3304 memset(pmConf, 0, sizeof(*pmConf));
3306 adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1);
3307 adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof(
3309 adapter->shared->devRead.pmConfDesc.confPA =
3310 cpu_to_le64(adapter->pm_conf_pa);
3312 netif_device_attach(netdev);
3313 pci_set_power_state(pdev, PCI_D0);
3314 pci_restore_state(pdev);
3315 err = pci_enable_device_mem(pdev);
3319 pci_enable_wake(pdev, PCI_D0, 0);
3321 spin_lock_irqsave(&adapter->cmd_lock, flags);
3322 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3323 VMXNET3_CMD_UPDATE_PMCFG);
3324 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3325 vmxnet3_alloc_intr_resources(adapter);
3326 vmxnet3_request_irqs(adapter);
3327 for (i = 0; i < adapter->num_rx_queues; i++)
3328 napi_enable(&adapter->rx_queue[i].napi);
3329 vmxnet3_enable_all_intrs(adapter);
3334 static const struct dev_pm_ops vmxnet3_pm_ops = {
3335 .suspend = vmxnet3_suspend,
3336 .resume = vmxnet3_resume,
3340 static struct pci_driver vmxnet3_driver = {
3341 .name = vmxnet3_driver_name,
3342 .id_table = vmxnet3_pciid_table,
3343 .probe = vmxnet3_probe_device,
3344 .remove = vmxnet3_remove_device,
3346 .driver.pm = &vmxnet3_pm_ops,
3352 vmxnet3_init_module(void)
3354 pr_info("%s - version %s\n", VMXNET3_DRIVER_DESC,
3355 VMXNET3_DRIVER_VERSION_REPORT);
3356 return pci_register_driver(&vmxnet3_driver);
3359 module_init(vmxnet3_init_module);
3363 vmxnet3_exit_module(void)
3365 pci_unregister_driver(&vmxnet3_driver);
3368 module_exit(vmxnet3_exit_module);
3370 MODULE_AUTHOR("VMware, Inc.");
3371 MODULE_DESCRIPTION(VMXNET3_DRIVER_DESC);
3372 MODULE_LICENSE("GPL v2");
3373 MODULE_VERSION(VMXNET3_DRIVER_VERSION_STRING);