projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
vmxnet3: use static RSS key
[linux-2.6-block.git] / drivers / net / vmxnet3 / vmxnet3_drv.c
CommitLineData
d1a890fa
SB		 1/*
2 * Linux driver for VMware's vmxnet3 ethernet NIC.
3 *
4 * Copyright (C) 2008-2009, VMware, Inc. All Rights Reserved.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License as published by the
8 * Free Software Foundation; version 2 of the License and no later version.
9 *
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
13 * NON INFRINGEMENT. See the GNU General Public License for more
14 * details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * The full GNU General Public License is included in this distribution in
21 * the file called "COPYING".
22 *
23 * Maintained by: Shreyas Bhatewara <pv-drivers@vmware.com>
24 *
25 */
26
9d9779e7 27#include <linux/module.h>
b038b040
SR		 28#include <net/ip6_checksum.h>
29
d1a890fa
SB		 30#include "vmxnet3_int.h"
31
32char vmxnet3_driver_name[] = "vmxnet3";
33#define VMXNET3_DRIVER_DESC "VMware vmxnet3 virtual NIC driver"
34
d1a890fa
SB		 35/*
36 * PCI Device ID Table
37 * Last entry must be all 0s
38 */
a3aa1884 39static DEFINE_PCI_DEVICE_TABLE(vmxnet3_pciid_table) = {
d1a890fa
SB		 40 {PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_VMXNET3)},
41 {0}
42};
43
44MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table);
45
09c5088e 46static int enable_mq = 1;
d1a890fa 47
f9f25026
SB		 48static void
49vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac);
50
d1a890fa
SB		 51/*
52 * Enable/Disable the given intr
53 */
54static void
55vmxnet3_enable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
56{
57 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 0);
58}
59
60
61static void
62vmxnet3_disable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
63{
64 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 1);
65}
66
67
68/*
69 * Enable/Disable all intrs used by the device
70 */
71static void
72vmxnet3_enable_all_intrs(struct vmxnet3_adapter *adapter)
73{
74 int i;
75
76 for (i = 0; i < adapter->intr.num_intrs; i++)
77 vmxnet3_enable_intr(adapter, i);
6929fe8a
RZ		 78 adapter->shared->devRead.intrConf.intrCtrl &=
79 cpu_to_le32(~VMXNET3_IC_DISABLE_ALL);
d1a890fa
SB		 80}
81
82
83static void
84vmxnet3_disable_all_intrs(struct vmxnet3_adapter *adapter)
85{
86 int i;
87
6929fe8a
RZ		 88 adapter->shared->devRead.intrConf.intrCtrl |=
89 cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
d1a890fa
SB		 90 for (i = 0; i < adapter->intr.num_intrs; i++)
91 vmxnet3_disable_intr(adapter, i);
92}
93
94
95static void
96vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events)
97{
98 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_ECR, events);
99}
100
101
102static bool
103vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
104{
09c5088e 105 return tq->stopped;
d1a890fa
SB		 106}
107
108
109static void
110vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
111{
112 tq->stopped = false;
09c5088e 113 netif_start_subqueue(adapter->netdev, tq - adapter->tx_queue);
d1a890fa
SB		 114}
115
116
117static void
118vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
119{
120 tq->stopped = false;
09c5088e 121 netif_wake_subqueue(adapter->netdev, (tq - adapter->tx_queue));
d1a890fa
SB		 122}
123
124
125static void
126vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
127{
128 tq->stopped = true;
129 tq->num_stop++;
09c5088e 130 netif_stop_subqueue(adapter->netdev, (tq - adapter->tx_queue));
d1a890fa
SB		 131}
132
133
134/*
135 * Check the link state. This may start or stop the tx queue.
136 */
137static void
4a1745fc 138vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue)
d1a890fa
SB		 139{
140 u32 ret;
09c5088e 141 int i;
83d0feff 142 unsigned long flags;
d1a890fa 143
83d0feff 144 spin_lock_irqsave(&adapter->cmd_lock, flags);
d1a890fa
SB		 145 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
146 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
83d0feff
SB		 147 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
148
d1a890fa
SB		 149 adapter->link_speed = ret >> 16;
150 if (ret & 1) { /* Link is up. */
204a6e65
SH		 151 netdev_info(adapter->netdev, "NIC Link is Up %d Mbps\n",
152 adapter->link_speed);
d1a890fa
SB		 153 if (!netif_carrier_ok(adapter->netdev))
154 netif_carrier_on(adapter->netdev);
155
09c5088e
SB		 156 if (affectTxQueue) {
157 for (i = 0; i < adapter->num_tx_queues; i++)
158 vmxnet3_tq_start(&adapter->tx_queue[i],
159 adapter);
160 }
d1a890fa 161 } else {
204a6e65 162 netdev_info(adapter->netdev, "NIC Link is Down\n");
d1a890fa
SB		 163 if (netif_carrier_ok(adapter->netdev))
164 netif_carrier_off(adapter->netdev);
165
09c5088e
SB		 166 if (affectTxQueue) {
167 for (i = 0; i < adapter->num_tx_queues; i++)
168 vmxnet3_tq_stop(&adapter->tx_queue[i], adapter);
169 }
d1a890fa
SB		 170 }
171}
172
d1a890fa
SB		 173static void
174vmxnet3_process_events(struct vmxnet3_adapter *adapter)
175{
09c5088e 176 int i;
e328d410 177 unsigned long flags;
115924b6 178 u32 events = le32_to_cpu(adapter->shared->ecr);
d1a890fa
SB		 179 if (!events)
180 return;
181
182 vmxnet3_ack_events(adapter, events);
183
184 /* Check if link state has changed */
185 if (events & VMXNET3_ECR_LINK)
4a1745fc 186 vmxnet3_check_link(adapter, true);
d1a890fa
SB		 187
188 /* Check if there is an error on xmit/recv queues */
189 if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) {
e328d410 190 spin_lock_irqsave(&adapter->cmd_lock, flags);
d1a890fa
SB		 191 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
192 VMXNET3_CMD_GET_QUEUE_STATUS);
e328d410 193 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
d1a890fa 194
09c5088e
SB		 195 for (i = 0; i < adapter->num_tx_queues; i++)
196 if (adapter->tqd_start[i].status.stopped)
197 dev_err(&adapter->netdev->dev,
198 "%s: tq[%d] error 0x%x\n",
199 adapter->netdev->name, i, le32_to_cpu(
200 adapter->tqd_start[i].status.error));
201 for (i = 0; i < adapter->num_rx_queues; i++)
202 if (adapter->rqd_start[i].status.stopped)
203 dev_err(&adapter->netdev->dev,
204 "%s: rq[%d] error 0x%x\n",
205 adapter->netdev->name, i,
206 adapter->rqd_start[i].status.error);
d1a890fa
SB		 207
208 schedule_work(&adapter->work);
209 }
210}
211
115924b6
SB		 212#ifdef __BIG_ENDIAN_BITFIELD
213/*
214 * The device expects the bitfields in shared structures to be written in
215 * little endian. When CPU is big endian, the following routines are used to
216 * correctly read and write into ABI.
217 * The general technique used here is : double word bitfields are defined in
218 * opposite order for big endian architecture. Then before reading them in
219 * driver the complete double word is translated using le32_to_cpu. Similarly
220 * After the driver writes into bitfields, cpu_to_le32 is used to translate the
221 * double words into required format.
222 * In order to avoid touching bits in shared structure more than once, temporary
223 * descriptors are used. These are passed as srcDesc to following functions.
224 */
225static void vmxnet3_RxDescToCPU(const struct Vmxnet3_RxDesc *srcDesc,
226 struct Vmxnet3_RxDesc *dstDesc)
227{
228 u32 *src = (u32 *)srcDesc + 2;
229 u32 *dst = (u32 *)dstDesc + 2;
230 dstDesc->addr = le64_to_cpu(srcDesc->addr);
231 *dst = le32_to_cpu(*src);
232 dstDesc->ext1 = le32_to_cpu(srcDesc->ext1);
233}
234
235static void vmxnet3_TxDescToLe(const struct Vmxnet3_TxDesc *srcDesc,
236 struct Vmxnet3_TxDesc *dstDesc)
237{
238 int i;
239 u32 *src = (u32 *)(srcDesc + 1);
240 u32 *dst = (u32 *)(dstDesc + 1);
241
242 /* Working backwards so that the gen bit is set at the end. */
243 for (i = 2; i > 0; i--) {
244 src--;
245 dst--;
246 *dst = cpu_to_le32(*src);
247 }
248}
249
250
251static void vmxnet3_RxCompToCPU(const struct Vmxnet3_RxCompDesc *srcDesc,
252 struct Vmxnet3_RxCompDesc *dstDesc)
253{
254 int i = 0;
255 u32 *src = (u32 *)srcDesc;
256 u32 *dst = (u32 *)dstDesc;
257 for (i = 0; i < sizeof(struct Vmxnet3_RxCompDesc) / sizeof(u32); i++) {
258 *dst = le32_to_cpu(*src);
259 src++;
260 dst++;
261 }
262}
263
264
265/* Used to read bitfield values from double words. */
266static u32 get_bitfield32(const __le32 *bitfield, u32 pos, u32 size)
267{
268 u32 temp = le32_to_cpu(*bitfield);
269 u32 mask = ((1 << size) - 1) << pos;
270 temp &= mask;
271 temp >>= pos;
272 return temp;
273}
274
275
276
277#endif /* __BIG_ENDIAN_BITFIELD */
278
279#ifdef __BIG_ENDIAN_BITFIELD
280
281# define VMXNET3_TXDESC_GET_GEN(txdesc) get_bitfield32(((const __le32 *) \
282 txdesc) + VMXNET3_TXD_GEN_DWORD_SHIFT, \
283 VMXNET3_TXD_GEN_SHIFT, VMXNET3_TXD_GEN_SIZE)
284# define VMXNET3_TXDESC_GET_EOP(txdesc) get_bitfield32(((const __le32 *) \
285 txdesc) + VMXNET3_TXD_EOP_DWORD_SHIFT, \
286 VMXNET3_TXD_EOP_SHIFT, VMXNET3_TXD_EOP_SIZE)
287# define VMXNET3_TCD_GET_GEN(tcd) get_bitfield32(((const __le32 *)tcd) + \
288 VMXNET3_TCD_GEN_DWORD_SHIFT, VMXNET3_TCD_GEN_SHIFT, \
289 VMXNET3_TCD_GEN_SIZE)
290# define VMXNET3_TCD_GET_TXIDX(tcd) get_bitfield32((const __le32 *)tcd, \
291 VMXNET3_TCD_TXIDX_SHIFT, VMXNET3_TCD_TXIDX_SIZE)
292# define vmxnet3_getRxComp(dstrcd, rcd, tmp) do { \
293 (dstrcd) = (tmp); \
294 vmxnet3_RxCompToCPU((rcd), (tmp)); \
295 } while (0)
296# define vmxnet3_getRxDesc(dstrxd, rxd, tmp) do { \
297 (dstrxd) = (tmp); \
298 vmxnet3_RxDescToCPU((rxd), (tmp)); \
299 } while (0)
300
301#else
302
303# define VMXNET3_TXDESC_GET_GEN(txdesc) ((txdesc)->gen)
304# define VMXNET3_TXDESC_GET_EOP(txdesc) ((txdesc)->eop)
305# define VMXNET3_TCD_GET_GEN(tcd) ((tcd)->gen)
306# define VMXNET3_TCD_GET_TXIDX(tcd) ((tcd)->txdIdx)
307# define vmxnet3_getRxComp(dstrcd, rcd, tmp) (dstrcd) = (rcd)
308# define vmxnet3_getRxDesc(dstrxd, rxd, tmp) (dstrxd) = (rxd)
309
310#endif /* __BIG_ENDIAN_BITFIELD */
311
d1a890fa
SB		 312
313static void
314vmxnet3_unmap_tx_buf(struct vmxnet3_tx_buf_info *tbi,
315 struct pci_dev *pdev)
316{
317 if (tbi->map_type == VMXNET3_MAP_SINGLE)
318 pci_unmap_single(pdev, tbi->dma_addr, tbi->len,
319 PCI_DMA_TODEVICE);
320 else if (tbi->map_type == VMXNET3_MAP_PAGE)
321 pci_unmap_page(pdev, tbi->dma_addr, tbi->len,
322 PCI_DMA_TODEVICE);
323 else
324 BUG_ON(tbi->map_type != VMXNET3_MAP_NONE);
325
326 tbi->map_type = VMXNET3_MAP_NONE; /* to help debugging */
327}
328
329
330static int
331vmxnet3_unmap_pkt(u32 eop_idx, struct vmxnet3_tx_queue *tq,
332 struct pci_dev *pdev, struct vmxnet3_adapter *adapter)
333{
334 struct sk_buff *skb;
335 int entries = 0;
336
337 /* no out of order completion */
338 BUG_ON(tq->buf_info[eop_idx].sop_idx != tq->tx_ring.next2comp);
115924b6 339 BUG_ON(VMXNET3_TXDESC_GET_EOP(&(tq->tx_ring.base[eop_idx].txd)) != 1);
d1a890fa
SB		 340
341 skb = tq->buf_info[eop_idx].skb;
342 BUG_ON(skb == NULL);
343 tq->buf_info[eop_idx].skb = NULL;
344
345 VMXNET3_INC_RING_IDX_ONLY(eop_idx, tq->tx_ring.size);
346
347 while (tq->tx_ring.next2comp != eop_idx) {
348 vmxnet3_unmap_tx_buf(tq->buf_info + tq->tx_ring.next2comp,
349 pdev);
350
351 /* update next2comp w/o tx_lock. Since we are marking more,
352 * instead of less, tx ring entries avail, the worst case is
353 * that the tx routine incorrectly re-queues a pkt due to
354 * insufficient tx ring entries.
355 */
356 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
357 entries++;
358 }
359
360 dev_kfree_skb_any(skb);
361 return entries;
362}
363
364
365static int
366vmxnet3_tq_tx_complete(struct vmxnet3_tx_queue *tq,
367 struct vmxnet3_adapter *adapter)
368{
369 int completed = 0;
370 union Vmxnet3_GenericDesc *gdesc;
371
372 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
115924b6
SB		 373 while (VMXNET3_TCD_GET_GEN(&gdesc->tcd) == tq->comp_ring.gen) {
374 completed += vmxnet3_unmap_pkt(VMXNET3_TCD_GET_TXIDX(
375 &gdesc->tcd), tq, adapter->pdev,
376 adapter);
d1a890fa
SB		 377
378 vmxnet3_comp_ring_adv_next2proc(&tq->comp_ring);
379 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
380 }
381
382 if (completed) {
383 spin_lock(&tq->tx_lock);
384 if (unlikely(vmxnet3_tq_stopped(tq, adapter) &&
385 vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) >
386 VMXNET3_WAKE_QUEUE_THRESHOLD(tq) &&
387 netif_carrier_ok(adapter->netdev))) {
388 vmxnet3_tq_wake(tq, adapter);
389 }
390 spin_unlock(&tq->tx_lock);
391 }
392 return completed;
393}
394
395
396static void
397vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq,
398 struct vmxnet3_adapter *adapter)
399{
400 int i;
401
402 while (tq->tx_ring.next2comp != tq->tx_ring.next2fill) {
403 struct vmxnet3_tx_buf_info *tbi;
d1a890fa
SB		 404
405 tbi = tq->buf_info + tq->tx_ring.next2comp;
d1a890fa
SB		 406
407 vmxnet3_unmap_tx_buf(tbi, adapter->pdev);
408 if (tbi->skb) {
409 dev_kfree_skb_any(tbi->skb);
410 tbi->skb = NULL;
411 }
412 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
413 }
414
415 /* sanity check, verify all buffers are indeed unmapped and freed */
416 for (i = 0; i < tq->tx_ring.size; i++) {
417 BUG_ON(tq->buf_info[i].skb != NULL ||
418 tq->buf_info[i].map_type != VMXNET3_MAP_NONE);
419 }
420
421 tq->tx_ring.gen = VMXNET3_INIT_GEN;
422 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
423
424 tq->comp_ring.gen = VMXNET3_INIT_GEN;
425 tq->comp_ring.next2proc = 0;
426}
427
428
09c5088e 429static void
d1a890fa
SB		 430vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq,
431 struct vmxnet3_adapter *adapter)
432{
433 if (tq->tx_ring.base) {
434 pci_free_consistent(adapter->pdev, tq->tx_ring.size *
435 sizeof(struct Vmxnet3_TxDesc),
436 tq->tx_ring.base, tq->tx_ring.basePA);
437 tq->tx_ring.base = NULL;
438 }
439 if (tq->data_ring.base) {
440 pci_free_consistent(adapter->pdev, tq->data_ring.size *
441 sizeof(struct Vmxnet3_TxDataDesc),
442 tq->data_ring.base, tq->data_ring.basePA);
443 tq->data_ring.base = NULL;
444 }
445 if (tq->comp_ring.base) {
446 pci_free_consistent(adapter->pdev, tq->comp_ring.size *
447 sizeof(struct Vmxnet3_TxCompDesc),
448 tq->comp_ring.base, tq->comp_ring.basePA);
449 tq->comp_ring.base = NULL;
450 }
451 kfree(tq->buf_info);
452 tq->buf_info = NULL;
453}
454
455
09c5088e
SB		 456/* Destroy all tx queues */
457void
458vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter)
459{
460 int i;
461
462 for (i = 0; i < adapter->num_tx_queues; i++)
463 vmxnet3_tq_destroy(&adapter->tx_queue[i], adapter);
464}
465
466
d1a890fa
SB		 467static void
468vmxnet3_tq_init(struct vmxnet3_tx_queue *tq,
469 struct vmxnet3_adapter *adapter)
470{
471 int i;
472
473 /* reset the tx ring contents to 0 and reset the tx ring states */
474 memset(tq->tx_ring.base, 0, tq->tx_ring.size *
475 sizeof(struct Vmxnet3_TxDesc));
476 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
477 tq->tx_ring.gen = VMXNET3_INIT_GEN;
478
479 memset(tq->data_ring.base, 0, tq->data_ring.size *
480 sizeof(struct Vmxnet3_TxDataDesc));
481
482 /* reset the tx comp ring contents to 0 and reset comp ring states */
483 memset(tq->comp_ring.base, 0, tq->comp_ring.size *
484 sizeof(struct Vmxnet3_TxCompDesc));
485 tq->comp_ring.next2proc = 0;
486 tq->comp_ring.gen = VMXNET3_INIT_GEN;
487
488 /* reset the bookkeeping data */
489 memset(tq->buf_info, 0, sizeof(tq->buf_info[0]) * tq->tx_ring.size);
490 for (i = 0; i < tq->tx_ring.size; i++)
491 tq->buf_info[i].map_type = VMXNET3_MAP_NONE;
492
493 /* stats are not reset */
494}
495
496
497static int
498vmxnet3_tq_create(struct vmxnet3_tx_queue *tq,
499 struct vmxnet3_adapter *adapter)
500{
501 BUG_ON(tq->tx_ring.base || tq->data_ring.base ||
502 tq->comp_ring.base || tq->buf_info);
503
504 tq->tx_ring.base = pci_alloc_consistent(adapter->pdev, tq->tx_ring.size
505 * sizeof(struct Vmxnet3_TxDesc),
506 &tq->tx_ring.basePA);
507 if (!tq->tx_ring.base) {
204a6e65 508 netdev_err(adapter->netdev, "failed to allocate tx ring\n");
d1a890fa
SB		 509 goto err;
510 }
511
512 tq->data_ring.base = pci_alloc_consistent(adapter->pdev,
513 tq->data_ring.size *
514 sizeof(struct Vmxnet3_TxDataDesc),
515 &tq->data_ring.basePA);
516 if (!tq->data_ring.base) {
204a6e65 517 netdev_err(adapter->netdev, "failed to allocate data ring\n");
d1a890fa
SB		 518 goto err;
519 }
520
521 tq->comp_ring.base = pci_alloc_consistent(adapter->pdev,
522 tq->comp_ring.size *
523 sizeof(struct Vmxnet3_TxCompDesc),
524 &tq->comp_ring.basePA);
525 if (!tq->comp_ring.base) {
204a6e65 526 netdev_err(adapter->netdev, "failed to allocate tx comp ring\n");
d1a890fa
SB		 527 goto err;
528 }
529
530 tq->buf_info = kcalloc(tq->tx_ring.size, sizeof(tq->buf_info[0]),
531 GFP_KERNEL);
e404decb 532 if (!tq->buf_info)
d1a890fa 533 goto err;
d1a890fa
SB		 534
535 return 0;
536
537err:
538 vmxnet3_tq_destroy(tq, adapter);
539 return -ENOMEM;
540}
541
09c5088e
SB		 542static void
543vmxnet3_tq_cleanup_all(struct vmxnet3_adapter *adapter)
544{
545 int i;
546
547 for (i = 0; i < adapter->num_tx_queues; i++)
548 vmxnet3_tq_cleanup(&adapter->tx_queue[i], adapter);
549}
d1a890fa
SB		 550
551/*
552 * starting from ring->next2fill, allocate rx buffers for the given ring
553 * of the rx queue and update the rx desc. stop after @num_to_alloc buffers
554 * are allocated or allocation fails
555 */
556
557static int
558vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx,
559 int num_to_alloc, struct vmxnet3_adapter *adapter)
560{
561 int num_allocated = 0;
562 struct vmxnet3_rx_buf_info *rbi_base = rq->buf_info[ring_idx];
563 struct vmxnet3_cmd_ring *ring = &rq->rx_ring[ring_idx];
564 u32 val;
565
5318d809 566 while (num_allocated <= num_to_alloc) {
d1a890fa
SB		 567 struct vmxnet3_rx_buf_info *rbi;
568 union Vmxnet3_GenericDesc *gd;
569
570 rbi = rbi_base + ring->next2fill;
571 gd = ring->base + ring->next2fill;
572
573 if (rbi->buf_type == VMXNET3_RX_BUF_SKB) {
574 if (rbi->skb == NULL) {
0d735f13
SH		 575 rbi->skb = __netdev_alloc_skb_ip_align(adapter->netdev,
576 rbi->len,
577 GFP_KERNEL);
d1a890fa
SB		 578 if (unlikely(rbi->skb == NULL)) {
579 rq->stats.rx_buf_alloc_failure++;
580 break;
581 }
d1a890fa 582
d1a890fa
SB		 583 rbi->dma_addr = pci_map_single(adapter->pdev,
584 rbi->skb->data, rbi->len,
585 PCI_DMA_FROMDEVICE);
586 } else {
587 /* rx buffer skipped by the device */
588 }
589 val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT;
590 } else {
591 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE ||
592 rbi->len != PAGE_SIZE);
593
594 if (rbi->page == NULL) {
595 rbi->page = alloc_page(GFP_ATOMIC);
596 if (unlikely(rbi->page == NULL)) {
597 rq->stats.rx_buf_alloc_failure++;
598 break;
599 }
600 rbi->dma_addr = pci_map_page(adapter->pdev,
601 rbi->page, 0, PAGE_SIZE,
602 PCI_DMA_FROMDEVICE);
603 } else {
604 /* rx buffers skipped by the device */
605 }
606 val = VMXNET3_RXD_BTYPE_BODY << VMXNET3_RXD_BTYPE_SHIFT;
607 }
608
609 BUG_ON(rbi->dma_addr == 0);
115924b6 610 gd->rxd.addr = cpu_to_le64(rbi->dma_addr);
5318d809 611 gd->dword[2] = cpu_to_le32((!ring->gen << VMXNET3_RXD_GEN_SHIFT)
115924b6 612 | val | rbi->len);
d1a890fa 613
5318d809
SB		 614 /* Fill the last buffer but dont mark it ready, or else the
615 * device will think that the queue is full */
616 if (num_allocated == num_to_alloc)
617 break;
618
619 gd->dword[2] |= cpu_to_le32(ring->gen << VMXNET3_RXD_GEN_SHIFT);
d1a890fa
SB		 620 num_allocated++;
621 vmxnet3_cmd_ring_adv_next2fill(ring);
622 }
d1a890fa 623
fdcd79b9 624 netdev_dbg(adapter->netdev,
69b9a712
SH		 625 "alloc_rx_buf: %d allocated, next2fill %u, next2comp %u\n",
626 num_allocated, ring->next2fill, ring->next2comp);
d1a890fa
SB		 627
628 /* so that the device can distinguish a full ring and an empty ring */
629 BUG_ON(num_allocated != 0 && ring->next2fill == ring->next2comp);
630
631 return num_allocated;
632}
633
634
635static void
636vmxnet3_append_frag(struct sk_buff *skb, struct Vmxnet3_RxCompDesc *rcd,
637 struct vmxnet3_rx_buf_info *rbi)
638{
639 struct skb_frag_struct *frag = skb_shinfo(skb)->frags +
640 skb_shinfo(skb)->nr_frags;
641
642 BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS);
643
0e0634d2 644 __skb_frag_set_page(frag, rbi->page);
d1a890fa 645 frag->page_offset = 0;
9e903e08
ED		 646 skb_frag_size_set(frag, rcd->len);
647 skb->data_len += rcd->len;
5e6c355c 648 skb->truesize += PAGE_SIZE;
d1a890fa
SB		 649 skb_shinfo(skb)->nr_frags++;
650}
651
652
653static void
654vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx,
655 struct vmxnet3_tx_queue *tq, struct pci_dev *pdev,
656 struct vmxnet3_adapter *adapter)
657{
658 u32 dw2, len;
659 unsigned long buf_offset;
660 int i;
661 union Vmxnet3_GenericDesc *gdesc;
662 struct vmxnet3_tx_buf_info *tbi = NULL;
663
664 BUG_ON(ctx->copy_size > skb_headlen(skb));
665
666 /* use the previous gen bit for the SOP desc */
667 dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT;
668
669 ctx->sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill;
670 gdesc = ctx->sop_txd; /* both loops below can be skipped */
671
672 /* no need to map the buffer if headers are copied */
673 if (ctx->copy_size) {
115924b6 674 ctx->sop_txd->txd.addr = cpu_to_le64(tq->data_ring.basePA +
d1a890fa 675 tq->tx_ring.next2fill *
115924b6
SB		 676 sizeof(struct Vmxnet3_TxDataDesc));
677 ctx->sop_txd->dword[2] = cpu_to_le32(dw2 | ctx->copy_size);
d1a890fa
SB		 678 ctx->sop_txd->dword[3] = 0;
679
680 tbi = tq->buf_info + tq->tx_ring.next2fill;
681 tbi->map_type = VMXNET3_MAP_NONE;
682
fdcd79b9 683 netdev_dbg(adapter->netdev,
f6965582 684 "txd[%u]: 0x%Lx 0x%x 0x%x\n",
115924b6
SB		 685 tq->tx_ring.next2fill,
686 le64_to_cpu(ctx->sop_txd->txd.addr),
d1a890fa
SB		 687 ctx->sop_txd->dword[2], ctx->sop_txd->dword[3]);
688 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
689
690 /* use the right gen for non-SOP desc */
691 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
692 }
693
694 /* linear part can use multiple tx desc if it's big */
695 len = skb_headlen(skb) - ctx->copy_size;
696 buf_offset = ctx->copy_size;
697 while (len) {
698 u32 buf_size;
699
1f4b1612
BD		 700 if (len < VMXNET3_MAX_TX_BUF_SIZE) {
701 buf_size = len;
702 dw2 |= len;
703 } else {
704 buf_size = VMXNET3_MAX_TX_BUF_SIZE;
705 /* spec says that for TxDesc.len, 0 == 2^14 */
706 }
d1a890fa
SB		 707
708 tbi = tq->buf_info + tq->tx_ring.next2fill;
709 tbi->map_type = VMXNET3_MAP_SINGLE;
710 tbi->dma_addr = pci_map_single(adapter->pdev,
711 skb->data + buf_offset, buf_size,
712 PCI_DMA_TODEVICE);
713
1f4b1612 714 tbi->len = buf_size;
d1a890fa
SB		 715
716 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
717 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
718
115924b6 719 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);
1f4b1612 720 gdesc->dword[2] = cpu_to_le32(dw2);
d1a890fa
SB		 721 gdesc->dword[3] = 0;
722
fdcd79b9 723 netdev_dbg(adapter->netdev,
f6965582 724 "txd[%u]: 0x%Lx 0x%x 0x%x\n",
115924b6
SB		 725 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),
726 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);
d1a890fa
SB		 727 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
728 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
729
730 len -= buf_size;
731 buf_offset += buf_size;
732 }
733
734 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
9e903e08 735 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
a4d7e485 736 u32 buf_size;
d1a890fa 737
a4d7e485
ED		 738 buf_offset = 0;
739 len = skb_frag_size(frag);
740 while (len) {
741 tbi = tq->buf_info + tq->tx_ring.next2fill;
742 if (len < VMXNET3_MAX_TX_BUF_SIZE) {
743 buf_size = len;
744 dw2 |= len;
745 } else {
746 buf_size = VMXNET3_MAX_TX_BUF_SIZE;
747 /* spec says that for TxDesc.len, 0 == 2^14 */
748 }
749 tbi->map_type = VMXNET3_MAP_PAGE;
750 tbi->dma_addr = skb_frag_dma_map(&adapter->pdev->dev, frag,
751 buf_offset, buf_size,
752 DMA_TO_DEVICE);
d1a890fa 753
a4d7e485 754 tbi->len = buf_size;
d1a890fa 755
a4d7e485
ED		 756 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
757 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
d1a890fa 758
a4d7e485
ED		 759 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);
760 gdesc->dword[2] = cpu_to_le32(dw2);
761 gdesc->dword[3] = 0;
d1a890fa 762
fdcd79b9 763 netdev_dbg(adapter->netdev,
a4d7e485
ED		 764 "txd[%u]: 0x%llu %u %u\n",
765 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),
766 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);
767 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
768 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
769
770 len -= buf_size;
771 buf_offset += buf_size;
772 }
d1a890fa
SB		 773 }
774
775 ctx->eop_txd = gdesc;
776
777 /* set the last buf_info for the pkt */
778 tbi->skb = skb;
779 tbi->sop_idx = ctx->sop_txd - tq->tx_ring.base;
780}
781
782
09c5088e
SB		 783/* Init all tx queues */
784static void
785vmxnet3_tq_init_all(struct vmxnet3_adapter *adapter)
786{
787 int i;
788
789 for (i = 0; i < adapter->num_tx_queues; i++)
790 vmxnet3_tq_init(&adapter->tx_queue[i], adapter);
791}
792
793
d1a890fa
SB		 794/*
795 * parse and copy relevant protocol headers:
796 * For a tso pkt, relevant headers are L2/3/4 including options
797 * For a pkt requesting csum offloading, they are L2/3 and may include L4
798 * if it's a TCP/UDP pkt
799 *
800 * Returns:
801 * -1: error happens during parsing
802 * 0: protocol headers parsed, but too big to be copied
803 * 1: protocol headers parsed and copied
804 *
805 * Other effects:
806 * 1. related *ctx fields are updated.
807 * 2. ctx->copy_size is # of bytes copied
808 * 3. the portion copied is guaranteed to be in the linear part
809 *
810 */
811static int
812vmxnet3_parse_and_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
813 struct vmxnet3_tx_ctx *ctx,
814 struct vmxnet3_adapter *adapter)
815{
816 struct Vmxnet3_TxDataDesc *tdd;
817
0d0b1672 818 if (ctx->mss) { /* TSO */
d1a890fa 819 ctx->eth_ip_hdr_size = skb_transport_offset(skb);
8bca5d1e 820 ctx->l4_hdr_size = tcp_hdrlen(skb);
d1a890fa
SB		 821 ctx->copy_size = ctx->eth_ip_hdr_size + ctx->l4_hdr_size;
822 } else {
d1a890fa 823 if (skb->ip_summed == CHECKSUM_PARTIAL) {
0d0b1672 824 ctx->eth_ip_hdr_size = skb_checksum_start_offset(skb);
d1a890fa
SB		 825
826 if (ctx->ipv4) {
8bca5d1e
ED		 827 const struct iphdr *iph = ip_hdr(skb);
828
39d4a96f 829 if (iph->protocol == IPPROTO_TCP)
8bca5d1e 830 ctx->l4_hdr_size = tcp_hdrlen(skb);
39d4a96f 831 else if (iph->protocol == IPPROTO_UDP)
f6a1ad42 832 ctx->l4_hdr_size = sizeof(struct udphdr);
39d4a96f 833 else
d1a890fa 834 ctx->l4_hdr_size = 0;
d1a890fa
SB		 835 } else {
836 /* for simplicity, don't copy L4 headers */
837 ctx->l4_hdr_size = 0;
838 }
b203262d
NH		 839 ctx->copy_size = min(ctx->eth_ip_hdr_size +
840 ctx->l4_hdr_size, skb->len);
d1a890fa
SB		 841 } else {
842 ctx->eth_ip_hdr_size = 0;
843 ctx->l4_hdr_size = 0;
844 /* copy as much as allowed */
845 ctx->copy_size = min((unsigned int)VMXNET3_HDR_COPY_SIZE
846 , skb_headlen(skb));
847 }
848
849 /* make sure headers are accessible directly */
850 if (unlikely(!pskb_may_pull(skb, ctx->copy_size)))
851 goto err;
852 }
853
854 if (unlikely(ctx->copy_size > VMXNET3_HDR_COPY_SIZE)) {
855 tq->stats.oversized_hdr++;
856 ctx->copy_size = 0;
857 return 0;
858 }
859
860 tdd = tq->data_ring.base + tq->tx_ring.next2fill;
861
862 memcpy(tdd->data, skb->data, ctx->copy_size);
fdcd79b9 863 netdev_dbg(adapter->netdev,
f6965582 864 "copy %u bytes to dataRing[%u]\n",
d1a890fa
SB		 865 ctx->copy_size, tq->tx_ring.next2fill);
866 return 1;
867
868err:
869 return -1;
870}
871
872
873static void
874vmxnet3_prepare_tso(struct sk_buff *skb,
875 struct vmxnet3_tx_ctx *ctx)
876{
8bca5d1e
ED		 877 struct tcphdr *tcph = tcp_hdr(skb);
878
d1a890fa 879 if (ctx->ipv4) {
8bca5d1e
ED		 880 struct iphdr *iph = ip_hdr(skb);
881
d1a890fa
SB		 882 iph->check = 0;
883 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0,
884 IPPROTO_TCP, 0);
885 } else {
8bca5d1e
ED		 886 struct ipv6hdr *iph = ipv6_hdr(skb);
887
d1a890fa
SB		 888 tcph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, 0,
889 IPPROTO_TCP, 0);
890 }
891}
892
a4d7e485
ED		 893static int txd_estimate(const struct sk_buff *skb)
894{
895 int count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
896 int i;
897
898 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
899 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
900
901 count += VMXNET3_TXD_NEEDED(skb_frag_size(frag));
902 }
903 return count;
904}
d1a890fa
SB		 905
906/*
907 * Transmits a pkt thru a given tq
908 * Returns:
909 * NETDEV_TX_OK: descriptors are setup successfully
25985edc 910 * NETDEV_TX_OK: error occurred, the pkt is dropped
d1a890fa
SB		 911 * NETDEV_TX_BUSY: tx ring is full, queue is stopped
912 *
913 * Side-effects:
914 * 1. tx ring may be changed
915 * 2. tq stats may be updated accordingly
916 * 3. shared->txNumDeferred may be updated
917 */
918
919static int
920vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
921 struct vmxnet3_adapter *adapter, struct net_device *netdev)
922{
923 int ret;
924 u32 count;
925 unsigned long flags;
926 struct vmxnet3_tx_ctx ctx;
927 union Vmxnet3_GenericDesc *gdesc;
115924b6
SB		 928#ifdef __BIG_ENDIAN_BITFIELD
929 /* Use temporary descriptor to avoid touching bits multiple times */
930 union Vmxnet3_GenericDesc tempTxDesc;
931#endif
d1a890fa 932
a4d7e485 933 count = txd_estimate(skb);
d1a890fa 934
72e85c45 935 ctx.ipv4 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IP));
d1a890fa
SB		 936
937 ctx.mss = skb_shinfo(skb)->gso_size;
938 if (ctx.mss) {
939 if (skb_header_cloned(skb)) {
940 if (unlikely(pskb_expand_head(skb, 0, 0,
941 GFP_ATOMIC) != 0)) {
942 tq->stats.drop_tso++;
943 goto drop_pkt;
944 }
945 tq->stats.copy_skb_header++;
946 }
947 vmxnet3_prepare_tso(skb, &ctx);
948 } else {
949 if (unlikely(count > VMXNET3_MAX_TXD_PER_PKT)) {
950
951 /* non-tso pkts must not use more than
952 * VMXNET3_MAX_TXD_PER_PKT entries
953 */
954 if (skb_linearize(skb) != 0) {
955 tq->stats.drop_too_many_frags++;
956 goto drop_pkt;
957 }
958 tq->stats.linearized++;
959
960 /* recalculate the # of descriptors to use */
961 count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
962 }
963 }
964
09c5088e
SB		 965 spin_lock_irqsave(&tq->tx_lock, flags);
966
967 if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) {
968 tq->stats.tx_ring_full++;
fdcd79b9 969 netdev_dbg(adapter->netdev,
09c5088e
SB		 970 "tx queue stopped on %s, next2comp %u"
971 " next2fill %u\n", adapter->netdev->name,
972 tq->tx_ring.next2comp, tq->tx_ring.next2fill);
973
974 vmxnet3_tq_stop(tq, adapter);
975 spin_unlock_irqrestore(&tq->tx_lock, flags);
976 return NETDEV_TX_BUSY;
977 }
978
979
d1a890fa
SB		 980 ret = vmxnet3_parse_and_copy_hdr(skb, tq, &ctx, adapter);
981 if (ret >= 0) {
982 BUG_ON(ret <= 0 && ctx.copy_size != 0);
983 /* hdrs parsed, check against other limits */
984 if (ctx.mss) {
985 if (unlikely(ctx.eth_ip_hdr_size + ctx.l4_hdr_size >
986 VMXNET3_MAX_TX_BUF_SIZE)) {
987 goto hdr_too_big;
988 }
989 } else {
990 if (skb->ip_summed == CHECKSUM_PARTIAL) {
991 if (unlikely(ctx.eth_ip_hdr_size +
992 skb->csum_offset >
993 VMXNET3_MAX_CSUM_OFFSET)) {
994 goto hdr_too_big;
995 }
996 }
997 }
998 } else {
999 tq->stats.drop_hdr_inspect_err++;
f955e141 1000 goto unlock_drop_pkt;
d1a890fa
SB		 1001 }
1002
d1a890fa
SB		 1003 /* fill tx descs related to addr & len */
1004 vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter);
1005
1006 /* setup the EOP desc */
115924b6 1007 ctx.eop_txd->dword[3] = cpu_to_le32(VMXNET3_TXD_CQ | VMXNET3_TXD_EOP);
d1a890fa
SB		 1008
1009 /* setup the SOP desc */
115924b6
SB		 1010#ifdef __BIG_ENDIAN_BITFIELD
1011 gdesc = &tempTxDesc;
1012 gdesc->dword[2] = ctx.sop_txd->dword[2];
1013 gdesc->dword[3] = ctx.sop_txd->dword[3];
1014#else
d1a890fa 1015 gdesc = ctx.sop_txd;
115924b6 1016#endif
d1a890fa
SB		 1017 if (ctx.mss) {
1018 gdesc->txd.hlen = ctx.eth_ip_hdr_size + ctx.l4_hdr_size;
1019 gdesc->txd.om = VMXNET3_OM_TSO;
1020 gdesc->txd.msscof = ctx.mss;
115924b6
SB		 1021 le32_add_cpu(&tq->shared->txNumDeferred, (skb->len -
1022 gdesc->txd.hlen + ctx.mss - 1) / ctx.mss);
d1a890fa
SB		 1023 } else {
1024 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1025 gdesc->txd.hlen = ctx.eth_ip_hdr_size;
1026 gdesc->txd.om = VMXNET3_OM_CSUM;
1027 gdesc->txd.msscof = ctx.eth_ip_hdr_size +
1028 skb->csum_offset;
1029 } else {
1030 gdesc->txd.om = 0;
1031 gdesc->txd.msscof = 0;
1032 }
115924b6 1033 le32_add_cpu(&tq->shared->txNumDeferred, 1);
d1a890fa
SB		 1034 }
1035
1036 if (vlan_tx_tag_present(skb)) {
1037 gdesc->txd.ti = 1;
1038 gdesc->txd.tci = vlan_tx_tag_get(skb);
1039 }
1040
115924b6
SB		 1041 /* finally flips the GEN bit of the SOP desc. */
1042 gdesc->dword[2] = cpu_to_le32(le32_to_cpu(gdesc->dword[2]) ^
1043 VMXNET3_TXD_GEN);
1044#ifdef __BIG_ENDIAN_BITFIELD
1045 /* Finished updating in bitfields of Tx Desc, so write them in original
1046 * place.
1047 */
1048 vmxnet3_TxDescToLe((struct Vmxnet3_TxDesc *)gdesc,
1049 (struct Vmxnet3_TxDesc *)ctx.sop_txd);
1050 gdesc = ctx.sop_txd;
1051#endif
fdcd79b9 1052 netdev_dbg(adapter->netdev,
f6965582 1053 "txd[%u]: SOP 0x%Lx 0x%x 0x%x\n",
c2fd03a0 1054 (u32)(ctx.sop_txd -
115924b6
SB		 1055 tq->tx_ring.base), le64_to_cpu(gdesc->txd.addr),
1056 le32_to_cpu(gdesc->dword[2]), le32_to_cpu(gdesc->dword[3]));
d1a890fa
SB		 1057
1058 spin_unlock_irqrestore(&tq->tx_lock, flags);
1059
115924b6
SB		 1060 if (le32_to_cpu(tq->shared->txNumDeferred) >=
1061 le32_to_cpu(tq->shared->txThreshold)) {
d1a890fa 1062 tq->shared->txNumDeferred = 0;
09c5088e
SB		 1063 VMXNET3_WRITE_BAR0_REG(adapter,
1064 VMXNET3_REG_TXPROD + tq->qid * 8,
d1a890fa
SB		 1065 tq->tx_ring.next2fill);
1066 }
d1a890fa
SB		 1067
1068 return NETDEV_TX_OK;
1069
1070hdr_too_big:
1071 tq->stats.drop_oversized_hdr++;
f955e141
DC		 1072unlock_drop_pkt:
1073 spin_unlock_irqrestore(&tq->tx_lock, flags);
d1a890fa
SB		 1074drop_pkt:
1075 tq->stats.drop_total++;
1076 dev_kfree_skb(skb);
1077 return NETDEV_TX_OK;
1078}
1079
1080
1081static netdev_tx_t
1082vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
1083{
1084 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
d1a890fa 1085
96800ee7 1086 BUG_ON(skb->queue_mapping > adapter->num_tx_queues);
1087 return vmxnet3_tq_xmit(skb,
1088 &adapter->tx_queue[skb->queue_mapping],
1089 adapter, netdev);
d1a890fa
SB		 1090}
1091
1092
1093static void
1094vmxnet3_rx_csum(struct vmxnet3_adapter *adapter,
1095 struct sk_buff *skb,
1096 union Vmxnet3_GenericDesc *gdesc)
1097{
a0d2730c 1098 if (!gdesc->rcd.cnc && adapter->netdev->features & NETIF_F_RXCSUM) {
d1a890fa 1099 /* typical case: TCP/UDP over IP and both csums are correct */
115924b6 1100 if ((le32_to_cpu(gdesc->dword[3]) & VMXNET3_RCD_CSUM_OK) ==
d1a890fa
SB		 1101 VMXNET3_RCD_CSUM_OK) {
1102 skb->ip_summed = CHECKSUM_UNNECESSARY;
1103 BUG_ON(!(gdesc->rcd.tcp || gdesc->rcd.udp));
1104 BUG_ON(!(gdesc->rcd.v4 || gdesc->rcd.v6));
1105 BUG_ON(gdesc->rcd.frg);
1106 } else {
1107 if (gdesc->rcd.csum) {
1108 skb->csum = htons(gdesc->rcd.csum);
1109 skb->ip_summed = CHECKSUM_PARTIAL;
1110 } else {
bc8acf2c 1111 skb_checksum_none_assert(skb);
d1a890fa
SB		 1112 }
1113 }
1114 } else {
bc8acf2c 1115 skb_checksum_none_assert(skb);
d1a890fa
SB		 1116 }
1117}
1118
1119
1120static void
1121vmxnet3_rx_error(struct vmxnet3_rx_queue *rq, struct Vmxnet3_RxCompDesc *rcd,
1122 struct vmxnet3_rx_ctx *ctx, struct vmxnet3_adapter *adapter)
1123{
1124 rq->stats.drop_err++;
1125 if (!rcd->fcs)
1126 rq->stats.drop_fcs++;
1127
1128 rq->stats.drop_total++;
1129
1130 /*
1131 * We do not unmap and chain the rx buffer to the skb.
1132 * We basically pretend this buffer is not used and will be recycled
1133 * by vmxnet3_rq_alloc_rx_buf()
1134 */
1135
1136 /*
1137 * ctx->skb may be NULL if this is the first and the only one
1138 * desc for the pkt
1139 */
1140 if (ctx->skb)
1141 dev_kfree_skb_irq(ctx->skb);
1142
1143 ctx->skb = NULL;
1144}
1145
1146
1147static int
1148vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,
1149 struct vmxnet3_adapter *adapter, int quota)
1150{
215faf9c
JP		 1151 static const u32 rxprod_reg[2] = {
1152 VMXNET3_REG_RXPROD, VMXNET3_REG_RXPROD2
1153 };
d1a890fa 1154 u32 num_rxd = 0;
5318d809 1155 bool skip_page_frags = false;
d1a890fa
SB		 1156 struct Vmxnet3_RxCompDesc *rcd;
1157 struct vmxnet3_rx_ctx *ctx = &rq->rx_ctx;
115924b6
SB		 1158#ifdef __BIG_ENDIAN_BITFIELD
1159 struct Vmxnet3_RxDesc rxCmdDesc;
1160 struct Vmxnet3_RxCompDesc rxComp;
1161#endif
1162 vmxnet3_getRxComp(rcd, &rq->comp_ring.base[rq->comp_ring.next2proc].rcd,
1163 &rxComp);
d1a890fa
SB		 1164 while (rcd->gen == rq->comp_ring.gen) {
1165 struct vmxnet3_rx_buf_info *rbi;
5318d809
SB		 1166 struct sk_buff *skb, *new_skb = NULL;
1167 struct page *new_page = NULL;
d1a890fa
SB		 1168 int num_to_alloc;
1169 struct Vmxnet3_RxDesc *rxd;
1170 u32 idx, ring_idx;
5318d809 1171 struct vmxnet3_cmd_ring *ring = NULL;
d1a890fa
SB		 1172 if (num_rxd >= quota) {
1173 /* we may stop even before we see the EOP desc of
1174 * the current pkt
1175 */
1176 break;
1177 }
1178 num_rxd++;
09c5088e 1179 BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2);
d1a890fa 1180 idx = rcd->rxdIdx;
09c5088e 1181 ring_idx = rcd->rqID < adapter->num_rx_queues ? 0 : 1;
5318d809 1182 ring = rq->rx_ring + ring_idx;
115924b6
SB		 1183 vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd,
1184 &rxCmdDesc);
d1a890fa
SB		 1185 rbi = rq->buf_info[ring_idx] + idx;
1186
115924b6
SB		 1187 BUG_ON(rxd->addr != rbi->dma_addr ||
1188 rxd->len != rbi->len);
d1a890fa
SB		 1189
1190 if (unlikely(rcd->eop && rcd->err)) {
1191 vmxnet3_rx_error(rq, rcd, ctx, adapter);
1192 goto rcd_done;
1193 }
1194
1195 if (rcd->sop) { /* first buf of the pkt */
1196 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_HEAD ||
1197 rcd->rqID != rq->qid);
1198
1199 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_SKB);
1200 BUG_ON(ctx->skb != NULL || rbi->skb == NULL);
1201
1202 if (unlikely(rcd->len == 0)) {
1203 /* Pretend the rx buffer is skipped. */
1204 BUG_ON(!(rcd->sop && rcd->eop));
fdcd79b9 1205 netdev_dbg(adapter->netdev,
f6965582 1206 "rxRing[%u][%u] 0 length\n",
d1a890fa
SB		 1207 ring_idx, idx);
1208 goto rcd_done;
1209 }
1210
5318d809 1211 skip_page_frags = false;
d1a890fa 1212 ctx->skb = rbi->skb;
0d735f13
SH		 1213 new_skb = netdev_alloc_skb_ip_align(adapter->netdev,
1214 rbi->len);
5318d809
SB		 1215 if (new_skb == NULL) {
1216 /* Skb allocation failed, do not handover this
1217 * skb to stack. Reuse it. Drop the existing pkt
1218 */
1219 rq->stats.rx_buf_alloc_failure++;
1220 ctx->skb = NULL;
1221 rq->stats.drop_total++;
1222 skip_page_frags = true;
1223 goto rcd_done;
1224 }
d1a890fa
SB		 1225
1226 pci_unmap_single(adapter->pdev, rbi->dma_addr, rbi->len,
1227 PCI_DMA_FROMDEVICE);
1228
1229 skb_put(ctx->skb, rcd->len);
5318d809
SB		 1230
1231 /* Immediate refill */
5318d809
SB		 1232 rbi->skb = new_skb;
1233 rbi->dma_addr = pci_map_single(adapter->pdev,
96800ee7 1234 rbi->skb->data, rbi->len,
1235 PCI_DMA_FROMDEVICE);
5318d809
SB		 1236 rxd->addr = cpu_to_le64(rbi->dma_addr);
1237 rxd->len = rbi->len;
1238
d1a890fa 1239 } else {
5318d809
SB		 1240 BUG_ON(ctx->skb == NULL && !skip_page_frags);
1241
d1a890fa 1242 /* non SOP buffer must be type 1 in most cases */
5318d809
SB		 1243 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE);
1244 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_BODY);
d1a890fa 1245
5318d809
SB		 1246 /* If an sop buffer was dropped, skip all
1247 * following non-sop fragments. They will be reused.
1248 */
1249 if (skip_page_frags)
1250 goto rcd_done;
d1a890fa 1251
5318d809
SB		 1252 new_page = alloc_page(GFP_ATOMIC);
1253 if (unlikely(new_page == NULL)) {
1254 /* Replacement page frag could not be allocated.
1255 * Reuse this page. Drop the pkt and free the
1256 * skb which contained this page as a frag. Skip
1257 * processing all the following non-sop frags.
d1a890fa 1258 */
5318d809
SB		 1259 rq->stats.rx_buf_alloc_failure++;
1260 dev_kfree_skb(ctx->skb);
1261 ctx->skb = NULL;
1262 skip_page_frags = true;
1263 goto rcd_done;
1264 }
1265
1266 if (rcd->len) {
1267 pci_unmap_page(adapter->pdev,
1268 rbi->dma_addr, rbi->len,
1269 PCI_DMA_FROMDEVICE);
1270
1271 vmxnet3_append_frag(ctx->skb, rcd, rbi);
d1a890fa 1272 }
5318d809
SB		 1273
1274 /* Immediate refill */
1275 rbi->page = new_page;
1276 rbi->dma_addr = pci_map_page(adapter->pdev, rbi->page,
1277 0, PAGE_SIZE,
1278 PCI_DMA_FROMDEVICE);
1279 rxd->addr = cpu_to_le64(rbi->dma_addr);
1280 rxd->len = rbi->len;
d1a890fa
SB		 1281 }
1282
5318d809 1283
d1a890fa
SB		 1284 skb = ctx->skb;
1285 if (rcd->eop) {
1286 skb->len += skb->data_len;
d1a890fa
SB		 1287
1288 vmxnet3_rx_csum(adapter, skb,
1289 (union Vmxnet3_GenericDesc *)rcd);
1290 skb->protocol = eth_type_trans(skb, adapter->netdev);
1291
72e85c45
JG		 1292 if (unlikely(rcd->ts))
1293 __vlan_hwaccel_put_tag(skb, rcd->tci);
1294
213ade8c
JG		 1295 if (adapter->netdev->features & NETIF_F_LRO)
1296 netif_receive_skb(skb);
1297 else
1298 napi_gro_receive(&rq->napi, skb);
d1a890fa 1299
d1a890fa
SB		 1300 ctx->skb = NULL;
1301 }
1302
1303rcd_done:
5318d809
SB		 1304 /* device may have skipped some rx descs */
1305 ring->next2comp = idx;
1306 num_to_alloc = vmxnet3_cmd_ring_desc_avail(ring);
1307 ring = rq->rx_ring + ring_idx;
1308 while (num_to_alloc) {
1309 vmxnet3_getRxDesc(rxd, &ring->base[ring->next2fill].rxd,
1310 &rxCmdDesc);
1311 BUG_ON(!rxd->addr);
1312
1313 /* Recv desc is ready to be used by the device */
1314 rxd->gen = ring->gen;
1315 vmxnet3_cmd_ring_adv_next2fill(ring);
1316 num_to_alloc--;
1317 }
1318
1319 /* if needed, update the register */
1320 if (unlikely(rq->shared->updateRxProd)) {
1321 VMXNET3_WRITE_BAR0_REG(adapter,
96800ee7 1322 rxprod_reg[ring_idx] + rq->qid * 8,
1323 ring->next2fill);
d1a890fa
SB		 1324 }
1325
1326 vmxnet3_comp_ring_adv_next2proc(&rq->comp_ring);
115924b6 1327 vmxnet3_getRxComp(rcd,
96800ee7 1328 &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, &rxComp);
d1a890fa
SB		 1329 }
1330
1331 return num_rxd;
1332}
1333
1334
1335static void
1336vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq,
1337 struct vmxnet3_adapter *adapter)
1338{
1339 u32 i, ring_idx;
1340 struct Vmxnet3_RxDesc *rxd;
1341
1342 for (ring_idx = 0; ring_idx < 2; ring_idx++) {
1343 for (i = 0; i < rq->rx_ring[ring_idx].size; i++) {
115924b6
SB		 1344#ifdef __BIG_ENDIAN_BITFIELD
1345 struct Vmxnet3_RxDesc rxDesc;
1346#endif
1347 vmxnet3_getRxDesc(rxd,
1348 &rq->rx_ring[ring_idx].base[i].rxd, &rxDesc);
d1a890fa
SB		 1349
1350 if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD &&
1351 rq->buf_info[ring_idx][i].skb) {
1352 pci_unmap_single(adapter->pdev, rxd->addr,
1353 rxd->len, PCI_DMA_FROMDEVICE);
1354 dev_kfree_skb(rq->buf_info[ring_idx][i].skb);
1355 rq->buf_info[ring_idx][i].skb = NULL;
1356 } else if (rxd->btype == VMXNET3_RXD_BTYPE_BODY &&
1357 rq->buf_info[ring_idx][i].page) {
1358 pci_unmap_page(adapter->pdev, rxd->addr,
1359 rxd->len, PCI_DMA_FROMDEVICE);
1360 put_page(rq->buf_info[ring_idx][i].page);
1361 rq->buf_info[ring_idx][i].page = NULL;
1362 }
1363 }
1364
1365 rq->rx_ring[ring_idx].gen = VMXNET3_INIT_GEN;
1366 rq->rx_ring[ring_idx].next2fill =
1367 rq->rx_ring[ring_idx].next2comp = 0;
d1a890fa
SB		 1368 }
1369
1370 rq->comp_ring.gen = VMXNET3_INIT_GEN;
1371 rq->comp_ring.next2proc = 0;
1372}
1373
1374
09c5088e
SB		 1375static void
1376vmxnet3_rq_cleanup_all(struct vmxnet3_adapter *adapter)
1377{
1378 int i;
1379
1380 for (i = 0; i < adapter->num_rx_queues; i++)
1381 vmxnet3_rq_cleanup(&adapter->rx_queue[i], adapter);
1382}
1383
1384
d1a890fa
SB		 1385void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq,
1386 struct vmxnet3_adapter *adapter)
1387{
1388 int i;
1389 int j;
1390
1391 /* all rx buffers must have already been freed */
1392 for (i = 0; i < 2; i++) {
1393 if (rq->buf_info[i]) {
1394 for (j = 0; j < rq->rx_ring[i].size; j++)
1395 BUG_ON(rq->buf_info[i][j].page != NULL);
1396 }
1397 }
1398
1399
1400 kfree(rq->buf_info[0]);
1401
1402 for (i = 0; i < 2; i++) {
1403 if (rq->rx_ring[i].base) {
1404 pci_free_consistent(adapter->pdev, rq->rx_ring[i].size
1405 * sizeof(struct Vmxnet3_RxDesc),
1406 rq->rx_ring[i].base,
1407 rq->rx_ring[i].basePA);
1408 rq->rx_ring[i].base = NULL;
1409 }
1410 rq->buf_info[i] = NULL;
1411 }
1412
1413 if (rq->comp_ring.base) {
1414 pci_free_consistent(adapter->pdev, rq->comp_ring.size *
1415 sizeof(struct Vmxnet3_RxCompDesc),
1416 rq->comp_ring.base, rq->comp_ring.basePA);
1417 rq->comp_ring.base = NULL;
1418 }
1419}
1420
1421
1422static int
1423vmxnet3_rq_init(struct vmxnet3_rx_queue *rq,
1424 struct vmxnet3_adapter *adapter)
1425{
1426 int i;
1427
1428 /* initialize buf_info */
1429 for (i = 0; i < rq->rx_ring[0].size; i++) {
1430
1431 /* 1st buf for a pkt is skbuff */
1432 if (i % adapter->rx_buf_per_pkt == 0) {
1433 rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_SKB;
1434 rq->buf_info[0][i].len = adapter->skb_buf_size;
1435 } else { /* subsequent bufs for a pkt is frag */
1436 rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_PAGE;
1437 rq->buf_info[0][i].len = PAGE_SIZE;
1438 }
1439 }
1440 for (i = 0; i < rq->rx_ring[1].size; i++) {
1441 rq->buf_info[1][i].buf_type = VMXNET3_RX_BUF_PAGE;
1442 rq->buf_info[1][i].len = PAGE_SIZE;
1443 }
1444
1445 /* reset internal state and allocate buffers for both rings */
1446 for (i = 0; i < 2; i++) {
1447 rq->rx_ring[i].next2fill = rq->rx_ring[i].next2comp = 0;
d1a890fa
SB		 1448
1449 memset(rq->rx_ring[i].base, 0, rq->rx_ring[i].size *
1450 sizeof(struct Vmxnet3_RxDesc));
1451 rq->rx_ring[i].gen = VMXNET3_INIT_GEN;
1452 }
1453 if (vmxnet3_rq_alloc_rx_buf(rq, 0, rq->rx_ring[0].size - 1,
1454 adapter) == 0) {
1455 /* at least has 1 rx buffer for the 1st ring */
1456 return -ENOMEM;
1457 }
1458 vmxnet3_rq_alloc_rx_buf(rq, 1, rq->rx_ring[1].size - 1, adapter);
1459
1460 /* reset the comp ring */
1461 rq->comp_ring.next2proc = 0;
1462 memset(rq->comp_ring.base, 0, rq->comp_ring.size *
1463 sizeof(struct Vmxnet3_RxCompDesc));
1464 rq->comp_ring.gen = VMXNET3_INIT_GEN;
1465
1466 /* reset rxctx */
1467 rq->rx_ctx.skb = NULL;
1468
1469 /* stats are not reset */
1470 return 0;
1471}
1472
1473
09c5088e
SB		 1474static int
1475vmxnet3_rq_init_all(struct vmxnet3_adapter *adapter)
1476{
1477 int i, err = 0;
1478
1479 for (i = 0; i < adapter->num_rx_queues; i++) {
1480 err = vmxnet3_rq_init(&adapter->rx_queue[i], adapter);
1481 if (unlikely(err)) {
1482 dev_err(&adapter->netdev->dev, "%s: failed to "
1483 "initialize rx queue%i\n",
1484 adapter->netdev->name, i);
1485 break;
1486 }
1487 }
1488 return err;
1489
1490}
1491
1492
d1a890fa
SB		 1493static int
1494vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter)
1495{
1496 int i;
1497 size_t sz;
1498 struct vmxnet3_rx_buf_info *bi;
1499
1500 for (i = 0; i < 2; i++) {
1501
1502 sz = rq->rx_ring[i].size * sizeof(struct Vmxnet3_RxDesc);
1503 rq->rx_ring[i].base = pci_alloc_consistent(adapter->pdev, sz,
1504 &rq->rx_ring[i].basePA);
1505 if (!rq->rx_ring[i].base) {
204a6e65
SH		 1506 netdev_err(adapter->netdev,
1507 "failed to allocate rx ring %d\n", i);
d1a890fa
SB		 1508 goto err;
1509 }
1510 }
1511
1512 sz = rq->comp_ring.size * sizeof(struct Vmxnet3_RxCompDesc);
1513 rq->comp_ring.base = pci_alloc_consistent(adapter->pdev, sz,
1514 &rq->comp_ring.basePA);
1515 if (!rq->comp_ring.base) {
204a6e65 1516 netdev_err(adapter->netdev, "failed to allocate rx comp ring\n");
d1a890fa
SB		 1517 goto err;
1518 }
1519
1520 sz = sizeof(struct vmxnet3_rx_buf_info) * (rq->rx_ring[0].size +
1521 rq->rx_ring[1].size);
476c609e 1522 bi = kzalloc(sz, GFP_KERNEL);
e404decb 1523 if (!bi)
d1a890fa 1524 goto err;
e404decb 1525
d1a890fa
SB		 1526 rq->buf_info[0] = bi;
1527 rq->buf_info[1] = bi + rq->rx_ring[0].size;
1528
1529 return 0;
1530
1531err:
1532 vmxnet3_rq_destroy(rq, adapter);
1533 return -ENOMEM;
1534}
1535
1536
09c5088e
SB		 1537static int
1538vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter)
1539{
1540 int i, err = 0;
1541
1542 for (i = 0; i < adapter->num_rx_queues; i++) {
1543 err = vmxnet3_rq_create(&adapter->rx_queue[i], adapter);
1544 if (unlikely(err)) {
1545 dev_err(&adapter->netdev->dev,
1546 "%s: failed to create rx queue%i\n",
1547 adapter->netdev->name, i);
1548 goto err_out;
1549 }
1550 }
1551 return err;
1552err_out:
1553 vmxnet3_rq_destroy_all(adapter);
1554 return err;
1555
1556}
1557
1558/* Multiple queue aware polling function for tx and rx */
1559
d1a890fa
SB		 1560static int
1561vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget)
1562{
09c5088e 1563 int rcd_done = 0, i;
d1a890fa
SB		 1564 if (unlikely(adapter->shared->ecr))
1565 vmxnet3_process_events(adapter);
09c5088e
SB		 1566 for (i = 0; i < adapter->num_tx_queues; i++)
1567 vmxnet3_tq_tx_complete(&adapter->tx_queue[i], adapter);
d1a890fa 1568
09c5088e
SB		 1569 for (i = 0; i < adapter->num_rx_queues; i++)
1570 rcd_done += vmxnet3_rq_rx_complete(&adapter->rx_queue[i],
1571 adapter, budget);
1572 return rcd_done;
d1a890fa
SB		 1573}
1574
1575
1576static int
1577vmxnet3_poll(struct napi_struct *napi, int budget)
1578{
09c5088e
SB		 1579 struct vmxnet3_rx_queue *rx_queue = container_of(napi,
1580 struct vmxnet3_rx_queue, napi);
1581 int rxd_done;
1582
1583 rxd_done = vmxnet3_do_poll(rx_queue->adapter, budget);
1584
1585 if (rxd_done < budget) {
1586 napi_complete(napi);
1587 vmxnet3_enable_all_intrs(rx_queue->adapter);
1588 }
1589 return rxd_done;
1590}
1591
1592/*
1593 * NAPI polling function for MSI-X mode with multiple Rx queues
1594 * Returns the # of the NAPI credit consumed (# of rx descriptors processed)
1595 */
1596
1597static int
1598vmxnet3_poll_rx_only(struct napi_struct *napi, int budget)
1599{
1600 struct vmxnet3_rx_queue *rq = container_of(napi,
1601 struct vmxnet3_rx_queue, napi);
1602 struct vmxnet3_adapter *adapter = rq->adapter;
d1a890fa
SB		 1603 int rxd_done;
1604
09c5088e
SB		 1605 /* When sharing interrupt with corresponding tx queue, process
1606 * tx completions in that queue as well
1607 */
1608 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) {
1609 struct vmxnet3_tx_queue *tq =
1610 &adapter->tx_queue[rq - adapter->rx_queue];
1611 vmxnet3_tq_tx_complete(tq, adapter);
1612 }
1613
1614 rxd_done = vmxnet3_rq_rx_complete(rq, adapter, budget);
d1a890fa
SB		 1615
1616 if (rxd_done < budget) {
1617 napi_complete(napi);
09c5088e 1618 vmxnet3_enable_intr(adapter, rq->comp_ring.intr_idx);
d1a890fa
SB		 1619 }
1620 return rxd_done;
1621}
1622
1623
09c5088e
SB		 1624#ifdef CONFIG_PCI_MSI
1625
1626/*
1627 * Handle completion interrupts on tx queues
1628 * Returns whether or not the intr is handled
1629 */
1630
1631static irqreturn_t
1632vmxnet3_msix_tx(int irq, void *data)
1633{
1634 struct vmxnet3_tx_queue *tq = data;
1635 struct vmxnet3_adapter *adapter = tq->adapter;
1636
1637 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1638 vmxnet3_disable_intr(adapter, tq->comp_ring.intr_idx);
1639
1640 /* Handle the case where only one irq is allocate for all tx queues */
1641 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
1642 int i;
1643 for (i = 0; i < adapter->num_tx_queues; i++) {
1644 struct vmxnet3_tx_queue *txq = &adapter->tx_queue[i];
1645 vmxnet3_tq_tx_complete(txq, adapter);
1646 }
1647 } else {
1648 vmxnet3_tq_tx_complete(tq, adapter);
1649 }
1650 vmxnet3_enable_intr(adapter, tq->comp_ring.intr_idx);
1651
1652 return IRQ_HANDLED;
1653}
1654
1655
1656/*
1657 * Handle completion interrupts on rx queues. Returns whether or not the
1658 * intr is handled
1659 */
1660
1661static irqreturn_t
1662vmxnet3_msix_rx(int irq, void *data)
1663{
1664 struct vmxnet3_rx_queue *rq = data;
1665 struct vmxnet3_adapter *adapter = rq->adapter;
1666
1667 /* disable intr if needed */
1668 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1669 vmxnet3_disable_intr(adapter, rq->comp_ring.intr_idx);
1670 napi_schedule(&rq->napi);
1671
1672 return IRQ_HANDLED;
1673}
1674
1675/*
1676 *----------------------------------------------------------------------------
1677 *
1678 * vmxnet3_msix_event --
1679 *
1680 * vmxnet3 msix event intr handler
1681 *
1682 * Result:
1683 * whether or not the intr is handled
1684 *
1685 *----------------------------------------------------------------------------
1686 */
1687
1688static irqreturn_t
1689vmxnet3_msix_event(int irq, void *data)
1690{
1691 struct net_device *dev = data;
1692 struct vmxnet3_adapter *adapter = netdev_priv(dev);
1693
1694 /* disable intr if needed */
1695 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1696 vmxnet3_disable_intr(adapter, adapter->intr.event_intr_idx);
1697
1698 if (adapter->shared->ecr)
1699 vmxnet3_process_events(adapter);
1700
1701 vmxnet3_enable_intr(adapter, adapter->intr.event_intr_idx);
1702
1703 return IRQ_HANDLED;
1704}
1705
1706#endif /* CONFIG_PCI_MSI */
1707
1708
d1a890fa
SB		 1709/* Interrupt handler for vmxnet3 */
1710static irqreturn_t
1711vmxnet3_intr(int irq, void *dev_id)
1712{
1713 struct net_device *dev = dev_id;
1714 struct vmxnet3_adapter *adapter = netdev_priv(dev);
1715
09c5088e 1716 if (adapter->intr.type == VMXNET3_IT_INTX) {
d1a890fa
SB		 1717 u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR);
1718 if (unlikely(icr == 0))
1719 /* not ours */
1720 return IRQ_NONE;
1721 }
1722
1723
1724 /* disable intr if needed */
1725 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
09c5088e 1726 vmxnet3_disable_all_intrs(adapter);
d1a890fa 1727
09c5088e 1728 napi_schedule(&adapter->rx_queue[0].napi);
d1a890fa
SB		 1729
1730 return IRQ_HANDLED;
1731}
1732
1733#ifdef CONFIG_NET_POLL_CONTROLLER
1734
d1a890fa
SB		 1735/* netpoll callback. */
1736static void
1737vmxnet3_netpoll(struct net_device *netdev)
1738{
1739 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
d1a890fa 1740
09c5088e
SB		 1741 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1742 vmxnet3_disable_all_intrs(adapter);
1743
1744 vmxnet3_do_poll(adapter, adapter->rx_queue[0].rx_ring[0].size);
1745 vmxnet3_enable_all_intrs(adapter);
d1a890fa 1746
d1a890fa 1747}
09c5088e 1748#endif /* CONFIG_NET_POLL_CONTROLLER */
d1a890fa
SB		 1749
1750static int
1751vmxnet3_request_irqs(struct vmxnet3_adapter *adapter)
1752{
09c5088e
SB		 1753 struct vmxnet3_intr *intr = &adapter->intr;
1754 int err = 0, i;
1755 int vector = 0;
d1a890fa 1756
8f7e524c 1757#ifdef CONFIG_PCI_MSI
d1a890fa 1758 if (adapter->intr.type == VMXNET3_IT_MSIX) {
09c5088e
SB		 1759 for (i = 0; i < adapter->num_tx_queues; i++) {
1760 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
1761 sprintf(adapter->tx_queue[i].name, "%s-tx-%d",
1762 adapter->netdev->name, vector);
1763 err = request_irq(
1764 intr->msix_entries[vector].vector,
1765 vmxnet3_msix_tx, 0,
1766 adapter->tx_queue[i].name,
1767 &adapter->tx_queue[i]);
1768 } else {
1769 sprintf(adapter->tx_queue[i].name, "%s-rxtx-%d",
1770 adapter->netdev->name, vector);
1771 }
1772 if (err) {
1773 dev_err(&adapter->netdev->dev,
1774 "Failed to request irq for MSIX, %s, "
1775 "error %d\n",
1776 adapter->tx_queue[i].name, err);
1777 return err;
1778 }
1779
1780 /* Handle the case where only 1 MSIx was allocated for
1781 * all tx queues */
1782 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
1783 for (; i < adapter->num_tx_queues; i++)
1784 adapter->tx_queue[i].comp_ring.intr_idx
1785 = vector;
1786 vector++;
1787 break;
1788 } else {
1789 adapter->tx_queue[i].comp_ring.intr_idx
1790 = vector++;
1791 }
1792 }
1793 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE)
1794 vector = 0;
1795
1796 for (i = 0; i < adapter->num_rx_queues; i++) {
1797 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE)
1798 sprintf(adapter->rx_queue[i].name, "%s-rx-%d",
1799 adapter->netdev->name, vector);
1800 else
1801 sprintf(adapter->rx_queue[i].name, "%s-rxtx-%d",
1802 adapter->netdev->name, vector);
1803 err = request_irq(intr->msix_entries[vector].vector,
1804 vmxnet3_msix_rx, 0,
1805 adapter->rx_queue[i].name,
1806 &(adapter->rx_queue[i]));
1807 if (err) {
204a6e65
SH		 1808 netdev_err(adapter->netdev,
1809 "Failed to request irq for MSIX, "
1810 "%s, error %d\n",
1811 adapter->rx_queue[i].name, err);
09c5088e
SB		 1812 return err;
1813 }
1814
1815 adapter->rx_queue[i].comp_ring.intr_idx = vector++;
1816 }
1817
1818 sprintf(intr->event_msi_vector_name, "%s-event-%d",
1819 adapter->netdev->name, vector);
1820 err = request_irq(intr->msix_entries[vector].vector,
1821 vmxnet3_msix_event, 0,
1822 intr->event_msi_vector_name, adapter->netdev);
1823 intr->event_intr_idx = vector;
1824
1825 } else if (intr->type == VMXNET3_IT_MSI) {
1826 adapter->num_rx_queues = 1;
d1a890fa
SB		 1827 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0,
1828 adapter->netdev->name, adapter->netdev);
09c5088e 1829 } else {
115924b6 1830#endif
09c5088e 1831 adapter->num_rx_queues = 1;
d1a890fa
SB		 1832 err = request_irq(adapter->pdev->irq, vmxnet3_intr,
1833 IRQF_SHARED, adapter->netdev->name,
1834 adapter->netdev);
09c5088e 1835#ifdef CONFIG_PCI_MSI
d1a890fa 1836 }
09c5088e
SB		 1837#endif
1838 intr->num_intrs = vector + 1;
1839 if (err) {
204a6e65
SH		 1840 netdev_err(adapter->netdev,
1841 "Failed to request irq (intr type:%d), error %d\n",
1842 intr->type, err);
09c5088e
SB		 1843 } else {
1844 /* Number of rx queues will not change after this */
1845 for (i = 0; i < adapter->num_rx_queues; i++) {
1846 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
1847 rq->qid = i;
1848 rq->qid2 = i + adapter->num_rx_queues;
1849 }
d1a890fa
SB		 1850
1851
d1a890fa 1852
09c5088e
SB		 1853 /* init our intr settings */
1854 for (i = 0; i < intr->num_intrs; i++)
1855 intr->mod_levels[i] = UPT1_IML_ADAPTIVE;
1856 if (adapter->intr.type != VMXNET3_IT_MSIX) {
1857 adapter->intr.event_intr_idx = 0;
1858 for (i = 0; i < adapter->num_tx_queues; i++)
1859 adapter->tx_queue[i].comp_ring.intr_idx = 0;
1860 adapter->rx_queue[0].comp_ring.intr_idx = 0;
1861 }
d1a890fa 1862
204a6e65
SH		 1863 netdev_info(adapter->netdev,
1864 "intr type %u, mode %u, %u vectors allocated\n",
1865 intr->type, intr->mask_mode, intr->num_intrs);
d1a890fa
SB		 1866 }
1867
1868 return err;
1869}
1870
1871
1872static void
1873vmxnet3_free_irqs(struct vmxnet3_adapter *adapter)
1874{
09c5088e
SB		 1875 struct vmxnet3_intr *intr = &adapter->intr;
1876 BUG_ON(intr->type == VMXNET3_IT_AUTO || intr->num_intrs <= 0);
d1a890fa 1877
09c5088e 1878 switch (intr->type) {
8f7e524c 1879#ifdef CONFIG_PCI_MSI
d1a890fa
SB		 1880 case VMXNET3_IT_MSIX:
1881 {
09c5088e 1882 int i, vector = 0;
d1a890fa 1883
09c5088e
SB		 1884 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
1885 for (i = 0; i < adapter->num_tx_queues; i++) {
1886 free_irq(intr->msix_entries[vector++].vector,
1887 &(adapter->tx_queue[i]));
1888 if (adapter->share_intr == VMXNET3_INTR_TXSHARE)
1889 break;
1890 }
1891 }
1892
1893 for (i = 0; i < adapter->num_rx_queues; i++) {
1894 free_irq(intr->msix_entries[vector++].vector,
1895 &(adapter->rx_queue[i]));
1896 }
1897
1898 free_irq(intr->msix_entries[vector].vector,
1899 adapter->netdev);
1900 BUG_ON(vector >= intr->num_intrs);
d1a890fa
SB		 1901 break;
1902 }
8f7e524c 1903#endif
d1a890fa
SB		 1904 case VMXNET3_IT_MSI:
1905 free_irq(adapter->pdev->irq, adapter->netdev);
1906 break;
1907 case VMXNET3_IT_INTX:
1908 free_irq(adapter->pdev->irq, adapter->netdev);
1909 break;
1910 default:
c068e777 1911 BUG();
d1a890fa
SB		 1912 }
1913}
1914
d1a890fa
SB		 1915
1916static void
1917vmxnet3_restore_vlan(struct vmxnet3_adapter *adapter)
1918{
72e85c45
JG		 1919 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1920 u16 vid;
d1a890fa 1921
72e85c45
JG		 1922 /* allow untagged pkts */
1923 VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0);
1924
1925 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
1926 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
d1a890fa
SB		 1927}
1928
1929
8e586137 1930static int
d1a890fa
SB		 1931vmxnet3_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
1932{
1933 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
d1a890fa 1934
f6957f88
JG		 1935 if (!(netdev->flags & IFF_PROMISC)) {
1936 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1937 unsigned long flags;
1938
1939 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
1940 spin_lock_irqsave(&adapter->cmd_lock, flags);
1941 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1942 VMXNET3_CMD_UPDATE_VLAN_FILTERS);
1943 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
1944 }
72e85c45
JG		 1945
1946 set_bit(vid, adapter->active_vlans);
8e586137
JP		 1947
1948 return 0;
d1a890fa
SB		 1949}
1950
1951
8e586137 1952static int
d1a890fa
SB		 1953vmxnet3_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
1954{
1955 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
d1a890fa 1956
f6957f88
JG		 1957 if (!(netdev->flags & IFF_PROMISC)) {
1958 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1959 unsigned long flags;
1960
1961 VMXNET3_CLEAR_VFTABLE_ENTRY(vfTable, vid);
1962 spin_lock_irqsave(&adapter->cmd_lock, flags);
1963 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1964 VMXNET3_CMD_UPDATE_VLAN_FILTERS);
1965 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
1966 }
72e85c45
JG		 1967
1968 clear_bit(vid, adapter->active_vlans);
8e586137
JP		 1969
1970 return 0;
d1a890fa
SB		 1971}
1972
1973
1974static u8 *
1975vmxnet3_copy_mc(struct net_device *netdev)
1976{
1977 u8 *buf = NULL;
4cd24eaf 1978 u32 sz = netdev_mc_count(netdev) * ETH_ALEN;
d1a890fa
SB		 1979
1980 /* struct Vmxnet3_RxFilterConf.mfTableLen is u16. */
1981 if (sz <= 0xffff) {
1982 /* We may be called with BH disabled */
1983 buf = kmalloc(sz, GFP_ATOMIC);
1984 if (buf) {
22bedad3 1985 struct netdev_hw_addr *ha;
567ec874 1986 int i = 0;
d1a890fa 1987
22bedad3
JP		 1988 netdev_for_each_mc_addr(ha, netdev)
1989 memcpy(buf + i++ * ETH_ALEN, ha->addr,
d1a890fa 1990 ETH_ALEN);
d1a890fa
SB		 1991 }
1992 }
1993 return buf;
1994}
1995
1996
1997static void
1998vmxnet3_set_mc(struct net_device *netdev)
1999{
2000 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
83d0feff 2001 unsigned long flags;
d1a890fa
SB		 2002 struct Vmxnet3_RxFilterConf *rxConf =
2003 &adapter->shared->devRead.rxFilterConf;
2004 u8 *new_table = NULL;
2005 u32 new_mode = VMXNET3_RXM_UCAST;
2006
72e85c45
JG		 2007 if (netdev->flags & IFF_PROMISC) {
2008 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
2009 memset(vfTable, 0, VMXNET3_VFT_SIZE * sizeof(*vfTable));
2010
d1a890fa 2011 new_mode |= VMXNET3_RXM_PROMISC;
72e85c45
JG		 2012 } else {
2013 vmxnet3_restore_vlan(adapter);
2014 }
d1a890fa
SB		 2015
2016 if (netdev->flags & IFF_BROADCAST)
2017 new_mode |= VMXNET3_RXM_BCAST;
2018
2019 if (netdev->flags & IFF_ALLMULTI)
2020 new_mode |= VMXNET3_RXM_ALL_MULTI;
2021 else
4cd24eaf 2022 if (!netdev_mc_empty(netdev)) {
d1a890fa
SB		 2023 new_table = vmxnet3_copy_mc(netdev);
2024 if (new_table) {
2025 new_mode |= VMXNET3_RXM_MCAST;
115924b6 2026 rxConf->mfTableLen = cpu_to_le16(
4cd24eaf 2027 netdev_mc_count(netdev) * ETH_ALEN);
115924b6
SB		 2028 rxConf->mfTablePA = cpu_to_le64(virt_to_phys(
2029 new_table));
d1a890fa 2030 } else {
204a6e65
SH		 2031 netdev_info(netdev, "failed to copy mcast list"
2032 ", setting ALL_MULTI\n");
d1a890fa
SB		 2033 new_mode |= VMXNET3_RXM_ALL_MULTI;
2034 }
2035 }
2036
2037
2038 if (!(new_mode & VMXNET3_RXM_MCAST)) {
2039 rxConf->mfTableLen = 0;
2040 rxConf->mfTablePA = 0;
2041 }
2042
83d0feff 2043 spin_lock_irqsave(&adapter->cmd_lock, flags);
d1a890fa 2044 if (new_mode != rxConf->rxMode) {
115924b6 2045 rxConf->rxMode = cpu_to_le32(new_mode);
d1a890fa
SB		 2046 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2047 VMXNET3_CMD_UPDATE_RX_MODE);
72e85c45
JG		 2048 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2049 VMXNET3_CMD_UPDATE_VLAN_FILTERS);
d1a890fa
SB		 2050 }
2051
2052 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2053 VMXNET3_CMD_UPDATE_MAC_FILTERS);
83d0feff 2054 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
d1a890fa
SB		 2055
2056 kfree(new_table);
2057}
2058
09c5088e
SB		 2059void
2060vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter)
2061{
2062 int i;
2063
2064 for (i = 0; i < adapter->num_rx_queues; i++)
2065 vmxnet3_rq_destroy(&adapter->rx_queue[i], adapter);
2066}
2067
d1a890fa
SB		 2068
2069/*
2070 * Set up driver_shared based on settings in adapter.
2071 */
2072
2073static void
2074vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter)
2075{
2076 struct Vmxnet3_DriverShared *shared = adapter->shared;
2077 struct Vmxnet3_DSDevRead *devRead = &shared->devRead;
2078 struct Vmxnet3_TxQueueConf *tqc;
2079 struct Vmxnet3_RxQueueConf *rqc;
2080 int i;
2081
2082 memset(shared, 0, sizeof(*shared));
2083
2084 /* driver settings */
115924b6
SB		 2085 shared->magic = cpu_to_le32(VMXNET3_REV1_MAGIC);
2086 devRead->misc.driverInfo.version = cpu_to_le32(
2087 VMXNET3_DRIVER_VERSION_NUM);
d1a890fa
SB		 2088 devRead->misc.driverInfo.gos.gosBits = (sizeof(void *) == 4 ?
2089 VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64);
2090 devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX;
115924b6
SB		 2091 *((u32 *)&devRead->misc.driverInfo.gos) = cpu_to_le32(
2092 *((u32 *)&devRead->misc.driverInfo.gos));
2093 devRead->misc.driverInfo.vmxnet3RevSpt = cpu_to_le32(1);
2094 devRead->misc.driverInfo.uptVerSpt = cpu_to_le32(1);
d1a890fa 2095
115924b6
SB		 2096 devRead->misc.ddPA = cpu_to_le64(virt_to_phys(adapter));
2097 devRead->misc.ddLen = cpu_to_le32(sizeof(struct vmxnet3_adapter));
d1a890fa
SB		 2098
2099 /* set up feature flags */
a0d2730c 2100 if (adapter->netdev->features & NETIF_F_RXCSUM)
3843e515 2101 devRead->misc.uptFeatures |= UPT1_F_RXCSUM;
d1a890fa 2102
a0d2730c 2103 if (adapter->netdev->features & NETIF_F_LRO) {
3843e515 2104 devRead->misc.uptFeatures |= UPT1_F_LRO;
115924b6 2105 devRead->misc.maxNumRxSG = cpu_to_le16(1 + MAX_SKB_FRAGS);
d1a890fa 2106 }
54da3d00 2107 if (adapter->netdev->features & NETIF_F_HW_VLAN_RX)
3843e515 2108 devRead->misc.uptFeatures |= UPT1_F_RXVLAN;
d1a890fa 2109
115924b6
SB		 2110 devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu);
2111 devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa);
2112 devRead->misc.queueDescLen = cpu_to_le32(
09c5088e
SB		 2113 adapter->num_tx_queues * sizeof(struct Vmxnet3_TxQueueDesc) +
2114 adapter->num_rx_queues * sizeof(struct Vmxnet3_RxQueueDesc));
d1a890fa
SB		 2115
2116 /* tx queue settings */
09c5088e
SB		 2117 devRead->misc.numTxQueues = adapter->num_tx_queues;
2118 for (i = 0; i < adapter->num_tx_queues; i++) {
2119 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i];
2120 BUG_ON(adapter->tx_queue[i].tx_ring.base == NULL);
2121 tqc = &adapter->tqd_start[i].conf;
2122 tqc->txRingBasePA = cpu_to_le64(tq->tx_ring.basePA);
2123 tqc->dataRingBasePA = cpu_to_le64(tq->data_ring.basePA);
2124 tqc->compRingBasePA = cpu_to_le64(tq->comp_ring.basePA);
2125 tqc->ddPA = cpu_to_le64(virt_to_phys(tq->buf_info));
2126 tqc->txRingSize = cpu_to_le32(tq->tx_ring.size);
2127 tqc->dataRingSize = cpu_to_le32(tq->data_ring.size);
2128 tqc->compRingSize = cpu_to_le32(tq->comp_ring.size);
2129 tqc->ddLen = cpu_to_le32(
2130 sizeof(struct vmxnet3_tx_buf_info) *
2131 tqc->txRingSize);
2132 tqc->intrIdx = tq->comp_ring.intr_idx;
2133 }
d1a890fa
SB		 2134
2135 /* rx queue settings */
09c5088e
SB		 2136 devRead->misc.numRxQueues = adapter->num_rx_queues;
2137 for (i = 0; i < adapter->num_rx_queues; i++) {
2138 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
2139 rqc = &adapter->rqd_start[i].conf;
2140 rqc->rxRingBasePA[0] = cpu_to_le64(rq->rx_ring[0].basePA);
2141 rqc->rxRingBasePA[1] = cpu_to_le64(rq->rx_ring[1].basePA);
2142 rqc->compRingBasePA = cpu_to_le64(rq->comp_ring.basePA);
2143 rqc->ddPA = cpu_to_le64(virt_to_phys(
2144 rq->buf_info));
2145 rqc->rxRingSize[0] = cpu_to_le32(rq->rx_ring[0].size);
2146 rqc->rxRingSize[1] = cpu_to_le32(rq->rx_ring[1].size);
2147 rqc->compRingSize = cpu_to_le32(rq->comp_ring.size);
2148 rqc->ddLen = cpu_to_le32(
2149 sizeof(struct vmxnet3_rx_buf_info) *
2150 (rqc->rxRingSize[0] +
2151 rqc->rxRingSize[1]));
2152 rqc->intrIdx = rq->comp_ring.intr_idx;
2153 }
2154
2155#ifdef VMXNET3_RSS
2156 memset(adapter->rss_conf, 0, sizeof(*adapter->rss_conf));
2157
2158 if (adapter->rss) {
2159 struct UPT1_RSSConf *rssConf = adapter->rss_conf;
66d35910
SH		 2160 static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = {
2161 0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac,
2162 0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28,
2163 0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70,
2164 0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3,
2165 0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9,
2166 };
2167
09c5088e
SB		 2168 devRead->misc.uptFeatures |= UPT1_F_RSS;
2169 devRead->misc.numRxQueues = adapter->num_rx_queues;
2170 rssConf->hashType = UPT1_RSS_HASH_TYPE_TCP_IPV4 |
2171 UPT1_RSS_HASH_TYPE_IPV4 |
2172 UPT1_RSS_HASH_TYPE_TCP_IPV6 |
2173 UPT1_RSS_HASH_TYPE_IPV6;
2174 rssConf->hashFunc = UPT1_RSS_HASH_FUNC_TOEPLITZ;
2175 rssConf->hashKeySize = UPT1_RSS_MAX_KEY_SIZE;
2176 rssConf->indTableSize = VMXNET3_RSS_IND_TABLE_SIZE;
66d35910
SH		 2177 memcpy(rssConf->hashKey, rss_key, sizeof(rss_key));
2178
09c5088e 2179 for (i = 0; i < rssConf->indTableSize; i++)
278bc429
BH		 2180 rssConf->indTable[i] = ethtool_rxfh_indir_default(
2181 i, adapter->num_rx_queues);
09c5088e
SB		 2182
2183 devRead->rssConfDesc.confVer = 1;
2184 devRead->rssConfDesc.confLen = sizeof(*rssConf);
2185 devRead->rssConfDesc.confPA = virt_to_phys(rssConf);
2186 }
2187
2188#endif /* VMXNET3_RSS */
d1a890fa
SB		 2189
2190 /* intr settings */
2191 devRead->intrConf.autoMask = adapter->intr.mask_mode ==
2192 VMXNET3_IMM_AUTO;
2193 devRead->intrConf.numIntrs = adapter->intr.num_intrs;
2194 for (i = 0; i < adapter->intr.num_intrs; i++)
2195 devRead->intrConf.modLevels[i] = adapter->intr.mod_levels[i];
2196
2197 devRead->intrConf.eventIntrIdx = adapter->intr.event_intr_idx;
6929fe8a 2198 devRead->intrConf.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
d1a890fa
SB		 2199
2200 /* rx filter settings */
2201 devRead->rxFilterConf.rxMode = 0;
2202 vmxnet3_restore_vlan(adapter);
f9f25026
SB		 2203 vmxnet3_write_mac_addr(adapter, adapter->netdev->dev_addr);
2204
d1a890fa
SB		 2205 /* the rest are already zeroed */
2206}
2207
2208
2209int
2210vmxnet3_activate_dev(struct vmxnet3_adapter *adapter)
2211{
09c5088e 2212 int err, i;
d1a890fa 2213 u32 ret;
83d0feff 2214 unsigned long flags;
d1a890fa 2215
fdcd79b9 2216 netdev_dbg(adapter->netdev, "%s: skb_buf_size %d, rx_buf_per_pkt %d,"
09c5088e
SB		 2217 " ring sizes %u %u %u\n", adapter->netdev->name,
2218 adapter->skb_buf_size, adapter->rx_buf_per_pkt,
2219 adapter->tx_queue[0].tx_ring.size,
2220 adapter->rx_queue[0].rx_ring[0].size,
2221 adapter->rx_queue[0].rx_ring[1].size);
2222
2223 vmxnet3_tq_init_all(adapter);
2224 err = vmxnet3_rq_init_all(adapter);
d1a890fa 2225 if (err) {
204a6e65
SH		 2226 netdev_err(adapter->netdev,
2227 "Failed to init rx queue error %d\n", err);
d1a890fa
SB		 2228 goto rq_err;
2229 }
2230
2231 err = vmxnet3_request_irqs(adapter);
2232 if (err) {
204a6e65
SH		 2233 netdev_err(adapter->netdev,
2234 "Failed to setup irq for error %d\n", err);
d1a890fa
SB		 2235 goto irq_err;
2236 }
2237
2238 vmxnet3_setup_driver_shared(adapter);
2239
115924b6
SB		 2240 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, VMXNET3_GET_ADDR_LO(
2241 adapter->shared_pa));
2242 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, VMXNET3_GET_ADDR_HI(
2243 adapter->shared_pa));
83d0feff 2244 spin_lock_irqsave(&adapter->cmd_lock, flags);
d1a890fa
SB		 2245 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2246 VMXNET3_CMD_ACTIVATE_DEV);
2247 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
83d0feff 2248 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
d1a890fa
SB		 2249
2250 if (ret != 0) {
204a6e65
SH		 2251 netdev_err(adapter->netdev,
2252 "Failed to activate dev: error %u\n", ret);
d1a890fa
SB		 2253 err = -EINVAL;
2254 goto activate_err;
2255 }
09c5088e
SB		 2256
2257 for (i = 0; i < adapter->num_rx_queues; i++) {
2258 VMXNET3_WRITE_BAR0_REG(adapter,
2259 VMXNET3_REG_RXPROD + i * VMXNET3_REG_ALIGN,
2260 adapter->rx_queue[i].rx_ring[0].next2fill);
2261 VMXNET3_WRITE_BAR0_REG(adapter, (VMXNET3_REG_RXPROD2 +
2262 (i * VMXNET3_REG_ALIGN)),
2263 adapter->rx_queue[i].rx_ring[1].next2fill);
2264 }
d1a890fa
SB		 2265
2266 /* Apply the rx filter settins last. */
2267 vmxnet3_set_mc(adapter->netdev);
2268
2269 /*
2270 * Check link state when first activating device. It will start the
2271 * tx queue if the link is up.
2272 */
4a1745fc 2273 vmxnet3_check_link(adapter, true);
09c5088e
SB		 2274 for (i = 0; i < adapter->num_rx_queues; i++)
2275 napi_enable(&adapter->rx_queue[i].napi);
d1a890fa
SB		 2276 vmxnet3_enable_all_intrs(adapter);
2277 clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
2278 return 0;
2279
2280activate_err:
2281 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, 0);
2282 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, 0);
2283 vmxnet3_free_irqs(adapter);
2284irq_err:
2285rq_err:
2286 /* free up buffers we allocated */
09c5088e 2287 vmxnet3_rq_cleanup_all(adapter);
d1a890fa
SB		 2288 return err;
2289}
2290
2291
2292void
2293vmxnet3_reset_dev(struct vmxnet3_adapter *adapter)
2294{
83d0feff
SB		 2295 unsigned long flags;
2296 spin_lock_irqsave(&adapter->cmd_lock, flags);
d1a890fa 2297 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
83d0feff 2298 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
d1a890fa
SB		 2299}
2300
2301
2302int
2303vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter)
2304{
09c5088e 2305 int i;
83d0feff 2306 unsigned long flags;
d1a890fa
SB		 2307 if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state))
2308 return 0;
2309
2310
83d0feff 2311 spin_lock_irqsave(&adapter->cmd_lock, flags);
d1a890fa
SB		 2312 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2313 VMXNET3_CMD_QUIESCE_DEV);
83d0feff 2314 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
d1a890fa
SB		 2315 vmxnet3_disable_all_intrs(adapter);
2316
09c5088e
SB		 2317 for (i = 0; i < adapter->num_rx_queues; i++)
2318 napi_disable(&adapter->rx_queue[i].napi);
d1a890fa
SB		 2319 netif_tx_disable(adapter->netdev);
2320 adapter->link_speed = 0;
2321 netif_carrier_off(adapter->netdev);
2322
09c5088e
SB		 2323 vmxnet3_tq_cleanup_all(adapter);
2324 vmxnet3_rq_cleanup_all(adapter);
d1a890fa
SB		 2325 vmxnet3_free_irqs(adapter);
2326 return 0;
2327}
2328
2329
2330static void
2331vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
2332{
2333 u32 tmp;
2334
2335 tmp = *(u32 *)mac;
2336 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACL, tmp);
2337
2338 tmp = (mac[5] << 8) | mac[4];
2339 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACH, tmp);
2340}
2341
2342
2343static int
2344vmxnet3_set_mac_addr(struct net_device *netdev, void *p)
2345{
2346 struct sockaddr *addr = p;
2347 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2348
2349 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2350 vmxnet3_write_mac_addr(adapter, addr->sa_data);
2351
2352 return 0;
2353}
2354
2355
2356/* ==================== initialization and cleanup routines ============ */
2357
2358static int
2359vmxnet3_alloc_pci_resources(struct vmxnet3_adapter *adapter, bool *dma64)
2360{
2361 int err;
2362 unsigned long mmio_start, mmio_len;
2363 struct pci_dev *pdev = adapter->pdev;
2364
2365 err = pci_enable_device(pdev);
2366 if (err) {
204a6e65 2367 dev_err(&pdev->dev, "Failed to enable adapter: error %d\n", err);
d1a890fa
SB		 2368 return err;
2369 }
2370
2371 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) == 0) {
2372 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)) != 0) {
204a6e65
SH		 2373 dev_err(&pdev->dev,
2374 "pci_set_consistent_dma_mask failed\n");
d1a890fa
SB		 2375 err = -EIO;
2376 goto err_set_mask;
2377 }
2378 *dma64 = true;
2379 } else {
2380 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(32)) != 0) {
204a6e65
SH		 2381 dev_err(&pdev->dev,
2382 "pci_set_dma_mask failed\n");
d1a890fa
SB		 2383 err = -EIO;
2384 goto err_set_mask;
2385 }
2386 *dma64 = false;
2387 }
2388
2389 err = pci_request_selected_regions(pdev, (1 << 2) - 1,
2390 vmxnet3_driver_name);
2391 if (err) {
204a6e65
SH		 2392 dev_err(&pdev->dev,
2393 "Failed to request region for adapter: error %d\n", err);
d1a890fa
SB		 2394 goto err_set_mask;
2395 }
2396
2397 pci_set_master(pdev);
2398
2399 mmio_start = pci_resource_start(pdev, 0);
2400 mmio_len = pci_resource_len(pdev, 0);
2401 adapter->hw_addr0 = ioremap(mmio_start, mmio_len);
2402 if (!adapter->hw_addr0) {
204a6e65 2403 dev_err(&pdev->dev, "Failed to map bar0\n");
d1a890fa
SB		 2404 err = -EIO;
2405 goto err_ioremap;
2406 }
2407
2408 mmio_start = pci_resource_start(pdev, 1);
2409 mmio_len = pci_resource_len(pdev, 1);
2410 adapter->hw_addr1 = ioremap(mmio_start, mmio_len);
2411 if (!adapter->hw_addr1) {
204a6e65 2412 dev_err(&pdev->dev, "Failed to map bar1\n");
d1a890fa
SB		 2413 err = -EIO;
2414 goto err_bar1;
2415 }
2416 return 0;
2417
2418err_bar1:
2419 iounmap(adapter->hw_addr0);
2420err_ioremap:
2421 pci_release_selected_regions(pdev, (1 << 2) - 1);
2422err_set_mask:
2423 pci_disable_device(pdev);
2424 return err;
2425}
2426
2427
2428static void
2429vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter)
2430{
2431 BUG_ON(!adapter->pdev);
2432
2433 iounmap(adapter->hw_addr0);
2434 iounmap(adapter->hw_addr1);
2435 pci_release_selected_regions(adapter->pdev, (1 << 2) - 1);
2436 pci_disable_device(adapter->pdev);
2437}
2438
2439
2440static void
2441vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter)
2442{
09c5088e
SB		 2443 size_t sz, i, ring0_size, ring1_size, comp_size;
2444 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[0];
2445
d1a890fa
SB		 2446
2447 if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE -
2448 VMXNET3_MAX_ETH_HDR_SIZE) {
2449 adapter->skb_buf_size = adapter->netdev->mtu +
2450 VMXNET3_MAX_ETH_HDR_SIZE;
2451 if (adapter->skb_buf_size < VMXNET3_MIN_T0_BUF_SIZE)
2452 adapter->skb_buf_size = VMXNET3_MIN_T0_BUF_SIZE;
2453
2454 adapter->rx_buf_per_pkt = 1;
2455 } else {
2456 adapter->skb_buf_size = VMXNET3_MAX_SKB_BUF_SIZE;
2457 sz = adapter->netdev->mtu - VMXNET3_MAX_SKB_BUF_SIZE +
2458 VMXNET3_MAX_ETH_HDR_SIZE;
2459 adapter->rx_buf_per_pkt = 1 + (sz + PAGE_SIZE - 1) / PAGE_SIZE;
2460 }
2461
2462 /*
2463 * for simplicity, force the ring0 size to be a multiple of
2464 * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN
2465 */
2466 sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN;
09c5088e
SB		 2467 ring0_size = adapter->rx_queue[0].rx_ring[0].size;
2468 ring0_size = (ring0_size + sz - 1) / sz * sz;
a53255d3 2469 ring0_size = min_t(u32, ring0_size, VMXNET3_RX_RING_MAX_SIZE /
09c5088e
SB		 2470 sz * sz);
2471 ring1_size = adapter->rx_queue[0].rx_ring[1].size;
2472 comp_size = ring0_size + ring1_size;
2473
2474 for (i = 0; i < adapter->num_rx_queues; i++) {
2475 rq = &adapter->rx_queue[i];
2476 rq->rx_ring[0].size = ring0_size;
2477 rq->rx_ring[1].size = ring1_size;
2478 rq->comp_ring.size = comp_size;
2479 }
d1a890fa
SB		 2480}
2481
2482
2483int
2484vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size,
2485 u32 rx_ring_size, u32 rx_ring2_size)
2486{
09c5088e
SB		 2487 int err = 0, i;
2488
2489 for (i = 0; i < adapter->num_tx_queues; i++) {
2490 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i];
2491 tq->tx_ring.size = tx_ring_size;
2492 tq->data_ring.size = tx_ring_size;
2493 tq->comp_ring.size = tx_ring_size;
2494 tq->shared = &adapter->tqd_start[i].ctrl;
2495 tq->stopped = true;
2496 tq->adapter = adapter;
2497 tq->qid = i;
2498 err = vmxnet3_tq_create(tq, adapter);
2499 /*
2500 * Too late to change num_tx_queues. We cannot do away with
2501 * lesser number of queues than what we asked for
2502 */
2503 if (err)
2504 goto queue_err;
2505 }
d1a890fa 2506
09c5088e
SB		 2507 adapter->rx_queue[0].rx_ring[0].size = rx_ring_size;
2508 adapter->rx_queue[0].rx_ring[1].size = rx_ring2_size;
d1a890fa 2509 vmxnet3_adjust_rx_ring_size(adapter);
09c5088e
SB		 2510 for (i = 0; i < adapter->num_rx_queues; i++) {
2511 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
2512 /* qid and qid2 for rx queues will be assigned later when num
2513 * of rx queues is finalized after allocating intrs */
2514 rq->shared = &adapter->rqd_start[i].ctrl;
2515 rq->adapter = adapter;
2516 err = vmxnet3_rq_create(rq, adapter);
2517 if (err) {
2518 if (i == 0) {
204a6e65
SH		 2519 netdev_err(adapter->netdev,
2520 "Could not allocate any rx queues. "
2521 "Aborting.\n");
09c5088e
SB		 2522 goto queue_err;
2523 } else {
204a6e65
SH		 2524 netdev_info(adapter->netdev,
2525 "Number of rx queues changed "
2526 "to : %d.\n", i);
09c5088e
SB		 2527 adapter->num_rx_queues = i;
2528 err = 0;
2529 break;
2530 }
2531 }
2532 }
2533 return err;
2534queue_err:
2535 vmxnet3_tq_destroy_all(adapter);
d1a890fa
SB		 2536 return err;
2537}
2538
2539static int
2540vmxnet3_open(struct net_device *netdev)
2541{
2542 struct vmxnet3_adapter *adapter;
09c5088e 2543 int err, i;
d1a890fa
SB		 2544
2545 adapter = netdev_priv(netdev);
2546
09c5088e
SB		 2547 for (i = 0; i < adapter->num_tx_queues; i++)
2548 spin_lock_init(&adapter->tx_queue[i].tx_lock);
d1a890fa
SB		 2549
2550 err = vmxnet3_create_queues(adapter, VMXNET3_DEF_TX_RING_SIZE,
2551 VMXNET3_DEF_RX_RING_SIZE,
2552 VMXNET3_DEF_RX_RING_SIZE);
2553 if (err)
2554 goto queue_err;
2555
2556 err = vmxnet3_activate_dev(adapter);
2557 if (err)
2558 goto activate_err;
2559
2560 return 0;
2561
2562activate_err:
09c5088e
SB		 2563 vmxnet3_rq_destroy_all(adapter);
2564 vmxnet3_tq_destroy_all(adapter);
d1a890fa
SB		 2565queue_err:
2566 return err;
2567}
2568
2569
2570static int
2571vmxnet3_close(struct net_device *netdev)
2572{
2573 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2574
2575 /*
2576 * Reset_work may be in the middle of resetting the device, wait for its
2577 * completion.
2578 */
2579 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
2580 msleep(1);
2581
2582 vmxnet3_quiesce_dev(adapter);
2583
09c5088e
SB		 2584 vmxnet3_rq_destroy_all(adapter);
2585 vmxnet3_tq_destroy_all(adapter);
d1a890fa
SB		 2586
2587 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2588
2589
2590 return 0;
2591}
2592
2593
2594void
2595vmxnet3_force_close(struct vmxnet3_adapter *adapter)
2596{
09c5088e
SB		 2597 int i;
2598
d1a890fa
SB		 2599 /*
2600 * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise
2601 * vmxnet3_close() will deadlock.
2602 */
2603 BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state));
2604
2605 /* we need to enable NAPI, otherwise dev_close will deadlock */
09c5088e
SB		 2606 for (i = 0; i < adapter->num_rx_queues; i++)
2607 napi_enable(&adapter->rx_queue[i].napi);
d1a890fa
SB		 2608 dev_close(adapter->netdev);
2609}
2610
2611
2612static int
2613vmxnet3_change_mtu(struct net_device *netdev, int new_mtu)
2614{
2615 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2616 int err = 0;
2617
2618 if (new_mtu < VMXNET3_MIN_MTU || new_mtu > VMXNET3_MAX_MTU)
2619 return -EINVAL;
2620
d1a890fa
SB		 2621 netdev->mtu = new_mtu;
2622
2623 /*
2624 * Reset_work may be in the middle of resetting the device, wait for its
2625 * completion.
2626 */
2627 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
2628 msleep(1);
2629
2630 if (netif_running(netdev)) {
2631 vmxnet3_quiesce_dev(adapter);
2632 vmxnet3_reset_dev(adapter);
2633
2634 /* we need to re-create the rx queue based on the new mtu */
09c5088e 2635 vmxnet3_rq_destroy_all(adapter);
d1a890fa 2636 vmxnet3_adjust_rx_ring_size(adapter);
09c5088e 2637 err = vmxnet3_rq_create_all(adapter);
d1a890fa 2638 if (err) {
204a6e65
SH		 2639 netdev_err(netdev,
2640 "failed to re-create rx queues, "
2641 " error %d. Closing it.\n", err);
d1a890fa
SB		 2642 goto out;
2643 }
2644
2645 err = vmxnet3_activate_dev(adapter);
2646 if (err) {
204a6e65
SH		 2647 netdev_err(netdev,
2648 "failed to re-activate, error %d. "
2649 "Closing it\n", err);
d1a890fa
SB		 2650 goto out;
2651 }
2652 }
2653
2654out:
2655 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2656 if (err)
2657 vmxnet3_force_close(adapter);
2658
2659 return err;
2660}
2661
2662
2663static void
2664vmxnet3_declare_features(struct vmxnet3_adapter *adapter, bool dma64)
2665{
2666 struct net_device *netdev = adapter->netdev;
2667
a0d2730c
MM		 2668 netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM |
2669 NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_TX |
72e85c45
JG		 2670 NETIF_F_HW_VLAN_RX | NETIF_F_TSO | NETIF_F_TSO6 |
2671 NETIF_F_LRO;
a0d2730c 2672 if (dma64)
ebbf9295 2673 netdev->hw_features |= NETIF_F_HIGHDMA;
72e85c45
JG		 2674 netdev->vlan_features = netdev->hw_features &
2675 ~(NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX);
2676 netdev->features = netdev->hw_features | NETIF_F_HW_VLAN_FILTER;
d1a890fa
SB		 2677}
2678
2679
2680static void
2681vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
2682{
2683 u32 tmp;
2684
2685 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACL);
2686 *(u32 *)mac = tmp;
2687
2688 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACH);
2689 mac[4] = tmp & 0xff;
2690 mac[5] = (tmp >> 8) & 0xff;
2691}
2692
09c5088e
SB		 2693#ifdef CONFIG_PCI_MSI
2694
2695/*
2696 * Enable MSIx vectors.
2697 * Returns :
2698 * 0 on successful enabling of required vectors,
25985edc 2699 * VMXNET3_LINUX_MIN_MSIX_VECT when only minimum number of vectors required
09c5088e
SB		 2700 * could be enabled.
2701 * number of vectors which can be enabled otherwise (this number is smaller
2702 * than VMXNET3_LINUX_MIN_MSIX_VECT)
2703 */
2704
2705static int
2706vmxnet3_acquire_msix_vectors(struct vmxnet3_adapter *adapter,
2707 int vectors)
2708{
2709 int err = 0, vector_threshold;
2710 vector_threshold = VMXNET3_LINUX_MIN_MSIX_VECT;
2711
2712 while (vectors >= vector_threshold) {
2713 err = pci_enable_msix(adapter->pdev, adapter->intr.msix_entries,
2714 vectors);
2715 if (!err) {
2716 adapter->intr.num_intrs = vectors;
2717 return 0;
2718 } else if (err < 0) {
4bad25fa 2719 dev_err(&adapter->netdev->dev,
4c1dc80a 2720 "Failed to enable MSI-X, error: %d\n", err);
09c5088e
SB		 2721 vectors = 0;
2722 } else if (err < vector_threshold) {
2723 break;
2724 } else {
2725 /* If fails to enable required number of MSI-x vectors
7e96fbf2 2726 * try enabling minimum number of vectors required.
09c5088e 2727 */
4bad25fa
SH		 2728 dev_err(&adapter->netdev->dev,
2729 "Failed to enable %d MSI-X, trying %d instead\n",
4c1dc80a 2730 vectors, vector_threshold);
09c5088e 2731 vectors = vector_threshold;
09c5088e
SB		 2732 }
2733 }
2734
4bad25fa
SH		 2735 dev_info(&adapter->pdev->dev,
2736 "Number of MSI-X interrupts which can be allocated "
2737 "is lower than min threshold required.\n");
09c5088e
SB		 2738 return err;
2739}
2740
2741
2742#endif /* CONFIG_PCI_MSI */
d1a890fa
SB		 2743
2744static void
2745vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter)
2746{
2747 u32 cfg;
e328d410 2748 unsigned long flags;
d1a890fa
SB		 2749
2750 /* intr settings */
e328d410 2751 spin_lock_irqsave(&adapter->cmd_lock, flags);
d1a890fa
SB		 2752 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2753 VMXNET3_CMD_GET_CONF_INTR);
2754 cfg = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
e328d410 2755 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
d1a890fa
SB		 2756 adapter->intr.type = cfg & 0x3;
2757 adapter->intr.mask_mode = (cfg >> 2) & 0x3;
2758
2759 if (adapter->intr.type == VMXNET3_IT_AUTO) {
0bdc0d70
SB		 2760 adapter->intr.type = VMXNET3_IT_MSIX;
2761 }
d1a890fa 2762
8f7e524c 2763#ifdef CONFIG_PCI_MSI
0bdc0d70 2764 if (adapter->intr.type == VMXNET3_IT_MSIX) {
09c5088e
SB		 2765 int vector, err = 0;
2766
2767 adapter->intr.num_intrs = (adapter->share_intr ==
2768 VMXNET3_INTR_TXSHARE) ? 1 :
2769 adapter->num_tx_queues;
2770 adapter->intr.num_intrs += (adapter->share_intr ==
2771 VMXNET3_INTR_BUDDYSHARE) ? 0 :
2772 adapter->num_rx_queues;
2773 adapter->intr.num_intrs += 1; /* for link event */
2774
2775 adapter->intr.num_intrs = (adapter->intr.num_intrs >
2776 VMXNET3_LINUX_MIN_MSIX_VECT
2777 ? adapter->intr.num_intrs :
2778 VMXNET3_LINUX_MIN_MSIX_VECT);
2779
2780 for (vector = 0; vector < adapter->intr.num_intrs; vector++)
2781 adapter->intr.msix_entries[vector].entry = vector;
2782
2783 err = vmxnet3_acquire_msix_vectors(adapter,
2784 adapter->intr.num_intrs);
2785 /* If we cannot allocate one MSIx vector per queue
2786 * then limit the number of rx queues to 1
2787 */
2788 if (err == VMXNET3_LINUX_MIN_MSIX_VECT) {
2789 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE
7e96fbf2 2790 || adapter->num_rx_queues != 1) {
09c5088e 2791 adapter->share_intr = VMXNET3_INTR_TXSHARE;
204a6e65
SH		 2792 netdev_err(adapter->netdev,
2793 "Number of rx queues : 1\n");
09c5088e
SB		 2794 adapter->num_rx_queues = 1;
2795 adapter->intr.num_intrs =
2796 VMXNET3_LINUX_MIN_MSIX_VECT;
2797 }
d1a890fa
SB		 2798 return;
2799 }
09c5088e
SB		 2800 if (!err)
2801 return;
2802
2803 /* If we cannot allocate MSIx vectors use only one rx queue */
4bad25fa
SH		 2804 dev_info(&adapter->pdev->dev,
2805 "Failed to enable MSI-X, error %d. "
2806 "Limiting #rx queues to 1, try MSI.\n", err);
09c5088e 2807
0bdc0d70
SB		 2808 adapter->intr.type = VMXNET3_IT_MSI;
2809 }
d1a890fa 2810
0bdc0d70
SB		 2811 if (adapter->intr.type == VMXNET3_IT_MSI) {
2812 int err;
d1a890fa
SB		 2813 err = pci_enable_msi(adapter->pdev);
2814 if (!err) {
09c5088e 2815 adapter->num_rx_queues = 1;
d1a890fa 2816 adapter->intr.num_intrs = 1;
d1a890fa
SB		 2817 return;
2818 }
2819 }
0bdc0d70 2820#endif /* CONFIG_PCI_MSI */
d1a890fa 2821
09c5088e 2822 adapter->num_rx_queues = 1;
204a6e65
SH		 2823 dev_info(&adapter->netdev->dev,
2824 "Using INTx interrupt, #Rx queues: 1.\n");
d1a890fa
SB		 2825 adapter->intr.type = VMXNET3_IT_INTX;
2826
2827 /* INT-X related setting */
2828 adapter->intr.num_intrs = 1;
2829}
2830
2831
2832static void
2833vmxnet3_free_intr_resources(struct vmxnet3_adapter *adapter)
2834{
2835 if (adapter->intr.type == VMXNET3_IT_MSIX)
2836 pci_disable_msix(adapter->pdev);
2837 else if (adapter->intr.type == VMXNET3_IT_MSI)
2838 pci_disable_msi(adapter->pdev);
2839 else
2840 BUG_ON(adapter->intr.type != VMXNET3_IT_INTX);
2841}
2842
2843
2844static void
2845vmxnet3_tx_timeout(struct net_device *netdev)
2846{
2847 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2848 adapter->tx_timeout_count++;
2849
204a6e65 2850 netdev_err(adapter->netdev, "tx hang\n");
d1a890fa 2851 schedule_work(&adapter->work);
09c5088e 2852 netif_wake_queue(adapter->netdev);
d1a890fa
SB		 2853}
2854
2855
2856static void
2857vmxnet3_reset_work(struct work_struct *data)
2858{
2859 struct vmxnet3_adapter *adapter;
2860
2861 adapter = container_of(data, struct vmxnet3_adapter, work);
2862
2863 /* if another thread is resetting the device, no need to proceed */
2864 if (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
2865 return;
2866
2867 /* if the device is closed, we must leave it alone */
d9a5f210 2868 rtnl_lock();
d1a890fa 2869 if (netif_running(adapter->netdev)) {
204a6e65 2870 netdev_notice(adapter->netdev, "resetting\n");
d1a890fa
SB		 2871 vmxnet3_quiesce_dev(adapter);
2872 vmxnet3_reset_dev(adapter);
2873 vmxnet3_activate_dev(adapter);
2874 } else {
204a6e65 2875 netdev_info(adapter->netdev, "already closed\n");
d1a890fa 2876 }
d9a5f210 2877 rtnl_unlock();
d1a890fa
SB		 2878
2879 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2880}
2881
2882
3a4751a3 2883static int
d1a890fa
SB		 2884vmxnet3_probe_device(struct pci_dev *pdev,
2885 const struct pci_device_id *id)
2886{
2887 static const struct net_device_ops vmxnet3_netdev_ops = {
2888 .ndo_open = vmxnet3_open,
2889 .ndo_stop = vmxnet3_close,
2890 .ndo_start_xmit = vmxnet3_xmit_frame,
2891 .ndo_set_mac_address = vmxnet3_set_mac_addr,
2892 .ndo_change_mtu = vmxnet3_change_mtu,
a0d2730c 2893 .ndo_set_features = vmxnet3_set_features,
95305f6c 2894 .ndo_get_stats64 = vmxnet3_get_stats64,
d1a890fa 2895 .ndo_tx_timeout = vmxnet3_tx_timeout,
afc4b13d 2896 .ndo_set_rx_mode = vmxnet3_set_mc,
d1a890fa
SB		 2897 .ndo_vlan_rx_add_vid = vmxnet3_vlan_rx_add_vid,
2898 .ndo_vlan_rx_kill_vid = vmxnet3_vlan_rx_kill_vid,
2899#ifdef CONFIG_NET_POLL_CONTROLLER
2900 .ndo_poll_controller = vmxnet3_netpoll,
2901#endif
2902 };
2903 int err;
2904 bool dma64 = false; /* stupid gcc */
2905 u32 ver;
2906 struct net_device *netdev;
2907 struct vmxnet3_adapter *adapter;
2908 u8 mac[ETH_ALEN];
09c5088e
SB		 2909 int size;
2910 int num_tx_queues;
2911 int num_rx_queues;
2912
e154b639
SB		 2913 if (!pci_msi_enabled())
2914 enable_mq = 0;
2915
09c5088e
SB		 2916#ifdef VMXNET3_RSS
2917 if (enable_mq)
2918 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
2919 (int)num_online_cpus());
2920 else
2921#endif
2922 num_rx_queues = 1;
eebb02b1 2923 num_rx_queues = rounddown_pow_of_two(num_rx_queues);
09c5088e
SB		 2924
2925 if (enable_mq)
2926 num_tx_queues = min(VMXNET3_DEVICE_MAX_TX_QUEUES,
2927 (int)num_online_cpus());
2928 else
2929 num_tx_queues = 1;
2930
eebb02b1 2931 num_tx_queues = rounddown_pow_of_two(num_tx_queues);
09c5088e
SB		 2932 netdev = alloc_etherdev_mq(sizeof(struct vmxnet3_adapter),
2933 max(num_tx_queues, num_rx_queues));
204a6e65
SH		 2934 dev_info(&pdev->dev,
2935 "# of Tx queues : %d, # of Rx queues : %d\n",
2936 num_tx_queues, num_rx_queues);
d1a890fa 2937
41de8d4c 2938 if (!netdev)
d1a890fa 2939 return -ENOMEM;
d1a890fa
SB		 2940
2941 pci_set_drvdata(pdev, netdev);
2942 adapter = netdev_priv(netdev);
2943 adapter->netdev = netdev;
2944 adapter->pdev = pdev;
2945
83d0feff 2946 spin_lock_init(&adapter->cmd_lock);
d1a890fa 2947 adapter->shared = pci_alloc_consistent(adapter->pdev,
96800ee7 2948 sizeof(struct Vmxnet3_DriverShared),
2949 &adapter->shared_pa);
d1a890fa 2950 if (!adapter->shared) {
204a6e65 2951 dev_err(&pdev->dev, "Failed to allocate memory\n");
d1a890fa
SB		 2952 err = -ENOMEM;
2953 goto err_alloc_shared;
2954 }
2955
09c5088e
SB		 2956 adapter->num_rx_queues = num_rx_queues;
2957 adapter->num_tx_queues = num_tx_queues;
2958
2959 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
2960 size += sizeof(struct Vmxnet3_RxQueueDesc) * adapter->num_rx_queues;
2961 adapter->tqd_start = pci_alloc_consistent(adapter->pdev, size,
96800ee7 2962 &adapter->queue_desc_pa);
d1a890fa
SB		 2963
2964 if (!adapter->tqd_start) {
204a6e65 2965 dev_err(&pdev->dev, "Failed to allocate memory\n");
d1a890fa
SB		 2966 err = -ENOMEM;
2967 goto err_alloc_queue_desc;
2968 }
09c5088e 2969 adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start +
96800ee7 2970 adapter->num_tx_queues);
d1a890fa
SB		 2971
2972 adapter->pm_conf = kmalloc(sizeof(struct Vmxnet3_PMConf), GFP_KERNEL);
2973 if (adapter->pm_conf == NULL) {
d1a890fa
SB		 2974 err = -ENOMEM;
2975 goto err_alloc_pm;
2976 }
2977
09c5088e
SB		 2978#ifdef VMXNET3_RSS
2979
2980 adapter->rss_conf = kmalloc(sizeof(struct UPT1_RSSConf), GFP_KERNEL);
2981 if (adapter->rss_conf == NULL) {
09c5088e
SB		 2982 err = -ENOMEM;
2983 goto err_alloc_rss;
2984 }
2985#endif /* VMXNET3_RSS */
2986
d1a890fa
SB		 2987 err = vmxnet3_alloc_pci_resources(adapter, &dma64);
2988 if (err < 0)
2989 goto err_alloc_pci;
2990
2991 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS);
2992 if (ver & 1) {
2993 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_VRRS, 1);
2994 } else {
204a6e65
SH		 2995 dev_err(&pdev->dev,
2996 "Incompatible h/w version (0x%x) for adapter\n", ver);
d1a890fa
SB		 2997 err = -EBUSY;
2998 goto err_ver;
2999 }
3000
3001 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_UVRS);
3002 if (ver & 1) {
3003 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_UVRS, 1);
3004 } else {
204a6e65
SH		 3005 dev_err(&pdev->dev,
3006 "Incompatible upt version (0x%x) for adapter\n", ver);
d1a890fa
SB		 3007 err = -EBUSY;
3008 goto err_ver;
3009 }
3010
e101e7dd 3011 SET_NETDEV_DEV(netdev, &pdev->dev);
d1a890fa
SB		 3012 vmxnet3_declare_features(adapter, dma64);
3013
4db37a78
SH		 3014 if (adapter->num_tx_queues == adapter->num_rx_queues)
3015 adapter->share_intr = VMXNET3_INTR_BUDDYSHARE;
3016 else
09c5088e
SB		 3017 adapter->share_intr = VMXNET3_INTR_DONTSHARE;
3018
d1a890fa
SB		 3019 vmxnet3_alloc_intr_resources(adapter);
3020
09c5088e
SB		 3021#ifdef VMXNET3_RSS
3022 if (adapter->num_rx_queues > 1 &&
3023 adapter->intr.type == VMXNET3_IT_MSIX) {
3024 adapter->rss = true;
204a6e65 3025 dev_dbg(&pdev->dev, "RSS is enabled.\n");
09c5088e
SB		 3026 } else {
3027 adapter->rss = false;
3028 }
3029#endif
3030
d1a890fa
SB		 3031 vmxnet3_read_mac_addr(adapter, mac);
3032 memcpy(netdev->dev_addr, mac, netdev->addr_len);
3033
3034 netdev->netdev_ops = &vmxnet3_netdev_ops;
d1a890fa 3035 vmxnet3_set_ethtool_ops(netdev);
09c5088e 3036 netdev->watchdog_timeo = 5 * HZ;
d1a890fa
SB		 3037
3038 INIT_WORK(&adapter->work, vmxnet3_reset_work);
e3bc4ffb 3039 set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
d1a890fa 3040
09c5088e
SB		 3041 if (adapter->intr.type == VMXNET3_IT_MSIX) {
3042 int i;
3043 for (i = 0; i < adapter->num_rx_queues; i++) {
3044 netif_napi_add(adapter->netdev,
3045 &adapter->rx_queue[i].napi,
3046 vmxnet3_poll_rx_only, 64);
3047 }
3048 } else {
3049 netif_napi_add(adapter->netdev, &adapter->rx_queue[0].napi,
3050 vmxnet3_poll, 64);
3051 }
3052
3053 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
3054 netif_set_real_num_rx_queues(adapter->netdev, adapter->num_rx_queues);
3055
d1a890fa
SB		 3056 err = register_netdev(netdev);
3057
3058 if (err) {
204a6e65 3059 dev_err(&pdev->dev, "Failed to register adapter\n");
d1a890fa
SB		 3060 goto err_register;
3061 }
3062
4a1745fc 3063 vmxnet3_check_link(adapter, false);
d1a890fa
SB		 3064 return 0;
3065
3066err_register:
3067 vmxnet3_free_intr_resources(adapter);
3068err_ver:
3069 vmxnet3_free_pci_resources(adapter);
3070err_alloc_pci:
09c5088e
SB		 3071#ifdef VMXNET3_RSS
3072 kfree(adapter->rss_conf);
3073err_alloc_rss:
3074#endif
d1a890fa
SB		 3075 kfree(adapter->pm_conf);
3076err_alloc_pm:
09c5088e
SB		 3077 pci_free_consistent(adapter->pdev, size, adapter->tqd_start,
3078 adapter->queue_desc_pa);
d1a890fa
SB		 3079err_alloc_queue_desc:
3080 pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_DriverShared),
3081 adapter->shared, adapter->shared_pa);
3082err_alloc_shared:
3083 pci_set_drvdata(pdev, NULL);
3084 free_netdev(netdev);
3085 return err;
3086}
3087
3088
3a4751a3 3089static void
d1a890fa
SB		 3090vmxnet3_remove_device(struct pci_dev *pdev)
3091{
3092 struct net_device *netdev = pci_get_drvdata(pdev);
3093 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
09c5088e
SB		 3094 int size = 0;
3095 int num_rx_queues;
3096
3097#ifdef VMXNET3_RSS
3098 if (enable_mq)
3099 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
3100 (int)num_online_cpus());
3101 else
3102#endif
3103 num_rx_queues = 1;
eebb02b1 3104 num_rx_queues = rounddown_pow_of_two(num_rx_queues);
d1a890fa 3105
23f333a2 3106 cancel_work_sync(&adapter->work);
d1a890fa
SB		 3107
3108 unregister_netdev(netdev);
3109
3110 vmxnet3_free_intr_resources(adapter);
3111 vmxnet3_free_pci_resources(adapter);
09c5088e
SB		 3112#ifdef VMXNET3_RSS
3113 kfree(adapter->rss_conf);
3114#endif
d1a890fa 3115 kfree(adapter->pm_conf);
09c5088e
SB		 3116
3117 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
3118 size += sizeof(struct Vmxnet3_RxQueueDesc) * num_rx_queues;
3119 pci_free_consistent(adapter->pdev, size, adapter->tqd_start,
3120 adapter->queue_desc_pa);
d1a890fa
SB		 3121 pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_DriverShared),
3122 adapter->shared, adapter->shared_pa);
3123 free_netdev(netdev);
3124}
3125
3126
3127#ifdef CONFIG_PM
3128
3129static int
3130vmxnet3_suspend(struct device *device)
3131{
3132 struct pci_dev *pdev = to_pci_dev(device);
3133 struct net_device *netdev = pci_get_drvdata(pdev);
3134 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3135 struct Vmxnet3_PMConf *pmConf;
3136 struct ethhdr *ehdr;
3137 struct arphdr *ahdr;
3138 u8 *arpreq;
3139 struct in_device *in_dev;
3140 struct in_ifaddr *ifa;
83d0feff 3141 unsigned long flags;
d1a890fa
SB		 3142 int i = 0;
3143
3144 if (!netif_running(netdev))
3145 return 0;
3146
51956cd6
SB		 3147 for (i = 0; i < adapter->num_rx_queues; i++)
3148 napi_disable(&adapter->rx_queue[i].napi);
3149
d1a890fa
SB		 3150 vmxnet3_disable_all_intrs(adapter);
3151 vmxnet3_free_irqs(adapter);
3152 vmxnet3_free_intr_resources(adapter);
3153
3154 netif_device_detach(netdev);
09c5088e 3155 netif_tx_stop_all_queues(netdev);
d1a890fa
SB		 3156
3157 /* Create wake-up filters. */
3158 pmConf = adapter->pm_conf;
3159 memset(pmConf, 0, sizeof(*pmConf));
3160
3161 if (adapter->wol & WAKE_UCAST) {
3162 pmConf->filters[i].patternSize = ETH_ALEN;
3163 pmConf->filters[i].maskSize = 1;
3164 memcpy(pmConf->filters[i].pattern, netdev->dev_addr, ETH_ALEN);
3165 pmConf->filters[i].mask[0] = 0x3F; /* LSB ETH_ALEN bits */
3166
3843e515 3167 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
d1a890fa
SB		 3168 i++;
3169 }
3170
3171 if (adapter->wol & WAKE_ARP) {
3172 in_dev = in_dev_get(netdev);
3173 if (!in_dev)
3174 goto skip_arp;
3175
3176 ifa = (struct in_ifaddr *)in_dev->ifa_list;
3177 if (!ifa)
3178 goto skip_arp;
3179
3180 pmConf->filters[i].patternSize = ETH_HLEN + /* Ethernet header*/
3181 sizeof(struct arphdr) + /* ARP header */
3182 2 * ETH_ALEN + /* 2 Ethernet addresses*/
3183 2 * sizeof(u32); /*2 IPv4 addresses */
3184 pmConf->filters[i].maskSize =
3185 (pmConf->filters[i].patternSize - 1) / 8 + 1;
3186
3187 /* ETH_P_ARP in Ethernet header. */
3188 ehdr = (struct ethhdr *)pmConf->filters[i].pattern;
3189 ehdr->h_proto = htons(ETH_P_ARP);
3190
3191 /* ARPOP_REQUEST in ARP header. */
3192 ahdr = (struct arphdr *)&pmConf->filters[i].pattern[ETH_HLEN];
3193 ahdr->ar_op = htons(ARPOP_REQUEST);
3194 arpreq = (u8 *)(ahdr + 1);
3195
3196 /* The Unicast IPv4 address in 'tip' field. */
3197 arpreq += 2 * ETH_ALEN + sizeof(u32);
3198 *(u32 *)arpreq = ifa->ifa_address;
3199
3200 /* The mask for the relevant bits. */
3201 pmConf->filters[i].mask[0] = 0x00;
3202 pmConf->filters[i].mask[1] = 0x30; /* ETH_P_ARP */
3203 pmConf->filters[i].mask[2] = 0x30; /* ARPOP_REQUEST */
3204 pmConf->filters[i].mask[3] = 0x00;
3205 pmConf->filters[i].mask[4] = 0xC0; /* IPv4 TIP */
3206 pmConf->filters[i].mask[5] = 0x03; /* IPv4 TIP */
3207 in_dev_put(in_dev);
3208
3843e515 3209 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
d1a890fa
SB		 3210 i++;
3211 }
3212
3213skip_arp:
3214 if (adapter->wol & WAKE_MAGIC)
3843e515 3215 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_MAGIC;
d1a890fa
SB		 3216
3217 pmConf->numFilters = i;
3218
115924b6
SB		 3219 adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1);
3220 adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof(
3221 *pmConf));
3222 adapter->shared->devRead.pmConfDesc.confPA = cpu_to_le64(virt_to_phys(
3223 pmConf));
d1a890fa 3224
83d0feff 3225 spin_lock_irqsave(&adapter->cmd_lock, flags);
d1a890fa
SB		 3226 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3227 VMXNET3_CMD_UPDATE_PMCFG);
83d0feff 3228 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
d1a890fa
SB		 3229
3230 pci_save_state(pdev);
3231 pci_enable_wake(pdev, pci_choose_state(pdev, PMSG_SUSPEND),
3232 adapter->wol);
3233 pci_disable_device(pdev);
3234 pci_set_power_state(pdev, pci_choose_state(pdev, PMSG_SUSPEND));
3235
3236 return 0;
3237}
3238
3239
3240static int
3241vmxnet3_resume(struct device *device)
3242{
51956cd6 3243 int err, i = 0;
83d0feff 3244 unsigned long flags;
d1a890fa
SB		 3245 struct pci_dev *pdev = to_pci_dev(device);
3246 struct net_device *netdev = pci_get_drvdata(pdev);
3247 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3248 struct Vmxnet3_PMConf *pmConf;
3249
3250 if (!netif_running(netdev))
3251 return 0;
3252
3253 /* Destroy wake-up filters. */
3254 pmConf = adapter->pm_conf;
3255 memset(pmConf, 0, sizeof(*pmConf));
3256
115924b6
SB		 3257 adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1);
3258 adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof(
3259 *pmConf));
0561cf3d 3260 adapter->shared->devRead.pmConfDesc.confPA = cpu_to_le64(virt_to_phys(
115924b6 3261 pmConf));
d1a890fa
SB		 3262
3263 netif_device_attach(netdev);
3264 pci_set_power_state(pdev, PCI_D0);
3265 pci_restore_state(pdev);
3266 err = pci_enable_device_mem(pdev);
3267 if (err != 0)
3268 return err;
3269
3270 pci_enable_wake(pdev, PCI_D0, 0);
3271
83d0feff 3272 spin_lock_irqsave(&adapter->cmd_lock, flags);
d1a890fa
SB		 3273 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3274 VMXNET3_CMD_UPDATE_PMCFG);
83d0feff 3275 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
d1a890fa
SB		 3276 vmxnet3_alloc_intr_resources(adapter);
3277 vmxnet3_request_irqs(adapter);
51956cd6
SB		 3278 for (i = 0; i < adapter->num_rx_queues; i++)
3279 napi_enable(&adapter->rx_queue[i].napi);
d1a890fa
SB		 3280 vmxnet3_enable_all_intrs(adapter);
3281
3282 return 0;
3283}
3284
47145210 3285static const struct dev_pm_ops vmxnet3_pm_ops = {
d1a890fa
SB		 3286 .suspend = vmxnet3_suspend,
3287 .resume = vmxnet3_resume,
3288};
3289#endif
3290
3291static struct pci_driver vmxnet3_driver = {
3292 .name = vmxnet3_driver_name,
3293 .id_table = vmxnet3_pciid_table,
3294 .probe = vmxnet3_probe_device,
3a4751a3 3295 .remove = vmxnet3_remove_device,
d1a890fa
SB		 3296#ifdef CONFIG_PM
3297 .driver.pm = &vmxnet3_pm_ops,
3298#endif
3299};
3300
3301
3302static int __init
3303vmxnet3_init_module(void)
3304{
204a6e65 3305 pr_info("%s - version %s\n", VMXNET3_DRIVER_DESC,
d1a890fa
SB		 3306 VMXNET3_DRIVER_VERSION_REPORT);
3307 return pci_register_driver(&vmxnet3_driver);
3308}
3309
3310module_init(vmxnet3_init_module);
3311
3312
3313static void
3314vmxnet3_exit_module(void)
3315{
3316 pci_unregister_driver(&vmxnet3_driver);
3317}
3318
3319module_exit(vmxnet3_exit_module);
3320
3321MODULE_AUTHOR("VMware, Inc.");
3322MODULE_DESCRIPTION(VMXNET3_DRIVER_DESC);
3323MODULE_LICENSE("GPL v2");
3324MODULE_VERSION(VMXNET3_DRIVER_VERSION_STRING);
Linux 6.x block layer and io_uring tree(s)