drivers/net/ethernet/cavium/liquidio/lio_main.c

   1 /**********************************************************************
   2 * Author: Cavium, Inc.
   3 *
   4 * Contact: support@cavium.com
   5 *          Please include "LiquidIO" in the subject.
   6 *
   7 * Copyright (c) 2003-2015 Cavium, Inc.
   8 *
   9 * This file is free software; you can redistribute it and/or modify
  10 * it under the terms of the GNU General Public License, Version 2, as
  11 * published by the Free Software Foundation.
  12 *
  13 * This file is distributed in the hope that it will be useful, but
  14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
  15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
  16 * NONINFRINGEMENT.  See the GNU General Public License for more
  17 * details.
  18 *
  19 * This file may also be available under a different license from Cavium.
  20 * Contact Cavium, Inc. for more information
  21 **********************************************************************/
  22 #include <linux/version.h>
  23 #include <linux/pci.h>
  24 #include <linux/firmware.h>
  25 #include <linux/ptp_clock_kernel.h>
  26 #include <net/vxlan.h>
  27 #include "liquidio_common.h"
  28 #include "octeon_droq.h"
  29 #include "octeon_iq.h"
  30 #include "response_manager.h"
  31 #include "octeon_device.h"
  32 #include "octeon_nic.h"
  33 #include "octeon_main.h"
  34 #include "octeon_network.h"
  35 #include "cn66xx_regs.h"
  36 #include "cn66xx_device.h"
  37 #include "cn68xx_device.h"
  38 #include "cn23xx_pf_device.h"
  39 #include "liquidio_image.h"
  40
  41 MODULE_AUTHOR("Cavium Networks, <support@cavium.com>");
  42 MODULE_DESCRIPTION("Cavium LiquidIO Intelligent Server Adapter Driver");
  43 MODULE_LICENSE("GPL");
  44 MODULE_VERSION(LIQUIDIO_VERSION);
  45 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210SV_NAME LIO_FW_NAME_SUFFIX);
  46 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210NV_NAME LIO_FW_NAME_SUFFIX);
  47 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_410NV_NAME LIO_FW_NAME_SUFFIX);
  48
  49 static int ddr_timeout = 10000;
  50 module_param(ddr_timeout, int, 0644);
  51 MODULE_PARM_DESC(ddr_timeout,
  52                  "Number of milliseconds to wait for DDR initialization. 0 waits for ddr_timeout to be set to non-zero value before starting to check");
  53
  54 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
  55
  56 #define INCR_INSTRQUEUE_PKT_COUNT(octeon_dev_ptr, iq_no, field, count)  \
  57         (octeon_dev_ptr->instr_queue[iq_no]->stats.field += count)
  58
  59 static int debug = -1;
  60 module_param(debug, int, 0644);
  61 MODULE_PARM_DESC(debug, "NETIF_MSG debug bits");
  62
  63 static char fw_type[LIO_MAX_FW_TYPE_LEN];
  64 module_param_string(fw_type, fw_type, sizeof(fw_type), 0000);
  65 MODULE_PARM_DESC(fw_type, "Type of firmware to be loaded. Default \"nic\"");
  66
  67 static int conf_type;
  68 module_param(conf_type, int, 0);
  69 MODULE_PARM_DESC(conf_type, "select octeon configuration 0 default 1 ovs");
  70
  71 static int ptp_enable = 1;
  72
  73 /* Bit mask values for lio->ifstate */
  74 #define   LIO_IFSTATE_DROQ_OPS             0x01
  75 #define   LIO_IFSTATE_REGISTERED           0x02
  76 #define   LIO_IFSTATE_RUNNING              0x04
  77 #define   LIO_IFSTATE_RX_TIMESTAMP_ENABLED 0x08
  78
  79 /* Polling interval for determining when NIC application is alive */
  80 #define LIQUIDIO_STARTER_POLL_INTERVAL_MS 100
  81
  82 /* runtime link query interval */
  83 #define LIQUIDIO_LINK_QUERY_INTERVAL_MS         1000
  84
  85 struct liquidio_if_cfg_context {
  86         int octeon_id;
  87
  88         wait_queue_head_t wc;
  89
  90         int cond;
  91 };
  92
  93 struct liquidio_if_cfg_resp {
  94         u64 rh;
  95         struct liquidio_if_cfg_info cfg_info;
  96         u64 status;
  97 };
  98
  99 struct oct_link_status_resp {
 100         u64 rh;
 101         struct oct_link_info link_info;
 102         u64 status;
 103 };
 104
 105 struct oct_timestamp_resp {
 106         u64 rh;
 107         u64 timestamp;
 108         u64 status;
 109 };
 110
 111 #define OCT_TIMESTAMP_RESP_SIZE (sizeof(struct oct_timestamp_resp))
 112
 113 union tx_info {
 114         u64 u64;
 115         struct {
 116 #ifdef __BIG_ENDIAN_BITFIELD
 117                 u16 gso_size;
 118                 u16 gso_segs;
 119                 u32 reserved;
 120 #else
 121                 u32 reserved;
 122                 u16 gso_segs;
 123                 u16 gso_size;
 124 #endif
 125         } s;
 126 };
 127
 128 /** Octeon device properties to be used by the NIC module.
 129  * Each octeon device in the system will be represented
 130  * by this structure in the NIC module.
 131  */
 132
 133 #define OCTNIC_MAX_SG  (MAX_SKB_FRAGS)
 134
 135 #define OCTNIC_GSO_MAX_HEADER_SIZE 128
 136 #define OCTNIC_GSO_MAX_SIZE                                                    \
 137         (CN23XX_DEFAULT_INPUT_JABBER - OCTNIC_GSO_MAX_HEADER_SIZE)
 138
 139 /** Structure of a node in list of gather components maintained by
 140  * NIC driver for each network device.
 141  */
 142 struct octnic_gather {
 143         /** List manipulation. Next and prev pointers. */
 144         struct list_head list;
 145
 146         /** Size of the gather component at sg in bytes. */
 147         int sg_size;
 148
 149         /** Number of bytes that sg was adjusted to make it 8B-aligned. */
 150         int adjust;
 151
 152         /** Gather component that can accommodate max sized fragment list
 153          *  received from the IP layer.
 154          */
 155         struct octeon_sg_entry *sg;
 156
 157         u64 sg_dma_ptr;
 158 };
 159
 160 struct handshake {
 161         struct completion init;
 162         struct completion started;
 163         struct pci_dev *pci_dev;
 164         int init_ok;
 165         int started_ok;
 166 };
 167
 168 struct octeon_device_priv {
 169         /** Tasklet structures for this device. */
 170         struct tasklet_struct droq_tasklet;
 171         unsigned long napi_mask;
 172 };
 173
 174 static int octeon_device_init(struct octeon_device *);
 175 static int liquidio_stop(struct net_device *netdev);
 176 static void liquidio_remove(struct pci_dev *pdev);
 177 static int liquidio_probe(struct pci_dev *pdev,
 178                           const struct pci_device_id *ent);
 179
 180 static struct handshake handshake[MAX_OCTEON_DEVICES];
 181 static struct completion first_stage;
 182
 183 static void octeon_droq_bh(unsigned long pdev)
 184 {
 185         int q_no;
 186         int reschedule = 0;
 187         struct octeon_device *oct = (struct octeon_device *)pdev;
 188         struct octeon_device_priv *oct_priv =
 189                 (struct octeon_device_priv *)oct->priv;
 190
 191         /* for (q_no = 0; q_no < oct->num_oqs; q_no++) { */
 192         for (q_no = 0; q_no < MAX_OCTEON_OUTPUT_QUEUES(oct); q_no++) {
 193                 if (!(oct->io_qmask.oq & (1ULL << q_no)))
 194                         continue;
 195                 reschedule |= octeon_droq_process_packets(oct, oct->droq[q_no],
 196                                                           MAX_PACKET_BUDGET);
 197                 lio_enable_irq(oct->droq[q_no], NULL);
 198         }
 199
 200         if (reschedule)
 201                 tasklet_schedule(&oct_priv->droq_tasklet);
 202 }
 203
 204 static int lio_wait_for_oq_pkts(struct octeon_device *oct)
 205 {
 206         struct octeon_device_priv *oct_priv =
 207                 (struct octeon_device_priv *)oct->priv;
 208         int retry = 100, pkt_cnt = 0, pending_pkts = 0;
 209         int i;
 210
 211         do {
 212                 pending_pkts = 0;
 213
 214                 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
 215                         if (!(oct->io_qmask.oq & (1ULL << i)))
 216                                 continue;
 217                         pkt_cnt += octeon_droq_check_hw_for_pkts(oct->droq[i]);
 218                 }
 219                 if (pkt_cnt > 0) {
 220                         pending_pkts += pkt_cnt;
 221                         tasklet_schedule(&oct_priv->droq_tasklet);
 222                 }
 223                 pkt_cnt = 0;
 224                 schedule_timeout_uninterruptible(1);
 225
 226         } while (retry-- && pending_pkts);
 227
 228         return pkt_cnt;
 229 }
 230
 231 /**
 232  * \brief Forces all IO queues off on a given device
 233  * @param oct Pointer to Octeon device
 234  */
 235 static void force_io_queues_off(struct octeon_device *oct)
 236 {
 237         if ((oct->chip_id == OCTEON_CN66XX) ||
 238             (oct->chip_id == OCTEON_CN68XX)) {
 239                 /* Reset the Enable bits for Input Queues. */
 240                 octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, 0);
 241
 242                 /* Reset the Enable bits for Output Queues. */
 243                 octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, 0);
 244         }
 245 }
 246
 247 /**
 248  * \brief wait for all pending requests to complete
 249  * @param oct Pointer to Octeon device
 250  *
 251  * Called during shutdown sequence
 252  */
 253 static int wait_for_pending_requests(struct octeon_device *oct)
 254 {
 255         int i, pcount = 0;
 256
 257         for (i = 0; i < 100; i++) {
 258                 pcount =
 259                         atomic_read(&oct->response_list
 260                                 [OCTEON_ORDERED_SC_LIST].pending_req_count);
 261                 if (pcount)
 262                         schedule_timeout_uninterruptible(HZ / 10);
 263                 else
 264                         break;
 265         }
 266
 267         if (pcount)
 268                 return 1;
 269
 270         return 0;
 271 }
 272
 273 /**
 274  * \brief Cause device to go quiet so it can be safely removed/reset/etc
 275  * @param oct Pointer to Octeon device
 276  */
 277 static inline void pcierror_quiesce_device(struct octeon_device *oct)
 278 {
 279         int i;
 280
 281         /* Disable the input and output queues now. No more packets will
 282          * arrive from Octeon, but we should wait for all packet processing
 283          * to finish.
 284          */
 285         force_io_queues_off(oct);
 286
 287         /* To allow for in-flight requests */
 288         schedule_timeout_uninterruptible(100);
 289
 290         if (wait_for_pending_requests(oct))
 291                 dev_err(&oct->pci_dev->dev, "There were pending requests\n");
 292
 293         /* Force all requests waiting to be fetched by OCTEON to complete. */
 294         for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
 295                 struct octeon_instr_queue *iq;
 296
 297                 if (!(oct->io_qmask.iq & (1ULL << i)))
 298                         continue;
 299                 iq = oct->instr_queue[i];
 300
 301                 if (atomic_read(&iq->instr_pending)) {
 302                         spin_lock_bh(&iq->lock);
 303                         iq->fill_cnt = 0;
 304                         iq->octeon_read_index = iq->host_write_index;
 305                         iq->stats.instr_processed +=
 306                                 atomic_read(&iq->instr_pending);
 307                         lio_process_iq_request_list(oct, iq, 0);
 308                         spin_unlock_bh(&iq->lock);
 309                 }
 310         }
 311
 312         /* Force all pending ordered list requests to time out. */
 313         lio_process_ordered_list(oct, 1);
 314
 315         /* We do not need to wait for output queue packets to be processed. */
 316 }
 317
 318 /**
 319  * \brief Cleanup PCI AER uncorrectable error status
 320  * @param dev Pointer to PCI device
 321  */
 322 static void cleanup_aer_uncorrect_error_status(struct pci_dev *dev)
 323 {
 324         int pos = 0x100;
 325         u32 status, mask;
 326
 327         pr_info("%s :\n", __func__);
 328
 329         pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
 330         pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask);
 331         if (dev->error_state == pci_channel_io_normal)
 332                 status &= ~mask;        /* Clear corresponding nonfatal bits */
 333         else
 334                 status &= mask;         /* Clear corresponding fatal bits */
 335         pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);
 336 }
 337
 338 /**
 339  * \brief Stop all PCI IO to a given device
 340  * @param dev Pointer to Octeon device
 341  */
 342 static void stop_pci_io(struct octeon_device *oct)
 343 {
 344         /* No more instructions will be forwarded. */
 345         atomic_set(&oct->status, OCT_DEV_IN_RESET);
 346
 347         pci_disable_device(oct->pci_dev);
 348
 349         /* Disable interrupts  */
 350         oct->fn_list.disable_interrupt(oct->chip);
 351
 352         pcierror_quiesce_device(oct);
 353
 354         /* Release the interrupt line */
 355         free_irq(oct->pci_dev->irq, oct);
 356
 357         if (oct->flags & LIO_FLAG_MSI_ENABLED)
 358                 pci_disable_msi(oct->pci_dev);
 359
 360         dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
 361                 lio_get_state_string(&oct->status));
 362
 363         /* cn63xx_cleanup_aer_uncorrect_error_status(oct->pci_dev); */
 364         /* making it a common function for all OCTEON models */
 365         cleanup_aer_uncorrect_error_status(oct->pci_dev);
 366 }
 367
 368 /**
 369  * \brief called when PCI error is detected
 370  * @param pdev Pointer to PCI device
 371  * @param state The current pci connection state
 372  *
 373  * This function is called after a PCI bus error affecting
 374  * this device has been detected.
 375  */
 376 static pci_ers_result_t liquidio_pcie_error_detected(struct pci_dev *pdev,
 377                                                      pci_channel_state_t state)
 378 {
 379         struct octeon_device *oct = pci_get_drvdata(pdev);
 380
 381         /* Non-correctable Non-fatal errors */
 382         if (state == pci_channel_io_normal) {
 383                 dev_err(&oct->pci_dev->dev, "Non-correctable non-fatal error reported:\n");
 384                 cleanup_aer_uncorrect_error_status(oct->pci_dev);
 385                 return PCI_ERS_RESULT_CAN_RECOVER;
 386         }
 387
 388         /* Non-correctable Fatal errors */
 389         dev_err(&oct->pci_dev->dev, "Non-correctable FATAL reported by PCI AER driver\n");
 390         stop_pci_io(oct);
 391
 392         /* Always return a DISCONNECT. There is no support for recovery but only
 393          * for a clean shutdown.
 394          */
 395         return PCI_ERS_RESULT_DISCONNECT;
 396 }
 397
 398 /**
 399  * \brief mmio handler
 400  * @param pdev Pointer to PCI device
 401  */
 402 static pci_ers_result_t liquidio_pcie_mmio_enabled(
 403                                 struct pci_dev *pdev __attribute__((unused)))
 404 {
 405         /* We should never hit this since we never ask for a reset for a Fatal
 406          * Error. We always return DISCONNECT in io_error above.
 407          * But play safe and return RECOVERED for now.
 408          */
 409         return PCI_ERS_RESULT_RECOVERED;
 410 }
 411
 412 /**
 413  * \brief called after the pci bus has been reset.
 414  * @param pdev Pointer to PCI device
 415  *
 416  * Restart the card from scratch, as if from a cold-boot. Implementation
 417  * resembles the first-half of the octeon_resume routine.
 418  */
 419 static pci_ers_result_t liquidio_pcie_slot_reset(
 420                                 struct pci_dev *pdev __attribute__((unused)))
 421 {
 422         /* We should never hit this since we never ask for a reset for a Fatal
 423          * Error. We always return DISCONNECT in io_error above.
 424          * But play safe and return RECOVERED for now.
 425          */
 426         return PCI_ERS_RESULT_RECOVERED;
 427 }
 428
 429 /**
 430  * \brief called when traffic can start flowing again.
 431  * @param pdev Pointer to PCI device
 432  *
 433  * This callback is called when the error recovery driver tells us that
 434  * its OK to resume normal operation. Implementation resembles the
 435  * second-half of the octeon_resume routine.
 436  */
 437 static void liquidio_pcie_resume(struct pci_dev *pdev __attribute__((unused)))
 438 {
 439         /* Nothing to be done here. */
 440 }
 441
 442 #ifdef CONFIG_PM
 443 /**
 444  * \brief called when suspending
 445  * @param pdev Pointer to PCI device
 446  * @param state state to suspend to
 447  */
 448 static int liquidio_suspend(struct pci_dev *pdev __attribute__((unused)),
 449                             pm_message_t state __attribute__((unused)))
 450 {
 451         return 0;
 452 }
 453
 454 /**
 455  * \brief called when resuming
 456  * @param pdev Pointer to PCI device
 457  */
 458 static int liquidio_resume(struct pci_dev *pdev __attribute__((unused)))
 459 {
 460         return 0;
 461 }
 462 #endif
 463
 464 /* For PCI-E Advanced Error Recovery (AER) Interface */
 465 static const struct pci_error_handlers liquidio_err_handler = {
 466         .error_detected = liquidio_pcie_error_detected,
 467         .mmio_enabled   = liquidio_pcie_mmio_enabled,
 468         .slot_reset     = liquidio_pcie_slot_reset,
 469         .resume         = liquidio_pcie_resume,
 470 };
 471
 472 static const struct pci_device_id liquidio_pci_tbl[] = {
 473         {       /* 68xx */
 474                 PCI_VENDOR_ID_CAVIUM, 0x91, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
 475         },
 476         {       /* 66xx */
 477                 PCI_VENDOR_ID_CAVIUM, 0x92, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
 478         },
 479         {       /* 23xx pf */
 480                 PCI_VENDOR_ID_CAVIUM, 0x9702, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
 481         },
 482         {
 483                 0, 0, 0, 0, 0, 0, 0
 484         }
 485 };
 486 MODULE_DEVICE_TABLE(pci, liquidio_pci_tbl);
 487
 488 static struct pci_driver liquidio_pci_driver = {
 489         .name           = "LiquidIO",
 490         .id_table       = liquidio_pci_tbl,
 491         .probe          = liquidio_probe,
 492         .remove         = liquidio_remove,
 493         .err_handler    = &liquidio_err_handler,    /* For AER */
 494
 495 #ifdef CONFIG_PM
 496         .suspend        = liquidio_suspend,
 497         .resume         = liquidio_resume,
 498 #endif
 499 };
 500
 501 /**
 502  * \brief register PCI driver
 503  */
 504 static int liquidio_init_pci(void)
 505 {
 506         return pci_register_driver(&liquidio_pci_driver);
 507 }
 508
 509 /**
 510  * \brief unregister PCI driver
 511  */
 512 static void liquidio_deinit_pci(void)
 513 {
 514         pci_unregister_driver(&liquidio_pci_driver);
 515 }
 516
 517 /**
 518  * \brief check interface state
 519  * @param lio per-network private data
 520  * @param state_flag flag state to check
 521  */
 522 static inline int ifstate_check(struct lio *lio, int state_flag)
 523 {
 524         return atomic_read(&lio->ifstate) & state_flag;
 525 }
 526
 527 /**
 528  * \brief set interface state
 529  * @param lio per-network private data
 530  * @param state_flag flag state to set
 531  */
 532 static inline void ifstate_set(struct lio *lio, int state_flag)
 533 {
 534         atomic_set(&lio->ifstate, (atomic_read(&lio->ifstate) | state_flag));
 535 }
 536
 537 /**
 538  * \brief clear interface state
 539  * @param lio per-network private data
 540  * @param state_flag flag state to clear
 541  */
 542 static inline void ifstate_reset(struct lio *lio, int state_flag)
 543 {
 544         atomic_set(&lio->ifstate, (atomic_read(&lio->ifstate) & ~(state_flag)));
 545 }
 546
 547 /**
 548  * \brief Stop Tx queues
 549  * @param netdev network device
 550  */
 551 static inline void txqs_stop(struct net_device *netdev)
 552 {
 553         if (netif_is_multiqueue(netdev)) {
 554                 int i;
 555
 556                 for (i = 0; i < netdev->num_tx_queues; i++)
 557                         netif_stop_subqueue(netdev, i);
 558         } else {
 559                 netif_stop_queue(netdev);
 560         }
 561 }
 562
 563 /**
 564  * \brief Start Tx queues
 565  * @param netdev network device
 566  */
 567 static inline void txqs_start(struct net_device *netdev)
 568 {
 569         if (netif_is_multiqueue(netdev)) {
 570                 int i;
 571
 572                 for (i = 0; i < netdev->num_tx_queues; i++)
 573                         netif_start_subqueue(netdev, i);
 574         } else {
 575                 netif_start_queue(netdev);
 576         }
 577 }
 578
 579 /**
 580  * \brief Wake Tx queues
 581  * @param netdev network device
 582  */
 583 static inline void txqs_wake(struct net_device *netdev)
 584 {
 585         struct lio *lio = GET_LIO(netdev);
 586
 587         if (netif_is_multiqueue(netdev)) {
 588                 int i;
 589
 590                 for (i = 0; i < netdev->num_tx_queues; i++) {
 591                         int qno = lio->linfo.txpciq[i %
 592                                 (lio->linfo.num_txpciq)].s.q_no;
 593
 594                         if (__netif_subqueue_stopped(netdev, i)) {
 595                                 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, qno,
 596                                                           tx_restart, 1);
 597                                 netif_wake_subqueue(netdev, i);
 598                         }
 599                 }
 600         } else {
 601                 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, lio->txq,
 602                                           tx_restart, 1);
 603                 netif_wake_queue(netdev);
 604         }
 605 }
 606
 607 /**
 608  * \brief Stop Tx queue
 609  * @param netdev network device
 610  */
 611 static void stop_txq(struct net_device *netdev)
 612 {
 613         txqs_stop(netdev);
 614 }
 615
 616 /**
 617  * \brief Start Tx queue
 618  * @param netdev network device
 619  */
 620 static void start_txq(struct net_device *netdev)
 621 {
 622         struct lio *lio = GET_LIO(netdev);
 623
 624         if (lio->linfo.link.s.link_up) {
 625                 txqs_start(netdev);
 626                 return;
 627         }
 628 }
 629
 630 /**
 631  * \brief Wake a queue
 632  * @param netdev network device
 633  * @param q which queue to wake
 634  */
 635 static inline void wake_q(struct net_device *netdev, int q)
 636 {
 637         if (netif_is_multiqueue(netdev))
 638                 netif_wake_subqueue(netdev, q);
 639         else
 640                 netif_wake_queue(netdev);
 641 }
 642
 643 /**
 644  * \brief Stop a queue
 645  * @param netdev network device
 646  * @param q which queue to stop
 647  */
 648 static inline void stop_q(struct net_device *netdev, int q)
 649 {
 650         if (netif_is_multiqueue(netdev))
 651                 netif_stop_subqueue(netdev, q);
 652         else
 653                 netif_stop_queue(netdev);
 654 }
 655
 656 /**
 657  * \brief Check Tx queue status, and take appropriate action
 658  * @param lio per-network private data
 659  * @returns 0 if full, number of queues woken up otherwise
 660  */
 661 static inline int check_txq_status(struct lio *lio)
 662 {
 663         int ret_val = 0;
 664
 665         if (netif_is_multiqueue(lio->netdev)) {
 666                 int numqs = lio->netdev->num_tx_queues;
 667                 int q, iq = 0;
 668
 669                 /* check each sub-queue state */
 670                 for (q = 0; q < numqs; q++) {
 671                         iq = lio->linfo.txpciq[q %
 672                                 (lio->linfo.num_txpciq)].s.q_no;
 673                         if (octnet_iq_is_full(lio->oct_dev, iq))
 674                                 continue;
 675                         if (__netif_subqueue_stopped(lio->netdev, q)) {
 676                                 wake_q(lio->netdev, q);
 677                                 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq,
 678                                                           tx_restart, 1);
 679                                 ret_val++;
 680                         }
 681                 }
 682         } else {
 683                 if (octnet_iq_is_full(lio->oct_dev, lio->txq))
 684                         return 0;
 685                 wake_q(lio->netdev, lio->txq);
 686                 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, lio->txq,
 687                                           tx_restart, 1);
 688                 ret_val = 1;
 689         }
 690         return ret_val;
 691 }
 692
 693 /**
 694  * Remove the node at the head of the list. The list would be empty at
 695  * the end of this call if there are no more nodes in the list.
 696  */
 697 static inline struct list_head *list_delete_head(struct list_head *root)
 698 {
 699         struct list_head *node;
 700
 701         if ((root->prev == root) && (root->next == root))
 702                 node = NULL;
 703         else
 704                 node = root->next;
 705
 706         if (node)
 707                 list_del(node);
 708
 709         return node;
 710 }
 711
 712 /**
 713  * \brief Delete gather lists
 714  * @param lio per-network private data
 715  */
 716 static void delete_glists(struct lio *lio)
 717 {
 718         struct octnic_gather *g;
 719         int i;
 720
 721         if (!lio->glist)
 722                 return;
 723
 724         for (i = 0; i < lio->linfo.num_txpciq; i++) {
 725                 do {
 726                         g = (struct octnic_gather *)
 727                                 list_delete_head(&lio->glist[i]);
 728                         if (g) {
 729                                 if (g->sg) {
 730                                         dma_unmap_single(&lio->oct_dev->
 731                                                          pci_dev->dev,
 732                                                          g->sg_dma_ptr,
 733                                                          g->sg_size,
 734                                                          DMA_TO_DEVICE);
 735                                         kfree((void *)((unsigned long)g->sg -
 736                                                        g->adjust));
 737                                 }
 738                                 kfree(g);
 739                         }
 740                 } while (g);
 741         }
 742
 743         kfree((void *)lio->glist);
 744 }
 745
 746 /**
 747  * \brief Setup gather lists
 748  * @param lio per-network private data
 749  */
 750 static int setup_glists(struct octeon_device *oct, struct lio *lio, int num_iqs)
 751 {
 752         int i, j;
 753         struct octnic_gather *g;
 754
 755         lio->glist_lock = kcalloc(num_iqs, sizeof(*lio->glist_lock),
 756                                   GFP_KERNEL);
 757         if (!lio->glist_lock)
 758                 return 1;
 759
 760         lio->glist = kcalloc(num_iqs, sizeof(*lio->glist),
 761                              GFP_KERNEL);
 762         if (!lio->glist) {
 763                 kfree((void *)lio->glist_lock);
 764                 return 1;
 765         }
 766
 767         for (i = 0; i < num_iqs; i++) {
 768                 int numa_node = cpu_to_node(i % num_online_cpus());
 769
 770                 spin_lock_init(&lio->glist_lock[i]);
 771
 772                 INIT_LIST_HEAD(&lio->glist[i]);
 773
 774                 for (j = 0; j < lio->tx_qsize; j++) {
 775                         g = kzalloc_node(sizeof(*g), GFP_KERNEL,
 776                                          numa_node);
 777                         if (!g)
 778                                 g = kzalloc(sizeof(*g), GFP_KERNEL);
 779                         if (!g)
 780                                 break;
 781
 782                         g->sg_size = ((ROUNDUP4(OCTNIC_MAX_SG) >> 2) *
 783                                       OCT_SG_ENTRY_SIZE);
 784
 785                         g->sg = kmalloc_node(g->sg_size + 8,
 786                                              GFP_KERNEL, numa_node);
 787                         if (!g->sg)
 788                                 g->sg = kmalloc(g->sg_size + 8, GFP_KERNEL);
 789                         if (!g->sg) {
 790                                 kfree(g);
 791                                 break;
 792                         }
 793
 794                         /* The gather component should be aligned on 64-bit
 795                          * boundary
 796                          */
 797                         if (((unsigned long)g->sg) & 7) {
 798                                 g->adjust = 8 - (((unsigned long)g->sg) & 7);
 799                                 g->sg = (struct octeon_sg_entry *)
 800                                         ((unsigned long)g->sg + g->adjust);
 801                         }
 802                         g->sg_dma_ptr = dma_map_single(&oct->pci_dev->dev,
 803                                                        g->sg, g->sg_size,
 804                                                        DMA_TO_DEVICE);
 805                         if (dma_mapping_error(&oct->pci_dev->dev,
 806                                               g->sg_dma_ptr)) {
 807                                 kfree((void *)((unsigned long)g->sg -
 808                                                g->adjust));
 809                                 kfree(g);
 810                                 break;
 811                         }
 812
 813                         list_add_tail(&g->list, &lio->glist[i]);
 814                 }
 815
 816                 if (j != lio->tx_qsize) {
 817                         delete_glists(lio);
 818                         return 1;
 819                 }
 820         }
 821
 822         return 0;
 823 }
 824
 825 /**
 826  * \brief Print link information
 827  * @param netdev network device
 828  */
 829 static void print_link_info(struct net_device *netdev)
 830 {
 831         struct lio *lio = GET_LIO(netdev);
 832
 833         if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED) {
 834                 struct oct_link_info *linfo = &lio->linfo;
 835
 836                 if (linfo->link.s.link_up) {
 837                         netif_info(lio, link, lio->netdev, "%d Mbps %s Duplex UP\n",
 838                                    linfo->link.s.speed,
 839                                    (linfo->link.s.duplex) ? "Full" : "Half");
 840                 } else {
 841                         netif_info(lio, link, lio->netdev, "Link Down\n");
 842                 }
 843         }
 844 }
 845
 846 /**
 847  * \brief Update link status
 848  * @param netdev network device
 849  * @param ls link status structure
 850  *
 851  * Called on receipt of a link status response from the core application to
 852  * update each interface's link status.
 853  */
 854 static inline void update_link_status(struct net_device *netdev,
 855                                       union oct_link_status *ls)
 856 {
 857         struct lio *lio = GET_LIO(netdev);
 858         int changed = (lio->linfo.link.u64 != ls->u64);
 859
 860         lio->linfo.link.u64 = ls->u64;
 861
 862         if ((lio->intf_open) && (changed)) {
 863                 print_link_info(netdev);
 864                 lio->link_changes++;
 865
 866                 if (lio->linfo.link.s.link_up) {
 867                         netif_carrier_on(netdev);
 868                         /* start_txq(netdev); */
 869                         txqs_wake(netdev);
 870                 } else {
 871                         netif_carrier_off(netdev);
 872                         stop_txq(netdev);
 873                 }
 874         }
 875 }
 876
 877 /* Runs in interrupt context. */
 878 static void update_txq_status(struct octeon_device *oct, int iq_num)
 879 {
 880         struct net_device *netdev;
 881         struct lio *lio;
 882         struct octeon_instr_queue *iq = oct->instr_queue[iq_num];
 883
 884         /*octeon_update_iq_read_idx(oct, iq);*/
 885
 886         netdev = oct->props[iq->ifidx].netdev;
 887
 888         /* This is needed because the first IQ does not have
 889          * a netdev associated with it.
 890          */
 891         if (!netdev)
 892                 return;
 893
 894         lio = GET_LIO(netdev);
 895         if (netif_is_multiqueue(netdev)) {
 896                 if (__netif_subqueue_stopped(netdev, iq->q_index) &&
 897                     lio->linfo.link.s.link_up &&
 898                     (!octnet_iq_is_full(oct, iq_num))) {
 899                         INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq_num,
 900                                                   tx_restart, 1);
 901                         netif_wake_subqueue(netdev, iq->q_index);
 902                 } else {
 903                         if (!octnet_iq_is_full(oct, lio->txq)) {
 904                                 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev,
 905                                                           lio->txq,
 906                                                           tx_restart, 1);
 907                                 wake_q(netdev, lio->txq);
 908                         }
 909                 }
 910         }
 911 }
 912
 913 /**
 914  * \brief Droq packet processor sceduler
 915  * @param oct octeon device
 916  */
 917 static
 918 void liquidio_schedule_droq_pkt_handlers(struct octeon_device *oct)
 919 {
 920         struct octeon_device_priv *oct_priv =
 921                 (struct octeon_device_priv *)oct->priv;
 922         u64 oq_no;
 923         struct octeon_droq *droq;
 924
 925         if (oct->int_status & OCT_DEV_INTR_PKT_DATA) {
 926                 for (oq_no = 0; oq_no < MAX_OCTEON_OUTPUT_QUEUES(oct);
 927                      oq_no++) {
 928                         if (!(oct->droq_intr & (1ULL << oq_no)))
 929                                 continue;
 930
 931                         droq = oct->droq[oq_no];
 932
 933                         if (droq->ops.poll_mode) {
 934                                 droq->ops.napi_fn(droq);
 935                                 oct_priv->napi_mask |= (1 << oq_no);
 936                         } else {
 937                                 tasklet_schedule(&oct_priv->droq_tasklet);
 938                         }
 939                 }
 940         }
 941 }
 942
 943 /**
 944  * \brief Interrupt handler for octeon
 945  * @param irq unused
 946  * @param dev octeon device
 947  */
 948 static
 949 irqreturn_t liquidio_intr_handler(int irq __attribute__((unused)), void *dev)
 950 {
 951         struct octeon_device *oct = (struct octeon_device *)dev;
 952         irqreturn_t ret;
 953
 954         /* Disable our interrupts for the duration of ISR */
 955         oct->fn_list.disable_interrupt(oct->chip);
 956
 957         ret = oct->fn_list.process_interrupt_regs(oct);
 958
 959         if (ret == IRQ_HANDLED)
 960                 liquidio_schedule_droq_pkt_handlers(oct);
 961
 962         /* Re-enable our interrupts  */
 963         if (!(atomic_read(&oct->status) == OCT_DEV_IN_RESET))
 964                 oct->fn_list.enable_interrupt(oct->chip);
 965
 966         return ret;
 967 }
 968
 969 /**
 970  * \brief Setup interrupt for octeon device
 971  * @param oct octeon device
 972  *
 973  *  Enable interrupt in Octeon device as given in the PCI interrupt mask.
 974  */
 975 static int octeon_setup_interrupt(struct octeon_device *oct)
 976 {
 977         int irqret, err;
 978
 979         err = pci_enable_msi(oct->pci_dev);
 980         if (err)
 981                 dev_warn(&oct->pci_dev->dev, "Reverting to legacy interrupts. Error: %d\n",
 982                          err);
 983         else
 984                 oct->flags |= LIO_FLAG_MSI_ENABLED;
 985
 986         irqret = request_irq(oct->pci_dev->irq, liquidio_intr_handler,
 987                              IRQF_SHARED, "octeon", oct);
 988         if (irqret) {
 989                 if (oct->flags & LIO_FLAG_MSI_ENABLED)
 990                         pci_disable_msi(oct->pci_dev);
 991                 dev_err(&oct->pci_dev->dev, "Request IRQ failed with code: %d\n",
 992                         irqret);
 993                 return 1;
 994         }
 995
 996         return 0;
 997 }
 998
 999 /**
1000  * \brief PCI probe handler
1001  * @param pdev PCI device structure
1002  * @param ent unused
1003  */
1004 static int
1005 liquidio_probe(struct pci_dev *pdev,
1006                const struct pci_device_id *ent __attribute__((unused)))
1007 {
1008         struct octeon_device *oct_dev = NULL;
1009         struct handshake *hs;
1010
1011         oct_dev = octeon_allocate_device(pdev->device,
1012                                          sizeof(struct octeon_device_priv));
1013         if (!oct_dev) {
1014                 dev_err(&pdev->dev, "Unable to allocate device\n");
1015                 return -ENOMEM;
1016         }
1017
1018         dev_info(&pdev->dev, "Initializing device %x:%x.\n",
1019                  (u32)pdev->vendor, (u32)pdev->device);
1020
1021         /* Assign octeon_device for this device to the private data area. */
1022         pci_set_drvdata(pdev, oct_dev);
1023
1024         /* set linux specific device pointer */
1025         oct_dev->pci_dev = (void *)pdev;
1026
1027         hs = &handshake[oct_dev->octeon_id];
1028         init_completion(&hs->init);
1029         init_completion(&hs->started);
1030         hs->pci_dev = pdev;
1031
1032         if (oct_dev->octeon_id == 0)
1033                 /* first LiquidIO NIC is detected */
1034                 complete(&first_stage);
1035
1036         if (octeon_device_init(oct_dev)) {
1037                 liquidio_remove(pdev);
1038                 return -ENOMEM;
1039         }
1040
1041         oct_dev->rx_pause = 1;
1042         oct_dev->tx_pause = 1;
1043
1044         dev_dbg(&oct_dev->pci_dev->dev, "Device is ready\n");
1045
1046         return 0;
1047 }
1048
1049 /**
1050  *\brief Destroy resources associated with octeon device
1051  * @param pdev PCI device structure
1052  * @param ent unused
1053  */
1054 static void octeon_destroy_resources(struct octeon_device *oct)
1055 {
1056         int i;
1057         struct octeon_device_priv *oct_priv =
1058                 (struct octeon_device_priv *)oct->priv;
1059
1060         struct handshake *hs;
1061
1062         switch (atomic_read(&oct->status)) {
1063         case OCT_DEV_RUNNING:
1064         case OCT_DEV_CORE_OK:
1065
1066                 /* No more instructions will be forwarded. */
1067                 atomic_set(&oct->status, OCT_DEV_IN_RESET);
1068
1069                 oct->app_mode = CVM_DRV_INVALID_APP;
1070                 dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
1071                         lio_get_state_string(&oct->status));
1072
1073                 schedule_timeout_uninterruptible(HZ / 10);
1074
1075                 /* fallthrough */
1076         case OCT_DEV_HOST_OK:
1077
1078                 /* fallthrough */
1079         case OCT_DEV_CONSOLE_INIT_DONE:
1080                 /* Remove any consoles */
1081                 octeon_remove_consoles(oct);
1082
1083                 /* fallthrough */
1084         case OCT_DEV_IO_QUEUES_DONE:
1085                 if (wait_for_pending_requests(oct))
1086                         dev_err(&oct->pci_dev->dev, "There were pending requests\n");
1087
1088                 if (lio_wait_for_instr_fetch(oct))
1089                         dev_err(&oct->pci_dev->dev, "IQ had pending instructions\n");
1090
1091                 /* Disable the input and output queues now. No more packets will
1092                  * arrive from Octeon, but we should wait for all packet
1093                  * processing to finish.
1094                  */
1095                 oct->fn_list.disable_io_queues(oct);
1096
1097                 if (lio_wait_for_oq_pkts(oct))
1098                         dev_err(&oct->pci_dev->dev, "OQ had pending packets\n");
1099
1100                 /* Disable interrupts  */
1101                 oct->fn_list.disable_interrupt(oct->chip);
1102
1103                 /* Release the interrupt line */
1104                 free_irq(oct->pci_dev->irq, oct);
1105
1106                 if (oct->flags & LIO_FLAG_MSI_ENABLED)
1107                         pci_disable_msi(oct->pci_dev);
1108
1109                 /* fallthrough */
1110         case OCT_DEV_IN_RESET:
1111         case OCT_DEV_DROQ_INIT_DONE:
1112                 /*atomic_set(&oct->status, OCT_DEV_DROQ_INIT_DONE);*/
1113                 mdelay(100);
1114                 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
1115                         if (!(oct->io_qmask.oq & (1ULL << i)))
1116                                 continue;
1117                         octeon_delete_droq(oct, i);
1118                 }
1119
1120                 /* Force any pending handshakes to complete */
1121                 for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
1122                         hs = &handshake[i];
1123
1124                         if (hs->pci_dev) {
1125                                 handshake[oct->octeon_id].init_ok = 0;
1126                                 complete(&handshake[oct->octeon_id].init);
1127                                 handshake[oct->octeon_id].started_ok = 0;
1128                                 complete(&handshake[oct->octeon_id].started);
1129                         }
1130                 }
1131
1132                 /* fallthrough */
1133         case OCT_DEV_RESP_LIST_INIT_DONE:
1134                 octeon_delete_response_list(oct);
1135
1136                 /* fallthrough */
1137         case OCT_DEV_SC_BUFF_POOL_INIT_DONE:
1138                 octeon_free_sc_buffer_pool(oct);
1139
1140                 /* fallthrough */
1141         case OCT_DEV_INSTR_QUEUE_INIT_DONE:
1142                 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
1143                         if (!(oct->io_qmask.iq & (1ULL << i)))
1144                                 continue;
1145                         octeon_delete_instr_queue(oct, i);
1146                 }
1147
1148                 /* fallthrough */
1149         case OCT_DEV_DISPATCH_INIT_DONE:
1150                 octeon_delete_dispatch_list(oct);
1151                 cancel_delayed_work_sync(&oct->nic_poll_work.work);
1152
1153                 /* fallthrough */
1154         case OCT_DEV_PCI_MAP_DONE:
1155
1156                 /* Soft reset the octeon device before exiting */
1157                 oct->fn_list.soft_reset(oct);
1158
1159                 octeon_unmap_pci_barx(oct, 0);
1160                 octeon_unmap_pci_barx(oct, 1);
1161
1162                 /* fallthrough */
1163         case OCT_DEV_BEGIN_STATE:
1164                 /* Disable the device, releasing the PCI INT */
1165                 pci_disable_device(oct->pci_dev);
1166
1167                 /* Nothing to be done here either */
1168                 break;
1169         }                       /* end switch (oct->status) */
1170
1171         tasklet_kill(&oct_priv->droq_tasklet);
1172 }
1173
1174 /**
1175  * \brief Send Rx control command
1176  * @param lio per-network private data
1177  * @param start_stop whether to start or stop
1178  */
1179 static void send_rx_ctrl_cmd(struct lio *lio, int start_stop)
1180 {
1181         struct octnic_ctrl_pkt nctrl;
1182
1183         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
1184
1185         nctrl.ncmd.s.cmd = OCTNET_CMD_RX_CTL;
1186         nctrl.ncmd.s.param1 = start_stop;
1187         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
1188         nctrl.netpndev = (u64)lio->netdev;
1189
1190         if (octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl) < 0)
1191                 netif_info(lio, rx_err, lio->netdev, "Failed to send RX Control message\n");
1192 }
1193
1194 /**
1195  * \brief Destroy NIC device interface
1196  * @param oct octeon device
1197  * @param ifidx which interface to destroy
1198  *
1199  * Cleanup associated with each interface for an Octeon device  when NIC
1200  * module is being unloaded or if initialization fails during load.
1201  */
1202 static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx)
1203 {
1204         struct net_device *netdev = oct->props[ifidx].netdev;
1205         struct lio *lio;
1206         struct napi_struct *napi, *n;
1207
1208         if (!netdev) {
1209                 dev_err(&oct->pci_dev->dev, "%s No netdevice ptr for index %d\n",
1210                         __func__, ifidx);
1211                 return;
1212         }
1213
1214         lio = GET_LIO(netdev);
1215
1216         dev_dbg(&oct->pci_dev->dev, "NIC device cleanup\n");
1217
1218         send_rx_ctrl_cmd(lio, 0);
1219
1220         if (atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING)
1221                 txqs_stop(netdev);
1222
1223         if (oct->props[lio->ifidx].napi_enabled == 1) {
1224                 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1225                         napi_disable(napi);
1226
1227                 oct->props[lio->ifidx].napi_enabled = 0;
1228         }
1229
1230         if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED)
1231                 unregister_netdev(netdev);
1232
1233         delete_glists(lio);
1234
1235         free_netdev(netdev);
1236
1237         oct->props[ifidx].gmxport = -1;
1238
1239         oct->props[ifidx].netdev = NULL;
1240 }
1241
1242 /**
1243  * \brief Stop complete NIC functionality
1244  * @param oct octeon device
1245  */
1246 static int liquidio_stop_nic_module(struct octeon_device *oct)
1247 {
1248         int i, j;
1249         struct lio *lio;
1250
1251         dev_dbg(&oct->pci_dev->dev, "Stopping network interfaces\n");
1252         if (!oct->ifcount) {
1253                 dev_err(&oct->pci_dev->dev, "Init for Octeon was not completed\n");
1254                 return 1;
1255         }
1256
1257         spin_lock_bh(&oct->cmd_resp_wqlock);
1258         oct->cmd_resp_state = OCT_DRV_OFFLINE;
1259         spin_unlock_bh(&oct->cmd_resp_wqlock);
1260
1261         for (i = 0; i < oct->ifcount; i++) {
1262                 lio = GET_LIO(oct->props[i].netdev);
1263                 for (j = 0; j < lio->linfo.num_rxpciq; j++)
1264                         octeon_unregister_droq_ops(oct,
1265                                                    lio->linfo.rxpciq[j].s.q_no);
1266         }
1267
1268         for (i = 0; i < oct->ifcount; i++)
1269                 liquidio_destroy_nic_device(oct, i);
1270
1271         dev_dbg(&oct->pci_dev->dev, "Network interfaces stopped\n");
1272         return 0;
1273 }
1274
1275 /**
1276  * \brief Cleans up resources at unload time
1277  * @param pdev PCI device structure
1278  */
1279 static void liquidio_remove(struct pci_dev *pdev)
1280 {
1281         struct octeon_device *oct_dev = pci_get_drvdata(pdev);
1282
1283         dev_dbg(&oct_dev->pci_dev->dev, "Stopping device\n");
1284
1285         if (oct_dev->app_mode && (oct_dev->app_mode == CVM_DRV_NIC_APP))
1286                 liquidio_stop_nic_module(oct_dev);
1287
1288         /* Reset the octeon device and cleanup all memory allocated for
1289          * the octeon device by driver.
1290          */
1291         octeon_destroy_resources(oct_dev);
1292
1293         dev_info(&oct_dev->pci_dev->dev, "Device removed\n");
1294
1295         /* This octeon device has been removed. Update the global
1296          * data structure to reflect this. Free the device structure.
1297          */
1298         octeon_free_device_mem(oct_dev);
1299 }
1300
1301 /**
1302  * \brief Identify the Octeon device and to map the BAR address space
1303  * @param oct octeon device
1304  */
1305 static int octeon_chip_specific_setup(struct octeon_device *oct)
1306 {
1307         u32 dev_id, rev_id;
1308         int ret = 1;
1309         char *s;
1310
1311         pci_read_config_dword(oct->pci_dev, 0, &dev_id);
1312         pci_read_config_dword(oct->pci_dev, 8, &rev_id);
1313         oct->rev_id = rev_id & 0xff;
1314
1315         switch (dev_id) {
1316         case OCTEON_CN68XX_PCIID:
1317                 oct->chip_id = OCTEON_CN68XX;
1318                 ret = lio_setup_cn68xx_octeon_device(oct);
1319                 s = "CN68XX";
1320                 break;
1321
1322         case OCTEON_CN66XX_PCIID:
1323                 oct->chip_id = OCTEON_CN66XX;
1324                 ret = lio_setup_cn66xx_octeon_device(oct);
1325                 s = "CN66XX";
1326                 break;
1327
1328         case OCTEON_CN23XX_PCIID_PF:
1329                 oct->chip_id = OCTEON_CN23XX_PF_VID;
1330                 ret = setup_cn23xx_octeon_pf_device(oct);
1331                 s = "CN23XX";
1332                 break;
1333
1334         default:
1335                 s = "?";
1336                 dev_err(&oct->pci_dev->dev, "Unknown device found (dev_id: %x)\n",
1337                         dev_id);
1338         }
1339
1340         if (!ret)
1341                 dev_info(&oct->pci_dev->dev, "%s PASS%d.%d %s Version: %s\n", s,
1342                          OCTEON_MAJOR_REV(oct),
1343                          OCTEON_MINOR_REV(oct),
1344                          octeon_get_conf(oct)->card_name,
1345                          LIQUIDIO_VERSION);
1346
1347         return ret;
1348 }
1349
1350 /**
1351  * \brief PCI initialization for each Octeon device.
1352  * @param oct octeon device
1353  */
1354 static int octeon_pci_os_setup(struct octeon_device *oct)
1355 {
1356         /* setup PCI stuff first */
1357         if (pci_enable_device(oct->pci_dev)) {
1358                 dev_err(&oct->pci_dev->dev, "pci_enable_device failed\n");
1359                 return 1;
1360         }
1361
1362         if (dma_set_mask_and_coherent(&oct->pci_dev->dev, DMA_BIT_MASK(64))) {
1363                 dev_err(&oct->pci_dev->dev, "Unexpected DMA device capability\n");
1364                 return 1;
1365         }
1366
1367         /* Enable PCI DMA Master. */
1368         pci_set_master(oct->pci_dev);
1369
1370         return 0;
1371 }
1372
1373 static inline int skb_iq(struct lio *lio, struct sk_buff *skb)
1374 {
1375         int q = 0;
1376
1377         if (netif_is_multiqueue(lio->netdev))
1378                 q = skb->queue_mapping % lio->linfo.num_txpciq;
1379
1380         return q;
1381 }
1382
1383 /**
1384  * \brief Check Tx queue state for a given network buffer
1385  * @param lio per-network private data
1386  * @param skb network buffer
1387  */
1388 static inline int check_txq_state(struct lio *lio, struct sk_buff *skb)
1389 {
1390         int q = 0, iq = 0;
1391
1392         if (netif_is_multiqueue(lio->netdev)) {
1393                 q = skb->queue_mapping;
1394                 iq = lio->linfo.txpciq[(q % (lio->linfo.num_txpciq))].s.q_no;
1395         } else {
1396                 iq = lio->txq;
1397                 q = iq;
1398         }
1399
1400         if (octnet_iq_is_full(lio->oct_dev, iq))
1401                 return 0;
1402
1403         if (__netif_subqueue_stopped(lio->netdev, q)) {
1404                 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq, tx_restart, 1);
1405                 wake_q(lio->netdev, q);
1406         }
1407         return 1;
1408 }
1409
1410 /**
1411  * \brief Unmap and free network buffer
1412  * @param buf buffer
1413  */
1414 static void free_netbuf(void *buf)
1415 {
1416         struct sk_buff *skb;
1417         struct octnet_buf_free_info *finfo;
1418         struct lio *lio;
1419
1420         finfo = (struct octnet_buf_free_info *)buf;
1421         skb = finfo->skb;
1422         lio = finfo->lio;
1423
1424         dma_unmap_single(&lio->oct_dev->pci_dev->dev, finfo->dptr, skb->len,
1425                          DMA_TO_DEVICE);
1426
1427         check_txq_state(lio, skb);
1428
1429         tx_buffer_free(skb);
1430 }
1431
1432 /**
1433  * \brief Unmap and free gather buffer
1434  * @param buf buffer
1435  */
1436 static void free_netsgbuf(void *buf)
1437 {
1438         struct octnet_buf_free_info *finfo;
1439         struct sk_buff *skb;
1440         struct lio *lio;
1441         struct octnic_gather *g;
1442         int i, frags, iq;
1443
1444         finfo = (struct octnet_buf_free_info *)buf;
1445         skb = finfo->skb;
1446         lio = finfo->lio;
1447         g = finfo->g;
1448         frags = skb_shinfo(skb)->nr_frags;
1449
1450         dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1451                          g->sg[0].ptr[0], (skb->len - skb->data_len),
1452                          DMA_TO_DEVICE);
1453
1454         i = 1;
1455         while (frags--) {
1456                 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1];
1457
1458                 pci_unmap_page((lio->oct_dev)->pci_dev,
1459                                g->sg[(i >> 2)].ptr[(i & 3)],
1460                                frag->size, DMA_TO_DEVICE);
1461                 i++;
1462         }
1463
1464         dma_sync_single_for_cpu(&lio->oct_dev->pci_dev->dev,
1465                                 g->sg_dma_ptr, g->sg_size, DMA_TO_DEVICE);
1466
1467         iq = skb_iq(lio, skb);
1468         spin_lock(&lio->glist_lock[iq]);
1469         list_add_tail(&g->list, &lio->glist[iq]);
1470         spin_unlock(&lio->glist_lock[iq]);
1471
1472         check_txq_state(lio, skb);     /* mq support: sub-queue state check */
1473
1474         tx_buffer_free(skb);
1475 }
1476
1477 /**
1478  * \brief Unmap and free gather buffer with response
1479  * @param buf buffer
1480  */
1481 static void free_netsgbuf_with_resp(void *buf)
1482 {
1483         struct octeon_soft_command *sc;
1484         struct octnet_buf_free_info *finfo;
1485         struct sk_buff *skb;
1486         struct lio *lio;
1487         struct octnic_gather *g;
1488         int i, frags, iq;
1489
1490         sc = (struct octeon_soft_command *)buf;
1491         skb = (struct sk_buff *)sc->callback_arg;
1492         finfo = (struct octnet_buf_free_info *)&skb->cb;
1493
1494         lio = finfo->lio;
1495         g = finfo->g;
1496         frags = skb_shinfo(skb)->nr_frags;
1497
1498         dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1499                          g->sg[0].ptr[0], (skb->len - skb->data_len),
1500                          DMA_TO_DEVICE);
1501
1502         i = 1;
1503         while (frags--) {
1504                 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1];
1505
1506                 pci_unmap_page((lio->oct_dev)->pci_dev,
1507                                g->sg[(i >> 2)].ptr[(i & 3)],
1508                                frag->size, DMA_TO_DEVICE);
1509                 i++;
1510         }
1511
1512         dma_sync_single_for_cpu(&lio->oct_dev->pci_dev->dev,
1513                                 g->sg_dma_ptr, g->sg_size, DMA_TO_DEVICE);
1514
1515         iq = skb_iq(lio, skb);
1516
1517         spin_lock(&lio->glist_lock[iq]);
1518         list_add_tail(&g->list, &lio->glist[iq]);
1519         spin_unlock(&lio->glist_lock[iq]);
1520
1521         /* Don't free the skb yet */
1522
1523         check_txq_state(lio, skb);
1524 }
1525
1526 /**
1527  * \brief Adjust ptp frequency
1528  * @param ptp PTP clock info
1529  * @param ppb how much to adjust by, in parts-per-billion
1530  */
1531 static int liquidio_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb)
1532 {
1533         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1534         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1535         u64 comp, delta;
1536         unsigned long flags;
1537         bool neg_adj = false;
1538
1539         if (ppb < 0) {
1540                 neg_adj = true;
1541                 ppb = -ppb;
1542         }
1543
1544         /* The hardware adds the clock compensation value to the
1545          * PTP clock on every coprocessor clock cycle, so we
1546          * compute the delta in terms of coprocessor clocks.
1547          */
1548         delta = (u64)ppb << 32;
1549         do_div(delta, oct->coproc_clock_rate);
1550
1551         spin_lock_irqsave(&lio->ptp_lock, flags);
1552         comp = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_COMP);
1553         if (neg_adj)
1554                 comp -= delta;
1555         else
1556                 comp += delta;
1557         lio_pci_writeq(oct, comp, CN6XXX_MIO_PTP_CLOCK_COMP);
1558         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1559
1560         return 0;
1561 }
1562
1563 /**
1564  * \brief Adjust ptp time
1565  * @param ptp PTP clock info
1566  * @param delta how much to adjust by, in nanosecs
1567  */
1568 static int liquidio_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
1569 {
1570         unsigned long flags;
1571         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1572
1573         spin_lock_irqsave(&lio->ptp_lock, flags);
1574         lio->ptp_adjust += delta;
1575         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1576
1577         return 0;
1578 }
1579
1580 /**
1581  * \brief Get hardware clock time, including any adjustment
1582  * @param ptp PTP clock info
1583  * @param ts timespec
1584  */
1585 static int liquidio_ptp_gettime(struct ptp_clock_info *ptp,
1586                                 struct timespec64 *ts)
1587 {
1588         u64 ns;
1589         unsigned long flags;
1590         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1591         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1592
1593         spin_lock_irqsave(&lio->ptp_lock, flags);
1594         ns = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_HI);
1595         ns += lio->ptp_adjust;
1596         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1597
1598         *ts = ns_to_timespec64(ns);
1599
1600         return 0;
1601 }
1602
1603 /**
1604  * \brief Set hardware clock time. Reset adjustment
1605  * @param ptp PTP clock info
1606  * @param ts timespec
1607  */
1608 static int liquidio_ptp_settime(struct ptp_clock_info *ptp,
1609                                 const struct timespec64 *ts)
1610 {
1611         u64 ns;
1612         unsigned long flags;
1613         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1614         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1615
1616         ns = timespec_to_ns(ts);
1617
1618         spin_lock_irqsave(&lio->ptp_lock, flags);
1619         lio_pci_writeq(oct, ns, CN6XXX_MIO_PTP_CLOCK_HI);
1620         lio->ptp_adjust = 0;
1621         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1622
1623         return 0;
1624 }
1625
1626 /**
1627  * \brief Check if PTP is enabled
1628  * @param ptp PTP clock info
1629  * @param rq request
1630  * @param on is it on
1631  */
1632 static int
1633 liquidio_ptp_enable(struct ptp_clock_info *ptp __attribute__((unused)),
1634                     struct ptp_clock_request *rq __attribute__((unused)),
1635                     int on __attribute__((unused)))
1636 {
1637         return -EOPNOTSUPP;
1638 }
1639
1640 /**
1641  * \brief Open PTP clock source
1642  * @param netdev network device
1643  */
1644 static void oct_ptp_open(struct net_device *netdev)
1645 {
1646         struct lio *lio = GET_LIO(netdev);
1647         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1648
1649         spin_lock_init(&lio->ptp_lock);
1650
1651         snprintf(lio->ptp_info.name, 16, "%s", netdev->name);
1652         lio->ptp_info.owner = THIS_MODULE;
1653         lio->ptp_info.max_adj = 250000000;
1654         lio->ptp_info.n_alarm = 0;
1655         lio->ptp_info.n_ext_ts = 0;
1656         lio->ptp_info.n_per_out = 0;
1657         lio->ptp_info.pps = 0;
1658         lio->ptp_info.adjfreq = liquidio_ptp_adjfreq;
1659         lio->ptp_info.adjtime = liquidio_ptp_adjtime;
1660         lio->ptp_info.gettime64 = liquidio_ptp_gettime;
1661         lio->ptp_info.settime64 = liquidio_ptp_settime;
1662         lio->ptp_info.enable = liquidio_ptp_enable;
1663
1664         lio->ptp_adjust = 0;
1665
1666         lio->ptp_clock = ptp_clock_register(&lio->ptp_info,
1667                                              &oct->pci_dev->dev);
1668
1669         if (IS_ERR(lio->ptp_clock))
1670                 lio->ptp_clock = NULL;
1671 }
1672
1673 /**
1674  * \brief Init PTP clock
1675  * @param oct octeon device
1676  */
1677 static void liquidio_ptp_init(struct octeon_device *oct)
1678 {
1679         u64 clock_comp, cfg;
1680
1681         clock_comp = (u64)NSEC_PER_SEC << 32;
1682         do_div(clock_comp, oct->coproc_clock_rate);
1683         lio_pci_writeq(oct, clock_comp, CN6XXX_MIO_PTP_CLOCK_COMP);
1684
1685         /* Enable */
1686         cfg = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_CFG);
1687         lio_pci_writeq(oct, cfg | 0x01, CN6XXX_MIO_PTP_CLOCK_CFG);
1688 }
1689
1690 /**
1691  * \brief Load firmware to device
1692  * @param oct octeon device
1693  *
1694  * Maps device to firmware filename, requests firmware, and downloads it
1695  */
1696 static int load_firmware(struct octeon_device *oct)
1697 {
1698         int ret = 0;
1699         const struct firmware *fw;
1700         char fw_name[LIO_MAX_FW_FILENAME_LEN];
1701         char *tmp_fw_type;
1702
1703         if (strncmp(fw_type, LIO_FW_NAME_TYPE_NONE,
1704                     sizeof(LIO_FW_NAME_TYPE_NONE)) == 0) {
1705                 dev_info(&oct->pci_dev->dev, "Skipping firmware load\n");
1706                 return ret;
1707         }
1708
1709         if (fw_type[0] == '\0')
1710                 tmp_fw_type = LIO_FW_NAME_TYPE_NIC;
1711         else
1712                 tmp_fw_type = fw_type;
1713
1714         sprintf(fw_name, "%s%s%s_%s%s", LIO_FW_DIR, LIO_FW_BASE_NAME,
1715                 octeon_get_conf(oct)->card_name, tmp_fw_type,
1716                 LIO_FW_NAME_SUFFIX);
1717
1718         ret = request_firmware(&fw, fw_name, &oct->pci_dev->dev);
1719         if (ret) {
1720                 dev_err(&oct->pci_dev->dev, "Request firmware failed. Could not find file %s.\n.",
1721                         fw_name);
1722                 release_firmware(fw);
1723                 return ret;
1724         }
1725
1726         ret = octeon_download_firmware(oct, fw->data, fw->size);
1727
1728         release_firmware(fw);
1729
1730         return ret;
1731 }
1732
1733 /**
1734  * \brief Setup output queue
1735  * @param oct octeon device
1736  * @param q_no which queue
1737  * @param num_descs how many descriptors
1738  * @param desc_size size of each descriptor
1739  * @param app_ctx application context
1740  */
1741 static int octeon_setup_droq(struct octeon_device *oct, int q_no, int num_descs,
1742                              int desc_size, void *app_ctx)
1743 {
1744         int ret_val = 0;
1745
1746         dev_dbg(&oct->pci_dev->dev, "Creating Droq: %d\n", q_no);
1747         /* droq creation and local register settings. */
1748         ret_val = octeon_create_droq(oct, q_no, num_descs, desc_size, app_ctx);
1749         if (ret_val < 0)
1750                 return ret_val;
1751
1752         if (ret_val == 1) {
1753                 dev_dbg(&oct->pci_dev->dev, "Using default droq %d\n", q_no);
1754                 return 0;
1755         }
1756         /* tasklet creation for the droq */
1757
1758         /* Enable the droq queues */
1759         octeon_set_droq_pkt_op(oct, q_no, 1);
1760
1761         /* Send Credit for Octeon Output queues. Credits are always
1762          * sent after the output queue is enabled.
1763          */
1764         writel(oct->droq[q_no]->max_count,
1765                oct->droq[q_no]->pkts_credit_reg);
1766
1767         return ret_val;
1768 }
1769
1770 /**
1771  * \brief Callback for getting interface configuration
1772  * @param status status of request
1773  * @param buf pointer to resp structure
1774  */
1775 static void if_cfg_callback(struct octeon_device *oct,
1776                             u32 status __attribute__((unused)),
1777                             void *buf)
1778 {
1779         struct octeon_soft_command *sc = (struct octeon_soft_command *)buf;
1780         struct liquidio_if_cfg_resp *resp;
1781         struct liquidio_if_cfg_context *ctx;
1782
1783         resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
1784         ctx  = (struct liquidio_if_cfg_context *)sc->ctxptr;
1785
1786         oct = lio_get_device(ctx->octeon_id);
1787         if (resp->status)
1788                 dev_err(&oct->pci_dev->dev, "nic if cfg instruction failed. Status: %llx\n",
1789                         CVM_CAST64(resp->status));
1790         WRITE_ONCE(ctx->cond, 1);
1791
1792         snprintf(oct->fw_info.liquidio_firmware_version, 32, "%s",
1793                  resp->cfg_info.liquidio_firmware_version);
1794
1795         /* This barrier is required to be sure that the response has been
1796          * written fully before waking up the handler
1797          */
1798         wmb();
1799
1800         wake_up_interruptible(&ctx->wc);
1801 }
1802
1803 /**
1804  * \brief Select queue based on hash
1805  * @param dev Net device
1806  * @param skb sk_buff structure
1807  * @returns selected queue number
1808  */
1809 static u16 select_q(struct net_device *dev, struct sk_buff *skb,
1810                     void *accel_priv __attribute__((unused)),
1811                     select_queue_fallback_t fallback __attribute__((unused)))
1812 {
1813         u32 qindex = 0;
1814         struct lio *lio;
1815
1816         lio = GET_LIO(dev);
1817         qindex = skb_tx_hash(dev, skb);
1818
1819         return (u16)(qindex % (lio->linfo.num_txpciq));
1820 }
1821
1822 /** Routine to push packets arriving on Octeon interface upto network layer.
1823  * @param oct_id   - octeon device id.
1824  * @param skbuff   - skbuff struct to be passed to network layer.
1825  * @param len      - size of total data received.
1826  * @param rh       - Control header associated with the packet
1827  * @param param    - additional control data with the packet
1828  * @param arg      - farg registered in droq_ops
1829  */
1830 static void
1831 liquidio_push_packet(u32 octeon_id __attribute__((unused)),
1832                      void *skbuff,
1833                      u32 len,
1834                      union octeon_rh *rh,
1835                      void *param,
1836                      void *arg)
1837 {
1838         struct napi_struct *napi = param;
1839         struct sk_buff *skb = (struct sk_buff *)skbuff;
1840         struct skb_shared_hwtstamps *shhwtstamps;
1841         u64 ns;
1842         u16 vtag = 0;
1843         struct net_device *netdev = (struct net_device *)arg;
1844         struct octeon_droq *droq = container_of(param, struct octeon_droq,
1845                                                 napi);
1846         if (netdev) {
1847                 int packet_was_received;
1848                 struct lio *lio = GET_LIO(netdev);
1849                 struct octeon_device *oct = lio->oct_dev;
1850
1851                 /* Do not proceed if the interface is not in RUNNING state. */
1852                 if (!ifstate_check(lio, LIO_IFSTATE_RUNNING)) {
1853                         recv_buffer_free(skb);
1854                         droq->stats.rx_dropped++;
1855                         return;
1856                 }
1857
1858                 skb->dev = netdev;
1859
1860                 skb_record_rx_queue(skb, droq->q_no);
1861                 if (likely(len > MIN_SKB_SIZE)) {
1862                         struct octeon_skb_page_info *pg_info;
1863                         unsigned char *va;
1864
1865                         pg_info = ((struct octeon_skb_page_info *)(skb->cb));
1866                         if (pg_info->page) {
1867                                 /* For Paged allocation use the frags */
1868                                 va = page_address(pg_info->page) +
1869                                         pg_info->page_offset;
1870                                 memcpy(skb->data, va, MIN_SKB_SIZE);
1871                                 skb_put(skb, MIN_SKB_SIZE);
1872                                 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
1873                                                 pg_info->page,
1874                                                 pg_info->page_offset +
1875                                                 MIN_SKB_SIZE,
1876                                                 len - MIN_SKB_SIZE,
1877                                                 LIO_RXBUFFER_SZ);
1878                         }
1879                 } else {
1880                         struct octeon_skb_page_info *pg_info =
1881                                 ((struct octeon_skb_page_info *)(skb->cb));
1882                         skb_copy_to_linear_data(skb, page_address(pg_info->page)
1883                                                 + pg_info->page_offset, len);
1884                         skb_put(skb, len);
1885                         put_page(pg_info->page);
1886                 }
1887
1888                 if (((oct->chip_id == OCTEON_CN66XX) ||
1889                      (oct->chip_id == OCTEON_CN68XX)) &&
1890                     ptp_enable) {
1891                         if (rh->r_dh.has_hwtstamp) {
1892                                 /* timestamp is included from the hardware at
1893                                  * the beginning of the packet.
1894                                  */
1895                                 if (ifstate_check
1896                                     (lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED)) {
1897                                         /* Nanoseconds are in the first 64-bits
1898                                          * of the packet.
1899                                          */
1900                                         memcpy(&ns, (skb->data), sizeof(ns));
1901                                         shhwtstamps = skb_hwtstamps(skb);
1902                                         shhwtstamps->hwtstamp =
1903                                                 ns_to_ktime(ns +
1904                                                             lio->ptp_adjust);
1905                                 }
1906                                 skb_pull(skb, sizeof(ns));
1907                         }
1908                 }
1909
1910                 skb->protocol = eth_type_trans(skb, skb->dev);
1911                 if ((netdev->features & NETIF_F_RXCSUM) &&
1912                     (((rh->r_dh.encap_on) &&
1913                       (rh->r_dh.csum_verified & CNNIC_TUN_CSUM_VERIFIED)) ||
1914                      (!(rh->r_dh.encap_on) &&
1915                       (rh->r_dh.csum_verified & CNNIC_CSUM_VERIFIED))))
1916                         /* checksum has already been verified */
1917                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1918                 else
1919                         skb->ip_summed = CHECKSUM_NONE;
1920
1921                 /* Setting Encapsulation field on basis of status received
1922                  * from the firmware
1923                  */
1924                 if (rh->r_dh.encap_on) {
1925                         skb->encapsulation = 1;
1926                         skb->csum_level = 1;
1927                         droq->stats.rx_vxlan++;
1928                 }
1929
1930                 /* inbound VLAN tag */
1931                 if ((netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
1932                     (rh->r_dh.vlan != 0)) {
1933                         u16 vid = rh->r_dh.vlan;
1934                         u16 priority = rh->r_dh.priority;
1935
1936                         vtag = priority << 13 | vid;
1937                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vtag);
1938                 }
1939
1940                 packet_was_received = napi_gro_receive(napi, skb) != GRO_DROP;
1941
1942                 if (packet_was_received) {
1943                         droq->stats.rx_bytes_received += len;
1944                         droq->stats.rx_pkts_received++;
1945                         netdev->last_rx = jiffies;
1946                 } else {
1947                         droq->stats.rx_dropped++;
1948                         netif_info(lio, rx_err, lio->netdev,
1949                                    "droq:%d  error rx_dropped:%llu\n",
1950                                    droq->q_no, droq->stats.rx_dropped);
1951                 }
1952
1953         } else {
1954                 recv_buffer_free(skb);
1955         }
1956 }
1957
1958 /**
1959  * \brief wrapper for calling napi_schedule
1960  * @param param parameters to pass to napi_schedule
1961  *
1962  * Used when scheduling on different CPUs
1963  */
1964 static void napi_schedule_wrapper(void *param)
1965 {
1966         struct napi_struct *napi = param;
1967
1968         napi_schedule(napi);
1969 }
1970
1971 /**
1972  * \brief callback when receive interrupt occurs and we are in NAPI mode
1973  * @param arg pointer to octeon output queue
1974  */
1975 static void liquidio_napi_drv_callback(void *arg)
1976 {
1977         struct octeon_droq *droq = arg;
1978         int this_cpu = smp_processor_id();
1979
1980         if (droq->cpu_id == this_cpu) {
1981                 napi_schedule(&droq->napi);
1982         } else {
1983                 struct call_single_data *csd = &droq->csd;
1984
1985                 csd->func = napi_schedule_wrapper;
1986                 csd->info = &droq->napi;
1987                 csd->flags = 0;
1988
1989                 smp_call_function_single_async(droq->cpu_id, csd);
1990         }
1991 }
1992
1993 /**
1994  * \brief Entry point for NAPI polling
1995  * @param napi NAPI structure
1996  * @param budget maximum number of items to process
1997  */
1998 static int liquidio_napi_poll(struct napi_struct *napi, int budget)
1999 {
2000         struct octeon_droq *droq;
2001         int work_done;
2002         int tx_done = 0, iq_no;
2003         struct octeon_instr_queue *iq;
2004         struct octeon_device *oct;
2005
2006         droq = container_of(napi, struct octeon_droq, napi);
2007         oct = droq->oct_dev;
2008         iq_no = droq->q_no;
2009         /* Handle Droq descriptors */
2010         work_done = octeon_process_droq_poll_cmd(oct, droq->q_no,
2011                                                  POLL_EVENT_PROCESS_PKTS,
2012                                                  budget);
2013
2014         /* Flush the instruction queue */
2015         iq = oct->instr_queue[iq_no];
2016         if (iq) {
2017                 /* Process iq buffers with in the budget limits */
2018                 tx_done = octeon_flush_iq(oct, iq, 1, budget);
2019                 /* Update iq read-index rather than waiting for next interrupt.
2020                  * Return back if tx_done is false.
2021                  */
2022                 update_txq_status(oct, iq_no);
2023                 /*tx_done = (iq->flush_index == iq->octeon_read_index);*/
2024         } else {
2025                 dev_err(&oct->pci_dev->dev, "%s:  iq (%d) num invalid\n",
2026                         __func__, iq_no);
2027         }
2028
2029         if ((work_done < budget) && (tx_done)) {
2030                 napi_complete(napi);
2031                 octeon_process_droq_poll_cmd(droq->oct_dev, droq->q_no,
2032                                              POLL_EVENT_ENABLE_INTR, 0);
2033                 return 0;
2034         }
2035
2036         return (!tx_done) ? (budget) : (work_done);
2037 }
2038
2039 /**
2040  * \brief Setup input and output queues
2041  * @param octeon_dev octeon device
2042  * @param ifidx  Interface Index
2043  *
2044  * Note: Queues are with respect to the octeon device. Thus
2045  * an input queue is for egress packets, and output queues
2046  * are for ingress packets.
2047  */
2048 static inline int setup_io_queues(struct octeon_device *octeon_dev,
2049                                   int ifidx)
2050 {
2051         struct octeon_droq_ops droq_ops;
2052         struct net_device *netdev;
2053         static int cpu_id;
2054         static int cpu_id_modulus;
2055         struct octeon_droq *droq;
2056         struct napi_struct *napi;
2057         int q, q_no, retval = 0;
2058         struct lio *lio;
2059         int num_tx_descs;
2060
2061         netdev = octeon_dev->props[ifidx].netdev;
2062
2063         lio = GET_LIO(netdev);
2064
2065         memset(&droq_ops, 0, sizeof(struct octeon_droq_ops));
2066
2067         droq_ops.fptr = liquidio_push_packet;
2068         droq_ops.farg = (void *)netdev;
2069
2070         droq_ops.poll_mode = 1;
2071         droq_ops.napi_fn = liquidio_napi_drv_callback;
2072         cpu_id = 0;
2073         cpu_id_modulus = num_present_cpus();
2074
2075         /* set up DROQs. */
2076         for (q = 0; q < lio->linfo.num_rxpciq; q++) {
2077                 q_no = lio->linfo.rxpciq[q].s.q_no;
2078                 dev_dbg(&octeon_dev->pci_dev->dev,
2079                         "setup_io_queues index:%d linfo.rxpciq.s.q_no:%d\n",
2080                         q, q_no);
2081                 retval = octeon_setup_droq(octeon_dev, q_no,
2082                                            CFG_GET_NUM_RX_DESCS_NIC_IF
2083                                                    (octeon_get_conf(octeon_dev),
2084                                                    lio->ifidx),
2085                                            CFG_GET_NUM_RX_BUF_SIZE_NIC_IF
2086                                                    (octeon_get_conf(octeon_dev),
2087                                                    lio->ifidx), NULL);
2088                 if (retval) {
2089                         dev_err(&octeon_dev->pci_dev->dev,
2090                                 "%s : Runtime DROQ(RxQ) creation failed.\n",
2091                                 __func__);
2092                         return 1;
2093                 }
2094
2095                 droq = octeon_dev->droq[q_no];
2096                 napi = &droq->napi;
2097                 dev_dbg(&octeon_dev->pci_dev->dev,
2098                         "netif_napi_add netdev:%llx oct:%llx\n",
2099                         (u64)netdev,
2100                         (u64)octeon_dev);
2101                 netif_napi_add(netdev, napi, liquidio_napi_poll, 64);
2102
2103                 /* designate a CPU for this droq */
2104                 droq->cpu_id = cpu_id;
2105                 cpu_id++;
2106                 if (cpu_id >= cpu_id_modulus)
2107                         cpu_id = 0;
2108
2109                 octeon_register_droq_ops(octeon_dev, q_no, &droq_ops);
2110         }
2111
2112         /* set up IQs. */
2113         for (q = 0; q < lio->linfo.num_txpciq; q++) {
2114                 num_tx_descs = CFG_GET_NUM_TX_DESCS_NIC_IF(octeon_get_conf
2115                                                            (octeon_dev),
2116                                                            lio->ifidx);
2117                 retval = octeon_setup_iq(octeon_dev, ifidx, q,
2118                                          lio->linfo.txpciq[q], num_tx_descs,
2119                                          netdev_get_tx_queue(netdev, q));
2120                 if (retval) {
2121                         dev_err(&octeon_dev->pci_dev->dev,
2122                                 " %s : Runtime IQ(TxQ) creation failed.\n",
2123                                 __func__);
2124                         return 1;
2125                 }
2126         }
2127
2128         return 0;
2129 }
2130
2131 /**
2132  * \brief Poll routine for checking transmit queue status
2133  * @param work work_struct data structure
2134  */
2135 static void octnet_poll_check_txq_status(struct work_struct *work)
2136 {
2137         struct cavium_wk *wk = (struct cavium_wk *)work;
2138         struct lio *lio = (struct lio *)wk->ctxptr;
2139
2140         if (!ifstate_check(lio, LIO_IFSTATE_RUNNING))
2141                 return;
2142
2143         check_txq_status(lio);
2144         queue_delayed_work(lio->txq_status_wq.wq,
2145                            &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
2146 }
2147
2148 /**
2149  * \brief Sets up the txq poll check
2150  * @param netdev network device
2151  */
2152 static inline void setup_tx_poll_fn(struct net_device *netdev)
2153 {
2154         struct lio *lio = GET_LIO(netdev);
2155         struct octeon_device *oct = lio->oct_dev;
2156
2157         lio->txq_status_wq.wq = alloc_workqueue("txq-status",
2158                                                 WQ_MEM_RECLAIM, 0);
2159         if (!lio->txq_status_wq.wq) {
2160                 dev_err(&oct->pci_dev->dev, "unable to create cavium txq status wq\n");
2161                 return;
2162         }
2163         INIT_DELAYED_WORK(&lio->txq_status_wq.wk.work,
2164                           octnet_poll_check_txq_status);
2165         lio->txq_status_wq.wk.ctxptr = lio;
2166         queue_delayed_work(lio->txq_status_wq.wq,
2167                            &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
2168 }
2169
2170 static inline void cleanup_tx_poll_fn(struct net_device *netdev)
2171 {
2172         struct lio *lio = GET_LIO(netdev);
2173
2174         cancel_delayed_work_sync(&lio->txq_status_wq.wk.work);
2175         destroy_workqueue(lio->txq_status_wq.wq);
2176 }
2177
2178 /**
2179  * \brief Net device open for LiquidIO
2180  * @param netdev network device
2181  */
2182 static int liquidio_open(struct net_device *netdev)
2183 {
2184         struct lio *lio = GET_LIO(netdev);
2185         struct octeon_device *oct = lio->oct_dev;
2186         struct napi_struct *napi, *n;
2187
2188         if (oct->props[lio->ifidx].napi_enabled == 0) {
2189                 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
2190                         napi_enable(napi);
2191
2192                 oct->props[lio->ifidx].napi_enabled = 1;
2193         }
2194
2195         oct_ptp_open(netdev);
2196
2197         ifstate_set(lio, LIO_IFSTATE_RUNNING);
2198
2199         setup_tx_poll_fn(netdev);
2200
2201         start_txq(netdev);
2202
2203         netif_info(lio, ifup, lio->netdev, "Interface Open, ready for traffic\n");
2204
2205         /* tell Octeon to start forwarding packets to host */
2206         send_rx_ctrl_cmd(lio, 1);
2207
2208         /* Ready for link status updates */
2209         lio->intf_open = 1;
2210
2211         dev_info(&oct->pci_dev->dev, "%s interface is opened\n",
2212                  netdev->name);
2213
2214         return 0;
2215 }
2216
2217 /**
2218  * \brief Net device stop for LiquidIO
2219  * @param netdev network device
2220  */
2221 static int liquidio_stop(struct net_device *netdev)
2222 {
2223         struct lio *lio = GET_LIO(netdev);
2224         struct octeon_device *oct = lio->oct_dev;
2225
2226         ifstate_reset(lio, LIO_IFSTATE_RUNNING);
2227
2228         netif_tx_disable(netdev);
2229
2230         /* Inform that netif carrier is down */
2231         netif_carrier_off(netdev);
2232         lio->intf_open = 0;
2233         lio->linfo.link.s.link_up = 0;
2234         lio->link_changes++;
2235
2236         /* Pause for a moment and wait for Octeon to flush out (to the wire) any
2237          * egress packets that are in-flight.
2238          */
2239         set_current_state(TASK_INTERRUPTIBLE);
2240         schedule_timeout(msecs_to_jiffies(100));
2241
2242         /* Now it should be safe to tell Octeon that nic interface is down. */
2243         send_rx_ctrl_cmd(lio, 0);
2244
2245         cleanup_tx_poll_fn(netdev);
2246
2247         if (lio->ptp_clock) {
2248                 ptp_clock_unregister(lio->ptp_clock);
2249                 lio->ptp_clock = NULL;
2250         }
2251
2252         dev_info(&oct->pci_dev->dev, "%s interface is stopped\n", netdev->name);
2253
2254         return 0;
2255 }
2256
2257 /**
2258  * \brief Converts a mask based on net device flags
2259  * @param netdev network device
2260  *
2261  * This routine generates a octnet_ifflags mask from the net device flags
2262  * received from the OS.
2263  */
2264 static inline enum octnet_ifflags get_new_flags(struct net_device *netdev)
2265 {
2266         enum octnet_ifflags f = OCTNET_IFFLAG_UNICAST;
2267
2268         if (netdev->flags & IFF_PROMISC)
2269                 f |= OCTNET_IFFLAG_PROMISC;
2270
2271         if (netdev->flags & IFF_ALLMULTI)
2272                 f |= OCTNET_IFFLAG_ALLMULTI;
2273
2274         if (netdev->flags & IFF_MULTICAST) {
2275                 f |= OCTNET_IFFLAG_MULTICAST;
2276
2277                 /* Accept all multicast addresses if there are more than we
2278                  * can handle
2279                  */
2280                 if (netdev_mc_count(netdev) > MAX_OCTEON_MULTICAST_ADDR)
2281                         f |= OCTNET_IFFLAG_ALLMULTI;
2282         }
2283
2284         if (netdev->flags & IFF_BROADCAST)
2285                 f |= OCTNET_IFFLAG_BROADCAST;
2286
2287         return f;
2288 }
2289
2290 /**
2291  * \brief Net device set_multicast_list
2292  * @param netdev network device
2293  */
2294 static void liquidio_set_mcast_list(struct net_device *netdev)
2295 {
2296         struct lio *lio = GET_LIO(netdev);
2297         struct octeon_device *oct = lio->oct_dev;
2298         struct octnic_ctrl_pkt nctrl;
2299         struct netdev_hw_addr *ha;
2300         u64 *mc;
2301         int ret;
2302         int mc_count = min(netdev_mc_count(netdev), MAX_OCTEON_MULTICAST_ADDR);
2303
2304         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2305
2306         /* Create a ctrl pkt command to be sent to core app. */
2307         nctrl.ncmd.u64 = 0;
2308         nctrl.ncmd.s.cmd = OCTNET_CMD_SET_MULTI_LIST;
2309         nctrl.ncmd.s.param1 = get_new_flags(netdev);
2310         nctrl.ncmd.s.param2 = mc_count;
2311         nctrl.ncmd.s.more = mc_count;
2312         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2313         nctrl.netpndev = (u64)netdev;
2314         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2315
2316         /* copy all the addresses into the udd */
2317         mc = &nctrl.udd[0];
2318         netdev_for_each_mc_addr(ha, netdev) {
2319                 *mc = 0;
2320                 memcpy(((u8 *)mc) + 2, ha->addr, ETH_ALEN);
2321                 /* no need to swap bytes */
2322
2323                 if (++mc > &nctrl.udd[mc_count])
2324                         break;
2325         }
2326
2327         /* Apparently, any activity in this call from the kernel has to
2328          * be atomic. So we won't wait for response.
2329          */
2330         nctrl.wait_time = 0;
2331
2332         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2333         if (ret < 0) {
2334                 dev_err(&oct->pci_dev->dev, "DEVFLAGS change failed in core (ret: 0x%x)\n",
2335                         ret);
2336         }
2337 }
2338
2339 /**
2340  * \brief Net device set_mac_address
2341  * @param netdev network device
2342  */
2343 static int liquidio_set_mac(struct net_device *netdev, void *p)
2344 {
2345         int ret = 0;
2346         struct lio *lio = GET_LIO(netdev);
2347         struct octeon_device *oct = lio->oct_dev;
2348         struct sockaddr *addr = (struct sockaddr *)p;
2349         struct octnic_ctrl_pkt nctrl;
2350
2351         if (!is_valid_ether_addr(addr->sa_data))
2352                 return -EADDRNOTAVAIL;
2353
2354         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2355
2356         nctrl.ncmd.u64 = 0;
2357         nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR;
2358         nctrl.ncmd.s.param1 = 0;
2359         nctrl.ncmd.s.more = 1;
2360         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2361         nctrl.netpndev = (u64)netdev;
2362         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2363         nctrl.wait_time = 100;
2364
2365         nctrl.udd[0] = 0;
2366         /* The MAC Address is presented in network byte order. */
2367         memcpy((u8 *)&nctrl.udd[0] + 2, addr->sa_data, ETH_ALEN);
2368
2369         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2370         if (ret < 0) {
2371                 dev_err(&oct->pci_dev->dev, "MAC Address change failed\n");
2372                 return -ENOMEM;
2373         }
2374         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2375         memcpy(((u8 *)&lio->linfo.hw_addr) + 2, addr->sa_data, ETH_ALEN);
2376
2377         return 0;
2378 }
2379
2380 /**
2381  * \brief Net device get_stats
2382  * @param netdev network device
2383  */
2384 static struct net_device_stats *liquidio_get_stats(struct net_device *netdev)
2385 {
2386         struct lio *lio = GET_LIO(netdev);
2387         struct net_device_stats *stats = &netdev->stats;
2388         struct octeon_device *oct;
2389         u64 pkts = 0, drop = 0, bytes = 0;
2390         struct oct_droq_stats *oq_stats;
2391         struct oct_iq_stats *iq_stats;
2392         int i, iq_no, oq_no;
2393
2394         oct = lio->oct_dev;
2395
2396         for (i = 0; i < lio->linfo.num_txpciq; i++) {
2397                 iq_no = lio->linfo.txpciq[i].s.q_no;
2398                 iq_stats = &oct->instr_queue[iq_no]->stats;
2399                 pkts += iq_stats->tx_done;
2400                 drop += iq_stats->tx_dropped;
2401                 bytes += iq_stats->tx_tot_bytes;
2402         }
2403
2404         stats->tx_packets = pkts;
2405         stats->tx_bytes = bytes;
2406         stats->tx_dropped = drop;
2407
2408         pkts = 0;
2409         drop = 0;
2410         bytes = 0;
2411
2412         for (i = 0; i < lio->linfo.num_rxpciq; i++) {
2413                 oq_no = lio->linfo.rxpciq[i].s.q_no;
2414                 oq_stats = &oct->droq[oq_no]->stats;
2415                 pkts += oq_stats->rx_pkts_received;
2416                 drop += (oq_stats->rx_dropped +
2417                          oq_stats->dropped_nodispatch +
2418                          oq_stats->dropped_toomany +
2419                          oq_stats->dropped_nomem);
2420                 bytes += oq_stats->rx_bytes_received;
2421         }
2422
2423         stats->rx_bytes = bytes;
2424         stats->rx_packets = pkts;
2425         stats->rx_dropped = drop;
2426
2427         return stats;
2428 }
2429
2430 /**
2431  * \brief Net device change_mtu
2432  * @param netdev network device
2433  */
2434 static int liquidio_change_mtu(struct net_device *netdev, int new_mtu)
2435 {
2436         struct lio *lio = GET_LIO(netdev);
2437         struct octeon_device *oct = lio->oct_dev;
2438         struct octnic_ctrl_pkt nctrl;
2439         int ret = 0;
2440
2441         /* Limit the MTU to make sure the ethernet packets are between 68 bytes
2442          * and 16000 bytes
2443          */
2444         if ((new_mtu < LIO_MIN_MTU_SIZE) ||
2445             (new_mtu > LIO_MAX_MTU_SIZE)) {
2446                 dev_err(&oct->pci_dev->dev, "Invalid MTU: %d\n", new_mtu);
2447                 dev_err(&oct->pci_dev->dev, "Valid range %d and %d\n",
2448                         LIO_MIN_MTU_SIZE, LIO_MAX_MTU_SIZE);
2449                 return -EINVAL;
2450         }
2451
2452         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2453
2454         nctrl.ncmd.u64 = 0;
2455         nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MTU;
2456         nctrl.ncmd.s.param1 = new_mtu;
2457         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2458         nctrl.wait_time = 100;
2459         nctrl.netpndev = (u64)netdev;
2460         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2461
2462         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2463         if (ret < 0) {
2464                 dev_err(&oct->pci_dev->dev, "Failed to set MTU\n");
2465                 return -1;
2466         }
2467
2468         lio->mtu = new_mtu;
2469
2470         return 0;
2471 }
2472
2473 /**
2474  * \brief Handler for SIOCSHWTSTAMP ioctl
2475  * @param netdev network device
2476  * @param ifr interface request
2477  * @param cmd command
2478  */
2479 static int hwtstamp_ioctl(struct net_device *netdev, struct ifreq *ifr)
2480 {
2481         struct hwtstamp_config conf;
2482         struct lio *lio = GET_LIO(netdev);
2483
2484         if (copy_from_user(&conf, ifr->ifr_data, sizeof(conf)))
2485                 return -EFAULT;
2486
2487         if (conf.flags)
2488                 return -EINVAL;
2489
2490         switch (conf.tx_type) {
2491         case HWTSTAMP_TX_ON:
2492         case HWTSTAMP_TX_OFF:
2493                 break;
2494         default:
2495                 return -ERANGE;
2496         }
2497
2498         switch (conf.rx_filter) {
2499         case HWTSTAMP_FILTER_NONE:
2500                 break;
2501         case HWTSTAMP_FILTER_ALL:
2502         case HWTSTAMP_FILTER_SOME:
2503         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
2504         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
2505         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
2506         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
2507         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
2508         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
2509         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
2510         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
2511         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
2512         case HWTSTAMP_FILTER_PTP_V2_EVENT:
2513         case HWTSTAMP_FILTER_PTP_V2_SYNC:
2514         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
2515                 conf.rx_filter = HWTSTAMP_FILTER_ALL;
2516                 break;
2517         default:
2518                 return -ERANGE;
2519         }
2520
2521         if (conf.rx_filter == HWTSTAMP_FILTER_ALL)
2522                 ifstate_set(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
2523
2524         else
2525                 ifstate_reset(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
2526
2527         return copy_to_user(ifr->ifr_data, &conf, sizeof(conf)) ? -EFAULT : 0;
2528 }
2529
2530 /**
2531  * \brief ioctl handler
2532  * @param netdev network device
2533  * @param ifr interface request
2534  * @param cmd command
2535  */
2536 static int liquidio_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2537 {
2538         switch (cmd) {
2539         case SIOCSHWTSTAMP:
2540                 return hwtstamp_ioctl(netdev, ifr);
2541         default:
2542                 return -EOPNOTSUPP;
2543         }
2544 }
2545
2546 /**
2547  * \brief handle a Tx timestamp response
2548  * @param status response status
2549  * @param buf pointer to skb
2550  */
2551 static void handle_timestamp(struct octeon_device *oct,
2552                              u32 status,
2553                              void *buf)
2554 {
2555         struct octnet_buf_free_info *finfo;
2556         struct octeon_soft_command *sc;
2557         struct oct_timestamp_resp *resp;
2558         struct lio *lio;
2559         struct sk_buff *skb = (struct sk_buff *)buf;
2560
2561         finfo = (struct octnet_buf_free_info *)skb->cb;
2562         lio = finfo->lio;
2563         sc = finfo->sc;
2564         oct = lio->oct_dev;
2565         resp = (struct oct_timestamp_resp *)sc->virtrptr;
2566
2567         if (status != OCTEON_REQUEST_DONE) {
2568                 dev_err(&oct->pci_dev->dev, "Tx timestamp instruction failed. Status: %llx\n",
2569                         CVM_CAST64(status));
2570                 resp->timestamp = 0;
2571         }
2572
2573         octeon_swap_8B_data(&resp->timestamp, 1);
2574
2575         if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) != 0)) {
2576                 struct skb_shared_hwtstamps ts;
2577                 u64 ns = resp->timestamp;
2578
2579                 netif_info(lio, tx_done, lio->netdev,
2580                            "Got resulting SKBTX_HW_TSTAMP skb=%p ns=%016llu\n",
2581                            skb, (unsigned long long)ns);
2582                 ts.hwtstamp = ns_to_ktime(ns + lio->ptp_adjust);
2583                 skb_tstamp_tx(skb, &ts);
2584         }
2585
2586         octeon_free_soft_command(oct, sc);
2587         tx_buffer_free(skb);
2588 }
2589
2590 /* \brief Send a data packet that will be timestamped
2591  * @param oct octeon device
2592  * @param ndata pointer to network data
2593  * @param finfo pointer to private network data
2594  */
2595 static inline int send_nic_timestamp_pkt(struct octeon_device *oct,
2596                                          struct octnic_data_pkt *ndata,
2597                                          struct octnet_buf_free_info *finfo)
2598 {
2599         int retval;
2600         struct octeon_soft_command *sc;
2601         struct lio *lio;
2602         int ring_doorbell;
2603         u32 len;
2604
2605         lio = finfo->lio;
2606
2607         sc = octeon_alloc_soft_command_resp(oct, &ndata->cmd,
2608                                             sizeof(struct oct_timestamp_resp));
2609         finfo->sc = sc;
2610
2611         if (!sc) {
2612                 dev_err(&oct->pci_dev->dev, "No memory for timestamped data packet\n");
2613                 return IQ_SEND_FAILED;
2614         }
2615
2616         if (ndata->reqtype == REQTYPE_NORESP_NET)
2617                 ndata->reqtype = REQTYPE_RESP_NET;
2618         else if (ndata->reqtype == REQTYPE_NORESP_NET_SG)
2619                 ndata->reqtype = REQTYPE_RESP_NET_SG;
2620
2621         sc->callback = handle_timestamp;
2622         sc->callback_arg = finfo->skb;
2623         sc->iq_no = ndata->q_no;
2624
2625         len = (u32)((struct octeon_instr_ih2 *)(&sc->cmd.cmd2.ih2))->dlengsz;
2626
2627         ring_doorbell = 1;
2628         retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd,
2629                                      sc, len, ndata->reqtype);
2630
2631         if (retval == IQ_SEND_FAILED) {
2632                 dev_err(&oct->pci_dev->dev, "timestamp data packet failed status: %x\n",
2633                         retval);
2634                 octeon_free_soft_command(oct, sc);
2635         } else {
2636                 netif_info(lio, tx_queued, lio->netdev, "Queued timestamp packet\n");
2637         }
2638
2639         return retval;
2640 }
2641
2642 /** \brief Transmit networks packets to the Octeon interface
2643  * @param skbuff   skbuff struct to be passed to network layer.
2644  * @param netdev    pointer to network device
2645  * @returns whether the packet was transmitted to the device okay or not
2646  *             (NETDEV_TX_OK or NETDEV_TX_BUSY)
2647  */
2648 static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
2649 {
2650         struct lio *lio;
2651         struct octnet_buf_free_info *finfo;
2652         union octnic_cmd_setup cmdsetup;
2653         struct octnic_data_pkt ndata;
2654         struct octeon_device *oct;
2655         struct oct_iq_stats *stats;
2656         struct octeon_instr_irh *irh;
2657         union tx_info *tx_info;
2658         int status = 0;
2659         int q_idx = 0, iq_no = 0;
2660         int j;
2661         u64 dptr = 0;
2662         u32 tag = 0;
2663
2664         lio = GET_LIO(netdev);
2665         oct = lio->oct_dev;
2666
2667         if (netif_is_multiqueue(netdev)) {
2668                 q_idx = skb->queue_mapping;
2669                 q_idx = (q_idx % (lio->linfo.num_txpciq));
2670                 tag = q_idx;
2671                 iq_no = lio->linfo.txpciq[q_idx].s.q_no;
2672         } else {
2673                 iq_no = lio->txq;
2674         }
2675
2676         stats = &oct->instr_queue[iq_no]->stats;
2677
2678         /* Check for all conditions in which the current packet cannot be
2679          * transmitted.
2680          */
2681         if (!(atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) ||
2682             (!lio->linfo.link.s.link_up) ||
2683             (skb->len <= 0)) {
2684                 netif_info(lio, tx_err, lio->netdev,
2685                            "Transmit failed link_status : %d\n",
2686                            lio->linfo.link.s.link_up);
2687                 goto lio_xmit_failed;
2688         }
2689
2690         /* Use space in skb->cb to store info used to unmap and
2691          * free the buffers.
2692          */
2693         finfo = (struct octnet_buf_free_info *)skb->cb;
2694         finfo->lio = lio;
2695         finfo->skb = skb;
2696         finfo->sc = NULL;
2697
2698         /* Prepare the attributes for the data to be passed to OSI. */
2699         memset(&ndata, 0, sizeof(struct octnic_data_pkt));
2700
2701         ndata.buf = (void *)finfo;
2702
2703         ndata.q_no = iq_no;
2704
2705         if (netif_is_multiqueue(netdev)) {
2706                 if (octnet_iq_is_full(oct, ndata.q_no)) {
2707                         /* defer sending if queue is full */
2708                         netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n",
2709                                    ndata.q_no);
2710                         stats->tx_iq_busy++;
2711                         return NETDEV_TX_BUSY;
2712                 }
2713         } else {
2714                 if (octnet_iq_is_full(oct, lio->txq)) {
2715                         /* defer sending if queue is full */
2716                         stats->tx_iq_busy++;
2717                         netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n",
2718                                    lio->txq);
2719                         return NETDEV_TX_BUSY;
2720                 }
2721         }
2722         /* pr_info(" XMIT - valid Qs: %d, 1st Q no: %d, cpu:  %d, q_no:%d\n",
2723          *      lio->linfo.num_txpciq, lio->txq, cpu, ndata.q_no);
2724          */
2725
2726         ndata.datasize = skb->len;
2727
2728         cmdsetup.u64 = 0;
2729         cmdsetup.s.iq_no = iq_no;
2730
2731         if (skb->ip_summed == CHECKSUM_PARTIAL) {
2732                 if (skb->encapsulation) {
2733                         cmdsetup.s.tnl_csum = 1;
2734                         stats->tx_vxlan++;
2735                 } else {
2736                         cmdsetup.s.transport_csum = 1;
2737                 }
2738         }
2739         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
2740                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2741                 cmdsetup.s.timestamp = 1;
2742         }
2743
2744         if (skb_shinfo(skb)->nr_frags == 0) {
2745                 cmdsetup.s.u.datasize = skb->len;
2746                 octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
2747
2748                 /* Offload checksum calculation for TCP/UDP packets */
2749                 dptr = dma_map_single(&oct->pci_dev->dev,
2750                                       skb->data,
2751                                       skb->len,
2752                                       DMA_TO_DEVICE);
2753                 if (dma_mapping_error(&oct->pci_dev->dev, dptr)) {
2754                         dev_err(&oct->pci_dev->dev, "%s DMA mapping error 1\n",
2755                                 __func__);
2756                         return NETDEV_TX_BUSY;
2757                 }
2758
2759                 ndata.cmd.cmd2.dptr = dptr;
2760                 finfo->dptr = dptr;
2761                 ndata.reqtype = REQTYPE_NORESP_NET;
2762
2763         } else {
2764                 int i, frags;
2765                 struct skb_frag_struct *frag;
2766                 struct octnic_gather *g;
2767
2768                 spin_lock(&lio->glist_lock[q_idx]);
2769                 g = (struct octnic_gather *)
2770                         list_delete_head(&lio->glist[q_idx]);
2771                 spin_unlock(&lio->glist_lock[q_idx]);
2772
2773                 if (!g) {
2774                         netif_info(lio, tx_err, lio->netdev,
2775                                    "Transmit scatter gather: glist null!\n");
2776                         goto lio_xmit_failed;
2777                 }
2778
2779                 cmdsetup.s.gather = 1;
2780                 cmdsetup.s.u.gatherptrs = (skb_shinfo(skb)->nr_frags + 1);
2781                 octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
2782
2783                 memset(g->sg, 0, g->sg_size);
2784
2785                 g->sg[0].ptr[0] = dma_map_single(&oct->pci_dev->dev,
2786                                                  skb->data,
2787                                                  (skb->len - skb->data_len),
2788                                                  DMA_TO_DEVICE);
2789                 if (dma_mapping_error(&oct->pci_dev->dev, g->sg[0].ptr[0])) {
2790                         dev_err(&oct->pci_dev->dev, "%s DMA mapping error 2\n",
2791                                 __func__);
2792                         return NETDEV_TX_BUSY;
2793                 }
2794                 add_sg_size(&g->sg[0], (skb->len - skb->data_len), 0);
2795
2796                 frags = skb_shinfo(skb)->nr_frags;
2797                 i = 1;
2798                 while (frags--) {
2799                         frag = &skb_shinfo(skb)->frags[i - 1];
2800
2801                         g->sg[(i >> 2)].ptr[(i & 3)] =
2802                                 dma_map_page(&oct->pci_dev->dev,
2803                                              frag->page.p,
2804                                              frag->page_offset,
2805                                              frag->size,
2806                                              DMA_TO_DEVICE);
2807
2808                         if (dma_mapping_error(&oct->pci_dev->dev,
2809                                               g->sg[i >> 2].ptr[i & 3])) {
2810                                 dma_unmap_single(&oct->pci_dev->dev,
2811                                                  g->sg[0].ptr[0],
2812                                                  skb->len - skb->data_len,
2813                                                  DMA_TO_DEVICE);
2814                                 for (j = 1; j < i; j++) {
2815                                         frag = &skb_shinfo(skb)->frags[j - 1];
2816                                         dma_unmap_page(&oct->pci_dev->dev,
2817                                                        g->sg[j >> 2].ptr[j & 3],
2818                                                        frag->size,
2819                                                        DMA_TO_DEVICE);
2820                                 }
2821                                 dev_err(&oct->pci_dev->dev, "%s DMA mapping error 3\n",
2822                                         __func__);
2823                                 return NETDEV_TX_BUSY;
2824                         }
2825
2826                         add_sg_size(&g->sg[(i >> 2)], frag->size, (i & 3));
2827                         i++;
2828                 }
2829
2830                 dma_sync_single_for_device(&oct->pci_dev->dev, g->sg_dma_ptr,
2831                                            g->sg_size, DMA_TO_DEVICE);
2832                 dptr = g->sg_dma_ptr;
2833
2834                 ndata.cmd.cmd2.dptr = dptr;
2835                 finfo->dptr = dptr;
2836                 finfo->g = g;
2837
2838                 ndata.reqtype = REQTYPE_NORESP_NET_SG;
2839         }
2840
2841         irh = (struct octeon_instr_irh *)&ndata.cmd.cmd2.irh;
2842         tx_info = (union tx_info *)&ndata.cmd.cmd2.ossp[0];
2843
2844         if (skb_shinfo(skb)->gso_size) {
2845                 tx_info->s.gso_size = skb_shinfo(skb)->gso_size;
2846                 tx_info->s.gso_segs = skb_shinfo(skb)->gso_segs;
2847                 stats->tx_gso++;
2848         }
2849
2850         /* HW insert VLAN tag */
2851         if (skb_vlan_tag_present(skb)) {
2852                 irh->priority = skb_vlan_tag_get(skb) >> 13;
2853                 irh->vlan = skb_vlan_tag_get(skb) & 0xfff;
2854         }
2855
2856         if (unlikely(cmdsetup.s.timestamp))
2857                 status = send_nic_timestamp_pkt(oct, &ndata, finfo);
2858         else
2859                 status = octnet_send_nic_data_pkt(oct, &ndata);
2860         if (status == IQ_SEND_FAILED)
2861                 goto lio_xmit_failed;
2862
2863         netif_info(lio, tx_queued, lio->netdev, "Transmit queued successfully\n");
2864
2865         if (status == IQ_SEND_STOP)
2866                 stop_q(lio->netdev, q_idx);
2867
2868         netif_trans_update(netdev);
2869
2870         if (skb_shinfo(skb)->gso_size)
2871                 stats->tx_done += skb_shinfo(skb)->gso_segs;
2872         else
2873                 stats->tx_done++;
2874         stats->tx_tot_bytes += skb->len;
2875
2876         return NETDEV_TX_OK;
2877
2878 lio_xmit_failed:
2879         stats->tx_dropped++;
2880         netif_info(lio, tx_err, lio->netdev, "IQ%d Transmit dropped:%llu\n",
2881                    iq_no, stats->tx_dropped);
2882         if (dptr)
2883                 dma_unmap_single(&oct->pci_dev->dev, dptr,
2884                                  ndata.datasize, DMA_TO_DEVICE);
2885         tx_buffer_free(skb);
2886         return NETDEV_TX_OK;
2887 }
2888
2889 /** \brief Network device Tx timeout
2890  * @param netdev    pointer to network device
2891  */
2892 static void liquidio_tx_timeout(struct net_device *netdev)
2893 {
2894         struct lio *lio;
2895
2896         lio = GET_LIO(netdev);
2897
2898         netif_info(lio, tx_err, lio->netdev,
2899                    "Transmit timeout tx_dropped:%ld, waking up queues now!!\n",
2900                    netdev->stats.tx_dropped);
2901         netif_trans_update(netdev);
2902         txqs_wake(netdev);
2903 }
2904
2905 static int liquidio_vlan_rx_add_vid(struct net_device *netdev,
2906                                     __be16 proto __attribute__((unused)),
2907                                     u16 vid)
2908 {
2909         struct lio *lio = GET_LIO(netdev);
2910         struct octeon_device *oct = lio->oct_dev;
2911         struct octnic_ctrl_pkt nctrl;
2912         int ret = 0;
2913
2914         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2915
2916         nctrl.ncmd.u64 = 0;
2917         nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER;
2918         nctrl.ncmd.s.param1 = vid;
2919         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2920         nctrl.wait_time = 100;
2921         nctrl.netpndev = (u64)netdev;
2922         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2923
2924         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2925         if (ret < 0) {
2926                 dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n",
2927                         ret);
2928         }
2929
2930         return ret;
2931 }
2932
2933 static int liquidio_vlan_rx_kill_vid(struct net_device *netdev,
2934                                      __be16 proto __attribute__((unused)),
2935                                      u16 vid)
2936 {
2937         struct lio *lio = GET_LIO(netdev);
2938         struct octeon_device *oct = lio->oct_dev;
2939         struct octnic_ctrl_pkt nctrl;
2940         int ret = 0;
2941
2942         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2943
2944         nctrl.ncmd.u64 = 0;
2945         nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER;
2946         nctrl.ncmd.s.param1 = vid;
2947         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2948         nctrl.wait_time = 100;
2949         nctrl.netpndev = (u64)netdev;
2950         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2951
2952         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2953         if (ret < 0) {
2954                 dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n",
2955                         ret);
2956         }
2957         return ret;
2958 }
2959
2960 /** Sending command to enable/disable RX checksum offload
2961  * @param netdev                pointer to network device
2962  * @param command               OCTNET_CMD_TNL_RX_CSUM_CTL
2963  * @param rx_cmd_bit            OCTNET_CMD_RXCSUM_ENABLE/
2964  *                              OCTNET_CMD_RXCSUM_DISABLE
2965  * @returns                     SUCCESS or FAILURE
2966  */
2967 static int liquidio_set_rxcsum_command(struct net_device *netdev, int command,
2968                                        u8 rx_cmd)
2969 {
2970         struct lio *lio = GET_LIO(netdev);
2971         struct octeon_device *oct = lio->oct_dev;
2972         struct octnic_ctrl_pkt nctrl;
2973         int ret = 0;
2974
2975         nctrl.ncmd.u64 = 0;
2976         nctrl.ncmd.s.cmd = command;
2977         nctrl.ncmd.s.param1 = rx_cmd;
2978         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2979         nctrl.wait_time = 100;
2980         nctrl.netpndev = (u64)netdev;
2981         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2982
2983         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2984         if (ret < 0) {
2985                 dev_err(&oct->pci_dev->dev,
2986                         "DEVFLAGS RXCSUM change failed in core(ret:0x%x)\n",
2987                         ret);
2988         }
2989         return ret;
2990 }
2991
2992 /** Sending command to add/delete VxLAN UDP port to firmware
2993  * @param netdev                pointer to network device
2994  * @param command               OCTNET_CMD_VXLAN_PORT_CONFIG
2995  * @param vxlan_port            VxLAN port to be added or deleted
2996  * @param vxlan_cmd_bit         OCTNET_CMD_VXLAN_PORT_ADD,
2997  *                              OCTNET_CMD_VXLAN_PORT_DEL
2998  * @returns                     SUCCESS or FAILURE
2999  */
3000 static int liquidio_vxlan_port_command(struct net_device *netdev, int command,
3001                                        u16 vxlan_port, u8 vxlan_cmd_bit)
3002 {
3003         struct lio *lio = GET_LIO(netdev);
3004         struct octeon_device *oct = lio->oct_dev;
3005         struct octnic_ctrl_pkt nctrl;
3006         int ret = 0;
3007
3008         nctrl.ncmd.u64 = 0;
3009         nctrl.ncmd.s.cmd = command;
3010         nctrl.ncmd.s.more = vxlan_cmd_bit;
3011         nctrl.ncmd.s.param1 = vxlan_port;
3012         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
3013         nctrl.wait_time = 100;
3014         nctrl.netpndev = (u64)netdev;
3015         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
3016
3017         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
3018         if (ret < 0) {
3019                 dev_err(&oct->pci_dev->dev,
3020                         "VxLAN port add/delete failed in core (ret:0x%x)\n",
3021                         ret);
3022         }
3023         return ret;
3024 }
3025
3026 /** \brief Net device fix features
3027  * @param netdev  pointer to network device
3028  * @param request features requested
3029  * @returns updated features list
3030  */
3031 static netdev_features_t liquidio_fix_features(struct net_device *netdev,
3032                                                netdev_features_t request)
3033 {
3034         struct lio *lio = netdev_priv(netdev);
3035
3036         if ((request & NETIF_F_RXCSUM) &&
3037             !(lio->dev_capability & NETIF_F_RXCSUM))
3038                 request &= ~NETIF_F_RXCSUM;
3039
3040         if ((request & NETIF_F_HW_CSUM) &&
3041             !(lio->dev_capability & NETIF_F_HW_CSUM))
3042                 request &= ~NETIF_F_HW_CSUM;
3043
3044         if ((request & NETIF_F_TSO) && !(lio->dev_capability & NETIF_F_TSO))
3045                 request &= ~NETIF_F_TSO;
3046
3047         if ((request & NETIF_F_TSO6) && !(lio->dev_capability & NETIF_F_TSO6))
3048                 request &= ~NETIF_F_TSO6;
3049
3050         if ((request & NETIF_F_LRO) && !(lio->dev_capability & NETIF_F_LRO))
3051                 request &= ~NETIF_F_LRO;
3052
3053         /*Disable LRO if RXCSUM is off */
3054         if (!(request & NETIF_F_RXCSUM) && (netdev->features & NETIF_F_LRO) &&
3055             (lio->dev_capability & NETIF_F_LRO))
3056                 request &= ~NETIF_F_LRO;
3057
3058         return request;
3059 }
3060
3061 /** \brief Net device set features
3062  * @param netdev  pointer to network device
3063  * @param features features to enable/disable
3064  */
3065 static int liquidio_set_features(struct net_device *netdev,
3066                                  netdev_features_t features)
3067 {
3068         struct lio *lio = netdev_priv(netdev);
3069
3070         if (!((netdev->features ^ features) & NETIF_F_LRO))
3071                 return 0;
3072
3073         if ((features & NETIF_F_LRO) && (lio->dev_capability & NETIF_F_LRO))
3074                 liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
3075                                      OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
3076         else if (!(features & NETIF_F_LRO) &&
3077                  (lio->dev_capability & NETIF_F_LRO))
3078                 liquidio_set_feature(netdev, OCTNET_CMD_LRO_DISABLE,
3079                                      OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
3080
3081         /* Sending command to firmware to enable/disable RX checksum
3082          * offload settings using ethtool
3083          */
3084         if (!(netdev->features & NETIF_F_RXCSUM) &&
3085             (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
3086             (features & NETIF_F_RXCSUM))
3087                 liquidio_set_rxcsum_command(netdev,
3088                                             OCTNET_CMD_TNL_RX_CSUM_CTL,
3089                                             OCTNET_CMD_RXCSUM_ENABLE);
3090         else if ((netdev->features & NETIF_F_RXCSUM) &&
3091                  (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
3092                  !(features & NETIF_F_RXCSUM))
3093                 liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
3094                                             OCTNET_CMD_RXCSUM_DISABLE);
3095
3096         return 0;
3097 }
3098
3099 static void liquidio_add_vxlan_port(struct net_device *netdev,
3100                                     struct udp_tunnel_info *ti)
3101 {
3102         if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
3103                 return;
3104
3105         liquidio_vxlan_port_command(netdev,
3106                                     OCTNET_CMD_VXLAN_PORT_CONFIG,
3107                                     htons(ti->port),
3108                                     OCTNET_CMD_VXLAN_PORT_ADD);
3109 }
3110
3111 static void liquidio_del_vxlan_port(struct net_device *netdev,
3112                                     struct udp_tunnel_info *ti)
3113 {
3114         if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
3115                 return;
3116
3117         liquidio_vxlan_port_command(netdev,
3118                                     OCTNET_CMD_VXLAN_PORT_CONFIG,
3119                                     htons(ti->port),
3120                                     OCTNET_CMD_VXLAN_PORT_DEL);
3121 }
3122
3123 static struct net_device_ops lionetdevops = {
3124         .ndo_open               = liquidio_open,
3125         .ndo_stop               = liquidio_stop,
3126         .ndo_start_xmit         = liquidio_xmit,
3127         .ndo_get_stats          = liquidio_get_stats,
3128         .ndo_set_mac_address    = liquidio_set_mac,
3129         .ndo_set_rx_mode        = liquidio_set_mcast_list,
3130         .ndo_tx_timeout         = liquidio_tx_timeout,
3131
3132         .ndo_vlan_rx_add_vid    = liquidio_vlan_rx_add_vid,
3133         .ndo_vlan_rx_kill_vid   = liquidio_vlan_rx_kill_vid,
3134         .ndo_change_mtu         = liquidio_change_mtu,
3135         .ndo_do_ioctl           = liquidio_ioctl,
3136         .ndo_fix_features       = liquidio_fix_features,
3137         .ndo_set_features       = liquidio_set_features,
3138         .ndo_udp_tunnel_add     = liquidio_add_vxlan_port,
3139         .ndo_udp_tunnel_del     = liquidio_del_vxlan_port,
3140 };
3141
3142 /** \brief Entry point for the liquidio module
3143  */
3144 static int __init liquidio_init(void)
3145 {
3146         int i;
3147         struct handshake *hs;
3148
3149         init_completion(&first_stage);
3150
3151         octeon_init_device_list(conf_type);
3152
3153         if (liquidio_init_pci())
3154                 return -EINVAL;
3155
3156         wait_for_completion_timeout(&first_stage, msecs_to_jiffies(1000));
3157
3158         for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3159                 hs = &handshake[i];
3160                 if (hs->pci_dev) {
3161                         wait_for_completion(&hs->init);
3162                         if (!hs->init_ok) {
3163                                 /* init handshake failed */
3164                                 dev_err(&hs->pci_dev->dev,
3165                                         "Failed to init device\n");
3166                                 liquidio_deinit_pci();
3167                                 return -EIO;
3168                         }
3169                 }
3170         }
3171
3172         for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3173                 hs = &handshake[i];
3174                 if (hs->pci_dev) {
3175                         wait_for_completion_timeout(&hs->started,
3176                                                     msecs_to_jiffies(30000));
3177                         if (!hs->started_ok) {
3178                                 /* starter handshake failed */
3179                                 dev_err(&hs->pci_dev->dev,
3180                                         "Firmware failed to start\n");
3181                                 liquidio_deinit_pci();
3182                                 return -EIO;
3183                         }
3184                 }
3185         }
3186
3187         return 0;
3188 }
3189
3190 static int lio_nic_info(struct octeon_recv_info *recv_info, void *buf)
3191 {
3192         struct octeon_device *oct = (struct octeon_device *)buf;
3193         struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt;
3194         int gmxport = 0;
3195         union oct_link_status *ls;
3196         int i;
3197
3198         if (recv_pkt->buffer_size[0] != sizeof(*ls)) {
3199                 dev_err(&oct->pci_dev->dev, "Malformed NIC_INFO, len=%d, ifidx=%d\n",
3200                         recv_pkt->buffer_size[0],
3201                         recv_pkt->rh.r_nic_info.gmxport);
3202                 goto nic_info_err;
3203         }
3204
3205         gmxport = recv_pkt->rh.r_nic_info.gmxport;
3206         ls = (union oct_link_status *)get_rbd(recv_pkt->buffer_ptr[0]);
3207
3208         octeon_swap_8B_data((u64 *)ls, (sizeof(union oct_link_status)) >> 3);
3209         for (i = 0; i < oct->ifcount; i++) {
3210                 if (oct->props[i].gmxport == gmxport) {
3211                         update_link_status(oct->props[i].netdev, ls);
3212                         break;
3213                 }
3214         }
3215
3216 nic_info_err:
3217         for (i = 0; i < recv_pkt->buffer_count; i++)
3218                 recv_buffer_free(recv_pkt->buffer_ptr[i]);
3219         octeon_free_recv_info(recv_info);
3220         return 0;
3221 }
3222
3223 /**
3224  * \brief Setup network interfaces
3225  * @param octeon_dev  octeon device
3226  *
3227  * Called during init time for each device. It assumes the NIC
3228  * is already up and running.  The link information for each
3229  * interface is passed in link_info.
3230  */
3231 static int setup_nic_devices(struct octeon_device *octeon_dev)
3232 {
3233         struct lio *lio = NULL;
3234         struct net_device *netdev;
3235         u8 mac[6], i, j;
3236         struct octeon_soft_command *sc;
3237         struct liquidio_if_cfg_context *ctx;
3238         struct liquidio_if_cfg_resp *resp;
3239         struct octdev_props *props;
3240         int retval, num_iqueues, num_oqueues;
3241         union oct_nic_if_cfg if_cfg;
3242         unsigned int base_queue;
3243         unsigned int gmx_port_id;
3244         u32 resp_size, ctx_size, data_size;
3245         u32 ifidx_or_pfnum;
3246         struct lio_version *vdata;
3247
3248         /* This is to handle link status changes */
3249         octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
3250                                     OPCODE_NIC_INFO,
3251                                     lio_nic_info, octeon_dev);
3252
3253         /* REQTYPE_RESP_NET and REQTYPE_SOFT_COMMAND do not have free functions.
3254          * They are handled directly.
3255          */
3256         octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET,
3257                                         free_netbuf);
3258
3259         octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET_SG,
3260                                         free_netsgbuf);
3261
3262         octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_RESP_NET_SG,
3263                                         free_netsgbuf_with_resp);
3264
3265         for (i = 0; i < octeon_dev->ifcount; i++) {
3266                 resp_size = sizeof(struct liquidio_if_cfg_resp);
3267                 ctx_size = sizeof(struct liquidio_if_cfg_context);
3268                 data_size = sizeof(struct lio_version);
3269                 sc = (struct octeon_soft_command *)
3270                         octeon_alloc_soft_command(octeon_dev, data_size,
3271                                                   resp_size, ctx_size);
3272                 resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
3273                 ctx  = (struct liquidio_if_cfg_context *)sc->ctxptr;
3274                 vdata = (struct lio_version *)sc->virtdptr;
3275
3276                 *((u64 *)vdata) = 0;
3277                 vdata->major = cpu_to_be16(LIQUIDIO_BASE_MAJOR_VERSION);
3278                 vdata->minor = cpu_to_be16(LIQUIDIO_BASE_MINOR_VERSION);
3279                 vdata->micro = cpu_to_be16(LIQUIDIO_BASE_MICRO_VERSION);
3280
3281                 if (OCTEON_CN23XX_PF(octeon_dev)) {
3282                         num_iqueues = octeon_dev->sriov_info.num_pf_rings;
3283                         num_oqueues = octeon_dev->sriov_info.num_pf_rings;
3284                         base_queue = octeon_dev->sriov_info.pf_srn;
3285
3286                         gmx_port_id = octeon_dev->pf_num;
3287                         ifidx_or_pfnum = octeon_dev->pf_num;
3288                 } else {
3289                         num_iqueues = CFG_GET_NUM_TXQS_NIC_IF(
3290                                                 octeon_get_conf(octeon_dev), i);
3291                         num_oqueues = CFG_GET_NUM_RXQS_NIC_IF(
3292                                                 octeon_get_conf(octeon_dev), i);
3293                         base_queue = CFG_GET_BASE_QUE_NIC_IF(
3294                                                 octeon_get_conf(octeon_dev), i);
3295                         gmx_port_id = CFG_GET_GMXID_NIC_IF(
3296                                                 octeon_get_conf(octeon_dev), i);
3297                         ifidx_or_pfnum = i;
3298                 }
3299
3300                 dev_dbg(&octeon_dev->pci_dev->dev,
3301                         "requesting config for interface %d, iqs %d, oqs %d\n",
3302                         ifidx_or_pfnum, num_iqueues, num_oqueues);
3303                 WRITE_ONCE(ctx->cond, 0);
3304                 ctx->octeon_id = lio_get_device_id(octeon_dev);
3305                 init_waitqueue_head(&ctx->wc);
3306
3307                 if_cfg.u64 = 0;
3308                 if_cfg.s.num_iqueues = num_iqueues;
3309                 if_cfg.s.num_oqueues = num_oqueues;
3310                 if_cfg.s.base_queue = base_queue;
3311                 if_cfg.s.gmx_port_id = gmx_port_id;
3312
3313                 sc->iq_no = 0;
3314
3315                 octeon_prepare_soft_command(octeon_dev, sc, OPCODE_NIC,
3316                                             OPCODE_NIC_IF_CFG, 0,
3317                                             if_cfg.u64, 0);
3318
3319                 sc->callback = if_cfg_callback;
3320                 sc->callback_arg = sc;
3321                 sc->wait_time = 3000;
3322
3323                 retval = octeon_send_soft_command(octeon_dev, sc);
3324                 if (retval == IQ_SEND_FAILED) {
3325                         dev_err(&octeon_dev->pci_dev->dev,
3326                                 "iq/oq config failed status: %x\n",
3327                                 retval);
3328                         /* Soft instr is freed by driver in case of failure. */
3329                         goto setup_nic_dev_fail;
3330                 }
3331
3332                 /* Sleep on a wait queue till the cond flag indicates that the
3333                  * response arrived or timed-out.
3334                  */
3335                 sleep_cond(&ctx->wc, &ctx->cond);
3336                 retval = resp->status;
3337                 if (retval) {
3338                         dev_err(&octeon_dev->pci_dev->dev, "iq/oq config failed\n");
3339                         goto setup_nic_dev_fail;
3340                 }
3341
3342                 octeon_swap_8B_data((u64 *)(&resp->cfg_info),
3343                                     (sizeof(struct liquidio_if_cfg_info)) >> 3);
3344
3345                 num_iqueues = hweight64(resp->cfg_info.iqmask);
3346                 num_oqueues = hweight64(resp->cfg_info.oqmask);
3347
3348                 if (!(num_iqueues) || !(num_oqueues)) {
3349                         dev_err(&octeon_dev->pci_dev->dev,
3350                                 "Got bad iqueues (%016llx) or oqueues (%016llx) from firmware.\n",
3351                                 resp->cfg_info.iqmask,
3352                                 resp->cfg_info.oqmask);
3353                         goto setup_nic_dev_fail;
3354                 }
3355                 dev_dbg(&octeon_dev->pci_dev->dev,
3356                         "interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d\n",
3357                         i, resp->cfg_info.iqmask, resp->cfg_info.oqmask,
3358                         num_iqueues, num_oqueues);
3359                 netdev = alloc_etherdev_mq(LIO_SIZE, num_iqueues);
3360
3361                 if (!netdev) {
3362                         dev_err(&octeon_dev->pci_dev->dev, "Device allocation failed\n");
3363                         goto setup_nic_dev_fail;
3364                 }
3365
3366                 SET_NETDEV_DEV(netdev, &octeon_dev->pci_dev->dev);
3367
3368                 if (num_iqueues > 1)
3369                         lionetdevops.ndo_select_queue = select_q;
3370
3371                 /* Associate the routines that will handle different
3372                  * netdev tasks.
3373                  */
3374                 netdev->netdev_ops = &lionetdevops;
3375
3376                 lio = GET_LIO(netdev);
3377
3378                 memset(lio, 0, sizeof(struct lio));
3379
3380                 lio->ifidx = ifidx_or_pfnum;
3381
3382                 props = &octeon_dev->props[i];
3383                 props->gmxport = resp->cfg_info.linfo.gmxport;
3384                 props->netdev = netdev;
3385
3386                 lio->linfo.num_rxpciq = num_oqueues;
3387                 lio->linfo.num_txpciq = num_iqueues;
3388                 for (j = 0; j < num_oqueues; j++) {
3389                         lio->linfo.rxpciq[j].u64 =
3390                                 resp->cfg_info.linfo.rxpciq[j].u64;
3391                 }
3392                 for (j = 0; j < num_iqueues; j++) {
3393                         lio->linfo.txpciq[j].u64 =
3394                                 resp->cfg_info.linfo.txpciq[j].u64;
3395                 }
3396                 lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr;
3397                 lio->linfo.gmxport = resp->cfg_info.linfo.gmxport;
3398                 lio->linfo.link.u64 = resp->cfg_info.linfo.link.u64;
3399
3400                 lio->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3401
3402                 if (OCTEON_CN23XX_PF(octeon_dev) ||
3403                     OCTEON_CN6XXX(octeon_dev)) {
3404                         lio->dev_capability = NETIF_F_HIGHDMA
3405                                               | NETIF_F_IP_CSUM
3406                                               | NETIF_F_IPV6_CSUM
3407                                               | NETIF_F_SG | NETIF_F_RXCSUM
3408                                               | NETIF_F_GRO
3409                                               | NETIF_F_TSO | NETIF_F_TSO6
3410                                               | NETIF_F_LRO;
3411                 }
3412                 netif_set_gso_max_size(netdev, OCTNIC_GSO_MAX_SIZE);
3413
3414                 /*  Copy of transmit encapsulation capabilities:
3415                  *  TSO, TSO6, Checksums for this device
3416                  */
3417                 lio->enc_dev_capability = NETIF_F_IP_CSUM
3418                                           | NETIF_F_IPV6_CSUM
3419                                           | NETIF_F_GSO_UDP_TUNNEL
3420                                           | NETIF_F_HW_CSUM | NETIF_F_SG
3421                                           | NETIF_F_RXCSUM
3422                                           | NETIF_F_TSO | NETIF_F_TSO6
3423                                           | NETIF_F_LRO;
3424
3425                 netdev->hw_enc_features = (lio->enc_dev_capability &
3426                                            ~NETIF_F_LRO);
3427
3428                 lio->dev_capability |= NETIF_F_GSO_UDP_TUNNEL;
3429
3430                 netdev->vlan_features = lio->dev_capability;
3431                 /* Add any unchangeable hw features */
3432                 lio->dev_capability |=  NETIF_F_HW_VLAN_CTAG_FILTER |
3433                                         NETIF_F_HW_VLAN_CTAG_RX |
3434                                         NETIF_F_HW_VLAN_CTAG_TX;
3435
3436                 netdev->features = (lio->dev_capability & ~NETIF_F_LRO);
3437
3438                 netdev->hw_features = lio->dev_capability;
3439                 /*HW_VLAN_RX and HW_VLAN_FILTER is always on*/
3440                 netdev->hw_features = netdev->hw_features &
3441                         ~NETIF_F_HW_VLAN_CTAG_RX;
3442
3443                 /* Point to the  properties for octeon device to which this
3444                  * interface belongs.
3445                  */
3446                 lio->oct_dev = octeon_dev;
3447                 lio->octprops = props;
3448                 lio->netdev = netdev;
3449
3450                 dev_dbg(&octeon_dev->pci_dev->dev,
3451                         "if%d gmx: %d hw_addr: 0x%llx\n", i,
3452                         lio->linfo.gmxport, CVM_CAST64(lio->linfo.hw_addr));
3453
3454                 /* 64-bit swap required on LE machines */
3455                 octeon_swap_8B_data(&lio->linfo.hw_addr, 1);
3456                 for (j = 0; j < 6; j++)
3457                         mac[j] = *((u8 *)(((u8 *)&lio->linfo.hw_addr) + 2 + j));
3458
3459                 /* Copy MAC Address to OS network device structure */
3460
3461                 ether_addr_copy(netdev->dev_addr, mac);
3462
3463                 /* By default all interfaces on a single Octeon uses the same
3464                  * tx and rx queues
3465                  */
3466                 lio->txq = lio->linfo.txpciq[0].s.q_no;
3467                 lio->rxq = lio->linfo.rxpciq[0].s.q_no;
3468                 if (setup_io_queues(octeon_dev, i)) {
3469                         dev_err(&octeon_dev->pci_dev->dev, "I/O queues creation failed\n");
3470                         goto setup_nic_dev_fail;
3471                 }
3472
3473                 ifstate_set(lio, LIO_IFSTATE_DROQ_OPS);
3474
3475                 lio->tx_qsize = octeon_get_tx_qsize(octeon_dev, lio->txq);
3476                 lio->rx_qsize = octeon_get_rx_qsize(octeon_dev, lio->rxq);
3477
3478                 if (setup_glists(octeon_dev, lio, num_iqueues)) {
3479                         dev_err(&octeon_dev->pci_dev->dev,
3480                                 "Gather list allocation failed\n");
3481                         goto setup_nic_dev_fail;
3482                 }
3483
3484                 /* Register ethtool support */
3485                 liquidio_set_ethtool_ops(netdev);
3486                 octeon_dev->priv_flags = 0x0;
3487
3488                 if (netdev->features & NETIF_F_LRO)
3489                         liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
3490                                              OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
3491
3492                 liquidio_set_feature(netdev, OCTNET_CMD_ENABLE_VLAN_FILTER, 0);
3493
3494                 if ((debug != -1) && (debug & NETIF_MSG_HW))
3495                         liquidio_set_feature(netdev,
3496                                              OCTNET_CMD_VERBOSE_ENABLE, 0);
3497
3498                 /* Register the network device with the OS */
3499                 if (register_netdev(netdev)) {
3500                         dev_err(&octeon_dev->pci_dev->dev, "Device registration failed\n");
3501                         goto setup_nic_dev_fail;
3502                 }
3503
3504                 dev_dbg(&octeon_dev->pci_dev->dev,
3505                         "Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n",
3506                         i, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
3507                 netif_carrier_off(netdev);
3508                 lio->link_changes++;
3509
3510                 ifstate_set(lio, LIO_IFSTATE_REGISTERED);
3511
3512                 /* Sending command to firmware to enable Rx checksum offload
3513                  * by default at the time of setup of Liquidio driver for
3514                  * this device
3515                  */
3516                 liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
3517                                             OCTNET_CMD_RXCSUM_ENABLE);
3518                 liquidio_set_feature(netdev, OCTNET_CMD_TNL_TX_CSUM_CTL,
3519                                      OCTNET_CMD_TXCSUM_ENABLE);
3520
3521                 dev_dbg(&octeon_dev->pci_dev->dev,
3522                         "NIC ifidx:%d Setup successful\n", i);
3523
3524                 octeon_free_soft_command(octeon_dev, sc);
3525         }
3526
3527         return 0;
3528
3529 setup_nic_dev_fail:
3530
3531         octeon_free_soft_command(octeon_dev, sc);
3532
3533         while (i--) {
3534                 dev_err(&octeon_dev->pci_dev->dev,
3535                         "NIC ifidx:%d Setup failed\n", i);
3536                 liquidio_destroy_nic_device(octeon_dev, i);
3537         }
3538         return -ENODEV;
3539 }
3540
3541 /**
3542  * \brief initialize the NIC
3543  * @param oct octeon device
3544  *
3545  * This initialization routine is called once the Octeon device application is
3546  * up and running
3547  */
3548 static int liquidio_init_nic_module(struct octeon_device *oct)
3549 {
3550         struct oct_intrmod_cfg *intrmod_cfg;
3551         int i, retval = 0;
3552         int num_nic_ports = CFG_GET_NUM_NIC_PORTS(octeon_get_conf(oct));
3553
3554         dev_dbg(&oct->pci_dev->dev, "Initializing network interfaces\n");
3555
3556         /* only default iq and oq were initialized
3557          * initialize the rest as well
3558          */
3559         /* run port_config command for each port */
3560         oct->ifcount = num_nic_ports;
3561
3562         memset(oct->props, 0,
3563                sizeof(struct octdev_props) * num_nic_ports);
3564
3565         for (i = 0; i < MAX_OCTEON_LINKS; i++)
3566                 oct->props[i].gmxport = -1;
3567
3568         retval = setup_nic_devices(oct);
3569         if (retval) {
3570                 dev_err(&oct->pci_dev->dev, "Setup NIC devices failed\n");
3571                 goto octnet_init_failure;
3572         }
3573
3574         liquidio_ptp_init(oct);
3575
3576         /* Initialize interrupt moderation params */
3577         intrmod_cfg = &((struct octeon_device *)oct)->intrmod;
3578         intrmod_cfg->rx_enable = 1;
3579         intrmod_cfg->check_intrvl =   LIO_INTRMOD_CHECK_INTERVAL;
3580         intrmod_cfg->maxpkt_ratethr = LIO_INTRMOD_MAXPKT_RATETHR;
3581         intrmod_cfg->minpkt_ratethr = LIO_INTRMOD_MINPKT_RATETHR;
3582         intrmod_cfg->rx_maxcnt_trigger = LIO_INTRMOD_RXMAXCNT_TRIGGER;
3583         intrmod_cfg->rx_maxtmr_trigger = LIO_INTRMOD_RXMAXTMR_TRIGGER;
3584         intrmod_cfg->rx_mintmr_trigger = LIO_INTRMOD_RXMINTMR_TRIGGER;
3585         intrmod_cfg->rx_mincnt_trigger = LIO_INTRMOD_RXMINCNT_TRIGGER;
3586         intrmod_cfg->tx_enable = 1;
3587         intrmod_cfg->tx_maxcnt_trigger = LIO_INTRMOD_TXMAXCNT_TRIGGER;
3588         intrmod_cfg->tx_mincnt_trigger = LIO_INTRMOD_TXMINCNT_TRIGGER;
3589         intrmod_cfg->rx_frames = CFG_GET_OQ_INTR_PKT(octeon_get_conf(oct));
3590         intrmod_cfg->rx_usecs = CFG_GET_OQ_INTR_TIME(octeon_get_conf(oct));
3591         dev_dbg(&oct->pci_dev->dev, "Network interfaces ready\n");
3592
3593         return retval;
3594
3595 octnet_init_failure:
3596
3597         oct->ifcount = 0;
3598
3599         return retval;
3600 }
3601
3602 /**
3603  * \brief starter callback that invokes the remaining initialization work after
3604  * the NIC is up and running.
3605  * @param octptr  work struct work_struct
3606  */
3607 static void nic_starter(struct work_struct *work)
3608 {
3609         struct octeon_device *oct;
3610         struct cavium_wk *wk = (struct cavium_wk *)work;
3611
3612         oct = (struct octeon_device *)wk->ctxptr;
3613
3614         if (atomic_read(&oct->status) == OCT_DEV_RUNNING)
3615                 return;
3616
3617         /* If the status of the device is CORE_OK, the core
3618          * application has reported its application type. Call
3619          * any registered handlers now and move to the RUNNING
3620          * state.
3621          */
3622         if (atomic_read(&oct->status) != OCT_DEV_CORE_OK) {
3623                 schedule_delayed_work(&oct->nic_poll_work.work,
3624                                       LIQUIDIO_STARTER_POLL_INTERVAL_MS);
3625                 return;
3626         }
3627
3628         atomic_set(&oct->status, OCT_DEV_RUNNING);
3629
3630         if (oct->app_mode && oct->app_mode == CVM_DRV_NIC_APP) {
3631                 dev_dbg(&oct->pci_dev->dev, "Starting NIC module\n");
3632
3633                 if (liquidio_init_nic_module(oct))
3634                         dev_err(&oct->pci_dev->dev, "NIC initialization failed\n");
3635                 else
3636                         handshake[oct->octeon_id].started_ok = 1;
3637         } else {
3638                 dev_err(&oct->pci_dev->dev,
3639                         "Unexpected application running on NIC (%d). Check firmware.\n",
3640                         oct->app_mode);
3641         }
3642
3643         complete(&handshake[oct->octeon_id].started);
3644 }
3645
3646 /**
3647  * \brief Device initialization for each Octeon device that is probed
3648  * @param octeon_dev  octeon device
3649  */
3650 static int octeon_device_init(struct octeon_device *octeon_dev)
3651 {
3652         int j, ret;
3653         char bootcmd[] = "\n";
3654         struct octeon_device_priv *oct_priv =
3655                 (struct octeon_device_priv *)octeon_dev->priv;
3656         atomic_set(&octeon_dev->status, OCT_DEV_BEGIN_STATE);
3657
3658         /* Enable access to the octeon device and make its DMA capability
3659          * known to the OS.
3660          */
3661         if (octeon_pci_os_setup(octeon_dev))
3662                 return 1;
3663
3664         /* Identify the Octeon type and map the BAR address space. */
3665         if (octeon_chip_specific_setup(octeon_dev)) {
3666                 dev_err(&octeon_dev->pci_dev->dev, "Chip specific setup failed\n");
3667                 return 1;
3668         }
3669
3670         atomic_set(&octeon_dev->status, OCT_DEV_PCI_MAP_DONE);
3671
3672         octeon_dev->app_mode = CVM_DRV_INVALID_APP;
3673
3674         /* Do a soft reset of the Octeon device. */
3675         if (octeon_dev->fn_list.soft_reset(octeon_dev))
3676                 return 1;
3677
3678         /* Initialize the dispatch mechanism used to push packets arriving on
3679          * Octeon Output queues.
3680          */
3681         if (octeon_init_dispatch_list(octeon_dev))
3682                 return 1;
3683
3684         octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
3685                                     OPCODE_NIC_CORE_DRV_ACTIVE,
3686                                     octeon_core_drv_init,
3687                                     octeon_dev);
3688
3689         INIT_DELAYED_WORK(&octeon_dev->nic_poll_work.work, nic_starter);
3690         octeon_dev->nic_poll_work.ctxptr = (void *)octeon_dev;
3691         schedule_delayed_work(&octeon_dev->nic_poll_work.work,
3692                               LIQUIDIO_STARTER_POLL_INTERVAL_MS);
3693
3694         atomic_set(&octeon_dev->status, OCT_DEV_DISPATCH_INIT_DONE);
3695
3696         octeon_set_io_queues_off(octeon_dev);
3697
3698         /*  Setup the data structures that manage this Octeon's Input queues. */
3699         if (octeon_setup_instr_queues(octeon_dev)) {
3700                 dev_err(&octeon_dev->pci_dev->dev,
3701                         "instruction queue initialization failed\n");
3702                 /* On error, release any previously allocated queues */
3703                 for (j = 0; j < octeon_dev->num_iqs; j++)
3704                         octeon_delete_instr_queue(octeon_dev, j);
3705                 return 1;
3706         }
3707         atomic_set(&octeon_dev->status, OCT_DEV_INSTR_QUEUE_INIT_DONE);
3708
3709         /* Initialize soft command buffer pool
3710          */
3711         if (octeon_setup_sc_buffer_pool(octeon_dev)) {
3712                 dev_err(&octeon_dev->pci_dev->dev, "sc buffer pool allocation failed\n");
3713                 return 1;
3714         }
3715         atomic_set(&octeon_dev->status, OCT_DEV_SC_BUFF_POOL_INIT_DONE);
3716
3717         /* Initialize lists to manage the requests of different types that
3718          * arrive from user & kernel applications for this octeon device.
3719          */
3720         if (octeon_setup_response_list(octeon_dev)) {
3721                 dev_err(&octeon_dev->pci_dev->dev, "Response list allocation failed\n");
3722                 return 1;
3723         }
3724         atomic_set(&octeon_dev->status, OCT_DEV_RESP_LIST_INIT_DONE);
3725
3726         if (octeon_setup_output_queues(octeon_dev)) {
3727                 dev_err(&octeon_dev->pci_dev->dev, "Output queue initialization failed\n");
3728                 /* Release any previously allocated queues */
3729                 for (j = 0; j < octeon_dev->num_oqs; j++)
3730                         octeon_delete_droq(octeon_dev, j);
3731                 return 1;
3732         }
3733
3734         atomic_set(&octeon_dev->status, OCT_DEV_DROQ_INIT_DONE);
3735
3736         /* The input and output queue registers were setup earlier (the queues
3737          * were not enabled). Any additional registers that need to be
3738          * programmed should be done now.
3739          */
3740         ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
3741         if (ret) {
3742                 dev_err(&octeon_dev->pci_dev->dev,
3743                         "Failed to configure device registers\n");
3744                 return ret;
3745         }
3746
3747         /* Initialize the tasklet that handles output queue packet processing.*/
3748         dev_dbg(&octeon_dev->pci_dev->dev, "Initializing droq tasklet\n");
3749         tasklet_init(&oct_priv->droq_tasklet, octeon_droq_bh,
3750                      (unsigned long)octeon_dev);
3751
3752         /* Setup the interrupt handler and record the INT SUM register address
3753          */
3754         if (octeon_setup_interrupt(octeon_dev))
3755                 return 1;
3756
3757         /* Enable Octeon device interrupts */
3758         octeon_dev->fn_list.enable_interrupt(octeon_dev->chip);
3759
3760         /* Enable the input and output queues for this Octeon device */
3761         octeon_dev->fn_list.enable_io_queues(octeon_dev);
3762
3763         atomic_set(&octeon_dev->status, OCT_DEV_IO_QUEUES_DONE);
3764
3765         dev_dbg(&octeon_dev->pci_dev->dev, "Waiting for DDR initialization...\n");
3766
3767         if (ddr_timeout == 0)
3768                 dev_info(&octeon_dev->pci_dev->dev, "WAITING. Set ddr_timeout to non-zero value to proceed with initialization.\n");
3769
3770         schedule_timeout_uninterruptible(HZ * LIO_RESET_SECS);
3771
3772         /* Wait for the octeon to initialize DDR after the soft-reset. */
3773         while (ddr_timeout == 0) {
3774                 set_current_state(TASK_INTERRUPTIBLE);
3775                 if (schedule_timeout(HZ / 10)) {
3776                         /* user probably pressed Control-C */
3777                         return 1;
3778                 }
3779         }
3780         ret = octeon_wait_for_ddr_init(octeon_dev, &ddr_timeout);
3781         if (ret) {
3782                 dev_err(&octeon_dev->pci_dev->dev,
3783                         "DDR not initialized. Please confirm that board is configured to boot from Flash, ret: %d\n",
3784                         ret);
3785                 return 1;
3786         }
3787
3788         if (octeon_wait_for_bootloader(octeon_dev, 1000) != 0) {
3789                 dev_err(&octeon_dev->pci_dev->dev, "Board not responding\n");
3790                 return 1;
3791         }
3792
3793         /* Divert uboot to take commands from host instead. */
3794         ret = octeon_console_send_cmd(octeon_dev, bootcmd, 50);
3795
3796         dev_dbg(&octeon_dev->pci_dev->dev, "Initializing consoles\n");
3797         ret = octeon_init_consoles(octeon_dev);
3798         if (ret) {
3799                 dev_err(&octeon_dev->pci_dev->dev, "Could not access board consoles\n");
3800                 return 1;
3801         }
3802         ret = octeon_add_console(octeon_dev, 0);
3803         if (ret) {
3804                 dev_err(&octeon_dev->pci_dev->dev, "Could not access board console\n");
3805                 return 1;
3806         }
3807
3808         atomic_set(&octeon_dev->status, OCT_DEV_CONSOLE_INIT_DONE);
3809
3810         dev_dbg(&octeon_dev->pci_dev->dev, "Loading firmware\n");
3811         ret = load_firmware(octeon_dev);
3812         if (ret) {
3813                 dev_err(&octeon_dev->pci_dev->dev, "Could not load firmware to board\n");
3814                 return 1;
3815         }
3816
3817         handshake[octeon_dev->octeon_id].init_ok = 1;
3818         complete(&handshake[octeon_dev->octeon_id].init);
3819
3820         atomic_set(&octeon_dev->status, OCT_DEV_HOST_OK);
3821
3822         /* Send Credit for Octeon Output queues. Credits are always sent after
3823          * the output queue is enabled.
3824          */
3825         for (j = 0; j < octeon_dev->num_oqs; j++)
3826                 writel(octeon_dev->droq[j]->max_count,
3827                        octeon_dev->droq[j]->pkts_credit_reg);
3828
3829         /* Packets can start arriving on the output queues from this point. */
3830
3831         return 0;
3832 }
3833
3834 /**
3835  * \brief Exits the module
3836  */
3837 static void __exit liquidio_exit(void)
3838 {
3839         liquidio_deinit_pci();
3840
3841         pr_info("LiquidIO network module is now unloaded\n");
3842 }
3843
3844 module_init(liquidio_init);
3845 module_exit(liquidio_exit);