drivers/net/ethernet/cavium/liquidio/lio_main.c

   1 /**********************************************************************
   2  * Author: Cavium, Inc.
   3  *
   4  * Contact: support@cavium.com
   5  *          Please include "LiquidIO" in the subject.
   6  *
   7  * Copyright (c) 2003-2016 Cavium, Inc.
   8  *
   9  * This file is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License, Version 2, as
  11  * published by the Free Software Foundation.
  12  *
  13  * This file is distributed in the hope that it will be useful, but
  14  * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
  15  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
  16  * NONINFRINGEMENT.  See the GNU General Public License for more details.
  17  ***********************************************************************/
  18 #include <linux/module.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/pci.h>
  21 #include <linux/firmware.h>
  22 #include <net/vxlan.h>
  23 #include <linux/kthread.h>
  24 #include "liquidio_common.h"
  25 #include "octeon_droq.h"
  26 #include "octeon_iq.h"
  27 #include "response_manager.h"
  28 #include "octeon_device.h"
  29 #include "octeon_nic.h"
  30 #include "octeon_main.h"
  31 #include "octeon_network.h"
  32 #include "cn66xx_regs.h"
  33 #include "cn66xx_device.h"
  34 #include "cn68xx_device.h"
  35 #include "cn23xx_pf_device.h"
  36 #include "liquidio_image.h"
  37 #include "lio_vf_rep.h"
  38
  39 MODULE_AUTHOR("Cavium Networks, <support@cavium.com>");
  40 MODULE_DESCRIPTION("Cavium LiquidIO Intelligent Server Adapter Driver");
  41 MODULE_LICENSE("GPL");
  42 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210SV_NAME
  43                 "_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
  44 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210NV_NAME
  45                 "_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
  46 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_410NV_NAME
  47                 "_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
  48 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_23XX_NAME
  49                 "_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
  50
  51 static int ddr_timeout = 10000;
  52 module_param(ddr_timeout, int, 0644);
  53 MODULE_PARM_DESC(ddr_timeout,
  54                  "Number of milliseconds to wait for DDR initialization. 0 waits for ddr_timeout to be set to non-zero value before starting to check");
  55
  56 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
  57
  58 static int debug = -1;
  59 module_param(debug, int, 0644);
  60 MODULE_PARM_DESC(debug, "NETIF_MSG debug bits");
  61
  62 static char fw_type[LIO_MAX_FW_TYPE_LEN] = LIO_FW_NAME_TYPE_AUTO;
  63 module_param_string(fw_type, fw_type, sizeof(fw_type), 0444);
  64 MODULE_PARM_DESC(fw_type, "Type of firmware to be loaded (default is \"auto\"), which uses firmware in flash, if present, else loads \"nic\".");
  65
  66 static u32 console_bitmask;
  67 module_param(console_bitmask, int, 0644);
  68 MODULE_PARM_DESC(console_bitmask,
  69                  "Bitmask indicating which consoles have debug output redirected to syslog.");
  70
  71 /**
  72  * \brief determines if a given console has debug enabled.
  73  * @param console console to check
  74  * @returns  1 = enabled. 0 otherwise
  75  */
  76 static int octeon_console_debug_enabled(u32 console)
  77 {
  78         return (console_bitmask >> (console)) & 0x1;
  79 }
  80
  81 /* Polling interval for determining when NIC application is alive */
  82 #define LIQUIDIO_STARTER_POLL_INTERVAL_MS 100
  83
  84 /* runtime link query interval */
  85 #define LIQUIDIO_LINK_QUERY_INTERVAL_MS         1000
  86 /* update localtime to octeon firmware every 60 seconds.
  87  * make firmware to use same time reference, so that it will be easy to
  88  * correlate firmware logged events/errors with host events, for debugging.
  89  */
  90 #define LIO_SYNC_OCTEON_TIME_INTERVAL_MS 60000
  91
  92 /* time to wait for possible in-flight requests in milliseconds */
  93 #define WAIT_INFLIGHT_REQUEST   msecs_to_jiffies(1000)
  94
  95 struct lio_trusted_vf_ctx {
  96         struct completion complete;
  97         int status;
  98 };
  99
 100 struct oct_link_status_resp {
 101         u64 rh;
 102         struct oct_link_info link_info;
 103         u64 status;
 104 };
 105
 106 struct oct_timestamp_resp {
 107         u64 rh;
 108         u64 timestamp;
 109         u64 status;
 110 };
 111
 112 #define OCT_TIMESTAMP_RESP_SIZE (sizeof(struct oct_timestamp_resp))
 113
 114 union tx_info {
 115         u64 u64;
 116         struct {
 117 #ifdef __BIG_ENDIAN_BITFIELD
 118                 u16 gso_size;
 119                 u16 gso_segs;
 120                 u32 reserved;
 121 #else
 122                 u32 reserved;
 123                 u16 gso_segs;
 124                 u16 gso_size;
 125 #endif
 126         } s;
 127 };
 128
 129 /** Octeon device properties to be used by the NIC module.
 130  * Each octeon device in the system will be represented
 131  * by this structure in the NIC module.
 132  */
 133
 134 #define OCTNIC_GSO_MAX_HEADER_SIZE 128
 135 #define OCTNIC_GSO_MAX_SIZE                                                    \
 136         (CN23XX_DEFAULT_INPUT_JABBER - OCTNIC_GSO_MAX_HEADER_SIZE)
 137
 138 struct handshake {
 139         struct completion init;
 140         struct completion started;
 141         struct pci_dev *pci_dev;
 142         int init_ok;
 143         int started_ok;
 144 };
 145
 146 #ifdef CONFIG_PCI_IOV
 147 static int liquidio_enable_sriov(struct pci_dev *dev, int num_vfs);
 148 #endif
 149
 150 static int octeon_dbg_console_print(struct octeon_device *oct, u32 console_num,
 151                                     char *prefix, char *suffix);
 152
 153 static int octeon_device_init(struct octeon_device *);
 154 static int liquidio_stop(struct net_device *netdev);
 155 static void liquidio_remove(struct pci_dev *pdev);
 156 static int liquidio_probe(struct pci_dev *pdev,
 157                           const struct pci_device_id *ent);
 158 static int liquidio_set_vf_link_state(struct net_device *netdev, int vfidx,
 159                                       int linkstate);
 160
 161 static struct handshake handshake[MAX_OCTEON_DEVICES];
 162 static struct completion first_stage;
 163
 164 static void octeon_droq_bh(unsigned long pdev)
 165 {
 166         int q_no;
 167         int reschedule = 0;
 168         struct octeon_device *oct = (struct octeon_device *)pdev;
 169         struct octeon_device_priv *oct_priv =
 170                 (struct octeon_device_priv *)oct->priv;
 171
 172         for (q_no = 0; q_no < MAX_OCTEON_OUTPUT_QUEUES(oct); q_no++) {
 173                 if (!(oct->io_qmask.oq & BIT_ULL(q_no)))
 174                         continue;
 175                 reschedule |= octeon_droq_process_packets(oct, oct->droq[q_no],
 176                                                           MAX_PACKET_BUDGET);
 177                 lio_enable_irq(oct->droq[q_no], NULL);
 178
 179                 if (OCTEON_CN23XX_PF(oct) && oct->msix_on) {
 180                         /* set time and cnt interrupt thresholds for this DROQ
 181                          * for NAPI
 182                          */
 183                         int adjusted_q_no = q_no + oct->sriov_info.pf_srn;
 184
 185                         octeon_write_csr64(
 186                             oct, CN23XX_SLI_OQ_PKT_INT_LEVELS(adjusted_q_no),
 187                             0x5700000040ULL);
 188                         octeon_write_csr64(
 189                             oct, CN23XX_SLI_OQ_PKTS_SENT(adjusted_q_no), 0);
 190                 }
 191         }
 192
 193         if (reschedule)
 194                 tasklet_schedule(&oct_priv->droq_tasklet);
 195 }
 196
 197 static int lio_wait_for_oq_pkts(struct octeon_device *oct)
 198 {
 199         struct octeon_device_priv *oct_priv =
 200                 (struct octeon_device_priv *)oct->priv;
 201         int retry = 100, pkt_cnt = 0, pending_pkts = 0;
 202         int i;
 203
 204         do {
 205                 pending_pkts = 0;
 206
 207                 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
 208                         if (!(oct->io_qmask.oq & BIT_ULL(i)))
 209                                 continue;
 210                         pkt_cnt += octeon_droq_check_hw_for_pkts(oct->droq[i]);
 211                 }
 212                 if (pkt_cnt > 0) {
 213                         pending_pkts += pkt_cnt;
 214                         tasklet_schedule(&oct_priv->droq_tasklet);
 215                 }
 216                 pkt_cnt = 0;
 217                 schedule_timeout_uninterruptible(1);
 218
 219         } while (retry-- && pending_pkts);
 220
 221         return pkt_cnt;
 222 }
 223
 224 /**
 225  * \brief Forces all IO queues off on a given device
 226  * @param oct Pointer to Octeon device
 227  */
 228 static void force_io_queues_off(struct octeon_device *oct)
 229 {
 230         if ((oct->chip_id == OCTEON_CN66XX) ||
 231             (oct->chip_id == OCTEON_CN68XX)) {
 232                 /* Reset the Enable bits for Input Queues. */
 233                 octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, 0);
 234
 235                 /* Reset the Enable bits for Output Queues. */
 236                 octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, 0);
 237         }
 238 }
 239
 240 /**
 241  * \brief Cause device to go quiet so it can be safely removed/reset/etc
 242  * @param oct Pointer to Octeon device
 243  */
 244 static inline void pcierror_quiesce_device(struct octeon_device *oct)
 245 {
 246         int i;
 247
 248         /* Disable the input and output queues now. No more packets will
 249          * arrive from Octeon, but we should wait for all packet processing
 250          * to finish.
 251          */
 252         force_io_queues_off(oct);
 253
 254         /* To allow for in-flight requests */
 255         schedule_timeout_uninterruptible(WAIT_INFLIGHT_REQUEST);
 256
 257         if (wait_for_pending_requests(oct))
 258                 dev_err(&oct->pci_dev->dev, "There were pending requests\n");
 259
 260         /* Force all requests waiting to be fetched by OCTEON to complete. */
 261         for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
 262                 struct octeon_instr_queue *iq;
 263
 264                 if (!(oct->io_qmask.iq & BIT_ULL(i)))
 265                         continue;
 266                 iq = oct->instr_queue[i];
 267
 268                 if (atomic_read(&iq->instr_pending)) {
 269                         spin_lock_bh(&iq->lock);
 270                         iq->fill_cnt = 0;
 271                         iq->octeon_read_index = iq->host_write_index;
 272                         iq->stats.instr_processed +=
 273                                 atomic_read(&iq->instr_pending);
 274                         lio_process_iq_request_list(oct, iq, 0);
 275                         spin_unlock_bh(&iq->lock);
 276                 }
 277         }
 278
 279         /* Force all pending ordered list requests to time out. */
 280         lio_process_ordered_list(oct, 1);
 281
 282         /* We do not need to wait for output queue packets to be processed. */
 283 }
 284
 285 /**
 286  * \brief Cleanup PCI AER uncorrectable error status
 287  * @param dev Pointer to PCI device
 288  */
 289 static void cleanup_aer_uncorrect_error_status(struct pci_dev *dev)
 290 {
 291         int pos = 0x100;
 292         u32 status, mask;
 293
 294         pr_info("%s :\n", __func__);
 295
 296         pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
 297         pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask);
 298         if (dev->error_state == pci_channel_io_normal)
 299                 status &= ~mask;        /* Clear corresponding nonfatal bits */
 300         else
 301                 status &= mask;         /* Clear corresponding fatal bits */
 302         pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);
 303 }
 304
 305 /**
 306  * \brief Stop all PCI IO to a given device
 307  * @param dev Pointer to Octeon device
 308  */
 309 static void stop_pci_io(struct octeon_device *oct)
 310 {
 311         /* No more instructions will be forwarded. */
 312         atomic_set(&oct->status, OCT_DEV_IN_RESET);
 313
 314         pci_disable_device(oct->pci_dev);
 315
 316         /* Disable interrupts  */
 317         oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
 318
 319         pcierror_quiesce_device(oct);
 320
 321         /* Release the interrupt line */
 322         free_irq(oct->pci_dev->irq, oct);
 323
 324         if (oct->flags & LIO_FLAG_MSI_ENABLED)
 325                 pci_disable_msi(oct->pci_dev);
 326
 327         dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
 328                 lio_get_state_string(&oct->status));
 329
 330         /* making it a common function for all OCTEON models */
 331         cleanup_aer_uncorrect_error_status(oct->pci_dev);
 332 }
 333
 334 /**
 335  * \brief called when PCI error is detected
 336  * @param pdev Pointer to PCI device
 337  * @param state The current pci connection state
 338  *
 339  * This function is called after a PCI bus error affecting
 340  * this device has been detected.
 341  */
 342 static pci_ers_result_t liquidio_pcie_error_detected(struct pci_dev *pdev,
 343                                                      pci_channel_state_t state)
 344 {
 345         struct octeon_device *oct = pci_get_drvdata(pdev);
 346
 347         /* Non-correctable Non-fatal errors */
 348         if (state == pci_channel_io_normal) {
 349                 dev_err(&oct->pci_dev->dev, "Non-correctable non-fatal error reported:\n");
 350                 cleanup_aer_uncorrect_error_status(oct->pci_dev);
 351                 return PCI_ERS_RESULT_CAN_RECOVER;
 352         }
 353
 354         /* Non-correctable Fatal errors */
 355         dev_err(&oct->pci_dev->dev, "Non-correctable FATAL reported by PCI AER driver\n");
 356         stop_pci_io(oct);
 357
 358         /* Always return a DISCONNECT. There is no support for recovery but only
 359          * for a clean shutdown.
 360          */
 361         return PCI_ERS_RESULT_DISCONNECT;
 362 }
 363
 364 /**
 365  * \brief mmio handler
 366  * @param pdev Pointer to PCI device
 367  */
 368 static pci_ers_result_t liquidio_pcie_mmio_enabled(
 369                                 struct pci_dev *pdev __attribute__((unused)))
 370 {
 371         /* We should never hit this since we never ask for a reset for a Fatal
 372          * Error. We always return DISCONNECT in io_error above.
 373          * But play safe and return RECOVERED for now.
 374          */
 375         return PCI_ERS_RESULT_RECOVERED;
 376 }
 377
 378 /**
 379  * \brief called after the pci bus has been reset.
 380  * @param pdev Pointer to PCI device
 381  *
 382  * Restart the card from scratch, as if from a cold-boot. Implementation
 383  * resembles the first-half of the octeon_resume routine.
 384  */
 385 static pci_ers_result_t liquidio_pcie_slot_reset(
 386                                 struct pci_dev *pdev __attribute__((unused)))
 387 {
 388         /* We should never hit this since we never ask for a reset for a Fatal
 389          * Error. We always return DISCONNECT in io_error above.
 390          * But play safe and return RECOVERED for now.
 391          */
 392         return PCI_ERS_RESULT_RECOVERED;
 393 }
 394
 395 /**
 396  * \brief called when traffic can start flowing again.
 397  * @param pdev Pointer to PCI device
 398  *
 399  * This callback is called when the error recovery driver tells us that
 400  * its OK to resume normal operation. Implementation resembles the
 401  * second-half of the octeon_resume routine.
 402  */
 403 static void liquidio_pcie_resume(struct pci_dev *pdev __attribute__((unused)))
 404 {
 405         /* Nothing to be done here. */
 406 }
 407
 408 #define liquidio_suspend NULL
 409 #define liquidio_resume NULL
 410
 411 /* For PCI-E Advanced Error Recovery (AER) Interface */
 412 static const struct pci_error_handlers liquidio_err_handler = {
 413         .error_detected = liquidio_pcie_error_detected,
 414         .mmio_enabled   = liquidio_pcie_mmio_enabled,
 415         .slot_reset     = liquidio_pcie_slot_reset,
 416         .resume         = liquidio_pcie_resume,
 417 };
 418
 419 static const struct pci_device_id liquidio_pci_tbl[] = {
 420         {       /* 68xx */
 421                 PCI_VENDOR_ID_CAVIUM, 0x91, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
 422         },
 423         {       /* 66xx */
 424                 PCI_VENDOR_ID_CAVIUM, 0x92, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
 425         },
 426         {       /* 23xx pf */
 427                 PCI_VENDOR_ID_CAVIUM, 0x9702, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
 428         },
 429         {
 430                 0, 0, 0, 0, 0, 0, 0
 431         }
 432 };
 433 MODULE_DEVICE_TABLE(pci, liquidio_pci_tbl);
 434
 435 static SIMPLE_DEV_PM_OPS(liquidio_pm_ops, liquidio_suspend, liquidio_resume);
 436
 437 static struct pci_driver liquidio_pci_driver = {
 438         .name           = "LiquidIO",
 439         .id_table       = liquidio_pci_tbl,
 440         .probe          = liquidio_probe,
 441         .remove         = liquidio_remove,
 442         .err_handler    = &liquidio_err_handler,    /* For AER */
 443         .driver.pm      = &liquidio_pm_ops,
 444 #ifdef CONFIG_PCI_IOV
 445         .sriov_configure = liquidio_enable_sriov,
 446 #endif
 447 };
 448
 449 /**
 450  * \brief register PCI driver
 451  */
 452 static int liquidio_init_pci(void)
 453 {
 454         return pci_register_driver(&liquidio_pci_driver);
 455 }
 456
 457 /**
 458  * \brief unregister PCI driver
 459  */
 460 static void liquidio_deinit_pci(void)
 461 {
 462         pci_unregister_driver(&liquidio_pci_driver);
 463 }
 464
 465 /**
 466  * \brief Check Tx queue status, and take appropriate action
 467  * @param lio per-network private data
 468  * @returns 0 if full, number of queues woken up otherwise
 469  */
 470 static inline int check_txq_status(struct lio *lio)
 471 {
 472         int numqs = lio->netdev->real_num_tx_queues;
 473         int ret_val = 0;
 474         int q, iq;
 475
 476         /* check each sub-queue state */
 477         for (q = 0; q < numqs; q++) {
 478                 iq = lio->linfo.txpciq[q %
 479                         lio->oct_dev->num_iqs].s.q_no;
 480                 if (octnet_iq_is_full(lio->oct_dev, iq))
 481                         continue;
 482                 if (__netif_subqueue_stopped(lio->netdev, q)) {
 483                         netif_wake_subqueue(lio->netdev, q);
 484                         INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq,
 485                                                   tx_restart, 1);
 486                         ret_val++;
 487                 }
 488         }
 489
 490         return ret_val;
 491 }
 492
 493 /**
 494  * \brief Print link information
 495  * @param netdev network device
 496  */
 497 static void print_link_info(struct net_device *netdev)
 498 {
 499         struct lio *lio = GET_LIO(netdev);
 500
 501         if (!ifstate_check(lio, LIO_IFSTATE_RESETTING) &&
 502             ifstate_check(lio, LIO_IFSTATE_REGISTERED)) {
 503                 struct oct_link_info *linfo = &lio->linfo;
 504
 505                 if (linfo->link.s.link_up) {
 506                         netif_info(lio, link, lio->netdev, "%d Mbps %s Duplex UP\n",
 507                                    linfo->link.s.speed,
 508                                    (linfo->link.s.duplex) ? "Full" : "Half");
 509                 } else {
 510                         netif_info(lio, link, lio->netdev, "Link Down\n");
 511                 }
 512         }
 513 }
 514
 515 /**
 516  * \brief Routine to notify MTU change
 517  * @param work work_struct data structure
 518  */
 519 static void octnet_link_status_change(struct work_struct *work)
 520 {
 521         struct cavium_wk *wk = (struct cavium_wk *)work;
 522         struct lio *lio = (struct lio *)wk->ctxptr;
 523
 524         /* lio->linfo.link.s.mtu always contains max MTU of the lio interface.
 525          * this API is invoked only when new max-MTU of the interface is
 526          * less than current MTU.
 527          */
 528         rtnl_lock();
 529         dev_set_mtu(lio->netdev, lio->linfo.link.s.mtu);
 530         rtnl_unlock();
 531 }
 532
 533 /**
 534  * \brief Sets up the mtu status change work
 535  * @param netdev network device
 536  */
 537 static inline int setup_link_status_change_wq(struct net_device *netdev)
 538 {
 539         struct lio *lio = GET_LIO(netdev);
 540         struct octeon_device *oct = lio->oct_dev;
 541
 542         lio->link_status_wq.wq = alloc_workqueue("link-status",
 543                                                  WQ_MEM_RECLAIM, 0);
 544         if (!lio->link_status_wq.wq) {
 545                 dev_err(&oct->pci_dev->dev, "unable to create cavium link status wq\n");
 546                 return -1;
 547         }
 548         INIT_DELAYED_WORK(&lio->link_status_wq.wk.work,
 549                           octnet_link_status_change);
 550         lio->link_status_wq.wk.ctxptr = lio;
 551
 552         return 0;
 553 }
 554
 555 static inline void cleanup_link_status_change_wq(struct net_device *netdev)
 556 {
 557         struct lio *lio = GET_LIO(netdev);
 558
 559         if (lio->link_status_wq.wq) {
 560                 cancel_delayed_work_sync(&lio->link_status_wq.wk.work);
 561                 destroy_workqueue(lio->link_status_wq.wq);
 562         }
 563 }
 564
 565 /**
 566  * \brief Update link status
 567  * @param netdev network device
 568  * @param ls link status structure
 569  *
 570  * Called on receipt of a link status response from the core application to
 571  * update each interface's link status.
 572  */
 573 static inline void update_link_status(struct net_device *netdev,
 574                                       union oct_link_status *ls)
 575 {
 576         struct lio *lio = GET_LIO(netdev);
 577         int changed = (lio->linfo.link.u64 != ls->u64);
 578         int current_max_mtu = lio->linfo.link.s.mtu;
 579         struct octeon_device *oct = lio->oct_dev;
 580
 581         dev_dbg(&oct->pci_dev->dev, "%s: lio->linfo.link.u64=%llx, ls->u64=%llx\n",
 582                 __func__, lio->linfo.link.u64, ls->u64);
 583         lio->linfo.link.u64 = ls->u64;
 584
 585         if ((lio->intf_open) && (changed)) {
 586                 print_link_info(netdev);
 587                 lio->link_changes++;
 588
 589                 if (lio->linfo.link.s.link_up) {
 590                         dev_dbg(&oct->pci_dev->dev, "%s: link_up", __func__);
 591                         netif_carrier_on(netdev);
 592                         wake_txqs(netdev);
 593                 } else {
 594                         dev_dbg(&oct->pci_dev->dev, "%s: link_off", __func__);
 595                         netif_carrier_off(netdev);
 596                         stop_txqs(netdev);
 597                 }
 598                 if (lio->linfo.link.s.mtu != current_max_mtu) {
 599                         netif_info(lio, probe, lio->netdev, "Max MTU changed from %d to %d\n",
 600                                    current_max_mtu, lio->linfo.link.s.mtu);
 601                         netdev->max_mtu = lio->linfo.link.s.mtu;
 602                 }
 603                 if (lio->linfo.link.s.mtu < netdev->mtu) {
 604                         dev_warn(&oct->pci_dev->dev,
 605                                  "Current MTU is higher than new max MTU; Reducing the current mtu from %d to %d\n",
 606                                      netdev->mtu, lio->linfo.link.s.mtu);
 607                         queue_delayed_work(lio->link_status_wq.wq,
 608                                            &lio->link_status_wq.wk.work, 0);
 609                 }
 610         }
 611 }
 612
 613 /**
 614  * lio_sync_octeon_time - send latest localtime to octeon firmware so that
 615  * firmware will correct it's time, in case there is a time skew
 616  *
 617  * @work: work scheduled to send time update to octeon firmware
 618  **/
 619 static void lio_sync_octeon_time(struct work_struct *work)
 620 {
 621         struct cavium_wk *wk = (struct cavium_wk *)work;
 622         struct lio *lio = (struct lio *)wk->ctxptr;
 623         struct octeon_device *oct = lio->oct_dev;
 624         struct octeon_soft_command *sc;
 625         struct timespec64 ts;
 626         struct lio_time *lt;
 627         int ret;
 628
 629         sc = octeon_alloc_soft_command(oct, sizeof(struct lio_time), 16, 0);
 630         if (!sc) {
 631                 dev_err(&oct->pci_dev->dev,
 632                         "Failed to sync time to octeon: soft command allocation failed\n");
 633                 return;
 634         }
 635
 636         lt = (struct lio_time *)sc->virtdptr;
 637
 638         /* Get time of the day */
 639         ktime_get_real_ts64(&ts);
 640         lt->sec = ts.tv_sec;
 641         lt->nsec = ts.tv_nsec;
 642         octeon_swap_8B_data((u64 *)lt, (sizeof(struct lio_time)) / 8);
 643
 644         sc->iq_no = lio->linfo.txpciq[0].s.q_no;
 645         octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
 646                                     OPCODE_NIC_SYNC_OCTEON_TIME, 0, 0, 0);
 647
 648         init_completion(&sc->complete);
 649         sc->sc_status = OCTEON_REQUEST_PENDING;
 650
 651         ret = octeon_send_soft_command(oct, sc);
 652         if (ret == IQ_SEND_FAILED) {
 653                 dev_err(&oct->pci_dev->dev,
 654                         "Failed to sync time to octeon: failed to send soft command\n");
 655                 octeon_free_soft_command(oct, sc);
 656         } else {
 657                 WRITE_ONCE(sc->caller_is_done, true);
 658         }
 659
 660         queue_delayed_work(lio->sync_octeon_time_wq.wq,
 661                            &lio->sync_octeon_time_wq.wk.work,
 662                            msecs_to_jiffies(LIO_SYNC_OCTEON_TIME_INTERVAL_MS));
 663 }
 664
 665 /**
 666  * setup_sync_octeon_time_wq - Sets up the work to periodically update
 667  * local time to octeon firmware
 668  *
 669  * @netdev - network device which should send time update to firmware
 670  **/
 671 static inline int setup_sync_octeon_time_wq(struct net_device *netdev)
 672 {
 673         struct lio *lio = GET_LIO(netdev);
 674         struct octeon_device *oct = lio->oct_dev;
 675
 676         lio->sync_octeon_time_wq.wq =
 677                 alloc_workqueue("update-octeon-time", WQ_MEM_RECLAIM, 0);
 678         if (!lio->sync_octeon_time_wq.wq) {
 679                 dev_err(&oct->pci_dev->dev, "Unable to create wq to update octeon time\n");
 680                 return -1;
 681         }
 682         INIT_DELAYED_WORK(&lio->sync_octeon_time_wq.wk.work,
 683                           lio_sync_octeon_time);
 684         lio->sync_octeon_time_wq.wk.ctxptr = lio;
 685         queue_delayed_work(lio->sync_octeon_time_wq.wq,
 686                            &lio->sync_octeon_time_wq.wk.work,
 687                            msecs_to_jiffies(LIO_SYNC_OCTEON_TIME_INTERVAL_MS));
 688
 689         return 0;
 690 }
 691
 692 /**
 693  * cleanup_sync_octeon_time_wq - stop scheduling and destroy the work created
 694  * to periodically update local time to octeon firmware
 695  *
 696  * @netdev - network device which should send time update to firmware
 697  **/
 698 static inline void cleanup_sync_octeon_time_wq(struct net_device *netdev)
 699 {
 700         struct lio *lio = GET_LIO(netdev);
 701         struct cavium_wq *time_wq = &lio->sync_octeon_time_wq;
 702
 703         if (time_wq->wq) {
 704                 cancel_delayed_work_sync(&time_wq->wk.work);
 705                 destroy_workqueue(time_wq->wq);
 706         }
 707 }
 708
 709 static struct octeon_device *get_other_octeon_device(struct octeon_device *oct)
 710 {
 711         struct octeon_device *other_oct;
 712
 713         other_oct = lio_get_device(oct->octeon_id + 1);
 714
 715         if (other_oct && other_oct->pci_dev) {
 716                 int oct_busnum, other_oct_busnum;
 717
 718                 oct_busnum = oct->pci_dev->bus->number;
 719                 other_oct_busnum = other_oct->pci_dev->bus->number;
 720
 721                 if (oct_busnum == other_oct_busnum) {
 722                         int oct_slot, other_oct_slot;
 723
 724                         oct_slot = PCI_SLOT(oct->pci_dev->devfn);
 725                         other_oct_slot = PCI_SLOT(other_oct->pci_dev->devfn);
 726
 727                         if (oct_slot == other_oct_slot)
 728                                 return other_oct;
 729                 }
 730         }
 731
 732         return NULL;
 733 }
 734
 735 static void disable_all_vf_links(struct octeon_device *oct)
 736 {
 737         struct net_device *netdev;
 738         int max_vfs, vf, i;
 739
 740         if (!oct)
 741                 return;
 742
 743         max_vfs = oct->sriov_info.max_vfs;
 744
 745         for (i = 0; i < oct->ifcount; i++) {
 746                 netdev = oct->props[i].netdev;
 747                 if (!netdev)
 748                         continue;
 749
 750                 for (vf = 0; vf < max_vfs; vf++)
 751                         liquidio_set_vf_link_state(netdev, vf,
 752                                                    IFLA_VF_LINK_STATE_DISABLE);
 753         }
 754 }
 755
 756 static int liquidio_watchdog(void *param)
 757 {
 758         bool err_msg_was_printed[LIO_MAX_CORES];
 759         u16 mask_of_crashed_or_stuck_cores = 0;
 760         bool all_vf_links_are_disabled = false;
 761         struct octeon_device *oct = param;
 762         struct octeon_device *other_oct;
 763 #ifdef CONFIG_MODULE_UNLOAD
 764         long refcount, vfs_referencing_pf;
 765         u64 vfs_mask1, vfs_mask2;
 766 #endif
 767         int core;
 768
 769         memset(err_msg_was_printed, 0, sizeof(err_msg_was_printed));
 770
 771         while (!kthread_should_stop()) {
 772                 /* sleep for a couple of seconds so that we don't hog the CPU */
 773                 set_current_state(TASK_INTERRUPTIBLE);
 774                 schedule_timeout(msecs_to_jiffies(2000));
 775
 776                 mask_of_crashed_or_stuck_cores =
 777                     (u16)octeon_read_csr64(oct, CN23XX_SLI_SCRATCH2);
 778
 779                 if (!mask_of_crashed_or_stuck_cores)
 780                         continue;
 781
 782                 WRITE_ONCE(oct->cores_crashed, true);
 783                 other_oct = get_other_octeon_device(oct);
 784                 if (other_oct)
 785                         WRITE_ONCE(other_oct->cores_crashed, true);
 786
 787                 for (core = 0; core < LIO_MAX_CORES; core++) {
 788                         bool core_crashed_or_got_stuck;
 789
 790                         core_crashed_or_got_stuck =
 791                                                 (mask_of_crashed_or_stuck_cores
 792                                                  >> core) & 1;
 793
 794                         if (core_crashed_or_got_stuck &&
 795                             !err_msg_was_printed[core]) {
 796                                 dev_err(&oct->pci_dev->dev,
 797                                         "ERROR: Octeon core %d crashed or got stuck!  See oct-fwdump for details.\n",
 798                                         core);
 799                                 err_msg_was_printed[core] = true;
 800                         }
 801                 }
 802
 803                 if (all_vf_links_are_disabled)
 804                         continue;
 805
 806                 disable_all_vf_links(oct);
 807                 disable_all_vf_links(other_oct);
 808                 all_vf_links_are_disabled = true;
 809
 810 #ifdef CONFIG_MODULE_UNLOAD
 811                 vfs_mask1 = READ_ONCE(oct->sriov_info.vf_drv_loaded_mask);
 812                 vfs_mask2 = READ_ONCE(other_oct->sriov_info.vf_drv_loaded_mask);
 813
 814                 vfs_referencing_pf  = hweight64(vfs_mask1);
 815                 vfs_referencing_pf += hweight64(vfs_mask2);
 816
 817                 refcount = module_refcount(THIS_MODULE);
 818                 if (refcount >= vfs_referencing_pf) {
 819                         while (vfs_referencing_pf) {
 820                                 module_put(THIS_MODULE);
 821                                 vfs_referencing_pf--;
 822                         }
 823                 }
 824 #endif
 825         }
 826
 827         return 0;
 828 }
 829
 830 /**
 831  * \brief PCI probe handler
 832  * @param pdev PCI device structure
 833  * @param ent unused
 834  */
 835 static int
 836 liquidio_probe(struct pci_dev *pdev,
 837                const struct pci_device_id *ent __attribute__((unused)))
 838 {
 839         struct octeon_device *oct_dev = NULL;
 840         struct handshake *hs;
 841
 842         oct_dev = octeon_allocate_device(pdev->device,
 843                                          sizeof(struct octeon_device_priv));
 844         if (!oct_dev) {
 845                 dev_err(&pdev->dev, "Unable to allocate device\n");
 846                 return -ENOMEM;
 847         }
 848
 849         if (pdev->device == OCTEON_CN23XX_PF_VID)
 850                 oct_dev->msix_on = LIO_FLAG_MSIX_ENABLED;
 851
 852         /* Enable PTP for 6XXX Device */
 853         if (((pdev->device == OCTEON_CN66XX) ||
 854              (pdev->device == OCTEON_CN68XX)))
 855                 oct_dev->ptp_enable = true;
 856         else
 857                 oct_dev->ptp_enable = false;
 858
 859         dev_info(&pdev->dev, "Initializing device %x:%x.\n",
 860                  (u32)pdev->vendor, (u32)pdev->device);
 861
 862         /* Assign octeon_device for this device to the private data area. */
 863         pci_set_drvdata(pdev, oct_dev);
 864
 865         /* set linux specific device pointer */
 866         oct_dev->pci_dev = (void *)pdev;
 867
 868         oct_dev->subsystem_id = pdev->subsystem_vendor |
 869                 (pdev->subsystem_device << 16);
 870
 871         hs = &handshake[oct_dev->octeon_id];
 872         init_completion(&hs->init);
 873         init_completion(&hs->started);
 874         hs->pci_dev = pdev;
 875
 876         if (oct_dev->octeon_id == 0)
 877                 /* first LiquidIO NIC is detected */
 878                 complete(&first_stage);
 879
 880         if (octeon_device_init(oct_dev)) {
 881                 complete(&hs->init);
 882                 liquidio_remove(pdev);
 883                 return -ENOMEM;
 884         }
 885
 886         if (OCTEON_CN23XX_PF(oct_dev)) {
 887                 u8 bus, device, function;
 888
 889                 if (atomic_read(oct_dev->adapter_refcount) == 1) {
 890                         /* Each NIC gets one watchdog kernel thread.  The first
 891                          * PF (of each NIC) that gets pci_driver->probe()'d
 892                          * creates that thread.
 893                          */
 894                         bus = pdev->bus->number;
 895                         device = PCI_SLOT(pdev->devfn);
 896                         function = PCI_FUNC(pdev->devfn);
 897                         oct_dev->watchdog_task = kthread_create(
 898                             liquidio_watchdog, oct_dev,
 899                             "liowd/%02hhx:%02hhx.%hhx", bus, device, function);
 900                         if (!IS_ERR(oct_dev->watchdog_task)) {
 901                                 wake_up_process(oct_dev->watchdog_task);
 902                         } else {
 903                                 oct_dev->watchdog_task = NULL;
 904                                 dev_err(&oct_dev->pci_dev->dev,
 905                                         "failed to create kernel_thread\n");
 906                                 liquidio_remove(pdev);
 907                                 return -1;
 908                         }
 909                 }
 910         }
 911
 912         oct_dev->rx_pause = 1;
 913         oct_dev->tx_pause = 1;
 914
 915         dev_dbg(&oct_dev->pci_dev->dev, "Device is ready\n");
 916
 917         return 0;
 918 }
 919
 920 static bool fw_type_is_auto(void)
 921 {
 922         return strncmp(fw_type, LIO_FW_NAME_TYPE_AUTO,
 923                        sizeof(LIO_FW_NAME_TYPE_AUTO)) == 0;
 924 }
 925
 926 /**
 927  * \brief PCI FLR for each Octeon device.
 928  * @param oct octeon device
 929  */
 930 static void octeon_pci_flr(struct octeon_device *oct)
 931 {
 932         int rc;
 933
 934         pci_save_state(oct->pci_dev);
 935
 936         pci_cfg_access_lock(oct->pci_dev);
 937
 938         /* Quiesce the device completely */
 939         pci_write_config_word(oct->pci_dev, PCI_COMMAND,
 940                               PCI_COMMAND_INTX_DISABLE);
 941
 942         rc = __pci_reset_function_locked(oct->pci_dev);
 943
 944         if (rc != 0)
 945                 dev_err(&oct->pci_dev->dev, "Error %d resetting PCI function %d\n",
 946                         rc, oct->pf_num);
 947
 948         pci_cfg_access_unlock(oct->pci_dev);
 949
 950         pci_restore_state(oct->pci_dev);
 951 }
 952
 953 /**
 954  *\brief Destroy resources associated with octeon device
 955  * @param pdev PCI device structure
 956  * @param ent unused
 957  */
 958 static void octeon_destroy_resources(struct octeon_device *oct)
 959 {
 960         int i, refcount;
 961         struct msix_entry *msix_entries;
 962         struct octeon_device_priv *oct_priv =
 963                 (struct octeon_device_priv *)oct->priv;
 964
 965         struct handshake *hs;
 966
 967         switch (atomic_read(&oct->status)) {
 968         case OCT_DEV_RUNNING:
 969         case OCT_DEV_CORE_OK:
 970
 971                 /* No more instructions will be forwarded. */
 972                 atomic_set(&oct->status, OCT_DEV_IN_RESET);
 973
 974                 oct->app_mode = CVM_DRV_INVALID_APP;
 975                 dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
 976                         lio_get_state_string(&oct->status));
 977
 978                 schedule_timeout_uninterruptible(HZ / 10);
 979
 980                 fallthrough;
 981         case OCT_DEV_HOST_OK:
 982
 983         case OCT_DEV_CONSOLE_INIT_DONE:
 984                 /* Remove any consoles */
 985                 octeon_remove_consoles(oct);
 986
 987                 fallthrough;
 988         case OCT_DEV_IO_QUEUES_DONE:
 989                 if (lio_wait_for_instr_fetch(oct))
 990                         dev_err(&oct->pci_dev->dev, "IQ had pending instructions\n");
 991
 992                 if (wait_for_pending_requests(oct))
 993                         dev_err(&oct->pci_dev->dev, "There were pending requests\n");
 994
 995                 /* Disable the input and output queues now. No more packets will
 996                  * arrive from Octeon, but we should wait for all packet
 997                  * processing to finish.
 998                  */
 999                 oct->fn_list.disable_io_queues(oct);
1000
1001                 if (lio_wait_for_oq_pkts(oct))
1002                         dev_err(&oct->pci_dev->dev, "OQ had pending packets\n");
1003
1004                 /* Force all requests waiting to be fetched by OCTEON to
1005                  * complete.
1006                  */
1007                 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
1008                         struct octeon_instr_queue *iq;
1009
1010                         if (!(oct->io_qmask.iq & BIT_ULL(i)))
1011                                 continue;
1012                         iq = oct->instr_queue[i];
1013
1014                         if (atomic_read(&iq->instr_pending)) {
1015                                 spin_lock_bh(&iq->lock);
1016                                 iq->fill_cnt = 0;
1017                                 iq->octeon_read_index = iq->host_write_index;
1018                                 iq->stats.instr_processed +=
1019                                         atomic_read(&iq->instr_pending);
1020                                 lio_process_iq_request_list(oct, iq, 0);
1021                                 spin_unlock_bh(&iq->lock);
1022                         }
1023                 }
1024
1025                 lio_process_ordered_list(oct, 1);
1026                 octeon_free_sc_done_list(oct);
1027                 octeon_free_sc_zombie_list(oct);
1028
1029                 fallthrough;
1030         case OCT_DEV_INTR_SET_DONE:
1031                 /* Disable interrupts  */
1032                 oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
1033
1034                 if (oct->msix_on) {
1035                         msix_entries = (struct msix_entry *)oct->msix_entries;
1036                         for (i = 0; i < oct->num_msix_irqs - 1; i++) {
1037                                 if (oct->ioq_vector[i].vector) {
1038                                         /* clear the affinity_cpumask */
1039                                         irq_set_affinity_hint(
1040                                                         msix_entries[i].vector,
1041                                                         NULL);
1042                                         free_irq(msix_entries[i].vector,
1043                                                  &oct->ioq_vector[i]);
1044                                         oct->ioq_vector[i].vector = 0;
1045                                 }
1046                         }
1047                         /* non-iov vector's argument is oct struct */
1048                         free_irq(msix_entries[i].vector, oct);
1049
1050                         pci_disable_msix(oct->pci_dev);
1051                         kfree(oct->msix_entries);
1052                         oct->msix_entries = NULL;
1053                 } else {
1054                         /* Release the interrupt line */
1055                         free_irq(oct->pci_dev->irq, oct);
1056
1057                         if (oct->flags & LIO_FLAG_MSI_ENABLED)
1058                                 pci_disable_msi(oct->pci_dev);
1059                 }
1060
1061                 kfree(oct->irq_name_storage);
1062                 oct->irq_name_storage = NULL;
1063
1064                 fallthrough;
1065         case OCT_DEV_MSIX_ALLOC_VECTOR_DONE:
1066                 if (OCTEON_CN23XX_PF(oct))
1067                         octeon_free_ioq_vector(oct);
1068
1069                 fallthrough;
1070         case OCT_DEV_MBOX_SETUP_DONE:
1071                 if (OCTEON_CN23XX_PF(oct))
1072                         oct->fn_list.free_mbox(oct);
1073
1074                 fallthrough;
1075         case OCT_DEV_IN_RESET:
1076         case OCT_DEV_DROQ_INIT_DONE:
1077                 /* Wait for any pending operations */
1078                 mdelay(100);
1079                 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
1080                         if (!(oct->io_qmask.oq & BIT_ULL(i)))
1081                                 continue;
1082                         octeon_delete_droq(oct, i);
1083                 }
1084
1085                 /* Force any pending handshakes to complete */
1086                 for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
1087                         hs = &handshake[i];
1088
1089                         if (hs->pci_dev) {
1090                                 handshake[oct->octeon_id].init_ok = 0;
1091                                 complete(&handshake[oct->octeon_id].init);
1092                                 handshake[oct->octeon_id].started_ok = 0;
1093                                 complete(&handshake[oct->octeon_id].started);
1094                         }
1095                 }
1096
1097                 fallthrough;
1098         case OCT_DEV_RESP_LIST_INIT_DONE:
1099                 octeon_delete_response_list(oct);
1100
1101                 fallthrough;
1102         case OCT_DEV_INSTR_QUEUE_INIT_DONE:
1103                 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
1104                         if (!(oct->io_qmask.iq & BIT_ULL(i)))
1105                                 continue;
1106                         octeon_delete_instr_queue(oct, i);
1107                 }
1108 #ifdef CONFIG_PCI_IOV
1109                 if (oct->sriov_info.sriov_enabled)
1110                         pci_disable_sriov(oct->pci_dev);
1111 #endif
1112                 fallthrough;
1113         case OCT_DEV_SC_BUFF_POOL_INIT_DONE:
1114                 octeon_free_sc_buffer_pool(oct);
1115
1116                 fallthrough;
1117         case OCT_DEV_DISPATCH_INIT_DONE:
1118                 octeon_delete_dispatch_list(oct);
1119                 cancel_delayed_work_sync(&oct->nic_poll_work.work);
1120
1121                 fallthrough;
1122         case OCT_DEV_PCI_MAP_DONE:
1123                 refcount = octeon_deregister_device(oct);
1124
1125                 /* Soft reset the octeon device before exiting.
1126                  * However, if fw was loaded from card (i.e. autoboot),
1127                  * perform an FLR instead.
1128                  * Implementation note: only soft-reset the device
1129                  * if it is a CN6XXX OR the LAST CN23XX device.
1130                  */
1131                 if (atomic_read(oct->adapter_fw_state) == FW_IS_PRELOADED)
1132                         octeon_pci_flr(oct);
1133                 else if (OCTEON_CN6XXX(oct) || !refcount)
1134                         oct->fn_list.soft_reset(oct);
1135
1136                 octeon_unmap_pci_barx(oct, 0);
1137                 octeon_unmap_pci_barx(oct, 1);
1138
1139                 fallthrough;
1140         case OCT_DEV_PCI_ENABLE_DONE:
1141                 pci_clear_master(oct->pci_dev);
1142                 /* Disable the device, releasing the PCI INT */
1143                 pci_disable_device(oct->pci_dev);
1144
1145                 fallthrough;
1146         case OCT_DEV_BEGIN_STATE:
1147                 /* Nothing to be done here either */
1148                 break;
1149         }                       /* end switch (oct->status) */
1150
1151         tasklet_kill(&oct_priv->droq_tasklet);
1152 }
1153
1154 /**
1155  * \brief Send Rx control command
1156  * @param lio per-network private data
1157  * @param start_stop whether to start or stop
1158  */
1159 static void send_rx_ctrl_cmd(struct lio *lio, int start_stop)
1160 {
1161         struct octeon_soft_command *sc;
1162         union octnet_cmd *ncmd;
1163         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1164         int retval;
1165
1166         if (oct->props[lio->ifidx].rx_on == start_stop)
1167                 return;
1168
1169         sc = (struct octeon_soft_command *)
1170                 octeon_alloc_soft_command(oct, OCTNET_CMD_SIZE,
1171                                           16, 0);
1172         if (!sc) {
1173                 netif_info(lio, rx_err, lio->netdev,
1174                            "Failed to allocate octeon_soft_command\n");
1175                 return;
1176         }
1177
1178         ncmd = (union octnet_cmd *)sc->virtdptr;
1179
1180         ncmd->u64 = 0;
1181         ncmd->s.cmd = OCTNET_CMD_RX_CTL;
1182         ncmd->s.param1 = start_stop;
1183
1184         octeon_swap_8B_data((u64 *)ncmd, (OCTNET_CMD_SIZE >> 3));
1185
1186         sc->iq_no = lio->linfo.txpciq[0].s.q_no;
1187
1188         octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
1189                                     OPCODE_NIC_CMD, 0, 0, 0);
1190
1191         init_completion(&sc->complete);
1192         sc->sc_status = OCTEON_REQUEST_PENDING;
1193
1194         retval = octeon_send_soft_command(oct, sc);
1195         if (retval == IQ_SEND_FAILED) {
1196                 netif_info(lio, rx_err, lio->netdev, "Failed to send RX Control message\n");
1197                 octeon_free_soft_command(oct, sc);
1198                 return;
1199         } else {
1200                 /* Sleep on a wait queue till the cond flag indicates that the
1201                  * response arrived or timed-out.
1202                  */
1203                 retval = wait_for_sc_completion_timeout(oct, sc, 0);
1204                 if (retval)
1205                         return;
1206
1207                 oct->props[lio->ifidx].rx_on = start_stop;
1208                 WRITE_ONCE(sc->caller_is_done, true);
1209         }
1210 }
1211
1212 /**
1213  * \brief Destroy NIC device interface
1214  * @param oct octeon device
1215  * @param ifidx which interface to destroy
1216  *
1217  * Cleanup associated with each interface for an Octeon device  when NIC
1218  * module is being unloaded or if initialization fails during load.
1219  */
1220 static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx)
1221 {
1222         struct net_device *netdev = oct->props[ifidx].netdev;
1223         struct octeon_device_priv *oct_priv =
1224                 (struct octeon_device_priv *)oct->priv;
1225         struct napi_struct *napi, *n;
1226         struct lio *lio;
1227
1228         if (!netdev) {
1229                 dev_err(&oct->pci_dev->dev, "%s No netdevice ptr for index %d\n",
1230                         __func__, ifidx);
1231                 return;
1232         }
1233
1234         lio = GET_LIO(netdev);
1235
1236         dev_dbg(&oct->pci_dev->dev, "NIC device cleanup\n");
1237
1238         if (atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING)
1239                 liquidio_stop(netdev);
1240
1241         if (oct->props[lio->ifidx].napi_enabled == 1) {
1242                 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1243                         napi_disable(napi);
1244
1245                 oct->props[lio->ifidx].napi_enabled = 0;
1246
1247                 if (OCTEON_CN23XX_PF(oct))
1248                         oct->droq[0]->ops.poll_mode = 0;
1249         }
1250
1251         /* Delete NAPI */
1252         list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1253                 netif_napi_del(napi);
1254
1255         tasklet_enable(&oct_priv->droq_tasklet);
1256
1257         if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED)
1258                 unregister_netdev(netdev);
1259
1260         cleanup_sync_octeon_time_wq(netdev);
1261         cleanup_link_status_change_wq(netdev);
1262
1263         cleanup_rx_oom_poll_fn(netdev);
1264
1265         lio_delete_glists(lio);
1266
1267         free_netdev(netdev);
1268
1269         oct->props[ifidx].gmxport = -1;
1270
1271         oct->props[ifidx].netdev = NULL;
1272 }
1273
1274 /**
1275  * \brief Stop complete NIC functionality
1276  * @param oct octeon device
1277  */
1278 static int liquidio_stop_nic_module(struct octeon_device *oct)
1279 {
1280         int i, j;
1281         struct lio *lio;
1282
1283         dev_dbg(&oct->pci_dev->dev, "Stopping network interfaces\n");
1284         if (!oct->ifcount) {
1285                 dev_err(&oct->pci_dev->dev, "Init for Octeon was not completed\n");
1286                 return 1;
1287         }
1288
1289         spin_lock_bh(&oct->cmd_resp_wqlock);
1290         oct->cmd_resp_state = OCT_DRV_OFFLINE;
1291         spin_unlock_bh(&oct->cmd_resp_wqlock);
1292
1293         lio_vf_rep_destroy(oct);
1294
1295         for (i = 0; i < oct->ifcount; i++) {
1296                 lio = GET_LIO(oct->props[i].netdev);
1297                 for (j = 0; j < oct->num_oqs; j++)
1298                         octeon_unregister_droq_ops(oct,
1299                                                    lio->linfo.rxpciq[j].s.q_no);
1300         }
1301
1302         for (i = 0; i < oct->ifcount; i++)
1303                 liquidio_destroy_nic_device(oct, i);
1304
1305         if (oct->devlink) {
1306                 devlink_unregister(oct->devlink);
1307                 devlink_free(oct->devlink);
1308                 oct->devlink = NULL;
1309         }
1310
1311         dev_dbg(&oct->pci_dev->dev, "Network interfaces stopped\n");
1312         return 0;
1313 }
1314
1315 /**
1316  * \brief Cleans up resources at unload time
1317  * @param pdev PCI device structure
1318  */
1319 static void liquidio_remove(struct pci_dev *pdev)
1320 {
1321         struct octeon_device *oct_dev = pci_get_drvdata(pdev);
1322
1323         dev_dbg(&oct_dev->pci_dev->dev, "Stopping device\n");
1324
1325         if (oct_dev->watchdog_task)
1326                 kthread_stop(oct_dev->watchdog_task);
1327
1328         if (!oct_dev->octeon_id &&
1329             oct_dev->fw_info.app_cap_flags & LIQUIDIO_SWITCHDEV_CAP)
1330                 lio_vf_rep_modexit();
1331
1332         if (oct_dev->app_mode && (oct_dev->app_mode == CVM_DRV_NIC_APP))
1333                 liquidio_stop_nic_module(oct_dev);
1334
1335         /* Reset the octeon device and cleanup all memory allocated for
1336          * the octeon device by driver.
1337          */
1338         octeon_destroy_resources(oct_dev);
1339
1340         dev_info(&oct_dev->pci_dev->dev, "Device removed\n");
1341
1342         /* This octeon device has been removed. Update the global
1343          * data structure to reflect this. Free the device structure.
1344          */
1345         octeon_free_device_mem(oct_dev);
1346 }
1347
1348 /**
1349  * \brief Identify the Octeon device and to map the BAR address space
1350  * @param oct octeon device
1351  */
1352 static int octeon_chip_specific_setup(struct octeon_device *oct)
1353 {
1354         u32 dev_id, rev_id;
1355         int ret = 1;
1356
1357         pci_read_config_dword(oct->pci_dev, 0, &dev_id);
1358         pci_read_config_dword(oct->pci_dev, 8, &rev_id);
1359         oct->rev_id = rev_id & 0xff;
1360
1361         switch (dev_id) {
1362         case OCTEON_CN68XX_PCIID:
1363                 oct->chip_id = OCTEON_CN68XX;
1364                 ret = lio_setup_cn68xx_octeon_device(oct);
1365                 break;
1366
1367         case OCTEON_CN66XX_PCIID:
1368                 oct->chip_id = OCTEON_CN66XX;
1369                 ret = lio_setup_cn66xx_octeon_device(oct);
1370                 break;
1371
1372         case OCTEON_CN23XX_PCIID_PF:
1373                 oct->chip_id = OCTEON_CN23XX_PF_VID;
1374                 ret = setup_cn23xx_octeon_pf_device(oct);
1375                 if (ret)
1376                         break;
1377 #ifdef CONFIG_PCI_IOV
1378                 if (!ret)
1379                         pci_sriov_set_totalvfs(oct->pci_dev,
1380                                                oct->sriov_info.max_vfs);
1381 #endif
1382                 break;
1383
1384         default:
1385                 dev_err(&oct->pci_dev->dev, "Unknown device found (dev_id: %x)\n",
1386                         dev_id);
1387         }
1388
1389         return ret;
1390 }
1391
1392 /**
1393  * \brief PCI initialization for each Octeon device.
1394  * @param oct octeon device
1395  */
1396 static int octeon_pci_os_setup(struct octeon_device *oct)
1397 {
1398         /* setup PCI stuff first */
1399         if (pci_enable_device(oct->pci_dev)) {
1400                 dev_err(&oct->pci_dev->dev, "pci_enable_device failed\n");
1401                 return 1;
1402         }
1403
1404         if (dma_set_mask_and_coherent(&oct->pci_dev->dev, DMA_BIT_MASK(64))) {
1405                 dev_err(&oct->pci_dev->dev, "Unexpected DMA device capability\n");
1406                 pci_disable_device(oct->pci_dev);
1407                 return 1;
1408         }
1409
1410         /* Enable PCI DMA Master. */
1411         pci_set_master(oct->pci_dev);
1412
1413         return 0;
1414 }
1415
1416 /**
1417  * \brief Unmap and free network buffer
1418  * @param buf buffer
1419  */
1420 static void free_netbuf(void *buf)
1421 {
1422         struct sk_buff *skb;
1423         struct octnet_buf_free_info *finfo;
1424         struct lio *lio;
1425
1426         finfo = (struct octnet_buf_free_info *)buf;
1427         skb = finfo->skb;
1428         lio = finfo->lio;
1429
1430         dma_unmap_single(&lio->oct_dev->pci_dev->dev, finfo->dptr, skb->len,
1431                          DMA_TO_DEVICE);
1432
1433         tx_buffer_free(skb);
1434 }
1435
1436 /**
1437  * \brief Unmap and free gather buffer
1438  * @param buf buffer
1439  */
1440 static void free_netsgbuf(void *buf)
1441 {
1442         struct octnet_buf_free_info *finfo;
1443         struct sk_buff *skb;
1444         struct lio *lio;
1445         struct octnic_gather *g;
1446         int i, frags, iq;
1447
1448         finfo = (struct octnet_buf_free_info *)buf;
1449         skb = finfo->skb;
1450         lio = finfo->lio;
1451         g = finfo->g;
1452         frags = skb_shinfo(skb)->nr_frags;
1453
1454         dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1455                          g->sg[0].ptr[0], (skb->len - skb->data_len),
1456                          DMA_TO_DEVICE);
1457
1458         i = 1;
1459         while (frags--) {
1460                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
1461
1462                 pci_unmap_page((lio->oct_dev)->pci_dev,
1463                                g->sg[(i >> 2)].ptr[(i & 3)],
1464                                skb_frag_size(frag), DMA_TO_DEVICE);
1465                 i++;
1466         }
1467
1468         iq = skb_iq(lio->oct_dev, skb);
1469         spin_lock(&lio->glist_lock[iq]);
1470         list_add_tail(&g->list, &lio->glist[iq]);
1471         spin_unlock(&lio->glist_lock[iq]);
1472
1473         tx_buffer_free(skb);
1474 }
1475
1476 /**
1477  * \brief Unmap and free gather buffer with response
1478  * @param buf buffer
1479  */
1480 static void free_netsgbuf_with_resp(void *buf)
1481 {
1482         struct octeon_soft_command *sc;
1483         struct octnet_buf_free_info *finfo;
1484         struct sk_buff *skb;
1485         struct lio *lio;
1486         struct octnic_gather *g;
1487         int i, frags, iq;
1488
1489         sc = (struct octeon_soft_command *)buf;
1490         skb = (struct sk_buff *)sc->callback_arg;
1491         finfo = (struct octnet_buf_free_info *)&skb->cb;
1492
1493         lio = finfo->lio;
1494         g = finfo->g;
1495         frags = skb_shinfo(skb)->nr_frags;
1496
1497         dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1498                          g->sg[0].ptr[0], (skb->len - skb->data_len),
1499                          DMA_TO_DEVICE);
1500
1501         i = 1;
1502         while (frags--) {
1503                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
1504
1505                 pci_unmap_page((lio->oct_dev)->pci_dev,
1506                                g->sg[(i >> 2)].ptr[(i & 3)],
1507                                skb_frag_size(frag), DMA_TO_DEVICE);
1508                 i++;
1509         }
1510
1511         iq = skb_iq(lio->oct_dev, skb);
1512
1513         spin_lock(&lio->glist_lock[iq]);
1514         list_add_tail(&g->list, &lio->glist[iq]);
1515         spin_unlock(&lio->glist_lock[iq]);
1516
1517         /* Don't free the skb yet */
1518 }
1519
1520 /**
1521  * \brief Adjust ptp frequency
1522  * @param ptp PTP clock info
1523  * @param ppb how much to adjust by, in parts-per-billion
1524  */
1525 static int liquidio_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb)
1526 {
1527         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1528         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1529         u64 comp, delta;
1530         unsigned long flags;
1531         bool neg_adj = false;
1532
1533         if (ppb < 0) {
1534                 neg_adj = true;
1535                 ppb = -ppb;
1536         }
1537
1538         /* The hardware adds the clock compensation value to the
1539          * PTP clock on every coprocessor clock cycle, so we
1540          * compute the delta in terms of coprocessor clocks.
1541          */
1542         delta = (u64)ppb << 32;
1543         do_div(delta, oct->coproc_clock_rate);
1544
1545         spin_lock_irqsave(&lio->ptp_lock, flags);
1546         comp = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_COMP);
1547         if (neg_adj)
1548                 comp -= delta;
1549         else
1550                 comp += delta;
1551         lio_pci_writeq(oct, comp, CN6XXX_MIO_PTP_CLOCK_COMP);
1552         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1553
1554         return 0;
1555 }
1556
1557 /**
1558  * \brief Adjust ptp time
1559  * @param ptp PTP clock info
1560  * @param delta how much to adjust by, in nanosecs
1561  */
1562 static int liquidio_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
1563 {
1564         unsigned long flags;
1565         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1566
1567         spin_lock_irqsave(&lio->ptp_lock, flags);
1568         lio->ptp_adjust += delta;
1569         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1570
1571         return 0;
1572 }
1573
1574 /**
1575  * \brief Get hardware clock time, including any adjustment
1576  * @param ptp PTP clock info
1577  * @param ts timespec
1578  */
1579 static int liquidio_ptp_gettime(struct ptp_clock_info *ptp,
1580                                 struct timespec64 *ts)
1581 {
1582         u64 ns;
1583         unsigned long flags;
1584         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1585         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1586
1587         spin_lock_irqsave(&lio->ptp_lock, flags);
1588         ns = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_HI);
1589         ns += lio->ptp_adjust;
1590         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1591
1592         *ts = ns_to_timespec64(ns);
1593
1594         return 0;
1595 }
1596
1597 /**
1598  * \brief Set hardware clock time. Reset adjustment
1599  * @param ptp PTP clock info
1600  * @param ts timespec
1601  */
1602 static int liquidio_ptp_settime(struct ptp_clock_info *ptp,
1603                                 const struct timespec64 *ts)
1604 {
1605         u64 ns;
1606         unsigned long flags;
1607         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1608         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1609
1610         ns = timespec64_to_ns(ts);
1611
1612         spin_lock_irqsave(&lio->ptp_lock, flags);
1613         lio_pci_writeq(oct, ns, CN6XXX_MIO_PTP_CLOCK_HI);
1614         lio->ptp_adjust = 0;
1615         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1616
1617         return 0;
1618 }
1619
1620 /**
1621  * \brief Check if PTP is enabled
1622  * @param ptp PTP clock info
1623  * @param rq request
1624  * @param on is it on
1625  */
1626 static int
1627 liquidio_ptp_enable(struct ptp_clock_info *ptp __attribute__((unused)),
1628                     struct ptp_clock_request *rq __attribute__((unused)),
1629                     int on __attribute__((unused)))
1630 {
1631         return -EOPNOTSUPP;
1632 }
1633
1634 /**
1635  * \brief Open PTP clock source
1636  * @param netdev network device
1637  */
1638 static void oct_ptp_open(struct net_device *netdev)
1639 {
1640         struct lio *lio = GET_LIO(netdev);
1641         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1642
1643         spin_lock_init(&lio->ptp_lock);
1644
1645         snprintf(lio->ptp_info.name, 16, "%s", netdev->name);
1646         lio->ptp_info.owner = THIS_MODULE;
1647         lio->ptp_info.max_adj = 250000000;
1648         lio->ptp_info.n_alarm = 0;
1649         lio->ptp_info.n_ext_ts = 0;
1650         lio->ptp_info.n_per_out = 0;
1651         lio->ptp_info.pps = 0;
1652         lio->ptp_info.adjfreq = liquidio_ptp_adjfreq;
1653         lio->ptp_info.adjtime = liquidio_ptp_adjtime;
1654         lio->ptp_info.gettime64 = liquidio_ptp_gettime;
1655         lio->ptp_info.settime64 = liquidio_ptp_settime;
1656         lio->ptp_info.enable = liquidio_ptp_enable;
1657
1658         lio->ptp_adjust = 0;
1659
1660         lio->ptp_clock = ptp_clock_register(&lio->ptp_info,
1661                                              &oct->pci_dev->dev);
1662
1663         if (IS_ERR(lio->ptp_clock))
1664                 lio->ptp_clock = NULL;
1665 }
1666
1667 /**
1668  * \brief Init PTP clock
1669  * @param oct octeon device
1670  */
1671 static void liquidio_ptp_init(struct octeon_device *oct)
1672 {
1673         u64 clock_comp, cfg;
1674
1675         clock_comp = (u64)NSEC_PER_SEC << 32;
1676         do_div(clock_comp, oct->coproc_clock_rate);
1677         lio_pci_writeq(oct, clock_comp, CN6XXX_MIO_PTP_CLOCK_COMP);
1678
1679         /* Enable */
1680         cfg = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_CFG);
1681         lio_pci_writeq(oct, cfg | 0x01, CN6XXX_MIO_PTP_CLOCK_CFG);
1682 }
1683
1684 /**
1685  * \brief Load firmware to device
1686  * @param oct octeon device
1687  *
1688  * Maps device to firmware filename, requests firmware, and downloads it
1689  */
1690 static int load_firmware(struct octeon_device *oct)
1691 {
1692         int ret = 0;
1693         const struct firmware *fw;
1694         char fw_name[LIO_MAX_FW_FILENAME_LEN];
1695         char *tmp_fw_type;
1696
1697         if (fw_type_is_auto()) {
1698                 tmp_fw_type = LIO_FW_NAME_TYPE_NIC;
1699                 strncpy(fw_type, tmp_fw_type, sizeof(fw_type));
1700         } else {
1701                 tmp_fw_type = fw_type;
1702         }
1703
1704         sprintf(fw_name, "%s%s%s_%s%s", LIO_FW_DIR, LIO_FW_BASE_NAME,
1705                 octeon_get_conf(oct)->card_name, tmp_fw_type,
1706                 LIO_FW_NAME_SUFFIX);
1707
1708         ret = request_firmware(&fw, fw_name, &oct->pci_dev->dev);
1709         if (ret) {
1710                 dev_err(&oct->pci_dev->dev, "Request firmware failed. Could not find file %s.\n",
1711                         fw_name);
1712                 release_firmware(fw);
1713                 return ret;
1714         }
1715
1716         ret = octeon_download_firmware(oct, fw->data, fw->size);
1717
1718         release_firmware(fw);
1719
1720         return ret;
1721 }
1722
1723 /**
1724  * \brief Poll routine for checking transmit queue status
1725  * @param work work_struct data structure
1726  */
1727 static void octnet_poll_check_txq_status(struct work_struct *work)
1728 {
1729         struct cavium_wk *wk = (struct cavium_wk *)work;
1730         struct lio *lio = (struct lio *)wk->ctxptr;
1731
1732         if (!ifstate_check(lio, LIO_IFSTATE_RUNNING))
1733                 return;
1734
1735         check_txq_status(lio);
1736         queue_delayed_work(lio->txq_status_wq.wq,
1737                            &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
1738 }
1739
1740 /**
1741  * \brief Sets up the txq poll check
1742  * @param netdev network device
1743  */
1744 static inline int setup_tx_poll_fn(struct net_device *netdev)
1745 {
1746         struct lio *lio = GET_LIO(netdev);
1747         struct octeon_device *oct = lio->oct_dev;
1748
1749         lio->txq_status_wq.wq = alloc_workqueue("txq-status",
1750                                                 WQ_MEM_RECLAIM, 0);
1751         if (!lio->txq_status_wq.wq) {
1752                 dev_err(&oct->pci_dev->dev, "unable to create cavium txq status wq\n");
1753                 return -1;
1754         }
1755         INIT_DELAYED_WORK(&lio->txq_status_wq.wk.work,
1756                           octnet_poll_check_txq_status);
1757         lio->txq_status_wq.wk.ctxptr = lio;
1758         queue_delayed_work(lio->txq_status_wq.wq,
1759                            &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
1760         return 0;
1761 }
1762
1763 static inline void cleanup_tx_poll_fn(struct net_device *netdev)
1764 {
1765         struct lio *lio = GET_LIO(netdev);
1766
1767         if (lio->txq_status_wq.wq) {
1768                 cancel_delayed_work_sync(&lio->txq_status_wq.wk.work);
1769                 destroy_workqueue(lio->txq_status_wq.wq);
1770         }
1771 }
1772
1773 /**
1774  * \brief Net device open for LiquidIO
1775  * @param netdev network device
1776  */
1777 static int liquidio_open(struct net_device *netdev)
1778 {
1779         struct lio *lio = GET_LIO(netdev);
1780         struct octeon_device *oct = lio->oct_dev;
1781         struct octeon_device_priv *oct_priv =
1782                 (struct octeon_device_priv *)oct->priv;
1783         struct napi_struct *napi, *n;
1784
1785         if (oct->props[lio->ifidx].napi_enabled == 0) {
1786                 tasklet_disable(&oct_priv->droq_tasklet);
1787
1788                 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1789                         napi_enable(napi);
1790
1791                 oct->props[lio->ifidx].napi_enabled = 1;
1792
1793                 if (OCTEON_CN23XX_PF(oct))
1794                         oct->droq[0]->ops.poll_mode = 1;
1795         }
1796
1797         if (oct->ptp_enable)
1798                 oct_ptp_open(netdev);
1799
1800         ifstate_set(lio, LIO_IFSTATE_RUNNING);
1801
1802         if (OCTEON_CN23XX_PF(oct)) {
1803                 if (!oct->msix_on)
1804                         if (setup_tx_poll_fn(netdev))
1805                                 return -1;
1806         } else {
1807                 if (setup_tx_poll_fn(netdev))
1808                         return -1;
1809         }
1810
1811         netif_tx_start_all_queues(netdev);
1812
1813         /* Ready for link status updates */
1814         lio->intf_open = 1;
1815
1816         netif_info(lio, ifup, lio->netdev, "Interface Open, ready for traffic\n");
1817
1818         /* tell Octeon to start forwarding packets to host */
1819         send_rx_ctrl_cmd(lio, 1);
1820
1821         /* start periodical statistics fetch */
1822         INIT_DELAYED_WORK(&lio->stats_wk.work, lio_fetch_stats);
1823         lio->stats_wk.ctxptr = lio;
1824         schedule_delayed_work(&lio->stats_wk.work, msecs_to_jiffies
1825                                         (LIQUIDIO_NDEV_STATS_POLL_TIME_MS));
1826
1827         dev_info(&oct->pci_dev->dev, "%s interface is opened\n",
1828                  netdev->name);
1829
1830         return 0;
1831 }
1832
1833 /**
1834  * \brief Net device stop for LiquidIO
1835  * @param netdev network device
1836  */
1837 static int liquidio_stop(struct net_device *netdev)
1838 {
1839         struct lio *lio = GET_LIO(netdev);
1840         struct octeon_device *oct = lio->oct_dev;
1841         struct octeon_device_priv *oct_priv =
1842                 (struct octeon_device_priv *)oct->priv;
1843         struct napi_struct *napi, *n;
1844
1845         ifstate_reset(lio, LIO_IFSTATE_RUNNING);
1846
1847         /* Stop any link updates */
1848         lio->intf_open = 0;
1849
1850         stop_txqs(netdev);
1851
1852         /* Inform that netif carrier is down */
1853         netif_carrier_off(netdev);
1854         netif_tx_disable(netdev);
1855
1856         lio->linfo.link.s.link_up = 0;
1857         lio->link_changes++;
1858
1859         /* Tell Octeon that nic interface is down. */
1860         send_rx_ctrl_cmd(lio, 0);
1861
1862         if (OCTEON_CN23XX_PF(oct)) {
1863                 if (!oct->msix_on)
1864                         cleanup_tx_poll_fn(netdev);
1865         } else {
1866                 cleanup_tx_poll_fn(netdev);
1867         }
1868
1869         cancel_delayed_work_sync(&lio->stats_wk.work);
1870
1871         if (lio->ptp_clock) {
1872                 ptp_clock_unregister(lio->ptp_clock);
1873                 lio->ptp_clock = NULL;
1874         }
1875
1876         /* Wait for any pending Rx descriptors */
1877         if (lio_wait_for_clean_oq(oct))
1878                 netif_info(lio, rx_err, lio->netdev,
1879                            "Proceeding with stop interface after partial RX desc processing\n");
1880
1881         if (oct->props[lio->ifidx].napi_enabled == 1) {
1882                 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1883                         napi_disable(napi);
1884
1885                 oct->props[lio->ifidx].napi_enabled = 0;
1886
1887                 if (OCTEON_CN23XX_PF(oct))
1888                         oct->droq[0]->ops.poll_mode = 0;
1889
1890                 tasklet_enable(&oct_priv->droq_tasklet);
1891         }
1892
1893         dev_info(&oct->pci_dev->dev, "%s interface is stopped\n", netdev->name);
1894
1895         return 0;
1896 }
1897
1898 /**
1899  * \brief Converts a mask based on net device flags
1900  * @param netdev network device
1901  *
1902  * This routine generates a octnet_ifflags mask from the net device flags
1903  * received from the OS.
1904  */
1905 static inline enum octnet_ifflags get_new_flags(struct net_device *netdev)
1906 {
1907         enum octnet_ifflags f = OCTNET_IFFLAG_UNICAST;
1908
1909         if (netdev->flags & IFF_PROMISC)
1910                 f |= OCTNET_IFFLAG_PROMISC;
1911
1912         if (netdev->flags & IFF_ALLMULTI)
1913                 f |= OCTNET_IFFLAG_ALLMULTI;
1914
1915         if (netdev->flags & IFF_MULTICAST) {
1916                 f |= OCTNET_IFFLAG_MULTICAST;
1917
1918                 /* Accept all multicast addresses if there are more than we
1919                  * can handle
1920                  */
1921                 if (netdev_mc_count(netdev) > MAX_OCTEON_MULTICAST_ADDR)
1922                         f |= OCTNET_IFFLAG_ALLMULTI;
1923         }
1924
1925         if (netdev->flags & IFF_BROADCAST)
1926                 f |= OCTNET_IFFLAG_BROADCAST;
1927
1928         return f;
1929 }
1930
1931 /**
1932  * \brief Net device set_multicast_list
1933  * @param netdev network device
1934  */
1935 static void liquidio_set_mcast_list(struct net_device *netdev)
1936 {
1937         struct lio *lio = GET_LIO(netdev);
1938         struct octeon_device *oct = lio->oct_dev;
1939         struct octnic_ctrl_pkt nctrl;
1940         struct netdev_hw_addr *ha;
1941         u64 *mc;
1942         int ret;
1943         int mc_count = min(netdev_mc_count(netdev), MAX_OCTEON_MULTICAST_ADDR);
1944
1945         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
1946
1947         /* Create a ctrl pkt command to be sent to core app. */
1948         nctrl.ncmd.u64 = 0;
1949         nctrl.ncmd.s.cmd = OCTNET_CMD_SET_MULTI_LIST;
1950         nctrl.ncmd.s.param1 = get_new_flags(netdev);
1951         nctrl.ncmd.s.param2 = mc_count;
1952         nctrl.ncmd.s.more = mc_count;
1953         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
1954         nctrl.netpndev = (u64)netdev;
1955         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
1956
1957         /* copy all the addresses into the udd */
1958         mc = &nctrl.udd[0];
1959         netdev_for_each_mc_addr(ha, netdev) {
1960                 *mc = 0;
1961                 memcpy(((u8 *)mc) + 2, ha->addr, ETH_ALEN);
1962                 /* no need to swap bytes */
1963
1964                 if (++mc > &nctrl.udd[mc_count])
1965                         break;
1966         }
1967
1968         /* Apparently, any activity in this call from the kernel has to
1969          * be atomic. So we won't wait for response.
1970          */
1971
1972         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
1973         if (ret) {
1974                 dev_err(&oct->pci_dev->dev, "DEVFLAGS change failed in core (ret: 0x%x)\n",
1975                         ret);
1976         }
1977 }
1978
1979 /**
1980  * \brief Net device set_mac_address
1981  * @param netdev network device
1982  */
1983 static int liquidio_set_mac(struct net_device *netdev, void *p)
1984 {
1985         int ret = 0;
1986         struct lio *lio = GET_LIO(netdev);
1987         struct octeon_device *oct = lio->oct_dev;
1988         struct sockaddr *addr = (struct sockaddr *)p;
1989         struct octnic_ctrl_pkt nctrl;
1990
1991         if (!is_valid_ether_addr(addr->sa_data))
1992                 return -EADDRNOTAVAIL;
1993
1994         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
1995
1996         nctrl.ncmd.u64 = 0;
1997         nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR;
1998         nctrl.ncmd.s.param1 = 0;
1999         nctrl.ncmd.s.more = 1;
2000         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2001         nctrl.netpndev = (u64)netdev;
2002
2003         nctrl.udd[0] = 0;
2004         /* The MAC Address is presented in network byte order. */
2005         memcpy((u8 *)&nctrl.udd[0] + 2, addr->sa_data, ETH_ALEN);
2006
2007         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2008         if (ret < 0) {
2009                 dev_err(&oct->pci_dev->dev, "MAC Address change failed\n");
2010                 return -ENOMEM;
2011         }
2012
2013         if (nctrl.sc_status) {
2014                 dev_err(&oct->pci_dev->dev,
2015                         "%s: MAC Address change failed. sc return=%x\n",
2016                          __func__, nctrl.sc_status);
2017                 return -EIO;
2018         }
2019
2020         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2021         memcpy(((u8 *)&lio->linfo.hw_addr) + 2, addr->sa_data, ETH_ALEN);
2022
2023         return 0;
2024 }
2025
2026 static void
2027 liquidio_get_stats64(struct net_device *netdev,
2028                      struct rtnl_link_stats64 *lstats)
2029 {
2030         struct lio *lio = GET_LIO(netdev);
2031         struct octeon_device *oct;
2032         u64 pkts = 0, drop = 0, bytes = 0;
2033         struct oct_droq_stats *oq_stats;
2034         struct oct_iq_stats *iq_stats;
2035         int i, iq_no, oq_no;
2036
2037         oct = lio->oct_dev;
2038
2039         if (ifstate_check(lio, LIO_IFSTATE_RESETTING))
2040                 return;
2041
2042         for (i = 0; i < oct->num_iqs; i++) {
2043                 iq_no = lio->linfo.txpciq[i].s.q_no;
2044                 iq_stats = &oct->instr_queue[iq_no]->stats;
2045                 pkts += iq_stats->tx_done;
2046                 drop += iq_stats->tx_dropped;
2047                 bytes += iq_stats->tx_tot_bytes;
2048         }
2049
2050         lstats->tx_packets = pkts;
2051         lstats->tx_bytes = bytes;
2052         lstats->tx_dropped = drop;
2053
2054         pkts = 0;
2055         drop = 0;
2056         bytes = 0;
2057
2058         for (i = 0; i < oct->num_oqs; i++) {
2059                 oq_no = lio->linfo.rxpciq[i].s.q_no;
2060                 oq_stats = &oct->droq[oq_no]->stats;
2061                 pkts += oq_stats->rx_pkts_received;
2062                 drop += (oq_stats->rx_dropped +
2063                          oq_stats->dropped_nodispatch +
2064                          oq_stats->dropped_toomany +
2065                          oq_stats->dropped_nomem);
2066                 bytes += oq_stats->rx_bytes_received;
2067         }
2068
2069         lstats->rx_bytes = bytes;
2070         lstats->rx_packets = pkts;
2071         lstats->rx_dropped = drop;
2072
2073         lstats->multicast = oct->link_stats.fromwire.fw_total_mcast;
2074         lstats->collisions = oct->link_stats.fromhost.total_collisions;
2075
2076         /* detailed rx_errors: */
2077         lstats->rx_length_errors = oct->link_stats.fromwire.l2_err;
2078         /* recved pkt with crc error    */
2079         lstats->rx_crc_errors = oct->link_stats.fromwire.fcs_err;
2080         /* recv'd frame alignment error */
2081         lstats->rx_frame_errors = oct->link_stats.fromwire.frame_err;
2082         /* recv'r fifo overrun */
2083         lstats->rx_fifo_errors = oct->link_stats.fromwire.fifo_err;
2084
2085         lstats->rx_errors = lstats->rx_length_errors + lstats->rx_crc_errors +
2086                 lstats->rx_frame_errors + lstats->rx_fifo_errors;
2087
2088         /* detailed tx_errors */
2089         lstats->tx_aborted_errors = oct->link_stats.fromhost.fw_err_pko;
2090         lstats->tx_carrier_errors = oct->link_stats.fromhost.fw_err_link;
2091         lstats->tx_fifo_errors = oct->link_stats.fromhost.fifo_err;
2092
2093         lstats->tx_errors = lstats->tx_aborted_errors +
2094                 lstats->tx_carrier_errors +
2095                 lstats->tx_fifo_errors;
2096 }
2097
2098 /**
2099  * \brief Handler for SIOCSHWTSTAMP ioctl
2100  * @param netdev network device
2101  * @param ifr interface request
2102  * @param cmd command
2103  */
2104 static int hwtstamp_ioctl(struct net_device *netdev, struct ifreq *ifr)
2105 {
2106         struct hwtstamp_config conf;
2107         struct lio *lio = GET_LIO(netdev);
2108
2109         if (copy_from_user(&conf, ifr->ifr_data, sizeof(conf)))
2110                 return -EFAULT;
2111
2112         if (conf.flags)
2113                 return -EINVAL;
2114
2115         switch (conf.tx_type) {
2116         case HWTSTAMP_TX_ON:
2117         case HWTSTAMP_TX_OFF:
2118                 break;
2119         default:
2120                 return -ERANGE;
2121         }
2122
2123         switch (conf.rx_filter) {
2124         case HWTSTAMP_FILTER_NONE:
2125                 break;
2126         case HWTSTAMP_FILTER_ALL:
2127         case HWTSTAMP_FILTER_SOME:
2128         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
2129         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
2130         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
2131         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
2132         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
2133         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
2134         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
2135         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
2136         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
2137         case HWTSTAMP_FILTER_PTP_V2_EVENT:
2138         case HWTSTAMP_FILTER_PTP_V2_SYNC:
2139         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
2140         case HWTSTAMP_FILTER_NTP_ALL:
2141                 conf.rx_filter = HWTSTAMP_FILTER_ALL;
2142                 break;
2143         default:
2144                 return -ERANGE;
2145         }
2146
2147         if (conf.rx_filter == HWTSTAMP_FILTER_ALL)
2148                 ifstate_set(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
2149
2150         else
2151                 ifstate_reset(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
2152
2153         return copy_to_user(ifr->ifr_data, &conf, sizeof(conf)) ? -EFAULT : 0;
2154 }
2155
2156 /**
2157  * \brief ioctl handler
2158  * @param netdev network device
2159  * @param ifr interface request
2160  * @param cmd command
2161  */
2162 static int liquidio_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2163 {
2164         struct lio *lio = GET_LIO(netdev);
2165
2166         switch (cmd) {
2167         case SIOCSHWTSTAMP:
2168                 if (lio->oct_dev->ptp_enable)
2169                         return hwtstamp_ioctl(netdev, ifr);
2170                 fallthrough;
2171         default:
2172                 return -EOPNOTSUPP;
2173         }
2174 }
2175
2176 /**
2177  * \brief handle a Tx timestamp response
2178  * @param status response status
2179  * @param buf pointer to skb
2180  */
2181 static void handle_timestamp(struct octeon_device *oct,
2182                              u32 status,
2183                              void *buf)
2184 {
2185         struct octnet_buf_free_info *finfo;
2186         struct octeon_soft_command *sc;
2187         struct oct_timestamp_resp *resp;
2188         struct lio *lio;
2189         struct sk_buff *skb = (struct sk_buff *)buf;
2190
2191         finfo = (struct octnet_buf_free_info *)skb->cb;
2192         lio = finfo->lio;
2193         sc = finfo->sc;
2194         oct = lio->oct_dev;
2195         resp = (struct oct_timestamp_resp *)sc->virtrptr;
2196
2197         if (status != OCTEON_REQUEST_DONE) {
2198                 dev_err(&oct->pci_dev->dev, "Tx timestamp instruction failed. Status: %llx\n",
2199                         CVM_CAST64(status));
2200                 resp->timestamp = 0;
2201         }
2202
2203         octeon_swap_8B_data(&resp->timestamp, 1);
2204
2205         if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) != 0)) {
2206                 struct skb_shared_hwtstamps ts;
2207                 u64 ns = resp->timestamp;
2208
2209                 netif_info(lio, tx_done, lio->netdev,
2210                            "Got resulting SKBTX_HW_TSTAMP skb=%p ns=%016llu\n",
2211                            skb, (unsigned long long)ns);
2212                 ts.hwtstamp = ns_to_ktime(ns + lio->ptp_adjust);
2213                 skb_tstamp_tx(skb, &ts);
2214         }
2215
2216         octeon_free_soft_command(oct, sc);
2217         tx_buffer_free(skb);
2218 }
2219
2220 /* \brief Send a data packet that will be timestamped
2221  * @param oct octeon device
2222  * @param ndata pointer to network data
2223  * @param finfo pointer to private network data
2224  */
2225 static inline int send_nic_timestamp_pkt(struct octeon_device *oct,
2226                                          struct octnic_data_pkt *ndata,
2227                                          struct octnet_buf_free_info *finfo,
2228                                          int xmit_more)
2229 {
2230         int retval;
2231         struct octeon_soft_command *sc;
2232         struct lio *lio;
2233         int ring_doorbell;
2234         u32 len;
2235
2236         lio = finfo->lio;
2237
2238         sc = octeon_alloc_soft_command_resp(oct, &ndata->cmd,
2239                                             sizeof(struct oct_timestamp_resp));
2240         finfo->sc = sc;
2241
2242         if (!sc) {
2243                 dev_err(&oct->pci_dev->dev, "No memory for timestamped data packet\n");
2244                 return IQ_SEND_FAILED;
2245         }
2246
2247         if (ndata->reqtype == REQTYPE_NORESP_NET)
2248                 ndata->reqtype = REQTYPE_RESP_NET;
2249         else if (ndata->reqtype == REQTYPE_NORESP_NET_SG)
2250                 ndata->reqtype = REQTYPE_RESP_NET_SG;
2251
2252         sc->callback = handle_timestamp;
2253         sc->callback_arg = finfo->skb;
2254         sc->iq_no = ndata->q_no;
2255
2256         if (OCTEON_CN23XX_PF(oct))
2257                 len = (u32)((struct octeon_instr_ih3 *)
2258                             (&sc->cmd.cmd3.ih3))->dlengsz;
2259         else
2260                 len = (u32)((struct octeon_instr_ih2 *)
2261                             (&sc->cmd.cmd2.ih2))->dlengsz;
2262
2263         ring_doorbell = !xmit_more;
2264
2265         retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd,
2266                                      sc, len, ndata->reqtype);
2267
2268         if (retval == IQ_SEND_FAILED) {
2269                 dev_err(&oct->pci_dev->dev, "timestamp data packet failed status: %x\n",
2270                         retval);
2271                 octeon_free_soft_command(oct, sc);
2272         } else {
2273                 netif_info(lio, tx_queued, lio->netdev, "Queued timestamp packet\n");
2274         }
2275
2276         return retval;
2277 }
2278
2279 /** \brief Transmit networks packets to the Octeon interface
2280  * @param skbuff   skbuff struct to be passed to network layer.
2281  * @param netdev    pointer to network device
2282  * @returns whether the packet was transmitted to the device okay or not
2283  *             (NETDEV_TX_OK or NETDEV_TX_BUSY)
2284  */
2285 static netdev_tx_t liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
2286 {
2287         struct lio *lio;
2288         struct octnet_buf_free_info *finfo;
2289         union octnic_cmd_setup cmdsetup;
2290         struct octnic_data_pkt ndata;
2291         struct octeon_device *oct;
2292         struct oct_iq_stats *stats;
2293         struct octeon_instr_irh *irh;
2294         union tx_info *tx_info;
2295         int status = 0;
2296         int q_idx = 0, iq_no = 0;
2297         int j, xmit_more = 0;
2298         u64 dptr = 0;
2299         u32 tag = 0;
2300
2301         lio = GET_LIO(netdev);
2302         oct = lio->oct_dev;
2303
2304         q_idx = skb_iq(oct, skb);
2305         tag = q_idx;
2306         iq_no = lio->linfo.txpciq[q_idx].s.q_no;
2307
2308         stats = &oct->instr_queue[iq_no]->stats;
2309
2310         /* Check for all conditions in which the current packet cannot be
2311          * transmitted.
2312          */
2313         if (!(atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) ||
2314             (!lio->linfo.link.s.link_up) ||
2315             (skb->len <= 0)) {
2316                 netif_info(lio, tx_err, lio->netdev,
2317                            "Transmit failed link_status : %d\n",
2318                            lio->linfo.link.s.link_up);
2319                 goto lio_xmit_failed;
2320         }
2321
2322         /* Use space in skb->cb to store info used to unmap and
2323          * free the buffers.
2324          */
2325         finfo = (struct octnet_buf_free_info *)skb->cb;
2326         finfo->lio = lio;
2327         finfo->skb = skb;
2328         finfo->sc = NULL;
2329
2330         /* Prepare the attributes for the data to be passed to OSI. */
2331         memset(&ndata, 0, sizeof(struct octnic_data_pkt));
2332
2333         ndata.buf = (void *)finfo;
2334
2335         ndata.q_no = iq_no;
2336
2337         if (octnet_iq_is_full(oct, ndata.q_no)) {
2338                 /* defer sending if queue is full */
2339                 netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n",
2340                            ndata.q_no);
2341                 stats->tx_iq_busy++;
2342                 return NETDEV_TX_BUSY;
2343         }
2344
2345         /* pr_info(" XMIT - valid Qs: %d, 1st Q no: %d, cpu:  %d, q_no:%d\n",
2346          *      lio->linfo.num_txpciq, lio->txq, cpu, ndata.q_no);
2347          */
2348
2349         ndata.datasize = skb->len;
2350
2351         cmdsetup.u64 = 0;
2352         cmdsetup.s.iq_no = iq_no;
2353
2354         if (skb->ip_summed == CHECKSUM_PARTIAL) {
2355                 if (skb->encapsulation) {
2356                         cmdsetup.s.tnl_csum = 1;
2357                         stats->tx_vxlan++;
2358                 } else {
2359                         cmdsetup.s.transport_csum = 1;
2360                 }
2361         }
2362         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
2363                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2364                 cmdsetup.s.timestamp = 1;
2365         }
2366
2367         if (skb_shinfo(skb)->nr_frags == 0) {
2368                 cmdsetup.s.u.datasize = skb->len;
2369                 octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
2370
2371                 /* Offload checksum calculation for TCP/UDP packets */
2372                 dptr = dma_map_single(&oct->pci_dev->dev,
2373                                       skb->data,
2374                                       skb->len,
2375                                       DMA_TO_DEVICE);
2376                 if (dma_mapping_error(&oct->pci_dev->dev, dptr)) {
2377                         dev_err(&oct->pci_dev->dev, "%s DMA mapping error 1\n",
2378                                 __func__);
2379                         stats->tx_dmamap_fail++;
2380                         return NETDEV_TX_BUSY;
2381                 }
2382
2383                 if (OCTEON_CN23XX_PF(oct))
2384                         ndata.cmd.cmd3.dptr = dptr;
2385                 else
2386                         ndata.cmd.cmd2.dptr = dptr;
2387                 finfo->dptr = dptr;
2388                 ndata.reqtype = REQTYPE_NORESP_NET;
2389
2390         } else {
2391                 int i, frags;
2392                 skb_frag_t *frag;
2393                 struct octnic_gather *g;
2394
2395                 spin_lock(&lio->glist_lock[q_idx]);
2396                 g = (struct octnic_gather *)
2397                         lio_list_delete_head(&lio->glist[q_idx]);
2398                 spin_unlock(&lio->glist_lock[q_idx]);
2399
2400                 if (!g) {
2401                         netif_info(lio, tx_err, lio->netdev,
2402                                    "Transmit scatter gather: glist null!\n");
2403                         goto lio_xmit_failed;
2404                 }
2405
2406                 cmdsetup.s.gather = 1;
2407                 cmdsetup.s.u.gatherptrs = (skb_shinfo(skb)->nr_frags + 1);
2408                 octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
2409
2410                 memset(g->sg, 0, g->sg_size);
2411
2412                 g->sg[0].ptr[0] = dma_map_single(&oct->pci_dev->dev,
2413                                                  skb->data,
2414                                                  (skb->len - skb->data_len),
2415                                                  DMA_TO_DEVICE);
2416                 if (dma_mapping_error(&oct->pci_dev->dev, g->sg[0].ptr[0])) {
2417                         dev_err(&oct->pci_dev->dev, "%s DMA mapping error 2\n",
2418                                 __func__);
2419                         stats->tx_dmamap_fail++;
2420                         return NETDEV_TX_BUSY;
2421                 }
2422                 add_sg_size(&g->sg[0], (skb->len - skb->data_len), 0);
2423
2424                 frags = skb_shinfo(skb)->nr_frags;
2425                 i = 1;
2426                 while (frags--) {
2427                         frag = &skb_shinfo(skb)->frags[i - 1];
2428
2429                         g->sg[(i >> 2)].ptr[(i & 3)] =
2430                                 skb_frag_dma_map(&oct->pci_dev->dev,
2431                                                  frag, 0, skb_frag_size(frag),
2432                                                  DMA_TO_DEVICE);
2433
2434                         if (dma_mapping_error(&oct->pci_dev->dev,
2435                                               g->sg[i >> 2].ptr[i & 3])) {
2436                                 dma_unmap_single(&oct->pci_dev->dev,
2437                                                  g->sg[0].ptr[0],
2438                                                  skb->len - skb->data_len,
2439                                                  DMA_TO_DEVICE);
2440                                 for (j = 1; j < i; j++) {
2441                                         frag = &skb_shinfo(skb)->frags[j - 1];
2442                                         dma_unmap_page(&oct->pci_dev->dev,
2443                                                        g->sg[j >> 2].ptr[j & 3],
2444                                                        skb_frag_size(frag),
2445                                                        DMA_TO_DEVICE);
2446                                 }
2447                                 dev_err(&oct->pci_dev->dev, "%s DMA mapping error 3\n",
2448                                         __func__);
2449                                 return NETDEV_TX_BUSY;
2450                         }
2451
2452                         add_sg_size(&g->sg[(i >> 2)], skb_frag_size(frag),
2453                                     (i & 3));
2454                         i++;
2455                 }
2456
2457                 dptr = g->sg_dma_ptr;
2458
2459                 if (OCTEON_CN23XX_PF(oct))
2460                         ndata.cmd.cmd3.dptr = dptr;
2461                 else
2462                         ndata.cmd.cmd2.dptr = dptr;
2463                 finfo->dptr = dptr;
2464                 finfo->g = g;
2465
2466                 ndata.reqtype = REQTYPE_NORESP_NET_SG;
2467         }
2468
2469         if (OCTEON_CN23XX_PF(oct)) {
2470                 irh = (struct octeon_instr_irh *)&ndata.cmd.cmd3.irh;
2471                 tx_info = (union tx_info *)&ndata.cmd.cmd3.ossp[0];
2472         } else {
2473                 irh = (struct octeon_instr_irh *)&ndata.cmd.cmd2.irh;
2474                 tx_info = (union tx_info *)&ndata.cmd.cmd2.ossp[0];
2475         }
2476
2477         if (skb_shinfo(skb)->gso_size) {
2478                 tx_info->s.gso_size = skb_shinfo(skb)->gso_size;
2479                 tx_info->s.gso_segs = skb_shinfo(skb)->gso_segs;
2480                 stats->tx_gso++;
2481         }
2482
2483         /* HW insert VLAN tag */
2484         if (skb_vlan_tag_present(skb)) {
2485                 irh->priority = skb_vlan_tag_get(skb) >> 13;
2486                 irh->vlan = skb_vlan_tag_get(skb) & 0xfff;
2487         }
2488
2489         xmit_more = netdev_xmit_more();
2490
2491         if (unlikely(cmdsetup.s.timestamp))
2492                 status = send_nic_timestamp_pkt(oct, &ndata, finfo, xmit_more);
2493         else
2494                 status = octnet_send_nic_data_pkt(oct, &ndata, xmit_more);
2495         if (status == IQ_SEND_FAILED)
2496                 goto lio_xmit_failed;
2497
2498         netif_info(lio, tx_queued, lio->netdev, "Transmit queued successfully\n");
2499
2500         if (status == IQ_SEND_STOP)
2501                 netif_stop_subqueue(netdev, q_idx);
2502
2503         netif_trans_update(netdev);
2504
2505         if (tx_info->s.gso_segs)
2506                 stats->tx_done += tx_info->s.gso_segs;
2507         else
2508                 stats->tx_done++;
2509         stats->tx_tot_bytes += ndata.datasize;
2510
2511         return NETDEV_TX_OK;
2512
2513 lio_xmit_failed:
2514         stats->tx_dropped++;
2515         netif_info(lio, tx_err, lio->netdev, "IQ%d Transmit dropped:%llu\n",
2516                    iq_no, stats->tx_dropped);
2517         if (dptr)
2518                 dma_unmap_single(&oct->pci_dev->dev, dptr,
2519                                  ndata.datasize, DMA_TO_DEVICE);
2520
2521         octeon_ring_doorbell_locked(oct, iq_no);
2522
2523         tx_buffer_free(skb);
2524         return NETDEV_TX_OK;
2525 }
2526
2527 /** \brief Network device Tx timeout
2528  * @param netdev    pointer to network device
2529  */
2530 static void liquidio_tx_timeout(struct net_device *netdev, unsigned int txqueue)
2531 {
2532         struct lio *lio;
2533
2534         lio = GET_LIO(netdev);
2535
2536         netif_info(lio, tx_err, lio->netdev,
2537                    "Transmit timeout tx_dropped:%ld, waking up queues now!!\n",
2538                    netdev->stats.tx_dropped);
2539         netif_trans_update(netdev);
2540         wake_txqs(netdev);
2541 }
2542
2543 static int liquidio_vlan_rx_add_vid(struct net_device *netdev,
2544                                     __be16 proto __attribute__((unused)),
2545                                     u16 vid)
2546 {
2547         struct lio *lio = GET_LIO(netdev);
2548         struct octeon_device *oct = lio->oct_dev;
2549         struct octnic_ctrl_pkt nctrl;
2550         int ret = 0;
2551
2552         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2553
2554         nctrl.ncmd.u64 = 0;
2555         nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER;
2556         nctrl.ncmd.s.param1 = vid;
2557         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2558         nctrl.netpndev = (u64)netdev;
2559         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2560
2561         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2562         if (ret) {
2563                 dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n",
2564                         ret);
2565                 if (ret > 0)
2566                         ret = -EIO;
2567         }
2568
2569         return ret;
2570 }
2571
2572 static int liquidio_vlan_rx_kill_vid(struct net_device *netdev,
2573                                      __be16 proto __attribute__((unused)),
2574                                      u16 vid)
2575 {
2576         struct lio *lio = GET_LIO(netdev);
2577         struct octeon_device *oct = lio->oct_dev;
2578         struct octnic_ctrl_pkt nctrl;
2579         int ret = 0;
2580
2581         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2582
2583         nctrl.ncmd.u64 = 0;
2584         nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER;
2585         nctrl.ncmd.s.param1 = vid;
2586         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2587         nctrl.netpndev = (u64)netdev;
2588         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2589
2590         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2591         if (ret) {
2592                 dev_err(&oct->pci_dev->dev, "Del VLAN filter failed in core (ret: 0x%x)\n",
2593                         ret);
2594                 if (ret > 0)
2595                         ret = -EIO;
2596         }
2597         return ret;
2598 }
2599
2600 /** Sending command to enable/disable RX checksum offload
2601  * @param netdev                pointer to network device
2602  * @param command               OCTNET_CMD_TNL_RX_CSUM_CTL
2603  * @param rx_cmd_bit            OCTNET_CMD_RXCSUM_ENABLE/
2604  *                              OCTNET_CMD_RXCSUM_DISABLE
2605  * @returns                     SUCCESS or FAILURE
2606  */
2607 static int liquidio_set_rxcsum_command(struct net_device *netdev, int command,
2608                                        u8 rx_cmd)
2609 {
2610         struct lio *lio = GET_LIO(netdev);
2611         struct octeon_device *oct = lio->oct_dev;
2612         struct octnic_ctrl_pkt nctrl;
2613         int ret = 0;
2614
2615         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2616
2617         nctrl.ncmd.u64 = 0;
2618         nctrl.ncmd.s.cmd = command;
2619         nctrl.ncmd.s.param1 = rx_cmd;
2620         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2621         nctrl.netpndev = (u64)netdev;
2622         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2623
2624         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2625         if (ret) {
2626                 dev_err(&oct->pci_dev->dev,
2627                         "DEVFLAGS RXCSUM change failed in core(ret:0x%x)\n",
2628                         ret);
2629                 if (ret > 0)
2630                         ret = -EIO;
2631         }
2632         return ret;
2633 }
2634
2635 /** Sending command to add/delete VxLAN UDP port to firmware
2636  * @param netdev                pointer to network device
2637  * @param command               OCTNET_CMD_VXLAN_PORT_CONFIG
2638  * @param vxlan_port            VxLAN port to be added or deleted
2639  * @param vxlan_cmd_bit         OCTNET_CMD_VXLAN_PORT_ADD,
2640  *                              OCTNET_CMD_VXLAN_PORT_DEL
2641  * @returns                     SUCCESS or FAILURE
2642  */
2643 static int liquidio_vxlan_port_command(struct net_device *netdev, int command,
2644                                        u16 vxlan_port, u8 vxlan_cmd_bit)
2645 {
2646         struct lio *lio = GET_LIO(netdev);
2647         struct octeon_device *oct = lio->oct_dev;
2648         struct octnic_ctrl_pkt nctrl;
2649         int ret = 0;
2650
2651         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2652
2653         nctrl.ncmd.u64 = 0;
2654         nctrl.ncmd.s.cmd = command;
2655         nctrl.ncmd.s.more = vxlan_cmd_bit;
2656         nctrl.ncmd.s.param1 = vxlan_port;
2657         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2658         nctrl.netpndev = (u64)netdev;
2659         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2660
2661         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2662         if (ret) {
2663                 dev_err(&oct->pci_dev->dev,
2664                         "VxLAN port add/delete failed in core (ret:0x%x)\n",
2665                         ret);
2666                 if (ret > 0)
2667                         ret = -EIO;
2668         }
2669         return ret;
2670 }
2671
2672 static int liquidio_udp_tunnel_set_port(struct net_device *netdev,
2673                                         unsigned int table, unsigned int entry,
2674                                         struct udp_tunnel_info *ti)
2675 {
2676         return liquidio_vxlan_port_command(netdev,
2677                                            OCTNET_CMD_VXLAN_PORT_CONFIG,
2678                                            htons(ti->port),
2679                                            OCTNET_CMD_VXLAN_PORT_ADD);
2680 }
2681
2682 static int liquidio_udp_tunnel_unset_port(struct net_device *netdev,
2683                                           unsigned int table,
2684                                           unsigned int entry,
2685                                           struct udp_tunnel_info *ti)
2686 {
2687         return liquidio_vxlan_port_command(netdev,
2688                                            OCTNET_CMD_VXLAN_PORT_CONFIG,
2689                                            htons(ti->port),
2690                                            OCTNET_CMD_VXLAN_PORT_DEL);
2691 }
2692
2693 static const struct udp_tunnel_nic_info liquidio_udp_tunnels = {
2694         .set_port       = liquidio_udp_tunnel_set_port,
2695         .unset_port     = liquidio_udp_tunnel_unset_port,
2696         .tables         = {
2697                 { .n_entries = 1024, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, },
2698         },
2699 };
2700
2701 /** \brief Net device fix features
2702  * @param netdev  pointer to network device
2703  * @param request features requested
2704  * @returns updated features list
2705  */
2706 static netdev_features_t liquidio_fix_features(struct net_device *netdev,
2707                                                netdev_features_t request)
2708 {
2709         struct lio *lio = netdev_priv(netdev);
2710
2711         if ((request & NETIF_F_RXCSUM) &&
2712             !(lio->dev_capability & NETIF_F_RXCSUM))
2713                 request &= ~NETIF_F_RXCSUM;
2714
2715         if ((request & NETIF_F_HW_CSUM) &&
2716             !(lio->dev_capability & NETIF_F_HW_CSUM))
2717                 request &= ~NETIF_F_HW_CSUM;
2718
2719         if ((request & NETIF_F_TSO) && !(lio->dev_capability & NETIF_F_TSO))
2720                 request &= ~NETIF_F_TSO;
2721
2722         if ((request & NETIF_F_TSO6) && !(lio->dev_capability & NETIF_F_TSO6))
2723                 request &= ~NETIF_F_TSO6;
2724
2725         if ((request & NETIF_F_LRO) && !(lio->dev_capability & NETIF_F_LRO))
2726                 request &= ~NETIF_F_LRO;
2727
2728         /*Disable LRO if RXCSUM is off */
2729         if (!(request & NETIF_F_RXCSUM) && (netdev->features & NETIF_F_LRO) &&
2730             (lio->dev_capability & NETIF_F_LRO))
2731                 request &= ~NETIF_F_LRO;
2732
2733         if ((request & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2734             !(lio->dev_capability & NETIF_F_HW_VLAN_CTAG_FILTER))
2735                 request &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
2736
2737         return request;
2738 }
2739
2740 /** \brief Net device set features
2741  * @param netdev  pointer to network device
2742  * @param features features to enable/disable
2743  */
2744 static int liquidio_set_features(struct net_device *netdev,
2745                                  netdev_features_t features)
2746 {
2747         struct lio *lio = netdev_priv(netdev);
2748
2749         if ((features & NETIF_F_LRO) &&
2750             (lio->dev_capability & NETIF_F_LRO) &&
2751             !(netdev->features & NETIF_F_LRO))
2752                 liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
2753                                      OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
2754         else if (!(features & NETIF_F_LRO) &&
2755                  (lio->dev_capability & NETIF_F_LRO) &&
2756                  (netdev->features & NETIF_F_LRO))
2757                 liquidio_set_feature(netdev, OCTNET_CMD_LRO_DISABLE,
2758                                      OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
2759
2760         /* Sending command to firmware to enable/disable RX checksum
2761          * offload settings using ethtool
2762          */
2763         if (!(netdev->features & NETIF_F_RXCSUM) &&
2764             (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
2765             (features & NETIF_F_RXCSUM))
2766                 liquidio_set_rxcsum_command(netdev,
2767                                             OCTNET_CMD_TNL_RX_CSUM_CTL,
2768                                             OCTNET_CMD_RXCSUM_ENABLE);
2769         else if ((netdev->features & NETIF_F_RXCSUM) &&
2770                  (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
2771                  !(features & NETIF_F_RXCSUM))
2772                 liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
2773                                             OCTNET_CMD_RXCSUM_DISABLE);
2774
2775         if ((features & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2776             (lio->dev_capability & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2777             !(netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER))
2778                 liquidio_set_feature(netdev, OCTNET_CMD_VLAN_FILTER_CTL,
2779                                      OCTNET_CMD_VLAN_FILTER_ENABLE);
2780         else if (!(features & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2781                  (lio->dev_capability & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2782                  (netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER))
2783                 liquidio_set_feature(netdev, OCTNET_CMD_VLAN_FILTER_CTL,
2784                                      OCTNET_CMD_VLAN_FILTER_DISABLE);
2785
2786         return 0;
2787 }
2788
2789 static int __liquidio_set_vf_mac(struct net_device *netdev, int vfidx,
2790                                  u8 *mac, bool is_admin_assigned)
2791 {
2792         struct lio *lio = GET_LIO(netdev);
2793         struct octeon_device *oct = lio->oct_dev;
2794         struct octnic_ctrl_pkt nctrl;
2795         int ret = 0;
2796
2797         if (!is_valid_ether_addr(mac))
2798                 return -EINVAL;
2799
2800         if (vfidx < 0 || vfidx >= oct->sriov_info.max_vfs)
2801                 return -EINVAL;
2802
2803         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2804
2805         nctrl.ncmd.u64 = 0;
2806         nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR;
2807         /* vfidx is 0 based, but vf_num (param1) is 1 based */
2808         nctrl.ncmd.s.param1 = vfidx + 1;
2809         nctrl.ncmd.s.more = 1;
2810         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2811         nctrl.netpndev = (u64)netdev;
2812         if (is_admin_assigned) {
2813                 nctrl.ncmd.s.param2 = true;
2814                 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2815         }
2816
2817         nctrl.udd[0] = 0;
2818         /* The MAC Address is presented in network byte order. */
2819         ether_addr_copy((u8 *)&nctrl.udd[0] + 2, mac);
2820
2821         oct->sriov_info.vf_macaddr[vfidx] = nctrl.udd[0];
2822
2823         ret = octnet_send_nic_ctrl_pkt(oct, &nctrl);
2824         if (ret > 0)
2825                 ret = -EIO;
2826
2827         return ret;
2828 }
2829
2830 static int liquidio_set_vf_mac(struct net_device *netdev, int vfidx, u8 *mac)
2831 {
2832         struct lio *lio = GET_LIO(netdev);
2833         struct octeon_device *oct = lio->oct_dev;
2834         int retval;
2835
2836         if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
2837                 return -EINVAL;
2838
2839         retval = __liquidio_set_vf_mac(netdev, vfidx, mac, true);
2840         if (!retval)
2841                 cn23xx_tell_vf_its_macaddr_changed(oct, vfidx, mac);
2842
2843         return retval;
2844 }
2845
2846 static int liquidio_set_vf_spoofchk(struct net_device *netdev, int vfidx,
2847                                     bool enable)
2848 {
2849         struct lio *lio = GET_LIO(netdev);
2850         struct octeon_device *oct = lio->oct_dev;
2851         struct octnic_ctrl_pkt nctrl;
2852         int retval;
2853
2854         if (!(oct->fw_info.app_cap_flags & LIQUIDIO_SPOOFCHK_CAP)) {
2855                 netif_info(lio, drv, lio->netdev,
2856                            "firmware does not support spoofchk\n");
2857                 return -EOPNOTSUPP;
2858         }
2859
2860         if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced) {
2861                 netif_info(lio, drv, lio->netdev, "Invalid vfidx %d\n", vfidx);
2862                 return -EINVAL;
2863         }
2864
2865         if (enable) {
2866                 if (oct->sriov_info.vf_spoofchk[vfidx])
2867                         return 0;
2868         } else {
2869                 /* Clear */
2870                 if (!oct->sriov_info.vf_spoofchk[vfidx])
2871                         return 0;
2872         }
2873
2874         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2875         nctrl.ncmd.s.cmdgroup = OCTNET_CMD_GROUP1;
2876         nctrl.ncmd.s.cmd = OCTNET_CMD_SET_VF_SPOOFCHK;
2877         nctrl.ncmd.s.param1 =
2878                 vfidx + 1; /* vfidx is 0 based,
2879                             * but vf_num (param1) is 1 based
2880                             */
2881         nctrl.ncmd.s.param2 = enable;
2882         nctrl.ncmd.s.more = 0;
2883         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2884         nctrl.cb_fn = NULL;
2885
2886         retval = octnet_send_nic_ctrl_pkt(oct, &nctrl);
2887
2888         if (retval) {
2889                 netif_info(lio, drv, lio->netdev,
2890                            "Failed to set VF %d spoofchk %s\n", vfidx,
2891                         enable ? "on" : "off");
2892                 return -1;
2893         }
2894
2895         oct->sriov_info.vf_spoofchk[vfidx] = enable;
2896         netif_info(lio, drv, lio->netdev, "VF %u spoofchk is %s\n", vfidx,
2897                    enable ? "on" : "off");
2898
2899         return 0;
2900 }
2901
2902 static int liquidio_set_vf_vlan(struct net_device *netdev, int vfidx,
2903                                 u16 vlan, u8 qos, __be16 vlan_proto)
2904 {
2905         struct lio *lio = GET_LIO(netdev);
2906         struct octeon_device *oct = lio->oct_dev;
2907         struct octnic_ctrl_pkt nctrl;
2908         u16 vlantci;
2909         int ret = 0;
2910
2911         if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
2912                 return -EINVAL;
2913
2914         if (vlan_proto != htons(ETH_P_8021Q))
2915                 return -EPROTONOSUPPORT;
2916
2917         if (vlan >= VLAN_N_VID || qos > 7)
2918                 return -EINVAL;
2919
2920         if (vlan)
2921                 vlantci = vlan | (u16)qos << VLAN_PRIO_SHIFT;
2922         else
2923                 vlantci = 0;
2924
2925         if (oct->sriov_info.vf_vlantci[vfidx] == vlantci)
2926                 return 0;
2927
2928         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2929
2930         if (vlan)
2931                 nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER;
2932         else
2933                 nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER;
2934
2935         nctrl.ncmd.s.param1 = vlantci;
2936         nctrl.ncmd.s.param2 =
2937             vfidx + 1; /* vfidx is 0 based, but vf_num (param2) is 1 based */
2938         nctrl.ncmd.s.more = 0;
2939         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2940         nctrl.cb_fn = NULL;
2941
2942         ret = octnet_send_nic_ctrl_pkt(oct, &nctrl);
2943         if (ret) {
2944                 if (ret > 0)
2945                         ret = -EIO;
2946                 return ret;
2947         }
2948
2949         oct->sriov_info.vf_vlantci[vfidx] = vlantci;
2950
2951         return ret;
2952 }
2953
2954 static int liquidio_get_vf_config(struct net_device *netdev, int vfidx,
2955                                   struct ifla_vf_info *ivi)
2956 {
2957         struct lio *lio = GET_LIO(netdev);
2958         struct octeon_device *oct = lio->oct_dev;
2959         u8 *macaddr;
2960
2961         if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
2962                 return -EINVAL;
2963
2964         memset(ivi, 0, sizeof(struct ifla_vf_info));
2965
2966         ivi->vf = vfidx;
2967         macaddr = 2 + (u8 *)&oct->sriov_info.vf_macaddr[vfidx];
2968         ether_addr_copy(&ivi->mac[0], macaddr);
2969         ivi->vlan = oct->sriov_info.vf_vlantci[vfidx] & VLAN_VID_MASK;
2970         ivi->qos = oct->sriov_info.vf_vlantci[vfidx] >> VLAN_PRIO_SHIFT;
2971         if (oct->sriov_info.trusted_vf.active &&
2972             oct->sriov_info.trusted_vf.id == vfidx)
2973                 ivi->trusted = true;
2974         else
2975                 ivi->trusted = false;
2976         ivi->linkstate = oct->sriov_info.vf_linkstate[vfidx];
2977         ivi->spoofchk = oct->sriov_info.vf_spoofchk[vfidx];
2978         ivi->max_tx_rate = lio->linfo.link.s.speed;
2979         ivi->min_tx_rate = 0;
2980
2981         return 0;
2982 }
2983
2984 static int liquidio_send_vf_trust_cmd(struct lio *lio, int vfidx, bool trusted)
2985 {
2986         struct octeon_device *oct = lio->oct_dev;
2987         struct octeon_soft_command *sc;
2988         int retval;
2989
2990         sc = octeon_alloc_soft_command(oct, 0, 16, 0);
2991         if (!sc)
2992                 return -ENOMEM;
2993
2994         sc->iq_no = lio->linfo.txpciq[0].s.q_no;
2995
2996         /* vfidx is 0 based, but vf_num (param1) is 1 based */
2997         octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
2998                                     OPCODE_NIC_SET_TRUSTED_VF, 0, vfidx + 1,
2999                                     trusted);
3000
3001         init_completion(&sc->complete);
3002         sc->sc_status = OCTEON_REQUEST_PENDING;
3003
3004         retval = octeon_send_soft_command(oct, sc);
3005         if (retval == IQ_SEND_FAILED) {
3006                 octeon_free_soft_command(oct, sc);
3007                 retval = -1;
3008         } else {
3009                 /* Wait for response or timeout */
3010                 retval = wait_for_sc_completion_timeout(oct, sc, 0);
3011                 if (retval)
3012                         return (retval);
3013
3014                 WRITE_ONCE(sc->caller_is_done, true);
3015         }
3016
3017         return retval;
3018 }
3019
3020 static int liquidio_set_vf_trust(struct net_device *netdev, int vfidx,
3021                                  bool setting)
3022 {
3023         struct lio *lio = GET_LIO(netdev);
3024         struct octeon_device *oct = lio->oct_dev;
3025
3026         if (strcmp(oct->fw_info.liquidio_firmware_version, "1.7.1") < 0) {
3027                 /* trusted vf is not supported by firmware older than 1.7.1 */
3028                 return -EOPNOTSUPP;
3029         }
3030
3031         if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced) {
3032                 netif_info(lio, drv, lio->netdev, "Invalid vfidx %d\n", vfidx);
3033                 return -EINVAL;
3034         }
3035
3036         if (setting) {
3037                 /* Set */
3038
3039                 if (oct->sriov_info.trusted_vf.active &&
3040                     oct->sriov_info.trusted_vf.id == vfidx)
3041                         return 0;
3042
3043                 if (oct->sriov_info.trusted_vf.active) {
3044                         netif_info(lio, drv, lio->netdev, "More than one trusted VF is not allowed\n");
3045                         return -EPERM;
3046                 }
3047         } else {
3048                 /* Clear */
3049
3050                 if (!oct->sriov_info.trusted_vf.active)
3051                         return 0;
3052         }
3053
3054         if (!liquidio_send_vf_trust_cmd(lio, vfidx, setting)) {
3055                 if (setting) {
3056                         oct->sriov_info.trusted_vf.id = vfidx;
3057                         oct->sriov_info.trusted_vf.active = true;
3058                 } else {
3059                         oct->sriov_info.trusted_vf.active = false;
3060                 }
3061
3062                 netif_info(lio, drv, lio->netdev, "VF %u is %strusted\n", vfidx,
3063                            setting ? "" : "not ");
3064         } else {
3065                 netif_info(lio, drv, lio->netdev, "Failed to set VF trusted\n");
3066                 return -1;
3067         }
3068
3069         return 0;
3070 }
3071
3072 static int liquidio_set_vf_link_state(struct net_device *netdev, int vfidx,
3073                                       int linkstate)
3074 {
3075         struct lio *lio = GET_LIO(netdev);
3076         struct octeon_device *oct = lio->oct_dev;
3077         struct octnic_ctrl_pkt nctrl;
3078         int ret = 0;
3079
3080         if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
3081                 return -EINVAL;
3082
3083         if (oct->sriov_info.vf_linkstate[vfidx] == linkstate)
3084                 return 0;
3085
3086         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
3087         nctrl.ncmd.s.cmd = OCTNET_CMD_SET_VF_LINKSTATE;
3088         nctrl.ncmd.s.param1 =
3089             vfidx + 1; /* vfidx is 0 based, but vf_num (param1) is 1 based */
3090         nctrl.ncmd.s.param2 = linkstate;
3091         nctrl.ncmd.s.more = 0;
3092         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
3093         nctrl.cb_fn = NULL;
3094
3095         ret = octnet_send_nic_ctrl_pkt(oct, &nctrl);
3096
3097         if (!ret)
3098                 oct->sriov_info.vf_linkstate[vfidx] = linkstate;
3099         else if (ret > 0)
3100                 ret = -EIO;
3101
3102         return ret;
3103 }
3104
3105 static int
3106 liquidio_eswitch_mode_get(struct devlink *devlink, u16 *mode)
3107 {
3108         struct lio_devlink_priv *priv;
3109         struct octeon_device *oct;
3110
3111         priv = devlink_priv(devlink);
3112         oct = priv->oct;
3113
3114         *mode = oct->eswitch_mode;
3115
3116         return 0;
3117 }
3118
3119 static int
3120 liquidio_eswitch_mode_set(struct devlink *devlink, u16 mode,
3121                           struct netlink_ext_ack *extack)
3122 {
3123         struct lio_devlink_priv *priv;
3124         struct octeon_device *oct;
3125         int ret = 0;
3126
3127         priv = devlink_priv(devlink);
3128         oct = priv->oct;
3129
3130         if (!(oct->fw_info.app_cap_flags & LIQUIDIO_SWITCHDEV_CAP))
3131                 return -EINVAL;
3132
3133         if (oct->eswitch_mode == mode)
3134                 return 0;
3135
3136         switch (mode) {
3137         case DEVLINK_ESWITCH_MODE_SWITCHDEV:
3138                 oct->eswitch_mode = mode;
3139                 ret = lio_vf_rep_create(oct);
3140                 break;
3141
3142         case DEVLINK_ESWITCH_MODE_LEGACY:
3143                 lio_vf_rep_destroy(oct);
3144                 oct->eswitch_mode = mode;
3145                 break;
3146
3147         default:
3148                 ret = -EINVAL;
3149         }
3150
3151         return ret;
3152 }
3153
3154 static const struct devlink_ops liquidio_devlink_ops = {
3155         .eswitch_mode_get = liquidio_eswitch_mode_get,
3156         .eswitch_mode_set = liquidio_eswitch_mode_set,
3157 };
3158
3159 static int
3160 liquidio_get_port_parent_id(struct net_device *dev,
3161                             struct netdev_phys_item_id *ppid)
3162 {
3163         struct lio *lio = GET_LIO(dev);
3164         struct octeon_device *oct = lio->oct_dev;
3165
3166         if (oct->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
3167                 return -EOPNOTSUPP;
3168
3169         ppid->id_len = ETH_ALEN;
3170         ether_addr_copy(ppid->id, (void *)&lio->linfo.hw_addr + 2);
3171
3172         return 0;
3173 }
3174
3175 static int liquidio_get_vf_stats(struct net_device *netdev, int vfidx,
3176                                  struct ifla_vf_stats *vf_stats)
3177 {
3178         struct lio *lio = GET_LIO(netdev);
3179         struct octeon_device *oct = lio->oct_dev;
3180         struct oct_vf_stats stats;
3181         int ret;
3182
3183         if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
3184                 return -EINVAL;
3185
3186         memset(&stats, 0, sizeof(struct oct_vf_stats));
3187         ret = cn23xx_get_vf_stats(oct, vfidx, &stats);
3188         if (!ret) {
3189                 vf_stats->rx_packets = stats.rx_packets;
3190                 vf_stats->tx_packets = stats.tx_packets;
3191                 vf_stats->rx_bytes = stats.rx_bytes;
3192                 vf_stats->tx_bytes = stats.tx_bytes;
3193                 vf_stats->broadcast = stats.broadcast;
3194                 vf_stats->multicast = stats.multicast;
3195         }
3196
3197         return ret;
3198 }
3199
3200 static const struct net_device_ops lionetdevops = {
3201         .ndo_open               = liquidio_open,
3202         .ndo_stop               = liquidio_stop,
3203         .ndo_start_xmit         = liquidio_xmit,
3204         .ndo_get_stats64        = liquidio_get_stats64,
3205         .ndo_set_mac_address    = liquidio_set_mac,
3206         .ndo_set_rx_mode        = liquidio_set_mcast_list,
3207         .ndo_tx_timeout         = liquidio_tx_timeout,
3208
3209         .ndo_vlan_rx_add_vid    = liquidio_vlan_rx_add_vid,
3210         .ndo_vlan_rx_kill_vid   = liquidio_vlan_rx_kill_vid,
3211         .ndo_change_mtu         = liquidio_change_mtu,
3212         .ndo_do_ioctl           = liquidio_ioctl,
3213         .ndo_fix_features       = liquidio_fix_features,
3214         .ndo_set_features       = liquidio_set_features,
3215         .ndo_udp_tunnel_add     = udp_tunnel_nic_add_port,
3216         .ndo_udp_tunnel_del     = udp_tunnel_nic_del_port,
3217         .ndo_set_vf_mac         = liquidio_set_vf_mac,
3218         .ndo_set_vf_vlan        = liquidio_set_vf_vlan,
3219         .ndo_get_vf_config      = liquidio_get_vf_config,
3220         .ndo_set_vf_spoofchk    = liquidio_set_vf_spoofchk,
3221         .ndo_set_vf_trust       = liquidio_set_vf_trust,
3222         .ndo_set_vf_link_state  = liquidio_set_vf_link_state,
3223         .ndo_get_vf_stats       = liquidio_get_vf_stats,
3224         .ndo_get_port_parent_id = liquidio_get_port_parent_id,
3225 };
3226
3227 /** \brief Entry point for the liquidio module
3228  */
3229 static int __init liquidio_init(void)
3230 {
3231         int i;
3232         struct handshake *hs;
3233
3234         init_completion(&first_stage);
3235
3236         octeon_init_device_list(OCTEON_CONFIG_TYPE_DEFAULT);
3237
3238         if (liquidio_init_pci())
3239                 return -EINVAL;
3240
3241         wait_for_completion_timeout(&first_stage, msecs_to_jiffies(1000));
3242
3243         for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3244                 hs = &handshake[i];
3245                 if (hs->pci_dev) {
3246                         wait_for_completion(&hs->init);
3247                         if (!hs->init_ok) {
3248                                 /* init handshake failed */
3249                                 dev_err(&hs->pci_dev->dev,
3250                                         "Failed to init device\n");
3251                                 liquidio_deinit_pci();
3252                                 return -EIO;
3253                         }
3254                 }
3255         }
3256
3257         for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3258                 hs = &handshake[i];
3259                 if (hs->pci_dev) {
3260                         wait_for_completion_timeout(&hs->started,
3261                                                     msecs_to_jiffies(30000));
3262                         if (!hs->started_ok) {
3263                                 /* starter handshake failed */
3264                                 dev_err(&hs->pci_dev->dev,
3265                                         "Firmware failed to start\n");
3266                                 liquidio_deinit_pci();
3267                                 return -EIO;
3268                         }
3269                 }
3270         }
3271
3272         return 0;
3273 }
3274
3275 static int lio_nic_info(struct octeon_recv_info *recv_info, void *buf)
3276 {
3277         struct octeon_device *oct = (struct octeon_device *)buf;
3278         struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt;
3279         int gmxport = 0;
3280         union oct_link_status *ls;
3281         int i;
3282
3283         if (recv_pkt->buffer_size[0] != (sizeof(*ls) + OCT_DROQ_INFO_SIZE)) {
3284                 dev_err(&oct->pci_dev->dev, "Malformed NIC_INFO, len=%d, ifidx=%d\n",
3285                         recv_pkt->buffer_size[0],
3286                         recv_pkt->rh.r_nic_info.gmxport);
3287                 goto nic_info_err;
3288         }
3289
3290         gmxport = recv_pkt->rh.r_nic_info.gmxport;
3291         ls = (union oct_link_status *)(get_rbd(recv_pkt->buffer_ptr[0]) +
3292                 OCT_DROQ_INFO_SIZE);
3293
3294         octeon_swap_8B_data((u64 *)ls, (sizeof(union oct_link_status)) >> 3);
3295         for (i = 0; i < oct->ifcount; i++) {
3296                 if (oct->props[i].gmxport == gmxport) {
3297                         update_link_status(oct->props[i].netdev, ls);
3298                         break;
3299                 }
3300         }
3301
3302 nic_info_err:
3303         for (i = 0; i < recv_pkt->buffer_count; i++)
3304                 recv_buffer_free(recv_pkt->buffer_ptr[i]);
3305         octeon_free_recv_info(recv_info);
3306         return 0;
3307 }
3308
3309 /**
3310  * \brief Setup network interfaces
3311  * @param octeon_dev  octeon device
3312  *
3313  * Called during init time for each device. It assumes the NIC
3314  * is already up and running.  The link information for each
3315  * interface is passed in link_info.
3316  */
3317 static int setup_nic_devices(struct octeon_device *octeon_dev)
3318 {
3319         struct lio *lio = NULL;
3320         struct net_device *netdev;
3321         u8 mac[6], i, j, *fw_ver, *micro_ver;
3322         unsigned long micro;
3323         u32 cur_ver;
3324         struct octeon_soft_command *sc;
3325         struct liquidio_if_cfg_resp *resp;
3326         struct octdev_props *props;
3327         int retval, num_iqueues, num_oqueues;
3328         int max_num_queues = 0;
3329         union oct_nic_if_cfg if_cfg;
3330         unsigned int base_queue;
3331         unsigned int gmx_port_id;
3332         u32 resp_size, data_size;
3333         u32 ifidx_or_pfnum;
3334         struct lio_version *vdata;
3335         struct devlink *devlink;
3336         struct lio_devlink_priv *lio_devlink;
3337
3338         /* This is to handle link status changes */
3339         octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
3340                                     OPCODE_NIC_INFO,
3341                                     lio_nic_info, octeon_dev);
3342
3343         /* REQTYPE_RESP_NET and REQTYPE_SOFT_COMMAND do not have free functions.
3344          * They are handled directly.
3345          */
3346         octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET,
3347                                         free_netbuf);
3348
3349         octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET_SG,
3350                                         free_netsgbuf);
3351
3352         octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_RESP_NET_SG,
3353                                         free_netsgbuf_with_resp);
3354
3355         for (i = 0; i < octeon_dev->ifcount; i++) {
3356                 resp_size = sizeof(struct liquidio_if_cfg_resp);
3357                 data_size = sizeof(struct lio_version);
3358                 sc = (struct octeon_soft_command *)
3359                         octeon_alloc_soft_command(octeon_dev, data_size,
3360                                                   resp_size, 0);
3361                 resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
3362                 vdata = (struct lio_version *)sc->virtdptr;
3363
3364                 *((u64 *)vdata) = 0;
3365                 vdata->major = cpu_to_be16(LIQUIDIO_BASE_MAJOR_VERSION);
3366                 vdata->minor = cpu_to_be16(LIQUIDIO_BASE_MINOR_VERSION);
3367                 vdata->micro = cpu_to_be16(LIQUIDIO_BASE_MICRO_VERSION);
3368
3369                 if (OCTEON_CN23XX_PF(octeon_dev)) {
3370                         num_iqueues = octeon_dev->sriov_info.num_pf_rings;
3371                         num_oqueues = octeon_dev->sriov_info.num_pf_rings;
3372                         base_queue = octeon_dev->sriov_info.pf_srn;
3373
3374                         gmx_port_id = octeon_dev->pf_num;
3375                         ifidx_or_pfnum = octeon_dev->pf_num;
3376                 } else {
3377                         num_iqueues = CFG_GET_NUM_TXQS_NIC_IF(
3378                                                 octeon_get_conf(octeon_dev), i);
3379                         num_oqueues = CFG_GET_NUM_RXQS_NIC_IF(
3380                                                 octeon_get_conf(octeon_dev), i);
3381                         base_queue = CFG_GET_BASE_QUE_NIC_IF(
3382                                                 octeon_get_conf(octeon_dev), i);
3383                         gmx_port_id = CFG_GET_GMXID_NIC_IF(
3384                                                 octeon_get_conf(octeon_dev), i);
3385                         ifidx_or_pfnum = i;
3386                 }
3387
3388                 dev_dbg(&octeon_dev->pci_dev->dev,
3389                         "requesting config for interface %d, iqs %d, oqs %d\n",
3390                         ifidx_or_pfnum, num_iqueues, num_oqueues);
3391
3392                 if_cfg.u64 = 0;
3393                 if_cfg.s.num_iqueues = num_iqueues;
3394                 if_cfg.s.num_oqueues = num_oqueues;
3395                 if_cfg.s.base_queue = base_queue;
3396                 if_cfg.s.gmx_port_id = gmx_port_id;
3397
3398                 sc->iq_no = 0;
3399
3400                 octeon_prepare_soft_command(octeon_dev, sc, OPCODE_NIC,
3401                                             OPCODE_NIC_IF_CFG, 0,
3402                                             if_cfg.u64, 0);
3403
3404                 init_completion(&sc->complete);
3405                 sc->sc_status = OCTEON_REQUEST_PENDING;
3406
3407                 retval = octeon_send_soft_command(octeon_dev, sc);
3408                 if (retval == IQ_SEND_FAILED) {
3409                         dev_err(&octeon_dev->pci_dev->dev,
3410                                 "iq/oq config failed status: %x\n",
3411                                 retval);
3412                         /* Soft instr is freed by driver in case of failure. */
3413                         octeon_free_soft_command(octeon_dev, sc);
3414                         return(-EIO);
3415                 }
3416
3417                 /* Sleep on a wait queue till the cond flag indicates that the
3418                  * response arrived or timed-out.
3419                  */
3420                 retval = wait_for_sc_completion_timeout(octeon_dev, sc, 0);
3421                 if (retval)
3422                         return retval;
3423
3424                 retval = resp->status;
3425                 if (retval) {
3426                         dev_err(&octeon_dev->pci_dev->dev, "iq/oq config failed\n");
3427                         WRITE_ONCE(sc->caller_is_done, true);
3428                         goto setup_nic_dev_done;
3429                 }
3430                 snprintf(octeon_dev->fw_info.liquidio_firmware_version,
3431                          32, "%s",
3432                          resp->cfg_info.liquidio_firmware_version);
3433
3434                 /* Verify f/w version (in case of 'auto' loading from flash) */
3435                 fw_ver = octeon_dev->fw_info.liquidio_firmware_version;
3436                 if (memcmp(LIQUIDIO_BASE_VERSION,
3437                            fw_ver,
3438                            strlen(LIQUIDIO_BASE_VERSION))) {
3439                         dev_err(&octeon_dev->pci_dev->dev,
3440                                 "Unmatched firmware version. Expected %s.x, got %s.\n",
3441                                 LIQUIDIO_BASE_VERSION, fw_ver);
3442                         WRITE_ONCE(sc->caller_is_done, true);
3443                         goto setup_nic_dev_done;
3444                 } else if (atomic_read(octeon_dev->adapter_fw_state) ==
3445                            FW_IS_PRELOADED) {
3446                         dev_info(&octeon_dev->pci_dev->dev,
3447                                  "Using auto-loaded firmware version %s.\n",
3448                                  fw_ver);
3449                 }
3450
3451                 /* extract micro version field; point past '<maj>.<min>.' */
3452                 micro_ver = fw_ver + strlen(LIQUIDIO_BASE_VERSION) + 1;
3453                 if (kstrtoul(micro_ver, 10, &micro) != 0)
3454                         micro = 0;
3455                 octeon_dev->fw_info.ver.maj = LIQUIDIO_BASE_MAJOR_VERSION;
3456                 octeon_dev->fw_info.ver.min = LIQUIDIO_BASE_MINOR_VERSION;
3457                 octeon_dev->fw_info.ver.rev = micro;
3458
3459                 octeon_swap_8B_data((u64 *)(&resp->cfg_info),
3460                                     (sizeof(struct liquidio_if_cfg_info)) >> 3);
3461
3462                 num_iqueues = hweight64(resp->cfg_info.iqmask);
3463                 num_oqueues = hweight64(resp->cfg_info.oqmask);
3464
3465                 if (!(num_iqueues) || !(num_oqueues)) {
3466                         dev_err(&octeon_dev->pci_dev->dev,
3467                                 "Got bad iqueues (%016llx) or oqueues (%016llx) from firmware.\n",
3468                                 resp->cfg_info.iqmask,
3469                                 resp->cfg_info.oqmask);
3470                         WRITE_ONCE(sc->caller_is_done, true);
3471                         goto setup_nic_dev_done;
3472                 }
3473
3474                 if (OCTEON_CN6XXX(octeon_dev)) {
3475                         max_num_queues = CFG_GET_IQ_MAX_Q(CHIP_CONF(octeon_dev,
3476                                                                     cn6xxx));
3477                 } else if (OCTEON_CN23XX_PF(octeon_dev)) {
3478                         max_num_queues = CFG_GET_IQ_MAX_Q(CHIP_CONF(octeon_dev,
3479                                                                     cn23xx_pf));
3480                 }
3481
3482                 dev_dbg(&octeon_dev->pci_dev->dev,
3483                         "interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d max_num_queues: %d\n",
3484                         i, resp->cfg_info.iqmask, resp->cfg_info.oqmask,
3485                         num_iqueues, num_oqueues, max_num_queues);
3486                 netdev = alloc_etherdev_mq(LIO_SIZE, max_num_queues);
3487
3488                 if (!netdev) {
3489                         dev_err(&octeon_dev->pci_dev->dev, "Device allocation failed\n");
3490                         WRITE_ONCE(sc->caller_is_done, true);
3491                         goto setup_nic_dev_done;
3492                 }
3493
3494                 SET_NETDEV_DEV(netdev, &octeon_dev->pci_dev->dev);
3495
3496                 /* Associate the routines that will handle different
3497                  * netdev tasks.
3498                  */
3499                 netdev->netdev_ops = &lionetdevops;
3500
3501                 retval = netif_set_real_num_rx_queues(netdev, num_oqueues);
3502                 if (retval) {
3503                         dev_err(&octeon_dev->pci_dev->dev,
3504                                 "setting real number rx failed\n");
3505                         WRITE_ONCE(sc->caller_is_done, true);
3506                         goto setup_nic_dev_free;
3507                 }
3508
3509                 retval = netif_set_real_num_tx_queues(netdev, num_iqueues);
3510                 if (retval) {
3511                         dev_err(&octeon_dev->pci_dev->dev,
3512                                 "setting real number tx failed\n");
3513                         WRITE_ONCE(sc->caller_is_done, true);
3514                         goto setup_nic_dev_free;
3515                 }
3516
3517                 lio = GET_LIO(netdev);
3518
3519                 memset(lio, 0, sizeof(struct lio));
3520
3521                 lio->ifidx = ifidx_or_pfnum;
3522
3523                 props = &octeon_dev->props[i];
3524                 props->gmxport = resp->cfg_info.linfo.gmxport;
3525                 props->netdev = netdev;
3526
3527                 lio->linfo.num_rxpciq = num_oqueues;
3528                 lio->linfo.num_txpciq = num_iqueues;
3529                 for (j = 0; j < num_oqueues; j++) {
3530                         lio->linfo.rxpciq[j].u64 =
3531                                 resp->cfg_info.linfo.rxpciq[j].u64;
3532                 }
3533                 for (j = 0; j < num_iqueues; j++) {
3534                         lio->linfo.txpciq[j].u64 =
3535                                 resp->cfg_info.linfo.txpciq[j].u64;
3536                 }
3537                 lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr;
3538                 lio->linfo.gmxport = resp->cfg_info.linfo.gmxport;
3539                 lio->linfo.link.u64 = resp->cfg_info.linfo.link.u64;
3540
3541                 WRITE_ONCE(sc->caller_is_done, true);
3542
3543                 lio->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3544
3545                 if (OCTEON_CN23XX_PF(octeon_dev) ||
3546                     OCTEON_CN6XXX(octeon_dev)) {
3547                         lio->dev_capability = NETIF_F_HIGHDMA
3548                                               | NETIF_F_IP_CSUM
3549                                               | NETIF_F_IPV6_CSUM
3550                                               | NETIF_F_SG | NETIF_F_RXCSUM
3551                                               | NETIF_F_GRO
3552                                               | NETIF_F_TSO | NETIF_F_TSO6
3553                                               | NETIF_F_LRO;
3554                 }
3555                 netif_set_gso_max_size(netdev, OCTNIC_GSO_MAX_SIZE);
3556
3557                 /*  Copy of transmit encapsulation capabilities:
3558                  *  TSO, TSO6, Checksums for this device
3559                  */
3560                 lio->enc_dev_capability = NETIF_F_IP_CSUM
3561                                           | NETIF_F_IPV6_CSUM
3562                                           | NETIF_F_GSO_UDP_TUNNEL
3563                                           | NETIF_F_HW_CSUM | NETIF_F_SG
3564                                           | NETIF_F_RXCSUM
3565                                           | NETIF_F_TSO | NETIF_F_TSO6
3566                                           | NETIF_F_LRO;
3567
3568                 netdev->hw_enc_features = (lio->enc_dev_capability &
3569                                            ~NETIF_F_LRO);
3570
3571                 netdev->udp_tunnel_nic_info = &liquidio_udp_tunnels;
3572
3573                 lio->dev_capability |= NETIF_F_GSO_UDP_TUNNEL;
3574
3575                 netdev->vlan_features = lio->dev_capability;
3576                 /* Add any unchangeable hw features */
3577                 lio->dev_capability |=  NETIF_F_HW_VLAN_CTAG_FILTER |
3578                                         NETIF_F_HW_VLAN_CTAG_RX |
3579                                         NETIF_F_HW_VLAN_CTAG_TX;
3580
3581                 netdev->features = (lio->dev_capability & ~NETIF_F_LRO);
3582
3583                 netdev->hw_features = lio->dev_capability;
3584                 /*HW_VLAN_RX and HW_VLAN_FILTER is always on*/
3585                 netdev->hw_features = netdev->hw_features &
3586                         ~NETIF_F_HW_VLAN_CTAG_RX;
3587
3588                 /* MTU range: 68 - 16000 */
3589                 netdev->min_mtu = LIO_MIN_MTU_SIZE;
3590                 netdev->max_mtu = LIO_MAX_MTU_SIZE;
3591
3592                 /* Point to the  properties for octeon device to which this
3593                  * interface belongs.
3594                  */
3595                 lio->oct_dev = octeon_dev;
3596                 lio->octprops = props;
3597                 lio->netdev = netdev;
3598
3599                 dev_dbg(&octeon_dev->pci_dev->dev,
3600                         "if%d gmx: %d hw_addr: 0x%llx\n", i,
3601                         lio->linfo.gmxport, CVM_CAST64(lio->linfo.hw_addr));
3602
3603                 for (j = 0; j < octeon_dev->sriov_info.max_vfs; j++) {
3604                         u8 vfmac[ETH_ALEN];
3605
3606                         eth_random_addr(vfmac);
3607                         if (__liquidio_set_vf_mac(netdev, j, vfmac, false)) {
3608                                 dev_err(&octeon_dev->pci_dev->dev,
3609                                         "Error setting VF%d MAC address\n",
3610                                         j);
3611                                 goto setup_nic_dev_free;
3612                         }
3613                 }
3614
3615                 /* 64-bit swap required on LE machines */
3616                 octeon_swap_8B_data(&lio->linfo.hw_addr, 1);
3617                 for (j = 0; j < 6; j++)
3618                         mac[j] = *((u8 *)(((u8 *)&lio->linfo.hw_addr) + 2 + j));
3619
3620                 /* Copy MAC Address to OS network device structure */
3621
3622                 ether_addr_copy(netdev->dev_addr, mac);
3623
3624                 /* By default all interfaces on a single Octeon uses the same
3625                  * tx and rx queues
3626                  */
3627                 lio->txq = lio->linfo.txpciq[0].s.q_no;
3628                 lio->rxq = lio->linfo.rxpciq[0].s.q_no;
3629                 if (liquidio_setup_io_queues(octeon_dev, i,
3630                                              lio->linfo.num_txpciq,
3631                                              lio->linfo.num_rxpciq)) {
3632                         dev_err(&octeon_dev->pci_dev->dev, "I/O queues creation failed\n");
3633                         goto setup_nic_dev_free;
3634                 }
3635
3636                 ifstate_set(lio, LIO_IFSTATE_DROQ_OPS);
3637
3638                 lio->tx_qsize = octeon_get_tx_qsize(octeon_dev, lio->txq);
3639                 lio->rx_qsize = octeon_get_rx_qsize(octeon_dev, lio->rxq);
3640
3641                 if (lio_setup_glists(octeon_dev, lio, num_iqueues)) {
3642                         dev_err(&octeon_dev->pci_dev->dev,
3643                                 "Gather list allocation failed\n");
3644                         goto setup_nic_dev_free;
3645                 }
3646
3647                 /* Register ethtool support */
3648                 liquidio_set_ethtool_ops(netdev);
3649                 if (lio->oct_dev->chip_id == OCTEON_CN23XX_PF_VID)
3650                         octeon_dev->priv_flags = OCT_PRIV_FLAG_DEFAULT;
3651                 else
3652                         octeon_dev->priv_flags = 0x0;
3653
3654                 if (netdev->features & NETIF_F_LRO)
3655                         liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
3656                                              OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
3657
3658                 liquidio_set_feature(netdev, OCTNET_CMD_VLAN_FILTER_CTL,
3659                                      OCTNET_CMD_VLAN_FILTER_ENABLE);
3660
3661                 if ((debug != -1) && (debug & NETIF_MSG_HW))
3662                         liquidio_set_feature(netdev,
3663                                              OCTNET_CMD_VERBOSE_ENABLE, 0);
3664
3665                 if (setup_link_status_change_wq(netdev))
3666                         goto setup_nic_dev_free;
3667
3668                 if ((octeon_dev->fw_info.app_cap_flags &
3669                      LIQUIDIO_TIME_SYNC_CAP) &&
3670                     setup_sync_octeon_time_wq(netdev))
3671                         goto setup_nic_dev_free;
3672
3673                 if (setup_rx_oom_poll_fn(netdev))
3674                         goto setup_nic_dev_free;
3675
3676                 /* Register the network device with the OS */
3677                 if (register_netdev(netdev)) {
3678                         dev_err(&octeon_dev->pci_dev->dev, "Device registration failed\n");
3679                         goto setup_nic_dev_free;
3680                 }
3681
3682                 dev_dbg(&octeon_dev->pci_dev->dev,
3683                         "Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n",
3684                         i, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
3685                 netif_carrier_off(netdev);
3686                 lio->link_changes++;
3687
3688                 ifstate_set(lio, LIO_IFSTATE_REGISTERED);
3689
3690                 /* Sending command to firmware to enable Rx checksum offload
3691                  * by default at the time of setup of Liquidio driver for
3692                  * this device
3693                  */
3694                 liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
3695                                             OCTNET_CMD_RXCSUM_ENABLE);
3696                 liquidio_set_feature(netdev, OCTNET_CMD_TNL_TX_CSUM_CTL,
3697                                      OCTNET_CMD_TXCSUM_ENABLE);
3698
3699                 dev_dbg(&octeon_dev->pci_dev->dev,
3700                         "NIC ifidx:%d Setup successful\n", i);
3701
3702                 if (octeon_dev->subsystem_id ==
3703                         OCTEON_CN2350_25GB_SUBSYS_ID ||
3704                     octeon_dev->subsystem_id ==
3705                         OCTEON_CN2360_25GB_SUBSYS_ID) {
3706                         cur_ver = OCT_FW_VER(octeon_dev->fw_info.ver.maj,
3707                                              octeon_dev->fw_info.ver.min,
3708                                              octeon_dev->fw_info.ver.rev);
3709
3710                         /* speed control unsupported in f/w older than 1.7.2 */
3711                         if (cur_ver < OCT_FW_VER(1, 7, 2)) {
3712                                 dev_info(&octeon_dev->pci_dev->dev,
3713                                          "speed setting not supported by f/w.");
3714                                 octeon_dev->speed_setting = 25;
3715                                 octeon_dev->no_speed_setting = 1;
3716                         } else {
3717                                 liquidio_get_speed(lio);
3718                         }
3719
3720                         if (octeon_dev->speed_setting == 0) {
3721                                 octeon_dev->speed_setting = 25;
3722                                 octeon_dev->no_speed_setting = 1;
3723                         }
3724                 } else {
3725                         octeon_dev->no_speed_setting = 1;
3726                         octeon_dev->speed_setting = 10;
3727                 }
3728                 octeon_dev->speed_boot = octeon_dev->speed_setting;
3729
3730                 /* don't read FEC setting if unsupported by f/w (see above) */
3731                 if (octeon_dev->speed_boot == 25 &&
3732                     !octeon_dev->no_speed_setting) {
3733                         liquidio_get_fec(lio);
3734                         octeon_dev->props[lio->ifidx].fec_boot =
3735                                 octeon_dev->props[lio->ifidx].fec;
3736                 }
3737         }
3738
3739         devlink = devlink_alloc(&liquidio_devlink_ops,
3740                                 sizeof(struct lio_devlink_priv));
3741         if (!devlink) {
3742                 dev_err(&octeon_dev->pci_dev->dev, "devlink alloc failed\n");
3743                 goto setup_nic_dev_free;
3744         }
3745
3746         lio_devlink = devlink_priv(devlink);
3747         lio_devlink->oct = octeon_dev;
3748
3749         if (devlink_register(devlink, &octeon_dev->pci_dev->dev)) {
3750                 devlink_free(devlink);
3751                 dev_err(&octeon_dev->pci_dev->dev,
3752                         "devlink registration failed\n");
3753                 goto setup_nic_dev_free;
3754         }
3755
3756         octeon_dev->devlink = devlink;
3757         octeon_dev->eswitch_mode = DEVLINK_ESWITCH_MODE_LEGACY;
3758
3759         return 0;
3760
3761 setup_nic_dev_free:
3762
3763         while (i--) {
3764                 dev_err(&octeon_dev->pci_dev->dev,
3765                         "NIC ifidx:%d Setup failed\n", i);
3766                 liquidio_destroy_nic_device(octeon_dev, i);
3767         }
3768
3769 setup_nic_dev_done:
3770
3771         return -ENODEV;
3772 }
3773
3774 #ifdef CONFIG_PCI_IOV
3775 static int octeon_enable_sriov(struct octeon_device *oct)
3776 {
3777         unsigned int num_vfs_alloced = oct->sriov_info.num_vfs_alloced;
3778         struct pci_dev *vfdev;
3779         int err;
3780         u32 u;
3781
3782         if (OCTEON_CN23XX_PF(oct) && num_vfs_alloced) {
3783                 err = pci_enable_sriov(oct->pci_dev,
3784                                        oct->sriov_info.num_vfs_alloced);
3785                 if (err) {
3786                         dev_err(&oct->pci_dev->dev,
3787                                 "OCTEON: Failed to enable PCI sriov: %d\n",
3788                                 err);
3789                         oct->sriov_info.num_vfs_alloced = 0;
3790                         return err;
3791                 }
3792                 oct->sriov_info.sriov_enabled = 1;
3793
3794                 /* init lookup table that maps DPI ring number to VF pci_dev
3795                  * struct pointer
3796                  */
3797                 u = 0;
3798                 vfdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
3799                                        OCTEON_CN23XX_VF_VID, NULL);
3800                 while (vfdev) {
3801                         if (vfdev->is_virtfn &&
3802                             (vfdev->physfn == oct->pci_dev)) {
3803                                 oct->sriov_info.dpiring_to_vfpcidev_lut[u] =
3804                                         vfdev;
3805                                 u += oct->sriov_info.rings_per_vf;
3806                         }
3807                         vfdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
3808                                                OCTEON_CN23XX_VF_VID, vfdev);
3809                 }
3810         }
3811
3812         return num_vfs_alloced;
3813 }
3814
3815 static int lio_pci_sriov_disable(struct octeon_device *oct)
3816 {
3817         int u;
3818
3819         if (pci_vfs_assigned(oct->pci_dev)) {
3820                 dev_err(&oct->pci_dev->dev, "VFs are still assigned to VMs.\n");
3821                 return -EPERM;
3822         }
3823
3824         pci_disable_sriov(oct->pci_dev);
3825
3826         u = 0;
3827         while (u < MAX_POSSIBLE_VFS) {
3828                 oct->sriov_info.dpiring_to_vfpcidev_lut[u] = NULL;
3829                 u += oct->sriov_info.rings_per_vf;
3830         }
3831
3832         oct->sriov_info.num_vfs_alloced = 0;
3833         dev_info(&oct->pci_dev->dev, "oct->pf_num:%d disabled VFs\n",
3834                  oct->pf_num);
3835
3836         return 0;
3837 }
3838
3839 static int liquidio_enable_sriov(struct pci_dev *dev, int num_vfs)
3840 {
3841         struct octeon_device *oct = pci_get_drvdata(dev);
3842         int ret = 0;
3843
3844         if ((num_vfs == oct->sriov_info.num_vfs_alloced) &&
3845             (oct->sriov_info.sriov_enabled)) {
3846                 dev_info(&oct->pci_dev->dev, "oct->pf_num:%d already enabled num_vfs:%d\n",
3847                          oct->pf_num, num_vfs);
3848                 return 0;
3849         }
3850
3851         if (!num_vfs) {
3852                 lio_vf_rep_destroy(oct);
3853                 ret = lio_pci_sriov_disable(oct);
3854         } else if (num_vfs > oct->sriov_info.max_vfs) {
3855                 dev_err(&oct->pci_dev->dev,
3856                         "OCTEON: Max allowed VFs:%d user requested:%d",
3857                         oct->sriov_info.max_vfs, num_vfs);
3858                 ret = -EPERM;
3859         } else {
3860                 oct->sriov_info.num_vfs_alloced = num_vfs;
3861                 ret = octeon_enable_sriov(oct);
3862                 dev_info(&oct->pci_dev->dev, "oct->pf_num:%d num_vfs:%d\n",
3863                          oct->pf_num, num_vfs);
3864                 ret = lio_vf_rep_create(oct);
3865                 if (ret)
3866                         dev_info(&oct->pci_dev->dev,
3867                                  "vf representor create failed");
3868         }
3869
3870         return ret;
3871 }
3872 #endif
3873
3874 /**
3875  * \brief initialize the NIC
3876  * @param oct octeon device
3877  *
3878  * This initialization routine is called once the Octeon device application is
3879  * up and running
3880  */
3881 static int liquidio_init_nic_module(struct octeon_device *oct)
3882 {
3883         int i, retval = 0;
3884         int num_nic_ports = CFG_GET_NUM_NIC_PORTS(octeon_get_conf(oct));
3885
3886         dev_dbg(&oct->pci_dev->dev, "Initializing network interfaces\n");
3887
3888         /* only default iq and oq were initialized
3889          * initialize the rest as well
3890          */
3891         /* run port_config command for each port */
3892         oct->ifcount = num_nic_ports;
3893
3894         memset(oct->props, 0, sizeof(struct octdev_props) * num_nic_ports);
3895
3896         for (i = 0; i < MAX_OCTEON_LINKS; i++)
3897                 oct->props[i].gmxport = -1;
3898
3899         retval = setup_nic_devices(oct);
3900         if (retval) {
3901                 dev_err(&oct->pci_dev->dev, "Setup NIC devices failed\n");
3902                 goto octnet_init_failure;
3903         }
3904
3905         /* Call vf_rep_modinit if the firmware is switchdev capable
3906          * and do it from the first liquidio function probed.
3907          */
3908         if (!oct->octeon_id &&
3909             oct->fw_info.app_cap_flags & LIQUIDIO_SWITCHDEV_CAP) {
3910                 retval = lio_vf_rep_modinit();
3911                 if (retval) {
3912                         liquidio_stop_nic_module(oct);
3913                         goto octnet_init_failure;
3914                 }
3915         }
3916
3917         liquidio_ptp_init(oct);
3918
3919         dev_dbg(&oct->pci_dev->dev, "Network interfaces ready\n");
3920
3921         return retval;
3922
3923 octnet_init_failure:
3924
3925         oct->ifcount = 0;
3926
3927         return retval;
3928 }
3929
3930 /**
3931  * \brief starter callback that invokes the remaining initialization work after
3932  * the NIC is up and running.
3933  * @param octptr  work struct work_struct
3934  */
3935 static void nic_starter(struct work_struct *work)
3936 {
3937         struct octeon_device *oct;
3938         struct cavium_wk *wk = (struct cavium_wk *)work;
3939
3940         oct = (struct octeon_device *)wk->ctxptr;
3941
3942         if (atomic_read(&oct->status) == OCT_DEV_RUNNING)
3943                 return;
3944
3945         /* If the status of the device is CORE_OK, the core
3946          * application has reported its application type. Call
3947          * any registered handlers now and move to the RUNNING
3948          * state.
3949          */
3950         if (atomic_read(&oct->status) != OCT_DEV_CORE_OK) {
3951                 schedule_delayed_work(&oct->nic_poll_work.work,
3952                                       LIQUIDIO_STARTER_POLL_INTERVAL_MS);
3953                 return;
3954         }
3955
3956         atomic_set(&oct->status, OCT_DEV_RUNNING);
3957
3958         if (oct->app_mode && oct->app_mode == CVM_DRV_NIC_APP) {
3959                 dev_dbg(&oct->pci_dev->dev, "Starting NIC module\n");
3960
3961                 if (liquidio_init_nic_module(oct))
3962                         dev_err(&oct->pci_dev->dev, "NIC initialization failed\n");
3963                 else
3964                         handshake[oct->octeon_id].started_ok = 1;
3965         } else {
3966                 dev_err(&oct->pci_dev->dev,
3967                         "Unexpected application running on NIC (%d). Check firmware.\n",
3968                         oct->app_mode);
3969         }
3970
3971         complete(&handshake[oct->octeon_id].started);
3972 }
3973
3974 static int
3975 octeon_recv_vf_drv_notice(struct octeon_recv_info *recv_info, void *buf)
3976 {
3977         struct octeon_device *oct = (struct octeon_device *)buf;
3978         struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt;
3979         int i, notice, vf_idx;
3980         bool cores_crashed;
3981         u64 *data, vf_num;
3982
3983         notice = recv_pkt->rh.r.ossp;
3984         data = (u64 *)(get_rbd(recv_pkt->buffer_ptr[0]) + OCT_DROQ_INFO_SIZE);
3985
3986         /* the first 64-bit word of data is the vf_num */
3987         vf_num = data[0];
3988         octeon_swap_8B_data(&vf_num, 1);
3989         vf_idx = (int)vf_num - 1;
3990
3991         cores_crashed = READ_ONCE(oct->cores_crashed);
3992
3993         if (notice == VF_DRV_LOADED) {
3994                 if (!(oct->sriov_info.vf_drv_loaded_mask & BIT_ULL(vf_idx))) {
3995                         oct->sriov_info.vf_drv_loaded_mask |= BIT_ULL(vf_idx);
3996                         dev_info(&oct->pci_dev->dev,
3997                                  "driver for VF%d was loaded\n", vf_idx);
3998                         if (!cores_crashed)
3999                                 try_module_get(THIS_MODULE);
4000                 }
4001         } else if (notice == VF_DRV_REMOVED) {
4002                 if (oct->sriov_info.vf_drv_loaded_mask & BIT_ULL(vf_idx)) {
4003                         oct->sriov_info.vf_drv_loaded_mask &= ~BIT_ULL(vf_idx);
4004                         dev_info(&oct->pci_dev->dev,
4005                                  "driver for VF%d was removed\n", vf_idx);
4006                         if (!cores_crashed)
4007                                 module_put(THIS_MODULE);
4008                 }
4009         } else if (notice == VF_DRV_MACADDR_CHANGED) {
4010                 u8 *b = (u8 *)&data[1];
4011
4012                 oct->sriov_info.vf_macaddr[vf_idx] = data[1];
4013                 dev_info(&oct->pci_dev->dev,
4014                          "VF driver changed VF%d's MAC address to %pM\n",
4015                          vf_idx, b + 2);
4016         }
4017
4018         for (i = 0; i < recv_pkt->buffer_count; i++)
4019                 recv_buffer_free(recv_pkt->buffer_ptr[i]);
4020         octeon_free_recv_info(recv_info);
4021
4022         return 0;
4023 }
4024
4025 /**
4026  * \brief Device initialization for each Octeon device that is probed
4027  * @param octeon_dev  octeon device
4028  */
4029 static int octeon_device_init(struct octeon_device *octeon_dev)
4030 {
4031         int j, ret;
4032         char bootcmd[] = "\n";
4033         char *dbg_enb = NULL;
4034         enum lio_fw_state fw_state;
4035         struct octeon_device_priv *oct_priv =
4036                 (struct octeon_device_priv *)octeon_dev->priv;
4037         atomic_set(&octeon_dev->status, OCT_DEV_BEGIN_STATE);
4038
4039         /* Enable access to the octeon device and make its DMA capability
4040          * known to the OS.
4041          */
4042         if (octeon_pci_os_setup(octeon_dev))
4043                 return 1;
4044
4045         atomic_set(&octeon_dev->status, OCT_DEV_PCI_ENABLE_DONE);
4046
4047         /* Identify the Octeon type and map the BAR address space. */
4048         if (octeon_chip_specific_setup(octeon_dev)) {
4049                 dev_err(&octeon_dev->pci_dev->dev, "Chip specific setup failed\n");
4050                 return 1;
4051         }
4052
4053         atomic_set(&octeon_dev->status, OCT_DEV_PCI_MAP_DONE);
4054
4055         /* Only add a reference after setting status 'OCT_DEV_PCI_MAP_DONE',
4056          * since that is what is required for the reference to be removed
4057          * during de-initialization (see 'octeon_destroy_resources').
4058          */
4059         octeon_register_device(octeon_dev, octeon_dev->pci_dev->bus->number,
4060                                PCI_SLOT(octeon_dev->pci_dev->devfn),
4061                                PCI_FUNC(octeon_dev->pci_dev->devfn),
4062                                true);
4063
4064         octeon_dev->app_mode = CVM_DRV_INVALID_APP;
4065
4066         /* CN23XX supports preloaded firmware if the following is true:
4067          *
4068          * The adapter indicates that firmware is currently running AND
4069          * 'fw_type' is 'auto'.
4070          *
4071          * (default state is NEEDS_TO_BE_LOADED, override it if appropriate).
4072          */
4073         if (OCTEON_CN23XX_PF(octeon_dev) &&
4074             cn23xx_fw_loaded(octeon_dev) && fw_type_is_auto()) {
4075                 atomic_cmpxchg(octeon_dev->adapter_fw_state,
4076                                FW_NEEDS_TO_BE_LOADED, FW_IS_PRELOADED);
4077         }
4078
4079         /* If loading firmware, only first device of adapter needs to do so. */
4080         fw_state = atomic_cmpxchg(octeon_dev->adapter_fw_state,
4081                                   FW_NEEDS_TO_BE_LOADED,
4082                                   FW_IS_BEING_LOADED);
4083
4084         /* Here, [local variable] 'fw_state' is set to one of:
4085          *
4086          *   FW_IS_PRELOADED:       No firmware is to be loaded (see above)
4087          *   FW_NEEDS_TO_BE_LOADED: The driver's first instance will load
4088          *                          firmware to the adapter.
4089          *   FW_IS_BEING_LOADED:    The driver's second instance will not load
4090          *                          firmware to the adapter.
4091          */
4092
4093         /* Prior to f/w load, perform a soft reset of the Octeon device;
4094          * if error resetting, return w/error.
4095          */
4096         if (fw_state == FW_NEEDS_TO_BE_LOADED)
4097                 if (octeon_dev->fn_list.soft_reset(octeon_dev))
4098                         return 1;
4099
4100         /* Initialize the dispatch mechanism used to push packets arriving on
4101          * Octeon Output queues.
4102          */
4103         if (octeon_init_dispatch_list(octeon_dev))
4104                 return 1;
4105
4106         octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
4107                                     OPCODE_NIC_CORE_DRV_ACTIVE,
4108                                     octeon_core_drv_init,
4109                                     octeon_dev);
4110
4111         octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
4112                                     OPCODE_NIC_VF_DRV_NOTICE,
4113                                     octeon_recv_vf_drv_notice, octeon_dev);
4114         INIT_DELAYED_WORK(&octeon_dev->nic_poll_work.work, nic_starter);
4115         octeon_dev->nic_poll_work.ctxptr = (void *)octeon_dev;
4116         schedule_delayed_work(&octeon_dev->nic_poll_work.work,
4117                               LIQUIDIO_STARTER_POLL_INTERVAL_MS);
4118
4119         atomic_set(&octeon_dev->status, OCT_DEV_DISPATCH_INIT_DONE);
4120
4121         if (octeon_set_io_queues_off(octeon_dev)) {
4122                 dev_err(&octeon_dev->pci_dev->dev, "setting io queues off failed\n");
4123                 return 1;
4124         }
4125
4126         if (OCTEON_CN23XX_PF(octeon_dev)) {
4127                 ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
4128                 if (ret) {
4129                         dev_err(&octeon_dev->pci_dev->dev, "OCTEON: Failed to configure device registers\n");
4130                         return ret;
4131                 }
4132         }
4133
4134         /* Initialize soft command buffer pool
4135          */
4136         if (octeon_setup_sc_buffer_pool(octeon_dev)) {
4137                 dev_err(&octeon_dev->pci_dev->dev, "sc buffer pool allocation failed\n");
4138                 return 1;
4139         }
4140         atomic_set(&octeon_dev->status, OCT_DEV_SC_BUFF_POOL_INIT_DONE);
4141
4142         /*  Setup the data structures that manage this Octeon's Input queues. */
4143         if (octeon_setup_instr_queues(octeon_dev)) {
4144                 dev_err(&octeon_dev->pci_dev->dev,
4145                         "instruction queue initialization failed\n");
4146                 return 1;
4147         }
4148         atomic_set(&octeon_dev->status, OCT_DEV_INSTR_QUEUE_INIT_DONE);
4149
4150         /* Initialize lists to manage the requests of different types that
4151          * arrive from user & kernel applications for this octeon device.
4152          */
4153         if (octeon_setup_response_list(octeon_dev)) {
4154                 dev_err(&octeon_dev->pci_dev->dev, "Response list allocation failed\n");
4155                 return 1;
4156         }
4157         atomic_set(&octeon_dev->status, OCT_DEV_RESP_LIST_INIT_DONE);
4158
4159         if (octeon_setup_output_queues(octeon_dev)) {
4160                 dev_err(&octeon_dev->pci_dev->dev, "Output queue initialization failed\n");
4161                 return 1;
4162         }
4163
4164         atomic_set(&octeon_dev->status, OCT_DEV_DROQ_INIT_DONE);
4165
4166         if (OCTEON_CN23XX_PF(octeon_dev)) {
4167                 if (octeon_dev->fn_list.setup_mbox(octeon_dev)) {
4168                         dev_err(&octeon_dev->pci_dev->dev, "OCTEON: Mailbox setup failed\n");
4169                         return 1;
4170                 }
4171                 atomic_set(&octeon_dev->status, OCT_DEV_MBOX_SETUP_DONE);
4172
4173                 if (octeon_allocate_ioq_vector
4174                                 (octeon_dev,
4175                                  octeon_dev->sriov_info.num_pf_rings)) {
4176                         dev_err(&octeon_dev->pci_dev->dev, "OCTEON: ioq vector allocation failed\n");
4177                         return 1;
4178                 }
4179                 atomic_set(&octeon_dev->status, OCT_DEV_MSIX_ALLOC_VECTOR_DONE);
4180
4181         } else {
4182                 /* The input and output queue registers were setup earlier (the
4183                  * queues were not enabled). Any additional registers
4184                  * that need to be programmed should be done now.
4185                  */
4186                 ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
4187                 if (ret) {
4188                         dev_err(&octeon_dev->pci_dev->dev,
4189                                 "Failed to configure device registers\n");
4190                         return ret;
4191                 }
4192         }
4193
4194         /* Initialize the tasklet that handles output queue packet processing.*/
4195         dev_dbg(&octeon_dev->pci_dev->dev, "Initializing droq tasklet\n");
4196         tasklet_init(&oct_priv->droq_tasklet, octeon_droq_bh,
4197                      (unsigned long)octeon_dev);
4198
4199         /* Setup the interrupt handler and record the INT SUM register address
4200          */
4201         if (octeon_setup_interrupt(octeon_dev,
4202                                    octeon_dev->sriov_info.num_pf_rings))
4203                 return 1;
4204
4205         /* Enable Octeon device interrupts */
4206         octeon_dev->fn_list.enable_interrupt(octeon_dev, OCTEON_ALL_INTR);
4207
4208         atomic_set(&octeon_dev->status, OCT_DEV_INTR_SET_DONE);
4209
4210         /* Send Credit for Octeon Output queues. Credits are always sent BEFORE
4211          * the output queue is enabled.
4212          * This ensures that we'll receive the f/w CORE DRV_ACTIVE message in
4213          * case we've configured CN23XX_SLI_GBL_CONTROL[NOPTR_D] = 0.
4214          * Otherwise, it is possible that the DRV_ACTIVE message will be sent
4215          * before any credits have been issued, causing the ring to be reset
4216          * (and the f/w appear to never have started).
4217          */
4218         for (j = 0; j < octeon_dev->num_oqs; j++)
4219                 writel(octeon_dev->droq[j]->max_count,
4220                        octeon_dev->droq[j]->pkts_credit_reg);
4221
4222         /* Enable the input and output queues for this Octeon device */
4223         ret = octeon_dev->fn_list.enable_io_queues(octeon_dev);
4224         if (ret) {
4225                 dev_err(&octeon_dev->pci_dev->dev, "Failed to enable input/output queues");
4226                 return ret;
4227         }
4228
4229         atomic_set(&octeon_dev->status, OCT_DEV_IO_QUEUES_DONE);
4230
4231         if (fw_state == FW_NEEDS_TO_BE_LOADED) {
4232                 dev_dbg(&octeon_dev->pci_dev->dev, "Waiting for DDR initialization...\n");
4233                 if (!ddr_timeout) {
4234                         dev_info(&octeon_dev->pci_dev->dev,
4235                                  "WAITING. Set ddr_timeout to non-zero value to proceed with initialization.\n");
4236                 }
4237
4238                 schedule_timeout_uninterruptible(HZ * LIO_RESET_SECS);
4239
4240                 /* Wait for the octeon to initialize DDR after the soft-reset.*/
4241                 while (!ddr_timeout) {
4242                         set_current_state(TASK_INTERRUPTIBLE);
4243                         if (schedule_timeout(HZ / 10)) {
4244                                 /* user probably pressed Control-C */
4245                                 return 1;
4246                         }
4247                 }
4248                 ret = octeon_wait_for_ddr_init(octeon_dev, &ddr_timeout);
4249                 if (ret) {
4250                         dev_err(&octeon_dev->pci_dev->dev,
4251                                 "DDR not initialized. Please confirm that board is configured to boot from Flash, ret: %d\n",
4252                                 ret);
4253                         return 1;
4254                 }
4255
4256                 if (octeon_wait_for_bootloader(octeon_dev, 1000)) {
4257                         dev_err(&octeon_dev->pci_dev->dev, "Board not responding\n");
4258                         return 1;
4259                 }
4260
4261                 /* Divert uboot to take commands from host instead. */
4262                 ret = octeon_console_send_cmd(octeon_dev, bootcmd, 50);
4263
4264                 dev_dbg(&octeon_dev->pci_dev->dev, "Initializing consoles\n");
4265                 ret = octeon_init_consoles(octeon_dev);
4266                 if (ret) {
4267                         dev_err(&octeon_dev->pci_dev->dev, "Could not access board consoles\n");
4268                         return 1;
4269                 }
4270                 /* If console debug enabled, specify empty string to use default
4271                  * enablement ELSE specify NULL string for 'disabled'.
4272                  */
4273                 dbg_enb = octeon_console_debug_enabled(0) ? "" : NULL;
4274                 ret = octeon_add_console(octeon_dev, 0, dbg_enb);
4275                 if (ret) {
4276                         dev_err(&octeon_dev->pci_dev->dev, "Could not access board console\n");
4277                         return 1;
4278                 } else if (octeon_console_debug_enabled(0)) {
4279                         /* If console was added AND we're logging console output
4280                          * then set our console print function.
4281                          */
4282                         octeon_dev->console[0].print = octeon_dbg_console_print;
4283                 }
4284
4285                 atomic_set(&octeon_dev->status, OCT_DEV_CONSOLE_INIT_DONE);
4286
4287                 dev_dbg(&octeon_dev->pci_dev->dev, "Loading firmware\n");
4288                 ret = load_firmware(octeon_dev);
4289                 if (ret) {
4290                         dev_err(&octeon_dev->pci_dev->dev, "Could not load firmware to board\n");
4291                         return 1;
4292                 }
4293
4294                 atomic_set(octeon_dev->adapter_fw_state, FW_HAS_BEEN_LOADED);
4295         }
4296
4297         handshake[octeon_dev->octeon_id].init_ok = 1;
4298         complete(&handshake[octeon_dev->octeon_id].init);
4299
4300         atomic_set(&octeon_dev->status, OCT_DEV_HOST_OK);
4301
4302         return 0;
4303 }
4304
4305 /**
4306  * \brief Debug console print function
4307  * @param octeon_dev  octeon device
4308  * @param console_num console number
4309  * @param prefix      first portion of line to display
4310  * @param suffix      second portion of line to display
4311  *
4312  * The OCTEON debug console outputs entire lines (excluding '\n').
4313  * Normally, the line will be passed in the 'prefix' parameter.
4314  * However, due to buffering, it is possible for a line to be split into two
4315  * parts, in which case they will be passed as the 'prefix' parameter and
4316  * 'suffix' parameter.
4317  */
4318 static int octeon_dbg_console_print(struct octeon_device *oct, u32 console_num,
4319                                     char *prefix, char *suffix)
4320 {
4321         if (prefix && suffix)
4322                 dev_info(&oct->pci_dev->dev, "%u: %s%s\n", console_num, prefix,
4323                          suffix);
4324         else if (prefix)
4325                 dev_info(&oct->pci_dev->dev, "%u: %s\n", console_num, prefix);
4326         else if (suffix)
4327                 dev_info(&oct->pci_dev->dev, "%u: %s\n", console_num, suffix);
4328
4329         return 0;
4330 }
4331
4332 /**
4333  * \brief Exits the module
4334  */
4335 static void __exit liquidio_exit(void)
4336 {
4337         liquidio_deinit_pci();
4338
4339         pr_info("LiquidIO network module is now unloaded\n");
4340 }
4341
4342 module_init(liquidio_init);
4343 module_exit(liquidio_exit);