projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge remote-tracking branches 'asoc/topic/rt5645', 'asoc/topic/sam9g20_wm8731',...
[linux-2.6-block.git] / drivers / net / ethernet / cavium / liquidio / lio_main.c
CommitLineData
f21fb3ed 1/**********************************************************************
50579d3d
RV		 2 * Author: Cavium, Inc.
3 *
4 * Contact: support@cavium.com
5 * Please include "LiquidIO" in the subject.
6 *
7 * Copyright (c) 2003-2016 Cavium, Inc.
8 *
9 * This file is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License, Version 2, as
11 * published by the Free Software Foundation.
12 *
13 * This file is distributed in the hope that it will be useful, but
14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16 * NONINFRINGEMENT. See the GNU General Public License for more details.
17 ***********************************************************************/
e3bfc6e7 18#include <linux/module.h>
282ccf6e 19#include <linux/interrupt.h>
f21fb3ed 20#include <linux/pci.h>
f21fb3ed 21#include <linux/firmware.h>
01fb237a 22#include <net/vxlan.h>
9ff1a9ba 23#include <linux/kthread.h>
1f233f32 24#include <net/switchdev.h>
f21fb3ed
RV		 25#include "liquidio_common.h"
26#include "octeon_droq.h"
27#include "octeon_iq.h"
28#include "response_manager.h"
29#include "octeon_device.h"
30#include "octeon_nic.h"
31#include "octeon_main.h"
32#include "octeon_network.h"
33#include "cn66xx_regs.h"
34#include "cn66xx_device.h"
f21fb3ed 35#include "cn68xx_device.h"
72c00912 36#include "cn23xx_pf_device.h"
f21fb3ed 37#include "liquidio_image.h"
d4be8ebe 38#include "lio_vf_rep.h"
f21fb3ed
RV		 39
40MODULE_AUTHOR("Cavium Networks, <support@cavium.com>");
41MODULE_DESCRIPTION("Cavium LiquidIO Intelligent Server Adapter Driver");
42MODULE_LICENSE("GPL");
43MODULE_VERSION(LIQUIDIO_VERSION);
ea6404c8
DC		 44MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210SV_NAME
45 "_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
46MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210NV_NAME
47 "_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
48MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_410NV_NAME
49 "_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
50MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_23XX_NAME
51 "_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
f21fb3ed
RV		 52
53static int ddr_timeout = 10000;
54module_param(ddr_timeout, int, 0644);
55MODULE_PARM_DESC(ddr_timeout,
56 "Number of milliseconds to wait for DDR initialization. 0 waits for ddr_timeout to be set to non-zero value before starting to check");
57
f21fb3ed
RV		 58#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
59
60static int debug = -1;
61module_param(debug, int, 0644);
62MODULE_PARM_DESC(debug, "NETIF_MSG debug bits");
63
088b8749 64static char fw_type[LIO_MAX_FW_TYPE_LEN] = LIO_FW_NAME_TYPE_AUTO;
d396179c 65module_param_string(fw_type, fw_type, sizeof(fw_type), 0444);
088b8749 66MODULE_PARM_DESC(fw_type, "Type of firmware to be loaded (default is \"auto\"), which uses firmware in flash, if present, else loads \"nic\".");
f21fb3ed 67
2470f3a2
IB		 68static u32 console_bitmask;
69module_param(console_bitmask, int, 0644);
70MODULE_PARM_DESC(console_bitmask,
71 "Bitmask indicating which consoles have debug output redirected to syslog.");
72
73/**
74 * \brief determines if a given console has debug enabled.
75 * @param console console to check
76 * @returns 1 = enabled. 0 otherwise
77 */
da1542b0 78static int octeon_console_debug_enabled(u32 console)
2470f3a2
IB		 79{
80 return (console_bitmask >> (console)) & 0x1;
81}
82
f21fb3ed
RV		 83/* Polling interval for determining when NIC application is alive */
84#define LIQUIDIO_STARTER_POLL_INTERVAL_MS 100
85
86/* runtime link query interval */
87#define LIQUIDIO_LINK_QUERY_INTERVAL_MS 1000
907aaa6b
VB		 88/* update localtime to octeon firmware every 60 seconds.
89 * make firmware to use same time reference, so that it will be easy to
90 * correlate firmware logged events/errors with host events, for debugging.
91 */
92#define LIO_SYNC_OCTEON_TIME_INTERVAL_MS 60000
f21fb3ed
RV		 93
94struct liquidio_if_cfg_context {
95 int octeon_id;
96
97 wait_queue_head_t wc;
98
99 int cond;
100};
101
102struct liquidio_if_cfg_resp {
103 u64 rh;
104 struct liquidio_if_cfg_info cfg_info;
105 u64 status;
106};
107
afdf841f
RV		 108struct liquidio_rx_ctl_context {
109 int octeon_id;
110
111 wait_queue_head_t wc;
112
113 int cond;
114};
115
f21fb3ed
RV		 116struct oct_link_status_resp {
117 u64 rh;
118 struct oct_link_info link_info;
119 u64 status;
120};
121
122struct oct_timestamp_resp {
123 u64 rh;
124 u64 timestamp;
125 u64 status;
126};
127
128#define OCT_TIMESTAMP_RESP_SIZE (sizeof(struct oct_timestamp_resp))
129
130union tx_info {
131 u64 u64;
132 struct {
133#ifdef __BIG_ENDIAN_BITFIELD
134 u16 gso_size;
135 u16 gso_segs;
136 u32 reserved;
137#else
138 u32 reserved;
139 u16 gso_segs;
140 u16 gso_size;
141#endif
142 } s;
143};
144
145/** Octeon device properties to be used by the NIC module.
146 * Each octeon device in the system will be represented
147 * by this structure in the NIC module.
148 */
149
150#define OCTNIC_MAX_SG (MAX_SKB_FRAGS)
151
152#define OCTNIC_GSO_MAX_HEADER_SIZE 128
72c00912
RV		 153#define OCTNIC_GSO_MAX_SIZE \
154 (CN23XX_DEFAULT_INPUT_JABBER - OCTNIC_GSO_MAX_HEADER_SIZE)
f21fb3ed
RV		 155
156/** Structure of a node in list of gather components maintained by
157 * NIC driver for each network device.
158 */
159struct octnic_gather {
160 /** List manipulation. Next and prev pointers. */
161 struct list_head list;
162
163 /** Size of the gather component at sg in bytes. */
164 int sg_size;
165
166 /** Number of bytes that sg was adjusted to make it 8B-aligned. */
167 int adjust;
168
169 /** Gather component that can accommodate max sized fragment list
170 * received from the IP layer.
171 */
172 struct octeon_sg_entry *sg;
fcd2b5e3 173
67e303e0 174 dma_addr_t sg_dma_ptr;
f21fb3ed
RV		 175};
176
f21fb3ed
RV		 177struct handshake {
178 struct completion init;
179 struct completion started;
180 struct pci_dev *pci_dev;
181 int init_ok;
182 int started_ok;
183};
184
ca6139ff
RV		 185#ifdef CONFIG_PCI_IOV
186static int liquidio_enable_sriov(struct pci_dev *dev, int num_vfs);
187#endif
188
da1542b0
RF		 189static int octeon_dbg_console_print(struct octeon_device *oct, u32 console_num,
190 char *prefix, char *suffix);
191
f21fb3ed 192static int octeon_device_init(struct octeon_device *);
32581245 193static int liquidio_stop(struct net_device *netdev);
f21fb3ed
RV		 194static void liquidio_remove(struct pci_dev *pdev);
195static int liquidio_probe(struct pci_dev *pdev,
196 const struct pci_device_id *ent);
bb54be58
FM		 197static int liquidio_set_vf_link_state(struct net_device *netdev, int vfidx,
198 int linkstate);
f21fb3ed
RV		 199
200static struct handshake handshake[MAX_OCTEON_DEVICES];
201static struct completion first_stage;
202
5b173cf9 203static void octeon_droq_bh(unsigned long pdev)
f21fb3ed
RV		 204{
205 int q_no;
206 int reschedule = 0;
207 struct octeon_device *oct = (struct octeon_device *)pdev;
208 struct octeon_device_priv *oct_priv =
209 (struct octeon_device_priv *)oct->priv;
210
63da8404 211 for (q_no = 0; q_no < MAX_OCTEON_OUTPUT_QUEUES(oct); q_no++) {
763185a3 212 if (!(oct->io_qmask.oq & BIT_ULL(q_no)))
f21fb3ed
RV		 213 continue;
214 reschedule |= octeon_droq_process_packets(oct, oct->droq[q_no],
215 MAX_PACKET_BUDGET);
cd8b1eb4 216 lio_enable_irq(oct->droq[q_no], NULL);
5b07aee1
RV		 217
218 if (OCTEON_CN23XX_PF(oct) && oct->msix_on) {
219 /* set time and cnt interrupt thresholds for this DROQ
220 * for NAPI
221 */
222 int adjusted_q_no = q_no + oct->sriov_info.pf_srn;
223
224 octeon_write_csr64(
225 oct, CN23XX_SLI_OQ_PKT_INT_LEVELS(adjusted_q_no),
226 0x5700000040ULL);
227 octeon_write_csr64(
228 oct, CN23XX_SLI_OQ_PKTS_SENT(adjusted_q_no), 0);
229 }
f21fb3ed
RV		 230 }
231
232 if (reschedule)
233 tasklet_schedule(&oct_priv->droq_tasklet);
234}
235
5b173cf9 236static int lio_wait_for_oq_pkts(struct octeon_device *oct)
f21fb3ed
RV		 237{
238 struct octeon_device_priv *oct_priv =
239 (struct octeon_device_priv *)oct->priv;
240 int retry = 100, pkt_cnt = 0, pending_pkts = 0;
241 int i;
242
243 do {
244 pending_pkts = 0;
245
63da8404 246 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
763185a3 247 if (!(oct->io_qmask.oq & BIT_ULL(i)))
f21fb3ed 248 continue;
a7d5a3dc 249 pkt_cnt += octeon_droq_check_hw_for_pkts(oct->droq[i]);
f21fb3ed
RV		 250 }
251 if (pkt_cnt > 0) {
252 pending_pkts += pkt_cnt;
253 tasklet_schedule(&oct_priv->droq_tasklet);
254 }
255 pkt_cnt = 0;
256 schedule_timeout_uninterruptible(1);
257
258 } while (retry-- && pending_pkts);
259
260 return pkt_cnt;
261}
262
f21fb3ed
RV		 263/**
264 * \brief Forces all IO queues off on a given device
265 * @param oct Pointer to Octeon device
266 */
267static void force_io_queues_off(struct octeon_device *oct)
268{
269 if ((oct->chip_id == OCTEON_CN66XX) ||
270 (oct->chip_id == OCTEON_CN68XX)) {
271 /* Reset the Enable bits for Input Queues. */
272 octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, 0);
273
274 /* Reset the Enable bits for Output Queues. */
275 octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, 0);
276 }
277}
278
f21fb3ed
RV		 279/**
280 * \brief Cause device to go quiet so it can be safely removed/reset/etc
281 * @param oct Pointer to Octeon device
282 */
283static inline void pcierror_quiesce_device(struct octeon_device *oct)
284{
285 int i;
286
287 /* Disable the input and output queues now. No more packets will
288 * arrive from Octeon, but we should wait for all packet processing
289 * to finish.
290 */
291 force_io_queues_off(oct);
292
293 /* To allow for in-flight requests */
294 schedule_timeout_uninterruptible(100);
295
296 if (wait_for_pending_requests(oct))
297 dev_err(&oct->pci_dev->dev, "There were pending requests\n");
298
299 /* Force all requests waiting to be fetched by OCTEON to complete. */
63da8404 300 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
f21fb3ed
RV		 301 struct octeon_instr_queue *iq;
302
763185a3 303 if (!(oct->io_qmask.iq & BIT_ULL(i)))
f21fb3ed
RV		 304 continue;
305 iq = oct->instr_queue[i];
306
307 if (atomic_read(&iq->instr_pending)) {
308 spin_lock_bh(&iq->lock);
309 iq->fill_cnt = 0;
310 iq->octeon_read_index = iq->host_write_index;
311 iq->stats.instr_processed +=
312 atomic_read(&iq->instr_pending);
9a96bde4 313 lio_process_iq_request_list(oct, iq, 0);
f21fb3ed
RV		 314 spin_unlock_bh(&iq->lock);
315 }
316 }
317
318 /* Force all pending ordered list requests to time out. */
319 lio_process_ordered_list(oct, 1);
320
321 /* We do not need to wait for output queue packets to be processed. */
322}
323
324/**
325 * \brief Cleanup PCI AER uncorrectable error status
326 * @param dev Pointer to PCI device
327 */
328static void cleanup_aer_uncorrect_error_status(struct pci_dev *dev)
329{
330 int pos = 0x100;
331 u32 status, mask;
332
333 pr_info("%s :\n", __func__);
334
335 pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
336 pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask);
337 if (dev->error_state == pci_channel_io_normal)
338 status &= ~mask; /* Clear corresponding nonfatal bits */
339 else
340 status &= mask; /* Clear corresponding fatal bits */
341 pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);
342}
343
344/**
345 * \brief Stop all PCI IO to a given device
346 * @param dev Pointer to Octeon device
347 */
348static void stop_pci_io(struct octeon_device *oct)
349{
350 /* No more instructions will be forwarded. */
351 atomic_set(&oct->status, OCT_DEV_IN_RESET);
352
353 pci_disable_device(oct->pci_dev);
354
355 /* Disable interrupts */
5b07aee1 356 oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
f21fb3ed
RV		 357
358 pcierror_quiesce_device(oct);
359
360 /* Release the interrupt line */
361 free_irq(oct->pci_dev->irq, oct);
362
363 if (oct->flags & LIO_FLAG_MSI_ENABLED)
364 pci_disable_msi(oct->pci_dev);
365
366 dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
367 lio_get_state_string(&oct->status));
368
f21fb3ed
RV		 369 /* making it a common function for all OCTEON models */
370 cleanup_aer_uncorrect_error_status(oct->pci_dev);
371}
372
373/**
374 * \brief called when PCI error is detected
375 * @param pdev Pointer to PCI device
376 * @param state The current pci connection state
377 *
378 * This function is called after a PCI bus error affecting
379 * this device has been detected.
380 */
381static pci_ers_result_t liquidio_pcie_error_detected(struct pci_dev *pdev,
382 pci_channel_state_t state)
383{
384 struct octeon_device *oct = pci_get_drvdata(pdev);
385
386 /* Non-correctable Non-fatal errors */
387 if (state == pci_channel_io_normal) {
388 dev_err(&oct->pci_dev->dev, "Non-correctable non-fatal error reported:\n");
389 cleanup_aer_uncorrect_error_status(oct->pci_dev);
390 return PCI_ERS_RESULT_CAN_RECOVER;
391 }
392
393 /* Non-correctable Fatal errors */
394 dev_err(&oct->pci_dev->dev, "Non-correctable FATAL reported by PCI AER driver\n");
395 stop_pci_io(oct);
396
397 /* Always return a DISCONNECT. There is no support for recovery but only
398 * for a clean shutdown.
399 */
400 return PCI_ERS_RESULT_DISCONNECT;
401}
402
403/**
404 * \brief mmio handler
405 * @param pdev Pointer to PCI device
406 */
a7d5a3dc
RV		 407static pci_ers_result_t liquidio_pcie_mmio_enabled(
408 struct pci_dev *pdev __attribute__((unused)))
f21fb3ed
RV		 409{
410 /* We should never hit this since we never ask for a reset for a Fatal
411 * Error. We always return DISCONNECT in io_error above.
412 * But play safe and return RECOVERED for now.
413 */
414 return PCI_ERS_RESULT_RECOVERED;
415}
416
417/**
418 * \brief called after the pci bus has been reset.
419 * @param pdev Pointer to PCI device
420 *
421 * Restart the card from scratch, as if from a cold-boot. Implementation
422 * resembles the first-half of the octeon_resume routine.
423 */
a7d5a3dc
RV		 424static pci_ers_result_t liquidio_pcie_slot_reset(
425 struct pci_dev *pdev __attribute__((unused)))
f21fb3ed
RV		 426{
427 /* We should never hit this since we never ask for a reset for a Fatal
428 * Error. We always return DISCONNECT in io_error above.
429 * But play safe and return RECOVERED for now.
430 */
431 return PCI_ERS_RESULT_RECOVERED;
432}
433
434/**
435 * \brief called when traffic can start flowing again.
436 * @param pdev Pointer to PCI device
437 *
438 * This callback is called when the error recovery driver tells us that
439 * its OK to resume normal operation. Implementation resembles the
440 * second-half of the octeon_resume routine.
441 */
a7d5a3dc 442static void liquidio_pcie_resume(struct pci_dev *pdev __attribute__((unused)))
f21fb3ed
RV		 443{
444 /* Nothing to be done here. */
445}
446
447#ifdef CONFIG_PM
448/**
449 * \brief called when suspending
450 * @param pdev Pointer to PCI device
451 * @param state state to suspend to
452 */
a7d5a3dc
RV		 453static int liquidio_suspend(struct pci_dev *pdev __attribute__((unused)),
454 pm_message_t state __attribute__((unused)))
f21fb3ed
RV		 455{
456 return 0;
457}
458
459/**
460 * \brief called when resuming
461 * @param pdev Pointer to PCI device
462 */
a7d5a3dc 463static int liquidio_resume(struct pci_dev *pdev __attribute__((unused)))
f21fb3ed
RV		 464{
465 return 0;
466}
467#endif
468
469/* For PCI-E Advanced Error Recovery (AER) Interface */
166e2362 470static const struct pci_error_handlers liquidio_err_handler = {
f21fb3ed
RV		 471 .error_detected = liquidio_pcie_error_detected,
472 .mmio_enabled = liquidio_pcie_mmio_enabled,
473 .slot_reset = liquidio_pcie_slot_reset,
474 .resume = liquidio_pcie_resume,
475};
476
477static const struct pci_device_id liquidio_pci_tbl[] = {
478 { /* 68xx */
479 PCI_VENDOR_ID_CAVIUM, 0x91, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
480 },
481 { /* 66xx */
482 PCI_VENDOR_ID_CAVIUM, 0x92, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
483 },
e86b1ab6
RV		 484 { /* 23xx pf */
485 PCI_VENDOR_ID_CAVIUM, 0x9702, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
486 },
f21fb3ed
RV		 487 {
488 0, 0, 0, 0, 0, 0, 0
489 }
490};
491MODULE_DEVICE_TABLE(pci, liquidio_pci_tbl);
492
493static struct pci_driver liquidio_pci_driver = {
494 .name = "LiquidIO",
495 .id_table = liquidio_pci_tbl,
496 .probe = liquidio_probe,
497 .remove = liquidio_remove,
498 .err_handler = &liquidio_err_handler, /* For AER */
499
500#ifdef CONFIG_PM
501 .suspend = liquidio_suspend,
502 .resume = liquidio_resume,
503#endif
ca6139ff
RV		 504#ifdef CONFIG_PCI_IOV
505 .sriov_configure = liquidio_enable_sriov,
506#endif
f21fb3ed
RV		 507};
508
509/**
510 * \brief register PCI driver
511 */
512static int liquidio_init_pci(void)
513{
514 return pci_register_driver(&liquidio_pci_driver);
515}
516
517/**
518 * \brief unregister PCI driver
519 */
520static void liquidio_deinit_pci(void)
521{
522 pci_unregister_driver(&liquidio_pci_driver);
523}
524
f21fb3ed
RV		 525/**
526 * \brief Stop Tx queues
527 * @param netdev network device
528 */
529static inline void txqs_stop(struct net_device *netdev)
530{
531 if (netif_is_multiqueue(netdev)) {
532 int i;
533
534 for (i = 0; i < netdev->num_tx_queues; i++)
535 netif_stop_subqueue(netdev, i);
536 } else {
537 netif_stop_queue(netdev);
538 }
539}
540
541/**
542 * \brief Start Tx queues
543 * @param netdev network device
544 */
545static inline void txqs_start(struct net_device *netdev)
546{
547 if (netif_is_multiqueue(netdev)) {
548 int i;
549
550 for (i = 0; i < netdev->num_tx_queues; i++)
551 netif_start_subqueue(netdev, i);
552 } else {
553 netif_start_queue(netdev);
554 }
555}
556
557/**
558 * \brief Wake Tx queues
559 * @param netdev network device
560 */
561static inline void txqs_wake(struct net_device *netdev)
562{
1f164717
RV		 563 struct lio *lio = GET_LIO(netdev);
564
f21fb3ed
RV		 565 if (netif_is_multiqueue(netdev)) {
566 int i;
567
1f164717
RV		 568 for (i = 0; i < netdev->num_tx_queues; i++) {
569 int qno = lio->linfo.txpciq[i %
a82457f1 570 lio->oct_dev->num_iqs].s.q_no;
1f164717
RV		 571
572 if (__netif_subqueue_stopped(netdev, i)) {
573 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, qno,
574 tx_restart, 1);
26236fa9 575 netif_wake_subqueue(netdev, i);
1f164717
RV		 576 }
577 }
f21fb3ed 578 } else {
1f164717
RV		 579 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, lio->txq,
580 tx_restart, 1);
f21fb3ed
RV		 581 netif_wake_queue(netdev);
582 }
583}
584
585/**
586 * \brief Stop Tx queue
587 * @param netdev network device
588 */
589static void stop_txq(struct net_device *netdev)
590{
591 txqs_stop(netdev);
592}
593
594/**
595 * \brief Start Tx queue
596 * @param netdev network device
597 */
598static void start_txq(struct net_device *netdev)
599{
600 struct lio *lio = GET_LIO(netdev);
601
0cece6c5 602 if (lio->linfo.link.s.link_up) {
f21fb3ed
RV		 603 txqs_start(netdev);
604 return;
605 }
606}
607
608/**
609 * \brief Wake a queue
610 * @param netdev network device
611 * @param q which queue to wake
612 */
613static inline void wake_q(struct net_device *netdev, int q)
614{
615 if (netif_is_multiqueue(netdev))
616 netif_wake_subqueue(netdev, q);
617 else
618 netif_wake_queue(netdev);
619}
620
621/**
622 * \brief Stop a queue
623 * @param netdev network device
624 * @param q which queue to stop
625 */
626static inline void stop_q(struct net_device *netdev, int q)
627{
628 if (netif_is_multiqueue(netdev))
629 netif_stop_subqueue(netdev, q);
630 else
631 netif_stop_queue(netdev);
632}
633
634/**
635 * \brief Check Tx queue status, and take appropriate action
636 * @param lio per-network private data
637 * @returns 0 if full, number of queues woken up otherwise
638 */
639static inline int check_txq_status(struct lio *lio)
640{
641 int ret_val = 0;
642
643 if (netif_is_multiqueue(lio->netdev)) {
644 int numqs = lio->netdev->num_tx_queues;
645 int q, iq = 0;
646
647 /* check each sub-queue state */
648 for (q = 0; q < numqs; q++) {
26236fa9 649 iq = lio->linfo.txpciq[q %
a82457f1 650 lio->oct_dev->num_iqs].s.q_no;
f21fb3ed
RV		 651 if (octnet_iq_is_full(lio->oct_dev, iq))
652 continue;
26236fa9
RV		 653 if (__netif_subqueue_stopped(lio->netdev, q)) {
654 wake_q(lio->netdev, q);
1f164717
RV		 655 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq,
656 tx_restart, 1);
26236fa9
RV		 657 ret_val++;
658 }
f21fb3ed
RV		 659 }
660 } else {
661 if (octnet_iq_is_full(lio->oct_dev, lio->txq))
662 return 0;
663 wake_q(lio->netdev, lio->txq);
1f164717
RV		 664 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, lio->txq,
665 tx_restart, 1);
f21fb3ed
RV		 666 ret_val = 1;
667 }
668 return ret_val;
669}
670
671/**
672 * Remove the node at the head of the list. The list would be empty at
673 * the end of this call if there are no more nodes in the list.
674 */
675static inline struct list_head *list_delete_head(struct list_head *root)
676{
677 struct list_head *node;
678
679 if ((root->prev == root) && (root->next == root))
680 node = NULL;
681 else
682 node = root->next;
683
684 if (node)
685 list_del(node);
686
687 return node;
688}
689
690/**
fcd2b5e3 691 * \brief Delete gather lists
f21fb3ed
RV		 692 * @param lio per-network private data
693 */
fcd2b5e3 694static void delete_glists(struct lio *lio)
f21fb3ed
RV		 695{
696 struct octnic_gather *g;
fcd2b5e3 697 int i;
f21fb3ed 698
67e303e0
VB		 699 kfree(lio->glist_lock);
700 lio->glist_lock = NULL;
701
fcd2b5e3
RV		 702 if (!lio->glist)
703 return;
704
705 for (i = 0; i < lio->linfo.num_txpciq; i++) {
706 do {
707 g = (struct octnic_gather *)
708 list_delete_head(&lio->glist[i]);
67e303e0 709 if (g)
fcd2b5e3 710 kfree(g);
fcd2b5e3 711 } while (g);
67e303e0 712
58ad3198
FM		 713 if (lio->glists_virt_base && lio->glists_virt_base[i] &&
714 lio->glists_dma_base && lio->glists_dma_base[i]) {
67e303e0
VB		 715 lio_dma_free(lio->oct_dev,
716 lio->glist_entry_size * lio->tx_qsize,
717 lio->glists_virt_base[i],
718 lio->glists_dma_base[i]);
719 }
fcd2b5e3
RV		 720 }
721
67e303e0
VB		 722 kfree(lio->glists_virt_base);
723 lio->glists_virt_base = NULL;
724
725 kfree(lio->glists_dma_base);
726 lio->glists_dma_base = NULL;
727
728 kfree(lio->glist);
729 lio->glist = NULL;
f21fb3ed
RV		 730}
731
732/**
fcd2b5e3 733 * \brief Setup gather lists
f21fb3ed
RV		 734 * @param lio per-network private data
735 */
fcd2b5e3 736static int setup_glists(struct octeon_device *oct, struct lio *lio, int num_iqs)
f21fb3ed 737{
fcd2b5e3 738 int i, j;
f21fb3ed
RV		 739 struct octnic_gather *g;
740
fcd2b5e3
RV		 741 lio->glist_lock = kcalloc(num_iqs, sizeof(*lio->glist_lock),
742 GFP_KERNEL);
743 if (!lio->glist_lock)
67e303e0 744 return -ENOMEM;
f21fb3ed 745
fcd2b5e3
RV		 746 lio->glist = kcalloc(num_iqs, sizeof(*lio->glist),
747 GFP_KERNEL);
748 if (!lio->glist) {
67e303e0
VB		 749 kfree(lio->glist_lock);
750 lio->glist_lock = NULL;
751 return -ENOMEM;
752 }
753
754 lio->glist_entry_size =
755 ROUNDUP8((ROUNDUP4(OCTNIC_MAX_SG) >> 2) * OCT_SG_ENTRY_SIZE);
756
757 /* allocate memory to store virtual and dma base address of
758 * per glist consistent memory
759 */
760 lio->glists_virt_base = kcalloc(num_iqs, sizeof(*lio->glists_virt_base),
761 GFP_KERNEL);
762 lio->glists_dma_base = kcalloc(num_iqs, sizeof(*lio->glists_dma_base),
763 GFP_KERNEL);
764
765 if (!lio->glists_virt_base || !lio->glists_dma_base) {
766 delete_glists(lio);
767 return -ENOMEM;
fcd2b5e3 768 }
f21fb3ed 769
fcd2b5e3 770 for (i = 0; i < num_iqs; i++) {
b3ca9af0 771 int numa_node = dev_to_node(&oct->pci_dev->dev);
f21fb3ed 772
fcd2b5e3
RV		 773 spin_lock_init(&lio->glist_lock[i]);
774
775 INIT_LIST_HEAD(&lio->glist[i]);
776
67e303e0
VB		 777 lio->glists_virt_base[i] =
778 lio_dma_alloc(oct,
779 lio->glist_entry_size * lio->tx_qsize,
780 &lio->glists_dma_base[i]);
781
782 if (!lio->glists_virt_base[i]) {
783 delete_glists(lio);
784 return -ENOMEM;
785 }
786
fcd2b5e3
RV		 787 for (j = 0; j < lio->tx_qsize; j++) {
788 g = kzalloc_node(sizeof(*g), GFP_KERNEL,
789 numa_node);
790 if (!g)
791 g = kzalloc(sizeof(*g), GFP_KERNEL);
792 if (!g)
793 break;
794
67e303e0
VB		 795 g->sg = lio->glists_virt_base[i] +
796 (j * lio->glist_entry_size);
fcd2b5e3 797
67e303e0
VB		 798 g->sg_dma_ptr = lio->glists_dma_base[i] +
799 (j * lio->glist_entry_size);
fcd2b5e3
RV		 800
801 list_add_tail(&g->list, &lio->glist[i]);
f21fb3ed
RV		 802 }
803
fcd2b5e3
RV		 804 if (j != lio->tx_qsize) {
805 delete_glists(lio);
67e303e0 806 return -ENOMEM;
f21fb3ed 807 }
f21fb3ed
RV		 808 }
809
fcd2b5e3 810 return 0;
f21fb3ed
RV		 811}
812
813/**
814 * \brief Print link information
815 * @param netdev network device
816 */
817static void print_link_info(struct net_device *netdev)
818{
819 struct lio *lio = GET_LIO(netdev);
820
d18ca7df
IB		 821 if (!ifstate_check(lio, LIO_IFSTATE_RESETTING) &&
822 ifstate_check(lio, LIO_IFSTATE_REGISTERED)) {
f21fb3ed
RV		 823 struct oct_link_info *linfo = &lio->linfo;
824
0cece6c5 825 if (linfo->link.s.link_up) {
f21fb3ed
RV		 826 netif_info(lio, link, lio->netdev, "%d Mbps %s Duplex UP\n",
827 linfo->link.s.speed,
828 (linfo->link.s.duplex) ? "Full" : "Half");
829 } else {
830 netif_info(lio, link, lio->netdev, "Link Down\n");
831 }
832 }
833}
834
7b6b6c95
RV		 835/**
836 * \brief Routine to notify MTU change
837 * @param work work_struct data structure
838 */
839static void octnet_link_status_change(struct work_struct *work)
840{
841 struct cavium_wk *wk = (struct cavium_wk *)work;
842 struct lio *lio = (struct lio *)wk->ctxptr;
843
844 rtnl_lock();
845 call_netdevice_notifiers(NETDEV_CHANGEMTU, lio->netdev);
846 rtnl_unlock();
847}
848
849/**
850 * \brief Sets up the mtu status change work
851 * @param netdev network device
852 */
853static inline int setup_link_status_change_wq(struct net_device *netdev)
854{
855 struct lio *lio = GET_LIO(netdev);
856 struct octeon_device *oct = lio->oct_dev;
857
858 lio->link_status_wq.wq = alloc_workqueue("link-status",
859 WQ_MEM_RECLAIM, 0);
860 if (!lio->link_status_wq.wq) {
861 dev_err(&oct->pci_dev->dev, "unable to create cavium link status wq\n");
862 return -1;
863 }
864 INIT_DELAYED_WORK(&lio->link_status_wq.wk.work,
865 octnet_link_status_change);
866 lio->link_status_wq.wk.ctxptr = lio;
867
868 return 0;
869}
870
871static inline void cleanup_link_status_change_wq(struct net_device *netdev)
872{
873 struct lio *lio = GET_LIO(netdev);
874
875 if (lio->link_status_wq.wq) {
876 cancel_delayed_work_sync(&lio->link_status_wq.wk.work);
877 destroy_workqueue(lio->link_status_wq.wq);
878 }
879}
880
f21fb3ed
RV		 881/**
882 * \brief Update link status
883 * @param netdev network device
884 * @param ls link status structure
885 *
886 * Called on receipt of a link status response from the core application to
887 * update each interface's link status.
888 */
889static inline void update_link_status(struct net_device *netdev,
890 union oct_link_status *ls)
891{
892 struct lio *lio = GET_LIO(netdev);
0cece6c5 893 int changed = (lio->linfo.link.u64 != ls->u64);
f21fb3ed 894
0cece6c5 895 lio->linfo.link.u64 = ls->u64;
f21fb3ed 896
0cece6c5 897 if ((lio->intf_open) && (changed)) {
f21fb3ed 898 print_link_info(netdev);
0cece6c5 899 lio->link_changes++;
f21fb3ed 900
0cece6c5 901 if (lio->linfo.link.s.link_up) {
f21fb3ed 902 netif_carrier_on(netdev);
f21fb3ed
RV		 903 txqs_wake(netdev);
904 } else {
905 netif_carrier_off(netdev);
906 stop_txq(netdev);
907 }
908 }
909}
910
907aaa6b
VB		 911/**
912 * lio_sync_octeon_time_cb - callback that is invoked when soft command
913 * sent by lio_sync_octeon_time() has completed successfully or failed
914 *
915 * @oct - octeon device structure
916 * @status - indicates success or failure
917 * @buf - pointer to the command that was sent to firmware
918 **/
919static void lio_sync_octeon_time_cb(struct octeon_device *oct,
920 u32 status, void *buf)
921{
922 struct octeon_soft_command *sc = (struct octeon_soft_command *)buf;
923
924 if (status)
925 dev_err(&oct->pci_dev->dev,
926 "Failed to sync time to octeon; error=%d\n", status);
927
928 octeon_free_soft_command(oct, sc);
929}
930
931/**
932 * lio_sync_octeon_time - send latest localtime to octeon firmware so that
933 * firmware will correct it's time, in case there is a time skew
934 *
935 * @work: work scheduled to send time update to octeon firmware
936 **/
937static void lio_sync_octeon_time(struct work_struct *work)
938{
939 struct cavium_wk *wk = (struct cavium_wk *)work;
940 struct lio *lio = (struct lio *)wk->ctxptr;
941 struct octeon_device *oct = lio->oct_dev;
942 struct octeon_soft_command *sc;
943 struct timespec64 ts;
944 struct lio_time *lt;
945 int ret;
946
947 sc = octeon_alloc_soft_command(oct, sizeof(struct lio_time), 0, 0);
948 if (!sc) {
949 dev_err(&oct->pci_dev->dev,
950 "Failed to sync time to octeon: soft command allocation failed\n");
951 return;
952 }
953
954 lt = (struct lio_time *)sc->virtdptr;
955
956 /* Get time of the day */
957 getnstimeofday64(&ts);
958 lt->sec = ts.tv_sec;
959 lt->nsec = ts.tv_nsec;
960 octeon_swap_8B_data((u64 *)lt, (sizeof(struct lio_time)) / 8);
961
962 sc->iq_no = lio->linfo.txpciq[0].s.q_no;
963 octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
964 OPCODE_NIC_SYNC_OCTEON_TIME, 0, 0, 0);
965
966 sc->callback = lio_sync_octeon_time_cb;
967 sc->callback_arg = sc;
968 sc->wait_time = 1000;
969
970 ret = octeon_send_soft_command(oct, sc);
971 if (ret == IQ_SEND_FAILED) {
972 dev_err(&oct->pci_dev->dev,
973 "Failed to sync time to octeon: failed to send soft command\n");
974 octeon_free_soft_command(oct, sc);
975 }
976
977 queue_delayed_work(lio->sync_octeon_time_wq.wq,
978 &lio->sync_octeon_time_wq.wk.work,
979 msecs_to_jiffies(LIO_SYNC_OCTEON_TIME_INTERVAL_MS));
980}
981
982/**
983 * setup_sync_octeon_time_wq - Sets up the work to periodically update
984 * local time to octeon firmware
985 *
986 * @netdev - network device which should send time update to firmware
987 **/
988static inline int setup_sync_octeon_time_wq(struct net_device *netdev)
989{
990 struct lio *lio = GET_LIO(netdev);
991 struct octeon_device *oct = lio->oct_dev;
992
993 lio->sync_octeon_time_wq.wq =
994 alloc_workqueue("update-octeon-time", WQ_MEM_RECLAIM, 0);
995 if (!lio->sync_octeon_time_wq.wq) {
996 dev_err(&oct->pci_dev->dev, "Unable to create wq to update octeon time\n");
997 return -1;
998 }
999 INIT_DELAYED_WORK(&lio->sync_octeon_time_wq.wk.work,
1000 lio_sync_octeon_time);
1001 lio->sync_octeon_time_wq.wk.ctxptr = lio;
1002 queue_delayed_work(lio->sync_octeon_time_wq.wq,
1003 &lio->sync_octeon_time_wq.wk.work,
1004 msecs_to_jiffies(LIO_SYNC_OCTEON_TIME_INTERVAL_MS));
1005
1006 return 0;
1007}
1008
1009/**
1010 * cleanup_sync_octeon_time_wq - stop scheduling and destroy the work created
1011 * to periodically update local time to octeon firmware
1012 *
1013 * @netdev - network device which should send time update to firmware
1014 **/
1015static inline void cleanup_sync_octeon_time_wq(struct net_device *netdev)
1016{
1017 struct lio *lio = GET_LIO(netdev);
1018 struct cavium_wq *time_wq = &lio->sync_octeon_time_wq;
1019
1020 if (time_wq->wq) {
1021 cancel_delayed_work_sync(&time_wq->wk.work);
1022 destroy_workqueue(time_wq->wq);
1023 }
1024}
1025
bb54be58
FM		 1026static struct octeon_device *get_other_octeon_device(struct octeon_device *oct)
1027{
1028 struct octeon_device *other_oct;
1029
1030 other_oct = lio_get_device(oct->octeon_id + 1);
1031
1032 if (other_oct && other_oct->pci_dev) {
1033 int oct_busnum, other_oct_busnum;
1034
1035 oct_busnum = oct->pci_dev->bus->number;
1036 other_oct_busnum = other_oct->pci_dev->bus->number;
1037
1038 if (oct_busnum == other_oct_busnum) {
1039 int oct_slot, other_oct_slot;
1040
1041 oct_slot = PCI_SLOT(oct->pci_dev->devfn);
1042 other_oct_slot = PCI_SLOT(other_oct->pci_dev->devfn);
1043
1044 if (oct_slot == other_oct_slot)
1045 return other_oct;
1046 }
1047 }
1048
1049 return NULL;
1050}
1051
1052static void disable_all_vf_links(struct octeon_device *oct)
1053{
1054 struct net_device *netdev;
1055 int max_vfs, vf, i;
1056
1057 if (!oct)
1058 return;
1059
1060 max_vfs = oct->sriov_info.max_vfs;
1061
1062 for (i = 0; i < oct->ifcount; i++) {
1063 netdev = oct->props[i].netdev;
1064 if (!netdev)
1065 continue;
1066
1067 for (vf = 0; vf < max_vfs; vf++)
1068 liquidio_set_vf_link_state(netdev, vf,
1069 IFLA_VF_LINK_STATE_DISABLE);
1070 }
1071}
1072
9ff1a9ba
RV		 1073static int liquidio_watchdog(void *param)
1074{
bb54be58
FM		 1075 bool err_msg_was_printed[LIO_MAX_CORES];
1076 u16 mask_of_crashed_or_stuck_cores = 0;
1077 bool all_vf_links_are_disabled = false;
9ff1a9ba 1078 struct octeon_device *oct = param;
bb54be58
FM		 1079 struct octeon_device *other_oct;
1080#ifdef CONFIG_MODULE_UNLOAD
1081 long refcount, vfs_referencing_pf;
1082 u64 vfs_mask1, vfs_mask2;
1083#endif
1084 int core;
9ff1a9ba 1085
bb54be58 1086 memset(err_msg_was_printed, 0, sizeof(err_msg_was_printed));
9ff1a9ba
RV		 1087
1088 while (!kthread_should_stop()) {
bb54be58
FM		 1089 /* sleep for a couple of seconds so that we don't hog the CPU */
1090 set_current_state(TASK_INTERRUPTIBLE);
1091 schedule_timeout(msecs_to_jiffies(2000));
1092
1093 mask_of_crashed_or_stuck_cores =
9ff1a9ba
RV		 1094 (u16)octeon_read_csr64(oct, CN23XX_SLI_SCRATCH2);
1095
bb54be58
FM		 1096 if (!mask_of_crashed_or_stuck_cores)
1097 continue;
9ff1a9ba 1098
bb54be58
FM		 1099 WRITE_ONCE(oct->cores_crashed, true);
1100 other_oct = get_other_octeon_device(oct);
1101 if (other_oct)
1102 WRITE_ONCE(other_oct->cores_crashed, true);
9ff1a9ba 1103
bb54be58
FM		 1104 for (core = 0; core < LIO_MAX_CORES; core++) {
1105 bool core_crashed_or_got_stuck;
9ff1a9ba 1106
bb54be58
FM		 1107 core_crashed_or_got_stuck =
1108 (mask_of_crashed_or_stuck_cores
1109 >> core) & 1;
1110
1111 if (core_crashed_or_got_stuck &&
1112 !err_msg_was_printed[core]) {
1113 dev_err(&oct->pci_dev->dev,
1114 "ERROR: Octeon core %d crashed or got stuck! See oct-fwdump for details.\n",
1115 core);
886afc1d 1116 err_msg_was_printed[core] = true;
9ff1a9ba
RV		 1117 }
1118 }
bb54be58
FM		 1119
1120 if (all_vf_links_are_disabled)
1121 continue;
1122
1123 disable_all_vf_links(oct);
1124 disable_all_vf_links(other_oct);
1125 all_vf_links_are_disabled = true;
1126
9ff1a9ba 1127#ifdef CONFIG_MODULE_UNLOAD
bb54be58
FM		 1128 vfs_mask1 = READ_ONCE(oct->sriov_info.vf_drv_loaded_mask);
1129 vfs_mask2 = READ_ONCE(other_oct->sriov_info.vf_drv_loaded_mask);
9ff1a9ba 1130
bb54be58
FM		 1131 vfs_referencing_pf = hweight64(vfs_mask1);
1132 vfs_referencing_pf += hweight64(vfs_mask2);
9ff1a9ba 1133
bb54be58
FM		 1134 refcount = module_refcount(THIS_MODULE);
1135 if (refcount >= vfs_referencing_pf) {
1136 while (vfs_referencing_pf) {
9ff1a9ba 1137 module_put(THIS_MODULE);
bb54be58 1138 vfs_referencing_pf--;
9ff1a9ba
RV		 1139 }
1140 }
1141#endif
9ff1a9ba
RV		 1142 }
1143
1144 return 0;
1145}
1146
f21fb3ed
RV		 1147/**
1148 * \brief PCI probe handler
1149 * @param pdev PCI device structure
1150 * @param ent unused
1151 */
a7d5a3dc
RV		 1152static int
1153liquidio_probe(struct pci_dev *pdev,
1154 const struct pci_device_id *ent __attribute__((unused)))
f21fb3ed
RV		 1155{
1156 struct octeon_device *oct_dev = NULL;
1157 struct handshake *hs;
1158
1159 oct_dev = octeon_allocate_device(pdev->device,
1160 sizeof(struct octeon_device_priv));
1161 if (!oct_dev) {
1162 dev_err(&pdev->dev, "Unable to allocate device\n");
1163 return -ENOMEM;
1164 }
1165
5b07aee1
RV		 1166 if (pdev->device == OCTEON_CN23XX_PF_VID)
1167 oct_dev->msix_on = LIO_FLAG_MSIX_ENABLED;
1168
aa69ff9e
IB		 1169 /* Enable PTP for 6XXX Device */
1170 if (((pdev->device == OCTEON_CN66XX) ||
1171 (pdev->device == OCTEON_CN68XX)))
1172 oct_dev->ptp_enable = true;
1173 else
1174 oct_dev->ptp_enable = false;
1175
f21fb3ed
RV		 1176 dev_info(&pdev->dev, "Initializing device %x:%x.\n",
1177 (u32)pdev->vendor, (u32)pdev->device);
1178
1179 /* Assign octeon_device for this device to the private data area. */
1180 pci_set_drvdata(pdev, oct_dev);
1181
1182 /* set linux specific device pointer */
1183 oct_dev->pci_dev = (void *)pdev;
1184
1185 hs = &handshake[oct_dev->octeon_id];
1186 init_completion(&hs->init);
1187 init_completion(&hs->started);
1188 hs->pci_dev = pdev;
1189
1190 if (oct_dev->octeon_id == 0)
1191 /* first LiquidIO NIC is detected */
1192 complete(&first_stage);
1193
1194 if (octeon_device_init(oct_dev)) {
515e752d 1195 complete(&hs->init);
f21fb3ed
RV		 1196 liquidio_remove(pdev);
1197 return -ENOMEM;
1198 }
1199
9ff1a9ba 1200 if (OCTEON_CN23XX_PF(oct_dev)) {
9ff1a9ba
RV		 1201 u8 bus, device, function;
1202
392209fa
FM		 1203 if (atomic_read(oct_dev->adapter_refcount) == 1) {
1204 /* Each NIC gets one watchdog kernel thread. The first
1205 * PF (of each NIC) that gets pci_driver->probe()'d
1206 * creates that thread.
9ff1a9ba 1207 */
9ff1a9ba
RV		 1208 bus = pdev->bus->number;
1209 device = PCI_SLOT(pdev->devfn);
1210 function = PCI_FUNC(pdev->devfn);
1211 oct_dev->watchdog_task = kthread_create(
1212 liquidio_watchdog, oct_dev,
1213 "liowd/%02hhx:%02hhx.%hhx", bus, device, function);
515e752d
RV		 1214 if (!IS_ERR(oct_dev->watchdog_task)) {
1215 wake_up_process(oct_dev->watchdog_task);
1216 } else {
1217 oct_dev->watchdog_task = NULL;
1218 dev_err(&oct_dev->pci_dev->dev,
1219 "failed to create kernel_thread\n");
1220 liquidio_remove(pdev);
1221 return -1;
1222 }
9ff1a9ba
RV		 1223 }
1224 }
1225
1f164717
RV		 1226 oct_dev->rx_pause = 1;
1227 oct_dev->tx_pause = 1;
1228
f21fb3ed
RV		 1229 dev_dbg(&oct_dev->pci_dev->dev, "Device is ready\n");
1230
1231 return 0;
1232}
1233
088b8749 1234static bool fw_type_is_auto(void)
7cc61db9 1235{
088b8749
RF		 1236 return strncmp(fw_type, LIO_FW_NAME_TYPE_AUTO,
1237 sizeof(LIO_FW_NAME_TYPE_AUTO)) == 0;
7cc61db9
FM		 1238}
1239
70535350
RF		 1240/**
1241 * \brief PCI FLR for each Octeon device.
1242 * @param oct octeon device
1243 */
1244static void octeon_pci_flr(struct octeon_device *oct)
1245{
1246 int rc;
1247
1248 pci_save_state(oct->pci_dev);
1249
1250 pci_cfg_access_lock(oct->pci_dev);
1251
1252 /* Quiesce the device completely */
1253 pci_write_config_word(oct->pci_dev, PCI_COMMAND,
1254 PCI_COMMAND_INTX_DISABLE);
1255
1256 rc = __pci_reset_function_locked(oct->pci_dev);
1257
1258 if (rc != 0)
1259 dev_err(&oct->pci_dev->dev, "Error %d resetting PCI function %d\n",
1260 rc, oct->pf_num);
1261
1262 pci_cfg_access_unlock(oct->pci_dev);
1263
1264 pci_restore_state(oct->pci_dev);
1265}
1266
f21fb3ed
RV		 1267/**
1268 *\brief Destroy resources associated with octeon device
1269 * @param pdev PCI device structure
1270 * @param ent unused
1271 */
1272static void octeon_destroy_resources(struct octeon_device *oct)
1273{
e1e3ce62 1274 int i, refcount;
5b07aee1 1275 struct msix_entry *msix_entries;
f21fb3ed
RV		 1276 struct octeon_device_priv *oct_priv =
1277 (struct octeon_device_priv *)oct->priv;
1278
1279 struct handshake *hs;
1280
1281 switch (atomic_read(&oct->status)) {
1282 case OCT_DEV_RUNNING:
1283 case OCT_DEV_CORE_OK:
1284
1285 /* No more instructions will be forwarded. */
1286 atomic_set(&oct->status, OCT_DEV_IN_RESET);
1287
1288 oct->app_mode = CVM_DRV_INVALID_APP;
1289 dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
1290 lio_get_state_string(&oct->status));
1291
1292 schedule_timeout_uninterruptible(HZ / 10);
1293
1294 /* fallthrough */
1295 case OCT_DEV_HOST_OK:
1296
1297 /* fallthrough */
1298 case OCT_DEV_CONSOLE_INIT_DONE:
1299 /* Remove any consoles */
1300 octeon_remove_consoles(oct);
1301
1302 /* fallthrough */
1303 case OCT_DEV_IO_QUEUES_DONE:
1304 if (wait_for_pending_requests(oct))
1305 dev_err(&oct->pci_dev->dev, "There were pending requests\n");
1306
1307 if (lio_wait_for_instr_fetch(oct))
1308 dev_err(&oct->pci_dev->dev, "IQ had pending instructions\n");
1309
1310 /* Disable the input and output queues now. No more packets will
1311 * arrive from Octeon, but we should wait for all packet
1312 * processing to finish.
1313 */
1314 oct->fn_list.disable_io_queues(oct);
1315
1316 if (lio_wait_for_oq_pkts(oct))
1317 dev_err(&oct->pci_dev->dev, "OQ had pending packets\n");
1318
515e752d
RV		 1319 /* fallthrough */
1320 case OCT_DEV_INTR_SET_DONE:
f21fb3ed 1321 /* Disable interrupts */
5b07aee1
RV		 1322 oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
1323
1324 if (oct->msix_on) {
1325 msix_entries = (struct msix_entry *)oct->msix_entries;
1326 for (i = 0; i < oct->num_msix_irqs - 1; i++) {
a82457f1
IB		 1327 if (oct->ioq_vector[i].vector) {
1328 /* clear the affinity_cpumask */
1329 irq_set_affinity_hint(
1330 msix_entries[i].vector,
1331 NULL);
1332 free_irq(msix_entries[i].vector,
1333 &oct->ioq_vector[i]);
1334 oct->ioq_vector[i].vector = 0;
1335 }
5b07aee1
RV		 1336 }
1337 /* non-iov vector's argument is oct struct */
1338 free_irq(msix_entries[i].vector, oct);
f21fb3ed 1339
5b07aee1
RV		 1340 pci_disable_msix(oct->pci_dev);
1341 kfree(oct->msix_entries);
1342 oct->msix_entries = NULL;
1343 } else {
1344 /* Release the interrupt line */
1345 free_irq(oct->pci_dev->irq, oct);
f21fb3ed 1346
5b07aee1
RV		 1347 if (oct->flags & LIO_FLAG_MSI_ENABLED)
1348 pci_disable_msi(oct->pci_dev);
1349 }
f21fb3ed 1350
0c88a761
RF		 1351 kfree(oct->irq_name_storage);
1352 oct->irq_name_storage = NULL;
1353
515e752d
RV		 1354 /* fallthrough */
1355 case OCT_DEV_MSIX_ALLOC_VECTOR_DONE:
5b07aee1
RV		 1356 if (OCTEON_CN23XX_PF(oct))
1357 octeon_free_ioq_vector(oct);
5d65556b
RV		 1358
1359 /* fallthrough */
1360 case OCT_DEV_MBOX_SETUP_DONE:
1361 if (OCTEON_CN23XX_PF(oct))
1362 oct->fn_list.free_mbox(oct);
1363
5b07aee1 1364 /* fallthrough */
f21fb3ed
RV		 1365 case OCT_DEV_IN_RESET:
1366 case OCT_DEV_DROQ_INIT_DONE:
763185a3 1367 /* Wait for any pending operations */
f21fb3ed 1368 mdelay(100);
63da8404 1369 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
5b07aee1 1370 if (!(oct->io_qmask.oq & BIT_ULL(i)))
f21fb3ed
RV		 1371 continue;
1372 octeon_delete_droq(oct, i);
1373 }
1374
1375 /* Force any pending handshakes to complete */
1376 for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
1377 hs = &handshake[i];
1378
1379 if (hs->pci_dev) {
1380 handshake[oct->octeon_id].init_ok = 0;
1381 complete(&handshake[oct->octeon_id].init);
1382 handshake[oct->octeon_id].started_ok = 0;
1383 complete(&handshake[oct->octeon_id].started);
1384 }
1385 }
1386
1387 /* fallthrough */
1388 case OCT_DEV_RESP_LIST_INIT_DONE:
1389 octeon_delete_response_list(oct);
1390
f21fb3ed
RV		 1391 /* fallthrough */
1392 case OCT_DEV_INSTR_QUEUE_INIT_DONE:
63da8404 1393 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
5b823514 1394 if (!(oct->io_qmask.iq & BIT_ULL(i)))
f21fb3ed
RV		 1395 continue;
1396 octeon_delete_instr_queue(oct, i);
1397 }
ca6139ff
RV		 1398#ifdef CONFIG_PCI_IOV
1399 if (oct->sriov_info.sriov_enabled)
1400 pci_disable_sriov(oct->pci_dev);
1401#endif
5b823514
RV		 1402 /* fallthrough */
1403 case OCT_DEV_SC_BUFF_POOL_INIT_DONE:
1404 octeon_free_sc_buffer_pool(oct);
f21fb3ed
RV		 1405
1406 /* fallthrough */
1407 case OCT_DEV_DISPATCH_INIT_DONE:
1408 octeon_delete_dispatch_list(oct);
1409 cancel_delayed_work_sync(&oct->nic_poll_work.work);
1410
1411 /* fallthrough */
1412 case OCT_DEV_PCI_MAP_DONE:
e1e3ce62
RF		 1413 refcount = octeon_deregister_device(oct);
1414
70535350
RF		 1415 /* Soft reset the octeon device before exiting.
1416 * However, if fw was loaded from card (i.e. autoboot),
1417 * perform an FLR instead.
1418 * Implementation note: only soft-reset the device
1419 * if it is a CN6XXX OR the LAST CN23XX device.
1420 */
088b8749 1421 if (atomic_read(oct->adapter_fw_state) == FW_IS_PRELOADED)
70535350
RF		 1422 octeon_pci_flr(oct);
1423 else if (OCTEON_CN6XXX(oct) || !refcount)
1424 oct->fn_list.soft_reset(oct);
60b48c5a 1425
f21fb3ed
RV		 1426 octeon_unmap_pci_barx(oct, 0);
1427 octeon_unmap_pci_barx(oct, 1);
1428
1429 /* fallthrough */
515e752d
RV		 1430 case OCT_DEV_PCI_ENABLE_DONE:
1431 pci_clear_master(oct->pci_dev);
60b48c5a
RV		 1432 /* Disable the device, releasing the PCI INT */
1433 pci_disable_device(oct->pci_dev);
1434
515e752d
RV		 1435 /* fallthrough */
1436 case OCT_DEV_BEGIN_STATE:
f21fb3ed
RV		 1437 /* Nothing to be done here either */
1438 break;
a2c64b67 1439 } /* end switch (oct->status) */
f21fb3ed
RV		 1440
1441 tasklet_kill(&oct_priv->droq_tasklet);
1442}
1443
afdf841f
RV		 1444/**
1445 * \brief Callback for rx ctrl
1446 * @param status status of request
1447 * @param buf pointer to resp structure
1448 */
1449static void rx_ctl_callback(struct octeon_device *oct,
1450 u32 status,
1451 void *buf)
1452{
1453 struct octeon_soft_command *sc = (struct octeon_soft_command *)buf;
1454 struct liquidio_rx_ctl_context *ctx;
1455
1456 ctx = (struct liquidio_rx_ctl_context *)sc->ctxptr;
1457
1458 oct = lio_get_device(ctx->octeon_id);
1459 if (status)
1460 dev_err(&oct->pci_dev->dev, "rx ctl instruction failed. Status: %llx\n",
1461 CVM_CAST64(status));
1462 WRITE_ONCE(ctx->cond, 1);
1463
1464 /* This barrier is required to be sure that the response has been
1465 * written fully before waking up the handler
1466 */
1467 wmb();
1468
1469 wake_up_interruptible(&ctx->wc);
1470}
1471
f21fb3ed
RV		 1472/**
1473 * \brief Send Rx control command
1474 * @param lio per-network private data
1475 * @param start_stop whether to start or stop
1476 */
1477static void send_rx_ctrl_cmd(struct lio *lio, int start_stop)
1478{
afdf841f
RV		 1479 struct octeon_soft_command *sc;
1480 struct liquidio_rx_ctl_context *ctx;
1481 union octnet_cmd *ncmd;
1482 int ctx_size = sizeof(struct liquidio_rx_ctl_context);
1483 struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1484 int retval;
f21fb3ed 1485
afdf841f
RV		 1486 if (oct->props[lio->ifidx].rx_on == start_stop)
1487 return;
f21fb3ed 1488
afdf841f
RV		 1489 sc = (struct octeon_soft_command *)
1490 octeon_alloc_soft_command(oct, OCTNET_CMD_SIZE,
1491 16, ctx_size);
1492
1493 ncmd = (union octnet_cmd *)sc->virtdptr;
1494 ctx = (struct liquidio_rx_ctl_context *)sc->ctxptr;
1495
1496 WRITE_ONCE(ctx->cond, 0);
1497 ctx->octeon_id = lio_get_device_id(oct);
1498 init_waitqueue_head(&ctx->wc);
1499
1500 ncmd->u64 = 0;
1501 ncmd->s.cmd = OCTNET_CMD_RX_CTL;
1502 ncmd->s.param1 = start_stop;
1503
1504 octeon_swap_8B_data((u64 *)ncmd, (OCTNET_CMD_SIZE >> 3));
1505
1506 sc->iq_no = lio->linfo.txpciq[0].s.q_no;
1507
1508 octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
1509 OPCODE_NIC_CMD, 0, 0, 0);
1510
1511 sc->callback = rx_ctl_callback;
1512 sc->callback_arg = sc;
1513 sc->wait_time = 5000;
f21fb3ed 1514
afdf841f
RV		 1515 retval = octeon_send_soft_command(oct, sc);
1516 if (retval == IQ_SEND_FAILED) {
f21fb3ed 1517 netif_info(lio, rx_err, lio->netdev, "Failed to send RX Control message\n");
afdf841f
RV		 1518 } else {
1519 /* Sleep on a wait queue till the cond flag indicates that the
1520 * response arrived or timed-out.
1521 */
1522 if (sleep_cond(&ctx->wc, &ctx->cond) == -EINTR)
1523 return;
1524 oct->props[lio->ifidx].rx_on = start_stop;
1525 }
1526
1527 octeon_free_soft_command(oct, sc);
f21fb3ed
RV		 1528}
1529
1530/**
1531 * \brief Destroy NIC device interface
1532 * @param oct octeon device
1533 * @param ifidx which interface to destroy
1534 *
1535 * Cleanup associated with each interface for an Octeon device when NIC
1536 * module is being unloaded or if initialization fails during load.
1537 */
1538static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx)
1539{
1540 struct net_device *netdev = oct->props[ifidx].netdev;
1541 struct lio *lio;
9a96bde4 1542 struct napi_struct *napi, *n;
f21fb3ed
RV		 1543
1544 if (!netdev) {
1545 dev_err(&oct->pci_dev->dev, "%s No netdevice ptr for index %d\n",
1546 __func__, ifidx);
1547 return;
1548 }
1549
1550 lio = GET_LIO(netdev);
1551
1552 dev_dbg(&oct->pci_dev->dev, "NIC device cleanup\n");
1553
f21fb3ed 1554 if (atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING)
afdf841f 1555 liquidio_stop(netdev);
f21fb3ed 1556
9a96bde4
RV		 1557 if (oct->props[lio->ifidx].napi_enabled == 1) {
1558 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1559 napi_disable(napi);
1560
1561 oct->props[lio->ifidx].napi_enabled = 0;
7b6b6c95
RV		 1562
1563 if (OCTEON_CN23XX_PF(oct))
1564 oct->droq[0]->ops.poll_mode = 0;
9a96bde4
RV		 1565 }
1566
42013e90
IB		 1567 /* Delete NAPI */
1568 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1569 netif_napi_del(napi);
1570
f21fb3ed
RV		 1571 if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED)
1572 unregister_netdev(netdev);
1573
907aaa6b 1574 cleanup_sync_octeon_time_wq(netdev);
7b6b6c95
RV		 1575 cleanup_link_status_change_wq(netdev);
1576
031d4f12
SB		 1577 cleanup_rx_oom_poll_fn(netdev);
1578
fcd2b5e3 1579 delete_glists(lio);
f21fb3ed
RV		 1580
1581 free_netdev(netdev);
1582
0cece6c5
RV		 1583 oct->props[ifidx].gmxport = -1;
1584
f21fb3ed
RV		 1585 oct->props[ifidx].netdev = NULL;
1586}
1587
1588/**
1589 * \brief Stop complete NIC functionality
1590 * @param oct octeon device
1591 */
1592static int liquidio_stop_nic_module(struct octeon_device *oct)
1593{
1594 int i, j;
1595 struct lio *lio;
1596
1597 dev_dbg(&oct->pci_dev->dev, "Stopping network interfaces\n");
1598 if (!oct->ifcount) {
1599 dev_err(&oct->pci_dev->dev, "Init for Octeon was not completed\n");
1600 return 1;
1601 }
1602
60441888
RV		 1603 spin_lock_bh(&oct->cmd_resp_wqlock);
1604 oct->cmd_resp_state = OCT_DRV_OFFLINE;
1605 spin_unlock_bh(&oct->cmd_resp_wqlock);
1606
d4be8ebe
VMG		 1607 lio_vf_rep_destroy(oct);
1608
f21fb3ed
RV		 1609 for (i = 0; i < oct->ifcount; i++) {
1610 lio = GET_LIO(oct->props[i].netdev);
a82457f1 1611 for (j = 0; j < oct->num_oqs; j++)
26236fa9
RV		 1612 octeon_unregister_droq_ops(oct,
1613 lio->linfo.rxpciq[j].s.q_no);
f21fb3ed
RV		 1614 }
1615
1616 for (i = 0; i < oct->ifcount; i++)
1617 liquidio_destroy_nic_device(oct, i);
1618
d4be8ebe
VMG		 1619 if (oct->devlink) {
1620 devlink_unregister(oct->devlink);
1621 devlink_free(oct->devlink);
1622 oct->devlink = NULL;
1623 }
1624
f21fb3ed
RV		 1625 dev_dbg(&oct->pci_dev->dev, "Network interfaces stopped\n");
1626 return 0;
1627}
1628
1629/**
1630 * \brief Cleans up resources at unload time
1631 * @param pdev PCI device structure
1632 */
1633static void liquidio_remove(struct pci_dev *pdev)
1634{
1635 struct octeon_device *oct_dev = pci_get_drvdata(pdev);
1636
1637 dev_dbg(&oct_dev->pci_dev->dev, "Stopping device\n");
1638
9ff1a9ba
RV		 1639 if (oct_dev->watchdog_task)
1640 kthread_stop(oct_dev->watchdog_task);
1641
e20f4696
VMG		 1642 if (!oct_dev->octeon_id &&
1643 oct_dev->fw_info.app_cap_flags & LIQUIDIO_SWITCHDEV_CAP)
1644 lio_vf_rep_modexit();
1645
f21fb3ed
RV		 1646 if (oct_dev->app_mode && (oct_dev->app_mode == CVM_DRV_NIC_APP))
1647 liquidio_stop_nic_module(oct_dev);
1648
1649 /* Reset the octeon device and cleanup all memory allocated for
1650 * the octeon device by driver.
1651 */
1652 octeon_destroy_resources(oct_dev);
1653
1654 dev_info(&oct_dev->pci_dev->dev, "Device removed\n");
1655
1656 /* This octeon device has been removed. Update the global
1657 * data structure to reflect this. Free the device structure.
1658 */
1659 octeon_free_device_mem(oct_dev);
1660}
1661
1662/**
1663 * \brief Identify the Octeon device and to map the BAR address space
1664 * @param oct octeon device
1665 */
1666static int octeon_chip_specific_setup(struct octeon_device *oct)
1667{
1668 u32 dev_id, rev_id;
1669 int ret = 1;
d3d7e6c6 1670 char *s;
f21fb3ed
RV		 1671
1672 pci_read_config_dword(oct->pci_dev, 0, &dev_id);
1673 pci_read_config_dword(oct->pci_dev, 8, &rev_id);
1674 oct->rev_id = rev_id & 0xff;
1675
1676 switch (dev_id) {
1677 case OCTEON_CN68XX_PCIID:
1678 oct->chip_id = OCTEON_CN68XX;
1679 ret = lio_setup_cn68xx_octeon_device(oct);
d3d7e6c6 1680 s = "CN68XX";
f21fb3ed
RV		 1681 break;
1682
1683 case OCTEON_CN66XX_PCIID:
1684 oct->chip_id = OCTEON_CN66XX;
1685 ret = lio_setup_cn66xx_octeon_device(oct);
d3d7e6c6 1686 s = "CN66XX";
f21fb3ed 1687 break;
d3d7e6c6 1688
72c00912
RV		 1689 case OCTEON_CN23XX_PCIID_PF:
1690 oct->chip_id = OCTEON_CN23XX_PF_VID;
1691 ret = setup_cn23xx_octeon_pf_device(oct);
0c45d7fe
RF		 1692 if (ret)
1693 break;
cf19a8c3
DC		 1694#ifdef CONFIG_PCI_IOV
1695 if (!ret)
1696 pci_sriov_set_totalvfs(oct->pci_dev,
1697 oct->sriov_info.max_vfs);
1698#endif
72c00912
RV		 1699 s = "CN23XX";
1700 break;
1701
f21fb3ed 1702 default:
d3d7e6c6 1703 s = "?";
f21fb3ed
RV		 1704 dev_err(&oct->pci_dev->dev, "Unknown device found (dev_id: %x)\n",
1705 dev_id);
1706 }
1707
1708 if (!ret)
d3d7e6c6 1709 dev_info(&oct->pci_dev->dev, "%s PASS%d.%d %s Version: %s\n", s,
f21fb3ed
RV		 1710 OCTEON_MAJOR_REV(oct),
1711 OCTEON_MINOR_REV(oct),
d3d7e6c6
RV		 1712 octeon_get_conf(oct)->card_name,
1713 LIQUIDIO_VERSION);
f21fb3ed
RV		 1714
1715 return ret;
1716}
1717
1718/**
1719 * \brief PCI initialization for each Octeon device.
1720 * @param oct octeon device
1721 */
1722static int octeon_pci_os_setup(struct octeon_device *oct)
1723{
1724 /* setup PCI stuff first */
1725 if (pci_enable_device(oct->pci_dev)) {
1726 dev_err(&oct->pci_dev->dev, "pci_enable_device failed\n");
1727 return 1;
1728 }
1729
1730 if (dma_set_mask_and_coherent(&oct->pci_dev->dev, DMA_BIT_MASK(64))) {
1731 dev_err(&oct->pci_dev->dev, "Unexpected DMA device capability\n");
515e752d 1732 pci_disable_device(oct->pci_dev);
f21fb3ed
RV		 1733 return 1;
1734 }
1735
1736 /* Enable PCI DMA Master. */
1737 pci_set_master(oct->pci_dev);
1738
1739 return 0;
1740}
1741
fcd2b5e3
RV		 1742static inline int skb_iq(struct lio *lio, struct sk_buff *skb)
1743{
1744 int q = 0;
1745
1746 if (netif_is_multiqueue(lio->netdev))
1747 q = skb->queue_mapping % lio->linfo.num_txpciq;
1748
1749 return q;
1750}
1751
f21fb3ed
RV		 1752/**
1753 * \brief Check Tx queue state for a given network buffer
1754 * @param lio per-network private data
1755 * @param skb network buffer
1756 */
1757static inline int check_txq_state(struct lio *lio, struct sk_buff *skb)
1758{
1759 int q = 0, iq = 0;
1760
1761 if (netif_is_multiqueue(lio->netdev)) {
1762 q = skb->queue_mapping;
a82457f1 1763 iq = lio->linfo.txpciq[(q % lio->oct_dev->num_iqs)].s.q_no;
f21fb3ed
RV		 1764 } else {
1765 iq = lio->txq;
26236fa9 1766 q = iq;
f21fb3ed
RV		 1767 }
1768
1769 if (octnet_iq_is_full(lio->oct_dev, iq))
1770 return 0;
26236fa9 1771
1f164717
RV		 1772 if (__netif_subqueue_stopped(lio->netdev, q)) {
1773 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq, tx_restart, 1);
26236fa9 1774 wake_q(lio->netdev, q);
1f164717 1775 }
f21fb3ed
RV		 1776 return 1;
1777}
1778
1779/**
1780 * \brief Unmap and free network buffer
1781 * @param buf buffer
1782 */
1783static void free_netbuf(void *buf)
1784{
1785 struct sk_buff *skb;
1786 struct octnet_buf_free_info *finfo;
1787 struct lio *lio;
1788
1789 finfo = (struct octnet_buf_free_info *)buf;
1790 skb = finfo->skb;
1791 lio = finfo->lio;
1792
1793 dma_unmap_single(&lio->oct_dev->pci_dev->dev, finfo->dptr, skb->len,
1794 DMA_TO_DEVICE);
1795
1796 check_txq_state(lio, skb);
1797
cabeb13b 1798 tx_buffer_free(skb);
f21fb3ed
RV		 1799}
1800
1801/**
1802 * \brief Unmap and free gather buffer
1803 * @param buf buffer
1804 */
1805static void free_netsgbuf(void *buf)
1806{
1807 struct octnet_buf_free_info *finfo;
1808 struct sk_buff *skb;
1809 struct lio *lio;
1810 struct octnic_gather *g;
fcd2b5e3 1811 int i, frags, iq;
f21fb3ed
RV		 1812
1813 finfo = (struct octnet_buf_free_info *)buf;
1814 skb = finfo->skb;
1815 lio = finfo->lio;
1816 g = finfo->g;
1817 frags = skb_shinfo(skb)->nr_frags;
1818
1819 dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1820 g->sg[0].ptr[0], (skb->len - skb->data_len),
1821 DMA_TO_DEVICE);
1822
1823 i = 1;
1824 while (frags--) {
1825 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1];
1826
1827 pci_unmap_page((lio->oct_dev)->pci_dev,
1828 g->sg[(i >> 2)].ptr[(i & 3)],
1829 frag->size, DMA_TO_DEVICE);
1830 i++;
1831 }
1832
fcd2b5e3
RV		 1833 iq = skb_iq(lio, skb);
1834 spin_lock(&lio->glist_lock[iq]);
1835 list_add_tail(&g->list, &lio->glist[iq]);
1836 spin_unlock(&lio->glist_lock[iq]);
f21fb3ed
RV		 1837
1838 check_txq_state(lio, skb); /* mq support: sub-queue state check */
1839
cabeb13b 1840 tx_buffer_free(skb);
f21fb3ed
RV		 1841}
1842
1843/**
1844 * \brief Unmap and free gather buffer with response
1845 * @param buf buffer
1846 */
1847static void free_netsgbuf_with_resp(void *buf)
1848{
1849 struct octeon_soft_command *sc;
1850 struct octnet_buf_free_info *finfo;
1851 struct sk_buff *skb;
1852 struct lio *lio;
1853 struct octnic_gather *g;
fcd2b5e3 1854 int i, frags, iq;
f21fb3ed
RV		 1855
1856 sc = (struct octeon_soft_command *)buf;
1857 skb = (struct sk_buff *)sc->callback_arg;
1858 finfo = (struct octnet_buf_free_info *)&skb->cb;
1859
1860 lio = finfo->lio;
1861 g = finfo->g;
1862 frags = skb_shinfo(skb)->nr_frags;
1863
1864 dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1865 g->sg[0].ptr[0], (skb->len - skb->data_len),
1866 DMA_TO_DEVICE);
1867
1868 i = 1;
1869 while (frags--) {
1870 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1];
1871
1872 pci_unmap_page((lio->oct_dev)->pci_dev,
1873 g->sg[(i >> 2)].ptr[(i & 3)],
1874 frag->size, DMA_TO_DEVICE);
1875 i++;
1876 }
1877
fcd2b5e3
RV		 1878 iq = skb_iq(lio, skb);
1879
1880 spin_lock(&lio->glist_lock[iq]);
1881 list_add_tail(&g->list, &lio->glist[iq]);
1882 spin_unlock(&lio->glist_lock[iq]);
f21fb3ed
RV		 1883
1884 /* Don't free the skb yet */
1885
1886 check_txq_state(lio, skb);
1887}
1888
1889/**
1890 * \brief Adjust ptp frequency
1891 * @param ptp PTP clock info
1892 * @param ppb how much to adjust by, in parts-per-billion
1893 */
1894static int liquidio_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb)
1895{
1896 struct lio *lio = container_of(ptp, struct lio, ptp_info);
1897 struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1898 u64 comp, delta;
1899 unsigned long flags;
1900 bool neg_adj = false;
1901
1902 if (ppb < 0) {
1903 neg_adj = true;
1904 ppb = -ppb;
1905 }
1906
1907 /* The hardware adds the clock compensation value to the
1908 * PTP clock on every coprocessor clock cycle, so we
1909 * compute the delta in terms of coprocessor clocks.
1910 */
1911 delta = (u64)ppb << 32;
1912 do_div(delta, oct->coproc_clock_rate);
1913
1914 spin_lock_irqsave(&lio->ptp_lock, flags);
1915 comp = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_COMP);
1916 if (neg_adj)
1917 comp -= delta;
1918 else
1919 comp += delta;
1920 lio_pci_writeq(oct, comp, CN6XXX_MIO_PTP_CLOCK_COMP);
1921 spin_unlock_irqrestore(&lio->ptp_lock, flags);
1922
1923 return 0;
1924}
1925
1926/**
1927 * \brief Adjust ptp time
1928 * @param ptp PTP clock info
1929 * @param delta how much to adjust by, in nanosecs
1930 */
1931static int liquidio_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
1932{
1933 unsigned long flags;
1934 struct lio *lio = container_of(ptp, struct lio, ptp_info);
1935
1936 spin_lock_irqsave(&lio->ptp_lock, flags);
1937 lio->ptp_adjust += delta;
1938 spin_unlock_irqrestore(&lio->ptp_lock, flags);
1939
1940 return 0;
1941}
1942
1943/**
1944 * \brief Get hardware clock time, including any adjustment
1945 * @param ptp PTP clock info
1946 * @param ts timespec
1947 */
1948static int liquidio_ptp_gettime(struct ptp_clock_info *ptp,
1949 struct timespec64 *ts)
1950{
1951 u64 ns;
f21fb3ed
RV		 1952 unsigned long flags;
1953 struct lio *lio = container_of(ptp, struct lio, ptp_info);
1954 struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1955
1956 spin_lock_irqsave(&lio->ptp_lock, flags);
1957 ns = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_HI);
1958 ns += lio->ptp_adjust;
1959 spin_unlock_irqrestore(&lio->ptp_lock, flags);
1960
286af315 1961 *ts = ns_to_timespec64(ns);
f21fb3ed
RV		 1962
1963 return 0;
1964}
1965
1966/**
1967 * \brief Set hardware clock time. Reset adjustment
1968 * @param ptp PTP clock info
1969 * @param ts timespec
1970 */
1971static int liquidio_ptp_settime(struct ptp_clock_info *ptp,
1972 const struct timespec64 *ts)
1973{
1974 u64 ns;
1975 unsigned long flags;
1976 struct lio *lio = container_of(ptp, struct lio, ptp_info);
1977 struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1978
e7ad9793 1979 ns = timespec64_to_ns(ts);
f21fb3ed
RV		 1980
1981 spin_lock_irqsave(&lio->ptp_lock, flags);
1982 lio_pci_writeq(oct, ns, CN6XXX_MIO_PTP_CLOCK_HI);
1983 lio->ptp_adjust = 0;
1984 spin_unlock_irqrestore(&lio->ptp_lock, flags);
1985
1986 return 0;
1987}
1988
1989/**
1990 * \brief Check if PTP is enabled
1991 * @param ptp PTP clock info
1992 * @param rq request
1993 * @param on is it on
1994 */
a7d5a3dc
RV		 1995static int
1996liquidio_ptp_enable(struct ptp_clock_info *ptp __attribute__((unused)),
1997 struct ptp_clock_request *rq __attribute__((unused)),
1998 int on __attribute__((unused)))
f21fb3ed
RV		 1999{
2000 return -EOPNOTSUPP;
2001}
2002
2003/**
2004 * \brief Open PTP clock source
2005 * @param netdev network device
2006 */
2007static void oct_ptp_open(struct net_device *netdev)
2008{
2009 struct lio *lio = GET_LIO(netdev);
2010 struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
2011
2012 spin_lock_init(&lio->ptp_lock);
2013
2014 snprintf(lio->ptp_info.name, 16, "%s", netdev->name);
2015 lio->ptp_info.owner = THIS_MODULE;
2016 lio->ptp_info.max_adj = 250000000;
2017 lio->ptp_info.n_alarm = 0;
2018 lio->ptp_info.n_ext_ts = 0;
2019 lio->ptp_info.n_per_out = 0;
2020 lio->ptp_info.pps = 0;
2021 lio->ptp_info.adjfreq = liquidio_ptp_adjfreq;
2022 lio->ptp_info.adjtime = liquidio_ptp_adjtime;
2023 lio->ptp_info.gettime64 = liquidio_ptp_gettime;
2024 lio->ptp_info.settime64 = liquidio_ptp_settime;
2025 lio->ptp_info.enable = liquidio_ptp_enable;
2026
2027 lio->ptp_adjust = 0;
2028
2029 lio->ptp_clock = ptp_clock_register(&lio->ptp_info,
2030 &oct->pci_dev->dev);
2031
2032 if (IS_ERR(lio->ptp_clock))
2033 lio->ptp_clock = NULL;
2034}
2035
2036/**
2037 * \brief Init PTP clock
2038 * @param oct octeon device
2039 */
2040static void liquidio_ptp_init(struct octeon_device *oct)
2041{
2042 u64 clock_comp, cfg;
2043
2044 clock_comp = (u64)NSEC_PER_SEC << 32;
2045 do_div(clock_comp, oct->coproc_clock_rate);
2046 lio_pci_writeq(oct, clock_comp, CN6XXX_MIO_PTP_CLOCK_COMP);
2047
2048 /* Enable */
2049 cfg = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_CFG);
2050 lio_pci_writeq(oct, cfg | 0x01, CN6XXX_MIO_PTP_CLOCK_CFG);
2051}
2052
2053/**
2054 * \brief Load firmware to device
2055 * @param oct octeon device
2056 *
2057 * Maps device to firmware filename, requests firmware, and downloads it
2058 */
2059static int load_firmware(struct octeon_device *oct)
2060{
2061 int ret = 0;
2062 const struct firmware *fw;
2063 char fw_name[LIO_MAX_FW_FILENAME_LEN];
2064 char *tmp_fw_type;
2065
429cbf6b 2066 if (fw_type_is_auto()) {
f21fb3ed 2067 tmp_fw_type = LIO_FW_NAME_TYPE_NIC;
429cbf6b
RF		 2068 strncpy(fw_type, tmp_fw_type, sizeof(fw_type));
2069 } else {
f21fb3ed 2070 tmp_fw_type = fw_type;
429cbf6b 2071 }
f21fb3ed
RV		 2072
2073 sprintf(fw_name, "%s%s%s_%s%s", LIO_FW_DIR, LIO_FW_BASE_NAME,
2074 octeon_get_conf(oct)->card_name, tmp_fw_type,
2075 LIO_FW_NAME_SUFFIX);
2076
2077 ret = request_firmware(&fw, fw_name, &oct->pci_dev->dev);
2078 if (ret) {
2079 dev_err(&oct->pci_dev->dev, "Request firmware failed. Could not find file %s.\n.",
2080 fw_name);
d3d7e6c6 2081 release_firmware(fw);
f21fb3ed
RV		 2082 return ret;
2083 }
2084
2085 ret = octeon_download_firmware(oct, fw->data, fw->size);
2086
2087 release_firmware(fw);
2088
2089 return ret;
2090}
2091
f21fb3ed
RV		 2092/**
2093 * \brief Callback for getting interface configuration
2094 * @param status status of request
2095 * @param buf pointer to resp structure
2096 */
2097static void if_cfg_callback(struct octeon_device *oct,
a7d5a3dc 2098 u32 status __attribute__((unused)),
f21fb3ed
RV		 2099 void *buf)
2100{
2101 struct octeon_soft_command *sc = (struct octeon_soft_command *)buf;
2102 struct liquidio_if_cfg_resp *resp;
2103 struct liquidio_if_cfg_context *ctx;
2104
2105 resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
30136395 2106 ctx = (struct liquidio_if_cfg_context *)sc->ctxptr;
f21fb3ed
RV		 2107
2108 oct = lio_get_device(ctx->octeon_id);
2109 if (resp->status)
c5b71e63
RF		 2110 dev_err(&oct->pci_dev->dev, "nic if cfg instruction failed. Status: 0x%llx (0x%08x)\n",
2111 CVM_CAST64(resp->status), status);
a7d5a3dc 2112 WRITE_ONCE(ctx->cond, 1);
f21fb3ed 2113
d3d7e6c6
RV		 2114 snprintf(oct->fw_info.liquidio_firmware_version, 32, "%s",
2115 resp->cfg_info.liquidio_firmware_version);
2116
f21fb3ed
RV		 2117 /* This barrier is required to be sure that the response has been
2118 * written fully before waking up the handler
2119 */
2120 wmb();
2121
2122 wake_up_interruptible(&ctx->wc);
2123}
2124
f21fb3ed
RV		 2125/**
2126 * \brief Poll routine for checking transmit queue status
2127 * @param work work_struct data structure
2128 */
2129static void octnet_poll_check_txq_status(struct work_struct *work)
2130{
2131 struct cavium_wk *wk = (struct cavium_wk *)work;
2132 struct lio *lio = (struct lio *)wk->ctxptr;
2133
2134 if (!ifstate_check(lio, LIO_IFSTATE_RUNNING))
2135 return;
2136
2137 check_txq_status(lio);
2138 queue_delayed_work(lio->txq_status_wq.wq,
2139 &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
2140}
2141
2142/**
2143 * \brief Sets up the txq poll check
2144 * @param netdev network device
2145 */
5b07aee1 2146static inline int setup_tx_poll_fn(struct net_device *netdev)
f21fb3ed
RV		 2147{
2148 struct lio *lio = GET_LIO(netdev);
2149 struct octeon_device *oct = lio->oct_dev;
2150
292b9dab
BS		 2151 lio->txq_status_wq.wq = alloc_workqueue("txq-status",
2152 WQ_MEM_RECLAIM, 0);
f21fb3ed
RV		 2153 if (!lio->txq_status_wq.wq) {
2154 dev_err(&oct->pci_dev->dev, "unable to create cavium txq status wq\n");
5b07aee1 2155 return -1;
f21fb3ed
RV		 2156 }
2157 INIT_DELAYED_WORK(&lio->txq_status_wq.wk.work,
2158 octnet_poll_check_txq_status);
2159 lio->txq_status_wq.wk.ctxptr = lio;
2160 queue_delayed_work(lio->txq_status_wq.wq,
2161 &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
5b07aee1 2162 return 0;
f21fb3ed
RV		 2163}
2164
9a96bde4
RV		 2165static inline void cleanup_tx_poll_fn(struct net_device *netdev)
2166{
2167 struct lio *lio = GET_LIO(netdev);
2168
5b07aee1
RV		 2169 if (lio->txq_status_wq.wq) {
2170 cancel_delayed_work_sync(&lio->txq_status_wq.wk.work);
2171 destroy_workqueue(lio->txq_status_wq.wq);
2172 }
9a96bde4
RV		 2173}
2174
f21fb3ed
RV		 2175/**
2176 * \brief Net device open for LiquidIO
2177 * @param netdev network device
2178 */
2179static int liquidio_open(struct net_device *netdev)
2180{
2181 struct lio *lio = GET_LIO(netdev);
2182 struct octeon_device *oct = lio->oct_dev;
2183 struct napi_struct *napi, *n;
2184
9a96bde4
RV		 2185 if (oct->props[lio->ifidx].napi_enabled == 0) {
2186 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
2187 napi_enable(napi);
2188
2189 oct->props[lio->ifidx].napi_enabled = 1;
7b6b6c95
RV		 2190
2191 if (OCTEON_CN23XX_PF(oct))
2192 oct->droq[0]->ops.poll_mode = 1;
9a96bde4 2193 }
f21fb3ed 2194
aa69ff9e 2195 if (oct->ptp_enable)
9feb16ae 2196 oct_ptp_open(netdev);
f21fb3ed
RV		 2197
2198 ifstate_set(lio, LIO_IFSTATE_RUNNING);
9a96bde4 2199
7b6b6c95
RV		 2200 /* Ready for link status updates */
2201 lio->intf_open = 1;
2202
2203 netif_info(lio, ifup, lio->netdev, "Interface Open, ready for traffic\n");
2204
5b07aee1
RV		 2205 if (OCTEON_CN23XX_PF(oct)) {
2206 if (!oct->msix_on)
2207 if (setup_tx_poll_fn(netdev))
2208 return -1;
2209 } else {
2210 if (setup_tx_poll_fn(netdev))
2211 return -1;
2212 }
9a96bde4 2213
f21fb3ed
RV		 2214 start_txq(netdev);
2215
f21fb3ed
RV		 2216 /* tell Octeon to start forwarding packets to host */
2217 send_rx_ctrl_cmd(lio, 1);
2218
f21fb3ed
RV		 2219 dev_info(&oct->pci_dev->dev, "%s interface is opened\n",
2220 netdev->name);
2221
2222 return 0;
2223}
2224
2225/**
2226 * \brief Net device stop for LiquidIO
2227 * @param netdev network device
2228 */
2229static int liquidio_stop(struct net_device *netdev)
2230{
f21fb3ed
RV		 2231 struct lio *lio = GET_LIO(netdev);
2232 struct octeon_device *oct = lio->oct_dev;
42013e90
IB		 2233 struct napi_struct *napi, *n;
2234
2235 if (oct->props[lio->ifidx].napi_enabled) {
2236 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
2237 napi_disable(napi);
2238
2239 oct->props[lio->ifidx].napi_enabled = 0;
2240
2241 if (OCTEON_CN23XX_PF(oct))
2242 oct->droq[0]->ops.poll_mode = 0;
2243 }
f21fb3ed 2244
9a96bde4
RV		 2245 ifstate_reset(lio, LIO_IFSTATE_RUNNING);
2246
2247 netif_tx_disable(netdev);
2248
f21fb3ed 2249 /* Inform that netif carrier is down */
9a96bde4 2250 netif_carrier_off(netdev);
f21fb3ed 2251 lio->intf_open = 0;
0cece6c5
RV		 2252 lio->linfo.link.s.link_up = 0;
2253 lio->link_changes++;
f21fb3ed 2254
cb2336b5 2255 /* Tell Octeon that nic interface is down. */
f21fb3ed
RV		 2256 send_rx_ctrl_cmd(lio, 0);
2257
5b07aee1
RV		 2258 if (OCTEON_CN23XX_PF(oct)) {
2259 if (!oct->msix_on)
2260 cleanup_tx_poll_fn(netdev);
2261 } else {
2262 cleanup_tx_poll_fn(netdev);
2263 }
f21fb3ed
RV		 2264
2265 if (lio->ptp_clock) {
2266 ptp_clock_unregister(lio->ptp_clock);
2267 lio->ptp_clock = NULL;
2268 }
2269
f21fb3ed 2270 dev_info(&oct->pci_dev->dev, "%s interface is stopped\n", netdev->name);
f21fb3ed
RV		 2271
2272 return 0;
2273}
2274
f21fb3ed
RV		 2275/**
2276 * \brief Converts a mask based on net device flags
2277 * @param netdev network device
2278 *
2279 * This routine generates a octnet_ifflags mask from the net device flags
2280 * received from the OS.
2281 */
2282static inline enum octnet_ifflags get_new_flags(struct net_device *netdev)
2283{
2284 enum octnet_ifflags f = OCTNET_IFFLAG_UNICAST;
2285
2286 if (netdev->flags & IFF_PROMISC)
2287 f |= OCTNET_IFFLAG_PROMISC;
2288
2289 if (netdev->flags & IFF_ALLMULTI)
2290 f |= OCTNET_IFFLAG_ALLMULTI;
2291
2292 if (netdev->flags & IFF_MULTICAST) {
2293 f |= OCTNET_IFFLAG_MULTICAST;
2294
2295 /* Accept all multicast addresses if there are more than we
2296 * can handle
2297 */
2298 if (netdev_mc_count(netdev) > MAX_OCTEON_MULTICAST_ADDR)
2299 f |= OCTNET_IFFLAG_ALLMULTI;
2300 }
2301
2302 if (netdev->flags & IFF_BROADCAST)
2303 f |= OCTNET_IFFLAG_BROADCAST;
2304
2305 return f;
2306}
2307
2308/**
2309 * \brief Net device set_multicast_list
2310 * @param netdev network device
2311 */
2312static void liquidio_set_mcast_list(struct net_device *netdev)
2313{
2314 struct lio *lio = GET_LIO(netdev);
2315 struct octeon_device *oct = lio->oct_dev;
2316 struct octnic_ctrl_pkt nctrl;
f21fb3ed
RV		 2317 struct netdev_hw_addr *ha;
2318 u64 *mc;
a7d5a3dc 2319 int ret;
f21fb3ed
RV		 2320 int mc_count = min(netdev_mc_count(netdev), MAX_OCTEON_MULTICAST_ADDR);
2321
2322 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2323
2324 /* Create a ctrl pkt command to be sent to core app. */
2325 nctrl.ncmd.u64 = 0;
2326 nctrl.ncmd.s.cmd = OCTNET_CMD_SET_MULTI_LIST;
0cece6c5
RV		 2327 nctrl.ncmd.s.param1 = get_new_flags(netdev);
2328 nctrl.ncmd.s.param2 = mc_count;
f21fb3ed 2329 nctrl.ncmd.s.more = mc_count;
0cece6c5 2330 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
f21fb3ed
RV		 2331 nctrl.netpndev = (u64)netdev;
2332 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2333
2334 /* copy all the addresses into the udd */
f21fb3ed
RV		 2335 mc = &nctrl.udd[0];
2336 netdev_for_each_mc_addr(ha, netdev) {
2337 *mc = 0;
2338 memcpy(((u8 *)mc) + 2, ha->addr, ETH_ALEN);
2339 /* no need to swap bytes */
2340
2341 if (++mc > &nctrl.udd[mc_count])
2342 break;
2343 }
2344
2345 /* Apparently, any activity in this call from the kernel has to
2346 * be atomic. So we won't wait for response.
2347 */
2348 nctrl.wait_time = 0;
2349
0cece6c5 2350 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
f21fb3ed
RV		 2351 if (ret < 0) {
2352 dev_err(&oct->pci_dev->dev, "DEVFLAGS change failed in core (ret: 0x%x)\n",
2353 ret);
2354 }
2355}
2356
2357/**
2358 * \brief Net device set_mac_address
2359 * @param netdev network device
2360 */
2361static int liquidio_set_mac(struct net_device *netdev, void *p)
2362{
2363 int ret = 0;
2364 struct lio *lio = GET_LIO(netdev);
2365 struct octeon_device *oct = lio->oct_dev;
2366 struct sockaddr *addr = (struct sockaddr *)p;
2367 struct octnic_ctrl_pkt nctrl;
f21fb3ed 2368
0cece6c5 2369 if (!is_valid_ether_addr(addr->sa_data))
f21fb3ed
RV		 2370 return -EADDRNOTAVAIL;
2371
2372 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2373
2374 nctrl.ncmd.u64 = 0;
2375 nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR;
0cece6c5 2376 nctrl.ncmd.s.param1 = 0;
f21fb3ed 2377 nctrl.ncmd.s.more = 1;
0cece6c5 2378 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
f21fb3ed
RV		 2379 nctrl.netpndev = (u64)netdev;
2380 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2381 nctrl.wait_time = 100;
2382
2383 nctrl.udd[0] = 0;
2384 /* The MAC Address is presented in network byte order. */
2385 memcpy((u8 *)&nctrl.udd[0] + 2, addr->sa_data, ETH_ALEN);
2386
0cece6c5 2387 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
f21fb3ed
RV		 2388 if (ret < 0) {
2389 dev_err(&oct->pci_dev->dev, "MAC Address change failed\n");
2390 return -ENOMEM;
2391 }
2392 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2393 memcpy(((u8 *)&lio->linfo.hw_addr) + 2, addr->sa_data, ETH_ALEN);
2394
2395 return 0;
2396}
2397
2398/**
2399 * \brief Net device get_stats
2400 * @param netdev network device
2401 */
2402static struct net_device_stats *liquidio_get_stats(struct net_device *netdev)
2403{
2404 struct lio *lio = GET_LIO(netdev);
2405 struct net_device_stats *stats = &netdev->stats;
2406 struct octeon_device *oct;
2407 u64 pkts = 0, drop = 0, bytes = 0;
2408 struct oct_droq_stats *oq_stats;
2409 struct oct_iq_stats *iq_stats;
2410 int i, iq_no, oq_no;
2411
2412 oct = lio->oct_dev;
2413
d18ca7df
IB		 2414 if (ifstate_check(lio, LIO_IFSTATE_RESETTING))
2415 return stats;
2416
a82457f1 2417 for (i = 0; i < oct->num_iqs; i++) {
26236fa9 2418 iq_no = lio->linfo.txpciq[i].s.q_no;
f21fb3ed
RV		 2419 iq_stats = &oct->instr_queue[iq_no]->stats;
2420 pkts += iq_stats->tx_done;
2421 drop += iq_stats->tx_dropped;
2422 bytes += iq_stats->tx_tot_bytes;
2423 }
2424
2425 stats->tx_packets = pkts;
2426 stats->tx_bytes = bytes;
2427 stats->tx_dropped = drop;
2428
2429 pkts = 0;
2430 drop = 0;
2431 bytes = 0;
2432
a82457f1 2433 for (i = 0; i < oct->num_oqs; i++) {
26236fa9 2434 oq_no = lio->linfo.rxpciq[i].s.q_no;
f21fb3ed
RV		 2435 oq_stats = &oct->droq[oq_no]->stats;
2436 pkts += oq_stats->rx_pkts_received;
2437 drop += (oq_stats->rx_dropped +
2438 oq_stats->dropped_nodispatch +
2439 oq_stats->dropped_toomany +
2440 oq_stats->dropped_nomem);
2441 bytes += oq_stats->rx_bytes_received;
2442 }
2443
2444 stats->rx_bytes = bytes;
2445 stats->rx_packets = pkts;
2446 stats->rx_dropped = drop;
2447
2448 return stats;
2449}
2450
2451/**
2452 * \brief Net device change_mtu
2453 * @param netdev network device
2454 */
2455static int liquidio_change_mtu(struct net_device *netdev, int new_mtu)
2456{
2457 struct lio *lio = GET_LIO(netdev);
2458 struct octeon_device *oct = lio->oct_dev;
2459 struct octnic_ctrl_pkt nctrl;
f21fb3ed
RV		 2460 int ret = 0;
2461
f21fb3ed
RV		 2462 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2463
2464 nctrl.ncmd.u64 = 0;
2465 nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MTU;
0cece6c5
RV		 2466 nctrl.ncmd.s.param1 = new_mtu;
2467 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
f21fb3ed
RV		 2468 nctrl.wait_time = 100;
2469 nctrl.netpndev = (u64)netdev;
2470 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2471
0cece6c5 2472 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
f21fb3ed
RV		 2473 if (ret < 0) {
2474 dev_err(&oct->pci_dev->dev, "Failed to set MTU\n");
2475 return -1;
2476 }
2477
2478 lio->mtu = new_mtu;
2479
2480 return 0;
2481}
2482
2483/**
2484 * \brief Handler for SIOCSHWTSTAMP ioctl
2485 * @param netdev network device
2486 * @param ifr interface request
2487 * @param cmd command
2488 */
a7d5a3dc 2489static int hwtstamp_ioctl(struct net_device *netdev, struct ifreq *ifr)
f21fb3ed
RV		 2490{
2491 struct hwtstamp_config conf;
2492 struct lio *lio = GET_LIO(netdev);
2493
2494 if (copy_from_user(&conf, ifr->ifr_data, sizeof(conf)))
2495 return -EFAULT;
2496
2497 if (conf.flags)
2498 return -EINVAL;
2499
2500 switch (conf.tx_type) {
2501 case HWTSTAMP_TX_ON:
2502 case HWTSTAMP_TX_OFF:
2503 break;
2504 default:
2505 return -ERANGE;
2506 }
2507
2508 switch (conf.rx_filter) {
2509 case HWTSTAMP_FILTER_NONE:
2510 break;
2511 case HWTSTAMP_FILTER_ALL:
2512 case HWTSTAMP_FILTER_SOME:
2513 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
2514 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
2515 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
2516 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
2517 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
2518 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
2519 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
2520 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
2521 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
2522 case HWTSTAMP_FILTER_PTP_V2_EVENT:
2523 case HWTSTAMP_FILTER_PTP_V2_SYNC:
2524 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
e3412575 2525 case HWTSTAMP_FILTER_NTP_ALL:
f21fb3ed
RV		 2526 conf.rx_filter = HWTSTAMP_FILTER_ALL;
2527 break;
2528 default:
2529 return -ERANGE;
2530 }
2531
2532 if (conf.rx_filter == HWTSTAMP_FILTER_ALL)
2533 ifstate_set(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
2534
2535 else
2536 ifstate_reset(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
2537
2538 return copy_to_user(ifr->ifr_data, &conf, sizeof(conf)) ? -EFAULT : 0;
2539}
2540
2541/**
2542 * \brief ioctl handler
2543 * @param netdev network device
2544 * @param ifr interface request
2545 * @param cmd command
2546 */
2547static int liquidio_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2548{
9feb16ae
PK		 2549 struct lio *lio = GET_LIO(netdev);
2550
f21fb3ed
RV		 2551 switch (cmd) {
2552 case SIOCSHWTSTAMP:
aa69ff9e 2553 if (lio->oct_dev->ptp_enable)
9feb16ae 2554 return hwtstamp_ioctl(netdev, ifr);
f21fb3ed
RV		 2555 default:
2556 return -EOPNOTSUPP;
2557 }
2558}
2559
2560/**
2561 * \brief handle a Tx timestamp response
2562 * @param status response status
2563 * @param buf pointer to skb
2564 */
2565static void handle_timestamp(struct octeon_device *oct,
2566 u32 status,
2567 void *buf)
2568{
2569 struct octnet_buf_free_info *finfo;
2570 struct octeon_soft_command *sc;
2571 struct oct_timestamp_resp *resp;
2572 struct lio *lio;
2573 struct sk_buff *skb = (struct sk_buff *)buf;
2574
2575 finfo = (struct octnet_buf_free_info *)skb->cb;
2576 lio = finfo->lio;
2577 sc = finfo->sc;
2578 oct = lio->oct_dev;
2579 resp = (struct oct_timestamp_resp *)sc->virtrptr;
2580
2581 if (status != OCTEON_REQUEST_DONE) {
2582 dev_err(&oct->pci_dev->dev, "Tx timestamp instruction failed. Status: %llx\n",
2583 CVM_CAST64(status));
2584 resp->timestamp = 0;
2585 }
2586
2587 octeon_swap_8B_data(&resp->timestamp, 1);
2588
19a6d156 2589 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) != 0)) {
f21fb3ed
RV		 2590 struct skb_shared_hwtstamps ts;
2591 u64 ns = resp->timestamp;
2592
2593 netif_info(lio, tx_done, lio->netdev,
2594 "Got resulting SKBTX_HW_TSTAMP skb=%p ns=%016llu\n",
2595 skb, (unsigned long long)ns);
2596 ts.hwtstamp = ns_to_ktime(ns + lio->ptp_adjust);
2597 skb_tstamp_tx(skb, &ts);
2598 }
2599
2600 octeon_free_soft_command(oct, sc);
cabeb13b 2601 tx_buffer_free(skb);
f21fb3ed
RV		 2602}
2603
2604/* \brief Send a data packet that will be timestamped
2605 * @param oct octeon device
2606 * @param ndata pointer to network data
2607 * @param finfo pointer to private network data
2608 */
2609static inline int send_nic_timestamp_pkt(struct octeon_device *oct,
2610 struct octnic_data_pkt *ndata,
c859e21a
IB		 2611 struct octnet_buf_free_info *finfo,
2612 int xmit_more)
f21fb3ed
RV		 2613{
2614 int retval;
2615 struct octeon_soft_command *sc;
f21fb3ed
RV		 2616 struct lio *lio;
2617 int ring_doorbell;
6a885b60 2618 u32 len;
f21fb3ed
RV		 2619
2620 lio = finfo->lio;
2621
2622 sc = octeon_alloc_soft_command_resp(oct, &ndata->cmd,
2623 sizeof(struct oct_timestamp_resp));
2624 finfo->sc = sc;
2625
2626 if (!sc) {
2627 dev_err(&oct->pci_dev->dev, "No memory for timestamped data packet\n");
2628 return IQ_SEND_FAILED;
2629 }
2630
2631 if (ndata->reqtype == REQTYPE_NORESP_NET)
2632 ndata->reqtype = REQTYPE_RESP_NET;
2633 else if (ndata->reqtype == REQTYPE_NORESP_NET_SG)
2634 ndata->reqtype = REQTYPE_RESP_NET_SG;
2635
2636 sc->callback = handle_timestamp;
2637 sc->callback_arg = finfo->skb;
2638 sc->iq_no = ndata->q_no;
2639
5b823514
RV		 2640 if (OCTEON_CN23XX_PF(oct))
2641 len = (u32)((struct octeon_instr_ih3 *)
2642 (&sc->cmd.cmd3.ih3))->dlengsz;
2643 else
2644 len = (u32)((struct octeon_instr_ih2 *)
2645 (&sc->cmd.cmd2.ih2))->dlengsz;
f21fb3ed 2646
c859e21a 2647 ring_doorbell = !xmit_more;
5b823514 2648
f21fb3ed 2649 retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd,
6a885b60 2650 sc, len, ndata->reqtype);
f21fb3ed 2651
ddc173a6 2652 if (retval == IQ_SEND_FAILED) {
f21fb3ed
RV		 2653 dev_err(&oct->pci_dev->dev, "timestamp data packet failed status: %x\n",
2654 retval);
2655 octeon_free_soft_command(oct, sc);
2656 } else {
2657 netif_info(lio, tx_queued, lio->netdev, "Queued timestamp packet\n");
2658 }
2659
2660 return retval;
2661}
2662
f21fb3ed
RV		 2663/** \brief Transmit networks packets to the Octeon interface
2664 * @param skbuff skbuff struct to be passed to network layer.
2665 * @param netdev pointer to network device
2666 * @returns whether the packet was transmitted to the device okay or not
2667 * (NETDEV_TX_OK or NETDEV_TX_BUSY)
2668 */
2669static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
2670{
2671 struct lio *lio;
2672 struct octnet_buf_free_info *finfo;
2673 union octnic_cmd_setup cmdsetup;
2674 struct octnic_data_pkt ndata;
2675 struct octeon_device *oct;
2676 struct oct_iq_stats *stats;
6a885b60
RV		 2677 struct octeon_instr_irh *irh;
2678 union tx_info *tx_info;
26236fa9 2679 int status = 0;
f21fb3ed 2680 int q_idx = 0, iq_no = 0;
c859e21a 2681 int j, xmit_more = 0;
fcd2b5e3 2682 u64 dptr = 0;
f21fb3ed
RV		 2683 u32 tag = 0;
2684
2685 lio = GET_LIO(netdev);
2686 oct = lio->oct_dev;
2687
2688 if (netif_is_multiqueue(netdev)) {
26236fa9
RV		 2689 q_idx = skb->queue_mapping;
2690 q_idx = (q_idx % (lio->linfo.num_txpciq));
2691 tag = q_idx;
2692 iq_no = lio->linfo.txpciq[q_idx].s.q_no;
f21fb3ed
RV		 2693 } else {
2694 iq_no = lio->txq;
2695 }
2696
2697 stats = &oct->instr_queue[iq_no]->stats;
2698
2699 /* Check for all conditions in which the current packet cannot be
2700 * transmitted.
2701 */
2702 if (!(atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) ||
0cece6c5 2703 (!lio->linfo.link.s.link_up) ||
f21fb3ed
RV		 2704 (skb->len <= 0)) {
2705 netif_info(lio, tx_err, lio->netdev,
2706 "Transmit failed link_status : %d\n",
0cece6c5 2707 lio->linfo.link.s.link_up);
f21fb3ed
RV		 2708 goto lio_xmit_failed;
2709 }
2710
2711 /* Use space in skb->cb to store info used to unmap and
2712 * free the buffers.
2713 */
2714 finfo = (struct octnet_buf_free_info *)skb->cb;
2715 finfo->lio = lio;
2716 finfo->skb = skb;
2717 finfo->sc = NULL;
2718
2719 /* Prepare the attributes for the data to be passed to OSI. */
2720 memset(&ndata, 0, sizeof(struct octnic_data_pkt));
2721
2722 ndata.buf = (void *)finfo;
2723
2724 ndata.q_no = iq_no;
2725
2726 if (netif_is_multiqueue(netdev)) {
2727 if (octnet_iq_is_full(oct, ndata.q_no)) {
2728 /* defer sending if queue is full */
2729 netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n",
2730 ndata.q_no);
2731 stats->tx_iq_busy++;
2732 return NETDEV_TX_BUSY;
2733 }
2734 } else {
2735 if (octnet_iq_is_full(oct, lio->txq)) {
2736 /* defer sending if queue is full */
2737 stats->tx_iq_busy++;
2738 netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n",
a2c64b67 2739 lio->txq);
f21fb3ed
RV		 2740 return NETDEV_TX_BUSY;
2741 }
2742 }
2743 /* pr_info(" XMIT - valid Qs: %d, 1st Q no: %d, cpu: %d, q_no:%d\n",
a2c64b67 2744 * lio->linfo.num_txpciq, lio->txq, cpu, ndata.q_no);
f21fb3ed
RV		 2745 */
2746
2747 ndata.datasize = skb->len;
2748
2749 cmdsetup.u64 = 0;
7275ebfc 2750 cmdsetup.s.iq_no = iq_no;
f21fb3ed 2751
01fb237a
RV		 2752 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2753 if (skb->encapsulation) {
2754 cmdsetup.s.tnl_csum = 1;
2755 stats->tx_vxlan++;
2756 } else {
2757 cmdsetup.s.transport_csum = 1;
2758 }
2759 }
f21fb3ed
RV		 2760 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
2761 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2762 cmdsetup.s.timestamp = 1;
2763 }
2764
2765 if (skb_shinfo(skb)->nr_frags == 0) {
2766 cmdsetup.s.u.datasize = skb->len;
0cece6c5 2767 octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
a2c64b67 2768
f21fb3ed 2769 /* Offload checksum calculation for TCP/UDP packets */
6a885b60
RV		 2770 dptr = dma_map_single(&oct->pci_dev->dev,
2771 skb->data,
2772 skb->len,
2773 DMA_TO_DEVICE);
2774 if (dma_mapping_error(&oct->pci_dev->dev, dptr)) {
f21fb3ed
RV		 2775 dev_err(&oct->pci_dev->dev, "%s DMA mapping error 1\n",
2776 __func__);
2777 return NETDEV_TX_BUSY;
2778 }
2779
5b823514
RV		 2780 if (OCTEON_CN23XX_PF(oct))
2781 ndata.cmd.cmd3.dptr = dptr;
2782 else
2783 ndata.cmd.cmd2.dptr = dptr;
6a885b60 2784 finfo->dptr = dptr;
f21fb3ed
RV		 2785 ndata.reqtype = REQTYPE_NORESP_NET;
2786
2787 } else {
2788 int i, frags;
2789 struct skb_frag_struct *frag;
2790 struct octnic_gather *g;
2791
fcd2b5e3
RV		 2792 spin_lock(&lio->glist_lock[q_idx]);
2793 g = (struct octnic_gather *)
2794 list_delete_head(&lio->glist[q_idx]);
2795 spin_unlock(&lio->glist_lock[q_idx]);
f21fb3ed
RV		 2796
2797 if (!g) {
2798 netif_info(lio, tx_err, lio->netdev,
2799 "Transmit scatter gather: glist null!\n");
2800 goto lio_xmit_failed;
2801 }
2802
2803 cmdsetup.s.gather = 1;
2804 cmdsetup.s.u.gatherptrs = (skb_shinfo(skb)->nr_frags + 1);
0cece6c5 2805 octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
f21fb3ed
RV		 2806
2807 memset(g->sg, 0, g->sg_size);
2808
2809 g->sg[0].ptr[0] = dma_map_single(&oct->pci_dev->dev,
2810 skb->data,
2811 (skb->len - skb->data_len),
2812 DMA_TO_DEVICE);
2813 if (dma_mapping_error(&oct->pci_dev->dev, g->sg[0].ptr[0])) {
2814 dev_err(&oct->pci_dev->dev, "%s DMA mapping error 2\n",
2815 __func__);
2816 return NETDEV_TX_BUSY;
2817 }
2818 add_sg_size(&g->sg[0], (skb->len - skb->data_len), 0);
2819
2820 frags = skb_shinfo(skb)->nr_frags;
2821 i = 1;
2822 while (frags--) {
2823 frag = &skb_shinfo(skb)->frags[i - 1];
2824
2825 g->sg[(i >> 2)].ptr[(i & 3)] =
2826 dma_map_page(&oct->pci_dev->dev,
2827 frag->page.p,
2828 frag->page_offset,
2829 frag->size,
2830 DMA_TO_DEVICE);
2831
fcd2b5e3
RV		 2832 if (dma_mapping_error(&oct->pci_dev->dev,
2833 g->sg[i >> 2].ptr[i & 3])) {
2834 dma_unmap_single(&oct->pci_dev->dev,
2835 g->sg[0].ptr[0],
2836 skb->len - skb->data_len,
2837 DMA_TO_DEVICE);
2838 for (j = 1; j < i; j++) {
2839 frag = &skb_shinfo(skb)->frags[j - 1];
2840 dma_unmap_page(&oct->pci_dev->dev,
2841 g->sg[j >> 2].ptr[j & 3],
2842 frag->size,
2843 DMA_TO_DEVICE);
2844 }
2845 dev_err(&oct->pci_dev->dev, "%s DMA mapping error 3\n",
2846 __func__);
2847 return NETDEV_TX_BUSY;
2848 }
2849
f21fb3ed
RV		 2850 add_sg_size(&g->sg[(i >> 2)], frag->size, (i & 3));
2851 i++;
2852 }
2853
fcd2b5e3 2854 dptr = g->sg_dma_ptr;
f21fb3ed 2855
5b823514
RV		 2856 if (OCTEON_CN23XX_PF(oct))
2857 ndata.cmd.cmd3.dptr = dptr;
2858 else
2859 ndata.cmd.cmd2.dptr = dptr;
6a885b60 2860 finfo->dptr = dptr;
f21fb3ed
RV		 2861 finfo->g = g;
2862
2863 ndata.reqtype = REQTYPE_NORESP_NET_SG;
2864 }
2865
5b823514
RV		 2866 if (OCTEON_CN23XX_PF(oct)) {
2867 irh = (struct octeon_instr_irh *)&ndata.cmd.cmd3.irh;
2868 tx_info = (union tx_info *)&ndata.cmd.cmd3.ossp[0];
2869 } else {
2870 irh = (struct octeon_instr_irh *)&ndata.cmd.cmd2.irh;
2871 tx_info = (union tx_info *)&ndata.cmd.cmd2.ossp[0];
2872 }
f21fb3ed 2873
6a885b60 2874 if (skb_shinfo(skb)->gso_size) {
f21fb3ed
RV		 2875 tx_info->s.gso_size = skb_shinfo(skb)->gso_size;
2876 tx_info->s.gso_segs = skb_shinfo(skb)->gso_segs;
1f164717 2877 stats->tx_gso++;
f21fb3ed 2878 }
1f164717 2879
0da0b77c
RV		 2880 /* HW insert VLAN tag */
2881 if (skb_vlan_tag_present(skb)) {
2882 irh->priority = skb_vlan_tag_get(skb) >> 13;
2883 irh->vlan = skb_vlan_tag_get(skb) & 0xfff;
2884 }
f21fb3ed 2885
c859e21a
IB		 2886 xmit_more = skb->xmit_more;
2887
f21fb3ed 2888 if (unlikely(cmdsetup.s.timestamp))
c859e21a 2889 status = send_nic_timestamp_pkt(oct, &ndata, finfo, xmit_more);
f21fb3ed 2890 else
c859e21a 2891 status = octnet_send_nic_data_pkt(oct, &ndata, xmit_more);
f21fb3ed
RV		 2892 if (status == IQ_SEND_FAILED)
2893 goto lio_xmit_failed;
2894
2895 netif_info(lio, tx_queued, lio->netdev, "Transmit queued successfully\n");
2896
2897 if (status == IQ_SEND_STOP)
c859e21a 2898 stop_q(netdev, q_idx);
f21fb3ed 2899
860e9538 2900 netif_trans_update(netdev);
f21fb3ed 2901
80c8eae6
SB		 2902 if (tx_info->s.gso_segs)
2903 stats->tx_done += tx_info->s.gso_segs;
1f164717
RV		 2904 else
2905 stats->tx_done++;
80c8eae6 2906 stats->tx_tot_bytes += ndata.datasize;
f21fb3ed
RV		 2907
2908 return NETDEV_TX_OK;
2909
2910lio_xmit_failed:
2911 stats->tx_dropped++;
2912 netif_info(lio, tx_err, lio->netdev, "IQ%d Transmit dropped:%llu\n",
2913 iq_no, stats->tx_dropped);
6a885b60
RV		 2914 if (dptr)
2915 dma_unmap_single(&oct->pci_dev->dev, dptr,
2916 ndata.datasize, DMA_TO_DEVICE);
c859e21a
IB		 2917
2918 octeon_ring_doorbell_locked(oct, iq_no);
2919
cabeb13b 2920 tx_buffer_free(skb);
f21fb3ed
RV		 2921 return NETDEV_TX_OK;
2922}
2923
2924/** \brief Network device Tx timeout
2925 * @param netdev pointer to network device
2926 */
2927static void liquidio_tx_timeout(struct net_device *netdev)
2928{
2929 struct lio *lio;
2930
2931 lio = GET_LIO(netdev);
2932
2933 netif_info(lio, tx_err, lio->netdev,
2934 "Transmit timeout tx_dropped:%ld, waking up queues now!!\n",
2935 netdev->stats.tx_dropped);
860e9538 2936 netif_trans_update(netdev);
f21fb3ed
RV		 2937 txqs_wake(netdev);
2938}
2939
63245f25
RV		 2940static int liquidio_vlan_rx_add_vid(struct net_device *netdev,
2941 __be16 proto __attribute__((unused)),
2942 u16 vid)
2943{
2944 struct lio *lio = GET_LIO(netdev);
2945 struct octeon_device *oct = lio->oct_dev;
2946 struct octnic_ctrl_pkt nctrl;
2947 int ret = 0;
2948
2949 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2950
2951 nctrl.ncmd.u64 = 0;
2952 nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER;
2953 nctrl.ncmd.s.param1 = vid;
2954 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2955 nctrl.wait_time = 100;
2956 nctrl.netpndev = (u64)netdev;
2957 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2958
2959 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2960 if (ret < 0) {
2961 dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n",
2962 ret);
2963 }
2964
2965 return ret;
2966}
2967
2968static int liquidio_vlan_rx_kill_vid(struct net_device *netdev,
2969 __be16 proto __attribute__((unused)),
2970 u16 vid)
2971{
2972 struct lio *lio = GET_LIO(netdev);
2973 struct octeon_device *oct = lio->oct_dev;
2974 struct octnic_ctrl_pkt nctrl;
2975 int ret = 0;
2976
2977 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2978
2979 nctrl.ncmd.u64 = 0;
2980 nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER;
2981 nctrl.ncmd.s.param1 = vid;
2982 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2983 nctrl.wait_time = 100;
2984 nctrl.netpndev = (u64)netdev;
2985 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2986
2987 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2988 if (ret < 0) {
2989 dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n",
2990 ret);
2991 }
2992 return ret;
2993}
2994
01fb237a
RV		 2995/** Sending command to enable/disable RX checksum offload
2996 * @param netdev pointer to network device
2997 * @param command OCTNET_CMD_TNL_RX_CSUM_CTL
2998 * @param rx_cmd_bit OCTNET_CMD_RXCSUM_ENABLE/
2999 * OCTNET_CMD_RXCSUM_DISABLE
3000 * @returns SUCCESS or FAILURE
3001 */
c41419b0
NMG		 3002static int liquidio_set_rxcsum_command(struct net_device *netdev, int command,
3003 u8 rx_cmd)
01fb237a
RV		 3004{
3005 struct lio *lio = GET_LIO(netdev);
3006 struct octeon_device *oct = lio->oct_dev;
3007 struct octnic_ctrl_pkt nctrl;
3008 int ret = 0;
3009
0c264588
FM		 3010 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
3011
01fb237a
RV		 3012 nctrl.ncmd.u64 = 0;
3013 nctrl.ncmd.s.cmd = command;
3014 nctrl.ncmd.s.param1 = rx_cmd;
3015 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
3016 nctrl.wait_time = 100;
3017 nctrl.netpndev = (u64)netdev;
3018 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
3019
3020 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
3021 if (ret < 0) {
3022 dev_err(&oct->pci_dev->dev,
3023 "DEVFLAGS RXCSUM change failed in core(ret:0x%x)\n",
3024 ret);
3025 }
3026 return ret;
3027}
3028
3029/** Sending command to add/delete VxLAN UDP port to firmware
3030 * @param netdev pointer to network device
3031 * @param command OCTNET_CMD_VXLAN_PORT_CONFIG
3032 * @param vxlan_port VxLAN port to be added or deleted
3033 * @param vxlan_cmd_bit OCTNET_CMD_VXLAN_PORT_ADD,
3034 * OCTNET_CMD_VXLAN_PORT_DEL
3035 * @returns SUCCESS or FAILURE
3036 */
3037static int liquidio_vxlan_port_command(struct net_device *netdev, int command,
3038 u16 vxlan_port, u8 vxlan_cmd_bit)
3039{
3040 struct lio *lio = GET_LIO(netdev);
3041 struct octeon_device *oct = lio->oct_dev;
3042 struct octnic_ctrl_pkt nctrl;
3043 int ret = 0;
3044
0c264588
FM		 3045 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
3046
01fb237a
RV		 3047 nctrl.ncmd.u64 = 0;
3048 nctrl.ncmd.s.cmd = command;
3049 nctrl.ncmd.s.more = vxlan_cmd_bit;
3050 nctrl.ncmd.s.param1 = vxlan_port;
3051 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
3052 nctrl.wait_time = 100;
3053 nctrl.netpndev = (u64)netdev;
3054 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
3055
3056 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
3057 if (ret < 0) {
3058 dev_err(&oct->pci_dev->dev,
3059 "VxLAN port add/delete failed in core (ret:0x%x)\n",
3060 ret);
3061 }
3062 return ret;
3063}
3064
f21fb3ed
RV		 3065/** \brief Net device fix features
3066 * @param netdev pointer to network device
3067 * @param request features requested
3068 * @returns updated features list
3069 */
3070static netdev_features_t liquidio_fix_features(struct net_device *netdev,
3071 netdev_features_t request)
3072{
3073 struct lio *lio = netdev_priv(netdev);
3074
3075 if ((request & NETIF_F_RXCSUM) &&
3076 !(lio->dev_capability & NETIF_F_RXCSUM))
3077 request &= ~NETIF_F_RXCSUM;
3078
3079 if ((request & NETIF_F_HW_CSUM) &&
3080 !(lio->dev_capability & NETIF_F_HW_CSUM))
3081 request &= ~NETIF_F_HW_CSUM;
3082
3083 if ((request & NETIF_F_TSO) && !(lio->dev_capability & NETIF_F_TSO))
3084 request &= ~NETIF_F_TSO;
3085
3086 if ((request & NETIF_F_TSO6) && !(lio->dev_capability & NETIF_F_TSO6))
3087 request &= ~NETIF_F_TSO6;
3088
3089 if ((request & NETIF_F_LRO) && !(lio->dev_capability & NETIF_F_LRO))
3090 request &= ~NETIF_F_LRO;
3091
3092 /*Disable LRO if RXCSUM is off */
3093 if (!(request & NETIF_F_RXCSUM) && (netdev->features & NETIF_F_LRO) &&
3094 (lio->dev_capability & NETIF_F_LRO))
3095 request &= ~NETIF_F_LRO;
3096
836d57e5
PK		 3097 if ((request & NETIF_F_HW_VLAN_CTAG_FILTER) &&
3098 !(lio->dev_capability & NETIF_F_HW_VLAN_CTAG_FILTER))
3099 request &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
3100
f21fb3ed
RV		 3101 return request;
3102}
3103
3104/** \brief Net device set features
3105 * @param netdev pointer to network device
3106 * @param features features to enable/disable
3107 */
3108static int liquidio_set_features(struct net_device *netdev,
3109 netdev_features_t features)
3110{
3111 struct lio *lio = netdev_priv(netdev);
3112
836d57e5
PK		 3113 if ((features & NETIF_F_LRO) &&
3114 (lio->dev_capability & NETIF_F_LRO) &&
3115 !(netdev->features & NETIF_F_LRO))
0cece6c5
RV		 3116 liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
3117 OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
f21fb3ed 3118 else if (!(features & NETIF_F_LRO) &&
836d57e5
PK		 3119 (lio->dev_capability & NETIF_F_LRO) &&
3120 (netdev->features & NETIF_F_LRO))
0cece6c5
RV		 3121 liquidio_set_feature(netdev, OCTNET_CMD_LRO_DISABLE,
3122 OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
f21fb3ed 3123
01fb237a
RV		 3124 /* Sending command to firmware to enable/disable RX checksum
3125 * offload settings using ethtool
3126 */
3127 if (!(netdev->features & NETIF_F_RXCSUM) &&
3128 (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
3129 (features & NETIF_F_RXCSUM))
3130 liquidio_set_rxcsum_command(netdev,
3131 OCTNET_CMD_TNL_RX_CSUM_CTL,
3132 OCTNET_CMD_RXCSUM_ENABLE);
3133 else if ((netdev->features & NETIF_F_RXCSUM) &&
3134 (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
3135 !(features & NETIF_F_RXCSUM))
3136 liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
3137 OCTNET_CMD_RXCSUM_DISABLE);
3138
836d57e5
PK		 3139 if ((features & NETIF_F_HW_VLAN_CTAG_FILTER) &&
3140 (lio->dev_capability & NETIF_F_HW_VLAN_CTAG_FILTER) &&
3141 !(netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER))
3142 liquidio_set_feature(netdev, OCTNET_CMD_VLAN_FILTER_CTL,
3143 OCTNET_CMD_VLAN_FILTER_ENABLE);
3144 else if (!(features & NETIF_F_HW_VLAN_CTAG_FILTER) &&
3145 (lio->dev_capability & NETIF_F_HW_VLAN_CTAG_FILTER) &&
3146 (netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER))
3147 liquidio_set_feature(netdev, OCTNET_CMD_VLAN_FILTER_CTL,
3148 OCTNET_CMD_VLAN_FILTER_DISABLE);
3149
f21fb3ed
RV		 3150 return 0;
3151}
3152
01fb237a
RV		 3153static void liquidio_add_vxlan_port(struct net_device *netdev,
3154 struct udp_tunnel_info *ti)
3155{
3156 if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
3157 return;
3158
3159 liquidio_vxlan_port_command(netdev,
3160 OCTNET_CMD_VXLAN_PORT_CONFIG,
3161 htons(ti->port),
3162 OCTNET_CMD_VXLAN_PORT_ADD);
3163}
3164
3165static void liquidio_del_vxlan_port(struct net_device *netdev,
3166 struct udp_tunnel_info *ti)
3167{
3168 if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
3169 return;
3170
3171 liquidio_vxlan_port_command(netdev,
3172 OCTNET_CMD_VXLAN_PORT_CONFIG,
3173 htons(ti->port),
3174 OCTNET_CMD_VXLAN_PORT_DEL);
3175}
3176
86dea55b
RV		 3177static int __liquidio_set_vf_mac(struct net_device *netdev, int vfidx,
3178 u8 *mac, bool is_admin_assigned)
3179{
3180 struct lio *lio = GET_LIO(netdev);
3181 struct octeon_device *oct = lio->oct_dev;
3182 struct octnic_ctrl_pkt nctrl;
3183
3184 if (!is_valid_ether_addr(mac))
3185 return -EINVAL;
3186
3187 if (vfidx < 0 || vfidx >= oct->sriov_info.max_vfs)
3188 return -EINVAL;
3189
3190 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
3191
3192 nctrl.ncmd.u64 = 0;
3193 nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR;
3194 /* vfidx is 0 based, but vf_num (param1) is 1 based */
3195 nctrl.ncmd.s.param1 = vfidx + 1;
3196 nctrl.ncmd.s.param2 = (is_admin_assigned ? 1 : 0);
3197 nctrl.ncmd.s.more = 1;
3198 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
9549c6c8
RF		 3199 nctrl.netpndev = (u64)netdev;
3200 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
86dea55b
RV		 3201 nctrl.wait_time = LIO_CMD_WAIT_TM;
3202
3203 nctrl.udd[0] = 0;
3204 /* The MAC Address is presented in network byte order. */
3205 ether_addr_copy((u8 *)&nctrl.udd[0] + 2, mac);
3206
3207 oct->sriov_info.vf_macaddr[vfidx] = nctrl.udd[0];
3208
3209 octnet_send_nic_ctrl_pkt(oct, &nctrl);
3210
3211 return 0;
3212}
3213
3214static int liquidio_set_vf_mac(struct net_device *netdev, int vfidx, u8 *mac)
3215{
3216 struct lio *lio = GET_LIO(netdev);
3217 struct octeon_device *oct = lio->oct_dev;
3218 int retval;
3219
0d9a5997
FM		 3220 if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
3221 return -EINVAL;
3222
86dea55b
RV		 3223 retval = __liquidio_set_vf_mac(netdev, vfidx, mac, true);
3224 if (!retval)
3225 cn23xx_tell_vf_its_macaddr_changed(oct, vfidx, mac);
3226
3227 return retval;
3228}
3229
3230static int liquidio_set_vf_vlan(struct net_device *netdev, int vfidx,
3231 u16 vlan, u8 qos, __be16 vlan_proto)
3232{
3233 struct lio *lio = GET_LIO(netdev);
3234 struct octeon_device *oct = lio->oct_dev;
3235 struct octnic_ctrl_pkt nctrl;
3236 u16 vlantci;
3237
3238 if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
3239 return -EINVAL;
3240
3241 if (vlan_proto != htons(ETH_P_8021Q))
3242 return -EPROTONOSUPPORT;
3243
3244 if (vlan >= VLAN_N_VID || qos > 7)
3245 return -EINVAL;
3246
3247 if (vlan)
3248 vlantci = vlan | (u16)qos << VLAN_PRIO_SHIFT;
3249 else
3250 vlantci = 0;
3251
3252 if (oct->sriov_info.vf_vlantci[vfidx] == vlantci)
3253 return 0;
3254
3255 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
3256
3257 if (vlan)
3258 nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER;
3259 else
3260 nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER;
3261
3262 nctrl.ncmd.s.param1 = vlantci;
3263 nctrl.ncmd.s.param2 =
3264 vfidx + 1; /* vfidx is 0 based, but vf_num (param2) is 1 based */
3265 nctrl.ncmd.s.more = 0;
3266 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
3267 nctrl.cb_fn = 0;
3268 nctrl.wait_time = LIO_CMD_WAIT_TM;
3269
3270 octnet_send_nic_ctrl_pkt(oct, &nctrl);
3271
3272 oct->sriov_info.vf_vlantci[vfidx] = vlantci;
3273
3274 return 0;
3275}
3276
3277static int liquidio_get_vf_config(struct net_device *netdev, int vfidx,
3278 struct ifla_vf_info *ivi)
3279{
3280 struct lio *lio = GET_LIO(netdev);
3281 struct octeon_device *oct = lio->oct_dev;
3282 u8 *macaddr;
3283
3284 if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
3285 return -EINVAL;
3286
3287 ivi->vf = vfidx;
3288 macaddr = 2 + (u8 *)&oct->sriov_info.vf_macaddr[vfidx];
3289 ether_addr_copy(&ivi->mac[0], macaddr);
3290 ivi->vlan = oct->sriov_info.vf_vlantci[vfidx] & VLAN_VID_MASK;
3291 ivi->qos = oct->sriov_info.vf_vlantci[vfidx] >> VLAN_PRIO_SHIFT;
3292 ivi->linkstate = oct->sriov_info.vf_linkstate[vfidx];
3293 return 0;
3294}
3295
3296static int liquidio_set_vf_link_state(struct net_device *netdev, int vfidx,
3297 int linkstate)
3298{
3299 struct lio *lio = GET_LIO(netdev);
3300 struct octeon_device *oct = lio->oct_dev;
3301 struct octnic_ctrl_pkt nctrl;
3302
3303 if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
3304 return -EINVAL;
3305
3306 if (oct->sriov_info.vf_linkstate[vfidx] == linkstate)
3307 return 0;
3308
3309 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
3310 nctrl.ncmd.s.cmd = OCTNET_CMD_SET_VF_LINKSTATE;
3311 nctrl.ncmd.s.param1 =
3312 vfidx + 1; /* vfidx is 0 based, but vf_num (param1) is 1 based */
3313 nctrl.ncmd.s.param2 = linkstate;
3314 nctrl.ncmd.s.more = 0;
3315 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
3316 nctrl.cb_fn = 0;
3317 nctrl.wait_time = LIO_CMD_WAIT_TM;
3318
3319 octnet_send_nic_ctrl_pkt(oct, &nctrl);
3320
3321 oct->sriov_info.vf_linkstate[vfidx] = linkstate;
3322
3323 return 0;
3324}
3325
d4be8ebe
VMG		 3326static int
3327liquidio_eswitch_mode_get(struct devlink *devlink, u16 *mode)
3328{
3329 struct lio_devlink_priv *priv;
3330 struct octeon_device *oct;
3331
3332 priv = devlink_priv(devlink);
3333 oct = priv->oct;
3334
3335 *mode = oct->eswitch_mode;
3336
3337 return 0;
3338}
3339
3340static int
3341liquidio_eswitch_mode_set(struct devlink *devlink, u16 mode)
3342{
3343 struct lio_devlink_priv *priv;
3344 struct octeon_device *oct;
3345 int ret = 0;
3346
3347 priv = devlink_priv(devlink);
3348 oct = priv->oct;
3349
3350 if (!(oct->fw_info.app_cap_flags & LIQUIDIO_SWITCHDEV_CAP))
3351 return -EINVAL;
3352
3353 if (oct->eswitch_mode == mode)
3354 return 0;
3355
3356 switch (mode) {
3357 case DEVLINK_ESWITCH_MODE_SWITCHDEV:
3358 oct->eswitch_mode = mode;
3359 ret = lio_vf_rep_create(oct);
3360 break;
3361
3362 case DEVLINK_ESWITCH_MODE_LEGACY:
3363 lio_vf_rep_destroy(oct);
3364 oct->eswitch_mode = mode;
3365 break;
3366
3367 default:
3368 ret = -EINVAL;
3369 }
3370
3371 return ret;
3372}
3373
3374static const struct devlink_ops liquidio_devlink_ops = {
3375 .eswitch_mode_get = liquidio_eswitch_mode_get,
3376 .eswitch_mode_set = liquidio_eswitch_mode_set,
3377};
3378
1f233f32
VMG		 3379static int
3380lio_pf_switchdev_attr_get(struct net_device *dev, struct switchdev_attr *attr)
3381{
3382 struct lio *lio = GET_LIO(dev);
d4be8ebe
VMG		 3383 struct octeon_device *oct = lio->oct_dev;
3384
3385 if (oct->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
3386 return -EOPNOTSUPP;
1f233f32
VMG		 3387
3388 switch (attr->id) {
3389 case SWITCHDEV_ATTR_ID_PORT_PARENT_ID:
3390 attr->u.ppid.id_len = ETH_ALEN;
3391 ether_addr_copy(attr->u.ppid.id,
3392 (void *)&lio->linfo.hw_addr + 2);
3393 break;
3394
3395 default:
3396 return -EOPNOTSUPP;
3397 }
3398
3399 return 0;
3400}
3401
3402static const struct switchdev_ops lio_pf_switchdev_ops = {
3403 .switchdev_port_attr_get = lio_pf_switchdev_attr_get,
3404};
3405
97a25326 3406static const struct net_device_ops lionetdevops = {
f21fb3ed
RV		 3407 .ndo_open = liquidio_open,
3408 .ndo_stop = liquidio_stop,
3409 .ndo_start_xmit = liquidio_xmit,
3410 .ndo_get_stats = liquidio_get_stats,
3411 .ndo_set_mac_address = liquidio_set_mac,
3412 .ndo_set_rx_mode = liquidio_set_mcast_list,
3413 .ndo_tx_timeout = liquidio_tx_timeout,
63245f25
RV		 3414
3415 .ndo_vlan_rx_add_vid = liquidio_vlan_rx_add_vid,
3416 .ndo_vlan_rx_kill_vid = liquidio_vlan_rx_kill_vid,
f21fb3ed
RV		 3417 .ndo_change_mtu = liquidio_change_mtu,
3418 .ndo_do_ioctl = liquidio_ioctl,
3419 .ndo_fix_features = liquidio_fix_features,
3420 .ndo_set_features = liquidio_set_features,
01fb237a
RV		 3421 .ndo_udp_tunnel_add = liquidio_add_vxlan_port,
3422 .ndo_udp_tunnel_del = liquidio_del_vxlan_port,
86dea55b
RV		 3423 .ndo_set_vf_mac = liquidio_set_vf_mac,
3424 .ndo_set_vf_vlan = liquidio_set_vf_vlan,
3425 .ndo_get_vf_config = liquidio_get_vf_config,
3426 .ndo_set_vf_link_state = liquidio_set_vf_link_state,
f21fb3ed
RV		 3427};
3428
3429/** \brief Entry point for the liquidio module
3430 */
3431static int __init liquidio_init(void)
3432{
3433 int i;
3434 struct handshake *hs;
3435
3436 init_completion(&first_stage);
3437
97a25326 3438 octeon_init_device_list(OCTEON_CONFIG_TYPE_DEFAULT);
f21fb3ed
RV		 3439
3440 if (liquidio_init_pci())
3441 return -EINVAL;
3442
3443 wait_for_completion_timeout(&first_stage, msecs_to_jiffies(1000));
3444
3445 for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3446 hs = &handshake[i];
3447 if (hs->pci_dev) {
3448 wait_for_completion(&hs->init);
3449 if (!hs->init_ok) {
3450 /* init handshake failed */
3451 dev_err(&hs->pci_dev->dev,
3452 "Failed to init device\n");
3453 liquidio_deinit_pci();
3454 return -EIO;
3455 }
3456 }
3457 }
3458
3459 for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3460 hs = &handshake[i];
3461 if (hs->pci_dev) {
3462 wait_for_completion_timeout(&hs->started,
3463 msecs_to_jiffies(30000));
3464 if (!hs->started_ok) {
3465 /* starter handshake failed */
3466 dev_err(&hs->pci_dev->dev,
3467 "Firmware failed to start\n");
3468 liquidio_deinit_pci();
3469 return -EIO;
3470 }
3471 }
3472 }
3473
3474 return 0;
3475}
3476
5b173cf9 3477static int lio_nic_info(struct octeon_recv_info *recv_info, void *buf)
f21fb3ed
RV		 3478{
3479 struct octeon_device *oct = (struct octeon_device *)buf;
3480 struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt;
0cece6c5 3481 int gmxport = 0;
f21fb3ed
RV		 3482 union oct_link_status *ls;
3483 int i;
3484
c4ee5d81 3485 if (recv_pkt->buffer_size[0] != (sizeof(*ls) + OCT_DROQ_INFO_SIZE)) {
f21fb3ed
RV		 3486 dev_err(&oct->pci_dev->dev, "Malformed NIC_INFO, len=%d, ifidx=%d\n",
3487 recv_pkt->buffer_size[0],
0cece6c5 3488 recv_pkt->rh.r_nic_info.gmxport);
f21fb3ed
RV		 3489 goto nic_info_err;
3490 }
3491
0cece6c5 3492 gmxport = recv_pkt->rh.r_nic_info.gmxport;
c4ee5d81
PK		 3493 ls = (union oct_link_status *)(get_rbd(recv_pkt->buffer_ptr[0]) +
3494 OCT_DROQ_INFO_SIZE);
f21fb3ed
RV		 3495
3496 octeon_swap_8B_data((u64 *)ls, (sizeof(union oct_link_status)) >> 3);
0cece6c5
RV		 3497 for (i = 0; i < oct->ifcount; i++) {
3498 if (oct->props[i].gmxport == gmxport) {
3499 update_link_status(oct->props[i].netdev, ls);
3500 break;
3501 }
3502 }
f21fb3ed
RV		 3503
3504nic_info_err:
3505 for (i = 0; i < recv_pkt->buffer_count; i++)
3506 recv_buffer_free(recv_pkt->buffer_ptr[i]);
3507 octeon_free_recv_info(recv_info);
3508 return 0;
3509}
3510
3511/**
3512 * \brief Setup network interfaces
3513 * @param octeon_dev octeon device
3514 *
3515 * Called during init time for each device. It assumes the NIC
3516 * is already up and running. The link information for each
3517 * interface is passed in link_info.
3518 */
3519static int setup_nic_devices(struct octeon_device *octeon_dev)
3520{
3521 struct lio *lio = NULL;
3522 struct net_device *netdev;
b36e4820 3523 u8 mac[6], i, j, *fw_ver;
f21fb3ed
RV		 3524 struct octeon_soft_command *sc;
3525 struct liquidio_if_cfg_context *ctx;
3526 struct liquidio_if_cfg_resp *resp;
3527 struct octdev_props *props;
26236fa9 3528 int retval, num_iqueues, num_oqueues;
f21fb3ed
RV		 3529 union oct_nic_if_cfg if_cfg;
3530 unsigned int base_queue;
3531 unsigned int gmx_port_id;
83101ce3 3532 u32 resp_size, ctx_size, data_size;
0cece6c5 3533 u32 ifidx_or_pfnum;
83101ce3 3534 struct lio_version *vdata;
d4be8ebe
VMG		 3535 struct devlink *devlink;
3536 struct lio_devlink_priv *lio_devlink;
f21fb3ed
RV		 3537
3538 /* This is to handle link status changes */
3539 octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
3540 OPCODE_NIC_INFO,
3541 lio_nic_info, octeon_dev);
3542
3543 /* REQTYPE_RESP_NET and REQTYPE_SOFT_COMMAND do not have free functions.
3544 * They are handled directly.
3545 */
3546 octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET,
3547 free_netbuf);
3548
3549 octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET_SG,
3550 free_netsgbuf);
3551
3552 octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_RESP_NET_SG,
3553 free_netsgbuf_with_resp);
3554
3555 for (i = 0; i < octeon_dev->ifcount; i++) {
3556 resp_size = sizeof(struct liquidio_if_cfg_resp);
3557 ctx_size = sizeof(struct liquidio_if_cfg_context);
83101ce3 3558 data_size = sizeof(struct lio_version);
f21fb3ed 3559 sc = (struct octeon_soft_command *)
83101ce3 3560 octeon_alloc_soft_command(octeon_dev, data_size,
f21fb3ed
RV		 3561 resp_size, ctx_size);
3562 resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
3563 ctx = (struct liquidio_if_cfg_context *)sc->ctxptr;
83101ce3
RV		 3564 vdata = (struct lio_version *)sc->virtdptr;
3565
3566 *((u64 *)vdata) = 0;
3567 vdata->major = cpu_to_be16(LIQUIDIO_BASE_MAJOR_VERSION);
3568 vdata->minor = cpu_to_be16(LIQUIDIO_BASE_MINOR_VERSION);
3569 vdata->micro = cpu_to_be16(LIQUIDIO_BASE_MICRO_VERSION);
f21fb3ed 3570
e86b1ab6
RV		 3571 if (OCTEON_CN23XX_PF(octeon_dev)) {
3572 num_iqueues = octeon_dev->sriov_info.num_pf_rings;
3573 num_oqueues = octeon_dev->sriov_info.num_pf_rings;
3574 base_queue = octeon_dev->sriov_info.pf_srn;
3575
3576 gmx_port_id = octeon_dev->pf_num;
3577 ifidx_or_pfnum = octeon_dev->pf_num;
3578 } else {
3579 num_iqueues = CFG_GET_NUM_TXQS_NIC_IF(
3580 octeon_get_conf(octeon_dev), i);
3581 num_oqueues = CFG_GET_NUM_RXQS_NIC_IF(
3582 octeon_get_conf(octeon_dev), i);
3583 base_queue = CFG_GET_BASE_QUE_NIC_IF(
3584 octeon_get_conf(octeon_dev), i);
3585 gmx_port_id = CFG_GET_GMXID_NIC_IF(
3586 octeon_get_conf(octeon_dev), i);
3587 ifidx_or_pfnum = i;
3588 }
3dcef2ca 3589
f21fb3ed
RV		 3590 dev_dbg(&octeon_dev->pci_dev->dev,
3591 "requesting config for interface %d, iqs %d, oqs %d\n",
0cece6c5 3592 ifidx_or_pfnum, num_iqueues, num_oqueues);
a7d5a3dc 3593 WRITE_ONCE(ctx->cond, 0);
f21fb3ed
RV		 3594 ctx->octeon_id = lio_get_device_id(octeon_dev);
3595 init_waitqueue_head(&ctx->wc);
3596
3597 if_cfg.u64 = 0;
3598 if_cfg.s.num_iqueues = num_iqueues;
3599 if_cfg.s.num_oqueues = num_oqueues;
3600 if_cfg.s.base_queue = base_queue;
3601 if_cfg.s.gmx_port_id = gmx_port_id;
0cece6c5
RV		 3602
3603 sc->iq_no = 0;
3604
f21fb3ed 3605 octeon_prepare_soft_command(octeon_dev, sc, OPCODE_NIC,
0cece6c5 3606 OPCODE_NIC_IF_CFG, 0,
f21fb3ed
RV		 3607 if_cfg.u64, 0);
3608
3609 sc->callback = if_cfg_callback;
3610 sc->callback_arg = sc;
55893a63 3611 sc->wait_time = 3000;
f21fb3ed
RV		 3612
3613 retval = octeon_send_soft_command(octeon_dev, sc);
ddc173a6 3614 if (retval == IQ_SEND_FAILED) {
f21fb3ed
RV		 3615 dev_err(&octeon_dev->pci_dev->dev,
3616 "iq/oq config failed status: %x\n",
3617 retval);
3618 /* Soft instr is freed by driver in case of failure. */
3619 goto setup_nic_dev_fail;
3620 }
3621
3622 /* Sleep on a wait queue till the cond flag indicates that the
3623 * response arrived or timed-out.
3624 */
afdf841f
RV		 3625 if (sleep_cond(&ctx->wc, &ctx->cond) == -EINTR) {
3626 dev_err(&octeon_dev->pci_dev->dev, "Wait interrupted\n");
3627 goto setup_nic_wait_intr;
3628 }
3629
f21fb3ed
RV		 3630 retval = resp->status;
3631 if (retval) {
3632 dev_err(&octeon_dev->pci_dev->dev, "iq/oq config failed\n");
3633 goto setup_nic_dev_fail;
3634 }
3635
b36e4820
RF		 3636 /* Verify f/w version (in case of 'auto' loading from flash) */
3637 fw_ver = octeon_dev->fw_info.liquidio_firmware_version;
3638 if (memcmp(LIQUIDIO_BASE_VERSION,
3639 fw_ver,
3640 strlen(LIQUIDIO_BASE_VERSION))) {
3641 dev_err(&octeon_dev->pci_dev->dev,
3642 "Unmatched firmware version. Expected %s.x, got %s.\n",
3643 LIQUIDIO_BASE_VERSION, fw_ver);
3644 goto setup_nic_dev_fail;
3645 } else if (atomic_read(octeon_dev->adapter_fw_state) ==
3646 FW_IS_PRELOADED) {
3647 dev_info(&octeon_dev->pci_dev->dev,
3648 "Using auto-loaded firmware version %s.\n",
3649 fw_ver);
3650 }
3651
f21fb3ed
RV		 3652 octeon_swap_8B_data((u64 *)(&resp->cfg_info),
3653 (sizeof(struct liquidio_if_cfg_info)) >> 3);
3654
3655 num_iqueues = hweight64(resp->cfg_info.iqmask);
3656 num_oqueues = hweight64(resp->cfg_info.oqmask);
3657
3658 if (!(num_iqueues) || !(num_oqueues)) {
3659 dev_err(&octeon_dev->pci_dev->dev,
3660 "Got bad iqueues (%016llx) or oqueues (%016llx) from firmware.\n",
3661 resp->cfg_info.iqmask,
3662 resp->cfg_info.oqmask);
3663 goto setup_nic_dev_fail;
3664 }
3665 dev_dbg(&octeon_dev->pci_dev->dev,
3666 "interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d\n",
3667 i, resp->cfg_info.iqmask, resp->cfg_info.oqmask,
3668 num_iqueues, num_oqueues);
3669 netdev = alloc_etherdev_mq(LIO_SIZE, num_iqueues);
3670
3671 if (!netdev) {
3672 dev_err(&octeon_dev->pci_dev->dev, "Device allocation failed\n");
3673 goto setup_nic_dev_fail;
3674 }
3675
0cece6c5 3676 SET_NETDEV_DEV(netdev, &octeon_dev->pci_dev->dev);
f21fb3ed 3677
f21fb3ed
RV		 3678 /* Associate the routines that will handle different
3679 * netdev tasks.
3680 */
3681 netdev->netdev_ops = &lionetdevops;
1f233f32 3682 SWITCHDEV_SET_OPS(netdev, &lio_pf_switchdev_ops);
f21fb3ed
RV		 3683
3684 lio = GET_LIO(netdev);
3685
3686 memset(lio, 0, sizeof(struct lio));
3687
0cece6c5
RV		 3688 lio->ifidx = ifidx_or_pfnum;
3689
3690 props = &octeon_dev->props[i];
3691 props->gmxport = resp->cfg_info.linfo.gmxport;
3692 props->netdev = netdev;
f21fb3ed
RV		 3693
3694 lio->linfo.num_rxpciq = num_oqueues;
3695 lio->linfo.num_txpciq = num_iqueues;
f21fb3ed 3696 for (j = 0; j < num_oqueues; j++) {
26236fa9
RV		 3697 lio->linfo.rxpciq[j].u64 =
3698 resp->cfg_info.linfo.rxpciq[j].u64;
f21fb3ed 3699 }
f21fb3ed 3700 for (j = 0; j < num_iqueues; j++) {
26236fa9
RV		 3701 lio->linfo.txpciq[j].u64 =
3702 resp->cfg_info.linfo.txpciq[j].u64;
f21fb3ed
RV		 3703 }
3704 lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr;
3705 lio->linfo.gmxport = resp->cfg_info.linfo.gmxport;
3706 lio->linfo.link.u64 = resp->cfg_info.linfo.link.u64;
3707
3708 lio->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3709
e86b1ab6
RV		 3710 if (OCTEON_CN23XX_PF(octeon_dev) ||
3711 OCTEON_CN6XXX(octeon_dev)) {
3712 lio->dev_capability = NETIF_F_HIGHDMA
3713 | NETIF_F_IP_CSUM
3714 | NETIF_F_IPV6_CSUM
3715 | NETIF_F_SG | NETIF_F_RXCSUM
3716 | NETIF_F_GRO
3717 | NETIF_F_TSO | NETIF_F_TSO6
3718 | NETIF_F_LRO;
3719 }
f21fb3ed
RV		 3720 netif_set_gso_max_size(netdev, OCTNIC_GSO_MAX_SIZE);
3721
01fb237a
RV		 3722 /* Copy of transmit encapsulation capabilities:
3723 * TSO, TSO6, Checksums for this device
3724 */
3725 lio->enc_dev_capability = NETIF_F_IP_CSUM
3726 | NETIF_F_IPV6_CSUM
3727 | NETIF_F_GSO_UDP_TUNNEL
3728 | NETIF_F_HW_CSUM | NETIF_F_SG
3729 | NETIF_F_RXCSUM
3730 | NETIF_F_TSO | NETIF_F_TSO6
3731 | NETIF_F_LRO;
3732
3733 netdev->hw_enc_features = (lio->enc_dev_capability &
3734 ~NETIF_F_LRO);
3735
3736 lio->dev_capability |= NETIF_F_GSO_UDP_TUNNEL;
3737
f21fb3ed 3738 netdev->vlan_features = lio->dev_capability;
0da0b77c 3739 /* Add any unchangeable hw features */
63245f25
RV		 3740 lio->dev_capability |= NETIF_F_HW_VLAN_CTAG_FILTER |
3741 NETIF_F_HW_VLAN_CTAG_RX |
0da0b77c
RV		 3742 NETIF_F_HW_VLAN_CTAG_TX;
3743
3744 netdev->features = (lio->dev_capability & ~NETIF_F_LRO);
f21fb3ed
RV		 3745
3746 netdev->hw_features = lio->dev_capability;
0da0b77c
RV		 3747 /*HW_VLAN_RX and HW_VLAN_FILTER is always on*/
3748 netdev->hw_features = netdev->hw_features &
3749 ~NETIF_F_HW_VLAN_CTAG_RX;
f21fb3ed 3750
109cc165
JW		 3751 /* MTU range: 68 - 16000 */
3752 netdev->min_mtu = LIO_MIN_MTU_SIZE;
3753 netdev->max_mtu = LIO_MAX_MTU_SIZE;
3754
f21fb3ed
RV		 3755 /* Point to the properties for octeon device to which this
3756 * interface belongs.
3757 */
3758 lio->oct_dev = octeon_dev;
3759 lio->octprops = props;
3760 lio->netdev = netdev;
f21fb3ed
RV		 3761
3762 dev_dbg(&octeon_dev->pci_dev->dev,
3763 "if%d gmx: %d hw_addr: 0x%llx\n", i,
3764 lio->linfo.gmxport, CVM_CAST64(lio->linfo.hw_addr));
3765
86dea55b
RV		 3766 for (j = 0; j < octeon_dev->sriov_info.max_vfs; j++) {
3767 u8 vfmac[ETH_ALEN];
3768
3769 random_ether_addr(&vfmac[0]);
3770 if (__liquidio_set_vf_mac(netdev, j,
3771 &vfmac[0], false)) {
3772 dev_err(&octeon_dev->pci_dev->dev,
3773 "Error setting VF%d MAC address\n",
3774 j);
3775 goto setup_nic_dev_fail;
3776 }
3777 }
3778
f21fb3ed
RV		 3779 /* 64-bit swap required on LE machines */
3780 octeon_swap_8B_data(&lio->linfo.hw_addr, 1);
3781 for (j = 0; j < 6; j++)
3782 mac[j] = *((u8 *)(((u8 *)&lio->linfo.hw_addr) + 2 + j));
3783
3784 /* Copy MAC Address to OS network device structure */
3785
3786 ether_addr_copy(netdev->dev_addr, mac);
3787
26236fa9
RV		 3788 /* By default all interfaces on a single Octeon uses the same
3789 * tx and rx queues
3790 */
3791 lio->txq = lio->linfo.txpciq[0].s.q_no;
3792 lio->rxq = lio->linfo.rxpciq[0].s.q_no;
a82457f1
IB		 3793 if (liquidio_setup_io_queues(octeon_dev, i,
3794 lio->linfo.num_txpciq,
3795 lio->linfo.num_rxpciq)) {
f21fb3ed
RV		 3796 dev_err(&octeon_dev->pci_dev->dev, "I/O queues creation failed\n");
3797 goto setup_nic_dev_fail;
3798 }
3799
3800 ifstate_set(lio, LIO_IFSTATE_DROQ_OPS);
3801
f21fb3ed
RV		 3802 lio->tx_qsize = octeon_get_tx_qsize(octeon_dev, lio->txq);
3803 lio->rx_qsize = octeon_get_rx_qsize(octeon_dev, lio->rxq);
3804
fcd2b5e3 3805 if (setup_glists(octeon_dev, lio, num_iqueues)) {
f21fb3ed
RV		 3806 dev_err(&octeon_dev->pci_dev->dev,
3807 "Gather list allocation failed\n");
3808 goto setup_nic_dev_fail;
3809 }
3810
3811 /* Register ethtool support */
3812 liquidio_set_ethtool_ops(netdev);
30136395
RV		 3813 if (lio->oct_dev->chip_id == OCTEON_CN23XX_PF_VID)
3814 octeon_dev->priv_flags = OCT_PRIV_FLAG_DEFAULT;
3815 else
3816 octeon_dev->priv_flags = 0x0;
f21fb3ed 3817
0cece6c5 3818 if (netdev->features & NETIF_F_LRO)
a2c64b67
RV		 3819 liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
3820 OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
f21fb3ed 3821
836d57e5
PK		 3822 liquidio_set_feature(netdev, OCTNET_CMD_VLAN_FILTER_CTL,
3823 OCTNET_CMD_VLAN_FILTER_ENABLE);
63245f25 3824
f21fb3ed 3825 if ((debug != -1) && (debug & NETIF_MSG_HW))
63245f25
RV		 3826 liquidio_set_feature(netdev,
3827 OCTNET_CMD_VERBOSE_ENABLE, 0);
f21fb3ed 3828
7b6b6c95
RV		 3829 if (setup_link_status_change_wq(netdev))
3830 goto setup_nic_dev_fail;
3831
907aaa6b
VB		 3832 if ((octeon_dev->fw_info.app_cap_flags &
3833 LIQUIDIO_TIME_SYNC_CAP) &&
3834 setup_sync_octeon_time_wq(netdev))
3835 goto setup_nic_dev_fail;
3836
031d4f12
SB		 3837 if (setup_rx_oom_poll_fn(netdev))
3838 goto setup_nic_dev_fail;
3839
f21fb3ed
RV		 3840 /* Register the network device with the OS */
3841 if (register_netdev(netdev)) {
3842 dev_err(&octeon_dev->pci_dev->dev, "Device registration failed\n");
3843 goto setup_nic_dev_fail;
3844 }
3845
3846 dev_dbg(&octeon_dev->pci_dev->dev,
3847 "Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n",
3848 i, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
3849 netif_carrier_off(netdev);
0cece6c5 3850 lio->link_changes++;
f21fb3ed
RV		 3851
3852 ifstate_set(lio, LIO_IFSTATE_REGISTERED);
3853
01fb237a
RV		 3854 /* Sending command to firmware to enable Rx checksum offload
3855 * by default at the time of setup of Liquidio driver for
3856 * this device
3857 */
3858 liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
3859 OCTNET_CMD_RXCSUM_ENABLE);
3860 liquidio_set_feature(netdev, OCTNET_CMD_TNL_TX_CSUM_CTL,
3861 OCTNET_CMD_TXCSUM_ENABLE);
3862
f21fb3ed
RV		 3863 dev_dbg(&octeon_dev->pci_dev->dev,
3864 "NIC ifidx:%d Setup successful\n", i);
3865
3866 octeon_free_soft_command(octeon_dev, sc);
3867 }
3868
d4be8ebe
VMG		 3869 devlink = devlink_alloc(&liquidio_devlink_ops,
3870 sizeof(struct lio_devlink_priv));
3871 if (!devlink) {
3872 dev_err(&octeon_dev->pci_dev->dev, "devlink alloc failed\n");
3873 goto setup_nic_wait_intr;
3874 }
3875
3876 lio_devlink = devlink_priv(devlink);
3877 lio_devlink->oct = octeon_dev;
3878
3879 if (devlink_register(devlink, &octeon_dev->pci_dev->dev)) {
3880 devlink_free(devlink);
3881 dev_err(&octeon_dev->pci_dev->dev,
3882 "devlink registration failed\n");
3883 goto setup_nic_wait_intr;
3884 }
3885
3886 octeon_dev->devlink = devlink;
3887 octeon_dev->eswitch_mode = DEVLINK_ESWITCH_MODE_LEGACY;
3888
f21fb3ed
RV		 3889 return 0;
3890
3891setup_nic_dev_fail:
3892
3893 octeon_free_soft_command(octeon_dev, sc);
3894
afdf841f
RV		 3895setup_nic_wait_intr:
3896
f21fb3ed
RV		 3897 while (i--) {
3898 dev_err(&octeon_dev->pci_dev->dev,
3899 "NIC ifidx:%d Setup failed\n", i);
3900 liquidio_destroy_nic_device(octeon_dev, i);
3901 }
3902 return -ENODEV;
3903}
3904
ca6139ff
RV		 3905#ifdef CONFIG_PCI_IOV
3906static int octeon_enable_sriov(struct octeon_device *oct)
3907{
3908 unsigned int num_vfs_alloced = oct->sriov_info.num_vfs_alloced;
3909 struct pci_dev *vfdev;
3910 int err;
3911 u32 u;
3912
3913 if (OCTEON_CN23XX_PF(oct) && num_vfs_alloced) {
3914 err = pci_enable_sriov(oct->pci_dev,
3915 oct->sriov_info.num_vfs_alloced);
3916 if (err) {
3917 dev_err(&oct->pci_dev->dev,
3918 "OCTEON: Failed to enable PCI sriov: %d\n",
3919 err);
3920 oct->sriov_info.num_vfs_alloced = 0;
3921 return err;
3922 }
3923 oct->sriov_info.sriov_enabled = 1;
3924
3925 /* init lookup table that maps DPI ring number to VF pci_dev
3926 * struct pointer
3927 */
3928 u = 0;
3929 vfdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
3930 OCTEON_CN23XX_VF_VID, NULL);
3931 while (vfdev) {
3932 if (vfdev->is_virtfn &&
3933 (vfdev->physfn == oct->pci_dev)) {
3934 oct->sriov_info.dpiring_to_vfpcidev_lut[u] =
3935 vfdev;
3936 u += oct->sriov_info.rings_per_vf;
3937 }
3938 vfdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
3939 OCTEON_CN23XX_VF_VID, vfdev);
3940 }
3941 }
3942
3943 return num_vfs_alloced;
3944}
3945
3946static int lio_pci_sriov_disable(struct octeon_device *oct)
3947{
3948 int u;
3949
3950 if (pci_vfs_assigned(oct->pci_dev)) {
3951 dev_err(&oct->pci_dev->dev, "VFs are still assigned to VMs.\n");
3952 return -EPERM;
3953 }
3954
3955 pci_disable_sriov(oct->pci_dev);
3956
3957 u = 0;
3958 while (u < MAX_POSSIBLE_VFS) {
3959 oct->sriov_info.dpiring_to_vfpcidev_lut[u] = NULL;
3960 u += oct->sriov_info.rings_per_vf;
3961 }
3962
3963 oct->sriov_info.num_vfs_alloced = 0;
3964 dev_info(&oct->pci_dev->dev, "oct->pf_num:%d disabled VFs\n",
3965 oct->pf_num);
3966
3967 return 0;
3968}
3969
3970static int liquidio_enable_sriov(struct pci_dev *dev, int num_vfs)
3971{
3972 struct octeon_device *oct = pci_get_drvdata(dev);
3973 int ret = 0;
3974
3975 if ((num_vfs == oct->sriov_info.num_vfs_alloced) &&
3976 (oct->sriov_info.sriov_enabled)) {
3977 dev_info(&oct->pci_dev->dev, "oct->pf_num:%d already enabled num_vfs:%d\n",
3978 oct->pf_num, num_vfs);
3979 return 0;
3980 }
3981
3982 if (!num_vfs) {
d4be8ebe 3983 lio_vf_rep_destroy(oct);
ca6139ff
RV		 3984 ret = lio_pci_sriov_disable(oct);
3985 } else if (num_vfs > oct->sriov_info.max_vfs) {
3986 dev_err(&oct->pci_dev->dev,
3987 "OCTEON: Max allowed VFs:%d user requested:%d",
3988 oct->sriov_info.max_vfs, num_vfs);
3989 ret = -EPERM;
3990 } else {
3991 oct->sriov_info.num_vfs_alloced = num_vfs;
3992 ret = octeon_enable_sriov(oct);
3993 dev_info(&oct->pci_dev->dev, "oct->pf_num:%d num_vfs:%d\n",
3994 oct->pf_num, num_vfs);
d4be8ebe
VMG		 3995 ret = lio_vf_rep_create(oct);
3996 if (ret)
3997 dev_info(&oct->pci_dev->dev,
3998 "vf representor create failed");
ca6139ff
RV		 3999 }
4000
4001 return ret;
4002}
4003#endif
4004
f21fb3ed
RV		 4005/**
4006 * \brief initialize the NIC
4007 * @param oct octeon device
4008 *
4009 * This initialization routine is called once the Octeon device application is
4010 * up and running
4011 */
4012static int liquidio_init_nic_module(struct octeon_device *oct)
4013{
0cece6c5 4014 int i, retval = 0;
f21fb3ed
RV		 4015 int num_nic_ports = CFG_GET_NUM_NIC_PORTS(octeon_get_conf(oct));
4016
4017 dev_dbg(&oct->pci_dev->dev, "Initializing network interfaces\n");
4018
4019 /* only default iq and oq were initialized
4020 * initialize the rest as well
4021 */
4022 /* run port_config command for each port */
4023 oct->ifcount = num_nic_ports;
4024
30136395 4025 memset(oct->props, 0, sizeof(struct octdev_props) * num_nic_ports);
f21fb3ed 4026
0cece6c5
RV		 4027 for (i = 0; i < MAX_OCTEON_LINKS; i++)
4028 oct->props[i].gmxport = -1;
4029
f21fb3ed
RV		 4030 retval = setup_nic_devices(oct);
4031 if (retval) {
4032 dev_err(&oct->pci_dev->dev, "Setup NIC devices failed\n");
4033 goto octnet_init_failure;
4034 }
4035
e20f4696
VMG		 4036 /* Call vf_rep_modinit if the firmware is switchdev capable
4037 * and do it from the first liquidio function probed.
4038 */
4039 if (!oct->octeon_id &&
4040 oct->fw_info.app_cap_flags & LIQUIDIO_SWITCHDEV_CAP) {
228aa012
DC		 4041 retval = lio_vf_rep_modinit();
4042 if (retval) {
e20f4696
VMG		 4043 liquidio_stop_nic_module(oct);
4044 goto octnet_init_failure;
4045 }
4046 }
4047
f21fb3ed
RV		 4048 liquidio_ptp_init(oct);
4049
f21fb3ed
RV		 4050 dev_dbg(&oct->pci_dev->dev, "Network interfaces ready\n");
4051
4052 return retval;
4053
4054octnet_init_failure:
4055
4056 oct->ifcount = 0;
4057
4058 return retval;
4059}
4060
4061/**
4062 * \brief starter callback that invokes the remaining initialization work after
4063 * the NIC is up and running.
4064 * @param octptr work struct work_struct
4065 */
4066static void nic_starter(struct work_struct *work)
4067{
4068 struct octeon_device *oct;
4069 struct cavium_wk *wk = (struct cavium_wk *)work;
4070
4071 oct = (struct octeon_device *)wk->ctxptr;
4072
4073 if (atomic_read(&oct->status) == OCT_DEV_RUNNING)
4074 return;
4075
4076 /* If the status of the device is CORE_OK, the core
4077 * application has reported its application type. Call
4078 * any registered handlers now and move to the RUNNING
4079 * state.
4080 */
4081 if (atomic_read(&oct->status) != OCT_DEV_CORE_OK) {
4082 schedule_delayed_work(&oct->nic_poll_work.work,
4083 LIQUIDIO_STARTER_POLL_INTERVAL_MS);
4084 return;
4085 }
4086
4087 atomic_set(&oct->status, OCT_DEV_RUNNING);
4088
4089 if (oct->app_mode && oct->app_mode == CVM_DRV_NIC_APP) {
4090 dev_dbg(&oct->pci_dev->dev, "Starting NIC module\n");
4091
4092 if (liquidio_init_nic_module(oct))
4093 dev_err(&oct->pci_dev->dev, "NIC initialization failed\n");
4094 else
4095 handshake[oct->octeon_id].started_ok = 1;
4096 } else {
4097 dev_err(&oct->pci_dev->dev,
4098 "Unexpected application running on NIC (%d). Check firmware.\n",
4099 oct->app_mode);
4100 }
4101
4102 complete(&handshake[oct->octeon_id].started);
4103}
4104
86dea55b
RV		 4105static int
4106octeon_recv_vf_drv_notice(struct octeon_recv_info *recv_info, void *buf)
4107{
4108 struct octeon_device *oct = (struct octeon_device *)buf;
4109 struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt;
4110 int i, notice, vf_idx;
bb54be58 4111 bool cores_crashed;
86dea55b
RV		 4112 u64 *data, vf_num;
4113
4114 notice = recv_pkt->rh.r.ossp;
c4ee5d81 4115 data = (u64 *)(get_rbd(recv_pkt->buffer_ptr[0]) + OCT_DROQ_INFO_SIZE);
86dea55b
RV		 4116
4117 /* the first 64-bit word of data is the vf_num */
4118 vf_num = data[0];
4119 octeon_swap_8B_data(&vf_num, 1);
4120 vf_idx = (int)vf_num - 1;
4121
bb54be58
FM		 4122 cores_crashed = READ_ONCE(oct->cores_crashed);
4123
86dea55b
RV		 4124 if (notice == VF_DRV_LOADED) {
4125 if (!(oct->sriov_info.vf_drv_loaded_mask & BIT_ULL(vf_idx))) {
4126 oct->sriov_info.vf_drv_loaded_mask |= BIT_ULL(vf_idx);
4127 dev_info(&oct->pci_dev->dev,
4128 "driver for VF%d was loaded\n", vf_idx);
bb54be58
FM		 4129 if (!cores_crashed)
4130 try_module_get(THIS_MODULE);
86dea55b
RV		 4131 }
4132 } else if (notice == VF_DRV_REMOVED) {
4133 if (oct->sriov_info.vf_drv_loaded_mask & BIT_ULL(vf_idx)) {
4134 oct->sriov_info.vf_drv_loaded_mask &= ~BIT_ULL(vf_idx);
4135 dev_info(&oct->pci_dev->dev,
4136 "driver for VF%d was removed\n", vf_idx);
bb54be58
FM		 4137 if (!cores_crashed)
4138 module_put(THIS_MODULE);
86dea55b
RV		 4139 }
4140 } else if (notice == VF_DRV_MACADDR_CHANGED) {
4141 u8 *b = (u8 *)&data[1];
4142
4143 oct->sriov_info.vf_macaddr[vf_idx] = data[1];
4144 dev_info(&oct->pci_dev->dev,
4145 "VF driver changed VF%d's MAC address to %pM\n",
4146 vf_idx, b + 2);
4147 }
4148
4149 for (i = 0; i < recv_pkt->buffer_count; i++)
4150 recv_buffer_free(recv_pkt->buffer_ptr[i]);
4151 octeon_free_recv_info(recv_info);
4152
4153 return 0;
4154}
4155
f21fb3ed
RV		 4156/**
4157 * \brief Device initialization for each Octeon device that is probed
4158 * @param octeon_dev octeon device
4159 */
4160static int octeon_device_init(struct octeon_device *octeon_dev)
4161{
4162 int j, ret;
d3d7e6c6 4163 char bootcmd[] = "\n";
da1542b0 4164 char *dbg_enb = NULL;
088b8749 4165 enum lio_fw_state fw_state;
f21fb3ed
RV		 4166 struct octeon_device_priv *oct_priv =
4167 (struct octeon_device_priv *)octeon_dev->priv;
4168 atomic_set(&octeon_dev->status, OCT_DEV_BEGIN_STATE);
4169
4170 /* Enable access to the octeon device and make its DMA capability
4171 * known to the OS.
4172 */
4173 if (octeon_pci_os_setup(octeon_dev))
4174 return 1;
4175
515e752d
RV		 4176 atomic_set(&octeon_dev->status, OCT_DEV_PCI_ENABLE_DONE);
4177
f21fb3ed
RV		 4178 /* Identify the Octeon type and map the BAR address space. */
4179 if (octeon_chip_specific_setup(octeon_dev)) {
4180 dev_err(&octeon_dev->pci_dev->dev, "Chip specific setup failed\n");
4181 return 1;
4182 }
4183
4184 atomic_set(&octeon_dev->status, OCT_DEV_PCI_MAP_DONE);
4185
e1e3ce62
RF		 4186 /* Only add a reference after setting status 'OCT_DEV_PCI_MAP_DONE',
4187 * since that is what is required for the reference to be removed
4188 * during de-initialization (see 'octeon_destroy_resources').
4189 */
4190 octeon_register_device(octeon_dev, octeon_dev->pci_dev->bus->number,
4191 PCI_SLOT(octeon_dev->pci_dev->devfn),
4192 PCI_FUNC(octeon_dev->pci_dev->devfn),
4193 true);
4194
f21fb3ed
RV		 4195 octeon_dev->app_mode = CVM_DRV_INVALID_APP;
4196
088b8749
RF		 4197 /* CN23XX supports preloaded firmware if the following is true:
4198 *
4199 * The adapter indicates that firmware is currently running AND
4200 * 'fw_type' is 'auto'.
4201 *
4202 * (default state is NEEDS_TO_BE_LOADED, override it if appropriate).
4203 */
4204 if (OCTEON_CN23XX_PF(octeon_dev) &&
4205 cn23xx_fw_loaded(octeon_dev) && fw_type_is_auto()) {
4206 atomic_cmpxchg(octeon_dev->adapter_fw_state,
4207 FW_NEEDS_TO_BE_LOADED, FW_IS_PRELOADED);
c0eab5b3 4208 }
f21fb3ed 4209
088b8749
RF		 4210 /* If loading firmware, only first device of adapter needs to do so. */
4211 fw_state = atomic_cmpxchg(octeon_dev->adapter_fw_state,
4212 FW_NEEDS_TO_BE_LOADED,
4213 FW_IS_BEING_LOADED);
4214
4215 /* Here, [local variable] 'fw_state' is set to one of:
4216 *
4217 * FW_IS_PRELOADED: No firmware is to be loaded (see above)
4218 * FW_NEEDS_TO_BE_LOADED: The driver's first instance will load
4219 * firmware to the adapter.
4220 * FW_IS_BEING_LOADED: The driver's second instance will not load
4221 * firmware to the adapter.
4222 */
4223
4224 /* Prior to f/w load, perform a soft reset of the Octeon device;
4225 * if error resetting, return w/error.
4226 */
4227 if (fw_state == FW_NEEDS_TO_BE_LOADED)
4228 if (octeon_dev->fn_list.soft_reset(octeon_dev))
4229 return 1;
4230
f21fb3ed
RV		 4231 /* Initialize the dispatch mechanism used to push packets arriving on
4232 * Octeon Output queues.
4233 */
4234 if (octeon_init_dispatch_list(octeon_dev))
4235 return 1;
4236
4237 octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
4238 OPCODE_NIC_CORE_DRV_ACTIVE,
4239 octeon_core_drv_init,
4240 octeon_dev);
4241
86dea55b
RV		 4242 octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
4243 OPCODE_NIC_VF_DRV_NOTICE,
4244 octeon_recv_vf_drv_notice, octeon_dev);
f21fb3ed
RV		 4245 INIT_DELAYED_WORK(&octeon_dev->nic_poll_work.work, nic_starter);
4246 octeon_dev->nic_poll_work.ctxptr = (void *)octeon_dev;
4247 schedule_delayed_work(&octeon_dev->nic_poll_work.work,
4248 LIQUIDIO_STARTER_POLL_INTERVAL_MS);
4249
4250 atomic_set(&octeon_dev->status, OCT_DEV_DISPATCH_INIT_DONE);
4251
c865cdf1
RV		 4252 if (octeon_set_io_queues_off(octeon_dev)) {
4253 dev_err(&octeon_dev->pci_dev->dev, "setting io queues off failed\n");
4254 return 1;
4255 }
f21fb3ed 4256
3451b97c
RV		 4257 if (OCTEON_CN23XX_PF(octeon_dev)) {
4258 ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
4259 if (ret) {
4260 dev_err(&octeon_dev->pci_dev->dev, "OCTEON: Failed to configure device registers\n");
4261 return ret;
4262 }
4263 }
4264
4265 /* Initialize soft command buffer pool
4266 */
4267 if (octeon_setup_sc_buffer_pool(octeon_dev)) {
4268 dev_err(&octeon_dev->pci_dev->dev, "sc buffer pool allocation failed\n");
4269 return 1;
4270 }
4271 atomic_set(&octeon_dev->status, OCT_DEV_SC_BUFF_POOL_INIT_DONE);
4272
f21fb3ed
RV		 4273 /* Setup the data structures that manage this Octeon's Input queues. */
4274 if (octeon_setup_instr_queues(octeon_dev)) {
4275 dev_err(&octeon_dev->pci_dev->dev,
4276 "instruction queue initialization failed\n");
f21fb3ed
RV		 4277 return 1;
4278 }
4279 atomic_set(&octeon_dev->status, OCT_DEV_INSTR_QUEUE_INIT_DONE);
4280
f21fb3ed
RV		 4281 /* Initialize lists to manage the requests of different types that
4282 * arrive from user & kernel applications for this octeon device.
4283 */
4284 if (octeon_setup_response_list(octeon_dev)) {
4285 dev_err(&octeon_dev->pci_dev->dev, "Response list allocation failed\n");
4286 return 1;
4287 }
4288 atomic_set(&octeon_dev->status, OCT_DEV_RESP_LIST_INIT_DONE);
4289
4290 if (octeon_setup_output_queues(octeon_dev)) {
4291 dev_err(&octeon_dev->pci_dev->dev, "Output queue initialization failed\n");
1e0d30fe 4292 return 1;
f21fb3ed
RV		 4293 }
4294
4295 atomic_set(&octeon_dev->status, OCT_DEV_DROQ_INIT_DONE);
4296
5b07aee1 4297 if (OCTEON_CN23XX_PF(octeon_dev)) {
5d65556b
RV		 4298 if (octeon_dev->fn_list.setup_mbox(octeon_dev)) {
4299 dev_err(&octeon_dev->pci_dev->dev, "OCTEON: Mailbox setup failed\n");
4300 return 1;
4301 }
4302 atomic_set(&octeon_dev->status, OCT_DEV_MBOX_SETUP_DONE);
4303
5b07aee1
RV		 4304 if (octeon_allocate_ioq_vector(octeon_dev)) {
4305 dev_err(&octeon_dev->pci_dev->dev, "OCTEON: ioq vector allocation failed\n");
4306 return 1;
4307 }
515e752d 4308 atomic_set(&octeon_dev->status, OCT_DEV_MSIX_ALLOC_VECTOR_DONE);
5b07aee1
RV		 4309
4310 } else {
4311 /* The input and output queue registers were setup earlier (the
4312 * queues were not enabled). Any additional registers
4313 * that need to be programmed should be done now.
4314 */
4315 ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
4316 if (ret) {
4317 dev_err(&octeon_dev->pci_dev->dev,
4318 "Failed to configure device registers\n");
4319 return ret;
4320 }
f21fb3ed
RV		 4321 }
4322
4323 /* Initialize the tasklet that handles output queue packet processing.*/
4324 dev_dbg(&octeon_dev->pci_dev->dev, "Initializing droq tasklet\n");
4325 tasklet_init(&oct_priv->droq_tasklet, octeon_droq_bh,
4326 (unsigned long)octeon_dev);
4327
4328 /* Setup the interrupt handler and record the INT SUM register address
4329 */
a82457f1
IB		 4330 if (octeon_setup_interrupt(octeon_dev,
4331 octeon_dev->sriov_info.num_pf_rings))
1e0d30fe 4332 return 1;
f21fb3ed
RV		 4333
4334 /* Enable Octeon device interrupts */
5b07aee1 4335 octeon_dev->fn_list.enable_interrupt(octeon_dev, OCTEON_ALL_INTR);
f21fb3ed 4336
515e752d
RV		 4337 atomic_set(&octeon_dev->status, OCT_DEV_INTR_SET_DONE);
4338
3c57f615
RF		 4339 /* Send Credit for Octeon Output queues. Credits are always sent BEFORE
4340 * the output queue is enabled.
4341 * This ensures that we'll receive the f/w CORE DRV_ACTIVE message in
4342 * case we've configured CN23XX_SLI_GBL_CONTROL[NOPTR_D] = 0.
4343 * Otherwise, it is possible that the DRV_ACTIVE message will be sent
4344 * before any credits have been issued, causing the ring to be reset
4345 * (and the f/w appear to never have started).
4346 */
4347 for (j = 0; j < octeon_dev->num_oqs; j++)
4348 writel(octeon_dev->droq[j]->max_count,
4349 octeon_dev->droq[j]->pkts_credit_reg);
4350
f21fb3ed 4351 /* Enable the input and output queues for this Octeon device */
1b7c55c4
RV		 4352 ret = octeon_dev->fn_list.enable_io_queues(octeon_dev);
4353 if (ret) {
4354 dev_err(&octeon_dev->pci_dev->dev, "Failed to enable input/output queues");
4355 return ret;
4356 }
f21fb3ed
RV		 4357
4358 atomic_set(&octeon_dev->status, OCT_DEV_IO_QUEUES_DONE);
4359
088b8749 4360 if (fw_state == FW_NEEDS_TO_BE_LOADED) {
c0eab5b3
RV		 4361 dev_dbg(&octeon_dev->pci_dev->dev, "Waiting for DDR initialization...\n");
4362 if (!ddr_timeout) {
4363 dev_info(&octeon_dev->pci_dev->dev,
4364 "WAITING. Set ddr_timeout to non-zero value to proceed with initialization.\n");
4365 }
f21fb3ed 4366
c0eab5b3 4367 schedule_timeout_uninterruptible(HZ * LIO_RESET_SECS);
f21fb3ed 4368
c0eab5b3
RV		 4369 /* Wait for the octeon to initialize DDR after the soft-reset.*/
4370 while (!ddr_timeout) {
4371 set_current_state(TASK_INTERRUPTIBLE);
4372 if (schedule_timeout(HZ / 10)) {
4373 /* user probably pressed Control-C */
4374 return 1;
4375 }
4376 }
4377 ret = octeon_wait_for_ddr_init(octeon_dev, &ddr_timeout);
4378 if (ret) {
4379 dev_err(&octeon_dev->pci_dev->dev,
4380 "DDR not initialized. Please confirm that board is configured to boot from Flash, ret: %d\n",
4381 ret);
4b129ae3
RV		 4382 return 1;
4383 }
f21fb3ed 4384
c0eab5b3
RV		 4385 if (octeon_wait_for_bootloader(octeon_dev, 1000)) {
4386 dev_err(&octeon_dev->pci_dev->dev, "Board not responding\n");
4387 return 1;
4388 }
f21fb3ed 4389
c0eab5b3
RV		 4390 /* Divert uboot to take commands from host instead. */
4391 ret = octeon_console_send_cmd(octeon_dev, bootcmd, 50);
d3d7e6c6 4392
c0eab5b3
RV		 4393 dev_dbg(&octeon_dev->pci_dev->dev, "Initializing consoles\n");
4394 ret = octeon_init_consoles(octeon_dev);
4395 if (ret) {
4396 dev_err(&octeon_dev->pci_dev->dev, "Could not access board consoles\n");
4397 return 1;
4398 }
da1542b0
RF		 4399 /* If console debug enabled, specify empty string to use default
4400 * enablement ELSE specify NULL string for 'disabled'.
4401 */
4402 dbg_enb = octeon_console_debug_enabled(0) ? "" : NULL;
4403 ret = octeon_add_console(octeon_dev, 0, dbg_enb);
c0eab5b3
RV		 4404 if (ret) {
4405 dev_err(&octeon_dev->pci_dev->dev, "Could not access board console\n");
4406 return 1;
da1542b0
RF		 4407 } else if (octeon_console_debug_enabled(0)) {
4408 /* If console was added AND we're logging console output
4409 * then set our console print function.
4410 */
4411 octeon_dev->console[0].print = octeon_dbg_console_print;
c0eab5b3 4412 }
f21fb3ed 4413
c0eab5b3 4414 atomic_set(&octeon_dev->status, OCT_DEV_CONSOLE_INIT_DONE);
f21fb3ed 4415
c0eab5b3
RV		 4416 dev_dbg(&octeon_dev->pci_dev->dev, "Loading firmware\n");
4417 ret = load_firmware(octeon_dev);
4418 if (ret) {
4419 dev_err(&octeon_dev->pci_dev->dev, "Could not load firmware to board\n");
4420 return 1;
4421 }
088b8749
RF		 4422
4423 atomic_set(octeon_dev->adapter_fw_state, FW_HAS_BEEN_LOADED);
f21fb3ed
RV		 4424 }
4425
4426 handshake[octeon_dev->octeon_id].init_ok = 1;
4427 complete(&handshake[octeon_dev->octeon_id].init);
4428
4429 atomic_set(&octeon_dev->status, OCT_DEV_HOST_OK);
4430
f21fb3ed
RV		 4431 return 0;
4432}
4433
da1542b0
RF		 4434/**
4435 * \brief Debug console print function
4436 * @param octeon_dev octeon device
4437 * @param console_num console number
4438 * @param prefix first portion of line to display
4439 * @param suffix second portion of line to display
4440 *
4441 * The OCTEON debug console outputs entire lines (excluding '\n').
4442 * Normally, the line will be passed in the 'prefix' parameter.
4443 * However, due to buffering, it is possible for a line to be split into two
4444 * parts, in which case they will be passed as the 'prefix' parameter and
4445 * 'suffix' parameter.
4446 */
4447static int octeon_dbg_console_print(struct octeon_device *oct, u32 console_num,
4448 char *prefix, char *suffix)
4449{
4450 if (prefix && suffix)
4451 dev_info(&oct->pci_dev->dev, "%u: %s%s\n", console_num, prefix,
4452 suffix);
4453 else if (prefix)
4454 dev_info(&oct->pci_dev->dev, "%u: %s\n", console_num, prefix);
4455 else if (suffix)
4456 dev_info(&oct->pci_dev->dev, "%u: %s\n", console_num, suffix);
4457
4458 return 0;
4459}
4460
f21fb3ed
RV		 4461/**
4462 * \brief Exits the module
4463 */
4464static void __exit liquidio_exit(void)
4465{
4466 liquidio_deinit_pci();
4467
4468 pr_info("LiquidIO network module is now unloaded\n");
4469}
4470
4471module_init(liquidio_init);
4472module_exit(liquidio_exit);
Linux 6.x block layer and io_uring tree(s)