2 * Copyright (C) 2015-2017 Netronome Systems, Inc.
4 * This software is dual licensed under the GNU General License Version 2,
5 * June 1991 as shown in the file COPYING in the top-level directory of this
6 * source tree or the BSD 2-Clause License provided below. You have the
7 * option to license this software under the complete terms of either license.
9 * The BSD 2-Clause License:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * 1. Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * 2. Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36 * Netronome network device driver: Main entry point
37 * Authors: Jakub Kicinski <jakub.kicinski@netronome.com>
38 * Alejandro Lucero <alejandro.lucero@netronome.com>
39 * Jason McMullan <jason.mcmullan@netronome.com>
40 * Rolf Neugebauer <rolf.neugebauer@netronome.com>
43 #include <linux/etherdevice.h>
44 #include <linux/kernel.h>
45 #include <linux/init.h>
46 #include <linux/pci.h>
47 #include <linux/pci_regs.h>
48 #include <linux/msi.h>
49 #include <linux/random.h>
50 #include <linux/rtnetlink.h>
52 #include "nfpcore/nfp.h"
53 #include "nfpcore/nfp_cpp.h"
54 #include "nfpcore/nfp_nffw.h"
55 #include "nfpcore/nfp_nsp.h"
56 #include "nfpcore/nfp6000_pcie.h"
58 #include "nfp_net_ctrl.h"
63 #define NFP_PF_CSR_SLICE_SIZE (32 * 1024)
65 static int nfp_is_ready(struct nfp_cpp *cpp)
71 cp = nfp_hwinfo_lookup(cpp, "board.state");
75 err = kstrtol(cp, 0, &state);
83 * nfp_net_map_area() - Help function to map an area
84 * @cpp: NFP CPP handler
85 * @name: Name for the area
88 * @size: Size of the area
89 * @area: Area handle (returned).
91 * This function is primarily to simplify the code in the main probe
92 * function. To undo the effect of this functions call
93 * @nfp_cpp_area_release_free(*area);
95 * Return: Pointer to memory mapped area or ERR_PTR
97 static u8 __iomem *nfp_net_map_area(struct nfp_cpp *cpp,
98 const char *name, int isl, int target,
99 unsigned long long addr, unsigned long size,
100 struct nfp_cpp_area **area)
106 dest = NFP_CPP_ISLAND_ID(target, NFP_CPP_ACTION_RW, 0, isl);
108 *area = nfp_cpp_area_alloc_with_name(cpp, dest, name, addr, size);
114 err = nfp_cpp_area_acquire(*area);
118 res = nfp_cpp_area_iomem(*area);
127 nfp_cpp_area_release(*area);
129 nfp_cpp_area_free(*area);
131 return (u8 __iomem *)ERR_PTR(err);
135 * nfp_net_get_mac_addr() - Get the MAC address.
136 * @nn: NFP Network structure
137 * @cpp: NFP CPP handle
140 * First try to get the MAC address from NSP ETH table. If that
141 * fails try HWInfo. As a last resort generate a random address.
144 nfp_net_get_mac_addr(struct nfp_net *nn, struct nfp_cpp *cpp, unsigned int id)
146 struct nfp_eth_table_port *eth_port;
147 struct nfp_net_dp *dp = &nn->dp;
148 u8 mac_addr[ETH_ALEN];
152 eth_port = __nfp_port_get_eth_port(nn->port);
154 ether_addr_copy(dp->netdev->dev_addr, eth_port->mac_addr);
155 ether_addr_copy(dp->netdev->perm_addr, eth_port->mac_addr);
159 snprintf(name, sizeof(name), "eth%d.mac", id);
161 mac_str = nfp_hwinfo_lookup(cpp, name);
163 dev_warn(dp->dev, "Can't lookup MAC address. Generate\n");
164 eth_hw_addr_random(dp->netdev);
168 if (sscanf(mac_str, "%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx",
169 &mac_addr[0], &mac_addr[1], &mac_addr[2],
170 &mac_addr[3], &mac_addr[4], &mac_addr[5]) != 6) {
172 "Can't parse MAC address (%s). Generate.\n", mac_str);
173 eth_hw_addr_random(dp->netdev);
177 ether_addr_copy(dp->netdev->dev_addr, mac_addr);
178 ether_addr_copy(dp->netdev->perm_addr, mac_addr);
181 static struct nfp_eth_table_port *
182 nfp_net_find_port(struct nfp_eth_table *eth_tbl, unsigned int id)
186 for (i = 0; eth_tbl && i < eth_tbl->count; i++)
187 if (eth_tbl->ports[i].eth_index == id)
188 return ð_tbl->ports[i];
193 static unsigned int nfp_net_pf_get_num_ports(struct nfp_pf *pf)
199 snprintf(name, sizeof(name), "nfd_cfg_pf%u_num_ports",
200 nfp_cppcore_pcie_unit(pf->cpp));
202 val = nfp_rtsym_read_le(pf->cpp, name, &err);
203 /* Default to one port/vNIC */
206 nfp_err(pf->cpp, "Unable to read adapter vNIC count\n");
214 nfp_net_pf_total_qcs(struct nfp_pf *pf, void __iomem *ctrl_bar,
215 unsigned int stride, u32 start_off, u32 num_off)
217 unsigned int i, min_qc, max_qc;
219 min_qc = readl(ctrl_bar + start_off);
222 for (i = 0; i < pf->max_data_vnics; i++) {
223 /* To make our lives simpler only accept configuration where
224 * queues are allocated to PFs in order (queues of PFn all have
225 * indexes lower than PFn+1).
227 if (max_qc > readl(ctrl_bar + start_off))
230 max_qc = readl(ctrl_bar + start_off);
231 max_qc += readl(ctrl_bar + num_off) * stride;
232 ctrl_bar += NFP_PF_CSR_SLICE_SIZE;
235 return max_qc - min_qc;
238 static u8 __iomem *nfp_net_pf_map_ctrl_bar(struct nfp_pf *pf)
240 const struct nfp_rtsym *ctrl_sym;
241 u8 __iomem *ctrl_bar;
244 snprintf(pf_symbol, sizeof(pf_symbol), "_pf%u_net_bar0",
245 nfp_cppcore_pcie_unit(pf->cpp));
247 ctrl_sym = nfp_rtsym_lookup(pf->cpp, pf_symbol);
249 dev_err(&pf->pdev->dev,
250 "Failed to find PF BAR0 symbol %s\n", pf_symbol);
254 if (ctrl_sym->size < pf->max_data_vnics * NFP_PF_CSR_SLICE_SIZE) {
255 dev_err(&pf->pdev->dev,
256 "PF BAR0 too small to contain %d vNICs\n",
261 ctrl_bar = nfp_net_map_area(pf->cpp, "net.ctrl",
262 ctrl_sym->domain, ctrl_sym->target,
263 ctrl_sym->addr, ctrl_sym->size,
265 if (IS_ERR(ctrl_bar)) {
266 dev_err(&pf->pdev->dev, "Failed to map PF BAR0: %ld\n",
274 static void nfp_net_pf_free_vnic(struct nfp_pf *pf, struct nfp_net *nn)
276 nfp_port_free(nn->port);
277 list_del(&nn->vnic_list);
282 static void nfp_net_pf_free_vnics(struct nfp_pf *pf)
286 while (!list_empty(&pf->vnics)) {
287 nn = list_first_entry(&pf->vnics, struct nfp_net, vnic_list);
288 nfp_net_pf_free_vnic(pf, nn);
292 static struct nfp_net *
293 nfp_net_pf_alloc_vnic(struct nfp_pf *pf, void __iomem *ctrl_bar,
294 void __iomem *tx_bar, void __iomem *rx_bar,
295 int stride, struct nfp_net_fw_version *fw_ver,
298 struct nfp_eth_table_port *eth_port;
299 u32 n_tx_rings, n_rx_rings;
302 n_tx_rings = readl(ctrl_bar + NFP_NET_CFG_MAX_TXRINGS);
303 n_rx_rings = readl(ctrl_bar + NFP_NET_CFG_MAX_RXRINGS);
305 /* Allocate and initialise the vNIC */
306 nn = nfp_net_alloc(pf->pdev, n_tx_rings, n_rx_rings);
311 nn->fw_ver = *fw_ver;
312 nn->dp.ctrl_bar = ctrl_bar;
316 nn->stride_rx = stride;
317 nn->stride_tx = stride;
319 eth_port = nfp_net_find_port(pf->eth_tbl, eth_id);
321 nn->port = nfp_port_alloc(pf->app, NFP_PORT_PHYS_PORT,
323 if (IS_ERR(nn->port)) {
325 return ERR_CAST(nn->port);
327 nn->port->eth_id = eth_id;
328 nn->port->eth_port = eth_port;
332 list_add_tail(&nn->vnic_list, &pf->vnics);
338 nfp_net_pf_init_vnic(struct nfp_pf *pf, struct nfp_net *nn, unsigned int id)
342 /* Get MAC address */
343 nfp_net_get_mac_addr(nn, pf->cpp, id);
345 /* Get ME clock frequency from ctrl BAR
346 * XXX for now frequency is hardcoded until we figure out how
347 * to get the value from nfp-hwinfo into ctrl bar
349 nn->me_freq_mhz = 1200;
351 err = nfp_net_init(nn);
355 nfp_net_debugfs_vnic_add(nn, pf->ddir, id);
363 nfp_net_pf_alloc_vnics(struct nfp_pf *pf, void __iomem *ctrl_bar,
364 void __iomem *tx_bar, void __iomem *rx_bar,
365 int stride, struct nfp_net_fw_version *fw_ver)
367 u32 prev_tx_base, prev_rx_base, tgt_tx_base, tgt_rx_base;
372 if (pf->eth_tbl && pf->max_data_vnics != pf->eth_tbl->count) {
373 nfp_err(pf->cpp, "ETH entries don't match vNICs (%d vs %d)\n",
374 pf->max_data_vnics, pf->eth_tbl->count);
378 prev_tx_base = readl(ctrl_bar + NFP_NET_CFG_START_TXQ);
379 prev_rx_base = readl(ctrl_bar + NFP_NET_CFG_START_RXQ);
381 for (i = 0; i < pf->max_data_vnics; i++) {
382 tgt_tx_base = readl(ctrl_bar + NFP_NET_CFG_START_TXQ);
383 tgt_rx_base = readl(ctrl_bar + NFP_NET_CFG_START_RXQ);
384 tx_bar += (tgt_tx_base - prev_tx_base) * NFP_QCP_QUEUE_ADDR_SZ;
385 rx_bar += (tgt_rx_base - prev_rx_base) * NFP_QCP_QUEUE_ADDR_SZ;
386 prev_tx_base = tgt_tx_base;
387 prev_rx_base = tgt_rx_base;
389 nn = nfp_net_pf_alloc_vnic(pf, ctrl_bar, tx_bar, rx_bar,
396 ctrl_bar += NFP_PF_CSR_SLICE_SIZE;
398 /* Check if vNIC has external port associated and cfg is OK */
399 if (pf->eth_tbl && !nn->port) {
400 nfp_err(pf->cpp, "NSP port entries don't match vNICs (no entry for port #%d)\n", i);
404 if (nn->port && nn->port->eth_port->override_changed) {
405 nfp_warn(pf->cpp, "Config changed for port #%d, reboot required before port will be operational\n", i);
406 nfp_net_pf_free_vnic(pf, nn);
411 if (list_empty(&pf->vnics))
417 nfp_net_pf_free_vnics(pf);
422 nfp_net_pf_spawn_vnics(struct nfp_pf *pf,
423 void __iomem *ctrl_bar, void __iomem *tx_bar,
424 void __iomem *rx_bar, int stride,
425 struct nfp_net_fw_version *fw_ver)
427 unsigned int id, wanted_irqs, num_irqs, vnics_left, irqs_left;
431 /* Allocate the vnics and do basic init */
432 err = nfp_net_pf_alloc_vnics(pf, ctrl_bar, tx_bar, rx_bar,
437 /* Get MSI-X vectors */
439 list_for_each_entry(nn, &pf->vnics, vnic_list)
440 wanted_irqs += NFP_NET_NON_Q_VECTORS + nn->dp.num_r_vecs;
441 pf->irq_entries = kcalloc(wanted_irqs, sizeof(*pf->irq_entries),
443 if (!pf->irq_entries) {
448 num_irqs = nfp_net_irqs_alloc(pf->pdev, pf->irq_entries,
449 NFP_NET_MIN_VNIC_IRQS * pf->num_vnics,
452 nn_warn(nn, "Unable to allocate MSI-X Vectors. Exiting\n");
457 /* Distribute IRQs to vNICs */
458 irqs_left = num_irqs;
459 vnics_left = pf->num_vnics;
460 list_for_each_entry(nn, &pf->vnics, vnic_list) {
463 n = DIV_ROUND_UP(irqs_left, vnics_left);
464 nfp_net_irqs_assign(nn, &pf->irq_entries[num_irqs - irqs_left],
470 /* Finish vNIC init and register */
472 list_for_each_entry(nn, &pf->vnics, vnic_list) {
473 err = nfp_net_pf_init_vnic(pf, nn, id);
475 goto err_prev_deinit;
483 list_for_each_entry_continue_reverse(nn, &pf->vnics, vnic_list) {
484 nfp_net_debugfs_dir_clean(&nn->debugfs_dir);
487 nfp_net_irqs_disable(pf->pdev);
489 kfree(pf->irq_entries);
491 nfp_net_pf_free_vnics(pf);
495 static int nfp_net_pf_app_init(struct nfp_pf *pf)
497 pf->app = nfp_app_alloc(pf);
499 return PTR_ERR_OR_ZERO(pf->app);
502 static void nfp_net_pf_app_clean(struct nfp_pf *pf)
504 nfp_app_free(pf->app);
508 static void nfp_net_pci_remove_finish(struct nfp_pf *pf)
510 nfp_net_debugfs_dir_clean(&pf->ddir);
512 nfp_net_irqs_disable(pf->pdev);
513 kfree(pf->irq_entries);
515 nfp_net_pf_app_clean(pf);
517 nfp_cpp_area_release_free(pf->rx_area);
518 nfp_cpp_area_release_free(pf->tx_area);
519 nfp_cpp_area_release_free(pf->data_vnic_bar);
523 nfp_net_eth_port_update(struct nfp_cpp *cpp, struct nfp_port *port,
524 struct nfp_eth_table *eth_table)
526 struct nfp_eth_table_port *eth_port;
530 eth_port = nfp_net_find_port(eth_table, port->eth_id);
532 set_bit(NFP_PORT_CHANGED, &port->flags);
533 nfp_warn(cpp, "Warning: port #%d not present after reconfig\n",
537 if (eth_port->override_changed) {
538 nfp_warn(cpp, "Port #%d config changed, unregistering. Reboot required before port will be operational again.\n", port->eth_id);
539 port->type = NFP_PORT_INVALID;
542 memcpy(port->eth_port, eth_port, sizeof(*eth_port));
547 static void nfp_net_refresh_vnics(struct work_struct *work)
549 struct nfp_pf *pf = container_of(work, struct nfp_pf,
551 struct nfp_eth_table *eth_table;
552 struct nfp_net *nn, *next;
553 struct nfp_port *port;
555 mutex_lock(&pf->lock);
557 /* Check for nfp_net_pci_remove() racing against us */
558 if (list_empty(&pf->vnics))
561 /* Update state of all ports */
563 list_for_each_entry(port, &pf->ports, port_list)
564 clear_bit(NFP_PORT_CHANGED, &port->flags);
566 eth_table = nfp_eth_read_ports(pf->cpp);
568 list_for_each_entry(port, &pf->ports, port_list)
569 if (__nfp_port_get_eth_port(port))
570 set_bit(NFP_PORT_CHANGED, &port->flags);
572 nfp_err(pf->cpp, "Error refreshing port config!\n");
576 list_for_each_entry(port, &pf->ports, port_list)
577 if (__nfp_port_get_eth_port(port))
578 nfp_net_eth_port_update(pf->cpp, port, eth_table);
583 /* Shoot off the ports which became invalid */
584 list_for_each_entry_safe(nn, next, &pf->vnics, vnic_list) {
585 if (!nn->port || nn->port->type != NFP_PORT_INVALID)
588 nfp_net_debugfs_dir_clean(&nn->debugfs_dir);
591 nfp_net_pf_free_vnic(pf, nn);
594 if (list_empty(&pf->vnics))
595 nfp_net_pci_remove_finish(pf);
597 mutex_unlock(&pf->lock);
600 void nfp_net_refresh_port_table(struct nfp_port *port)
602 struct nfp_pf *pf = port->app->pf;
604 set_bit(NFP_PORT_CHANGED, &port->flags);
606 schedule_work(&pf->port_refresh_work);
609 int nfp_net_refresh_eth_port(struct nfp_port *port)
611 struct nfp_cpp *cpp = port->app->cpp;
612 struct nfp_eth_table *eth_table;
615 clear_bit(NFP_PORT_CHANGED, &port->flags);
617 eth_table = nfp_eth_read_ports(cpp);
619 set_bit(NFP_PORT_CHANGED, &port->flags);
620 nfp_err(cpp, "Error refreshing port state table!\n");
624 ret = nfp_net_eth_port_update(cpp, port, eth_table);
632 * PCI device functions
634 int nfp_net_pci_probe(struct nfp_pf *pf)
636 u8 __iomem *ctrl_bar, *tx_bar, *rx_bar;
637 u32 total_tx_qcs, total_rx_qcs;
638 struct nfp_net_fw_version fw_ver;
639 u32 tx_area_sz, rx_area_sz;
644 INIT_WORK(&pf->port_refresh_work, nfp_net_refresh_vnics);
646 /* Verify that the board has completed initialization */
647 if (!nfp_is_ready(pf->cpp)) {
648 nfp_err(pf->cpp, "NFP is not ready for NIC operation.\n");
652 mutex_lock(&pf->lock);
653 pf->max_data_vnics = nfp_net_pf_get_num_ports(pf);
655 ctrl_bar = nfp_net_pf_map_ctrl_bar(pf);
657 err = pf->fw_loaded ? -EINVAL : -EPROBE_DEFER;
661 nfp_net_get_fw_version(&fw_ver, ctrl_bar);
662 if (fw_ver.resv || fw_ver.class != NFP_NET_CFG_VERSION_CLASS_GENERIC) {
663 nfp_err(pf->cpp, "Unknown Firmware ABI %d.%d.%d.%d\n",
664 fw_ver.resv, fw_ver.class, fw_ver.major, fw_ver.minor);
669 /* Determine stride */
670 if (nfp_net_fw_ver_eq(&fw_ver, 0, 0, 0, 1)) {
672 nfp_warn(pf->cpp, "OBSOLETE Firmware detected - VF isolation not available\n");
674 switch (fw_ver.major) {
679 nfp_err(pf->cpp, "Unsupported Firmware ABI %d.%d.%d.%d\n",
680 fw_ver.resv, fw_ver.class,
681 fw_ver.major, fw_ver.minor);
687 /* Find how many QC structs need to be mapped */
688 total_tx_qcs = nfp_net_pf_total_qcs(pf, ctrl_bar, stride,
689 NFP_NET_CFG_START_TXQ,
690 NFP_NET_CFG_MAX_TXRINGS);
691 total_rx_qcs = nfp_net_pf_total_qcs(pf, ctrl_bar, stride,
692 NFP_NET_CFG_START_RXQ,
693 NFP_NET_CFG_MAX_RXRINGS);
694 if (!total_tx_qcs || !total_rx_qcs) {
695 nfp_err(pf->cpp, "Invalid PF QC configuration [%d,%d]\n",
696 total_tx_qcs, total_rx_qcs);
701 tx_area_sz = NFP_QCP_QUEUE_ADDR_SZ * total_tx_qcs;
702 rx_area_sz = NFP_QCP_QUEUE_ADDR_SZ * total_rx_qcs;
705 start_q = readl(ctrl_bar + NFP_NET_CFG_START_TXQ);
706 tx_bar = nfp_net_map_area(pf->cpp, "net.tx", 0, 0,
707 NFP_PCIE_QUEUE(start_q),
708 tx_area_sz, &pf->tx_area);
709 if (IS_ERR(tx_bar)) {
710 nfp_err(pf->cpp, "Failed to map TX area.\n");
711 err = PTR_ERR(tx_bar);
716 start_q = readl(ctrl_bar + NFP_NET_CFG_START_RXQ);
717 rx_bar = nfp_net_map_area(pf->cpp, "net.rx", 0, 0,
718 NFP_PCIE_QUEUE(start_q),
719 rx_area_sz, &pf->rx_area);
720 if (IS_ERR(rx_bar)) {
721 nfp_err(pf->cpp, "Failed to map RX area.\n");
722 err = PTR_ERR(rx_bar);
726 err = nfp_net_pf_app_init(pf);
730 pf->ddir = nfp_net_debugfs_device_add(pf->pdev);
732 err = nfp_net_pf_spawn_vnics(pf, ctrl_bar, tx_bar, rx_bar,
737 mutex_unlock(&pf->lock);
742 nfp_net_debugfs_dir_clean(&pf->ddir);
743 nfp_net_pf_app_clean(pf);
745 nfp_cpp_area_release_free(pf->rx_area);
747 nfp_cpp_area_release_free(pf->tx_area);
749 nfp_cpp_area_release_free(pf->data_vnic_bar);
751 mutex_unlock(&pf->lock);
755 void nfp_net_pci_remove(struct nfp_pf *pf)
759 mutex_lock(&pf->lock);
760 if (list_empty(&pf->vnics))
763 list_for_each_entry(nn, &pf->vnics, vnic_list) {
764 nfp_net_debugfs_dir_clean(&nn->debugfs_dir);
769 nfp_net_pf_free_vnics(pf);
771 nfp_net_pci_remove_finish(pf);
773 mutex_unlock(&pf->lock);
775 cancel_work_sync(&pf->port_refresh_work);