-* Marvell Armada 375 Ethernet Controller (PPv2)
+* Marvell Armada 375 Ethernet Controller (PPv2.1)
+ Marvell Armada 7K/8K Ethernet Controller (PPv2.2)
Required properties:
-- compatible: should be "marvell,armada-375-pp2"
+- compatible: should be one of:
+ "marvell,armada-375-pp2"
+ "marvell,armada-7k-pp2"
- reg: addresses and length of the register sets for the device.
- Must contain the following register sets:
+ For "marvell,armada-375-pp2", must contain the following register
+ sets:
- common controller registers
- LMS registers
- In addition, at least one port register set is required.
-- clocks: a pointer to the reference clocks for this device, consequently:
- - main controller clock
- - GOP clock
-- clock-names: names of used clocks, must be "pp_clk" and "gop_clk".
+ - one register area per Ethernet port
+ For "marvell,armada-7k-pp2", must contain the following register
+ sets:
+ - packet processor registers
+ - networking interfaces registers
+
+- clocks: pointers to the reference clocks for this device, consequently:
+ - main controller clock (for both armada-375-pp2 and armada-7k-pp2)
+ - GOP clock (for both armada-375-pp2 and armada-7k-pp2)
+ - MG clock (only for armada-7k-pp2)
+- clock-names: names of used clocks, must be "pp_clk", "gop_clk" and
+ "mg_clk" (the latter only for armada-7k-pp2).
The ethernet ports are represented by subnodes. At least one port is
required.
Required properties (port):
- interrupts: interrupt for the port
-- port-id: should be '0' or '1' for ethernet ports, and '2' for the
- loopback port
+- port-id: ID of the port from the MAC point of view
+- gop-port-id: only for marvell,armada-7k-pp2, ID of the port from the
+ GOP (Group Of Ports) point of view. This ID is used to index the
+ per-port registers in the second register area.
- phy-mode: See ethernet.txt file in the same directory
Optional properties (port):
- phy: a phandle to a phy node defining the PHY address (as the reg
property, a single integer).
-Example:
+Example for marvell,armada-375-pp2:
ethernet@f0000 {
compatible = "marvell,armada-375-pp2";
phy-mode = "gmii";
};
};
+
+Example for marvell,armada-7k-pp2:
+
+cpm_ethernet: ethernet@0 {
+ compatible = "marvell,armada-7k-pp22";
+ reg = <0x0 0x100000>, <0x129000 0xb000>;
+ clocks = <&cpm_syscon0 1 3>, <&cpm_syscon0 1 9>, <&cpm_syscon0 1 5>;
+ clock-names = "pp_clk", "gop_clk", "gp_clk";
+
+ eth0: eth0 {
+ interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>;
+ port-id = <0>;
+ gop-port-id = <0>;
+ };
+
+ eth1: eth1 {
+ interrupts = <GIC_SPI 38 IRQ_TYPE_LEVEL_HIGH>;
+ port-id = <1>;
+ gop-port-id = <2>;
+ };
+
+ eth2: eth2 {
+ interrupts = <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>;
+ port-id = <2>;
+ gop-port-id = <3>;
+ };
+};
F: Documentation/devicetree/bindings/net/apm-xgene-enet.txt
F: Documentation/devicetree/bindings/net/apm-xgene-mdio.txt
+APPLIED MICRO (APM) X-GENE SOC ETHERNET (V2) DRIVER
+M: Iyappan Subramanian <isubramanian@apm.com>
+M: Keyur Chudgar <kchudgar@apm.com>
+S: Supported
+F: drivers/net/ethernet/apm/xgene-v2/
+
APPLIED MICRO (APM) X-GENE SOC PMU
M: Tai Nguyen <ttnguyen@apm.com>
S: Supported
F: include/linux/platform_data/dma-dw.h
F: drivers/dma/dw/
+SYNOPSYS DESIGNWARE ENTERPRISE ETHERNET DRIVER
+M: Jie Deng <jiedeng@synopsys.com>
+L: netdev@vger.kernel.org
+S: Supported
+F: drivers/net/ethernet/synopsys/
+
SYNOPSYS DESIGNWARE I2C DRIVER
M: Jarkko Nikula <jarkko.nikula@linux.intel.com>
R: Andy Shevchenko <andriy.shevchenko@linux.intel.com>
port->sm_rx_state = AD_RX_INITIALIZE;
port->sm_vars |= AD_PORT_CHURNED;
/* check if port is not enabled */
- } else if (!(port->sm_vars & AD_PORT_BEGIN)
- && !port->is_enabled && !(port->sm_vars & AD_PORT_MOVED))
+ } else if (!(port->sm_vars & AD_PORT_BEGIN) && !port->is_enabled)
port->sm_rx_state = AD_RX_PORT_DISABLED;
/* check if new lacpdu arrived */
else if (lacpdu && ((port->sm_rx_state == AD_RX_EXPIRED) ||
/* if no lacpdu arrived and no timer is on */
switch (port->sm_rx_state) {
case AD_RX_PORT_DISABLED:
- if (port->sm_vars & AD_PORT_MOVED)
- port->sm_rx_state = AD_RX_INITIALIZE;
- else if (port->is_enabled
- && (port->sm_vars
- & AD_PORT_LACP_ENABLED))
+ if (port->is_enabled &&
+ (port->sm_vars & AD_PORT_LACP_ENABLED))
port->sm_rx_state = AD_RX_EXPIRED;
else if (port->is_enabled
&& ((port->sm_vars
port->sm_vars &= ~AD_PORT_SELECTED;
__record_default(port);
port->actor_oper_port_state &= ~AD_STATE_EXPIRED;
- port->sm_vars &= ~AD_PORT_MOVED;
port->sm_rx_state = AD_RX_PORT_DISABLED;
/* Fall Through */
unsigned int bond_net_id __read_mostly;
-static __be32 arp_target[BOND_MAX_ARP_TARGETS];
-static int arp_ip_count;
-static int bond_mode = BOND_MODE_ROUNDROBIN;
-static int xmit_hashtype = BOND_XMIT_POLICY_LAYER2;
-static int lacp_fast;
-
/*-------------------------- Forward declarations ---------------------------*/
static int bond_init(struct net_device *bond_dev);
* arp is transmitted to generate traffic. see activebackup_arp_monitor for
* arp monitoring in active backup mode.
*/
-static void bond_loadbalance_arp_mon(struct work_struct *work)
+static void bond_loadbalance_arp_mon(struct bonding *bond)
{
- struct bonding *bond = container_of(work, struct bonding,
- arp_work.work);
struct slave *slave, *oldcurrent;
struct list_head *iter;
int do_failover = 0, slave_state_changed = 0;
return should_notify_rtnl;
}
-static void bond_activebackup_arp_mon(struct work_struct *work)
+static void bond_activebackup_arp_mon(struct bonding *bond)
{
- struct bonding *bond = container_of(work, struct bonding,
- arp_work.work);
bool should_notify_peers = false;
bool should_notify_rtnl = false;
int delta_in_ticks;
}
}
+static void bond_arp_monitor(struct work_struct *work)
+{
+ struct bonding *bond = container_of(work, struct bonding,
+ arp_work.work);
+
+ if (BOND_MODE(bond) == BOND_MODE_ACTIVEBACKUP)
+ bond_activebackup_arp_mon(bond);
+ else
+ bond_loadbalance_arp_mon(bond);
+}
+
/*-------------------------- netdev event handling --------------------------*/
/* Change device name */
bond_resend_igmp_join_requests_delayed);
INIT_DELAYED_WORK(&bond->alb_work, bond_alb_monitor);
INIT_DELAYED_WORK(&bond->mii_work, bond_mii_monitor);
- if (BOND_MODE(bond) == BOND_MODE_ACTIVEBACKUP)
- INIT_DELAYED_WORK(&bond->arp_work, bond_activebackup_arp_mon);
- else
- INIT_DELAYED_WORK(&bond->arp_work, bond_loadbalance_arp_mon);
+ INIT_DELAYED_WORK(&bond->arp_work, bond_arp_monitor);
INIT_DELAYED_WORK(&bond->ad_work, bond_3ad_state_machine_handler);
INIT_DELAYED_WORK(&bond->slave_arr_work, bond_slave_arr_handler);
}
}
}
- bond_work_init_all(bond);
-
if (bond_is_lb(bond)) {
/* bond_alb_initialize must be called before the timer
* is started.
int arp_all_targets_value;
u16 ad_actor_sys_prio = 0;
u16 ad_user_port_key = 0;
+ __be32 arp_target[BOND_MAX_ARP_TARGETS];
+ int arp_ip_count;
+ int bond_mode = BOND_MODE_ROUNDROBIN;
+ int xmit_hashtype = BOND_XMIT_POLICY_LAYER2;
+ int lacp_fast = 0;
+ int tlb_dynamic_lb = 0;
/* Convert string parameters. */
if (mode) {
}
ad_user_port_key = valptr->value;
+ if (bond_mode == BOND_MODE_TLB) {
+ bond_opt_initstr(&newval, "default");
+ valptr = bond_opt_parse(bond_opt_get(BOND_OPT_TLB_DYNAMIC_LB),
+ &newval);
+ if (!valptr) {
+ pr_err("Error: No tlb_dynamic_lb default value");
+ return -EINVAL;
+ }
+ tlb_dynamic_lb = valptr->value;
+ }
+
if (lp_interval == 0) {
pr_warn("Warning: ip_interval must be between 1 and %d, so it was reset to %d\n",
INT_MAX, BOND_ALB_DEFAULT_LP_INTERVAL);
params->min_links = min_links;
params->lp_interval = lp_interval;
params->packets_per_slave = packets_per_slave;
- params->tlb_dynamic_lb = 1; /* Default value */
+ params->tlb_dynamic_lb = tlb_dynamic_lb;
params->ad_actor_sys_prio = ad_actor_sys_prio;
eth_zero_addr(params->ad_actor_system);
params->ad_user_port_key = ad_user_port_key;
netif_carrier_off(bond_dev);
+ bond_work_init_all(bond);
+
rtnl_unlock();
if (res < 0)
bond_destructor(bond_dev);
source "drivers/net/ethernet/wiznet/Kconfig"
source "drivers/net/ethernet/xilinx/Kconfig"
source "drivers/net/ethernet/xircom/Kconfig"
+source "drivers/net/ethernet/synopsys/Kconfig"
endif # ETHERNET
obj-$(CONFIG_NET_VENDOR_WIZNET) += wiznet/
obj-$(CONFIG_NET_VENDOR_XILINX) += xilinx/
obj-$(CONFIG_NET_VENDOR_XIRCOM) += xircom/
+obj-$(CONFIG_NET_VENDOR_SYNOPSYS) += synopsys/
source "drivers/net/ethernet/apm/xgene/Kconfig"
+source "drivers/net/ethernet/apm/xgene-v2/Kconfig"
#
obj-$(CONFIG_NET_XGENE) += xgene/
+obj-$(CONFIG_NET_XGENE_V2) += xgene-v2/
--- /dev/null
+config NET_XGENE_V2
+ tristate "APM X-Gene SoC Ethernet-v2 Driver"
+ depends on HAS_DMA
+ depends on ARCH_XGENE || COMPILE_TEST
+ help
+ This is the Ethernet driver for the on-chip ethernet interface
+ which uses a linked list of DMA descriptor architecture (v2) for
+ APM X-Gene SoCs.
+
+ To compile this driver as a module, choose M here. This module will
+ be called xgene-enet-v2.
--- /dev/null
+#
+# Makefile for APM X-Gene Ethernet v2 driver
+#
+
+xgene-enet-v2-objs := main.o mac.o enet.o ring.o
+obj-$(CONFIG_NET_XGENE_V2) += xgene-enet-v2.o
--- /dev/null
+/*
+ * Applied Micro X-Gene SoC Ethernet v2 Driver
+ *
+ * Copyright (c) 2017, Applied Micro Circuits Corporation
+ * Author(s): Iyappan Subramanian <isubramanian@apm.com>
+ * Keyur Chudgar <kchudgar@apm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "main.h"
+
+void xge_wr_csr(struct xge_pdata *pdata, u32 offset, u32 val)
+{
+ void __iomem *addr = pdata->resources.base_addr + offset;
+
+ iowrite32(val, addr);
+}
+
+u32 xge_rd_csr(struct xge_pdata *pdata, u32 offset)
+{
+ void __iomem *addr = pdata->resources.base_addr + offset;
+
+ return ioread32(addr);
+}
+
+int xge_port_reset(struct net_device *ndev)
+{
+ struct xge_pdata *pdata = netdev_priv(ndev);
+
+ xge_wr_csr(pdata, ENET_SRST, 0x3);
+ xge_wr_csr(pdata, ENET_SRST, 0x2);
+ xge_wr_csr(pdata, ENET_SRST, 0x0);
+
+ xge_wr_csr(pdata, ENET_SHIM, DEVM_ARAUX_COH | DEVM_AWAUX_COH);
+
+ return 0;
+}
+
+static void xge_traffic_resume(struct net_device *ndev)
+{
+ struct xge_pdata *pdata = netdev_priv(ndev);
+
+ xge_wr_csr(pdata, CFG_FORCE_LINK_STATUS_EN, 1);
+ xge_wr_csr(pdata, FORCE_LINK_STATUS, 1);
+
+ xge_wr_csr(pdata, CFG_LINK_AGGR_RESUME, 1);
+ xge_wr_csr(pdata, RX_DV_GATE_REG, 1);
+}
+
+int xge_port_init(struct net_device *ndev)
+{
+ struct xge_pdata *pdata = netdev_priv(ndev);
+
+ pdata->phy_speed = SPEED_1000;
+ xge_mac_init(pdata);
+ xge_traffic_resume(ndev);
+
+ return 0;
+}
--- /dev/null
+/*
+ * Applied Micro X-Gene SoC Ethernet v2 Driver
+ *
+ * Copyright (c) 2017, Applied Micro Circuits Corporation
+ * Author(s): Iyappan Subramanian <isubramanian@apm.com>
+ * Keyur Chudgar <kchudgar@apm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __XGENE_ENET_V2_ENET_H__
+#define __XGENE_ENET_V2_ENET_H__
+
+#define ENET_CLKEN 0xc008
+#define ENET_SRST 0xc000
+#define ENET_SHIM 0xc010
+#define CFG_MEM_RAM_SHUTDOWN 0xd070
+#define BLOCK_MEM_RDY 0xd074
+
+#define DEVM_ARAUX_COH BIT(19)
+#define DEVM_AWAUX_COH BIT(3)
+
+#define CFG_FORCE_LINK_STATUS_EN 0x229c
+#define FORCE_LINK_STATUS 0x22a0
+#define CFG_LINK_AGGR_RESUME 0x27c8
+#define RX_DV_GATE_REG 0x2dfc
+
+void xge_wr_csr(struct xge_pdata *pdata, u32 offset, u32 val);
+u32 xge_rd_csr(struct xge_pdata *pdata, u32 offset);
+int xge_port_reset(struct net_device *ndev);
+
+#endif /* __XGENE_ENET_V2_ENET__H__ */
--- /dev/null
+/*
+ * Applied Micro X-Gene SoC Ethernet v2 Driver
+ *
+ * Copyright (c) 2017, Applied Micro Circuits Corporation
+ * Author(s): Iyappan Subramanian <isubramanian@apm.com>
+ * Keyur Chudgar <kchudgar@apm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "main.h"
+
+void xge_mac_reset(struct xge_pdata *pdata)
+{
+ xge_wr_csr(pdata, MAC_CONFIG_1, SOFT_RESET);
+ xge_wr_csr(pdata, MAC_CONFIG_1, 0);
+}
+
+static void xge_mac_set_speed(struct xge_pdata *pdata)
+{
+ u32 icm0, icm2, ecm0, mc2;
+ u32 intf_ctrl, rgmii;
+
+ icm0 = xge_rd_csr(pdata, ICM_CONFIG0_REG_0);
+ icm2 = xge_rd_csr(pdata, ICM_CONFIG2_REG_0);
+ ecm0 = xge_rd_csr(pdata, ECM_CONFIG0_REG_0);
+ rgmii = xge_rd_csr(pdata, RGMII_REG_0);
+ mc2 = xge_rd_csr(pdata, MAC_CONFIG_2);
+ intf_ctrl = xge_rd_csr(pdata, INTERFACE_CONTROL);
+ icm2 |= CFG_WAITASYNCRD_EN;
+
+ switch (pdata->phy_speed) {
+ case SPEED_10:
+ SET_REG_BITS(&mc2, INTF_MODE, 1);
+ SET_REG_BITS(&intf_ctrl, HD_MODE, 0);
+ SET_REG_BITS(&icm0, CFG_MACMODE, 0);
+ SET_REG_BITS(&icm2, CFG_WAITASYNCRD, 500);
+ SET_REG_BIT(&rgmii, CFG_SPEED_125, 0);
+ break;
+ case SPEED_100:
+ SET_REG_BITS(&mc2, INTF_MODE, 1);
+ SET_REG_BITS(&intf_ctrl, HD_MODE, 1);
+ SET_REG_BITS(&icm0, CFG_MACMODE, 1);
+ SET_REG_BITS(&icm2, CFG_WAITASYNCRD, 80);
+ SET_REG_BIT(&rgmii, CFG_SPEED_125, 0);
+ break;
+ default:
+ SET_REG_BITS(&mc2, INTF_MODE, 2);
+ SET_REG_BITS(&intf_ctrl, HD_MODE, 2);
+ SET_REG_BITS(&icm0, CFG_MACMODE, 2);
+ SET_REG_BITS(&icm2, CFG_WAITASYNCRD, 16);
+ SET_REG_BIT(&rgmii, CFG_SPEED_125, 1);
+ break;
+ }
+
+ mc2 |= FULL_DUPLEX | CRC_EN | PAD_CRC;
+ SET_REG_BITS(&ecm0, CFG_WFIFOFULLTHR, 0x32);
+
+ xge_wr_csr(pdata, MAC_CONFIG_2, mc2);
+ xge_wr_csr(pdata, INTERFACE_CONTROL, intf_ctrl);
+ xge_wr_csr(pdata, RGMII_REG_0, rgmii);
+ xge_wr_csr(pdata, ICM_CONFIG0_REG_0, icm0);
+ xge_wr_csr(pdata, ICM_CONFIG2_REG_0, icm2);
+ xge_wr_csr(pdata, ECM_CONFIG0_REG_0, ecm0);
+}
+
+void xge_mac_set_station_addr(struct xge_pdata *pdata)
+{
+ u8 *dev_addr = pdata->ndev->dev_addr;
+ u32 addr0, addr1;
+
+ addr0 = (dev_addr[3] << 24) | (dev_addr[2] << 16) |
+ (dev_addr[1] << 8) | dev_addr[0];
+ addr1 = (dev_addr[5] << 24) | (dev_addr[4] << 16);
+
+ xge_wr_csr(pdata, STATION_ADDR0, addr0);
+ xge_wr_csr(pdata, STATION_ADDR1, addr1);
+}
+
+void xge_mac_init(struct xge_pdata *pdata)
+{
+ xge_mac_reset(pdata);
+ xge_mac_set_speed(pdata);
+ xge_mac_set_station_addr(pdata);
+}
+
+void xge_mac_enable(struct xge_pdata *pdata)
+{
+ u32 data;
+
+ data = xge_rd_csr(pdata, MAC_CONFIG_1);
+ data |= TX_EN | RX_EN;
+ xge_wr_csr(pdata, MAC_CONFIG_1, data);
+
+ data = xge_rd_csr(pdata, MAC_CONFIG_1);
+}
+
+void xge_mac_disable(struct xge_pdata *pdata)
+{
+ u32 data;
+
+ data = xge_rd_csr(pdata, MAC_CONFIG_1);
+ data &= ~(TX_EN | RX_EN);
+ xge_wr_csr(pdata, MAC_CONFIG_1, data);
+}
--- /dev/null
+/*
+ * Applied Micro X-Gene SoC Ethernet v2 Driver
+ *
+ * Copyright (c) 2017, Applied Micro Circuits Corporation
+ * Author(s): Iyappan Subramanian <isubramanian@apm.com>
+ * Keyur Chudgar <kchudgar@apm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __XGENE_ENET_V2_MAC_H__
+#define __XGENE_ENET_V2_MAC_H__
+
+/* Register offsets */
+#define MAC_CONFIG_1 0xa000
+#define MAC_CONFIG_2 0xa004
+#define MII_MGMT_CONFIG 0xa020
+#define MII_MGMT_COMMAND 0xa024
+#define MII_MGMT_ADDRESS 0xa028
+#define MII_MGMT_CONTROL 0xa02c
+#define MII_MGMT_STATUS 0xa030
+#define MII_MGMT_INDICATORS 0xa034
+#define INTERFACE_CONTROL 0xa038
+#define STATION_ADDR0 0xa040
+#define STATION_ADDR1 0xa044
+#define RBYT 0xa09c
+#define RPKT 0xa0a0
+#define RFCS 0xa0a4
+
+#define RGMII_REG_0 0x27e0
+#define ICM_CONFIG0_REG_0 0x2c00
+#define ICM_CONFIG2_REG_0 0x2c08
+#define ECM_CONFIG0_REG_0 0x2d00
+
+/* Register fields */
+#define SOFT_RESET BIT(31)
+#define TX_EN BIT(0)
+#define RX_EN BIT(2)
+#define PAD_CRC BIT(2)
+#define CRC_EN BIT(1)
+#define FULL_DUPLEX BIT(0)
+
+#define INTF_MODE_POS 8
+#define INTF_MODE_LEN 2
+#define HD_MODE_POS 25
+#define HD_MODE_LEN 2
+#define CFG_MACMODE_POS 18
+#define CFG_MACMODE_LEN 2
+#define CFG_WAITASYNCRD_POS 0
+#define CFG_WAITASYNCRD_LEN 16
+#define CFG_SPEED_125_POS 24
+#define CFG_WFIFOFULLTHR_POS 0
+#define CFG_WFIFOFULLTHR_LEN 7
+#define MGMT_CLOCK_SEL_POS 0
+#define MGMT_CLOCK_SEL_LEN 3
+#define PHY_ADDR_POS 8
+#define PHY_ADDR_LEN 5
+#define REG_ADDR_POS 0
+#define REG_ADDR_LEN 5
+#define MII_MGMT_BUSY BIT(0)
+#define MII_READ_CYCLE BIT(0)
+#define CFG_WAITASYNCRD_EN BIT(16)
+
+static inline void xgene_set_reg_bits(u32 *var, int pos, int len, u32 val)
+{
+ u32 mask = GENMASK(pos + len, pos);
+
+ *var &= ~mask;
+ *var |= ((val << pos) & mask);
+}
+
+static inline u32 xgene_get_reg_bits(u32 var, int pos, int len)
+{
+ u32 mask = GENMASK(pos + len, pos);
+
+ return (var & mask) >> pos;
+}
+
+#define SET_REG_BITS(var, field, val) \
+ xgene_set_reg_bits(var, field ## _POS, field ## _LEN, val)
+
+#define SET_REG_BIT(var, field, val) \
+ xgene_set_reg_bits(var, field ## _POS, 1, val)
+
+#define GET_REG_BITS(var, field) \
+ xgene_get_reg_bits(var, field ## _POS, field ## _LEN)
+
+#define GET_REG_BIT(var, field) ((var) & (field))
+
+struct xge_pdata;
+
+void xge_mac_reset(struct xge_pdata *pdata);
+void xge_mac_enable(struct xge_pdata *pdata);
+void xge_mac_disable(struct xge_pdata *pdata);
+void xge_mac_init(struct xge_pdata *pdata);
+int xge_port_init(struct net_device *ndev);
+void xge_mac_set_station_addr(struct xge_pdata *pdata);
+
+#endif /* __XGENE_ENET_V2_MAC_H__ */
--- /dev/null
+/*
+ * Applied Micro X-Gene SoC Ethernet v2 Driver
+ *
+ * Copyright (c) 2017, Applied Micro Circuits Corporation
+ * Author(s): Iyappan Subramanian <isubramanian@apm.com>
+ * Keyur Chudgar <kchudgar@apm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "main.h"
+
+static const struct acpi_device_id xge_acpi_match[];
+
+static int xge_get_resources(struct xge_pdata *pdata)
+{
+ struct platform_device *pdev;
+ struct net_device *ndev;
+ int phy_mode, ret = 0;
+ struct resource *res;
+ struct device *dev;
+
+ pdev = pdata->pdev;
+ dev = &pdev->dev;
+ ndev = pdata->ndev;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!res) {
+ dev_err(dev, "Resource enet_csr not defined\n");
+ return -ENODEV;
+ }
+
+ pdata->resources.base_addr = devm_ioremap(dev, res->start,
+ resource_size(res));
+ if (!pdata->resources.base_addr) {
+ dev_err(dev, "Unable to retrieve ENET Port CSR region\n");
+ return -ENOMEM;
+ }
+
+ if (!device_get_mac_address(dev, ndev->dev_addr, ETH_ALEN))
+ eth_hw_addr_random(ndev);
+
+ memcpy(ndev->perm_addr, ndev->dev_addr, ndev->addr_len);
+
+ phy_mode = device_get_phy_mode(dev);
+ if (phy_mode < 0) {
+ dev_err(dev, "Unable to get phy-connection-type\n");
+ return phy_mode;
+ }
+ pdata->resources.phy_mode = phy_mode;
+
+ if (pdata->resources.phy_mode != PHY_INTERFACE_MODE_RGMII) {
+ dev_err(dev, "Incorrect phy-connection-type specified\n");
+ return -ENODEV;
+ }
+
+ ret = platform_get_irq(pdev, 0);
+ if (ret <= 0) {
+ dev_err(dev, "Unable to get ENET IRQ\n");
+ ret = ret ? : -ENXIO;
+ return ret;
+ }
+ pdata->resources.irq = ret;
+
+ return 0;
+}
+
+static int xge_refill_buffers(struct net_device *ndev, u32 nbuf)
+{
+ struct xge_pdata *pdata = netdev_priv(ndev);
+ struct xge_desc_ring *ring = pdata->rx_ring;
+ const u8 slots = XGENE_ENET_NUM_DESC - 1;
+ struct device *dev = &pdata->pdev->dev;
+ struct xge_raw_desc *raw_desc;
+ u64 addr_lo, addr_hi;
+ u8 tail = ring->tail;
+ struct sk_buff *skb;
+ dma_addr_t dma_addr;
+ u16 len;
+ int i;
+
+ for (i = 0; i < nbuf; i++) {
+ raw_desc = &ring->raw_desc[tail];
+
+ len = XGENE_ENET_STD_MTU;
+ skb = netdev_alloc_skb(ndev, len);
+ if (unlikely(!skb))
+ return -ENOMEM;
+
+ dma_addr = dma_map_single(dev, skb->data, len, DMA_FROM_DEVICE);
+ if (dma_mapping_error(dev, dma_addr)) {
+ netdev_err(ndev, "DMA mapping error\n");
+ dev_kfree_skb_any(skb);
+ return -EINVAL;
+ }
+
+ ring->pkt_info[tail].skb = skb;
+ ring->pkt_info[tail].dma_addr = dma_addr;
+
+ addr_hi = GET_BITS(NEXT_DESC_ADDRH, le64_to_cpu(raw_desc->m1));
+ addr_lo = GET_BITS(NEXT_DESC_ADDRL, le64_to_cpu(raw_desc->m1));
+ raw_desc->m1 = cpu_to_le64(SET_BITS(NEXT_DESC_ADDRL, addr_lo) |
+ SET_BITS(NEXT_DESC_ADDRH, addr_hi) |
+ SET_BITS(PKT_ADDRH,
+ upper_32_bits(dma_addr)));
+
+ dma_wmb();
+ raw_desc->m0 = cpu_to_le64(SET_BITS(PKT_ADDRL, dma_addr) |
+ SET_BITS(E, 1));
+ tail = (tail + 1) & slots;
+ }
+
+ ring->tail = tail;
+
+ return 0;
+}
+
+static int xge_init_hw(struct net_device *ndev)
+{
+ struct xge_pdata *pdata = netdev_priv(ndev);
+ int ret;
+
+ ret = xge_port_reset(ndev);
+ if (ret)
+ return ret;
+
+ xge_port_init(ndev);
+ pdata->nbufs = NUM_BUFS;
+
+ return 0;
+}
+
+static irqreturn_t xge_irq(const int irq, void *data)
+{
+ struct xge_pdata *pdata = data;
+
+ if (napi_schedule_prep(&pdata->napi)) {
+ xge_intr_disable(pdata);
+ __napi_schedule(&pdata->napi);
+ }
+
+ return IRQ_HANDLED;
+}
+
+static int xge_request_irq(struct net_device *ndev)
+{
+ struct xge_pdata *pdata = netdev_priv(ndev);
+ struct device *dev = &pdata->pdev->dev;
+ int ret;
+
+ snprintf(pdata->irq_name, IRQ_ID_SIZE, "%s", ndev->name);
+
+ ret = devm_request_irq(dev, pdata->resources.irq, xge_irq,
+ 0, pdata->irq_name, pdata);
+ if (ret)
+ netdev_err(ndev, "Failed to request irq %s\n", pdata->irq_name);
+
+ return ret;
+}
+
+static void xge_free_irq(struct net_device *ndev)
+{
+ struct xge_pdata *pdata = netdev_priv(ndev);
+ struct device *dev = &pdata->pdev->dev;
+
+ devm_free_irq(dev, pdata->resources.irq, pdata);
+}
+
+static bool is_tx_slot_available(struct xge_raw_desc *raw_desc)
+{
+ if (GET_BITS(E, le64_to_cpu(raw_desc->m0)) &&
+ (GET_BITS(PKT_SIZE, le64_to_cpu(raw_desc->m0)) == SLOT_EMPTY))
+ return true;
+
+ return false;
+}
+
+static netdev_tx_t xge_start_xmit(struct sk_buff *skb, struct net_device *ndev)
+{
+ struct xge_pdata *pdata = netdev_priv(ndev);
+ struct device *dev = &pdata->pdev->dev;
+ struct xge_desc_ring *tx_ring;
+ struct xge_raw_desc *raw_desc;
+ static dma_addr_t dma_addr;
+ u64 addr_lo, addr_hi;
+ void *pkt_buf;
+ u8 tail;
+ u16 len;
+
+ tx_ring = pdata->tx_ring;
+ tail = tx_ring->tail;
+ len = skb_headlen(skb);
+ raw_desc = &tx_ring->raw_desc[tail];
+
+ if (!is_tx_slot_available(raw_desc)) {
+ netif_stop_queue(ndev);
+ return NETDEV_TX_BUSY;
+ }
+
+ /* Packet buffers should be 64B aligned */
+ pkt_buf = dma_zalloc_coherent(dev, XGENE_ENET_STD_MTU, &dma_addr,
+ GFP_ATOMIC);
+ if (unlikely(!pkt_buf)) {
+ dev_kfree_skb_any(skb);
+ return NETDEV_TX_OK;
+ }
+ memcpy(pkt_buf, skb->data, len);
+
+ addr_hi = GET_BITS(NEXT_DESC_ADDRH, le64_to_cpu(raw_desc->m1));
+ addr_lo = GET_BITS(NEXT_DESC_ADDRL, le64_to_cpu(raw_desc->m1));
+ raw_desc->m1 = cpu_to_le64(SET_BITS(NEXT_DESC_ADDRL, addr_lo) |
+ SET_BITS(NEXT_DESC_ADDRH, addr_hi) |
+ SET_BITS(PKT_ADDRH,
+ upper_32_bits(dma_addr)));
+
+ tx_ring->pkt_info[tail].skb = skb;
+ tx_ring->pkt_info[tail].dma_addr = dma_addr;
+ tx_ring->pkt_info[tail].pkt_buf = pkt_buf;
+
+ dma_wmb();
+
+ raw_desc->m0 = cpu_to_le64(SET_BITS(PKT_ADDRL, dma_addr) |
+ SET_BITS(PKT_SIZE, len) |
+ SET_BITS(E, 0));
+ skb_tx_timestamp(skb);
+ xge_wr_csr(pdata, DMATXCTRL, 1);
+
+ tx_ring->tail = (tail + 1) & (XGENE_ENET_NUM_DESC - 1);
+
+ return NETDEV_TX_OK;
+}
+
+static bool is_tx_hw_done(struct xge_raw_desc *raw_desc)
+{
+ if (GET_BITS(E, le64_to_cpu(raw_desc->m0)) &&
+ !GET_BITS(PKT_SIZE, le64_to_cpu(raw_desc->m0)))
+ return true;
+
+ return false;
+}
+
+static void xge_txc_poll(struct net_device *ndev)
+{
+ struct xge_pdata *pdata = netdev_priv(ndev);
+ struct device *dev = &pdata->pdev->dev;
+ struct xge_desc_ring *tx_ring;
+ struct xge_raw_desc *raw_desc;
+ dma_addr_t dma_addr;
+ struct sk_buff *skb;
+ void *pkt_buf;
+ u32 data;
+ u8 head;
+
+ tx_ring = pdata->tx_ring;
+ head = tx_ring->head;
+
+ data = xge_rd_csr(pdata, DMATXSTATUS);
+ if (!GET_BITS(TXPKTCOUNT, data))
+ return;
+
+ while (1) {
+ raw_desc = &tx_ring->raw_desc[head];
+
+ if (!is_tx_hw_done(raw_desc))
+ break;
+
+ dma_rmb();
+
+ skb = tx_ring->pkt_info[head].skb;
+ dma_addr = tx_ring->pkt_info[head].dma_addr;
+ pkt_buf = tx_ring->pkt_info[head].pkt_buf;
+ pdata->stats.tx_packets++;
+ pdata->stats.tx_bytes += skb->len;
+ dma_free_coherent(dev, XGENE_ENET_STD_MTU, pkt_buf, dma_addr);
+ dev_kfree_skb_any(skb);
+
+ /* clear pktstart address and pktsize */
+ raw_desc->m0 = cpu_to_le64(SET_BITS(E, 1) |
+ SET_BITS(PKT_SIZE, SLOT_EMPTY));
+ xge_wr_csr(pdata, DMATXSTATUS, 1);
+
+ head = (head + 1) & (XGENE_ENET_NUM_DESC - 1);
+ }
+
+ if (netif_queue_stopped(ndev))
+ netif_wake_queue(ndev);
+
+ tx_ring->head = head;
+}
+
+static int xge_rx_poll(struct net_device *ndev, unsigned int budget)
+{
+ struct xge_pdata *pdata = netdev_priv(ndev);
+ struct device *dev = &pdata->pdev->dev;
+ struct xge_desc_ring *rx_ring;
+ struct xge_raw_desc *raw_desc;
+ struct sk_buff *skb;
+ dma_addr_t dma_addr;
+ int processed = 0;
+ u8 head, rx_error;
+ int i, ret;
+ u32 data;
+ u16 len;
+
+ rx_ring = pdata->rx_ring;
+ head = rx_ring->head;
+
+ data = xge_rd_csr(pdata, DMARXSTATUS);
+ if (!GET_BITS(RXPKTCOUNT, data))
+ return 0;
+
+ for (i = 0; i < budget; i++) {
+ raw_desc = &rx_ring->raw_desc[head];
+
+ if (GET_BITS(E, le64_to_cpu(raw_desc->m0)))
+ break;
+
+ dma_rmb();
+
+ skb = rx_ring->pkt_info[head].skb;
+ rx_ring->pkt_info[head].skb = NULL;
+ dma_addr = rx_ring->pkt_info[head].dma_addr;
+ len = GET_BITS(PKT_SIZE, le64_to_cpu(raw_desc->m0));
+ dma_unmap_single(dev, dma_addr, XGENE_ENET_STD_MTU,
+ DMA_FROM_DEVICE);
+
+ rx_error = GET_BITS(D, le64_to_cpu(raw_desc->m2));
+ if (unlikely(rx_error)) {
+ pdata->stats.rx_errors++;
+ dev_kfree_skb_any(skb);
+ goto out;
+ }
+
+ skb_put(skb, len);
+ skb->protocol = eth_type_trans(skb, ndev);
+
+ pdata->stats.rx_packets++;
+ pdata->stats.rx_bytes += len;
+ napi_gro_receive(&pdata->napi, skb);
+out:
+ ret = xge_refill_buffers(ndev, 1);
+ xge_wr_csr(pdata, DMARXSTATUS, 1);
+ xge_wr_csr(pdata, DMARXCTRL, 1);
+
+ if (ret)
+ break;
+
+ head = (head + 1) & (XGENE_ENET_NUM_DESC - 1);
+ processed++;
+ }
+
+ rx_ring->head = head;
+
+ return processed;
+}
+
+static void xge_delete_desc_ring(struct net_device *ndev,
+ struct xge_desc_ring *ring)
+{
+ struct xge_pdata *pdata = netdev_priv(ndev);
+ struct device *dev = &pdata->pdev->dev;
+ u16 size;
+
+ if (!ring)
+ return;
+
+ size = XGENE_ENET_DESC_SIZE * XGENE_ENET_NUM_DESC;
+ if (ring->desc_addr)
+ dma_free_coherent(dev, size, ring->desc_addr, ring->dma_addr);
+
+ kfree(ring->pkt_info);
+ kfree(ring);
+}
+
+static void xge_free_buffers(struct net_device *ndev)
+{
+ struct xge_pdata *pdata = netdev_priv(ndev);
+ struct xge_desc_ring *ring = pdata->rx_ring;
+ struct device *dev = &pdata->pdev->dev;
+ struct sk_buff *skb;
+ dma_addr_t dma_addr;
+ int i;
+
+ for (i = 0; i < XGENE_ENET_NUM_DESC; i++) {
+ skb = ring->pkt_info[i].skb;
+ dma_addr = ring->pkt_info[i].dma_addr;
+
+ if (!skb)
+ continue;
+
+ dma_unmap_single(dev, dma_addr, XGENE_ENET_STD_MTU,
+ DMA_FROM_DEVICE);
+ dev_kfree_skb_any(skb);
+ }
+}
+
+static void xge_delete_desc_rings(struct net_device *ndev)
+{
+ struct xge_pdata *pdata = netdev_priv(ndev);
+
+ xge_txc_poll(ndev);
+ xge_delete_desc_ring(ndev, pdata->tx_ring);
+
+ xge_rx_poll(ndev, 64);
+ xge_free_buffers(ndev);
+ xge_delete_desc_ring(ndev, pdata->rx_ring);
+}
+
+static struct xge_desc_ring *xge_create_desc_ring(struct net_device *ndev)
+{
+ struct xge_pdata *pdata = netdev_priv(ndev);
+ struct device *dev = &pdata->pdev->dev;
+ struct xge_desc_ring *ring;
+ u16 size;
+
+ ring = kzalloc(sizeof(struct xge_desc_ring), GFP_KERNEL);
+ if (!ring)
+ return NULL;
+
+ ring->ndev = ndev;
+
+ size = XGENE_ENET_DESC_SIZE * XGENE_ENET_NUM_DESC;
+ ring->desc_addr = dma_zalloc_coherent(dev, size, &ring->dma_addr,
+ GFP_KERNEL);
+ if (!ring->desc_addr)
+ goto err;
+
+ ring->pkt_info = kcalloc(XGENE_ENET_NUM_DESC, sizeof(struct pkt_info),
+ GFP_KERNEL);
+ if (!ring->pkt_info)
+ goto err;
+
+ xge_setup_desc(ring);
+
+ return ring;
+
+err:
+ xge_delete_desc_ring(ndev, ring);
+
+ return NULL;
+}
+
+static int xge_create_desc_rings(struct net_device *ndev)
+{
+ struct xge_pdata *pdata = netdev_priv(ndev);
+ struct xge_desc_ring *ring;
+ int ret;
+
+ /* create tx ring */
+ ring = xge_create_desc_ring(ndev);
+ if (!ring)
+ goto err;
+
+ pdata->tx_ring = ring;
+ xge_update_tx_desc_addr(pdata);
+
+ /* create rx ring */
+ ring = xge_create_desc_ring(ndev);
+ if (!ring)
+ goto err;
+
+ pdata->rx_ring = ring;
+ xge_update_rx_desc_addr(pdata);
+
+ ret = xge_refill_buffers(ndev, XGENE_ENET_NUM_DESC);
+ if (ret)
+ goto err;
+
+ return 0;
+err:
+ xge_delete_desc_rings(ndev);
+
+ return -ENOMEM;
+}
+
+static int xge_open(struct net_device *ndev)
+{
+ struct xge_pdata *pdata = netdev_priv(ndev);
+ int ret;
+
+ ret = xge_create_desc_rings(ndev);
+ if (ret)
+ return ret;
+
+ napi_enable(&pdata->napi);
+ ret = xge_request_irq(ndev);
+ if (ret)
+ return ret;
+
+ xge_intr_enable(pdata);
+ xge_wr_csr(pdata, DMARXCTRL, 1);
+ xge_mac_enable(pdata);
+ netif_start_queue(ndev);
+ netif_carrier_on(ndev);
+
+ return 0;
+}
+
+static int xge_close(struct net_device *ndev)
+{
+ struct xge_pdata *pdata = netdev_priv(ndev);
+
+ netif_carrier_off(ndev);
+ netif_stop_queue(ndev);
+ xge_mac_disable(pdata);
+
+ xge_intr_disable(pdata);
+ xge_free_irq(ndev);
+ napi_disable(&pdata->napi);
+ xge_delete_desc_rings(ndev);
+
+ return 0;
+}
+
+static int xge_napi(struct napi_struct *napi, const int budget)
+{
+ struct net_device *ndev = napi->dev;
+ struct xge_pdata *pdata;
+ int processed;
+
+ pdata = netdev_priv(ndev);
+
+ xge_txc_poll(ndev);
+ processed = xge_rx_poll(ndev, budget);
+
+ if (processed < budget) {
+ napi_complete_done(napi, processed);
+ xge_intr_enable(pdata);
+ }
+
+ return processed;
+}
+
+static int xge_set_mac_addr(struct net_device *ndev, void *addr)
+{
+ struct xge_pdata *pdata = netdev_priv(ndev);
+ int ret;
+
+ ret = eth_mac_addr(ndev, addr);
+ if (ret)
+ return ret;
+
+ xge_mac_set_station_addr(pdata);
+
+ return 0;
+}
+
+static bool is_tx_pending(struct xge_raw_desc *raw_desc)
+{
+ if (!GET_BITS(E, le64_to_cpu(raw_desc->m0)))
+ return true;
+
+ return false;
+}
+
+static void xge_free_pending_skb(struct net_device *ndev)
+{
+ struct xge_pdata *pdata = netdev_priv(ndev);
+ struct device *dev = &pdata->pdev->dev;
+ struct xge_desc_ring *tx_ring;
+ struct xge_raw_desc *raw_desc;
+ dma_addr_t dma_addr;
+ struct sk_buff *skb;
+ void *pkt_buf;
+ int i;
+
+ tx_ring = pdata->tx_ring;
+
+ for (i = 0; i < XGENE_ENET_NUM_DESC; i++) {
+ raw_desc = &tx_ring->raw_desc[i];
+
+ if (!is_tx_pending(raw_desc))
+ continue;
+
+ skb = tx_ring->pkt_info[i].skb;
+ dma_addr = tx_ring->pkt_info[i].dma_addr;
+ pkt_buf = tx_ring->pkt_info[i].pkt_buf;
+ dma_free_coherent(dev, XGENE_ENET_STD_MTU, pkt_buf, dma_addr);
+ dev_kfree_skb_any(skb);
+ }
+}
+
+static void xge_timeout(struct net_device *ndev)
+{
+ struct xge_pdata *pdata = netdev_priv(ndev);
+
+ rtnl_lock();
+
+ if (netif_running(ndev)) {
+ netif_carrier_off(ndev);
+ netif_stop_queue(ndev);
+ xge_intr_disable(pdata);
+ napi_disable(&pdata->napi);
+
+ xge_wr_csr(pdata, DMATXCTRL, 0);
+ xge_txc_poll(ndev);
+ xge_free_pending_skb(ndev);
+ xge_wr_csr(pdata, DMATXSTATUS, ~0U);
+
+ xge_setup_desc(pdata->tx_ring);
+ xge_update_tx_desc_addr(pdata);
+ xge_mac_init(pdata);
+
+ napi_enable(&pdata->napi);
+ xge_intr_enable(pdata);
+ xge_mac_enable(pdata);
+ netif_start_queue(ndev);
+ netif_carrier_on(ndev);
+ }
+
+ rtnl_unlock();
+}
+
+static void xge_get_stats64(struct net_device *ndev,
+ struct rtnl_link_stats64 *storage)
+{
+ struct xge_pdata *pdata = netdev_priv(ndev);
+ struct xge_stats *stats = &pdata->stats;
+
+ storage->tx_packets += stats->tx_packets;
+ storage->tx_bytes += stats->tx_bytes;
+
+ storage->rx_packets += stats->rx_packets;
+ storage->rx_bytes += stats->rx_bytes;
+ storage->rx_errors += stats->rx_errors;
+}
+
+static const struct net_device_ops xgene_ndev_ops = {
+ .ndo_open = xge_open,
+ .ndo_stop = xge_close,
+ .ndo_start_xmit = xge_start_xmit,
+ .ndo_set_mac_address = xge_set_mac_addr,
+ .ndo_tx_timeout = xge_timeout,
+ .ndo_get_stats64 = xge_get_stats64,
+};
+
+static int xge_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct net_device *ndev;
+ struct xge_pdata *pdata;
+ int ret;
+
+ ndev = alloc_etherdev(sizeof(struct xge_pdata));
+ if (!ndev)
+ return -ENOMEM;
+
+ pdata = netdev_priv(ndev);
+
+ pdata->pdev = pdev;
+ pdata->ndev = ndev;
+ SET_NETDEV_DEV(ndev, dev);
+ platform_set_drvdata(pdev, pdata);
+ ndev->netdev_ops = &xgene_ndev_ops;
+
+ ndev->features |= NETIF_F_GSO |
+ NETIF_F_GRO;
+
+ ret = xge_get_resources(pdata);
+ if (ret)
+ goto err;
+
+ ndev->hw_features = ndev->features;
+
+ ret = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(64));
+ if (ret) {
+ netdev_err(ndev, "No usable DMA configuration\n");
+ goto err;
+ }
+
+ ret = xge_init_hw(ndev);
+ if (ret)
+ goto err;
+
+ netif_napi_add(ndev, &pdata->napi, xge_napi, NAPI_POLL_WEIGHT);
+
+ netif_carrier_off(ndev);
+ ret = register_netdev(ndev);
+ if (ret) {
+ netdev_err(ndev, "Failed to register netdev\n");
+ goto err;
+ }
+
+ return 0;
+
+err:
+ free_netdev(ndev);
+
+ return ret;
+}
+
+static int xge_remove(struct platform_device *pdev)
+{
+ struct xge_pdata *pdata;
+ struct net_device *ndev;
+
+ pdata = platform_get_drvdata(pdev);
+ ndev = pdata->ndev;
+
+ rtnl_lock();
+ if (netif_running(ndev))
+ dev_close(ndev);
+ rtnl_unlock();
+
+ unregister_netdev(ndev);
+ free_netdev(ndev);
+
+ return 0;
+}
+
+static void xge_shutdown(struct platform_device *pdev)
+{
+ struct xge_pdata *pdata;
+
+ pdata = platform_get_drvdata(pdev);
+ if (!pdata)
+ return;
+
+ if (!pdata->ndev)
+ return;
+
+ xge_remove(pdev);
+}
+
+static const struct acpi_device_id xge_acpi_match[] = {
+ { "APMC0D80" },
+ { }
+};
+MODULE_DEVICE_TABLE(acpi, xge_acpi_match);
+
+static struct platform_driver xge_driver = {
+ .driver = {
+ .name = "xgene-enet-v2",
+ .acpi_match_table = ACPI_PTR(xge_acpi_match),
+ },
+ .probe = xge_probe,
+ .remove = xge_remove,
+ .shutdown = xge_shutdown,
+};
+module_platform_driver(xge_driver);
+
+MODULE_DESCRIPTION("APM X-Gene SoC Ethernet v2 driver");
+MODULE_AUTHOR("Iyappan Subramanian <isubramanian@apm.com>");
+MODULE_VERSION(XGENE_ENET_V2_VERSION);
+MODULE_LICENSE("GPL");
--- /dev/null
+/*
+ * Applied Micro X-Gene SoC Ethernet v2 Driver
+ *
+ * Copyright (c) 2017, Applied Micro Circuits Corporation
+ * Author(s): Iyappan Subramanian <isubramanian@apm.com>
+ * Keyur Chudgar <kchudgar@apm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __XGENE_ENET_V2_MAIN_H__
+#define __XGENE_ENET_V2_MAIN_H__
+
+#include <linux/acpi.h>
+#include <linux/clk.h>
+#include <linux/efi.h>
+#include <linux/if_vlan.h>
+#include <linux/irq.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/of_net.h>
+#include <linux/of_mdio.h>
+#include <linux/prefetch.h>
+#include <linux/phy.h>
+#include <net/ip.h>
+#include "mac.h"
+#include "enet.h"
+#include "ring.h"
+
+#define XGENE_ENET_V2_VERSION "v1.0"
+#define XGENE_ENET_STD_MTU 1536
+#define XGENE_ENET_MIN_FRAME 60
+#define IRQ_ID_SIZE 16
+
+struct xge_resource {
+ void __iomem *base_addr;
+ int phy_mode;
+ u32 irq;
+};
+
+struct xge_stats {
+ u64 tx_packets;
+ u64 tx_bytes;
+ u64 rx_packets;
+ u64 rx_bytes;
+ u64 rx_errors;
+};
+
+/* ethernet private data */
+struct xge_pdata {
+ struct xge_resource resources;
+ struct xge_desc_ring *tx_ring;
+ struct xge_desc_ring *rx_ring;
+ struct platform_device *pdev;
+ char irq_name[IRQ_ID_SIZE];
+ struct net_device *ndev;
+ struct napi_struct napi;
+ struct xge_stats stats;
+ int phy_speed;
+ u8 nbufs;
+};
+
+#endif /* __XGENE_ENET_V2_MAIN_H__ */
--- /dev/null
+/*
+ * Applied Micro X-Gene SoC Ethernet v2 Driver
+ *
+ * Copyright (c) 2017, Applied Micro Circuits Corporation
+ * Author(s): Iyappan Subramanian <isubramanian@apm.com>
+ * Keyur Chudgar <kchudgar@apm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "main.h"
+
+/* create circular linked list of descriptors */
+void xge_setup_desc(struct xge_desc_ring *ring)
+{
+ struct xge_raw_desc *raw_desc;
+ dma_addr_t dma_h, next_dma;
+ u16 offset;
+ int i;
+
+ for (i = 0; i < XGENE_ENET_NUM_DESC; i++) {
+ raw_desc = &ring->raw_desc[i];
+
+ offset = (i + 1) & (XGENE_ENET_NUM_DESC - 1);
+ next_dma = ring->dma_addr + (offset * XGENE_ENET_DESC_SIZE);
+
+ raw_desc->m0 = cpu_to_le64(SET_BITS(E, 1) |
+ SET_BITS(PKT_SIZE, SLOT_EMPTY));
+ dma_h = upper_32_bits(next_dma);
+ raw_desc->m1 = cpu_to_le64(SET_BITS(NEXT_DESC_ADDRL, next_dma) |
+ SET_BITS(NEXT_DESC_ADDRH, dma_h));
+ }
+}
+
+void xge_update_tx_desc_addr(struct xge_pdata *pdata)
+{
+ struct xge_desc_ring *ring = pdata->tx_ring;
+ dma_addr_t dma_addr = ring->dma_addr;
+
+ xge_wr_csr(pdata, DMATXDESCL, dma_addr);
+ xge_wr_csr(pdata, DMATXDESCH, upper_32_bits(dma_addr));
+
+ ring->head = 0;
+ ring->tail = 0;
+}
+
+void xge_update_rx_desc_addr(struct xge_pdata *pdata)
+{
+ struct xge_desc_ring *ring = pdata->rx_ring;
+ dma_addr_t dma_addr = ring->dma_addr;
+
+ xge_wr_csr(pdata, DMARXDESCL, dma_addr);
+ xge_wr_csr(pdata, DMARXDESCH, upper_32_bits(dma_addr));
+
+ ring->head = 0;
+ ring->tail = 0;
+}
+
+void xge_intr_enable(struct xge_pdata *pdata)
+{
+ u32 data;
+
+ data = RX_PKT_RCVD | TX_PKT_SENT;
+ xge_wr_csr(pdata, DMAINTRMASK, data);
+}
+
+void xge_intr_disable(struct xge_pdata *pdata)
+{
+ xge_wr_csr(pdata, DMAINTRMASK, 0);
+}
--- /dev/null
+/*
+ * Applied Micro X-Gene SoC Ethernet v2 Driver
+ *
+ * Copyright (c) 2017, Applied Micro Circuits Corporation
+ * Author(s): Iyappan Subramanian <isubramanian@apm.com>
+ * Keyur Chudgar <kchudgar@apm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __XGENE_ENET_V2_RING_H__
+#define __XGENE_ENET_V2_RING_H__
+
+#define XGENE_ENET_DESC_SIZE 64
+#define XGENE_ENET_NUM_DESC 256
+#define NUM_BUFS 8
+#define SLOT_EMPTY 0xfff
+
+#define DMATXCTRL 0xa180
+#define DMATXDESCL 0xa184
+#define DMATXDESCH 0xa1a0
+#define DMATXSTATUS 0xa188
+#define DMARXCTRL 0xa18c
+#define DMARXDESCL 0xa190
+#define DMARXDESCH 0xa1a4
+#define DMARXSTATUS 0xa194
+#define DMAINTRMASK 0xa198
+#define DMAINTERRUPT 0xa19c
+
+#define D_POS 62
+#define D_LEN 2
+#define E_POS 63
+#define E_LEN 1
+#define PKT_ADDRL_POS 0
+#define PKT_ADDRL_LEN 32
+#define PKT_ADDRH_POS 32
+#define PKT_ADDRH_LEN 10
+#define PKT_SIZE_POS 32
+#define PKT_SIZE_LEN 12
+#define NEXT_DESC_ADDRL_POS 0
+#define NEXT_DESC_ADDRL_LEN 32
+#define NEXT_DESC_ADDRH_POS 48
+#define NEXT_DESC_ADDRH_LEN 10
+
+#define TXPKTCOUNT_POS 16
+#define TXPKTCOUNT_LEN 8
+#define RXPKTCOUNT_POS 16
+#define RXPKTCOUNT_LEN 8
+
+#define TX_PKT_SENT BIT(0)
+#define TX_BUS_ERROR BIT(3)
+#define RX_PKT_RCVD BIT(4)
+#define RX_BUS_ERROR BIT(7)
+#define RXSTATUS_RXPKTRCVD BIT(0)
+
+struct xge_raw_desc {
+ __le64 m0;
+ __le64 m1;
+ __le64 m2;
+ __le64 m3;
+ __le64 m4;
+ __le64 m5;
+ __le64 m6;
+ __le64 m7;
+};
+
+struct pkt_info {
+ struct sk_buff *skb;
+ dma_addr_t dma_addr;
+ void *pkt_buf;
+};
+
+/* software context of a descriptor ring */
+struct xge_desc_ring {
+ struct net_device *ndev;
+ dma_addr_t dma_addr;
+ u8 head;
+ u8 tail;
+ union {
+ void *desc_addr;
+ struct xge_raw_desc *raw_desc;
+ };
+ struct pkt_info (*pkt_info);
+};
+
+static inline u64 xge_set_desc_bits(int pos, int len, u64 val)
+{
+ return (val & ((1ULL << len) - 1)) << pos;
+}
+
+static inline u64 xge_get_desc_bits(int pos, int len, u64 src)
+{
+ return (src >> pos) & ((1ULL << len) - 1);
+}
+
+#define SET_BITS(field, val) \
+ xge_set_desc_bits(field ## _POS, field ## _LEN, val)
+
+#define GET_BITS(field, src) \
+ xge_get_desc_bits(field ## _POS, field ## _LEN, src)
+
+void xge_setup_desc(struct xge_desc_ring *ring);
+void xge_update_tx_desc_addr(struct xge_pdata *pdata);
+void xge_update_rx_desc_addr(struct xge_pdata *pdata);
+void xge_intr_enable(struct xge_pdata *pdata);
+void xge_intr_disable(struct xge_pdata *pdata);
+
+#endif /* __XGENE_ENET_V2_RING_H__ */
tristate "Broadcom Tigon3 support"
depends on PCI
select PHYLIB
- select HWMON
imply PTP_1588_CLOCK
---help---
This driver supports Broadcom Tigon3 based gigabit Ethernet cards.
To compile this driver as a module, choose M here: the module
will be called tg3. This is recommended.
+config TIGON3_HWMON
+ bool "Broadcom Tigon3 HWMON support"
+ default y
+ depends on TIGON3 && HWMON && !(TIGON3=y && HWMON=m)
+ ---help---
+ Say Y if you want to expose the thermal sensor on Tigon3 devices.
+
config BNX2X
tristate "Broadcom NetXtremeII 10Gb support"
depends on PCI
return timeout_us ? 0 : -EBUSY;
}
+#ifdef CONFIG_TIGON3_HWMON
static int tg3_ape_wait_for_event(struct tg3 *tp, u32 timeout_us)
{
u32 i, apedata;
return 0;
}
+#endif
static int tg3_ape_send_event(struct tg3 *tp, u32 event)
{
return tg3_reset_hw(tp, reset_phy);
}
+#ifdef CONFIG_TIGON3_HWMON
static void tg3_sd_scan_scratchpad(struct tg3 *tp, struct tg3_ocir *ocir)
{
int i;
dev_err(&pdev->dev, "Cannot register hwmon device, aborting\n");
}
}
+#else
+static inline void tg3_hwmon_close(struct tg3 *tp) { }
+static inline void tg3_hwmon_open(struct tg3 *tp) { }
+#endif /* CONFIG_TIGON3_HWMON */
#define TG3_STAT_ADD32(PSTAT, REG) \
__func__);
return 1;
}
+
+ if (octeon_dev->ioq_vector) {
+ struct octeon_ioq_vector *ioq_vector;
+
+ ioq_vector = &octeon_dev->ioq_vector[q];
+ netif_set_xps_queue(netdev,
+ &ioq_vector->affinity_mask,
+ ioq_vector->iq_index);
+ }
}
return 0;
}
}
-static int ixgbe_get_settings(struct net_device *netdev,
- struct ethtool_cmd *ecmd)
+static int ixgbe_get_link_ksettings(struct net_device *netdev,
+ struct ethtool_link_ksettings *cmd)
{
struct ixgbe_adapter *adapter = netdev_priv(netdev);
struct ixgbe_hw *hw = &adapter->hw;
ixgbe_link_speed supported_link;
bool autoneg = false;
+ u32 supported, advertising;
+
+ ethtool_convert_link_mode_to_legacy_u32(&supported,
+ cmd->link_modes.supported);
hw->mac.ops.get_link_capabilities(hw, &supported_link, &autoneg);
/* set the supported link speeds */
if (supported_link & IXGBE_LINK_SPEED_10GB_FULL)
- ecmd->supported |= ixgbe_get_supported_10gtypes(hw);
+ supported |= ixgbe_get_supported_10gtypes(hw);
if (supported_link & IXGBE_LINK_SPEED_1GB_FULL)
- ecmd->supported |= (ixgbe_isbackplane(hw->phy.media_type)) ?
+ supported |= (ixgbe_isbackplane(hw->phy.media_type)) ?
SUPPORTED_1000baseKX_Full :
SUPPORTED_1000baseT_Full;
if (supported_link & IXGBE_LINK_SPEED_100_FULL)
- ecmd->supported |= SUPPORTED_100baseT_Full;
+ supported |= SUPPORTED_100baseT_Full;
if (supported_link & IXGBE_LINK_SPEED_10_FULL)
- ecmd->supported |= SUPPORTED_10baseT_Full;
+ supported |= SUPPORTED_10baseT_Full;
/* default advertised speed if phy.autoneg_advertised isn't set */
- ecmd->advertising = ecmd->supported;
+ advertising = supported;
/* set the advertised speeds */
if (hw->phy.autoneg_advertised) {
- ecmd->advertising = 0;
+ advertising = 0;
if (hw->phy.autoneg_advertised & IXGBE_LINK_SPEED_10_FULL)
- ecmd->advertising |= ADVERTISED_10baseT_Full;
+ advertising |= ADVERTISED_10baseT_Full;
if (hw->phy.autoneg_advertised & IXGBE_LINK_SPEED_100_FULL)
- ecmd->advertising |= ADVERTISED_100baseT_Full;
+ advertising |= ADVERTISED_100baseT_Full;
if (hw->phy.autoneg_advertised & IXGBE_LINK_SPEED_10GB_FULL)
- ecmd->advertising |= ecmd->supported & ADVRTSD_MSK_10G;
+ advertising |= supported & ADVRTSD_MSK_10G;
if (hw->phy.autoneg_advertised & IXGBE_LINK_SPEED_1GB_FULL) {
- if (ecmd->supported & SUPPORTED_1000baseKX_Full)
- ecmd->advertising |= ADVERTISED_1000baseKX_Full;
+ if (supported & SUPPORTED_1000baseKX_Full)
+ advertising |= ADVERTISED_1000baseKX_Full;
else
- ecmd->advertising |= ADVERTISED_1000baseT_Full;
+ advertising |= ADVERTISED_1000baseT_Full;
}
} else {
if (hw->phy.multispeed_fiber && !autoneg) {
if (supported_link & IXGBE_LINK_SPEED_10GB_FULL)
- ecmd->advertising = ADVERTISED_10000baseT_Full;
+ advertising = ADVERTISED_10000baseT_Full;
}
}
if (autoneg) {
- ecmd->supported |= SUPPORTED_Autoneg;
- ecmd->advertising |= ADVERTISED_Autoneg;
- ecmd->autoneg = AUTONEG_ENABLE;
+ supported |= SUPPORTED_Autoneg;
+ advertising |= ADVERTISED_Autoneg;
+ cmd->base.autoneg = AUTONEG_ENABLE;
} else
- ecmd->autoneg = AUTONEG_DISABLE;
-
- ecmd->transceiver = XCVR_EXTERNAL;
+ cmd->base.autoneg = AUTONEG_DISABLE;
/* Determine the remaining settings based on the PHY type. */
switch (adapter->hw.phy.type) {
case ixgbe_phy_x550em_ext_t:
case ixgbe_phy_fw:
case ixgbe_phy_cu_unknown:
- ecmd->supported |= SUPPORTED_TP;
- ecmd->advertising |= ADVERTISED_TP;
- ecmd->port = PORT_TP;
+ supported |= SUPPORTED_TP;
+ advertising |= ADVERTISED_TP;
+ cmd->base.port = PORT_TP;
break;
case ixgbe_phy_qt:
- ecmd->supported |= SUPPORTED_FIBRE;
- ecmd->advertising |= ADVERTISED_FIBRE;
- ecmd->port = PORT_FIBRE;
+ supported |= SUPPORTED_FIBRE;
+ advertising |= ADVERTISED_FIBRE;
+ cmd->base.port = PORT_FIBRE;
break;
case ixgbe_phy_nl:
case ixgbe_phy_sfp_passive_tyco:
case ixgbe_sfp_type_da_cu:
case ixgbe_sfp_type_da_cu_core0:
case ixgbe_sfp_type_da_cu_core1:
- ecmd->supported |= SUPPORTED_FIBRE;
- ecmd->advertising |= ADVERTISED_FIBRE;
- ecmd->port = PORT_DA;
+ supported |= SUPPORTED_FIBRE;
+ advertising |= ADVERTISED_FIBRE;
+ cmd->base.port = PORT_DA;
break;
case ixgbe_sfp_type_sr:
case ixgbe_sfp_type_lr:
case ixgbe_sfp_type_1g_sx_core1:
case ixgbe_sfp_type_1g_lx_core0:
case ixgbe_sfp_type_1g_lx_core1:
- ecmd->supported |= SUPPORTED_FIBRE;
- ecmd->advertising |= ADVERTISED_FIBRE;
- ecmd->port = PORT_FIBRE;
+ supported |= SUPPORTED_FIBRE;
+ advertising |= ADVERTISED_FIBRE;
+ cmd->base.port = PORT_FIBRE;
break;
case ixgbe_sfp_type_not_present:
- ecmd->supported |= SUPPORTED_FIBRE;
- ecmd->advertising |= ADVERTISED_FIBRE;
- ecmd->port = PORT_NONE;
+ supported |= SUPPORTED_FIBRE;
+ advertising |= ADVERTISED_FIBRE;
+ cmd->base.port = PORT_NONE;
break;
case ixgbe_sfp_type_1g_cu_core0:
case ixgbe_sfp_type_1g_cu_core1:
- ecmd->supported |= SUPPORTED_TP;
- ecmd->advertising |= ADVERTISED_TP;
- ecmd->port = PORT_TP;
+ supported |= SUPPORTED_TP;
+ advertising |= ADVERTISED_TP;
+ cmd->base.port = PORT_TP;
break;
case ixgbe_sfp_type_unknown:
default:
- ecmd->supported |= SUPPORTED_FIBRE;
- ecmd->advertising |= ADVERTISED_FIBRE;
- ecmd->port = PORT_OTHER;
+ supported |= SUPPORTED_FIBRE;
+ advertising |= ADVERTISED_FIBRE;
+ cmd->base.port = PORT_OTHER;
break;
}
break;
case ixgbe_phy_xaui:
- ecmd->supported |= SUPPORTED_FIBRE;
- ecmd->advertising |= ADVERTISED_FIBRE;
- ecmd->port = PORT_NONE;
+ supported |= SUPPORTED_FIBRE;
+ advertising |= ADVERTISED_FIBRE;
+ cmd->base.port = PORT_NONE;
break;
case ixgbe_phy_unknown:
case ixgbe_phy_generic:
case ixgbe_phy_sfp_unsupported:
default:
- ecmd->supported |= SUPPORTED_FIBRE;
- ecmd->advertising |= ADVERTISED_FIBRE;
- ecmd->port = PORT_OTHER;
+ supported |= SUPPORTED_FIBRE;
+ advertising |= ADVERTISED_FIBRE;
+ cmd->base.port = PORT_OTHER;
break;
}
/* Indicate pause support */
- ecmd->supported |= SUPPORTED_Pause;
+ supported |= SUPPORTED_Pause;
switch (hw->fc.requested_mode) {
case ixgbe_fc_full:
- ecmd->advertising |= ADVERTISED_Pause;
+ advertising |= ADVERTISED_Pause;
break;
case ixgbe_fc_rx_pause:
- ecmd->advertising |= ADVERTISED_Pause |
+ advertising |= ADVERTISED_Pause |
ADVERTISED_Asym_Pause;
break;
case ixgbe_fc_tx_pause:
- ecmd->advertising |= ADVERTISED_Asym_Pause;
+ advertising |= ADVERTISED_Asym_Pause;
break;
default:
- ecmd->advertising &= ~(ADVERTISED_Pause |
+ advertising &= ~(ADVERTISED_Pause |
ADVERTISED_Asym_Pause);
}
if (netif_carrier_ok(netdev)) {
switch (adapter->link_speed) {
case IXGBE_LINK_SPEED_10GB_FULL:
- ethtool_cmd_speed_set(ecmd, SPEED_10000);
+ cmd->base.speed = SPEED_10000;
break;
case IXGBE_LINK_SPEED_5GB_FULL:
- ethtool_cmd_speed_set(ecmd, SPEED_5000);
+ cmd->base.speed = SPEED_5000;
break;
case IXGBE_LINK_SPEED_2_5GB_FULL:
- ethtool_cmd_speed_set(ecmd, SPEED_2500);
+ cmd->base.speed = SPEED_2500;
break;
case IXGBE_LINK_SPEED_1GB_FULL:
- ethtool_cmd_speed_set(ecmd, SPEED_1000);
+ cmd->base.speed = SPEED_1000;
break;
case IXGBE_LINK_SPEED_100_FULL:
- ethtool_cmd_speed_set(ecmd, SPEED_100);
+ cmd->base.speed = SPEED_100;
break;
case IXGBE_LINK_SPEED_10_FULL:
- ethtool_cmd_speed_set(ecmd, SPEED_10);
+ cmd->base.speed = SPEED_10;
break;
default:
break;
}
- ecmd->duplex = DUPLEX_FULL;
+ cmd->base.duplex = DUPLEX_FULL;
} else {
- ethtool_cmd_speed_set(ecmd, SPEED_UNKNOWN);
- ecmd->duplex = DUPLEX_UNKNOWN;
+ cmd->base.speed = SPEED_UNKNOWN;
+ cmd->base.duplex = DUPLEX_UNKNOWN;
}
+ ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported,
+ supported);
+ ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising,
+ advertising);
+
return 0;
}
-static int ixgbe_set_settings(struct net_device *netdev,
- struct ethtool_cmd *ecmd)
+static int ixgbe_set_link_ksettings(struct net_device *netdev,
+ const struct ethtool_link_ksettings *cmd)
{
struct ixgbe_adapter *adapter = netdev_priv(netdev);
struct ixgbe_hw *hw = &adapter->hw;
u32 advertised, old;
s32 err = 0;
+ u32 supported, advertising;
+
+ ethtool_convert_link_mode_to_legacy_u32(&supported,
+ cmd->link_modes.supported);
+ ethtool_convert_link_mode_to_legacy_u32(&advertising,
+ cmd->link_modes.advertising);
if ((hw->phy.media_type == ixgbe_media_type_copper) ||
(hw->phy.multispeed_fiber)) {
* this function does not support duplex forcing, but can
* limit the advertising of the adapter to the specified speed
*/
- if (ecmd->advertising & ~ecmd->supported)
+ if (advertising & ~supported)
return -EINVAL;
/* only allow one speed at a time if no autoneg */
- if (!ecmd->autoneg && hw->phy.multispeed_fiber) {
- if (ecmd->advertising ==
+ if (!cmd->base.autoneg && hw->phy.multispeed_fiber) {
+ if (advertising ==
(ADVERTISED_10000baseT_Full |
ADVERTISED_1000baseT_Full))
return -EINVAL;
old = hw->phy.autoneg_advertised;
advertised = 0;
- if (ecmd->advertising & ADVERTISED_10000baseT_Full)
+ if (advertising & ADVERTISED_10000baseT_Full)
advertised |= IXGBE_LINK_SPEED_10GB_FULL;
- if (ecmd->advertising & ADVERTISED_1000baseT_Full)
+ if (advertising & ADVERTISED_1000baseT_Full)
advertised |= IXGBE_LINK_SPEED_1GB_FULL;
- if (ecmd->advertising & ADVERTISED_100baseT_Full)
+ if (advertising & ADVERTISED_100baseT_Full)
advertised |= IXGBE_LINK_SPEED_100_FULL;
- if (ecmd->advertising & ADVERTISED_10baseT_Full)
+ if (advertising & ADVERTISED_10baseT_Full)
advertised |= IXGBE_LINK_SPEED_10_FULL;
if (old == advertised)
clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);
} else {
/* in this case we currently only support 10Gb/FULL */
- u32 speed = ethtool_cmd_speed(ecmd);
- if ((ecmd->autoneg == AUTONEG_ENABLE) ||
- (ecmd->advertising != ADVERTISED_10000baseT_Full) ||
- (speed + ecmd->duplex != SPEED_10000 + DUPLEX_FULL))
+ u32 speed = cmd->base.speed;
+
+ if ((cmd->base.autoneg == AUTONEG_ENABLE) ||
+ (advertising != ADVERTISED_10000baseT_Full) ||
+ (speed + cmd->base.duplex != SPEED_10000 + DUPLEX_FULL))
return -EINVAL;
}
}
static const struct ethtool_ops ixgbe_ethtool_ops = {
- .get_settings = ixgbe_get_settings,
- .set_settings = ixgbe_set_settings,
.get_drvinfo = ixgbe_get_drvinfo,
.get_regs_len = ixgbe_get_regs_len,
.get_regs = ixgbe_get_regs,
.get_ts_info = ixgbe_get_ts_info,
.get_module_info = ixgbe_get_module_info,
.get_module_eeprom = ixgbe_get_module_eeprom,
+ .get_link_ksettings = ixgbe_get_link_ksettings,
+ .set_link_ksettings = ixgbe_set_link_ksettings,
};
void ixgbe_set_ethtool_ops(struct net_device *netdev)
that all dependencies are met.
config MVPP2
- tristate "Marvell Armada 375 network interface support"
+ tristate "Marvell Armada 375/7K/8K network interface support"
depends on ARCH_MVEBU || COMPILE_TEST
depends on HAS_DMA
select MVMDIO
---help---
This driver supports the network interface units in the
- Marvell ARMADA 375 SoC.
+ Marvell ARMADA 375, 7K and 8K SoCs.
config PXA168_ETH
tristate "Marvell pxa168 ethernet support"
#include <linux/of_mdio.h>
#include <linux/of_net.h>
#include <linux/of_address.h>
+#include <linux/of_device.h>
#include <linux/phy.h>
#include <linux/clk.h>
#include <linux/hrtimer.h>
#define MVPP2_SNOOP_PKT_SIZE_MASK 0x1ff
#define MVPP2_SNOOP_BUF_HDR_MASK BIT(9)
#define MVPP2_RXQ_POOL_SHORT_OFFS 20
-#define MVPP2_RXQ_POOL_SHORT_MASK 0x700000
+#define MVPP21_RXQ_POOL_SHORT_MASK 0x700000
+#define MVPP22_RXQ_POOL_SHORT_MASK 0xf00000
#define MVPP2_RXQ_POOL_LONG_OFFS 24
-#define MVPP2_RXQ_POOL_LONG_MASK 0x7000000
+#define MVPP21_RXQ_POOL_LONG_MASK 0x7000000
+#define MVPP22_RXQ_POOL_LONG_MASK 0xf000000
#define MVPP2_RXQ_PACKET_OFFSET_OFFS 28
#define MVPP2_RXQ_PACKET_OFFSET_MASK 0x70000000
#define MVPP2_RXQ_DISABLE_MASK BIT(31)
/* Descriptor Manager Top Registers */
#define MVPP2_RXQ_NUM_REG 0x2040
#define MVPP2_RXQ_DESC_ADDR_REG 0x2044
+#define MVPP22_DESC_ADDR_OFFS 8
#define MVPP2_RXQ_DESC_SIZE_REG 0x2048
#define MVPP2_RXQ_DESC_SIZE_MASK 0x3ff0
#define MVPP2_RXQ_STATUS_UPDATE_REG(rxq) (0x3000 + 4 * (rxq))
#define MVPP2_TXQ_DESC_SIZE_REG 0x2088
#define MVPP2_TXQ_DESC_SIZE_MASK 0x3ff0
#define MVPP2_AGGR_TXQ_UPDATE_REG 0x2090
-#define MVPP2_TXQ_THRESH_REG 0x2094
-#define MVPP2_TRANSMITTED_THRESH_OFFSET 16
-#define MVPP2_TRANSMITTED_THRESH_MASK 0x3fff0000
#define MVPP2_TXQ_INDEX_REG 0x2098
#define MVPP2_TXQ_PREF_BUF_REG 0x209c
#define MVPP2_PREF_BUF_PTR(desc) ((desc) & 0xfff)
#define MVPP2_TXQ_RSVD_CLR_REG 0x20b8
#define MVPP2_TXQ_RSVD_CLR_OFFSET 16
#define MVPP2_AGGR_TXQ_DESC_ADDR_REG(cpu) (0x2100 + 4 * (cpu))
+#define MVPP22_AGGR_TXQ_DESC_ADDR_OFFS 8
#define MVPP2_AGGR_TXQ_DESC_SIZE_REG(cpu) (0x2140 + 4 * (cpu))
#define MVPP2_AGGR_TXQ_DESC_SIZE_MASK 0x3ff0
#define MVPP2_AGGR_TXQ_STATUS_REG(cpu) (0x2180 + 4 * (cpu))
#define MVPP2_WIN_REMAP(w) (0x4040 + ((w) << 2))
#define MVPP2_BASE_ADDR_ENABLE 0x4060
+/* AXI Bridge Registers */
+#define MVPP22_AXI_BM_WR_ATTR_REG 0x4100
+#define MVPP22_AXI_BM_RD_ATTR_REG 0x4104
+#define MVPP22_AXI_AGGRQ_DESCR_RD_ATTR_REG 0x4110
+#define MVPP22_AXI_TXQ_DESCR_WR_ATTR_REG 0x4114
+#define MVPP22_AXI_TXQ_DESCR_RD_ATTR_REG 0x4118
+#define MVPP22_AXI_RXQ_DESCR_WR_ATTR_REG 0x411c
+#define MVPP22_AXI_RX_DATA_WR_ATTR_REG 0x4120
+#define MVPP22_AXI_TX_DATA_RD_ATTR_REG 0x4130
+#define MVPP22_AXI_RD_NORMAL_CODE_REG 0x4150
+#define MVPP22_AXI_RD_SNOOP_CODE_REG 0x4154
+#define MVPP22_AXI_WR_NORMAL_CODE_REG 0x4160
+#define MVPP22_AXI_WR_SNOOP_CODE_REG 0x4164
+
+/* Values for AXI Bridge registers */
+#define MVPP22_AXI_ATTR_CACHE_OFFS 0
+#define MVPP22_AXI_ATTR_DOMAIN_OFFS 12
+
+#define MVPP22_AXI_CODE_CACHE_OFFS 0
+#define MVPP22_AXI_CODE_DOMAIN_OFFS 4
+
+#define MVPP22_AXI_CODE_CACHE_NON_CACHE 0x3
+#define MVPP22_AXI_CODE_CACHE_WR_CACHE 0x7
+#define MVPP22_AXI_CODE_CACHE_RD_CACHE 0xb
+
+#define MVPP22_AXI_CODE_DOMAIN_OUTER_DOM 2
+#define MVPP22_AXI_CODE_DOMAIN_SYSTEM 3
+
/* Interrupt Cause and Mask registers */
#define MVPP2_ISR_RX_THRESHOLD_REG(rxq) (0x5200 + 4 * (rxq))
#define MVPP2_MAX_ISR_RX_THRESHOLD 0xfffff0
-#define MVPP2_ISR_RXQ_GROUP_REG(rxq) (0x5400 + 4 * (rxq))
+#define MVPP21_ISR_RXQ_GROUP_REG(rxq) (0x5400 + 4 * (rxq))
+
+#define MVPP22_ISR_RXQ_GROUP_INDEX_REG 0x5400
+#define MVPP22_ISR_RXQ_GROUP_INDEX_SUBGROUP_MASK 0xf
+#define MVPP22_ISR_RXQ_GROUP_INDEX_GROUP_MASK 0x380
+#define MVPP22_ISR_RXQ_GROUP_INDEX_GROUP_OFFSET 7
+
+#define MVPP22_ISR_RXQ_GROUP_INDEX_SUBGROUP_MASK 0xf
+#define MVPP22_ISR_RXQ_GROUP_INDEX_GROUP_MASK 0x380
+
+#define MVPP22_ISR_RXQ_SUB_GROUP_CONFIG_REG 0x5404
+#define MVPP22_ISR_RXQ_SUB_GROUP_STARTQ_MASK 0x1f
+#define MVPP22_ISR_RXQ_SUB_GROUP_SIZE_MASK 0xf00
+#define MVPP22_ISR_RXQ_SUB_GROUP_SIZE_OFFSET 8
+
#define MVPP2_ISR_ENABLE_REG(port) (0x5420 + 4 * (port))
#define MVPP2_ISR_ENABLE_INTERRUPT(mask) ((mask) & 0xffff)
#define MVPP2_ISR_DISABLE_INTERRUPT(mask) (((mask) << 16) & 0xffff0000)
#define MVPP2_BM_PHY_ALLOC_REG(pool) (0x6400 + ((pool) * 4))
#define MVPP2_BM_PHY_ALLOC_GRNTD_MASK BIT(0)
#define MVPP2_BM_VIRT_ALLOC_REG 0x6440
+#define MVPP22_BM_ADDR_HIGH_ALLOC 0x6444
+#define MVPP22_BM_ADDR_HIGH_PHYS_MASK 0xff
+#define MVPP22_BM_ADDR_HIGH_VIRT_MASK 0xff00
+#define MVPP22_BM_ADDR_HIGH_VIRT_SHIFT 8
#define MVPP2_BM_PHY_RLS_REG(pool) (0x6480 + ((pool) * 4))
#define MVPP2_BM_PHY_RLS_MC_BUFF_MASK BIT(0)
#define MVPP2_BM_PHY_RLS_PRIO_EN_MASK BIT(1)
#define MVPP2_BM_PHY_RLS_GRNTD_MASK BIT(2)
#define MVPP2_BM_VIRT_RLS_REG 0x64c0
-#define MVPP2_BM_MC_RLS_REG 0x64c4
-#define MVPP2_BM_MC_ID_MASK 0xfff
-#define MVPP2_BM_FORCE_RELEASE_MASK BIT(12)
+#define MVPP22_BM_ADDR_HIGH_RLS_REG 0x64c4
+#define MVPP22_BM_ADDR_HIGH_PHYS_RLS_MASK 0xff
+#define MVPP22_BM_ADDR_HIGH_VIRT_RLS_MASK 0xff00
+#define MVPP22_BM_ADDR_HIGH_VIRT_RLS_SHIFT 8
/* TX Scheduler registers */
#define MVPP2_TXP_SCHED_PORT_INDEX_REG 0x8000
#define MVPP2_GMAC_TX_FIFO_MIN_TH_ALL_MASK 0x1fc0
#define MVPP2_GMAC_TX_FIFO_MIN_TH_MASK(v) (((v) << 6) & \
MVPP2_GMAC_TX_FIFO_MIN_TH_ALL_MASK)
+#define MVPP22_GMAC_CTRL_4_REG 0x90
+#define MVPP22_CTRL4_EXT_PIN_GMII_SEL BIT(0)
+#define MVPP22_CTRL4_DP_CLK_SEL BIT(5)
+#define MVPP22_CTRL4_SYNC_BYPASS BIT(6)
+#define MVPP22_CTRL4_QSGMII_BYPASS_ACTIVE BIT(7)
+
+/* Per-port XGMAC registers. PPv2.2 only, only for GOP port 0,
+ * relative to port->base.
+ */
+#define MVPP22_XLG_CTRL3_REG 0x11c
+#define MVPP22_XLG_CTRL3_MACMODESELECT_MASK (7 << 13)
+#define MVPP22_XLG_CTRL3_MACMODESELECT_GMAC (0 << 13)
+
+/* SMI registers. PPv2.2 only, relative to priv->iface_base. */
+#define MVPP22_SMI_MISC_CFG_REG 0x1204
+#define MVPP22_SMI_POLLING_EN BIT(10)
+
+#define MVPP22_GMAC_BASE(port) (0x7000 + (port) * 0x1000 + 0xe00)
#define MVPP2_CAUSE_TXQ_SENT_DESC_ALL_MASK 0xff
/* Maximum number of TXQs used by single port */
#define MVPP2_MAX_TXQ 8
-/* Maximum number of RXQs used by single port */
-#define MVPP2_MAX_RXQ 8
-
/* Dfault number of RXQs in use */
#define MVPP2_DEFAULT_RXQ 4
-/* Total number of RXQs available to all ports */
-#define MVPP2_RXQ_TOTAL_NUM (MVPP2_MAX_PORTS * MVPP2_MAX_RXQ)
-
/* Max number of Rx descriptors */
#define MVPP2_MAX_RXD 128
*/
#define MVPP2_BM_SHORT_PKT_SIZE MVPP2_RX_MAX_PKT_SIZE(512)
+#define MVPP21_ADDR_SPACE_SZ 0
+#define MVPP22_ADDR_SPACE_SZ SZ_64K
+
+#define MVPP2_MAX_CPUS 4
+
enum mvpp2_bm_type {
MVPP2_BM_FREE,
MVPP2_BM_SWF_LONG,
/* Shared Packet Processor resources */
struct mvpp2 {
/* Shared registers' base addresses */
- void __iomem *base;
void __iomem *lms_base;
+ void __iomem *iface_base;
+
+ /* On PPv2.2, each CPU can access the base register through a
+ * separate address space, each 64 KB apart from each
+ * other.
+ */
+ void __iomem *cpu_base[MVPP2_MAX_CPUS];
/* Common clocks */
struct clk *pp_clk;
struct clk *gop_clk;
+ struct clk *mg_clk;
/* List of pointers to port structures */
struct mvpp2_port **port_list;
/* Tclk value */
u32 tclk;
+
+ /* HW version */
+ enum { MVPP21, MVPP22 } hw_version;
+
+ /* Maximum number of RXQs per port */
+ unsigned int max_port_rxqs;
};
struct mvpp2_pcpu_stats {
struct mvpp2_port {
u8 id;
+ /* Index of the port from the "group of ports" complex point
+ * of view
+ */
+ int gop_id;
+
int irq;
struct mvpp2 *priv;
#define MVPP2_RXD_L3_IP6 BIT(30)
#define MVPP2_RXD_BUF_HDR BIT(31)
-struct mvpp2_tx_desc {
+/* HW TX descriptor for PPv2.1 */
+struct mvpp21_tx_desc {
u32 command; /* Options used by HW for packet transmitting.*/
u8 packet_offset; /* the offset from the buffer beginning */
u8 phys_txq; /* destination queue ID */
u16 data_size; /* data size of transmitted packet in bytes */
- u32 buf_phys_addr; /* physical addr of transmitted buffer */
+ u32 buf_dma_addr; /* physical addr of transmitted buffer */
u32 buf_cookie; /* cookie for access to TX buffer in tx path */
u32 reserved1[3]; /* hw_cmd (for future use, BM, PON, PNC) */
u32 reserved2; /* reserved (for future use) */
};
-struct mvpp2_rx_desc {
+/* HW RX descriptor for PPv2.1 */
+struct mvpp21_rx_desc {
u32 status; /* info about received packet */
u16 reserved1; /* parser_info (for future use, PnC) */
u16 data_size; /* size of received packet in bytes */
- u32 buf_phys_addr; /* physical address of the buffer */
+ u32 buf_dma_addr; /* physical address of the buffer */
u32 buf_cookie; /* cookie for access to RX buffer in rx path */
u16 reserved2; /* gem_port_id (for future use, PON) */
u16 reserved3; /* csum_l4 (for future use, PnC) */
u32 reserved8;
};
+/* HW TX descriptor for PPv2.2 */
+struct mvpp22_tx_desc {
+ u32 command;
+ u8 packet_offset;
+ u8 phys_txq;
+ u16 data_size;
+ u64 reserved1;
+ u64 buf_dma_addr_ptp;
+ u64 buf_cookie_misc;
+};
+
+/* HW RX descriptor for PPv2.2 */
+struct mvpp22_rx_desc {
+ u32 status;
+ u16 reserved1;
+ u16 data_size;
+ u32 reserved2;
+ u32 reserved3;
+ u64 buf_dma_addr_key_hash;
+ u64 buf_cookie_misc;
+};
+
+/* Opaque type used by the driver to manipulate the HW TX and RX
+ * descriptors
+ */
+struct mvpp2_tx_desc {
+ union {
+ struct mvpp21_tx_desc pp21;
+ struct mvpp22_tx_desc pp22;
+ };
+};
+
+struct mvpp2_rx_desc {
+ union {
+ struct mvpp21_rx_desc pp21;
+ struct mvpp22_rx_desc pp22;
+ };
+};
+
struct mvpp2_txq_pcpu_buf {
/* Transmitted SKB */
struct sk_buff *skb;
/* Physical address of transmitted buffer */
- dma_addr_t phys;
+ dma_addr_t dma;
/* Size transmitted */
size_t size;
struct mvpp2_tx_desc *descs;
/* DMA address of the Tx DMA descriptors array */
- dma_addr_t descs_phys;
+ dma_addr_t descs_dma;
/* Index of the last Tx DMA descriptor */
int last_desc;
struct mvpp2_rx_desc *descs;
/* DMA address of the RX DMA descriptors array */
- dma_addr_t descs_phys;
+ dma_addr_t descs_dma;
/* Index of the last RX DMA descriptor */
int last_desc;
/* Buffer Pointers Pool External (BPPE) size */
int size;
+ /* BPPE size in bytes */
+ int size_bytes;
/* Number of buffers for this pool */
int buf_num;
/* Pool buffer size */
/* BPPE virtual base address */
u32 *virt_addr;
- /* BPPE physical base address */
- dma_addr_t phys_addr;
+ /* BPPE DMA base address */
+ dma_addr_t dma_addr;
/* Ports using BM pool */
u32 port_map;
};
-struct mvpp2_buff_hdr {
- u32 next_buff_phys_addr;
- u32 next_buff_virt_addr;
- u16 byte_count;
- u16 info;
- u8 reserved1; /* bm_qset (for future use, BM) */
-};
-
-/* Buffer header info bits */
-#define MVPP2_B_HDR_INFO_MC_ID_MASK 0xfff
-#define MVPP2_B_HDR_INFO_MC_ID(info) ((info) & MVPP2_B_HDR_INFO_MC_ID_MASK)
-#define MVPP2_B_HDR_INFO_LAST_OFFS 12
-#define MVPP2_B_HDR_INFO_LAST_MASK BIT(12)
-#define MVPP2_B_HDR_INFO_IS_LAST(info) \
- ((info & MVPP2_B_HDR_INFO_LAST_MASK) >> MVPP2_B_HDR_INFO_LAST_OFFS)
-
/* Static declaractions */
/* Number of RXQs used by single port */
static void mvpp2_write(struct mvpp2 *priv, u32 offset, u32 data)
{
- writel(data, priv->base + offset);
+ writel(data, priv->cpu_base[0] + offset);
}
static u32 mvpp2_read(struct mvpp2 *priv, u32 offset)
{
- return readl(priv->base + offset);
+ return readl(priv->cpu_base[0] + offset);
+}
+
+/* These accessors should be used to access:
+ *
+ * - per-CPU registers, where each CPU has its own copy of the
+ * register.
+ *
+ * MVPP2_BM_VIRT_ALLOC_REG
+ * MVPP2_BM_ADDR_HIGH_ALLOC
+ * MVPP22_BM_ADDR_HIGH_RLS_REG
+ * MVPP2_BM_VIRT_RLS_REG
+ * MVPP2_ISR_RX_TX_CAUSE_REG
+ * MVPP2_ISR_RX_TX_MASK_REG
+ * MVPP2_TXQ_NUM_REG
+ * MVPP2_AGGR_TXQ_UPDATE_REG
+ * MVPP2_TXQ_RSVD_REQ_REG
+ * MVPP2_TXQ_RSVD_RSLT_REG
+ * MVPP2_TXQ_SENT_REG
+ * MVPP2_RXQ_NUM_REG
+ *
+ * - global registers that must be accessed through a specific CPU
+ * window, because they are related to an access to a per-CPU
+ * register
+ *
+ * MVPP2_BM_PHY_ALLOC_REG (related to MVPP2_BM_VIRT_ALLOC_REG)
+ * MVPP2_BM_PHY_RLS_REG (related to MVPP2_BM_VIRT_RLS_REG)
+ * MVPP2_RXQ_THRESH_REG (related to MVPP2_RXQ_NUM_REG)
+ * MVPP2_RXQ_DESC_ADDR_REG (related to MVPP2_RXQ_NUM_REG)
+ * MVPP2_RXQ_DESC_SIZE_REG (related to MVPP2_RXQ_NUM_REG)
+ * MVPP2_RXQ_INDEX_REG (related to MVPP2_RXQ_NUM_REG)
+ * MVPP2_TXQ_PENDING_REG (related to MVPP2_TXQ_NUM_REG)
+ * MVPP2_TXQ_DESC_ADDR_REG (related to MVPP2_TXQ_NUM_REG)
+ * MVPP2_TXQ_DESC_SIZE_REG (related to MVPP2_TXQ_NUM_REG)
+ * MVPP2_TXQ_INDEX_REG (related to MVPP2_TXQ_NUM_REG)
+ * MVPP2_TXQ_PENDING_REG (related to MVPP2_TXQ_NUM_REG)
+ * MVPP2_TXQ_PREF_BUF_REG (related to MVPP2_TXQ_NUM_REG)
+ * MVPP2_TXQ_PREF_BUF_REG (related to MVPP2_TXQ_NUM_REG)
+ */
+static void mvpp2_percpu_write(struct mvpp2 *priv, int cpu,
+ u32 offset, u32 data)
+{
+ writel(data, priv->cpu_base[cpu] + offset);
+}
+
+static u32 mvpp2_percpu_read(struct mvpp2 *priv, int cpu,
+ u32 offset)
+{
+ return readl(priv->cpu_base[cpu] + offset);
+}
+
+static dma_addr_t mvpp2_txdesc_dma_addr_get(struct mvpp2_port *port,
+ struct mvpp2_tx_desc *tx_desc)
+{
+ if (port->priv->hw_version == MVPP21)
+ return tx_desc->pp21.buf_dma_addr;
+ else
+ return tx_desc->pp22.buf_dma_addr_ptp & GENMASK_ULL(40, 0);
+}
+
+static void mvpp2_txdesc_dma_addr_set(struct mvpp2_port *port,
+ struct mvpp2_tx_desc *tx_desc,
+ dma_addr_t dma_addr)
+{
+ if (port->priv->hw_version == MVPP21) {
+ tx_desc->pp21.buf_dma_addr = dma_addr;
+ } else {
+ u64 val = (u64)dma_addr;
+
+ tx_desc->pp22.buf_dma_addr_ptp &= ~GENMASK_ULL(40, 0);
+ tx_desc->pp22.buf_dma_addr_ptp |= val;
+ }
+}
+
+static size_t mvpp2_txdesc_size_get(struct mvpp2_port *port,
+ struct mvpp2_tx_desc *tx_desc)
+{
+ if (port->priv->hw_version == MVPP21)
+ return tx_desc->pp21.data_size;
+ else
+ return tx_desc->pp22.data_size;
+}
+
+static void mvpp2_txdesc_size_set(struct mvpp2_port *port,
+ struct mvpp2_tx_desc *tx_desc,
+ size_t size)
+{
+ if (port->priv->hw_version == MVPP21)
+ tx_desc->pp21.data_size = size;
+ else
+ tx_desc->pp22.data_size = size;
+}
+
+static void mvpp2_txdesc_txq_set(struct mvpp2_port *port,
+ struct mvpp2_tx_desc *tx_desc,
+ unsigned int txq)
+{
+ if (port->priv->hw_version == MVPP21)
+ tx_desc->pp21.phys_txq = txq;
+ else
+ tx_desc->pp22.phys_txq = txq;
+}
+
+static void mvpp2_txdesc_cmd_set(struct mvpp2_port *port,
+ struct mvpp2_tx_desc *tx_desc,
+ unsigned int command)
+{
+ if (port->priv->hw_version == MVPP21)
+ tx_desc->pp21.command = command;
+ else
+ tx_desc->pp22.command = command;
+}
+
+static void mvpp2_txdesc_offset_set(struct mvpp2_port *port,
+ struct mvpp2_tx_desc *tx_desc,
+ unsigned int offset)
+{
+ if (port->priv->hw_version == MVPP21)
+ tx_desc->pp21.packet_offset = offset;
+ else
+ tx_desc->pp22.packet_offset = offset;
+}
+
+static unsigned int mvpp2_txdesc_offset_get(struct mvpp2_port *port,
+ struct mvpp2_tx_desc *tx_desc)
+{
+ if (port->priv->hw_version == MVPP21)
+ return tx_desc->pp21.packet_offset;
+ else
+ return tx_desc->pp22.packet_offset;
+}
+
+static dma_addr_t mvpp2_rxdesc_dma_addr_get(struct mvpp2_port *port,
+ struct mvpp2_rx_desc *rx_desc)
+{
+ if (port->priv->hw_version == MVPP21)
+ return rx_desc->pp21.buf_dma_addr;
+ else
+ return rx_desc->pp22.buf_dma_addr_key_hash & GENMASK_ULL(40, 0);
+}
+
+static unsigned long mvpp2_rxdesc_cookie_get(struct mvpp2_port *port,
+ struct mvpp2_rx_desc *rx_desc)
+{
+ if (port->priv->hw_version == MVPP21)
+ return rx_desc->pp21.buf_cookie;
+ else
+ return rx_desc->pp22.buf_cookie_misc & GENMASK_ULL(40, 0);
+}
+
+static size_t mvpp2_rxdesc_size_get(struct mvpp2_port *port,
+ struct mvpp2_rx_desc *rx_desc)
+{
+ if (port->priv->hw_version == MVPP21)
+ return rx_desc->pp21.data_size;
+ else
+ return rx_desc->pp22.data_size;
+}
+
+static u32 mvpp2_rxdesc_status_get(struct mvpp2_port *port,
+ struct mvpp2_rx_desc *rx_desc)
+{
+ if (port->priv->hw_version == MVPP21)
+ return rx_desc->pp21.status;
+ else
+ return rx_desc->pp22.status;
}
static void mvpp2_txq_inc_get(struct mvpp2_txq_pcpu *txq_pcpu)
txq_pcpu->txq_get_index = 0;
}
-static void mvpp2_txq_inc_put(struct mvpp2_txq_pcpu *txq_pcpu,
+static void mvpp2_txq_inc_put(struct mvpp2_port *port,
+ struct mvpp2_txq_pcpu *txq_pcpu,
struct sk_buff *skb,
struct mvpp2_tx_desc *tx_desc)
{
struct mvpp2_txq_pcpu_buf *tx_buf =
txq_pcpu->buffs + txq_pcpu->txq_put_index;
tx_buf->skb = skb;
- tx_buf->size = tx_desc->data_size;
- tx_buf->phys = tx_desc->buf_phys_addr + tx_desc->packet_offset;
+ tx_buf->size = mvpp2_txdesc_size_get(port, tx_desc);
+ tx_buf->dma = mvpp2_txdesc_dma_addr_get(port, tx_desc) +
+ mvpp2_txdesc_offset_get(port, tx_desc);
txq_pcpu->txq_put_index++;
if (txq_pcpu->txq_put_index == txq_pcpu->size)
txq_pcpu->txq_put_index = 0;
struct mvpp2 *priv,
struct mvpp2_bm_pool *bm_pool, int size)
{
- int size_bytes;
u32 val;
- size_bytes = sizeof(u32) * size;
- bm_pool->virt_addr = dma_alloc_coherent(&pdev->dev, size_bytes,
- &bm_pool->phys_addr,
+ /* Number of buffer pointers must be a multiple of 16, as per
+ * hardware constraints
+ */
+ if (!IS_ALIGNED(size, 16))
+ return -EINVAL;
+
+ /* PPv2.1 needs 8 bytes per buffer pointer, PPv2.2 needs 16
+ * bytes per buffer pointer
+ */
+ if (priv->hw_version == MVPP21)
+ bm_pool->size_bytes = 2 * sizeof(u32) * size;
+ else
+ bm_pool->size_bytes = 2 * sizeof(u64) * size;
+
+ bm_pool->virt_addr = dma_alloc_coherent(&pdev->dev, bm_pool->size_bytes,
+ &bm_pool->dma_addr,
GFP_KERNEL);
if (!bm_pool->virt_addr)
return -ENOMEM;
if (!IS_ALIGNED((unsigned long)bm_pool->virt_addr,
MVPP2_BM_POOL_PTR_ALIGN)) {
- dma_free_coherent(&pdev->dev, size_bytes, bm_pool->virt_addr,
- bm_pool->phys_addr);
+ dma_free_coherent(&pdev->dev, bm_pool->size_bytes,
+ bm_pool->virt_addr, bm_pool->dma_addr);
dev_err(&pdev->dev, "BM pool %d is not %d bytes aligned\n",
bm_pool->id, MVPP2_BM_POOL_PTR_ALIGN);
return -ENOMEM;
}
mvpp2_write(priv, MVPP2_BM_POOL_BASE_REG(bm_pool->id),
- bm_pool->phys_addr);
+ lower_32_bits(bm_pool->dma_addr));
mvpp2_write(priv, MVPP2_BM_POOL_SIZE_REG(bm_pool->id), size);
val = mvpp2_read(priv, MVPP2_BM_POOL_CTRL_REG(bm_pool->id));
mvpp2_write(priv, MVPP2_POOL_BUF_SIZE_REG(bm_pool->id), val);
}
+static void mvpp2_bm_bufs_get_addrs(struct device *dev, struct mvpp2 *priv,
+ struct mvpp2_bm_pool *bm_pool,
+ dma_addr_t *dma_addr,
+ phys_addr_t *phys_addr)
+{
+ int cpu = smp_processor_id();
+
+ *dma_addr = mvpp2_percpu_read(priv, cpu,
+ MVPP2_BM_PHY_ALLOC_REG(bm_pool->id));
+ *phys_addr = mvpp2_percpu_read(priv, cpu, MVPP2_BM_VIRT_ALLOC_REG);
+
+ if (priv->hw_version == MVPP22) {
+ u32 val;
+ u32 dma_addr_highbits, phys_addr_highbits;
+
+ val = mvpp2_percpu_read(priv, cpu, MVPP22_BM_ADDR_HIGH_ALLOC);
+ dma_addr_highbits = (val & MVPP22_BM_ADDR_HIGH_PHYS_MASK);
+ phys_addr_highbits = (val & MVPP22_BM_ADDR_HIGH_VIRT_MASK) >>
+ MVPP22_BM_ADDR_HIGH_VIRT_SHIFT;
+
+ if (sizeof(dma_addr_t) == 8)
+ *dma_addr |= (u64)dma_addr_highbits << 32;
+
+ if (sizeof(phys_addr_t) == 8)
+ *phys_addr |= (u64)phys_addr_highbits << 32;
+ }
+}
+
/* Free all buffers from the pool */
static void mvpp2_bm_bufs_free(struct device *dev, struct mvpp2 *priv,
struct mvpp2_bm_pool *bm_pool)
int i;
for (i = 0; i < bm_pool->buf_num; i++) {
- dma_addr_t buf_phys_addr;
- unsigned long vaddr;
+ dma_addr_t buf_dma_addr;
+ phys_addr_t buf_phys_addr;
+ void *data;
- /* Get buffer virtual address (indirect access) */
- buf_phys_addr = mvpp2_read(priv,
- MVPP2_BM_PHY_ALLOC_REG(bm_pool->id));
- vaddr = mvpp2_read(priv, MVPP2_BM_VIRT_ALLOC_REG);
+ mvpp2_bm_bufs_get_addrs(dev, priv, bm_pool,
+ &buf_dma_addr, &buf_phys_addr);
- dma_unmap_single(dev, buf_phys_addr,
+ dma_unmap_single(dev, buf_dma_addr,
bm_pool->buf_size, DMA_FROM_DEVICE);
- if (!vaddr)
+ data = (void *)phys_to_virt(buf_phys_addr);
+ if (!data)
break;
- mvpp2_frag_free(bm_pool, (void *)vaddr);
+ mvpp2_frag_free(bm_pool, data);
}
/* Update BM driver with number of buffers removed from pool */
val |= MVPP2_BM_STOP_MASK;
mvpp2_write(priv, MVPP2_BM_POOL_CTRL_REG(bm_pool->id), val);
- dma_free_coherent(&pdev->dev, sizeof(u32) * bm_pool->size,
+ dma_free_coherent(&pdev->dev, bm_pool->size_bytes,
bm_pool->virt_addr,
- bm_pool->phys_addr);
+ bm_pool->dma_addr);
return 0;
}
static void mvpp2_rxq_long_pool_set(struct mvpp2_port *port,
int lrxq, int long_pool)
{
- u32 val;
+ u32 val, mask;
int prxq;
/* Get queue physical ID */
prxq = port->rxqs[lrxq]->id;
- val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(prxq));
- val &= ~MVPP2_RXQ_POOL_LONG_MASK;
- val |= ((long_pool << MVPP2_RXQ_POOL_LONG_OFFS) &
- MVPP2_RXQ_POOL_LONG_MASK);
+ if (port->priv->hw_version == MVPP21)
+ mask = MVPP21_RXQ_POOL_LONG_MASK;
+ else
+ mask = MVPP22_RXQ_POOL_LONG_MASK;
+ val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(prxq));
+ val &= ~mask;
+ val |= (long_pool << MVPP2_RXQ_POOL_LONG_OFFS) & mask;
mvpp2_write(port->priv, MVPP2_RXQ_CONFIG_REG(prxq), val);
}
static void mvpp2_rxq_short_pool_set(struct mvpp2_port *port,
int lrxq, int short_pool)
{
- u32 val;
+ u32 val, mask;
int prxq;
/* Get queue physical ID */
prxq = port->rxqs[lrxq]->id;
- val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(prxq));
- val &= ~MVPP2_RXQ_POOL_SHORT_MASK;
- val |= ((short_pool << MVPP2_RXQ_POOL_SHORT_OFFS) &
- MVPP2_RXQ_POOL_SHORT_MASK);
+ if (port->priv->hw_version == MVPP21)
+ mask = MVPP21_RXQ_POOL_SHORT_MASK;
+ else
+ mask = MVPP22_RXQ_POOL_SHORT_MASK;
+ val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(prxq));
+ val &= ~mask;
+ val |= (short_pool << MVPP2_RXQ_POOL_SHORT_OFFS) & mask;
mvpp2_write(port->priv, MVPP2_RXQ_CONFIG_REG(prxq), val);
}
static void *mvpp2_buf_alloc(struct mvpp2_port *port,
struct mvpp2_bm_pool *bm_pool,
- dma_addr_t *buf_phys_addr,
+ dma_addr_t *buf_dma_addr,
+ phys_addr_t *buf_phys_addr,
gfp_t gfp_mask)
{
- dma_addr_t phys_addr;
+ dma_addr_t dma_addr;
void *data;
data = mvpp2_frag_alloc(bm_pool);
if (!data)
return NULL;
- phys_addr = dma_map_single(port->dev->dev.parent, data,
- MVPP2_RX_BUF_SIZE(bm_pool->pkt_size),
- DMA_FROM_DEVICE);
- if (unlikely(dma_mapping_error(port->dev->dev.parent, phys_addr))) {
+ dma_addr = dma_map_single(port->dev->dev.parent, data,
+ MVPP2_RX_BUF_SIZE(bm_pool->pkt_size),
+ DMA_FROM_DEVICE);
+ if (unlikely(dma_mapping_error(port->dev->dev.parent, dma_addr))) {
mvpp2_frag_free(bm_pool, data);
return NULL;
}
- *buf_phys_addr = phys_addr;
+ *buf_dma_addr = dma_addr;
+ *buf_phys_addr = virt_to_phys(data);
return data;
}
/* Release buffer to BM */
static inline void mvpp2_bm_pool_put(struct mvpp2_port *port, int pool,
- dma_addr_t buf_phys_addr,
- unsigned long buf_virt_addr)
+ dma_addr_t buf_dma_addr,
+ phys_addr_t buf_phys_addr)
{
- mvpp2_write(port->priv, MVPP2_BM_VIRT_RLS_REG, buf_virt_addr);
- mvpp2_write(port->priv, MVPP2_BM_PHY_RLS_REG(pool), buf_phys_addr);
-}
+ int cpu = smp_processor_id();
-/* Release multicast buffer */
-static void mvpp2_bm_pool_mc_put(struct mvpp2_port *port, int pool,
- dma_addr_t buf_phys_addr,
- unsigned long buf_virt_addr,
- int mc_id)
-{
- u32 val = 0;
+ if (port->priv->hw_version == MVPP22) {
+ u32 val = 0;
+
+ if (sizeof(dma_addr_t) == 8)
+ val |= upper_32_bits(buf_dma_addr) &
+ MVPP22_BM_ADDR_HIGH_PHYS_RLS_MASK;
+
+ if (sizeof(phys_addr_t) == 8)
+ val |= (upper_32_bits(buf_phys_addr)
+ << MVPP22_BM_ADDR_HIGH_VIRT_RLS_SHIFT) &
+ MVPP22_BM_ADDR_HIGH_VIRT_RLS_MASK;
- val |= (mc_id & MVPP2_BM_MC_ID_MASK);
- mvpp2_write(port->priv, MVPP2_BM_MC_RLS_REG, val);
+ mvpp2_percpu_write(port->priv, cpu,
+ MVPP22_BM_ADDR_HIGH_RLS_REG, val);
+ }
- mvpp2_bm_pool_put(port, pool,
- buf_phys_addr | MVPP2_BM_PHY_RLS_MC_BUFF_MASK,
- buf_virt_addr);
+ /* MVPP2_BM_VIRT_RLS_REG is not interpreted by HW, and simply
+ * returned in the "cookie" field of the RX
+ * descriptor. Instead of storing the virtual address, we
+ * store the physical address
+ */
+ mvpp2_percpu_write(port->priv, cpu,
+ MVPP2_BM_VIRT_RLS_REG, buf_phys_addr);
+ mvpp2_percpu_write(port->priv, cpu,
+ MVPP2_BM_PHY_RLS_REG(pool), buf_dma_addr);
}
/* Refill BM pool */
static void mvpp2_pool_refill(struct mvpp2_port *port, u32 bm,
- dma_addr_t phys_addr,
- unsigned long cookie)
+ dma_addr_t dma_addr,
+ phys_addr_t phys_addr)
{
int pool = mvpp2_bm_cookie_pool_get(bm);
- mvpp2_bm_pool_put(port, pool, phys_addr, cookie);
+ mvpp2_bm_pool_put(port, pool, dma_addr, phys_addr);
}
/* Allocate buffers for the pool */
struct mvpp2_bm_pool *bm_pool, int buf_num)
{
int i, buf_size, total_size;
- dma_addr_t phys_addr;
+ dma_addr_t dma_addr;
+ phys_addr_t phys_addr;
void *buf;
buf_size = MVPP2_RX_BUF_SIZE(bm_pool->pkt_size);
}
for (i = 0; i < buf_num; i++) {
- buf = mvpp2_buf_alloc(port, bm_pool, &phys_addr, GFP_KERNEL);
+ buf = mvpp2_buf_alloc(port, bm_pool, &dma_addr,
+ &phys_addr, GFP_KERNEL);
if (!buf)
break;
- mvpp2_bm_pool_put(port, bm_pool->id, phys_addr,
- (unsigned long)buf);
+ mvpp2_bm_pool_put(port, bm_pool->id, dma_addr,
+ phys_addr);
}
/* Update BM driver with number of buffers added to pool */
{
struct mvpp2_port *port = arg;
- mvpp2_write(port->priv, MVPP2_ISR_RX_TX_MASK_REG(port->id), 0);
+ mvpp2_percpu_write(port->priv, smp_processor_id(),
+ MVPP2_ISR_RX_TX_MASK_REG(port->id), 0);
}
/* Unmask the current CPU's Rx/Tx interrupts */
{
struct mvpp2_port *port = arg;
- mvpp2_write(port->priv, MVPP2_ISR_RX_TX_MASK_REG(port->id),
- (MVPP2_CAUSE_MISC_SUM_MASK |
- MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK));
+ mvpp2_percpu_write(port->priv, smp_processor_id(),
+ MVPP2_ISR_RX_TX_MASK_REG(port->id),
+ (MVPP2_CAUSE_MISC_SUM_MASK |
+ MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK));
}
/* Port configuration routines */
+static void mvpp22_port_mii_set(struct mvpp2_port *port)
+{
+ u32 val;
+
+ return;
+
+ /* Only GOP port 0 has an XLG MAC */
+ if (port->gop_id == 0) {
+ val = readl(port->base + MVPP22_XLG_CTRL3_REG);
+ val &= ~MVPP22_XLG_CTRL3_MACMODESELECT_MASK;
+ val |= MVPP22_XLG_CTRL3_MACMODESELECT_GMAC;
+ writel(val, port->base + MVPP22_XLG_CTRL3_REG);
+ }
+
+ val = readl(port->base + MVPP22_GMAC_CTRL_4_REG);
+ if (port->phy_interface == PHY_INTERFACE_MODE_RGMII)
+ val |= MVPP22_CTRL4_EXT_PIN_GMII_SEL;
+ else
+ val &= ~MVPP22_CTRL4_EXT_PIN_GMII_SEL;
+ val &= ~MVPP22_CTRL4_DP_CLK_SEL;
+ val |= MVPP22_CTRL4_SYNC_BYPASS;
+ val |= MVPP22_CTRL4_QSGMII_BYPASS_ACTIVE;
+ writel(val, port->base + MVPP22_GMAC_CTRL_4_REG);
+}
+
static void mvpp2_port_mii_set(struct mvpp2_port *port)
{
u32 val;
+ if (port->priv->hw_version == MVPP22)
+ mvpp22_port_mii_set(port);
+
val = readl(port->base + MVPP2_GMAC_CTRL_2_REG);
switch (port->phy_interface) {
{
int tx_port_num, val, queue, ptxq, lrxq;
- /* Configure port to loopback if needed */
- if (port->flags & MVPP2_F_LOOPBACK)
- mvpp2_port_loopback_set(port);
+ if (port->priv->hw_version == MVPP21) {
+ /* Configure port to loopback if needed */
+ if (port->flags & MVPP2_F_LOOPBACK)
+ mvpp2_port_loopback_set(port);
- /* Update TX FIFO MIN Threshold */
- val = readl(port->base + MVPP2_GMAC_PORT_FIFO_CFG_1_REG);
- val &= ~MVPP2_GMAC_TX_FIFO_MIN_TH_ALL_MASK;
- /* Min. TX threshold must be less than minimal packet length */
- val |= MVPP2_GMAC_TX_FIFO_MIN_TH_MASK(64 - 4 - 2);
- writel(val, port->base + MVPP2_GMAC_PORT_FIFO_CFG_1_REG);
+ /* Update TX FIFO MIN Threshold */
+ val = readl(port->base + MVPP2_GMAC_PORT_FIFO_CFG_1_REG);
+ val &= ~MVPP2_GMAC_TX_FIFO_MIN_TH_ALL_MASK;
+ /* Min. TX threshold must be less than minimal packet length */
+ val |= MVPP2_GMAC_TX_FIFO_MIN_TH_MASK(64 - 4 - 2);
+ writel(val, port->base + MVPP2_GMAC_PORT_FIFO_CFG_1_REG);
+ }
/* Disable Legacy WRR, Disable EJP, Release from reset */
tx_port_num = mvpp2_egress_port(port);
}
/* Obtain BM cookie information from descriptor */
-static u32 mvpp2_bm_cookie_build(struct mvpp2_rx_desc *rx_desc)
+static u32 mvpp2_bm_cookie_build(struct mvpp2_port *port,
+ struct mvpp2_rx_desc *rx_desc)
{
- int pool = (rx_desc->status & MVPP2_RXD_BM_POOL_ID_MASK) >>
- MVPP2_RXD_BM_POOL_ID_OFFS;
int cpu = smp_processor_id();
+ int pool;
+
+ pool = (mvpp2_rxdesc_status_get(port, rx_desc) &
+ MVPP2_RXD_BM_POOL_ID_MASK) >>
+ MVPP2_RXD_BM_POOL_ID_OFFS;
return ((pool & 0xFF) << MVPP2_BM_COOKIE_POOL_OFFS) |
((cpu & 0xFF) << MVPP2_BM_COOKIE_CPU_OFFS);
/* Tx descriptors helper methods */
-/* Get number of Tx descriptors waiting to be transmitted by HW */
-static int mvpp2_txq_pend_desc_num_get(struct mvpp2_port *port,
- struct mvpp2_tx_queue *txq)
-{
- u32 val;
-
- mvpp2_write(port->priv, MVPP2_TXQ_NUM_REG, txq->id);
- val = mvpp2_read(port->priv, MVPP2_TXQ_PENDING_REG);
-
- return val & MVPP2_TXQ_PENDING_MASK;
-}
-
/* Get pointer to next Tx descriptor to be processed (send) by HW */
static struct mvpp2_tx_desc *
mvpp2_txq_next_desc_get(struct mvpp2_tx_queue *txq)
static void mvpp2_aggr_txq_pend_desc_add(struct mvpp2_port *port, int pending)
{
/* aggregated access - relevant TXQ number is written in TX desc */
- mvpp2_write(port->priv, MVPP2_AGGR_TXQ_UPDATE_REG, pending);
+ mvpp2_percpu_write(port->priv, smp_processor_id(),
+ MVPP2_AGGR_TXQ_UPDATE_REG, pending);
}
struct mvpp2_tx_queue *txq, int num)
{
u32 val;
+ int cpu = smp_processor_id();
val = (txq->id << MVPP2_TXQ_RSVD_REQ_Q_OFFSET) | num;
- mvpp2_write(priv, MVPP2_TXQ_RSVD_REQ_REG, val);
+ mvpp2_percpu_write(priv, cpu, MVPP2_TXQ_RSVD_REQ_REG, val);
- val = mvpp2_read(priv, MVPP2_TXQ_RSVD_RSLT_REG);
+ val = mvpp2_percpu_read(priv, cpu, MVPP2_TXQ_RSVD_RSLT_REG);
return val & MVPP2_TXQ_RSVD_RSLT_MASK;
}
u32 val;
/* Reading status reg resets transmitted descriptor counter */
- val = mvpp2_read(port->priv, MVPP2_TXQ_SENT_REG(txq->id));
+ val = mvpp2_percpu_read(port->priv, smp_processor_id(),
+ MVPP2_TXQ_SENT_REG(txq->id));
return (val & MVPP2_TRANSMITTED_COUNT_MASK) >>
MVPP2_TRANSMITTED_COUNT_OFFSET;
for (queue = 0; queue < txq_number; queue++) {
int id = port->txqs[queue]->id;
- mvpp2_read(port->priv, MVPP2_TXQ_SENT_REG(id));
+ mvpp2_percpu_read(port->priv, smp_processor_id(),
+ MVPP2_TXQ_SENT_REG(id));
}
}
static void mvpp2_rx_pkts_coal_set(struct mvpp2_port *port,
struct mvpp2_rx_queue *rxq)
{
+ int cpu = smp_processor_id();
+
if (rxq->pkts_coal > MVPP2_OCCUPIED_THRESH_MASK)
rxq->pkts_coal = MVPP2_OCCUPIED_THRESH_MASK;
- mvpp2_write(port->priv, MVPP2_RXQ_NUM_REG, rxq->id);
- mvpp2_write(port->priv, MVPP2_RXQ_THRESH_REG,
- rxq->pkts_coal);
+ mvpp2_percpu_write(port->priv, cpu, MVPP2_RXQ_NUM_REG, rxq->id);
+ mvpp2_percpu_write(port->priv, cpu, MVPP2_RXQ_THRESH_REG,
+ rxq->pkts_coal);
}
static u32 mvpp2_usec_to_cycles(u32 usec, unsigned long clk_hz)
struct mvpp2_txq_pcpu_buf *tx_buf =
txq_pcpu->buffs + txq_pcpu->txq_get_index;
- dma_unmap_single(port->dev->dev.parent, tx_buf->phys,
+ dma_unmap_single(port->dev->dev.parent, tx_buf->dma,
tx_buf->size, DMA_TO_DEVICE);
if (tx_buf->skb)
dev_kfree_skb_any(tx_buf->skb);
int desc_num, int cpu,
struct mvpp2 *priv)
{
+ u32 txq_dma;
+
/* Allocate memory for TX descriptors */
aggr_txq->descs = dma_alloc_coherent(&pdev->dev,
desc_num * MVPP2_DESC_ALIGNED_SIZE,
- &aggr_txq->descs_phys, GFP_KERNEL);
+ &aggr_txq->descs_dma, GFP_KERNEL);
if (!aggr_txq->descs)
return -ENOMEM;
aggr_txq->next_desc_to_proc = mvpp2_read(priv,
MVPP2_AGGR_TXQ_INDEX_REG(cpu));
- /* Set Tx descriptors queue starting address */
- /* indirect access */
- mvpp2_write(priv, MVPP2_AGGR_TXQ_DESC_ADDR_REG(cpu),
- aggr_txq->descs_phys);
+ /* Set Tx descriptors queue starting address indirect
+ * access
+ */
+ if (priv->hw_version == MVPP21)
+ txq_dma = aggr_txq->descs_dma;
+ else
+ txq_dma = aggr_txq->descs_dma >>
+ MVPP22_AGGR_TXQ_DESC_ADDR_OFFS;
+
+ mvpp2_write(priv, MVPP2_AGGR_TXQ_DESC_ADDR_REG(cpu), txq_dma);
mvpp2_write(priv, MVPP2_AGGR_TXQ_DESC_SIZE_REG(cpu), desc_num);
return 0;
struct mvpp2_rx_queue *rxq)
{
+ u32 rxq_dma;
+ int cpu;
+
rxq->size = port->rx_ring_size;
/* Allocate memory for RX descriptors */
rxq->descs = dma_alloc_coherent(port->dev->dev.parent,
rxq->size * MVPP2_DESC_ALIGNED_SIZE,
- &rxq->descs_phys, GFP_KERNEL);
+ &rxq->descs_dma, GFP_KERNEL);
if (!rxq->descs)
return -ENOMEM;
mvpp2_write(port->priv, MVPP2_RXQ_STATUS_REG(rxq->id), 0);
/* Set Rx descriptors queue starting address - indirect access */
- mvpp2_write(port->priv, MVPP2_RXQ_NUM_REG, rxq->id);
- mvpp2_write(port->priv, MVPP2_RXQ_DESC_ADDR_REG, rxq->descs_phys);
- mvpp2_write(port->priv, MVPP2_RXQ_DESC_SIZE_REG, rxq->size);
- mvpp2_write(port->priv, MVPP2_RXQ_INDEX_REG, 0);
+ cpu = smp_processor_id();
+ mvpp2_percpu_write(port->priv, cpu, MVPP2_RXQ_NUM_REG, rxq->id);
+ if (port->priv->hw_version == MVPP21)
+ rxq_dma = rxq->descs_dma;
+ else
+ rxq_dma = rxq->descs_dma >> MVPP22_DESC_ADDR_OFFS;
+ mvpp2_percpu_write(port->priv, cpu, MVPP2_RXQ_DESC_ADDR_REG, rxq_dma);
+ mvpp2_percpu_write(port->priv, cpu, MVPP2_RXQ_DESC_SIZE_REG, rxq->size);
+ mvpp2_percpu_write(port->priv, cpu, MVPP2_RXQ_INDEX_REG, 0);
/* Set Offset */
mvpp2_rxq_offset_set(port, rxq->id, NET_SKB_PAD);
for (i = 0; i < rx_received; i++) {
struct mvpp2_rx_desc *rx_desc = mvpp2_rxq_next_desc_get(rxq);
- u32 bm = mvpp2_bm_cookie_build(rx_desc);
+ u32 bm = mvpp2_bm_cookie_build(port, rx_desc);
- mvpp2_pool_refill(port, bm, rx_desc->buf_phys_addr,
- rx_desc->buf_cookie);
+ mvpp2_pool_refill(port, bm,
+ mvpp2_rxdesc_dma_addr_get(port, rx_desc),
+ mvpp2_rxdesc_cookie_get(port, rx_desc));
}
mvpp2_rxq_status_update(port, rxq->id, rx_received, rx_received);
}
static void mvpp2_rxq_deinit(struct mvpp2_port *port,
struct mvpp2_rx_queue *rxq)
{
+ int cpu;
+
mvpp2_rxq_drop_pkts(port, rxq);
if (rxq->descs)
dma_free_coherent(port->dev->dev.parent,
rxq->size * MVPP2_DESC_ALIGNED_SIZE,
rxq->descs,
- rxq->descs_phys);
+ rxq->descs_dma);
rxq->descs = NULL;
rxq->last_desc = 0;
rxq->next_desc_to_proc = 0;
- rxq->descs_phys = 0;
+ rxq->descs_dma = 0;
/* Clear Rx descriptors queue starting address and size;
* free descriptor number
*/
mvpp2_write(port->priv, MVPP2_RXQ_STATUS_REG(rxq->id), 0);
- mvpp2_write(port->priv, MVPP2_RXQ_NUM_REG, rxq->id);
- mvpp2_write(port->priv, MVPP2_RXQ_DESC_ADDR_REG, 0);
- mvpp2_write(port->priv, MVPP2_RXQ_DESC_SIZE_REG, 0);
+ cpu = smp_processor_id();
+ mvpp2_percpu_write(port->priv, cpu, MVPP2_RXQ_NUM_REG, rxq->id);
+ mvpp2_percpu_write(port->priv, cpu, MVPP2_RXQ_DESC_ADDR_REG, 0);
+ mvpp2_percpu_write(port->priv, cpu, MVPP2_RXQ_DESC_SIZE_REG, 0);
}
/* Create and initialize a Tx queue */
/* Allocate memory for Tx descriptors */
txq->descs = dma_alloc_coherent(port->dev->dev.parent,
txq->size * MVPP2_DESC_ALIGNED_SIZE,
- &txq->descs_phys, GFP_KERNEL);
+ &txq->descs_dma, GFP_KERNEL);
if (!txq->descs)
return -ENOMEM;
txq->last_desc = txq->size - 1;
/* Set Tx descriptors queue starting address - indirect access */
- mvpp2_write(port->priv, MVPP2_TXQ_NUM_REG, txq->id);
- mvpp2_write(port->priv, MVPP2_TXQ_DESC_ADDR_REG, txq->descs_phys);
- mvpp2_write(port->priv, MVPP2_TXQ_DESC_SIZE_REG, txq->size &
- MVPP2_TXQ_DESC_SIZE_MASK);
- mvpp2_write(port->priv, MVPP2_TXQ_INDEX_REG, 0);
- mvpp2_write(port->priv, MVPP2_TXQ_RSVD_CLR_REG,
- txq->id << MVPP2_TXQ_RSVD_CLR_OFFSET);
- val = mvpp2_read(port->priv, MVPP2_TXQ_PENDING_REG);
+ cpu = smp_processor_id();
+ mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_NUM_REG, txq->id);
+ mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_DESC_ADDR_REG,
+ txq->descs_dma);
+ mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_DESC_SIZE_REG,
+ txq->size & MVPP2_TXQ_DESC_SIZE_MASK);
+ mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_INDEX_REG, 0);
+ mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_RSVD_CLR_REG,
+ txq->id << MVPP2_TXQ_RSVD_CLR_OFFSET);
+ val = mvpp2_percpu_read(port->priv, cpu, MVPP2_TXQ_PENDING_REG);
val &= ~MVPP2_TXQ_PENDING_MASK;
- mvpp2_write(port->priv, MVPP2_TXQ_PENDING_REG, val);
+ mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_PENDING_REG, val);
/* Calculate base address in prefetch buffer. We reserve 16 descriptors
* for each existing TXQ.
desc = (port->id * MVPP2_MAX_TXQ * desc_per_txq) +
(txq->log_id * desc_per_txq);
- mvpp2_write(port->priv, MVPP2_TXQ_PREF_BUF_REG,
- MVPP2_PREF_BUF_PTR(desc) | MVPP2_PREF_BUF_SIZE_16 |
- MVPP2_PREF_BUF_THRESH(desc_per_txq/2));
+ mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_PREF_BUF_REG,
+ MVPP2_PREF_BUF_PTR(desc) | MVPP2_PREF_BUF_SIZE_16 |
+ MVPP2_PREF_BUF_THRESH(desc_per_txq / 2));
/* WRR / EJP configuration - indirect access */
tx_port_num = mvpp2_egress_port(port);
dma_free_coherent(port->dev->dev.parent,
txq->size * MVPP2_DESC_ALIGNED_SIZE,
- txq->descs, txq->descs_phys);
+ txq->descs, txq->descs_dma);
return -ENOMEM;
}
if (txq->descs)
dma_free_coherent(port->dev->dev.parent,
txq->size * MVPP2_DESC_ALIGNED_SIZE,
- txq->descs, txq->descs_phys);
+ txq->descs, txq->descs_dma);
txq->descs = NULL;
txq->last_desc = 0;
txq->next_desc_to_proc = 0;
- txq->descs_phys = 0;
+ txq->descs_dma = 0;
/* Set minimum bandwidth for disabled TXQs */
mvpp2_write(port->priv, MVPP2_TXQ_SCHED_TOKEN_CNTR_REG(txq->id), 0);
/* Set Tx descriptors queue starting address and size */
- mvpp2_write(port->priv, MVPP2_TXQ_NUM_REG, txq->id);
- mvpp2_write(port->priv, MVPP2_TXQ_DESC_ADDR_REG, 0);
- mvpp2_write(port->priv, MVPP2_TXQ_DESC_SIZE_REG, 0);
+ cpu = smp_processor_id();
+ mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_NUM_REG, txq->id);
+ mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_DESC_ADDR_REG, 0);
+ mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_DESC_SIZE_REG, 0);
}
/* Cleanup Tx ports */
int delay, pending, cpu;
u32 val;
- mvpp2_write(port->priv, MVPP2_TXQ_NUM_REG, txq->id);
- val = mvpp2_read(port->priv, MVPP2_TXQ_PREF_BUF_REG);
+ cpu = smp_processor_id();
+ mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_NUM_REG, txq->id);
+ val = mvpp2_percpu_read(port->priv, cpu, MVPP2_TXQ_PREF_BUF_REG);
val |= MVPP2_TXQ_DRAIN_EN_MASK;
- mvpp2_write(port->priv, MVPP2_TXQ_PREF_BUF_REG, val);
+ mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_PREF_BUF_REG, val);
/* The napi queue has been stopped so wait for all packets
* to be transmitted.
mdelay(1);
delay++;
- pending = mvpp2_txq_pend_desc_num_get(port, txq);
+ pending = mvpp2_percpu_read(port->priv, cpu,
+ MVPP2_TXQ_PENDING_REG);
+ pending &= MVPP2_TXQ_PENDING_MASK;
} while (pending);
val &= ~MVPP2_TXQ_DRAIN_EN_MASK;
- mvpp2_write(port->priv, MVPP2_TXQ_PREF_BUF_REG, val);
+ mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_PREF_BUF_REG, val);
for_each_present_cpu(cpu) {
txq_pcpu = per_cpu_ptr(txq->pcpu, cpu);
static void mvpp2_rx_error(struct mvpp2_port *port,
struct mvpp2_rx_desc *rx_desc)
{
- u32 status = rx_desc->status;
+ u32 status = mvpp2_rxdesc_status_get(port, rx_desc);
+ size_t sz = mvpp2_rxdesc_size_get(port, rx_desc);
switch (status & MVPP2_RXD_ERR_CODE_MASK) {
case MVPP2_RXD_ERR_CRC:
- netdev_err(port->dev, "bad rx status %08x (crc error), size=%d\n",
- status, rx_desc->data_size);
+ netdev_err(port->dev, "bad rx status %08x (crc error), size=%zu\n",
+ status, sz);
break;
case MVPP2_RXD_ERR_OVERRUN:
- netdev_err(port->dev, "bad rx status %08x (overrun error), size=%d\n",
- status, rx_desc->data_size);
+ netdev_err(port->dev, "bad rx status %08x (overrun error), size=%zu\n",
+ status, sz);
break;
case MVPP2_RXD_ERR_RESOURCE:
- netdev_err(port->dev, "bad rx status %08x (resource error), size=%d\n",
- status, rx_desc->data_size);
+ netdev_err(port->dev, "bad rx status %08x (resource error), size=%zu\n",
+ status, sz);
break;
}
}
static int mvpp2_rx_refill(struct mvpp2_port *port,
struct mvpp2_bm_pool *bm_pool, u32 bm)
{
- dma_addr_t phys_addr;
+ dma_addr_t dma_addr;
+ phys_addr_t phys_addr;
void *buf;
/* No recycle or too many buffers are in use, so allocate a new skb */
- buf = mvpp2_buf_alloc(port, bm_pool, &phys_addr, GFP_ATOMIC);
+ buf = mvpp2_buf_alloc(port, bm_pool, &dma_addr, &phys_addr,
+ GFP_ATOMIC);
if (!buf)
return -ENOMEM;
- mvpp2_pool_refill(port, bm, phys_addr, (unsigned long)buf);
+ mvpp2_pool_refill(port, bm, dma_addr, phys_addr);
return 0;
}
return MVPP2_TXD_L4_CSUM_NOT | MVPP2_TXD_IP_CSUM_DISABLE;
}
-static void mvpp2_buff_hdr_rx(struct mvpp2_port *port,
- struct mvpp2_rx_desc *rx_desc)
-{
- struct mvpp2_buff_hdr *buff_hdr;
- struct sk_buff *skb;
- u32 rx_status = rx_desc->status;
- dma_addr_t buff_phys_addr;
- unsigned long buff_virt_addr;
- dma_addr_t buff_phys_addr_next;
- unsigned long buff_virt_addr_next;
- int mc_id;
- int pool_id;
-
- pool_id = (rx_status & MVPP2_RXD_BM_POOL_ID_MASK) >>
- MVPP2_RXD_BM_POOL_ID_OFFS;
- buff_phys_addr = rx_desc->buf_phys_addr;
- buff_virt_addr = rx_desc->buf_cookie;
-
- do {
- skb = (struct sk_buff *)buff_virt_addr;
- buff_hdr = (struct mvpp2_buff_hdr *)skb->head;
-
- mc_id = MVPP2_B_HDR_INFO_MC_ID(buff_hdr->info);
-
- buff_phys_addr_next = buff_hdr->next_buff_phys_addr;
- buff_virt_addr_next = buff_hdr->next_buff_virt_addr;
-
- /* Release buffer */
- mvpp2_bm_pool_mc_put(port, pool_id, buff_phys_addr,
- buff_virt_addr, mc_id);
-
- buff_phys_addr = buff_phys_addr_next;
- buff_virt_addr = buff_virt_addr_next;
-
- } while (!MVPP2_B_HDR_INFO_IS_LAST(buff_hdr->info));
-}
-
/* Main rx processing */
static int mvpp2_rx(struct mvpp2_port *port, int rx_todo,
struct mvpp2_rx_queue *rxq)
struct mvpp2_bm_pool *bm_pool;
struct sk_buff *skb;
unsigned int frag_size;
- dma_addr_t phys_addr;
+ dma_addr_t dma_addr;
+ phys_addr_t phys_addr;
u32 bm, rx_status;
int pool, rx_bytes, err;
void *data;
rx_done++;
- rx_status = rx_desc->status;
- rx_bytes = rx_desc->data_size - MVPP2_MH_SIZE;
- phys_addr = rx_desc->buf_phys_addr;
- data = (void *)(uintptr_t)rx_desc->buf_cookie;
-
- bm = mvpp2_bm_cookie_build(rx_desc);
+ rx_status = mvpp2_rxdesc_status_get(port, rx_desc);
+ rx_bytes = mvpp2_rxdesc_size_get(port, rx_desc);
+ rx_bytes -= MVPP2_MH_SIZE;
+ dma_addr = mvpp2_rxdesc_dma_addr_get(port, rx_desc);
+ phys_addr = mvpp2_rxdesc_cookie_get(port, rx_desc);
+ data = (void *)phys_to_virt(phys_addr);
+
+ bm = mvpp2_bm_cookie_build(port, rx_desc);
pool = mvpp2_bm_cookie_pool_get(bm);
bm_pool = &port->priv->bm_pools[pool];
- /* Check if buffer header is used */
- if (rx_status & MVPP2_RXD_BUF_HDR) {
- mvpp2_buff_hdr_rx(port, rx_desc);
- continue;
- }
/* In case of an error, release the requested buffer pointer
* to the Buffer Manager. This request process is controlled
dev->stats.rx_errors++;
mvpp2_rx_error(port, rx_desc);
/* Return the buffer to the pool */
-
- mvpp2_pool_refill(port, bm, rx_desc->buf_phys_addr,
- rx_desc->buf_cookie);
+ mvpp2_pool_refill(port, bm, dma_addr, phys_addr);
continue;
}
goto err_drop_frame;
}
- dma_unmap_single(dev->dev.parent, phys_addr,
+ dma_unmap_single(dev->dev.parent, dma_addr,
bm_pool->buf_size, DMA_FROM_DEVICE);
rcvd_pkts++;
}
static inline void
-tx_desc_unmap_put(struct device *dev, struct mvpp2_tx_queue *txq,
+tx_desc_unmap_put(struct mvpp2_port *port, struct mvpp2_tx_queue *txq,
struct mvpp2_tx_desc *desc)
{
- dma_unmap_single(dev, desc->buf_phys_addr,
- desc->data_size, DMA_TO_DEVICE);
+ dma_addr_t buf_dma_addr =
+ mvpp2_txdesc_dma_addr_get(port, desc);
+ size_t buf_sz =
+ mvpp2_txdesc_size_get(port, desc);
+ dma_unmap_single(port->dev->dev.parent, buf_dma_addr,
+ buf_sz, DMA_TO_DEVICE);
mvpp2_txq_desc_put(txq);
}
struct mvpp2_txq_pcpu *txq_pcpu = this_cpu_ptr(txq->pcpu);
struct mvpp2_tx_desc *tx_desc;
int i;
- dma_addr_t buf_phys_addr;
+ dma_addr_t buf_dma_addr;
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
void *addr = page_address(frag->page.p) + frag->page_offset;
tx_desc = mvpp2_txq_next_desc_get(aggr_txq);
- tx_desc->phys_txq = txq->id;
- tx_desc->data_size = frag->size;
+ mvpp2_txdesc_txq_set(port, tx_desc, txq->id);
+ mvpp2_txdesc_size_set(port, tx_desc, frag->size);
- buf_phys_addr = dma_map_single(port->dev->dev.parent, addr,
- tx_desc->data_size,
+ buf_dma_addr = dma_map_single(port->dev->dev.parent, addr,
+ frag->size,
DMA_TO_DEVICE);
- if (dma_mapping_error(port->dev->dev.parent, buf_phys_addr)) {
+ if (dma_mapping_error(port->dev->dev.parent, buf_dma_addr)) {
mvpp2_txq_desc_put(txq);
goto error;
}
- tx_desc->packet_offset = buf_phys_addr & MVPP2_TX_DESC_ALIGN;
- tx_desc->buf_phys_addr = buf_phys_addr & (~MVPP2_TX_DESC_ALIGN);
+ mvpp2_txdesc_offset_set(port, tx_desc,
+ buf_dma_addr & MVPP2_TX_DESC_ALIGN);
+ mvpp2_txdesc_dma_addr_set(port, tx_desc,
+ buf_dma_addr & ~MVPP2_TX_DESC_ALIGN);
if (i == (skb_shinfo(skb)->nr_frags - 1)) {
/* Last descriptor */
- tx_desc->command = MVPP2_TXD_L_DESC;
- mvpp2_txq_inc_put(txq_pcpu, skb, tx_desc);
+ mvpp2_txdesc_cmd_set(port, tx_desc,
+ MVPP2_TXD_L_DESC);
+ mvpp2_txq_inc_put(port, txq_pcpu, skb, tx_desc);
} else {
/* Descriptor in the middle: Not First, Not Last */
- tx_desc->command = 0;
- mvpp2_txq_inc_put(txq_pcpu, NULL, tx_desc);
+ mvpp2_txdesc_cmd_set(port, tx_desc, 0);
+ mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc);
}
}
*/
for (i = i - 1; i >= 0; i--) {
tx_desc = txq->descs + i;
- tx_desc_unmap_put(port->dev->dev.parent, txq, tx_desc);
+ tx_desc_unmap_put(port, txq, tx_desc);
}
return -ENOMEM;
struct mvpp2_tx_queue *txq, *aggr_txq;
struct mvpp2_txq_pcpu *txq_pcpu;
struct mvpp2_tx_desc *tx_desc;
- dma_addr_t buf_phys_addr;
+ dma_addr_t buf_dma_addr;
int frags = 0;
u16 txq_id;
u32 tx_cmd;
/* Get a descriptor for the first part of the packet */
tx_desc = mvpp2_txq_next_desc_get(aggr_txq);
- tx_desc->phys_txq = txq->id;
- tx_desc->data_size = skb_headlen(skb);
+ mvpp2_txdesc_txq_set(port, tx_desc, txq->id);
+ mvpp2_txdesc_size_set(port, tx_desc, skb_headlen(skb));
- buf_phys_addr = dma_map_single(dev->dev.parent, skb->data,
- tx_desc->data_size, DMA_TO_DEVICE);
- if (unlikely(dma_mapping_error(dev->dev.parent, buf_phys_addr))) {
+ buf_dma_addr = dma_map_single(dev->dev.parent, skb->data,
+ skb_headlen(skb), DMA_TO_DEVICE);
+ if (unlikely(dma_mapping_error(dev->dev.parent, buf_dma_addr))) {
mvpp2_txq_desc_put(txq);
frags = 0;
goto out;
}
- tx_desc->packet_offset = buf_phys_addr & MVPP2_TX_DESC_ALIGN;
- tx_desc->buf_phys_addr = buf_phys_addr & ~MVPP2_TX_DESC_ALIGN;
+
+ mvpp2_txdesc_offset_set(port, tx_desc,
+ buf_dma_addr & MVPP2_TX_DESC_ALIGN);
+ mvpp2_txdesc_dma_addr_set(port, tx_desc,
+ buf_dma_addr & ~MVPP2_TX_DESC_ALIGN);
tx_cmd = mvpp2_skb_tx_csum(port, skb);
if (frags == 1) {
/* First and Last descriptor */
tx_cmd |= MVPP2_TXD_F_DESC | MVPP2_TXD_L_DESC;
- tx_desc->command = tx_cmd;
- mvpp2_txq_inc_put(txq_pcpu, skb, tx_desc);
+ mvpp2_txdesc_cmd_set(port, tx_desc, tx_cmd);
+ mvpp2_txq_inc_put(port, txq_pcpu, skb, tx_desc);
} else {
/* First but not Last */
tx_cmd |= MVPP2_TXD_F_DESC | MVPP2_TXD_PADDING_DISABLE;
- tx_desc->command = tx_cmd;
- mvpp2_txq_inc_put(txq_pcpu, NULL, tx_desc);
+ mvpp2_txdesc_cmd_set(port, tx_desc, tx_cmd);
+ mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc);
/* Continue with other skb fragments */
if (mvpp2_tx_frag_process(port, skb, aggr_txq, txq)) {
- tx_desc_unmap_put(port->dev->dev.parent, txq, tx_desc);
+ tx_desc_unmap_put(port, txq, tx_desc);
frags = 0;
goto out;
}
u32 cause_rx_tx, cause_rx, cause_misc;
int rx_done = 0;
struct mvpp2_port *port = netdev_priv(napi->dev);
+ int cpu = smp_processor_id();
/* Rx/Tx cause register
*
*
* Each CPU has its own Rx/Tx cause register
*/
- cause_rx_tx = mvpp2_read(port->priv,
- MVPP2_ISR_RX_TX_CAUSE_REG(port->id));
+ cause_rx_tx = mvpp2_percpu_read(port->priv, cpu,
+ MVPP2_ISR_RX_TX_CAUSE_REG(port->id));
cause_rx_tx &= ~MVPP2_CAUSE_TXQ_OCCUP_DESC_ALL_MASK;
cause_misc = cause_rx_tx & MVPP2_CAUSE_MISC_SUM_MASK;
/* Clear the cause register */
mvpp2_write(port->priv, MVPP2_ISR_MISC_CAUSE_REG, 0);
- mvpp2_write(port->priv, MVPP2_ISR_RX_TX_CAUSE_REG(port->id),
- cause_rx_tx & ~MVPP2_CAUSE_MISC_SUM_MASK);
+ mvpp2_percpu_write(port->priv, cpu,
+ MVPP2_ISR_RX_TX_CAUSE_REG(port->id),
+ cause_rx_tx & ~MVPP2_CAUSE_MISC_SUM_MASK);
}
cause_rx = cause_rx_tx & MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK;
return 0;
}
-static void mvpp2_get_mac_address(struct mvpp2_port *port, unsigned char *addr)
+static void mvpp21_get_mac_address(struct mvpp2_port *port, unsigned char *addr)
{
u32 mac_addr_l, mac_addr_m, mac_addr_h;
.set_link_ksettings = phy_ethtool_set_link_ksettings,
};
-/* Driver initialization */
-
-static void mvpp2_port_power_up(struct mvpp2_port *port)
-{
- mvpp2_port_mii_set(port);
- mvpp2_port_periodic_xon_disable(port);
- mvpp2_port_fc_adv_enable(port);
- mvpp2_port_reset(port);
-}
-
/* Initialize port HW */
static int mvpp2_port_init(struct mvpp2_port *port)
{
struct mvpp2_txq_pcpu *txq_pcpu;
int queue, cpu, err;
- if (port->first_rxq + rxq_number > MVPP2_RXQ_TOTAL_NUM)
+ if (port->first_rxq + rxq_number >
+ MVPP2_MAX_PORTS * priv->max_port_rxqs)
return -EINVAL;
/* Disable port */
}
/* Configure Rx queue group interrupt for this port */
- mvpp2_write(priv, MVPP2_ISR_RXQ_GROUP_REG(port->id), rxq_number);
+ if (priv->hw_version == MVPP21) {
+ mvpp2_write(priv, MVPP21_ISR_RXQ_GROUP_REG(port->id),
+ rxq_number);
+ } else {
+ u32 val;
+
+ val = (port->id << MVPP22_ISR_RXQ_GROUP_INDEX_GROUP_OFFSET);
+ mvpp2_write(priv, MVPP22_ISR_RXQ_GROUP_INDEX_REG, val);
+
+ val = (rxq_number << MVPP22_ISR_RXQ_SUB_GROUP_SIZE_OFFSET);
+ mvpp2_write(priv, MVPP22_ISR_RXQ_SUB_GROUP_CONFIG_REG, val);
+ }
/* Create Rx descriptor rings */
for (queue = 0; queue < rxq_number; queue++) {
/* Ports initialization */
static int mvpp2_port_probe(struct platform_device *pdev,
struct device_node *port_node,
- struct mvpp2 *priv,
- int *next_first_rxq)
+ struct mvpp2 *priv)
{
struct device_node *phy_node;
struct mvpp2_port *port;
u32 id;
int features;
int phy_mode;
- int priv_common_regs_num = 2;
int err, i, cpu;
dev = alloc_etherdev_mqs(sizeof(struct mvpp2_port), txq_number,
port->priv = priv;
port->id = id;
- port->first_rxq = *next_first_rxq;
+ if (priv->hw_version == MVPP21)
+ port->first_rxq = port->id * rxq_number;
+ else
+ port->first_rxq = port->id * priv->max_port_rxqs;
+
port->phy_node = phy_node;
port->phy_interface = phy_mode;
- res = platform_get_resource(pdev, IORESOURCE_MEM,
- priv_common_regs_num + id);
- port->base = devm_ioremap_resource(&pdev->dev, res);
- if (IS_ERR(port->base)) {
- err = PTR_ERR(port->base);
- goto err_free_irq;
+ if (priv->hw_version == MVPP21) {
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 2 + id);
+ port->base = devm_ioremap_resource(&pdev->dev, res);
+ if (IS_ERR(port->base)) {
+ err = PTR_ERR(port->base);
+ goto err_free_irq;
+ }
+ } else {
+ if (of_property_read_u32(port_node, "gop-port-id",
+ &port->gop_id)) {
+ err = -EINVAL;
+ dev_err(&pdev->dev, "missing gop-port-id value\n");
+ goto err_free_irq;
+ }
+
+ port->base = priv->iface_base + MVPP22_GMAC_BASE(port->gop_id);
}
/* Alloc per-cpu stats */
mac_from = "device tree";
ether_addr_copy(dev->dev_addr, dt_mac_addr);
} else {
- mvpp2_get_mac_address(port, hw_mac_addr);
+ if (priv->hw_version == MVPP21)
+ mvpp21_get_mac_address(port, hw_mac_addr);
if (is_valid_ether_addr(hw_mac_addr)) {
mac_from = "hardware";
ether_addr_copy(dev->dev_addr, hw_mac_addr);
dev_err(&pdev->dev, "failed to init port %d\n", id);
goto err_free_stats;
}
- mvpp2_port_power_up(port);
+
+ mvpp2_port_mii_set(port);
+ mvpp2_port_periodic_xon_disable(port);
+
+ if (priv->hw_version == MVPP21)
+ mvpp2_port_fc_adv_enable(port);
+
+ mvpp2_port_reset(port);
port->pcpu = alloc_percpu(struct mvpp2_port_pcpu);
if (!port->pcpu) {
}
netdev_info(dev, "Using %s mac address %pM\n", mac_from, dev->dev_addr);
- /* Increment the first Rx queue number to be used by the next port */
- *next_first_rxq += rxq_number;
priv->port_list[id] = port;
return 0;
mvpp2_write(priv, MVPP2_RX_FIFO_INIT_REG, 0x1);
}
+static void mvpp2_axi_init(struct mvpp2 *priv)
+{
+ u32 val, rdval, wrval;
+
+ mvpp2_write(priv, MVPP22_BM_ADDR_HIGH_RLS_REG, 0x0);
+
+ /* AXI Bridge Configuration */
+
+ rdval = MVPP22_AXI_CODE_CACHE_RD_CACHE
+ << MVPP22_AXI_ATTR_CACHE_OFFS;
+ rdval |= MVPP22_AXI_CODE_DOMAIN_OUTER_DOM
+ << MVPP22_AXI_ATTR_DOMAIN_OFFS;
+
+ wrval = MVPP22_AXI_CODE_CACHE_WR_CACHE
+ << MVPP22_AXI_ATTR_CACHE_OFFS;
+ wrval |= MVPP22_AXI_CODE_DOMAIN_OUTER_DOM
+ << MVPP22_AXI_ATTR_DOMAIN_OFFS;
+
+ /* BM */
+ mvpp2_write(priv, MVPP22_AXI_BM_WR_ATTR_REG, wrval);
+ mvpp2_write(priv, MVPP22_AXI_BM_RD_ATTR_REG, rdval);
+
+ /* Descriptors */
+ mvpp2_write(priv, MVPP22_AXI_AGGRQ_DESCR_RD_ATTR_REG, rdval);
+ mvpp2_write(priv, MVPP22_AXI_TXQ_DESCR_WR_ATTR_REG, wrval);
+ mvpp2_write(priv, MVPP22_AXI_TXQ_DESCR_RD_ATTR_REG, rdval);
+ mvpp2_write(priv, MVPP22_AXI_RXQ_DESCR_WR_ATTR_REG, wrval);
+
+ /* Buffer Data */
+ mvpp2_write(priv, MVPP22_AXI_TX_DATA_RD_ATTR_REG, rdval);
+ mvpp2_write(priv, MVPP22_AXI_RX_DATA_WR_ATTR_REG, wrval);
+
+ val = MVPP22_AXI_CODE_CACHE_NON_CACHE
+ << MVPP22_AXI_CODE_CACHE_OFFS;
+ val |= MVPP22_AXI_CODE_DOMAIN_SYSTEM
+ << MVPP22_AXI_CODE_DOMAIN_OFFS;
+ mvpp2_write(priv, MVPP22_AXI_RD_NORMAL_CODE_REG, val);
+ mvpp2_write(priv, MVPP22_AXI_WR_NORMAL_CODE_REG, val);
+
+ val = MVPP22_AXI_CODE_CACHE_RD_CACHE
+ << MVPP22_AXI_CODE_CACHE_OFFS;
+ val |= MVPP22_AXI_CODE_DOMAIN_OUTER_DOM
+ << MVPP22_AXI_CODE_DOMAIN_OFFS;
+
+ mvpp2_write(priv, MVPP22_AXI_RD_SNOOP_CODE_REG, val);
+
+ val = MVPP22_AXI_CODE_CACHE_WR_CACHE
+ << MVPP22_AXI_CODE_CACHE_OFFS;
+ val |= MVPP22_AXI_CODE_DOMAIN_OUTER_DOM
+ << MVPP22_AXI_CODE_DOMAIN_OFFS;
+
+ mvpp2_write(priv, MVPP22_AXI_WR_SNOOP_CODE_REG, val);
+}
+
/* Initialize network controller common part HW */
static int mvpp2_init(struct platform_device *pdev, struct mvpp2 *priv)
{
u32 val;
/* Checks for hardware constraints */
- if (rxq_number % 4 || (rxq_number > MVPP2_MAX_RXQ) ||
+ if (rxq_number % 4 || (rxq_number > priv->max_port_rxqs) ||
(txq_number > MVPP2_MAX_TXQ)) {
dev_err(&pdev->dev, "invalid queue size parameter\n");
return -EINVAL;
if (dram_target_info)
mvpp2_conf_mbus_windows(dram_target_info, priv);
+ if (priv->hw_version == MVPP22)
+ mvpp2_axi_init(priv);
+
/* Disable HW PHY polling */
- val = readl(priv->lms_base + MVPP2_PHY_AN_CFG0_REG);
- val |= MVPP2_PHY_AN_STOP_SMI0_MASK;
- writel(val, priv->lms_base + MVPP2_PHY_AN_CFG0_REG);
+ if (priv->hw_version == MVPP21) {
+ val = readl(priv->lms_base + MVPP2_PHY_AN_CFG0_REG);
+ val |= MVPP2_PHY_AN_STOP_SMI0_MASK;
+ writel(val, priv->lms_base + MVPP2_PHY_AN_CFG0_REG);
+ } else {
+ val = readl(priv->iface_base + MVPP22_SMI_MISC_CFG_REG);
+ val &= ~MVPP22_SMI_POLLING_EN;
+ writel(val, priv->iface_base + MVPP22_SMI_MISC_CFG_REG);
+ }
/* Allocate and initialize aggregated TXQs */
priv->aggr_txqs = devm_kcalloc(&pdev->dev, num_present_cpus(),
mvpp2_rx_fifo_init(priv);
/* Reset Rx queue group interrupt configuration */
- for (i = 0; i < MVPP2_MAX_PORTS; i++)
- mvpp2_write(priv, MVPP2_ISR_RXQ_GROUP_REG(i), rxq_number);
+ for (i = 0; i < MVPP2_MAX_PORTS; i++) {
+ if (priv->hw_version == MVPP21) {
+ mvpp2_write(priv, MVPP21_ISR_RXQ_GROUP_REG(i),
+ rxq_number);
+ continue;
+ } else {
+ u32 val;
+
+ val = (i << MVPP22_ISR_RXQ_GROUP_INDEX_GROUP_OFFSET);
+ mvpp2_write(priv, MVPP22_ISR_RXQ_GROUP_INDEX_REG, val);
- writel(MVPP2_EXT_GLOBAL_CTRL_DEFAULT,
- priv->lms_base + MVPP2_MNG_EXTENDED_GLOBAL_CTRL_REG);
+ val = (rxq_number << MVPP22_ISR_RXQ_SUB_GROUP_SIZE_OFFSET);
+ mvpp2_write(priv, MVPP22_ISR_RXQ_SUB_GROUP_CONFIG_REG, val);
+ }
+ }
+
+ if (priv->hw_version == MVPP21)
+ writel(MVPP2_EXT_GLOBAL_CTRL_DEFAULT,
+ priv->lms_base + MVPP2_MNG_EXTENDED_GLOBAL_CTRL_REG);
/* Allow cache snoop when transmiting packets */
mvpp2_write(priv, MVPP2_TX_SNOOP_REG, 0x1);
struct device_node *port_node;
struct mvpp2 *priv;
struct resource *res;
- int port_count, first_rxq;
+ void __iomem *base;
+ int port_count, cpu;
int err;
priv = devm_kzalloc(&pdev->dev, sizeof(struct mvpp2), GFP_KERNEL);
if (!priv)
return -ENOMEM;
+ priv->hw_version =
+ (unsigned long)of_device_get_match_data(&pdev->dev);
+
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- priv->base = devm_ioremap_resource(&pdev->dev, res);
- if (IS_ERR(priv->base))
- return PTR_ERR(priv->base);
+ base = devm_ioremap_resource(&pdev->dev, res);
+ if (IS_ERR(base))
+ return PTR_ERR(base);
+
+ if (priv->hw_version == MVPP21) {
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+ priv->lms_base = devm_ioremap_resource(&pdev->dev, res);
+ if (IS_ERR(priv->lms_base))
+ return PTR_ERR(priv->lms_base);
+ } else {
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+ priv->iface_base = devm_ioremap_resource(&pdev->dev, res);
+ if (IS_ERR(priv->iface_base))
+ return PTR_ERR(priv->iface_base);
+ }
+
+ for_each_present_cpu(cpu) {
+ u32 addr_space_sz;
+
+ addr_space_sz = (priv->hw_version == MVPP21 ?
+ MVPP21_ADDR_SPACE_SZ : MVPP22_ADDR_SPACE_SZ);
+ priv->cpu_base[cpu] = base + cpu * addr_space_sz;
+ }
- res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
- priv->lms_base = devm_ioremap_resource(&pdev->dev, res);
- if (IS_ERR(priv->lms_base))
- return PTR_ERR(priv->lms_base);
+ if (priv->hw_version == MVPP21)
+ priv->max_port_rxqs = 8;
+ else
+ priv->max_port_rxqs = 32;
priv->pp_clk = devm_clk_get(&pdev->dev, "pp_clk");
if (IS_ERR(priv->pp_clk))
if (err < 0)
goto err_pp_clk;
+ if (priv->hw_version == MVPP22) {
+ priv->mg_clk = devm_clk_get(&pdev->dev, "mg_clk");
+ if (IS_ERR(priv->mg_clk)) {
+ err = PTR_ERR(priv->mg_clk);
+ goto err_gop_clk;
+ }
+
+ err = clk_prepare_enable(priv->mg_clk);
+ if (err < 0)
+ goto err_gop_clk;
+ }
+
/* Get system's tclk rate */
priv->tclk = clk_get_rate(priv->pp_clk);
+ if (priv->hw_version == MVPP22) {
+ err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(40));
+ if (err)
+ goto err_mg_clk;
+ /* Sadly, the BM pools all share the same register to
+ * store the high 32 bits of their address. So they
+ * must all have the same high 32 bits, which forces
+ * us to restrict coherent memory to DMA_BIT_MASK(32).
+ */
+ err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
+ if (err)
+ goto err_mg_clk;
+ }
+
/* Initialize network controller */
err = mvpp2_init(pdev, priv);
if (err < 0) {
dev_err(&pdev->dev, "failed to initialize controller\n");
- goto err_gop_clk;
+ goto err_mg_clk;
}
port_count = of_get_available_child_count(dn);
if (port_count == 0) {
dev_err(&pdev->dev, "no ports enabled\n");
err = -ENODEV;
- goto err_gop_clk;
+ goto err_mg_clk;
}
priv->port_list = devm_kcalloc(&pdev->dev, port_count,
GFP_KERNEL);
if (!priv->port_list) {
err = -ENOMEM;
- goto err_gop_clk;
+ goto err_mg_clk;
}
/* Initialize ports */
- first_rxq = 0;
for_each_available_child_of_node(dn, port_node) {
- err = mvpp2_port_probe(pdev, port_node, priv, &first_rxq);
+ err = mvpp2_port_probe(pdev, port_node, priv);
if (err < 0)
- goto err_gop_clk;
+ goto err_mg_clk;
}
platform_set_drvdata(pdev, priv);
return 0;
+err_mg_clk:
+ if (priv->hw_version == MVPP22)
+ clk_disable_unprepare(priv->mg_clk);
err_gop_clk:
clk_disable_unprepare(priv->gop_clk);
err_pp_clk:
dma_free_coherent(&pdev->dev,
MVPP2_AGGR_TXQ_SIZE * MVPP2_DESC_ALIGNED_SIZE,
aggr_txq->descs,
- aggr_txq->descs_phys);
+ aggr_txq->descs_dma);
}
+ clk_disable_unprepare(priv->mg_clk);
clk_disable_unprepare(priv->pp_clk);
clk_disable_unprepare(priv->gop_clk);
}
static const struct of_device_id mvpp2_match[] = {
- { .compatible = "marvell,armada-375-pp2" },
+ {
+ .compatible = "marvell,armada-375-pp2",
+ .data = (void *)MVPP21,
+ },
+ {
+ .compatible = "marvell,armada-7k-pp22",
+ .data = (void *)MVPP22,
+ },
{ }
};
MODULE_DEVICE_TABLE(of, mvpp2_match);
/* If the mac address is invalid, use random mac address */
if (!is_valid_ether_addr(dev->dev_addr)) {
- random_ether_addr(dev->dev_addr);
+ eth_hw_addr_random(dev);
dev_err(eth->dev, "generated random MAC address %pM\n",
dev->dev_addr);
- dev->addr_assign_type = NET_ADDR_RANDOM;
}
return mtk_phy_connect(dev);
/* port statistics */
"tso_packets",
"xmit_more",
- "queue_stopped", "wake_queue", "tx_timeout", "rx_alloc_failed",
+ "queue_stopped", "wake_queue", "tx_timeout", "rx_alloc_pages",
"rx_csum_good", "rx_csum_none", "rx_csum_complete", "tx_chksum_offload",
/* pf statistics */
priv->port_stats.rx_chksum_good = 0;
priv->port_stats.rx_chksum_none = 0;
priv->port_stats.rx_chksum_complete = 0;
+ priv->port_stats.rx_alloc_pages = 0;
priv->xdp_stats.rx_xdp_drop = 0;
priv->xdp_stats.rx_xdp_tx = 0;
priv->xdp_stats.rx_xdp_tx_full = 0;
priv->port_stats.rx_chksum_good += READ_ONCE(ring->csum_ok);
priv->port_stats.rx_chksum_none += READ_ONCE(ring->csum_none);
priv->port_stats.rx_chksum_complete += READ_ONCE(ring->csum_complete);
+ priv->port_stats.rx_alloc_pages += READ_ONCE(ring->rx_alloc_pages);
priv->xdp_stats.rx_xdp_drop += READ_ONCE(ring->xdp_drop);
priv->xdp_stats.rx_xdp_tx += READ_ONCE(ring->xdp_tx);
priv->xdp_stats.rx_xdp_tx_full += READ_ONCE(ring->xdp_tx_full);
#include "mlx4_en.h"
-static int mlx4_alloc_pages(struct mlx4_en_priv *priv,
- struct mlx4_en_rx_alloc *page_alloc,
- const struct mlx4_en_frag_info *frag_info,
- gfp_t _gfp)
+static int mlx4_alloc_page(struct mlx4_en_priv *priv,
+ struct mlx4_en_rx_alloc *frag,
+ gfp_t gfp)
{
- int order;
struct page *page;
dma_addr_t dma;
- for (order = frag_info->order; ;) {
- gfp_t gfp = _gfp;
-
- if (order)
- gfp |= __GFP_COMP | __GFP_NOWARN | __GFP_NOMEMALLOC;
- page = alloc_pages(gfp, order);
- if (likely(page))
- break;
- if (--order < 0 ||
- ((PAGE_SIZE << order) < frag_info->frag_size))
- return -ENOMEM;
- }
- dma = dma_map_page(priv->ddev, page, 0, PAGE_SIZE << order,
- frag_info->dma_dir);
+ page = alloc_page(gfp);
+ if (unlikely(!page))
+ return -ENOMEM;
+ dma = dma_map_page(priv->ddev, page, 0, PAGE_SIZE, priv->dma_dir);
if (unlikely(dma_mapping_error(priv->ddev, dma))) {
- put_page(page);
+ __free_page(page);
return -ENOMEM;
}
- page_alloc->page_size = PAGE_SIZE << order;
- page_alloc->page = page;
- page_alloc->dma = dma;
- page_alloc->page_offset = 0;
- /* Not doing get_page() for each frag is a big win
- * on asymetric workloads. Note we can not use atomic_set().
- */
- page_ref_add(page, page_alloc->page_size / frag_info->frag_stride - 1);
+ frag->page = page;
+ frag->dma = dma;
+ frag->page_offset = priv->rx_headroom;
return 0;
}
static int mlx4_en_alloc_frags(struct mlx4_en_priv *priv,
+ struct mlx4_en_rx_ring *ring,
struct mlx4_en_rx_desc *rx_desc,
struct mlx4_en_rx_alloc *frags,
- struct mlx4_en_rx_alloc *ring_alloc,
gfp_t gfp)
{
- struct mlx4_en_rx_alloc page_alloc[MLX4_EN_MAX_RX_FRAGS];
- const struct mlx4_en_frag_info *frag_info;
- struct page *page;
int i;
- for (i = 0; i < priv->num_frags; i++) {
- frag_info = &priv->frag_info[i];
- page_alloc[i] = ring_alloc[i];
- page_alloc[i].page_offset += frag_info->frag_stride;
-
- if (page_alloc[i].page_offset + frag_info->frag_stride <=
- ring_alloc[i].page_size)
- continue;
-
- if (unlikely(mlx4_alloc_pages(priv, &page_alloc[i],
- frag_info, gfp)))
- goto out;
- }
-
- for (i = 0; i < priv->num_frags; i++) {
- frags[i] = ring_alloc[i];
- frags[i].page_offset += priv->frag_info[i].rx_headroom;
- rx_desc->data[i].addr = cpu_to_be64(frags[i].dma +
- frags[i].page_offset);
- ring_alloc[i] = page_alloc[i];
- }
-
- return 0;
-
-out:
- while (i--) {
- if (page_alloc[i].page != ring_alloc[i].page) {
- dma_unmap_page(priv->ddev, page_alloc[i].dma,
- page_alloc[i].page_size,
- priv->frag_info[i].dma_dir);
- page = page_alloc[i].page;
- /* Revert changes done by mlx4_alloc_pages */
- page_ref_sub(page, page_alloc[i].page_size /
- priv->frag_info[i].frag_stride - 1);
- put_page(page);
+ for (i = 0; i < priv->num_frags; i++, frags++) {
+ if (!frags->page) {
+ if (mlx4_alloc_page(priv, frags, gfp))
+ return -ENOMEM;
+ ring->rx_alloc_pages++;
}
- }
- return -ENOMEM;
-}
-
-static void mlx4_en_free_frag(struct mlx4_en_priv *priv,
- struct mlx4_en_rx_alloc *frags,
- int i)
-{
- const struct mlx4_en_frag_info *frag_info = &priv->frag_info[i];
- u32 next_frag_end = frags[i].page_offset + 2 * frag_info->frag_stride;
-
-
- if (next_frag_end > frags[i].page_size)
- dma_unmap_page(priv->ddev, frags[i].dma, frags[i].page_size,
- frag_info->dma_dir);
-
- if (frags[i].page)
- put_page(frags[i].page);
-}
-
-static int mlx4_en_init_allocator(struct mlx4_en_priv *priv,
- struct mlx4_en_rx_ring *ring)
-{
- int i;
- struct mlx4_en_rx_alloc *page_alloc;
-
- for (i = 0; i < priv->num_frags; i++) {
- const struct mlx4_en_frag_info *frag_info = &priv->frag_info[i];
-
- if (mlx4_alloc_pages(priv, &ring->page_alloc[i],
- frag_info, GFP_KERNEL | __GFP_COLD))
- goto out;
-
- en_dbg(DRV, priv, " frag %d allocator: - size:%d frags:%d\n",
- i, ring->page_alloc[i].page_size,
- page_ref_count(ring->page_alloc[i].page));
+ rx_desc->data[i].addr = cpu_to_be64(frags->dma +
+ frags->page_offset);
}
return 0;
-
-out:
- while (i--) {
- struct page *page;
-
- page_alloc = &ring->page_alloc[i];
- dma_unmap_page(priv->ddev, page_alloc->dma,
- page_alloc->page_size,
- priv->frag_info[i].dma_dir);
- page = page_alloc->page;
- /* Revert changes done by mlx4_alloc_pages */
- page_ref_sub(page, page_alloc->page_size /
- priv->frag_info[i].frag_stride - 1);
- put_page(page);
- page_alloc->page = NULL;
- }
- return -ENOMEM;
}
-static void mlx4_en_destroy_allocator(struct mlx4_en_priv *priv,
- struct mlx4_en_rx_ring *ring)
+static void mlx4_en_free_frag(const struct mlx4_en_priv *priv,
+ struct mlx4_en_rx_alloc *frag)
{
- struct mlx4_en_rx_alloc *page_alloc;
- int i;
-
- for (i = 0; i < priv->num_frags; i++) {
- const struct mlx4_en_frag_info *frag_info = &priv->frag_info[i];
-
- page_alloc = &ring->page_alloc[i];
- en_dbg(DRV, priv, "Freeing allocator:%d count:%d\n",
- i, page_count(page_alloc->page));
-
- dma_unmap_page(priv->ddev, page_alloc->dma,
- page_alloc->page_size, frag_info->dma_dir);
- while (page_alloc->page_offset + frag_info->frag_stride <
- page_alloc->page_size) {
- put_page(page_alloc->page);
- page_alloc->page_offset += frag_info->frag_stride;
- }
- page_alloc->page = NULL;
+ if (frag->page) {
+ dma_unmap_page(priv->ddev, frag->dma,
+ PAGE_SIZE, priv->dma_dir);
+ __free_page(frag->page);
}
+ /* We need to clear all fields, otherwise a change of priv->log_rx_info
+ * could lead to see garbage later in frag->page.
+ */
+ memset(frag, 0, sizeof(*frag));
}
-static void mlx4_en_init_rx_desc(struct mlx4_en_priv *priv,
+static void mlx4_en_init_rx_desc(const struct mlx4_en_priv *priv,
struct mlx4_en_rx_ring *ring, int index)
{
struct mlx4_en_rx_desc *rx_desc = ring->buf + ring->stride * index;
struct mlx4_en_rx_desc *rx_desc = ring->buf + (index * ring->stride);
struct mlx4_en_rx_alloc *frags = ring->rx_info +
(index << priv->log_rx_info);
-
if (ring->page_cache.index > 0) {
- frags[0] = ring->page_cache.buf[--ring->page_cache.index];
- rx_desc->data[0].addr = cpu_to_be64(frags[0].dma +
- frags[0].page_offset);
+ /* XDP uses a single page per frame */
+ if (!frags->page) {
+ ring->page_cache.index--;
+ frags->page = ring->page_cache.buf[ring->page_cache.index].page;
+ frags->dma = ring->page_cache.buf[ring->page_cache.index].dma;
+ }
+ frags->page_offset = XDP_PACKET_HEADROOM;
+ rx_desc->data[0].addr = cpu_to_be64(frags->dma +
+ XDP_PACKET_HEADROOM);
return 0;
}
- return mlx4_en_alloc_frags(priv, rx_desc, frags, ring->page_alloc, gfp);
+ return mlx4_en_alloc_frags(priv, ring, rx_desc, frags, gfp);
}
-static inline bool mlx4_en_is_ring_empty(struct mlx4_en_rx_ring *ring)
+static bool mlx4_en_is_ring_empty(const struct mlx4_en_rx_ring *ring)
{
return ring->prod == ring->cons;
}
*ring->wqres.db.db = cpu_to_be32(ring->prod & 0xffff);
}
-static void mlx4_en_free_rx_desc(struct mlx4_en_priv *priv,
+/* slow path */
+static void mlx4_en_free_rx_desc(const struct mlx4_en_priv *priv,
struct mlx4_en_rx_ring *ring,
int index)
{
frags = ring->rx_info + (index << priv->log_rx_info);
for (nr = 0; nr < priv->num_frags; nr++) {
en_dbg(DRV, priv, "Freeing fragment:%d\n", nr);
- mlx4_en_free_frag(priv, frags, nr);
+ mlx4_en_free_frag(priv, frags + nr);
}
}
ring->cons, ring->prod);
/* Unmap and free Rx buffers */
- while (!mlx4_en_is_ring_empty(ring)) {
- index = ring->cons & ring->size_mask;
+ for (index = 0; index < ring->size; index++) {
en_dbg(DRV, priv, "Processing descriptor:%d\n", index);
mlx4_en_free_rx_desc(priv, ring, index);
- ++ring->cons;
}
+ ring->cons = 0;
+ ring->prod = 0;
}
void mlx4_en_set_num_rx_rings(struct mlx4_en_dev *mdev)
tmp = size * roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS *
sizeof(struct mlx4_en_rx_alloc));
- ring->rx_info = vmalloc_node(tmp, node);
+ ring->rx_info = vzalloc_node(tmp, node);
if (!ring->rx_info) {
- ring->rx_info = vmalloc(tmp);
+ ring->rx_info = vzalloc(tmp);
if (!ring->rx_info) {
err = -ENOMEM;
goto err_ring;
/* Initialize all descriptors */
for (i = 0; i < ring->size; i++)
mlx4_en_init_rx_desc(priv, ring, i);
-
- /* Initialize page allocators */
- err = mlx4_en_init_allocator(priv, ring);
- if (err) {
- en_err(priv, "Failed initializing ring allocator\n");
- if (ring->stride <= TXBB_SIZE)
- ring->buf -= TXBB_SIZE;
- ring_ind--;
- goto err_allocator;
- }
}
err = mlx4_en_fill_rx_buffers(priv);
if (err)
mlx4_en_free_rx_buf(priv, priv->rx_ring[ring_ind]);
ring_ind = priv->rx_ring_num - 1;
-err_allocator:
while (ring_ind >= 0) {
if (priv->rx_ring[ring_ind]->stride <= TXBB_SIZE)
priv->rx_ring[ring_ind]->buf -= TXBB_SIZE;
- mlx4_en_destroy_allocator(priv, priv->rx_ring[ring_ind]);
ring_ind--;
}
return err;
if (cache->index >= MLX4_EN_CACHE_SIZE)
return false;
- cache->buf[cache->index++] = *frame;
+ cache->buf[cache->index].page = frame->page;
+ cache->buf[cache->index].dma = frame->dma;
+ cache->index++;
return true;
}
int i;
for (i = 0; i < ring->page_cache.index; i++) {
- struct mlx4_en_rx_alloc *frame = &ring->page_cache.buf[i];
-
- dma_unmap_page(priv->ddev, frame->dma, frame->page_size,
- priv->frag_info[0].dma_dir);
- put_page(frame->page);
+ dma_unmap_page(priv->ddev, ring->page_cache.buf[i].dma,
+ PAGE_SIZE, priv->dma_dir);
+ put_page(ring->page_cache.buf[i].page);
}
ring->page_cache.index = 0;
mlx4_en_free_rx_buf(priv, ring);
if (ring->stride <= TXBB_SIZE)
ring->buf -= TXBB_SIZE;
- mlx4_en_destroy_allocator(priv, ring);
}
static int mlx4_en_complete_rx_desc(struct mlx4_en_priv *priv,
- struct mlx4_en_rx_desc *rx_desc,
struct mlx4_en_rx_alloc *frags,
struct sk_buff *skb,
int length)
{
- struct skb_frag_struct *skb_frags_rx = skb_shinfo(skb)->frags;
- struct mlx4_en_frag_info *frag_info;
- int nr;
+ const struct mlx4_en_frag_info *frag_info = priv->frag_info;
+ unsigned int truesize = 0;
+ int nr, frag_size;
+ struct page *page;
dma_addr_t dma;
+ bool release;
/* Collect used fragments while replacing them in the HW descriptors */
- for (nr = 0; nr < priv->num_frags; nr++) {
- frag_info = &priv->frag_info[nr];
- if (length <= frag_info->frag_prefix_size)
- break;
- if (unlikely(!frags[nr].page))
+ for (nr = 0;; frags++) {
+ frag_size = min_t(int, length, frag_info->frag_size);
+
+ page = frags->page;
+ if (unlikely(!page))
goto fail;
- dma = be64_to_cpu(rx_desc->data[nr].addr);
- dma_sync_single_for_cpu(priv->ddev, dma, frag_info->frag_size,
- DMA_FROM_DEVICE);
+ dma = frags->dma;
+ dma_sync_single_range_for_cpu(priv->ddev, dma, frags->page_offset,
+ frag_size, priv->dma_dir);
+
+ __skb_fill_page_desc(skb, nr, page, frags->page_offset,
+ frag_size);
- __skb_fill_page_desc(skb, nr, frags[nr].page,
- frags[nr].page_offset,
- frag_info->frag_size);
+ truesize += frag_info->frag_stride;
+ if (frag_info->frag_stride == PAGE_SIZE / 2) {
+ frags->page_offset ^= PAGE_SIZE / 2;
+ release = page_count(page) != 1 ||
+ page_is_pfmemalloc(page) ||
+ page_to_nid(page) != numa_mem_id();
+ } else {
+ u32 sz_align = ALIGN(frag_size, SMP_CACHE_BYTES);
- skb->truesize += frag_info->frag_stride;
- frags[nr].page = NULL;
+ frags->page_offset += sz_align;
+ release = frags->page_offset + frag_info->frag_size > PAGE_SIZE;
+ }
+ if (release) {
+ dma_unmap_page(priv->ddev, dma, PAGE_SIZE, priv->dma_dir);
+ frags->page = NULL;
+ } else {
+ page_ref_inc(page);
+ }
+
+ nr++;
+ length -= frag_size;
+ if (!length)
+ break;
+ frag_info++;
}
- /* Adjust size of last fragment to match actual length */
- if (nr > 0)
- skb_frag_size_set(&skb_frags_rx[nr - 1],
- length - priv->frag_info[nr - 1].frag_prefix_size);
+ skb->truesize += truesize;
return nr;
fail:
while (nr > 0) {
nr--;
- __skb_frag_unref(&skb_frags_rx[nr]);
+ __skb_frag_unref(skb_shinfo(skb)->frags + nr);
}
return 0;
}
-
-static struct sk_buff *mlx4_en_rx_skb(struct mlx4_en_priv *priv,
- struct mlx4_en_rx_desc *rx_desc,
- struct mlx4_en_rx_alloc *frags,
- unsigned int length)
-{
- struct sk_buff *skb;
- void *va;
- int used_frags;
- dma_addr_t dma;
-
- skb = netdev_alloc_skb(priv->dev, SMALL_PACKET_SIZE + NET_IP_ALIGN);
- if (unlikely(!skb)) {
- en_dbg(RX_ERR, priv, "Failed allocating skb\n");
- return NULL;
- }
- skb_reserve(skb, NET_IP_ALIGN);
- skb->len = length;
-
- /* Get pointer to first fragment so we could copy the headers into the
- * (linear part of the) skb */
- va = page_address(frags[0].page) + frags[0].page_offset;
-
- if (length <= SMALL_PACKET_SIZE) {
- /* We are copying all relevant data to the skb - temporarily
- * sync buffers for the copy */
- dma = be64_to_cpu(rx_desc->data[0].addr);
- dma_sync_single_for_cpu(priv->ddev, dma, length,
- DMA_FROM_DEVICE);
- skb_copy_to_linear_data(skb, va, length);
- skb->tail += length;
- } else {
- unsigned int pull_len;
-
- /* Move relevant fragments to skb */
- used_frags = mlx4_en_complete_rx_desc(priv, rx_desc, frags,
- skb, length);
- if (unlikely(!used_frags)) {
- kfree_skb(skb);
- return NULL;
- }
- skb_shinfo(skb)->nr_frags = used_frags;
-
- pull_len = eth_get_headlen(va, SMALL_PACKET_SIZE);
- /* Copy headers into the skb linear buffer */
- memcpy(skb->data, va, pull_len);
- skb->tail += pull_len;
-
- /* Skip headers in first fragment */
- skb_shinfo(skb)->frags[0].page_offset += pull_len;
-
- /* Adjust size of first fragment */
- skb_frag_size_sub(&skb_shinfo(skb)->frags[0], pull_len);
- skb->data_len = length - pull_len;
- }
- return skb;
-}
-
-static void validate_loopback(struct mlx4_en_priv *priv, struct sk_buff *skb)
+static void validate_loopback(struct mlx4_en_priv *priv, void *va)
{
+ const unsigned char *data = va + ETH_HLEN;
int i;
- int offset = ETH_HLEN;
- for (i = 0; i < MLX4_LOOPBACK_TEST_PAYLOAD; i++, offset++) {
- if (*(skb->data + offset) != (unsigned char) (i & 0xff))
- goto out_loopback;
+ for (i = 0; i < MLX4_LOOPBACK_TEST_PAYLOAD; i++) {
+ if (data[i] != (unsigned char)i)
+ return;
}
/* Loopback found */
priv->loopback_ok = 1;
-
-out_loopback:
- dev_kfree_skb_any(skb);
}
static bool mlx4_en_refill_rx_buffers(struct mlx4_en_priv *priv,
struct mlx4_cqe *cqe;
struct mlx4_en_rx_ring *ring = priv->rx_ring[cq->ring];
struct mlx4_en_rx_alloc *frags;
- struct mlx4_en_rx_desc *rx_desc;
struct bpf_prog *xdp_prog;
int doorbell_pending;
struct sk_buff *skb;
/* Process all completed CQEs */
while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,
cq->mcq.cons_index & cq->size)) {
+ void *va;
frags = ring->rx_info + (index << priv->log_rx_info);
- rx_desc = ring->buf + (index << ring->log_stride);
-
+ va = page_address(frags[0].page) + frags[0].page_offset;
/*
* make sure we read the CQE after we read the ownership bit
*/
* and not performing the selftest or flb disabled
*/
if (priv->flags & MLX4_EN_FLAG_RX_FILTER_NEEDED) {
- struct ethhdr *ethh;
+ const struct ethhdr *ethh = va;
dma_addr_t dma;
/* Get pointer to first fragment since we haven't
* skb yet and cast it to ethhdr struct
*/
- dma = be64_to_cpu(rx_desc->data[0].addr);
+ dma = frags[0].dma + frags[0].page_offset;
dma_sync_single_for_cpu(priv->ddev, dma, sizeof(*ethh),
DMA_FROM_DEVICE);
- ethh = (struct ethhdr *)(page_address(frags[0].page) +
- frags[0].page_offset);
if (is_multicast_ether_addr(ethh->h_dest)) {
struct mlx4_mac_entry *entry;
}
}
+ if (unlikely(priv->validate_loopback)) {
+ validate_loopback(priv, va);
+ goto next;
+ }
+
/*
* Packet is OK - process it.
*/
length = be32_to_cpu(cqe->byte_cnt);
length -= ring->fcs_del;
- l2_tunnel = (dev->hw_enc_features & NETIF_F_RXCSUM) &&
- (cqe->vlan_my_qpn & cpu_to_be32(MLX4_CQE_L2_TUNNEL));
/* A bpf program gets first chance to drop the packet. It may
* read bytes but not past the end of the frag.
void *orig_data;
u32 act;
- dma = be64_to_cpu(rx_desc->data[0].addr);
+ dma = frags[0].dma + frags[0].page_offset;
dma_sync_single_for_cpu(priv->ddev, dma,
priv->frag_info[0].frag_size,
DMA_FROM_DEVICE);
- xdp.data_hard_start = page_address(frags[0].page);
- xdp.data = xdp.data_hard_start + frags[0].page_offset;
+ xdp.data_hard_start = va - frags[0].page_offset;
+ xdp.data = va;
xdp.data_end = xdp.data + length;
orig_data = xdp.data;
length = xdp.data_end - xdp.data;
frags[0].page_offset = xdp.data -
xdp.data_hard_start;
+ va = xdp.data;
}
switch (act) {
case XDP_TX:
if (likely(!mlx4_en_xmit_frame(ring, frags, dev,
length, cq->ring,
- &doorbell_pending)))
- goto consumed;
+ &doorbell_pending))) {
+ frags[0].page = NULL;
+ goto next;
+ }
trace_xdp_exception(dev, xdp_prog, act);
goto xdp_drop_no_cnt; /* Drop on xmit failure */
default:
case XDP_DROP:
ring->xdp_drop++;
xdp_drop_no_cnt:
- if (likely(mlx4_en_rx_recycle(ring, frags)))
- goto consumed;
goto next;
}
}
ring->bytes += length;
ring->packets++;
+ skb = napi_get_frags(&cq->napi);
+ if (!skb)
+ goto next;
+
+ if (unlikely(ring->hwtstamp_rx_filter == HWTSTAMP_FILTER_ALL)) {
+ timestamp = mlx4_en_get_cqe_ts(cqe);
+ mlx4_en_fill_hwtstamps(mdev, skb_hwtstamps(skb),
+ timestamp);
+ }
+ skb_record_rx_queue(skb, cq->ring);
+
if (likely(dev->features & NETIF_F_RXCSUM)) {
if (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_TCP |
MLX4_CQE_STATUS_UDP)) {
if ((cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPOK)) &&
cqe->checksum == cpu_to_be16(0xffff)) {
ip_summed = CHECKSUM_UNNECESSARY;
+ l2_tunnel = (dev->hw_enc_features & NETIF_F_RXCSUM) &&
+ (cqe->vlan_my_qpn & cpu_to_be32(MLX4_CQE_L2_TUNNEL));
+ if (l2_tunnel)
+ skb->csum_level = 1;
ring->csum_ok++;
} else {
- ip_summed = CHECKSUM_NONE;
- ring->csum_none++;
+ goto csum_none;
}
} else {
if (priv->flags & MLX4_EN_FLAG_RX_CSUM_NON_TCP_UDP &&
(cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPV4 |
MLX4_CQE_STATUS_IPV6))) {
- ip_summed = CHECKSUM_COMPLETE;
- ring->csum_complete++;
+ if (check_csum(cqe, skb, va, dev->features)) {
+ goto csum_none;
+ } else {
+ ip_summed = CHECKSUM_COMPLETE;
+ ring->csum_complete++;
+ }
} else {
- ip_summed = CHECKSUM_NONE;
- ring->csum_none++;
+ goto csum_none;
}
}
} else {
+csum_none:
ip_summed = CHECKSUM_NONE;
ring->csum_none++;
}
-
- /* This packet is eligible for GRO if it is:
- * - DIX Ethernet (type interpretation)
- * - TCP/IP (v4)
- * - without IP options
- * - not an IP fragment
- */
- if (dev->features & NETIF_F_GRO) {
- struct sk_buff *gro_skb = napi_get_frags(&cq->napi);
- if (!gro_skb)
- goto next;
-
- nr = mlx4_en_complete_rx_desc(priv,
- rx_desc, frags, gro_skb,
- length);
- if (!nr)
- goto next;
-
- if (ip_summed == CHECKSUM_COMPLETE) {
- void *va = skb_frag_address(skb_shinfo(gro_skb)->frags);
- if (check_csum(cqe, gro_skb, va,
- dev->features)) {
- ip_summed = CHECKSUM_NONE;
- ring->csum_none++;
- ring->csum_complete--;
- }
- }
-
- skb_shinfo(gro_skb)->nr_frags = nr;
- gro_skb->len = length;
- gro_skb->data_len = length;
- gro_skb->ip_summed = ip_summed;
-
- if (l2_tunnel && ip_summed == CHECKSUM_UNNECESSARY)
- gro_skb->csum_level = 1;
-
- if ((cqe->vlan_my_qpn &
- cpu_to_be32(MLX4_CQE_CVLAN_PRESENT_MASK)) &&
- (dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
- u16 vid = be16_to_cpu(cqe->sl_vid);
-
- __vlan_hwaccel_put_tag(gro_skb, htons(ETH_P_8021Q), vid);
- } else if ((be32_to_cpu(cqe->vlan_my_qpn) &
- MLX4_CQE_SVLAN_PRESENT_MASK) &&
- (dev->features & NETIF_F_HW_VLAN_STAG_RX)) {
- __vlan_hwaccel_put_tag(gro_skb,
- htons(ETH_P_8021AD),
- be16_to_cpu(cqe->sl_vid));
- }
-
- if (dev->features & NETIF_F_RXHASH)
- skb_set_hash(gro_skb,
- be32_to_cpu(cqe->immed_rss_invalid),
- (ip_summed == CHECKSUM_UNNECESSARY) ?
- PKT_HASH_TYPE_L4 :
- PKT_HASH_TYPE_L3);
-
- skb_record_rx_queue(gro_skb, cq->ring);
-
- if (ring->hwtstamp_rx_filter == HWTSTAMP_FILTER_ALL) {
- timestamp = mlx4_en_get_cqe_ts(cqe);
- mlx4_en_fill_hwtstamps(mdev,
- skb_hwtstamps(gro_skb),
- timestamp);
- }
-
- napi_gro_frags(&cq->napi);
- goto next;
- }
-
- /* GRO not possible, complete processing here */
- skb = mlx4_en_rx_skb(priv, rx_desc, frags, length);
- if (unlikely(!skb)) {
- ring->dropped++;
- goto next;
- }
-
- if (unlikely(priv->validate_loopback)) {
- validate_loopback(priv, skb);
- goto next;
- }
-
- if (ip_summed == CHECKSUM_COMPLETE) {
- if (check_csum(cqe, skb, skb->data, dev->features)) {
- ip_summed = CHECKSUM_NONE;
- ring->csum_complete--;
- ring->csum_none++;
- }
- }
-
skb->ip_summed = ip_summed;
- skb->protocol = eth_type_trans(skb, dev);
- skb_record_rx_queue(skb, cq->ring);
-
- if (l2_tunnel && ip_summed == CHECKSUM_UNNECESSARY)
- skb->csum_level = 1;
-
if (dev->features & NETIF_F_RXHASH)
skb_set_hash(skb,
be32_to_cpu(cqe->immed_rss_invalid),
PKT_HASH_TYPE_L4 :
PKT_HASH_TYPE_L3);
- if ((be32_to_cpu(cqe->vlan_my_qpn) &
- MLX4_CQE_CVLAN_PRESENT_MASK) &&
+
+ if ((cqe->vlan_my_qpn &
+ cpu_to_be32(MLX4_CQE_CVLAN_PRESENT_MASK)) &&
(dev->features & NETIF_F_HW_VLAN_CTAG_RX))
- __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), be16_to_cpu(cqe->sl_vid));
- else if ((be32_to_cpu(cqe->vlan_my_qpn) &
- MLX4_CQE_SVLAN_PRESENT_MASK) &&
+ __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
+ be16_to_cpu(cqe->sl_vid));
+ else if ((cqe->vlan_my_qpn &
+ cpu_to_be32(MLX4_CQE_SVLAN_PRESENT_MASK)) &&
(dev->features & NETIF_F_HW_VLAN_STAG_RX))
__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021AD),
be16_to_cpu(cqe->sl_vid));
- if (ring->hwtstamp_rx_filter == HWTSTAMP_FILTER_ALL) {
- timestamp = mlx4_en_get_cqe_ts(cqe);
- mlx4_en_fill_hwtstamps(mdev, skb_hwtstamps(skb),
- timestamp);
+ nr = mlx4_en_complete_rx_desc(priv, frags, skb, length);
+ if (likely(nr)) {
+ skb_shinfo(skb)->nr_frags = nr;
+ skb->len = length;
+ skb->data_len = length;
+ napi_gro_frags(&cq->napi);
+ } else {
+ skb->vlan_tci = 0;
+ skb_clear_hash(skb);
}
-
- napi_gro_receive(&cq->napi, skb);
next:
- for (nr = 0; nr < priv->num_frags; nr++)
- mlx4_en_free_frag(priv, frags, nr);
-
-consumed:
++cq->mcq.cons_index;
index = (cq->mcq.cons_index) & ring->size_mask;
cqe = mlx4_en_get_cqe(cq->buf, index, priv->cqe_size) + factor;
if (++polled == budget)
- goto out;
+ break;
}
-out:
rcu_read_unlock();
if (polled) {
return done;
}
-static const int frag_sizes[] = {
- FRAG_SZ0,
- FRAG_SZ1,
- FRAG_SZ2,
- FRAG_SZ3
-};
-
void mlx4_en_calc_rx_buf(struct net_device *dev)
{
struct mlx4_en_priv *priv = netdev_priv(dev);
* This only works when num_frags == 1.
*/
if (priv->tx_ring_num[TX_XDP]) {
- priv->frag_info[0].order = 0;
priv->frag_info[0].frag_size = eff_mtu;
- priv->frag_info[0].frag_prefix_size = 0;
/* This will gain efficient xdp frame recycling at the
* expense of more costly truesize accounting
*/
priv->frag_info[0].frag_stride = PAGE_SIZE;
- priv->frag_info[0].dma_dir = PCI_DMA_BIDIRECTIONAL;
- priv->frag_info[0].rx_headroom = XDP_PACKET_HEADROOM;
+ priv->dma_dir = PCI_DMA_BIDIRECTIONAL;
+ priv->rx_headroom = XDP_PACKET_HEADROOM;
i = 1;
} else {
- int buf_size = 0;
+ int frag_size_max = 2048, buf_size = 0;
+
+ /* should not happen, right ? */
+ if (eff_mtu > PAGE_SIZE + (MLX4_EN_MAX_RX_FRAGS - 1) * 2048)
+ frag_size_max = PAGE_SIZE;
while (buf_size < eff_mtu) {
- priv->frag_info[i].order = MLX4_EN_ALLOC_PREFER_ORDER;
- priv->frag_info[i].frag_size =
- (eff_mtu > buf_size + frag_sizes[i]) ?
- frag_sizes[i] : eff_mtu - buf_size;
- priv->frag_info[i].frag_prefix_size = buf_size;
- priv->frag_info[i].frag_stride =
- ALIGN(priv->frag_info[i].frag_size,
- SMP_CACHE_BYTES);
- priv->frag_info[i].dma_dir = PCI_DMA_FROMDEVICE;
- priv->frag_info[i].rx_headroom = 0;
- buf_size += priv->frag_info[i].frag_size;
+ int frag_stride, frag_size = eff_mtu - buf_size;
+ int pad, nb;
+
+ if (i < MLX4_EN_MAX_RX_FRAGS - 1)
+ frag_size = min(frag_size, frag_size_max);
+
+ priv->frag_info[i].frag_size = frag_size;
+ frag_stride = ALIGN(frag_size, SMP_CACHE_BYTES);
+ /* We can only pack 2 1536-bytes frames in on 4K page
+ * Therefore, each frame would consume more bytes (truesize)
+ */
+ nb = PAGE_SIZE / frag_stride;
+ pad = (PAGE_SIZE - nb * frag_stride) / nb;
+ pad &= ~(SMP_CACHE_BYTES - 1);
+ priv->frag_info[i].frag_stride = frag_stride + pad;
+
+ buf_size += frag_size;
i++;
}
+ priv->dma_dir = PCI_DMA_FROMDEVICE;
+ priv->rx_headroom = 0;
}
priv->num_frags = i;
eff_mtu, priv->num_frags);
for (i = 0; i < priv->num_frags; i++) {
en_err(priv,
- " frag:%d - size:%d prefix:%d stride:%d\n",
+ " frag:%d - size:%d stride:%d\n",
i,
priv->frag_info[i].frag_size,
- priv->frag_info[i].frag_prefix_size,
priv->frag_info[i].frag_stride);
}
}
{
u32 loopback_ok = 0;
int i;
- bool gro_enabled;
priv->loopback_ok = 0;
priv->validate_loopback = 1;
- gro_enabled = priv->dev->features & NETIF_F_GRO;
mlx4_en_update_loopback_state(priv->dev, priv->dev->features);
- priv->dev->features &= ~NETIF_F_GRO;
/* xmit */
if (mlx4_en_test_loopback_xmit(priv)) {
priv->validate_loopback = 0;
- if (gro_enabled)
- priv->dev->features |= NETIF_F_GRO;
-
mlx4_en_update_loopback_state(priv->dev, priv->dev->features);
return !loopback_ok;
}
struct mlx4_en_rx_alloc frame = {
.page = tx_info->page,
.dma = tx_info->map0_dma,
- .page_offset = XDP_PACKET_HEADROOM,
- .page_size = PAGE_SIZE,
};
if (!mlx4_en_rx_recycle(ring->recycle_ring, &frame)) {
dma_unmap_page(priv->ddev, tx_info->map0_dma,
- PAGE_SIZE, priv->frag_info[0].dma_dir);
+ PAGE_SIZE, priv->dma_dir);
put_page(tx_info->page);
}
/* Use the maximum between 16384 and a single page */
#define MLX4_EN_ALLOC_SIZE PAGE_ALIGN(16384)
-#define MLX4_EN_ALLOC_PREFER_ORDER min_t(int, get_order(32768), \
- PAGE_ALLOC_COSTLY_ORDER)
-
-/* Receive fragment sizes; we use at most 3 fragments (for 9600 byte MTU
- * and 4K allocations) */
-enum {
- FRAG_SZ0 = 1536 - NET_IP_ALIGN,
- FRAG_SZ1 = 4096,
- FRAG_SZ2 = 4096,
- FRAG_SZ3 = MLX4_EN_ALLOC_SIZE
-};
#define MLX4_EN_MAX_RX_FRAGS 4
/* Maximum ring sizes */
struct page *page;
dma_addr_t dma;
u32 page_offset;
- u32 page_size;
};
#define MLX4_EN_CACHE_SIZE (2 * NAPI_POLL_WEIGHT)
+
struct mlx4_en_page_cache {
u32 index;
- struct mlx4_en_rx_alloc buf[MLX4_EN_CACHE_SIZE];
+ struct {
+ struct page *page;
+ dma_addr_t dma;
+ } buf[MLX4_EN_CACHE_SIZE];
};
struct mlx4_en_priv;
struct mlx4_en_rx_ring {
struct mlx4_hwq_resources wqres;
- struct mlx4_en_rx_alloc page_alloc[MLX4_EN_MAX_RX_FRAGS];
u32 size ; /* number of Rx descs*/
u32 actual_size;
u32 size_mask;
unsigned long csum_ok;
unsigned long csum_none;
unsigned long csum_complete;
+ unsigned long rx_alloc_pages;
unsigned long xdp_drop;
unsigned long xdp_tx;
unsigned long xdp_tx_full;
struct mlx4_en_frag_info {
u16 frag_size;
- u16 frag_prefix_size;
u32 frag_stride;
- enum dma_data_direction dma_dir;
- u16 order;
- u16 rx_headroom;
};
#ifdef CONFIG_MLX4_EN_DCB
u32 rx_ring_num;
u32 rx_skb_size;
struct mlx4_en_frag_info frag_info[MLX4_EN_MAX_RX_FRAGS];
- u16 num_frags;
- u16 log_rx_info;
+ u8 num_frags;
+ u8 log_rx_info;
+ u8 dma_dir;
+ u16 rx_headroom;
struct mlx4_en_tx_ring **tx_ring[MLX4_EN_NUM_TX_TYPES];
struct mlx4_en_rx_ring *rx_ring[MAX_RX_RINGS];
unsigned long queue_stopped;
unsigned long wake_queue;
unsigned long tx_timeout;
- unsigned long rx_alloc_failed;
+ unsigned long rx_alloc_pages;
unsigned long rx_chksum_good;
unsigned long rx_chksum_none;
unsigned long rx_chksum_complete;
*/
MLXSW_ITEM32(cmd_mbox, sw2hw_cq, c_eqn, 0x00, 24, 1);
-/* cmd_mbox_sw2hw_cq_oi
- * When set, overrun ignore is enabled. When set, updates of
- * CQ consumer counter (poll for completion) or Request completion
- * notifications (Arm CQ) DoorBells should not be rung on that CQ.
- */
-MLXSW_ITEM32(cmd_mbox, sw2hw_cq, oi, 0x00, 12, 1);
-
/* cmd_mbox_sw2hw_cq_st
* Event delivery state machine
* 0x0 - FIRED
*/
MLXSW_ITEM32(cmd_mbox, sw2hw_eq, int_msix, 0x00, 24, 1);
-/* cmd_mbox_sw2hw_eq_oi
- * When set, overrun ignore is enabled.
- */
-MLXSW_ITEM32(cmd_mbox, sw2hw_eq, oi, 0x00, 12, 1);
-
/* cmd_mbox_sw2hw_eq_st
* Event delivery state machine
* 0x0 - FIRED
return oneact + MLXSW_AFA_PAYLOAD_OFFSET;
}
+/* VLAN Action
+ * -----------
+ * VLAN action is used for manipulating VLANs. It can be used to implement QinQ,
+ * VLAN translation, change of PCP bits of the VLAN tag, push, pop as swap VLANs
+ * and more.
+ */
+
+#define MLXSW_AFA_VLAN_CODE 0x02
+#define MLXSW_AFA_VLAN_SIZE 1
+
+enum mlxsw_afa_vlan_vlan_tag_cmd {
+ MLXSW_AFA_VLAN_VLAN_TAG_CMD_NOP,
+ MLXSW_AFA_VLAN_VLAN_TAG_CMD_PUSH_TAG,
+ MLXSW_AFA_VLAN_VLAN_TAG_CMD_POP_TAG,
+};
+
+enum mlxsw_afa_vlan_cmd {
+ MLXSW_AFA_VLAN_CMD_NOP,
+ MLXSW_AFA_VLAN_CMD_SET_OUTER,
+ MLXSW_AFA_VLAN_CMD_SET_INNER,
+ MLXSW_AFA_VLAN_CMD_COPY_OUTER_TO_INNER,
+ MLXSW_AFA_VLAN_CMD_COPY_INNER_TO_OUTER,
+ MLXSW_AFA_VLAN_CMD_SWAP,
+};
+
+/* afa_vlan_vlan_tag_cmd
+ * Tag command: push, pop, nop VLAN header.
+ */
+MLXSW_ITEM32(afa, vlan, vlan_tag_cmd, 0x00, 29, 3);
+
+/* afa_vlan_vid_cmd */
+MLXSW_ITEM32(afa, vlan, vid_cmd, 0x04, 29, 3);
+
+/* afa_vlan_vid */
+MLXSW_ITEM32(afa, vlan, vid, 0x04, 0, 12);
+
+/* afa_vlan_ethertype_cmd */
+MLXSW_ITEM32(afa, vlan, ethertype_cmd, 0x08, 29, 3);
+
+/* afa_vlan_ethertype
+ * Index to EtherTypes in Switch VLAN EtherType Register (SVER).
+ */
+MLXSW_ITEM32(afa, vlan, ethertype, 0x08, 24, 3);
+
+/* afa_vlan_pcp_cmd */
+MLXSW_ITEM32(afa, vlan, pcp_cmd, 0x08, 13, 3);
+
+/* afa_vlan_pcp */
+MLXSW_ITEM32(afa, vlan, pcp, 0x08, 8, 3);
+
+static inline void
+mlxsw_afa_vlan_pack(char *payload,
+ enum mlxsw_afa_vlan_vlan_tag_cmd vlan_tag_cmd,
+ enum mlxsw_afa_vlan_cmd vid_cmd, u16 vid,
+ enum mlxsw_afa_vlan_cmd pcp_cmd, u8 pcp,
+ enum mlxsw_afa_vlan_cmd ethertype_cmd, u8 ethertype)
+{
+ mlxsw_afa_vlan_vlan_tag_cmd_set(payload, vlan_tag_cmd);
+ mlxsw_afa_vlan_vid_cmd_set(payload, vid_cmd);
+ mlxsw_afa_vlan_vid_set(payload, vid);
+ mlxsw_afa_vlan_pcp_cmd_set(payload, pcp_cmd);
+ mlxsw_afa_vlan_pcp_set(payload, pcp);
+ mlxsw_afa_vlan_ethertype_cmd_set(payload, ethertype_cmd);
+ mlxsw_afa_vlan_ethertype_set(payload, ethertype);
+}
+
+int mlxsw_afa_block_append_vlan_modify(struct mlxsw_afa_block *block,
+ u16 vid, u8 pcp, u8 et)
+{
+ char *act = mlxsw_afa_block_append_action(block,
+ MLXSW_AFA_VLAN_CODE,
+ MLXSW_AFA_VLAN_SIZE);
+
+ if (!act)
+ return -ENOBUFS;
+ mlxsw_afa_vlan_pack(act, MLXSW_AFA_VLAN_VLAN_TAG_CMD_NOP,
+ MLXSW_AFA_VLAN_CMD_SET_OUTER, vid,
+ MLXSW_AFA_VLAN_CMD_SET_OUTER, pcp,
+ MLXSW_AFA_VLAN_CMD_SET_OUTER, et);
+ return 0;
+}
+EXPORT_SYMBOL(mlxsw_afa_block_append_vlan_modify);
+
/* Trap / Discard Action
* ---------------------
* The Trap / Discard action enables trapping / mirroring packets to the CPU
int mlxsw_afa_block_append_drop(struct mlxsw_afa_block *block);
int mlxsw_afa_block_append_fwd(struct mlxsw_afa_block *block,
u8 local_port, bool in_port);
+int mlxsw_afa_block_append_vlan_modify(struct mlxsw_afa_block *block,
+ u16 vid, u8 pcp, u8 et);
#endif
MLXSW_AFK_ELEMENT_DST_IP6_LO,
MLXSW_AFK_ELEMENT_DST_L4_PORT,
MLXSW_AFK_ELEMENT_SRC_L4_PORT,
+ MLXSW_AFK_ELEMENT_VID,
+ MLXSW_AFK_ELEMENT_PCP,
MLXSW_AFK_ELEMENT_MAX,
};
MLXSW_AFK_ELEMENT_INFO(MLXSW_AFK_ELEMENT_TYPE_BUF, \
_element, _offset, 0, _size)
-/* For the purpose of the driver, define a internal storage scratchpad
+/* For the purpose of the driver, define an internal storage scratchpad
* that will be used to store key/mask values. For each defined element type
* define an internal storage geometry.
*/
MLXSW_AFK_ELEMENT_INFO_BUF(SMAC, 0x0A, 6),
MLXSW_AFK_ELEMENT_INFO_U32(ETHERTYPE, 0x00, 0, 16),
MLXSW_AFK_ELEMENT_INFO_U32(IP_PROTO, 0x10, 0, 8),
+ MLXSW_AFK_ELEMENT_INFO_U32(VID, 0x10, 8, 12),
+ MLXSW_AFK_ELEMENT_INFO_U32(PCP, 0x10, 20, 3),
MLXSW_AFK_ELEMENT_INFO_U32(SRC_IP4, 0x18, 0, 32),
MLXSW_AFK_ELEMENT_INFO_U32(DST_IP4, 0x1C, 0, 32),
MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP6_HI, 0x18, 8),
mlxsw_cmd_mbox_sw2hw_cq_cv_set(mbox, 0); /* CQE ver 0 */
mlxsw_cmd_mbox_sw2hw_cq_c_eqn_set(mbox, MLXSW_PCI_EQ_COMP_NUM);
- mlxsw_cmd_mbox_sw2hw_cq_oi_set(mbox, 0);
mlxsw_cmd_mbox_sw2hw_cq_st_set(mbox, 0);
mlxsw_cmd_mbox_sw2hw_cq_log_cq_size_set(mbox, ilog2(q->count));
for (i = 0; i < MLXSW_PCI_AQ_PAGES; i++) {
}
mlxsw_cmd_mbox_sw2hw_eq_int_msix_set(mbox, 1); /* MSI-X used */
- mlxsw_cmd_mbox_sw2hw_eq_oi_set(mbox, 0);
mlxsw_cmd_mbox_sw2hw_eq_st_set(mbox, 1); /* armed */
mlxsw_cmd_mbox_sw2hw_eq_log_eq_size_set(mbox, ilog2(q->count));
for (i = 0; i < MLXSW_PCI_AQ_PAGES; i++) {
static inline void mlxsw_reg_ritr_pack(char *payload, bool enable,
enum mlxsw_reg_ritr_if_type type,
- u16 rif, u16 mtu, const char *mac)
+ u16 rif, u16 vr_id, u16 mtu,
+ const char *mac)
{
bool op = enable ? MLXSW_REG_RITR_RIF_CREATE : MLXSW_REG_RITR_RIF_DEL;
mlxsw_reg_ritr_rif_set(payload, rif);
mlxsw_reg_ritr_ipv4_fe_set(payload, 1);
mlxsw_reg_ritr_lb_en_set(payload, 1);
+ mlxsw_reg_ritr_virtual_router_set(payload, vr_id);
mlxsw_reg_ritr_mtu_set(payload, mtu);
mlxsw_reg_ritr_if_mac_memcpy_to(payload, mac);
}
return dev->netdev_ops == &mlxsw_sp_port_netdev_ops;
}
-static int mlxsw_lower_dev_walk(struct net_device *lower_dev, void *data)
+static int mlxsw_sp_lower_dev_walk(struct net_device *lower_dev, void *data)
{
- struct mlxsw_sp_port **port = data;
+ struct mlxsw_sp_port **p_mlxsw_sp_port = data;
int ret = 0;
if (mlxsw_sp_port_dev_check(lower_dev)) {
- *port = netdev_priv(lower_dev);
+ *p_mlxsw_sp_port = netdev_priv(lower_dev);
ret = 1;
}
static struct mlxsw_sp_port *mlxsw_sp_port_dev_lower_find(struct net_device *dev)
{
- struct mlxsw_sp_port *port;
+ struct mlxsw_sp_port *mlxsw_sp_port;
if (mlxsw_sp_port_dev_check(dev))
return netdev_priv(dev);
- port = NULL;
- netdev_walk_all_lower_dev(dev, mlxsw_lower_dev_walk, &port);
+ mlxsw_sp_port = NULL;
+ netdev_walk_all_lower_dev(dev, mlxsw_sp_lower_dev_walk, &mlxsw_sp_port);
- return port;
+ return mlxsw_sp_port;
}
-static struct mlxsw_sp *mlxsw_sp_lower_get(struct net_device *dev)
+struct mlxsw_sp *mlxsw_sp_lower_get(struct net_device *dev)
{
struct mlxsw_sp_port *mlxsw_sp_port;
static struct mlxsw_sp_port *mlxsw_sp_port_dev_lower_find_rcu(struct net_device *dev)
{
- struct mlxsw_sp_port *port;
+ struct mlxsw_sp_port *mlxsw_sp_port;
if (mlxsw_sp_port_dev_check(dev))
return netdev_priv(dev);
- port = NULL;
- netdev_walk_all_lower_dev_rcu(dev, mlxsw_lower_dev_walk, &port);
+ mlxsw_sp_port = NULL;
+ netdev_walk_all_lower_dev_rcu(dev, mlxsw_sp_lower_dev_walk,
+ &mlxsw_sp_port);
- return port;
+ return mlxsw_sp_port;
}
struct mlxsw_sp_port *mlxsw_sp_port_lower_dev_hold(struct net_device *dev)
dev_put(mlxsw_sp_port->dev);
}
-static bool mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *r,
- unsigned long event)
-{
- switch (event) {
- case NETDEV_UP:
- if (!r)
- return true;
- r->ref_count++;
- return false;
- case NETDEV_DOWN:
- if (r && --r->ref_count == 0)
- return true;
- /* It is possible we already removed the RIF ourselves
- * if it was assigned to a netdev that is now a bridge
- * or LAG slave.
- */
- return false;
- }
-
- return false;
-}
-
-static int mlxsw_sp_avail_rif_get(struct mlxsw_sp *mlxsw_sp)
-{
- int i;
-
- for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
- if (!mlxsw_sp->rifs[i])
- return i;
-
- return MLXSW_SP_INVALID_RIF;
-}
-
-static void mlxsw_sp_vport_rif_sp_attr_get(struct mlxsw_sp_port *mlxsw_sp_vport,
- bool *p_lagged, u16 *p_system_port)
-{
- u8 local_port = mlxsw_sp_vport->local_port;
-
- *p_lagged = mlxsw_sp_vport->lagged;
- *p_system_port = *p_lagged ? mlxsw_sp_vport->lag_id : local_port;
-}
-
-static int mlxsw_sp_vport_rif_sp_op(struct mlxsw_sp_port *mlxsw_sp_vport,
- struct net_device *l3_dev, u16 rif,
- bool create)
-{
- struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
- bool lagged = mlxsw_sp_vport->lagged;
- char ritr_pl[MLXSW_REG_RITR_LEN];
- u16 system_port;
-
- mlxsw_reg_ritr_pack(ritr_pl, create, MLXSW_REG_RITR_SP_IF, rif,
- l3_dev->mtu, l3_dev->dev_addr);
-
- mlxsw_sp_vport_rif_sp_attr_get(mlxsw_sp_vport, &lagged, &system_port);
- mlxsw_reg_ritr_sp_if_pack(ritr_pl, lagged, system_port,
- mlxsw_sp_vport_vid_get(mlxsw_sp_vport));
-
- return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
-}
-
-static void mlxsw_sp_vport_rif_sp_leave(struct mlxsw_sp_port *mlxsw_sp_vport);
-
-static struct mlxsw_sp_fid *
-mlxsw_sp_rfid_alloc(u16 fid, struct net_device *l3_dev)
-{
- struct mlxsw_sp_fid *f;
-
- f = kzalloc(sizeof(*f), GFP_KERNEL);
- if (!f)
- return NULL;
-
- f->leave = mlxsw_sp_vport_rif_sp_leave;
- f->ref_count = 0;
- f->dev = l3_dev;
- f->fid = fid;
-
- return f;
-}
-
-static struct mlxsw_sp_rif *
-mlxsw_sp_rif_alloc(u16 rif, struct net_device *l3_dev, struct mlxsw_sp_fid *f)
-{
- struct mlxsw_sp_rif *r;
-
- r = kzalloc(sizeof(*r), GFP_KERNEL);
- if (!r)
- return NULL;
-
- INIT_LIST_HEAD(&r->nexthop_list);
- INIT_LIST_HEAD(&r->neigh_list);
- ether_addr_copy(r->addr, l3_dev->dev_addr);
- r->mtu = l3_dev->mtu;
- r->ref_count = 1;
- r->dev = l3_dev;
- r->rif = rif;
- r->f = f;
-
- return r;
-}
-
-static struct mlxsw_sp_rif *
-mlxsw_sp_vport_rif_sp_create(struct mlxsw_sp_port *mlxsw_sp_vport,
- struct net_device *l3_dev)
-{
- struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
- struct mlxsw_sp_fid *f;
- struct mlxsw_sp_rif *r;
- u16 fid, rif;
- int err;
-
- rif = mlxsw_sp_avail_rif_get(mlxsw_sp);
- if (rif == MLXSW_SP_INVALID_RIF)
- return ERR_PTR(-ERANGE);
-
- err = mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, l3_dev, rif, true);
- if (err)
- return ERR_PTR(err);
-
- fid = mlxsw_sp_rif_sp_to_fid(rif);
- err = mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, fid, true);
- if (err)
- goto err_rif_fdb_op;
-
- f = mlxsw_sp_rfid_alloc(fid, l3_dev);
- if (!f) {
- err = -ENOMEM;
- goto err_rfid_alloc;
- }
-
- r = mlxsw_sp_rif_alloc(rif, l3_dev, f);
- if (!r) {
- err = -ENOMEM;
- goto err_rif_alloc;
- }
-
- f->r = r;
- mlxsw_sp->rifs[rif] = r;
-
- return r;
-
-err_rif_alloc:
- kfree(f);
-err_rfid_alloc:
- mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, fid, false);
-err_rif_fdb_op:
- mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, l3_dev, rif, false);
- return ERR_PTR(err);
-}
-
-static void mlxsw_sp_vport_rif_sp_destroy(struct mlxsw_sp_port *mlxsw_sp_vport,
- struct mlxsw_sp_rif *r)
-{
- struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
- struct net_device *l3_dev = r->dev;
- struct mlxsw_sp_fid *f = r->f;
- u16 fid = f->fid;
- u16 rif = r->rif;
-
- mlxsw_sp_router_rif_gone_sync(mlxsw_sp, r);
-
- mlxsw_sp->rifs[rif] = NULL;
- f->r = NULL;
-
- kfree(r);
-
- kfree(f);
-
- mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, fid, false);
-
- mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, l3_dev, rif, false);
-}
-
-static int mlxsw_sp_vport_rif_sp_join(struct mlxsw_sp_port *mlxsw_sp_vport,
- struct net_device *l3_dev)
-{
- struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
- struct mlxsw_sp_rif *r;
-
- r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
- if (!r) {
- r = mlxsw_sp_vport_rif_sp_create(mlxsw_sp_vport, l3_dev);
- if (IS_ERR(r))
- return PTR_ERR(r);
- }
-
- mlxsw_sp_vport_fid_set(mlxsw_sp_vport, r->f);
- r->f->ref_count++;
-
- netdev_dbg(mlxsw_sp_vport->dev, "Joined FID=%d\n", r->f->fid);
-
- return 0;
-}
-
-static void mlxsw_sp_vport_rif_sp_leave(struct mlxsw_sp_port *mlxsw_sp_vport)
-{
- struct mlxsw_sp_fid *f = mlxsw_sp_vport_fid_get(mlxsw_sp_vport);
-
- netdev_dbg(mlxsw_sp_vport->dev, "Left FID=%d\n", f->fid);
-
- mlxsw_sp_vport_fid_set(mlxsw_sp_vport, NULL);
- if (--f->ref_count == 0)
- mlxsw_sp_vport_rif_sp_destroy(mlxsw_sp_vport, f->r);
-}
-
-static int mlxsw_sp_inetaddr_vport_event(struct net_device *l3_dev,
- struct net_device *port_dev,
- unsigned long event, u16 vid)
-{
- struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
- struct mlxsw_sp_port *mlxsw_sp_vport;
-
- mlxsw_sp_vport = mlxsw_sp_port_vport_find(mlxsw_sp_port, vid);
- if (WARN_ON(!mlxsw_sp_vport))
- return -EINVAL;
-
- switch (event) {
- case NETDEV_UP:
- return mlxsw_sp_vport_rif_sp_join(mlxsw_sp_vport, l3_dev);
- case NETDEV_DOWN:
- mlxsw_sp_vport_rif_sp_leave(mlxsw_sp_vport);
- break;
- }
-
- return 0;
-}
-
-static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
- unsigned long event)
-{
- if (netif_is_bridge_port(port_dev) || netif_is_lag_port(port_dev))
- return 0;
-
- return mlxsw_sp_inetaddr_vport_event(port_dev, port_dev, event, 1);
-}
-
-static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
- struct net_device *lag_dev,
- unsigned long event, u16 vid)
-{
- struct net_device *port_dev;
- struct list_head *iter;
- int err;
-
- netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
- if (mlxsw_sp_port_dev_check(port_dev)) {
- err = mlxsw_sp_inetaddr_vport_event(l3_dev, port_dev,
- event, vid);
- if (err)
- return err;
- }
- }
-
- return 0;
-}
-
-static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
- unsigned long event)
-{
- if (netif_is_bridge_port(lag_dev))
- return 0;
-
- return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1);
-}
-
-static struct mlxsw_sp_fid *mlxsw_sp_bridge_fid_get(struct mlxsw_sp *mlxsw_sp,
- struct net_device *l3_dev)
-{
- u16 fid;
-
- if (is_vlan_dev(l3_dev))
- fid = vlan_dev_vlan_id(l3_dev);
- else if (mlxsw_sp->master_bridge.dev == l3_dev)
- fid = 1;
- else
- return mlxsw_sp_vfid_find(mlxsw_sp, l3_dev);
-
- return mlxsw_sp_fid_find(mlxsw_sp, fid);
-}
-
-static enum mlxsw_flood_table_type mlxsw_sp_flood_table_type_get(u16 fid)
-{
- return mlxsw_sp_fid_is_vfid(fid) ? MLXSW_REG_SFGC_TABLE_TYPE_FID :
- MLXSW_REG_SFGC_TABLE_TYPE_FID_OFFEST;
-}
-
-static u16 mlxsw_sp_flood_table_index_get(u16 fid)
-{
- return mlxsw_sp_fid_is_vfid(fid) ? mlxsw_sp_fid_to_vfid(fid) : fid;
-}
-
-static int mlxsw_sp_router_port_flood_set(struct mlxsw_sp *mlxsw_sp, u16 fid,
- bool set)
-{
- enum mlxsw_flood_table_type table_type;
- char *sftr_pl;
- u16 index;
- int err;
-
- sftr_pl = kmalloc(MLXSW_REG_SFTR_LEN, GFP_KERNEL);
- if (!sftr_pl)
- return -ENOMEM;
-
- table_type = mlxsw_sp_flood_table_type_get(fid);
- index = mlxsw_sp_flood_table_index_get(fid);
- mlxsw_reg_sftr_pack(sftr_pl, MLXSW_SP_FLOOD_TABLE_BC, index, table_type,
- 1, MLXSW_PORT_ROUTER_PORT, set);
- err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sftr), sftr_pl);
-
- kfree(sftr_pl);
- return err;
-}
-
-static enum mlxsw_reg_ritr_if_type mlxsw_sp_rif_type_get(u16 fid)
-{
- if (mlxsw_sp_fid_is_vfid(fid))
- return MLXSW_REG_RITR_FID_IF;
- else
- return MLXSW_REG_RITR_VLAN_IF;
-}
-
-static int mlxsw_sp_rif_bridge_op(struct mlxsw_sp *mlxsw_sp,
- struct net_device *l3_dev,
- u16 fid, u16 rif,
- bool create)
-{
- enum mlxsw_reg_ritr_if_type rif_type;
- char ritr_pl[MLXSW_REG_RITR_LEN];
-
- rif_type = mlxsw_sp_rif_type_get(fid);
- mlxsw_reg_ritr_pack(ritr_pl, create, rif_type, rif, l3_dev->mtu,
- l3_dev->dev_addr);
- mlxsw_reg_ritr_fid_set(ritr_pl, rif_type, fid);
-
- return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
-}
-
-static int mlxsw_sp_rif_bridge_create(struct mlxsw_sp *mlxsw_sp,
- struct net_device *l3_dev,
- struct mlxsw_sp_fid *f)
-{
- struct mlxsw_sp_rif *r;
- u16 rif;
- int err;
-
- rif = mlxsw_sp_avail_rif_get(mlxsw_sp);
- if (rif == MLXSW_SP_INVALID_RIF)
- return -ERANGE;
-
- err = mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, true);
- if (err)
- return err;
-
- err = mlxsw_sp_rif_bridge_op(mlxsw_sp, l3_dev, f->fid, rif, true);
- if (err)
- goto err_rif_bridge_op;
-
- err = mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, true);
- if (err)
- goto err_rif_fdb_op;
-
- r = mlxsw_sp_rif_alloc(rif, l3_dev, f);
- if (!r) {
- err = -ENOMEM;
- goto err_rif_alloc;
- }
-
- f->r = r;
- mlxsw_sp->rifs[rif] = r;
-
- netdev_dbg(l3_dev, "RIF=%d created\n", rif);
-
- return 0;
-
-err_rif_alloc:
- mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, false);
-err_rif_fdb_op:
- mlxsw_sp_rif_bridge_op(mlxsw_sp, l3_dev, f->fid, rif, false);
-err_rif_bridge_op:
- mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, false);
- return err;
-}
-
-void mlxsw_sp_rif_bridge_destroy(struct mlxsw_sp *mlxsw_sp,
- struct mlxsw_sp_rif *r)
-{
- struct net_device *l3_dev = r->dev;
- struct mlxsw_sp_fid *f = r->f;
- u16 rif = r->rif;
-
- mlxsw_sp_router_rif_gone_sync(mlxsw_sp, r);
-
- mlxsw_sp->rifs[rif] = NULL;
- f->r = NULL;
-
- kfree(r);
-
- mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, false);
-
- mlxsw_sp_rif_bridge_op(mlxsw_sp, l3_dev, f->fid, rif, false);
-
- mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, false);
-
- netdev_dbg(l3_dev, "RIF=%d destroyed\n", rif);
-}
-
-static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev,
- struct net_device *br_dev,
- unsigned long event)
-{
- struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
- struct mlxsw_sp_fid *f;
-
- /* FID can either be an actual FID if the L3 device is the
- * VLAN-aware bridge or a VLAN device on top. Otherwise, the
- * L3 device is a VLAN-unaware bridge and we get a vFID.
- */
- f = mlxsw_sp_bridge_fid_get(mlxsw_sp, l3_dev);
- if (WARN_ON(!f))
- return -EINVAL;
-
- switch (event) {
- case NETDEV_UP:
- return mlxsw_sp_rif_bridge_create(mlxsw_sp, l3_dev, f);
- case NETDEV_DOWN:
- mlxsw_sp_rif_bridge_destroy(mlxsw_sp, f->r);
- break;
- }
-
- return 0;
-}
-
-static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev,
- unsigned long event)
-{
- struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
- struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(vlan_dev);
- u16 vid = vlan_dev_vlan_id(vlan_dev);
-
- if (mlxsw_sp_port_dev_check(real_dev))
- return mlxsw_sp_inetaddr_vport_event(vlan_dev, real_dev, event,
- vid);
- else if (netif_is_lag_master(real_dev))
- return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
- vid);
- else if (netif_is_bridge_master(real_dev) &&
- mlxsw_sp->master_bridge.dev == real_dev)
- return mlxsw_sp_inetaddr_bridge_event(vlan_dev, real_dev,
- event);
-
- return 0;
-}
-
-static int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
- unsigned long event, void *ptr)
-{
- struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
- struct net_device *dev = ifa->ifa_dev->dev;
- struct mlxsw_sp *mlxsw_sp;
- struct mlxsw_sp_rif *r;
- int err = 0;
-
- mlxsw_sp = mlxsw_sp_lower_get(dev);
- if (!mlxsw_sp)
- goto out;
-
- r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
- if (!mlxsw_sp_rif_should_config(r, event))
- goto out;
-
- if (mlxsw_sp_port_dev_check(dev))
- err = mlxsw_sp_inetaddr_port_event(dev, event);
- else if (netif_is_lag_master(dev))
- err = mlxsw_sp_inetaddr_lag_event(dev, event);
- else if (netif_is_bridge_master(dev))
- err = mlxsw_sp_inetaddr_bridge_event(dev, dev, event);
- else if (is_vlan_dev(dev))
- err = mlxsw_sp_inetaddr_vlan_event(dev, event);
-
-out:
- return notifier_from_errno(err);
-}
-
-static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif,
- const char *mac, int mtu)
-{
- char ritr_pl[MLXSW_REG_RITR_LEN];
- int err;
-
- mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
- err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
- if (err)
- return err;
-
- mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
- mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
- mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
- return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
-}
-
-static int mlxsw_sp_netdevice_router_port_event(struct net_device *dev)
-{
- struct mlxsw_sp *mlxsw_sp;
- struct mlxsw_sp_rif *r;
- int err;
-
- mlxsw_sp = mlxsw_sp_lower_get(dev);
- if (!mlxsw_sp)
- return 0;
-
- r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
- if (!r)
- return 0;
-
- err = mlxsw_sp_rif_fdb_op(mlxsw_sp, r->addr, r->f->fid, false);
- if (err)
- return err;
-
- err = mlxsw_sp_rif_edit(mlxsw_sp, r->rif, dev->dev_addr, dev->mtu);
- if (err)
- goto err_rif_edit;
-
- err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, r->f->fid, true);
- if (err)
- goto err_rif_fdb_op;
-
- ether_addr_copy(r->addr, dev->dev_addr);
- r->mtu = dev->mtu;
-
- netdev_dbg(dev, "Updated RIF=%d\n", r->rif);
-
- return 0;
-
-err_rif_fdb_op:
- mlxsw_sp_rif_edit(mlxsw_sp, r->rif, r->addr, r->mtu);
-err_rif_edit:
- mlxsw_sp_rif_fdb_op(mlxsw_sp, r->addr, r->f->fid, true);
- return err;
-}
-
static bool mlxsw_sp_lag_port_fid_member(struct mlxsw_sp_port *lag_port,
u16 fid)
{
static void
mlxsw_sp_port_pvid_vport_lag_join(struct mlxsw_sp_port *mlxsw_sp_port,
- u16 lag_id)
+ struct net_device *lag_dev, u16 lag_id)
{
struct mlxsw_sp_port *mlxsw_sp_vport;
struct mlxsw_sp_fid *f;
mlxsw_sp_vport->lag_id = lag_id;
mlxsw_sp_vport->lagged = 1;
+ mlxsw_sp_vport->dev = lag_dev;
}
static void
if (f)
f->leave(mlxsw_sp_vport);
+ mlxsw_sp_vport->dev = mlxsw_sp_port->dev;
mlxsw_sp_vport->lagged = 0;
}
mlxsw_sp_port->lagged = 1;
lag->ref_count++;
- mlxsw_sp_port_pvid_vport_lag_join(mlxsw_sp_port, lag_id);
+ mlxsw_sp_port_pvid_vport_lag_join(mlxsw_sp_port, lag_dev, lag_id);
return 0;
struct netdev_notifier_changeupper_info *info;
struct net_device *upper_dev;
struct mlxsw_sp *mlxsw_sp;
- int err;
+ int err = 0;
mlxsw_sp = mlxsw_sp_lower_get(br_dev);
if (!mlxsw_sp)
return 0;
- if (br_dev != mlxsw_sp->master_bridge.dev)
- return 0;
info = ptr;
switch (event) {
- case NETDEV_CHANGEUPPER:
+ case NETDEV_PRECHANGEUPPER:
upper_dev = info->upper_dev;
if (!is_vlan_dev(upper_dev))
- break;
- if (info->linking) {
- err = mlxsw_sp_master_bridge_vlan_link(mlxsw_sp,
- upper_dev);
- if (err)
- return err;
+ return -EINVAL;
+ if (is_vlan_dev(upper_dev) &&
+ br_dev != mlxsw_sp->master_bridge.dev)
+ return -EINVAL;
+ break;
+ case NETDEV_CHANGEUPPER:
+ upper_dev = info->upper_dev;
+ if (is_vlan_dev(upper_dev)) {
+ if (info->linking)
+ err = mlxsw_sp_master_bridge_vlan_link(mlxsw_sp,
+ upper_dev);
+ else
+ mlxsw_sp_master_bridge_vlan_unlink(mlxsw_sp,
+ upper_dev);
} else {
- mlxsw_sp_master_bridge_vlan_unlink(mlxsw_sp, upper_dev);
+ err = -EINVAL;
+ WARN_ON(1);
}
break;
}
- return 0;
+ return err;
}
static u16 mlxsw_sp_avail_vfid_get(const struct mlxsw_sp *mlxsw_sp)
int err = 0;
mlxsw_sp_vport = mlxsw_sp_port_vport_find(mlxsw_sp_port, vid);
+ if (!mlxsw_sp_vport)
+ return 0;
switch (event) {
case NETDEV_PRECHANGEUPPER:
break;
case NETDEV_CHANGEUPPER:
upper_dev = info->upper_dev;
- if (info->linking) {
- if (WARN_ON(!mlxsw_sp_vport))
- return -EINVAL;
- err = mlxsw_sp_vport_bridge_join(mlxsw_sp_vport,
- upper_dev);
+ if (netif_is_bridge_master(upper_dev)) {
+ if (info->linking)
+ err = mlxsw_sp_vport_bridge_join(mlxsw_sp_vport,
+ upper_dev);
+ else
+ mlxsw_sp_vport_bridge_leave(mlxsw_sp_vport);
} else {
- if (!mlxsw_sp_vport)
- return 0;
- mlxsw_sp_vport_bridge_leave(mlxsw_sp_vport);
+ err = -EINVAL;
+ WARN_ON(1);
}
+ break;
}
return err;
#define MLXSW_SP_VFID_MAX 1024 /* Bridged VLAN interfaces */
#define MLXSW_SP_RFID_BASE 15360
-#define MLXSW_SP_INVALID_RIF 0xffff
#define MLXSW_SP_MID_MAX 7000
}
struct mlxsw_sp_port;
+struct mlxsw_sp_rif;
struct mlxsw_sp_upper {
struct net_device *dev;
u16 fid;
};
-struct mlxsw_sp_rif {
- struct list_head nexthop_list;
- struct list_head neigh_list;
- struct net_device *dev;
- unsigned int ref_count;
- struct mlxsw_sp_fid *f;
- unsigned char addr[ETH_ALEN];
- int mtu;
- u16 rif;
-};
-
struct mlxsw_sp_mid {
struct list_head list;
unsigned char addr[ETH_ALEN];
return fid >= MLXSW_SP_VFID_BASE && fid < MLXSW_SP_RFID_BASE;
}
-static inline bool mlxsw_sp_fid_is_rfid(u16 fid)
-{
- return fid >= MLXSW_SP_RFID_BASE;
-}
-
-static inline u16 mlxsw_sp_rif_sp_to_fid(u16 rif)
-{
- return MLXSW_SP_RFID_BASE + rif;
-}
-
struct mlxsw_sp_sb_pr {
enum mlxsw_reg_sbpr_mode mode;
u32 size;
struct mlxsw_sp_vr {
u16 id; /* virtual router ID */
- bool used;
- enum mlxsw_sp_l3proto proto;
u32 tb_id; /* kernel fib table id */
- struct mlxsw_sp_lpm_tree *lpm_tree;
- struct mlxsw_sp_fib *fib;
+ unsigned int rif_count;
+ struct mlxsw_sp_fib *fib4;
};
enum mlxsw_sp_span_type {
};
bool mlxsw_sp_port_dev_check(const struct net_device *dev);
+struct mlxsw_sp *mlxsw_sp_lower_get(struct net_device *dev);
struct mlxsw_sp_port *mlxsw_sp_port_lower_dev_hold(struct net_device *dev);
void mlxsw_sp_port_dev_put(struct mlxsw_sp_port *mlxsw_sp_port);
return NULL;
}
-static inline struct mlxsw_sp_rif *
-mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
- const struct net_device *dev)
-{
- int i;
-
- for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
- if (mlxsw_sp->rifs[i] && mlxsw_sp->rifs[i]->dev == dev)
- return mlxsw_sp->rifs[i];
-
- return NULL;
-}
-
enum mlxsw_sp_flood_table {
MLXSW_SP_FLOOD_TABLE_UC,
MLXSW_SP_FLOOD_TABLE_BC,
bool adding);
struct mlxsw_sp_fid *mlxsw_sp_fid_create(struct mlxsw_sp *mlxsw_sp, u16 fid);
void mlxsw_sp_fid_destroy(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_fid *f);
-void mlxsw_sp_rif_bridge_destroy(struct mlxsw_sp *mlxsw_sp,
- struct mlxsw_sp_rif *r);
int mlxsw_sp_port_ets_set(struct mlxsw_sp_port *mlxsw_sp_port,
enum mlxsw_reg_qeec_hr hr, u8 index, u8 next_index,
bool dwrr, u8 dwrr_weight);
void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp);
int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
unsigned long event, void *ptr);
-void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
- struct mlxsw_sp_rif *r);
+int mlxsw_sp_netdevice_router_port_event(struct net_device *dev);
+int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
+ unsigned long event, void *ptr);
+void mlxsw_sp_rif_bridge_destroy(struct mlxsw_sp *mlxsw_sp,
+ struct mlxsw_sp_rif *r);
int mlxsw_sp_kvdl_alloc(struct mlxsw_sp *mlxsw_sp, unsigned int entry_count);
void mlxsw_sp_kvdl_free(struct mlxsw_sp *mlxsw_sp, int entry_index);
int mlxsw_sp_acl_rulei_act_fwd(struct mlxsw_sp *mlxsw_sp,
struct mlxsw_sp_acl_rule_info *rulei,
struct net_device *out_dev);
+int mlxsw_sp_acl_rulei_act_vlan(struct mlxsw_sp *mlxsw_sp,
+ struct mlxsw_sp_acl_rule_info *rulei,
+ u32 action, u16 vid, u16 proto, u8 prio);
struct mlxsw_sp_acl_rule;
#include <linux/string.h>
#include <linux/rhashtable.h>
#include <linux/netdevice.h>
+#include <net/tc_act/tc_vlan.h>
#include "reg.h"
#include "core.h"
local_port, in_port);
}
+int mlxsw_sp_acl_rulei_act_vlan(struct mlxsw_sp *mlxsw_sp,
+ struct mlxsw_sp_acl_rule_info *rulei,
+ u32 action, u16 vid, u16 proto, u8 prio)
+{
+ u8 ethertype;
+
+ if (action == TCA_VLAN_ACT_MODIFY) {
+ switch (proto) {
+ case ETH_P_8021Q:
+ ethertype = 0;
+ break;
+ case ETH_P_8021AD:
+ ethertype = 1;
+ break;
+ default:
+ dev_err(mlxsw_sp->bus_info->dev, "Unsupported VLAN protocol %#04x\n",
+ proto);
+ return -EINVAL;
+ }
+
+ return mlxsw_afa_block_append_vlan_modify(rulei->act_block,
+ vid, prio, ethertype);
+ } else {
+ dev_err(mlxsw_sp->bus_info->dev, "Unsupported VLAN action\n");
+ return -EINVAL;
+ }
+}
+
struct mlxsw_sp_acl_rule *
mlxsw_sp_acl_rule_create(struct mlxsw_sp *mlxsw_sp,
struct mlxsw_sp_acl_ruleset *ruleset,
static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_l2_dmac[] = {
MLXSW_AFK_ELEMENT_INST_BUF(DMAC, 0x00, 6),
+ MLXSW_AFK_ELEMENT_INST_U32(PCP, 0x08, 13, 3),
+ MLXSW_AFK_ELEMENT_INST_U32(VID, 0x08, 0, 12),
MLXSW_AFK_ELEMENT_INST_U32(SRC_SYS_PORT, 0x0C, 0, 16),
};
static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_l2_smac[] = {
MLXSW_AFK_ELEMENT_INST_BUF(SMAC, 0x00, 6),
+ MLXSW_AFK_ELEMENT_INST_U32(PCP, 0x08, 13, 3),
+ MLXSW_AFK_ELEMENT_INST_U32(VID, 0x08, 0, 12),
MLXSW_AFK_ELEMENT_INST_U32(SRC_SYS_PORT, 0x0C, 0, 16),
};
};
static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_ipv4_ex[] = {
+ MLXSW_AFK_ELEMENT_INST_U32(VID, 0x00, 0, 12),
+ MLXSW_AFK_ELEMENT_INST_U32(PCP, 0x08, 29, 3),
MLXSW_AFK_ELEMENT_INST_U32(SRC_L4_PORT, 0x08, 0, 16),
MLXSW_AFK_ELEMENT_INST_U32(DST_L4_PORT, 0x0C, 0, 16),
};
MLXSW_AFK_ELEMENT_DST_IP4,
MLXSW_AFK_ELEMENT_DST_L4_PORT,
MLXSW_AFK_ELEMENT_SRC_L4_PORT,
+ MLXSW_AFK_ELEMENT_VID,
+ MLXSW_AFK_ELEMENT_PCP,
};
static const enum mlxsw_afk_element mlxsw_sp_acl_tcam_pattern_ipv6[] = {
#include <net/pkt_cls.h>
#include <net/tc_act/tc_gact.h>
#include <net/tc_act/tc_mirred.h>
+#include <net/tc_act/tc_vlan.h>
#include "spectrum.h"
#include "core_acl_flex_keys.h"
out_dev);
if (err)
return err;
+ } else if (is_tcf_vlan(a)) {
+ u16 proto = be16_to_cpu(tcf_vlan_push_proto(a));
+ u32 action = tcf_vlan_action(a);
+ u8 prio = tcf_vlan_push_prio(a);
+ u16 vid = tcf_vlan_push_vid(a);
+
+ return mlxsw_sp_acl_rulei_act_vlan(mlxsw_sp, rulei,
+ action, vid,
+ proto, prio);
} else {
dev_err(mlxsw_sp->bus_info->dev, "Unsupported action\n");
return -EOPNOTSUPP;
BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
- BIT(FLOW_DISSECTOR_KEY_PORTS))) {
+ BIT(FLOW_DISSECTOR_KEY_PORTS) |
+ BIT(FLOW_DISSECTOR_KEY_VLAN))) {
dev_err(mlxsw_sp->bus_info->dev, "Unsupported key\n");
return -EOPNOTSUPP;
}
sizeof(key->src));
}
+ if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_VLAN)) {
+ struct flow_dissector_key_vlan *key =
+ skb_flow_dissector_target(f->dissector,
+ FLOW_DISSECTOR_KEY_VLAN,
+ f->key);
+ struct flow_dissector_key_vlan *mask =
+ skb_flow_dissector_target(f->dissector,
+ FLOW_DISSECTOR_KEY_VLAN,
+ f->mask);
+ if (mask->vlan_id != 0)
+ mlxsw_sp_acl_rulei_keymask_u32(rulei,
+ MLXSW_AFK_ELEMENT_VID,
+ key->vlan_id,
+ mask->vlan_id);
+ if (mask->vlan_priority != 0)
+ mlxsw_sp_acl_rulei_keymask_u32(rulei,
+ MLXSW_AFK_ELEMENT_PCP,
+ key->vlan_priority,
+ mask->vlan_priority);
+ }
+
if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS)
mlxsw_sp_flower_parse_ipv4(rulei, f);
#include "core.h"
#include "reg.h"
+struct mlxsw_sp_rif {
+ struct list_head nexthop_list;
+ struct list_head neigh_list;
+ struct net_device *dev;
+ struct mlxsw_sp_fid *f;
+ unsigned char addr[ETH_ALEN];
+ int mtu;
+ u16 rif;
+ u16 vr_id;
+};
+
+static struct mlxsw_sp_rif *
+mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
+ const struct net_device *dev);
+
#define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
}
-static void
-mlxsw_sp_prefix_usage_zero(struct mlxsw_sp_prefix_usage *prefix_usage)
-{
- memset(prefix_usage, 0, sizeof(*prefix_usage));
-}
-
static void
mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
unsigned char prefix_len)
struct list_head entry_list;
struct list_head list;
struct rhash_head ht_node;
- struct mlxsw_sp_vr *vr;
+ struct mlxsw_sp_fib *fib;
struct mlxsw_sp_fib_key key;
};
struct mlxsw_sp_fib {
struct rhashtable ht;
struct list_head node_list;
+ struct mlxsw_sp_vr *vr;
+ struct mlxsw_sp_lpm_tree *lpm_tree;
unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
struct mlxsw_sp_prefix_usage prefix_usage;
+ enum mlxsw_sp_l3proto proto;
};
static const struct rhashtable_params mlxsw_sp_fib_ht_params;
-static struct mlxsw_sp_fib *mlxsw_sp_fib_create(void)
+static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr,
+ enum mlxsw_sp_l3proto proto)
{
struct mlxsw_sp_fib *fib;
int err;
if (err)
goto err_rhashtable_init;
INIT_LIST_HEAD(&fib->node_list);
+ fib->proto = proto;
+ fib->vr = vr;
return fib;
err_rhashtable_init:
static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
{
WARN_ON(!list_empty(&fib->node_list));
+ WARN_ON(fib->lpm_tree);
rhashtable_destroy(&fib->ht);
kfree(fib);
}
static struct mlxsw_sp_lpm_tree *
-mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp, bool one_reserved)
+mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
{
static struct mlxsw_sp_lpm_tree *lpm_tree;
int i;
for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
lpm_tree = &mlxsw_sp->router.lpm_trees[i];
- if (lpm_tree->ref_count == 0) {
- if (one_reserved)
- one_reserved = false;
- else
- return lpm_tree;
- }
+ if (lpm_tree->ref_count == 0)
+ return lpm_tree;
}
return NULL;
}
static struct mlxsw_sp_lpm_tree *
mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
struct mlxsw_sp_prefix_usage *prefix_usage,
- enum mlxsw_sp_l3proto proto, bool one_reserved)
+ enum mlxsw_sp_l3proto proto)
{
struct mlxsw_sp_lpm_tree *lpm_tree;
int err;
- lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp, one_reserved);
+ lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
if (!lpm_tree)
return ERR_PTR(-EBUSY);
lpm_tree->proto = proto;
static struct mlxsw_sp_lpm_tree *
mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
struct mlxsw_sp_prefix_usage *prefix_usage,
- enum mlxsw_sp_l3proto proto, bool one_reserved)
+ enum mlxsw_sp_l3proto proto)
{
struct mlxsw_sp_lpm_tree *lpm_tree;
int i;
goto inc_ref_count;
}
lpm_tree = mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage,
- proto, one_reserved);
+ proto);
if (IS_ERR(lpm_tree))
return lpm_tree;
}
}
+static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
+{
+ return !!vr->fib4;
+}
+
static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
{
struct mlxsw_sp_vr *vr;
for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
vr = &mlxsw_sp->router.vrs[i];
- if (!vr->used)
+ if (!mlxsw_sp_vr_is_used(vr))
return vr;
}
return NULL;
}
static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
- struct mlxsw_sp_vr *vr)
+ const struct mlxsw_sp_fib *fib)
{
char raltb_pl[MLXSW_REG_RALTB_LEN];
- mlxsw_reg_raltb_pack(raltb_pl, vr->id,
- (enum mlxsw_reg_ralxx_protocol) vr->proto,
- vr->lpm_tree->id);
+ mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
+ (enum mlxsw_reg_ralxx_protocol) fib->proto,
+ fib->lpm_tree->id);
return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
}
static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
- struct mlxsw_sp_vr *vr)
+ const struct mlxsw_sp_fib *fib)
{
char raltb_pl[MLXSW_REG_RALTB_LEN];
/* Bind to tree 0 which is default */
- mlxsw_reg_raltb_pack(raltb_pl, vr->id,
- (enum mlxsw_reg_ralxx_protocol) vr->proto, 0);
+ mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
+ (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
}
}
static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
- u32 tb_id,
- enum mlxsw_sp_l3proto proto)
+ u32 tb_id)
{
struct mlxsw_sp_vr *vr;
int i;
for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
vr = &mlxsw_sp->router.vrs[i];
- if (vr->used && vr->proto == proto && vr->tb_id == tb_id)
+ if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
return vr;
}
return NULL;
}
+static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
+ enum mlxsw_sp_l3proto proto)
+{
+ switch (proto) {
+ case MLXSW_SP_L3_PROTO_IPV4:
+ return vr->fib4;
+ case MLXSW_SP_L3_PROTO_IPV6:
+ BUG_ON(1);
+ }
+ return NULL;
+}
+
static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
- unsigned char prefix_len,
- u32 tb_id,
- enum mlxsw_sp_l3proto proto)
+ u32 tb_id)
{
- struct mlxsw_sp_prefix_usage req_prefix_usage;
- struct mlxsw_sp_lpm_tree *lpm_tree;
struct mlxsw_sp_vr *vr;
- int err;
vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
if (!vr)
return ERR_PTR(-EBUSY);
- vr->fib = mlxsw_sp_fib_create();
- if (IS_ERR(vr->fib))
- return ERR_CAST(vr->fib);
-
- vr->proto = proto;
+ vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4);
+ if (IS_ERR(vr->fib4))
+ return ERR_CAST(vr->fib4);
vr->tb_id = tb_id;
- mlxsw_sp_prefix_usage_zero(&req_prefix_usage);
- mlxsw_sp_prefix_usage_set(&req_prefix_usage, prefix_len);
- lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
- proto, true);
- if (IS_ERR(lpm_tree)) {
- err = PTR_ERR(lpm_tree);
- goto err_tree_get;
- }
- vr->lpm_tree = lpm_tree;
- err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, vr);
- if (err)
- goto err_tree_bind;
-
- vr->used = true;
return vr;
-
-err_tree_bind:
- mlxsw_sp_lpm_tree_put(mlxsw_sp, vr->lpm_tree);
-err_tree_get:
- mlxsw_sp_fib_destroy(vr->fib);
-
- return ERR_PTR(err);
}
-static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp,
- struct mlxsw_sp_vr *vr)
+static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr)
{
- mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, vr);
- mlxsw_sp_lpm_tree_put(mlxsw_sp, vr->lpm_tree);
- mlxsw_sp_fib_destroy(vr->fib);
- vr->used = false;
+ mlxsw_sp_fib_destroy(vr->fib4);
+ vr->fib4 = NULL;
}
static int
-mlxsw_sp_vr_lpm_tree_check(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr,
+mlxsw_sp_vr_lpm_tree_check(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_fib *fib,
struct mlxsw_sp_prefix_usage *req_prefix_usage)
{
- struct mlxsw_sp_lpm_tree *lpm_tree = vr->lpm_tree;
+ struct mlxsw_sp_lpm_tree *lpm_tree = fib->lpm_tree;
struct mlxsw_sp_lpm_tree *new_tree;
int err;
return 0;
new_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, req_prefix_usage,
- vr->proto, false);
+ fib->proto);
if (IS_ERR(new_tree)) {
/* We failed to get a tree according to the required
* prefix usage. However, the current tree might be still good
}
/* Prevent packet loss by overwriting existing binding */
- vr->lpm_tree = new_tree;
- err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, vr);
+ fib->lpm_tree = new_tree;
+ err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
if (err)
goto err_tree_bind;
mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
return 0;
err_tree_bind:
- vr->lpm_tree = lpm_tree;
+ fib->lpm_tree = lpm_tree;
mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
return err;
}
-static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp,
- unsigned char prefix_len,
- u32 tb_id,
- enum mlxsw_sp_l3proto proto)
+static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id)
{
struct mlxsw_sp_vr *vr;
- int err;
tb_id = mlxsw_sp_fix_tb_id(tb_id);
- vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id, proto);
- if (!vr) {
- vr = mlxsw_sp_vr_create(mlxsw_sp, prefix_len, tb_id, proto);
- if (IS_ERR(vr))
- return vr;
- } else {
- struct mlxsw_sp_prefix_usage req_prefix_usage;
-
- mlxsw_sp_prefix_usage_cpy(&req_prefix_usage,
- &vr->fib->prefix_usage);
- mlxsw_sp_prefix_usage_set(&req_prefix_usage, prefix_len);
- /* Need to replace LPM tree in case new prefix is required. */
- err = mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, vr,
- &req_prefix_usage);
- if (err)
- return ERR_PTR(err);
- }
+ vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
+ if (!vr)
+ vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id);
return vr;
}
-static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr)
+static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr)
{
- /* Destroy virtual router entity in case the associated FIB is empty
- * and allow it to be used for other tables in future. Otherwise,
- * check if some prefix usage did not disappear and change tree if
- * that is the case. Note that in case new, smaller tree cannot be
- * allocated, the original one will be kept being used.
- */
- if (mlxsw_sp_prefix_usage_none(&vr->fib->prefix_usage))
- mlxsw_sp_vr_destroy(mlxsw_sp, vr);
- else
- mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, vr,
- &vr->fib->prefix_usage);
+ if (!vr->rif_count && list_empty(&vr->fib4->node_list))
+ mlxsw_sp_vr_destroy(vr);
}
static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
}
static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
- struct mlxsw_sp_vr *vr,
+ const struct mlxsw_sp_fib *fib,
u32 adj_index, u16 ecmp_size,
u32 new_adj_index,
u16 new_ecmp_size)
char raleu_pl[MLXSW_REG_RALEU_LEN];
mlxsw_reg_raleu_pack(raleu_pl,
- (enum mlxsw_reg_ralxx_protocol) vr->proto, vr->id,
- adj_index, ecmp_size, new_adj_index,
+ (enum mlxsw_reg_ralxx_protocol) fib->proto,
+ fib->vr->id, adj_index, ecmp_size, new_adj_index,
new_ecmp_size);
return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
}
u32 old_adj_index, u16 old_ecmp_size)
{
struct mlxsw_sp_fib_entry *fib_entry;
- struct mlxsw_sp_vr *vr = NULL;
+ struct mlxsw_sp_fib *fib = NULL;
int err;
list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
- if (vr == fib_entry->fib_node->vr)
+ if (fib == fib_entry->fib_node->fib)
continue;
- vr = fib_entry->fib_node->vr;
- err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, vr,
+ fib = fib_entry->fib_node->fib;
+ err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
old_adj_index,
old_ecmp_size,
nh_grp->adj_index,
if (err)
return err;
+ if (!dev)
+ return 0;
+
in_dev = __in_dev_get_rtnl(dev);
if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
fib_nh->nh_flags & RTNH_F_LINKDOWN)
{
fib_entry->offloaded = true;
- switch (fib_entry->fib_node->vr->proto) {
+ switch (fib_entry->fib_node->fib->proto) {
case MLXSW_SP_L3_PROTO_IPV4:
fib_info_offload_inc(fib_entry->nh_group->key.fi);
break;
static void
mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
{
- switch (fib_entry->fib_node->vr->proto) {
+ switch (fib_entry->fib_node->fib->proto) {
case MLXSW_SP_L3_PROTO_IPV4:
fib_info_offload_dec(fib_entry->nh_group->key.fi);
break;
enum mlxsw_reg_ralue_op op)
{
char ralue_pl[MLXSW_REG_RALUE_LEN];
+ struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
- struct mlxsw_sp_vr *vr = fib_entry->fib_node->vr;
enum mlxsw_reg_ralue_trap_action trap_action;
u16 trap_id = 0;
u32 adjacency_index = 0;
}
mlxsw_reg_ralue_pack4(ralue_pl,
- (enum mlxsw_reg_ralxx_protocol) vr->proto, op,
- vr->id, fib_entry->fib_node->key.prefix_len,
+ (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
+ fib->vr->id, fib_entry->fib_node->key.prefix_len,
*p_dip);
mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
adjacency_index, ecmp_size);
enum mlxsw_reg_ralue_op op)
{
struct mlxsw_sp_rif *r = fib_entry->nh_group->nh_rif;
+ struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
enum mlxsw_reg_ralue_trap_action trap_action;
char ralue_pl[MLXSW_REG_RALUE_LEN];
u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
- struct mlxsw_sp_vr *vr = fib_entry->fib_node->vr;
u16 trap_id = 0;
u16 rif = 0;
}
mlxsw_reg_ralue_pack4(ralue_pl,
- (enum mlxsw_reg_ralxx_protocol) vr->proto, op,
- vr->id, fib_entry->fib_node->key.prefix_len,
+ (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
+ fib->vr->id, fib_entry->fib_node->key.prefix_len,
*p_dip);
mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id, rif);
return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
struct mlxsw_sp_fib_entry *fib_entry,
enum mlxsw_reg_ralue_op op)
{
+ struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
char ralue_pl[MLXSW_REG_RALUE_LEN];
u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
- struct mlxsw_sp_vr *vr = fib_entry->fib_node->vr;
mlxsw_reg_ralue_pack4(ralue_pl,
- (enum mlxsw_reg_ralxx_protocol) vr->proto, op,
- vr->id, fib_entry->fib_node->key.prefix_len,
+ (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
+ fib->vr->id, fib_entry->fib_node->key.prefix_len,
*p_dip);
mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
{
int err = -EINVAL;
- switch (fib_entry->fib_node->vr->proto) {
+ switch (fib_entry->fib_node->fib->proto) {
case MLXSW_SP_L3_PROTO_IPV4:
err = mlxsw_sp_fib_entry_op4(mlxsw_sp, fib_entry, op);
break;
{
struct fib_info *fi = fen_info->fi;
- if (fen_info->type == RTN_LOCAL || fen_info->type == RTN_BROADCAST) {
+ switch (fen_info->type) {
+ case RTN_BROADCAST: /* fall through */
+ case RTN_LOCAL:
fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
return 0;
- }
- if (fen_info->type != RTN_UNICAST)
- return -EINVAL;
- if (fi->fib_nh->nh_scope != RT_SCOPE_LINK)
+ case RTN_UNREACHABLE: /* fall through */
+ case RTN_BLACKHOLE: /* fall through */
+ case RTN_PROHIBIT:
+ /* Packets hitting these routes need to be trapped, but
+ * can do so with a lower priority than packets directed
+ * at the host, so use action type local instead of trap.
+ */
fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
- else
- fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
- return 0;
+ return 0;
+ case RTN_UNICAST:
+ if (fi->fib_nh->nh_scope != RT_SCOPE_LINK)
+ fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
+ else
+ fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
+ return 0;
+ default:
+ return -EINVAL;
+ }
}
static struct mlxsw_sp_fib_entry *
}
static struct mlxsw_sp_fib_node *
-mlxsw_sp_fib_node_create(struct mlxsw_sp_vr *vr, const void *addr,
+mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
size_t addr_len, unsigned char prefix_len)
{
struct mlxsw_sp_fib_node *fib_node;
return NULL;
INIT_LIST_HEAD(&fib_node->entry_list);
- list_add(&fib_node->list, &vr->fib->node_list);
+ list_add(&fib_node->list, &fib->node_list);
memcpy(fib_node->key.addr, addr, addr_len);
fib_node->key.prefix_len = prefix_len;
- mlxsw_sp_fib_node_insert(vr->fib, fib_node);
- fib_node->vr = vr;
return fib_node;
}
static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
{
- mlxsw_sp_fib_node_remove(fib_node->vr->fib, fib_node);
list_del(&fib_node->list);
WARN_ON(!list_empty(&fib_node->entry_list));
kfree(fib_node);
static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node)
{
unsigned char prefix_len = fib_node->key.prefix_len;
- struct mlxsw_sp_fib *fib = fib_node->vr->fib;
+ struct mlxsw_sp_fib *fib = fib_node->fib;
if (fib->prefix_ref_count[prefix_len]++ == 0)
mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node)
{
unsigned char prefix_len = fib_node->key.prefix_len;
- struct mlxsw_sp_fib *fib = fib_node->vr->fib;
+ struct mlxsw_sp_fib *fib = fib_node->fib;
if (--fib->prefix_ref_count[prefix_len] == 0)
mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
}
+static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
+ struct mlxsw_sp_fib_node *fib_node,
+ struct mlxsw_sp_fib *fib)
+{
+ struct mlxsw_sp_prefix_usage req_prefix_usage;
+ struct mlxsw_sp_lpm_tree *lpm_tree;
+ int err;
+
+ err = mlxsw_sp_fib_node_insert(fib, fib_node);
+ if (err)
+ return err;
+ fib_node->fib = fib;
+
+ mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &fib->prefix_usage);
+ mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
+
+ if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
+ err = mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib,
+ &req_prefix_usage);
+ if (err)
+ goto err_tree_check;
+ } else {
+ lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
+ fib->proto);
+ if (IS_ERR(lpm_tree))
+ return PTR_ERR(lpm_tree);
+ fib->lpm_tree = lpm_tree;
+ err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
+ if (err)
+ goto err_tree_bind;
+ }
+
+ mlxsw_sp_fib_node_prefix_inc(fib_node);
+
+ return 0;
+
+err_tree_bind:
+ fib->lpm_tree = NULL;
+ mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
+err_tree_check:
+ fib_node->fib = NULL;
+ mlxsw_sp_fib_node_remove(fib, fib_node);
+ return err;
+}
+
+static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
+ struct mlxsw_sp_fib_node *fib_node)
+{
+ struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
+ struct mlxsw_sp_fib *fib = fib_node->fib;
+
+ mlxsw_sp_fib_node_prefix_dec(fib_node);
+
+ if (mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
+ mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
+ fib->lpm_tree = NULL;
+ mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
+ } else {
+ mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib, &fib->prefix_usage);
+ }
+
+ fib_node->fib = NULL;
+ mlxsw_sp_fib_node_remove(fib, fib_node);
+}
+
static struct mlxsw_sp_fib_node *
mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp,
const struct fib_entry_notifier_info *fen_info)
{
struct mlxsw_sp_fib_node *fib_node;
+ struct mlxsw_sp_fib *fib;
struct mlxsw_sp_vr *vr;
int err;
- vr = mlxsw_sp_vr_get(mlxsw_sp, fen_info->dst_len, fen_info->tb_id,
- MLXSW_SP_L3_PROTO_IPV4);
+ vr = mlxsw_sp_vr_get(mlxsw_sp, fen_info->tb_id);
if (IS_ERR(vr))
return ERR_CAST(vr);
+ fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
- fib_node = mlxsw_sp_fib_node_lookup(vr->fib, &fen_info->dst,
+ fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
sizeof(fen_info->dst),
fen_info->dst_len);
if (fib_node)
return fib_node;
- fib_node = mlxsw_sp_fib_node_create(vr, &fen_info->dst,
+ fib_node = mlxsw_sp_fib_node_create(fib, &fen_info->dst,
sizeof(fen_info->dst),
fen_info->dst_len);
if (!fib_node) {
goto err_fib_node_create;
}
+ err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
+ if (err)
+ goto err_fib_node_init;
+
return fib_node;
+err_fib_node_init:
+ mlxsw_sp_fib_node_destroy(fib_node);
err_fib_node_create:
- mlxsw_sp_vr_put(mlxsw_sp, vr);
+ mlxsw_sp_vr_put(vr);
return ERR_PTR(err);
}
static void mlxsw_sp_fib4_node_put(struct mlxsw_sp *mlxsw_sp,
struct mlxsw_sp_fib_node *fib_node)
{
- struct mlxsw_sp_vr *vr = fib_node->vr;
+ struct mlxsw_sp_vr *vr = fib_node->fib->vr;
if (!list_empty(&fib_node->entry_list))
return;
+ mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
mlxsw_sp_fib_node_destroy(fib_node);
- mlxsw_sp_vr_put(mlxsw_sp, vr);
+ mlxsw_sp_vr_put(vr);
}
static struct mlxsw_sp_fib_entry *
if (err)
goto err_fib4_node_entry_add;
- mlxsw_sp_fib_node_prefix_inc(fib_node);
-
return 0;
err_fib4_node_entry_add:
{
struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
- mlxsw_sp_fib_node_prefix_dec(fib_node);
mlxsw_sp_fib4_node_entry_del(mlxsw_sp, fib_node, fib_entry);
mlxsw_sp_fib4_node_list_remove(fib_entry);
}
{
char ralta_pl[MLXSW_REG_RALTA_LEN];
char ralst_pl[MLXSW_REG_RALST_LEN];
- char raltb_pl[MLXSW_REG_RALTB_LEN];
- char ralue_pl[MLXSW_REG_RALUE_LEN];
- int err;
+ int i, err;
mlxsw_reg_ralta_pack(ralta_pl, true, MLXSW_REG_RALXX_PROTOCOL_IPV4,
MLXSW_SP_LPM_TREE_MIN);
if (err)
return err;
- mlxsw_reg_raltb_pack(raltb_pl, 0, MLXSW_REG_RALXX_PROTOCOL_IPV4,
- MLXSW_SP_LPM_TREE_MIN);
- err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
- if (err)
- return err;
+ for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
+ struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[i];
+ char raltb_pl[MLXSW_REG_RALTB_LEN];
+ char ralue_pl[MLXSW_REG_RALUE_LEN];
- mlxsw_reg_ralue_pack4(ralue_pl, MLXSW_SP_L3_PROTO_IPV4,
- MLXSW_REG_RALUE_OP_WRITE_WRITE, 0, 0, 0);
- mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
- return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
+ if (!mlxsw_sp_vr_is_used(vr))
+ continue;
+
+ mlxsw_reg_raltb_pack(raltb_pl, vr->id,
+ MLXSW_REG_RALXX_PROTOCOL_IPV4,
+ MLXSW_SP_LPM_TREE_MIN);
+ err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
+ raltb_pl);
+ if (err)
+ return err;
+
+ mlxsw_reg_ralue_pack4(ralue_pl, MLXSW_SP_L3_PROTO_IPV4,
+ MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0,
+ 0);
+ mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
+ err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
+ ralue_pl);
+ if (err)
+ return err;
+ }
+
+ return 0;
}
static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
struct mlxsw_sp_fib_node *fib_node)
{
- switch (fib_node->vr->proto) {
+ switch (fib_node->fib->proto) {
case MLXSW_SP_L3_PROTO_IPV4:
mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
break;
}
}
-static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
+static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
+ struct mlxsw_sp_vr *vr,
+ enum mlxsw_sp_l3proto proto)
{
+ struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
struct mlxsw_sp_fib_node *fib_node, *tmp;
- struct mlxsw_sp_vr *vr;
+
+ list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
+ bool do_break = &tmp->list == &fib->node_list;
+
+ mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
+ if (do_break)
+ break;
+ }
+}
+
+static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
+{
int i;
for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
- vr = &mlxsw_sp->router.vrs[i];
+ struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[i];
- if (!vr->used)
+ if (!mlxsw_sp_vr_is_used(vr))
continue;
-
- list_for_each_entry_safe(fib_node, tmp, &vr->fib->node_list,
- list) {
- bool do_break = &tmp->list == &vr->fib->node_list;
-
- mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
- if (do_break)
- break;
- }
+ mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
}
}
dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
}
-static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
-{
- char ritr_pl[MLXSW_REG_RITR_LEN];
- int err;
-
- mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
- err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
- if (WARN_ON_ONCE(err))
- return err;
-
- mlxsw_reg_ritr_enable_set(ritr_pl, false);
- return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
-}
-
-void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
- struct mlxsw_sp_rif *r)
-{
- mlxsw_sp_router_rif_disable(mlxsw_sp, r->rif);
- mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, r);
- mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, r);
-}
-
-static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
-{
- char rgcr_pl[MLXSW_REG_RGCR_LEN];
- u64 max_rifs;
- int err;
-
- if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
- return -EIO;
-
- max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
- mlxsw_sp->rifs = kcalloc(max_rifs, sizeof(struct mlxsw_sp_rif *),
- GFP_KERNEL);
- if (!mlxsw_sp->rifs)
- return -ENOMEM;
-
- mlxsw_reg_rgcr_pack(rgcr_pl, true);
- mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
- err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
- if (err)
- goto err_rgcr_fail;
-
- return 0;
-
-err_rgcr_fail:
- kfree(mlxsw_sp->rifs);
- return err;
-}
-
-static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
-{
- char rgcr_pl[MLXSW_REG_RGCR_LEN];
- int i;
-
- mlxsw_reg_rgcr_pack(rgcr_pl, false);
- mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
-
- for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
- WARN_ON_ONCE(mlxsw_sp->rifs[i]);
-
- kfree(mlxsw_sp->rifs);
-}
-
struct mlxsw_sp_fib_event_work {
struct work_struct work;
union {
return NOTIFY_DONE;
}
-static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
+static struct mlxsw_sp_rif *
+mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
+ const struct net_device *dev)
{
- struct mlxsw_sp *mlxsw_sp = container_of(nb, struct mlxsw_sp, fib_nb);
+ int i;
- /* Flush pending FIB notifications and then flush the device's
- * table before requesting another dump. The FIB notification
- * block is unregistered, so no need to take RTNL.
- */
- mlxsw_core_flush_owq();
- mlxsw_sp_router_fib_flush(mlxsw_sp);
+ for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
+ if (mlxsw_sp->rifs[i] && mlxsw_sp->rifs[i]->dev == dev)
+ return mlxsw_sp->rifs[i];
+
+ return NULL;
+}
+
+static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
+{
+ char ritr_pl[MLXSW_REG_RITR_LEN];
+ int err;
+
+ mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
+ err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
+ if (WARN_ON_ONCE(err))
+ return err;
+
+ mlxsw_reg_ritr_enable_set(ritr_pl, false);
+ return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
+}
+
+static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
+ struct mlxsw_sp_rif *r)
+{
+ mlxsw_sp_router_rif_disable(mlxsw_sp, r->rif);
+ mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, r);
+ mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, r);
+}
+
+static bool mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *r,
+ const struct in_device *in_dev,
+ unsigned long event)
+{
+ switch (event) {
+ case NETDEV_UP:
+ if (!r)
+ return true;
+ return false;
+ case NETDEV_DOWN:
+ if (r && !in_dev->ifa_list)
+ return true;
+ /* It is possible we already removed the RIF ourselves
+ * if it was assigned to a netdev that is now a bridge
+ * or LAG slave.
+ */
+ return false;
+ }
+
+ return false;
+}
+
+#define MLXSW_SP_INVALID_RIF 0xffff
+static int mlxsw_sp_avail_rif_get(struct mlxsw_sp *mlxsw_sp)
+{
+ int i;
+
+ for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
+ if (!mlxsw_sp->rifs[i])
+ return i;
+
+ return MLXSW_SP_INVALID_RIF;
+}
+
+static void mlxsw_sp_vport_rif_sp_attr_get(struct mlxsw_sp_port *mlxsw_sp_vport,
+ bool *p_lagged, u16 *p_system_port)
+{
+ u8 local_port = mlxsw_sp_vport->local_port;
+
+ *p_lagged = mlxsw_sp_vport->lagged;
+ *p_system_port = *p_lagged ? mlxsw_sp_vport->lag_id : local_port;
+}
+
+static int mlxsw_sp_vport_rif_sp_op(struct mlxsw_sp_port *mlxsw_sp_vport,
+ u16 vr_id, struct net_device *l3_dev,
+ u16 rif, bool create)
+{
+ struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
+ bool lagged = mlxsw_sp_vport->lagged;
+ char ritr_pl[MLXSW_REG_RITR_LEN];
+ u16 system_port;
+
+ mlxsw_reg_ritr_pack(ritr_pl, create, MLXSW_REG_RITR_SP_IF, rif, vr_id,
+ l3_dev->mtu, l3_dev->dev_addr);
+
+ mlxsw_sp_vport_rif_sp_attr_get(mlxsw_sp_vport, &lagged, &system_port);
+ mlxsw_reg_ritr_sp_if_pack(ritr_pl, lagged, system_port,
+ mlxsw_sp_vport_vid_get(mlxsw_sp_vport));
+
+ return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
+}
+
+static void mlxsw_sp_vport_rif_sp_leave(struct mlxsw_sp_port *mlxsw_sp_vport);
+
+static u16 mlxsw_sp_rif_sp_to_fid(u16 rif)
+{
+ return MLXSW_SP_RFID_BASE + rif;
+}
+
+static struct mlxsw_sp_fid *
+mlxsw_sp_rfid_alloc(u16 fid, struct net_device *l3_dev)
+{
+ struct mlxsw_sp_fid *f;
+
+ f = kzalloc(sizeof(*f), GFP_KERNEL);
+ if (!f)
+ return NULL;
+
+ f->leave = mlxsw_sp_vport_rif_sp_leave;
+ f->ref_count = 0;
+ f->dev = l3_dev;
+ f->fid = fid;
+
+ return f;
+}
+
+static struct mlxsw_sp_rif *
+mlxsw_sp_rif_alloc(u16 rif, u16 vr_id, struct net_device *l3_dev,
+ struct mlxsw_sp_fid *f)
+{
+ struct mlxsw_sp_rif *r;
+
+ r = kzalloc(sizeof(*r), GFP_KERNEL);
+ if (!r)
+ return NULL;
+
+ INIT_LIST_HEAD(&r->nexthop_list);
+ INIT_LIST_HEAD(&r->neigh_list);
+ ether_addr_copy(r->addr, l3_dev->dev_addr);
+ r->mtu = l3_dev->mtu;
+ r->vr_id = vr_id;
+ r->dev = l3_dev;
+ r->rif = rif;
+ r->f = f;
+
+ return r;
+}
+
+static struct mlxsw_sp_rif *
+mlxsw_sp_vport_rif_sp_create(struct mlxsw_sp_port *mlxsw_sp_vport,
+ struct net_device *l3_dev)
+{
+ struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
+ struct mlxsw_sp_vr *vr;
+ struct mlxsw_sp_fid *f;
+ struct mlxsw_sp_rif *r;
+ u16 fid, rif;
+ int err;
+
+ rif = mlxsw_sp_avail_rif_get(mlxsw_sp);
+ if (rif == MLXSW_SP_INVALID_RIF)
+ return ERR_PTR(-ERANGE);
+
+ vr = mlxsw_sp_vr_get(mlxsw_sp, RT_TABLE_MAIN);
+ if (IS_ERR(vr))
+ return ERR_CAST(vr);
+
+ err = mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, vr->id, l3_dev, rif,
+ true);
+ if (err)
+ goto err_vport_rif_sp_op;
+
+ fid = mlxsw_sp_rif_sp_to_fid(rif);
+ err = mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, fid, true);
+ if (err)
+ goto err_rif_fdb_op;
+
+ f = mlxsw_sp_rfid_alloc(fid, l3_dev);
+ if (!f) {
+ err = -ENOMEM;
+ goto err_rfid_alloc;
+ }
+
+ r = mlxsw_sp_rif_alloc(rif, vr->id, l3_dev, f);
+ if (!r) {
+ err = -ENOMEM;
+ goto err_rif_alloc;
+ }
+
+ f->r = r;
+ mlxsw_sp->rifs[rif] = r;
+ vr->rif_count++;
+
+ return r;
+
+err_rif_alloc:
+ kfree(f);
+err_rfid_alloc:
+ mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, fid, false);
+err_rif_fdb_op:
+ mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, vr->id, l3_dev, rif, false);
+err_vport_rif_sp_op:
+ mlxsw_sp_vr_put(vr);
+ return ERR_PTR(err);
+}
+
+static void mlxsw_sp_vport_rif_sp_destroy(struct mlxsw_sp_port *mlxsw_sp_vport,
+ struct mlxsw_sp_rif *r)
+{
+ struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
+ struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[r->vr_id];
+ struct net_device *l3_dev = r->dev;
+ struct mlxsw_sp_fid *f = r->f;
+ u16 fid = f->fid;
+ u16 rif = r->rif;
+
+ mlxsw_sp_router_rif_gone_sync(mlxsw_sp, r);
+
+ vr->rif_count--;
+ mlxsw_sp->rifs[rif] = NULL;
+ f->r = NULL;
+
+ kfree(r);
+
+ kfree(f);
+
+ mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, fid, false);
+
+ mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, vr->id, l3_dev, rif, false);
+
+ mlxsw_sp_vr_put(vr);
+}
+
+static int mlxsw_sp_vport_rif_sp_join(struct mlxsw_sp_port *mlxsw_sp_vport,
+ struct net_device *l3_dev)
+{
+ struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
+ struct mlxsw_sp_rif *r;
+
+ r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
+ if (!r) {
+ r = mlxsw_sp_vport_rif_sp_create(mlxsw_sp_vport, l3_dev);
+ if (IS_ERR(r))
+ return PTR_ERR(r);
+ }
+
+ mlxsw_sp_vport_fid_set(mlxsw_sp_vport, r->f);
+ r->f->ref_count++;
+
+ netdev_dbg(mlxsw_sp_vport->dev, "Joined FID=%d\n", r->f->fid);
+
+ return 0;
+}
+
+static void mlxsw_sp_vport_rif_sp_leave(struct mlxsw_sp_port *mlxsw_sp_vport)
+{
+ struct mlxsw_sp_fid *f = mlxsw_sp_vport_fid_get(mlxsw_sp_vport);
+
+ netdev_dbg(mlxsw_sp_vport->dev, "Left FID=%d\n", f->fid);
+
+ mlxsw_sp_vport_fid_set(mlxsw_sp_vport, NULL);
+ if (--f->ref_count == 0)
+ mlxsw_sp_vport_rif_sp_destroy(mlxsw_sp_vport, f->r);
+}
+
+static int mlxsw_sp_inetaddr_vport_event(struct net_device *l3_dev,
+ struct net_device *port_dev,
+ unsigned long event, u16 vid)
+{
+ struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
+ struct mlxsw_sp_port *mlxsw_sp_vport;
+
+ mlxsw_sp_vport = mlxsw_sp_port_vport_find(mlxsw_sp_port, vid);
+ if (WARN_ON(!mlxsw_sp_vport))
+ return -EINVAL;
+
+ switch (event) {
+ case NETDEV_UP:
+ return mlxsw_sp_vport_rif_sp_join(mlxsw_sp_vport, l3_dev);
+ case NETDEV_DOWN:
+ mlxsw_sp_vport_rif_sp_leave(mlxsw_sp_vport);
+ break;
+ }
+
+ return 0;
+}
+
+static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
+ unsigned long event)
+{
+ if (netif_is_bridge_port(port_dev) || netif_is_lag_port(port_dev))
+ return 0;
+
+ return mlxsw_sp_inetaddr_vport_event(port_dev, port_dev, event, 1);
+}
+
+static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
+ struct net_device *lag_dev,
+ unsigned long event, u16 vid)
+{
+ struct net_device *port_dev;
+ struct list_head *iter;
+ int err;
+
+ netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
+ if (mlxsw_sp_port_dev_check(port_dev)) {
+ err = mlxsw_sp_inetaddr_vport_event(l3_dev, port_dev,
+ event, vid);
+ if (err)
+ return err;
+ }
+ }
+
+ return 0;
+}
+
+static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
+ unsigned long event)
+{
+ if (netif_is_bridge_port(lag_dev))
+ return 0;
+
+ return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1);
+}
+
+static struct mlxsw_sp_fid *mlxsw_sp_bridge_fid_get(struct mlxsw_sp *mlxsw_sp,
+ struct net_device *l3_dev)
+{
+ u16 fid;
+
+ if (is_vlan_dev(l3_dev))
+ fid = vlan_dev_vlan_id(l3_dev);
+ else if (mlxsw_sp->master_bridge.dev == l3_dev)
+ fid = 1;
+ else
+ return mlxsw_sp_vfid_find(mlxsw_sp, l3_dev);
+
+ return mlxsw_sp_fid_find(mlxsw_sp, fid);
+}
+
+static enum mlxsw_flood_table_type mlxsw_sp_flood_table_type_get(u16 fid)
+{
+ return mlxsw_sp_fid_is_vfid(fid) ? MLXSW_REG_SFGC_TABLE_TYPE_FID :
+ MLXSW_REG_SFGC_TABLE_TYPE_FID_OFFEST;
+}
+
+static u16 mlxsw_sp_flood_table_index_get(u16 fid)
+{
+ return mlxsw_sp_fid_is_vfid(fid) ? mlxsw_sp_fid_to_vfid(fid) : fid;
+}
+
+static int mlxsw_sp_router_port_flood_set(struct mlxsw_sp *mlxsw_sp, u16 fid,
+ bool set)
+{
+ enum mlxsw_flood_table_type table_type;
+ char *sftr_pl;
+ u16 index;
+ int err;
+
+ sftr_pl = kmalloc(MLXSW_REG_SFTR_LEN, GFP_KERNEL);
+ if (!sftr_pl)
+ return -ENOMEM;
+
+ table_type = mlxsw_sp_flood_table_type_get(fid);
+ index = mlxsw_sp_flood_table_index_get(fid);
+ mlxsw_reg_sftr_pack(sftr_pl, MLXSW_SP_FLOOD_TABLE_BC, index, table_type,
+ 1, MLXSW_PORT_ROUTER_PORT, set);
+ err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sftr), sftr_pl);
+
+ kfree(sftr_pl);
+ return err;
+}
+
+static enum mlxsw_reg_ritr_if_type mlxsw_sp_rif_type_get(u16 fid)
+{
+ if (mlxsw_sp_fid_is_vfid(fid))
+ return MLXSW_REG_RITR_FID_IF;
+ else
+ return MLXSW_REG_RITR_VLAN_IF;
+}
+
+static int mlxsw_sp_rif_bridge_op(struct mlxsw_sp *mlxsw_sp, u16 vr_id,
+ struct net_device *l3_dev,
+ u16 fid, u16 rif,
+ bool create)
+{
+ enum mlxsw_reg_ritr_if_type rif_type;
+ char ritr_pl[MLXSW_REG_RITR_LEN];
+
+ rif_type = mlxsw_sp_rif_type_get(fid);
+ mlxsw_reg_ritr_pack(ritr_pl, create, rif_type, rif, vr_id, l3_dev->mtu,
+ l3_dev->dev_addr);
+ mlxsw_reg_ritr_fid_set(ritr_pl, rif_type, fid);
+
+ return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
+}
+
+static int mlxsw_sp_rif_bridge_create(struct mlxsw_sp *mlxsw_sp,
+ struct net_device *l3_dev,
+ struct mlxsw_sp_fid *f)
+{
+ struct mlxsw_sp_vr *vr;
+ struct mlxsw_sp_rif *r;
+ u16 rif;
+ int err;
+
+ rif = mlxsw_sp_avail_rif_get(mlxsw_sp);
+ if (rif == MLXSW_SP_INVALID_RIF)
+ return -ERANGE;
+
+ vr = mlxsw_sp_vr_get(mlxsw_sp, RT_TABLE_MAIN);
+ if (IS_ERR(vr))
+ return PTR_ERR(vr);
+
+ err = mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, true);
+ if (err)
+ goto err_port_flood_set;
+
+ err = mlxsw_sp_rif_bridge_op(mlxsw_sp, vr->id, l3_dev, f->fid, rif,
+ true);
+ if (err)
+ goto err_rif_bridge_op;
+
+ err = mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, true);
+ if (err)
+ goto err_rif_fdb_op;
+
+ r = mlxsw_sp_rif_alloc(rif, vr->id, l3_dev, f);
+ if (!r) {
+ err = -ENOMEM;
+ goto err_rif_alloc;
+ }
+
+ f->r = r;
+ mlxsw_sp->rifs[rif] = r;
+ vr->rif_count++;
+
+ netdev_dbg(l3_dev, "RIF=%d created\n", rif);
+
+ return 0;
+
+err_rif_alloc:
+ mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, false);
+err_rif_fdb_op:
+ mlxsw_sp_rif_bridge_op(mlxsw_sp, vr->id, l3_dev, f->fid, rif, false);
+err_rif_bridge_op:
+ mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, false);
+err_port_flood_set:
+ mlxsw_sp_vr_put(vr);
+ return err;
+}
+
+void mlxsw_sp_rif_bridge_destroy(struct mlxsw_sp *mlxsw_sp,
+ struct mlxsw_sp_rif *r)
+{
+ struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[r->vr_id];
+ struct net_device *l3_dev = r->dev;
+ struct mlxsw_sp_fid *f = r->f;
+ u16 rif = r->rif;
+
+ mlxsw_sp_router_rif_gone_sync(mlxsw_sp, r);
+
+ vr->rif_count--;
+ mlxsw_sp->rifs[rif] = NULL;
+ f->r = NULL;
+
+ kfree(r);
+
+ mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, false);
+
+ mlxsw_sp_rif_bridge_op(mlxsw_sp, vr->id, l3_dev, f->fid, rif, false);
+
+ mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, false);
+
+ mlxsw_sp_vr_put(vr);
+
+ netdev_dbg(l3_dev, "RIF=%d destroyed\n", rif);
+}
+
+static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev,
+ struct net_device *br_dev,
+ unsigned long event)
+{
+ struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
+ struct mlxsw_sp_fid *f;
+
+ /* FID can either be an actual FID if the L3 device is the
+ * VLAN-aware bridge or a VLAN device on top. Otherwise, the
+ * L3 device is a VLAN-unaware bridge and we get a vFID.
+ */
+ f = mlxsw_sp_bridge_fid_get(mlxsw_sp, l3_dev);
+ if (WARN_ON(!f))
+ return -EINVAL;
+
+ switch (event) {
+ case NETDEV_UP:
+ return mlxsw_sp_rif_bridge_create(mlxsw_sp, l3_dev, f);
+ case NETDEV_DOWN:
+ mlxsw_sp_rif_bridge_destroy(mlxsw_sp, f->r);
+ break;
+ }
+
+ return 0;
+}
+
+static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev,
+ unsigned long event)
+{
+ struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
+ struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(vlan_dev);
+ u16 vid = vlan_dev_vlan_id(vlan_dev);
+
+ if (mlxsw_sp_port_dev_check(real_dev))
+ return mlxsw_sp_inetaddr_vport_event(vlan_dev, real_dev, event,
+ vid);
+ else if (netif_is_lag_master(real_dev))
+ return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
+ vid);
+ else if (netif_is_bridge_master(real_dev) &&
+ mlxsw_sp->master_bridge.dev == real_dev)
+ return mlxsw_sp_inetaddr_bridge_event(vlan_dev, real_dev,
+ event);
+
+ return 0;
+}
+
+int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
+ unsigned long event, void *ptr)
+{
+ struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
+ struct net_device *dev = ifa->ifa_dev->dev;
+ struct mlxsw_sp *mlxsw_sp;
+ struct mlxsw_sp_rif *r;
+ int err = 0;
+
+ mlxsw_sp = mlxsw_sp_lower_get(dev);
+ if (!mlxsw_sp)
+ goto out;
+
+ r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
+ if (!mlxsw_sp_rif_should_config(r, ifa->ifa_dev, event))
+ goto out;
+
+ if (mlxsw_sp_port_dev_check(dev))
+ err = mlxsw_sp_inetaddr_port_event(dev, event);
+ else if (netif_is_lag_master(dev))
+ err = mlxsw_sp_inetaddr_lag_event(dev, event);
+ else if (netif_is_bridge_master(dev))
+ err = mlxsw_sp_inetaddr_bridge_event(dev, dev, event);
+ else if (is_vlan_dev(dev))
+ err = mlxsw_sp_inetaddr_vlan_event(dev, event);
+
+out:
+ return notifier_from_errno(err);
+}
+
+static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif,
+ const char *mac, int mtu)
+{
+ char ritr_pl[MLXSW_REG_RITR_LEN];
+ int err;
+
+ mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
+ err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
+ if (err)
+ return err;
+
+ mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
+ mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
+ mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
+ return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
+}
+
+int mlxsw_sp_netdevice_router_port_event(struct net_device *dev)
+{
+ struct mlxsw_sp *mlxsw_sp;
+ struct mlxsw_sp_rif *r;
+ int err;
+
+ mlxsw_sp = mlxsw_sp_lower_get(dev);
+ if (!mlxsw_sp)
+ return 0;
+
+ r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
+ if (!r)
+ return 0;
+
+ err = mlxsw_sp_rif_fdb_op(mlxsw_sp, r->addr, r->f->fid, false);
+ if (err)
+ return err;
+
+ err = mlxsw_sp_rif_edit(mlxsw_sp, r->rif, dev->dev_addr, dev->mtu);
+ if (err)
+ goto err_rif_edit;
+
+ err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, r->f->fid, true);
+ if (err)
+ goto err_rif_fdb_op;
+
+ ether_addr_copy(r->addr, dev->dev_addr);
+ r->mtu = dev->mtu;
+
+ netdev_dbg(dev, "Updated RIF=%d\n", r->rif);
+
+ return 0;
+
+err_rif_fdb_op:
+ mlxsw_sp_rif_edit(mlxsw_sp, r->rif, r->addr, r->mtu);
+err_rif_edit:
+ mlxsw_sp_rif_fdb_op(mlxsw_sp, r->addr, r->f->fid, true);
+ return err;
+}
+
+static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
+{
+ struct mlxsw_sp *mlxsw_sp = container_of(nb, struct mlxsw_sp, fib_nb);
+
+ /* Flush pending FIB notifications and then flush the device's
+ * table before requesting another dump. The FIB notification
+ * block is unregistered, so no need to take RTNL.
+ */
+ mlxsw_core_flush_owq();
+ mlxsw_sp_router_fib_flush(mlxsw_sp);
+}
+
+static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
+{
+ char rgcr_pl[MLXSW_REG_RGCR_LEN];
+ u64 max_rifs;
+ int err;
+
+ if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
+ return -EIO;
+
+ max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
+ mlxsw_sp->rifs = kcalloc(max_rifs, sizeof(struct mlxsw_sp_rif *),
+ GFP_KERNEL);
+ if (!mlxsw_sp->rifs)
+ return -ENOMEM;
+
+ mlxsw_reg_rgcr_pack(rgcr_pl, true);
+ mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
+ err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
+ if (err)
+ goto err_rgcr_fail;
+
+ return 0;
+
+err_rgcr_fail:
+ kfree(mlxsw_sp->rifs);
+ return err;
+}
+
+static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
+{
+ char rgcr_pl[MLXSW_REG_RGCR_LEN];
+ int i;
+
+ mlxsw_reg_rgcr_pack(rgcr_pl, false);
+ mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
+
+ for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
+ WARN_ON_ONCE(mlxsw_sp->rifs[i]);
+
+ kfree(mlxsw_sp->rifs);
}
int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
netdev_err(ks->netdev, "spi_sync() failed\n");
}
-/**
- * ks8851_rx_1msg - select whether to use one or two messages for spi read
- * @ks: The device structure
- *
- * Return whether to generate a single message with a tx and rx buffer
- * supplied to spi_sync(), or alternatively send the tx and rx buffers
- * as separate messages.
- *
- * Depending on the hardware in use, a single message may be more efficient
- * on interrupts or work done by the driver.
- *
- * This currently always returns true until we add some per-device data passed
- * from the platform code to specify which mode is better.
- */
-static inline bool ks8851_rx_1msg(struct ks8851_net *ks)
-{
- return true;
-}
-
/**
* ks8851_rdreg - issue read register command and return the data
* @ks: The device state
txb[0] = cpu_to_le16(op | KS_SPIOP_RD);
- if (ks8851_rx_1msg(ks)) {
- msg = &ks->spi_msg1;
- xfer = &ks->spi_xfer1;
-
- xfer->tx_buf = txb;
- xfer->rx_buf = trx;
- xfer->len = rxl + 2;
- } else {
+ if (ks->spidev->master->flags & SPI_MASTER_HALF_DUPLEX) {
msg = &ks->spi_msg2;
xfer = ks->spi_xfer2;
xfer->tx_buf = NULL;
xfer->rx_buf = trx;
xfer->len = rxl;
+ } else {
+ msg = &ks->spi_msg1;
+ xfer = &ks->spi_xfer1;
+
+ xfer->tx_buf = txb;
+ xfer->rx_buf = trx;
+ xfer->len = rxl + 2;
}
ret = spi_sync(ks->spidev, msg);
if (ret < 0)
netdev_err(ks->netdev, "read: spi_sync() failed\n");
- else if (ks8851_rx_1msg(ks))
- memcpy(rxb, trx + 2, rxl);
- else
+ else if (ks->spidev->master->flags & SPI_MASTER_HALF_DUPLEX)
memcpy(rxb, trx, rxl);
+ else
+ memcpy(rxb, trx + 2, rxl);
}
/**
/* Forward declarations */
struct nfp_cpp;
+struct nfp_eth_table_port;
struct nfp_net;
struct nfp_net_r_vector;
/**
* struct nfp_net - NFP network device structure
- * @pdev: Backpointer to PCI device
+ * @dev: Backpointer to struct device
* @netdev: Backpointer to net_device structure
* @is_vf: Is the driver attached to a VF?
* @bpf_offload_skip_sw: Offloaded BPF program will not be rerun by cls_bpf
* @bpf_offload_xdp: Offloaded BPF program is XDP
+ * @chained_metadata_format: Firemware will use new metadata format
* @ctrl: Local copy of the control register/word.
* @fl_bufsz: Currently configured size of the freelist buffers
* @rx_offset: Offset in the RX buffers where packet data starts
* @fw_ver: Firmware version
* @cap: Capabilities advertised by the Firmware
* @max_mtu: Maximum support MTU advertised by the Firmware
+ * @rss_hfunc: RSS selected hash function
* @rss_cfg: RSS configuration
* @rss_key: RSS secret key
* @rss_itbl: RSS indirection table
* @debugfs_dir: Device directory in debugfs
* @ethtool_dump_flag: Ethtool dump flag
* @port_list: Entry on device port list
+ * @pdev: Backpointer to PCI device
* @cpp: CPP device handle if available
+ * @eth_port: Translated ETH Table port entry
*/
struct nfp_net {
- struct pci_dev *pdev;
+ struct device *dev;
struct net_device *netdev;
unsigned is_vf:1;
unsigned bpf_offload_skip_sw:1;
unsigned bpf_offload_xdp:1;
+ unsigned chained_metadata_format:1;
u32 ctrl;
u32 fl_bufsz;
u32 cap;
u32 max_mtu;
+ u8 rss_hfunc;
u32 rss_cfg;
u8 rss_key[NFP_NET_CFG_RSS_KEY_SZ];
u8 rss_itbl[NFP_NET_CFG_RSS_ITBL_SZ];
struct list_head port_list;
+ struct pci_dev *pdev;
struct nfp_cpp *cpp;
+
+ struct nfp_eth_table_port *eth_port;
};
struct nfp_net_ring_set {
void nfp_net_set_ethtool_ops(struct net_device *netdev);
void nfp_net_info(struct nfp_net *nn);
int nfp_net_reconfig(struct nfp_net *nn, u32 update);
+unsigned int nfp_net_rss_key_sz(struct nfp_net *nn);
void nfp_net_rss_write_itbl(struct nfp_net *nn);
void nfp_net_rss_write_key(struct nfp_net *nn);
void nfp_net_coalesce_write_cfg(struct nfp_net *nn);
* Chris Telfer <chris.telfer@netronome.com>
*/
+#include <linux/bitfield.h>
#include <linux/bpf.h>
#include <linux/bpf_trace.h>
#include <linux/module.h>
#include <net/pkt_cls.h>
#include <net/vxlan.h>
+#include "nfpcore/nfp_nsp_eth.h"
#include "nfp_net_ctrl.h"
#include "nfp_net.h"
nfp_net_dma_map_rx(struct nfp_net *nn, void *frag, unsigned int bufsz,
int direction)
{
- return dma_map_single(&nn->pdev->dev, frag + NFP_NET_RX_BUF_HEADROOM,
+ return dma_map_single(nn->dev, frag + NFP_NET_RX_BUF_HEADROOM,
bufsz - NFP_NET_RX_BUF_NON_DATA, direction);
}
nfp_net_dma_unmap_rx(struct nfp_net *nn, dma_addr_t dma_addr,
unsigned int bufsz, int direction)
{
- dma_unmap_single(&nn->pdev->dev, dma_addr,
+ dma_unmap_single(nn->dev, dma_addr,
bufsz - NFP_NET_RX_BUF_NON_DATA, direction);
}
{
struct nfp_net *nn = netdev_priv(netdev);
const struct skb_frag_struct *frag;
- struct nfp_net_r_vector *r_vec;
struct nfp_net_tx_desc *txd, txdg;
- struct nfp_net_tx_buf *txbuf;
struct nfp_net_tx_ring *tx_ring;
+ struct nfp_net_r_vector *r_vec;
+ struct nfp_net_tx_buf *txbuf;
struct netdev_queue *nd_q;
dma_addr_t dma_addr;
unsigned int fsize;
}
/* Start with the head skbuf */
- dma_addr = dma_map_single(&nn->pdev->dev, skb->data, skb_headlen(skb),
+ dma_addr = dma_map_single(nn->dev, skb->data, skb_headlen(skb),
DMA_TO_DEVICE);
- if (dma_mapping_error(&nn->pdev->dev, dma_addr))
+ if (dma_mapping_error(nn->dev, dma_addr))
goto err_free;
wr_idx = tx_ring->wr_p & (tx_ring->cnt - 1);
frag = &skb_shinfo(skb)->frags[f];
fsize = skb_frag_size(frag);
- dma_addr = skb_frag_dma_map(&nn->pdev->dev, frag, 0,
+ dma_addr = skb_frag_dma_map(nn->dev, frag, 0,
fsize, DMA_TO_DEVICE);
- if (dma_mapping_error(&nn->pdev->dev, dma_addr))
+ if (dma_mapping_error(nn->dev, dma_addr))
goto err_unmap;
wr_idx = (wr_idx + 1) & (tx_ring->cnt - 1);
--f;
while (f >= 0) {
frag = &skb_shinfo(skb)->frags[f];
- dma_unmap_page(&nn->pdev->dev,
- tx_ring->txbufs[wr_idx].dma_addr,
+ dma_unmap_page(nn->dev, tx_ring->txbufs[wr_idx].dma_addr,
skb_frag_size(frag), DMA_TO_DEVICE);
tx_ring->txbufs[wr_idx].skb = NULL;
tx_ring->txbufs[wr_idx].dma_addr = 0;
if (wr_idx < 0)
wr_idx += tx_ring->cnt;
}
- dma_unmap_single(&nn->pdev->dev, tx_ring->txbufs[wr_idx].dma_addr,
+ dma_unmap_single(nn->dev, tx_ring->txbufs[wr_idx].dma_addr,
skb_headlen(skb), DMA_TO_DEVICE);
tx_ring->txbufs[wr_idx].skb = NULL;
tx_ring->txbufs[wr_idx].dma_addr = 0;
if (fidx == -1) {
/* unmap head */
- dma_unmap_single(&nn->pdev->dev,
- tx_ring->txbufs[idx].dma_addr,
+ dma_unmap_single(nn->dev, tx_ring->txbufs[idx].dma_addr,
skb_headlen(skb), DMA_TO_DEVICE);
done_pkts += tx_ring->txbufs[idx].pkt_cnt;
} else {
/* unmap fragment */
frag = &skb_shinfo(skb)->frags[fidx];
- dma_unmap_page(&nn->pdev->dev,
- tx_ring->txbufs[idx].dma_addr,
+ dma_unmap_page(nn->dev, tx_ring->txbufs[idx].dma_addr,
skb_frag_size(frag), DMA_TO_DEVICE);
}
{
struct nfp_net_r_vector *r_vec = tx_ring->r_vec;
const struct skb_frag_struct *frag;
- struct pci_dev *pdev = nn->pdev;
struct netdev_queue *nd_q;
while (tx_ring->rd_p != tx_ring->wr_p) {
if (tx_buf->fidx == -1) {
/* unmap head */
- dma_unmap_single(&pdev->dev, tx_buf->dma_addr,
+ dma_unmap_single(nn->dev, tx_buf->dma_addr,
skb_headlen(skb),
DMA_TO_DEVICE);
} else {
/* unmap fragment */
frag = &skb_shinfo(skb)->frags[tx_buf->fidx];
- dma_unmap_page(&pdev->dev, tx_buf->dma_addr,
+ dma_unmap_page(nn->dev, tx_buf->dma_addr,
skb_frag_size(frag),
DMA_TO_DEVICE);
}
direction = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
*dma_addr = nfp_net_dma_map_rx(nn, frag, fl_bufsz, direction);
- if (dma_mapping_error(&nn->pdev->dev, *dma_addr)) {
+ if (dma_mapping_error(nn->dev, *dma_addr)) {
nfp_net_free_frag(frag, xdp);
nn_warn_ratelimit(nn, "Failed to map DMA RX buffer\n");
return NULL;
}
*dma_addr = nfp_net_dma_map_rx(nn, frag, nn->fl_bufsz, direction);
- if (dma_mapping_error(&nn->pdev->dev, *dma_addr)) {
+ if (dma_mapping_error(nn->dev, *dma_addr)) {
nfp_net_free_frag(frag, nn->xdp_prog);
nn_warn_ratelimit(nn, "Failed to map DMA RX buffer\n");
return NULL;
txbuf->pkt_cnt = 1;
txbuf->real_len = pkt_len;
- dma_sync_single_for_device(&nn->pdev->dev, rxbuf->dma_addr + pkt_off,
+ dma_sync_single_for_device(nn->dev, rxbuf->dma_addr + pkt_off,
pkt_len, DMA_BIDIRECTIONAL);
/* Build TX descriptor */
nn->bpf_offload_xdp)) {
int act;
- dma_sync_single_for_cpu(&nn->pdev->dev,
+ dma_sync_single_for_cpu(nn->dev,
rxbuf->dma_addr + pkt_off,
pkt_len, DMA_BIDIRECTIONAL);
act = nfp_net_run_xdp(xdp_prog, rxbuf->frag + data_off,
skb_reserve(skb, data_off);
skb_put(skb, pkt_len);
- if (nn->fw_ver.major <= 3) {
+ if (!nn->chained_metadata_format) {
nfp_net_set_hash_desc(nn->netdev, skb, rxd);
} else if (meta_len) {
void *end;
nfp_net_xdp_complete(r_vec->xdp_ring);
}
- if (pkts_polled < budget) {
- napi_complete_done(napi, pkts_polled);
- nfp_net_irq_unmask(r_vec->nfp_net, r_vec->irq_entry);
- }
+ if (pkts_polled < budget)
+ if (napi_complete_done(napi, pkts_polled))
+ nfp_net_irq_unmask(r_vec->nfp_net, r_vec->irq_entry);
return pkts_polled;
}
{
struct nfp_net_r_vector *r_vec = tx_ring->r_vec;
struct nfp_net *nn = r_vec->nfp_net;
- struct pci_dev *pdev = nn->pdev;
kfree(tx_ring->txbufs);
if (tx_ring->txds)
- dma_free_coherent(&pdev->dev, tx_ring->size,
+ dma_free_coherent(nn->dev, tx_ring->size,
tx_ring->txds, tx_ring->dma);
tx_ring->cnt = 0;
{
struct nfp_net_r_vector *r_vec = tx_ring->r_vec;
struct nfp_net *nn = r_vec->nfp_net;
- struct pci_dev *pdev = nn->pdev;
int sz;
tx_ring->cnt = cnt;
tx_ring->size = sizeof(*tx_ring->txds) * tx_ring->cnt;
- tx_ring->txds = dma_zalloc_coherent(&pdev->dev, tx_ring->size,
+ tx_ring->txds = dma_zalloc_coherent(nn->dev, tx_ring->size,
&tx_ring->dma, GFP_KERNEL);
if (!tx_ring->txds)
goto err_alloc;
netif_set_xps_queue(nn->netdev, &r_vec->affinity_mask,
tx_ring->idx);
- nn_dbg(nn, "TxQ%02d: QCidx=%02d cnt=%d dma=%#llx host=%p %s\n",
- tx_ring->idx, tx_ring->qcidx,
- tx_ring->cnt, (unsigned long long)tx_ring->dma, tx_ring->txds,
- is_xdp ? "XDP" : "");
-
return 0;
err_alloc:
{
struct nfp_net_r_vector *r_vec = rx_ring->r_vec;
struct nfp_net *nn = r_vec->nfp_net;
- struct pci_dev *pdev = nn->pdev;
kfree(rx_ring->rxbufs);
if (rx_ring->rxds)
- dma_free_coherent(&pdev->dev, rx_ring->size,
+ dma_free_coherent(nn->dev, rx_ring->size,
rx_ring->rxds, rx_ring->dma);
rx_ring->cnt = 0;
{
struct nfp_net_r_vector *r_vec = rx_ring->r_vec;
struct nfp_net *nn = r_vec->nfp_net;
- struct pci_dev *pdev = nn->pdev;
int sz;
rx_ring->cnt = cnt;
rx_ring->bufsz = fl_bufsz;
rx_ring->size = sizeof(*rx_ring->rxds) * rx_ring->cnt;
- rx_ring->rxds = dma_zalloc_coherent(&pdev->dev, rx_ring->size,
+ rx_ring->rxds = dma_zalloc_coherent(nn->dev, rx_ring->size,
&rx_ring->dma, GFP_KERNEL);
if (!rx_ring->rxds)
goto err_alloc;
if (!rx_ring->rxbufs)
goto err_alloc;
- nn_dbg(nn, "RxQ%02d: FlQCidx=%02d RxQCidx=%02d cnt=%d dma=%#llx host=%p\n",
- rx_ring->idx, rx_ring->fl_qcidx, rx_ring->rx_qcidx,
- rx_ring->cnt, (unsigned long long)rx_ring->dma, rx_ring->rxds);
-
return 0;
err_alloc:
{
int i;
- for (i = 0; i < NFP_NET_CFG_RSS_KEY_SZ; i += 4)
+ for (i = 0; i < nfp_net_rss_key_sz(nn); i += 4)
nn_writel(nn, NFP_NET_CFG_RSS_KEY + i,
get_unaligned_le32(nn->rss_key + i));
}
return features;
}
+static int
+nfp_net_get_phys_port_name(struct net_device *netdev, char *name, size_t len)
+{
+ struct nfp_net *nn = netdev_priv(netdev);
+ int err;
+
+ if (!nn->eth_port)
+ return -EOPNOTSUPP;
+
+ if (!nn->eth_port->is_split)
+ err = snprintf(name, len, "p%d", nn->eth_port->label_port);
+ else
+ err = snprintf(name, len, "p%ds%d", nn->eth_port->label_port,
+ nn->eth_port->label_subport);
+ if (err >= len)
+ return -EINVAL;
+
+ return 0;
+}
+
/**
* nfp_net_set_vxlan_port() - set vxlan port in SW and reconfigure HW
* @nn: NFP Net device to reconfigure
.ndo_set_mac_address = eth_mac_addr,
.ndo_set_features = nfp_net_set_features,
.ndo_features_check = nfp_net_features_check,
+ .ndo_get_phys_port_name = nfp_net_get_phys_port_name,
.ndo_udp_tunnel_add = nfp_net_add_vxlan_port,
.ndo_udp_tunnel_del = nfp_net_del_vxlan_port,
.ndo_xdp = nfp_net_xdp,
nn = netdev_priv(netdev);
nn->netdev = netdev;
+ nn->dev = &pdev->dev;
nn->pdev = pdev;
nn->max_tx_rings = max_tx_rings;
free_netdev(nn->netdev);
}
+/**
+ * nfp_net_rss_key_sz() - Get current size of the RSS key
+ * @nn: NFP Net device instance
+ *
+ * Return: size of the RSS key for currently selected hash function.
+ */
+unsigned int nfp_net_rss_key_sz(struct nfp_net *nn)
+{
+ switch (nn->rss_hfunc) {
+ case ETH_RSS_HASH_TOP:
+ return NFP_NET_CFG_RSS_KEY_SZ;
+ case ETH_RSS_HASH_XOR:
+ return 0;
+ case ETH_RSS_HASH_CRC32:
+ return 4;
+ }
+
+ nn_warn(nn, "Unknown hash function: %u\n", nn->rss_hfunc);
+ return 0;
+}
+
/**
* nfp_net_rss_init() - Set the initial RSS parameters
* @nn: NFP Net device to reconfigure
*/
static void nfp_net_rss_init(struct nfp_net *nn)
{
- netdev_rss_key_fill(nn->rss_key, NFP_NET_CFG_RSS_KEY_SZ);
+ unsigned long func_bit, rss_cap_hfunc;
+ u32 reg;
+
+ /* Read the RSS function capability and select first supported func */
+ reg = nn_readl(nn, NFP_NET_CFG_RSS_CAP);
+ rss_cap_hfunc = FIELD_GET(NFP_NET_CFG_RSS_CAP_HFUNC, reg);
+ if (!rss_cap_hfunc)
+ rss_cap_hfunc = FIELD_GET(NFP_NET_CFG_RSS_CAP_HFUNC,
+ NFP_NET_CFG_RSS_TOEPLITZ);
+
+ func_bit = find_first_bit(&rss_cap_hfunc, NFP_NET_CFG_RSS_HFUNCS);
+ if (func_bit == NFP_NET_CFG_RSS_HFUNCS) {
+ dev_warn(nn->dev,
+ "Bad RSS config, defaulting to Toeplitz hash\n");
+ func_bit = ETH_RSS_HASH_TOP_BIT;
+ }
+ nn->rss_hfunc = 1 << func_bit;
+
+ netdev_rss_key_fill(nn->rss_key, nfp_net_rss_key_sz(nn));
nfp_net_rss_init_itbl(nn);
/* Enable IPv4/IPv6 TCP by default */
nn->rss_cfg = NFP_NET_CFG_RSS_IPV4_TCP |
NFP_NET_CFG_RSS_IPV6_TCP |
- NFP_NET_CFG_RSS_TOEPLITZ |
+ FIELD_PREP(NFP_NET_CFG_RSS_HFUNC, nn->rss_hfunc) |
NFP_NET_CFG_RSS_MASK;
}
struct nfp_net *nn = netdev_priv(netdev);
int err;
+ nn->chained_metadata_format = nn->fw_ver.major > 3;
+
/* Get some of the read-only fields from the BAR */
nn->cap = nn_readl(nn, NFP_NET_CFG_CAP);
nn->max_mtu = nn_readl(nn, NFP_NET_CFG_MAX_MTU);
/*
- * Copyright (C) 2015 Netronome Systems, Inc.
+ * Copyright (C) 2015-2017 Netronome Systems, Inc.
*
* This software is dual licensed under the GNU General License Version 2,
* June 1991 as shown in the file COPYING in the top-level directory of this
#define NFP_NET_CFG_RX_OFFSET 0x0050
#define NFP_NET_CFG_RX_OFFSET_DYNAMIC 0 /* Prepend mode */
+/**
+ * RSS capabilities
+ * @NFP_NET_CFG_RSS_CAP_HFUNC: supported hash functions (same bits as
+ * @NFP_NET_CFG_RSS_HFUNC)
+ */
+#define NFP_NET_CFG_RSS_CAP 0x0054
+#define NFP_NET_CFG_RSS_CAP_HFUNC 0xff000000
+
/**
* VXLAN/UDP encap configuration
* @NFP_NET_CFG_VXLAN_PORT: Base address of table of tunnels' UDP dst ports
#define NFP_NET_CFG_RSS_IPV4_UDP (1 << 11) /* RSS for IPv4/UDP */
#define NFP_NET_CFG_RSS_IPV6_TCP (1 << 12) /* RSS for IPv6/TCP */
#define NFP_NET_CFG_RSS_IPV6_UDP (1 << 13) /* RSS for IPv6/UDP */
+#define NFP_NET_CFG_RSS_HFUNC 0xff000000
#define NFP_NET_CFG_RSS_TOEPLITZ (1 << 24) /* Use Toeplitz hash */
+#define NFP_NET_CFG_RSS_XOR (1 << 25) /* Use XOR as hash */
+#define NFP_NET_CFG_RSS_CRC32 (1 << 26) /* Use CRC32 as hash */
+#define NFP_NET_CFG_RSS_HFUNCS 3
#define NFP_NET_CFG_RSS_KEY (NFP_NET_CFG_RSS_BASE + 0x4)
#define NFP_NET_CFG_RSS_KEY_SZ 0x28
#define NFP_NET_CFG_RSS_ITBL (NFP_NET_CFG_RSS_BASE + 0x4 + \
rx_rd_p = nfp_qcp_rd_ptr_read(rx_ring->qcp_rx);
rx_wr_p = nfp_qcp_wr_ptr_read(rx_ring->qcp_rx);
- seq_printf(file, "RX[%02d]: H_RD=%d H_WR=%d FL_RD=%d FL_WR=%d RX_RD=%d RX_WR=%d\n",
- rx_ring->idx, rx_ring->rd_p, rx_ring->wr_p,
+ seq_printf(file, "RX[%02d,%02d,%02d]: cnt=%d dma=%pad host=%p H_RD=%d H_WR=%d FL_RD=%d FL_WR=%d RX_RD=%d RX_WR=%d\n",
+ rx_ring->idx, rx_ring->fl_qcidx, rx_ring->rx_qcidx,
+ rx_ring->cnt, &rx_ring->dma, rx_ring->rxds,
+ rx_ring->rd_p, rx_ring->wr_p,
fl_rd_p, fl_wr_p, rx_rd_p, rx_wr_p);
for (i = 0; i < rxd_cnt; i++) {
d_rd_p = nfp_qcp_rd_ptr_read(tx_ring->qcp_q);
d_wr_p = nfp_qcp_wr_ptr_read(tx_ring->qcp_q);
- seq_printf(file, "TX[%02d]: H_RD=%d H_WR=%d D_RD=%d D_WR=%d\n",
- tx_ring->idx, tx_ring->rd_p, tx_ring->wr_p, d_rd_p, d_wr_p);
+ seq_printf(file, "TX[%02d,%02d%s]: cnt=%d dma=%pad host=%p H_RD=%d H_WR=%d D_RD=%d D_WR=%d\n",
+ tx_ring->idx, tx_ring->qcidx,
+ tx_ring == r_vec->tx_ring ? "" : "xdp",
+ tx_ring->cnt, &tx_ring->dma, tx_ring->txds,
+ tx_ring->rd_p, tx_ring->wr_p, d_rd_p, d_wr_p);
for (i = 0; i < txd_cnt; i++) {
txd = &tx_ring->txds[i];
* Brad Petrus <brad.petrus@netronome.com>
*/
+#include <linux/bitfield.h>
#include <linux/kernel.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
return -EINVAL;
}
- new_rss_cfg |= NFP_NET_CFG_RSS_TOEPLITZ;
+ new_rss_cfg |= FIELD_PREP(NFP_NET_CFG_RSS_HFUNC, nn->rss_hfunc);
new_rss_cfg |= NFP_NET_CFG_RSS_MASK;
if (new_rss_cfg == nn->rss_cfg)
static u32 nfp_net_get_rxfh_key_size(struct net_device *netdev)
{
- return NFP_NET_CFG_RSS_KEY_SZ;
+ struct nfp_net *nn = netdev_priv(netdev);
+
+ if (!(nn->cap & NFP_NET_CFG_CTRL_RSS))
+ return -EOPNOTSUPP;
+
+ return nfp_net_rss_key_sz(nn);
}
static int nfp_net_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key,
for (i = 0; i < ARRAY_SIZE(nn->rss_itbl); i++)
indir[i] = nn->rss_itbl[i];
if (key)
- memcpy(key, nn->rss_key, NFP_NET_CFG_RSS_KEY_SZ);
- if (hfunc)
- *hfunc = ETH_RSS_HASH_TOP;
+ memcpy(key, nn->rss_key, nfp_net_rss_key_sz(nn));
+ if (hfunc) {
+ *hfunc = nn->rss_hfunc;
+ if (*hfunc >= 1 << ETH_RSS_HASH_FUNCS_COUNT)
+ *hfunc = ETH_RSS_HASH_UNKNOWN;
+ }
return 0;
}
int i;
if (!(nn->cap & NFP_NET_CFG_CTRL_RSS) ||
- !(hfunc == ETH_RSS_HASH_NO_CHANGE || hfunc == ETH_RSS_HASH_TOP))
+ !(hfunc == ETH_RSS_HASH_NO_CHANGE || hfunc == nn->rss_hfunc))
return -EOPNOTSUPP;
if (!key && !indir)
return 0;
if (key) {
- memcpy(nn->rss_key, key, NFP_NET_CFG_RSS_KEY_SZ);
+ memcpy(nn->rss_key, key, nfp_net_rss_key_sz(nn));
nfp_net_rss_write_key(nn);
}
if (indir) {
mac_str = nfp_hwinfo_lookup(cpp, name);
if (!mac_str) {
- dev_warn(&nn->pdev->dev,
- "Can't lookup MAC address. Generate\n");
+ dev_warn(nn->dev, "Can't lookup MAC address. Generate\n");
eth_hw_addr_random(nn->netdev);
return;
}
if (sscanf(mac_str, "%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx",
&mac_addr[0], &mac_addr[1], &mac_addr[2],
&mac_addr[3], &mac_addr[4], &mac_addr[5]) != 6) {
- dev_warn(&nn->pdev->dev,
+ dev_warn(nn->dev,
"Can't parse MAC address (%s). Generate.\n", mac_str);
eth_hw_addr_random(nn->netdev);
return;
if (pf->eth_tbl->ports[i].eth_index == id) {
const u8 *mac_addr = pf->eth_tbl->ports[i].mac_addr;
+ nn->eth_port = &pf->eth_tbl->ports[i];
+
ether_addr_copy(nn->netdev->dev_addr, mac_addr);
ether_addr_copy(nn->netdev->perm_addr, mac_addr);
return;
start_off = nn_readw(nn, NFP_NET_CFG_BPF_START);
done_off = nn_readw(nn, NFP_NET_CFG_BPF_DONE);
- *code = dma_zalloc_coherent(&nn->pdev->dev, code_sz, dma_addr,
- GFP_KERNEL);
+ *code = dma_zalloc_coherent(nn->dev, code_sz, dma_addr, GFP_KERNEL);
if (!*code)
return -ENOMEM;
return 0;
out:
- dma_free_coherent(&nn->pdev->dev, code_sz, *code, *dma_addr);
+ dma_free_coherent(nn->dev, code_sz, *code, *dma_addr);
return ret;
}
if (err)
nn_err(nn, "FW command error while enabling BPF: %d\n", err);
- dma_free_coherent(&nn->pdev->dev, code_sz, code, dma_addr);
+ dma_free_coherent(nn->dev, code_sz, code, dma_addr);
nfp_net_bpf_stats_reset(nn);
mod_timer(&nn->rx_filter_stats_timer, jiffies + NFP_NET_STAT_POLL_IVL);
nfp_eth_copy_mac_reverse(dst->mac_addr, src->mac_addr);
- snprintf(dst->label, sizeof(dst->label) - 1, "%llu.%llu",
- FIELD_GET(NSP_ETH_PORT_PHYLABEL, port),
- FIELD_GET(NSP_ETH_PORT_LABEL, port));
+ dst->label_port = FIELD_GET(NSP_ETH_PORT_PHYLABEL, port);
+ dst->label_subport = FIELD_GET(NSP_ETH_PORT_LABEL, port);
+}
+
+static void
+nfp_eth_mark_split_ports(struct nfp_cpp *cpp, struct nfp_eth_table *table)
+{
+ unsigned int i, j;
+
+ for (i = 0; i < table->count; i++)
+ for (j = 0; j < table->count; j++) {
+ if (i == j)
+ continue;
+ if (table->ports[i].label_port !=
+ table->ports[j].label_port)
+ continue;
+ if (table->ports[i].label_subport ==
+ table->ports[j].label_subport)
+ nfp_warn(cpp,
+ "Port %d subport %d is a duplicate\n",
+ table->ports[i].label_port,
+ table->ports[i].label_subport);
+
+ table->ports[i].is_split = true;
+ break;
+ }
}
/**
{
struct eth_table_entry *entries;
struct nfp_eth_table *table;
- unsigned int cnt;
- int i, j, ret;
+ int i, j, ret, cnt = 0;
entries = kzalloc(NSP_ETH_TABLE_SIZE, GFP_KERNEL);
if (!entries)
ret = nfp_nsp_read_eth_table(nsp, entries, NSP_ETH_TABLE_SIZE);
if (ret < 0) {
nfp_err(cpp, "reading port table failed %d\n", ret);
- kfree(entries);
- return NULL;
+ goto err;
}
- /* Some versions of flash will give us 0 instead of port count */
- cnt = ret;
- if (!cnt) {
- for (i = 0; i < NSP_ETH_MAX_COUNT; i++)
- if (entries[i].port & NSP_ETH_PORT_LANES_MASK)
- cnt++;
+ for (i = 0; i < NSP_ETH_MAX_COUNT; i++)
+ if (entries[i].port & NSP_ETH_PORT_LANES_MASK)
+ cnt++;
+
+ /* Some versions of flash will give us 0 instead of port count.
+ * For those that give a port count, verify it against the value
+ * calculated above.
+ */
+ if (ret && ret != cnt) {
+ nfp_err(cpp, "table entry count reported (%d) does not match entries present (%d)\n",
+ ret, cnt);
+ goto err;
}
table = kzalloc(sizeof(*table) +
sizeof(struct nfp_eth_table_port) * cnt, GFP_KERNEL);
- if (!table) {
- kfree(entries);
- return NULL;
- }
+ if (!table)
+ goto err;
table->count = cnt;
for (i = 0, j = 0; i < NSP_ETH_MAX_COUNT; i++)
nfp_eth_port_translate(&entries[i], i,
&table->ports[j++]);
+ nfp_eth_mark_split_ports(cpp, table);
+
kfree(entries);
return table;
+
+err:
+ kfree(entries);
+ return NULL;
}
/**
* @lanes: number of channels
* @speed: interface speed (in Mbps)
* @mac_addr: interface MAC address
- * @label: interface id string
+ * @label_port: port id
+ * @label_subport: id of interface within port (for split ports)
* @enabled: is enabled?
* @tx_enabled: is TX enabled?
* @rx_enabled: is RX enabled?
+ *
+ * @is_split: is interface part of a split port
*/
struct nfp_eth_table {
unsigned int count;
unsigned int speed;
u8 mac_addr[ETH_ALEN];
- char label[8];
+
+ u8 label_port;
+ u8 label_subport;
bool enabled;
bool tx_enabled;
bool rx_enabled;
+
+ /* Computed fields */
+ bool is_split;
} ports[0];
};
+struct nfp_cpp;
+struct nfp_nsp;
+
struct nfp_eth_table *nfp_eth_read_ports(struct nfp_cpp *cpp);
struct nfp_eth_table *
__nfp_eth_read_ports(struct nfp_cpp *cpp, struct nfp_nsp *nsp);
if (!np)
return NULL;
- axi = kzalloc(sizeof(*axi), GFP_KERNEL);
+ axi = devm_kzalloc(&pdev->dev, sizeof(*axi), GFP_KERNEL);
if (!axi) {
of_node_put(np);
return ERR_PTR(-ENOMEM);
--- /dev/null
+#
+# Synopsys network device configuration
+#
+
+config NET_VENDOR_SYNOPSYS
+ bool "Synopsys devices"
+ default y
+ ---help---
+ If you have a network (Ethernet) device belonging to this class, say Y.
+
+ Note that the answer to this question doesn't directly affect the
+ kernel: saying N will just cause the configurator to skip all
+ the questions about Synopsys devices. If you say Y, you will be asked
+ for your specific device in the following questions.
+
+if NET_VENDOR_SYNOPSYS
+
+config DWC_XLGMAC
+ tristate "Synopsys DWC Enterprise Ethernet (XLGMAC) driver support"
+ depends on HAS_IOMEM && HAS_DMA
+ select BITREVERSE
+ select CRC32
+ ---help---
+ This driver supports the Synopsys DesignWare Cores Enterprise
+ Ethernet (dwc-xlgmac).
+
+if DWC_XLGMAC
+
+config DWC_XLGMAC_PCI
+ tristate "XLGMAC PCI bus support"
+ depends on DWC_XLGMAC && PCI
+ ---help---
+ This selects the pci bus support for the dwc-xlgmac driver.
+ This driver was tested on Synopsys XLGMAC IP Prototyping Kit.
+
+ If you have a controller with this interface, say Y or M here.
+ If unsure, say N.
+
+endif # DWC_XLGMAC
+
+endif # NET_VENDOR_SYNOPSYS
--- /dev/null
+#
+# Makefile for the Synopsys network device drivers.
+#
+
+obj-$(CONFIG_DWC_XLGMAC) += dwc-xlgmac.o
+dwc-xlgmac-objs := dwc-xlgmac-net.o dwc-xlgmac-desc.o \
+ dwc-xlgmac-hw.o dwc-xlgmac-common.o
+
+dwc-xlgmac-$(CONFIG_DWC_XLGMAC_PCI) += dwc-xlgmac-pci.o
--- /dev/null
+/* Synopsys DesignWare Core Enterprise Ethernet (XLGMAC) Driver
+ *
+ * Copyright (c) 2017 Synopsys, Inc. (www.synopsys.com)
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This Synopsys DWC XLGMAC software driver and associated documentation
+ * (hereinafter the "Software") is an unsupported proprietary work of
+ * Synopsys, Inc. unless otherwise expressly agreed to in writing between
+ * Synopsys and you. The Software IS NOT an item of Licensed Software or a
+ * Licensed Product under any End User Software License Agreement or
+ * Agreement for Licensed Products with Synopsys or any supplement thereto.
+ * Synopsys is a registered trademark of Synopsys, Inc. Other names included
+ * in the SOFTWARE may be the trademarks of their respective owners.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+
+#include "dwc-xlgmac.h"
+#include "dwc-xlgmac-reg.h"
+
+static int debug = -1;
+module_param(debug, int, 0644);
+MODULE_PARM_DESC(debug, "DWC ethernet debug level (0=none,...,16=all)");
+static const u32 default_msg_level = (NETIF_MSG_LINK | NETIF_MSG_IFDOWN |
+ NETIF_MSG_IFUP);
+
+static unsigned char dev_addr[6] = {0, 0x55, 0x7b, 0xb5, 0x7d, 0xf7};
+
+static void xlgmac_read_mac_addr(struct xlgmac_pdata *pdata)
+{
+ struct net_device *netdev = pdata->netdev;
+
+ /* Currently it uses a static mac address for test */
+ memcpy(pdata->mac_addr, dev_addr, netdev->addr_len);
+}
+
+static void xlgmac_default_config(struct xlgmac_pdata *pdata)
+{
+ pdata->tx_osp_mode = DMA_OSP_ENABLE;
+ pdata->tx_sf_mode = MTL_TSF_ENABLE;
+ pdata->rx_sf_mode = MTL_RSF_DISABLE;
+ pdata->pblx8 = DMA_PBL_X8_ENABLE;
+ pdata->tx_pbl = DMA_PBL_32;
+ pdata->rx_pbl = DMA_PBL_32;
+ pdata->tx_threshold = MTL_TX_THRESHOLD_128;
+ pdata->rx_threshold = MTL_RX_THRESHOLD_128;
+ pdata->tx_pause = 1;
+ pdata->rx_pause = 1;
+ pdata->phy_speed = SPEED_25000;
+ pdata->sysclk_rate = XLGMAC_SYSCLOCK;
+
+ strlcpy(pdata->drv_name, XLGMAC_DRV_NAME, sizeof(pdata->drv_name));
+ strlcpy(pdata->drv_ver, XLGMAC_DRV_VERSION, sizeof(pdata->drv_ver));
+}
+
+static void xlgmac_init_all_ops(struct xlgmac_pdata *pdata)
+{
+ xlgmac_init_desc_ops(&pdata->desc_ops);
+ xlgmac_init_hw_ops(&pdata->hw_ops);
+}
+
+static int xlgmac_init(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;
+ struct net_device *netdev = pdata->netdev;
+ unsigned int i;
+ int ret;
+
+ /* Set default configuration data */
+ xlgmac_default_config(pdata);
+
+ /* Set irq, base_addr, MAC address, */
+ netdev->irq = pdata->dev_irq;
+ netdev->base_addr = (unsigned long)pdata->mac_regs;
+ xlgmac_read_mac_addr(pdata);
+ memcpy(netdev->dev_addr, pdata->mac_addr, netdev->addr_len);
+
+ /* Set all the function pointers */
+ xlgmac_init_all_ops(pdata);
+
+ /* Issue software reset to device */
+ hw_ops->exit(pdata);
+
+ /* Populate the hardware features */
+ xlgmac_get_all_hw_features(pdata);
+ xlgmac_print_all_hw_features(pdata);
+
+ /* TODO: Set the PHY mode to XLGMII */
+
+ /* Set the DMA mask */
+ ret = dma_set_mask_and_coherent(pdata->dev,
+ DMA_BIT_MASK(pdata->hw_feat.dma_width));
+ if (ret) {
+ dev_err(pdata->dev, "dma_set_mask_and_coherent failed\n");
+ return ret;
+ }
+
+ /* Channel and ring params initializtion
+ * pdata->channel_count;
+ * pdata->tx_ring_count;
+ * pdata->rx_ring_count;
+ * pdata->tx_desc_count;
+ * pdata->rx_desc_count;
+ */
+ BUILD_BUG_ON_NOT_POWER_OF_2(XLGMAC_TX_DESC_CNT);
+ pdata->tx_desc_count = XLGMAC_TX_DESC_CNT;
+ if (pdata->tx_desc_count & (pdata->tx_desc_count - 1)) {
+ dev_err(pdata->dev, "tx descriptor count (%d) is not valid\n",
+ pdata->tx_desc_count);
+ ret = -EINVAL;
+ return ret;
+ }
+ BUILD_BUG_ON_NOT_POWER_OF_2(XLGMAC_RX_DESC_CNT);
+ pdata->rx_desc_count = XLGMAC_RX_DESC_CNT;
+ if (pdata->rx_desc_count & (pdata->rx_desc_count - 1)) {
+ dev_err(pdata->dev, "rx descriptor count (%d) is not valid\n",
+ pdata->rx_desc_count);
+ ret = -EINVAL;
+ return ret;
+ }
+
+ pdata->tx_ring_count = min_t(unsigned int, num_online_cpus(),
+ pdata->hw_feat.tx_ch_cnt);
+ pdata->tx_ring_count = min_t(unsigned int, pdata->tx_ring_count,
+ pdata->hw_feat.tx_q_cnt);
+ pdata->tx_q_count = pdata->tx_ring_count;
+ ret = netif_set_real_num_tx_queues(netdev, pdata->tx_q_count);
+ if (ret) {
+ dev_err(pdata->dev, "error setting real tx queue count\n");
+ return ret;
+ }
+
+ pdata->rx_ring_count = min_t(unsigned int,
+ netif_get_num_default_rss_queues(),
+ pdata->hw_feat.rx_ch_cnt);
+ pdata->rx_ring_count = min_t(unsigned int, pdata->rx_ring_count,
+ pdata->hw_feat.rx_q_cnt);
+ pdata->rx_q_count = pdata->rx_ring_count;
+ ret = netif_set_real_num_rx_queues(netdev, pdata->rx_q_count);
+ if (ret) {
+ dev_err(pdata->dev, "error setting real rx queue count\n");
+ return ret;
+ }
+
+ pdata->channel_count =
+ max_t(unsigned int, pdata->tx_ring_count, pdata->rx_ring_count);
+
+ /* Initialize RSS hash key and lookup table */
+ netdev_rss_key_fill(pdata->rss_key, sizeof(pdata->rss_key));
+
+ for (i = 0; i < XLGMAC_RSS_MAX_TABLE_SIZE; i++)
+ pdata->rss_table[i] = XLGMAC_SET_REG_BITS(
+ pdata->rss_table[i],
+ MAC_RSSDR_DMCH_POS,
+ MAC_RSSDR_DMCH_LEN,
+ i % pdata->rx_ring_count);
+
+ pdata->rss_options = XLGMAC_SET_REG_BITS(
+ pdata->rss_options,
+ MAC_RSSCR_IP2TE_POS,
+ MAC_RSSCR_IP2TE_LEN, 1);
+ pdata->rss_options = XLGMAC_SET_REG_BITS(
+ pdata->rss_options,
+ MAC_RSSCR_TCP4TE_POS,
+ MAC_RSSCR_TCP4TE_LEN, 1);
+ pdata->rss_options = XLGMAC_SET_REG_BITS(
+ pdata->rss_options,
+ MAC_RSSCR_UDP4TE_POS,
+ MAC_RSSCR_UDP4TE_LEN, 1);
+
+ /* Set device operations */
+ netdev->netdev_ops = xlgmac_get_netdev_ops();
+
+ /* Set device features */
+ if (pdata->hw_feat.tso) {
+ netdev->hw_features = NETIF_F_TSO;
+ netdev->hw_features |= NETIF_F_TSO6;
+ netdev->hw_features |= NETIF_F_SG;
+ netdev->hw_features |= NETIF_F_IP_CSUM;
+ netdev->hw_features |= NETIF_F_IPV6_CSUM;
+ } else if (pdata->hw_feat.tx_coe) {
+ netdev->hw_features = NETIF_F_IP_CSUM;
+ netdev->hw_features |= NETIF_F_IPV6_CSUM;
+ }
+
+ if (pdata->hw_feat.rx_coe) {
+ netdev->hw_features |= NETIF_F_RXCSUM;
+ netdev->hw_features |= NETIF_F_GRO;
+ }
+
+ if (pdata->hw_feat.rss)
+ netdev->hw_features |= NETIF_F_RXHASH;
+
+ netdev->vlan_features |= netdev->hw_features;
+
+ netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
+ if (pdata->hw_feat.sa_vlan_ins)
+ netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
+ if (pdata->hw_feat.vlhash)
+ netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
+
+ netdev->features |= netdev->hw_features;
+ pdata->netdev_features = netdev->features;
+
+ netdev->priv_flags |= IFF_UNICAST_FLT;
+
+ /* Use default watchdog timeout */
+ netdev->watchdog_timeo = 0;
+
+ /* Tx coalesce parameters initialization */
+ pdata->tx_usecs = XLGMAC_INIT_DMA_TX_USECS;
+ pdata->tx_frames = XLGMAC_INIT_DMA_TX_FRAMES;
+
+ /* Rx coalesce parameters initialization */
+ pdata->rx_riwt = hw_ops->usec_to_riwt(pdata, XLGMAC_INIT_DMA_RX_USECS);
+ pdata->rx_usecs = XLGMAC_INIT_DMA_RX_USECS;
+ pdata->rx_frames = XLGMAC_INIT_DMA_RX_FRAMES;
+
+ return 0;
+}
+
+int xlgmac_drv_probe(struct device *dev, struct xlgmac_resources *res)
+{
+ struct xlgmac_pdata *pdata;
+ struct net_device *netdev;
+ int ret;
+
+ netdev = alloc_etherdev_mq(sizeof(struct xlgmac_pdata),
+ XLGMAC_MAX_DMA_CHANNELS);
+
+ if (!netdev) {
+ dev_err(dev, "alloc_etherdev failed\n");
+ return -ENOMEM;
+ }
+
+ SET_NETDEV_DEV(netdev, dev);
+ dev_set_drvdata(dev, netdev);
+ pdata = netdev_priv(netdev);
+ pdata->dev = dev;
+ pdata->netdev = netdev;
+
+ pdata->dev_irq = res->irq;
+ pdata->mac_regs = res->addr;
+
+ mutex_init(&pdata->rss_mutex);
+ pdata->msg_enable = netif_msg_init(debug, default_msg_level);
+
+ ret = xlgmac_init(pdata);
+ if (ret) {
+ dev_err(dev, "xlgmac init failed\n");
+ goto err_free_netdev;
+ }
+
+ ret = register_netdev(netdev);
+ if (ret) {
+ dev_err(dev, "net device registration failed\n");
+ goto err_free_netdev;
+ }
+
+ return 0;
+
+err_free_netdev:
+ free_netdev(netdev);
+
+ return ret;
+}
+
+int xlgmac_drv_remove(struct device *dev)
+{
+ struct net_device *netdev = dev_get_drvdata(dev);
+
+ unregister_netdev(netdev);
+ free_netdev(netdev);
+
+ return 0;
+}
+
+void xlgmac_dump_tx_desc(struct xlgmac_pdata *pdata,
+ struct xlgmac_ring *ring,
+ unsigned int idx,
+ unsigned int count,
+ unsigned int flag)
+{
+ struct xlgmac_desc_data *desc_data;
+ struct xlgmac_dma_desc *dma_desc;
+
+ while (count--) {
+ desc_data = XLGMAC_GET_DESC_DATA(ring, idx);
+ dma_desc = desc_data->dma_desc;
+
+ netdev_dbg(pdata->netdev, "TX: dma_desc=%p, dma_desc_addr=%pad\n",
+ desc_data->dma_desc, &desc_data->dma_desc_addr);
+ netdev_dbg(pdata->netdev,
+ "TX_NORMAL_DESC[%d %s] = %08x:%08x:%08x:%08x\n", idx,
+ (flag == 1) ? "QUEUED FOR TX" : "TX BY DEVICE",
+ le32_to_cpu(dma_desc->desc0),
+ le32_to_cpu(dma_desc->desc1),
+ le32_to_cpu(dma_desc->desc2),
+ le32_to_cpu(dma_desc->desc3));
+
+ idx++;
+ }
+}
+
+void xlgmac_dump_rx_desc(struct xlgmac_pdata *pdata,
+ struct xlgmac_ring *ring,
+ unsigned int idx)
+{
+ struct xlgmac_desc_data *desc_data;
+ struct xlgmac_dma_desc *dma_desc;
+
+ desc_data = XLGMAC_GET_DESC_DATA(ring, idx);
+ dma_desc = desc_data->dma_desc;
+
+ netdev_dbg(pdata->netdev, "RX: dma_desc=%p, dma_desc_addr=%pad\n",
+ desc_data->dma_desc, &desc_data->dma_desc_addr);
+ netdev_dbg(pdata->netdev,
+ "RX_NORMAL_DESC[%d RX BY DEVICE] = %08x:%08x:%08x:%08x\n",
+ idx,
+ le32_to_cpu(dma_desc->desc0),
+ le32_to_cpu(dma_desc->desc1),
+ le32_to_cpu(dma_desc->desc2),
+ le32_to_cpu(dma_desc->desc3));
+}
+
+void xlgmac_print_pkt(struct net_device *netdev,
+ struct sk_buff *skb, bool tx_rx)
+{
+ struct ethhdr *eth = (struct ethhdr *)skb->data;
+ unsigned char *buf = skb->data;
+ unsigned char buffer[128];
+ unsigned int i, j;
+
+ netdev_dbg(netdev, "\n************** SKB dump ****************\n");
+
+ netdev_dbg(netdev, "%s packet of %d bytes\n",
+ (tx_rx ? "TX" : "RX"), skb->len);
+
+ netdev_dbg(netdev, "Dst MAC addr: %pM\n", eth->h_dest);
+ netdev_dbg(netdev, "Src MAC addr: %pM\n", eth->h_source);
+ netdev_dbg(netdev, "Protocol: %#06hx\n", ntohs(eth->h_proto));
+
+ for (i = 0, j = 0; i < skb->len;) {
+ j += snprintf(buffer + j, sizeof(buffer) - j, "%02hhx",
+ buf[i++]);
+
+ if ((i % 32) == 0) {
+ netdev_dbg(netdev, " %#06x: %s\n", i - 32, buffer);
+ j = 0;
+ } else if ((i % 16) == 0) {
+ buffer[j++] = ' ';
+ buffer[j++] = ' ';
+ } else if ((i % 4) == 0) {
+ buffer[j++] = ' ';
+ }
+ }
+ if (i % 32)
+ netdev_dbg(netdev, " %#06x: %s\n", i - (i % 32), buffer);
+
+ netdev_dbg(netdev, "\n************** SKB dump ****************\n");
+}
+
+void xlgmac_get_all_hw_features(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_hw_features *hw_feat = &pdata->hw_feat;
+ unsigned int mac_hfr0, mac_hfr1, mac_hfr2;
+
+ mac_hfr0 = readl(pdata->mac_regs + MAC_HWF0R);
+ mac_hfr1 = readl(pdata->mac_regs + MAC_HWF1R);
+ mac_hfr2 = readl(pdata->mac_regs + MAC_HWF2R);
+
+ memset(hw_feat, 0, sizeof(*hw_feat));
+
+ hw_feat->version = readl(pdata->mac_regs + MAC_VR);
+
+ /* Hardware feature register 0 */
+ hw_feat->phyifsel = XLGMAC_GET_REG_BITS(mac_hfr0,
+ MAC_HWF0R_PHYIFSEL_POS,
+ MAC_HWF0R_PHYIFSEL_LEN);
+ hw_feat->vlhash = XLGMAC_GET_REG_BITS(mac_hfr0,
+ MAC_HWF0R_VLHASH_POS,
+ MAC_HWF0R_VLHASH_LEN);
+ hw_feat->sma = XLGMAC_GET_REG_BITS(mac_hfr0,
+ MAC_HWF0R_SMASEL_POS,
+ MAC_HWF0R_SMASEL_LEN);
+ hw_feat->rwk = XLGMAC_GET_REG_BITS(mac_hfr0,
+ MAC_HWF0R_RWKSEL_POS,
+ MAC_HWF0R_RWKSEL_LEN);
+ hw_feat->mgk = XLGMAC_GET_REG_BITS(mac_hfr0,
+ MAC_HWF0R_MGKSEL_POS,
+ MAC_HWF0R_MGKSEL_LEN);
+ hw_feat->mmc = XLGMAC_GET_REG_BITS(mac_hfr0,
+ MAC_HWF0R_MMCSEL_POS,
+ MAC_HWF0R_MMCSEL_LEN);
+ hw_feat->aoe = XLGMAC_GET_REG_BITS(mac_hfr0,
+ MAC_HWF0R_ARPOFFSEL_POS,
+ MAC_HWF0R_ARPOFFSEL_LEN);
+ hw_feat->ts = XLGMAC_GET_REG_BITS(mac_hfr0,
+ MAC_HWF0R_TSSEL_POS,
+ MAC_HWF0R_TSSEL_LEN);
+ hw_feat->eee = XLGMAC_GET_REG_BITS(mac_hfr0,
+ MAC_HWF0R_EEESEL_POS,
+ MAC_HWF0R_EEESEL_LEN);
+ hw_feat->tx_coe = XLGMAC_GET_REG_BITS(mac_hfr0,
+ MAC_HWF0R_TXCOESEL_POS,
+ MAC_HWF0R_TXCOESEL_LEN);
+ hw_feat->rx_coe = XLGMAC_GET_REG_BITS(mac_hfr0,
+ MAC_HWF0R_RXCOESEL_POS,
+ MAC_HWF0R_RXCOESEL_LEN);
+ hw_feat->addn_mac = XLGMAC_GET_REG_BITS(mac_hfr0,
+ MAC_HWF0R_ADDMACADRSEL_POS,
+ MAC_HWF0R_ADDMACADRSEL_LEN);
+ hw_feat->ts_src = XLGMAC_GET_REG_BITS(mac_hfr0,
+ MAC_HWF0R_TSSTSSEL_POS,
+ MAC_HWF0R_TSSTSSEL_LEN);
+ hw_feat->sa_vlan_ins = XLGMAC_GET_REG_BITS(mac_hfr0,
+ MAC_HWF0R_SAVLANINS_POS,
+ MAC_HWF0R_SAVLANINS_LEN);
+
+ /* Hardware feature register 1 */
+ hw_feat->rx_fifo_size = XLGMAC_GET_REG_BITS(mac_hfr1,
+ MAC_HWF1R_RXFIFOSIZE_POS,
+ MAC_HWF1R_RXFIFOSIZE_LEN);
+ hw_feat->tx_fifo_size = XLGMAC_GET_REG_BITS(mac_hfr1,
+ MAC_HWF1R_TXFIFOSIZE_POS,
+ MAC_HWF1R_TXFIFOSIZE_LEN);
+ hw_feat->adv_ts_hi = XLGMAC_GET_REG_BITS(mac_hfr1,
+ MAC_HWF1R_ADVTHWORD_POS,
+ MAC_HWF1R_ADVTHWORD_LEN);
+ hw_feat->dma_width = XLGMAC_GET_REG_BITS(mac_hfr1,
+ MAC_HWF1R_ADDR64_POS,
+ MAC_HWF1R_ADDR64_LEN);
+ hw_feat->dcb = XLGMAC_GET_REG_BITS(mac_hfr1,
+ MAC_HWF1R_DCBEN_POS,
+ MAC_HWF1R_DCBEN_LEN);
+ hw_feat->sph = XLGMAC_GET_REG_BITS(mac_hfr1,
+ MAC_HWF1R_SPHEN_POS,
+ MAC_HWF1R_SPHEN_LEN);
+ hw_feat->tso = XLGMAC_GET_REG_BITS(mac_hfr1,
+ MAC_HWF1R_TSOEN_POS,
+ MAC_HWF1R_TSOEN_LEN);
+ hw_feat->dma_debug = XLGMAC_GET_REG_BITS(mac_hfr1,
+ MAC_HWF1R_DBGMEMA_POS,
+ MAC_HWF1R_DBGMEMA_LEN);
+ hw_feat->rss = XLGMAC_GET_REG_BITS(mac_hfr1,
+ MAC_HWF1R_RSSEN_POS,
+ MAC_HWF1R_RSSEN_LEN);
+ hw_feat->tc_cnt = XLGMAC_GET_REG_BITS(mac_hfr1,
+ MAC_HWF1R_NUMTC_POS,
+ MAC_HWF1R_NUMTC_LEN);
+ hw_feat->hash_table_size = XLGMAC_GET_REG_BITS(mac_hfr1,
+ MAC_HWF1R_HASHTBLSZ_POS,
+ MAC_HWF1R_HASHTBLSZ_LEN);
+ hw_feat->l3l4_filter_num = XLGMAC_GET_REG_BITS(mac_hfr1,
+ MAC_HWF1R_L3L4FNUM_POS,
+ MAC_HWF1R_L3L4FNUM_LEN);
+
+ /* Hardware feature register 2 */
+ hw_feat->rx_q_cnt = XLGMAC_GET_REG_BITS(mac_hfr2,
+ MAC_HWF2R_RXQCNT_POS,
+ MAC_HWF2R_RXQCNT_LEN);
+ hw_feat->tx_q_cnt = XLGMAC_GET_REG_BITS(mac_hfr2,
+ MAC_HWF2R_TXQCNT_POS,
+ MAC_HWF2R_TXQCNT_LEN);
+ hw_feat->rx_ch_cnt = XLGMAC_GET_REG_BITS(mac_hfr2,
+ MAC_HWF2R_RXCHCNT_POS,
+ MAC_HWF2R_RXCHCNT_LEN);
+ hw_feat->tx_ch_cnt = XLGMAC_GET_REG_BITS(mac_hfr2,
+ MAC_HWF2R_TXCHCNT_POS,
+ MAC_HWF2R_TXCHCNT_LEN);
+ hw_feat->pps_out_num = XLGMAC_GET_REG_BITS(mac_hfr2,
+ MAC_HWF2R_PPSOUTNUM_POS,
+ MAC_HWF2R_PPSOUTNUM_LEN);
+ hw_feat->aux_snap_num = XLGMAC_GET_REG_BITS(mac_hfr2,
+ MAC_HWF2R_AUXSNAPNUM_POS,
+ MAC_HWF2R_AUXSNAPNUM_LEN);
+
+ /* Translate the Hash Table size into actual number */
+ switch (hw_feat->hash_table_size) {
+ case 0:
+ break;
+ case 1:
+ hw_feat->hash_table_size = 64;
+ break;
+ case 2:
+ hw_feat->hash_table_size = 128;
+ break;
+ case 3:
+ hw_feat->hash_table_size = 256;
+ break;
+ }
+
+ /* Translate the address width setting into actual number */
+ switch (hw_feat->dma_width) {
+ case 0:
+ hw_feat->dma_width = 32;
+ break;
+ case 1:
+ hw_feat->dma_width = 40;
+ break;
+ case 2:
+ hw_feat->dma_width = 48;
+ break;
+ default:
+ hw_feat->dma_width = 32;
+ }
+
+ /* The Queue, Channel and TC counts are zero based so increment them
+ * to get the actual number
+ */
+ hw_feat->rx_q_cnt++;
+ hw_feat->tx_q_cnt++;
+ hw_feat->rx_ch_cnt++;
+ hw_feat->tx_ch_cnt++;
+ hw_feat->tc_cnt++;
+}
+
+void xlgmac_print_all_hw_features(struct xlgmac_pdata *pdata)
+{
+ char *str = NULL;
+
+ XLGMAC_PR("\n");
+ XLGMAC_PR("=====================================================\n");
+ XLGMAC_PR("\n");
+ XLGMAC_PR("HW support following features\n");
+ XLGMAC_PR("\n");
+ /* HW Feature Register0 */
+ XLGMAC_PR("VLAN Hash Filter Selected : %s\n",
+ pdata->hw_feat.vlhash ? "YES" : "NO");
+ XLGMAC_PR("SMA (MDIO) Interface : %s\n",
+ pdata->hw_feat.sma ? "YES" : "NO");
+ XLGMAC_PR("PMT Remote Wake-up Packet Enable : %s\n",
+ pdata->hw_feat.rwk ? "YES" : "NO");
+ XLGMAC_PR("PMT Magic Packet Enable : %s\n",
+ pdata->hw_feat.mgk ? "YES" : "NO");
+ XLGMAC_PR("RMON/MMC Module Enable : %s\n",
+ pdata->hw_feat.mmc ? "YES" : "NO");
+ XLGMAC_PR("ARP Offload Enabled : %s\n",
+ pdata->hw_feat.aoe ? "YES" : "NO");
+ XLGMAC_PR("IEEE 1588-2008 Timestamp Enabled : %s\n",
+ pdata->hw_feat.ts ? "YES" : "NO");
+ XLGMAC_PR("Energy Efficient Ethernet Enabled : %s\n",
+ pdata->hw_feat.eee ? "YES" : "NO");
+ XLGMAC_PR("Transmit Checksum Offload Enabled : %s\n",
+ pdata->hw_feat.tx_coe ? "YES" : "NO");
+ XLGMAC_PR("Receive Checksum Offload Enabled : %s\n",
+ pdata->hw_feat.rx_coe ? "YES" : "NO");
+ XLGMAC_PR("Additional MAC Addresses 1-31 Selected : %s\n",
+ pdata->hw_feat.addn_mac ? "YES" : "NO");
+
+ switch (pdata->hw_feat.ts_src) {
+ case 0:
+ str = "RESERVED";
+ break;
+ case 1:
+ str = "INTERNAL";
+ break;
+ case 2:
+ str = "EXTERNAL";
+ break;
+ case 3:
+ str = "BOTH";
+ break;
+ }
+ XLGMAC_PR("Timestamp System Time Source : %s\n", str);
+
+ XLGMAC_PR("Source Address or VLAN Insertion Enable : %s\n",
+ pdata->hw_feat.sa_vlan_ins ? "YES" : "NO");
+
+ /* HW Feature Register1 */
+ switch (pdata->hw_feat.rx_fifo_size) {
+ case 0:
+ str = "128 bytes";
+ break;
+ case 1:
+ str = "256 bytes";
+ break;
+ case 2:
+ str = "512 bytes";
+ break;
+ case 3:
+ str = "1 KBytes";
+ break;
+ case 4:
+ str = "2 KBytes";
+ break;
+ case 5:
+ str = "4 KBytes";
+ break;
+ case 6:
+ str = "8 KBytes";
+ break;
+ case 7:
+ str = "16 KBytes";
+ break;
+ case 8:
+ str = "32 kBytes";
+ break;
+ case 9:
+ str = "64 KBytes";
+ break;
+ case 10:
+ str = "128 KBytes";
+ break;
+ case 11:
+ str = "256 KBytes";
+ break;
+ default:
+ str = "RESERVED";
+ }
+ XLGMAC_PR("MTL Receive FIFO Size : %s\n", str);
+
+ switch (pdata->hw_feat.tx_fifo_size) {
+ case 0:
+ str = "128 bytes";
+ break;
+ case 1:
+ str = "256 bytes";
+ break;
+ case 2:
+ str = "512 bytes";
+ break;
+ case 3:
+ str = "1 KBytes";
+ break;
+ case 4:
+ str = "2 KBytes";
+ break;
+ case 5:
+ str = "4 KBytes";
+ break;
+ case 6:
+ str = "8 KBytes";
+ break;
+ case 7:
+ str = "16 KBytes";
+ break;
+ case 8:
+ str = "32 kBytes";
+ break;
+ case 9:
+ str = "64 KBytes";
+ break;
+ case 10:
+ str = "128 KBytes";
+ break;
+ case 11:
+ str = "256 KBytes";
+ break;
+ default:
+ str = "RESERVED";
+ }
+ XLGMAC_PR("MTL Transmit FIFO Size : %s\n", str);
+
+ XLGMAC_PR("IEEE 1588 High Word Register Enable : %s\n",
+ pdata->hw_feat.adv_ts_hi ? "YES" : "NO");
+ XLGMAC_PR("Address width : %u\n",
+ pdata->hw_feat.dma_width);
+ XLGMAC_PR("DCB Feature Enable : %s\n",
+ pdata->hw_feat.dcb ? "YES" : "NO");
+ XLGMAC_PR("Split Header Feature Enable : %s\n",
+ pdata->hw_feat.sph ? "YES" : "NO");
+ XLGMAC_PR("TCP Segmentation Offload Enable : %s\n",
+ pdata->hw_feat.tso ? "YES" : "NO");
+ XLGMAC_PR("DMA Debug Registers Enabled : %s\n",
+ pdata->hw_feat.dma_debug ? "YES" : "NO");
+ XLGMAC_PR("RSS Feature Enabled : %s\n",
+ pdata->hw_feat.rss ? "YES" : "NO");
+ XLGMAC_PR("Number of Traffic classes : %u\n",
+ (pdata->hw_feat.tc_cnt));
+ XLGMAC_PR("Hash Table Size : %u\n",
+ pdata->hw_feat.hash_table_size);
+ XLGMAC_PR("Total number of L3 or L4 Filters : %u\n",
+ pdata->hw_feat.l3l4_filter_num);
+
+ /* HW Feature Register2 */
+ XLGMAC_PR("Number of MTL Receive Queues : %u\n",
+ pdata->hw_feat.rx_q_cnt);
+ XLGMAC_PR("Number of MTL Transmit Queues : %u\n",
+ pdata->hw_feat.tx_q_cnt);
+ XLGMAC_PR("Number of DMA Receive Channels : %u\n",
+ pdata->hw_feat.rx_ch_cnt);
+ XLGMAC_PR("Number of DMA Transmit Channels : %u\n",
+ pdata->hw_feat.tx_ch_cnt);
+
+ switch (pdata->hw_feat.pps_out_num) {
+ case 0:
+ str = "No PPS output";
+ break;
+ case 1:
+ str = "1 PPS output";
+ break;
+ case 2:
+ str = "2 PPS output";
+ break;
+ case 3:
+ str = "3 PPS output";
+ break;
+ case 4:
+ str = "4 PPS output";
+ break;
+ default:
+ str = "RESERVED";
+ }
+ XLGMAC_PR("Number of PPS Outputs : %s\n", str);
+
+ switch (pdata->hw_feat.aux_snap_num) {
+ case 0:
+ str = "No auxiliary input";
+ break;
+ case 1:
+ str = "1 auxiliary input";
+ break;
+ case 2:
+ str = "2 auxiliary input";
+ break;
+ case 3:
+ str = "3 auxiliary input";
+ break;
+ case 4:
+ str = "4 auxiliary input";
+ break;
+ default:
+ str = "RESERVED";
+ }
+ XLGMAC_PR("Number of Auxiliary Snapshot Inputs : %s", str);
+
+ XLGMAC_PR("\n");
+ XLGMAC_PR("=====================================================\n");
+ XLGMAC_PR("\n");
+}
--- /dev/null
+/* Synopsys DesignWare Core Enterprise Ethernet (XLGMAC) Driver
+ *
+ * Copyright (c) 2017 Synopsys, Inc. (www.synopsys.com)
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This Synopsys DWC XLGMAC software driver and associated documentation
+ * (hereinafter the "Software") is an unsupported proprietary work of
+ * Synopsys, Inc. unless otherwise expressly agreed to in writing between
+ * Synopsys and you. The Software IS NOT an item of Licensed Software or a
+ * Licensed Product under any End User Software License Agreement or
+ * Agreement for Licensed Products with Synopsys or any supplement thereto.
+ * Synopsys is a registered trademark of Synopsys, Inc. Other names included
+ * in the SOFTWARE may be the trademarks of their respective owners.
+ */
+
+#include "dwc-xlgmac.h"
+#include "dwc-xlgmac-reg.h"
+
+static void xlgmac_unmap_desc_data(struct xlgmac_pdata *pdata,
+ struct xlgmac_desc_data *desc_data)
+{
+ if (desc_data->skb_dma) {
+ if (desc_data->mapped_as_page) {
+ dma_unmap_page(pdata->dev, desc_data->skb_dma,
+ desc_data->skb_dma_len, DMA_TO_DEVICE);
+ } else {
+ dma_unmap_single(pdata->dev, desc_data->skb_dma,
+ desc_data->skb_dma_len, DMA_TO_DEVICE);
+ }
+ desc_data->skb_dma = 0;
+ desc_data->skb_dma_len = 0;
+ }
+
+ if (desc_data->skb) {
+ dev_kfree_skb_any(desc_data->skb);
+ desc_data->skb = NULL;
+ }
+
+ if (desc_data->rx.hdr.pa.pages)
+ put_page(desc_data->rx.hdr.pa.pages);
+
+ if (desc_data->rx.hdr.pa_unmap.pages) {
+ dma_unmap_page(pdata->dev, desc_data->rx.hdr.pa_unmap.pages_dma,
+ desc_data->rx.hdr.pa_unmap.pages_len,
+ DMA_FROM_DEVICE);
+ put_page(desc_data->rx.hdr.pa_unmap.pages);
+ }
+
+ if (desc_data->rx.buf.pa.pages)
+ put_page(desc_data->rx.buf.pa.pages);
+
+ if (desc_data->rx.buf.pa_unmap.pages) {
+ dma_unmap_page(pdata->dev, desc_data->rx.buf.pa_unmap.pages_dma,
+ desc_data->rx.buf.pa_unmap.pages_len,
+ DMA_FROM_DEVICE);
+ put_page(desc_data->rx.buf.pa_unmap.pages);
+ }
+
+ memset(&desc_data->tx, 0, sizeof(desc_data->tx));
+ memset(&desc_data->rx, 0, sizeof(desc_data->rx));
+
+ desc_data->mapped_as_page = 0;
+
+ if (desc_data->state_saved) {
+ desc_data->state_saved = 0;
+ desc_data->state.skb = NULL;
+ desc_data->state.len = 0;
+ desc_data->state.error = 0;
+ }
+}
+
+static void xlgmac_free_ring(struct xlgmac_pdata *pdata,
+ struct xlgmac_ring *ring)
+{
+ struct xlgmac_desc_data *desc_data;
+ unsigned int i;
+
+ if (!ring)
+ return;
+
+ if (ring->desc_data_head) {
+ for (i = 0; i < ring->dma_desc_count; i++) {
+ desc_data = XLGMAC_GET_DESC_DATA(ring, i);
+ xlgmac_unmap_desc_data(pdata, desc_data);
+ }
+
+ kfree(ring->desc_data_head);
+ ring->desc_data_head = NULL;
+ }
+
+ if (ring->rx_hdr_pa.pages) {
+ dma_unmap_page(pdata->dev, ring->rx_hdr_pa.pages_dma,
+ ring->rx_hdr_pa.pages_len, DMA_FROM_DEVICE);
+ put_page(ring->rx_hdr_pa.pages);
+
+ ring->rx_hdr_pa.pages = NULL;
+ ring->rx_hdr_pa.pages_len = 0;
+ ring->rx_hdr_pa.pages_offset = 0;
+ ring->rx_hdr_pa.pages_dma = 0;
+ }
+
+ if (ring->rx_buf_pa.pages) {
+ dma_unmap_page(pdata->dev, ring->rx_buf_pa.pages_dma,
+ ring->rx_buf_pa.pages_len, DMA_FROM_DEVICE);
+ put_page(ring->rx_buf_pa.pages);
+
+ ring->rx_buf_pa.pages = NULL;
+ ring->rx_buf_pa.pages_len = 0;
+ ring->rx_buf_pa.pages_offset = 0;
+ ring->rx_buf_pa.pages_dma = 0;
+ }
+
+ if (ring->dma_desc_head) {
+ dma_free_coherent(pdata->dev,
+ (sizeof(struct xlgmac_dma_desc) *
+ ring->dma_desc_count),
+ ring->dma_desc_head,
+ ring->dma_desc_head_addr);
+ ring->dma_desc_head = NULL;
+ }
+}
+
+static int xlgmac_init_ring(struct xlgmac_pdata *pdata,
+ struct xlgmac_ring *ring,
+ unsigned int dma_desc_count)
+{
+ if (!ring)
+ return 0;
+
+ /* Descriptors */
+ ring->dma_desc_count = dma_desc_count;
+ ring->dma_desc_head = dma_alloc_coherent(pdata->dev,
+ (sizeof(struct xlgmac_dma_desc) *
+ dma_desc_count),
+ &ring->dma_desc_head_addr,
+ GFP_KERNEL);
+ if (!ring->dma_desc_head)
+ return -ENOMEM;
+
+ /* Array of descriptor data */
+ ring->desc_data_head = kcalloc(dma_desc_count,
+ sizeof(struct xlgmac_desc_data),
+ GFP_KERNEL);
+ if (!ring->desc_data_head)
+ return -ENOMEM;
+
+ netif_dbg(pdata, drv, pdata->netdev,
+ "dma_desc_head=%p, dma_desc_head_addr=%pad, desc_data_head=%p\n",
+ ring->dma_desc_head,
+ &ring->dma_desc_head_addr,
+ ring->desc_data_head);
+
+ return 0;
+}
+
+static void xlgmac_free_rings(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_channel *channel;
+ unsigned int i;
+
+ if (!pdata->channel_head)
+ return;
+
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ xlgmac_free_ring(pdata, channel->tx_ring);
+ xlgmac_free_ring(pdata, channel->rx_ring);
+ }
+}
+
+static int xlgmac_alloc_rings(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_channel *channel;
+ unsigned int i;
+ int ret;
+
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ netif_dbg(pdata, drv, pdata->netdev, "%s - Tx ring:\n",
+ channel->name);
+
+ ret = xlgmac_init_ring(pdata, channel->tx_ring,
+ pdata->tx_desc_count);
+
+ if (ret) {
+ netdev_alert(pdata->netdev,
+ "error initializing Tx ring");
+ goto err_init_ring;
+ }
+
+ netif_dbg(pdata, drv, pdata->netdev, "%s - Rx ring:\n",
+ channel->name);
+
+ ret = xlgmac_init_ring(pdata, channel->rx_ring,
+ pdata->rx_desc_count);
+ if (ret) {
+ netdev_alert(pdata->netdev,
+ "error initializing Rx ring\n");
+ goto err_init_ring;
+ }
+ }
+
+ return 0;
+
+err_init_ring:
+ xlgmac_free_rings(pdata);
+
+ return ret;
+}
+
+static void xlgmac_free_channels(struct xlgmac_pdata *pdata)
+{
+ if (!pdata->channel_head)
+ return;
+
+ kfree(pdata->channel_head->tx_ring);
+ pdata->channel_head->tx_ring = NULL;
+
+ kfree(pdata->channel_head->rx_ring);
+ pdata->channel_head->rx_ring = NULL;
+
+ kfree(pdata->channel_head);
+
+ pdata->channel_head = NULL;
+ pdata->channel_count = 0;
+}
+
+static int xlgmac_alloc_channels(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_channel *channel_head, *channel;
+ struct xlgmac_ring *tx_ring, *rx_ring;
+ int ret = -ENOMEM;
+ unsigned int i;
+
+ channel_head = kcalloc(pdata->channel_count,
+ sizeof(struct xlgmac_channel), GFP_KERNEL);
+ if (!channel_head)
+ return ret;
+
+ netif_dbg(pdata, drv, pdata->netdev,
+ "channel_head=%p\n", channel_head);
+
+ tx_ring = kcalloc(pdata->tx_ring_count, sizeof(struct xlgmac_ring),
+ GFP_KERNEL);
+ if (!tx_ring)
+ goto err_tx_ring;
+
+ rx_ring = kcalloc(pdata->rx_ring_count, sizeof(struct xlgmac_ring),
+ GFP_KERNEL);
+ if (!rx_ring)
+ goto err_rx_ring;
+
+ for (i = 0, channel = channel_head; i < pdata->channel_count;
+ i++, channel++) {
+ snprintf(channel->name, sizeof(channel->name), "channel-%u", i);
+ channel->pdata = pdata;
+ channel->queue_index = i;
+ channel->dma_regs = pdata->mac_regs + DMA_CH_BASE +
+ (DMA_CH_INC * i);
+
+ if (pdata->per_channel_irq) {
+ /* Get the per DMA interrupt */
+ ret = pdata->channel_irq[i];
+ if (ret < 0) {
+ netdev_err(pdata->netdev,
+ "get_irq %u failed\n",
+ i + 1);
+ goto err_irq;
+ }
+ channel->dma_irq = ret;
+ }
+
+ if (i < pdata->tx_ring_count)
+ channel->tx_ring = tx_ring++;
+
+ if (i < pdata->rx_ring_count)
+ channel->rx_ring = rx_ring++;
+
+ netif_dbg(pdata, drv, pdata->netdev,
+ "%s: dma_regs=%p, tx_ring=%p, rx_ring=%p\n",
+ channel->name, channel->dma_regs,
+ channel->tx_ring, channel->rx_ring);
+ }
+
+ pdata->channel_head = channel_head;
+
+ return 0;
+
+err_irq:
+ kfree(rx_ring);
+
+err_rx_ring:
+ kfree(tx_ring);
+
+err_tx_ring:
+ kfree(channel_head);
+
+ return ret;
+}
+
+static void xlgmac_free_channels_and_rings(struct xlgmac_pdata *pdata)
+{
+ xlgmac_free_rings(pdata);
+
+ xlgmac_free_channels(pdata);
+}
+
+static int xlgmac_alloc_channels_and_rings(struct xlgmac_pdata *pdata)
+{
+ int ret;
+
+ ret = xlgmac_alloc_channels(pdata);
+ if (ret)
+ goto err_alloc;
+
+ ret = xlgmac_alloc_rings(pdata);
+ if (ret)
+ goto err_alloc;
+
+ return 0;
+
+err_alloc:
+ xlgmac_free_channels_and_rings(pdata);
+
+ return ret;
+}
+
+static int xlgmac_alloc_pages(struct xlgmac_pdata *pdata,
+ struct xlgmac_page_alloc *pa,
+ gfp_t gfp, int order)
+{
+ struct page *pages = NULL;
+ dma_addr_t pages_dma;
+ int ret;
+
+ /* Try to obtain pages, decreasing order if necessary */
+ gfp |= __GFP_COLD | __GFP_COMP | __GFP_NOWARN;
+ while (order >= 0) {
+ pages = alloc_pages(gfp, order);
+ if (pages)
+ break;
+
+ order--;
+ }
+ if (!pages)
+ return -ENOMEM;
+
+ /* Map the pages */
+ pages_dma = dma_map_page(pdata->dev, pages, 0,
+ PAGE_SIZE << order, DMA_FROM_DEVICE);
+ ret = dma_mapping_error(pdata->dev, pages_dma);
+ if (ret) {
+ put_page(pages);
+ return ret;
+ }
+
+ pa->pages = pages;
+ pa->pages_len = PAGE_SIZE << order;
+ pa->pages_offset = 0;
+ pa->pages_dma = pages_dma;
+
+ return 0;
+}
+
+static void xlgmac_set_buffer_data(struct xlgmac_buffer_data *bd,
+ struct xlgmac_page_alloc *pa,
+ unsigned int len)
+{
+ get_page(pa->pages);
+ bd->pa = *pa;
+
+ bd->dma_base = pa->pages_dma;
+ bd->dma_off = pa->pages_offset;
+ bd->dma_len = len;
+
+ pa->pages_offset += len;
+ if ((pa->pages_offset + len) > pa->pages_len) {
+ /* This data descriptor is responsible for unmapping page(s) */
+ bd->pa_unmap = *pa;
+
+ /* Get a new allocation next time */
+ pa->pages = NULL;
+ pa->pages_len = 0;
+ pa->pages_offset = 0;
+ pa->pages_dma = 0;
+ }
+}
+
+static int xlgmac_map_rx_buffer(struct xlgmac_pdata *pdata,
+ struct xlgmac_ring *ring,
+ struct xlgmac_desc_data *desc_data)
+{
+ int order, ret;
+
+ if (!ring->rx_hdr_pa.pages) {
+ ret = xlgmac_alloc_pages(pdata, &ring->rx_hdr_pa,
+ GFP_ATOMIC, 0);
+ if (ret)
+ return ret;
+ }
+
+ if (!ring->rx_buf_pa.pages) {
+ order = max_t(int, PAGE_ALLOC_COSTLY_ORDER - 1, 0);
+ ret = xlgmac_alloc_pages(pdata, &ring->rx_buf_pa,
+ GFP_ATOMIC, order);
+ if (ret)
+ return ret;
+ }
+
+ /* Set up the header page info */
+ xlgmac_set_buffer_data(&desc_data->rx.hdr, &ring->rx_hdr_pa,
+ XLGMAC_SKB_ALLOC_SIZE);
+
+ /* Set up the buffer page info */
+ xlgmac_set_buffer_data(&desc_data->rx.buf, &ring->rx_buf_pa,
+ pdata->rx_buf_size);
+
+ return 0;
+}
+
+static void xlgmac_tx_desc_init(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;
+ struct xlgmac_desc_data *desc_data;
+ struct xlgmac_dma_desc *dma_desc;
+ struct xlgmac_channel *channel;
+ struct xlgmac_ring *ring;
+ dma_addr_t dma_desc_addr;
+ unsigned int i, j;
+
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ ring = channel->tx_ring;
+ if (!ring)
+ break;
+
+ dma_desc = ring->dma_desc_head;
+ dma_desc_addr = ring->dma_desc_head_addr;
+
+ for (j = 0; j < ring->dma_desc_count; j++) {
+ desc_data = XLGMAC_GET_DESC_DATA(ring, j);
+
+ desc_data->dma_desc = dma_desc;
+ desc_data->dma_desc_addr = dma_desc_addr;
+
+ dma_desc++;
+ dma_desc_addr += sizeof(struct xlgmac_dma_desc);
+ }
+
+ ring->cur = 0;
+ ring->dirty = 0;
+ memset(&ring->tx, 0, sizeof(ring->tx));
+
+ hw_ops->tx_desc_init(channel);
+ }
+}
+
+static void xlgmac_rx_desc_init(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;
+ struct xlgmac_desc_data *desc_data;
+ struct xlgmac_dma_desc *dma_desc;
+ struct xlgmac_channel *channel;
+ struct xlgmac_ring *ring;
+ dma_addr_t dma_desc_addr;
+ unsigned int i, j;
+
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ ring = channel->rx_ring;
+ if (!ring)
+ break;
+
+ dma_desc = ring->dma_desc_head;
+ dma_desc_addr = ring->dma_desc_head_addr;
+
+ for (j = 0; j < ring->dma_desc_count; j++) {
+ desc_data = XLGMAC_GET_DESC_DATA(ring, j);
+
+ desc_data->dma_desc = dma_desc;
+ desc_data->dma_desc_addr = dma_desc_addr;
+
+ if (xlgmac_map_rx_buffer(pdata, ring, desc_data))
+ break;
+
+ dma_desc++;
+ dma_desc_addr += sizeof(struct xlgmac_dma_desc);
+ }
+
+ ring->cur = 0;
+ ring->dirty = 0;
+
+ hw_ops->rx_desc_init(channel);
+ }
+}
+
+static int xlgmac_map_tx_skb(struct xlgmac_channel *channel,
+ struct sk_buff *skb)
+{
+ struct xlgmac_pdata *pdata = channel->pdata;
+ struct xlgmac_ring *ring = channel->tx_ring;
+ unsigned int start_index, cur_index;
+ struct xlgmac_desc_data *desc_data;
+ unsigned int offset, datalen, len;
+ struct xlgmac_pkt_info *pkt_info;
+ struct skb_frag_struct *frag;
+ unsigned int tso, vlan;
+ dma_addr_t skb_dma;
+ unsigned int i;
+
+ offset = 0;
+ start_index = ring->cur;
+ cur_index = ring->cur;
+
+ pkt_info = &ring->pkt_info;
+ pkt_info->desc_count = 0;
+ pkt_info->length = 0;
+
+ tso = XLGMAC_GET_REG_BITS(pkt_info->attributes,
+ TX_PACKET_ATTRIBUTES_TSO_ENABLE_POS,
+ TX_PACKET_ATTRIBUTES_TSO_ENABLE_LEN);
+ vlan = XLGMAC_GET_REG_BITS(pkt_info->attributes,
+ TX_PACKET_ATTRIBUTES_VLAN_CTAG_POS,
+ TX_PACKET_ATTRIBUTES_VLAN_CTAG_LEN);
+
+ /* Save space for a context descriptor if needed */
+ if ((tso && (pkt_info->mss != ring->tx.cur_mss)) ||
+ (vlan && (pkt_info->vlan_ctag != ring->tx.cur_vlan_ctag)))
+ cur_index++;
+ desc_data = XLGMAC_GET_DESC_DATA(ring, cur_index);
+
+ if (tso) {
+ /* Map the TSO header */
+ skb_dma = dma_map_single(pdata->dev, skb->data,
+ pkt_info->header_len, DMA_TO_DEVICE);
+ if (dma_mapping_error(pdata->dev, skb_dma)) {
+ netdev_alert(pdata->netdev, "dma_map_single failed\n");
+ goto err_out;
+ }
+ desc_data->skb_dma = skb_dma;
+ desc_data->skb_dma_len = pkt_info->header_len;
+ netif_dbg(pdata, tx_queued, pdata->netdev,
+ "skb header: index=%u, dma=%pad, len=%u\n",
+ cur_index, &skb_dma, pkt_info->header_len);
+
+ offset = pkt_info->header_len;
+
+ pkt_info->length += pkt_info->header_len;
+
+ cur_index++;
+ desc_data = XLGMAC_GET_DESC_DATA(ring, cur_index);
+ }
+
+ /* Map the (remainder of the) packet */
+ for (datalen = skb_headlen(skb) - offset; datalen; ) {
+ len = min_t(unsigned int, datalen, XLGMAC_TX_MAX_BUF_SIZE);
+
+ skb_dma = dma_map_single(pdata->dev, skb->data + offset, len,
+ DMA_TO_DEVICE);
+ if (dma_mapping_error(pdata->dev, skb_dma)) {
+ netdev_alert(pdata->netdev, "dma_map_single failed\n");
+ goto err_out;
+ }
+ desc_data->skb_dma = skb_dma;
+ desc_data->skb_dma_len = len;
+ netif_dbg(pdata, tx_queued, pdata->netdev,
+ "skb data: index=%u, dma=%pad, len=%u\n",
+ cur_index, &skb_dma, len);
+
+ datalen -= len;
+ offset += len;
+
+ pkt_info->length += len;
+
+ cur_index++;
+ desc_data = XLGMAC_GET_DESC_DATA(ring, cur_index);
+ }
+
+ for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
+ netif_dbg(pdata, tx_queued, pdata->netdev,
+ "mapping frag %u\n", i);
+
+ frag = &skb_shinfo(skb)->frags[i];
+ offset = 0;
+
+ for (datalen = skb_frag_size(frag); datalen; ) {
+ len = min_t(unsigned int, datalen,
+ XLGMAC_TX_MAX_BUF_SIZE);
+
+ skb_dma = skb_frag_dma_map(pdata->dev, frag, offset,
+ len, DMA_TO_DEVICE);
+ if (dma_mapping_error(pdata->dev, skb_dma)) {
+ netdev_alert(pdata->netdev,
+ "skb_frag_dma_map failed\n");
+ goto err_out;
+ }
+ desc_data->skb_dma = skb_dma;
+ desc_data->skb_dma_len = len;
+ desc_data->mapped_as_page = 1;
+ netif_dbg(pdata, tx_queued, pdata->netdev,
+ "skb frag: index=%u, dma=%pad, len=%u\n",
+ cur_index, &skb_dma, len);
+
+ datalen -= len;
+ offset += len;
+
+ pkt_info->length += len;
+
+ cur_index++;
+ desc_data = XLGMAC_GET_DESC_DATA(ring, cur_index);
+ }
+ }
+
+ /* Save the skb address in the last entry. We always have some data
+ * that has been mapped so desc_data is always advanced past the last
+ * piece of mapped data - use the entry pointed to by cur_index - 1.
+ */
+ desc_data = XLGMAC_GET_DESC_DATA(ring, cur_index - 1);
+ desc_data->skb = skb;
+
+ /* Save the number of descriptor entries used */
+ pkt_info->desc_count = cur_index - start_index;
+
+ return pkt_info->desc_count;
+
+err_out:
+ while (start_index < cur_index) {
+ desc_data = XLGMAC_GET_DESC_DATA(ring, start_index++);
+ xlgmac_unmap_desc_data(pdata, desc_data);
+ }
+
+ return 0;
+}
+
+void xlgmac_init_desc_ops(struct xlgmac_desc_ops *desc_ops)
+{
+ desc_ops->alloc_channles_and_rings = xlgmac_alloc_channels_and_rings;
+ desc_ops->free_channels_and_rings = xlgmac_free_channels_and_rings;
+ desc_ops->map_tx_skb = xlgmac_map_tx_skb;
+ desc_ops->map_rx_buffer = xlgmac_map_rx_buffer;
+ desc_ops->unmap_desc_data = xlgmac_unmap_desc_data;
+ desc_ops->tx_desc_init = xlgmac_tx_desc_init;
+ desc_ops->rx_desc_init = xlgmac_rx_desc_init;
+}
--- /dev/null
+/* Synopsys DesignWare Core Enterprise Ethernet (XLGMAC) Driver
+ *
+ * Copyright (c) 2017 Synopsys, Inc. (www.synopsys.com)
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This Synopsys DWC XLGMAC software driver and associated documentation
+ * (hereinafter the "Software") is an unsupported proprietary work of
+ * Synopsys, Inc. unless otherwise expressly agreed to in writing between
+ * Synopsys and you. The Software IS NOT an item of Licensed Software or a
+ * Licensed Product under any End User Software License Agreement or
+ * Agreement for Licensed Products with Synopsys or any supplement thereto.
+ * Synopsys is a registered trademark of Synopsys, Inc. Other names included
+ * in the SOFTWARE may be the trademarks of their respective owners.
+ */
+
+#include <linux/phy.h>
+#include <linux/mdio.h>
+#include <linux/clk.h>
+#include <linux/bitrev.h>
+#include <linux/crc32.h>
+
+#include "dwc-xlgmac.h"
+#include "dwc-xlgmac-reg.h"
+
+static int xlgmac_tx_complete(struct xlgmac_dma_desc *dma_desc)
+{
+ return !XLGMAC_GET_REG_BITS_LE(dma_desc->desc3,
+ TX_NORMAL_DESC3_OWN_POS,
+ TX_NORMAL_DESC3_OWN_LEN);
+}
+
+static int xlgmac_disable_rx_csum(struct xlgmac_pdata *pdata)
+{
+ u32 regval;
+
+ regval = readl(pdata->mac_regs + MAC_RCR);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_RCR_IPC_POS,
+ MAC_RCR_IPC_LEN, 0);
+ writel(regval, pdata->mac_regs + MAC_RCR);
+
+ return 0;
+}
+
+static int xlgmac_enable_rx_csum(struct xlgmac_pdata *pdata)
+{
+ u32 regval;
+
+ regval = readl(pdata->mac_regs + MAC_RCR);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_RCR_IPC_POS,
+ MAC_RCR_IPC_LEN, 1);
+ writel(regval, pdata->mac_regs + MAC_RCR);
+
+ return 0;
+}
+
+static int xlgmac_set_mac_address(struct xlgmac_pdata *pdata, u8 *addr)
+{
+ unsigned int mac_addr_hi, mac_addr_lo;
+
+ mac_addr_hi = (addr[5] << 8) | (addr[4] << 0);
+ mac_addr_lo = (addr[3] << 24) | (addr[2] << 16) |
+ (addr[1] << 8) | (addr[0] << 0);
+
+ writel(mac_addr_hi, pdata->mac_regs + MAC_MACA0HR);
+ writel(mac_addr_lo, pdata->mac_regs + MAC_MACA0LR);
+
+ return 0;
+}
+
+static void xlgmac_set_mac_reg(struct xlgmac_pdata *pdata,
+ struct netdev_hw_addr *ha,
+ unsigned int *mac_reg)
+{
+ unsigned int mac_addr_hi, mac_addr_lo;
+ u8 *mac_addr;
+
+ mac_addr_lo = 0;
+ mac_addr_hi = 0;
+
+ if (ha) {
+ mac_addr = (u8 *)&mac_addr_lo;
+ mac_addr[0] = ha->addr[0];
+ mac_addr[1] = ha->addr[1];
+ mac_addr[2] = ha->addr[2];
+ mac_addr[3] = ha->addr[3];
+ mac_addr = (u8 *)&mac_addr_hi;
+ mac_addr[0] = ha->addr[4];
+ mac_addr[1] = ha->addr[5];
+
+ netif_dbg(pdata, drv, pdata->netdev,
+ "adding mac address %pM at %#x\n",
+ ha->addr, *mac_reg);
+
+ mac_addr_hi = XLGMAC_SET_REG_BITS(mac_addr_hi,
+ MAC_MACA1HR_AE_POS,
+ MAC_MACA1HR_AE_LEN,
+ 1);
+ }
+
+ writel(mac_addr_hi, pdata->mac_regs + *mac_reg);
+ *mac_reg += MAC_MACA_INC;
+ writel(mac_addr_lo, pdata->mac_regs + *mac_reg);
+ *mac_reg += MAC_MACA_INC;
+}
+
+static int xlgmac_enable_rx_vlan_stripping(struct xlgmac_pdata *pdata)
+{
+ u32 regval;
+
+ regval = readl(pdata->mac_regs + MAC_VLANTR);
+ /* Put the VLAN tag in the Rx descriptor */
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_VLANTR_EVLRXS_POS,
+ MAC_VLANTR_EVLRXS_LEN, 1);
+ /* Don't check the VLAN type */
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_VLANTR_DOVLTC_POS,
+ MAC_VLANTR_DOVLTC_LEN, 1);
+ /* Check only C-TAG (0x8100) packets */
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_VLANTR_ERSVLM_POS,
+ MAC_VLANTR_ERSVLM_LEN, 0);
+ /* Don't consider an S-TAG (0x88A8) packet as a VLAN packet */
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_VLANTR_ESVL_POS,
+ MAC_VLANTR_ESVL_LEN, 0);
+ /* Enable VLAN tag stripping */
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_VLANTR_EVLS_POS,
+ MAC_VLANTR_EVLS_LEN, 0x3);
+ writel(regval, pdata->mac_regs + MAC_VLANTR);
+
+ return 0;
+}
+
+static int xlgmac_disable_rx_vlan_stripping(struct xlgmac_pdata *pdata)
+{
+ u32 regval;
+
+ regval = readl(pdata->mac_regs + MAC_VLANTR);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_VLANTR_EVLS_POS,
+ MAC_VLANTR_EVLS_LEN, 0);
+ writel(regval, pdata->mac_regs + MAC_VLANTR);
+
+ return 0;
+}
+
+static int xlgmac_enable_rx_vlan_filtering(struct xlgmac_pdata *pdata)
+{
+ u32 regval;
+
+ regval = readl(pdata->mac_regs + MAC_PFR);
+ /* Enable VLAN filtering */
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_PFR_VTFE_POS,
+ MAC_PFR_VTFE_LEN, 1);
+ writel(regval, pdata->mac_regs + MAC_PFR);
+
+ regval = readl(pdata->mac_regs + MAC_VLANTR);
+ /* Enable VLAN Hash Table filtering */
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_VLANTR_VTHM_POS,
+ MAC_VLANTR_VTHM_LEN, 1);
+ /* Disable VLAN tag inverse matching */
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_VLANTR_VTIM_POS,
+ MAC_VLANTR_VTIM_LEN, 0);
+ /* Only filter on the lower 12-bits of the VLAN tag */
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_VLANTR_ETV_POS,
+ MAC_VLANTR_ETV_LEN, 1);
+ /* In order for the VLAN Hash Table filtering to be effective,
+ * the VLAN tag identifier in the VLAN Tag Register must not
+ * be zero. Set the VLAN tag identifier to "1" to enable the
+ * VLAN Hash Table filtering. This implies that a VLAN tag of
+ * 1 will always pass filtering.
+ */
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_VLANTR_VL_POS,
+ MAC_VLANTR_VL_LEN, 1);
+ writel(regval, pdata->mac_regs + MAC_VLANTR);
+
+ return 0;
+}
+
+static int xlgmac_disable_rx_vlan_filtering(struct xlgmac_pdata *pdata)
+{
+ u32 regval;
+
+ regval = readl(pdata->mac_regs + MAC_PFR);
+ /* Disable VLAN filtering */
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_PFR_VTFE_POS,
+ MAC_PFR_VTFE_LEN, 0);
+ writel(regval, pdata->mac_regs + MAC_PFR);
+
+ return 0;
+}
+
+static u32 xlgmac_vid_crc32_le(__le16 vid_le)
+{
+ unsigned char *data = (unsigned char *)&vid_le;
+ unsigned char data_byte = 0;
+ u32 poly = 0xedb88320;
+ u32 crc = ~0;
+ u32 temp = 0;
+ int i, bits;
+
+ bits = get_bitmask_order(VLAN_VID_MASK);
+ for (i = 0; i < bits; i++) {
+ if ((i % 8) == 0)
+ data_byte = data[i / 8];
+
+ temp = ((crc & 1) ^ data_byte) & 1;
+ crc >>= 1;
+ data_byte >>= 1;
+
+ if (temp)
+ crc ^= poly;
+ }
+
+ return crc;
+}
+
+static int xlgmac_update_vlan_hash_table(struct xlgmac_pdata *pdata)
+{
+ u16 vlan_hash_table = 0;
+ __le16 vid_le;
+ u32 regval;
+ u32 crc;
+ u16 vid;
+
+ /* Generate the VLAN Hash Table value */
+ for_each_set_bit(vid, pdata->active_vlans, VLAN_N_VID) {
+ /* Get the CRC32 value of the VLAN ID */
+ vid_le = cpu_to_le16(vid);
+ crc = bitrev32(~xlgmac_vid_crc32_le(vid_le)) >> 28;
+
+ vlan_hash_table |= (1 << crc);
+ }
+
+ regval = readl(pdata->mac_regs + MAC_VLANHTR);
+ /* Set the VLAN Hash Table filtering register */
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_VLANHTR_VLHT_POS,
+ MAC_VLANHTR_VLHT_LEN, vlan_hash_table);
+ writel(regval, pdata->mac_regs + MAC_VLANHTR);
+
+ return 0;
+}
+
+static int xlgmac_set_promiscuous_mode(struct xlgmac_pdata *pdata,
+ unsigned int enable)
+{
+ unsigned int val = enable ? 1 : 0;
+ u32 regval;
+
+ regval = XLGMAC_GET_REG_BITS(readl(pdata->mac_regs + MAC_PFR),
+ MAC_PFR_PR_POS, MAC_PFR_PR_LEN);
+ if (regval == val)
+ return 0;
+
+ netif_dbg(pdata, drv, pdata->netdev, "%s promiscuous mode\n",
+ enable ? "entering" : "leaving");
+
+ regval = readl(pdata->mac_regs + MAC_PFR);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_PFR_PR_POS,
+ MAC_PFR_PR_LEN, val);
+ writel(regval, pdata->mac_regs + MAC_PFR);
+
+ /* Hardware will still perform VLAN filtering in promiscuous mode */
+ if (enable) {
+ xlgmac_disable_rx_vlan_filtering(pdata);
+ } else {
+ if (pdata->netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER)
+ xlgmac_enable_rx_vlan_filtering(pdata);
+ }
+
+ return 0;
+}
+
+static int xlgmac_set_all_multicast_mode(struct xlgmac_pdata *pdata,
+ unsigned int enable)
+{
+ unsigned int val = enable ? 1 : 0;
+ u32 regval;
+
+ regval = XLGMAC_GET_REG_BITS(readl(pdata->mac_regs + MAC_PFR),
+ MAC_PFR_PM_POS, MAC_PFR_PM_LEN);
+ if (regval == val)
+ return 0;
+
+ netif_dbg(pdata, drv, pdata->netdev, "%s allmulti mode\n",
+ enable ? "entering" : "leaving");
+
+ regval = readl(pdata->mac_regs + MAC_PFR);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_PFR_PM_POS,
+ MAC_PFR_PM_LEN, val);
+ writel(regval, pdata->mac_regs + MAC_PFR);
+
+ return 0;
+}
+
+static void xlgmac_set_mac_addn_addrs(struct xlgmac_pdata *pdata)
+{
+ struct net_device *netdev = pdata->netdev;
+ struct netdev_hw_addr *ha;
+ unsigned int addn_macs;
+ unsigned int mac_reg;
+
+ mac_reg = MAC_MACA1HR;
+ addn_macs = pdata->hw_feat.addn_mac;
+
+ if (netdev_uc_count(netdev) > addn_macs) {
+ xlgmac_set_promiscuous_mode(pdata, 1);
+ } else {
+ netdev_for_each_uc_addr(ha, netdev) {
+ xlgmac_set_mac_reg(pdata, ha, &mac_reg);
+ addn_macs--;
+ }
+
+ if (netdev_mc_count(netdev) > addn_macs) {
+ xlgmac_set_all_multicast_mode(pdata, 1);
+ } else {
+ netdev_for_each_mc_addr(ha, netdev) {
+ xlgmac_set_mac_reg(pdata, ha, &mac_reg);
+ addn_macs--;
+ }
+ }
+ }
+
+ /* Clear remaining additional MAC address entries */
+ while (addn_macs--)
+ xlgmac_set_mac_reg(pdata, NULL, &mac_reg);
+}
+
+static void xlgmac_set_mac_hash_table(struct xlgmac_pdata *pdata)
+{
+ unsigned int hash_table_shift, hash_table_count;
+ u32 hash_table[XLGMAC_MAC_HASH_TABLE_SIZE];
+ struct net_device *netdev = pdata->netdev;
+ struct netdev_hw_addr *ha;
+ unsigned int hash_reg;
+ unsigned int i;
+ u32 crc;
+
+ hash_table_shift = 26 - (pdata->hw_feat.hash_table_size >> 7);
+ hash_table_count = pdata->hw_feat.hash_table_size / 32;
+ memset(hash_table, 0, sizeof(hash_table));
+
+ /* Build the MAC Hash Table register values */
+ netdev_for_each_uc_addr(ha, netdev) {
+ crc = bitrev32(~crc32_le(~0, ha->addr, ETH_ALEN));
+ crc >>= hash_table_shift;
+ hash_table[crc >> 5] |= (1 << (crc & 0x1f));
+ }
+
+ netdev_for_each_mc_addr(ha, netdev) {
+ crc = bitrev32(~crc32_le(~0, ha->addr, ETH_ALEN));
+ crc >>= hash_table_shift;
+ hash_table[crc >> 5] |= (1 << (crc & 0x1f));
+ }
+
+ /* Set the MAC Hash Table registers */
+ hash_reg = MAC_HTR0;
+ for (i = 0; i < hash_table_count; i++) {
+ writel(hash_table[i], pdata->mac_regs + hash_reg);
+ hash_reg += MAC_HTR_INC;
+ }
+}
+
+static int xlgmac_add_mac_addresses(struct xlgmac_pdata *pdata)
+{
+ if (pdata->hw_feat.hash_table_size)
+ xlgmac_set_mac_hash_table(pdata);
+ else
+ xlgmac_set_mac_addn_addrs(pdata);
+
+ return 0;
+}
+
+static void xlgmac_config_mac_address(struct xlgmac_pdata *pdata)
+{
+ u32 regval;
+
+ xlgmac_set_mac_address(pdata, pdata->netdev->dev_addr);
+
+ /* Filtering is done using perfect filtering and hash filtering */
+ if (pdata->hw_feat.hash_table_size) {
+ regval = readl(pdata->mac_regs + MAC_PFR);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_PFR_HPF_POS,
+ MAC_PFR_HPF_LEN, 1);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_PFR_HUC_POS,
+ MAC_PFR_HUC_LEN, 1);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_PFR_HMC_POS,
+ MAC_PFR_HMC_LEN, 1);
+ writel(regval, pdata->mac_regs + MAC_PFR);
+ }
+}
+
+static void xlgmac_config_jumbo_enable(struct xlgmac_pdata *pdata)
+{
+ unsigned int val;
+ u32 regval;
+
+ val = (pdata->netdev->mtu > XLGMAC_STD_PACKET_MTU) ? 1 : 0;
+
+ regval = readl(pdata->mac_regs + MAC_RCR);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_RCR_JE_POS,
+ MAC_RCR_JE_LEN, val);
+ writel(regval, pdata->mac_regs + MAC_RCR);
+}
+
+static void xlgmac_config_checksum_offload(struct xlgmac_pdata *pdata)
+{
+ if (pdata->netdev->features & NETIF_F_RXCSUM)
+ xlgmac_enable_rx_csum(pdata);
+ else
+ xlgmac_disable_rx_csum(pdata);
+}
+
+static void xlgmac_config_vlan_support(struct xlgmac_pdata *pdata)
+{
+ u32 regval;
+
+ regval = readl(pdata->mac_regs + MAC_VLANIR);
+ /* Indicate that VLAN Tx CTAGs come from context descriptors */
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_VLANIR_CSVL_POS,
+ MAC_VLANIR_CSVL_LEN, 0);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_VLANIR_VLTI_POS,
+ MAC_VLANIR_VLTI_LEN, 1);
+ writel(regval, pdata->mac_regs + MAC_VLANIR);
+
+ /* Set the current VLAN Hash Table register value */
+ xlgmac_update_vlan_hash_table(pdata);
+
+ if (pdata->netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER)
+ xlgmac_enable_rx_vlan_filtering(pdata);
+ else
+ xlgmac_disable_rx_vlan_filtering(pdata);
+
+ if (pdata->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
+ xlgmac_enable_rx_vlan_stripping(pdata);
+ else
+ xlgmac_disable_rx_vlan_stripping(pdata);
+}
+
+static int xlgmac_config_rx_mode(struct xlgmac_pdata *pdata)
+{
+ struct net_device *netdev = pdata->netdev;
+ unsigned int pr_mode, am_mode;
+
+ pr_mode = ((netdev->flags & IFF_PROMISC) != 0);
+ am_mode = ((netdev->flags & IFF_ALLMULTI) != 0);
+
+ xlgmac_set_promiscuous_mode(pdata, pr_mode);
+ xlgmac_set_all_multicast_mode(pdata, am_mode);
+
+ xlgmac_add_mac_addresses(pdata);
+
+ return 0;
+}
+
+static void xlgmac_prepare_tx_stop(struct xlgmac_pdata *pdata,
+ struct xlgmac_channel *channel)
+{
+ unsigned int tx_dsr, tx_pos, tx_qidx;
+ unsigned long tx_timeout;
+ unsigned int tx_status;
+
+ /* Calculate the status register to read and the position within */
+ if (channel->queue_index < DMA_DSRX_FIRST_QUEUE) {
+ tx_dsr = DMA_DSR0;
+ tx_pos = (channel->queue_index * DMA_DSR_Q_LEN) +
+ DMA_DSR0_TPS_START;
+ } else {
+ tx_qidx = channel->queue_index - DMA_DSRX_FIRST_QUEUE;
+
+ tx_dsr = DMA_DSR1 + ((tx_qidx / DMA_DSRX_QPR) * DMA_DSRX_INC);
+ tx_pos = ((tx_qidx % DMA_DSRX_QPR) * DMA_DSR_Q_LEN) +
+ DMA_DSRX_TPS_START;
+ }
+
+ /* The Tx engine cannot be stopped if it is actively processing
+ * descriptors. Wait for the Tx engine to enter the stopped or
+ * suspended state. Don't wait forever though...
+ */
+ tx_timeout = jiffies + (XLGMAC_DMA_STOP_TIMEOUT * HZ);
+ while (time_before(jiffies, tx_timeout)) {
+ tx_status = readl(pdata->mac_regs + tx_dsr);
+ tx_status = XLGMAC_GET_REG_BITS(tx_status, tx_pos,
+ DMA_DSR_TPS_LEN);
+ if ((tx_status == DMA_TPS_STOPPED) ||
+ (tx_status == DMA_TPS_SUSPENDED))
+ break;
+
+ usleep_range(500, 1000);
+ }
+
+ if (!time_before(jiffies, tx_timeout))
+ netdev_info(pdata->netdev,
+ "timed out waiting for Tx DMA channel %u to stop\n",
+ channel->queue_index);
+}
+
+static void xlgmac_enable_tx(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_channel *channel;
+ unsigned int i;
+ u32 regval;
+
+ /* Enable each Tx DMA channel */
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ if (!channel->tx_ring)
+ break;
+
+ regval = readl(XLGMAC_DMA_REG(channel, DMA_CH_TCR));
+ regval = XLGMAC_SET_REG_BITS(regval, DMA_CH_TCR_ST_POS,
+ DMA_CH_TCR_ST_LEN, 1);
+ writel(regval, XLGMAC_DMA_REG(channel, DMA_CH_TCR));
+ }
+
+ /* Enable each Tx queue */
+ for (i = 0; i < pdata->tx_q_count; i++) {
+ regval = readl(XLGMAC_MTL_REG(pdata, i, MTL_Q_TQOMR));
+ regval = XLGMAC_SET_REG_BITS(regval, MTL_Q_TQOMR_TXQEN_POS,
+ MTL_Q_TQOMR_TXQEN_LEN,
+ MTL_Q_ENABLED);
+ writel(regval, XLGMAC_MTL_REG(pdata, i, MTL_Q_TQOMR));
+ }
+
+ /* Enable MAC Tx */
+ regval = readl(pdata->mac_regs + MAC_TCR);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_TCR_TE_POS,
+ MAC_TCR_TE_LEN, 1);
+ writel(regval, pdata->mac_regs + MAC_TCR);
+}
+
+static void xlgmac_disable_tx(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_channel *channel;
+ unsigned int i;
+ u32 regval;
+
+ /* Prepare for Tx DMA channel stop */
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ if (!channel->tx_ring)
+ break;
+
+ xlgmac_prepare_tx_stop(pdata, channel);
+ }
+
+ /* Disable MAC Tx */
+ regval = readl(pdata->mac_regs + MAC_TCR);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_TCR_TE_POS,
+ MAC_TCR_TE_LEN, 0);
+ writel(regval, pdata->mac_regs + MAC_TCR);
+
+ /* Disable each Tx queue */
+ for (i = 0; i < pdata->tx_q_count; i++) {
+ regval = readl(XLGMAC_MTL_REG(pdata, i, MTL_Q_TQOMR));
+ regval = XLGMAC_SET_REG_BITS(regval, MTL_Q_TQOMR_TXQEN_POS,
+ MTL_Q_TQOMR_TXQEN_LEN, 0);
+ writel(regval, XLGMAC_MTL_REG(pdata, i, MTL_Q_TQOMR));
+ }
+
+ /* Disable each Tx DMA channel */
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ if (!channel->tx_ring)
+ break;
+
+ regval = readl(XLGMAC_DMA_REG(channel, DMA_CH_TCR));
+ regval = XLGMAC_SET_REG_BITS(regval, DMA_CH_TCR_ST_POS,
+ DMA_CH_TCR_ST_LEN, 0);
+ writel(regval, XLGMAC_DMA_REG(channel, DMA_CH_TCR));
+ }
+}
+
+static void xlgmac_prepare_rx_stop(struct xlgmac_pdata *pdata,
+ unsigned int queue)
+{
+ unsigned int rx_status, prxq, rxqsts;
+ unsigned long rx_timeout;
+
+ /* The Rx engine cannot be stopped if it is actively processing
+ * packets. Wait for the Rx queue to empty the Rx fifo. Don't
+ * wait forever though...
+ */
+ rx_timeout = jiffies + (XLGMAC_DMA_STOP_TIMEOUT * HZ);
+ while (time_before(jiffies, rx_timeout)) {
+ rx_status = readl(XLGMAC_MTL_REG(pdata, queue, MTL_Q_RQDR));
+ prxq = XLGMAC_GET_REG_BITS(rx_status, MTL_Q_RQDR_PRXQ_POS,
+ MTL_Q_RQDR_PRXQ_LEN);
+ rxqsts = XLGMAC_GET_REG_BITS(rx_status, MTL_Q_RQDR_RXQSTS_POS,
+ MTL_Q_RQDR_RXQSTS_LEN);
+ if ((prxq == 0) && (rxqsts == 0))
+ break;
+
+ usleep_range(500, 1000);
+ }
+
+ if (!time_before(jiffies, rx_timeout))
+ netdev_info(pdata->netdev,
+ "timed out waiting for Rx queue %u to empty\n",
+ queue);
+}
+
+static void xlgmac_enable_rx(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_channel *channel;
+ unsigned int regval, i;
+
+ /* Enable each Rx DMA channel */
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ if (!channel->rx_ring)
+ break;
+
+ regval = readl(XLGMAC_DMA_REG(channel, DMA_CH_RCR));
+ regval = XLGMAC_SET_REG_BITS(regval, DMA_CH_RCR_SR_POS,
+ DMA_CH_RCR_SR_LEN, 1);
+ writel(regval, XLGMAC_DMA_REG(channel, DMA_CH_RCR));
+ }
+
+ /* Enable each Rx queue */
+ regval = 0;
+ for (i = 0; i < pdata->rx_q_count; i++)
+ regval |= (0x02 << (i << 1));
+ writel(regval, pdata->mac_regs + MAC_RQC0R);
+
+ /* Enable MAC Rx */
+ regval = readl(pdata->mac_regs + MAC_RCR);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_RCR_DCRCC_POS,
+ MAC_RCR_DCRCC_LEN, 1);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_RCR_CST_POS,
+ MAC_RCR_CST_LEN, 1);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_RCR_ACS_POS,
+ MAC_RCR_ACS_LEN, 1);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_RCR_RE_POS,
+ MAC_RCR_RE_LEN, 1);
+ writel(regval, pdata->mac_regs + MAC_RCR);
+}
+
+static void xlgmac_disable_rx(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_channel *channel;
+ unsigned int i;
+ u32 regval;
+
+ /* Disable MAC Rx */
+ regval = readl(pdata->mac_regs + MAC_RCR);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_RCR_DCRCC_POS,
+ MAC_RCR_DCRCC_LEN, 0);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_RCR_CST_POS,
+ MAC_RCR_CST_LEN, 0);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_RCR_ACS_POS,
+ MAC_RCR_ACS_LEN, 0);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_RCR_RE_POS,
+ MAC_RCR_RE_LEN, 0);
+ writel(regval, pdata->mac_regs + MAC_RCR);
+
+ /* Prepare for Rx DMA channel stop */
+ for (i = 0; i < pdata->rx_q_count; i++)
+ xlgmac_prepare_rx_stop(pdata, i);
+
+ /* Disable each Rx queue */
+ writel(0, pdata->mac_regs + MAC_RQC0R);
+
+ /* Disable each Rx DMA channel */
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ if (!channel->rx_ring)
+ break;
+
+ regval = readl(XLGMAC_DMA_REG(channel, DMA_CH_RCR));
+ regval = XLGMAC_SET_REG_BITS(regval, DMA_CH_RCR_SR_POS,
+ DMA_CH_RCR_SR_LEN, 0);
+ writel(regval, XLGMAC_DMA_REG(channel, DMA_CH_RCR));
+ }
+}
+
+static void xlgmac_tx_start_xmit(struct xlgmac_channel *channel,
+ struct xlgmac_ring *ring)
+{
+ struct xlgmac_pdata *pdata = channel->pdata;
+ struct xlgmac_desc_data *desc_data;
+
+ /* Make sure everything is written before the register write */
+ wmb();
+
+ /* Issue a poll command to Tx DMA by writing address
+ * of next immediate free descriptor
+ */
+ desc_data = XLGMAC_GET_DESC_DATA(ring, ring->cur);
+ writel(lower_32_bits(desc_data->dma_desc_addr),
+ XLGMAC_DMA_REG(channel, DMA_CH_TDTR_LO));
+
+ /* Start the Tx timer */
+ if (pdata->tx_usecs && !channel->tx_timer_active) {
+ channel->tx_timer_active = 1;
+ mod_timer(&channel->tx_timer,
+ jiffies + usecs_to_jiffies(pdata->tx_usecs));
+ }
+
+ ring->tx.xmit_more = 0;
+}
+
+static void xlgmac_dev_xmit(struct xlgmac_channel *channel)
+{
+ struct xlgmac_pdata *pdata = channel->pdata;
+ struct xlgmac_ring *ring = channel->tx_ring;
+ unsigned int tso_context, vlan_context;
+ struct xlgmac_desc_data *desc_data;
+ struct xlgmac_dma_desc *dma_desc;
+ struct xlgmac_pkt_info *pkt_info;
+ unsigned int csum, tso, vlan;
+ int start_index = ring->cur;
+ int cur_index = ring->cur;
+ unsigned int tx_set_ic;
+ int i;
+
+ pkt_info = &ring->pkt_info;
+ csum = XLGMAC_GET_REG_BITS(pkt_info->attributes,
+ TX_PACKET_ATTRIBUTES_CSUM_ENABLE_POS,
+ TX_PACKET_ATTRIBUTES_CSUM_ENABLE_LEN);
+ tso = XLGMAC_GET_REG_BITS(pkt_info->attributes,
+ TX_PACKET_ATTRIBUTES_TSO_ENABLE_POS,
+ TX_PACKET_ATTRIBUTES_TSO_ENABLE_LEN);
+ vlan = XLGMAC_GET_REG_BITS(pkt_info->attributes,
+ TX_PACKET_ATTRIBUTES_VLAN_CTAG_POS,
+ TX_PACKET_ATTRIBUTES_VLAN_CTAG_LEN);
+
+ if (tso && (pkt_info->mss != ring->tx.cur_mss))
+ tso_context = 1;
+ else
+ tso_context = 0;
+
+ if (vlan && (pkt_info->vlan_ctag != ring->tx.cur_vlan_ctag))
+ vlan_context = 1;
+ else
+ vlan_context = 0;
+
+ /* Determine if an interrupt should be generated for this Tx:
+ * Interrupt:
+ * - Tx frame count exceeds the frame count setting
+ * - Addition of Tx frame count to the frame count since the
+ * last interrupt was set exceeds the frame count setting
+ * No interrupt:
+ * - No frame count setting specified (ethtool -C ethX tx-frames 0)
+ * - Addition of Tx frame count to the frame count since the
+ * last interrupt was set does not exceed the frame count setting
+ */
+ ring->coalesce_count += pkt_info->tx_packets;
+ if (!pdata->tx_frames)
+ tx_set_ic = 0;
+ else if (pkt_info->tx_packets > pdata->tx_frames)
+ tx_set_ic = 1;
+ else if ((ring->coalesce_count % pdata->tx_frames) <
+ pkt_info->tx_packets)
+ tx_set_ic = 1;
+ else
+ tx_set_ic = 0;
+
+ desc_data = XLGMAC_GET_DESC_DATA(ring, cur_index);
+ dma_desc = desc_data->dma_desc;
+
+ /* Create a context descriptor if this is a TSO pkt_info */
+ if (tso_context || vlan_context) {
+ if (tso_context) {
+ netif_dbg(pdata, tx_queued, pdata->netdev,
+ "TSO context descriptor, mss=%u\n",
+ pkt_info->mss);
+
+ /* Set the MSS size */
+ dma_desc->desc2 = XLGMAC_SET_REG_BITS_LE(
+ dma_desc->desc2,
+ TX_CONTEXT_DESC2_MSS_POS,
+ TX_CONTEXT_DESC2_MSS_LEN,
+ pkt_info->mss);
+
+ /* Mark it as a CONTEXT descriptor */
+ dma_desc->desc3 = XLGMAC_SET_REG_BITS_LE(
+ dma_desc->desc3,
+ TX_CONTEXT_DESC3_CTXT_POS,
+ TX_CONTEXT_DESC3_CTXT_LEN,
+ 1);
+
+ /* Indicate this descriptor contains the MSS */
+ dma_desc->desc3 = XLGMAC_SET_REG_BITS_LE(
+ dma_desc->desc3,
+ TX_CONTEXT_DESC3_TCMSSV_POS,
+ TX_CONTEXT_DESC3_TCMSSV_LEN,
+ 1);
+
+ ring->tx.cur_mss = pkt_info->mss;
+ }
+
+ if (vlan_context) {
+ netif_dbg(pdata, tx_queued, pdata->netdev,
+ "VLAN context descriptor, ctag=%u\n",
+ pkt_info->vlan_ctag);
+
+ /* Mark it as a CONTEXT descriptor */
+ dma_desc->desc3 = XLGMAC_SET_REG_BITS_LE(
+ dma_desc->desc3,
+ TX_CONTEXT_DESC3_CTXT_POS,
+ TX_CONTEXT_DESC3_CTXT_LEN,
+ 1);
+
+ /* Set the VLAN tag */
+ dma_desc->desc3 = XLGMAC_SET_REG_BITS_LE(
+ dma_desc->desc3,
+ TX_CONTEXT_DESC3_VT_POS,
+ TX_CONTEXT_DESC3_VT_LEN,
+ pkt_info->vlan_ctag);
+
+ /* Indicate this descriptor contains the VLAN tag */
+ dma_desc->desc3 = XLGMAC_SET_REG_BITS_LE(
+ dma_desc->desc3,
+ TX_CONTEXT_DESC3_VLTV_POS,
+ TX_CONTEXT_DESC3_VLTV_LEN,
+ 1);
+
+ ring->tx.cur_vlan_ctag = pkt_info->vlan_ctag;
+ }
+
+ cur_index++;
+ desc_data = XLGMAC_GET_DESC_DATA(ring, cur_index);
+ dma_desc = desc_data->dma_desc;
+ }
+
+ /* Update buffer address (for TSO this is the header) */
+ dma_desc->desc0 = cpu_to_le32(lower_32_bits(desc_data->skb_dma));
+ dma_desc->desc1 = cpu_to_le32(upper_32_bits(desc_data->skb_dma));
+
+ /* Update the buffer length */
+ dma_desc->desc2 = XLGMAC_SET_REG_BITS_LE(
+ dma_desc->desc2,
+ TX_NORMAL_DESC2_HL_B1L_POS,
+ TX_NORMAL_DESC2_HL_B1L_LEN,
+ desc_data->skb_dma_len);
+
+ /* VLAN tag insertion check */
+ if (vlan)
+ dma_desc->desc2 = XLGMAC_SET_REG_BITS_LE(
+ dma_desc->desc2,
+ TX_NORMAL_DESC2_VTIR_POS,
+ TX_NORMAL_DESC2_VTIR_LEN,
+ TX_NORMAL_DESC2_VLAN_INSERT);
+
+ /* Timestamp enablement check */
+ if (XLGMAC_GET_REG_BITS(pkt_info->attributes,
+ TX_PACKET_ATTRIBUTES_PTP_POS,
+ TX_PACKET_ATTRIBUTES_PTP_LEN))
+ dma_desc->desc2 = XLGMAC_SET_REG_BITS_LE(
+ dma_desc->desc2,
+ TX_NORMAL_DESC2_TTSE_POS,
+ TX_NORMAL_DESC2_TTSE_LEN,
+ 1);
+
+ /* Mark it as First Descriptor */
+ dma_desc->desc3 = XLGMAC_SET_REG_BITS_LE(
+ dma_desc->desc3,
+ TX_NORMAL_DESC3_FD_POS,
+ TX_NORMAL_DESC3_FD_LEN,
+ 1);
+
+ /* Mark it as a NORMAL descriptor */
+ dma_desc->desc3 = XLGMAC_SET_REG_BITS_LE(
+ dma_desc->desc3,
+ TX_NORMAL_DESC3_CTXT_POS,
+ TX_NORMAL_DESC3_CTXT_LEN,
+ 0);
+
+ /* Set OWN bit if not the first descriptor */
+ if (cur_index != start_index)
+ dma_desc->desc3 = XLGMAC_SET_REG_BITS_LE(
+ dma_desc->desc3,
+ TX_NORMAL_DESC3_OWN_POS,
+ TX_NORMAL_DESC3_OWN_LEN,
+ 1);
+
+ if (tso) {
+ /* Enable TSO */
+ dma_desc->desc3 = XLGMAC_SET_REG_BITS_LE(
+ dma_desc->desc3,
+ TX_NORMAL_DESC3_TSE_POS,
+ TX_NORMAL_DESC3_TSE_LEN, 1);
+ dma_desc->desc3 = XLGMAC_SET_REG_BITS_LE(
+ dma_desc->desc3,
+ TX_NORMAL_DESC3_TCPPL_POS,
+ TX_NORMAL_DESC3_TCPPL_LEN,
+ pkt_info->tcp_payload_len);
+ dma_desc->desc3 = XLGMAC_SET_REG_BITS_LE(
+ dma_desc->desc3,
+ TX_NORMAL_DESC3_TCPHDRLEN_POS,
+ TX_NORMAL_DESC3_TCPHDRLEN_LEN,
+ pkt_info->tcp_header_len / 4);
+
+ pdata->stats.tx_tso_packets++;
+ } else {
+ /* Enable CRC and Pad Insertion */
+ dma_desc->desc3 = XLGMAC_SET_REG_BITS_LE(
+ dma_desc->desc3,
+ TX_NORMAL_DESC3_CPC_POS,
+ TX_NORMAL_DESC3_CPC_LEN, 0);
+
+ /* Enable HW CSUM */
+ if (csum)
+ dma_desc->desc3 = XLGMAC_SET_REG_BITS_LE(
+ dma_desc->desc3,
+ TX_NORMAL_DESC3_CIC_POS,
+ TX_NORMAL_DESC3_CIC_LEN,
+ 0x3);
+
+ /* Set the total length to be transmitted */
+ dma_desc->desc3 = XLGMAC_SET_REG_BITS_LE(
+ dma_desc->desc3,
+ TX_NORMAL_DESC3_FL_POS,
+ TX_NORMAL_DESC3_FL_LEN,
+ pkt_info->length);
+ }
+
+ for (i = cur_index - start_index + 1; i < pkt_info->desc_count; i++) {
+ cur_index++;
+ desc_data = XLGMAC_GET_DESC_DATA(ring, cur_index);
+ dma_desc = desc_data->dma_desc;
+
+ /* Update buffer address */
+ dma_desc->desc0 =
+ cpu_to_le32(lower_32_bits(desc_data->skb_dma));
+ dma_desc->desc1 =
+ cpu_to_le32(upper_32_bits(desc_data->skb_dma));
+
+ /* Update the buffer length */
+ dma_desc->desc2 = XLGMAC_SET_REG_BITS_LE(
+ dma_desc->desc2,
+ TX_NORMAL_DESC2_HL_B1L_POS,
+ TX_NORMAL_DESC2_HL_B1L_LEN,
+ desc_data->skb_dma_len);
+
+ /* Set OWN bit */
+ dma_desc->desc3 = XLGMAC_SET_REG_BITS_LE(
+ dma_desc->desc3,
+ TX_NORMAL_DESC3_OWN_POS,
+ TX_NORMAL_DESC3_OWN_LEN, 1);
+
+ /* Mark it as NORMAL descriptor */
+ dma_desc->desc3 = XLGMAC_SET_REG_BITS_LE(
+ dma_desc->desc3,
+ TX_NORMAL_DESC3_CTXT_POS,
+ TX_NORMAL_DESC3_CTXT_LEN, 0);
+
+ /* Enable HW CSUM */
+ if (csum)
+ dma_desc->desc3 = XLGMAC_SET_REG_BITS_LE(
+ dma_desc->desc3,
+ TX_NORMAL_DESC3_CIC_POS,
+ TX_NORMAL_DESC3_CIC_LEN,
+ 0x3);
+ }
+
+ /* Set LAST bit for the last descriptor */
+ dma_desc->desc3 = XLGMAC_SET_REG_BITS_LE(
+ dma_desc->desc3,
+ TX_NORMAL_DESC3_LD_POS,
+ TX_NORMAL_DESC3_LD_LEN, 1);
+
+ /* Set IC bit based on Tx coalescing settings */
+ if (tx_set_ic)
+ dma_desc->desc2 = XLGMAC_SET_REG_BITS_LE(
+ dma_desc->desc2,
+ TX_NORMAL_DESC2_IC_POS,
+ TX_NORMAL_DESC2_IC_LEN, 1);
+
+ /* Save the Tx info to report back during cleanup */
+ desc_data->tx.packets = pkt_info->tx_packets;
+ desc_data->tx.bytes = pkt_info->tx_bytes;
+
+ /* In case the Tx DMA engine is running, make sure everything
+ * is written to the descriptor(s) before setting the OWN bit
+ * for the first descriptor
+ */
+ dma_wmb();
+
+ /* Set OWN bit for the first descriptor */
+ desc_data = XLGMAC_GET_DESC_DATA(ring, start_index);
+ dma_desc = desc_data->dma_desc;
+ dma_desc->desc3 = XLGMAC_SET_REG_BITS_LE(
+ dma_desc->desc3,
+ TX_NORMAL_DESC3_OWN_POS,
+ TX_NORMAL_DESC3_OWN_LEN, 1);
+
+ if (netif_msg_tx_queued(pdata))
+ xlgmac_dump_tx_desc(pdata, ring, start_index,
+ pkt_info->desc_count, 1);
+
+ /* Make sure ownership is written to the descriptor */
+ smp_wmb();
+
+ ring->cur = cur_index + 1;
+ if (!pkt_info->skb->xmit_more ||
+ netif_xmit_stopped(netdev_get_tx_queue(pdata->netdev,
+ channel->queue_index)))
+ xlgmac_tx_start_xmit(channel, ring);
+ else
+ ring->tx.xmit_more = 1;
+
+ XLGMAC_PR("%s: descriptors %u to %u written\n",
+ channel->name, start_index & (ring->dma_desc_count - 1),
+ (ring->cur - 1) & (ring->dma_desc_count - 1));
+}
+
+static void xlgmac_get_rx_tstamp(struct xlgmac_pkt_info *pkt_info,
+ struct xlgmac_dma_desc *dma_desc)
+{
+ u32 tsa, tsd;
+ u64 nsec;
+
+ tsa = XLGMAC_GET_REG_BITS_LE(dma_desc->desc3,
+ RX_CONTEXT_DESC3_TSA_POS,
+ RX_CONTEXT_DESC3_TSA_LEN);
+ tsd = XLGMAC_GET_REG_BITS_LE(dma_desc->desc3,
+ RX_CONTEXT_DESC3_TSD_POS,
+ RX_CONTEXT_DESC3_TSD_LEN);
+ if (tsa && !tsd) {
+ nsec = le32_to_cpu(dma_desc->desc1);
+ nsec <<= 32;
+ nsec |= le32_to_cpu(dma_desc->desc0);
+ if (nsec != 0xffffffffffffffffULL) {
+ pkt_info->rx_tstamp = nsec;
+ pkt_info->attributes = XLGMAC_SET_REG_BITS(
+ pkt_info->attributes,
+ RX_PACKET_ATTRIBUTES_RX_TSTAMP_POS,
+ RX_PACKET_ATTRIBUTES_RX_TSTAMP_LEN,
+ 1);
+ }
+ }
+}
+
+static void xlgmac_tx_desc_reset(struct xlgmac_desc_data *desc_data)
+{
+ struct xlgmac_dma_desc *dma_desc = desc_data->dma_desc;
+
+ /* Reset the Tx descriptor
+ * Set buffer 1 (lo) address to zero
+ * Set buffer 1 (hi) address to zero
+ * Reset all other control bits (IC, TTSE, B2L & B1L)
+ * Reset all other control bits (OWN, CTXT, FD, LD, CPC, CIC, etc)
+ */
+ dma_desc->desc0 = 0;
+ dma_desc->desc1 = 0;
+ dma_desc->desc2 = 0;
+ dma_desc->desc3 = 0;
+
+ /* Make sure ownership is written to the descriptor */
+ dma_wmb();
+}
+
+static void xlgmac_tx_desc_init(struct xlgmac_channel *channel)
+{
+ struct xlgmac_ring *ring = channel->tx_ring;
+ struct xlgmac_desc_data *desc_data;
+ int start_index = ring->cur;
+ int i;
+
+ /* Initialze all descriptors */
+ for (i = 0; i < ring->dma_desc_count; i++) {
+ desc_data = XLGMAC_GET_DESC_DATA(ring, i);
+
+ /* Initialize Tx descriptor */
+ xlgmac_tx_desc_reset(desc_data);
+ }
+
+ /* Update the total number of Tx descriptors */
+ writel(ring->dma_desc_count - 1, XLGMAC_DMA_REG(channel, DMA_CH_TDRLR));
+
+ /* Update the starting address of descriptor ring */
+ desc_data = XLGMAC_GET_DESC_DATA(ring, start_index);
+ writel(upper_32_bits(desc_data->dma_desc_addr),
+ XLGMAC_DMA_REG(channel, DMA_CH_TDLR_HI));
+ writel(lower_32_bits(desc_data->dma_desc_addr),
+ XLGMAC_DMA_REG(channel, DMA_CH_TDLR_LO));
+}
+
+static void xlgmac_rx_desc_reset(struct xlgmac_pdata *pdata,
+ struct xlgmac_desc_data *desc_data,
+ unsigned int index)
+{
+ struct xlgmac_dma_desc *dma_desc = desc_data->dma_desc;
+ unsigned int rx_frames = pdata->rx_frames;
+ unsigned int rx_usecs = pdata->rx_usecs;
+ dma_addr_t hdr_dma, buf_dma;
+ unsigned int inte;
+
+ if (!rx_usecs && !rx_frames) {
+ /* No coalescing, interrupt for every descriptor */
+ inte = 1;
+ } else {
+ /* Set interrupt based on Rx frame coalescing setting */
+ if (rx_frames && !((index + 1) % rx_frames))
+ inte = 1;
+ else
+ inte = 0;
+ }
+
+ /* Reset the Rx descriptor
+ * Set buffer 1 (lo) address to header dma address (lo)
+ * Set buffer 1 (hi) address to header dma address (hi)
+ * Set buffer 2 (lo) address to buffer dma address (lo)
+ * Set buffer 2 (hi) address to buffer dma address (hi) and
+ * set control bits OWN and INTE
+ */
+ hdr_dma = desc_data->rx.hdr.dma_base + desc_data->rx.hdr.dma_off;
+ buf_dma = desc_data->rx.buf.dma_base + desc_data->rx.buf.dma_off;
+ dma_desc->desc0 = cpu_to_le32(lower_32_bits(hdr_dma));
+ dma_desc->desc1 = cpu_to_le32(upper_32_bits(hdr_dma));
+ dma_desc->desc2 = cpu_to_le32(lower_32_bits(buf_dma));
+ dma_desc->desc3 = cpu_to_le32(upper_32_bits(buf_dma));
+
+ dma_desc->desc3 = XLGMAC_SET_REG_BITS_LE(
+ dma_desc->desc3,
+ RX_NORMAL_DESC3_INTE_POS,
+ RX_NORMAL_DESC3_INTE_LEN,
+ inte);
+
+ /* Since the Rx DMA engine is likely running, make sure everything
+ * is written to the descriptor(s) before setting the OWN bit
+ * for the descriptor
+ */
+ dma_wmb();
+
+ dma_desc->desc3 = XLGMAC_SET_REG_BITS_LE(
+ dma_desc->desc3,
+ RX_NORMAL_DESC3_OWN_POS,
+ RX_NORMAL_DESC3_OWN_LEN,
+ 1);
+
+ /* Make sure ownership is written to the descriptor */
+ dma_wmb();
+}
+
+static void xlgmac_rx_desc_init(struct xlgmac_channel *channel)
+{
+ struct xlgmac_pdata *pdata = channel->pdata;
+ struct xlgmac_ring *ring = channel->rx_ring;
+ unsigned int start_index = ring->cur;
+ struct xlgmac_desc_data *desc_data;
+ unsigned int i;
+
+ /* Initialize all descriptors */
+ for (i = 0; i < ring->dma_desc_count; i++) {
+ desc_data = XLGMAC_GET_DESC_DATA(ring, i);
+
+ /* Initialize Rx descriptor */
+ xlgmac_rx_desc_reset(pdata, desc_data, i);
+ }
+
+ /* Update the total number of Rx descriptors */
+ writel(ring->dma_desc_count - 1, XLGMAC_DMA_REG(channel, DMA_CH_RDRLR));
+
+ /* Update the starting address of descriptor ring */
+ desc_data = XLGMAC_GET_DESC_DATA(ring, start_index);
+ writel(upper_32_bits(desc_data->dma_desc_addr),
+ XLGMAC_DMA_REG(channel, DMA_CH_RDLR_HI));
+ writel(lower_32_bits(desc_data->dma_desc_addr),
+ XLGMAC_DMA_REG(channel, DMA_CH_RDLR_LO));
+
+ /* Update the Rx Descriptor Tail Pointer */
+ desc_data = XLGMAC_GET_DESC_DATA(ring, start_index +
+ ring->dma_desc_count - 1);
+ writel(lower_32_bits(desc_data->dma_desc_addr),
+ XLGMAC_DMA_REG(channel, DMA_CH_RDTR_LO));
+}
+
+static int xlgmac_is_context_desc(struct xlgmac_dma_desc *dma_desc)
+{
+ /* Rx and Tx share CTXT bit, so check TDES3.CTXT bit */
+ return XLGMAC_GET_REG_BITS_LE(dma_desc->desc3,
+ TX_NORMAL_DESC3_CTXT_POS,
+ TX_NORMAL_DESC3_CTXT_LEN);
+}
+
+static int xlgmac_is_last_desc(struct xlgmac_dma_desc *dma_desc)
+{
+ /* Rx and Tx share LD bit, so check TDES3.LD bit */
+ return XLGMAC_GET_REG_BITS_LE(dma_desc->desc3,
+ TX_NORMAL_DESC3_LD_POS,
+ TX_NORMAL_DESC3_LD_LEN);
+}
+
+static int xlgmac_disable_tx_flow_control(struct xlgmac_pdata *pdata)
+{
+ unsigned int max_q_count, q_count;
+ unsigned int reg, regval;
+ unsigned int i;
+
+ /* Clear MTL flow control */
+ for (i = 0; i < pdata->rx_q_count; i++) {
+ regval = readl(XLGMAC_MTL_REG(pdata, i, MTL_Q_RQOMR));
+ regval = XLGMAC_SET_REG_BITS(regval, MTL_Q_RQOMR_EHFC_POS,
+ MTL_Q_RQOMR_EHFC_LEN, 0);
+ writel(regval, XLGMAC_MTL_REG(pdata, i, MTL_Q_RQOMR));
+ }
+
+ /* Clear MAC flow control */
+ max_q_count = XLGMAC_MAX_FLOW_CONTROL_QUEUES;
+ q_count = min_t(unsigned int, pdata->tx_q_count, max_q_count);
+ reg = MAC_Q0TFCR;
+ for (i = 0; i < q_count; i++) {
+ regval = readl(pdata->mac_regs + reg);
+ regval = XLGMAC_SET_REG_BITS(regval,
+ MAC_Q0TFCR_TFE_POS,
+ MAC_Q0TFCR_TFE_LEN,
+ 0);
+ writel(regval, pdata->mac_regs + reg);
+
+ reg += MAC_QTFCR_INC;
+ }
+
+ return 0;
+}
+
+static int xlgmac_enable_tx_flow_control(struct xlgmac_pdata *pdata)
+{
+ unsigned int max_q_count, q_count;
+ unsigned int reg, regval;
+ unsigned int i;
+
+ /* Set MTL flow control */
+ for (i = 0; i < pdata->rx_q_count; i++) {
+ regval = readl(XLGMAC_MTL_REG(pdata, i, MTL_Q_RQOMR));
+ regval = XLGMAC_SET_REG_BITS(regval, MTL_Q_RQOMR_EHFC_POS,
+ MTL_Q_RQOMR_EHFC_LEN, 1);
+ writel(regval, XLGMAC_MTL_REG(pdata, i, MTL_Q_RQOMR));
+ }
+
+ /* Set MAC flow control */
+ max_q_count = XLGMAC_MAX_FLOW_CONTROL_QUEUES;
+ q_count = min_t(unsigned int, pdata->tx_q_count, max_q_count);
+ reg = MAC_Q0TFCR;
+ for (i = 0; i < q_count; i++) {
+ regval = readl(pdata->mac_regs + reg);
+
+ /* Enable transmit flow control */
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_Q0TFCR_TFE_POS,
+ MAC_Q0TFCR_TFE_LEN, 1);
+ /* Set pause time */
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_Q0TFCR_PT_POS,
+ MAC_Q0TFCR_PT_LEN, 0xffff);
+
+ writel(regval, pdata->mac_regs + reg);
+
+ reg += MAC_QTFCR_INC;
+ }
+
+ return 0;
+}
+
+static int xlgmac_disable_rx_flow_control(struct xlgmac_pdata *pdata)
+{
+ u32 regval;
+
+ regval = readl(pdata->mac_regs + MAC_RFCR);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_RFCR_RFE_POS,
+ MAC_RFCR_RFE_LEN, 0);
+ writel(regval, pdata->mac_regs + MAC_RFCR);
+
+ return 0;
+}
+
+static int xlgmac_enable_rx_flow_control(struct xlgmac_pdata *pdata)
+{
+ u32 regval;
+
+ regval = readl(pdata->mac_regs + MAC_RFCR);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_RFCR_RFE_POS,
+ MAC_RFCR_RFE_LEN, 1);
+ writel(regval, pdata->mac_regs + MAC_RFCR);
+
+ return 0;
+}
+
+static int xlgmac_config_tx_flow_control(struct xlgmac_pdata *pdata)
+{
+ if (pdata->tx_pause)
+ xlgmac_enable_tx_flow_control(pdata);
+ else
+ xlgmac_disable_tx_flow_control(pdata);
+
+ return 0;
+}
+
+static int xlgmac_config_rx_flow_control(struct xlgmac_pdata *pdata)
+{
+ if (pdata->rx_pause)
+ xlgmac_enable_rx_flow_control(pdata);
+ else
+ xlgmac_disable_rx_flow_control(pdata);
+
+ return 0;
+}
+
+static int xlgmac_config_rx_coalesce(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_channel *channel;
+ unsigned int i;
+ u32 regval;
+
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ if (!channel->rx_ring)
+ break;
+
+ regval = readl(XLGMAC_DMA_REG(channel, DMA_CH_RIWT));
+ regval = XLGMAC_SET_REG_BITS(regval, DMA_CH_RIWT_RWT_POS,
+ DMA_CH_RIWT_RWT_LEN,
+ pdata->rx_riwt);
+ writel(regval, XLGMAC_DMA_REG(channel, DMA_CH_RIWT));
+ }
+
+ return 0;
+}
+
+static void xlgmac_config_flow_control(struct xlgmac_pdata *pdata)
+{
+ xlgmac_config_tx_flow_control(pdata);
+ xlgmac_config_rx_flow_control(pdata);
+}
+
+static void xlgmac_config_rx_fep_enable(struct xlgmac_pdata *pdata)
+{
+ unsigned int i;
+ u32 regval;
+
+ for (i = 0; i < pdata->rx_q_count; i++) {
+ regval = readl(XLGMAC_MTL_REG(pdata, i, MTL_Q_RQOMR));
+ regval = XLGMAC_SET_REG_BITS(regval, MTL_Q_RQOMR_FEP_POS,
+ MTL_Q_RQOMR_FEP_LEN, 1);
+ writel(regval, XLGMAC_MTL_REG(pdata, i, MTL_Q_RQOMR));
+ }
+}
+
+static void xlgmac_config_rx_fup_enable(struct xlgmac_pdata *pdata)
+{
+ unsigned int i;
+ u32 regval;
+
+ for (i = 0; i < pdata->rx_q_count; i++) {
+ regval = readl(XLGMAC_MTL_REG(pdata, i, MTL_Q_RQOMR));
+ regval = XLGMAC_SET_REG_BITS(regval, MTL_Q_RQOMR_FUP_POS,
+ MTL_Q_RQOMR_FUP_LEN, 1);
+ writel(regval, XLGMAC_MTL_REG(pdata, i, MTL_Q_RQOMR));
+ }
+}
+
+static int xlgmac_config_tx_coalesce(struct xlgmac_pdata *pdata)
+{
+ return 0;
+}
+
+static void xlgmac_config_rx_buffer_size(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_channel *channel;
+ unsigned int i;
+ u32 regval;
+
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ if (!channel->rx_ring)
+ break;
+
+ regval = readl(XLGMAC_DMA_REG(channel, DMA_CH_RCR));
+ regval = XLGMAC_SET_REG_BITS(regval, DMA_CH_RCR_RBSZ_POS,
+ DMA_CH_RCR_RBSZ_LEN,
+ pdata->rx_buf_size);
+ writel(regval, XLGMAC_DMA_REG(channel, DMA_CH_RCR));
+ }
+}
+
+static void xlgmac_config_tso_mode(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_channel *channel;
+ unsigned int i;
+ u32 regval;
+
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ if (!channel->tx_ring)
+ break;
+
+ if (pdata->hw_feat.tso) {
+ regval = readl(XLGMAC_DMA_REG(channel, DMA_CH_TCR));
+ regval = XLGMAC_SET_REG_BITS(regval, DMA_CH_TCR_TSE_POS,
+ DMA_CH_TCR_TSE_LEN, 1);
+ writel(regval, XLGMAC_DMA_REG(channel, DMA_CH_TCR));
+ }
+ }
+}
+
+static void xlgmac_config_sph_mode(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_channel *channel;
+ unsigned int i;
+ u32 regval;
+
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ if (!channel->rx_ring)
+ break;
+
+ regval = readl(XLGMAC_DMA_REG(channel, DMA_CH_CR));
+ regval = XLGMAC_SET_REG_BITS(regval, DMA_CH_CR_SPH_POS,
+ DMA_CH_CR_SPH_LEN, 1);
+ writel(regval, XLGMAC_DMA_REG(channel, DMA_CH_CR));
+ }
+
+ regval = readl(pdata->mac_regs + MAC_RCR);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_RCR_HDSMS_POS,
+ MAC_RCR_HDSMS_LEN,
+ XLGMAC_SPH_HDSMS_SIZE);
+ writel(regval, pdata->mac_regs + MAC_RCR);
+}
+
+static unsigned int xlgmac_usec_to_riwt(struct xlgmac_pdata *pdata,
+ unsigned int usec)
+{
+ unsigned long rate;
+ unsigned int ret;
+
+ rate = pdata->sysclk_rate;
+
+ /* Convert the input usec value to the watchdog timer value. Each
+ * watchdog timer value is equivalent to 256 clock cycles.
+ * Calculate the required value as:
+ * ( usec * ( system_clock_mhz / 10^6 ) / 256
+ */
+ ret = (usec * (rate / 1000000)) / 256;
+
+ return ret;
+}
+
+static unsigned int xlgmac_riwt_to_usec(struct xlgmac_pdata *pdata,
+ unsigned int riwt)
+{
+ unsigned long rate;
+ unsigned int ret;
+
+ rate = pdata->sysclk_rate;
+
+ /* Convert the input watchdog timer value to the usec value. Each
+ * watchdog timer value is equivalent to 256 clock cycles.
+ * Calculate the required value as:
+ * ( riwt * 256 ) / ( system_clock_mhz / 10^6 )
+ */
+ ret = (riwt * 256) / (rate / 1000000);
+
+ return ret;
+}
+
+static int xlgmac_config_rx_threshold(struct xlgmac_pdata *pdata,
+ unsigned int val)
+{
+ unsigned int i;
+ u32 regval;
+
+ for (i = 0; i < pdata->rx_q_count; i++) {
+ regval = readl(XLGMAC_MTL_REG(pdata, i, MTL_Q_RQOMR));
+ regval = XLGMAC_SET_REG_BITS(regval, MTL_Q_RQOMR_RTC_POS,
+ MTL_Q_RQOMR_RTC_LEN, val);
+ writel(regval, XLGMAC_MTL_REG(pdata, i, MTL_Q_RQOMR));
+ }
+
+ return 0;
+}
+
+static void xlgmac_config_mtl_mode(struct xlgmac_pdata *pdata)
+{
+ unsigned int i;
+ u32 regval;
+
+ /* Set Tx to weighted round robin scheduling algorithm */
+ regval = readl(pdata->mac_regs + MTL_OMR);
+ regval = XLGMAC_SET_REG_BITS(regval, MTL_OMR_ETSALG_POS,
+ MTL_OMR_ETSALG_LEN, MTL_ETSALG_WRR);
+ writel(regval, pdata->mac_regs + MTL_OMR);
+
+ /* Set Tx traffic classes to use WRR algorithm with equal weights */
+ for (i = 0; i < pdata->hw_feat.tc_cnt; i++) {
+ regval = readl(XLGMAC_MTL_REG(pdata, i, MTL_TC_ETSCR));
+ regval = XLGMAC_SET_REG_BITS(regval, MTL_TC_ETSCR_TSA_POS,
+ MTL_TC_ETSCR_TSA_LEN, MTL_TSA_ETS);
+ writel(regval, XLGMAC_MTL_REG(pdata, i, MTL_TC_ETSCR));
+
+ regval = readl(XLGMAC_MTL_REG(pdata, i, MTL_TC_QWR));
+ regval = XLGMAC_SET_REG_BITS(regval, MTL_TC_QWR_QW_POS,
+ MTL_TC_QWR_QW_LEN, 1);
+ writel(regval, XLGMAC_MTL_REG(pdata, i, MTL_TC_QWR));
+ }
+
+ /* Set Rx to strict priority algorithm */
+ regval = readl(pdata->mac_regs + MTL_OMR);
+ regval = XLGMAC_SET_REG_BITS(regval, MTL_OMR_RAA_POS,
+ MTL_OMR_RAA_LEN, MTL_RAA_SP);
+ writel(regval, pdata->mac_regs + MTL_OMR);
+}
+
+static void xlgmac_config_queue_mapping(struct xlgmac_pdata *pdata)
+{
+ unsigned int ppq, ppq_extra, prio, prio_queues;
+ unsigned int qptc, qptc_extra, queue;
+ unsigned int reg, regval;
+ unsigned int mask;
+ unsigned int i, j;
+
+ /* Map the MTL Tx Queues to Traffic Classes
+ * Note: Tx Queues >= Traffic Classes
+ */
+ qptc = pdata->tx_q_count / pdata->hw_feat.tc_cnt;
+ qptc_extra = pdata->tx_q_count % pdata->hw_feat.tc_cnt;
+
+ for (i = 0, queue = 0; i < pdata->hw_feat.tc_cnt; i++) {
+ for (j = 0; j < qptc; j++) {
+ netif_dbg(pdata, drv, pdata->netdev,
+ "TXq%u mapped to TC%u\n", queue, i);
+ regval = readl(XLGMAC_MTL_REG(pdata, queue,
+ MTL_Q_TQOMR));
+ regval = XLGMAC_SET_REG_BITS(regval,
+ MTL_Q_TQOMR_Q2TCMAP_POS,
+ MTL_Q_TQOMR_Q2TCMAP_LEN,
+ i);
+ writel(regval, XLGMAC_MTL_REG(pdata, queue,
+ MTL_Q_TQOMR));
+ queue++;
+ }
+
+ if (i < qptc_extra) {
+ netif_dbg(pdata, drv, pdata->netdev,
+ "TXq%u mapped to TC%u\n", queue, i);
+ regval = readl(XLGMAC_MTL_REG(pdata, queue,
+ MTL_Q_TQOMR));
+ regval = XLGMAC_SET_REG_BITS(regval,
+ MTL_Q_TQOMR_Q2TCMAP_POS,
+ MTL_Q_TQOMR_Q2TCMAP_LEN,
+ i);
+ writel(regval, XLGMAC_MTL_REG(pdata, queue,
+ MTL_Q_TQOMR));
+ queue++;
+ }
+ }
+
+ /* Map the 8 VLAN priority values to available MTL Rx queues */
+ prio_queues = min_t(unsigned int, IEEE_8021QAZ_MAX_TCS,
+ pdata->rx_q_count);
+ ppq = IEEE_8021QAZ_MAX_TCS / prio_queues;
+ ppq_extra = IEEE_8021QAZ_MAX_TCS % prio_queues;
+
+ reg = MAC_RQC2R;
+ regval = 0;
+ for (i = 0, prio = 0; i < prio_queues;) {
+ mask = 0;
+ for (j = 0; j < ppq; j++) {
+ netif_dbg(pdata, drv, pdata->netdev,
+ "PRIO%u mapped to RXq%u\n", prio, i);
+ mask |= (1 << prio);
+ prio++;
+ }
+
+ if (i < ppq_extra) {
+ netif_dbg(pdata, drv, pdata->netdev,
+ "PRIO%u mapped to RXq%u\n", prio, i);
+ mask |= (1 << prio);
+ prio++;
+ }
+
+ regval |= (mask << ((i++ % MAC_RQC2_Q_PER_REG) << 3));
+
+ if ((i % MAC_RQC2_Q_PER_REG) && (i != prio_queues))
+ continue;
+
+ writel(regval, pdata->mac_regs + reg);
+ reg += MAC_RQC2_INC;
+ regval = 0;
+ }
+
+ /* Configure one to one, MTL Rx queue to DMA Rx channel mapping
+ * ie Q0 <--> CH0, Q1 <--> CH1 ... Q11 <--> CH11
+ */
+ reg = MTL_RQDCM0R;
+ regval = readl(pdata->mac_regs + reg);
+ regval |= (MTL_RQDCM0R_Q0MDMACH | MTL_RQDCM0R_Q1MDMACH |
+ MTL_RQDCM0R_Q2MDMACH | MTL_RQDCM0R_Q3MDMACH);
+ writel(regval, pdata->mac_regs + reg);
+
+ reg += MTL_RQDCM_INC;
+ regval = readl(pdata->mac_regs + reg);
+ regval |= (MTL_RQDCM1R_Q4MDMACH | MTL_RQDCM1R_Q5MDMACH |
+ MTL_RQDCM1R_Q6MDMACH | MTL_RQDCM1R_Q7MDMACH);
+ writel(regval, pdata->mac_regs + reg);
+
+ reg += MTL_RQDCM_INC;
+ regval = readl(pdata->mac_regs + reg);
+ regval |= (MTL_RQDCM2R_Q8MDMACH | MTL_RQDCM2R_Q9MDMACH |
+ MTL_RQDCM2R_Q10MDMACH | MTL_RQDCM2R_Q11MDMACH);
+ writel(regval, pdata->mac_regs + reg);
+}
+
+static unsigned int xlgmac_calculate_per_queue_fifo(
+ unsigned int fifo_size,
+ unsigned int queue_count)
+{
+ unsigned int q_fifo_size;
+ unsigned int p_fifo;
+
+ /* Calculate the configured fifo size */
+ q_fifo_size = 1 << (fifo_size + 7);
+
+ /* The configured value may not be the actual amount of fifo RAM */
+ q_fifo_size = min_t(unsigned int, XLGMAC_MAX_FIFO, q_fifo_size);
+
+ q_fifo_size = q_fifo_size / queue_count;
+
+ /* Each increment in the queue fifo size represents 256 bytes of
+ * fifo, with 0 representing 256 bytes. Distribute the fifo equally
+ * between the queues.
+ */
+ p_fifo = q_fifo_size / 256;
+ if (p_fifo)
+ p_fifo--;
+
+ return p_fifo;
+}
+
+static void xlgmac_config_tx_fifo_size(struct xlgmac_pdata *pdata)
+{
+ unsigned int fifo_size;
+ unsigned int i;
+ u32 regval;
+
+ fifo_size = xlgmac_calculate_per_queue_fifo(
+ pdata->hw_feat.tx_fifo_size,
+ pdata->tx_q_count);
+
+ for (i = 0; i < pdata->tx_q_count; i++) {
+ regval = readl(XLGMAC_MTL_REG(pdata, i, MTL_Q_TQOMR));
+ regval = XLGMAC_SET_REG_BITS(regval, MTL_Q_TQOMR_TQS_POS,
+ MTL_Q_TQOMR_TQS_LEN, fifo_size);
+ writel(regval, XLGMAC_MTL_REG(pdata, i, MTL_Q_TQOMR));
+ }
+
+ netif_info(pdata, drv, pdata->netdev,
+ "%d Tx hardware queues, %d byte fifo per queue\n",
+ pdata->tx_q_count, ((fifo_size + 1) * 256));
+}
+
+static void xlgmac_config_rx_fifo_size(struct xlgmac_pdata *pdata)
+{
+ unsigned int fifo_size;
+ unsigned int i;
+ u32 regval;
+
+ fifo_size = xlgmac_calculate_per_queue_fifo(
+ pdata->hw_feat.rx_fifo_size,
+ pdata->rx_q_count);
+
+ for (i = 0; i < pdata->rx_q_count; i++) {
+ regval = readl(XLGMAC_MTL_REG(pdata, i, MTL_Q_RQOMR));
+ regval = XLGMAC_SET_REG_BITS(regval, MTL_Q_RQOMR_RQS_POS,
+ MTL_Q_RQOMR_RQS_LEN, fifo_size);
+ writel(regval, XLGMAC_MTL_REG(pdata, i, MTL_Q_RQOMR));
+ }
+
+ netif_info(pdata, drv, pdata->netdev,
+ "%d Rx hardware queues, %d byte fifo per queue\n",
+ pdata->rx_q_count, ((fifo_size + 1) * 256));
+}
+
+static void xlgmac_config_flow_control_threshold(struct xlgmac_pdata *pdata)
+{
+ unsigned int i;
+ u32 regval;
+
+ for (i = 0; i < pdata->rx_q_count; i++) {
+ regval = readl(XLGMAC_MTL_REG(pdata, i, MTL_Q_RQFCR));
+ /* Activate flow control when less than 4k left in fifo */
+ regval = XLGMAC_SET_REG_BITS(regval, MTL_Q_RQFCR_RFA_POS,
+ MTL_Q_RQFCR_RFA_LEN, 2);
+ /* De-activate flow control when more than 6k left in fifo */
+ regval = XLGMAC_SET_REG_BITS(regval, MTL_Q_RQFCR_RFD_POS,
+ MTL_Q_RQFCR_RFD_LEN, 4);
+ writel(regval, XLGMAC_MTL_REG(pdata, i, MTL_Q_RQFCR));
+ }
+}
+
+static int xlgmac_config_tx_threshold(struct xlgmac_pdata *pdata,
+ unsigned int val)
+{
+ unsigned int i;
+ u32 regval;
+
+ for (i = 0; i < pdata->tx_q_count; i++) {
+ regval = readl(XLGMAC_MTL_REG(pdata, i, MTL_Q_TQOMR));
+ regval = XLGMAC_SET_REG_BITS(regval, MTL_Q_TQOMR_TTC_POS,
+ MTL_Q_TQOMR_TTC_LEN, val);
+ writel(regval, XLGMAC_MTL_REG(pdata, i, MTL_Q_TQOMR));
+ }
+
+ return 0;
+}
+
+static int xlgmac_config_rsf_mode(struct xlgmac_pdata *pdata,
+ unsigned int val)
+{
+ unsigned int i;
+ u32 regval;
+
+ for (i = 0; i < pdata->rx_q_count; i++) {
+ regval = readl(XLGMAC_MTL_REG(pdata, i, MTL_Q_RQOMR));
+ regval = XLGMAC_SET_REG_BITS(regval, MTL_Q_RQOMR_RSF_POS,
+ MTL_Q_RQOMR_RSF_LEN, val);
+ writel(regval, XLGMAC_MTL_REG(pdata, i, MTL_Q_RQOMR));
+ }
+
+ return 0;
+}
+
+static int xlgmac_config_tsf_mode(struct xlgmac_pdata *pdata,
+ unsigned int val)
+{
+ unsigned int i;
+ u32 regval;
+
+ for (i = 0; i < pdata->tx_q_count; i++) {
+ regval = readl(XLGMAC_MTL_REG(pdata, i, MTL_Q_TQOMR));
+ regval = XLGMAC_SET_REG_BITS(regval, MTL_Q_TQOMR_TSF_POS,
+ MTL_Q_TQOMR_TSF_LEN, val);
+ writel(regval, XLGMAC_MTL_REG(pdata, i, MTL_Q_TQOMR));
+ }
+
+ return 0;
+}
+
+static int xlgmac_config_osp_mode(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_channel *channel;
+ unsigned int i;
+ u32 regval;
+
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ if (!channel->tx_ring)
+ break;
+
+ regval = readl(XLGMAC_DMA_REG(channel, DMA_CH_TCR));
+ regval = XLGMAC_SET_REG_BITS(regval, DMA_CH_TCR_OSP_POS,
+ DMA_CH_TCR_OSP_LEN,
+ pdata->tx_osp_mode);
+ writel(regval, XLGMAC_DMA_REG(channel, DMA_CH_TCR));
+ }
+
+ return 0;
+}
+
+static int xlgmac_config_pblx8(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_channel *channel;
+ unsigned int i;
+ u32 regval;
+
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ regval = readl(XLGMAC_DMA_REG(channel, DMA_CH_CR));
+ regval = XLGMAC_SET_REG_BITS(regval, DMA_CH_CR_PBLX8_POS,
+ DMA_CH_CR_PBLX8_LEN,
+ pdata->pblx8);
+ writel(regval, XLGMAC_DMA_REG(channel, DMA_CH_CR));
+ }
+
+ return 0;
+}
+
+static int xlgmac_get_tx_pbl_val(struct xlgmac_pdata *pdata)
+{
+ u32 regval;
+
+ regval = readl(XLGMAC_DMA_REG(pdata->channel_head, DMA_CH_TCR));
+ regval = XLGMAC_GET_REG_BITS(regval, DMA_CH_TCR_PBL_POS,
+ DMA_CH_TCR_PBL_LEN);
+ return regval;
+}
+
+static int xlgmac_config_tx_pbl_val(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_channel *channel;
+ unsigned int i;
+ u32 regval;
+
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ if (!channel->tx_ring)
+ break;
+
+ regval = readl(XLGMAC_DMA_REG(channel, DMA_CH_TCR));
+ regval = XLGMAC_SET_REG_BITS(regval, DMA_CH_TCR_PBL_POS,
+ DMA_CH_TCR_PBL_LEN,
+ pdata->tx_pbl);
+ writel(regval, XLGMAC_DMA_REG(channel, DMA_CH_TCR));
+ }
+
+ return 0;
+}
+
+static int xlgmac_get_rx_pbl_val(struct xlgmac_pdata *pdata)
+{
+ u32 regval;
+
+ regval = readl(XLGMAC_DMA_REG(pdata->channel_head, DMA_CH_RCR));
+ regval = XLGMAC_GET_REG_BITS(regval, DMA_CH_RCR_PBL_POS,
+ DMA_CH_RCR_PBL_LEN);
+ return regval;
+}
+
+static int xlgmac_config_rx_pbl_val(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_channel *channel;
+ unsigned int i;
+ u32 regval;
+
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ if (!channel->rx_ring)
+ break;
+
+ regval = readl(XLGMAC_DMA_REG(channel, DMA_CH_RCR));
+ regval = XLGMAC_SET_REG_BITS(regval, DMA_CH_RCR_PBL_POS,
+ DMA_CH_RCR_PBL_LEN,
+ pdata->rx_pbl);
+ writel(regval, XLGMAC_DMA_REG(channel, DMA_CH_RCR));
+ }
+
+ return 0;
+}
+
+static u64 xlgmac_mmc_read(struct xlgmac_pdata *pdata, unsigned int reg_lo)
+{
+ bool read_hi;
+ u64 val;
+
+ switch (reg_lo) {
+ /* These registers are always 64 bit */
+ case MMC_TXOCTETCOUNT_GB_LO:
+ case MMC_TXOCTETCOUNT_G_LO:
+ case MMC_RXOCTETCOUNT_GB_LO:
+ case MMC_RXOCTETCOUNT_G_LO:
+ read_hi = true;
+ break;
+
+ default:
+ read_hi = false;
+ }
+
+ val = (u64)readl(pdata->mac_regs + reg_lo);
+
+ if (read_hi)
+ val |= ((u64)readl(pdata->mac_regs + reg_lo + 4) << 32);
+
+ return val;
+}
+
+static void xlgmac_tx_mmc_int(struct xlgmac_pdata *pdata)
+{
+ unsigned int mmc_isr = readl(pdata->mac_regs + MMC_TISR);
+ struct xlgmac_stats *stats = &pdata->stats;
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_TISR_TXOCTETCOUNT_GB_POS,
+ MMC_TISR_TXOCTETCOUNT_GB_LEN))
+ stats->txoctetcount_gb +=
+ xlgmac_mmc_read(pdata, MMC_TXOCTETCOUNT_GB_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_TISR_TXFRAMECOUNT_GB_POS,
+ MMC_TISR_TXFRAMECOUNT_GB_LEN))
+ stats->txframecount_gb +=
+ xlgmac_mmc_read(pdata, MMC_TXFRAMECOUNT_GB_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_TISR_TXBROADCASTFRAMES_G_POS,
+ MMC_TISR_TXBROADCASTFRAMES_G_LEN))
+ stats->txbroadcastframes_g +=
+ xlgmac_mmc_read(pdata, MMC_TXBROADCASTFRAMES_G_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_TISR_TXMULTICASTFRAMES_G_POS,
+ MMC_TISR_TXMULTICASTFRAMES_G_LEN))
+ stats->txmulticastframes_g +=
+ xlgmac_mmc_read(pdata, MMC_TXMULTICASTFRAMES_G_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_TISR_TX64OCTETS_GB_POS,
+ MMC_TISR_TX64OCTETS_GB_LEN))
+ stats->tx64octets_gb +=
+ xlgmac_mmc_read(pdata, MMC_TX64OCTETS_GB_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_TISR_TX65TO127OCTETS_GB_POS,
+ MMC_TISR_TX65TO127OCTETS_GB_LEN))
+ stats->tx65to127octets_gb +=
+ xlgmac_mmc_read(pdata, MMC_TX65TO127OCTETS_GB_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_TISR_TX128TO255OCTETS_GB_POS,
+ MMC_TISR_TX128TO255OCTETS_GB_LEN))
+ stats->tx128to255octets_gb +=
+ xlgmac_mmc_read(pdata, MMC_TX128TO255OCTETS_GB_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_TISR_TX256TO511OCTETS_GB_POS,
+ MMC_TISR_TX256TO511OCTETS_GB_LEN))
+ stats->tx256to511octets_gb +=
+ xlgmac_mmc_read(pdata, MMC_TX256TO511OCTETS_GB_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_TISR_TX512TO1023OCTETS_GB_POS,
+ MMC_TISR_TX512TO1023OCTETS_GB_LEN))
+ stats->tx512to1023octets_gb +=
+ xlgmac_mmc_read(pdata, MMC_TX512TO1023OCTETS_GB_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_TISR_TX1024TOMAXOCTETS_GB_POS,
+ MMC_TISR_TX1024TOMAXOCTETS_GB_LEN))
+ stats->tx1024tomaxoctets_gb +=
+ xlgmac_mmc_read(pdata, MMC_TX1024TOMAXOCTETS_GB_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_TISR_TXUNICASTFRAMES_GB_POS,
+ MMC_TISR_TXUNICASTFRAMES_GB_LEN))
+ stats->txunicastframes_gb +=
+ xlgmac_mmc_read(pdata, MMC_TXUNICASTFRAMES_GB_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_TISR_TXMULTICASTFRAMES_GB_POS,
+ MMC_TISR_TXMULTICASTFRAMES_GB_LEN))
+ stats->txmulticastframes_gb +=
+ xlgmac_mmc_read(pdata, MMC_TXMULTICASTFRAMES_GB_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_TISR_TXBROADCASTFRAMES_GB_POS,
+ MMC_TISR_TXBROADCASTFRAMES_GB_LEN))
+ stats->txbroadcastframes_g +=
+ xlgmac_mmc_read(pdata, MMC_TXBROADCASTFRAMES_GB_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_TISR_TXUNDERFLOWERROR_POS,
+ MMC_TISR_TXUNDERFLOWERROR_LEN))
+ stats->txunderflowerror +=
+ xlgmac_mmc_read(pdata, MMC_TXUNDERFLOWERROR_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_TISR_TXOCTETCOUNT_G_POS,
+ MMC_TISR_TXOCTETCOUNT_G_LEN))
+ stats->txoctetcount_g +=
+ xlgmac_mmc_read(pdata, MMC_TXOCTETCOUNT_G_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_TISR_TXFRAMECOUNT_G_POS,
+ MMC_TISR_TXFRAMECOUNT_G_LEN))
+ stats->txframecount_g +=
+ xlgmac_mmc_read(pdata, MMC_TXFRAMECOUNT_G_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_TISR_TXPAUSEFRAMES_POS,
+ MMC_TISR_TXPAUSEFRAMES_LEN))
+ stats->txpauseframes +=
+ xlgmac_mmc_read(pdata, MMC_TXPAUSEFRAMES_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_TISR_TXVLANFRAMES_G_POS,
+ MMC_TISR_TXVLANFRAMES_G_LEN))
+ stats->txvlanframes_g +=
+ xlgmac_mmc_read(pdata, MMC_TXVLANFRAMES_G_LO);
+}
+
+static void xlgmac_rx_mmc_int(struct xlgmac_pdata *pdata)
+{
+ unsigned int mmc_isr = readl(pdata->mac_regs + MMC_RISR);
+ struct xlgmac_stats *stats = &pdata->stats;
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_RISR_RXFRAMECOUNT_GB_POS,
+ MMC_RISR_RXFRAMECOUNT_GB_LEN))
+ stats->rxframecount_gb +=
+ xlgmac_mmc_read(pdata, MMC_RXFRAMECOUNT_GB_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_RISR_RXOCTETCOUNT_GB_POS,
+ MMC_RISR_RXOCTETCOUNT_GB_LEN))
+ stats->rxoctetcount_gb +=
+ xlgmac_mmc_read(pdata, MMC_RXOCTETCOUNT_GB_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_RISR_RXOCTETCOUNT_G_POS,
+ MMC_RISR_RXOCTETCOUNT_G_LEN))
+ stats->rxoctetcount_g +=
+ xlgmac_mmc_read(pdata, MMC_RXOCTETCOUNT_G_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_RISR_RXBROADCASTFRAMES_G_POS,
+ MMC_RISR_RXBROADCASTFRAMES_G_LEN))
+ stats->rxbroadcastframes_g +=
+ xlgmac_mmc_read(pdata, MMC_RXBROADCASTFRAMES_G_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_RISR_RXMULTICASTFRAMES_G_POS,
+ MMC_RISR_RXMULTICASTFRAMES_G_LEN))
+ stats->rxmulticastframes_g +=
+ xlgmac_mmc_read(pdata, MMC_RXMULTICASTFRAMES_G_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_RISR_RXCRCERROR_POS,
+ MMC_RISR_RXCRCERROR_LEN))
+ stats->rxcrcerror +=
+ xlgmac_mmc_read(pdata, MMC_RXCRCERROR_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_RISR_RXRUNTERROR_POS,
+ MMC_RISR_RXRUNTERROR_LEN))
+ stats->rxrunterror +=
+ xlgmac_mmc_read(pdata, MMC_RXRUNTERROR);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_RISR_RXJABBERERROR_POS,
+ MMC_RISR_RXJABBERERROR_LEN))
+ stats->rxjabbererror +=
+ xlgmac_mmc_read(pdata, MMC_RXJABBERERROR);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_RISR_RXUNDERSIZE_G_POS,
+ MMC_RISR_RXUNDERSIZE_G_LEN))
+ stats->rxundersize_g +=
+ xlgmac_mmc_read(pdata, MMC_RXUNDERSIZE_G);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_RISR_RXOVERSIZE_G_POS,
+ MMC_RISR_RXOVERSIZE_G_LEN))
+ stats->rxoversize_g +=
+ xlgmac_mmc_read(pdata, MMC_RXOVERSIZE_G);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_RISR_RX64OCTETS_GB_POS,
+ MMC_RISR_RX64OCTETS_GB_LEN))
+ stats->rx64octets_gb +=
+ xlgmac_mmc_read(pdata, MMC_RX64OCTETS_GB_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_RISR_RX65TO127OCTETS_GB_POS,
+ MMC_RISR_RX65TO127OCTETS_GB_LEN))
+ stats->rx65to127octets_gb +=
+ xlgmac_mmc_read(pdata, MMC_RX65TO127OCTETS_GB_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_RISR_RX128TO255OCTETS_GB_POS,
+ MMC_RISR_RX128TO255OCTETS_GB_LEN))
+ stats->rx128to255octets_gb +=
+ xlgmac_mmc_read(pdata, MMC_RX128TO255OCTETS_GB_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_RISR_RX256TO511OCTETS_GB_POS,
+ MMC_RISR_RX256TO511OCTETS_GB_LEN))
+ stats->rx256to511octets_gb +=
+ xlgmac_mmc_read(pdata, MMC_RX256TO511OCTETS_GB_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_RISR_RX512TO1023OCTETS_GB_POS,
+ MMC_RISR_RX512TO1023OCTETS_GB_LEN))
+ stats->rx512to1023octets_gb +=
+ xlgmac_mmc_read(pdata, MMC_RX512TO1023OCTETS_GB_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_RISR_RX1024TOMAXOCTETS_GB_POS,
+ MMC_RISR_RX1024TOMAXOCTETS_GB_LEN))
+ stats->rx1024tomaxoctets_gb +=
+ xlgmac_mmc_read(pdata, MMC_RX1024TOMAXOCTETS_GB_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_RISR_RXUNICASTFRAMES_G_POS,
+ MMC_RISR_RXUNICASTFRAMES_G_LEN))
+ stats->rxunicastframes_g +=
+ xlgmac_mmc_read(pdata, MMC_RXUNICASTFRAMES_G_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_RISR_RXLENGTHERROR_POS,
+ MMC_RISR_RXLENGTHERROR_LEN))
+ stats->rxlengtherror +=
+ xlgmac_mmc_read(pdata, MMC_RXLENGTHERROR_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_RISR_RXOUTOFRANGETYPE_POS,
+ MMC_RISR_RXOUTOFRANGETYPE_LEN))
+ stats->rxoutofrangetype +=
+ xlgmac_mmc_read(pdata, MMC_RXOUTOFRANGETYPE_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_RISR_RXPAUSEFRAMES_POS,
+ MMC_RISR_RXPAUSEFRAMES_LEN))
+ stats->rxpauseframes +=
+ xlgmac_mmc_read(pdata, MMC_RXPAUSEFRAMES_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_RISR_RXFIFOOVERFLOW_POS,
+ MMC_RISR_RXFIFOOVERFLOW_LEN))
+ stats->rxfifooverflow +=
+ xlgmac_mmc_read(pdata, MMC_RXFIFOOVERFLOW_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_RISR_RXVLANFRAMES_GB_POS,
+ MMC_RISR_RXVLANFRAMES_GB_LEN))
+ stats->rxvlanframes_gb +=
+ xlgmac_mmc_read(pdata, MMC_RXVLANFRAMES_GB_LO);
+
+ if (XLGMAC_GET_REG_BITS(mmc_isr,
+ MMC_RISR_RXWATCHDOGERROR_POS,
+ MMC_RISR_RXWATCHDOGERROR_LEN))
+ stats->rxwatchdogerror +=
+ xlgmac_mmc_read(pdata, MMC_RXWATCHDOGERROR);
+}
+
+static void xlgmac_read_mmc_stats(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_stats *stats = &pdata->stats;
+ u32 regval;
+
+ /* Freeze counters */
+ regval = readl(pdata->mac_regs + MMC_CR);
+ regval = XLGMAC_SET_REG_BITS(regval, MMC_CR_MCF_POS,
+ MMC_CR_MCF_LEN, 1);
+ writel(regval, pdata->mac_regs + MMC_CR);
+
+ stats->txoctetcount_gb +=
+ xlgmac_mmc_read(pdata, MMC_TXOCTETCOUNT_GB_LO);
+
+ stats->txframecount_gb +=
+ xlgmac_mmc_read(pdata, MMC_TXFRAMECOUNT_GB_LO);
+
+ stats->txbroadcastframes_g +=
+ xlgmac_mmc_read(pdata, MMC_TXBROADCASTFRAMES_G_LO);
+
+ stats->txmulticastframes_g +=
+ xlgmac_mmc_read(pdata, MMC_TXMULTICASTFRAMES_G_LO);
+
+ stats->tx64octets_gb +=
+ xlgmac_mmc_read(pdata, MMC_TX64OCTETS_GB_LO);
+
+ stats->tx65to127octets_gb +=
+ xlgmac_mmc_read(pdata, MMC_TX65TO127OCTETS_GB_LO);
+
+ stats->tx128to255octets_gb +=
+ xlgmac_mmc_read(pdata, MMC_TX128TO255OCTETS_GB_LO);
+
+ stats->tx256to511octets_gb +=
+ xlgmac_mmc_read(pdata, MMC_TX256TO511OCTETS_GB_LO);
+
+ stats->tx512to1023octets_gb +=
+ xlgmac_mmc_read(pdata, MMC_TX512TO1023OCTETS_GB_LO);
+
+ stats->tx1024tomaxoctets_gb +=
+ xlgmac_mmc_read(pdata, MMC_TX1024TOMAXOCTETS_GB_LO);
+
+ stats->txunicastframes_gb +=
+ xlgmac_mmc_read(pdata, MMC_TXUNICASTFRAMES_GB_LO);
+
+ stats->txmulticastframes_gb +=
+ xlgmac_mmc_read(pdata, MMC_TXMULTICASTFRAMES_GB_LO);
+
+ stats->txbroadcastframes_g +=
+ xlgmac_mmc_read(pdata, MMC_TXBROADCASTFRAMES_GB_LO);
+
+ stats->txunderflowerror +=
+ xlgmac_mmc_read(pdata, MMC_TXUNDERFLOWERROR_LO);
+
+ stats->txoctetcount_g +=
+ xlgmac_mmc_read(pdata, MMC_TXOCTETCOUNT_G_LO);
+
+ stats->txframecount_g +=
+ xlgmac_mmc_read(pdata, MMC_TXFRAMECOUNT_G_LO);
+
+ stats->txpauseframes +=
+ xlgmac_mmc_read(pdata, MMC_TXPAUSEFRAMES_LO);
+
+ stats->txvlanframes_g +=
+ xlgmac_mmc_read(pdata, MMC_TXVLANFRAMES_G_LO);
+
+ stats->rxframecount_gb +=
+ xlgmac_mmc_read(pdata, MMC_RXFRAMECOUNT_GB_LO);
+
+ stats->rxoctetcount_gb +=
+ xlgmac_mmc_read(pdata, MMC_RXOCTETCOUNT_GB_LO);
+
+ stats->rxoctetcount_g +=
+ xlgmac_mmc_read(pdata, MMC_RXOCTETCOUNT_G_LO);
+
+ stats->rxbroadcastframes_g +=
+ xlgmac_mmc_read(pdata, MMC_RXBROADCASTFRAMES_G_LO);
+
+ stats->rxmulticastframes_g +=
+ xlgmac_mmc_read(pdata, MMC_RXMULTICASTFRAMES_G_LO);
+
+ stats->rxcrcerror +=
+ xlgmac_mmc_read(pdata, MMC_RXCRCERROR_LO);
+
+ stats->rxrunterror +=
+ xlgmac_mmc_read(pdata, MMC_RXRUNTERROR);
+
+ stats->rxjabbererror +=
+ xlgmac_mmc_read(pdata, MMC_RXJABBERERROR);
+
+ stats->rxundersize_g +=
+ xlgmac_mmc_read(pdata, MMC_RXUNDERSIZE_G);
+
+ stats->rxoversize_g +=
+ xlgmac_mmc_read(pdata, MMC_RXOVERSIZE_G);
+
+ stats->rx64octets_gb +=
+ xlgmac_mmc_read(pdata, MMC_RX64OCTETS_GB_LO);
+
+ stats->rx65to127octets_gb +=
+ xlgmac_mmc_read(pdata, MMC_RX65TO127OCTETS_GB_LO);
+
+ stats->rx128to255octets_gb +=
+ xlgmac_mmc_read(pdata, MMC_RX128TO255OCTETS_GB_LO);
+
+ stats->rx256to511octets_gb +=
+ xlgmac_mmc_read(pdata, MMC_RX256TO511OCTETS_GB_LO);
+
+ stats->rx512to1023octets_gb +=
+ xlgmac_mmc_read(pdata, MMC_RX512TO1023OCTETS_GB_LO);
+
+ stats->rx1024tomaxoctets_gb +=
+ xlgmac_mmc_read(pdata, MMC_RX1024TOMAXOCTETS_GB_LO);
+
+ stats->rxunicastframes_g +=
+ xlgmac_mmc_read(pdata, MMC_RXUNICASTFRAMES_G_LO);
+
+ stats->rxlengtherror +=
+ xlgmac_mmc_read(pdata, MMC_RXLENGTHERROR_LO);
+
+ stats->rxoutofrangetype +=
+ xlgmac_mmc_read(pdata, MMC_RXOUTOFRANGETYPE_LO);
+
+ stats->rxpauseframes +=
+ xlgmac_mmc_read(pdata, MMC_RXPAUSEFRAMES_LO);
+
+ stats->rxfifooverflow +=
+ xlgmac_mmc_read(pdata, MMC_RXFIFOOVERFLOW_LO);
+
+ stats->rxvlanframes_gb +=
+ xlgmac_mmc_read(pdata, MMC_RXVLANFRAMES_GB_LO);
+
+ stats->rxwatchdogerror +=
+ xlgmac_mmc_read(pdata, MMC_RXWATCHDOGERROR);
+
+ /* Un-freeze counters */
+ regval = readl(pdata->mac_regs + MMC_CR);
+ regval = XLGMAC_SET_REG_BITS(regval, MMC_CR_MCF_POS,
+ MMC_CR_MCF_LEN, 0);
+ writel(regval, pdata->mac_regs + MMC_CR);
+}
+
+static void xlgmac_config_mmc(struct xlgmac_pdata *pdata)
+{
+ u32 regval;
+
+ regval = readl(pdata->mac_regs + MMC_CR);
+ /* Set counters to reset on read */
+ regval = XLGMAC_SET_REG_BITS(regval, MMC_CR_ROR_POS,
+ MMC_CR_ROR_LEN, 1);
+ /* Reset the counters */
+ regval = XLGMAC_SET_REG_BITS(regval, MMC_CR_CR_POS,
+ MMC_CR_CR_LEN, 1);
+ writel(regval, pdata->mac_regs + MMC_CR);
+}
+
+static int xlgmac_write_rss_reg(struct xlgmac_pdata *pdata, unsigned int type,
+ unsigned int index, unsigned int val)
+{
+ unsigned int wait;
+ int ret = 0;
+ u32 regval;
+
+ mutex_lock(&pdata->rss_mutex);
+
+ regval = XLGMAC_GET_REG_BITS(readl(pdata->mac_regs + MAC_RSSAR),
+ MAC_RSSAR_OB_POS, MAC_RSSAR_OB_LEN);
+ if (regval) {
+ ret = -EBUSY;
+ goto unlock;
+ }
+
+ writel(val, pdata->mac_regs + MAC_RSSDR);
+
+ regval = readl(pdata->mac_regs + MAC_RSSAR);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_RSSAR_RSSIA_POS,
+ MAC_RSSAR_RSSIA_LEN, index);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_RSSAR_ADDRT_POS,
+ MAC_RSSAR_ADDRT_LEN, type);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_RSSAR_CT_POS,
+ MAC_RSSAR_CT_LEN, 0);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_RSSAR_OB_POS,
+ MAC_RSSAR_OB_LEN, 1);
+ writel(regval, pdata->mac_regs + MAC_RSSAR);
+
+ wait = 1000;
+ while (wait--) {
+ regval = XLGMAC_GET_REG_BITS(readl(pdata->mac_regs + MAC_RSSAR),
+ MAC_RSSAR_OB_POS,
+ MAC_RSSAR_OB_LEN);
+ if (!regval)
+ goto unlock;
+
+ usleep_range(1000, 1500);
+ }
+
+ ret = -EBUSY;
+
+unlock:
+ mutex_unlock(&pdata->rss_mutex);
+
+ return ret;
+}
+
+static int xlgmac_write_rss_hash_key(struct xlgmac_pdata *pdata)
+{
+ unsigned int key_regs = sizeof(pdata->rss_key) / sizeof(u32);
+ unsigned int *key = (unsigned int *)&pdata->rss_key;
+ int ret;
+
+ while (key_regs--) {
+ ret = xlgmac_write_rss_reg(pdata, XLGMAC_RSS_HASH_KEY_TYPE,
+ key_regs, *key++);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
+static int xlgmac_write_rss_lookup_table(struct xlgmac_pdata *pdata)
+{
+ unsigned int i;
+ int ret;
+
+ for (i = 0; i < ARRAY_SIZE(pdata->rss_table); i++) {
+ ret = xlgmac_write_rss_reg(pdata,
+ XLGMAC_RSS_LOOKUP_TABLE_TYPE, i,
+ pdata->rss_table[i]);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
+static int xlgmac_set_rss_hash_key(struct xlgmac_pdata *pdata, const u8 *key)
+{
+ memcpy(pdata->rss_key, key, sizeof(pdata->rss_key));
+
+ return xlgmac_write_rss_hash_key(pdata);
+}
+
+static int xlgmac_set_rss_lookup_table(struct xlgmac_pdata *pdata,
+ const u32 *table)
+{
+ unsigned int i;
+ u32 tval;
+
+ for (i = 0; i < ARRAY_SIZE(pdata->rss_table); i++) {
+ tval = table[i];
+ pdata->rss_table[i] = XLGMAC_SET_REG_BITS(
+ pdata->rss_table[i],
+ MAC_RSSDR_DMCH_POS,
+ MAC_RSSDR_DMCH_LEN,
+ tval);
+ }
+
+ return xlgmac_write_rss_lookup_table(pdata);
+}
+
+static int xlgmac_enable_rss(struct xlgmac_pdata *pdata)
+{
+ u32 regval;
+ int ret;
+
+ if (!pdata->hw_feat.rss)
+ return -EOPNOTSUPP;
+
+ /* Program the hash key */
+ ret = xlgmac_write_rss_hash_key(pdata);
+ if (ret)
+ return ret;
+
+ /* Program the lookup table */
+ ret = xlgmac_write_rss_lookup_table(pdata);
+ if (ret)
+ return ret;
+
+ /* Set the RSS options */
+ writel(pdata->rss_options, pdata->mac_regs + MAC_RSSCR);
+
+ /* Enable RSS */
+ regval = readl(pdata->mac_regs + MAC_RSSCR);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_RSSCR_RSSE_POS,
+ MAC_RSSCR_RSSE_LEN, 1);
+ writel(regval, pdata->mac_regs + MAC_RSSCR);
+
+ return 0;
+}
+
+static int xlgmac_disable_rss(struct xlgmac_pdata *pdata)
+{
+ u32 regval;
+
+ if (!pdata->hw_feat.rss)
+ return -EOPNOTSUPP;
+
+ regval = readl(pdata->mac_regs + MAC_RSSCR);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_RSSCR_RSSE_POS,
+ MAC_RSSCR_RSSE_LEN, 0);
+ writel(regval, pdata->mac_regs + MAC_RSSCR);
+
+ return 0;
+}
+
+static void xlgmac_config_rss(struct xlgmac_pdata *pdata)
+{
+ int ret;
+
+ if (!pdata->hw_feat.rss)
+ return;
+
+ if (pdata->netdev->features & NETIF_F_RXHASH)
+ ret = xlgmac_enable_rss(pdata);
+ else
+ ret = xlgmac_disable_rss(pdata);
+
+ if (ret)
+ netdev_err(pdata->netdev,
+ "error configuring RSS, RSS disabled\n");
+}
+
+static void xlgmac_enable_dma_interrupts(struct xlgmac_pdata *pdata)
+{
+ unsigned int dma_ch_isr, dma_ch_ier;
+ struct xlgmac_channel *channel;
+ unsigned int i;
+
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ /* Clear all the interrupts which are set */
+ dma_ch_isr = readl(XLGMAC_DMA_REG(channel, DMA_CH_SR));
+ writel(dma_ch_isr, XLGMAC_DMA_REG(channel, DMA_CH_SR));
+
+ /* Clear all interrupt enable bits */
+ dma_ch_ier = 0;
+
+ /* Enable following interrupts
+ * NIE - Normal Interrupt Summary Enable
+ * AIE - Abnormal Interrupt Summary Enable
+ * FBEE - Fatal Bus Error Enable
+ */
+ dma_ch_ier = XLGMAC_SET_REG_BITS(dma_ch_ier,
+ DMA_CH_IER_NIE_POS,
+ DMA_CH_IER_NIE_LEN, 1);
+ dma_ch_ier = XLGMAC_SET_REG_BITS(dma_ch_ier,
+ DMA_CH_IER_AIE_POS,
+ DMA_CH_IER_AIE_LEN, 1);
+ dma_ch_ier = XLGMAC_SET_REG_BITS(dma_ch_ier,
+ DMA_CH_IER_FBEE_POS,
+ DMA_CH_IER_FBEE_LEN, 1);
+
+ if (channel->tx_ring) {
+ /* Enable the following Tx interrupts
+ * TIE - Transmit Interrupt Enable (unless using
+ * per channel interrupts)
+ */
+ if (!pdata->per_channel_irq)
+ dma_ch_ier = XLGMAC_SET_REG_BITS(
+ dma_ch_ier,
+ DMA_CH_IER_TIE_POS,
+ DMA_CH_IER_TIE_LEN,
+ 1);
+ }
+ if (channel->rx_ring) {
+ /* Enable following Rx interrupts
+ * RBUE - Receive Buffer Unavailable Enable
+ * RIE - Receive Interrupt Enable (unless using
+ * per channel interrupts)
+ */
+ dma_ch_ier = XLGMAC_SET_REG_BITS(
+ dma_ch_ier,
+ DMA_CH_IER_RBUE_POS,
+ DMA_CH_IER_RBUE_LEN,
+ 1);
+ if (!pdata->per_channel_irq)
+ dma_ch_ier = XLGMAC_SET_REG_BITS(
+ dma_ch_ier,
+ DMA_CH_IER_RIE_POS,
+ DMA_CH_IER_RIE_LEN,
+ 1);
+ }
+
+ writel(dma_ch_isr, XLGMAC_DMA_REG(channel, DMA_CH_IER));
+ }
+}
+
+static void xlgmac_enable_mtl_interrupts(struct xlgmac_pdata *pdata)
+{
+ unsigned int q_count, i;
+ unsigned int mtl_q_isr;
+
+ q_count = max(pdata->hw_feat.tx_q_cnt, pdata->hw_feat.rx_q_cnt);
+ for (i = 0; i < q_count; i++) {
+ /* Clear all the interrupts which are set */
+ mtl_q_isr = readl(XLGMAC_MTL_REG(pdata, i, MTL_Q_ISR));
+ writel(mtl_q_isr, XLGMAC_MTL_REG(pdata, i, MTL_Q_ISR));
+
+ /* No MTL interrupts to be enabled */
+ writel(0, XLGMAC_MTL_REG(pdata, i, MTL_Q_IER));
+ }
+}
+
+static void xlgmac_enable_mac_interrupts(struct xlgmac_pdata *pdata)
+{
+ unsigned int mac_ier = 0;
+ u32 regval;
+
+ /* Enable Timestamp interrupt */
+ mac_ier = XLGMAC_SET_REG_BITS(mac_ier, MAC_IER_TSIE_POS,
+ MAC_IER_TSIE_LEN, 1);
+
+ writel(mac_ier, pdata->mac_regs + MAC_IER);
+
+ /* Enable all counter interrupts */
+ regval = readl(pdata->mac_regs + MMC_RIER);
+ regval = XLGMAC_SET_REG_BITS(regval, MMC_RIER_ALL_INTERRUPTS_POS,
+ MMC_RIER_ALL_INTERRUPTS_LEN, 0xffffffff);
+ writel(regval, pdata->mac_regs + MMC_RIER);
+ regval = readl(pdata->mac_regs + MMC_TIER);
+ regval = XLGMAC_SET_REG_BITS(regval, MMC_TIER_ALL_INTERRUPTS_POS,
+ MMC_TIER_ALL_INTERRUPTS_LEN, 0xffffffff);
+ writel(regval, pdata->mac_regs + MMC_TIER);
+}
+
+static int xlgmac_set_xlgmii_25000_speed(struct xlgmac_pdata *pdata)
+{
+ u32 regval;
+
+ regval = XLGMAC_GET_REG_BITS(readl(pdata->mac_regs + MAC_TCR),
+ MAC_TCR_SS_POS, MAC_TCR_SS_LEN);
+ if (regval == 0x1)
+ return 0;
+
+ regval = readl(pdata->mac_regs + MAC_TCR);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_TCR_SS_POS,
+ MAC_TCR_SS_LEN, 0x1);
+ writel(regval, pdata->mac_regs + MAC_TCR);
+
+ return 0;
+}
+
+static int xlgmac_set_xlgmii_40000_speed(struct xlgmac_pdata *pdata)
+{
+ u32 regval;
+
+ regval = XLGMAC_GET_REG_BITS(readl(pdata->mac_regs + MAC_TCR),
+ MAC_TCR_SS_POS, MAC_TCR_SS_LEN);
+ if (regval == 0)
+ return 0;
+
+ regval = readl(pdata->mac_regs + MAC_TCR);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_TCR_SS_POS,
+ MAC_TCR_SS_LEN, 0);
+ writel(regval, pdata->mac_regs + MAC_TCR);
+
+ return 0;
+}
+
+static int xlgmac_set_xlgmii_50000_speed(struct xlgmac_pdata *pdata)
+{
+ u32 regval;
+
+ regval = XLGMAC_GET_REG_BITS(readl(pdata->mac_regs + MAC_TCR),
+ MAC_TCR_SS_POS, MAC_TCR_SS_LEN);
+ if (regval == 0x2)
+ return 0;
+
+ regval = readl(pdata->mac_regs + MAC_TCR);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_TCR_SS_POS,
+ MAC_TCR_SS_LEN, 0x2);
+ writel(regval, pdata->mac_regs + MAC_TCR);
+
+ return 0;
+}
+
+static int xlgmac_set_xlgmii_100000_speed(struct xlgmac_pdata *pdata)
+{
+ u32 regval;
+
+ regval = XLGMAC_GET_REG_BITS(readl(pdata->mac_regs + MAC_TCR),
+ MAC_TCR_SS_POS, MAC_TCR_SS_LEN);
+ if (regval == 0x3)
+ return 0;
+
+ regval = readl(pdata->mac_regs + MAC_TCR);
+ regval = XLGMAC_SET_REG_BITS(regval, MAC_TCR_SS_POS,
+ MAC_TCR_SS_LEN, 0x3);
+ writel(regval, pdata->mac_regs + MAC_TCR);
+
+ return 0;
+}
+
+static void xlgmac_config_mac_speed(struct xlgmac_pdata *pdata)
+{
+ switch (pdata->phy_speed) {
+ case SPEED_100000:
+ xlgmac_set_xlgmii_100000_speed(pdata);
+ break;
+
+ case SPEED_50000:
+ xlgmac_set_xlgmii_50000_speed(pdata);
+ break;
+
+ case SPEED_40000:
+ xlgmac_set_xlgmii_40000_speed(pdata);
+ break;
+
+ case SPEED_25000:
+ xlgmac_set_xlgmii_25000_speed(pdata);
+ break;
+ }
+}
+
+static int xlgmac_dev_read(struct xlgmac_channel *channel)
+{
+ struct xlgmac_pdata *pdata = channel->pdata;
+ struct xlgmac_ring *ring = channel->rx_ring;
+ struct net_device *netdev = pdata->netdev;
+ struct xlgmac_desc_data *desc_data;
+ struct xlgmac_dma_desc *dma_desc;
+ struct xlgmac_pkt_info *pkt_info;
+ unsigned int err, etlt, l34t;
+
+ desc_data = XLGMAC_GET_DESC_DATA(ring, ring->cur);
+ dma_desc = desc_data->dma_desc;
+ pkt_info = &ring->pkt_info;
+
+ /* Check for data availability */
+ if (XLGMAC_GET_REG_BITS_LE(dma_desc->desc3,
+ RX_NORMAL_DESC3_OWN_POS,
+ RX_NORMAL_DESC3_OWN_LEN))
+ return 1;
+
+ /* Make sure descriptor fields are read after reading the OWN bit */
+ dma_rmb();
+
+ if (netif_msg_rx_status(pdata))
+ xlgmac_dump_rx_desc(pdata, ring, ring->cur);
+
+ if (XLGMAC_GET_REG_BITS_LE(dma_desc->desc3,
+ RX_NORMAL_DESC3_CTXT_POS,
+ RX_NORMAL_DESC3_CTXT_LEN)) {
+ /* Timestamp Context Descriptor */
+ xlgmac_get_rx_tstamp(pkt_info, dma_desc);
+
+ pkt_info->attributes = XLGMAC_SET_REG_BITS(
+ pkt_info->attributes,
+ RX_PACKET_ATTRIBUTES_CONTEXT_POS,
+ RX_PACKET_ATTRIBUTES_CONTEXT_LEN,
+ 1);
+ pkt_info->attributes = XLGMAC_SET_REG_BITS(
+ pkt_info->attributes,
+ RX_PACKET_ATTRIBUTES_CONTEXT_NEXT_POS,
+ RX_PACKET_ATTRIBUTES_CONTEXT_NEXT_LEN,
+ 0);
+ return 0;
+ }
+
+ /* Normal Descriptor, be sure Context Descriptor bit is off */
+ pkt_info->attributes = XLGMAC_SET_REG_BITS(
+ pkt_info->attributes,
+ RX_PACKET_ATTRIBUTES_CONTEXT_POS,
+ RX_PACKET_ATTRIBUTES_CONTEXT_LEN,
+ 0);
+
+ /* Indicate if a Context Descriptor is next */
+ if (XLGMAC_GET_REG_BITS_LE(dma_desc->desc3,
+ RX_NORMAL_DESC3_CDA_POS,
+ RX_NORMAL_DESC3_CDA_LEN))
+ pkt_info->attributes = XLGMAC_SET_REG_BITS(
+ pkt_info->attributes,
+ RX_PACKET_ATTRIBUTES_CONTEXT_NEXT_POS,
+ RX_PACKET_ATTRIBUTES_CONTEXT_NEXT_LEN,
+ 1);
+
+ /* Get the header length */
+ if (XLGMAC_GET_REG_BITS_LE(dma_desc->desc3,
+ RX_NORMAL_DESC3_FD_POS,
+ RX_NORMAL_DESC3_FD_LEN)) {
+ desc_data->rx.hdr_len = XLGMAC_GET_REG_BITS_LE(dma_desc->desc2,
+ RX_NORMAL_DESC2_HL_POS,
+ RX_NORMAL_DESC2_HL_LEN);
+ if (desc_data->rx.hdr_len)
+ pdata->stats.rx_split_header_packets++;
+ }
+
+ /* Get the RSS hash */
+ if (XLGMAC_GET_REG_BITS_LE(dma_desc->desc3,
+ RX_NORMAL_DESC3_RSV_POS,
+ RX_NORMAL_DESC3_RSV_LEN)) {
+ pkt_info->attributes = XLGMAC_SET_REG_BITS(
+ pkt_info->attributes,
+ RX_PACKET_ATTRIBUTES_RSS_HASH_POS,
+ RX_PACKET_ATTRIBUTES_RSS_HASH_LEN,
+ 1);
+
+ pkt_info->rss_hash = le32_to_cpu(dma_desc->desc1);
+
+ l34t = XLGMAC_GET_REG_BITS_LE(dma_desc->desc3,
+ RX_NORMAL_DESC3_L34T_POS,
+ RX_NORMAL_DESC3_L34T_LEN);
+ switch (l34t) {
+ case RX_DESC3_L34T_IPV4_TCP:
+ case RX_DESC3_L34T_IPV4_UDP:
+ case RX_DESC3_L34T_IPV6_TCP:
+ case RX_DESC3_L34T_IPV6_UDP:
+ pkt_info->rss_hash_type = PKT_HASH_TYPE_L4;
+ break;
+ default:
+ pkt_info->rss_hash_type = PKT_HASH_TYPE_L3;
+ }
+ }
+
+ /* Get the pkt_info length */
+ desc_data->rx.len = XLGMAC_GET_REG_BITS_LE(dma_desc->desc3,
+ RX_NORMAL_DESC3_PL_POS,
+ RX_NORMAL_DESC3_PL_LEN);
+
+ if (!XLGMAC_GET_REG_BITS_LE(dma_desc->desc3,
+ RX_NORMAL_DESC3_LD_POS,
+ RX_NORMAL_DESC3_LD_LEN)) {
+ /* Not all the data has been transferred for this pkt_info */
+ pkt_info->attributes = XLGMAC_SET_REG_BITS(
+ pkt_info->attributes,
+ RX_PACKET_ATTRIBUTES_INCOMPLETE_POS,
+ RX_PACKET_ATTRIBUTES_INCOMPLETE_LEN,
+ 1);
+ return 0;
+ }
+
+ /* This is the last of the data for this pkt_info */
+ pkt_info->attributes = XLGMAC_SET_REG_BITS(
+ pkt_info->attributes,
+ RX_PACKET_ATTRIBUTES_INCOMPLETE_POS,
+ RX_PACKET_ATTRIBUTES_INCOMPLETE_LEN,
+ 0);
+
+ /* Set checksum done indicator as appropriate */
+ if (netdev->features & NETIF_F_RXCSUM)
+ pkt_info->attributes = XLGMAC_SET_REG_BITS(
+ pkt_info->attributes,
+ RX_PACKET_ATTRIBUTES_CSUM_DONE_POS,
+ RX_PACKET_ATTRIBUTES_CSUM_DONE_LEN,
+ 1);
+
+ /* Check for errors (only valid in last descriptor) */
+ err = XLGMAC_GET_REG_BITS_LE(dma_desc->desc3,
+ RX_NORMAL_DESC3_ES_POS,
+ RX_NORMAL_DESC3_ES_LEN);
+ etlt = XLGMAC_GET_REG_BITS_LE(dma_desc->desc3,
+ RX_NORMAL_DESC3_ETLT_POS,
+ RX_NORMAL_DESC3_ETLT_LEN);
+ netif_dbg(pdata, rx_status, netdev, "err=%u, etlt=%#x\n", err, etlt);
+
+ if (!err || !etlt) {
+ /* No error if err is 0 or etlt is 0 */
+ if ((etlt == 0x09) &&
+ (netdev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
+ pkt_info->attributes = XLGMAC_SET_REG_BITS(
+ pkt_info->attributes,
+ RX_PACKET_ATTRIBUTES_VLAN_CTAG_POS,
+ RX_PACKET_ATTRIBUTES_VLAN_CTAG_LEN,
+ 1);
+ pkt_info->vlan_ctag =
+ XLGMAC_GET_REG_BITS_LE(dma_desc->desc0,
+ RX_NORMAL_DESC0_OVT_POS,
+ RX_NORMAL_DESC0_OVT_LEN);
+ netif_dbg(pdata, rx_status, netdev, "vlan-ctag=%#06x\n",
+ pkt_info->vlan_ctag);
+ }
+ } else {
+ if ((etlt == 0x05) || (etlt == 0x06))
+ pkt_info->attributes = XLGMAC_SET_REG_BITS(
+ pkt_info->attributes,
+ RX_PACKET_ATTRIBUTES_CSUM_DONE_POS,
+ RX_PACKET_ATTRIBUTES_CSUM_DONE_LEN,
+ 0);
+ else
+ pkt_info->errors = XLGMAC_SET_REG_BITS(
+ pkt_info->errors,
+ RX_PACKET_ERRORS_FRAME_POS,
+ RX_PACKET_ERRORS_FRAME_LEN,
+ 1);
+ }
+
+ XLGMAC_PR("%s - descriptor=%u (cur=%d)\n", channel->name,
+ ring->cur & (ring->dma_desc_count - 1), ring->cur);
+
+ return 0;
+}
+
+static int xlgmac_enable_int(struct xlgmac_channel *channel,
+ enum xlgmac_int int_id)
+{
+ unsigned int dma_ch_ier;
+
+ dma_ch_ier = readl(XLGMAC_DMA_REG(channel, DMA_CH_IER));
+
+ switch (int_id) {
+ case XLGMAC_INT_DMA_CH_SR_TI:
+ dma_ch_ier = XLGMAC_SET_REG_BITS(
+ dma_ch_ier, DMA_CH_IER_TIE_POS,
+ DMA_CH_IER_TIE_LEN, 1);
+ break;
+ case XLGMAC_INT_DMA_CH_SR_TPS:
+ dma_ch_ier = XLGMAC_SET_REG_BITS(
+ dma_ch_ier, DMA_CH_IER_TXSE_POS,
+ DMA_CH_IER_TXSE_LEN, 1);
+ break;
+ case XLGMAC_INT_DMA_CH_SR_TBU:
+ dma_ch_ier = XLGMAC_SET_REG_BITS(
+ dma_ch_ier, DMA_CH_IER_TBUE_POS,
+ DMA_CH_IER_TBUE_LEN, 1);
+ break;
+ case XLGMAC_INT_DMA_CH_SR_RI:
+ dma_ch_ier = XLGMAC_SET_REG_BITS(
+ dma_ch_ier, DMA_CH_IER_RIE_POS,
+ DMA_CH_IER_RIE_LEN, 1);
+ break;
+ case XLGMAC_INT_DMA_CH_SR_RBU:
+ dma_ch_ier = XLGMAC_SET_REG_BITS(
+ dma_ch_ier, DMA_CH_IER_RBUE_POS,
+ DMA_CH_IER_RBUE_LEN, 1);
+ break;
+ case XLGMAC_INT_DMA_CH_SR_RPS:
+ dma_ch_ier = XLGMAC_SET_REG_BITS(
+ dma_ch_ier, DMA_CH_IER_RSE_POS,
+ DMA_CH_IER_RSE_LEN, 1);
+ break;
+ case XLGMAC_INT_DMA_CH_SR_TI_RI:
+ dma_ch_ier = XLGMAC_SET_REG_BITS(
+ dma_ch_ier, DMA_CH_IER_TIE_POS,
+ DMA_CH_IER_TIE_LEN, 1);
+ dma_ch_ier = XLGMAC_SET_REG_BITS(
+ dma_ch_ier, DMA_CH_IER_RIE_POS,
+ DMA_CH_IER_RIE_LEN, 1);
+ break;
+ case XLGMAC_INT_DMA_CH_SR_FBE:
+ dma_ch_ier = XLGMAC_SET_REG_BITS(
+ dma_ch_ier, DMA_CH_IER_FBEE_POS,
+ DMA_CH_IER_FBEE_LEN, 1);
+ break;
+ case XLGMAC_INT_DMA_ALL:
+ dma_ch_ier |= channel->saved_ier;
+ break;
+ default:
+ return -1;
+ }
+
+ writel(dma_ch_ier, XLGMAC_DMA_REG(channel, DMA_CH_IER));
+
+ return 0;
+}
+
+static int xlgmac_disable_int(struct xlgmac_channel *channel,
+ enum xlgmac_int int_id)
+{
+ unsigned int dma_ch_ier;
+
+ dma_ch_ier = readl(XLGMAC_DMA_REG(channel, DMA_CH_IER));
+
+ switch (int_id) {
+ case XLGMAC_INT_DMA_CH_SR_TI:
+ dma_ch_ier = XLGMAC_SET_REG_BITS(
+ dma_ch_ier, DMA_CH_IER_TIE_POS,
+ DMA_CH_IER_TIE_LEN, 0);
+ break;
+ case XLGMAC_INT_DMA_CH_SR_TPS:
+ dma_ch_ier = XLGMAC_SET_REG_BITS(
+ dma_ch_ier, DMA_CH_IER_TXSE_POS,
+ DMA_CH_IER_TXSE_LEN, 0);
+ break;
+ case XLGMAC_INT_DMA_CH_SR_TBU:
+ dma_ch_ier = XLGMAC_SET_REG_BITS(
+ dma_ch_ier, DMA_CH_IER_TBUE_POS,
+ DMA_CH_IER_TBUE_LEN, 0);
+ break;
+ case XLGMAC_INT_DMA_CH_SR_RI:
+ dma_ch_ier = XLGMAC_SET_REG_BITS(
+ dma_ch_ier, DMA_CH_IER_RIE_POS,
+ DMA_CH_IER_RIE_LEN, 0);
+ break;
+ case XLGMAC_INT_DMA_CH_SR_RBU:
+ dma_ch_ier = XLGMAC_SET_REG_BITS(
+ dma_ch_ier, DMA_CH_IER_RBUE_POS,
+ DMA_CH_IER_RBUE_LEN, 0);
+ break;
+ case XLGMAC_INT_DMA_CH_SR_RPS:
+ dma_ch_ier = XLGMAC_SET_REG_BITS(
+ dma_ch_ier, DMA_CH_IER_RSE_POS,
+ DMA_CH_IER_RSE_LEN, 0);
+ break;
+ case XLGMAC_INT_DMA_CH_SR_TI_RI:
+ dma_ch_ier = XLGMAC_SET_REG_BITS(
+ dma_ch_ier, DMA_CH_IER_TIE_POS,
+ DMA_CH_IER_TIE_LEN, 0);
+ dma_ch_ier = XLGMAC_SET_REG_BITS(
+ dma_ch_ier, DMA_CH_IER_RIE_POS,
+ DMA_CH_IER_RIE_LEN, 0);
+ break;
+ case XLGMAC_INT_DMA_CH_SR_FBE:
+ dma_ch_ier = XLGMAC_SET_REG_BITS(
+ dma_ch_ier, DMA_CH_IER_FBEE_POS,
+ DMA_CH_IER_FBEE_LEN, 0);
+ break;
+ case XLGMAC_INT_DMA_ALL:
+ channel->saved_ier = dma_ch_ier & XLGMAC_DMA_INTERRUPT_MASK;
+ dma_ch_ier &= ~XLGMAC_DMA_INTERRUPT_MASK;
+ break;
+ default:
+ return -1;
+ }
+
+ writel(dma_ch_ier, XLGMAC_DMA_REG(channel, DMA_CH_IER));
+
+ return 0;
+}
+
+static int xlgmac_flush_tx_queues(struct xlgmac_pdata *pdata)
+{
+ unsigned int i, count;
+ u32 regval;
+
+ for (i = 0; i < pdata->tx_q_count; i++) {
+ regval = readl(XLGMAC_MTL_REG(pdata, i, MTL_Q_TQOMR));
+ regval = XLGMAC_SET_REG_BITS(regval, MTL_Q_TQOMR_FTQ_POS,
+ MTL_Q_TQOMR_FTQ_LEN, 1);
+ writel(regval, XLGMAC_MTL_REG(pdata, i, MTL_Q_TQOMR));
+ }
+
+ /* Poll Until Poll Condition */
+ for (i = 0; i < pdata->tx_q_count; i++) {
+ count = 2000;
+ regval = readl(XLGMAC_MTL_REG(pdata, i, MTL_Q_TQOMR));
+ regval = XLGMAC_GET_REG_BITS(regval, MTL_Q_TQOMR_FTQ_POS,
+ MTL_Q_TQOMR_FTQ_LEN);
+ while (--count && regval)
+ usleep_range(500, 600);
+
+ if (!count)
+ return -EBUSY;
+ }
+
+ return 0;
+}
+
+static void xlgmac_config_dma_bus(struct xlgmac_pdata *pdata)
+{
+ u32 regval;
+
+ regval = readl(pdata->mac_regs + DMA_SBMR);
+ /* Set enhanced addressing mode */
+ regval = XLGMAC_SET_REG_BITS(regval, DMA_SBMR_EAME_POS,
+ DMA_SBMR_EAME_LEN, 1);
+ /* Set the System Bus mode */
+ regval = XLGMAC_SET_REG_BITS(regval, DMA_SBMR_UNDEF_POS,
+ DMA_SBMR_UNDEF_LEN, 1);
+ regval = XLGMAC_SET_REG_BITS(regval, DMA_SBMR_BLEN_256_POS,
+ DMA_SBMR_BLEN_256_LEN, 1);
+ writel(regval, pdata->mac_regs + DMA_SBMR);
+}
+
+static int xlgmac_hw_init(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_desc_ops *desc_ops = &pdata->desc_ops;
+ int ret;
+
+ /* Flush Tx queues */
+ ret = xlgmac_flush_tx_queues(pdata);
+ if (ret)
+ return ret;
+
+ /* Initialize DMA related features */
+ xlgmac_config_dma_bus(pdata);
+ xlgmac_config_osp_mode(pdata);
+ xlgmac_config_pblx8(pdata);
+ xlgmac_config_tx_pbl_val(pdata);
+ xlgmac_config_rx_pbl_val(pdata);
+ xlgmac_config_rx_coalesce(pdata);
+ xlgmac_config_tx_coalesce(pdata);
+ xlgmac_config_rx_buffer_size(pdata);
+ xlgmac_config_tso_mode(pdata);
+ xlgmac_config_sph_mode(pdata);
+ xlgmac_config_rss(pdata);
+ desc_ops->tx_desc_init(pdata);
+ desc_ops->rx_desc_init(pdata);
+ xlgmac_enable_dma_interrupts(pdata);
+
+ /* Initialize MTL related features */
+ xlgmac_config_mtl_mode(pdata);
+ xlgmac_config_queue_mapping(pdata);
+ xlgmac_config_tsf_mode(pdata, pdata->tx_sf_mode);
+ xlgmac_config_rsf_mode(pdata, pdata->rx_sf_mode);
+ xlgmac_config_tx_threshold(pdata, pdata->tx_threshold);
+ xlgmac_config_rx_threshold(pdata, pdata->rx_threshold);
+ xlgmac_config_tx_fifo_size(pdata);
+ xlgmac_config_rx_fifo_size(pdata);
+ xlgmac_config_flow_control_threshold(pdata);
+ xlgmac_config_rx_fep_enable(pdata);
+ xlgmac_config_rx_fup_enable(pdata);
+ xlgmac_enable_mtl_interrupts(pdata);
+
+ /* Initialize MAC related features */
+ xlgmac_config_mac_address(pdata);
+ xlgmac_config_rx_mode(pdata);
+ xlgmac_config_jumbo_enable(pdata);
+ xlgmac_config_flow_control(pdata);
+ xlgmac_config_mac_speed(pdata);
+ xlgmac_config_checksum_offload(pdata);
+ xlgmac_config_vlan_support(pdata);
+ xlgmac_config_mmc(pdata);
+ xlgmac_enable_mac_interrupts(pdata);
+
+ return 0;
+}
+
+static int xlgmac_hw_exit(struct xlgmac_pdata *pdata)
+{
+ unsigned int count = 2000;
+ u32 regval;
+
+ /* Issue a software reset */
+ regval = readl(pdata->mac_regs + DMA_MR);
+ regval = XLGMAC_SET_REG_BITS(regval, DMA_MR_SWR_POS,
+ DMA_MR_SWR_LEN, 1);
+ writel(regval, pdata->mac_regs + DMA_MR);
+ usleep_range(10, 15);
+
+ /* Poll Until Poll Condition */
+ while (--count &&
+ XLGMAC_GET_REG_BITS(readl(pdata->mac_regs + DMA_MR),
+ DMA_MR_SWR_POS, DMA_MR_SWR_LEN))
+ usleep_range(500, 600);
+
+ if (!count)
+ return -EBUSY;
+
+ return 0;
+}
+
+void xlgmac_init_hw_ops(struct xlgmac_hw_ops *hw_ops)
+{
+ hw_ops->init = xlgmac_hw_init;
+ hw_ops->exit = xlgmac_hw_exit;
+
+ hw_ops->tx_complete = xlgmac_tx_complete;
+
+ hw_ops->enable_tx = xlgmac_enable_tx;
+ hw_ops->disable_tx = xlgmac_disable_tx;
+ hw_ops->enable_rx = xlgmac_enable_rx;
+ hw_ops->disable_rx = xlgmac_disable_rx;
+
+ hw_ops->dev_xmit = xlgmac_dev_xmit;
+ hw_ops->dev_read = xlgmac_dev_read;
+ hw_ops->enable_int = xlgmac_enable_int;
+ hw_ops->disable_int = xlgmac_disable_int;
+
+ hw_ops->set_mac_address = xlgmac_set_mac_address;
+ hw_ops->config_rx_mode = xlgmac_config_rx_mode;
+ hw_ops->enable_rx_csum = xlgmac_enable_rx_csum;
+ hw_ops->disable_rx_csum = xlgmac_disable_rx_csum;
+
+ /* For MII speed configuration */
+ hw_ops->set_xlgmii_25000_speed = xlgmac_set_xlgmii_25000_speed;
+ hw_ops->set_xlgmii_40000_speed = xlgmac_set_xlgmii_40000_speed;
+ hw_ops->set_xlgmii_50000_speed = xlgmac_set_xlgmii_50000_speed;
+ hw_ops->set_xlgmii_100000_speed = xlgmac_set_xlgmii_100000_speed;
+
+ /* For descriptor related operation */
+ hw_ops->tx_desc_init = xlgmac_tx_desc_init;
+ hw_ops->rx_desc_init = xlgmac_rx_desc_init;
+ hw_ops->tx_desc_reset = xlgmac_tx_desc_reset;
+ hw_ops->rx_desc_reset = xlgmac_rx_desc_reset;
+ hw_ops->is_last_desc = xlgmac_is_last_desc;
+ hw_ops->is_context_desc = xlgmac_is_context_desc;
+ hw_ops->tx_start_xmit = xlgmac_tx_start_xmit;
+
+ /* For Flow Control */
+ hw_ops->config_tx_flow_control = xlgmac_config_tx_flow_control;
+ hw_ops->config_rx_flow_control = xlgmac_config_rx_flow_control;
+
+ /* For Vlan related config */
+ hw_ops->enable_rx_vlan_stripping = xlgmac_enable_rx_vlan_stripping;
+ hw_ops->disable_rx_vlan_stripping = xlgmac_disable_rx_vlan_stripping;
+ hw_ops->enable_rx_vlan_filtering = xlgmac_enable_rx_vlan_filtering;
+ hw_ops->disable_rx_vlan_filtering = xlgmac_disable_rx_vlan_filtering;
+ hw_ops->update_vlan_hash_table = xlgmac_update_vlan_hash_table;
+
+ /* For RX coalescing */
+ hw_ops->config_rx_coalesce = xlgmac_config_rx_coalesce;
+ hw_ops->config_tx_coalesce = xlgmac_config_tx_coalesce;
+ hw_ops->usec_to_riwt = xlgmac_usec_to_riwt;
+ hw_ops->riwt_to_usec = xlgmac_riwt_to_usec;
+
+ /* For RX and TX threshold config */
+ hw_ops->config_rx_threshold = xlgmac_config_rx_threshold;
+ hw_ops->config_tx_threshold = xlgmac_config_tx_threshold;
+
+ /* For RX and TX Store and Forward Mode config */
+ hw_ops->config_rsf_mode = xlgmac_config_rsf_mode;
+ hw_ops->config_tsf_mode = xlgmac_config_tsf_mode;
+
+ /* For TX DMA Operating on Second Frame config */
+ hw_ops->config_osp_mode = xlgmac_config_osp_mode;
+
+ /* For RX and TX PBL config */
+ hw_ops->config_rx_pbl_val = xlgmac_config_rx_pbl_val;
+ hw_ops->get_rx_pbl_val = xlgmac_get_rx_pbl_val;
+ hw_ops->config_tx_pbl_val = xlgmac_config_tx_pbl_val;
+ hw_ops->get_tx_pbl_val = xlgmac_get_tx_pbl_val;
+ hw_ops->config_pblx8 = xlgmac_config_pblx8;
+
+ /* For MMC statistics support */
+ hw_ops->tx_mmc_int = xlgmac_tx_mmc_int;
+ hw_ops->rx_mmc_int = xlgmac_rx_mmc_int;
+ hw_ops->read_mmc_stats = xlgmac_read_mmc_stats;
+
+ /* For Receive Side Scaling */
+ hw_ops->enable_rss = xlgmac_enable_rss;
+ hw_ops->disable_rss = xlgmac_disable_rss;
+ hw_ops->set_rss_hash_key = xlgmac_set_rss_hash_key;
+ hw_ops->set_rss_lookup_table = xlgmac_set_rss_lookup_table;
+}
--- /dev/null
+/* Synopsys DesignWare Core Enterprise Ethernet (XLGMAC) Driver
+ *
+ * Copyright (c) 2017 Synopsys, Inc. (www.synopsys.com)
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This Synopsys DWC XLGMAC software driver and associated documentation
+ * (hereinafter the "Software") is an unsupported proprietary work of
+ * Synopsys, Inc. unless otherwise expressly agreed to in writing between
+ * Synopsys and you. The Software IS NOT an item of Licensed Software or a
+ * Licensed Product under any End User Software License Agreement or
+ * Agreement for Licensed Products with Synopsys or any supplement thereto.
+ * Synopsys is a registered trademark of Synopsys, Inc. Other names included
+ * in the SOFTWARE may be the trademarks of their respective owners.
+ */
+
+#include <linux/netdevice.h>
+#include <linux/tcp.h>
+
+#include "dwc-xlgmac.h"
+#include "dwc-xlgmac-reg.h"
+
+static int xlgmac_one_poll(struct napi_struct *, int);
+static int xlgmac_all_poll(struct napi_struct *, int);
+
+static inline unsigned int xlgmac_tx_avail_desc(struct xlgmac_ring *ring)
+{
+ return (ring->dma_desc_count - (ring->cur - ring->dirty));
+}
+
+static inline unsigned int xlgmac_rx_dirty_desc(struct xlgmac_ring *ring)
+{
+ return (ring->cur - ring->dirty);
+}
+
+static int xlgmac_maybe_stop_tx_queue(
+ struct xlgmac_channel *channel,
+ struct xlgmac_ring *ring,
+ unsigned int count)
+{
+ struct xlgmac_pdata *pdata = channel->pdata;
+
+ if (count > xlgmac_tx_avail_desc(ring)) {
+ netif_info(pdata, drv, pdata->netdev,
+ "Tx queue stopped, not enough descriptors available\n");
+ netif_stop_subqueue(pdata->netdev, channel->queue_index);
+ ring->tx.queue_stopped = 1;
+
+ /* If we haven't notified the hardware because of xmit_more
+ * support, tell it now
+ */
+ if (ring->tx.xmit_more)
+ pdata->hw_ops.tx_start_xmit(channel, ring);
+
+ return NETDEV_TX_BUSY;
+ }
+
+ return 0;
+}
+
+static void xlgmac_prep_vlan(struct sk_buff *skb,
+ struct xlgmac_pkt_info *pkt_info)
+{
+ if (skb_vlan_tag_present(skb))
+ pkt_info->vlan_ctag = skb_vlan_tag_get(skb);
+}
+
+static int xlgmac_prep_tso(struct sk_buff *skb,
+ struct xlgmac_pkt_info *pkt_info)
+{
+ int ret;
+
+ if (!XLGMAC_GET_REG_BITS(pkt_info->attributes,
+ TX_PACKET_ATTRIBUTES_TSO_ENABLE_POS,
+ TX_PACKET_ATTRIBUTES_TSO_ENABLE_LEN))
+ return 0;
+
+ ret = skb_cow_head(skb, 0);
+ if (ret)
+ return ret;
+
+ pkt_info->header_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
+ pkt_info->tcp_header_len = tcp_hdrlen(skb);
+ pkt_info->tcp_payload_len = skb->len - pkt_info->header_len;
+ pkt_info->mss = skb_shinfo(skb)->gso_size;
+
+ XLGMAC_PR("header_len=%u\n", pkt_info->header_len);
+ XLGMAC_PR("tcp_header_len=%u, tcp_payload_len=%u\n",
+ pkt_info->tcp_header_len, pkt_info->tcp_payload_len);
+ XLGMAC_PR("mss=%u\n", pkt_info->mss);
+
+ /* Update the number of packets that will ultimately be transmitted
+ * along with the extra bytes for each extra packet
+ */
+ pkt_info->tx_packets = skb_shinfo(skb)->gso_segs;
+ pkt_info->tx_bytes += (pkt_info->tx_packets - 1) * pkt_info->header_len;
+
+ return 0;
+}
+
+static int xlgmac_is_tso(struct sk_buff *skb)
+{
+ if (skb->ip_summed != CHECKSUM_PARTIAL)
+ return 0;
+
+ if (!skb_is_gso(skb))
+ return 0;
+
+ return 1;
+}
+
+static void xlgmac_prep_tx_pkt(struct xlgmac_pdata *pdata,
+ struct xlgmac_ring *ring,
+ struct sk_buff *skb,
+ struct xlgmac_pkt_info *pkt_info)
+{
+ struct skb_frag_struct *frag;
+ unsigned int context_desc;
+ unsigned int len;
+ unsigned int i;
+
+ pkt_info->skb = skb;
+
+ context_desc = 0;
+ pkt_info->desc_count = 0;
+
+ pkt_info->tx_packets = 1;
+ pkt_info->tx_bytes = skb->len;
+
+ if (xlgmac_is_tso(skb)) {
+ /* TSO requires an extra descriptor if mss is different */
+ if (skb_shinfo(skb)->gso_size != ring->tx.cur_mss) {
+ context_desc = 1;
+ pkt_info->desc_count++;
+ }
+
+ /* TSO requires an extra descriptor for TSO header */
+ pkt_info->desc_count++;
+
+ pkt_info->attributes = XLGMAC_SET_REG_BITS(
+ pkt_info->attributes,
+ TX_PACKET_ATTRIBUTES_TSO_ENABLE_POS,
+ TX_PACKET_ATTRIBUTES_TSO_ENABLE_LEN,
+ 1);
+ pkt_info->attributes = XLGMAC_SET_REG_BITS(
+ pkt_info->attributes,
+ TX_PACKET_ATTRIBUTES_CSUM_ENABLE_POS,
+ TX_PACKET_ATTRIBUTES_CSUM_ENABLE_LEN,
+ 1);
+ } else if (skb->ip_summed == CHECKSUM_PARTIAL)
+ pkt_info->attributes = XLGMAC_SET_REG_BITS(
+ pkt_info->attributes,
+ TX_PACKET_ATTRIBUTES_CSUM_ENABLE_POS,
+ TX_PACKET_ATTRIBUTES_CSUM_ENABLE_LEN,
+ 1);
+
+ if (skb_vlan_tag_present(skb)) {
+ /* VLAN requires an extra descriptor if tag is different */
+ if (skb_vlan_tag_get(skb) != ring->tx.cur_vlan_ctag)
+ /* We can share with the TSO context descriptor */
+ if (!context_desc) {
+ context_desc = 1;
+ pkt_info->desc_count++;
+ }
+
+ pkt_info->attributes = XLGMAC_SET_REG_BITS(
+ pkt_info->attributes,
+ TX_PACKET_ATTRIBUTES_VLAN_CTAG_POS,
+ TX_PACKET_ATTRIBUTES_VLAN_CTAG_LEN,
+ 1);
+ }
+
+ for (len = skb_headlen(skb); len;) {
+ pkt_info->desc_count++;
+ len -= min_t(unsigned int, len, XLGMAC_TX_MAX_BUF_SIZE);
+ }
+
+ for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
+ frag = &skb_shinfo(skb)->frags[i];
+ for (len = skb_frag_size(frag); len; ) {
+ pkt_info->desc_count++;
+ len -= min_t(unsigned int, len, XLGMAC_TX_MAX_BUF_SIZE);
+ }
+ }
+}
+
+static int xlgmac_calc_rx_buf_size(struct net_device *netdev, unsigned int mtu)
+{
+ unsigned int rx_buf_size;
+
+ if (mtu > XLGMAC_JUMBO_PACKET_MTU) {
+ netdev_alert(netdev, "MTU exceeds maximum supported value\n");
+ return -EINVAL;
+ }
+
+ rx_buf_size = mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
+ rx_buf_size = clamp_val(rx_buf_size, XLGMAC_RX_MIN_BUF_SIZE, PAGE_SIZE);
+
+ rx_buf_size = (rx_buf_size + XLGMAC_RX_BUF_ALIGN - 1) &
+ ~(XLGMAC_RX_BUF_ALIGN - 1);
+
+ return rx_buf_size;
+}
+
+static void xlgmac_enable_rx_tx_ints(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;
+ struct xlgmac_channel *channel;
+ enum xlgmac_int int_id;
+ unsigned int i;
+
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ if (channel->tx_ring && channel->rx_ring)
+ int_id = XLGMAC_INT_DMA_CH_SR_TI_RI;
+ else if (channel->tx_ring)
+ int_id = XLGMAC_INT_DMA_CH_SR_TI;
+ else if (channel->rx_ring)
+ int_id = XLGMAC_INT_DMA_CH_SR_RI;
+ else
+ continue;
+
+ hw_ops->enable_int(channel, int_id);
+ }
+}
+
+static void xlgmac_disable_rx_tx_ints(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;
+ struct xlgmac_channel *channel;
+ enum xlgmac_int int_id;
+ unsigned int i;
+
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ if (channel->tx_ring && channel->rx_ring)
+ int_id = XLGMAC_INT_DMA_CH_SR_TI_RI;
+ else if (channel->tx_ring)
+ int_id = XLGMAC_INT_DMA_CH_SR_TI;
+ else if (channel->rx_ring)
+ int_id = XLGMAC_INT_DMA_CH_SR_RI;
+ else
+ continue;
+
+ hw_ops->disable_int(channel, int_id);
+ }
+}
+
+static irqreturn_t xlgmac_isr(int irq, void *data)
+{
+ unsigned int dma_isr, dma_ch_isr, mac_isr;
+ struct xlgmac_pdata *pdata = data;
+ struct xlgmac_channel *channel;
+ struct xlgmac_hw_ops *hw_ops;
+ unsigned int i, ti, ri;
+
+ hw_ops = &pdata->hw_ops;
+
+ /* The DMA interrupt status register also reports MAC and MTL
+ * interrupts. So for polling mode, we just need to check for
+ * this register to be non-zero
+ */
+ dma_isr = readl(pdata->mac_regs + DMA_ISR);
+ if (!dma_isr)
+ return IRQ_HANDLED;
+
+ netif_dbg(pdata, intr, pdata->netdev, "DMA_ISR=%#010x\n", dma_isr);
+
+ for (i = 0; i < pdata->channel_count; i++) {
+ if (!(dma_isr & (1 << i)))
+ continue;
+
+ channel = pdata->channel_head + i;
+
+ dma_ch_isr = readl(XLGMAC_DMA_REG(channel, DMA_CH_SR));
+ netif_dbg(pdata, intr, pdata->netdev, "DMA_CH%u_ISR=%#010x\n",
+ i, dma_ch_isr);
+
+ /* The TI or RI interrupt bits may still be set even if using
+ * per channel DMA interrupts. Check to be sure those are not
+ * enabled before using the private data napi structure.
+ */
+ ti = XLGMAC_GET_REG_BITS(dma_ch_isr, DMA_CH_SR_TI_POS,
+ DMA_CH_SR_TI_LEN);
+ ri = XLGMAC_GET_REG_BITS(dma_ch_isr, DMA_CH_SR_RI_POS,
+ DMA_CH_SR_RI_LEN);
+ if (!pdata->per_channel_irq && (ti || ri)) {
+ if (napi_schedule_prep(&pdata->napi)) {
+ /* Disable Tx and Rx interrupts */
+ xlgmac_disable_rx_tx_ints(pdata);
+
+ /* Turn on polling */
+ __napi_schedule_irqoff(&pdata->napi);
+ }
+ }
+
+ if (XLGMAC_GET_REG_BITS(dma_ch_isr, DMA_CH_SR_RBU_POS,
+ DMA_CH_SR_RBU_LEN))
+ pdata->stats.rx_buffer_unavailable++;
+
+ /* Restart the device on a Fatal Bus Error */
+ if (XLGMAC_GET_REG_BITS(dma_ch_isr, DMA_CH_SR_FBE_POS,
+ DMA_CH_SR_FBE_LEN))
+ schedule_work(&pdata->restart_work);
+
+ /* Clear all interrupt signals */
+ writel(dma_ch_isr, XLGMAC_DMA_REG(channel, DMA_CH_SR));
+ }
+
+ if (XLGMAC_GET_REG_BITS(dma_isr, DMA_ISR_MACIS_POS,
+ DMA_ISR_MACIS_LEN)) {
+ mac_isr = readl(pdata->mac_regs + MAC_ISR);
+
+ if (XLGMAC_GET_REG_BITS(mac_isr, MAC_ISR_MMCTXIS_POS,
+ MAC_ISR_MMCTXIS_LEN))
+ hw_ops->tx_mmc_int(pdata);
+
+ if (XLGMAC_GET_REG_BITS(mac_isr, MAC_ISR_MMCRXIS_POS,
+ MAC_ISR_MMCRXIS_LEN))
+ hw_ops->rx_mmc_int(pdata);
+ }
+
+ return IRQ_HANDLED;
+}
+
+static irqreturn_t xlgmac_dma_isr(int irq, void *data)
+{
+ struct xlgmac_channel *channel = data;
+
+ /* Per channel DMA interrupts are enabled, so we use the per
+ * channel napi structure and not the private data napi structure
+ */
+ if (napi_schedule_prep(&channel->napi)) {
+ /* Disable Tx and Rx interrupts */
+ disable_irq_nosync(channel->dma_irq);
+
+ /* Turn on polling */
+ __napi_schedule_irqoff(&channel->napi);
+ }
+
+ return IRQ_HANDLED;
+}
+
+static void xlgmac_tx_timer(unsigned long data)
+{
+ struct xlgmac_channel *channel = (struct xlgmac_channel *)data;
+ struct xlgmac_pdata *pdata = channel->pdata;
+ struct napi_struct *napi;
+
+ napi = (pdata->per_channel_irq) ? &channel->napi : &pdata->napi;
+
+ if (napi_schedule_prep(napi)) {
+ /* Disable Tx and Rx interrupts */
+ if (pdata->per_channel_irq)
+ disable_irq_nosync(channel->dma_irq);
+ else
+ xlgmac_disable_rx_tx_ints(pdata);
+
+ /* Turn on polling */
+ __napi_schedule(napi);
+ }
+
+ channel->tx_timer_active = 0;
+}
+
+static void xlgmac_init_timers(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_channel *channel;
+ unsigned int i;
+
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ if (!channel->tx_ring)
+ break;
+
+ setup_timer(&channel->tx_timer, xlgmac_tx_timer,
+ (unsigned long)channel);
+ }
+}
+
+static void xlgmac_stop_timers(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_channel *channel;
+ unsigned int i;
+
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ if (!channel->tx_ring)
+ break;
+
+ del_timer_sync(&channel->tx_timer);
+ }
+}
+
+static void xlgmac_napi_enable(struct xlgmac_pdata *pdata, unsigned int add)
+{
+ struct xlgmac_channel *channel;
+ unsigned int i;
+
+ if (pdata->per_channel_irq) {
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ if (add)
+ netif_napi_add(pdata->netdev, &channel->napi,
+ xlgmac_one_poll,
+ NAPI_POLL_WEIGHT);
+
+ napi_enable(&channel->napi);
+ }
+ } else {
+ if (add)
+ netif_napi_add(pdata->netdev, &pdata->napi,
+ xlgmac_all_poll, NAPI_POLL_WEIGHT);
+
+ napi_enable(&pdata->napi);
+ }
+}
+
+static void xlgmac_napi_disable(struct xlgmac_pdata *pdata, unsigned int del)
+{
+ struct xlgmac_channel *channel;
+ unsigned int i;
+
+ if (pdata->per_channel_irq) {
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ napi_disable(&channel->napi);
+
+ if (del)
+ netif_napi_del(&channel->napi);
+ }
+ } else {
+ napi_disable(&pdata->napi);
+
+ if (del)
+ netif_napi_del(&pdata->napi);
+ }
+}
+
+static int xlgmac_request_irqs(struct xlgmac_pdata *pdata)
+{
+ struct net_device *netdev = pdata->netdev;
+ struct xlgmac_channel *channel;
+ unsigned int i;
+ int ret;
+
+ ret = devm_request_irq(pdata->dev, pdata->dev_irq, xlgmac_isr,
+ IRQF_SHARED, netdev->name, pdata);
+ if (ret) {
+ netdev_alert(netdev, "error requesting irq %d\n",
+ pdata->dev_irq);
+ return ret;
+ }
+
+ if (!pdata->per_channel_irq)
+ return 0;
+
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ snprintf(channel->dma_irq_name,
+ sizeof(channel->dma_irq_name) - 1,
+ "%s-TxRx-%u", netdev_name(netdev),
+ channel->queue_index);
+
+ ret = devm_request_irq(pdata->dev, channel->dma_irq,
+ xlgmac_dma_isr, 0,
+ channel->dma_irq_name, channel);
+ if (ret) {
+ netdev_alert(netdev, "error requesting irq %d\n",
+ channel->dma_irq);
+ goto err_irq;
+ }
+ }
+
+ return 0;
+
+err_irq:
+ /* Using an unsigned int, 'i' will go to UINT_MAX and exit */
+ for (i--, channel--; i < pdata->channel_count; i--, channel--)
+ devm_free_irq(pdata->dev, channel->dma_irq, channel);
+
+ devm_free_irq(pdata->dev, pdata->dev_irq, pdata);
+
+ return ret;
+}
+
+static void xlgmac_free_irqs(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_channel *channel;
+ unsigned int i;
+
+ devm_free_irq(pdata->dev, pdata->dev_irq, pdata);
+
+ if (!pdata->per_channel_irq)
+ return;
+
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++)
+ devm_free_irq(pdata->dev, channel->dma_irq, channel);
+}
+
+static void xlgmac_free_tx_data(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_desc_ops *desc_ops = &pdata->desc_ops;
+ struct xlgmac_desc_data *desc_data;
+ struct xlgmac_channel *channel;
+ struct xlgmac_ring *ring;
+ unsigned int i, j;
+
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ ring = channel->tx_ring;
+ if (!ring)
+ break;
+
+ for (j = 0; j < ring->dma_desc_count; j++) {
+ desc_data = XLGMAC_GET_DESC_DATA(ring, j);
+ desc_ops->unmap_desc_data(pdata, desc_data);
+ }
+ }
+}
+
+static void xlgmac_free_rx_data(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_desc_ops *desc_ops = &pdata->desc_ops;
+ struct xlgmac_desc_data *desc_data;
+ struct xlgmac_channel *channel;
+ struct xlgmac_ring *ring;
+ unsigned int i, j;
+
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ ring = channel->rx_ring;
+ if (!ring)
+ break;
+
+ for (j = 0; j < ring->dma_desc_count; j++) {
+ desc_data = XLGMAC_GET_DESC_DATA(ring, j);
+ desc_ops->unmap_desc_data(pdata, desc_data);
+ }
+ }
+}
+
+static int xlgmac_start(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;
+ struct net_device *netdev = pdata->netdev;
+ int ret;
+
+ hw_ops->init(pdata);
+ xlgmac_napi_enable(pdata, 1);
+
+ ret = xlgmac_request_irqs(pdata);
+ if (ret)
+ goto err_napi;
+
+ hw_ops->enable_tx(pdata);
+ hw_ops->enable_rx(pdata);
+ netif_tx_start_all_queues(netdev);
+
+ return 0;
+
+err_napi:
+ xlgmac_napi_disable(pdata, 1);
+ hw_ops->exit(pdata);
+
+ return ret;
+}
+
+static void xlgmac_stop(struct xlgmac_pdata *pdata)
+{
+ struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;
+ struct net_device *netdev = pdata->netdev;
+ struct xlgmac_channel *channel;
+ struct netdev_queue *txq;
+ unsigned int i;
+
+ netif_tx_stop_all_queues(netdev);
+ xlgmac_stop_timers(pdata);
+ hw_ops->disable_tx(pdata);
+ hw_ops->disable_rx(pdata);
+ xlgmac_free_irqs(pdata);
+ xlgmac_napi_disable(pdata, 1);
+ hw_ops->exit(pdata);
+
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ if (!channel->tx_ring)
+ continue;
+
+ txq = netdev_get_tx_queue(netdev, channel->queue_index);
+ netdev_tx_reset_queue(txq);
+ }
+}
+
+static void xlgmac_restart_dev(struct xlgmac_pdata *pdata)
+{
+ /* If not running, "restart" will happen on open */
+ if (!netif_running(pdata->netdev))
+ return;
+
+ xlgmac_stop(pdata);
+
+ xlgmac_free_tx_data(pdata);
+ xlgmac_free_rx_data(pdata);
+
+ xlgmac_start(pdata);
+}
+
+static void xlgmac_restart(struct work_struct *work)
+{
+ struct xlgmac_pdata *pdata = container_of(work,
+ struct xlgmac_pdata,
+ restart_work);
+
+ rtnl_lock();
+
+ xlgmac_restart_dev(pdata);
+
+ rtnl_unlock();
+}
+
+static int xlgmac_open(struct net_device *netdev)
+{
+ struct xlgmac_pdata *pdata = netdev_priv(netdev);
+ struct xlgmac_desc_ops *desc_ops;
+ int ret;
+
+ desc_ops = &pdata->desc_ops;
+
+ /* TODO: Initialize the phy */
+
+ /* Calculate the Rx buffer size before allocating rings */
+ ret = xlgmac_calc_rx_buf_size(netdev, netdev->mtu);
+ if (ret < 0)
+ return ret;
+ pdata->rx_buf_size = ret;
+
+ /* Allocate the channels and rings */
+ ret = desc_ops->alloc_channles_and_rings(pdata);
+ if (ret)
+ return ret;
+
+ INIT_WORK(&pdata->restart_work, xlgmac_restart);
+ xlgmac_init_timers(pdata);
+
+ ret = xlgmac_start(pdata);
+ if (ret)
+ goto err_channels_and_rings;
+
+ return 0;
+
+err_channels_and_rings:
+ desc_ops->free_channels_and_rings(pdata);
+
+ return ret;
+}
+
+static int xlgmac_close(struct net_device *netdev)
+{
+ struct xlgmac_pdata *pdata = netdev_priv(netdev);
+ struct xlgmac_desc_ops *desc_ops;
+
+ desc_ops = &pdata->desc_ops;
+
+ /* Stop the device */
+ xlgmac_stop(pdata);
+
+ /* Free the channels and rings */
+ desc_ops->free_channels_and_rings(pdata);
+
+ return 0;
+}
+
+static void xlgmac_tx_timeout(struct net_device *netdev)
+{
+ struct xlgmac_pdata *pdata = netdev_priv(netdev);
+
+ netdev_warn(netdev, "tx timeout, device restarting\n");
+ schedule_work(&pdata->restart_work);
+}
+
+static int xlgmac_xmit(struct sk_buff *skb, struct net_device *netdev)
+{
+ struct xlgmac_pdata *pdata = netdev_priv(netdev);
+ struct xlgmac_pkt_info *tx_pkt_info;
+ struct xlgmac_desc_ops *desc_ops;
+ struct xlgmac_channel *channel;
+ struct xlgmac_hw_ops *hw_ops;
+ struct netdev_queue *txq;
+ struct xlgmac_ring *ring;
+ int ret;
+
+ desc_ops = &pdata->desc_ops;
+ hw_ops = &pdata->hw_ops;
+
+ XLGMAC_PR("skb->len = %d\n", skb->len);
+
+ channel = pdata->channel_head + skb->queue_mapping;
+ txq = netdev_get_tx_queue(netdev, channel->queue_index);
+ ring = channel->tx_ring;
+ tx_pkt_info = &ring->pkt_info;
+
+ if (skb->len == 0) {
+ netif_err(pdata, tx_err, netdev,
+ "empty skb received from stack\n");
+ dev_kfree_skb_any(skb);
+ return NETDEV_TX_OK;
+ }
+
+ /* Prepare preliminary packet info for TX */
+ memset(tx_pkt_info, 0, sizeof(*tx_pkt_info));
+ xlgmac_prep_tx_pkt(pdata, ring, skb, tx_pkt_info);
+
+ /* Check that there are enough descriptors available */
+ ret = xlgmac_maybe_stop_tx_queue(channel, ring,
+ tx_pkt_info->desc_count);
+ if (ret)
+ return ret;
+
+ ret = xlgmac_prep_tso(skb, tx_pkt_info);
+ if (ret) {
+ netif_err(pdata, tx_err, netdev,
+ "error processing TSO packet\n");
+ dev_kfree_skb_any(skb);
+ return ret;
+ }
+ xlgmac_prep_vlan(skb, tx_pkt_info);
+
+ if (!desc_ops->map_tx_skb(channel, skb)) {
+ dev_kfree_skb_any(skb);
+ return NETDEV_TX_OK;
+ }
+
+ /* Report on the actual number of bytes (to be) sent */
+ netdev_tx_sent_queue(txq, tx_pkt_info->tx_bytes);
+
+ /* Configure required descriptor fields for transmission */
+ hw_ops->dev_xmit(channel);
+
+ if (netif_msg_pktdata(pdata))
+ xlgmac_print_pkt(netdev, skb, true);
+
+ /* Stop the queue in advance if there may not be enough descriptors */
+ xlgmac_maybe_stop_tx_queue(channel, ring, XLGMAC_TX_MAX_DESC_NR);
+
+ return NETDEV_TX_OK;
+}
+
+static void xlgmac_get_stats64(struct net_device *netdev,
+ struct rtnl_link_stats64 *s)
+{
+ struct xlgmac_pdata *pdata = netdev_priv(netdev);
+ struct xlgmac_stats *pstats = &pdata->stats;
+
+ pdata->hw_ops.read_mmc_stats(pdata);
+
+ s->rx_packets = pstats->rxframecount_gb;
+ s->rx_bytes = pstats->rxoctetcount_gb;
+ s->rx_errors = pstats->rxframecount_gb -
+ pstats->rxbroadcastframes_g -
+ pstats->rxmulticastframes_g -
+ pstats->rxunicastframes_g;
+ s->multicast = pstats->rxmulticastframes_g;
+ s->rx_length_errors = pstats->rxlengtherror;
+ s->rx_crc_errors = pstats->rxcrcerror;
+ s->rx_fifo_errors = pstats->rxfifooverflow;
+
+ s->tx_packets = pstats->txframecount_gb;
+ s->tx_bytes = pstats->txoctetcount_gb;
+ s->tx_errors = pstats->txframecount_gb - pstats->txframecount_g;
+ s->tx_dropped = netdev->stats.tx_dropped;
+}
+
+static int xlgmac_set_mac_address(struct net_device *netdev, void *addr)
+{
+ struct xlgmac_pdata *pdata = netdev_priv(netdev);
+ struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;
+ struct sockaddr *saddr = addr;
+
+ if (!is_valid_ether_addr(saddr->sa_data))
+ return -EADDRNOTAVAIL;
+
+ memcpy(netdev->dev_addr, saddr->sa_data, netdev->addr_len);
+
+ hw_ops->set_mac_address(pdata, netdev->dev_addr);
+
+ return 0;
+}
+
+static int xlgmac_ioctl(struct net_device *netdev,
+ struct ifreq *ifreq, int cmd)
+{
+ if (!netif_running(netdev))
+ return -ENODEV;
+
+ return 0;
+}
+
+static int xlgmac_change_mtu(struct net_device *netdev, int mtu)
+{
+ struct xlgmac_pdata *pdata = netdev_priv(netdev);
+ int ret;
+
+ ret = xlgmac_calc_rx_buf_size(netdev, mtu);
+ if (ret < 0)
+ return ret;
+
+ pdata->rx_buf_size = ret;
+ netdev->mtu = mtu;
+
+ xlgmac_restart_dev(pdata);
+
+ return 0;
+}
+
+static int xlgmac_vlan_rx_add_vid(struct net_device *netdev,
+ __be16 proto,
+ u16 vid)
+{
+ struct xlgmac_pdata *pdata = netdev_priv(netdev);
+ struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;
+
+ set_bit(vid, pdata->active_vlans);
+ hw_ops->update_vlan_hash_table(pdata);
+
+ return 0;
+}
+
+static int xlgmac_vlan_rx_kill_vid(struct net_device *netdev,
+ __be16 proto,
+ u16 vid)
+{
+ struct xlgmac_pdata *pdata = netdev_priv(netdev);
+ struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;
+
+ clear_bit(vid, pdata->active_vlans);
+ hw_ops->update_vlan_hash_table(pdata);
+
+ return 0;
+}
+
+#ifdef CONFIG_NET_POLL_CONTROLLER
+static void xlgmac_poll_controller(struct net_device *netdev)
+{
+ struct xlgmac_pdata *pdata = netdev_priv(netdev);
+ struct xlgmac_channel *channel;
+ unsigned int i;
+
+ if (pdata->per_channel_irq) {
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++)
+ xlgmac_dma_isr(channel->dma_irq, channel);
+ } else {
+ disable_irq(pdata->dev_irq);
+ xlgmac_isr(pdata->dev_irq, pdata);
+ enable_irq(pdata->dev_irq);
+ }
+}
+#endif /* CONFIG_NET_POLL_CONTROLLER */
+
+static int xlgmac_set_features(struct net_device *netdev,
+ netdev_features_t features)
+{
+ netdev_features_t rxhash, rxcsum, rxvlan, rxvlan_filter;
+ struct xlgmac_pdata *pdata = netdev_priv(netdev);
+ struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;
+ int ret = 0;
+
+ rxhash = pdata->netdev_features & NETIF_F_RXHASH;
+ rxcsum = pdata->netdev_features & NETIF_F_RXCSUM;
+ rxvlan = pdata->netdev_features & NETIF_F_HW_VLAN_CTAG_RX;
+ rxvlan_filter = pdata->netdev_features & NETIF_F_HW_VLAN_CTAG_FILTER;
+
+ if ((features & NETIF_F_RXHASH) && !rxhash)
+ ret = hw_ops->enable_rss(pdata);
+ else if (!(features & NETIF_F_RXHASH) && rxhash)
+ ret = hw_ops->disable_rss(pdata);
+ if (ret)
+ return ret;
+
+ if ((features & NETIF_F_RXCSUM) && !rxcsum)
+ hw_ops->enable_rx_csum(pdata);
+ else if (!(features & NETIF_F_RXCSUM) && rxcsum)
+ hw_ops->disable_rx_csum(pdata);
+
+ if ((features & NETIF_F_HW_VLAN_CTAG_RX) && !rxvlan)
+ hw_ops->enable_rx_vlan_stripping(pdata);
+ else if (!(features & NETIF_F_HW_VLAN_CTAG_RX) && rxvlan)
+ hw_ops->disable_rx_vlan_stripping(pdata);
+
+ if ((features & NETIF_F_HW_VLAN_CTAG_FILTER) && !rxvlan_filter)
+ hw_ops->enable_rx_vlan_filtering(pdata);
+ else if (!(features & NETIF_F_HW_VLAN_CTAG_FILTER) && rxvlan_filter)
+ hw_ops->disable_rx_vlan_filtering(pdata);
+
+ pdata->netdev_features = features;
+
+ return 0;
+}
+
+static void xlgmac_set_rx_mode(struct net_device *netdev)
+{
+ struct xlgmac_pdata *pdata = netdev_priv(netdev);
+ struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;
+
+ hw_ops->config_rx_mode(pdata);
+}
+
+static const struct net_device_ops xlgmac_netdev_ops = {
+ .ndo_open = xlgmac_open,
+ .ndo_stop = xlgmac_close,
+ .ndo_start_xmit = xlgmac_xmit,
+ .ndo_tx_timeout = xlgmac_tx_timeout,
+ .ndo_get_stats64 = xlgmac_get_stats64,
+ .ndo_change_mtu = xlgmac_change_mtu,
+ .ndo_set_mac_address = xlgmac_set_mac_address,
+ .ndo_validate_addr = eth_validate_addr,
+ .ndo_do_ioctl = xlgmac_ioctl,
+ .ndo_vlan_rx_add_vid = xlgmac_vlan_rx_add_vid,
+ .ndo_vlan_rx_kill_vid = xlgmac_vlan_rx_kill_vid,
+#ifdef CONFIG_NET_POLL_CONTROLLER
+ .ndo_poll_controller = xlgmac_poll_controller,
+#endif
+ .ndo_set_features = xlgmac_set_features,
+ .ndo_set_rx_mode = xlgmac_set_rx_mode,
+};
+
+const struct net_device_ops *xlgmac_get_netdev_ops(void)
+{
+ return &xlgmac_netdev_ops;
+}
+
+static void xlgmac_rx_refresh(struct xlgmac_channel *channel)
+{
+ struct xlgmac_pdata *pdata = channel->pdata;
+ struct xlgmac_ring *ring = channel->rx_ring;
+ struct xlgmac_desc_data *desc_data;
+ struct xlgmac_desc_ops *desc_ops;
+ struct xlgmac_hw_ops *hw_ops;
+
+ desc_ops = &pdata->desc_ops;
+ hw_ops = &pdata->hw_ops;
+
+ while (ring->dirty != ring->cur) {
+ desc_data = XLGMAC_GET_DESC_DATA(ring, ring->dirty);
+
+ /* Reset desc_data values */
+ desc_ops->unmap_desc_data(pdata, desc_data);
+
+ if (desc_ops->map_rx_buffer(pdata, ring, desc_data))
+ break;
+
+ hw_ops->rx_desc_reset(pdata, desc_data, ring->dirty);
+
+ ring->dirty++;
+ }
+
+ /* Make sure everything is written before the register write */
+ wmb();
+
+ /* Update the Rx Tail Pointer Register with address of
+ * the last cleaned entry
+ */
+ desc_data = XLGMAC_GET_DESC_DATA(ring, ring->dirty - 1);
+ writel(lower_32_bits(desc_data->dma_desc_addr),
+ XLGMAC_DMA_REG(channel, DMA_CH_RDTR_LO));
+}
+
+static struct sk_buff *xlgmac_create_skb(struct xlgmac_pdata *pdata,
+ struct napi_struct *napi,
+ struct xlgmac_desc_data *desc_data,
+ unsigned int len)
+{
+ unsigned int copy_len;
+ struct sk_buff *skb;
+ u8 *packet;
+
+ skb = napi_alloc_skb(napi, desc_data->rx.hdr.dma_len);
+ if (!skb)
+ return NULL;
+
+ /* Start with the header buffer which may contain just the header
+ * or the header plus data
+ */
+ dma_sync_single_range_for_cpu(pdata->dev, desc_data->rx.hdr.dma_base,
+ desc_data->rx.hdr.dma_off,
+ desc_data->rx.hdr.dma_len,
+ DMA_FROM_DEVICE);
+
+ packet = page_address(desc_data->rx.hdr.pa.pages) +
+ desc_data->rx.hdr.pa.pages_offset;
+ copy_len = (desc_data->rx.hdr_len) ? desc_data->rx.hdr_len : len;
+ copy_len = min(desc_data->rx.hdr.dma_len, copy_len);
+ skb_copy_to_linear_data(skb, packet, copy_len);
+ skb_put(skb, copy_len);
+
+ len -= copy_len;
+ if (len) {
+ /* Add the remaining data as a frag */
+ dma_sync_single_range_for_cpu(pdata->dev,
+ desc_data->rx.buf.dma_base,
+ desc_data->rx.buf.dma_off,
+ desc_data->rx.buf.dma_len,
+ DMA_FROM_DEVICE);
+
+ skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
+ desc_data->rx.buf.pa.pages,
+ desc_data->rx.buf.pa.pages_offset,
+ len, desc_data->rx.buf.dma_len);
+ desc_data->rx.buf.pa.pages = NULL;
+ }
+
+ return skb;
+}
+
+static int xlgmac_tx_poll(struct xlgmac_channel *channel)
+{
+ struct xlgmac_pdata *pdata = channel->pdata;
+ struct xlgmac_ring *ring = channel->tx_ring;
+ struct net_device *netdev = pdata->netdev;
+ unsigned int tx_packets = 0, tx_bytes = 0;
+ struct xlgmac_desc_data *desc_data;
+ struct xlgmac_dma_desc *dma_desc;
+ struct xlgmac_desc_ops *desc_ops;
+ struct xlgmac_hw_ops *hw_ops;
+ struct netdev_queue *txq;
+ int processed = 0;
+ unsigned int cur;
+
+ desc_ops = &pdata->desc_ops;
+ hw_ops = &pdata->hw_ops;
+
+ /* Nothing to do if there isn't a Tx ring for this channel */
+ if (!ring)
+ return 0;
+
+ cur = ring->cur;
+
+ /* Be sure we get ring->cur before accessing descriptor data */
+ smp_rmb();
+
+ txq = netdev_get_tx_queue(netdev, channel->queue_index);
+
+ while ((processed < XLGMAC_TX_DESC_MAX_PROC) &&
+ (ring->dirty != cur)) {
+ desc_data = XLGMAC_GET_DESC_DATA(ring, ring->dirty);
+ dma_desc = desc_data->dma_desc;
+
+ if (!hw_ops->tx_complete(dma_desc))
+ break;
+
+ /* Make sure descriptor fields are read after reading
+ * the OWN bit
+ */
+ dma_rmb();
+
+ if (netif_msg_tx_done(pdata))
+ xlgmac_dump_tx_desc(pdata, ring, ring->dirty, 1, 0);
+
+ if (hw_ops->is_last_desc(dma_desc)) {
+ tx_packets += desc_data->tx.packets;
+ tx_bytes += desc_data->tx.bytes;
+ }
+
+ /* Free the SKB and reset the descriptor for re-use */
+ desc_ops->unmap_desc_data(pdata, desc_data);
+ hw_ops->tx_desc_reset(desc_data);
+
+ processed++;
+ ring->dirty++;
+ }
+
+ if (!processed)
+ return 0;
+
+ netdev_tx_completed_queue(txq, tx_packets, tx_bytes);
+
+ if ((ring->tx.queue_stopped == 1) &&
+ (xlgmac_tx_avail_desc(ring) > XLGMAC_TX_DESC_MIN_FREE)) {
+ ring->tx.queue_stopped = 0;
+ netif_tx_wake_queue(txq);
+ }
+
+ XLGMAC_PR("processed=%d\n", processed);
+
+ return processed;
+}
+
+static int xlgmac_rx_poll(struct xlgmac_channel *channel, int budget)
+{
+ struct xlgmac_pdata *pdata = channel->pdata;
+ struct xlgmac_ring *ring = channel->rx_ring;
+ struct net_device *netdev = pdata->netdev;
+ unsigned int len, dma_desc_len, max_len;
+ unsigned int context_next, context;
+ struct xlgmac_desc_data *desc_data;
+ struct xlgmac_pkt_info *pkt_info;
+ unsigned int incomplete, error;
+ struct xlgmac_hw_ops *hw_ops;
+ unsigned int received = 0;
+ struct napi_struct *napi;
+ struct sk_buff *skb;
+ int packet_count = 0;
+
+ hw_ops = &pdata->hw_ops;
+
+ /* Nothing to do if there isn't a Rx ring for this channel */
+ if (!ring)
+ return 0;
+
+ incomplete = 0;
+ context_next = 0;
+
+ napi = (pdata->per_channel_irq) ? &channel->napi : &pdata->napi;
+
+ desc_data = XLGMAC_GET_DESC_DATA(ring, ring->cur);
+ pkt_info = &ring->pkt_info;
+ while (packet_count < budget) {
+ /* First time in loop see if we need to restore state */
+ if (!received && desc_data->state_saved) {
+ skb = desc_data->state.skb;
+ error = desc_data->state.error;
+ len = desc_data->state.len;
+ } else {
+ memset(pkt_info, 0, sizeof(*pkt_info));
+ skb = NULL;
+ error = 0;
+ len = 0;
+ }
+
+read_again:
+ desc_data = XLGMAC_GET_DESC_DATA(ring, ring->cur);
+
+ if (xlgmac_rx_dirty_desc(ring) > XLGMAC_RX_DESC_MAX_DIRTY)
+ xlgmac_rx_refresh(channel);
+
+ if (hw_ops->dev_read(channel))
+ break;
+
+ received++;
+ ring->cur++;
+
+ incomplete = XLGMAC_GET_REG_BITS(
+ pkt_info->attributes,
+ RX_PACKET_ATTRIBUTES_INCOMPLETE_POS,
+ RX_PACKET_ATTRIBUTES_INCOMPLETE_LEN);
+ context_next = XLGMAC_GET_REG_BITS(
+ pkt_info->attributes,
+ RX_PACKET_ATTRIBUTES_CONTEXT_NEXT_POS,
+ RX_PACKET_ATTRIBUTES_CONTEXT_NEXT_LEN);
+ context = XLGMAC_GET_REG_BITS(
+ pkt_info->attributes,
+ RX_PACKET_ATTRIBUTES_CONTEXT_POS,
+ RX_PACKET_ATTRIBUTES_CONTEXT_LEN);
+
+ /* Earlier error, just drain the remaining data */
+ if ((incomplete || context_next) && error)
+ goto read_again;
+
+ if (error || pkt_info->errors) {
+ if (pkt_info->errors)
+ netif_err(pdata, rx_err, netdev,
+ "error in received packet\n");
+ dev_kfree_skb(skb);
+ goto next_packet;
+ }
+
+ if (!context) {
+ /* Length is cumulative, get this descriptor's length */
+ dma_desc_len = desc_data->rx.len - len;
+ len += dma_desc_len;
+
+ if (dma_desc_len && !skb) {
+ skb = xlgmac_create_skb(pdata, napi, desc_data,
+ dma_desc_len);
+ if (!skb)
+ error = 1;
+ } else if (dma_desc_len) {
+ dma_sync_single_range_for_cpu(
+ pdata->dev,
+ desc_data->rx.buf.dma_base,
+ desc_data->rx.buf.dma_off,
+ desc_data->rx.buf.dma_len,
+ DMA_FROM_DEVICE);
+
+ skb_add_rx_frag(
+ skb, skb_shinfo(skb)->nr_frags,
+ desc_data->rx.buf.pa.pages,
+ desc_data->rx.buf.pa.pages_offset,
+ dma_desc_len,
+ desc_data->rx.buf.dma_len);
+ desc_data->rx.buf.pa.pages = NULL;
+ }
+ }
+
+ if (incomplete || context_next)
+ goto read_again;
+
+ if (!skb)
+ goto next_packet;
+
+ /* Be sure we don't exceed the configured MTU */
+ max_len = netdev->mtu + ETH_HLEN;
+ if (!(netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
+ (skb->protocol == htons(ETH_P_8021Q)))
+ max_len += VLAN_HLEN;
+
+ if (skb->len > max_len) {
+ netif_err(pdata, rx_err, netdev,
+ "packet length exceeds configured MTU\n");
+ dev_kfree_skb(skb);
+ goto next_packet;
+ }
+
+ if (netif_msg_pktdata(pdata))
+ xlgmac_print_pkt(netdev, skb, false);
+
+ skb_checksum_none_assert(skb);
+ if (XLGMAC_GET_REG_BITS(pkt_info->attributes,
+ RX_PACKET_ATTRIBUTES_CSUM_DONE_POS,
+ RX_PACKET_ATTRIBUTES_CSUM_DONE_LEN))
+ skb->ip_summed = CHECKSUM_UNNECESSARY;
+
+ if (XLGMAC_GET_REG_BITS(pkt_info->attributes,
+ RX_PACKET_ATTRIBUTES_VLAN_CTAG_POS,
+ RX_PACKET_ATTRIBUTES_VLAN_CTAG_LEN))
+ __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
+ pkt_info->vlan_ctag);
+
+ if (XLGMAC_GET_REG_BITS(pkt_info->attributes,
+ RX_PACKET_ATTRIBUTES_RSS_HASH_POS,
+ RX_PACKET_ATTRIBUTES_RSS_HASH_LEN))
+ skb_set_hash(skb, pkt_info->rss_hash,
+ pkt_info->rss_hash_type);
+
+ skb->dev = netdev;
+ skb->protocol = eth_type_trans(skb, netdev);
+ skb_record_rx_queue(skb, channel->queue_index);
+
+ napi_gro_receive(napi, skb);
+
+next_packet:
+ packet_count++;
+ }
+
+ /* Check if we need to save state before leaving */
+ if (received && (incomplete || context_next)) {
+ desc_data = XLGMAC_GET_DESC_DATA(ring, ring->cur);
+ desc_data->state_saved = 1;
+ desc_data->state.skb = skb;
+ desc_data->state.len = len;
+ desc_data->state.error = error;
+ }
+
+ XLGMAC_PR("packet_count = %d\n", packet_count);
+
+ return packet_count;
+}
+
+static int xlgmac_one_poll(struct napi_struct *napi, int budget)
+{
+ struct xlgmac_channel *channel = container_of(napi,
+ struct xlgmac_channel,
+ napi);
+ int processed = 0;
+
+ XLGMAC_PR("budget=%d\n", budget);
+
+ /* Cleanup Tx ring first */
+ xlgmac_tx_poll(channel);
+
+ /* Process Rx ring next */
+ processed = xlgmac_rx_poll(channel, budget);
+
+ /* If we processed everything, we are done */
+ if (processed < budget) {
+ /* Turn off polling */
+ napi_complete_done(napi, processed);
+
+ /* Enable Tx and Rx interrupts */
+ enable_irq(channel->dma_irq);
+ }
+
+ XLGMAC_PR("received = %d\n", processed);
+
+ return processed;
+}
+
+static int xlgmac_all_poll(struct napi_struct *napi, int budget)
+{
+ struct xlgmac_pdata *pdata = container_of(napi,
+ struct xlgmac_pdata,
+ napi);
+ struct xlgmac_channel *channel;
+ int processed, last_processed;
+ int ring_budget;
+ unsigned int i;
+
+ XLGMAC_PR("budget=%d\n", budget);
+
+ processed = 0;
+ ring_budget = budget / pdata->rx_ring_count;
+ do {
+ last_processed = processed;
+
+ channel = pdata->channel_head;
+ for (i = 0; i < pdata->channel_count; i++, channel++) {
+ /* Cleanup Tx ring first */
+ xlgmac_tx_poll(channel);
+
+ /* Process Rx ring next */
+ if (ring_budget > (budget - processed))
+ ring_budget = budget - processed;
+ processed += xlgmac_rx_poll(channel, ring_budget);
+ }
+ } while ((processed < budget) && (processed != last_processed));
+
+ /* If we processed everything, we are done */
+ if (processed < budget) {
+ /* Turn off polling */
+ napi_complete_done(napi, processed);
+
+ /* Enable Tx and Rx interrupts */
+ xlgmac_enable_rx_tx_ints(pdata);
+ }
+
+ XLGMAC_PR("received = %d\n", processed);
+
+ return processed;
+}
--- /dev/null
+/* Synopsys DesignWare Core Enterprise Ethernet (XLGMAC) Driver
+ *
+ * Copyright (c) 2017 Synopsys, Inc. (www.synopsys.com)
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This Synopsys DWC XLGMAC software driver and associated documentation
+ * (hereinafter the "Software") is an unsupported proprietary work of
+ * Synopsys, Inc. unless otherwise expressly agreed to in writing between
+ * Synopsys and you. The Software IS NOT an item of Licensed Software or a
+ * Licensed Product under any End User Software License Agreement or
+ * Agreement for Licensed Products with Synopsys or any supplement thereto.
+ * Synopsys is a registered trademark of Synopsys, Inc. Other names included
+ * in the SOFTWARE may be the trademarks of their respective owners.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+
+#include "dwc-xlgmac.h"
+#include "dwc-xlgmac-reg.h"
+
+static int xlgmac_probe(struct pci_dev *pcidev, const struct pci_device_id *id)
+{
+ struct device *dev = &pcidev->dev;
+ struct xlgmac_resources res;
+ int i, ret;
+
+ ret = pcim_enable_device(pcidev);
+ if (ret) {
+ dev_err(dev, "ERROR: failed to enable device\n");
+ return ret;
+ }
+
+ for (i = 0; i <= PCI_STD_RESOURCE_END; i++) {
+ if (pci_resource_len(pcidev, i) == 0)
+ continue;
+ ret = pcim_iomap_regions(pcidev, BIT(i), XLGMAC_DRV_NAME);
+ if (ret)
+ return ret;
+ break;
+ }
+
+ pci_set_master(pcidev);
+
+ memset(&res, 0, sizeof(res));
+ res.irq = pcidev->irq;
+ res.addr = pcim_iomap_table(pcidev)[i];
+
+ return xlgmac_drv_probe(&pcidev->dev, &res);
+}
+
+static void xlgmac_remove(struct pci_dev *pcidev)
+{
+ xlgmac_drv_remove(&pcidev->dev);
+}
+
+static const struct pci_device_id xlgmac_pci_tbl[] = {
+ { PCI_DEVICE(PCI_VENDOR_ID_SYNOPSYS, 0x7302) },
+ { 0 }
+};
+MODULE_DEVICE_TABLE(pci, xlgmac_pci_tbl);
+
+static struct pci_driver xlgmac_pci_driver = {
+ .name = XLGMAC_DRV_NAME,
+ .id_table = xlgmac_pci_tbl,
+ .probe = xlgmac_probe,
+ .remove = xlgmac_remove,
+};
+
+module_pci_driver(xlgmac_pci_driver);
+
+MODULE_DESCRIPTION(XLGMAC_DRV_DESC);
+MODULE_VERSION(XLGMAC_DRV_VERSION);
+MODULE_AUTHOR("Jie Deng <jiedeng@synopsys.com>");
+MODULE_LICENSE("Dual BSD/GPL");
--- /dev/null
+/* Synopsys DesignWare Core Enterprise Ethernet (XLGMAC) Driver
+ *
+ * Copyright (c) 2017 Synopsys, Inc. (www.synopsys.com)
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This Synopsys DWC XLGMAC software driver and associated documentation
+ * (hereinafter the "Software") is an unsupported proprietary work of
+ * Synopsys, Inc. unless otherwise expressly agreed to in writing between
+ * Synopsys and you. The Software IS NOT an item of Licensed Software or a
+ * Licensed Product under any End User Software License Agreement or
+ * Agreement for Licensed Products with Synopsys or any supplement thereto.
+ * Synopsys is a registered trademark of Synopsys, Inc. Other names included
+ * in the SOFTWARE may be the trademarks of their respective owners.
+ */
+
+#ifndef __DWC_XLGMAC_REG_H__
+#define __DWC_XLGMAC_REG_H__
+
+/* MAC register offsets */
+#define MAC_TCR 0x0000
+#define MAC_RCR 0x0004
+#define MAC_PFR 0x0008
+#define MAC_HTR0 0x0010
+#define MAC_VLANTR 0x0050
+#define MAC_VLANHTR 0x0058
+#define MAC_VLANIR 0x0060
+#define MAC_Q0TFCR 0x0070
+#define MAC_RFCR 0x0090
+#define MAC_RQC0R 0x00a0
+#define MAC_RQC1R 0x00a4
+#define MAC_RQC2R 0x00a8
+#define MAC_RQC3R 0x00ac
+#define MAC_ISR 0x00b0
+#define MAC_IER 0x00b4
+#define MAC_VR 0x0110
+#define MAC_HWF0R 0x011c
+#define MAC_HWF1R 0x0120
+#define MAC_HWF2R 0x0124
+#define MAC_MACA0HR 0x0300
+#define MAC_MACA0LR 0x0304
+#define MAC_MACA1HR 0x0308
+#define MAC_MACA1LR 0x030c
+#define MAC_RSSCR 0x0c80
+#define MAC_RSSAR 0x0c88
+#define MAC_RSSDR 0x0c8c
+
+#define MAC_QTFCR_INC 4
+#define MAC_MACA_INC 4
+#define MAC_HTR_INC 4
+#define MAC_RQC2_INC 4
+#define MAC_RQC2_Q_PER_REG 4
+
+/* MAC register entry bit positions and sizes */
+#define MAC_HWF0R_ADDMACADRSEL_POS 18
+#define MAC_HWF0R_ADDMACADRSEL_LEN 5
+#define MAC_HWF0R_ARPOFFSEL_POS 9
+#define MAC_HWF0R_ARPOFFSEL_LEN 1
+#define MAC_HWF0R_EEESEL_POS 13
+#define MAC_HWF0R_EEESEL_LEN 1
+#define MAC_HWF0R_PHYIFSEL_POS 1
+#define MAC_HWF0R_PHYIFSEL_LEN 2
+#define MAC_HWF0R_MGKSEL_POS 7
+#define MAC_HWF0R_MGKSEL_LEN 1
+#define MAC_HWF0R_MMCSEL_POS 8
+#define MAC_HWF0R_MMCSEL_LEN 1
+#define MAC_HWF0R_RWKSEL_POS 6
+#define MAC_HWF0R_RWKSEL_LEN 1
+#define MAC_HWF0R_RXCOESEL_POS 16
+#define MAC_HWF0R_RXCOESEL_LEN 1
+#define MAC_HWF0R_SAVLANINS_POS 27
+#define MAC_HWF0R_SAVLANINS_LEN 1
+#define MAC_HWF0R_SMASEL_POS 5
+#define MAC_HWF0R_SMASEL_LEN 1
+#define MAC_HWF0R_TSSEL_POS 12
+#define MAC_HWF0R_TSSEL_LEN 1
+#define MAC_HWF0R_TSSTSSEL_POS 25
+#define MAC_HWF0R_TSSTSSEL_LEN 2
+#define MAC_HWF0R_TXCOESEL_POS 14
+#define MAC_HWF0R_TXCOESEL_LEN 1
+#define MAC_HWF0R_VLHASH_POS 4
+#define MAC_HWF0R_VLHASH_LEN 1
+#define MAC_HWF1R_ADDR64_POS 14
+#define MAC_HWF1R_ADDR64_LEN 2
+#define MAC_HWF1R_ADVTHWORD_POS 13
+#define MAC_HWF1R_ADVTHWORD_LEN 1
+#define MAC_HWF1R_DBGMEMA_POS 19
+#define MAC_HWF1R_DBGMEMA_LEN 1
+#define MAC_HWF1R_DCBEN_POS 16
+#define MAC_HWF1R_DCBEN_LEN 1
+#define MAC_HWF1R_HASHTBLSZ_POS 24
+#define MAC_HWF1R_HASHTBLSZ_LEN 3
+#define MAC_HWF1R_L3L4FNUM_POS 27
+#define MAC_HWF1R_L3L4FNUM_LEN 4
+#define MAC_HWF1R_NUMTC_POS 21
+#define MAC_HWF1R_NUMTC_LEN 3
+#define MAC_HWF1R_RSSEN_POS 20
+#define MAC_HWF1R_RSSEN_LEN 1
+#define MAC_HWF1R_RXFIFOSIZE_POS 0
+#define MAC_HWF1R_RXFIFOSIZE_LEN 5
+#define MAC_HWF1R_SPHEN_POS 17
+#define MAC_HWF1R_SPHEN_LEN 1
+#define MAC_HWF1R_TSOEN_POS 18
+#define MAC_HWF1R_TSOEN_LEN 1
+#define MAC_HWF1R_TXFIFOSIZE_POS 6
+#define MAC_HWF1R_TXFIFOSIZE_LEN 5
+#define MAC_HWF2R_AUXSNAPNUM_POS 28
+#define MAC_HWF2R_AUXSNAPNUM_LEN 3
+#define MAC_HWF2R_PPSOUTNUM_POS 24
+#define MAC_HWF2R_PPSOUTNUM_LEN 3
+#define MAC_HWF2R_RXCHCNT_POS 12
+#define MAC_HWF2R_RXCHCNT_LEN 4
+#define MAC_HWF2R_RXQCNT_POS 0
+#define MAC_HWF2R_RXQCNT_LEN 4
+#define MAC_HWF2R_TXCHCNT_POS 18
+#define MAC_HWF2R_TXCHCNT_LEN 4
+#define MAC_HWF2R_TXQCNT_POS 6
+#define MAC_HWF2R_TXQCNT_LEN 4
+#define MAC_IER_TSIE_POS 12
+#define MAC_IER_TSIE_LEN 1
+#define MAC_ISR_MMCRXIS_POS 9
+#define MAC_ISR_MMCRXIS_LEN 1
+#define MAC_ISR_MMCTXIS_POS 10
+#define MAC_ISR_MMCTXIS_LEN 1
+#define MAC_ISR_PMTIS_POS 4
+#define MAC_ISR_PMTIS_LEN 1
+#define MAC_ISR_TSIS_POS 12
+#define MAC_ISR_TSIS_LEN 1
+#define MAC_MACA1HR_AE_POS 31
+#define MAC_MACA1HR_AE_LEN 1
+#define MAC_PFR_HMC_POS 2
+#define MAC_PFR_HMC_LEN 1
+#define MAC_PFR_HPF_POS 10
+#define MAC_PFR_HPF_LEN 1
+#define MAC_PFR_HUC_POS 1
+#define MAC_PFR_HUC_LEN 1
+#define MAC_PFR_PM_POS 4
+#define MAC_PFR_PM_LEN 1
+#define MAC_PFR_PR_POS 0
+#define MAC_PFR_PR_LEN 1
+#define MAC_PFR_VTFE_POS 16
+#define MAC_PFR_VTFE_LEN 1
+#define MAC_Q0TFCR_PT_POS 16
+#define MAC_Q0TFCR_PT_LEN 16
+#define MAC_Q0TFCR_TFE_POS 1
+#define MAC_Q0TFCR_TFE_LEN 1
+#define MAC_RCR_ACS_POS 1
+#define MAC_RCR_ACS_LEN 1
+#define MAC_RCR_CST_POS 2
+#define MAC_RCR_CST_LEN 1
+#define MAC_RCR_DCRCC_POS 3
+#define MAC_RCR_DCRCC_LEN 1
+#define MAC_RCR_HDSMS_POS 12
+#define MAC_RCR_HDSMS_LEN 3
+#define MAC_RCR_IPC_POS 9
+#define MAC_RCR_IPC_LEN 1
+#define MAC_RCR_JE_POS 8
+#define MAC_RCR_JE_LEN 1
+#define MAC_RCR_LM_POS 10
+#define MAC_RCR_LM_LEN 1
+#define MAC_RCR_RE_POS 0
+#define MAC_RCR_RE_LEN 1
+#define MAC_RFCR_PFCE_POS 8
+#define MAC_RFCR_PFCE_LEN 1
+#define MAC_RFCR_RFE_POS 0
+#define MAC_RFCR_RFE_LEN 1
+#define MAC_RFCR_UP_POS 1
+#define MAC_RFCR_UP_LEN 1
+#define MAC_RQC0R_RXQ0EN_POS 0
+#define MAC_RQC0R_RXQ0EN_LEN 2
+#define MAC_RSSAR_ADDRT_POS 2
+#define MAC_RSSAR_ADDRT_LEN 1
+#define MAC_RSSAR_CT_POS 1
+#define MAC_RSSAR_CT_LEN 1
+#define MAC_RSSAR_OB_POS 0
+#define MAC_RSSAR_OB_LEN 1
+#define MAC_RSSAR_RSSIA_POS 8
+#define MAC_RSSAR_RSSIA_LEN 8
+#define MAC_RSSCR_IP2TE_POS 1
+#define MAC_RSSCR_IP2TE_LEN 1
+#define MAC_RSSCR_RSSE_POS 0
+#define MAC_RSSCR_RSSE_LEN 1
+#define MAC_RSSCR_TCP4TE_POS 2
+#define MAC_RSSCR_TCP4TE_LEN 1
+#define MAC_RSSCR_UDP4TE_POS 3
+#define MAC_RSSCR_UDP4TE_LEN 1
+#define MAC_RSSDR_DMCH_POS 0
+#define MAC_RSSDR_DMCH_LEN 4
+#define MAC_TCR_SS_POS 28
+#define MAC_TCR_SS_LEN 3
+#define MAC_TCR_TE_POS 0
+#define MAC_TCR_TE_LEN 1
+#define MAC_VLANHTR_VLHT_POS 0
+#define MAC_VLANHTR_VLHT_LEN 16
+#define MAC_VLANIR_VLTI_POS 20
+#define MAC_VLANIR_VLTI_LEN 1
+#define MAC_VLANIR_CSVL_POS 19
+#define MAC_VLANIR_CSVL_LEN 1
+#define MAC_VLANTR_DOVLTC_POS 20
+#define MAC_VLANTR_DOVLTC_LEN 1
+#define MAC_VLANTR_ERSVLM_POS 19
+#define MAC_VLANTR_ERSVLM_LEN 1
+#define MAC_VLANTR_ESVL_POS 18
+#define MAC_VLANTR_ESVL_LEN 1
+#define MAC_VLANTR_ETV_POS 16
+#define MAC_VLANTR_ETV_LEN 1
+#define MAC_VLANTR_EVLS_POS 21
+#define MAC_VLANTR_EVLS_LEN 2
+#define MAC_VLANTR_EVLRXS_POS 24
+#define MAC_VLANTR_EVLRXS_LEN 1
+#define MAC_VLANTR_VL_POS 0
+#define MAC_VLANTR_VL_LEN 16
+#define MAC_VLANTR_VTHM_POS 25
+#define MAC_VLANTR_VTHM_LEN 1
+#define MAC_VLANTR_VTIM_POS 17
+#define MAC_VLANTR_VTIM_LEN 1
+#define MAC_VR_DEVID_POS 8
+#define MAC_VR_DEVID_LEN 8
+#define MAC_VR_SNPSVER_POS 0
+#define MAC_VR_SNPSVER_LEN 8
+#define MAC_VR_USERVER_POS 16
+#define MAC_VR_USERVER_LEN 8
+
+/* MMC register offsets */
+#define MMC_CR 0x0800
+#define MMC_RISR 0x0804
+#define MMC_TISR 0x0808
+#define MMC_RIER 0x080c
+#define MMC_TIER 0x0810
+#define MMC_TXOCTETCOUNT_GB_LO 0x0814
+#define MMC_TXFRAMECOUNT_GB_LO 0x081c
+#define MMC_TXBROADCASTFRAMES_G_LO 0x0824
+#define MMC_TXMULTICASTFRAMES_G_LO 0x082c
+#define MMC_TX64OCTETS_GB_LO 0x0834
+#define MMC_TX65TO127OCTETS_GB_LO 0x083c
+#define MMC_TX128TO255OCTETS_GB_LO 0x0844
+#define MMC_TX256TO511OCTETS_GB_LO 0x084c
+#define MMC_TX512TO1023OCTETS_GB_LO 0x0854
+#define MMC_TX1024TOMAXOCTETS_GB_LO 0x085c
+#define MMC_TXUNICASTFRAMES_GB_LO 0x0864
+#define MMC_TXMULTICASTFRAMES_GB_LO 0x086c
+#define MMC_TXBROADCASTFRAMES_GB_LO 0x0874
+#define MMC_TXUNDERFLOWERROR_LO 0x087c
+#define MMC_TXOCTETCOUNT_G_LO 0x0884
+#define MMC_TXFRAMECOUNT_G_LO 0x088c
+#define MMC_TXPAUSEFRAMES_LO 0x0894
+#define MMC_TXVLANFRAMES_G_LO 0x089c
+#define MMC_RXFRAMECOUNT_GB_LO 0x0900
+#define MMC_RXOCTETCOUNT_GB_LO 0x0908
+#define MMC_RXOCTETCOUNT_G_LO 0x0910
+#define MMC_RXBROADCASTFRAMES_G_LO 0x0918
+#define MMC_RXMULTICASTFRAMES_G_LO 0x0920
+#define MMC_RXCRCERROR_LO 0x0928
+#define MMC_RXRUNTERROR 0x0930
+#define MMC_RXJABBERERROR 0x0934
+#define MMC_RXUNDERSIZE_G 0x0938
+#define MMC_RXOVERSIZE_G 0x093c
+#define MMC_RX64OCTETS_GB_LO 0x0940
+#define MMC_RX65TO127OCTETS_GB_LO 0x0948
+#define MMC_RX128TO255OCTETS_GB_LO 0x0950
+#define MMC_RX256TO511OCTETS_GB_LO 0x0958
+#define MMC_RX512TO1023OCTETS_GB_LO 0x0960
+#define MMC_RX1024TOMAXOCTETS_GB_LO 0x0968
+#define MMC_RXUNICASTFRAMES_G_LO 0x0970
+#define MMC_RXLENGTHERROR_LO 0x0978
+#define MMC_RXOUTOFRANGETYPE_LO 0x0980
+#define MMC_RXPAUSEFRAMES_LO 0x0988
+#define MMC_RXFIFOOVERFLOW_LO 0x0990
+#define MMC_RXVLANFRAMES_GB_LO 0x0998
+#define MMC_RXWATCHDOGERROR 0x09a0
+
+/* MMC register entry bit positions and sizes */
+#define MMC_CR_CR_POS 0
+#define MMC_CR_CR_LEN 1
+#define MMC_CR_CSR_POS 1
+#define MMC_CR_CSR_LEN 1
+#define MMC_CR_ROR_POS 2
+#define MMC_CR_ROR_LEN 1
+#define MMC_CR_MCF_POS 3
+#define MMC_CR_MCF_LEN 1
+#define MMC_CR_MCT_POS 4
+#define MMC_CR_MCT_LEN 2
+#define MMC_RIER_ALL_INTERRUPTS_POS 0
+#define MMC_RIER_ALL_INTERRUPTS_LEN 23
+#define MMC_RISR_RXFRAMECOUNT_GB_POS 0
+#define MMC_RISR_RXFRAMECOUNT_GB_LEN 1
+#define MMC_RISR_RXOCTETCOUNT_GB_POS 1
+#define MMC_RISR_RXOCTETCOUNT_GB_LEN 1
+#define MMC_RISR_RXOCTETCOUNT_G_POS 2
+#define MMC_RISR_RXOCTETCOUNT_G_LEN 1
+#define MMC_RISR_RXBROADCASTFRAMES_G_POS 3
+#define MMC_RISR_RXBROADCASTFRAMES_G_LEN 1
+#define MMC_RISR_RXMULTICASTFRAMES_G_POS 4
+#define MMC_RISR_RXMULTICASTFRAMES_G_LEN 1
+#define MMC_RISR_RXCRCERROR_POS 5
+#define MMC_RISR_RXCRCERROR_LEN 1
+#define MMC_RISR_RXRUNTERROR_POS 6
+#define MMC_RISR_RXRUNTERROR_LEN 1
+#define MMC_RISR_RXJABBERERROR_POS 7
+#define MMC_RISR_RXJABBERERROR_LEN 1
+#define MMC_RISR_RXUNDERSIZE_G_POS 8
+#define MMC_RISR_RXUNDERSIZE_G_LEN 1
+#define MMC_RISR_RXOVERSIZE_G_POS 9
+#define MMC_RISR_RXOVERSIZE_G_LEN 1
+#define MMC_RISR_RX64OCTETS_GB_POS 10
+#define MMC_RISR_RX64OCTETS_GB_LEN 1
+#define MMC_RISR_RX65TO127OCTETS_GB_POS 11
+#define MMC_RISR_RX65TO127OCTETS_GB_LEN 1
+#define MMC_RISR_RX128TO255OCTETS_GB_POS 12
+#define MMC_RISR_RX128TO255OCTETS_GB_LEN 1
+#define MMC_RISR_RX256TO511OCTETS_GB_POS 13
+#define MMC_RISR_RX256TO511OCTETS_GB_LEN 1
+#define MMC_RISR_RX512TO1023OCTETS_GB_POS 14
+#define MMC_RISR_RX512TO1023OCTETS_GB_LEN 1
+#define MMC_RISR_RX1024TOMAXOCTETS_GB_POS 15
+#define MMC_RISR_RX1024TOMAXOCTETS_GB_LEN 1
+#define MMC_RISR_RXUNICASTFRAMES_G_POS 16
+#define MMC_RISR_RXUNICASTFRAMES_G_LEN 1
+#define MMC_RISR_RXLENGTHERROR_POS 17
+#define MMC_RISR_RXLENGTHERROR_LEN 1
+#define MMC_RISR_RXOUTOFRANGETYPE_POS 18
+#define MMC_RISR_RXOUTOFRANGETYPE_LEN 1
+#define MMC_RISR_RXPAUSEFRAMES_POS 19
+#define MMC_RISR_RXPAUSEFRAMES_LEN 1
+#define MMC_RISR_RXFIFOOVERFLOW_POS 20
+#define MMC_RISR_RXFIFOOVERFLOW_LEN 1
+#define MMC_RISR_RXVLANFRAMES_GB_POS 21
+#define MMC_RISR_RXVLANFRAMES_GB_LEN 1
+#define MMC_RISR_RXWATCHDOGERROR_POS 22
+#define MMC_RISR_RXWATCHDOGERROR_LEN 1
+#define MMC_TIER_ALL_INTERRUPTS_POS 0
+#define MMC_TIER_ALL_INTERRUPTS_LEN 18
+#define MMC_TISR_TXOCTETCOUNT_GB_POS 0
+#define MMC_TISR_TXOCTETCOUNT_GB_LEN 1
+#define MMC_TISR_TXFRAMECOUNT_GB_POS 1
+#define MMC_TISR_TXFRAMECOUNT_GB_LEN 1
+#define MMC_TISR_TXBROADCASTFRAMES_G_POS 2
+#define MMC_TISR_TXBROADCASTFRAMES_G_LEN 1
+#define MMC_TISR_TXMULTICASTFRAMES_G_POS 3
+#define MMC_TISR_TXMULTICASTFRAMES_G_LEN 1
+#define MMC_TISR_TX64OCTETS_GB_POS 4
+#define MMC_TISR_TX64OCTETS_GB_LEN 1
+#define MMC_TISR_TX65TO127OCTETS_GB_POS 5
+#define MMC_TISR_TX65TO127OCTETS_GB_LEN 1
+#define MMC_TISR_TX128TO255OCTETS_GB_POS 6
+#define MMC_TISR_TX128TO255OCTETS_GB_LEN 1
+#define MMC_TISR_TX256TO511OCTETS_GB_POS 7
+#define MMC_TISR_TX256TO511OCTETS_GB_LEN 1
+#define MMC_TISR_TX512TO1023OCTETS_GB_POS 8
+#define MMC_TISR_TX512TO1023OCTETS_GB_LEN 1
+#define MMC_TISR_TX1024TOMAXOCTETS_GB_POS 9
+#define MMC_TISR_TX1024TOMAXOCTETS_GB_LEN 1
+#define MMC_TISR_TXUNICASTFRAMES_GB_POS 10
+#define MMC_TISR_TXUNICASTFRAMES_GB_LEN 1
+#define MMC_TISR_TXMULTICASTFRAMES_GB_POS 11
+#define MMC_TISR_TXMULTICASTFRAMES_GB_LEN 1
+#define MMC_TISR_TXBROADCASTFRAMES_GB_POS 12
+#define MMC_TISR_TXBROADCASTFRAMES_GB_LEN 1
+#define MMC_TISR_TXUNDERFLOWERROR_POS 13
+#define MMC_TISR_TXUNDERFLOWERROR_LEN 1
+#define MMC_TISR_TXOCTETCOUNT_G_POS 14
+#define MMC_TISR_TXOCTETCOUNT_G_LEN 1
+#define MMC_TISR_TXFRAMECOUNT_G_POS 15
+#define MMC_TISR_TXFRAMECOUNT_G_LEN 1
+#define MMC_TISR_TXPAUSEFRAMES_POS 16
+#define MMC_TISR_TXPAUSEFRAMES_LEN 1
+#define MMC_TISR_TXVLANFRAMES_G_POS 17
+#define MMC_TISR_TXVLANFRAMES_G_LEN 1
+
+/* MTL register offsets */
+#define MTL_OMR 0x1000
+#define MTL_FDDR 0x1010
+#define MTL_RQDCM0R 0x1030
+
+#define MTL_RQDCM_INC 4
+#define MTL_RQDCM_Q_PER_REG 4
+
+/* MTL register entry bit positions and sizes */
+#define MTL_OMR_ETSALG_POS 5
+#define MTL_OMR_ETSALG_LEN 2
+#define MTL_OMR_RAA_POS 2
+#define MTL_OMR_RAA_LEN 1
+
+/* MTL queue register offsets
+ * Multiple queues can be active. The first queue has registers
+ * that begin at 0x1100. Each subsequent queue has registers that
+ * are accessed using an offset of 0x80 from the previous queue.
+ */
+#define MTL_Q_BASE 0x1100
+#define MTL_Q_INC 0x80
+
+#define MTL_Q_TQOMR 0x00
+#define MTL_Q_RQOMR 0x40
+#define MTL_Q_RQDR 0x48
+#define MTL_Q_RQFCR 0x50
+#define MTL_Q_IER 0x70
+#define MTL_Q_ISR 0x74
+
+/* MTL queue register entry bit positions and sizes */
+#define MTL_Q_RQDR_PRXQ_POS 16
+#define MTL_Q_RQDR_PRXQ_LEN 14
+#define MTL_Q_RQDR_RXQSTS_POS 4
+#define MTL_Q_RQDR_RXQSTS_LEN 2
+#define MTL_Q_RQFCR_RFA_POS 1
+#define MTL_Q_RQFCR_RFA_LEN 6
+#define MTL_Q_RQFCR_RFD_POS 17
+#define MTL_Q_RQFCR_RFD_LEN 6
+#define MTL_Q_RQOMR_EHFC_POS 7
+#define MTL_Q_RQOMR_EHFC_LEN 1
+#define MTL_Q_RQOMR_RQS_POS 16
+#define MTL_Q_RQOMR_RQS_LEN 9
+#define MTL_Q_RQOMR_RSF_POS 5
+#define MTL_Q_RQOMR_RSF_LEN 1
+#define MTL_Q_RQOMR_FEP_POS 4
+#define MTL_Q_RQOMR_FEP_LEN 1
+#define MTL_Q_RQOMR_FUP_POS 3
+#define MTL_Q_RQOMR_FUP_LEN 1
+#define MTL_Q_RQOMR_RTC_POS 0
+#define MTL_Q_RQOMR_RTC_LEN 2
+#define MTL_Q_TQOMR_FTQ_POS 0
+#define MTL_Q_TQOMR_FTQ_LEN 1
+#define MTL_Q_TQOMR_Q2TCMAP_POS 8
+#define MTL_Q_TQOMR_Q2TCMAP_LEN 3
+#define MTL_Q_TQOMR_TQS_POS 16
+#define MTL_Q_TQOMR_TQS_LEN 10
+#define MTL_Q_TQOMR_TSF_POS 1
+#define MTL_Q_TQOMR_TSF_LEN 1
+#define MTL_Q_TQOMR_TTC_POS 4
+#define MTL_Q_TQOMR_TTC_LEN 3
+#define MTL_Q_TQOMR_TXQEN_POS 2
+#define MTL_Q_TQOMR_TXQEN_LEN 2
+
+/* MTL queue register value */
+#define MTL_RSF_DISABLE 0x00
+#define MTL_RSF_ENABLE 0x01
+#define MTL_TSF_DISABLE 0x00
+#define MTL_TSF_ENABLE 0x01
+
+#define MTL_RX_THRESHOLD_64 0x00
+#define MTL_RX_THRESHOLD_96 0x02
+#define MTL_RX_THRESHOLD_128 0x03
+#define MTL_TX_THRESHOLD_64 0x00
+#define MTL_TX_THRESHOLD_96 0x02
+#define MTL_TX_THRESHOLD_128 0x03
+#define MTL_TX_THRESHOLD_192 0x04
+#define MTL_TX_THRESHOLD_256 0x05
+#define MTL_TX_THRESHOLD_384 0x06
+#define MTL_TX_THRESHOLD_512 0x07
+
+#define MTL_ETSALG_WRR 0x00
+#define MTL_ETSALG_WFQ 0x01
+#define MTL_ETSALG_DWRR 0x02
+#define MTL_RAA_SP 0x00
+#define MTL_RAA_WSP 0x01
+
+#define MTL_Q_DISABLED 0x00
+#define MTL_Q_ENABLED 0x02
+
+#define MTL_RQDCM0R_Q0MDMACH 0x0
+#define MTL_RQDCM0R_Q1MDMACH 0x00000100
+#define MTL_RQDCM0R_Q2MDMACH 0x00020000
+#define MTL_RQDCM0R_Q3MDMACH 0x03000000
+#define MTL_RQDCM1R_Q4MDMACH 0x00000004
+#define MTL_RQDCM1R_Q5MDMACH 0x00000500
+#define MTL_RQDCM1R_Q6MDMACH 0x00060000
+#define MTL_RQDCM1R_Q7MDMACH 0x07000000
+#define MTL_RQDCM2R_Q8MDMACH 0x00000008
+#define MTL_RQDCM2R_Q9MDMACH 0x00000900
+#define MTL_RQDCM2R_Q10MDMACH 0x000A0000
+#define MTL_RQDCM2R_Q11MDMACH 0x0B000000
+
+/* MTL traffic class register offsets
+ * Multiple traffic classes can be active. The first class has registers
+ * that begin at 0x1100. Each subsequent queue has registers that
+ * are accessed using an offset of 0x80 from the previous queue.
+ */
+#define MTL_TC_BASE MTL_Q_BASE
+#define MTL_TC_INC MTL_Q_INC
+
+#define MTL_TC_ETSCR 0x10
+#define MTL_TC_ETSSR 0x14
+#define MTL_TC_QWR 0x18
+
+/* MTL traffic class register entry bit positions and sizes */
+#define MTL_TC_ETSCR_TSA_POS 0
+#define MTL_TC_ETSCR_TSA_LEN 2
+#define MTL_TC_QWR_QW_POS 0
+#define MTL_TC_QWR_QW_LEN 21
+
+/* MTL traffic class register value */
+#define MTL_TSA_SP 0x00
+#define MTL_TSA_ETS 0x02
+
+/* DMA register offsets */
+#define DMA_MR 0x3000
+#define DMA_SBMR 0x3004
+#define DMA_ISR 0x3008
+#define DMA_DSR0 0x3020
+#define DMA_DSR1 0x3024
+
+/* DMA register entry bit positions and sizes */
+#define DMA_ISR_MACIS_POS 17
+#define DMA_ISR_MACIS_LEN 1
+#define DMA_ISR_MTLIS_POS 16
+#define DMA_ISR_MTLIS_LEN 1
+#define DMA_MR_SWR_POS 0
+#define DMA_MR_SWR_LEN 1
+#define DMA_SBMR_EAME_POS 11
+#define DMA_SBMR_EAME_LEN 1
+#define DMA_SBMR_BLEN_64_POS 5
+#define DMA_SBMR_BLEN_64_LEN 1
+#define DMA_SBMR_BLEN_128_POS 6
+#define DMA_SBMR_BLEN_128_LEN 1
+#define DMA_SBMR_BLEN_256_POS 7
+#define DMA_SBMR_BLEN_256_LEN 1
+#define DMA_SBMR_UNDEF_POS 0
+#define DMA_SBMR_UNDEF_LEN 1
+
+/* DMA register values */
+#define DMA_DSR_RPS_LEN 4
+#define DMA_DSR_TPS_LEN 4
+#define DMA_DSR_Q_LEN (DMA_DSR_RPS_LEN + DMA_DSR_TPS_LEN)
+#define DMA_DSR0_TPS_START 12
+#define DMA_DSRX_FIRST_QUEUE 3
+#define DMA_DSRX_INC 4
+#define DMA_DSRX_QPR 4
+#define DMA_DSRX_TPS_START 4
+#define DMA_TPS_STOPPED 0x00
+#define DMA_TPS_SUSPENDED 0x06
+
+/* DMA channel register offsets
+ * Multiple channels can be active. The first channel has registers
+ * that begin at 0x3100. Each subsequent channel has registers that
+ * are accessed using an offset of 0x80 from the previous channel.
+ */
+#define DMA_CH_BASE 0x3100
+#define DMA_CH_INC 0x80
+
+#define DMA_CH_CR 0x00
+#define DMA_CH_TCR 0x04
+#define DMA_CH_RCR 0x08
+#define DMA_CH_TDLR_HI 0x10
+#define DMA_CH_TDLR_LO 0x14
+#define DMA_CH_RDLR_HI 0x18
+#define DMA_CH_RDLR_LO 0x1c
+#define DMA_CH_TDTR_LO 0x24
+#define DMA_CH_RDTR_LO 0x2c
+#define DMA_CH_TDRLR 0x30
+#define DMA_CH_RDRLR 0x34
+#define DMA_CH_IER 0x38
+#define DMA_CH_RIWT 0x3c
+#define DMA_CH_SR 0x60
+
+/* DMA channel register entry bit positions and sizes */
+#define DMA_CH_CR_PBLX8_POS 16
+#define DMA_CH_CR_PBLX8_LEN 1
+#define DMA_CH_CR_SPH_POS 24
+#define DMA_CH_CR_SPH_LEN 1
+#define DMA_CH_IER_AIE_POS 15
+#define DMA_CH_IER_AIE_LEN 1
+#define DMA_CH_IER_FBEE_POS 12
+#define DMA_CH_IER_FBEE_LEN 1
+#define DMA_CH_IER_NIE_POS 16
+#define DMA_CH_IER_NIE_LEN 1
+#define DMA_CH_IER_RBUE_POS 7
+#define DMA_CH_IER_RBUE_LEN 1
+#define DMA_CH_IER_RIE_POS 6
+#define DMA_CH_IER_RIE_LEN 1
+#define DMA_CH_IER_RSE_POS 8
+#define DMA_CH_IER_RSE_LEN 1
+#define DMA_CH_IER_TBUE_POS 2
+#define DMA_CH_IER_TBUE_LEN 1
+#define DMA_CH_IER_TIE_POS 0
+#define DMA_CH_IER_TIE_LEN 1
+#define DMA_CH_IER_TXSE_POS 1
+#define DMA_CH_IER_TXSE_LEN 1
+#define DMA_CH_RCR_PBL_POS 16
+#define DMA_CH_RCR_PBL_LEN 6
+#define DMA_CH_RCR_RBSZ_POS 1
+#define DMA_CH_RCR_RBSZ_LEN 14
+#define DMA_CH_RCR_SR_POS 0
+#define DMA_CH_RCR_SR_LEN 1
+#define DMA_CH_RIWT_RWT_POS 0
+#define DMA_CH_RIWT_RWT_LEN 8
+#define DMA_CH_SR_FBE_POS 12
+#define DMA_CH_SR_FBE_LEN 1
+#define DMA_CH_SR_RBU_POS 7
+#define DMA_CH_SR_RBU_LEN 1
+#define DMA_CH_SR_RI_POS 6
+#define DMA_CH_SR_RI_LEN 1
+#define DMA_CH_SR_RPS_POS 8
+#define DMA_CH_SR_RPS_LEN 1
+#define DMA_CH_SR_TBU_POS 2
+#define DMA_CH_SR_TBU_LEN 1
+#define DMA_CH_SR_TI_POS 0
+#define DMA_CH_SR_TI_LEN 1
+#define DMA_CH_SR_TPS_POS 1
+#define DMA_CH_SR_TPS_LEN 1
+#define DMA_CH_TCR_OSP_POS 4
+#define DMA_CH_TCR_OSP_LEN 1
+#define DMA_CH_TCR_PBL_POS 16
+#define DMA_CH_TCR_PBL_LEN 6
+#define DMA_CH_TCR_ST_POS 0
+#define DMA_CH_TCR_ST_LEN 1
+#define DMA_CH_TCR_TSE_POS 12
+#define DMA_CH_TCR_TSE_LEN 1
+
+/* DMA channel register values */
+#define DMA_OSP_DISABLE 0x00
+#define DMA_OSP_ENABLE 0x01
+#define DMA_PBL_1 1
+#define DMA_PBL_2 2
+#define DMA_PBL_4 4
+#define DMA_PBL_8 8
+#define DMA_PBL_16 16
+#define DMA_PBL_32 32
+#define DMA_PBL_64 64
+#define DMA_PBL_128 128
+#define DMA_PBL_256 256
+#define DMA_PBL_X8_DISABLE 0x00
+#define DMA_PBL_X8_ENABLE 0x01
+
+/* Descriptor/Packet entry bit positions and sizes */
+#define RX_PACKET_ERRORS_CRC_POS 2
+#define RX_PACKET_ERRORS_CRC_LEN 1
+#define RX_PACKET_ERRORS_FRAME_POS 3
+#define RX_PACKET_ERRORS_FRAME_LEN 1
+#define RX_PACKET_ERRORS_LENGTH_POS 0
+#define RX_PACKET_ERRORS_LENGTH_LEN 1
+#define RX_PACKET_ERRORS_OVERRUN_POS 1
+#define RX_PACKET_ERRORS_OVERRUN_LEN 1
+
+#define RX_PACKET_ATTRIBUTES_CSUM_DONE_POS 0
+#define RX_PACKET_ATTRIBUTES_CSUM_DONE_LEN 1
+#define RX_PACKET_ATTRIBUTES_VLAN_CTAG_POS 1
+#define RX_PACKET_ATTRIBUTES_VLAN_CTAG_LEN 1
+#define RX_PACKET_ATTRIBUTES_INCOMPLETE_POS 2
+#define RX_PACKET_ATTRIBUTES_INCOMPLETE_LEN 1
+#define RX_PACKET_ATTRIBUTES_CONTEXT_NEXT_POS 3
+#define RX_PACKET_ATTRIBUTES_CONTEXT_NEXT_LEN 1
+#define RX_PACKET_ATTRIBUTES_CONTEXT_POS 4
+#define RX_PACKET_ATTRIBUTES_CONTEXT_LEN 1
+#define RX_PACKET_ATTRIBUTES_RX_TSTAMP_POS 5
+#define RX_PACKET_ATTRIBUTES_RX_TSTAMP_LEN 1
+#define RX_PACKET_ATTRIBUTES_RSS_HASH_POS 6
+#define RX_PACKET_ATTRIBUTES_RSS_HASH_LEN 1
+
+#define RX_NORMAL_DESC0_OVT_POS 0
+#define RX_NORMAL_DESC0_OVT_LEN 16
+#define RX_NORMAL_DESC2_HL_POS 0
+#define RX_NORMAL_DESC2_HL_LEN 10
+#define RX_NORMAL_DESC3_CDA_POS 27
+#define RX_NORMAL_DESC3_CDA_LEN 1
+#define RX_NORMAL_DESC3_CTXT_POS 30
+#define RX_NORMAL_DESC3_CTXT_LEN 1
+#define RX_NORMAL_DESC3_ES_POS 15
+#define RX_NORMAL_DESC3_ES_LEN 1
+#define RX_NORMAL_DESC3_ETLT_POS 16
+#define RX_NORMAL_DESC3_ETLT_LEN 4
+#define RX_NORMAL_DESC3_FD_POS 29
+#define RX_NORMAL_DESC3_FD_LEN 1
+#define RX_NORMAL_DESC3_INTE_POS 30
+#define RX_NORMAL_DESC3_INTE_LEN 1
+#define RX_NORMAL_DESC3_L34T_POS 20
+#define RX_NORMAL_DESC3_L34T_LEN 4
+#define RX_NORMAL_DESC3_LD_POS 28
+#define RX_NORMAL_DESC3_LD_LEN 1
+#define RX_NORMAL_DESC3_OWN_POS 31
+#define RX_NORMAL_DESC3_OWN_LEN 1
+#define RX_NORMAL_DESC3_PL_POS 0
+#define RX_NORMAL_DESC3_PL_LEN 14
+#define RX_NORMAL_DESC3_RSV_POS 26
+#define RX_NORMAL_DESC3_RSV_LEN 1
+
+#define RX_DESC3_L34T_IPV4_TCP 1
+#define RX_DESC3_L34T_IPV4_UDP 2
+#define RX_DESC3_L34T_IPV4_ICMP 3
+#define RX_DESC3_L34T_IPV6_TCP 9
+#define RX_DESC3_L34T_IPV6_UDP 10
+#define RX_DESC3_L34T_IPV6_ICMP 11
+
+#define RX_CONTEXT_DESC3_TSA_POS 4
+#define RX_CONTEXT_DESC3_TSA_LEN 1
+#define RX_CONTEXT_DESC3_TSD_POS 6
+#define RX_CONTEXT_DESC3_TSD_LEN 1
+
+#define TX_PACKET_ATTRIBUTES_CSUM_ENABLE_POS 0
+#define TX_PACKET_ATTRIBUTES_CSUM_ENABLE_LEN 1
+#define TX_PACKET_ATTRIBUTES_TSO_ENABLE_POS 1
+#define TX_PACKET_ATTRIBUTES_TSO_ENABLE_LEN 1
+#define TX_PACKET_ATTRIBUTES_VLAN_CTAG_POS 2
+#define TX_PACKET_ATTRIBUTES_VLAN_CTAG_LEN 1
+#define TX_PACKET_ATTRIBUTES_PTP_POS 3
+#define TX_PACKET_ATTRIBUTES_PTP_LEN 1
+
+#define TX_CONTEXT_DESC2_MSS_POS 0
+#define TX_CONTEXT_DESC2_MSS_LEN 15
+#define TX_CONTEXT_DESC3_CTXT_POS 30
+#define TX_CONTEXT_DESC3_CTXT_LEN 1
+#define TX_CONTEXT_DESC3_TCMSSV_POS 26
+#define TX_CONTEXT_DESC3_TCMSSV_LEN 1
+#define TX_CONTEXT_DESC3_VLTV_POS 16
+#define TX_CONTEXT_DESC3_VLTV_LEN 1
+#define TX_CONTEXT_DESC3_VT_POS 0
+#define TX_CONTEXT_DESC3_VT_LEN 16
+
+#define TX_NORMAL_DESC2_HL_B1L_POS 0
+#define TX_NORMAL_DESC2_HL_B1L_LEN 14
+#define TX_NORMAL_DESC2_IC_POS 31
+#define TX_NORMAL_DESC2_IC_LEN 1
+#define TX_NORMAL_DESC2_TTSE_POS 30
+#define TX_NORMAL_DESC2_TTSE_LEN 1
+#define TX_NORMAL_DESC2_VTIR_POS 14
+#define TX_NORMAL_DESC2_VTIR_LEN 2
+#define TX_NORMAL_DESC3_CIC_POS 16
+#define TX_NORMAL_DESC3_CIC_LEN 2
+#define TX_NORMAL_DESC3_CPC_POS 26
+#define TX_NORMAL_DESC3_CPC_LEN 2
+#define TX_NORMAL_DESC3_CTXT_POS 30
+#define TX_NORMAL_DESC3_CTXT_LEN 1
+#define TX_NORMAL_DESC3_FD_POS 29
+#define TX_NORMAL_DESC3_FD_LEN 1
+#define TX_NORMAL_DESC3_FL_POS 0
+#define TX_NORMAL_DESC3_FL_LEN 15
+#define TX_NORMAL_DESC3_LD_POS 28
+#define TX_NORMAL_DESC3_LD_LEN 1
+#define TX_NORMAL_DESC3_OWN_POS 31
+#define TX_NORMAL_DESC3_OWN_LEN 1
+#define TX_NORMAL_DESC3_TCPHDRLEN_POS 19
+#define TX_NORMAL_DESC3_TCPHDRLEN_LEN 4
+#define TX_NORMAL_DESC3_TCPPL_POS 0
+#define TX_NORMAL_DESC3_TCPPL_LEN 18
+#define TX_NORMAL_DESC3_TSE_POS 18
+#define TX_NORMAL_DESC3_TSE_LEN 1
+
+#define TX_NORMAL_DESC2_VLAN_INSERT 0x2
+
+#define XLGMAC_MTL_REG(pdata, n, reg) \
+ ((pdata)->mac_regs + MTL_Q_BASE + ((n) * MTL_Q_INC) + (reg))
+
+#define XLGMAC_DMA_REG(channel, reg) ((channel)->dma_regs + (reg))
+
+#endif /* __DWC_XLGMAC_REG_H__ */
--- /dev/null
+/* Synopsys DesignWare Core Enterprise Ethernet (XLGMAC) Driver
+ *
+ * Copyright (c) 2017 Synopsys, Inc. (www.synopsys.com)
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This Synopsys DWC XLGMAC software driver and associated documentation
+ * (hereinafter the "Software") is an unsupported proprietary work of
+ * Synopsys, Inc. unless otherwise expressly agreed to in writing between
+ * Synopsys and you. The Software IS NOT an item of Licensed Software or a
+ * Licensed Product under any End User Software License Agreement or
+ * Agreement for Licensed Products with Synopsys or any supplement thereto.
+ * Synopsys is a registered trademark of Synopsys, Inc. Other names included
+ * in the SOFTWARE may be the trademarks of their respective owners.
+ */
+
+#ifndef __DWC_XLGMAC_H__
+#define __DWC_XLGMAC_H__
+
+#include <linux/dma-mapping.h>
+#include <linux/netdevice.h>
+#include <linux/workqueue.h>
+#include <linux/phy.h>
+#include <linux/if_vlan.h>
+#include <linux/bitops.h>
+#include <linux/timecounter.h>
+
+#define XLGMAC_DRV_NAME "dwc-xlgmac"
+#define XLGMAC_DRV_VERSION "1.0.0"
+#define XLGMAC_DRV_DESC "Synopsys DWC XLGMAC Driver"
+
+/* Descriptor related parameters */
+#define XLGMAC_TX_DESC_CNT 1024
+#define XLGMAC_TX_DESC_MIN_FREE (XLGMAC_TX_DESC_CNT >> 3)
+#define XLGMAC_TX_DESC_MAX_PROC (XLGMAC_TX_DESC_CNT >> 1)
+#define XLGMAC_RX_DESC_CNT 1024
+#define XLGMAC_RX_DESC_MAX_DIRTY (XLGMAC_RX_DESC_CNT >> 3)
+
+/* Descriptors required for maximum contiguous TSO/GSO packet */
+#define XLGMAC_TX_MAX_SPLIT ((GSO_MAX_SIZE / XLGMAC_TX_MAX_BUF_SIZE) + 1)
+
+/* Maximum possible descriptors needed for a SKB */
+#define XLGMAC_TX_MAX_DESC_NR (MAX_SKB_FRAGS + XLGMAC_TX_MAX_SPLIT + 2)
+
+#define XLGMAC_TX_MAX_BUF_SIZE (0x3fff & ~(64 - 1))
+#define XLGMAC_RX_MIN_BUF_SIZE (ETH_FRAME_LEN + ETH_FCS_LEN + VLAN_HLEN)
+#define XLGMAC_RX_BUF_ALIGN 64
+
+/* Maximum Size for Splitting the Header Data
+ * Keep in sync with SKB_ALLOC_SIZE
+ * 3'b000: 64 bytes, 3'b001: 128 bytes
+ * 3'b010: 256 bytes, 3'b011: 512 bytes
+ * 3'b100: 1023 bytes , 3'b101'3'b111: Reserved
+ */
+#define XLGMAC_SPH_HDSMS_SIZE 3
+#define XLGMAC_SKB_ALLOC_SIZE 512
+
+#define XLGMAC_MAX_FIFO 81920
+
+#define XLGMAC_MAX_DMA_CHANNELS 16
+#define XLGMAC_DMA_STOP_TIMEOUT 5
+#define XLGMAC_DMA_INTERRUPT_MASK 0x31c7
+
+/* Default coalescing parameters */
+#define XLGMAC_INIT_DMA_TX_USECS 1000
+#define XLGMAC_INIT_DMA_TX_FRAMES 25
+#define XLGMAC_INIT_DMA_RX_USECS 30
+#define XLGMAC_INIT_DMA_RX_FRAMES 25
+
+/* Flow control queue count */
+#define XLGMAC_MAX_FLOW_CONTROL_QUEUES 8
+
+/* System clock is 125 MHz */
+#define XLGMAC_SYSCLOCK 125000000
+
+/* Maximum MAC address hash table size (256 bits = 8 bytes) */
+#define XLGMAC_MAC_HASH_TABLE_SIZE 8
+
+/* Receive Side Scaling */
+#define XLGMAC_RSS_HASH_KEY_SIZE 40
+#define XLGMAC_RSS_MAX_TABLE_SIZE 256
+#define XLGMAC_RSS_LOOKUP_TABLE_TYPE 0
+#define XLGMAC_RSS_HASH_KEY_TYPE 1
+
+#define XLGMAC_STD_PACKET_MTU 1500
+#define XLGMAC_JUMBO_PACKET_MTU 9000
+
+/* Helper macro for descriptor handling
+ * Always use XLGMAC_GET_DESC_DATA to access the descriptor data
+ */
+#define XLGMAC_GET_DESC_DATA(ring, idx) ({ \
+ typeof(ring) _ring = (ring); \
+ ((_ring)->desc_data_head + \
+ ((idx) & ((_ring)->dma_desc_count - 1))); \
+})
+
+#define XLGMAC_GET_REG_BITS(var, pos, len) ({ \
+ typeof(pos) _pos = (pos); \
+ typeof(len) _len = (len); \
+ ((var) & GENMASK(_pos + _len - 1, _pos)) >> (_pos); \
+})
+
+#define XLGMAC_GET_REG_BITS_LE(var, pos, len) ({ \
+ typeof(pos) _pos = (pos); \
+ typeof(len) _len = (len); \
+ typeof(var) _var = le32_to_cpu((var)); \
+ ((_var) & GENMASK(_pos + _len - 1, _pos)) >> (_pos); \
+})
+
+#define XLGMAC_SET_REG_BITS(var, pos, len, val) ({ \
+ typeof(var) _var = (var); \
+ typeof(pos) _pos = (pos); \
+ typeof(len) _len = (len); \
+ typeof(val) _val = (val); \
+ _val = (_val << _pos) & GENMASK(_pos + _len - 1, _pos); \
+ _var = (_var & ~GENMASK(_pos + _len - 1, _pos)) | _val; \
+})
+
+#define XLGMAC_SET_REG_BITS_LE(var, pos, len, val) ({ \
+ typeof(var) _var = (var); \
+ typeof(pos) _pos = (pos); \
+ typeof(len) _len = (len); \
+ typeof(val) _val = (val); \
+ _val = (_val << _pos) & GENMASK(_pos + _len - 1, _pos); \
+ _var = (_var & ~GENMASK(_pos + _len - 1, _pos)) | _val; \
+ cpu_to_le32(_var); \
+})
+
+struct xlgmac_pdata;
+
+enum xlgmac_int {
+ XLGMAC_INT_DMA_CH_SR_TI,
+ XLGMAC_INT_DMA_CH_SR_TPS,
+ XLGMAC_INT_DMA_CH_SR_TBU,
+ XLGMAC_INT_DMA_CH_SR_RI,
+ XLGMAC_INT_DMA_CH_SR_RBU,
+ XLGMAC_INT_DMA_CH_SR_RPS,
+ XLGMAC_INT_DMA_CH_SR_TI_RI,
+ XLGMAC_INT_DMA_CH_SR_FBE,
+ XLGMAC_INT_DMA_ALL,
+};
+
+struct xlgmac_stats {
+ /* MMC TX counters */
+ u64 txoctetcount_gb;
+ u64 txframecount_gb;
+ u64 txbroadcastframes_g;
+ u64 txmulticastframes_g;
+ u64 tx64octets_gb;
+ u64 tx65to127octets_gb;
+ u64 tx128to255octets_gb;
+ u64 tx256to511octets_gb;
+ u64 tx512to1023octets_gb;
+ u64 tx1024tomaxoctets_gb;
+ u64 txunicastframes_gb;
+ u64 txmulticastframes_gb;
+ u64 txbroadcastframes_gb;
+ u64 txunderflowerror;
+ u64 txoctetcount_g;
+ u64 txframecount_g;
+ u64 txpauseframes;
+ u64 txvlanframes_g;
+
+ /* MMC RX counters */
+ u64 rxframecount_gb;
+ u64 rxoctetcount_gb;
+ u64 rxoctetcount_g;
+ u64 rxbroadcastframes_g;
+ u64 rxmulticastframes_g;
+ u64 rxcrcerror;
+ u64 rxrunterror;
+ u64 rxjabbererror;
+ u64 rxundersize_g;
+ u64 rxoversize_g;
+ u64 rx64octets_gb;
+ u64 rx65to127octets_gb;
+ u64 rx128to255octets_gb;
+ u64 rx256to511octets_gb;
+ u64 rx512to1023octets_gb;
+ u64 rx1024tomaxoctets_gb;
+ u64 rxunicastframes_g;
+ u64 rxlengtherror;
+ u64 rxoutofrangetype;
+ u64 rxpauseframes;
+ u64 rxfifooverflow;
+ u64 rxvlanframes_gb;
+ u64 rxwatchdogerror;
+
+ /* Extra counters */
+ u64 tx_tso_packets;
+ u64 rx_split_header_packets;
+ u64 rx_buffer_unavailable;
+};
+
+struct xlgmac_ring_buf {
+ struct sk_buff *skb;
+ dma_addr_t skb_dma;
+ unsigned int skb_len;
+};
+
+/* Common Tx and Rx DMA hardware descriptor */
+struct xlgmac_dma_desc {
+ __le32 desc0;
+ __le32 desc1;
+ __le32 desc2;
+ __le32 desc3;
+};
+
+/* Page allocation related values */
+struct xlgmac_page_alloc {
+ struct page *pages;
+ unsigned int pages_len;
+ unsigned int pages_offset;
+
+ dma_addr_t pages_dma;
+};
+
+/* Ring entry buffer data */
+struct xlgmac_buffer_data {
+ struct xlgmac_page_alloc pa;
+ struct xlgmac_page_alloc pa_unmap;
+
+ dma_addr_t dma_base;
+ unsigned long dma_off;
+ unsigned int dma_len;
+};
+
+/* Tx-related desc data */
+struct xlgmac_tx_desc_data {
+ unsigned int packets; /* BQL packet count */
+ unsigned int bytes; /* BQL byte count */
+};
+
+/* Rx-related desc data */
+struct xlgmac_rx_desc_data {
+ struct xlgmac_buffer_data hdr; /* Header locations */
+ struct xlgmac_buffer_data buf; /* Payload locations */
+
+ unsigned short hdr_len; /* Length of received header */
+ unsigned short len; /* Length of received packet */
+};
+
+struct xlgmac_pkt_info {
+ struct sk_buff *skb;
+
+ unsigned int attributes;
+
+ unsigned int errors;
+
+ /* descriptors needed for this packet */
+ unsigned int desc_count;
+ unsigned int length;
+
+ unsigned int tx_packets;
+ unsigned int tx_bytes;
+
+ unsigned int header_len;
+ unsigned int tcp_header_len;
+ unsigned int tcp_payload_len;
+ unsigned short mss;
+
+ unsigned short vlan_ctag;
+
+ u64 rx_tstamp;
+
+ u32 rss_hash;
+ enum pkt_hash_types rss_hash_type;
+};
+
+struct xlgmac_desc_data {
+ /* dma_desc: Virtual address of descriptor
+ * dma_desc_addr: DMA address of descriptor
+ */
+ struct xlgmac_dma_desc *dma_desc;
+ dma_addr_t dma_desc_addr;
+
+ /* skb: Virtual address of SKB
+ * skb_dma: DMA address of SKB data
+ * skb_dma_len: Length of SKB DMA area
+ */
+ struct sk_buff *skb;
+ dma_addr_t skb_dma;
+ unsigned int skb_dma_len;
+
+ /* Tx/Rx -related data */
+ struct xlgmac_tx_desc_data tx;
+ struct xlgmac_rx_desc_data rx;
+
+ unsigned int mapped_as_page;
+
+ /* Incomplete receive save location. If the budget is exhausted
+ * or the last descriptor (last normal descriptor or a following
+ * context descriptor) has not been DMA'd yet the current state
+ * of the receive processing needs to be saved.
+ */
+ unsigned int state_saved;
+ struct {
+ struct sk_buff *skb;
+ unsigned int len;
+ unsigned int error;
+ } state;
+};
+
+struct xlgmac_ring {
+ /* Per packet related information */
+ struct xlgmac_pkt_info pkt_info;
+
+ /* Virtual/DMA addresses of DMA descriptor list and the total count */
+ struct xlgmac_dma_desc *dma_desc_head;
+ dma_addr_t dma_desc_head_addr;
+ unsigned int dma_desc_count;
+
+ /* Array of descriptor data corresponding the DMA descriptor
+ * (always use the XLGMAC_GET_DESC_DATA macro to access this data)
+ */
+ struct xlgmac_desc_data *desc_data_head;
+
+ /* Page allocation for RX buffers */
+ struct xlgmac_page_alloc rx_hdr_pa;
+ struct xlgmac_page_alloc rx_buf_pa;
+
+ /* Ring index values
+ * cur - Tx: index of descriptor to be used for current transfer
+ * Rx: index of descriptor to check for packet availability
+ * dirty - Tx: index of descriptor to check for transfer complete
+ * Rx: index of descriptor to check for buffer reallocation
+ */
+ unsigned int cur;
+ unsigned int dirty;
+
+ /* Coalesce frame count used for interrupt bit setting */
+ unsigned int coalesce_count;
+
+ union {
+ struct {
+ unsigned int xmit_more;
+ unsigned int queue_stopped;
+ unsigned short cur_mss;
+ unsigned short cur_vlan_ctag;
+ } tx;
+ };
+} ____cacheline_aligned;
+
+struct xlgmac_channel {
+ char name[16];
+
+ /* Address of private data area for device */
+ struct xlgmac_pdata *pdata;
+
+ /* Queue index and base address of queue's DMA registers */
+ unsigned int queue_index;
+ void __iomem *dma_regs;
+
+ /* Per channel interrupt irq number */
+ int dma_irq;
+ char dma_irq_name[IFNAMSIZ + 32];
+
+ /* Netdev related settings */
+ struct napi_struct napi;
+
+ unsigned int saved_ier;
+
+ unsigned int tx_timer_active;
+ struct timer_list tx_timer;
+
+ struct xlgmac_ring *tx_ring;
+ struct xlgmac_ring *rx_ring;
+} ____cacheline_aligned;
+
+struct xlgmac_desc_ops {
+ int (*alloc_channles_and_rings)(struct xlgmac_pdata *pdata);
+ void (*free_channels_and_rings)(struct xlgmac_pdata *pdata);
+ int (*map_tx_skb)(struct xlgmac_channel *channel,
+ struct sk_buff *skb);
+ int (*map_rx_buffer)(struct xlgmac_pdata *pdata,
+ struct xlgmac_ring *ring,
+ struct xlgmac_desc_data *desc_data);
+ void (*unmap_desc_data)(struct xlgmac_pdata *pdata,
+ struct xlgmac_desc_data *desc_data);
+ void (*tx_desc_init)(struct xlgmac_pdata *pdata);
+ void (*rx_desc_init)(struct xlgmac_pdata *pdata);
+};
+
+struct xlgmac_hw_ops {
+ int (*init)(struct xlgmac_pdata *pdata);
+ int (*exit)(struct xlgmac_pdata *pdata);
+
+ int (*tx_complete)(struct xlgmac_dma_desc *dma_desc);
+
+ void (*enable_tx)(struct xlgmac_pdata *pdata);
+ void (*disable_tx)(struct xlgmac_pdata *pdata);
+ void (*enable_rx)(struct xlgmac_pdata *pdata);
+ void (*disable_rx)(struct xlgmac_pdata *pdata);
+
+ int (*enable_int)(struct xlgmac_channel *channel,
+ enum xlgmac_int int_id);
+ int (*disable_int)(struct xlgmac_channel *channel,
+ enum xlgmac_int int_id);
+ void (*dev_xmit)(struct xlgmac_channel *channel);
+ int (*dev_read)(struct xlgmac_channel *channel);
+
+ int (*set_mac_address)(struct xlgmac_pdata *pdata, u8 *addr);
+ int (*config_rx_mode)(struct xlgmac_pdata *pdata);
+ int (*enable_rx_csum)(struct xlgmac_pdata *pdata);
+ int (*disable_rx_csum)(struct xlgmac_pdata *pdata);
+
+ /* For MII speed configuration */
+ int (*set_xlgmii_25000_speed)(struct xlgmac_pdata *pdata);
+ int (*set_xlgmii_40000_speed)(struct xlgmac_pdata *pdata);
+ int (*set_xlgmii_50000_speed)(struct xlgmac_pdata *pdata);
+ int (*set_xlgmii_100000_speed)(struct xlgmac_pdata *pdata);
+
+ /* For descriptor related operation */
+ void (*tx_desc_init)(struct xlgmac_channel *channel);
+ void (*rx_desc_init)(struct xlgmac_channel *channel);
+ void (*tx_desc_reset)(struct xlgmac_desc_data *desc_data);
+ void (*rx_desc_reset)(struct xlgmac_pdata *pdata,
+ struct xlgmac_desc_data *desc_data,
+ unsigned int index);
+ int (*is_last_desc)(struct xlgmac_dma_desc *dma_desc);
+ int (*is_context_desc)(struct xlgmac_dma_desc *dma_desc);
+ void (*tx_start_xmit)(struct xlgmac_channel *channel,
+ struct xlgmac_ring *ring);
+
+ /* For Flow Control */
+ int (*config_tx_flow_control)(struct xlgmac_pdata *pdata);
+ int (*config_rx_flow_control)(struct xlgmac_pdata *pdata);
+
+ /* For Vlan related config */
+ int (*enable_rx_vlan_stripping)(struct xlgmac_pdata *pdata);
+ int (*disable_rx_vlan_stripping)(struct xlgmac_pdata *pdata);
+ int (*enable_rx_vlan_filtering)(struct xlgmac_pdata *pdata);
+ int (*disable_rx_vlan_filtering)(struct xlgmac_pdata *pdata);
+ int (*update_vlan_hash_table)(struct xlgmac_pdata *pdata);
+
+ /* For RX coalescing */
+ int (*config_rx_coalesce)(struct xlgmac_pdata *pdata);
+ int (*config_tx_coalesce)(struct xlgmac_pdata *pdata);
+ unsigned int (*usec_to_riwt)(struct xlgmac_pdata *pdata,
+ unsigned int usec);
+ unsigned int (*riwt_to_usec)(struct xlgmac_pdata *pdata,
+ unsigned int riwt);
+
+ /* For RX and TX threshold config */
+ int (*config_rx_threshold)(struct xlgmac_pdata *pdata,
+ unsigned int val);
+ int (*config_tx_threshold)(struct xlgmac_pdata *pdata,
+ unsigned int val);
+
+ /* For RX and TX Store and Forward Mode config */
+ int (*config_rsf_mode)(struct xlgmac_pdata *pdata,
+ unsigned int val);
+ int (*config_tsf_mode)(struct xlgmac_pdata *pdata,
+ unsigned int val);
+
+ /* For TX DMA Operate on Second Frame config */
+ int (*config_osp_mode)(struct xlgmac_pdata *pdata);
+
+ /* For RX and TX PBL config */
+ int (*config_rx_pbl_val)(struct xlgmac_pdata *pdata);
+ int (*get_rx_pbl_val)(struct xlgmac_pdata *pdata);
+ int (*config_tx_pbl_val)(struct xlgmac_pdata *pdata);
+ int (*get_tx_pbl_val)(struct xlgmac_pdata *pdata);
+ int (*config_pblx8)(struct xlgmac_pdata *pdata);
+
+ /* For MMC statistics */
+ void (*rx_mmc_int)(struct xlgmac_pdata *pdata);
+ void (*tx_mmc_int)(struct xlgmac_pdata *pdata);
+ void (*read_mmc_stats)(struct xlgmac_pdata *pdata);
+
+ /* For Receive Side Scaling */
+ int (*enable_rss)(struct xlgmac_pdata *pdata);
+ int (*disable_rss)(struct xlgmac_pdata *pdata);
+ int (*set_rss_hash_key)(struct xlgmac_pdata *pdata,
+ const u8 *key);
+ int (*set_rss_lookup_table)(struct xlgmac_pdata *pdata,
+ const u32 *table);
+};
+
+/* This structure contains flags that indicate what hardware features
+ * or configurations are present in the device.
+ */
+struct xlgmac_hw_features {
+ /* HW Version */
+ unsigned int version;
+
+ /* HW Feature Register0 */
+ unsigned int phyifsel; /* PHY interface support */
+ unsigned int vlhash; /* VLAN Hash Filter */
+ unsigned int sma; /* SMA(MDIO) Interface */
+ unsigned int rwk; /* PMT remote wake-up packet */
+ unsigned int mgk; /* PMT magic packet */
+ unsigned int mmc; /* RMON module */
+ unsigned int aoe; /* ARP Offload */
+ unsigned int ts; /* IEEE 1588-2008 Advanced Timestamp */
+ unsigned int eee; /* Energy Efficient Ethernet */
+ unsigned int tx_coe; /* Tx Checksum Offload */
+ unsigned int rx_coe; /* Rx Checksum Offload */
+ unsigned int addn_mac; /* Additional MAC Addresses */
+ unsigned int ts_src; /* Timestamp Source */
+ unsigned int sa_vlan_ins; /* Source Address or VLAN Insertion */
+
+ /* HW Feature Register1 */
+ unsigned int rx_fifo_size; /* MTL Receive FIFO Size */
+ unsigned int tx_fifo_size; /* MTL Transmit FIFO Size */
+ unsigned int adv_ts_hi; /* Advance Timestamping High Word */
+ unsigned int dma_width; /* DMA width */
+ unsigned int dcb; /* DCB Feature */
+ unsigned int sph; /* Split Header Feature */
+ unsigned int tso; /* TCP Segmentation Offload */
+ unsigned int dma_debug; /* DMA Debug Registers */
+ unsigned int rss; /* Receive Side Scaling */
+ unsigned int tc_cnt; /* Number of Traffic Classes */
+ unsigned int hash_table_size; /* Hash Table Size */
+ unsigned int l3l4_filter_num; /* Number of L3-L4 Filters */
+
+ /* HW Feature Register2 */
+ unsigned int rx_q_cnt; /* Number of MTL Receive Queues */
+ unsigned int tx_q_cnt; /* Number of MTL Transmit Queues */
+ unsigned int rx_ch_cnt; /* Number of DMA Receive Channels */
+ unsigned int tx_ch_cnt; /* Number of DMA Transmit Channels */
+ unsigned int pps_out_num; /* Number of PPS outputs */
+ unsigned int aux_snap_num; /* Number of Aux snapshot inputs */
+};
+
+struct xlgmac_resources {
+ void __iomem *addr;
+ int irq;
+};
+
+struct xlgmac_pdata {
+ struct net_device *netdev;
+ struct device *dev;
+
+ struct xlgmac_hw_ops hw_ops;
+ struct xlgmac_desc_ops desc_ops;
+
+ /* Device statistics */
+ struct xlgmac_stats stats;
+
+ u32 msg_enable;
+
+ /* MAC registers base */
+ void __iomem *mac_regs;
+
+ /* Hardware features of the device */
+ struct xlgmac_hw_features hw_feat;
+
+ struct work_struct restart_work;
+
+ /* Rings for Tx/Rx on a DMA channel */
+ struct xlgmac_channel *channel_head;
+ unsigned int channel_count;
+ unsigned int tx_ring_count;
+ unsigned int rx_ring_count;
+ unsigned int tx_desc_count;
+ unsigned int rx_desc_count;
+ unsigned int tx_q_count;
+ unsigned int rx_q_count;
+
+ /* Tx/Rx common settings */
+ unsigned int pblx8;
+
+ /* Tx settings */
+ unsigned int tx_sf_mode;
+ unsigned int tx_threshold;
+ unsigned int tx_pbl;
+ unsigned int tx_osp_mode;
+
+ /* Rx settings */
+ unsigned int rx_sf_mode;
+ unsigned int rx_threshold;
+ unsigned int rx_pbl;
+
+ /* Tx coalescing settings */
+ unsigned int tx_usecs;
+ unsigned int tx_frames;
+
+ /* Rx coalescing settings */
+ unsigned int rx_riwt;
+ unsigned int rx_usecs;
+ unsigned int rx_frames;
+
+ /* Current Rx buffer size */
+ unsigned int rx_buf_size;
+
+ /* Flow control settings */
+ unsigned int tx_pause;
+ unsigned int rx_pause;
+
+ /* Device interrupt number */
+ int dev_irq;
+ unsigned int per_channel_irq;
+ int channel_irq[XLGMAC_MAX_DMA_CHANNELS];
+
+ /* Netdev related settings */
+ unsigned char mac_addr[ETH_ALEN];
+ netdev_features_t netdev_features;
+ struct napi_struct napi;
+
+ /* Filtering support */
+ unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
+
+ /* Device clocks */
+ unsigned long sysclk_rate;
+
+ /* RSS addressing mutex */
+ struct mutex rss_mutex;
+
+ /* Receive Side Scaling settings */
+ u8 rss_key[XLGMAC_RSS_HASH_KEY_SIZE];
+ u32 rss_table[XLGMAC_RSS_MAX_TABLE_SIZE];
+ u32 rss_options;
+
+ int phy_speed;
+
+ char drv_name[32];
+ char drv_ver[32];
+};
+
+void xlgmac_init_desc_ops(struct xlgmac_desc_ops *desc_ops);
+void xlgmac_init_hw_ops(struct xlgmac_hw_ops *hw_ops);
+const struct net_device_ops *xlgmac_get_netdev_ops(void);
+void xlgmac_dump_tx_desc(struct xlgmac_pdata *pdata,
+ struct xlgmac_ring *ring,
+ unsigned int idx,
+ unsigned int count,
+ unsigned int flag);
+void xlgmac_dump_rx_desc(struct xlgmac_pdata *pdata,
+ struct xlgmac_ring *ring,
+ unsigned int idx);
+void xlgmac_print_pkt(struct net_device *netdev,
+ struct sk_buff *skb, bool tx_rx);
+void xlgmac_get_all_hw_features(struct xlgmac_pdata *pdata);
+void xlgmac_print_all_hw_features(struct xlgmac_pdata *pdata);
+int xlgmac_drv_probe(struct device *dev,
+ struct xlgmac_resources *res);
+int xlgmac_drv_remove(struct device *dev);
+
+/* For debug prints */
+#ifdef XLGMAC_DEBUG
+#define XLGMAC_PR(fmt, args...) \
+ pr_alert("[%s,%d]:" fmt, __func__, __LINE__, ## args)
+#else
+#define XLGMAC_PR(x...) do { } while (0)
+#endif
+
+#endif /* __DWC_XLGMAC_H__ */
TSI_WRITE(TSI108_EC_INTMASK, ~0);
}
-static int tsi108_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
+static int tsi108_get_link_ksettings(struct net_device *dev,
+ struct ethtool_link_ksettings *cmd)
{
struct tsi108_prv_data *data = netdev_priv(dev);
unsigned long flags;
int rc;
spin_lock_irqsave(&data->txlock, flags);
- rc = mii_ethtool_gset(&data->mii_if, cmd);
+ rc = mii_ethtool_get_link_ksettings(&data->mii_if, cmd);
spin_unlock_irqrestore(&data->txlock, flags);
return rc;
}
-static int tsi108_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
+static int tsi108_set_link_ksettings(struct net_device *dev,
+ const struct ethtool_link_ksettings *cmd)
{
struct tsi108_prv_data *data = netdev_priv(dev);
unsigned long flags;
int rc;
spin_lock_irqsave(&data->txlock, flags);
- rc = mii_ethtool_sset(&data->mii_if, cmd);
+ rc = mii_ethtool_set_link_ksettings(&data->mii_if, cmd);
spin_unlock_irqrestore(&data->txlock, flags);
return rc;
static const struct ethtool_ops tsi108_ethtool_ops = {
.get_link = ethtool_op_get_link,
- .get_settings = tsi108_get_settings,
- .set_settings = tsi108_set_settings,
+ .get_link_ksettings = tsi108_get_link_ksettings,
+ .set_link_ksettings = tsi108_set_link_ksettings,
};
static const struct net_device_ops tsi108_netdev_ops = {
strlcpy(info->bus_info, dev_name(hwdev), sizeof(info->bus_info));
}
-static int netdev_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
+static int netdev_get_link_ksettings(struct net_device *dev,
+ struct ethtool_link_ksettings *cmd)
{
struct rhine_private *rp = netdev_priv(dev);
int rc;
mutex_lock(&rp->task_lock);
- rc = mii_ethtool_gset(&rp->mii_if, cmd);
+ rc = mii_ethtool_get_link_ksettings(&rp->mii_if, cmd);
mutex_unlock(&rp->task_lock);
return rc;
}
-static int netdev_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
+static int netdev_set_link_ksettings(struct net_device *dev,
+ const struct ethtool_link_ksettings *cmd)
{
struct rhine_private *rp = netdev_priv(dev);
int rc;
mutex_lock(&rp->task_lock);
- rc = mii_ethtool_sset(&rp->mii_if, cmd);
+ rc = mii_ethtool_set_link_ksettings(&rp->mii_if, cmd);
rhine_set_carrier(&rp->mii_if);
mutex_unlock(&rp->task_lock);
static const struct ethtool_ops netdev_ethtool_ops = {
.get_drvinfo = netdev_get_drvinfo,
- .get_settings = netdev_get_settings,
- .set_settings = netdev_set_settings,
.nway_reset = netdev_nway_reset,
.get_link = netdev_get_link,
.get_msglevel = netdev_get_msglevel,
.set_msglevel = netdev_set_msglevel,
.get_wol = rhine_get_wol,
.set_wol = rhine_set_wol,
+ .get_link_ksettings = netdev_get_link_ksettings,
+ .set_link_ksettings = netdev_set_link_ksettings,
};
static int netdev_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
velocity_set_power_state(vptr, PCI_D3hot);
}
-static int velocity_get_settings(struct net_device *dev,
- struct ethtool_cmd *cmd)
+static int velocity_get_link_ksettings(struct net_device *dev,
+ struct ethtool_link_ksettings *cmd)
{
struct velocity_info *vptr = netdev_priv(dev);
struct mac_regs __iomem *regs = vptr->mac_regs;
u32 status;
+ u32 supported, advertising;
+
status = check_connection_type(vptr->mac_regs);
- cmd->supported = SUPPORTED_TP |
+ supported = SUPPORTED_TP |
SUPPORTED_Autoneg |
SUPPORTED_10baseT_Half |
SUPPORTED_10baseT_Full |
SUPPORTED_1000baseT_Half |
SUPPORTED_1000baseT_Full;
- cmd->advertising = ADVERTISED_TP | ADVERTISED_Autoneg;
+ advertising = ADVERTISED_TP | ADVERTISED_Autoneg;
if (vptr->options.spd_dpx == SPD_DPX_AUTO) {
- cmd->advertising |=
+ advertising |=
ADVERTISED_10baseT_Half |
ADVERTISED_10baseT_Full |
ADVERTISED_100baseT_Half |
} else {
switch (vptr->options.spd_dpx) {
case SPD_DPX_1000_FULL:
- cmd->advertising |= ADVERTISED_1000baseT_Full;
+ advertising |= ADVERTISED_1000baseT_Full;
break;
case SPD_DPX_100_HALF:
- cmd->advertising |= ADVERTISED_100baseT_Half;
+ advertising |= ADVERTISED_100baseT_Half;
break;
case SPD_DPX_100_FULL:
- cmd->advertising |= ADVERTISED_100baseT_Full;
+ advertising |= ADVERTISED_100baseT_Full;
break;
case SPD_DPX_10_HALF:
- cmd->advertising |= ADVERTISED_10baseT_Half;
+ advertising |= ADVERTISED_10baseT_Half;
break;
case SPD_DPX_10_FULL:
- cmd->advertising |= ADVERTISED_10baseT_Full;
+ advertising |= ADVERTISED_10baseT_Full;
break;
default:
break;
}
if (status & VELOCITY_SPEED_1000)
- ethtool_cmd_speed_set(cmd, SPEED_1000);
+ cmd->base.speed = SPEED_1000;
else if (status & VELOCITY_SPEED_100)
- ethtool_cmd_speed_set(cmd, SPEED_100);
+ cmd->base.speed = SPEED_100;
else
- ethtool_cmd_speed_set(cmd, SPEED_10);
+ cmd->base.speed = SPEED_10;
- cmd->autoneg = (status & VELOCITY_AUTONEG_ENABLE) ? AUTONEG_ENABLE : AUTONEG_DISABLE;
- cmd->port = PORT_TP;
- cmd->transceiver = XCVR_INTERNAL;
- cmd->phy_address = readb(®s->MIIADR) & 0x1F;
+ cmd->base.autoneg = (status & VELOCITY_AUTONEG_ENABLE) ?
+ AUTONEG_ENABLE : AUTONEG_DISABLE;
+ cmd->base.port = PORT_TP;
+ cmd->base.phy_address = readb(®s->MIIADR) & 0x1F;
if (status & VELOCITY_DUPLEX_FULL)
- cmd->duplex = DUPLEX_FULL;
+ cmd->base.duplex = DUPLEX_FULL;
else
- cmd->duplex = DUPLEX_HALF;
+ cmd->base.duplex = DUPLEX_HALF;
+
+ ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported,
+ supported);
+ ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising,
+ advertising);
return 0;
}
-static int velocity_set_settings(struct net_device *dev,
- struct ethtool_cmd *cmd)
+static int velocity_set_link_ksettings(struct net_device *dev,
+ const struct ethtool_link_ksettings *cmd)
{
struct velocity_info *vptr = netdev_priv(dev);
- u32 speed = ethtool_cmd_speed(cmd);
+ u32 speed = cmd->base.speed;
u32 curr_status;
u32 new_status = 0;
int ret = 0;
curr_status = check_connection_type(vptr->mac_regs);
curr_status &= (~VELOCITY_LINK_FAIL);
- new_status |= ((cmd->autoneg) ? VELOCITY_AUTONEG_ENABLE : 0);
+ new_status |= ((cmd->base.autoneg) ? VELOCITY_AUTONEG_ENABLE : 0);
new_status |= ((speed == SPEED_1000) ? VELOCITY_SPEED_1000 : 0);
new_status |= ((speed == SPEED_100) ? VELOCITY_SPEED_100 : 0);
new_status |= ((speed == SPEED_10) ? VELOCITY_SPEED_10 : 0);
- new_status |= ((cmd->duplex == DUPLEX_FULL) ? VELOCITY_DUPLEX_FULL : 0);
+ new_status |= ((cmd->base.duplex == DUPLEX_FULL) ?
+ VELOCITY_DUPLEX_FULL : 0);
if ((new_status & VELOCITY_AUTONEG_ENABLE) &&
(new_status != (curr_status | VELOCITY_AUTONEG_ENABLE))) {
}
static const struct ethtool_ops velocity_ethtool_ops = {
- .get_settings = velocity_get_settings,
- .set_settings = velocity_set_settings,
.get_drvinfo = velocity_get_drvinfo,
.get_wol = velocity_ethtool_get_wol,
.set_wol = velocity_ethtool_set_wol,
.get_coalesce = velocity_get_coalesce,
.set_coalesce = velocity_set_coalesce,
.begin = velocity_ethtool_up,
- .complete = velocity_ethtool_down
+ .complete = velocity_ethtool_down,
+ .get_link_ksettings = velocity_get_link_ksettings,
+ .set_link_ksettings = velocity_set_link_ksettings,
};
#if defined(CONFIG_PM) && defined(CONFIG_INET)
"platform:%s", plat_dev->name);
}
-static int fjes_get_settings(struct net_device *netdev,
- struct ethtool_cmd *ecmd)
+static int fjes_get_link_ksettings(struct net_device *netdev,
+ struct ethtool_link_ksettings *ecmd)
{
- ecmd->supported = 0;
- ecmd->advertising = 0;
- ecmd->duplex = DUPLEX_FULL;
- ecmd->autoneg = AUTONEG_DISABLE;
- ecmd->transceiver = XCVR_DUMMY1;
- ecmd->port = PORT_NONE;
- ethtool_cmd_speed_set(ecmd, 20000); /* 20Gb/s */
+ ethtool_link_ksettings_zero_link_mode(ecmd, supported);
+ ethtool_link_ksettings_zero_link_mode(ecmd, advertising);
+ ecmd->base.duplex = DUPLEX_FULL;
+ ecmd->base.autoneg = AUTONEG_DISABLE;
+ ecmd->base.port = PORT_NONE;
+ ecmd->base.speed = 20000; /* 20Gb/s */
return 0;
}
}
static const struct ethtool_ops fjes_ethtool_ops = {
- .get_settings = fjes_get_settings,
.get_drvinfo = fjes_get_drvinfo,
.get_ethtool_stats = fjes_get_ethtool_stats,
.get_strings = fjes_get_strings,
.set_dump = fjes_set_dump,
.get_dump_flag = fjes_get_dump_flag,
.get_dump_data = fjes_get_dump_data,
+ .get_link_ksettings = fjes_get_link_ksettings,
};
void fjes_set_ethtool_ops(struct net_device *netdev)
return ret;
}
-static bool netvsc_validate_ethtool_ss_cmd(const struct ethtool_cmd *cmd)
+static bool
+netvsc_validate_ethtool_ss_cmd(const struct ethtool_link_ksettings *cmd)
{
- struct ethtool_cmd diff1 = *cmd;
- struct ethtool_cmd diff2 = {};
+ struct ethtool_link_ksettings diff1 = *cmd;
+ struct ethtool_link_ksettings diff2 = {};
- ethtool_cmd_speed_set(&diff1, 0);
- diff1.duplex = 0;
+ diff1.base.speed = 0;
+ diff1.base.duplex = 0;
/* advertising and cmd are usually set */
- diff1.advertising = 0;
- diff1.cmd = 0;
+ ethtool_link_ksettings_zero_link_mode(&diff1, advertising);
+ diff1.base.cmd = 0;
/* We set port to PORT_OTHER */
- diff2.port = PORT_OTHER;
+ diff2.base.port = PORT_OTHER;
return !memcmp(&diff1, &diff2, sizeof(diff1));
}
ndc->duplex = DUPLEX_UNKNOWN;
}
-static int netvsc_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
+static int netvsc_get_link_ksettings(struct net_device *dev,
+ struct ethtool_link_ksettings *cmd)
{
struct net_device_context *ndc = netdev_priv(dev);
- ethtool_cmd_speed_set(cmd, ndc->speed);
- cmd->duplex = ndc->duplex;
- cmd->port = PORT_OTHER;
+ cmd->base.speed = ndc->speed;
+ cmd->base.duplex = ndc->duplex;
+ cmd->base.port = PORT_OTHER;
return 0;
}
-static int netvsc_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
+static int netvsc_set_link_ksettings(struct net_device *dev,
+ const struct ethtool_link_ksettings *cmd)
{
struct net_device_context *ndc = netdev_priv(dev);
u32 speed;
- speed = ethtool_cmd_speed(cmd);
+ speed = cmd->base.speed;
if (!ethtool_validate_speed(speed) ||
- !ethtool_validate_duplex(cmd->duplex) ||
+ !ethtool_validate_duplex(cmd->base.duplex) ||
!netvsc_validate_ethtool_ss_cmd(cmd))
return -EINVAL;
ndc->speed = speed;
- ndc->duplex = cmd->duplex;
+ ndc->duplex = cmd->base.duplex;
return 0;
}
.get_channels = netvsc_get_channels,
.set_channels = netvsc_set_channels,
.get_ts_info = ethtool_op_get_ts_info,
- .get_settings = netvsc_get_settings,
- .set_settings = netvsc_set_settings,
.get_rxnfc = netvsc_get_rxnfc,
.get_rxfh_key_size = netvsc_get_rxfh_key_size,
.get_rxfh_indir_size = netvsc_rss_indir_size,
.get_rxfh = netvsc_get_rxfh,
.set_rxfh = netvsc_set_rxfh,
+ .get_link_ksettings = netvsc_get_link_ksettings,
+ .set_link_ksettings = netvsc_set_link_ksettings,
};
static const struct net_device_ops device_ops = {
enum {
ETH_RSS_HASH_TOP_BIT, /* Configurable RSS hash function - Toeplitz */
ETH_RSS_HASH_XOR_BIT, /* Configurable RSS hash function - Xor */
+ ETH_RSS_HASH_CRC32_BIT, /* Configurable RSS hash function - Crc32 */
/*
* Add your fresh new hash function bits above and remember to update
#define ETH_RSS_HASH_TOP __ETH_RSS_HASH(TOP)
#define ETH_RSS_HASH_XOR __ETH_RSS_HASH(XOR)
+#define ETH_RSS_HASH_CRC32 __ETH_RSS_HASH(CRC32)
#define ETH_RSS_HASH_UNKNOWN 0
#define ETH_RSS_HASH_NO_CHANGE 0
int register_fib_notifier(struct notifier_block *nb,
void (*cb)(struct notifier_block *nb));
int unregister_fib_notifier(struct notifier_block *nb);
+int call_fib_notifier(struct notifier_block *nb, struct net *net,
+ enum fib_event_type event_type,
+ struct fib_notifier_info *info);
int call_fib_notifiers(struct net *net, enum fib_event_type event_type,
struct fib_notifier_info *info);
+void fib_notify(struct net *net, struct notifier_block *nb);
+#ifdef CONFIG_IP_MULTIPLE_TABLES
+void fib_rules_notify(struct net *net, struct notifier_block *nb);
+#else
+static inline void fib_rules_notify(struct net *net, struct notifier_block *nb)
+{
+}
+#endif
+
struct fib_table {
struct hlist_node tb_hlist;
u32 tb_id;
void qdisc_get_default(char *id, size_t len);
int qdisc_set_default(const char *id);
-void qdisc_hash_add(struct Qdisc *q);
+void qdisc_hash_add(struct Qdisc *q, bool invisible);
void qdisc_hash_del(struct Qdisc *q);
struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle);
struct Qdisc *qdisc_lookup_class(struct net_device *dev, u32 handle);
#define TCQ_F_NOPARENT 0x40 /* root of its hierarchy :
* qdisc_tree_decrease_qlen() should stop.
*/
+#define TCQ_F_INVISIBLE 0x80 /* invisible by default in dump */
u32 limit;
const struct Qdisc_ops *ops;
struct qdisc_size_table __rcu *stab;
u32 secure_ipv4_port_ephemeral(__be32 saddr, __be32 daddr, __be16 dport);
u32 secure_ipv6_port_ephemeral(const __be32 *saddr, const __be32 *daddr,
__be16 dport);
-u32 secure_tcp_sequence_number(__be32 saddr, __be32 daddr,
+u32 secure_tcp_seq_and_tsoff(__be32 saddr, __be32 daddr,
+ __be16 sport, __be16 dport, u32 *tsoff);
+u32 secure_tcpv6_seq_and_tsoff(const __be32 *saddr, const __be32 *daddr,
__be16 sport, __be16 dport, u32 *tsoff);
-u32 secure_tcpv6_sequence_number(const __be32 *saddr, const __be32 *daddr,
- __be16 sport, __be16 dport, u32 *tsoff);
u64 secure_dccp_sequence_number(__be32 saddr, __be32 daddr,
__be16 sport, __be16 dport);
u64 secure_dccpv6_sequence_number(__be32 *saddr, __be32 *daddr,
sk_tx_queue_clear(sk);
sk->sk_dst_pending_confirm = 0;
- /*
- * This can be called while sk is owned by the caller only,
- * with no state that can be checked in a rcu_dereference_check() cond
- */
- old_dst = rcu_dereference_raw(sk->sk_dst_cache);
+ old_dst = rcu_dereference_protected(sk->sk_dst_cache,
+ lockdep_sock_is_held(sk));
rcu_assign_pointer(sk->sk_dst_cache, dst);
dst_release(old_dst);
}
return to_vlan(a)->tcfv_push_proto;
}
+static inline u8 tcf_vlan_push_prio(const struct tc_action *a)
+{
+ return to_vlan(a)->tcfv_push_prio;
+}
+
#endif /* __NET_TC_VLAN_H */
struct dst_entry *(*route_req)(const struct sock *sk, struct flowi *fl,
const struct request_sock *req,
bool *strict);
- __u32 (*init_seq)(const struct sk_buff *skb, u32 *tsoff);
+ __u32 (*init_seq_tsoff)(const struct sk_buff *skb, u32 *tsoff);
int (*send_synack)(const struct sock *sk, struct dst_entry *dst,
struct flowi *fl, struct request_sock *req,
struct tcp_fastopen_cookie *foc,
TCA_STATS2,
TCA_STAB,
TCA_PAD,
+ TCA_DUMP_INVISIBLE,
__TCA_MAX
};
rss_hash_func_strings[ETH_RSS_HASH_FUNCS_COUNT][ETH_GSTRING_LEN] = {
[ETH_RSS_HASH_TOP_BIT] = "toeplitz",
[ETH_RSS_HASH_XOR_BIT] = "xor",
+ [ETH_RSS_HASH_CRC32_BIT] = "crc32",
};
static const char
}
EXPORT_SYMBOL(__skb_flow_get_ports);
+enum flow_dissect_ret {
+ FLOW_DISSECT_RET_OUT_GOOD,
+ FLOW_DISSECT_RET_OUT_BAD,
+ FLOW_DISSECT_RET_OUT_PROTO_AGAIN,
+};
+
+static enum flow_dissect_ret
+__skb_flow_dissect_mpls(const struct sk_buff *skb,
+ struct flow_dissector *flow_dissector,
+ void *target_container, void *data, int nhoff, int hlen)
+{
+ struct flow_dissector_key_keyid *key_keyid;
+ struct mpls_label *hdr, _hdr[2];
+
+ if (!dissector_uses_key(flow_dissector,
+ FLOW_DISSECTOR_KEY_MPLS_ENTROPY))
+ return FLOW_DISSECT_RET_OUT_GOOD;
+
+ hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data,
+ hlen, &_hdr);
+ if (!hdr)
+ return FLOW_DISSECT_RET_OUT_BAD;
+
+ if ((ntohl(hdr[0].entry) & MPLS_LS_LABEL_MASK) >>
+ MPLS_LS_LABEL_SHIFT == MPLS_LABEL_ENTROPY) {
+ key_keyid = skb_flow_dissector_target(flow_dissector,
+ FLOW_DISSECTOR_KEY_MPLS_ENTROPY,
+ target_container);
+ key_keyid->keyid = hdr[1].entry & htonl(MPLS_LS_LABEL_MASK);
+ }
+ return FLOW_DISSECT_RET_OUT_GOOD;
+}
+
+static enum flow_dissect_ret
+__skb_flow_dissect_arp(const struct sk_buff *skb,
+ struct flow_dissector *flow_dissector,
+ void *target_container, void *data, int nhoff, int hlen)
+{
+ struct flow_dissector_key_arp *key_arp;
+ struct {
+ unsigned char ar_sha[ETH_ALEN];
+ unsigned char ar_sip[4];
+ unsigned char ar_tha[ETH_ALEN];
+ unsigned char ar_tip[4];
+ } *arp_eth, _arp_eth;
+ const struct arphdr *arp;
+ struct arphdr *_arp;
+
+ if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ARP))
+ return FLOW_DISSECT_RET_OUT_GOOD;
+
+ arp = __skb_header_pointer(skb, nhoff, sizeof(_arp), data,
+ hlen, &_arp);
+ if (!arp)
+ return FLOW_DISSECT_RET_OUT_BAD;
+
+ if (arp->ar_hrd != htons(ARPHRD_ETHER) ||
+ arp->ar_pro != htons(ETH_P_IP) ||
+ arp->ar_hln != ETH_ALEN ||
+ arp->ar_pln != 4 ||
+ (arp->ar_op != htons(ARPOP_REPLY) &&
+ arp->ar_op != htons(ARPOP_REQUEST)))
+ return FLOW_DISSECT_RET_OUT_BAD;
+
+ arp_eth = __skb_header_pointer(skb, nhoff + sizeof(_arp),
+ sizeof(_arp_eth), data,
+ hlen, &_arp_eth);
+ if (!arp_eth)
+ return FLOW_DISSECT_RET_OUT_BAD;
+
+ key_arp = skb_flow_dissector_target(flow_dissector,
+ FLOW_DISSECTOR_KEY_ARP,
+ target_container);
+
+ memcpy(&key_arp->sip, arp_eth->ar_sip, sizeof(key_arp->sip));
+ memcpy(&key_arp->tip, arp_eth->ar_tip, sizeof(key_arp->tip));
+
+ /* Only store the lower byte of the opcode;
+ * this covers ARPOP_REPLY and ARPOP_REQUEST.
+ */
+ key_arp->op = ntohs(arp->ar_op) & 0xff;
+
+ ether_addr_copy(key_arp->sha, arp_eth->ar_sha);
+ ether_addr_copy(key_arp->tha, arp_eth->ar_tha);
+
+ return FLOW_DISSECT_RET_OUT_GOOD;
+}
+
+static enum flow_dissect_ret
+__skb_flow_dissect_gre(const struct sk_buff *skb,
+ struct flow_dissector_key_control *key_control,
+ struct flow_dissector *flow_dissector,
+ void *target_container, void *data,
+ __be16 *p_proto, int *p_nhoff, int *p_hlen,
+ unsigned int flags)
+{
+ struct flow_dissector_key_keyid *key_keyid;
+ struct gre_base_hdr *hdr, _hdr;
+ int offset = 0;
+ u16 gre_ver;
+
+ hdr = __skb_header_pointer(skb, *p_nhoff, sizeof(_hdr),
+ data, *p_hlen, &_hdr);
+ if (!hdr)
+ return FLOW_DISSECT_RET_OUT_BAD;
+
+ /* Only look inside GRE without routing */
+ if (hdr->flags & GRE_ROUTING)
+ return FLOW_DISSECT_RET_OUT_GOOD;
+
+ /* Only look inside GRE for version 0 and 1 */
+ gre_ver = ntohs(hdr->flags & GRE_VERSION);
+ if (gre_ver > 1)
+ return FLOW_DISSECT_RET_OUT_GOOD;
+
+ *p_proto = hdr->protocol;
+ if (gre_ver) {
+ /* Version1 must be PPTP, and check the flags */
+ if (!(*p_proto == GRE_PROTO_PPP && (hdr->flags & GRE_KEY)))
+ return FLOW_DISSECT_RET_OUT_GOOD;
+ }
+
+ offset += sizeof(struct gre_base_hdr);
+
+ if (hdr->flags & GRE_CSUM)
+ offset += sizeof(((struct gre_full_hdr *) 0)->csum) +
+ sizeof(((struct gre_full_hdr *) 0)->reserved1);
+
+ if (hdr->flags & GRE_KEY) {
+ const __be32 *keyid;
+ __be32 _keyid;
+
+ keyid = __skb_header_pointer(skb, *p_nhoff + offset,
+ sizeof(_keyid),
+ data, *p_hlen, &_keyid);
+ if (!keyid)
+ return FLOW_DISSECT_RET_OUT_BAD;
+
+ if (dissector_uses_key(flow_dissector,
+ FLOW_DISSECTOR_KEY_GRE_KEYID)) {
+ key_keyid = skb_flow_dissector_target(flow_dissector,
+ FLOW_DISSECTOR_KEY_GRE_KEYID,
+ target_container);
+ if (gre_ver == 0)
+ key_keyid->keyid = *keyid;
+ else
+ key_keyid->keyid = *keyid & GRE_PPTP_KEY_MASK;
+ }
+ offset += sizeof(((struct gre_full_hdr *) 0)->key);
+ }
+
+ if (hdr->flags & GRE_SEQ)
+ offset += sizeof(((struct pptp_gre_header *) 0)->seq);
+
+ if (gre_ver == 0) {
+ if (*p_proto == htons(ETH_P_TEB)) {
+ const struct ethhdr *eth;
+ struct ethhdr _eth;
+
+ eth = __skb_header_pointer(skb, *p_nhoff + offset,
+ sizeof(_eth),
+ data, *p_hlen, &_eth);
+ if (!eth)
+ return FLOW_DISSECT_RET_OUT_BAD;
+ *p_proto = eth->h_proto;
+ offset += sizeof(*eth);
+
+ /* Cap headers that we access via pointers at the
+ * end of the Ethernet header as our maximum alignment
+ * at that point is only 2 bytes.
+ */
+ if (NET_IP_ALIGN)
+ *p_hlen = *p_nhoff + offset;
+ }
+ } else { /* version 1, must be PPTP */
+ u8 _ppp_hdr[PPP_HDRLEN];
+ u8 *ppp_hdr;
+
+ if (hdr->flags & GRE_ACK)
+ offset += sizeof(((struct pptp_gre_header *) 0)->ack);
+
+ ppp_hdr = __skb_header_pointer(skb, *p_nhoff + offset,
+ sizeof(_ppp_hdr),
+ data, *p_hlen, _ppp_hdr);
+ if (!ppp_hdr)
+ return FLOW_DISSECT_RET_OUT_BAD;
+
+ switch (PPP_PROTOCOL(ppp_hdr)) {
+ case PPP_IP:
+ *p_proto = htons(ETH_P_IP);
+ break;
+ case PPP_IPV6:
+ *p_proto = htons(ETH_P_IPV6);
+ break;
+ default:
+ /* Could probably catch some more like MPLS */
+ break;
+ }
+
+ offset += PPP_HDRLEN;
+ }
+
+ *p_nhoff += offset;
+ key_control->flags |= FLOW_DIS_ENCAPSULATION;
+ if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP)
+ return FLOW_DISSECT_RET_OUT_GOOD;
+
+ return FLOW_DISSECT_RET_OUT_PROTO_AGAIN;
+}
+
/**
* __skb_flow_dissect - extract the flow_keys struct and return it
* @skb: sk_buff to extract the flow from, can be NULL if the rest are specified
struct flow_dissector_key_control *key_control;
struct flow_dissector_key_basic *key_basic;
struct flow_dissector_key_addrs *key_addrs;
- struct flow_dissector_key_arp *key_arp;
struct flow_dissector_key_ports *key_ports;
struct flow_dissector_key_icmp *key_icmp;
struct flow_dissector_key_tags *key_tags;
struct flow_dissector_key_vlan *key_vlan;
- struct flow_dissector_key_keyid *key_keyid;
bool skip_vlan = false;
u8 ip_proto = 0;
bool ret;
memcpy(key_eth_addrs, ð->h_dest, sizeof(*key_eth_addrs));
}
-again:
+proto_again:
switch (proto) {
case htons(ETH_P_IP): {
const struct iphdr *iph;
proto = vlan->h_vlan_encapsulated_proto;
nhoff += sizeof(*vlan);
if (skip_vlan)
- goto again;
+ goto proto_again;
}
skip_vlan = true;
}
}
- goto again;
+ goto proto_again;
}
case htons(ETH_P_PPP_SES): {
struct {
}
case htons(ETH_P_MPLS_UC):
- case htons(ETH_P_MPLS_MC): {
- struct mpls_label *hdr, _hdr[2];
+ case htons(ETH_P_MPLS_MC):
mpls:
- hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data,
- hlen, &_hdr);
- if (!hdr)
- goto out_bad;
-
- if ((ntohl(hdr[0].entry) & MPLS_LS_LABEL_MASK) >>
- MPLS_LS_LABEL_SHIFT == MPLS_LABEL_ENTROPY) {
- if (dissector_uses_key(flow_dissector,
- FLOW_DISSECTOR_KEY_MPLS_ENTROPY)) {
- key_keyid = skb_flow_dissector_target(flow_dissector,
- FLOW_DISSECTOR_KEY_MPLS_ENTROPY,
- target_container);
- key_keyid->keyid = hdr[1].entry &
- htonl(MPLS_LS_LABEL_MASK);
- }
-
+ switch (__skb_flow_dissect_mpls(skb, flow_dissector,
+ target_container, data,
+ nhoff, hlen)) {
+ case FLOW_DISSECT_RET_OUT_GOOD:
goto out_good;
+ case FLOW_DISSECT_RET_OUT_BAD:
+ default:
+ goto out_bad;
}
-
- goto out_good;
- }
-
case htons(ETH_P_FCOE):
if ((hlen - nhoff) < FCOE_HEADER_LEN)
goto out_bad;
goto out_good;
case htons(ETH_P_ARP):
- case htons(ETH_P_RARP): {
- struct {
- unsigned char ar_sha[ETH_ALEN];
- unsigned char ar_sip[4];
- unsigned char ar_tha[ETH_ALEN];
- unsigned char ar_tip[4];
- } *arp_eth, _arp_eth;
- const struct arphdr *arp;
- struct arphdr *_arp;
-
- arp = __skb_header_pointer(skb, nhoff, sizeof(_arp), data,
- hlen, &_arp);
- if (!arp)
- goto out_bad;
-
- if (arp->ar_hrd != htons(ARPHRD_ETHER) ||
- arp->ar_pro != htons(ETH_P_IP) ||
- arp->ar_hln != ETH_ALEN ||
- arp->ar_pln != 4 ||
- (arp->ar_op != htons(ARPOP_REPLY) &&
- arp->ar_op != htons(ARPOP_REQUEST)))
- goto out_bad;
-
- arp_eth = __skb_header_pointer(skb, nhoff + sizeof(_arp),
- sizeof(_arp_eth), data,
- hlen,
- &_arp_eth);
- if (!arp_eth)
+ case htons(ETH_P_RARP):
+ switch (__skb_flow_dissect_arp(skb, flow_dissector,
+ target_container, data,
+ nhoff, hlen)) {
+ case FLOW_DISSECT_RET_OUT_GOOD:
+ goto out_good;
+ case FLOW_DISSECT_RET_OUT_BAD:
+ default:
goto out_bad;
-
- if (dissector_uses_key(flow_dissector,
- FLOW_DISSECTOR_KEY_ARP)) {
-
- key_arp = skb_flow_dissector_target(flow_dissector,
- FLOW_DISSECTOR_KEY_ARP,
- target_container);
-
- memcpy(&key_arp->sip, arp_eth->ar_sip,
- sizeof(key_arp->sip));
- memcpy(&key_arp->tip, arp_eth->ar_tip,
- sizeof(key_arp->tip));
-
- /* Only store the lower byte of the opcode;
- * this covers ARPOP_REPLY and ARPOP_REQUEST.
- */
- key_arp->op = ntohs(arp->ar_op) & 0xff;
-
- ether_addr_copy(key_arp->sha, arp_eth->ar_sha);
- ether_addr_copy(key_arp->tha, arp_eth->ar_tha);
}
-
- goto out_good;
- }
-
default:
goto out_bad;
}
ip_proto_again:
switch (ip_proto) {
- case IPPROTO_GRE: {
- struct gre_base_hdr *hdr, _hdr;
- u16 gre_ver;
- int offset = 0;
-
- hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
- if (!hdr)
+ case IPPROTO_GRE:
+ switch (__skb_flow_dissect_gre(skb, key_control, flow_dissector,
+ target_container, data,
+ &proto, &nhoff, &hlen, flags)) {
+ case FLOW_DISSECT_RET_OUT_GOOD:
+ goto out_good;
+ case FLOW_DISSECT_RET_OUT_BAD:
goto out_bad;
-
- /* Only look inside GRE without routing */
- if (hdr->flags & GRE_ROUTING)
- break;
-
- /* Only look inside GRE for version 0 and 1 */
- gre_ver = ntohs(hdr->flags & GRE_VERSION);
- if (gre_ver > 1)
- break;
-
- proto = hdr->protocol;
- if (gre_ver) {
- /* Version1 must be PPTP, and check the flags */
- if (!(proto == GRE_PROTO_PPP && (hdr->flags & GRE_KEY)))
- break;
- }
-
- offset += sizeof(struct gre_base_hdr);
-
- if (hdr->flags & GRE_CSUM)
- offset += sizeof(((struct gre_full_hdr *)0)->csum) +
- sizeof(((struct gre_full_hdr *)0)->reserved1);
-
- if (hdr->flags & GRE_KEY) {
- const __be32 *keyid;
- __be32 _keyid;
-
- keyid = __skb_header_pointer(skb, nhoff + offset, sizeof(_keyid),
- data, hlen, &_keyid);
- if (!keyid)
- goto out_bad;
-
- if (dissector_uses_key(flow_dissector,
- FLOW_DISSECTOR_KEY_GRE_KEYID)) {
- key_keyid = skb_flow_dissector_target(flow_dissector,
- FLOW_DISSECTOR_KEY_GRE_KEYID,
- target_container);
- if (gre_ver == 0)
- key_keyid->keyid = *keyid;
- else
- key_keyid->keyid = *keyid & GRE_PPTP_KEY_MASK;
- }
- offset += sizeof(((struct gre_full_hdr *)0)->key);
+ case FLOW_DISSECT_RET_OUT_PROTO_AGAIN:
+ goto proto_again;
}
-
- if (hdr->flags & GRE_SEQ)
- offset += sizeof(((struct pptp_gre_header *)0)->seq);
-
- if (gre_ver == 0) {
- if (proto == htons(ETH_P_TEB)) {
- const struct ethhdr *eth;
- struct ethhdr _eth;
-
- eth = __skb_header_pointer(skb, nhoff + offset,
- sizeof(_eth),
- data, hlen, &_eth);
- if (!eth)
- goto out_bad;
- proto = eth->h_proto;
- offset += sizeof(*eth);
-
- /* Cap headers that we access via pointers at the
- * end of the Ethernet header as our maximum alignment
- * at that point is only 2 bytes.
- */
- if (NET_IP_ALIGN)
- hlen = (nhoff + offset);
- }
- } else { /* version 1, must be PPTP */
- u8 _ppp_hdr[PPP_HDRLEN];
- u8 *ppp_hdr;
-
- if (hdr->flags & GRE_ACK)
- offset += sizeof(((struct pptp_gre_header *)0)->ack);
-
- ppp_hdr = __skb_header_pointer(skb, nhoff + offset,
- sizeof(_ppp_hdr),
- data, hlen, _ppp_hdr);
- if (!ppp_hdr)
- goto out_bad;
-
- switch (PPP_PROTOCOL(ppp_hdr)) {
- case PPP_IP:
- proto = htons(ETH_P_IP);
- break;
- case PPP_IPV6:
- proto = htons(ETH_P_IPV6);
- break;
- default:
- /* Could probably catch some more like MPLS */
- break;
- }
-
- offset += PPP_HDRLEN;
- }
-
- nhoff += offset;
- key_control->flags |= FLOW_DIS_ENCAPSULATION;
- if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP)
- goto out_good;
-
- goto again;
- }
case NEXTHDR_HOP:
case NEXTHDR_ROUTING:
case NEXTHDR_DEST: {
#endif
#if IS_ENABLED(CONFIG_IPV6)
-u32 secure_tcpv6_sequence_number(const __be32 *saddr, const __be32 *daddr,
- __be16 sport, __be16 dport, u32 *tsoff)
+u32 secure_tcpv6_seq_and_tsoff(const __be32 *saddr, const __be32 *daddr,
+ __be16 sport, __be16 dport, u32 *tsoff)
{
const struct {
struct in6_addr saddr;
*tsoff = sysctl_tcp_timestamps == 1 ? (hash >> 32) : 0;
return seq_scale(hash);
}
-EXPORT_SYMBOL(secure_tcpv6_sequence_number);
+EXPORT_SYMBOL(secure_tcpv6_seq_and_tsoff);
u32 secure_ipv6_port_ephemeral(const __be32 *saddr, const __be32 *daddr,
__be16 dport)
#ifdef CONFIG_INET
-/* secure_tcp_sequence_number(a, b, 0, d) == secure_ipv4_port_ephemeral(a, b, d),
+/* secure_tcp_seq_and_tsoff(a, b, 0, d) == secure_ipv4_port_ephemeral(a, b, d),
* but fortunately, `sport' cannot be 0 in any circumstances. If this changes,
* it would be easy enough to have the former function use siphash_4u32, passing
* the arguments as separate u32.
*/
-
-u32 secure_tcp_sequence_number(__be32 saddr, __be32 daddr,
- __be16 sport, __be16 dport, u32 *tsoff)
+u32 secure_tcp_seq_and_tsoff(__be32 saddr, __be32 daddr,
+ __be16 sport, __be16 dport, u32 *tsoff)
{
u64 hash;
net_secret_init();
"clock-AF_NFC" , "clock-AF_VSOCK" , "clock-AF_KCM" ,
"clock-AF_QIPCRTR", "clock-AF_SMC" , "clock-AF_MAX"
};
+static const char *const af_family_rlock_key_strings[AF_MAX+1] = {
+ "rlock-AF_UNSPEC", "rlock-AF_UNIX" , "rlock-AF_INET" ,
+ "rlock-AF_AX25" , "rlock-AF_IPX" , "rlock-AF_APPLETALK",
+ "rlock-AF_NETROM", "rlock-AF_BRIDGE" , "rlock-AF_ATMPVC" ,
+ "rlock-AF_X25" , "rlock-AF_INET6" , "rlock-AF_ROSE" ,
+ "rlock-AF_DECnet", "rlock-AF_NETBEUI" , "rlock-AF_SECURITY" ,
+ "rlock-AF_KEY" , "rlock-AF_NETLINK" , "rlock-AF_PACKET" ,
+ "rlock-AF_ASH" , "rlock-AF_ECONET" , "rlock-AF_ATMSVC" ,
+ "rlock-AF_RDS" , "rlock-AF_SNA" , "rlock-AF_IRDA" ,
+ "rlock-AF_PPPOX" , "rlock-AF_WANPIPE" , "rlock-AF_LLC" ,
+ "rlock-27" , "rlock-28" , "rlock-AF_CAN" ,
+ "rlock-AF_TIPC" , "rlock-AF_BLUETOOTH", "rlock-AF_IUCV" ,
+ "rlock-AF_RXRPC" , "rlock-AF_ISDN" , "rlock-AF_PHONET" ,
+ "rlock-AF_IEEE802154", "rlock-AF_CAIF" , "rlock-AF_ALG" ,
+ "rlock-AF_NFC" , "rlock-AF_VSOCK" , "rlock-AF_KCM" ,
+ "rlock-AF_QIPCRTR", "rlock-AF_SMC" , "rlock-AF_MAX"
+};
+static const char *const af_family_wlock_key_strings[AF_MAX+1] = {
+ "wlock-AF_UNSPEC", "wlock-AF_UNIX" , "wlock-AF_INET" ,
+ "wlock-AF_AX25" , "wlock-AF_IPX" , "wlock-AF_APPLETALK",
+ "wlock-AF_NETROM", "wlock-AF_BRIDGE" , "wlock-AF_ATMPVC" ,
+ "wlock-AF_X25" , "wlock-AF_INET6" , "wlock-AF_ROSE" ,
+ "wlock-AF_DECnet", "wlock-AF_NETBEUI" , "wlock-AF_SECURITY" ,
+ "wlock-AF_KEY" , "wlock-AF_NETLINK" , "wlock-AF_PACKET" ,
+ "wlock-AF_ASH" , "wlock-AF_ECONET" , "wlock-AF_ATMSVC" ,
+ "wlock-AF_RDS" , "wlock-AF_SNA" , "wlock-AF_IRDA" ,
+ "wlock-AF_PPPOX" , "wlock-AF_WANPIPE" , "wlock-AF_LLC" ,
+ "wlock-27" , "wlock-28" , "wlock-AF_CAN" ,
+ "wlock-AF_TIPC" , "wlock-AF_BLUETOOTH", "wlock-AF_IUCV" ,
+ "wlock-AF_RXRPC" , "wlock-AF_ISDN" , "wlock-AF_PHONET" ,
+ "wlock-AF_IEEE802154", "wlock-AF_CAIF" , "wlock-AF_ALG" ,
+ "wlock-AF_NFC" , "wlock-AF_VSOCK" , "wlock-AF_KCM" ,
+ "wlock-AF_QIPCRTR", "wlock-AF_SMC" , "wlock-AF_MAX"
+};
+static const char *const af_family_elock_key_strings[AF_MAX+1] = {
+ "elock-AF_UNSPEC", "elock-AF_UNIX" , "elock-AF_INET" ,
+ "elock-AF_AX25" , "elock-AF_IPX" , "elock-AF_APPLETALK",
+ "elock-AF_NETROM", "elock-AF_BRIDGE" , "elock-AF_ATMPVC" ,
+ "elock-AF_X25" , "elock-AF_INET6" , "elock-AF_ROSE" ,
+ "elock-AF_DECnet", "elock-AF_NETBEUI" , "elock-AF_SECURITY" ,
+ "elock-AF_KEY" , "elock-AF_NETLINK" , "elock-AF_PACKET" ,
+ "elock-AF_ASH" , "elock-AF_ECONET" , "elock-AF_ATMSVC" ,
+ "elock-AF_RDS" , "elock-AF_SNA" , "elock-AF_IRDA" ,
+ "elock-AF_PPPOX" , "elock-AF_WANPIPE" , "elock-AF_LLC" ,
+ "elock-27" , "elock-28" , "elock-AF_CAN" ,
+ "elock-AF_TIPC" , "elock-AF_BLUETOOTH", "elock-AF_IUCV" ,
+ "elock-AF_RXRPC" , "elock-AF_ISDN" , "elock-AF_PHONET" ,
+ "elock-AF_IEEE802154", "elock-AF_CAIF" , "elock-AF_ALG" ,
+ "elock-AF_NFC" , "elock-AF_VSOCK" , "elock-AF_KCM" ,
+ "elock-AF_QIPCRTR", "elock-AF_SMC" , "elock-AF_MAX"
+};
/*
- * sk_callback_lock locking rules are per-address-family,
+ * sk_callback_lock and sk queues locking rules are per-address-family,
* so split the lock classes by using a per-AF key:
*/
static struct lock_class_key af_callback_keys[AF_MAX];
+static struct lock_class_key af_rlock_keys[AF_MAX];
+static struct lock_class_key af_wlock_keys[AF_MAX];
+static struct lock_class_key af_elock_keys[AF_MAX];
/* Take into consideration the size of the struct sk_buff overhead in the
* determination of these values, since that is non-constant across
}
EXPORT_SYMBOL(sk_free);
+static void sk_init_common(struct sock *sk)
+{
+ skb_queue_head_init(&sk->sk_receive_queue);
+ skb_queue_head_init(&sk->sk_write_queue);
+ skb_queue_head_init(&sk->sk_error_queue);
+
+ rwlock_init(&sk->sk_callback_lock);
+ lockdep_set_class_and_name(&sk->sk_receive_queue.lock,
+ af_rlock_keys + sk->sk_family,
+ af_family_rlock_key_strings[sk->sk_family]);
+ lockdep_set_class_and_name(&sk->sk_write_queue.lock,
+ af_wlock_keys + sk->sk_family,
+ af_family_wlock_key_strings[sk->sk_family]);
+ lockdep_set_class_and_name(&sk->sk_error_queue.lock,
+ af_elock_keys + sk->sk_family,
+ af_family_elock_key_strings[sk->sk_family]);
+ lockdep_set_class_and_name(&sk->sk_callback_lock,
+ af_callback_keys + sk->sk_family,
+ af_family_clock_key_strings[sk->sk_family]);
+}
+
/**
* sk_clone_lock - clone a socket, and lock its clone
* @sk: the socket to clone
*/
atomic_set(&newsk->sk_wmem_alloc, 1);
atomic_set(&newsk->sk_omem_alloc, 0);
- skb_queue_head_init(&newsk->sk_receive_queue);
- skb_queue_head_init(&newsk->sk_write_queue);
-
- rwlock_init(&newsk->sk_callback_lock);
- lockdep_set_class_and_name(&newsk->sk_callback_lock,
- af_callback_keys + newsk->sk_family,
- af_family_clock_key_strings[newsk->sk_family]);
+ sk_init_common(newsk);
newsk->sk_dst_cache = NULL;
newsk->sk_dst_pending_confirm = 0;
newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK;
sock_reset_flag(newsk, SOCK_DONE);
- skb_queue_head_init(&newsk->sk_error_queue);
filter = rcu_dereference_protected(newsk->sk_filter, 1);
if (filter != NULL)
void sock_init_data(struct socket *sock, struct sock *sk)
{
- skb_queue_head_init(&sk->sk_receive_queue);
- skb_queue_head_init(&sk->sk_write_queue);
- skb_queue_head_init(&sk->sk_error_queue);
-
+ sk_init_common(sk);
sk->sk_send_head = NULL;
init_timer(&sk->sk_timer);
sk->sk_uid = make_kuid(sock_net(sk)->user_ns, 0);
}
- rwlock_init(&sk->sk_callback_lock);
- lockdep_set_class_and_name(&sk->sk_callback_lock,
- af_callback_keys + sk->sk_family,
- af_family_clock_key_strings[sk->sk_family]);
-
sk->sk_state_change = sock_def_wakeup;
sk->sk_data_ready = sock_def_readable;
sk->sk_write_space = sock_def_write_space;
tcp_rate.o tcp_recovery.o \
tcp_offload.o datagram.o raw.o udp.o udplite.o \
udp_offload.o arp.o icmp.o devinet.o af_inet.o igmp.o \
- fib_frontend.o fib_semantics.o fib_trie.o \
+ fib_frontend.o fib_semantics.o fib_trie.o fib_notifier.o \
inet_fragment.o ping.o ip_tunnel_core.o gre_offload.o
obj-$(CONFIG_NET_IP_TUNNEL) += ip_tunnel.o
--- /dev/null
+#include <linux/rtnetlink.h>
+#include <linux/notifier.h>
+#include <linux/rcupdate.h>
+#include <linux/kernel.h>
+#include <net/net_namespace.h>
+#include <net/netns/ipv4.h>
+#include <net/ip_fib.h>
+
+static ATOMIC_NOTIFIER_HEAD(fib_chain);
+
+int call_fib_notifier(struct notifier_block *nb, struct net *net,
+ enum fib_event_type event_type,
+ struct fib_notifier_info *info)
+{
+ info->net = net;
+ return nb->notifier_call(nb, event_type, info);
+}
+
+int call_fib_notifiers(struct net *net, enum fib_event_type event_type,
+ struct fib_notifier_info *info)
+{
+ net->ipv4.fib_seq++;
+ info->net = net;
+ return atomic_notifier_call_chain(&fib_chain, event_type, info);
+}
+
+static unsigned int fib_seq_sum(void)
+{
+ unsigned int fib_seq = 0;
+ struct net *net;
+
+ rtnl_lock();
+ for_each_net(net)
+ fib_seq += net->ipv4.fib_seq;
+ rtnl_unlock();
+
+ return fib_seq;
+}
+
+static bool fib_dump_is_consistent(struct notifier_block *nb,
+ void (*cb)(struct notifier_block *nb),
+ unsigned int fib_seq)
+{
+ atomic_notifier_chain_register(&fib_chain, nb);
+ if (fib_seq == fib_seq_sum())
+ return true;
+ atomic_notifier_chain_unregister(&fib_chain, nb);
+ if (cb)
+ cb(nb);
+ return false;
+}
+
+#define FIB_DUMP_MAX_RETRIES 5
+int register_fib_notifier(struct notifier_block *nb,
+ void (*cb)(struct notifier_block *nb))
+{
+ int retries = 0;
+
+ do {
+ unsigned int fib_seq = fib_seq_sum();
+ struct net *net;
+
+ /* Mutex semantics guarantee that every change done to
+ * FIB tries before we read the change sequence counter
+ * is now visible to us.
+ */
+ rcu_read_lock();
+ for_each_net_rcu(net) {
+ fib_rules_notify(net, nb);
+ fib_notify(net, nb);
+ }
+ rcu_read_unlock();
+
+ if (fib_dump_is_consistent(nb, cb, fib_seq))
+ return 0;
+ } while (++retries < FIB_DUMP_MAX_RETRIES);
+
+ return -EBUSY;
+}
+EXPORT_SYMBOL(register_fib_notifier);
+
+int unregister_fib_notifier(struct notifier_block *nb)
+{
+ return atomic_notifier_chain_unregister(&fib_chain, nb);
+}
+EXPORT_SYMBOL(unregister_fib_notifier);
return call_fib_notifiers(net, event_type, &info);
}
+void fib_rules_notify(struct net *net, struct notifier_block *nb)
+{
+ struct fib_notifier_info info;
+
+ if (net->ipv4.fib_has_custom_rules)
+ call_fib_notifier(nb, net, FIB_EVENT_RULE_ADD, &info);
+}
+
static const struct nla_policy fib4_rule_policy[FRA_MAX+1] = {
FRA_GENERIC_POLICY,
[FRA_FLOW] = { .type = NLA_U32 },
#include <trace/events/fib.h>
#include "fib_lookup.h"
-static unsigned int fib_seq_sum(void)
-{
- unsigned int fib_seq = 0;
- struct net *net;
-
- rtnl_lock();
- for_each_net(net)
- fib_seq += net->ipv4.fib_seq;
- rtnl_unlock();
-
- return fib_seq;
-}
-
-static ATOMIC_NOTIFIER_HEAD(fib_chain);
-
-static int call_fib_notifier(struct notifier_block *nb, struct net *net,
- enum fib_event_type event_type,
- struct fib_notifier_info *info)
-{
- info->net = net;
- return nb->notifier_call(nb, event_type, info);
-}
-
-static void fib_rules_notify(struct net *net, struct notifier_block *nb,
- enum fib_event_type event_type)
-{
-#ifdef CONFIG_IP_MULTIPLE_TABLES
- struct fib_notifier_info info;
-
- if (net->ipv4.fib_has_custom_rules)
- call_fib_notifier(nb, net, event_type, &info);
-#endif
-}
-
-static void fib_notify(struct net *net, struct notifier_block *nb,
- enum fib_event_type event_type);
-
static int call_fib_entry_notifier(struct notifier_block *nb, struct net *net,
enum fib_event_type event_type, u32 dst,
int dst_len, struct fib_info *fi,
return call_fib_notifier(nb, net, event_type, &info.info);
}
-static bool fib_dump_is_consistent(struct notifier_block *nb,
- void (*cb)(struct notifier_block *nb),
- unsigned int fib_seq)
-{
- atomic_notifier_chain_register(&fib_chain, nb);
- if (fib_seq == fib_seq_sum())
- return true;
- atomic_notifier_chain_unregister(&fib_chain, nb);
- if (cb)
- cb(nb);
- return false;
-}
-
-#define FIB_DUMP_MAX_RETRIES 5
-int register_fib_notifier(struct notifier_block *nb,
- void (*cb)(struct notifier_block *nb))
-{
- int retries = 0;
-
- do {
- unsigned int fib_seq = fib_seq_sum();
- struct net *net;
-
- /* Mutex semantics guarantee that every change done to
- * FIB tries before we read the change sequence counter
- * is now visible to us.
- */
- rcu_read_lock();
- for_each_net_rcu(net) {
- fib_rules_notify(net, nb, FIB_EVENT_RULE_ADD);
- fib_notify(net, nb, FIB_EVENT_ENTRY_ADD);
- }
- rcu_read_unlock();
-
- if (fib_dump_is_consistent(nb, cb, fib_seq))
- return 0;
- } while (++retries < FIB_DUMP_MAX_RETRIES);
-
- return -EBUSY;
-}
-EXPORT_SYMBOL(register_fib_notifier);
-
-int unregister_fib_notifier(struct notifier_block *nb)
-{
- return atomic_notifier_chain_unregister(&fib_chain, nb);
-}
-EXPORT_SYMBOL(unregister_fib_notifier);
-
-int call_fib_notifiers(struct net *net, enum fib_event_type event_type,
- struct fib_notifier_info *info)
-{
- net->ipv4.fib_seq++;
- info->net = net;
- return atomic_notifier_call_chain(&fib_chain, event_type, info);
-}
-
static int call_fib_entry_notifiers(struct net *net,
enum fib_event_type event_type, u32 dst,
int dst_len, struct fib_info *fi,
}
static void fib_leaf_notify(struct net *net, struct key_vector *l,
- struct fib_table *tb, struct notifier_block *nb,
- enum fib_event_type event_type)
+ struct fib_table *tb, struct notifier_block *nb)
{
struct fib_alias *fa;
if (tb->tb_id != fa->tb_id)
continue;
- call_fib_entry_notifier(nb, net, event_type, l->key,
+ call_fib_entry_notifier(nb, net, FIB_EVENT_ENTRY_ADD, l->key,
KEYLENGTH - fa->fa_slen, fi, fa->fa_tos,
fa->fa_type, fa->tb_id);
}
}
static void fib_table_notify(struct net *net, struct fib_table *tb,
- struct notifier_block *nb,
- enum fib_event_type event_type)
+ struct notifier_block *nb)
{
struct trie *t = (struct trie *)tb->tb_data;
struct key_vector *l, *tp = t->kv;
t_key key = 0;
while ((l = leaf_walk_rcu(&tp, key)) != NULL) {
- fib_leaf_notify(net, l, tb, nb, event_type);
+ fib_leaf_notify(net, l, tb, nb);
key = l->key + 1;
/* stop in case of wrap around */
}
}
-static void fib_notify(struct net *net, struct notifier_block *nb,
- enum fib_event_type event_type)
+void fib_notify(struct net *net, struct notifier_block *nb)
{
unsigned int h;
struct fib_table *tb;
hlist_for_each_entry_rcu(tb, head, tb_hlist)
- fib_table_notify(net, tb, nb, event_type);
+ fib_table_notify(net, tb, nb);
}
}
goto drop_and_free;
if (isn && tmp_opt.tstamp_ok)
- af_ops->init_seq(skb, &tcp_rsk(req)->ts_off);
+ af_ops->init_seq_tsoff(skb, &tcp_rsk(req)->ts_off);
if (!want_cookie && !isn) {
/* VJ's idea. We save last timestamp seen
goto drop_and_release;
}
- isn = af_ops->init_seq(skb, &tcp_rsk(req)->ts_off);
+ isn = af_ops->init_seq_tsoff(skb, &tcp_rsk(req)->ts_off);
}
if (!dst) {
dst = af_ops->route_req(sk, &fl, req, NULL);
struct inet_hashinfo tcp_hashinfo;
EXPORT_SYMBOL(tcp_hashinfo);
-static u32 tcp_v4_init_sequence(const struct sk_buff *skb, u32 *tsoff)
+static u32 tcp_v4_init_seq_and_tsoff(const struct sk_buff *skb, u32 *tsoff)
{
- return secure_tcp_sequence_number(ip_hdr(skb)->daddr,
- ip_hdr(skb)->saddr,
- tcp_hdr(skb)->dest,
- tcp_hdr(skb)->source, tsoff);
+ return secure_tcp_seq_and_tsoff(ip_hdr(skb)->daddr,
+ ip_hdr(skb)->saddr,
+ tcp_hdr(skb)->dest,
+ tcp_hdr(skb)->source, tsoff);
}
int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp)
rt = NULL;
if (likely(!tp->repair)) {
- seq = secure_tcp_sequence_number(inet->inet_saddr,
- inet->inet_daddr,
- inet->inet_sport,
- usin->sin_port,
- &tp->tsoffset);
+ seq = secure_tcp_seq_and_tsoff(inet->inet_saddr,
+ inet->inet_daddr,
+ inet->inet_sport,
+ usin->sin_port,
+ &tp->tsoffset);
if (!tp->write_seq)
tp->write_seq = seq;
}
.cookie_init_seq = cookie_v4_init_sequence,
#endif
.route_req = tcp_v4_route_req,
- .init_seq = tcp_v4_init_sequence,
+ .init_seq_tsoff = tcp_v4_init_seq_and_tsoff,
.send_synack = tcp_v4_send_synack,
};
}
}
-static u32 tcp_v6_init_sequence(const struct sk_buff *skb, u32 *tsoff)
+static u32 tcp_v6_init_seq_and_tsoff(const struct sk_buff *skb, u32 *tsoff)
{
- return secure_tcpv6_sequence_number(ipv6_hdr(skb)->daddr.s6_addr32,
- ipv6_hdr(skb)->saddr.s6_addr32,
- tcp_hdr(skb)->dest,
- tcp_hdr(skb)->source, tsoff);
+ return secure_tcpv6_seq_and_tsoff(ipv6_hdr(skb)->daddr.s6_addr32,
+ ipv6_hdr(skb)->saddr.s6_addr32,
+ tcp_hdr(skb)->dest,
+ tcp_hdr(skb)->source, tsoff);
}
static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
sk_set_txhash(sk);
if (likely(!tp->repair)) {
- seq = secure_tcpv6_sequence_number(np->saddr.s6_addr32,
- sk->sk_v6_daddr.s6_addr32,
- inet->inet_sport,
- inet->inet_dport,
- &tp->tsoffset);
+ seq = secure_tcpv6_seq_and_tsoff(np->saddr.s6_addr32,
+ sk->sk_v6_daddr.s6_addr32,
+ inet->inet_sport,
+ inet->inet_dport,
+ &tp->tsoffset);
if (!tp->write_seq)
tp->write_seq = seq;
}
.cookie_init_seq = cookie_v6_init_sequence,
#endif
.route_req = tcp_v6_route_req,
- .init_seq = tcp_v6_init_sequence,
+ .init_seq_tsoff = tcp_v6_init_seq_and_tsoff,
.send_synack = tcp_v6_send_synack,
};
return 0;
}
+static struct sock *__udp6_lib_demux_lookup(struct net *net,
+ __be16 loc_port, const struct in6_addr *loc_addr,
+ __be16 rmt_port, const struct in6_addr *rmt_addr,
+ int dif)
+{
+ struct sock *sk;
+
+ rcu_read_lock();
+ sk = __udp6_lib_lookup(net, rmt_addr, rmt_port, loc_addr, loc_port,
+ dif, &udp_table, NULL);
+ if (sk && !atomic_inc_not_zero(&sk->sk_refcnt))
+ sk = NULL;
+ rcu_read_unlock();
+
+ return sk;
+}
+
+static void udp_v6_early_demux(struct sk_buff *skb)
+{
+ struct net *net = dev_net(skb->dev);
+ const struct udphdr *uh;
+ struct sock *sk;
+ struct dst_entry *dst;
+ int dif = skb->dev->ifindex;
+
+ if (!pskb_may_pull(skb, skb_transport_offset(skb) +
+ sizeof(struct udphdr)))
+ return;
+
+ uh = udp_hdr(skb);
+
+ if (skb->pkt_type == PACKET_HOST)
+ sk = __udp6_lib_demux_lookup(net, uh->dest,
+ &ipv6_hdr(skb)->daddr,
+ uh->source, &ipv6_hdr(skb)->saddr,
+ dif);
+ else
+ return;
+
+ if (!sk)
+ return;
+
+ skb->sk = sk;
+ skb->destructor = sock_efree;
+ dst = READ_ONCE(sk->sk_rx_dst);
+
+ if (dst)
+ dst = dst_check(dst, inet6_sk(sk)->rx_dst_cookie);
+ if (dst) {
+ if (dst->flags & DST_NOCACHE) {
+ if (likely(atomic_inc_not_zero(&dst->__refcnt)))
+ skb_dst_set(skb, dst);
+ } else {
+ skb_dst_set_noref(skb, dst);
+ }
+ }
+}
+
static __inline__ int udpv6_rcv(struct sk_buff *skb)
{
return __udp6_lib_rcv(skb, &udp_table, IPPROTO_UDP);
#endif
static const struct inet6_protocol udpv6_protocol = {
+ .early_demux = udp_v6_early_demux,
.handler = udpv6_rcv,
.err_handler = udpv6_err,
.flags = INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL,
return NULL;
}
-void qdisc_hash_add(struct Qdisc *q)
+void qdisc_hash_add(struct Qdisc *q, bool invisible)
{
if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) {
struct Qdisc *root = qdisc_dev(q)->qdisc;
WARN_ON_ONCE(root == &noop_qdisc);
ASSERT_RTNL();
hash_add_rcu(qdisc_dev(q)->qdisc_hash, &q->hash, q->handle);
+ if (invisible)
+ q->flags |= TCQ_F_INVISIBLE;
}
}
EXPORT_SYMBOL(qdisc_hash_add);
goto err_out4;
}
- qdisc_hash_add(sch);
+ qdisc_hash_add(sch, false);
return sch;
}
return -1;
}
-static bool tc_qdisc_dump_ignore(struct Qdisc *q)
+static bool tc_qdisc_dump_ignore(struct Qdisc *q, bool dump_invisible)
{
- return (q->flags & TCQ_F_BUILTIN) ? true : false;
+ if (q->flags & TCQ_F_BUILTIN)
+ return true;
+ if ((q->flags & TCQ_F_INVISIBLE) && !dump_invisible)
+ return true;
+
+ return false;
}
static int qdisc_notify(struct net *net, struct sk_buff *oskb,
if (!skb)
return -ENOBUFS;
- if (old && !tc_qdisc_dump_ignore(old)) {
+ if (old && !tc_qdisc_dump_ignore(old, false)) {
if (tc_fill_qdisc(skb, old, clid, portid, n->nlmsg_seq,
0, RTM_DELQDISC) < 0)
goto err_out;
}
- if (new && !tc_qdisc_dump_ignore(new)) {
+ if (new && !tc_qdisc_dump_ignore(new, false)) {
if (tc_fill_qdisc(skb, new, clid, portid, n->nlmsg_seq,
old ? NLM_F_REPLACE : 0, RTM_NEWQDISC) < 0)
goto err_out;
static int tc_dump_qdisc_root(struct Qdisc *root, struct sk_buff *skb,
struct netlink_callback *cb,
- int *q_idx_p, int s_q_idx, bool recur)
+ int *q_idx_p, int s_q_idx, bool recur,
+ bool dump_invisible)
{
int ret = 0, q_idx = *q_idx_p;
struct Qdisc *q;
if (q_idx < s_q_idx) {
q_idx++;
} else {
- if (!tc_qdisc_dump_ignore(q) &&
+ if (!tc_qdisc_dump_ignore(q, dump_invisible) &&
tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid,
cb->nlh->nlmsg_seq, NLM_F_MULTI,
RTM_NEWQDISC) <= 0)
q_idx++;
continue;
}
- if (!tc_qdisc_dump_ignore(q) &&
+ if (!tc_qdisc_dump_ignore(q, dump_invisible) &&
tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid,
cb->nlh->nlmsg_seq, NLM_F_MULTI,
RTM_NEWQDISC) <= 0)
int idx, q_idx;
int s_idx, s_q_idx;
struct net_device *dev;
+ const struct nlmsghdr *nlh = cb->nlh;
+ struct tcmsg *tcm = nlmsg_data(nlh);
+ struct nlattr *tca[TCA_MAX + 1];
+ int err;
s_idx = cb->args[0];
s_q_idx = q_idx = cb->args[1];
idx = 0;
ASSERT_RTNL();
+
+ err = nlmsg_parse(nlh, sizeof(*tcm), tca, TCA_MAX, NULL);
+ if (err < 0)
+ return err;
+
for_each_netdev(net, dev) {
struct netdev_queue *dev_queue;
q_idx = 0;
if (tc_dump_qdisc_root(dev->qdisc, skb, cb, &q_idx, s_q_idx,
- true) < 0)
+ true, tca[TCA_DUMP_INVISIBLE]) < 0)
goto done;
dev_queue = dev_ingress_queue(dev);
if (dev_queue &&
tc_dump_qdisc_root(dev_queue->qdisc_sleeping, skb, cb,
- &q_idx, s_q_idx, false) < 0)
+ &q_idx, s_q_idx, false,
+ tca[TCA_DUMP_INVISIBLE]) < 0)
goto done;
cont:
{
struct qdisc_dump_args arg;
- if (tc_qdisc_dump_ignore(q) ||
+ if (tc_qdisc_dump_ignore(q, false) ||
*t_p < s_t || !q->ops->cl_ops ||
(tcm->tcm_parent &&
TC_H_MAJ(tcm->tcm_parent) != q->handle)) {
sch->handle);
if (!q->link.q)
q->link.q = &noop_qdisc;
+ else
+ qdisc_hash_add(q->link.q, true);
q->link.priority = TC_CBQ_MAXPRIO - 1;
q->link.priority2 = TC_CBQ_MAXPRIO - 1;
cl->q = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops, classid);
if (!cl->q)
cl->q = &noop_qdisc;
+ else
+ qdisc_hash_add(cl->q, true);
+
cl->common.classid = classid;
cl->tparent = parent;
cl->qdisc = sch;
&pfifo_qdisc_ops, classid);
if (cl->qdisc == NULL)
cl->qdisc = &noop_qdisc;
+ else
+ qdisc_hash_add(cl->qdisc, true);
if (tca[TCA_RATE]) {
err = gen_replace_estimator(&cl->bstats, NULL, &cl->rate_est,
p->q = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops, sch->handle);
if (p->q == NULL)
p->q = &noop_qdisc;
+ else
+ qdisc_hash_add(p->q, true);
pr_debug("%s: qdisc %p\n", __func__, p->q);
}
#ifdef CONFIG_NET_SCHED
if (dev->qdisc)
- qdisc_hash_add(dev->qdisc);
+ qdisc_hash_add(dev->qdisc, false);
#endif
}
&pfifo_qdisc_ops, classid);
if (cl->qdisc == NULL)
cl->qdisc = &noop_qdisc;
+ else
+ qdisc_hash_add(cl->qdisc, true);
INIT_LIST_HEAD(&cl->children);
cl->vt_tree = RB_ROOT;
cl->cf_tree = RB_ROOT;
sch->handle);
if (q->root.qdisc == NULL)
q->root.qdisc = &noop_qdisc;
+ else
+ qdisc_hash_add(q->root.qdisc, true);
INIT_LIST_HEAD(&q->root.children);
q->root.vt_tree = RB_ROOT;
q->root.cf_tree = RB_ROOT;
qdisc_class_hash_insert(&q->clhash, &cl->common);
if (parent)
parent->children++;
+ if (cl->un.leaf.q != &noop_qdisc)
+ qdisc_hash_add(cl->un.leaf.q, true);
} else {
if (tca[TCA_RATE]) {
err = gen_replace_estimator(&cl->bstats, NULL,
qdisc_destroy(old);
#ifdef CONFIG_NET_SCHED
if (ntx < dev->real_num_tx_queues)
- qdisc_hash_add(qdisc);
+ qdisc_hash_add(qdisc, false);
#endif
}
if (old)
qdisc_destroy(old);
if (ntx < dev->real_num_tx_queues)
- qdisc_hash_add(qdisc);
+ qdisc_hash_add(qdisc, false);
}
kfree(priv->qdiscs);
priv->qdiscs = NULL;
sch_tree_lock(sch);
old = q->queues[i];
q->queues[i] = child;
+ if (child != &noop_qdisc)
+ qdisc_hash_add(child, true);
if (old != &noop_qdisc) {
qdisc_tree_reduce_backlog(old,
qdisc_destroy(child);
}
- for (i = oldbands; i < q->bands; i++)
+ for (i = oldbands; i < q->bands; i++) {
q->queues[i] = queues[i];
+ if (q->queues[i] != &noop_qdisc)
+ qdisc_hash_add(q->queues[i], true);
+ }
sch_tree_unlock(sch);
return 0;
goto destroy_class;
}
+ if (cl->qdisc != &noop_qdisc)
+ qdisc_hash_add(cl->qdisc, true);
sch_tree_lock(sch);
qdisc_class_hash_insert(&q->clhash, &cl->common);
sch_tree_unlock(sch);
return PTR_ERR(child);
}
+ if (child != &noop_qdisc)
+ qdisc_hash_add(child, true);
sch_tree_lock(sch);
q->flags = ctl->flags;
q->limit = ctl->limit;
if (IS_ERR(child))
return PTR_ERR(child);
+ if (child != &noop_qdisc)
+ qdisc_hash_add(child, true);
sch_tree_lock(sch);
qdisc_tree_reduce_backlog(q->qdisc, q->qdisc->q.qlen,
q->qdisc->qstats.backlog);
qdisc_destroy(q->qdisc);
q->qdisc = child;
+ if (child != &noop_qdisc);
+ qdisc_hash_add(child, true);
}
q->limit = qopt->limit;
if (tb[TCA_TBF_PBURST])
projects / linux-2.6-block.git / commitdiff