[linux-2.6-block.git] / drivers / net / ethernet / stmicro / stmmac / dwmac4_descs.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * This contains the functions to handle the descriptors for DesignWare databook
 * 4.xx.
 *
 * Copyright (C) 2015  STMicroelectronics Ltd
 *
 * Author: Alexandre Torgue <alexandre.torgue@st.com>
 */

#include <linux/stmmac.h>
#include "common.h"
#include "dwmac4_descs.h"

static int dwmac4_wrback_get_tx_status(void *data, struct stmmac_extra_stats *x,
				       struct dma_desc *p,
				       void __iomem *ioaddr)
{
	struct net_device_stats *stats = (struct net_device_stats *)data;
	unsigned int tdes3;
	int ret = tx_done;

	tdes3 = le32_to_cpu(p->des3);

	/* Get tx owner first */
	if (unlikely(tdes3 & TDES3_OWN))
		return tx_dma_own;

	/* Verify tx error by looking at the last segment. */
	if (likely(!(tdes3 & TDES3_LAST_DESCRIPTOR)))
		return tx_not_ls;

	if (unlikely(tdes3 & TDES3_ERROR_SUMMARY)) {
		if (unlikely(tdes3 & TDES3_JABBER_TIMEOUT))
			x->tx_jabber++;
		if (unlikely(tdes3 & TDES3_PACKET_FLUSHED))
			x->tx_frame_flushed++;
		if (unlikely(tdes3 & TDES3_LOSS_CARRIER)) {
			x->tx_losscarrier++;
			stats->tx_carrier_errors++;
		}
		if (unlikely(tdes3 & TDES3_NO_CARRIER)) {
			x->tx_carrier++;
			stats->tx_carrier_errors++;
		}
		if (unlikely((tdes3 & TDES3_LATE_COLLISION) ||
			     (tdes3 & TDES3_EXCESSIVE_COLLISION)))
			stats->collisions +=
			    (tdes3 & TDES3_COLLISION_COUNT_MASK)
			    >> TDES3_COLLISION_COUNT_SHIFT;

		if (unlikely(tdes3 & TDES3_EXCESSIVE_DEFERRAL))
			x->tx_deferred++;

		if (unlikely(tdes3 & TDES3_UNDERFLOW_ERROR))
			x->tx_underflow++;

		if (unlikely(tdes3 & TDES3_IP_HDR_ERROR))
			x->tx_ip_header_error++;

		if (unlikely(tdes3 & TDES3_PAYLOAD_ERROR))
			x->tx_payload_error++;

		ret = tx_err;
	}

	if (unlikely(tdes3 & TDES3_DEFERRED))
		x->tx_deferred++;

	return ret;
}

static int dwmac4_wrback_get_rx_status(void *data, struct stmmac_extra_stats *x,
				       struct dma_desc *p)
{
	struct net_device_stats *stats = (struct net_device_stats *)data;
	unsigned int rdes1 = le32_to_cpu(p->des1);
	unsigned int rdes2 = le32_to_cpu(p->des2);
	unsigned int rdes3 = le32_to_cpu(p->des3);
	int message_type;
	int ret = good_frame;

	if (unlikely(rdes3 & RDES3_OWN))
		return dma_own;

	/* Verify rx error by looking at the last segment. */
	if (likely(!(rdes3 & RDES3_LAST_DESCRIPTOR)))
		return discard_frame;

	if (unlikely(rdes3 & RDES3_ERROR_SUMMARY)) {
		if (unlikely(rdes3 & RDES3_GIANT_PACKET))
			stats->rx_length_errors++;
		if (unlikely(rdes3 & RDES3_OVERFLOW_ERROR))
			x->rx_gmac_overflow++;

		if (unlikely(rdes3 & RDES3_RECEIVE_WATCHDOG))
			x->rx_watchdog++;

		if (unlikely(rdes3 & RDES3_RECEIVE_ERROR))
			x->rx_mii++;

		if (unlikely(rdes3 & RDES3_CRC_ERROR)) {
			x->rx_crc_errors++;
			stats->rx_crc_errors++;
		}

		if (unlikely(rdes3 & RDES3_DRIBBLE_ERROR))
			x->dribbling_bit++;

		ret = discard_frame;
	}

	message_type = (rdes1 & ERDES4_MSG_TYPE_MASK) >> 8;

	if (rdes1 & RDES1_IP_HDR_ERROR)
		x->ip_hdr_err++;
	if (rdes1 & RDES1_IP_CSUM_BYPASSED)
		x->ip_csum_bypassed++;
	if (rdes1 & RDES1_IPV4_HEADER)
		x->ipv4_pkt_rcvd++;
	if (rdes1 & RDES1_IPV6_HEADER)
		x->ipv6_pkt_rcvd++;

	if (message_type == RDES_EXT_NO_PTP)
		x->no_ptp_rx_msg_type_ext++;
	else if (message_type == RDES_EXT_SYNC)
		x->ptp_rx_msg_type_sync++;
	else if (message_type == RDES_EXT_FOLLOW_UP)
		x->ptp_rx_msg_type_follow_up++;
	else if (message_type == RDES_EXT_DELAY_REQ)
		x->ptp_rx_msg_type_delay_req++;
	else if (message_type == RDES_EXT_DELAY_RESP)
		x->ptp_rx_msg_type_delay_resp++;
	else if (message_type == RDES_EXT_PDELAY_REQ)
		x->ptp_rx_msg_type_pdelay_req++;
	else if (message_type == RDES_EXT_PDELAY_RESP)
		x->ptp_rx_msg_type_pdelay_resp++;
	else if (message_type == RDES_EXT_PDELAY_FOLLOW_UP)
		x->ptp_rx_msg_type_pdelay_follow_up++;
	else if (message_type == RDES_PTP_ANNOUNCE)
		x->ptp_rx_msg_type_announce++;
	else if (message_type == RDES_PTP_MANAGEMENT)
		x->ptp_rx_msg_type_management++;
	else if (message_type == RDES_PTP_PKT_RESERVED_TYPE)
		x->ptp_rx_msg_pkt_reserved_type++;

	if (rdes1 & RDES1_PTP_PACKET_TYPE)
		x->ptp_frame_type++;
	if (rdes1 & RDES1_PTP_VER)
		x->ptp_ver++;
	if (rdes1 & RDES1_TIMESTAMP_DROPPED)
		x->timestamp_dropped++;

	if (unlikely(rdes2 & RDES2_SA_FILTER_FAIL)) {
		x->sa_rx_filter_fail++;
		ret = discard_frame;
	}
	if (unlikely(rdes2 & RDES2_DA_FILTER_FAIL)) {
		x->da_rx_filter_fail++;
		ret = discard_frame;
	}

	if (rdes2 & RDES2_L3_FILTER_MATCH)
		x->l3_filter_match++;
	if (rdes2 & RDES2_L4_FILTER_MATCH)
		x->l4_filter_match++;
	if ((rdes2 & RDES2_L3_L4_FILT_NB_MATCH_MASK)
	    >> RDES2_L3_L4_FILT_NB_MATCH_SHIFT)
		x->l3_l4_filter_no_match++;

	return ret;
}

static int dwmac4_rd_get_tx_len(struct dma_desc *p)
{
	return (le32_to_cpu(p->des2) & TDES2_BUFFER1_SIZE_MASK);
}

static int dwmac4_get_tx_owner(struct dma_desc *p)
{
	return (le32_to_cpu(p->des3) & TDES3_OWN) >> TDES3_OWN_SHIFT;
}

static void dwmac4_set_tx_owner(struct dma_desc *p)
{
	p->des3 |= cpu_to_le32(TDES3_OWN);
}

static void dwmac4_set_rx_owner(struct dma_desc *p, int disable_rx_ic)
{
	p->des3 = cpu_to_le32(RDES3_OWN | RDES3_BUFFER1_VALID_ADDR);

	if (!disable_rx_ic)
		p->des3 |= cpu_to_le32(RDES3_INT_ON_COMPLETION_EN);
}

static int dwmac4_get_tx_ls(struct dma_desc *p)
{
	return (le32_to_cpu(p->des3) & TDES3_LAST_DESCRIPTOR)
		>> TDES3_LAST_DESCRIPTOR_SHIFT;
}

static int dwmac4_wrback_get_rx_frame_len(struct dma_desc *p, int rx_coe)
{
	return (le32_to_cpu(p->des3) & RDES3_PACKET_SIZE_MASK);
}

static void dwmac4_rd_enable_tx_timestamp(struct dma_desc *p)
{
	p->des2 |= cpu_to_le32(TDES2_TIMESTAMP_ENABLE);
}

static int dwmac4_wrback_get_tx_timestamp_status(struct dma_desc *p)
{
	/* Context type from W/B descriptor must be zero */
	if (le32_to_cpu(p->des3) & TDES3_CONTEXT_TYPE)
		return 0;

	/* Tx Timestamp Status is 1 so des0 and des1'll have valid values */
	if (le32_to_cpu(p->des3) & TDES3_TIMESTAMP_STATUS)
		return 1;

	return 0;
}

static inline void dwmac4_get_timestamp(void *desc, u32 ats, u64 *ts)
{
	struct dma_desc *p = (struct dma_desc *)desc;
	u64 ns;

	ns = le32_to_cpu(p->des0);
	/* convert high/sec time stamp value to nanosecond */
	ns += le32_to_cpu(p->des1) * 1000000000ULL;

	*ts = ns;
}

static int dwmac4_rx_check_timestamp(void *desc)
{
	struct dma_desc *p = (struct dma_desc *)desc;
	unsigned int rdes0 = le32_to_cpu(p->des0);
	unsigned int rdes1 = le32_to_cpu(p->des1);
	unsigned int rdes3 = le32_to_cpu(p->des3);
	u32 own, ctxt;
	int ret = 1;

	own = rdes3 & RDES3_OWN;
	ctxt = ((rdes3 & RDES3_CONTEXT_DESCRIPTOR)
		>> RDES3_CONTEXT_DESCRIPTOR_SHIFT);

	if (likely(!own && ctxt)) {
		if ((rdes0 == 0xffffffff) && (rdes1 == 0xffffffff))
			/* Corrupted value */
			ret = -EINVAL;
		else
			/* A valid Timestamp is ready to be read */
			ret = 0;
	}

	/* Timestamp not ready */
	return ret;
}

static int dwmac4_wrback_get_rx_timestamp_status(void *desc, void *next_desc,
						 u32 ats)
{
	struct dma_desc *p = (struct dma_desc *)desc;
	int ret = -EINVAL;

	/* Get the status from normal w/b descriptor */
	if (likely(le32_to_cpu(p->des3) & RDES3_RDES1_VALID)) {
		if (likely(le32_to_cpu(p->des1) & RDES1_TIMESTAMP_AVAILABLE)) {
			int i = 0;

			/* Check if timestamp is OK from context descriptor */
			do {
				ret = dwmac4_rx_check_timestamp(next_desc);
				if (ret < 0)
					goto exit;
				i++;

			} while ((ret == 1) && (i < 10));

			if (i == 10)
				ret = -EBUSY;
		}
	}
exit:
	if (likely(ret == 0))
		return 1;

	return 0;
}

static void dwmac4_rd_init_rx_desc(struct dma_desc *p, int disable_rx_ic,
				   int mode, int end, int bfsize)
{
	dwmac4_set_rx_owner(p, disable_rx_ic);
}

static void dwmac4_rd_init_tx_desc(struct dma_desc *p, int mode, int end)
{
	p->des0 = 0;
	p->des1 = 0;
	p->des2 = 0;
	p->des3 = 0;
}

static void dwmac4_rd_prepare_tx_desc(struct dma_desc *p, int is_fs, int len,
				      bool csum_flag, int mode, bool tx_own,
				      bool ls, unsigned int tot_pkt_len)
{
	unsigned int tdes3 = le32_to_cpu(p->des3);

	p->des2 |= cpu_to_le32(len & TDES2_BUFFER1_SIZE_MASK);

	tdes3 |= tot_pkt_len & TDES3_PACKET_SIZE_MASK;
	if (is_fs)
		tdes3 |= TDES3_FIRST_DESCRIPTOR;
	else
		tdes3 &= ~TDES3_FIRST_DESCRIPTOR;

	if (likely(csum_flag))
		tdes3 |= (TX_CIC_FULL << TDES3_CHECKSUM_INSERTION_SHIFT);
	else
		tdes3 &= ~(TX_CIC_FULL << TDES3_CHECKSUM_INSERTION_SHIFT);

	if (ls)
		tdes3 |= TDES3_LAST_DESCRIPTOR;
	else
		tdes3 &= ~TDES3_LAST_DESCRIPTOR;

	/* Finally set the OWN bit. Later the DMA will start! */
	if (tx_own)
		tdes3 |= TDES3_OWN;

	if (is_fs && tx_own)
		/* When the own bit, for the first frame, has to be set, all
		 * descriptors for the same frame has to be set before, to
		 * avoid race condition.
		 */
		dma_wmb();

	p->des3 = cpu_to_le32(tdes3);
}

static void dwmac4_rd_prepare_tso_tx_desc(struct dma_desc *p, int is_fs,
					  int len1, int len2, bool tx_own,
					  bool ls, unsigned int tcphdrlen,
					  unsigned int tcppayloadlen)
{
	unsigned int tdes3 = le32_to_cpu(p->des3);

	if (len1)
		p->des2 |= cpu_to_le32((len1 & TDES2_BUFFER1_SIZE_MASK));

	if (len2)
		p->des2 |= cpu_to_le32((len2 << TDES2_BUFFER2_SIZE_MASK_SHIFT)
			    & TDES2_BUFFER2_SIZE_MASK);

	if (is_fs) {
		tdes3 |= TDES3_FIRST_DESCRIPTOR |
			 TDES3_TCP_SEGMENTATION_ENABLE |
			 ((tcphdrlen << TDES3_HDR_LEN_SHIFT) &
			  TDES3_SLOT_NUMBER_MASK) |
			 ((tcppayloadlen & TDES3_TCP_PKT_PAYLOAD_MASK));
	} else {
		tdes3 &= ~TDES3_FIRST_DESCRIPTOR;
	}

	if (ls)
		tdes3 |= TDES3_LAST_DESCRIPTOR;
	else
		tdes3 &= ~TDES3_LAST_DESCRIPTOR;

	/* Finally set the OWN bit. Later the DMA will start! */
	if (tx_own)
		tdes3 |= TDES3_OWN;

	if (is_fs && tx_own)
		/* When the own bit, for the first frame, has to be set, all
		 * descriptors for the same frame has to be set before, to
		 * avoid race condition.
		 */
		dma_wmb();

	p->des3 = cpu_to_le32(tdes3);
}

static void dwmac4_release_tx_desc(struct dma_desc *p, int mode)
{
	p->des0 = 0;
	p->des1 = 0;
	p->des2 = 0;
	p->des3 = 0;
}

static void dwmac4_rd_set_tx_ic(struct dma_desc *p)
{
	p->des2 |= cpu_to_le32(TDES2_INTERRUPT_ON_COMPLETION);
}

static void dwmac4_display_ring(void *head, unsigned int size, bool rx)
{
	struct dma_desc *p = (struct dma_desc *)head;
	int i;

	pr_info("%s descriptor ring:\n", rx ? "RX" : "TX");

	for (i = 0; i < size; i++) {
		pr_info("%03d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
			i, (unsigned int)virt_to_phys(p),
			le32_to_cpu(p->des0), le32_to_cpu(p->des1),
			le32_to_cpu(p->des2), le32_to_cpu(p->des3));
		p++;
	}
}

static void dwmac4_set_mss_ctxt(struct dma_desc *p, unsigned int mss)
{
	p->des0 = 0;
	p->des1 = 0;
	p->des2 = cpu_to_le32(mss);
	p->des3 = cpu_to_le32(TDES3_CONTEXT_TYPE | TDES3_CTXT_TCMSSV);
}

static void dwmac4_get_addr(struct dma_desc *p, unsigned int *addr)
{
	*addr = le32_to_cpu(p->des0);
}

static void dwmac4_set_addr(struct dma_desc *p, dma_addr_t addr)
{
	p->des0 = cpu_to_le32(lower_32_bits(addr));
	p->des1 = cpu_to_le32(upper_32_bits(addr));
}

static void dwmac4_clear(struct dma_desc *p)
{
	p->des0 = 0;
	p->des1 = 0;
	p->des2 = 0;
	p->des3 = 0;
}

static void dwmac4_set_sarc(struct dma_desc *p, u32 sarc_type)
{
	sarc_type <<= TDES3_SA_INSERT_CTRL_SHIFT;

	p->des3 |= cpu_to_le32(sarc_type & TDES3_SA_INSERT_CTRL_MASK);
}

static int set_16kib_bfsize(int mtu)
{
	int ret = 0;

	if (unlikely(mtu >= BUF_SIZE_8KiB))
		ret = BUF_SIZE_16KiB;
	return ret;
}

static void dwmac4_set_vlan_tag(struct dma_desc *p, u16 tag, u16 inner_tag,
				u32 inner_type)
{
	p->des0 = 0;
	p->des1 = 0;
	p->des2 = 0;
	p->des3 = 0;

	/* Inner VLAN */
	if (inner_type) {
		u32 des = inner_tag << TDES2_IVT_SHIFT;

		des &= TDES2_IVT_MASK;
		p->des2 = cpu_to_le32(des);

		des = inner_type << TDES3_IVTIR_SHIFT;
		des &= TDES3_IVTIR_MASK;
		p->des3 = cpu_to_le32(des | TDES3_IVLTV);
	}

	/* Outer VLAN */
	p->des3 |= cpu_to_le32(tag & TDES3_VLAN_TAG);
	p->des3 |= cpu_to_le32(TDES3_VLTV);

	p->des3 |= cpu_to_le32(TDES3_CONTEXT_TYPE);
}

static void dwmac4_set_vlan(struct dma_desc *p, u32 type)
{
	type <<= TDES2_VLAN_TAG_SHIFT;
	p->des2 |= cpu_to_le32(type & TDES2_VLAN_TAG_MASK);
}

const struct stmmac_desc_ops dwmac4_desc_ops = {
	.tx_status = dwmac4_wrback_get_tx_status,
	.rx_status = dwmac4_wrback_get_rx_status,
	.get_tx_len = dwmac4_rd_get_tx_len,
	.get_tx_owner = dwmac4_get_tx_owner,
	.set_tx_owner = dwmac4_set_tx_owner,
	.set_rx_owner = dwmac4_set_rx_owner,
	.get_tx_ls = dwmac4_get_tx_ls,
	.get_rx_frame_len = dwmac4_wrback_get_rx_frame_len,
	.enable_tx_timestamp = dwmac4_rd_enable_tx_timestamp,
	.get_tx_timestamp_status = dwmac4_wrback_get_tx_timestamp_status,
	.get_rx_timestamp_status = dwmac4_wrback_get_rx_timestamp_status,
	.get_timestamp = dwmac4_get_timestamp,
	.set_tx_ic = dwmac4_rd_set_tx_ic,
	.prepare_tx_desc = dwmac4_rd_prepare_tx_desc,
	.prepare_tso_tx_desc = dwmac4_rd_prepare_tso_tx_desc,
	.release_tx_desc = dwmac4_release_tx_desc,
	.init_rx_desc = dwmac4_rd_init_rx_desc,
	.init_tx_desc = dwmac4_rd_init_tx_desc,
	.display_ring = dwmac4_display_ring,
	.set_mss = dwmac4_set_mss_ctxt,
	.get_addr = dwmac4_get_addr,
	.set_addr = dwmac4_set_addr,
	.clear = dwmac4_clear,
	.set_sarc = dwmac4_set_sarc,
	.set_vlan_tag = dwmac4_set_vlan_tag,
	.set_vlan = dwmac4_set_vlan,
};

const struct stmmac_mode_ops dwmac4_ring_mode_ops = {
	.set_16kib_bfsize = set_16kib_bfsize,
};
Commit	Line	Data
75a6faf6	1	// SPDX-License-Identifier: GPL-2.0-only
753a7109 AT	2	/*
	3	* This contains the functions to handle the descriptors for DesignWare databook
	4	* 4.xx.
	5	*
	6	* Copyright (C) 2015 STMicroelectronics Ltd
	7	*
753a7109 AT	8	* Author: Alexandre Torgue <alexandre.torgue@st.com>
	9	*/
	10
	11	#include <linux/stmmac.h>
	12	#include "common.h"
	13	#include "dwmac4_descs.h"
	14
	15	static int dwmac4_wrback_get_tx_status(void data, struct stmmac_extra_stats x,
	16	struct dma_desc *p,
	17	void __iomem *ioaddr)
	18	{
	19	struct net_device_stats stats = (struct net_device_stats )data;
	20	unsigned int tdes3;
	21	int ret = tx_done;
	22
f8be0d78	23	tdes3 = le32_to_cpu(p->des3);
753a7109 AT	24
	25	/* Get tx owner first */
	26	if (unlikely(tdes3 & TDES3_OWN))
	27	return tx_dma_own;
	28
	29	/* Verify tx error by looking at the last segment. */
	30	if (likely(!(tdes3 & TDES3_LAST_DESCRIPTOR)))
	31	return tx_not_ls;
	32
	33	if (unlikely(tdes3 & TDES3_ERROR_SUMMARY)) {
	34	if (unlikely(tdes3 & TDES3_JABBER_TIMEOUT))
	35	x->tx_jabber++;
	36	if (unlikely(tdes3 & TDES3_PACKET_FLUSHED))
	37	x->tx_frame_flushed++;
	38	if (unlikely(tdes3 & TDES3_LOSS_CARRIER)) {
	39	x->tx_losscarrier++;
	40	stats->tx_carrier_errors++;
	41	}
	42	if (unlikely(tdes3 & TDES3_NO_CARRIER)) {
	43	x->tx_carrier++;
	44	stats->tx_carrier_errors++;
	45	}
	46	if (unlikely((tdes3 & TDES3_LATE_COLLISION) \|\|
	47	(tdes3 & TDES3_EXCESSIVE_COLLISION)))
	48	stats->collisions +=
	49	(tdes3 & TDES3_COLLISION_COUNT_MASK)
	50	>> TDES3_COLLISION_COUNT_SHIFT;
	51
	52	if (unlikely(tdes3 & TDES3_EXCESSIVE_DEFERRAL))
	53	x->tx_deferred++;
	54
	55	if (unlikely(tdes3 & TDES3_UNDERFLOW_ERROR))
	56	x->tx_underflow++;
	57
	58	if (unlikely(tdes3 & TDES3_IP_HDR_ERROR))
	59	x->tx_ip_header_error++;
	60
	61	if (unlikely(tdes3 & TDES3_PAYLOAD_ERROR))
	62	x->tx_payload_error++;
	63
	64	ret = tx_err;
	65	}
	66
	67	if (unlikely(tdes3 & TDES3_DEFERRED))
	68	x->tx_deferred++;
	69
	70	return ret;
	71	}
	72
	73	static int dwmac4_wrback_get_rx_status(void data, struct stmmac_extra_stats x,
	74	struct dma_desc *p)
	75	{
	76	struct net_device_stats stats = (struct net_device_stats )data;
f8be0d78 MW	77	unsigned int rdes1 = le32_to_cpu(p->des1);
	78	unsigned int rdes2 = le32_to_cpu(p->des2);
	79	unsigned int rdes3 = le32_to_cpu(p->des3);
753a7109 AT	80	int message_type;
	81	int ret = good_frame;
	82
	83	if (unlikely(rdes3 & RDES3_OWN))
	84	return dma_own;
	85
	86	/* Verify rx error by looking at the last segment. */
	87	if (likely(!(rdes3 & RDES3_LAST_DESCRIPTOR)))
	88	return discard_frame;
	89
	90	if (unlikely(rdes3 & RDES3_ERROR_SUMMARY)) {
	91	if (unlikely(rdes3 & RDES3_GIANT_PACKET))
	92	stats->rx_length_errors++;
	93	if (unlikely(rdes3 & RDES3_OVERFLOW_ERROR))
	94	x->rx_gmac_overflow++;
	95
	96	if (unlikely(rdes3 & RDES3_RECEIVE_WATCHDOG))
	97	x->rx_watchdog++;
	98
	99	if (unlikely(rdes3 & RDES3_RECEIVE_ERROR))
	100	x->rx_mii++;
	101
	102	if (unlikely(rdes3 & RDES3_CRC_ERROR)) {
e0a76606	103	x->rx_crc_errors++;
753a7109 AT	104	stats->rx_crc_errors++;
	105	}
	106
	107	if (unlikely(rdes3 & RDES3_DRIBBLE_ERROR))
	108	x->dribbling_bit++;
	109
	110	ret = discard_frame;
	111	}
	112
	113	message_type = (rdes1 & ERDES4_MSG_TYPE_MASK) >> 8;
	114
	115	if (rdes1 & RDES1_IP_HDR_ERROR)
	116	x->ip_hdr_err++;
	117	if (rdes1 & RDES1_IP_CSUM_BYPASSED)
	118	x->ip_csum_bypassed++;
	119	if (rdes1 & RDES1_IPV4_HEADER)
	120	x->ipv4_pkt_rcvd++;
	121	if (rdes1 & RDES1_IPV6_HEADER)
	122	x->ipv6_pkt_rcvd++;
ee112c12 GC	123
	124	if (message_type == RDES_EXT_NO_PTP)
	125	x->no_ptp_rx_msg_type_ext++;
	126	else if (message_type == RDES_EXT_SYNC)
	127	x->ptp_rx_msg_type_sync++;
753a7109	128	else if (message_type == RDES_EXT_FOLLOW_UP)
ee112c12	129	x->ptp_rx_msg_type_follow_up++;
753a7109	130	else if (message_type == RDES_EXT_DELAY_REQ)
ee112c12	131	x->ptp_rx_msg_type_delay_req++;
753a7109	132	else if (message_type == RDES_EXT_DELAY_RESP)
ee112c12	133	x->ptp_rx_msg_type_delay_resp++;
753a7109	134	else if (message_type == RDES_EXT_PDELAY_REQ)
ee112c12	135	x->ptp_rx_msg_type_pdelay_req++;
753a7109	136	else if (message_type == RDES_EXT_PDELAY_RESP)
ee112c12	137	x->ptp_rx_msg_type_pdelay_resp++;
753a7109	138	else if (message_type == RDES_EXT_PDELAY_FOLLOW_UP)
ee112c12 GC	139	x->ptp_rx_msg_type_pdelay_follow_up++;
	140	else if (message_type == RDES_PTP_ANNOUNCE)
	141	x->ptp_rx_msg_type_announce++;
	142	else if (message_type == RDES_PTP_MANAGEMENT)
	143	x->ptp_rx_msg_type_management++;
	144	else if (message_type == RDES_PTP_PKT_RESERVED_TYPE)
	145	x->ptp_rx_msg_pkt_reserved_type++;
753a7109 AT	146
	147	if (rdes1 & RDES1_PTP_PACKET_TYPE)
	148	x->ptp_frame_type++;
	149	if (rdes1 & RDES1_PTP_VER)
	150	x->ptp_ver++;
	151	if (rdes1 & RDES1_TIMESTAMP_DROPPED)
	152	x->timestamp_dropped++;
	153
	154	if (unlikely(rdes2 & RDES2_SA_FILTER_FAIL)) {
	155	x->sa_rx_filter_fail++;
	156	ret = discard_frame;
	157	}
	158	if (unlikely(rdes2 & RDES2_DA_FILTER_FAIL)) {
	159	x->da_rx_filter_fail++;
	160	ret = discard_frame;
	161	}
	162
	163	if (rdes2 & RDES2_L3_FILTER_MATCH)
	164	x->l3_filter_match++;
	165	if (rdes2 & RDES2_L4_FILTER_MATCH)
	166	x->l4_filter_match++;
	167	if ((rdes2 & RDES2_L3_L4_FILT_NB_MATCH_MASK)
	168	>> RDES2_L3_L4_FILT_NB_MATCH_SHIFT)
	169	x->l3_l4_filter_no_match++;
	170
	171	return ret;
	172	}
	173
	174	static int dwmac4_rd_get_tx_len(struct dma_desc *p)
	175	{
f8be0d78	176	return (le32_to_cpu(p->des2) & TDES2_BUFFER1_SIZE_MASK);
753a7109 AT	177	}
	178
	179	static int dwmac4_get_tx_owner(struct dma_desc *p)
	180	{
f8be0d78	181	return (le32_to_cpu(p->des3) & TDES3_OWN) >> TDES3_OWN_SHIFT;
753a7109 AT	182	}
	183
	184	static void dwmac4_set_tx_owner(struct dma_desc *p)
	185	{
f8be0d78	186	p->des3 \|= cpu_to_le32(TDES3_OWN);
753a7109 AT	187	}
753a7109 AT	188
357951cd	189	static void dwmac4_set_rx_owner(struct dma_desc *p, int disable_rx_ic)
753a7109	190	{
357951cd JA	191	p->des3 = cpu_to_le32(RDES3_OWN \| RDES3_BUFFER1_VALID_ADDR);
	192
	193	if (!disable_rx_ic)
	194	p->des3 \|= cpu_to_le32(RDES3_INT_ON_COMPLETION_EN);
753a7109 AT	195	}
	196
	197	static int dwmac4_get_tx_ls(struct dma_desc *p)
	198	{
f8be0d78 MW	199	return (le32_to_cpu(p->des3) & TDES3_LAST_DESCRIPTOR)
f8be0d78 MW	200	>> TDES3_LAST_DESCRIPTOR_SHIFT;
753a7109 AT	201	}
	202
	203	static int dwmac4_wrback_get_rx_frame_len(struct dma_desc *p, int rx_coe)
	204	{
f8be0d78	205	return (le32_to_cpu(p->des3) & RDES3_PACKET_SIZE_MASK);
753a7109 AT	206	}
	207
	208	static void dwmac4_rd_enable_tx_timestamp(struct dma_desc *p)
	209	{
f8be0d78	210	p->des2 \|= cpu_to_le32(TDES2_TIMESTAMP_ENABLE);
753a7109 AT	211	}
	212
	213	static int dwmac4_wrback_get_tx_timestamp_status(struct dma_desc *p)
	214	{
ba1ffd74	215	/* Context type from W/B descriptor must be zero */
f9aa9dc7	216	if (le32_to_cpu(p->des3) & TDES3_CONTEXT_TYPE)
33d4c482	217	return 0;
ba1ffd74 GC	218
ba1ffd74 GC	219	/* Tx Timestamp Status is 1 so des0 and des1'll have valid values */
f9aa9dc7	220	if (le32_to_cpu(p->des3) & TDES3_TIMESTAMP_STATUS)
33d4c482	221	return 1;
ba1ffd74	222
33d4c482	223	return 0;
753a7109 AT	224	}
753a7109 AT	225
42de047d	226	static inline void dwmac4_get_timestamp(void desc, u32 ats, u64 ts)
753a7109 AT	227	{
	228	struct dma_desc p = (struct dma_desc )desc;
	229	u64 ns;
	230
f8be0d78	231	ns = le32_to_cpu(p->des0);
753a7109	232	/* convert high/sec time stamp value to nanosecond */
f8be0d78	233	ns += le32_to_cpu(p->des1) * 1000000000ULL;
753a7109	234
42de047d	235	*ts = ns;
753a7109 AT	236	}
753a7109 AT	237
ba1ffd74 GC	238	static int dwmac4_rx_check_timestamp(void *desc)
	239	{
	240	struct dma_desc p = (struct dma_desc )desc;
4012e7d0 AT	241	unsigned int rdes0 = le32_to_cpu(p->des0);
	242	unsigned int rdes1 = le32_to_cpu(p->des1);
	243	unsigned int rdes3 = le32_to_cpu(p->des3);
ba1ffd74 GC	244	u32 own, ctxt;
	245	int ret = 1;
	246
4012e7d0 AT	247	own = rdes3 & RDES3_OWN;
4012e7d0 AT	248	ctxt = ((rdes3 & RDES3_CONTEXT_DESCRIPTOR)
ba1ffd74 GC	249	>> RDES3_CONTEXT_DESCRIPTOR_SHIFT);
	250
	251	if (likely(!own && ctxt)) {
4012e7d0	252	if ((rdes0 == 0xffffffff) && (rdes1 == 0xffffffff))
ba1ffd74 GC	253	/* Corrupted value */
	254	ret = -EINVAL;
	255	else
	256	/* A valid Timestamp is ready to be read */
	257	ret = 0;
	258	}
	259
	260	/* Timestamp not ready */
	261	return ret;
	262	}
	263
a1762456 FH	264	static int dwmac4_wrback_get_rx_timestamp_status(void desc, void next_desc,
a1762456 FH	265	u32 ats)
753a7109 AT	266	{
753a7109 AT	267	struct dma_desc p = (struct dma_desc )desc;
ba1ffd74 GC	268	int ret = -EINVAL;
	269
	270	/* Get the status from normal w/b descriptor */
f186a82b	271	if (likely(le32_to_cpu(p->des3) & RDES3_RDES1_VALID)) {
f9aa9dc7	272	if (likely(le32_to_cpu(p->des1) & RDES1_TIMESTAMP_AVAILABLE)) {
ba1ffd74 GC	273	int i = 0;
	274
	275	/* Check if timestamp is OK from context descriptor */
	276	do {
a1762456	277	ret = dwmac4_rx_check_timestamp(next_desc);
ba1ffd74 GC	278	if (ret < 0)
	279	goto exit;
	280	i++;
753a7109	281
9454360d	282	} while ((ret == 1) && (i < 10));
ba1ffd74 GC	283
	284	if (i == 10)
	285	ret = -EBUSY;
	286	}
	287	}
	288	exit:
33d4c482 MM	289	if (likely(ret == 0))
	290	return 1;
	291
	292	return 0;
753a7109 AT	293	}
	294
	295	static void dwmac4_rd_init_rx_desc(struct dma_desc *p, int disable_rx_ic,
583e6361	296	int mode, int end, int bfsize)
753a7109	297	{
357951cd	298	dwmac4_set_rx_owner(p, disable_rx_ic);
753a7109 AT	299	}
	300
	301	static void dwmac4_rd_init_tx_desc(struct dma_desc *p, int mode, int end)
	302	{
	303	p->des0 = 0;
	304	p->des1 = 0;
	305	p->des2 = 0;
	306	p->des3 = 0;
	307	}
	308
	309	static void dwmac4_rd_prepare_tx_desc(struct dma_desc *p, int is_fs, int len,
	310	bool csum_flag, int mode, bool tx_own,
fe6af0e1	311	bool ls, unsigned int tot_pkt_len)
753a7109	312	{
f8be0d78	313	unsigned int tdes3 = le32_to_cpu(p->des3);
753a7109	314
f8be0d78	315	p->des2 \|= cpu_to_le32(len & TDES2_BUFFER1_SIZE_MASK);
753a7109	316
fe6af0e1	317	tdes3 \|= tot_pkt_len & TDES3_PACKET_SIZE_MASK;
753a7109 AT	318	if (is_fs)
	319	tdes3 \|= TDES3_FIRST_DESCRIPTOR;
	320	else
	321	tdes3 &= ~TDES3_FIRST_DESCRIPTOR;
	322
	323	if (likely(csum_flag))
	324	tdes3 \|= (TX_CIC_FULL << TDES3_CHECKSUM_INSERTION_SHIFT);
	325	else
	326	tdes3 &= ~(TX_CIC_FULL << TDES3_CHECKSUM_INSERTION_SHIFT);
	327
	328	if (ls)
	329	tdes3 \|= TDES3_LAST_DESCRIPTOR;
	330	else
	331	tdes3 &= ~TDES3_LAST_DESCRIPTOR;
	332
	333	/* Finally set the OWN bit. Later the DMA will start! */
	334	if (tx_own)
	335	tdes3 \|= TDES3_OWN;
	336
d8f8b954	337	if (is_fs && tx_own)
753a7109 AT	338	/* When the own bit, for the first frame, has to be set, all
	339	* descriptors for the same frame has to be set before, to
	340	* avoid race condition.
	341	*/
ad688cdb	342	dma_wmb();
753a7109	343
f8be0d78	344	p->des3 = cpu_to_le32(tdes3);
753a7109 AT	345	}
	346
	347	static void dwmac4_rd_prepare_tso_tx_desc(struct dma_desc *p, int is_fs,
	348	int len1, int len2, bool tx_own,
	349	bool ls, unsigned int tcphdrlen,
	350	unsigned int tcppayloadlen)
	351	{
f8be0d78	352	unsigned int tdes3 = le32_to_cpu(p->des3);
753a7109 AT	353
753a7109 AT	354	if (len1)
f8be0d78	355	p->des2 \|= cpu_to_le32((len1 & TDES2_BUFFER1_SIZE_MASK));
753a7109 AT	356
753a7109 AT	357	if (len2)
f8be0d78 MW	358	p->des2 \|= cpu_to_le32((len2 << TDES2_BUFFER2_SIZE_MASK_SHIFT)
f8be0d78 MW	359	& TDES2_BUFFER2_SIZE_MASK);
753a7109 AT	360
	361	if (is_fs) {
	362	tdes3 \|= TDES3_FIRST_DESCRIPTOR \|
	363	TDES3_TCP_SEGMENTATION_ENABLE \|
	364	((tcphdrlen << TDES3_HDR_LEN_SHIFT) &
	365	TDES3_SLOT_NUMBER_MASK) \|
	366	((tcppayloadlen & TDES3_TCP_PKT_PAYLOAD_MASK));
	367	} else {
	368	tdes3 &= ~TDES3_FIRST_DESCRIPTOR;
	369	}
	370
	371	if (ls)
	372	tdes3 \|= TDES3_LAST_DESCRIPTOR;
	373	else
	374	tdes3 &= ~TDES3_LAST_DESCRIPTOR;
	375
	376	/* Finally set the OWN bit. Later the DMA will start! */
	377	if (tx_own)
	378	tdes3 \|= TDES3_OWN;
	379
d8f8b954	380	if (is_fs && tx_own)
753a7109 AT	381	/* When the own bit, for the first frame, has to be set, all
	382	* descriptors for the same frame has to be set before, to
	383	* avoid race condition.
	384	*/
ad688cdb	385	dma_wmb();
753a7109	386
f8be0d78	387	p->des3 = cpu_to_le32(tdes3);
753a7109 AT	388	}
	389
	390	static void dwmac4_release_tx_desc(struct dma_desc *p, int mode)
	391	{
1e88f6e0 NC	392	p->des0 = 0;
1e88f6e0 NC	393	p->des1 = 0;
753a7109 AT	394	p->des2 = 0;
	395	p->des3 = 0;
	396	}
	397
	398	static void dwmac4_rd_set_tx_ic(struct dma_desc *p)
	399	{
f8be0d78	400	p->des2 \|= cpu_to_le32(TDES2_INTERRUPT_ON_COMPLETION);
753a7109 AT	401	}
	402
	403	static void dwmac4_display_ring(void *head, unsigned int size, bool rx)
	404	{
	405	struct dma_desc p = (struct dma_desc )head;
	406	int i;
	407
	408	pr_info("%s descriptor ring:\n", rx ? "RX" : "TX");
	409
	410	for (i = 0; i < size; i++) {
bdb42166	411	pr_info("%03d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
8be0328e	412	i, (unsigned int)virt_to_phys(p),
f8be0d78 MW	413	le32_to_cpu(p->des0), le32_to_cpu(p->des1),
f8be0d78 MW	414	le32_to_cpu(p->des2), le32_to_cpu(p->des3));
753a7109 AT	415	p++;
	416	}
	417	}
	418
	419	static void dwmac4_set_mss_ctxt(struct dma_desc *p, unsigned int mss)
	420	{
	421	p->des0 = 0;
	422	p->des1 = 0;
f8be0d78 MW	423	p->des2 = cpu_to_le32(mss);
f8be0d78 MW	424	p->des3 = cpu_to_le32(TDES3_CONTEXT_TYPE \| TDES3_CTXT_TCMSSV);
753a7109 AT	425	}
753a7109 AT	426
d2df9ea0 JA	427	static void dwmac4_get_addr(struct dma_desc p, unsigned int addr)
	428	{
	429	*addr = le32_to_cpu(p->des0);
	430	}
	431
6844171d JA	432	static void dwmac4_set_addr(struct dma_desc *p, dma_addr_t addr)
6844171d JA	433	{
560c07cb TR	434	p->des0 = cpu_to_le32(lower_32_bits(addr));
560c07cb TR	435	p->des1 = cpu_to_le32(upper_32_bits(addr));
6844171d JA	436	}
6844171d JA	437
44c67f85 JA	438	static void dwmac4_clear(struct dma_desc *p)
	439	{
	440	p->des0 = 0;
	441	p->des1 = 0;
	442	p->des2 = 0;
	443	p->des3 = 0;
	444	}
	445
1d982e93 JA	446	static void dwmac4_set_sarc(struct dma_desc *p, u32 sarc_type)
	447	{
	448	sarc_type <<= TDES3_SA_INSERT_CTRL_SHIFT;
	449
	450	p->des3 \|= cpu_to_le32(sarc_type & TDES3_SA_INSERT_CTRL_MASK);
	451	}
	452
c3efed5a WV	453	static int set_16kib_bfsize(int mtu)
	454	{
	455	int ret = 0;
	456
	457	if (unlikely(mtu >= BUF_SIZE_8KiB))
	458	ret = BUF_SIZE_16KiB;
	459	return ret;
	460	}
	461
e94e3f3b JA	462	static void dwmac4_set_vlan_tag(struct dma_desc *p, u16 tag, u16 inner_tag,
	463	u32 inner_type)
	464	{
	465	p->des0 = 0;
	466	p->des1 = 0;
	467	p->des2 = 0;
	468	p->des3 = 0;
	469
	470	/* Inner VLAN */
	471	if (inner_type) {
	472	u32 des = inner_tag << TDES2_IVT_SHIFT;
	473
	474	des &= TDES2_IVT_MASK;
	475	p->des2 = cpu_to_le32(des);
	476
	477	des = inner_type << TDES3_IVTIR_SHIFT;
	478	des &= TDES3_IVTIR_MASK;
	479	p->des3 = cpu_to_le32(des \| TDES3_IVLTV);
	480	}
	481
	482	/* Outer VLAN */
	483	p->des3 \|= cpu_to_le32(tag & TDES3_VLAN_TAG);
	484	p->des3 \|= cpu_to_le32(TDES3_VLTV);
	485
	486	p->des3 \|= cpu_to_le32(TDES3_CONTEXT_TYPE);
	487	}
	488
	489	static void dwmac4_set_vlan(struct dma_desc *p, u32 type)
	490	{
	491	type <<= TDES2_VLAN_TAG_SHIFT;
	492	p->des2 \|= cpu_to_le32(type & TDES2_VLAN_TAG_MASK);
	493	}
	494
753a7109 AT	495	const struct stmmac_desc_ops dwmac4_desc_ops = {
	496	.tx_status = dwmac4_wrback_get_tx_status,
	497	.rx_status = dwmac4_wrback_get_rx_status,
	498	.get_tx_len = dwmac4_rd_get_tx_len,
	499	.get_tx_owner = dwmac4_get_tx_owner,
	500	.set_tx_owner = dwmac4_set_tx_owner,
	501	.set_rx_owner = dwmac4_set_rx_owner,
	502	.get_tx_ls = dwmac4_get_tx_ls,
	503	.get_rx_frame_len = dwmac4_wrback_get_rx_frame_len,
	504	.enable_tx_timestamp = dwmac4_rd_enable_tx_timestamp,
	505	.get_tx_timestamp_status = dwmac4_wrback_get_tx_timestamp_status,
ba1ffd74 GC	506	.get_rx_timestamp_status = dwmac4_wrback_get_rx_timestamp_status,
ba1ffd74 GC	507	.get_timestamp = dwmac4_get_timestamp,
753a7109 AT	508	.set_tx_ic = dwmac4_rd_set_tx_ic,
	509	.prepare_tx_desc = dwmac4_rd_prepare_tx_desc,
	510	.prepare_tso_tx_desc = dwmac4_rd_prepare_tso_tx_desc,
	511	.release_tx_desc = dwmac4_release_tx_desc,
	512	.init_rx_desc = dwmac4_rd_init_rx_desc,
	513	.init_tx_desc = dwmac4_rd_init_tx_desc,
	514	.display_ring = dwmac4_display_ring,
	515	.set_mss = dwmac4_set_mss_ctxt,
d2df9ea0	516	.get_addr = dwmac4_get_addr,
6844171d	517	.set_addr = dwmac4_set_addr,
44c67f85	518	.clear = dwmac4_clear,
1d982e93	519	.set_sarc = dwmac4_set_sarc,
e94e3f3b JA	520	.set_vlan_tag = dwmac4_set_vlan_tag,
e94e3f3b JA	521	.set_vlan = dwmac4_set_vlan,
753a7109 AT	522	};
753a7109 AT	523
c3efed5a WV	524	const struct stmmac_mode_ops dwmac4_ring_mode_ops = {
	525	.set_16kib_bfsize = set_16kib_bfsize,
	526	};