1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (C) 2021 Gerhard Engleder <gerhard@engleder-embedded.com> */
4 /* TSN endpoint Ethernet MAC driver
6 * The TSN endpoint Ethernet MAC is a FPGA based network device for real-time
7 * communication. It is designed for endpoints within TSN (Time Sensitive
8 * Networking) networks; e.g., for PLCs in the industrial automation case.
10 * It supports multiple TX/RX queue pairs. The first TX/RX queue pair is used
13 * More information can be found here:
14 * - www.embedded-experts.at/tsn
15 * - www.engleder-embedded.com
21 #include <linux/module.h>
23 #include <linux/of_net.h>
24 #include <linux/of_mdio.h>
25 #include <linux/interrupt.h>
26 #include <linux/etherdevice.h>
27 #include <linux/phy.h>
28 #include <linux/iopoll.h>
29 #include <linux/bpf.h>
30 #include <linux/bpf_trace.h>
31 #include <net/page_pool/helpers.h>
32 #include <net/xdp_sock_drv.h>
34 #define TSNEP_RX_OFFSET (max(NET_SKB_PAD, XDP_PACKET_HEADROOM) + NET_IP_ALIGN)
35 #define TSNEP_HEADROOM ALIGN(TSNEP_RX_OFFSET, 4)
36 #define TSNEP_MAX_RX_BUF_SIZE (PAGE_SIZE - TSNEP_HEADROOM - \
37 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
38 /* XSK buffer shall store at least Q-in-Q frame */
39 #define TSNEP_XSK_RX_BUF_SIZE (ALIGN(TSNEP_RX_INLINE_METADATA_SIZE + \
40 ETH_FRAME_LEN + ETH_FCS_LEN + \
43 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
44 #define DMA_ADDR_HIGH(dma_addr) ((u32)(((dma_addr) >> 32) & 0xFFFFFFFF))
46 #define DMA_ADDR_HIGH(dma_addr) ((u32)(0))
48 #define DMA_ADDR_LOW(dma_addr) ((u32)((dma_addr) & 0xFFFFFFFF))
50 #define TSNEP_COALESCE_USECS_DEFAULT 64
51 #define TSNEP_COALESCE_USECS_MAX ((ECM_INT_DELAY_MASK >> ECM_INT_DELAY_SHIFT) * \
52 ECM_INT_DELAY_BASE_US + ECM_INT_DELAY_BASE_US - 1)
55 #define TSNEP_TX_TYPE_MAP BIT(0)
56 #define TSNEP_TX_TYPE_MAP_PAGE BIT(1)
57 #define TSNEP_TX_TYPE_INLINE BIT(2)
59 #define TSNEP_TX_TYPE_SKB BIT(8)
60 #define TSNEP_TX_TYPE_SKB_MAP (TSNEP_TX_TYPE_SKB | TSNEP_TX_TYPE_MAP)
61 #define TSNEP_TX_TYPE_SKB_INLINE (TSNEP_TX_TYPE_SKB | TSNEP_TX_TYPE_INLINE)
62 #define TSNEP_TX_TYPE_SKB_FRAG BIT(9)
63 #define TSNEP_TX_TYPE_SKB_FRAG_MAP_PAGE (TSNEP_TX_TYPE_SKB_FRAG | TSNEP_TX_TYPE_MAP_PAGE)
64 #define TSNEP_TX_TYPE_SKB_FRAG_INLINE (TSNEP_TX_TYPE_SKB_FRAG | TSNEP_TX_TYPE_INLINE)
65 #define TSNEP_TX_TYPE_XDP_TX BIT(10)
66 #define TSNEP_TX_TYPE_XDP_NDO BIT(11)
67 #define TSNEP_TX_TYPE_XDP_NDO_MAP_PAGE (TSNEP_TX_TYPE_XDP_NDO | TSNEP_TX_TYPE_MAP_PAGE)
68 #define TSNEP_TX_TYPE_XDP (TSNEP_TX_TYPE_XDP_TX | TSNEP_TX_TYPE_XDP_NDO)
69 #define TSNEP_TX_TYPE_XSK BIT(12)
71 #define TSNEP_XDP_TX BIT(0)
72 #define TSNEP_XDP_REDIRECT BIT(1)
74 static void tsnep_enable_irq(struct tsnep_adapter *adapter, u32 mask)
76 iowrite32(mask, adapter->addr + ECM_INT_ENABLE);
79 static void tsnep_disable_irq(struct tsnep_adapter *adapter, u32 mask)
81 mask |= ECM_INT_DISABLE;
82 iowrite32(mask, adapter->addr + ECM_INT_ENABLE);
85 static irqreturn_t tsnep_irq(int irq, void *arg)
87 struct tsnep_adapter *adapter = arg;
88 u32 active = ioread32(adapter->addr + ECM_INT_ACTIVE);
90 /* acknowledge interrupt */
92 iowrite32(active, adapter->addr + ECM_INT_ACKNOWLEDGE);
94 /* handle link interrupt */
95 if ((active & ECM_INT_LINK) != 0)
96 phy_mac_interrupt(adapter->netdev->phydev);
98 /* handle TX/RX queue 0 interrupt */
99 if ((active & adapter->queue[0].irq_mask) != 0) {
100 if (napi_schedule_prep(&adapter->queue[0].napi)) {
101 tsnep_disable_irq(adapter, adapter->queue[0].irq_mask);
102 /* schedule after masking to avoid races */
103 __napi_schedule(&adapter->queue[0].napi);
110 static irqreturn_t tsnep_irq_txrx(int irq, void *arg)
112 struct tsnep_queue *queue = arg;
114 /* handle TX/RX queue interrupt */
115 if (napi_schedule_prep(&queue->napi)) {
116 tsnep_disable_irq(queue->adapter, queue->irq_mask);
117 /* schedule after masking to avoid races */
118 __napi_schedule(&queue->napi);
124 int tsnep_set_irq_coalesce(struct tsnep_queue *queue, u32 usecs)
126 if (usecs > TSNEP_COALESCE_USECS_MAX)
129 usecs /= ECM_INT_DELAY_BASE_US;
130 usecs <<= ECM_INT_DELAY_SHIFT;
131 usecs &= ECM_INT_DELAY_MASK;
133 queue->irq_delay &= ~ECM_INT_DELAY_MASK;
134 queue->irq_delay |= usecs;
135 iowrite8(queue->irq_delay, queue->irq_delay_addr);
140 u32 tsnep_get_irq_coalesce(struct tsnep_queue *queue)
144 usecs = (queue->irq_delay & ECM_INT_DELAY_MASK);
145 usecs >>= ECM_INT_DELAY_SHIFT;
146 usecs *= ECM_INT_DELAY_BASE_US;
151 static int tsnep_mdiobus_read(struct mii_bus *bus, int addr, int regnum)
153 struct tsnep_adapter *adapter = bus->priv;
158 if (!adapter->suppress_preamble)
159 md |= ECM_MD_PREAMBLE;
160 md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK;
161 md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK;
162 iowrite32(md, adapter->addr + ECM_MD_CONTROL);
163 retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md,
164 !(md & ECM_MD_BUSY), 16, 1000);
168 return (md & ECM_MD_DATA_MASK) >> ECM_MD_DATA_SHIFT;
171 static int tsnep_mdiobus_write(struct mii_bus *bus, int addr, int regnum,
174 struct tsnep_adapter *adapter = bus->priv;
179 if (!adapter->suppress_preamble)
180 md |= ECM_MD_PREAMBLE;
181 md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK;
182 md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK;
183 md |= ((u32)val << ECM_MD_DATA_SHIFT) & ECM_MD_DATA_MASK;
184 iowrite32(md, adapter->addr + ECM_MD_CONTROL);
185 retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md,
186 !(md & ECM_MD_BUSY), 16, 1000);
193 static void tsnep_set_link_mode(struct tsnep_adapter *adapter)
197 switch (adapter->phydev->speed) {
199 mode = ECM_LINK_MODE_100;
202 mode = ECM_LINK_MODE_1000;
205 mode = ECM_LINK_MODE_OFF;
208 iowrite32(mode, adapter->addr + ECM_STATUS);
211 static void tsnep_phy_link_status_change(struct net_device *netdev)
213 struct tsnep_adapter *adapter = netdev_priv(netdev);
214 struct phy_device *phydev = netdev->phydev;
217 tsnep_set_link_mode(adapter);
219 phy_print_status(netdev->phydev);
222 static int tsnep_phy_loopback(struct tsnep_adapter *adapter, bool enable)
226 retval = phy_loopback(adapter->phydev, enable);
228 /* PHY link state change is not signaled if loopback is enabled, it
229 * would delay a working loopback anyway, let's ensure that loopback
230 * is working immediately by setting link mode directly
232 if (!retval && enable)
233 tsnep_set_link_mode(adapter);
238 static int tsnep_phy_open(struct tsnep_adapter *adapter)
240 struct phy_device *phydev;
241 struct ethtool_eee ethtool_eee;
244 retval = phy_connect_direct(adapter->netdev, adapter->phydev,
245 tsnep_phy_link_status_change,
249 phydev = adapter->netdev->phydev;
251 /* MAC supports only 100Mbps|1000Mbps full duplex
252 * SPE (Single Pair Ethernet) is also an option but not implemented yet
254 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Half_BIT);
255 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Full_BIT);
256 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_100baseT_Half_BIT);
257 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_1000baseT_Half_BIT);
259 /* disable EEE autoneg, EEE not supported by TSNEP */
260 memset(ðtool_eee, 0, sizeof(ethtool_eee));
261 phy_ethtool_set_eee(adapter->phydev, ðtool_eee);
263 adapter->phydev->irq = PHY_MAC_INTERRUPT;
264 phy_start(adapter->phydev);
269 static void tsnep_phy_close(struct tsnep_adapter *adapter)
271 phy_stop(adapter->netdev->phydev);
272 phy_disconnect(adapter->netdev->phydev);
275 static void tsnep_tx_ring_cleanup(struct tsnep_tx *tx)
277 struct device *dmadev = tx->adapter->dmadev;
280 memset(tx->entry, 0, sizeof(tx->entry));
282 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {
284 dma_free_coherent(dmadev, PAGE_SIZE, tx->page[i],
292 static int tsnep_tx_ring_create(struct tsnep_tx *tx)
294 struct device *dmadev = tx->adapter->dmadev;
295 struct tsnep_tx_entry *entry;
296 struct tsnep_tx_entry *next_entry;
300 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {
302 dma_alloc_coherent(dmadev, PAGE_SIZE, &tx->page_dma[i],
308 for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) {
309 entry = &tx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j];
310 entry->desc_wb = (struct tsnep_tx_desc_wb *)
311 (((u8 *)tx->page[i]) + TSNEP_DESC_SIZE * j);
312 entry->desc = (struct tsnep_tx_desc *)
313 (((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET);
314 entry->desc_dma = tx->page_dma[i] + TSNEP_DESC_SIZE * j;
315 entry->owner_user_flag = false;
318 for (i = 0; i < TSNEP_RING_SIZE; i++) {
319 entry = &tx->entry[i];
320 next_entry = &tx->entry[(i + 1) & TSNEP_RING_MASK];
321 entry->desc->next = __cpu_to_le64(next_entry->desc_dma);
327 tsnep_tx_ring_cleanup(tx);
331 static void tsnep_tx_init(struct tsnep_tx *tx)
335 dma = tx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER;
336 iowrite32(DMA_ADDR_LOW(dma), tx->addr + TSNEP_TX_DESC_ADDR_LOW);
337 iowrite32(DMA_ADDR_HIGH(dma), tx->addr + TSNEP_TX_DESC_ADDR_HIGH);
340 tx->owner_counter = 1;
341 tx->increment_owner_counter = TSNEP_RING_SIZE - 1;
344 static void tsnep_tx_enable(struct tsnep_tx *tx)
346 struct netdev_queue *nq;
348 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);
350 __netif_tx_lock_bh(nq);
351 netif_tx_wake_queue(nq);
352 __netif_tx_unlock_bh(nq);
355 static void tsnep_tx_disable(struct tsnep_tx *tx, struct napi_struct *napi)
357 struct netdev_queue *nq;
360 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);
362 __netif_tx_lock_bh(nq);
363 netif_tx_stop_queue(nq);
364 __netif_tx_unlock_bh(nq);
366 /* wait until TX is done in hardware */
367 readx_poll_timeout(ioread32, tx->addr + TSNEP_CONTROL, val,
368 ((val & TSNEP_CONTROL_TX_ENABLE) == 0), 10000,
371 /* wait until TX is also done in software */
372 while (READ_ONCE(tx->read) != tx->write) {
374 napi_synchronize(napi);
378 static void tsnep_tx_activate(struct tsnep_tx *tx, int index, int length,
381 struct tsnep_tx_entry *entry = &tx->entry[index];
383 entry->properties = 0;
384 /* xdpf and zc are union with skb */
386 entry->properties = length & TSNEP_DESC_LENGTH_MASK;
387 entry->properties |= TSNEP_DESC_INTERRUPT_FLAG;
388 if ((entry->type & TSNEP_TX_TYPE_SKB) &&
389 (skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS))
390 entry->properties |= TSNEP_DESC_EXTENDED_WRITEBACK_FLAG;
392 /* toggle user flag to prevent false acknowledge
394 * Only the first fragment is acknowledged. For all other
395 * fragments no acknowledge is done and the last written owner
396 * counter stays in the writeback descriptor. Therefore, it is
397 * possible that the last written owner counter is identical to
398 * the new incremented owner counter and a false acknowledge is
399 * detected before the real acknowledge has been done by
402 * The user flag is used to prevent this situation. The user
403 * flag is copied to the writeback descriptor by the hardware
404 * and is used as additional acknowledge data. By toggeling the
405 * user flag only for the first fragment (which is
406 * acknowledged), it is guaranteed that the last acknowledge
407 * done for this descriptor has used a different user flag and
408 * cannot be detected as false acknowledge.
410 entry->owner_user_flag = !entry->owner_user_flag;
413 entry->properties |= TSNEP_TX_DESC_LAST_FRAGMENT_FLAG;
414 if (index == tx->increment_owner_counter) {
416 if (tx->owner_counter == 4)
417 tx->owner_counter = 1;
418 tx->increment_owner_counter--;
419 if (tx->increment_owner_counter < 0)
420 tx->increment_owner_counter = TSNEP_RING_SIZE - 1;
423 (tx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) &
424 TSNEP_DESC_OWNER_COUNTER_MASK;
425 if (entry->owner_user_flag)
426 entry->properties |= TSNEP_TX_DESC_OWNER_USER_FLAG;
427 entry->desc->more_properties =
428 __cpu_to_le32(entry->len & TSNEP_DESC_LENGTH_MASK);
429 if (entry->type & TSNEP_TX_TYPE_INLINE)
430 entry->properties |= TSNEP_TX_DESC_DATA_AFTER_DESC_FLAG;
432 /* descriptor properties shall be written last, because valid data is
437 entry->desc->properties = __cpu_to_le32(entry->properties);
440 static int tsnep_tx_desc_available(struct tsnep_tx *tx)
442 if (tx->read <= tx->write)
443 return TSNEP_RING_SIZE - tx->write + tx->read - 1;
445 return tx->read - tx->write - 1;
448 static int tsnep_tx_map_frag(skb_frag_t *frag, struct tsnep_tx_entry *entry,
449 struct device *dmadev, dma_addr_t *dma)
454 len = skb_frag_size(frag);
455 if (likely(len > TSNEP_DESC_SIZE_DATA_AFTER_INLINE)) {
456 *dma = skb_frag_dma_map(dmadev, frag, 0, len, DMA_TO_DEVICE);
457 if (dma_mapping_error(dmadev, *dma))
459 entry->type = TSNEP_TX_TYPE_SKB_FRAG_MAP_PAGE;
462 void *fragdata = skb_frag_address_safe(frag);
464 if (likely(fragdata)) {
465 memcpy(&entry->desc->tx, fragdata, len);
467 struct page *page = skb_frag_page(frag);
469 fragdata = kmap_local_page(page);
470 memcpy(&entry->desc->tx, fragdata + skb_frag_off(frag),
472 kunmap_local(fragdata);
474 entry->type = TSNEP_TX_TYPE_SKB_FRAG_INLINE;
481 static int tsnep_tx_map(struct sk_buff *skb, struct tsnep_tx *tx, int count)
483 struct device *dmadev = tx->adapter->dmadev;
484 struct tsnep_tx_entry *entry;
490 for (i = 0; i < count; i++) {
491 entry = &tx->entry[(tx->write + i) & TSNEP_RING_MASK];
494 len = skb_headlen(skb);
495 if (likely(len > TSNEP_DESC_SIZE_DATA_AFTER_INLINE)) {
496 dma = dma_map_single(dmadev, skb->data, len,
498 if (dma_mapping_error(dmadev, dma))
500 entry->type = TSNEP_TX_TYPE_SKB_MAP;
503 memcpy(&entry->desc->tx, skb->data, len);
504 entry->type = TSNEP_TX_TYPE_SKB_INLINE;
508 skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
510 len = skb_frag_size(frag);
511 mapped = tsnep_tx_map_frag(frag, entry, dmadev, &dma);
517 if (likely(mapped)) {
518 dma_unmap_addr_set(entry, dma, dma);
519 entry->desc->tx = __cpu_to_le64(dma);
528 static int tsnep_tx_unmap(struct tsnep_tx *tx, int index, int count)
530 struct device *dmadev = tx->adapter->dmadev;
531 struct tsnep_tx_entry *entry;
535 for (i = 0; i < count; i++) {
536 entry = &tx->entry[(index + i) & TSNEP_RING_MASK];
539 if (entry->type & TSNEP_TX_TYPE_MAP)
540 dma_unmap_single(dmadev,
541 dma_unmap_addr(entry, dma),
542 dma_unmap_len(entry, len),
544 else if (entry->type & TSNEP_TX_TYPE_MAP_PAGE)
545 dma_unmap_page(dmadev,
546 dma_unmap_addr(entry, dma),
547 dma_unmap_len(entry, len),
549 map_len += entry->len;
557 static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb,
561 struct tsnep_tx_entry *entry;
566 if (skb_shinfo(skb)->nr_frags > 0)
567 count += skb_shinfo(skb)->nr_frags;
569 if (tsnep_tx_desc_available(tx) < count) {
570 /* ring full, shall not happen because queue is stopped if full
573 netif_stop_subqueue(tx->adapter->netdev, tx->queue_index);
575 return NETDEV_TX_BUSY;
578 entry = &tx->entry[tx->write];
581 retval = tsnep_tx_map(skb, tx, count);
583 tsnep_tx_unmap(tx, tx->write, count);
584 dev_kfree_skb_any(entry->skb);
593 if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)
594 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
596 for (i = 0; i < count; i++)
597 tsnep_tx_activate(tx, (tx->write + i) & TSNEP_RING_MASK, length,
599 tx->write = (tx->write + count) & TSNEP_RING_MASK;
601 skb_tx_timestamp(skb);
603 /* descriptor properties shall be valid before hardware is notified */
606 iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL);
608 if (tsnep_tx_desc_available(tx) < (MAX_SKB_FRAGS + 1)) {
609 /* ring can get full with next frame */
610 netif_stop_subqueue(tx->adapter->netdev, tx->queue_index);
616 static int tsnep_xdp_tx_map(struct xdp_frame *xdpf, struct tsnep_tx *tx,
617 struct skb_shared_info *shinfo, int count, u32 type)
619 struct device *dmadev = tx->adapter->dmadev;
620 struct tsnep_tx_entry *entry;
631 for (i = 0; i < count; i++) {
632 entry = &tx->entry[(tx->write + i) & TSNEP_RING_MASK];
633 if (type & TSNEP_TX_TYPE_XDP_NDO) {
634 data = unlikely(frag) ? skb_frag_address(frag) :
636 dma = dma_map_single(dmadev, data, len, DMA_TO_DEVICE);
637 if (dma_mapping_error(dmadev, dma))
640 entry->type = TSNEP_TX_TYPE_XDP_NDO_MAP_PAGE;
642 page = unlikely(frag) ? skb_frag_page(frag) :
643 virt_to_page(xdpf->data);
644 dma = page_pool_get_dma_addr(page);
646 dma += skb_frag_off(frag);
648 dma += sizeof(*xdpf) + xdpf->headroom;
649 dma_sync_single_for_device(dmadev, dma, len,
652 entry->type = TSNEP_TX_TYPE_XDP_TX;
656 dma_unmap_addr_set(entry, dma, dma);
658 entry->desc->tx = __cpu_to_le64(dma);
663 frag = &shinfo->frags[i];
664 len = skb_frag_size(frag);
671 /* This function requires __netif_tx_lock is held by the caller. */
672 static bool tsnep_xdp_xmit_frame_ring(struct xdp_frame *xdpf,
673 struct tsnep_tx *tx, u32 type)
675 struct skb_shared_info *shinfo = xdp_get_shared_info_from_frame(xdpf);
676 struct tsnep_tx_entry *entry;
677 int count, length, retval, i;
680 if (unlikely(xdp_frame_has_frags(xdpf)))
681 count += shinfo->nr_frags;
683 /* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS
684 * will be available for normal TX path and queue is stopped there if
687 if (tsnep_tx_desc_available(tx) < (MAX_SKB_FRAGS + 1 + count))
690 entry = &tx->entry[tx->write];
693 retval = tsnep_xdp_tx_map(xdpf, tx, shinfo, count, type);
695 tsnep_tx_unmap(tx, tx->write, count);
704 for (i = 0; i < count; i++)
705 tsnep_tx_activate(tx, (tx->write + i) & TSNEP_RING_MASK, length,
707 tx->write = (tx->write + count) & TSNEP_RING_MASK;
709 /* descriptor properties shall be valid before hardware is notified */
715 static void tsnep_xdp_xmit_flush(struct tsnep_tx *tx)
717 iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL);
720 static bool tsnep_xdp_xmit_back(struct tsnep_adapter *adapter,
721 struct xdp_buff *xdp,
722 struct netdev_queue *tx_nq, struct tsnep_tx *tx)
724 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);
730 __netif_tx_lock(tx_nq, smp_processor_id());
732 xmit = tsnep_xdp_xmit_frame_ring(xdpf, tx, TSNEP_TX_TYPE_XDP_TX);
734 /* Avoid transmit queue timeout since we share it with the slow path */
736 txq_trans_cond_update(tx_nq);
738 __netif_tx_unlock(tx_nq);
743 static int tsnep_xdp_tx_map_zc(struct xdp_desc *xdpd, struct tsnep_tx *tx)
745 struct tsnep_tx_entry *entry;
748 entry = &tx->entry[tx->write];
751 dma = xsk_buff_raw_get_dma(tx->xsk_pool, xdpd->addr);
752 xsk_buff_raw_dma_sync_for_device(tx->xsk_pool, dma, xdpd->len);
754 entry->type = TSNEP_TX_TYPE_XSK;
755 entry->len = xdpd->len;
757 entry->desc->tx = __cpu_to_le64(dma);
762 static void tsnep_xdp_xmit_frame_ring_zc(struct xdp_desc *xdpd,
767 length = tsnep_xdp_tx_map_zc(xdpd, tx);
769 tsnep_tx_activate(tx, tx->write, length, true);
770 tx->write = (tx->write + 1) & TSNEP_RING_MASK;
773 static void tsnep_xdp_xmit_zc(struct tsnep_tx *tx)
775 int desc_available = tsnep_tx_desc_available(tx);
776 struct xdp_desc *descs = tx->xsk_pool->tx_descs;
779 /* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS
780 * will be available for normal TX path and queue is stopped there if
783 if (desc_available <= (MAX_SKB_FRAGS + 1))
785 desc_available -= MAX_SKB_FRAGS + 1;
787 batch = xsk_tx_peek_release_desc_batch(tx->xsk_pool, desc_available);
788 for (i = 0; i < batch; i++)
789 tsnep_xdp_xmit_frame_ring_zc(&descs[i], tx);
792 /* descriptor properties shall be valid before hardware is
797 tsnep_xdp_xmit_flush(tx);
801 static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget)
803 struct tsnep_tx_entry *entry;
804 struct netdev_queue *nq;
810 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);
811 __netif_tx_lock(nq, smp_processor_id());
814 if (tx->read == tx->write)
817 entry = &tx->entry[tx->read];
818 if ((__le32_to_cpu(entry->desc_wb->properties) &
819 TSNEP_TX_DESC_OWNER_MASK) !=
820 (entry->properties & TSNEP_TX_DESC_OWNER_MASK))
823 /* descriptor properties shall be read first, because valid data
829 if ((entry->type & TSNEP_TX_TYPE_SKB) &&
830 skb_shinfo(entry->skb)->nr_frags > 0)
831 count += skb_shinfo(entry->skb)->nr_frags;
832 else if ((entry->type & TSNEP_TX_TYPE_XDP) &&
833 xdp_frame_has_frags(entry->xdpf))
834 count += xdp_get_shared_info_from_frame(entry->xdpf)->nr_frags;
836 length = tsnep_tx_unmap(tx, tx->read, count);
838 if ((entry->type & TSNEP_TX_TYPE_SKB) &&
839 (skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS) &&
840 (__le32_to_cpu(entry->desc_wb->properties) &
841 TSNEP_DESC_EXTENDED_WRITEBACK_FLAG)) {
842 struct skb_shared_hwtstamps hwtstamps;
845 if (skb_shinfo(entry->skb)->tx_flags &
846 SKBTX_HW_TSTAMP_USE_CYCLES)
848 __le64_to_cpu(entry->desc_wb->counter);
851 __le64_to_cpu(entry->desc_wb->timestamp);
853 memset(&hwtstamps, 0, sizeof(hwtstamps));
854 hwtstamps.hwtstamp = ns_to_ktime(timestamp);
856 skb_tstamp_tx(entry->skb, &hwtstamps);
859 if (entry->type & TSNEP_TX_TYPE_SKB)
860 napi_consume_skb(entry->skb, napi_budget);
861 else if (entry->type & TSNEP_TX_TYPE_XDP)
862 xdp_return_frame_rx_napi(entry->xdpf);
865 /* xdpf and zc are union with skb */
868 tx->read = (tx->read + count) & TSNEP_RING_MASK;
871 tx->bytes += length + ETH_FCS_LEN;
874 } while (likely(budget));
878 xsk_tx_completed(tx->xsk_pool, xsk_frames);
879 if (xsk_uses_need_wakeup(tx->xsk_pool))
880 xsk_set_tx_need_wakeup(tx->xsk_pool);
881 tsnep_xdp_xmit_zc(tx);
884 if ((tsnep_tx_desc_available(tx) >= ((MAX_SKB_FRAGS + 1) * 2)) &&
885 netif_tx_queue_stopped(nq)) {
886 netif_tx_wake_queue(nq);
889 __netif_tx_unlock(nq);
894 static bool tsnep_tx_pending(struct tsnep_tx *tx)
896 struct tsnep_tx_entry *entry;
897 struct netdev_queue *nq;
898 bool pending = false;
900 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);
901 __netif_tx_lock(nq, smp_processor_id());
903 if (tx->read != tx->write) {
904 entry = &tx->entry[tx->read];
905 if ((__le32_to_cpu(entry->desc_wb->properties) &
906 TSNEP_TX_DESC_OWNER_MASK) ==
907 (entry->properties & TSNEP_TX_DESC_OWNER_MASK))
911 __netif_tx_unlock(nq);
916 static int tsnep_tx_open(struct tsnep_tx *tx)
920 retval = tsnep_tx_ring_create(tx);
929 static void tsnep_tx_close(struct tsnep_tx *tx)
931 tsnep_tx_ring_cleanup(tx);
934 static void tsnep_rx_ring_cleanup(struct tsnep_rx *rx)
936 struct device *dmadev = rx->adapter->dmadev;
937 struct tsnep_rx_entry *entry;
940 for (i = 0; i < TSNEP_RING_SIZE; i++) {
941 entry = &rx->entry[i];
942 if (!rx->xsk_pool && entry->page)
943 page_pool_put_full_page(rx->page_pool, entry->page,
945 if (rx->xsk_pool && entry->xdp)
946 xsk_buff_free(entry->xdp);
947 /* xdp is union with page */
952 page_pool_destroy(rx->page_pool);
954 memset(rx->entry, 0, sizeof(rx->entry));
956 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {
958 dma_free_coherent(dmadev, PAGE_SIZE, rx->page[i],
966 static int tsnep_rx_ring_create(struct tsnep_rx *rx)
968 struct device *dmadev = rx->adapter->dmadev;
969 struct tsnep_rx_entry *entry;
970 struct page_pool_params pp_params = { 0 };
971 struct tsnep_rx_entry *next_entry;
975 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {
977 dma_alloc_coherent(dmadev, PAGE_SIZE, &rx->page_dma[i],
983 for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) {
984 entry = &rx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j];
985 entry->desc_wb = (struct tsnep_rx_desc_wb *)
986 (((u8 *)rx->page[i]) + TSNEP_DESC_SIZE * j);
987 entry->desc = (struct tsnep_rx_desc *)
988 (((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET);
989 entry->desc_dma = rx->page_dma[i] + TSNEP_DESC_SIZE * j;
993 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
995 pp_params.pool_size = TSNEP_RING_SIZE;
996 pp_params.nid = dev_to_node(dmadev);
997 pp_params.dev = dmadev;
998 pp_params.dma_dir = DMA_BIDIRECTIONAL;
999 pp_params.max_len = TSNEP_MAX_RX_BUF_SIZE;
1000 pp_params.offset = TSNEP_RX_OFFSET;
1001 rx->page_pool = page_pool_create(&pp_params);
1002 if (IS_ERR(rx->page_pool)) {
1003 retval = PTR_ERR(rx->page_pool);
1004 rx->page_pool = NULL;
1008 for (i = 0; i < TSNEP_RING_SIZE; i++) {
1009 entry = &rx->entry[i];
1010 next_entry = &rx->entry[(i + 1) & TSNEP_RING_MASK];
1011 entry->desc->next = __cpu_to_le64(next_entry->desc_dma);
1017 tsnep_rx_ring_cleanup(rx);
1021 static void tsnep_rx_init(struct tsnep_rx *rx)
1025 dma = rx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER;
1026 iowrite32(DMA_ADDR_LOW(dma), rx->addr + TSNEP_RX_DESC_ADDR_LOW);
1027 iowrite32(DMA_ADDR_HIGH(dma), rx->addr + TSNEP_RX_DESC_ADDR_HIGH);
1030 rx->owner_counter = 1;
1031 rx->increment_owner_counter = TSNEP_RING_SIZE - 1;
1034 static void tsnep_rx_enable(struct tsnep_rx *rx)
1036 /* descriptor properties shall be valid before hardware is notified */
1039 iowrite32(TSNEP_CONTROL_RX_ENABLE, rx->addr + TSNEP_CONTROL);
1042 static void tsnep_rx_disable(struct tsnep_rx *rx)
1046 iowrite32(TSNEP_CONTROL_RX_DISABLE, rx->addr + TSNEP_CONTROL);
1047 readx_poll_timeout(ioread32, rx->addr + TSNEP_CONTROL, val,
1048 ((val & TSNEP_CONTROL_RX_ENABLE) == 0), 10000,
1052 static int tsnep_rx_desc_available(struct tsnep_rx *rx)
1054 if (rx->read <= rx->write)
1055 return TSNEP_RING_SIZE - rx->write + rx->read - 1;
1057 return rx->read - rx->write - 1;
1060 static void tsnep_rx_free_page_buffer(struct tsnep_rx *rx)
1064 /* last entry of page_buffer is always zero, because ring cannot be
1067 page = rx->page_buffer;
1069 page_pool_put_full_page(rx->page_pool, *page, false);
1075 static int tsnep_rx_alloc_page_buffer(struct tsnep_rx *rx)
1079 /* alloc for all ring entries except the last one, because ring cannot
1080 * be filled completely
1082 for (i = 0; i < TSNEP_RING_SIZE - 1; i++) {
1083 rx->page_buffer[i] = page_pool_dev_alloc_pages(rx->page_pool);
1084 if (!rx->page_buffer[i]) {
1085 tsnep_rx_free_page_buffer(rx);
1094 static void tsnep_rx_set_page(struct tsnep_rx *rx, struct tsnep_rx_entry *entry,
1098 entry->len = TSNEP_MAX_RX_BUF_SIZE;
1099 entry->dma = page_pool_get_dma_addr(entry->page);
1100 entry->desc->rx = __cpu_to_le64(entry->dma + TSNEP_RX_OFFSET);
1103 static int tsnep_rx_alloc_buffer(struct tsnep_rx *rx, int index)
1105 struct tsnep_rx_entry *entry = &rx->entry[index];
1108 page = page_pool_dev_alloc_pages(rx->page_pool);
1109 if (unlikely(!page))
1111 tsnep_rx_set_page(rx, entry, page);
1116 static void tsnep_rx_reuse_buffer(struct tsnep_rx *rx, int index)
1118 struct tsnep_rx_entry *entry = &rx->entry[index];
1119 struct tsnep_rx_entry *read = &rx->entry[rx->read];
1121 tsnep_rx_set_page(rx, entry, read->page);
1125 static void tsnep_rx_activate(struct tsnep_rx *rx, int index)
1127 struct tsnep_rx_entry *entry = &rx->entry[index];
1129 /* TSNEP_MAX_RX_BUF_SIZE and TSNEP_XSK_RX_BUF_SIZE are multiple of 4 */
1130 entry->properties = entry->len & TSNEP_DESC_LENGTH_MASK;
1131 entry->properties |= TSNEP_DESC_INTERRUPT_FLAG;
1132 if (index == rx->increment_owner_counter) {
1133 rx->owner_counter++;
1134 if (rx->owner_counter == 4)
1135 rx->owner_counter = 1;
1136 rx->increment_owner_counter--;
1137 if (rx->increment_owner_counter < 0)
1138 rx->increment_owner_counter = TSNEP_RING_SIZE - 1;
1140 entry->properties |=
1141 (rx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) &
1142 TSNEP_DESC_OWNER_COUNTER_MASK;
1144 /* descriptor properties shall be written last, because valid data is
1149 entry->desc->properties = __cpu_to_le32(entry->properties);
1152 static int tsnep_rx_alloc(struct tsnep_rx *rx, int count, bool reuse)
1154 bool alloc_failed = false;
1157 for (i = 0; i < count && !alloc_failed; i++) {
1158 index = (rx->write + i) & TSNEP_RING_MASK;
1160 if (unlikely(tsnep_rx_alloc_buffer(rx, index))) {
1162 alloc_failed = true;
1164 /* reuse only if no other allocation was successful */
1165 if (i == 0 && reuse)
1166 tsnep_rx_reuse_buffer(rx, index);
1171 tsnep_rx_activate(rx, index);
1175 rx->write = (rx->write + i) & TSNEP_RING_MASK;
1180 static int tsnep_rx_refill(struct tsnep_rx *rx, int count, bool reuse)
1184 desc_refilled = tsnep_rx_alloc(rx, count, reuse);
1186 tsnep_rx_enable(rx);
1188 return desc_refilled;
1191 static void tsnep_rx_set_xdp(struct tsnep_rx *rx, struct tsnep_rx_entry *entry,
1192 struct xdp_buff *xdp)
1195 entry->len = TSNEP_XSK_RX_BUF_SIZE;
1196 entry->dma = xsk_buff_xdp_get_dma(entry->xdp);
1197 entry->desc->rx = __cpu_to_le64(entry->dma);
1200 static void tsnep_rx_reuse_buffer_zc(struct tsnep_rx *rx, int index)
1202 struct tsnep_rx_entry *entry = &rx->entry[index];
1203 struct tsnep_rx_entry *read = &rx->entry[rx->read];
1205 tsnep_rx_set_xdp(rx, entry, read->xdp);
1209 static int tsnep_rx_alloc_zc(struct tsnep_rx *rx, int count, bool reuse)
1214 allocated = xsk_buff_alloc_batch(rx->xsk_pool, rx->xdp_batch, count);
1215 for (i = 0; i < allocated; i++) {
1216 int index = (rx->write + i) & TSNEP_RING_MASK;
1217 struct tsnep_rx_entry *entry = &rx->entry[index];
1219 tsnep_rx_set_xdp(rx, entry, rx->xdp_batch[i]);
1220 tsnep_rx_activate(rx, index);
1226 tsnep_rx_reuse_buffer_zc(rx, rx->write);
1227 tsnep_rx_activate(rx, rx->write);
1232 rx->write = (rx->write + i) & TSNEP_RING_MASK;
1237 static void tsnep_rx_free_zc(struct tsnep_rx *rx)
1241 for (i = 0; i < TSNEP_RING_SIZE; i++) {
1242 struct tsnep_rx_entry *entry = &rx->entry[i];
1245 xsk_buff_free(entry->xdp);
1250 static int tsnep_rx_refill_zc(struct tsnep_rx *rx, int count, bool reuse)
1254 desc_refilled = tsnep_rx_alloc_zc(rx, count, reuse);
1256 tsnep_rx_enable(rx);
1258 return desc_refilled;
1261 static bool tsnep_xdp_run_prog(struct tsnep_rx *rx, struct bpf_prog *prog,
1262 struct xdp_buff *xdp, int *status,
1263 struct netdev_queue *tx_nq, struct tsnep_tx *tx)
1265 unsigned int length;
1269 length = xdp->data_end - xdp->data_hard_start - XDP_PACKET_HEADROOM;
1271 act = bpf_prog_run_xdp(prog, xdp);
1276 if (!tsnep_xdp_xmit_back(rx->adapter, xdp, tx_nq, tx))
1278 *status |= TSNEP_XDP_TX;
1281 if (xdp_do_redirect(rx->adapter->netdev, xdp, prog) < 0)
1283 *status |= TSNEP_XDP_REDIRECT;
1286 bpf_warn_invalid_xdp_action(rx->adapter->netdev, prog, act);
1290 trace_xdp_exception(rx->adapter->netdev, prog, act);
1293 /* Due xdp_adjust_tail: DMA sync for_device cover max len CPU
1296 sync = xdp->data_end - xdp->data_hard_start -
1297 XDP_PACKET_HEADROOM;
1298 sync = max(sync, length);
1299 page_pool_put_page(rx->page_pool, virt_to_head_page(xdp->data),
1305 static bool tsnep_xdp_run_prog_zc(struct tsnep_rx *rx, struct bpf_prog *prog,
1306 struct xdp_buff *xdp, int *status,
1307 struct netdev_queue *tx_nq,
1308 struct tsnep_tx *tx)
1312 act = bpf_prog_run_xdp(prog, xdp);
1314 /* XDP_REDIRECT is the main action for zero-copy */
1315 if (likely(act == XDP_REDIRECT)) {
1316 if (xdp_do_redirect(rx->adapter->netdev, xdp, prog) < 0)
1318 *status |= TSNEP_XDP_REDIRECT;
1326 if (!tsnep_xdp_xmit_back(rx->adapter, xdp, tx_nq, tx))
1328 *status |= TSNEP_XDP_TX;
1331 bpf_warn_invalid_xdp_action(rx->adapter->netdev, prog, act);
1335 trace_xdp_exception(rx->adapter->netdev, prog, act);
1343 static void tsnep_finalize_xdp(struct tsnep_adapter *adapter, int status,
1344 struct netdev_queue *tx_nq, struct tsnep_tx *tx)
1346 if (status & TSNEP_XDP_TX) {
1347 __netif_tx_lock(tx_nq, smp_processor_id());
1348 tsnep_xdp_xmit_flush(tx);
1349 __netif_tx_unlock(tx_nq);
1352 if (status & TSNEP_XDP_REDIRECT)
1356 static struct sk_buff *tsnep_build_skb(struct tsnep_rx *rx, struct page *page,
1359 struct sk_buff *skb;
1361 skb = napi_build_skb(page_address(page), PAGE_SIZE);
1365 /* update pointers within the skb to store the data */
1366 skb_reserve(skb, TSNEP_RX_OFFSET + TSNEP_RX_INLINE_METADATA_SIZE);
1367 __skb_put(skb, length - ETH_FCS_LEN);
1369 if (rx->adapter->hwtstamp_config.rx_filter == HWTSTAMP_FILTER_ALL) {
1370 struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb);
1371 struct tsnep_rx_inline *rx_inline =
1372 (struct tsnep_rx_inline *)(page_address(page) +
1375 skb_shinfo(skb)->tx_flags |=
1376 SKBTX_HW_TSTAMP_NETDEV;
1377 memset(hwtstamps, 0, sizeof(*hwtstamps));
1378 hwtstamps->netdev_data = rx_inline;
1381 skb_record_rx_queue(skb, rx->queue_index);
1382 skb->protocol = eth_type_trans(skb, rx->adapter->netdev);
1387 static void tsnep_rx_page(struct tsnep_rx *rx, struct napi_struct *napi,
1388 struct page *page, int length)
1390 struct sk_buff *skb;
1392 skb = tsnep_build_skb(rx, page, length);
1394 skb_mark_for_recycle(skb);
1397 rx->bytes += length;
1398 if (skb->pkt_type == PACKET_MULTICAST)
1401 napi_gro_receive(napi, skb);
1403 page_pool_recycle_direct(rx->page_pool, page);
1409 static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi,
1412 struct device *dmadev = rx->adapter->dmadev;
1413 enum dma_data_direction dma_dir;
1414 struct tsnep_rx_entry *entry;
1415 struct netdev_queue *tx_nq;
1416 struct bpf_prog *prog;
1417 struct xdp_buff xdp;
1418 struct tsnep_tx *tx;
1424 desc_available = tsnep_rx_desc_available(rx);
1425 dma_dir = page_pool_get_dma_dir(rx->page_pool);
1426 prog = READ_ONCE(rx->adapter->xdp_prog);
1428 tx_nq = netdev_get_tx_queue(rx->adapter->netdev,
1429 rx->tx_queue_index);
1430 tx = &rx->adapter->tx[rx->tx_queue_index];
1432 xdp_init_buff(&xdp, PAGE_SIZE, &rx->xdp_rxq);
1435 while (likely(done < budget) && (rx->read != rx->write)) {
1436 entry = &rx->entry[rx->read];
1437 if ((__le32_to_cpu(entry->desc_wb->properties) &
1438 TSNEP_DESC_OWNER_COUNTER_MASK) !=
1439 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK))
1443 if (desc_available >= TSNEP_RING_RX_REFILL) {
1444 bool reuse = desc_available >= TSNEP_RING_RX_REUSE;
1446 desc_available -= tsnep_rx_refill(rx, desc_available,
1449 /* buffer has been reused for refill to prevent
1450 * empty RX ring, thus buffer cannot be used for
1453 rx->read = (rx->read + 1) & TSNEP_RING_MASK;
1462 /* descriptor properties shall be read first, because valid data
1467 prefetch(page_address(entry->page) + TSNEP_RX_OFFSET);
1468 length = __le32_to_cpu(entry->desc_wb->properties) &
1469 TSNEP_DESC_LENGTH_MASK;
1470 dma_sync_single_range_for_cpu(dmadev, entry->dma,
1471 TSNEP_RX_OFFSET, length, dma_dir);
1473 /* RX metadata with timestamps is in front of actual data,
1474 * subtract metadata size to get length of actual data and
1475 * consider metadata size as offset of actual data during RX
1478 length -= TSNEP_RX_INLINE_METADATA_SIZE;
1480 rx->read = (rx->read + 1) & TSNEP_RING_MASK;
1486 xdp_prepare_buff(&xdp, page_address(entry->page),
1487 XDP_PACKET_HEADROOM + TSNEP_RX_INLINE_METADATA_SIZE,
1488 length - ETH_FCS_LEN, false);
1490 consume = tsnep_xdp_run_prog(rx, prog, &xdp,
1491 &xdp_status, tx_nq, tx);
1494 rx->bytes += length;
1502 tsnep_rx_page(rx, napi, entry->page, length);
1507 tsnep_finalize_xdp(rx->adapter, xdp_status, tx_nq, tx);
1510 tsnep_rx_refill(rx, desc_available, false);
1515 static int tsnep_rx_poll_zc(struct tsnep_rx *rx, struct napi_struct *napi,
1518 struct tsnep_rx_entry *entry;
1519 struct netdev_queue *tx_nq;
1520 struct bpf_prog *prog;
1521 struct tsnep_tx *tx;
1528 desc_available = tsnep_rx_desc_available(rx);
1529 prog = READ_ONCE(rx->adapter->xdp_prog);
1531 tx_nq = netdev_get_tx_queue(rx->adapter->netdev,
1532 rx->tx_queue_index);
1533 tx = &rx->adapter->tx[rx->tx_queue_index];
1536 while (likely(done < budget) && (rx->read != rx->write)) {
1537 entry = &rx->entry[rx->read];
1538 if ((__le32_to_cpu(entry->desc_wb->properties) &
1539 TSNEP_DESC_OWNER_COUNTER_MASK) !=
1540 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK))
1544 if (desc_available >= TSNEP_RING_RX_REFILL) {
1545 bool reuse = desc_available >= TSNEP_RING_RX_REUSE;
1547 desc_available -= tsnep_rx_refill_zc(rx, desc_available,
1550 /* buffer has been reused for refill to prevent
1551 * empty RX ring, thus buffer cannot be used for
1554 rx->read = (rx->read + 1) & TSNEP_RING_MASK;
1563 /* descriptor properties shall be read first, because valid data
1568 prefetch(entry->xdp->data);
1569 length = __le32_to_cpu(entry->desc_wb->properties) &
1570 TSNEP_DESC_LENGTH_MASK;
1571 xsk_buff_set_size(entry->xdp, length - ETH_FCS_LEN);
1572 xsk_buff_dma_sync_for_cpu(entry->xdp, rx->xsk_pool);
1574 /* RX metadata with timestamps is in front of actual data,
1575 * subtract metadata size to get length of actual data and
1576 * consider metadata size as offset of actual data during RX
1579 length -= TSNEP_RX_INLINE_METADATA_SIZE;
1581 rx->read = (rx->read + 1) & TSNEP_RING_MASK;
1587 entry->xdp->data += TSNEP_RX_INLINE_METADATA_SIZE;
1588 entry->xdp->data_meta += TSNEP_RX_INLINE_METADATA_SIZE;
1590 consume = tsnep_xdp_run_prog_zc(rx, prog, entry->xdp,
1591 &xdp_status, tx_nq, tx);
1594 rx->bytes += length;
1602 page = page_pool_dev_alloc_pages(rx->page_pool);
1604 memcpy(page_address(page) + TSNEP_RX_OFFSET,
1605 entry->xdp->data - TSNEP_RX_INLINE_METADATA_SIZE,
1606 length + TSNEP_RX_INLINE_METADATA_SIZE);
1607 tsnep_rx_page(rx, napi, page, length);
1611 xsk_buff_free(entry->xdp);
1616 tsnep_finalize_xdp(rx->adapter, xdp_status, tx_nq, tx);
1619 desc_available -= tsnep_rx_refill_zc(rx, desc_available, false);
1621 if (xsk_uses_need_wakeup(rx->xsk_pool)) {
1623 xsk_set_rx_need_wakeup(rx->xsk_pool);
1625 xsk_clear_rx_need_wakeup(rx->xsk_pool);
1630 return desc_available ? budget : done;
1633 static bool tsnep_rx_pending(struct tsnep_rx *rx)
1635 struct tsnep_rx_entry *entry;
1637 if (rx->read != rx->write) {
1638 entry = &rx->entry[rx->read];
1639 if ((__le32_to_cpu(entry->desc_wb->properties) &
1640 TSNEP_DESC_OWNER_COUNTER_MASK) ==
1641 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK))
1648 static int tsnep_rx_open(struct tsnep_rx *rx)
1653 retval = tsnep_rx_ring_create(rx);
1659 desc_available = tsnep_rx_desc_available(rx);
1661 retval = tsnep_rx_alloc_zc(rx, desc_available, false);
1663 retval = tsnep_rx_alloc(rx, desc_available, false);
1664 if (retval != desc_available) {
1670 /* prealloc pages to prevent allocation failures when XSK pool is
1671 * disabled at runtime
1674 retval = tsnep_rx_alloc_page_buffer(rx);
1682 tsnep_rx_ring_cleanup(rx);
1686 static void tsnep_rx_close(struct tsnep_rx *rx)
1689 tsnep_rx_free_page_buffer(rx);
1691 tsnep_rx_ring_cleanup(rx);
1694 static void tsnep_rx_reopen(struct tsnep_rx *rx)
1696 struct page **page = rx->page_buffer;
1701 for (i = 0; i < TSNEP_RING_SIZE; i++) {
1702 struct tsnep_rx_entry *entry = &rx->entry[i];
1704 /* defined initial values for properties are required for
1705 * correct owner counter checking
1707 entry->desc->properties = 0;
1708 entry->desc_wb->properties = 0;
1710 /* prevent allocation failures by reusing kept pages */
1712 tsnep_rx_set_page(rx, entry, *page);
1713 tsnep_rx_activate(rx, rx->write);
1722 static void tsnep_rx_reopen_xsk(struct tsnep_rx *rx)
1724 struct page **page = rx->page_buffer;
1730 /* alloc all ring entries except the last one, because ring cannot be
1731 * filled completely, as many buffers as possible is enough as wakeup is
1732 * done if new buffers are available
1734 allocated = xsk_buff_alloc_batch(rx->xsk_pool, rx->xdp_batch,
1735 TSNEP_RING_SIZE - 1);
1737 for (i = 0; i < TSNEP_RING_SIZE; i++) {
1738 struct tsnep_rx_entry *entry = &rx->entry[i];
1740 /* keep pages to prevent allocation failures when xsk is
1744 *page = entry->page;
1750 /* defined initial values for properties are required for
1751 * correct owner counter checking
1753 entry->desc->properties = 0;
1754 entry->desc_wb->properties = 0;
1757 tsnep_rx_set_xdp(rx, entry,
1758 rx->xdp_batch[allocated - 1]);
1759 tsnep_rx_activate(rx, rx->write);
1767 static bool tsnep_pending(struct tsnep_queue *queue)
1769 if (queue->tx && tsnep_tx_pending(queue->tx))
1772 if (queue->rx && tsnep_rx_pending(queue->rx))
1778 static int tsnep_poll(struct napi_struct *napi, int budget)
1780 struct tsnep_queue *queue = container_of(napi, struct tsnep_queue,
1782 bool complete = true;
1786 complete = tsnep_tx_poll(queue->tx, budget);
1788 /* handle case where we are called by netpoll with a budget of 0 */
1789 if (unlikely(budget <= 0))
1793 done = queue->rx->xsk_pool ?
1794 tsnep_rx_poll_zc(queue->rx, napi, budget) :
1795 tsnep_rx_poll(queue->rx, napi, budget);
1800 /* if all work not completed, return budget and keep polling */
1804 if (likely(napi_complete_done(napi, done))) {
1805 tsnep_enable_irq(queue->adapter, queue->irq_mask);
1807 /* reschedule if work is already pending, prevent rotten packets
1808 * which are transmitted or received after polling but before
1811 if (tsnep_pending(queue)) {
1812 tsnep_disable_irq(queue->adapter, queue->irq_mask);
1813 napi_schedule(napi);
1817 return min(done, budget - 1);
1820 static int tsnep_request_irq(struct tsnep_queue *queue, bool first)
1822 const char *name = netdev_name(queue->adapter->netdev);
1823 irq_handler_t handler;
1828 sprintf(queue->name, "%s-mac", name);
1829 handler = tsnep_irq;
1830 dev = queue->adapter;
1832 if (queue->tx && queue->rx)
1833 snprintf(queue->name, sizeof(queue->name), "%s-txrx-%d",
1834 name, queue->rx->queue_index);
1836 snprintf(queue->name, sizeof(queue->name), "%s-tx-%d",
1837 name, queue->tx->queue_index);
1839 snprintf(queue->name, sizeof(queue->name), "%s-rx-%d",
1840 name, queue->rx->queue_index);
1841 handler = tsnep_irq_txrx;
1845 retval = request_irq(queue->irq, handler, 0, queue->name, dev);
1847 /* if name is empty, then interrupt won't be freed */
1848 memset(queue->name, 0, sizeof(queue->name));
1854 static void tsnep_free_irq(struct tsnep_queue *queue, bool first)
1858 if (!strlen(queue->name))
1862 dev = queue->adapter;
1866 free_irq(queue->irq, dev);
1867 memset(queue->name, 0, sizeof(queue->name));
1870 static void tsnep_queue_close(struct tsnep_queue *queue, bool first)
1872 struct tsnep_rx *rx = queue->rx;
1874 tsnep_free_irq(queue, first);
1877 if (xdp_rxq_info_is_reg(&rx->xdp_rxq))
1878 xdp_rxq_info_unreg(&rx->xdp_rxq);
1879 if (xdp_rxq_info_is_reg(&rx->xdp_rxq_zc))
1880 xdp_rxq_info_unreg(&rx->xdp_rxq_zc);
1883 netif_napi_del(&queue->napi);
1886 static int tsnep_queue_open(struct tsnep_adapter *adapter,
1887 struct tsnep_queue *queue, bool first)
1889 struct tsnep_rx *rx = queue->rx;
1890 struct tsnep_tx *tx = queue->tx;
1893 netif_napi_add(adapter->netdev, &queue->napi, tsnep_poll);
1896 /* choose TX queue for XDP_TX */
1898 rx->tx_queue_index = tx->queue_index;
1899 else if (rx->queue_index < adapter->num_tx_queues)
1900 rx->tx_queue_index = rx->queue_index;
1902 rx->tx_queue_index = 0;
1904 /* prepare both memory models to eliminate possible registration
1905 * errors when memory model is switched between page pool and
1906 * XSK pool during runtime
1908 retval = xdp_rxq_info_reg(&rx->xdp_rxq, adapter->netdev,
1909 rx->queue_index, queue->napi.napi_id);
1912 retval = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq,
1917 retval = xdp_rxq_info_reg(&rx->xdp_rxq_zc, adapter->netdev,
1918 rx->queue_index, queue->napi.napi_id);
1921 retval = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq_zc,
1922 MEM_TYPE_XSK_BUFF_POOL,
1927 xsk_pool_set_rxq_info(rx->xsk_pool, &rx->xdp_rxq_zc);
1930 retval = tsnep_request_irq(queue, first);
1932 netif_err(adapter, drv, adapter->netdev,
1933 "can't get assigned irq %d.\n", queue->irq);
1940 tsnep_queue_close(queue, first);
1945 static void tsnep_queue_enable(struct tsnep_queue *queue)
1947 napi_enable(&queue->napi);
1948 tsnep_enable_irq(queue->adapter, queue->irq_mask);
1951 tsnep_tx_enable(queue->tx);
1954 tsnep_rx_enable(queue->rx);
1957 static void tsnep_queue_disable(struct tsnep_queue *queue)
1960 tsnep_tx_disable(queue->tx, &queue->napi);
1962 napi_disable(&queue->napi);
1963 tsnep_disable_irq(queue->adapter, queue->irq_mask);
1965 /* disable RX after NAPI polling has been disabled, because RX can be
1966 * enabled during NAPI polling
1969 tsnep_rx_disable(queue->rx);
1972 static int tsnep_netdev_open(struct net_device *netdev)
1974 struct tsnep_adapter *adapter = netdev_priv(netdev);
1977 for (i = 0; i < adapter->num_queues; i++) {
1978 if (adapter->queue[i].tx) {
1979 retval = tsnep_tx_open(adapter->queue[i].tx);
1983 if (adapter->queue[i].rx) {
1984 retval = tsnep_rx_open(adapter->queue[i].rx);
1989 retval = tsnep_queue_open(adapter, &adapter->queue[i], i == 0);
1994 retval = netif_set_real_num_tx_queues(adapter->netdev,
1995 adapter->num_tx_queues);
1998 retval = netif_set_real_num_rx_queues(adapter->netdev,
1999 adapter->num_rx_queues);
2003 tsnep_enable_irq(adapter, ECM_INT_LINK);
2004 retval = tsnep_phy_open(adapter);
2008 for (i = 0; i < adapter->num_queues; i++)
2009 tsnep_queue_enable(&adapter->queue[i]);
2014 tsnep_disable_irq(adapter, ECM_INT_LINK);
2016 for (i = 0; i < adapter->num_queues; i++) {
2017 tsnep_queue_close(&adapter->queue[i], i == 0);
2019 if (adapter->queue[i].rx)
2020 tsnep_rx_close(adapter->queue[i].rx);
2021 if (adapter->queue[i].tx)
2022 tsnep_tx_close(adapter->queue[i].tx);
2027 static int tsnep_netdev_close(struct net_device *netdev)
2029 struct tsnep_adapter *adapter = netdev_priv(netdev);
2032 tsnep_disable_irq(adapter, ECM_INT_LINK);
2033 tsnep_phy_close(adapter);
2035 for (i = 0; i < adapter->num_queues; i++) {
2036 tsnep_queue_disable(&adapter->queue[i]);
2038 tsnep_queue_close(&adapter->queue[i], i == 0);
2040 if (adapter->queue[i].rx)
2041 tsnep_rx_close(adapter->queue[i].rx);
2042 if (adapter->queue[i].tx)
2043 tsnep_tx_close(adapter->queue[i].tx);
2049 int tsnep_enable_xsk(struct tsnep_queue *queue, struct xsk_buff_pool *pool)
2051 bool running = netif_running(queue->adapter->netdev);
2054 frame_size = xsk_pool_get_rx_frame_size(pool);
2055 if (frame_size < TSNEP_XSK_RX_BUF_SIZE)
2058 queue->rx->page_buffer = kcalloc(TSNEP_RING_SIZE,
2059 sizeof(*queue->rx->page_buffer),
2061 if (!queue->rx->page_buffer)
2063 queue->rx->xdp_batch = kcalloc(TSNEP_RING_SIZE,
2064 sizeof(*queue->rx->xdp_batch),
2066 if (!queue->rx->xdp_batch) {
2067 kfree(queue->rx->page_buffer);
2068 queue->rx->page_buffer = NULL;
2073 xsk_pool_set_rxq_info(pool, &queue->rx->xdp_rxq_zc);
2076 tsnep_queue_disable(queue);
2078 queue->tx->xsk_pool = pool;
2079 queue->rx->xsk_pool = pool;
2082 tsnep_rx_reopen_xsk(queue->rx);
2083 tsnep_queue_enable(queue);
2089 void tsnep_disable_xsk(struct tsnep_queue *queue)
2091 bool running = netif_running(queue->adapter->netdev);
2094 tsnep_queue_disable(queue);
2096 tsnep_rx_free_zc(queue->rx);
2098 queue->rx->xsk_pool = NULL;
2099 queue->tx->xsk_pool = NULL;
2102 tsnep_rx_reopen(queue->rx);
2103 tsnep_queue_enable(queue);
2106 kfree(queue->rx->xdp_batch);
2107 queue->rx->xdp_batch = NULL;
2108 kfree(queue->rx->page_buffer);
2109 queue->rx->page_buffer = NULL;
2112 static netdev_tx_t tsnep_netdev_xmit_frame(struct sk_buff *skb,
2113 struct net_device *netdev)
2115 struct tsnep_adapter *adapter = netdev_priv(netdev);
2116 u16 queue_mapping = skb_get_queue_mapping(skb);
2118 if (queue_mapping >= adapter->num_tx_queues)
2121 return tsnep_xmit_frame_ring(skb, &adapter->tx[queue_mapping]);
2124 static int tsnep_netdev_ioctl(struct net_device *netdev, struct ifreq *ifr,
2127 if (!netif_running(netdev))
2129 if (cmd == SIOCSHWTSTAMP || cmd == SIOCGHWTSTAMP)
2130 return tsnep_ptp_ioctl(netdev, ifr, cmd);
2131 return phy_mii_ioctl(netdev->phydev, ifr, cmd);
2134 static void tsnep_netdev_set_multicast(struct net_device *netdev)
2136 struct tsnep_adapter *adapter = netdev_priv(netdev);
2140 /* configured MAC address and broadcasts are never filtered */
2141 if (netdev->flags & IFF_PROMISC) {
2142 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS;
2143 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_UNICASTS;
2144 } else if (!netdev_mc_empty(netdev) || (netdev->flags & IFF_ALLMULTI)) {
2145 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS;
2147 iowrite16(rx_filter, adapter->addr + TSNEP_RX_FILTER);
2150 static void tsnep_netdev_get_stats64(struct net_device *netdev,
2151 struct rtnl_link_stats64 *stats)
2153 struct tsnep_adapter *adapter = netdev_priv(netdev);
2158 for (i = 0; i < adapter->num_tx_queues; i++) {
2159 stats->tx_packets += adapter->tx[i].packets;
2160 stats->tx_bytes += adapter->tx[i].bytes;
2161 stats->tx_dropped += adapter->tx[i].dropped;
2163 for (i = 0; i < adapter->num_rx_queues; i++) {
2164 stats->rx_packets += adapter->rx[i].packets;
2165 stats->rx_bytes += adapter->rx[i].bytes;
2166 stats->rx_dropped += adapter->rx[i].dropped;
2167 stats->multicast += adapter->rx[i].multicast;
2169 reg = ioread32(adapter->addr + TSNEP_QUEUE(i) +
2170 TSNEP_RX_STATISTIC);
2171 val = (reg & TSNEP_RX_STATISTIC_NO_DESC_MASK) >>
2172 TSNEP_RX_STATISTIC_NO_DESC_SHIFT;
2173 stats->rx_dropped += val;
2174 val = (reg & TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_MASK) >>
2175 TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_SHIFT;
2176 stats->rx_dropped += val;
2177 val = (reg & TSNEP_RX_STATISTIC_FIFO_OVERFLOW_MASK) >>
2178 TSNEP_RX_STATISTIC_FIFO_OVERFLOW_SHIFT;
2179 stats->rx_errors += val;
2180 stats->rx_fifo_errors += val;
2181 val = (reg & TSNEP_RX_STATISTIC_INVALID_FRAME_MASK) >>
2182 TSNEP_RX_STATISTIC_INVALID_FRAME_SHIFT;
2183 stats->rx_errors += val;
2184 stats->rx_frame_errors += val;
2187 reg = ioread32(adapter->addr + ECM_STAT);
2188 val = (reg & ECM_STAT_RX_ERR_MASK) >> ECM_STAT_RX_ERR_SHIFT;
2189 stats->rx_errors += val;
2190 val = (reg & ECM_STAT_INV_FRM_MASK) >> ECM_STAT_INV_FRM_SHIFT;
2191 stats->rx_errors += val;
2192 stats->rx_crc_errors += val;
2193 val = (reg & ECM_STAT_FWD_RX_ERR_MASK) >> ECM_STAT_FWD_RX_ERR_SHIFT;
2194 stats->rx_errors += val;
2197 static void tsnep_mac_set_address(struct tsnep_adapter *adapter, u8 *addr)
2199 iowrite32(*(u32 *)addr, adapter->addr + TSNEP_MAC_ADDRESS_LOW);
2200 iowrite16(*(u16 *)(addr + sizeof(u32)),
2201 adapter->addr + TSNEP_MAC_ADDRESS_HIGH);
2203 ether_addr_copy(adapter->mac_address, addr);
2204 netif_info(adapter, drv, adapter->netdev, "MAC address set to %pM\n",
2208 static int tsnep_netdev_set_mac_address(struct net_device *netdev, void *addr)
2210 struct tsnep_adapter *adapter = netdev_priv(netdev);
2211 struct sockaddr *sock_addr = addr;
2214 retval = eth_prepare_mac_addr_change(netdev, sock_addr);
2217 eth_hw_addr_set(netdev, sock_addr->sa_data);
2218 tsnep_mac_set_address(adapter, sock_addr->sa_data);
2223 static int tsnep_netdev_set_features(struct net_device *netdev,
2224 netdev_features_t features)
2226 struct tsnep_adapter *adapter = netdev_priv(netdev);
2227 netdev_features_t changed = netdev->features ^ features;
2231 if (changed & NETIF_F_LOOPBACK) {
2232 enable = !!(features & NETIF_F_LOOPBACK);
2233 retval = tsnep_phy_loopback(adapter, enable);
2239 static ktime_t tsnep_netdev_get_tstamp(struct net_device *netdev,
2240 const struct skb_shared_hwtstamps *hwtstamps,
2243 struct tsnep_rx_inline *rx_inline = hwtstamps->netdev_data;
2247 timestamp = __le64_to_cpu(rx_inline->counter);
2249 timestamp = __le64_to_cpu(rx_inline->timestamp);
2251 return ns_to_ktime(timestamp);
2254 static int tsnep_netdev_bpf(struct net_device *dev, struct netdev_bpf *bpf)
2256 struct tsnep_adapter *adapter = netdev_priv(dev);
2258 switch (bpf->command) {
2259 case XDP_SETUP_PROG:
2260 return tsnep_xdp_setup_prog(adapter, bpf->prog, bpf->extack);
2261 case XDP_SETUP_XSK_POOL:
2262 return tsnep_xdp_setup_pool(adapter, bpf->xsk.pool,
2269 static struct tsnep_tx *tsnep_xdp_get_tx(struct tsnep_adapter *adapter, u32 cpu)
2271 if (cpu >= TSNEP_MAX_QUEUES)
2272 cpu &= TSNEP_MAX_QUEUES - 1;
2274 while (cpu >= adapter->num_tx_queues)
2275 cpu -= adapter->num_tx_queues;
2277 return &adapter->tx[cpu];
2280 static int tsnep_netdev_xdp_xmit(struct net_device *dev, int n,
2281 struct xdp_frame **xdp, u32 flags)
2283 struct tsnep_adapter *adapter = netdev_priv(dev);
2284 u32 cpu = smp_processor_id();
2285 struct netdev_queue *nq;
2286 struct tsnep_tx *tx;
2290 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
2293 tx = tsnep_xdp_get_tx(adapter, cpu);
2294 nq = netdev_get_tx_queue(adapter->netdev, tx->queue_index);
2296 __netif_tx_lock(nq, cpu);
2298 for (nxmit = 0; nxmit < n; nxmit++) {
2299 xmit = tsnep_xdp_xmit_frame_ring(xdp[nxmit], tx,
2300 TSNEP_TX_TYPE_XDP_NDO);
2304 /* avoid transmit queue timeout since we share it with the slow
2307 txq_trans_cond_update(nq);
2310 if (flags & XDP_XMIT_FLUSH)
2311 tsnep_xdp_xmit_flush(tx);
2313 __netif_tx_unlock(nq);
2318 static int tsnep_netdev_xsk_wakeup(struct net_device *dev, u32 queue_id,
2321 struct tsnep_adapter *adapter = netdev_priv(dev);
2322 struct tsnep_queue *queue;
2324 if (queue_id >= adapter->num_rx_queues ||
2325 queue_id >= adapter->num_tx_queues)
2328 queue = &adapter->queue[queue_id];
2330 if (!napi_if_scheduled_mark_missed(&queue->napi))
2331 napi_schedule(&queue->napi);
2336 static const struct net_device_ops tsnep_netdev_ops = {
2337 .ndo_open = tsnep_netdev_open,
2338 .ndo_stop = tsnep_netdev_close,
2339 .ndo_start_xmit = tsnep_netdev_xmit_frame,
2340 .ndo_eth_ioctl = tsnep_netdev_ioctl,
2341 .ndo_set_rx_mode = tsnep_netdev_set_multicast,
2342 .ndo_get_stats64 = tsnep_netdev_get_stats64,
2343 .ndo_set_mac_address = tsnep_netdev_set_mac_address,
2344 .ndo_set_features = tsnep_netdev_set_features,
2345 .ndo_get_tstamp = tsnep_netdev_get_tstamp,
2346 .ndo_setup_tc = tsnep_tc_setup,
2347 .ndo_bpf = tsnep_netdev_bpf,
2348 .ndo_xdp_xmit = tsnep_netdev_xdp_xmit,
2349 .ndo_xsk_wakeup = tsnep_netdev_xsk_wakeup,
2352 static int tsnep_mac_init(struct tsnep_adapter *adapter)
2356 /* initialize RX filtering, at least configured MAC address and
2357 * broadcast are not filtered
2359 iowrite16(0, adapter->addr + TSNEP_RX_FILTER);
2361 /* try to get MAC address in the following order:
2363 * - valid MAC address already set
2364 * - MAC address register if valid
2365 * - random MAC address
2367 retval = of_get_mac_address(adapter->pdev->dev.of_node,
2368 adapter->mac_address);
2369 if (retval == -EPROBE_DEFER)
2371 if (retval && !is_valid_ether_addr(adapter->mac_address)) {
2372 *(u32 *)adapter->mac_address =
2373 ioread32(adapter->addr + TSNEP_MAC_ADDRESS_LOW);
2374 *(u16 *)(adapter->mac_address + sizeof(u32)) =
2375 ioread16(adapter->addr + TSNEP_MAC_ADDRESS_HIGH);
2376 if (!is_valid_ether_addr(adapter->mac_address))
2377 eth_random_addr(adapter->mac_address);
2380 tsnep_mac_set_address(adapter, adapter->mac_address);
2381 eth_hw_addr_set(adapter->netdev, adapter->mac_address);
2386 static int tsnep_mdio_init(struct tsnep_adapter *adapter)
2388 struct device_node *np = adapter->pdev->dev.of_node;
2392 np = of_get_child_by_name(np, "mdio");
2396 adapter->suppress_preamble =
2397 of_property_read_bool(np, "suppress-preamble");
2400 adapter->mdiobus = devm_mdiobus_alloc(&adapter->pdev->dev);
2401 if (!adapter->mdiobus) {
2407 adapter->mdiobus->priv = (void *)adapter;
2408 adapter->mdiobus->parent = &adapter->pdev->dev;
2409 adapter->mdiobus->read = tsnep_mdiobus_read;
2410 adapter->mdiobus->write = tsnep_mdiobus_write;
2411 adapter->mdiobus->name = TSNEP "-mdiobus";
2412 snprintf(adapter->mdiobus->id, MII_BUS_ID_SIZE, "%s",
2413 adapter->pdev->name);
2415 /* do not scan broadcast address */
2416 adapter->mdiobus->phy_mask = 0x0000001;
2418 retval = of_mdiobus_register(adapter->mdiobus, np);
2426 static int tsnep_phy_init(struct tsnep_adapter *adapter)
2428 struct device_node *phy_node;
2431 retval = of_get_phy_mode(adapter->pdev->dev.of_node,
2432 &adapter->phy_mode);
2434 adapter->phy_mode = PHY_INTERFACE_MODE_GMII;
2436 phy_node = of_parse_phandle(adapter->pdev->dev.of_node, "phy-handle",
2438 adapter->phydev = of_phy_find_device(phy_node);
2439 of_node_put(phy_node);
2440 if (!adapter->phydev && adapter->mdiobus)
2441 adapter->phydev = phy_find_first(adapter->mdiobus);
2442 if (!adapter->phydev)
2448 static int tsnep_queue_init(struct tsnep_adapter *adapter, int queue_count)
2450 u32 irq_mask = ECM_INT_TX_0 | ECM_INT_RX_0;
2455 /* one TX/RX queue pair for netdev is mandatory */
2456 if (platform_irq_count(adapter->pdev) == 1)
2457 retval = platform_get_irq(adapter->pdev, 0);
2459 retval = platform_get_irq_byname(adapter->pdev, "mac");
2462 adapter->num_tx_queues = 1;
2463 adapter->num_rx_queues = 1;
2464 adapter->num_queues = 1;
2465 adapter->queue[0].adapter = adapter;
2466 adapter->queue[0].irq = retval;
2467 adapter->queue[0].tx = &adapter->tx[0];
2468 adapter->queue[0].tx->adapter = adapter;
2469 adapter->queue[0].tx->addr = adapter->addr + TSNEP_QUEUE(0);
2470 adapter->queue[0].tx->queue_index = 0;
2471 adapter->queue[0].rx = &adapter->rx[0];
2472 adapter->queue[0].rx->adapter = adapter;
2473 adapter->queue[0].rx->addr = adapter->addr + TSNEP_QUEUE(0);
2474 adapter->queue[0].rx->queue_index = 0;
2475 adapter->queue[0].irq_mask = irq_mask;
2476 adapter->queue[0].irq_delay_addr = adapter->addr + ECM_INT_DELAY;
2477 retval = tsnep_set_irq_coalesce(&adapter->queue[0],
2478 TSNEP_COALESCE_USECS_DEFAULT);
2482 adapter->netdev->irq = adapter->queue[0].irq;
2484 /* add additional TX/RX queue pairs only if dedicated interrupt is
2487 for (i = 1; i < queue_count; i++) {
2488 sprintf(name, "txrx-%d", i);
2489 retval = platform_get_irq_byname_optional(adapter->pdev, name);
2493 adapter->num_tx_queues++;
2494 adapter->num_rx_queues++;
2495 adapter->num_queues++;
2496 adapter->queue[i].adapter = adapter;
2497 adapter->queue[i].irq = retval;
2498 adapter->queue[i].tx = &adapter->tx[i];
2499 adapter->queue[i].tx->adapter = adapter;
2500 adapter->queue[i].tx->addr = adapter->addr + TSNEP_QUEUE(i);
2501 adapter->queue[i].tx->queue_index = i;
2502 adapter->queue[i].rx = &adapter->rx[i];
2503 adapter->queue[i].rx->adapter = adapter;
2504 adapter->queue[i].rx->addr = adapter->addr + TSNEP_QUEUE(i);
2505 adapter->queue[i].rx->queue_index = i;
2506 adapter->queue[i].irq_mask =
2507 irq_mask << (ECM_INT_TXRX_SHIFT * i);
2508 adapter->queue[i].irq_delay_addr =
2509 adapter->addr + ECM_INT_DELAY + ECM_INT_DELAY_OFFSET * i;
2510 retval = tsnep_set_irq_coalesce(&adapter->queue[i],
2511 TSNEP_COALESCE_USECS_DEFAULT);
2519 static int tsnep_probe(struct platform_device *pdev)
2521 struct tsnep_adapter *adapter;
2522 struct net_device *netdev;
2523 struct resource *io;
2530 netdev = devm_alloc_etherdev_mqs(&pdev->dev,
2531 sizeof(struct tsnep_adapter),
2532 TSNEP_MAX_QUEUES, TSNEP_MAX_QUEUES);
2535 SET_NETDEV_DEV(netdev, &pdev->dev);
2536 adapter = netdev_priv(netdev);
2537 platform_set_drvdata(pdev, adapter);
2538 adapter->pdev = pdev;
2539 adapter->dmadev = &pdev->dev;
2540 adapter->netdev = netdev;
2541 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE |
2542 NETIF_MSG_LINK | NETIF_MSG_IFUP |
2543 NETIF_MSG_IFDOWN | NETIF_MSG_TX_QUEUED;
2545 netdev->min_mtu = ETH_MIN_MTU;
2546 netdev->max_mtu = TSNEP_MAX_FRAME_SIZE;
2548 mutex_init(&adapter->gate_control_lock);
2549 mutex_init(&adapter->rxnfc_lock);
2550 INIT_LIST_HEAD(&adapter->rxnfc_rules);
2552 io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2553 adapter->addr = devm_ioremap_resource(&pdev->dev, io);
2554 if (IS_ERR(adapter->addr))
2555 return PTR_ERR(adapter->addr);
2556 netdev->mem_start = io->start;
2557 netdev->mem_end = io->end;
2559 type = ioread32(adapter->addr + ECM_TYPE);
2560 revision = (type & ECM_REVISION_MASK) >> ECM_REVISION_SHIFT;
2561 version = (type & ECM_VERSION_MASK) >> ECM_VERSION_SHIFT;
2562 queue_count = (type & ECM_QUEUE_COUNT_MASK) >> ECM_QUEUE_COUNT_SHIFT;
2563 adapter->gate_control = type & ECM_GATE_CONTROL;
2564 adapter->rxnfc_max = TSNEP_RX_ASSIGN_ETHER_TYPE_COUNT;
2566 tsnep_disable_irq(adapter, ECM_INT_ALL);
2568 retval = tsnep_queue_init(adapter, queue_count);
2572 retval = dma_set_mask_and_coherent(&adapter->pdev->dev,
2575 dev_err(&adapter->pdev->dev, "no usable DMA configuration.\n");
2579 retval = tsnep_mac_init(adapter);
2583 retval = tsnep_mdio_init(adapter);
2585 goto mdio_init_failed;
2587 retval = tsnep_phy_init(adapter);
2589 goto phy_init_failed;
2591 retval = tsnep_ptp_init(adapter);
2593 goto ptp_init_failed;
2595 retval = tsnep_tc_init(adapter);
2597 goto tc_init_failed;
2599 retval = tsnep_rxnfc_init(adapter);
2601 goto rxnfc_init_failed;
2603 netdev->netdev_ops = &tsnep_netdev_ops;
2604 netdev->ethtool_ops = &tsnep_ethtool_ops;
2605 netdev->features = NETIF_F_SG;
2606 netdev->hw_features = netdev->features | NETIF_F_LOOPBACK;
2608 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
2609 NETDEV_XDP_ACT_NDO_XMIT |
2610 NETDEV_XDP_ACT_NDO_XMIT_SG |
2611 NETDEV_XDP_ACT_XSK_ZEROCOPY;
2613 /* carrier off reporting is important to ethtool even BEFORE open */
2614 netif_carrier_off(netdev);
2616 retval = register_netdev(netdev);
2618 goto register_failed;
2620 dev_info(&adapter->pdev->dev, "device version %d.%02d\n", version,
2622 if (adapter->gate_control)
2623 dev_info(&adapter->pdev->dev, "gate control detected\n");
2628 tsnep_rxnfc_cleanup(adapter);
2630 tsnep_tc_cleanup(adapter);
2632 tsnep_ptp_cleanup(adapter);
2635 if (adapter->mdiobus)
2636 mdiobus_unregister(adapter->mdiobus);
2641 static void tsnep_remove(struct platform_device *pdev)
2643 struct tsnep_adapter *adapter = platform_get_drvdata(pdev);
2645 unregister_netdev(adapter->netdev);
2647 tsnep_rxnfc_cleanup(adapter);
2649 tsnep_tc_cleanup(adapter);
2651 tsnep_ptp_cleanup(adapter);
2653 if (adapter->mdiobus)
2654 mdiobus_unregister(adapter->mdiobus);
2656 tsnep_disable_irq(adapter, ECM_INT_ALL);
2659 static const struct of_device_id tsnep_of_match[] = {
2660 { .compatible = "engleder,tsnep", },
2663 MODULE_DEVICE_TABLE(of, tsnep_of_match);
2665 static struct platform_driver tsnep_driver = {
2668 .of_match_table = tsnep_of_match,
2670 .probe = tsnep_probe,
2671 .remove_new = tsnep_remove,
2673 module_platform_driver(tsnep_driver);
2675 MODULE_AUTHOR("Gerhard Engleder <gerhard@engleder-embedded.com>");
2676 MODULE_DESCRIPTION("TSN endpoint Ethernet MAC driver");
2677 MODULE_LICENSE("GPL");