[linux-2.6-block.git] / drivers / net / wireless / mediatek / mt76 / mt76x02_mmio.c

// SPDX-License-Identifier: ISC
/*
 * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
 * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com>
 */

#include <linux/kernel.h>
#include <linux/irq.h>

#include "mt76x02.h"
#include "mt76x02_mcu.h"
#include "mt76x02_trace.h"

static void mt76x02_pre_tbtt_tasklet(unsigned long arg)
{
	struct mt76x02_dev *dev = (struct mt76x02_dev *)arg;
	struct mt76_queue *q = dev->mt76.q_tx[MT_TXQ_PSD].q;
	struct beacon_bc_data data = {};
	struct sk_buff *skb;
	int i;

	if (mt76_hw(dev)->conf.flags & IEEE80211_CONF_OFFCHANNEL)
		return;

	mt76x02_resync_beacon_timer(dev);

	ieee80211_iterate_active_interfaces_atomic(mt76_hw(dev),
		IEEE80211_IFACE_ITER_RESUME_ALL,
		mt76x02_update_beacon_iter, dev);

	mt76_csa_check(&dev->mt76);

	if (dev->mt76.csa_complete)
		return;

	mt76x02_enqueue_buffered_bc(dev, &data, 8);

	if (!skb_queue_len(&data.q))
		return;

	for (i = 0; i < ARRAY_SIZE(data.tail); i++) {
		if (!data.tail[i])
			continue;

		mt76_skb_set_moredata(data.tail[i], false);
	}

	spin_lock_bh(&q->lock);
	while ((skb = __skb_dequeue(&data.q)) != NULL) {
		struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
		struct ieee80211_vif *vif = info->control.vif;
		struct mt76x02_vif *mvif = (struct mt76x02_vif *)vif->drv_priv;

		mt76_tx_queue_skb(dev, MT_TXQ_PSD, skb, &mvif->group_wcid,
				  NULL);
	}
	spin_unlock_bh(&q->lock);
}

static void mt76x02e_pre_tbtt_enable(struct mt76x02_dev *dev, bool en)
{
	if (en)
		tasklet_enable(&dev->mt76.pre_tbtt_tasklet);
	else
		tasklet_disable(&dev->mt76.pre_tbtt_tasklet);
}

static void mt76x02e_beacon_enable(struct mt76x02_dev *dev, bool en)
{
	mt76_rmw_field(dev, MT_INT_TIMER_EN, MT_INT_TIMER_EN_PRE_TBTT_EN, en);
	if (en)
		mt76x02_irq_enable(dev, MT_INT_PRE_TBTT | MT_INT_TBTT);
	else
		mt76x02_irq_disable(dev, MT_INT_PRE_TBTT | MT_INT_TBTT);
}

void mt76x02e_init_beacon_config(struct mt76x02_dev *dev)
{
	static const struct mt76x02_beacon_ops beacon_ops = {
		.nslots = 8,
		.slot_size = 1024,
		.pre_tbtt_enable = mt76x02e_pre_tbtt_enable,
		.beacon_enable = mt76x02e_beacon_enable,
	};

	dev->beacon_ops = &beacon_ops;

	/* Fire a pre-TBTT interrupt 8 ms before TBTT */
	mt76_rmw_field(dev, MT_INT_TIMER_CFG, MT_INT_TIMER_CFG_PRE_TBTT,
		       8 << 4);
	mt76_rmw_field(dev, MT_INT_TIMER_CFG, MT_INT_TIMER_CFG_GP_TIMER,
		       MT_DFS_GP_INTERVAL);
	mt76_wr(dev, MT_INT_TIMER_EN, 0);

	mt76x02_init_beacon_config(dev);
}
EXPORT_SYMBOL_GPL(mt76x02e_init_beacon_config);

static int
mt76x02_init_tx_queue(struct mt76x02_dev *dev, struct mt76_sw_queue *q,
		      int idx, int n_desc)
{
	struct mt76_queue *hwq;
	int err;

	hwq = devm_kzalloc(dev->mt76.dev, sizeof(*hwq), GFP_KERNEL);
	if (!hwq)
		return -ENOMEM;

	err = mt76_queue_alloc(dev, hwq, idx, n_desc, 0, MT_TX_RING_BASE);
	if (err < 0)
		return err;

	INIT_LIST_HEAD(&q->swq);
	q->q = hwq;

	mt76x02_irq_enable(dev, MT_INT_TX_DONE(idx));

	return 0;
}

static int
mt76x02_init_rx_queue(struct mt76x02_dev *dev, struct mt76_queue *q,
		      int idx, int n_desc, int bufsize)
{
	int err;

	err = mt76_queue_alloc(dev, q, idx, n_desc, bufsize,
			       MT_RX_RING_BASE);
	if (err < 0)
		return err;

	mt76x02_irq_enable(dev, MT_INT_RX_DONE(idx));

	return 0;
}

static void mt76x02_process_tx_status_fifo(struct mt76x02_dev *dev)
{
	struct mt76x02_tx_status stat;
	u8 update = 1;

	while (kfifo_get(&dev->txstatus_fifo, &stat))
		mt76x02_send_tx_status(dev, &stat, &update);
}

static void mt76x02_tx_tasklet(unsigned long data)
{
	struct mt76x02_dev *dev = (struct mt76x02_dev *)data;

	mt76x02_mac_poll_tx_status(dev, false);
	mt76x02_process_tx_status_fifo(dev);

	mt76_txq_schedule_all(&dev->mt76);
}

static int mt76x02_poll_tx(struct napi_struct *napi, int budget)
{
	struct mt76x02_dev *dev = container_of(napi, struct mt76x02_dev,
					       mt76.tx_napi);
	int i;

	mt76x02_mac_poll_tx_status(dev, false);

	for (i = MT_TXQ_MCU; i >= 0; i--)
		mt76_queue_tx_cleanup(dev, i, false);

	if (napi_complete_done(napi, 0))
		mt76x02_irq_enable(dev, MT_INT_TX_DONE_ALL);

	for (i = MT_TXQ_MCU; i >= 0; i--)
		mt76_queue_tx_cleanup(dev, i, false);

	tasklet_schedule(&dev->mt76.tx_tasklet);

	return 0;
}

int mt76x02_dma_init(struct mt76x02_dev *dev)
{
	struct mt76_txwi_cache __maybe_unused *t;
	int i, ret, fifo_size;
	struct mt76_queue *q;
	void *status_fifo;

	BUILD_BUG_ON(sizeof(struct mt76x02_rxwi) > MT_RX_HEADROOM);

	fifo_size = roundup_pow_of_two(32 * sizeof(struct mt76x02_tx_status));
	status_fifo = devm_kzalloc(dev->mt76.dev, fifo_size, GFP_KERNEL);
	if (!status_fifo)
		return -ENOMEM;

	tasklet_init(&dev->mt76.tx_tasklet, mt76x02_tx_tasklet,
		     (unsigned long)dev);
	tasklet_init(&dev->mt76.pre_tbtt_tasklet, mt76x02_pre_tbtt_tasklet,
		     (unsigned long)dev);

	spin_lock_init(&dev->txstatus_fifo_lock);
	kfifo_init(&dev->txstatus_fifo, status_fifo, fifo_size);

	mt76_dma_attach(&dev->mt76);

	mt76_wr(dev, MT_WPDMA_RST_IDX, ~0);

	for (i = 0; i < IEEE80211_NUM_ACS; i++) {
		ret = mt76x02_init_tx_queue(dev, &dev->mt76.q_tx[i],
					    mt76_ac_to_hwq(i),
					    MT_TX_RING_SIZE);
		if (ret)
			return ret;
	}

	ret = mt76x02_init_tx_queue(dev, &dev->mt76.q_tx[MT_TXQ_PSD],
				    MT_TX_HW_QUEUE_MGMT, MT_TX_RING_SIZE);
	if (ret)
		return ret;

	ret = mt76x02_init_tx_queue(dev, &dev->mt76.q_tx[MT_TXQ_MCU],
				    MT_TX_HW_QUEUE_MCU, MT_MCU_RING_SIZE);
	if (ret)
		return ret;

	ret = mt76x02_init_rx_queue(dev, &dev->mt76.q_rx[MT_RXQ_MCU], 1,
				    MT_MCU_RING_SIZE, MT_RX_BUF_SIZE);
	if (ret)
		return ret;

	q = &dev->mt76.q_rx[MT_RXQ_MAIN];
	q->buf_offset = MT_RX_HEADROOM - sizeof(struct mt76x02_rxwi);
	ret = mt76x02_init_rx_queue(dev, q, 0, MT76X02_RX_RING_SIZE,
				    MT_RX_BUF_SIZE);
	if (ret)
		return ret;

	ret = mt76_init_queues(dev);
	if (ret)
		return ret;

	netif_tx_napi_add(&dev->mt76.napi_dev, &dev->mt76.tx_napi,
			  mt76x02_poll_tx, NAPI_POLL_WEIGHT);
	napi_enable(&dev->mt76.tx_napi);

	return 0;
}
EXPORT_SYMBOL_GPL(mt76x02_dma_init);

void mt76x02_rx_poll_complete(struct mt76_dev *mdev, enum mt76_rxq_id q)
{
	struct mt76x02_dev *dev;

	dev = container_of(mdev, struct mt76x02_dev, mt76);
	mt76x02_irq_enable(dev, MT_INT_RX_DONE(q));
}
EXPORT_SYMBOL_GPL(mt76x02_rx_poll_complete);

irqreturn_t mt76x02_irq_handler(int irq, void *dev_instance)
{
	struct mt76x02_dev *dev = dev_instance;
	u32 intr;

	intr = mt76_rr(dev, MT_INT_SOURCE_CSR);
	mt76_wr(dev, MT_INT_SOURCE_CSR, intr);

	if (!test_bit(MT76_STATE_INITIALIZED, &dev->mt76.state))
		return IRQ_NONE;

	trace_dev_irq(dev, intr, dev->mt76.mmio.irqmask);

	intr &= dev->mt76.mmio.irqmask;

	if (intr & MT_INT_RX_DONE(0)) {
		mt76x02_irq_disable(dev, MT_INT_RX_DONE(0));
		napi_schedule(&dev->mt76.napi[0]);
	}

	if (intr & MT_INT_RX_DONE(1)) {
		mt76x02_irq_disable(dev, MT_INT_RX_DONE(1));
		napi_schedule(&dev->mt76.napi[1]);
	}

	if (intr & MT_INT_PRE_TBTT)
		tasklet_schedule(&dev->mt76.pre_tbtt_tasklet);

	/* send buffered multicast frames now */
	if (intr & MT_INT_TBTT) {
		if (dev->mt76.csa_complete)
			mt76_csa_finish(&dev->mt76);
		else
			mt76_queue_kick(dev, dev->mt76.q_tx[MT_TXQ_PSD].q);
	}

	if (intr & MT_INT_TX_STAT)
		mt76x02_mac_poll_tx_status(dev, true);

	if (intr & (MT_INT_TX_STAT | MT_INT_TX_DONE_ALL)) {
		mt76x02_irq_disable(dev, MT_INT_TX_DONE_ALL);
		napi_schedule(&dev->mt76.tx_napi);
	}

	if (intr & MT_INT_GPTIMER) {
		mt76x02_irq_disable(dev, MT_INT_GPTIMER);
		tasklet_schedule(&dev->dfs_pd.dfs_tasklet);
	}

	return IRQ_HANDLED;
}
EXPORT_SYMBOL_GPL(mt76x02_irq_handler);

static void mt76x02_dma_enable(struct mt76x02_dev *dev)
{
	u32 val;

	mt76_wr(dev, MT_MAC_SYS_CTRL, MT_MAC_SYS_CTRL_ENABLE_TX);
	mt76x02_wait_for_wpdma(&dev->mt76, 1000);
	usleep_range(50, 100);

	val = FIELD_PREP(MT_WPDMA_GLO_CFG_DMA_BURST_SIZE, 3) |
	      MT_WPDMA_GLO_CFG_TX_DMA_EN |
	      MT_WPDMA_GLO_CFG_RX_DMA_EN;
	mt76_set(dev, MT_WPDMA_GLO_CFG, val);
	mt76_clear(dev, MT_WPDMA_GLO_CFG,
		   MT_WPDMA_GLO_CFG_TX_WRITEBACK_DONE);
}

void mt76x02_dma_cleanup(struct mt76x02_dev *dev)
{
	tasklet_kill(&dev->mt76.tx_tasklet);
	mt76_dma_cleanup(&dev->mt76);
}
EXPORT_SYMBOL_GPL(mt76x02_dma_cleanup);

void mt76x02_dma_disable(struct mt76x02_dev *dev)
{
	u32 val = mt76_rr(dev, MT_WPDMA_GLO_CFG);

	val &= MT_WPDMA_GLO_CFG_DMA_BURST_SIZE |
	       MT_WPDMA_GLO_CFG_BIG_ENDIAN |
	       MT_WPDMA_GLO_CFG_HDR_SEG_LEN;
	val |= MT_WPDMA_GLO_CFG_TX_WRITEBACK_DONE;
	mt76_wr(dev, MT_WPDMA_GLO_CFG, val);
}
EXPORT_SYMBOL_GPL(mt76x02_dma_disable);

void mt76x02_mac_start(struct mt76x02_dev *dev)
{
	mt76x02_dma_enable(dev);
	mt76_wr(dev, MT_RX_FILTR_CFG, dev->mt76.rxfilter);
	mt76_wr(dev, MT_MAC_SYS_CTRL,
		MT_MAC_SYS_CTRL_ENABLE_TX |
		MT_MAC_SYS_CTRL_ENABLE_RX);
	mt76x02_irq_enable(dev,
			   MT_INT_RX_DONE_ALL | MT_INT_TX_DONE_ALL |
			   MT_INT_TX_STAT);
}
EXPORT_SYMBOL_GPL(mt76x02_mac_start);

static bool mt76x02_tx_hang(struct mt76x02_dev *dev)
{
	u32 dma_idx, prev_dma_idx;
	struct mt76_queue *q;
	int i;

	for (i = 0; i < 4; i++) {
		q = dev->mt76.q_tx[i].q;

		if (!q->queued)
			continue;

		prev_dma_idx = dev->mt76.tx_dma_idx[i];
		dma_idx = readl(&q->regs->dma_idx);
		dev->mt76.tx_dma_idx[i] = dma_idx;

		if (prev_dma_idx == dma_idx)
			break;
	}

	return i < 4;
}

static void mt76x02_key_sync(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
			     struct ieee80211_sta *sta,
			     struct ieee80211_key_conf *key, void *data)
{
	struct mt76x02_dev *dev = hw->priv;
	struct mt76_wcid *wcid;

	if (!sta)
		return;

	wcid = (struct mt76_wcid *)sta->drv_priv;

	if (wcid->hw_key_idx != key->keyidx || wcid->sw_iv)
		return;

	mt76x02_mac_wcid_sync_pn(dev, wcid->idx, key);
}

static void mt76x02_reset_state(struct mt76x02_dev *dev)
{
	int i;

	lockdep_assert_held(&dev->mt76.mutex);

	clear_bit(MT76_STATE_RUNNING, &dev->mt76.state);

	rcu_read_lock();
	ieee80211_iter_keys_rcu(dev->mt76.hw, NULL, mt76x02_key_sync, NULL);
	rcu_read_unlock();

	for (i = 0; i < ARRAY_SIZE(dev->mt76.wcid); i++) {
		struct ieee80211_sta *sta;
		struct ieee80211_vif *vif;
		struct mt76x02_sta *msta;
		struct mt76_wcid *wcid;
		void *priv;

		wcid = rcu_dereference_protected(dev->mt76.wcid[i],
					lockdep_is_held(&dev->mt76.mutex));
		if (!wcid)
			continue;

		priv = msta = container_of(wcid, struct mt76x02_sta, wcid);
		sta = container_of(priv, struct ieee80211_sta, drv_priv);

		priv = msta->vif;
		vif = container_of(priv, struct ieee80211_vif, drv_priv);

		__mt76_sta_remove(&dev->mt76, vif, sta);
		memset(msta, 0, sizeof(*msta));
	}

	dev->vif_mask = 0;
	dev->mt76.beacon_mask = 0;
}

static void mt76x02_watchdog_reset(struct mt76x02_dev *dev)
{
	u32 mask = dev->mt76.mmio.irqmask;
	bool restart = dev->mt76.mcu_ops->mcu_restart;
	int i;

	ieee80211_stop_queues(dev->mt76.hw);
	set_bit(MT76_RESET, &dev->mt76.state);

	tasklet_disable(&dev->mt76.pre_tbtt_tasklet);
	tasklet_disable(&dev->mt76.tx_tasklet);
	napi_disable(&dev->mt76.tx_napi);

	for (i = 0; i < ARRAY_SIZE(dev->mt76.napi); i++)
		napi_disable(&dev->mt76.napi[i]);

	mutex_lock(&dev->mt76.mutex);

	if (restart)
		mt76x02_reset_state(dev);

	if (dev->mt76.beacon_mask)
		mt76_clear(dev, MT_BEACON_TIME_CFG,
			   MT_BEACON_TIME_CFG_BEACON_TX |
			   MT_BEACON_TIME_CFG_TBTT_EN);

	mt76x02_irq_disable(dev, mask);

	/* perform device reset */
	mt76_clear(dev, MT_TXOP_CTRL_CFG, MT_TXOP_ED_CCA_EN);
	mt76_wr(dev, MT_MAC_SYS_CTRL, 0);
	mt76_clear(dev, MT_WPDMA_GLO_CFG,
		   MT_WPDMA_GLO_CFG_TX_DMA_EN | MT_WPDMA_GLO_CFG_RX_DMA_EN);
	usleep_range(5000, 10000);
	mt76_wr(dev, MT_INT_SOURCE_CSR, 0xffffffff);

	/* let fw reset DMA */
	mt76_set(dev, 0x734, 0x3);

	if (restart)
		mt76_mcu_restart(dev);

	for (i = 0; i < ARRAY_SIZE(dev->mt76.q_tx); i++)
		mt76_queue_tx_cleanup(dev, i, true);

	for (i = 0; i < ARRAY_SIZE(dev->mt76.q_rx); i++)
		mt76_queue_rx_reset(dev, i);

	mt76x02_mac_start(dev);

	if (dev->ed_monitor)
		mt76_set(dev, MT_TXOP_CTRL_CFG, MT_TXOP_ED_CCA_EN);

	if (dev->mt76.beacon_mask && !restart)
		mt76_set(dev, MT_BEACON_TIME_CFG,
			 MT_BEACON_TIME_CFG_BEACON_TX |
			 MT_BEACON_TIME_CFG_TBTT_EN);

	mt76x02_irq_enable(dev, mask);

	mutex_unlock(&dev->mt76.mutex);

	clear_bit(MT76_RESET, &dev->mt76.state);

	tasklet_enable(&dev->mt76.tx_tasklet);
	napi_enable(&dev->mt76.tx_napi);
	napi_schedule(&dev->mt76.tx_napi);

	tasklet_enable(&dev->mt76.pre_tbtt_tasklet);

	for (i = 0; i < ARRAY_SIZE(dev->mt76.napi); i++) {
		napi_enable(&dev->mt76.napi[i]);
		napi_schedule(&dev->mt76.napi[i]);
	}

	if (restart) {
		mt76x02_mcu_function_select(dev, Q_SELECT, 1);
		ieee80211_restart_hw(dev->mt76.hw);
	} else {
		ieee80211_wake_queues(dev->mt76.hw);
		mt76_txq_schedule_all(&dev->mt76);
	}
}

static void mt76x02_check_tx_hang(struct mt76x02_dev *dev)
{
	if (mt76x02_tx_hang(dev)) {
		if (++dev->tx_hang_check >= MT_TX_HANG_TH)
			goto restart;
	} else {
		dev->tx_hang_check = 0;
	}

	if (dev->mcu_timeout)
		goto restart;

	return;

restart:
	mt76x02_watchdog_reset(dev);

	mutex_lock(&dev->mt76.mmio.mcu.mutex);
	dev->mcu_timeout = 0;
	mutex_unlock(&dev->mt76.mmio.mcu.mutex);

	dev->tx_hang_reset++;
	dev->tx_hang_check = 0;
	memset(dev->mt76.tx_dma_idx, 0xff,
	       sizeof(dev->mt76.tx_dma_idx));
}

void mt76x02_wdt_work(struct work_struct *work)
{
	struct mt76x02_dev *dev = container_of(work, struct mt76x02_dev,
					       wdt_work.work);

	mt76x02_check_tx_hang(dev);

	ieee80211_queue_delayed_work(mt76_hw(dev), &dev->wdt_work,
				     MT_WATCHDOG_TIME);
}
Commit	Line	Data
0e3d6777	1	// SPDX-License-Identifier: ISC
20885649 LB	2	/*
	3	* Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
	4	* Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com>
20885649 LB	5	*/
	6
	7	#include <linux/kernel.h>
9b43960b	8	#include <linux/irq.h>
20885649	9
7a07adcd	10	#include "mt76x02.h"
00496042	11	#include "mt76x02_mcu.h"
9b43960b	12	#include "mt76x02_trace.h"
b2eabd4c	13
82b5c239	14	static void mt76x02_pre_tbtt_tasklet(unsigned long arg)
dfe6c80c LB	15	{
dfe6c80c LB	16	struct mt76x02_dev dev = (struct mt76x02_dev )arg;
af005f26	17	struct mt76_queue *q = dev->mt76.q_tx[MT_TXQ_PSD].q;
dfe6c80c LB	18	struct beacon_bc_data data = {};
dfe6c80c LB	19	struct sk_buff *skb;
31cdd442	20	int i;
dfe6c80c	21
ae66068f LB	22	if (mt76_hw(dev)->conf.flags & IEEE80211_CONF_OFFCHANNEL)
	23	return;
	24
dfe6c80c LB	25	mt76x02_resync_beacon_timer(dev);
dfe6c80c LB	26
dfe6c80c LB	27	ieee80211_iterate_active_interfaces_atomic(mt76_hw(dev),
	28	IEEE80211_IFACE_ITER_RESUME_ALL,
	29	mt76x02_update_beacon_iter, dev);
	30
e7173858 FF	31	mt76_csa_check(&dev->mt76);
	32
	33	if (dev->mt76.csa_complete)
	34	return;
	35
31cdd442	36	mt76x02_enqueue_buffered_bc(dev, &data, 8);
dfe6c80c	37
cbbfd737	38	if (!skb_queue_len(&data.q))
dfe6c80c LB	39	return;
	40
	41	for (i = 0; i < ARRAY_SIZE(data.tail); i++) {
	42	if (!data.tail[i])
	43	continue;
	44
	45	mt76_skb_set_moredata(data.tail[i], false);
	46	}
	47
	48	spin_lock_bh(&q->lock);
	49	while ((skb = __skb_dequeue(&data.q)) != NULL) {
	50	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
	51	struct ieee80211_vif *vif = info->control.vif;
	52	struct mt76x02_vif mvif = (struct mt76x02_vif )vif->drv_priv;
	53
eb9ca7ec LB	54	mt76_tx_queue_skb(dev, MT_TXQ_PSD, skb, &mvif->group_wcid,
eb9ca7ec LB	55	NULL);
dfe6c80c LB	56	}
	57	spin_unlock_bh(&q->lock);
	58	}
dfe6c80c	59
c004b881 SG	60	static void mt76x02e_pre_tbtt_enable(struct mt76x02_dev *dev, bool en)
	61	{
	62	if (en)
dc6057f4	63	tasklet_enable(&dev->mt76.pre_tbtt_tasklet);
c004b881	64	else
dc6057f4	65	tasklet_disable(&dev->mt76.pre_tbtt_tasklet);
c004b881 SG	66	}
	67
	68	static void mt76x02e_beacon_enable(struct mt76x02_dev *dev, bool en)
	69	{
	70	mt76_rmw_field(dev, MT_INT_TIMER_EN, MT_INT_TIMER_EN_PRE_TBTT_EN, en);
	71	if (en)
	72	mt76x02_irq_enable(dev, MT_INT_PRE_TBTT \| MT_INT_TBTT);
	73	else
	74	mt76x02_irq_disable(dev, MT_INT_PRE_TBTT \| MT_INT_TBTT);
	75	}
	76
8d71aef9 SG	77	void mt76x02e_init_beacon_config(struct mt76x02_dev *dev)
8d71aef9 SG	78	{
c004b881	79	static const struct mt76x02_beacon_ops beacon_ops = {
f2276c29 SG	80	.nslots = 8,
f2276c29 SG	81	.slot_size = 1024,
c004b881 SG	82	.pre_tbtt_enable = mt76x02e_pre_tbtt_enable,
	83	.beacon_enable = mt76x02e_beacon_enable,
	84	};
	85
	86	dev->beacon_ops = &beacon_ops;
	87
8d71aef9	88	/* Fire a pre-TBTT interrupt 8 ms before TBTT */
ff97c52a RL	89	mt76_rmw_field(dev, MT_INT_TIMER_CFG, MT_INT_TIMER_CFG_PRE_TBTT,
ff97c52a RL	90	8 << 4);
8d71aef9 SG	91	mt76_rmw_field(dev, MT_INT_TIMER_CFG, MT_INT_TIMER_CFG_GP_TIMER,
	92	MT_DFS_GP_INTERVAL);
	93	mt76_wr(dev, MT_INT_TIMER_EN, 0);
	94
	95	mt76x02_init_beacon_config(dev);
	96	}
	97	EXPORT_SYMBOL_GPL(mt76x02e_init_beacon_config);
	98
b2eabd4c	99	static int
af005f26	100	mt76x02_init_tx_queue(struct mt76x02_dev dev, struct mt76_sw_queue q,
b2eabd4c LB	101	int idx, int n_desc)
b2eabd4c LB	102	{
af005f26	103	struct mt76_queue *hwq;
b1bfbe70	104	int err;
b2eabd4c	105
af005f26 LB	106	hwq = devm_kzalloc(dev->mt76.dev, sizeof(*hwq), GFP_KERNEL);
	107	if (!hwq)
	108	return -ENOMEM;
	109
	110	err = mt76_queue_alloc(dev, hwq, idx, n_desc, 0, MT_TX_RING_BASE);
b1bfbe70 LB	111	if (err < 0)
b1bfbe70 LB	112	return err;
b2eabd4c	113
af005f26 LB	114	INIT_LIST_HEAD(&q->swq);
	115	q->q = hwq;
	116
b2eabd4c LB	117	mt76x02_irq_enable(dev, MT_INT_TX_DONE(idx));
	118
	119	return 0;
	120	}
	121
	122	static int
a23fde09	123	mt76x02_init_rx_queue(struct mt76x02_dev dev, struct mt76_queue q,
b2eabd4c LB	124	int idx, int n_desc, int bufsize)
b2eabd4c LB	125	{
b1bfbe70	126	int err;
b2eabd4c	127
b1bfbe70 LB	128	err = mt76_queue_alloc(dev, q, idx, n_desc, bufsize,
	129	MT_RX_RING_BASE);
	130	if (err < 0)
	131	return err;
b2eabd4c LB	132
	133	mt76x02_irq_enable(dev, MT_INT_RX_DONE(idx));
	134
	135	return 0;
	136	}
	137
53d20fdb LB	138	static void mt76x02_process_tx_status_fifo(struct mt76x02_dev *dev)
	139	{
	140	struct mt76x02_tx_status stat;
	141	u8 update = 1;
	142
	143	while (kfifo_get(&dev->txstatus_fifo, &stat))
8d66af49	144	mt76x02_send_tx_status(dev, &stat, &update);
53d20fdb LB	145	}
	146
	147	static void mt76x02_tx_tasklet(unsigned long data)
	148	{
	149	struct mt76x02_dev dev = (struct mt76x02_dev )data;
53d20fdb	150
2ac515a5	151	mt76x02_mac_poll_tx_status(dev, false);
53d20fdb LB	152	mt76x02_process_tx_status_fifo(dev);
53d20fdb LB	153
2ac515a5 FF	154	mt76_txq_schedule_all(&dev->mt76);
	155	}
	156
9c7c756e	157	static int mt76x02_poll_tx(struct napi_struct *napi, int budget)
2ac515a5	158	{
8402650a LB	159	struct mt76x02_dev *dev = container_of(napi, struct mt76x02_dev,
8402650a LB	160	mt76.tx_napi);
2ac515a5 FF	161	int i;
	162
	163	mt76x02_mac_poll_tx_status(dev, false);
	164
53d20fdb LB	165	for (i = MT_TXQ_MCU; i >= 0; i--)
	166	mt76_queue_tx_cleanup(dev, i, false);
	167
2ac515a5 FF	168	if (napi_complete_done(napi, 0))
2ac515a5 FF	169	mt76x02_irq_enable(dev, MT_INT_TX_DONE_ALL);
41634aa8	170
2ac515a5 FF	171	for (i = MT_TXQ_MCU; i >= 0; i--)
2ac515a5 FF	172	mt76_queue_tx_cleanup(dev, i, false);
41634aa8	173
2ac515a5 FF	174	tasklet_schedule(&dev->mt76.tx_tasklet);
	175
	176	return 0;
53d20fdb LB	177	}
53d20fdb LB	178
a23fde09	179	int mt76x02_dma_init(struct mt76x02_dev *dev)
b2eabd4c LB	180	{
b2eabd4c LB	181	struct mt76_txwi_cache __maybe_unused *t;
53d20fdb	182	int i, ret, fifo_size;
b2eabd4c	183	struct mt76_queue *q;
53d20fdb	184	void *status_fifo;
b2eabd4c	185
b2eabd4c LB	186	BUILD_BUG_ON(sizeof(struct mt76x02_rxwi) > MT_RX_HEADROOM);
b2eabd4c LB	187
53d20fdb LB	188	fifo_size = roundup_pow_of_two(32 * sizeof(struct mt76x02_tx_status));
	189	status_fifo = devm_kzalloc(dev->mt76.dev, fifo_size, GFP_KERNEL);
	190	if (!status_fifo)
	191	return -ENOMEM;
	192
a33b8ab8	193	tasklet_init(&dev->mt76.tx_tasklet, mt76x02_tx_tasklet,
ff97c52a	194	(unsigned long)dev);
dc6057f4	195	tasklet_init(&dev->mt76.pre_tbtt_tasklet, mt76x02_pre_tbtt_tasklet,
82b5c239 LB	196	(unsigned long)dev);
82b5c239 LB	197
6fe53337	198	spin_lock_init(&dev->txstatus_fifo_lock);
53d20fdb LB	199	kfifo_init(&dev->txstatus_fifo, status_fifo, fifo_size);
53d20fdb LB	200
a23fde09	201	mt76_dma_attach(&dev->mt76);
53d20fdb	202
a23fde09	203	mt76_wr(dev, MT_WPDMA_RST_IDX, ~0);
b2eabd4c LB	204
b2eabd4c LB	205	for (i = 0; i < IEEE80211_NUM_ACS; i++) {
a23fde09	206	ret = mt76x02_init_tx_queue(dev, &dev->mt76.q_tx[i],
b2eabd4c LB	207	mt76_ac_to_hwq(i),
	208	MT_TX_RING_SIZE);
	209	if (ret)
	210	return ret;
	211	}
	212
a23fde09	213	ret = mt76x02_init_tx_queue(dev, &dev->mt76.q_tx[MT_TXQ_PSD],
b2eabd4c LB	214	MT_TX_HW_QUEUE_MGMT, MT_TX_RING_SIZE);
	215	if (ret)
	216	return ret;
	217
a23fde09	218	ret = mt76x02_init_tx_queue(dev, &dev->mt76.q_tx[MT_TXQ_MCU],
b2eabd4c LB	219	MT_TX_HW_QUEUE_MCU, MT_MCU_RING_SIZE);
	220	if (ret)
	221	return ret;
	222
a23fde09	223	ret = mt76x02_init_rx_queue(dev, &dev->mt76.q_rx[MT_RXQ_MCU], 1,
b2eabd4c LB	224	MT_MCU_RING_SIZE, MT_RX_BUF_SIZE);
	225	if (ret)
	226	return ret;
	227
a23fde09	228	q = &dev->mt76.q_rx[MT_RXQ_MAIN];
b2eabd4c LB	229	q->buf_offset = MT_RX_HEADROOM - sizeof(struct mt76x02_rxwi);
	230	ret = mt76x02_init_rx_queue(dev, q, 0, MT76X02_RX_RING_SIZE,
	231	MT_RX_BUF_SIZE);
	232	if (ret)
	233	return ret;
	234
2ac515a5 FF	235	ret = mt76_init_queues(dev);
	236	if (ret)
	237	return ret;
	238
8402650a LB	239	netif_tx_napi_add(&dev->mt76.napi_dev, &dev->mt76.tx_napi,
	240	mt76x02_poll_tx, NAPI_POLL_WEIGHT);
	241	napi_enable(&dev->mt76.tx_napi);
2ac515a5 FF	242
2ac515a5 FF	243	return 0;
b2eabd4c LB	244	}
b2eabd4c LB	245	EXPORT_SYMBOL_GPL(mt76x02_dma_init);
20885649	246
9b43960b LB	247	void mt76x02_rx_poll_complete(struct mt76_dev *mdev, enum mt76_rxq_id q)
	248	{
	249	struct mt76x02_dev *dev;
	250
	251	dev = container_of(mdev, struct mt76x02_dev, mt76);
	252	mt76x02_irq_enable(dev, MT_INT_RX_DONE(q));
	253	}
	254	EXPORT_SYMBOL_GPL(mt76x02_rx_poll_complete);
	255
	256	irqreturn_t mt76x02_irq_handler(int irq, void *dev_instance)
	257	{
	258	struct mt76x02_dev *dev = dev_instance;
	259	u32 intr;
	260
	261	intr = mt76_rr(dev, MT_INT_SOURCE_CSR);
	262	mt76_wr(dev, MT_INT_SOURCE_CSR, intr);
	263
	264	if (!test_bit(MT76_STATE_INITIALIZED, &dev->mt76.state))
	265	return IRQ_NONE;
	266
	267	trace_dev_irq(dev, intr, dev->mt76.mmio.irqmask);
	268
	269	intr &= dev->mt76.mmio.irqmask;
	270
9b43960b LB	271	if (intr & MT_INT_RX_DONE(0)) {
	272	mt76x02_irq_disable(dev, MT_INT_RX_DONE(0));
	273	napi_schedule(&dev->mt76.napi[0]);
	274	}
	275
	276	if (intr & MT_INT_RX_DONE(1)) {
	277	mt76x02_irq_disable(dev, MT_INT_RX_DONE(1));
	278	napi_schedule(&dev->mt76.napi[1]);
	279	}
	280
	281	if (intr & MT_INT_PRE_TBTT)
dc6057f4	282	tasklet_schedule(&dev->mt76.pre_tbtt_tasklet);
9b43960b LB	283
9b43960b LB	284	/* send buffered multicast frames now */
e7173858 FF	285	if (intr & MT_INT_TBTT) {
	286	if (dev->mt76.csa_complete)
	287	mt76_csa_finish(&dev->mt76);
	288	else
af005f26	289	mt76_queue_kick(dev, dev->mt76.q_tx[MT_TXQ_PSD].q);
e7173858	290	}
9b43960b	291
2ac515a5	292	if (intr & MT_INT_TX_STAT)
9b43960b	293	mt76x02_mac_poll_tx_status(dev, true);
2ac515a5 FF	294
	295	if (intr & (MT_INT_TX_STAT \| MT_INT_TX_DONE_ALL)) {
	296	mt76x02_irq_disable(dev, MT_INT_TX_DONE_ALL);
8402650a	297	napi_schedule(&dev->mt76.tx_napi);
9b43960b LB	298	}
	299
	300	if (intr & MT_INT_GPTIMER) {
	301	mt76x02_irq_disable(dev, MT_INT_GPTIMER);
	302	tasklet_schedule(&dev->dfs_pd.dfs_tasklet);
	303	}
	304
	305	return IRQ_HANDLED;
	306	}
	307	EXPORT_SYMBOL_GPL(mt76x02_irq_handler);
	308
a23fde09	309	static void mt76x02_dma_enable(struct mt76x02_dev *dev)
20885649 LB	310	{
	311	u32 val;
	312
a23fde09 LB	313	mt76_wr(dev, MT_MAC_SYS_CTRL, MT_MAC_SYS_CTRL_ENABLE_TX);
a23fde09 LB	314	mt76x02_wait_for_wpdma(&dev->mt76, 1000);
20885649 LB	315	usleep_range(50, 100);
	316
	317	val = FIELD_PREP(MT_WPDMA_GLO_CFG_DMA_BURST_SIZE, 3) \|
	318	MT_WPDMA_GLO_CFG_TX_DMA_EN \|
	319	MT_WPDMA_GLO_CFG_RX_DMA_EN;
a23fde09 LB	320	mt76_set(dev, MT_WPDMA_GLO_CFG, val);
	321	mt76_clear(dev, MT_WPDMA_GLO_CFG,
	322	MT_WPDMA_GLO_CFG_TX_WRITEBACK_DONE);
20885649	323	}
20885649	324
53d20fdb LB	325	void mt76x02_dma_cleanup(struct mt76x02_dev *dev)
53d20fdb LB	326	{
a33b8ab8	327	tasklet_kill(&dev->mt76.tx_tasklet);
53d20fdb LB	328	mt76_dma_cleanup(&dev->mt76);
	329	}
	330	EXPORT_SYMBOL_GPL(mt76x02_dma_cleanup);
	331
a23fde09	332	void mt76x02_dma_disable(struct mt76x02_dev *dev)
20885649	333	{
a23fde09	334	u32 val = mt76_rr(dev, MT_WPDMA_GLO_CFG);
20885649 LB	335
	336	val &= MT_WPDMA_GLO_CFG_DMA_BURST_SIZE \|
	337	MT_WPDMA_GLO_CFG_BIG_ENDIAN \|
	338	MT_WPDMA_GLO_CFG_HDR_SEG_LEN;
	339	val \|= MT_WPDMA_GLO_CFG_TX_WRITEBACK_DONE;
a23fde09	340	mt76_wr(dev, MT_WPDMA_GLO_CFG, val);
20885649 LB	341	}
20885649 LB	342	EXPORT_SYMBOL_GPL(mt76x02_dma_disable);
9f04eb7b	343
a23fde09	344	void mt76x02_mac_start(struct mt76x02_dev *dev)
9f04eb7b LB	345	{
9f04eb7b LB	346	mt76x02_dma_enable(dev);
a23fde09 LB	347	mt76_wr(dev, MT_RX_FILTR_CFG, dev->mt76.rxfilter);
	348	mt76_wr(dev, MT_MAC_SYS_CTRL,
	349	MT_MAC_SYS_CTRL_ENABLE_TX \|
	350	MT_MAC_SYS_CTRL_ENABLE_RX);
9f04eb7b LB	351	mt76x02_irq_enable(dev,
	352	MT_INT_RX_DONE_ALL \| MT_INT_TX_DONE_ALL \|
	353	MT_INT_TX_STAT);
	354	}
	355	EXPORT_SYMBOL_GPL(mt76x02_mac_start);
c1e0d2be LB	356
	357	static bool mt76x02_tx_hang(struct mt76x02_dev *dev)
	358	{
	359	u32 dma_idx, prev_dma_idx;
	360	struct mt76_queue *q;
	361	int i;
	362
	363	for (i = 0; i < 4; i++) {
af005f26	364	q = dev->mt76.q_tx[i].q;
c1e0d2be LB	365
	366	if (!q->queued)
	367	continue;
	368
	369	prev_dma_idx = dev->mt76.tx_dma_idx[i];
d908d4ec	370	dma_idx = readl(&q->regs->dma_idx);
c1e0d2be LB	371	dev->mt76.tx_dma_idx[i] = dma_idx;
	372
	373	if (prev_dma_idx == dma_idx)
	374	break;
	375	}
	376
	377	return i < 4;
	378	}
	379
00496042 FF	380	static void mt76x02_key_sync(struct ieee80211_hw hw, struct ieee80211_vif vif,
	381	struct ieee80211_sta *sta,
	382	struct ieee80211_key_conf key, void data)
	383	{
	384	struct mt76x02_dev *dev = hw->priv;
	385	struct mt76_wcid *wcid;
	386
	387	if (!sta)
ff97c52a	388	return;
00496042	389
ff97c52a	390	wcid = (struct mt76_wcid *)sta->drv_priv;
00496042 FF	391
00496042 FF	392	if (wcid->hw_key_idx != key->keyidx \|\| wcid->sw_iv)
ff97c52a	393	return;
00496042 FF	394
	395	mt76x02_mac_wcid_sync_pn(dev, wcid->idx, key);
	396	}
	397
	398	static void mt76x02_reset_state(struct mt76x02_dev *dev)
	399	{
	400	int i;
	401
13f61dfc LB	402	lockdep_assert_held(&dev->mt76.mutex);
13f61dfc LB	403
00496042 FF	404	clear_bit(MT76_STATE_RUNNING, &dev->mt76.state);
	405
	406	rcu_read_lock();
00496042	407	ieee80211_iter_keys_rcu(dev->mt76.hw, NULL, mt76x02_key_sync, NULL);
13f61dfc	408	rcu_read_unlock();
00496042 FF	409
00496042 FF	410	for (i = 0; i < ARRAY_SIZE(dev->mt76.wcid); i++) {
00496042 FF	411	struct ieee80211_sta *sta;
00496042 FF	412	struct ieee80211_vif *vif;
13f61dfc LB	413	struct mt76x02_sta *msta;
13f61dfc LB	414	struct mt76_wcid *wcid;
00496042 FF	415	void *priv;
00496042 FF	416
13f61dfc LB	417	wcid = rcu_dereference_protected(dev->mt76.wcid[i],
13f61dfc LB	418	lockdep_is_held(&dev->mt76.mutex));
00496042 FF	419	if (!wcid)
	420	continue;
	421
	422	priv = msta = container_of(wcid, struct mt76x02_sta, wcid);
	423	sta = container_of(priv, struct ieee80211_sta, drv_priv);
	424
	425	priv = msta->vif;
	426	vif = container_of(priv, struct ieee80211_vif, drv_priv);
	427
13f61dfc	428	__mt76_sta_remove(&dev->mt76, vif, sta);
00496042 FF	429	memset(msta, 0, sizeof(*msta));
	430	}
	431
00496042	432	dev->vif_mask = 0;
c8a04d98	433	dev->mt76.beacon_mask = 0;
00496042 FF	434	}
00496042 FF	435
c1e0d2be LB	436	static void mt76x02_watchdog_reset(struct mt76x02_dev *dev)
	437	{
	438	u32 mask = dev->mt76.mmio.irqmask;
00496042	439	bool restart = dev->mt76.mcu_ops->mcu_restart;
c1e0d2be LB	440	int i;
	441
	442	ieee80211_stop_queues(dev->mt76.hw);
	443	set_bit(MT76_RESET, &dev->mt76.state);
	444
dc6057f4	445	tasklet_disable(&dev->mt76.pre_tbtt_tasklet);
a33b8ab8	446	tasklet_disable(&dev->mt76.tx_tasklet);
8402650a	447	napi_disable(&dev->mt76.tx_napi);
c1e0d2be LB	448
	449	for (i = 0; i < ARRAY_SIZE(dev->mt76.napi); i++)
	450	napi_disable(&dev->mt76.napi[i]);
	451
13f61dfc LB	452	mutex_lock(&dev->mt76.mutex);
13f61dfc LB	453
00496042 FF	454	if (restart)
	455	mt76x02_reset_state(dev);
	456
c8a04d98	457	if (dev->mt76.beacon_mask)
c1e0d2be LB	458	mt76_clear(dev, MT_BEACON_TIME_CFG,
	459	MT_BEACON_TIME_CFG_BEACON_TX \|
	460	MT_BEACON_TIME_CFG_TBTT_EN);
	461
	462	mt76x02_irq_disable(dev, mask);
	463
	464	/* perform device reset */
	465	mt76_clear(dev, MT_TXOP_CTRL_CFG, MT_TXOP_ED_CCA_EN);
	466	mt76_wr(dev, MT_MAC_SYS_CTRL, 0);
	467	mt76_clear(dev, MT_WPDMA_GLO_CFG,
	468	MT_WPDMA_GLO_CFG_TX_DMA_EN \| MT_WPDMA_GLO_CFG_RX_DMA_EN);
	469	usleep_range(5000, 10000);
	470	mt76_wr(dev, MT_INT_SOURCE_CSR, 0xffffffff);
	471
	472	/* let fw reset DMA */
	473	mt76_set(dev, 0x734, 0x3);
	474
00496042	475	if (restart)
e2c2fd0f	476	mt76_mcu_restart(dev);
00496042	477
c1e0d2be LB	478	for (i = 0; i < ARRAY_SIZE(dev->mt76.q_tx); i++)
	479	mt76_queue_tx_cleanup(dev, i, true);
	480
	481	for (i = 0; i < ARRAY_SIZE(dev->mt76.q_rx); i++)
	482	mt76_queue_rx_reset(dev, i);
	483
00496042 FF	484	mt76x02_mac_start(dev);
00496042 FF	485
c1e0d2be LB	486	if (dev->ed_monitor)
	487	mt76_set(dev, MT_TXOP_CTRL_CFG, MT_TXOP_ED_CCA_EN);
	488
c8a04d98	489	if (dev->mt76.beacon_mask && !restart)
c1e0d2be LB	490	mt76_set(dev, MT_BEACON_TIME_CFG,
	491	MT_BEACON_TIME_CFG_BEACON_TX \|
	492	MT_BEACON_TIME_CFG_TBTT_EN);
	493
	494	mt76x02_irq_enable(dev, mask);
	495
	496	mutex_unlock(&dev->mt76.mutex);
	497
	498	clear_bit(MT76_RESET, &dev->mt76.state);
	499
a33b8ab8	500	tasklet_enable(&dev->mt76.tx_tasklet);
8402650a LB	501	napi_enable(&dev->mt76.tx_napi);
8402650a LB	502	napi_schedule(&dev->mt76.tx_napi);
c1e0d2be	503
dc6057f4	504	tasklet_enable(&dev->mt76.pre_tbtt_tasklet);
c1e0d2be LB	505
	506	for (i = 0; i < ARRAY_SIZE(dev->mt76.napi); i++) {
	507	napi_enable(&dev->mt76.napi[i]);
	508	napi_schedule(&dev->mt76.napi[i]);
	509	}
	510
00496042 FF	511	if (restart) {
	512	mt76x02_mcu_function_select(dev, Q_SELECT, 1);
	513	ieee80211_restart_hw(dev->mt76.hw);
	514	} else {
	515	ieee80211_wake_queues(dev->mt76.hw);
	516	mt76_txq_schedule_all(&dev->mt76);
	517	}
c1e0d2be LB	518	}
	519
	520	static void mt76x02_check_tx_hang(struct mt76x02_dev *dev)
	521	{
	522	if (mt76x02_tx_hang(dev)) {
72e5d479 FF	523	if (++dev->tx_hang_check >= MT_TX_HANG_TH)
72e5d479 FF	524	goto restart;
c1e0d2be LB	525	} else {
	526	dev->tx_hang_check = 0;
	527	}
72e5d479 FF	528
	529	if (dev->mcu_timeout)
	530	goto restart;
	531
	532	return;
	533
	534	restart:
	535	mt76x02_watchdog_reset(dev);
	536
	537	mutex_lock(&dev->mt76.mmio.mcu.mutex);
	538	dev->mcu_timeout = 0;
	539	mutex_unlock(&dev->mt76.mmio.mcu.mutex);
	540
	541	dev->tx_hang_reset++;
	542	dev->tx_hang_check = 0;
	543	memset(dev->mt76.tx_dma_idx, 0xff,
	544	sizeof(dev->mt76.tx_dma_idx));
c1e0d2be LB	545	}
	546
	547	void mt76x02_wdt_work(struct work_struct *work)
	548	{
	549	struct mt76x02_dev *dev = container_of(work, struct mt76x02_dev,
	550	wdt_work.work);
	551
	552	mt76x02_check_tx_hang(dev);
	553
	554	ieee80211_queue_delayed_work(mt76_hw(dev), &dev->wdt_work,
	555	MT_WATCHDOG_TIME);
	556	}