1 // SPDX-License-Identifier: ISC
3 * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
6 #include <linux/dma-mapping.h>
10 #if IS_ENABLED(CONFIG_NET_MEDIATEK_SOC_WED)
12 #define Q_READ(_dev, _q, _field) ({ \
13 u32 _offset = offsetof(struct mt76_queue_regs, _field); \
15 if ((_q)->flags & MT_QFLAG_WED) \
16 _val = mtk_wed_device_reg_read(&(_dev)->mmio.wed, \
20 _val = readl(&(_q)->regs->_field); \
24 #define Q_WRITE(_dev, _q, _field, _val) do { \
25 u32 _offset = offsetof(struct mt76_queue_regs, _field); \
26 if ((_q)->flags & MT_QFLAG_WED) \
27 mtk_wed_device_reg_write(&(_dev)->mmio.wed, \
28 ((_q)->wed_regs + _offset), \
31 writel(_val, &(_q)->regs->_field); \
36 #define Q_READ(_dev, _q, _field) readl(&(_q)->regs->_field)
37 #define Q_WRITE(_dev, _q, _field, _val) writel(_val, &(_q)->regs->_field)
41 static struct mt76_txwi_cache *
42 mt76_alloc_txwi(struct mt76_dev *dev)
44 struct mt76_txwi_cache *t;
49 size = L1_CACHE_ALIGN(dev->drv->txwi_size + sizeof(*t));
50 txwi = kzalloc(size, GFP_ATOMIC);
54 addr = dma_map_single(dev->dma_dev, txwi, dev->drv->txwi_size,
56 t = (struct mt76_txwi_cache *)(txwi + dev->drv->txwi_size);
62 static struct mt76_txwi_cache *
63 mt76_alloc_rxwi(struct mt76_dev *dev)
65 struct mt76_txwi_cache *t;
67 t = kzalloc(L1_CACHE_ALIGN(sizeof(*t)), GFP_ATOMIC);
75 static struct mt76_txwi_cache *
76 __mt76_get_txwi(struct mt76_dev *dev)
78 struct mt76_txwi_cache *t = NULL;
80 spin_lock(&dev->lock);
81 if (!list_empty(&dev->txwi_cache)) {
82 t = list_first_entry(&dev->txwi_cache, struct mt76_txwi_cache,
86 spin_unlock(&dev->lock);
91 static struct mt76_txwi_cache *
92 __mt76_get_rxwi(struct mt76_dev *dev)
94 struct mt76_txwi_cache *t = NULL;
96 spin_lock(&dev->wed_lock);
97 if (!list_empty(&dev->rxwi_cache)) {
98 t = list_first_entry(&dev->rxwi_cache, struct mt76_txwi_cache,
102 spin_unlock(&dev->wed_lock);
107 static struct mt76_txwi_cache *
108 mt76_get_txwi(struct mt76_dev *dev)
110 struct mt76_txwi_cache *t = __mt76_get_txwi(dev);
115 return mt76_alloc_txwi(dev);
118 struct mt76_txwi_cache *
119 mt76_get_rxwi(struct mt76_dev *dev)
121 struct mt76_txwi_cache *t = __mt76_get_rxwi(dev);
126 return mt76_alloc_rxwi(dev);
128 EXPORT_SYMBOL_GPL(mt76_get_rxwi);
131 mt76_put_txwi(struct mt76_dev *dev, struct mt76_txwi_cache *t)
136 spin_lock(&dev->lock);
137 list_add(&t->list, &dev->txwi_cache);
138 spin_unlock(&dev->lock);
140 EXPORT_SYMBOL_GPL(mt76_put_txwi);
143 mt76_put_rxwi(struct mt76_dev *dev, struct mt76_txwi_cache *t)
148 spin_lock(&dev->wed_lock);
149 list_add(&t->list, &dev->rxwi_cache);
150 spin_unlock(&dev->wed_lock);
152 EXPORT_SYMBOL_GPL(mt76_put_rxwi);
155 mt76_free_pending_txwi(struct mt76_dev *dev)
157 struct mt76_txwi_cache *t;
160 while ((t = __mt76_get_txwi(dev)) != NULL) {
161 dma_unmap_single(dev->dma_dev, t->dma_addr, dev->drv->txwi_size,
163 kfree(mt76_get_txwi_ptr(dev, t));
169 mt76_free_pending_rxwi(struct mt76_dev *dev)
171 struct mt76_txwi_cache *t;
174 while ((t = __mt76_get_rxwi(dev)) != NULL) {
176 skb_free_frag(t->ptr);
183 mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q)
185 Q_WRITE(dev, q, desc_base, q->desc_dma);
186 Q_WRITE(dev, q, ring_size, q->ndesc);
187 q->head = Q_READ(dev, q, dma_idx);
192 mt76_dma_queue_reset(struct mt76_dev *dev, struct mt76_queue *q)
199 /* clear descriptors */
200 for (i = 0; i < q->ndesc; i++)
201 q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE);
203 Q_WRITE(dev, q, cpu_idx, 0);
204 Q_WRITE(dev, q, dma_idx, 0);
205 mt76_dma_sync_idx(dev, q);
209 mt76_dma_add_buf(struct mt76_dev *dev, struct mt76_queue *q,
210 struct mt76_queue_buf *buf, int nbufs, u32 info,
211 struct sk_buff *skb, void *txwi)
213 struct mt76_queue_entry *entry;
214 struct mt76_desc *desc;
218 for (i = 0; i < nbufs; i += 2, buf += 2) {
219 u32 buf0 = buf[0].addr, buf1 = 0;
222 next = (q->head + 1) % q->ndesc;
224 desc = &q->desc[idx];
225 entry = &q->entry[idx];
227 if ((q->flags & MT_QFLAG_WED) &&
228 FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) == MT76_WED_Q_RX) {
229 struct mt76_txwi_cache *t = txwi;
235 rx_token = mt76_rx_token_consume(dev, (void *)skb, t,
240 buf1 |= FIELD_PREP(MT_DMA_CTL_TOKEN, rx_token);
241 ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len) |
245 q->entry[next].txwi = DMA_DUMMY_DATA;
246 q->entry[next].skip_buf0 = true;
249 if (buf[0].skip_unmap)
250 entry->skip_buf0 = true;
251 entry->skip_buf1 = i == nbufs - 1;
253 entry->dma_addr[0] = buf[0].addr;
254 entry->dma_len[0] = buf[0].len;
256 ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len);
258 entry->dma_addr[1] = buf[1].addr;
259 entry->dma_len[1] = buf[1].len;
261 ctrl |= FIELD_PREP(MT_DMA_CTL_SD_LEN1, buf[1].len);
262 if (buf[1].skip_unmap)
263 entry->skip_buf1 = true;
267 ctrl |= MT_DMA_CTL_LAST_SEC0;
268 else if (i == nbufs - 2)
269 ctrl |= MT_DMA_CTL_LAST_SEC1;
272 WRITE_ONCE(desc->buf0, cpu_to_le32(buf0));
273 WRITE_ONCE(desc->buf1, cpu_to_le32(buf1));
274 WRITE_ONCE(desc->info, cpu_to_le32(info));
275 WRITE_ONCE(desc->ctrl, cpu_to_le32(ctrl));
281 q->entry[idx].txwi = txwi;
282 q->entry[idx].skb = skb;
283 q->entry[idx].wcid = 0xffff;
289 mt76_dma_tx_cleanup_idx(struct mt76_dev *dev, struct mt76_queue *q, int idx,
290 struct mt76_queue_entry *prev_e)
292 struct mt76_queue_entry *e = &q->entry[idx];
295 dma_unmap_single(dev->dma_dev, e->dma_addr[0], e->dma_len[0],
299 dma_unmap_single(dev->dma_dev, e->dma_addr[1], e->dma_len[1],
302 if (e->txwi == DMA_DUMMY_DATA)
305 if (e->skb == DMA_DUMMY_DATA)
309 memset(e, 0, sizeof(*e));
313 mt76_dma_kick_queue(struct mt76_dev *dev, struct mt76_queue *q)
316 Q_WRITE(dev, q, cpu_idx, q->head);
320 mt76_dma_tx_cleanup(struct mt76_dev *dev, struct mt76_queue *q, bool flush)
322 struct mt76_queue_entry entry;
328 spin_lock_bh(&q->cleanup_lock);
332 last = Q_READ(dev, q, dma_idx);
334 while (q->queued > 0 && q->tail != last) {
335 mt76_dma_tx_cleanup_idx(dev, q, q->tail, &entry);
336 mt76_queue_tx_complete(dev, q, &entry);
339 if (!(dev->drv->drv_flags & MT_DRV_TXWI_NO_FREE))
340 mt76_put_txwi(dev, entry.txwi);
343 if (!flush && q->tail == last)
344 last = Q_READ(dev, q, dma_idx);
346 spin_unlock_bh(&q->cleanup_lock);
349 spin_lock_bh(&q->lock);
350 mt76_dma_sync_idx(dev, q);
351 mt76_dma_kick_queue(dev, q);
352 spin_unlock_bh(&q->lock);
356 wake_up(&dev->tx_wait);
360 mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx,
361 int *len, u32 *info, bool *more, bool *drop)
363 struct mt76_queue_entry *e = &q->entry[idx];
364 struct mt76_desc *desc = &q->desc[idx];
368 u32 ctrl = le32_to_cpu(READ_ONCE(desc->ctrl));
369 *len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctrl);
370 *more = !(ctrl & MT_DMA_CTL_LAST_SEC0);
374 *info = le32_to_cpu(desc->info);
376 if ((q->flags & MT_QFLAG_WED) &&
377 FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) == MT76_WED_Q_RX) {
378 u32 token = FIELD_GET(MT_DMA_CTL_TOKEN,
379 le32_to_cpu(desc->buf1));
380 struct mt76_txwi_cache *t = mt76_rx_token_release(dev, token);
385 dma_unmap_single(dev->dma_dev, t->dma_addr,
386 SKB_WITH_OVERHEAD(q->buf_size),
393 mt76_put_rxwi(dev, t);
396 u32 ctrl = le32_to_cpu(READ_ONCE(desc->ctrl));
398 *drop = !!(ctrl & (MT_DMA_CTL_TO_HOST_A |
404 dma_unmap_single(dev->dma_dev, e->dma_addr[0],
405 SKB_WITH_OVERHEAD(q->buf_size),
413 mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush,
414 int *len, u32 *info, bool *more, bool *drop)
423 q->desc[idx].ctrl |= cpu_to_le32(MT_DMA_CTL_DMA_DONE);
424 else if (!(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_DMA_DONE)))
427 q->tail = (q->tail + 1) % q->ndesc;
430 return mt76_dma_get_buf(dev, q, idx, len, info, more, drop);
434 mt76_dma_tx_queue_skb_raw(struct mt76_dev *dev, struct mt76_queue *q,
435 struct sk_buff *skb, u32 tx_info)
437 struct mt76_queue_buf buf = {};
440 if (q->queued + 1 >= q->ndesc - 1)
443 addr = dma_map_single(dev->dma_dev, skb->data, skb->len,
445 if (unlikely(dma_mapping_error(dev->dma_dev, addr)))
451 spin_lock_bh(&q->lock);
452 mt76_dma_add_buf(dev, q, &buf, 1, tx_info, skb, NULL);
453 mt76_dma_kick_queue(dev, q);
454 spin_unlock_bh(&q->lock);
464 mt76_dma_tx_queue_skb(struct mt76_dev *dev, struct mt76_queue *q,
465 enum mt76_txq_id qid, struct sk_buff *skb,
466 struct mt76_wcid *wcid, struct ieee80211_sta *sta)
468 struct ieee80211_tx_status status = {
471 struct mt76_tx_info tx_info = {
474 struct ieee80211_hw *hw;
475 int len, n = 0, ret = -ENOMEM;
476 struct mt76_txwi_cache *t;
477 struct sk_buff *iter;
481 t = mt76_get_txwi(dev);
485 txwi = mt76_get_txwi_ptr(dev, t);
487 skb->prev = skb->next = NULL;
488 if (dev->drv->drv_flags & MT_DRV_TX_ALIGNED4_SKBS)
489 mt76_insert_hdr_pad(skb);
491 len = skb_headlen(skb);
492 addr = dma_map_single(dev->dma_dev, skb->data, len, DMA_TO_DEVICE);
493 if (unlikely(dma_mapping_error(dev->dma_dev, addr)))
496 tx_info.buf[n].addr = t->dma_addr;
497 tx_info.buf[n++].len = dev->drv->txwi_size;
498 tx_info.buf[n].addr = addr;
499 tx_info.buf[n++].len = len;
501 skb_walk_frags(skb, iter) {
502 if (n == ARRAY_SIZE(tx_info.buf))
505 addr = dma_map_single(dev->dma_dev, iter->data, iter->len,
507 if (unlikely(dma_mapping_error(dev->dma_dev, addr)))
510 tx_info.buf[n].addr = addr;
511 tx_info.buf[n++].len = iter->len;
515 if (q->queued + (tx_info.nbuf + 1) / 2 >= q->ndesc - 1) {
520 dma_sync_single_for_cpu(dev->dma_dev, t->dma_addr, dev->drv->txwi_size,
522 ret = dev->drv->tx_prepare_skb(dev, txwi, qid, wcid, sta, &tx_info);
523 dma_sync_single_for_device(dev->dma_dev, t->dma_addr, dev->drv->txwi_size,
528 return mt76_dma_add_buf(dev, q, tx_info.buf, tx_info.nbuf,
529 tx_info.info, tx_info.skb, t);
532 for (n--; n > 0; n--)
533 dma_unmap_single(dev->dma_dev, tx_info.buf[n].addr,
534 tx_info.buf[n].len, DMA_TO_DEVICE);
537 #ifdef CONFIG_NL80211_TESTMODE
538 /* fix tx_done accounting on queue overflow */
539 if (mt76_is_testmode_skb(dev, skb, &hw)) {
540 struct mt76_phy *phy = hw->priv;
542 if (tx_info.skb == phy->test.tx_skb)
547 mt76_put_txwi(dev, t);
550 status.skb = tx_info.skb;
551 hw = mt76_tx_status_get_hw(dev, tx_info.skb);
552 ieee80211_tx_status_ext(hw, &status);
558 mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q)
560 int len = SKB_WITH_OVERHEAD(q->buf_size);
561 int frames = 0, offset = q->buf_offset;
567 spin_lock_bh(&q->lock);
569 while (q->queued < q->ndesc - 1) {
570 struct mt76_txwi_cache *t = NULL;
571 struct mt76_queue_buf qbuf;
574 if ((q->flags & MT_QFLAG_WED) &&
575 FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) == MT76_WED_Q_RX) {
576 t = mt76_get_rxwi(dev);
581 buf = page_frag_alloc(&q->rx_page, q->buf_size, GFP_ATOMIC);
585 addr = dma_map_single(dev->dma_dev, buf, len, DMA_FROM_DEVICE);
586 if (unlikely(dma_mapping_error(dev->dma_dev, addr))) {
591 qbuf.addr = addr + offset;
592 qbuf.len = len - offset;
593 qbuf.skip_unmap = false;
594 if (mt76_dma_add_buf(dev, q, &qbuf, 1, 0, buf, t) < 0) {
595 dma_unmap_single(dev->dma_dev, addr, len,
604 mt76_dma_kick_queue(dev, q);
606 spin_unlock_bh(&q->lock);
612 mt76_dma_wed_setup(struct mt76_dev *dev, struct mt76_queue *q)
614 #ifdef CONFIG_NET_MEDIATEK_SOC_WED
615 struct mtk_wed_device *wed = &dev->mmio.wed;
619 if (!mtk_wed_device_active(wed))
620 q->flags &= ~MT_QFLAG_WED;
622 if (!(q->flags & MT_QFLAG_WED))
625 type = FIELD_GET(MT_QFLAG_WED_TYPE, q->flags);
626 ring = FIELD_GET(MT_QFLAG_WED_RING, q->flags);
630 ret = mtk_wed_device_tx_ring_setup(wed, ring, q->regs, false);
632 q->wed_regs = wed->tx_ring[ring].reg_base;
634 case MT76_WED_Q_TXFREE:
635 /* WED txfree queue needs ring to be initialized before setup */
637 mt76_dma_queue_reset(dev, q);
638 mt76_dma_rx_fill(dev, q);
641 ret = mtk_wed_device_txfree_ring_setup(wed, q->regs);
643 q->wed_regs = wed->txfree_ring.reg_base;
646 ret = mtk_wed_device_rx_ring_setup(wed, ring, q->regs, false);
648 q->wed_regs = wed->rx_ring[ring].reg_base;
661 mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q,
662 int idx, int n_desc, int bufsize,
667 spin_lock_init(&q->lock);
668 spin_lock_init(&q->cleanup_lock);
670 q->regs = dev->mmio.regs + ring_base + idx * MT_RING_SIZE;
672 q->buf_size = bufsize;
675 size = q->ndesc * sizeof(struct mt76_desc);
676 q->desc = dmam_alloc_coherent(dev->dma_dev, size, &q->desc_dma, GFP_KERNEL);
680 size = q->ndesc * sizeof(*q->entry);
681 q->entry = devm_kzalloc(dev->dev, size, GFP_KERNEL);
685 ret = mt76_dma_wed_setup(dev, q);
689 if (q->flags != MT_WED_Q_TXFREE)
690 mt76_dma_queue_reset(dev, q);
696 mt76_dma_rx_cleanup(struct mt76_dev *dev, struct mt76_queue *q)
705 spin_lock_bh(&q->lock);
707 buf = mt76_dma_dequeue(dev, q, true, NULL, NULL, &more, NULL);
713 spin_unlock_bh(&q->lock);
718 page = virt_to_page(q->rx_page.va);
719 __page_frag_cache_drain(page, q->rx_page.pagecnt_bias);
720 memset(&q->rx_page, 0, sizeof(q->rx_page));
724 mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid)
726 struct mt76_queue *q = &dev->q_rx[qid];
732 for (i = 0; i < q->ndesc; i++)
733 q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE);
735 mt76_dma_rx_cleanup(dev, q);
736 mt76_dma_sync_idx(dev, q);
737 mt76_dma_rx_fill(dev, q);
742 dev_kfree_skb(q->rx_head);
747 mt76_add_fragment(struct mt76_dev *dev, struct mt76_queue *q, void *data,
748 int len, bool more, u32 info)
750 struct sk_buff *skb = q->rx_head;
751 struct skb_shared_info *shinfo = skb_shinfo(skb);
752 int nr_frags = shinfo->nr_frags;
754 if (nr_frags < ARRAY_SIZE(shinfo->frags)) {
755 struct page *page = virt_to_head_page(data);
756 int offset = data - page_address(page) + q->buf_offset;
758 skb_add_rx_frag(skb, nr_frags, page, offset, len, q->buf_size);
767 if (nr_frags < ARRAY_SIZE(shinfo->frags))
768 dev->drv->rx_skb(dev, q - dev->q_rx, skb, &info);
774 mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget)
776 int len, data_len, done = 0, dma_idx;
779 bool check_ddone = false;
782 if (IS_ENABLED(CONFIG_NET_MEDIATEK_SOC_WED) &&
783 q->flags == MT_WED_Q_TXFREE) {
784 dma_idx = Q_READ(dev, q, dma_idx);
788 while (done < budget) {
793 if (q->tail == dma_idx)
794 dma_idx = Q_READ(dev, q, dma_idx);
796 if (q->tail == dma_idx)
800 data = mt76_dma_dequeue(dev, q, false, &len, &info, &more,
809 data_len = q->buf_size;
811 data_len = SKB_WITH_OVERHEAD(q->buf_size);
813 if (data_len < len + q->buf_offset) {
814 dev_kfree_skb(q->rx_head);
820 mt76_add_fragment(dev, q, data, len, more, info);
824 if (!more && dev->drv->rx_check &&
825 !(dev->drv->rx_check(dev, data, len)))
828 skb = build_skb(data, q->buf_size);
832 skb_reserve(skb, q->buf_offset);
834 *(u32 *)skb->cb = info;
844 dev->drv->rx_skb(dev, q - dev->q_rx, skb, &info);
851 mt76_dma_rx_fill(dev, q);
855 int mt76_dma_rx_poll(struct napi_struct *napi, int budget)
857 struct mt76_dev *dev;
858 int qid, done = 0, cur;
860 dev = container_of(napi->dev, struct mt76_dev, napi_dev);
861 qid = napi - dev->napi;
866 cur = mt76_dma_rx_process(dev, &dev->q_rx[qid], budget - done);
867 mt76_rx_poll_complete(dev, qid, napi);
869 } while (cur && done < budget);
873 if (done < budget && napi_complete(napi))
874 dev->drv->rx_poll_complete(dev, qid);
878 EXPORT_SYMBOL_GPL(mt76_dma_rx_poll);
881 mt76_dma_init(struct mt76_dev *dev,
882 int (*poll)(struct napi_struct *napi, int budget))
886 init_dummy_netdev(&dev->napi_dev);
887 init_dummy_netdev(&dev->tx_napi_dev);
888 snprintf(dev->napi_dev.name, sizeof(dev->napi_dev.name), "%s",
889 wiphy_name(dev->hw->wiphy));
890 dev->napi_dev.threaded = 1;
892 mt76_for_each_q_rx(dev, i) {
893 netif_napi_add(&dev->napi_dev, &dev->napi[i], poll);
894 mt76_dma_rx_fill(dev, &dev->q_rx[i]);
895 napi_enable(&dev->napi[i]);
901 static const struct mt76_queue_ops mt76_dma_ops = {
902 .init = mt76_dma_init,
903 .alloc = mt76_dma_alloc_queue,
904 .reset_q = mt76_dma_queue_reset,
905 .tx_queue_skb_raw = mt76_dma_tx_queue_skb_raw,
906 .tx_queue_skb = mt76_dma_tx_queue_skb,
907 .tx_cleanup = mt76_dma_tx_cleanup,
908 .rx_cleanup = mt76_dma_rx_cleanup,
909 .rx_reset = mt76_dma_rx_reset,
910 .kick = mt76_dma_kick_queue,
913 void mt76_dma_attach(struct mt76_dev *dev)
915 dev->queue_ops = &mt76_dma_ops;
917 EXPORT_SYMBOL_GPL(mt76_dma_attach);
919 void mt76_dma_cleanup(struct mt76_dev *dev)
923 mt76_worker_disable(&dev->tx_worker);
924 netif_napi_del(&dev->tx_napi);
926 for (i = 0; i < ARRAY_SIZE(dev->phys); i++) {
927 struct mt76_phy *phy = dev->phys[i];
933 for (j = 0; j < ARRAY_SIZE(phy->q_tx); j++)
934 mt76_dma_tx_cleanup(dev, phy->q_tx[j], true);
937 for (i = 0; i < ARRAY_SIZE(dev->q_mcu); i++)
938 mt76_dma_tx_cleanup(dev, dev->q_mcu[i], true);
940 mt76_for_each_q_rx(dev, i) {
941 struct mt76_queue *q = &dev->q_rx[i];
943 netif_napi_del(&dev->napi[i]);
944 if (FIELD_GET(MT_QFLAG_WED_TYPE, q->flags))
945 mt76_dma_rx_cleanup(dev, q);
948 mt76_free_pending_txwi(dev);
949 mt76_free_pending_rxwi(dev);
951 if (mtk_wed_device_active(&dev->mmio.wed))
952 mtk_wed_device_detach(&dev->mmio.wed);
954 EXPORT_SYMBOL_GPL(mt76_dma_cleanup);