projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
dmaengine: Add Actions Semi Owl family S900 DMA driver
[linux-block.git] / drivers / dma / owl-dma.c
CommitLineData
47e20577
MS		 1// SPDX-License-Identifier: GPL-2.0+
2//
3// Actions Semi Owl SoCs DMA driver
4//
5// Copyright (c) 2014 Actions Semi Inc.
6// Author: David Liu <liuwei@actions-semi.com>
7//
8// Copyright (c) 2018 Linaro Ltd.
9// Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
10
11#include <linux/bitops.h>
12#include <linux/clk.h>
13#include <linux/delay.h>
14#include <linux/dmaengine.h>
15#include <linux/dma-mapping.h>
16#include <linux/dmapool.h>
17#include <linux/err.h>
18#include <linux/init.h>
19#include <linux/interrupt.h>
20#include <linux/io.h>
21#include <linux/mm.h>
22#include <linux/module.h>
23#include <linux/of_device.h>
24#include <linux/slab.h>
25#include "virt-dma.h"
26
27#define OWL_DMA_FRAME_MAX_LENGTH 0xfffff
28
29/* Global DMA Controller Registers */
30#define OWL_DMA_IRQ_PD0 0x00
31#define OWL_DMA_IRQ_PD1 0x04
32#define OWL_DMA_IRQ_PD2 0x08
33#define OWL_DMA_IRQ_PD3 0x0C
34#define OWL_DMA_IRQ_EN0 0x10
35#define OWL_DMA_IRQ_EN1 0x14
36#define OWL_DMA_IRQ_EN2 0x18
37#define OWL_DMA_IRQ_EN3 0x1C
38#define OWL_DMA_SECURE_ACCESS_CTL 0x20
39#define OWL_DMA_NIC_QOS 0x24
40#define OWL_DMA_DBGSEL 0x28
41#define OWL_DMA_IDLE_STAT 0x2C
42
43/* Channel Registers */
44#define OWL_DMA_CHAN_BASE(i) (0x100 + (i) * 0x100)
45#define OWL_DMAX_MODE 0x00
46#define OWL_DMAX_SOURCE 0x04
47#define OWL_DMAX_DESTINATION 0x08
48#define OWL_DMAX_FRAME_LEN 0x0C
49#define OWL_DMAX_FRAME_CNT 0x10
50#define OWL_DMAX_REMAIN_FRAME_CNT 0x14
51#define OWL_DMAX_REMAIN_CNT 0x18
52#define OWL_DMAX_SOURCE_STRIDE 0x1C
53#define OWL_DMAX_DESTINATION_STRIDE 0x20
54#define OWL_DMAX_START 0x24
55#define OWL_DMAX_PAUSE 0x28
56#define OWL_DMAX_CHAINED_CTL 0x2C
57#define OWL_DMAX_CONSTANT 0x30
58#define OWL_DMAX_LINKLIST_CTL 0x34
59#define OWL_DMAX_NEXT_DESCRIPTOR 0x38
60#define OWL_DMAX_CURRENT_DESCRIPTOR_NUM 0x3C
61#define OWL_DMAX_INT_CTL 0x40
62#define OWL_DMAX_INT_STATUS 0x44
63#define OWL_DMAX_CURRENT_SOURCE_POINTER 0x48
64#define OWL_DMAX_CURRENT_DESTINATION_POINTER 0x4C
65
66/* OWL_DMAX_MODE Bits */
67#define OWL_DMA_MODE_TS(x) (((x) & GENMASK(5, 0)) << 0)
68#define OWL_DMA_MODE_ST(x) (((x) & GENMASK(1, 0)) << 8)
69#define OWL_DMA_MODE_ST_DEV OWL_DMA_MODE_ST(0)
70#define OWL_DMA_MODE_ST_DCU OWL_DMA_MODE_ST(2)
71#define OWL_DMA_MODE_ST_SRAM OWL_DMA_MODE_ST(3)
72#define OWL_DMA_MODE_DT(x) (((x) & GENMASK(1, 0)) << 10)
73#define OWL_DMA_MODE_DT_DEV OWL_DMA_MODE_DT(0)
74#define OWL_DMA_MODE_DT_DCU OWL_DMA_MODE_DT(2)
75#define OWL_DMA_MODE_DT_SRAM OWL_DMA_MODE_DT(3)
76#define OWL_DMA_MODE_SAM(x) (((x) & GENMASK(1, 0)) << 16)
77#define OWL_DMA_MODE_SAM_CONST OWL_DMA_MODE_SAM(0)
78#define OWL_DMA_MODE_SAM_INC OWL_DMA_MODE_SAM(1)
79#define OWL_DMA_MODE_SAM_STRIDE OWL_DMA_MODE_SAM(2)
80#define OWL_DMA_MODE_DAM(x) (((x) & GENMASK(1, 0)) << 18)
81#define OWL_DMA_MODE_DAM_CONST OWL_DMA_MODE_DAM(0)
82#define OWL_DMA_MODE_DAM_INC OWL_DMA_MODE_DAM(1)
83#define OWL_DMA_MODE_DAM_STRIDE OWL_DMA_MODE_DAM(2)
84#define OWL_DMA_MODE_PW(x) (((x) & GENMASK(2, 0)) << 20)
85#define OWL_DMA_MODE_CB BIT(23)
86#define OWL_DMA_MODE_NDDBW(x) (((x) & 0x1) << 28)
87#define OWL_DMA_MODE_NDDBW_32BIT OWL_DMA_MODE_NDDBW(0)
88#define OWL_DMA_MODE_NDDBW_8BIT OWL_DMA_MODE_NDDBW(1)
89#define OWL_DMA_MODE_CFE BIT(29)
90#define OWL_DMA_MODE_LME BIT(30)
91#define OWL_DMA_MODE_CME BIT(31)
92
93/* OWL_DMAX_LINKLIST_CTL Bits */
94#define OWL_DMA_LLC_SAV(x) (((x) & GENMASK(1, 0)) << 8)
95#define OWL_DMA_LLC_SAV_INC OWL_DMA_LLC_SAV(0)
96#define OWL_DMA_LLC_SAV_LOAD_NEXT OWL_DMA_LLC_SAV(1)
97#define OWL_DMA_LLC_SAV_LOAD_PREV OWL_DMA_LLC_SAV(2)
98#define OWL_DMA_LLC_DAV(x) (((x) & GENMASK(1, 0)) << 10)
99#define OWL_DMA_LLC_DAV_INC OWL_DMA_LLC_DAV(0)
100#define OWL_DMA_LLC_DAV_LOAD_NEXT OWL_DMA_LLC_DAV(1)
101#define OWL_DMA_LLC_DAV_LOAD_PREV OWL_DMA_LLC_DAV(2)
102#define OWL_DMA_LLC_SUSPEND BIT(16)
103
104/* OWL_DMAX_INT_CTL Bits */
105#define OWL_DMA_INTCTL_BLOCK BIT(0)
106#define OWL_DMA_INTCTL_SUPER_BLOCK BIT(1)
107#define OWL_DMA_INTCTL_FRAME BIT(2)
108#define OWL_DMA_INTCTL_HALF_FRAME BIT(3)
109#define OWL_DMA_INTCTL_LAST_FRAME BIT(4)
110
111/* OWL_DMAX_INT_STATUS Bits */
112#define OWL_DMA_INTSTAT_BLOCK BIT(0)
113#define OWL_DMA_INTSTAT_SUPER_BLOCK BIT(1)
114#define OWL_DMA_INTSTAT_FRAME BIT(2)
115#define OWL_DMA_INTSTAT_HALF_FRAME BIT(3)
116#define OWL_DMA_INTSTAT_LAST_FRAME BIT(4)
117
118/* Pack shift and newshift in a single word */
119#define BIT_FIELD(val, width, shift, newshift) \
120 ((((val) >> (shift)) & ((BIT(width)) - 1)) << (newshift))
121
122/**
123 * struct owl_dma_lli_hw - Hardware link list for dma transfer
124 * @next_lli: physical address of the next link list
125 * @saddr: source physical address
126 * @daddr: destination physical address
127 * @flen: frame length
128 * @fcnt: frame count
129 * @src_stride: source stride
130 * @dst_stride: destination stride
131 * @ctrla: dma_mode and linklist ctrl config
132 * @ctrlb: interrupt config
133 * @const_num: data for constant fill
134 */
135struct owl_dma_lli_hw {
136 u32 next_lli;
137 u32 saddr;
138 u32 daddr;
139 u32 flen:20;
140 u32 fcnt:12;
141 u32 src_stride;
142 u32 dst_stride;
143 u32 ctrla;
144 u32 ctrlb;
145 u32 const_num;
146};
147
148/**
149 * struct owl_dma_lli - Link list for dma transfer
150 * @hw: hardware link list
151 * @phys: physical address of hardware link list
152 * @node: node for txd's lli_list
153 */
154struct owl_dma_lli {
155 struct owl_dma_lli_hw hw;
156 dma_addr_t phys;
157 struct list_head node;
158};
159
160/**
161 * struct owl_dma_txd - Wrapper for struct dma_async_tx_descriptor
162 * @vd: virtual DMA descriptor
163 * @lli_list: link list of lli nodes
164 */
165struct owl_dma_txd {
166 struct virt_dma_desc vd;
167 struct list_head lli_list;
168};
169
170/**
171 * struct owl_dma_pchan - Holder for the physical channels
172 * @id: physical index to this channel
173 * @base: virtual memory base for the dma channel
174 * @vchan: the virtual channel currently being served by this physical channel
175 * @lock: a lock to use when altering an instance of this struct
176 */
177struct owl_dma_pchan {
178 u32 id;
179 void __iomem *base;
180 struct owl_dma_vchan *vchan;
181 spinlock_t lock;
182};
183
184/**
185 * struct owl_dma_pchan - Wrapper for DMA ENGINE channel
186 * @vc: wrappped virtual channel
187 * @pchan: the physical channel utilized by this channel
188 * @txd: active transaction on this channel
189 */
190struct owl_dma_vchan {
191 struct virt_dma_chan vc;
192 struct owl_dma_pchan *pchan;
193 struct owl_dma_txd *txd;
194};
195
196/**
197 * struct owl_dma - Holder for the Owl DMA controller
198 * @dma: dma engine for this instance
199 * @base: virtual memory base for the DMA controller
200 * @clk: clock for the DMA controller
201 * @lock: a lock to use when change DMA controller global register
202 * @lli_pool: a pool for the LLI descriptors
203 * @nr_pchans: the number of physical channels
204 * @pchans: array of data for the physical channels
205 * @nr_vchans: the number of physical channels
206 * @vchans: array of data for the physical channels
207 */
208struct owl_dma {
209 struct dma_device dma;
210 void __iomem *base;
211 struct clk *clk;
212 spinlock_t lock;
213 struct dma_pool *lli_pool;
214 int irq;
215
216 unsigned int nr_pchans;
217 struct owl_dma_pchan *pchans;
218
219 unsigned int nr_vchans;
220 struct owl_dma_vchan *vchans;
221};
222
223static void pchan_update(struct owl_dma_pchan *pchan, u32 reg,
224 u32 val, bool state)
225{
226 u32 regval;
227
228 regval = readl(pchan->base + reg);
229
230 if (state)
231 regval |= val;
232 else
233 regval &= ~val;
234
235 writel(val, pchan->base + reg);
236}
237
238static void pchan_writel(struct owl_dma_pchan *pchan, u32 reg, u32 data)
239{
240 writel(data, pchan->base + reg);
241}
242
243static u32 pchan_readl(struct owl_dma_pchan *pchan, u32 reg)
244{
245 return readl(pchan->base + reg);
246}
247
248static void dma_update(struct owl_dma *od, u32 reg, u32 val, bool state)
249{
250 u32 regval;
251
252 regval = readl(od->base + reg);
253
254 if (state)
255 regval |= val;
256 else
257 regval &= ~val;
258
259 writel(val, od->base + reg);
260}
261
262static void dma_writel(struct owl_dma *od, u32 reg, u32 data)
263{
264 writel(data, od->base + reg);
265}
266
267static u32 dma_readl(struct owl_dma *od, u32 reg)
268{
269 return readl(od->base + reg);
270}
271
272static inline struct owl_dma *to_owl_dma(struct dma_device *dd)
273{
274 return container_of(dd, struct owl_dma, dma);
275}
276
277static struct device *chan2dev(struct dma_chan *chan)
278{
279 return &chan->dev->device;
280}
281
282static inline struct owl_dma_vchan *to_owl_vchan(struct dma_chan *chan)
283{
284 return container_of(chan, struct owl_dma_vchan, vc.chan);
285}
286
287static inline struct owl_dma_txd *to_owl_txd(struct dma_async_tx_descriptor *tx)
288{
289 return container_of(tx, struct owl_dma_txd, vd.tx);
290}
291
292static inline u32 llc_hw_ctrla(u32 mode, u32 llc_ctl)
293{
294 u32 ctl;
295
296 ctl = BIT_FIELD(mode, 4, 28, 28) |
297 BIT_FIELD(mode, 8, 16, 20) |
298 BIT_FIELD(mode, 4, 8, 16) |
299 BIT_FIELD(mode, 6, 0, 10) |
300 BIT_FIELD(llc_ctl, 2, 10, 8) |
301 BIT_FIELD(llc_ctl, 2, 8, 6);
302
303 return ctl;
304}
305
306static inline u32 llc_hw_ctrlb(u32 int_ctl)
307{
308 u32 ctl;
309
310 ctl = BIT_FIELD(int_ctl, 7, 0, 18);
311
312 return ctl;
313}
314
315static void owl_dma_free_lli(struct owl_dma *od,
316 struct owl_dma_lli *lli)
317{
318 list_del(&lli->node);
319 dma_pool_free(od->lli_pool, lli, lli->phys);
320}
321
322static struct owl_dma_lli *owl_dma_alloc_lli(struct owl_dma *od)
323{
324 struct owl_dma_lli *lli;
325 dma_addr_t phys;
326
327 lli = dma_pool_alloc(od->lli_pool, GFP_NOWAIT, &phys);
328 if (!lli)
329 return NULL;
330
331 INIT_LIST_HEAD(&lli->node);
332 lli->phys = phys;
333
334 return lli;
335}
336
337static struct owl_dma_lli *owl_dma_add_lli(struct owl_dma_txd *txd,
338 struct owl_dma_lli *prev,
339 struct owl_dma_lli *next)
340{
341 list_add_tail(&next->node, &txd->lli_list);
342
343 if (prev) {
344 prev->hw.next_lli = next->phys;
345 prev->hw.ctrla |= llc_hw_ctrla(OWL_DMA_MODE_LME, 0);
346 }
347
348 return next;
349}
350
351static inline int owl_dma_cfg_lli(struct owl_dma_vchan *vchan,
352 struct owl_dma_lli *lli,
353 dma_addr_t src, dma_addr_t dst,
354 u32 len, enum dma_transfer_direction dir)
355{
356 struct owl_dma_lli_hw *hw = &lli->hw;
357 u32 mode;
358
359 mode = OWL_DMA_MODE_PW(0);
360
361 switch (dir) {
362 case DMA_MEM_TO_MEM:
363 mode |= OWL_DMA_MODE_TS(0) | OWL_DMA_MODE_ST_DCU |
364 OWL_DMA_MODE_DT_DCU | OWL_DMA_MODE_SAM_INC |
365 OWL_DMA_MODE_DAM_INC;
366
367 break;
368 default:
369 return -EINVAL;
370 }
371
372 hw->next_lli = 0; /* One link list by default */
373 hw->saddr = src;
374 hw->daddr = dst;
375
376 hw->fcnt = 1; /* Frame count fixed as 1 */
377 hw->flen = len; /* Max frame length is 1MB */
378 hw->src_stride = 0;
379 hw->dst_stride = 0;
380 hw->ctrla = llc_hw_ctrla(mode,
381 OWL_DMA_LLC_SAV_LOAD_NEXT |
382 OWL_DMA_LLC_DAV_LOAD_NEXT);
383
384 hw->ctrlb = llc_hw_ctrlb(OWL_DMA_INTCTL_SUPER_BLOCK);
385
386 return 0;
387}
388
389static struct owl_dma_pchan *owl_dma_get_pchan(struct owl_dma *od,
390 struct owl_dma_vchan *vchan)
391{
392 struct owl_dma_pchan *pchan = NULL;
393 unsigned long flags;
394 int i;
395
396 for (i = 0; i < od->nr_pchans; i++) {
397 pchan = &od->pchans[i];
398
399 spin_lock_irqsave(&pchan->lock, flags);
400 if (!pchan->vchan) {
401 pchan->vchan = vchan;
402 spin_unlock_irqrestore(&pchan->lock, flags);
403 break;
404 }
405
406 spin_unlock_irqrestore(&pchan->lock, flags);
407 }
408
409 return pchan;
410}
411
412static int owl_dma_pchan_busy(struct owl_dma *od, struct owl_dma_pchan *pchan)
413{
414 unsigned int val;
415
416 val = dma_readl(od, OWL_DMA_IDLE_STAT);
417
418 return !(val & (1 << pchan->id));
419}
420
421static void owl_dma_terminate_pchan(struct owl_dma *od,
422 struct owl_dma_pchan *pchan)
423{
424 unsigned long flags;
425 u32 irq_pd;
426
427 pchan_writel(pchan, OWL_DMAX_START, 0);
428 pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false);
429
430 spin_lock_irqsave(&od->lock, flags);
431 dma_update(od, OWL_DMA_IRQ_EN0, (1 << pchan->id), false);
432
433 irq_pd = dma_readl(od, OWL_DMA_IRQ_PD0);
434 if (irq_pd & (1 << pchan->id)) {
435 dev_warn(od->dma.dev,
436 "terminating pchan %d that still has pending irq\n",
437 pchan->id);
438 dma_writel(od, OWL_DMA_IRQ_PD0, (1 << pchan->id));
439 }
440
441 pchan->vchan = NULL;
442
443 spin_unlock_irqrestore(&od->lock, flags);
444}
445
446static int owl_dma_start_next_txd(struct owl_dma_vchan *vchan)
447{
448 struct owl_dma *od = to_owl_dma(vchan->vc.chan.device);
449 struct virt_dma_desc *vd = vchan_next_desc(&vchan->vc);
450 struct owl_dma_pchan *pchan = vchan->pchan;
451 struct owl_dma_txd *txd = to_owl_txd(&vd->tx);
452 struct owl_dma_lli *lli;
453 unsigned long flags;
454 u32 int_ctl;
455
456 list_del(&vd->node);
457
458 vchan->txd = txd;
459
460 /* Wait for channel inactive */
461 while (owl_dma_pchan_busy(od, pchan))
462 cpu_relax();
463
464 lli = list_first_entry(&txd->lli_list,
465 struct owl_dma_lli, node);
466
467 int_ctl = OWL_DMA_INTCTL_SUPER_BLOCK;
468
469 pchan_writel(pchan, OWL_DMAX_MODE, OWL_DMA_MODE_LME);
470 pchan_writel(pchan, OWL_DMAX_LINKLIST_CTL,
471 OWL_DMA_LLC_SAV_LOAD_NEXT | OWL_DMA_LLC_DAV_LOAD_NEXT);
472 pchan_writel(pchan, OWL_DMAX_NEXT_DESCRIPTOR, lli->phys);
473 pchan_writel(pchan, OWL_DMAX_INT_CTL, int_ctl);
474
475 /* Clear IRQ status for this pchan */
476 pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false);
477
478 spin_lock_irqsave(&od->lock, flags);
479
480 dma_update(od, OWL_DMA_IRQ_EN0, (1 << pchan->id), true);
481
482 spin_unlock_irqrestore(&od->lock, flags);
483
484 dev_dbg(chan2dev(&vchan->vc.chan), "starting pchan %d\n", pchan->id);
485
486 /* Start DMA transfer for this pchan */
487 pchan_writel(pchan, OWL_DMAX_START, 0x1);
488
489 return 0;
490}
491
492static void owl_dma_phy_free(struct owl_dma *od, struct owl_dma_vchan *vchan)
493{
494 /* Ensure that the physical channel is stopped */
495 owl_dma_terminate_pchan(od, vchan->pchan);
496
497 vchan->pchan = NULL;
498}
499
500static irqreturn_t owl_dma_interrupt(int irq, void *dev_id)
501{
502 struct owl_dma *od = dev_id;
503 struct owl_dma_vchan *vchan;
504 struct owl_dma_pchan *pchan;
505 unsigned long pending;
506 int i;
507 unsigned int global_irq_pending, chan_irq_pending;
508
509 spin_lock(&od->lock);
510
511 pending = dma_readl(od, OWL_DMA_IRQ_PD0);
512
513 /* Clear IRQ status for each pchan */
514 for_each_set_bit(i, &pending, od->nr_pchans) {
515 pchan = &od->pchans[i];
516 pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false);
517 }
518
519 /* Clear pending IRQ */
520 dma_writel(od, OWL_DMA_IRQ_PD0, pending);
521
522 /* Check missed pending IRQ */
523 for (i = 0; i < od->nr_pchans; i++) {
524 pchan = &od->pchans[i];
525 chan_irq_pending = pchan_readl(pchan, OWL_DMAX_INT_CTL) &
526 pchan_readl(pchan, OWL_DMAX_INT_STATUS);
527
528 /* Dummy read to ensure OWL_DMA_IRQ_PD0 value is updated */
529 dma_readl(od, OWL_DMA_IRQ_PD0);
530
531 global_irq_pending = dma_readl(od, OWL_DMA_IRQ_PD0);
532
533 if (chan_irq_pending && !(global_irq_pending & BIT(i))) {
534 dev_dbg(od->dma.dev,
535 "global and channel IRQ pending match err\n");
536
537 /* Clear IRQ status for this pchan */
538 pchan_update(pchan, OWL_DMAX_INT_STATUS,
539 0xff, false);
540
541 /* Update global IRQ pending */
542 pending |= BIT(i);
543 }
544 }
545
546 spin_unlock(&od->lock);
547
548 for_each_set_bit(i, &pending, od->nr_pchans) {
549 struct owl_dma_txd *txd;
550
551 pchan = &od->pchans[i];
552
553 vchan = pchan->vchan;
554 if (!vchan) {
555 dev_warn(od->dma.dev, "no vchan attached on pchan %d\n",
556 pchan->id);
557 continue;
558 }
559
560 spin_lock(&vchan->vc.lock);
561
562 txd = vchan->txd;
563 if (txd) {
564 vchan->txd = NULL;
565
566 vchan_cookie_complete(&txd->vd);
567
568 /*
569 * Start the next descriptor (if any),
570 * otherwise free this channel.
571 */
572 if (vchan_next_desc(&vchan->vc))
573 owl_dma_start_next_txd(vchan);
574 else
575 owl_dma_phy_free(od, vchan);
576 }
577
578 spin_unlock(&vchan->vc.lock);
579 }
580
581 return IRQ_HANDLED;
582}
583
584static void owl_dma_free_txd(struct owl_dma *od, struct owl_dma_txd *txd)
585{
586 struct owl_dma_lli *lli, *_lli;
587
588 if (unlikely(!txd))
589 return;
590
591 list_for_each_entry_safe(lli, _lli, &txd->lli_list, node)
592 owl_dma_free_lli(od, lli);
593
594 kfree(txd);
595}
596
597static void owl_dma_desc_free(struct virt_dma_desc *vd)
598{
599 struct owl_dma *od = to_owl_dma(vd->tx.chan->device);
600 struct owl_dma_txd *txd = to_owl_txd(&vd->tx);
601
602 owl_dma_free_txd(od, txd);
603}
604
605static int owl_dma_terminate_all(struct dma_chan *chan)
606{
607 struct owl_dma *od = to_owl_dma(chan->device);
608 struct owl_dma_vchan *vchan = to_owl_vchan(chan);
609 unsigned long flags;
610 LIST_HEAD(head);
611
612 spin_lock_irqsave(&vchan->vc.lock, flags);
613
614 if (vchan->pchan)
615 owl_dma_phy_free(od, vchan);
616
617 if (vchan->txd) {
618 owl_dma_desc_free(&vchan->txd->vd);
619 vchan->txd = NULL;
620 }
621
622 vchan_get_all_descriptors(&vchan->vc, &head);
623 vchan_dma_desc_free_list(&vchan->vc, &head);
624
625 spin_unlock_irqrestore(&vchan->vc.lock, flags);
626
627 return 0;
628}
629
630static u32 owl_dma_getbytes_chan(struct owl_dma_vchan *vchan)
631{
632 struct owl_dma_pchan *pchan;
633 struct owl_dma_txd *txd;
634 struct owl_dma_lli *lli;
635 unsigned int next_lli_phy;
636 size_t bytes;
637
638 pchan = vchan->pchan;
639 txd = vchan->txd;
640
641 if (!pchan || !txd)
642 return 0;
643
644 /* Get remain count of current node in link list */
645 bytes = pchan_readl(pchan, OWL_DMAX_REMAIN_CNT);
646
647 /* Loop through the preceding nodes to get total remaining bytes */
648 if (pchan_readl(pchan, OWL_DMAX_MODE) & OWL_DMA_MODE_LME) {
649 next_lli_phy = pchan_readl(pchan, OWL_DMAX_NEXT_DESCRIPTOR);
650 list_for_each_entry(lli, &txd->lli_list, node) {
651 /* Start from the next active node */
652 if (lli->phys == next_lli_phy) {
653 list_for_each_entry(lli, &txd->lli_list, node)
654 bytes += lli->hw.flen;
655 break;
656 }
657 }
658 }
659
660 return bytes;
661}
662
663static enum dma_status owl_dma_tx_status(struct dma_chan *chan,
664 dma_cookie_t cookie,
665 struct dma_tx_state *state)
666{
667 struct owl_dma_vchan *vchan = to_owl_vchan(chan);
668 struct owl_dma_lli *lli;
669 struct virt_dma_desc *vd;
670 struct owl_dma_txd *txd;
671 enum dma_status ret;
672 unsigned long flags;
673 size_t bytes = 0;
674
675 ret = dma_cookie_status(chan, cookie, state);
676 if (ret == DMA_COMPLETE || !state)
677 return ret;
678
679 spin_lock_irqsave(&vchan->vc.lock, flags);
680
681 vd = vchan_find_desc(&vchan->vc, cookie);
682 if (vd) {
683 txd = to_owl_txd(&vd->tx);
684 list_for_each_entry(lli, &txd->lli_list, node)
685 bytes += lli->hw.flen;
686 } else {
687 bytes = owl_dma_getbytes_chan(vchan);
688 }
689
690 spin_unlock_irqrestore(&vchan->vc.lock, flags);
691
692 dma_set_residue(state, bytes);
693
694 return ret;
695}
696
697static void owl_dma_phy_alloc_and_start(struct owl_dma_vchan *vchan)
698{
699 struct owl_dma *od = to_owl_dma(vchan->vc.chan.device);
700 struct owl_dma_pchan *pchan;
701
702 pchan = owl_dma_get_pchan(od, vchan);
703 if (!pchan)
704 return;
705
706 dev_dbg(od->dma.dev, "allocated pchan %d\n", pchan->id);
707
708 vchan->pchan = pchan;
709 owl_dma_start_next_txd(vchan);
710}
711
712static void owl_dma_issue_pending(struct dma_chan *chan)
713{
714 struct owl_dma_vchan *vchan = to_owl_vchan(chan);
715 unsigned long flags;
716
717 spin_lock_irqsave(&vchan->vc.lock, flags);
718 if (vchan_issue_pending(&vchan->vc)) {
719 if (!vchan->pchan)
720 owl_dma_phy_alloc_and_start(vchan);
721 }
722 spin_unlock_irqrestore(&vchan->vc.lock, flags);
723}
724
725static struct dma_async_tx_descriptor
726 *owl_dma_prep_memcpy(struct dma_chan *chan,
727 dma_addr_t dst, dma_addr_t src,
728 size_t len, unsigned long flags)
729{
730 struct owl_dma *od = to_owl_dma(chan->device);
731 struct owl_dma_vchan *vchan = to_owl_vchan(chan);
732 struct owl_dma_txd *txd;
733 struct owl_dma_lli *lli, *prev = NULL;
734 size_t offset, bytes;
735 int ret;
736
737 if (!len)
738 return NULL;
739
740 txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
741 if (!txd)
742 return NULL;
743
744 INIT_LIST_HEAD(&txd->lli_list);
745
746 /* Process the transfer as frame by frame */
747 for (offset = 0; offset < len; offset += bytes) {
748 lli = owl_dma_alloc_lli(od);
749 if (!lli) {
750 dev_warn(chan2dev(chan), "failed to allocate lli\n");
751 goto err_txd_free;
752 }
753
754 bytes = min_t(size_t, (len - offset), OWL_DMA_FRAME_MAX_LENGTH);
755
756 ret = owl_dma_cfg_lli(vchan, lli, src + offset, dst + offset,
757 bytes, DMA_MEM_TO_MEM);
758 if (ret) {
759 dev_warn(chan2dev(chan), "failed to config lli\n");
760 goto err_txd_free;
761 }
762
763 prev = owl_dma_add_lli(txd, prev, lli);
764 }
765
766 return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
767
768err_txd_free:
769 owl_dma_free_txd(od, txd);
770 return NULL;
771}
772
773static void owl_dma_free_chan_resources(struct dma_chan *chan)
774{
775 struct owl_dma_vchan *vchan = to_owl_vchan(chan);
776
777 /* Ensure all queued descriptors are freed */
778 vchan_free_chan_resources(&vchan->vc);
779}
780
781static inline void owl_dma_free(struct owl_dma *od)
782{
783 struct owl_dma_vchan *vchan = NULL;
784 struct owl_dma_vchan *next;
785
786 list_for_each_entry_safe(vchan,
787 next, &od->dma.channels, vc.chan.device_node) {
788 list_del(&vchan->vc.chan.device_node);
789 tasklet_kill(&vchan->vc.task);
790 }
791}
792
793static int owl_dma_probe(struct platform_device *pdev)
794{
795 struct device_node *np = pdev->dev.of_node;
796 struct owl_dma *od;
797 struct resource *res;
798 int ret, i, nr_channels, nr_requests;
799
800 od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);
801 if (!od)
802 return -ENOMEM;
803
804 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
805 if (!res)
806 return -EINVAL;
807
808 od->base = devm_ioremap_resource(&pdev->dev, res);
809 if (IS_ERR(od->base))
810 return PTR_ERR(od->base);
811
812 ret = of_property_read_u32(np, "dma-channels", &nr_channels);
813 if (ret) {
814 dev_err(&pdev->dev, "can't get dma-channels\n");
815 return ret;
816 }
817
818 ret = of_property_read_u32(np, "dma-requests", &nr_requests);
819 if (ret) {
820 dev_err(&pdev->dev, "can't get dma-requests\n");
821 return ret;
822 }
823
824 dev_info(&pdev->dev, "dma-channels %d, dma-requests %d\n",
825 nr_channels, nr_requests);
826
827 od->nr_pchans = nr_channels;
828 od->nr_vchans = nr_requests;
829
830 pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32);
831
832 platform_set_drvdata(pdev, od);
833 spin_lock_init(&od->lock);
834
835 dma_cap_set(DMA_MEMCPY, od->dma.cap_mask);
836
837 od->dma.dev = &pdev->dev;
838 od->dma.device_free_chan_resources = owl_dma_free_chan_resources;
839 od->dma.device_tx_status = owl_dma_tx_status;
840 od->dma.device_issue_pending = owl_dma_issue_pending;
841 od->dma.device_prep_dma_memcpy = owl_dma_prep_memcpy;
842 od->dma.device_terminate_all = owl_dma_terminate_all;
843 od->dma.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
844 od->dma.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
845 od->dma.directions = BIT(DMA_MEM_TO_MEM);
846 od->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
847
848 INIT_LIST_HEAD(&od->dma.channels);
849
850 od->clk = devm_clk_get(&pdev->dev, NULL);
851 if (IS_ERR(od->clk)) {
852 dev_err(&pdev->dev, "unable to get clock\n");
853 return PTR_ERR(od->clk);
854 }
855
856 /*
857 * Eventhough the DMA controller is capable of generating 4
858 * IRQ's for DMA priority feature, we only use 1 IRQ for
859 * simplification.
860 */
861 od->irq = platform_get_irq(pdev, 0);
862 ret = devm_request_irq(&pdev->dev, od->irq, owl_dma_interrupt, 0,
863 dev_name(&pdev->dev), od);
864 if (ret) {
865 dev_err(&pdev->dev, "unable to request IRQ\n");
866 return ret;
867 }
868
869 /* Init physical channel */
870 od->pchans = devm_kcalloc(&pdev->dev, od->nr_pchans,
871 sizeof(struct owl_dma_pchan), GFP_KERNEL);
872 if (!od->pchans)
873 return -ENOMEM;
874
875 for (i = 0; i < od->nr_pchans; i++) {
876 struct owl_dma_pchan *pchan = &od->pchans[i];
877
878 pchan->id = i;
879 pchan->base = od->base + OWL_DMA_CHAN_BASE(i);
880 }
881
882 /* Init virtual channel */
883 od->vchans = devm_kcalloc(&pdev->dev, od->nr_vchans,
884 sizeof(struct owl_dma_vchan), GFP_KERNEL);
885 if (!od->vchans)
886 return -ENOMEM;
887
888 for (i = 0; i < od->nr_vchans; i++) {
889 struct owl_dma_vchan *vchan = &od->vchans[i];
890
891 vchan->vc.desc_free = owl_dma_desc_free;
892 vchan_init(&vchan->vc, &od->dma);
893 }
894
895 /* Create a pool of consistent memory blocks for hardware descriptors */
896 od->lli_pool = dma_pool_create(dev_name(od->dma.dev), od->dma.dev,
897 sizeof(struct owl_dma_lli),
898 __alignof__(struct owl_dma_lli),
899 0);
900 if (!od->lli_pool) {
901 dev_err(&pdev->dev, "unable to allocate DMA descriptor pool\n");
902 return -ENOMEM;
903 }
904
905 clk_prepare_enable(od->clk);
906
907 ret = dma_async_device_register(&od->dma);
908 if (ret) {
909 dev_err(&pdev->dev, "failed to register DMA engine device\n");
910 goto err_pool_free;
911 }
912
913 return 0;
914
915err_pool_free:
916 clk_disable_unprepare(od->clk);
917 dma_pool_destroy(od->lli_pool);
918
919 return ret;
920}
921
922static int owl_dma_remove(struct platform_device *pdev)
923{
924 struct owl_dma *od = platform_get_drvdata(pdev);
925
926 dma_async_device_unregister(&od->dma);
927
928 /* Mask all interrupts for this execution environment */
929 dma_writel(od, OWL_DMA_IRQ_EN0, 0x0);
930
931 /* Make sure we won't have any further interrupts */
932 devm_free_irq(od->dma.dev, od->irq, od);
933
934 owl_dma_free(od);
935
936 clk_disable_unprepare(od->clk);
937
938 return 0;
939}
940
941static const struct of_device_id owl_dma_match[] = {
942 { .compatible = "actions,s900-dma", },
943 { /* sentinel */ }
944};
945MODULE_DEVICE_TABLE(of, owl_dma_match);
946
947static struct platform_driver owl_dma_driver = {
948 .probe = owl_dma_probe,
949 .remove = owl_dma_remove,
950 .driver = {
951 .name = "dma-owl",
952 .of_match_table = of_match_ptr(owl_dma_match),
953 },
954};
955
956static int owl_dma_init(void)
957{
958 return platform_driver_register(&owl_dma_driver);
959}
960subsys_initcall(owl_dma_init);
961
962static void __exit owl_dma_exit(void)
963{
964 platform_driver_unregister(&owl_dma_driver);
965}
966module_exit(owl_dma_exit);
967
968MODULE_AUTHOR("David Liu <liuwei@actions-semi.com>");
969MODULE_AUTHOR("Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>");
970MODULE_DESCRIPTION("Actions Semi Owl SoCs DMA driver");
971MODULE_LICENSE("GPL");
Linux 6.x block layer and io_uring tree(s)