projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ARM: tegra: add clk_prepare/clk_unprepare
[linux-2.6-block.git] / arch / arm / mach-tegra / dma.c
CommitLineData
4de3a8fa
CC		 1/*
2 * arch/arm/mach-tegra/dma.c
3 *
4 * System DMA driver for NVIDIA Tegra SoCs
5 *
6 * Copyright (c) 2008-2009, NVIDIA Corporation.
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * more details.
17 *
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 */
22
23#include <linux/io.h>
24#include <linux/interrupt.h>
25#include <linux/module.h>
26#include <linux/spinlock.h>
27#include <linux/err.h>
28#include <linux/irq.h>
29#include <linux/delay.h>
1ca00347 30#include <linux/clk.h>
4de3a8fa
CC		 31#include <mach/dma.h>
32#include <mach/irqs.h>
33#include <mach/iomap.h>
2ea67fd1 34#include <mach/suspend.h>
4de3a8fa 35
e2f91578
OJ		 36#include "apbio.h"
37
4de3a8fa
CC		 38#define APB_DMA_GEN 0x000
39#define GEN_ENABLE (1<<31)
40
41#define APB_DMA_CNTRL 0x010
42
43#define APB_DMA_IRQ_MASK 0x01c
44
45#define APB_DMA_IRQ_MASK_SET 0x020
46
47#define APB_DMA_CHAN_CSR 0x000
48#define CSR_ENB (1<<31)
49#define CSR_IE_EOC (1<<30)
50#define CSR_HOLD (1<<29)
51#define CSR_DIR (1<<28)
52#define CSR_ONCE (1<<27)
53#define CSR_FLOW (1<<21)
54#define CSR_REQ_SEL_SHIFT 16
4de3a8fa
CC		 55#define CSR_WCOUNT_SHIFT 2
56#define CSR_WCOUNT_MASK 0xFFFC
57
58#define APB_DMA_CHAN_STA 0x004
59#define STA_BUSY (1<<31)
60#define STA_ISE_EOC (1<<30)
61#define STA_HALT (1<<29)
62#define STA_PING_PONG (1<<28)
63#define STA_COUNT_SHIFT 2
64#define STA_COUNT_MASK 0xFFFC
65
66#define APB_DMA_CHAN_AHB_PTR 0x010
67
68#define APB_DMA_CHAN_AHB_SEQ 0x014
69#define AHB_SEQ_INTR_ENB (1<<31)
70#define AHB_SEQ_BUS_WIDTH_SHIFT 28
71#define AHB_SEQ_BUS_WIDTH_MASK (0x7<<AHB_SEQ_BUS_WIDTH_SHIFT)
72#define AHB_SEQ_BUS_WIDTH_8 (0<<AHB_SEQ_BUS_WIDTH_SHIFT)
73#define AHB_SEQ_BUS_WIDTH_16 (1<<AHB_SEQ_BUS_WIDTH_SHIFT)
74#define AHB_SEQ_BUS_WIDTH_32 (2<<AHB_SEQ_BUS_WIDTH_SHIFT)
75#define AHB_SEQ_BUS_WIDTH_64 (3<<AHB_SEQ_BUS_WIDTH_SHIFT)
76#define AHB_SEQ_BUS_WIDTH_128 (4<<AHB_SEQ_BUS_WIDTH_SHIFT)
77#define AHB_SEQ_DATA_SWAP (1<<27)
78#define AHB_SEQ_BURST_MASK (0x7<<24)
79#define AHB_SEQ_BURST_1 (4<<24)
80#define AHB_SEQ_BURST_4 (5<<24)
81#define AHB_SEQ_BURST_8 (6<<24)
82#define AHB_SEQ_DBL_BUF (1<<19)
83#define AHB_SEQ_WRAP_SHIFT 16
84#define AHB_SEQ_WRAP_MASK (0x7<<AHB_SEQ_WRAP_SHIFT)
85
86#define APB_DMA_CHAN_APB_PTR 0x018
87
88#define APB_DMA_CHAN_APB_SEQ 0x01c
89#define APB_SEQ_BUS_WIDTH_SHIFT 28
90#define APB_SEQ_BUS_WIDTH_MASK (0x7<<APB_SEQ_BUS_WIDTH_SHIFT)
91#define APB_SEQ_BUS_WIDTH_8 (0<<APB_SEQ_BUS_WIDTH_SHIFT)
92#define APB_SEQ_BUS_WIDTH_16 (1<<APB_SEQ_BUS_WIDTH_SHIFT)
93#define APB_SEQ_BUS_WIDTH_32 (2<<APB_SEQ_BUS_WIDTH_SHIFT)
94#define APB_SEQ_BUS_WIDTH_64 (3<<APB_SEQ_BUS_WIDTH_SHIFT)
95#define APB_SEQ_BUS_WIDTH_128 (4<<APB_SEQ_BUS_WIDTH_SHIFT)
96#define APB_SEQ_DATA_SWAP (1<<27)
97#define APB_SEQ_WRAP_SHIFT 16
98#define APB_SEQ_WRAP_MASK (0x7<<APB_SEQ_WRAP_SHIFT)
99
100#define TEGRA_SYSTEM_DMA_CH_NR 16
101#define TEGRA_SYSTEM_DMA_AVP_CH_NUM 4
102#define TEGRA_SYSTEM_DMA_CH_MIN 0
103#define TEGRA_SYSTEM_DMA_CH_MAX \
104 (TEGRA_SYSTEM_DMA_CH_NR - TEGRA_SYSTEM_DMA_AVP_CH_NUM - 1)
105
106#define NV_DMA_MAX_TRASFER_SIZE 0x10000
107
cf28cba0 108static const unsigned int ahb_addr_wrap_table[8] = {
4de3a8fa
CC		 109 0, 32, 64, 128, 256, 512, 1024, 2048
110};
111
cf28cba0
OJ		 112static const unsigned int apb_addr_wrap_table[8] = {
113 0, 1, 2, 4, 8, 16, 32, 64
114};
4de3a8fa 115
cf28cba0
OJ		 116static const unsigned int bus_width_table[5] = {
117 8, 16, 32, 64, 128
118};
4de3a8fa
CC		 119
120#define TEGRA_DMA_NAME_SIZE 16
121struct tegra_dma_channel {
122 struct list_head list;
123 int id;
124 spinlock_t lock;
125 char name[TEGRA_DMA_NAME_SIZE];
126 void __iomem *addr;
127 int mode;
128 int irq;
5789fee9 129 int req_transfer_count;
4de3a8fa
CC		 130};
131
132#define NV_DMA_MAX_CHANNELS 32
133
ccac0515 134static bool tegra_dma_initialized;
5789fee9 135static DEFINE_MUTEX(tegra_dma_lock);
cb3732d0 136static DEFINE_SPINLOCK(enable_lock);
5789fee9 137
4de3a8fa
CC		 138static DECLARE_BITMAP(channel_usage, NV_DMA_MAX_CHANNELS);
139static struct tegra_dma_channel dma_channels[NV_DMA_MAX_CHANNELS];
140
141static void tegra_dma_update_hw(struct tegra_dma_channel *ch,
142 struct tegra_dma_req *req);
143static void tegra_dma_update_hw_partial(struct tegra_dma_channel *ch,
144 struct tegra_dma_req *req);
4de3a8fa
CC		 145static void tegra_dma_stop(struct tegra_dma_channel *ch);
146
147void tegra_dma_flush(struct tegra_dma_channel *ch)
148{
149}
150EXPORT_SYMBOL(tegra_dma_flush);
151
152void tegra_dma_dequeue(struct tegra_dma_channel *ch)
153{
154 struct tegra_dma_req *req;
155
5789fee9
CC		 156 if (tegra_dma_is_empty(ch))
157 return;
158
4de3a8fa
CC		 159 req = list_entry(ch->list.next, typeof(*req), node);
160
161 tegra_dma_dequeue_req(ch, req);
162 return;
163}
164
cf28cba0 165static void tegra_dma_stop(struct tegra_dma_channel *ch)
4de3a8fa 166{
5789fee9
CC		 167 u32 csr;
168 u32 status;
4de3a8fa 169
5789fee9 170 csr = readl(ch->addr + APB_DMA_CHAN_CSR);
4de3a8fa
CC		 171 csr &= ~CSR_IE_EOC;
172 writel(csr, ch->addr + APB_DMA_CHAN_CSR);
173
174 csr &= ~CSR_ENB;
175 writel(csr, ch->addr + APB_DMA_CHAN_CSR);
176
177 status = readl(ch->addr + APB_DMA_CHAN_STA);
178 if (status & STA_ISE_EOC)
179 writel(status, ch->addr + APB_DMA_CHAN_STA);
180}
181
cf28cba0 182static int tegra_dma_cancel(struct tegra_dma_channel *ch)
4de3a8fa 183{
4de3a8fa
CC		 184 unsigned long irq_flags;
185
186 spin_lock_irqsave(&ch->lock, irq_flags);
187 while (!list_empty(&ch->list))
188 list_del(ch->list.next);
189
4de3a8fa
CC		 190 tegra_dma_stop(ch);
191
192 spin_unlock_irqrestore(&ch->lock, irq_flags);
193 return 0;
194}
195
cb3732d0
LD		 196static unsigned int get_channel_status(struct tegra_dma_channel *ch,
197 struct tegra_dma_req *req, bool is_stop_dma)
198{
199 void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
200 unsigned int status;
201
202 if (is_stop_dma) {
203 /*
204 * STOP the DMA and get the transfer count.
205 * Getting the transfer count is tricky.
206 * - Globally disable DMA on all channels
207 * - Read the channel's status register to know the number
208 * of pending bytes to be transfered.
209 * - Stop the dma channel
210 * - Globally re-enable DMA to resume other transfers
211 */
212 spin_lock(&enable_lock);
213 writel(0, addr + APB_DMA_GEN);
214 udelay(20);
215 status = readl(ch->addr + APB_DMA_CHAN_STA);
216 tegra_dma_stop(ch);
217 writel(GEN_ENABLE, addr + APB_DMA_GEN);
218 spin_unlock(&enable_lock);
219 if (status & STA_ISE_EOC) {
220 pr_err("Got Dma Int here clearing");
221 writel(status, ch->addr + APB_DMA_CHAN_STA);
222 }
223 req->status = TEGRA_DMA_REQ_ERROR_ABORTED;
224 } else {
225 status = readl(ch->addr + APB_DMA_CHAN_STA);
226 }
227 return status;
228}
229
230/* should be called with the channel lock held */
231static unsigned int dma_active_count(struct tegra_dma_channel *ch,
232 struct tegra_dma_req *req, unsigned int status)
233{
234 unsigned int to_transfer;
235 unsigned int req_transfer_count;
236 unsigned int bytes_transferred;
237
238 to_transfer = ((status & STA_COUNT_MASK) >> STA_COUNT_SHIFT) + 1;
239 req_transfer_count = ch->req_transfer_count + 1;
240 bytes_transferred = req_transfer_count;
241 if (status & STA_BUSY)
242 bytes_transferred -= to_transfer;
243 /*
244 * In continuous transfer mode, DMA only tracks the count of the
245 * half DMA buffer. So, if the DMA already finished half the DMA
246 * then add the half buffer to the completed count.
247 */
248 if (ch->mode & TEGRA_DMA_MODE_CONTINOUS) {
249 if (req->buffer_status == TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL)
250 bytes_transferred += req_transfer_count;
251 if (status & STA_ISE_EOC)
252 bytes_transferred += req_transfer_count;
253 }
254 bytes_transferred *= 4;
255 return bytes_transferred;
256}
257
4de3a8fa
CC		 258int tegra_dma_dequeue_req(struct tegra_dma_channel *ch,
259 struct tegra_dma_req *_req)
260{
4de3a8fa
CC		 261 unsigned int status;
262 struct tegra_dma_req *req = NULL;
263 int found = 0;
264 unsigned long irq_flags;
cb3732d0 265 int stop = 0;
4de3a8fa
CC		 266
267 spin_lock_irqsave(&ch->lock, irq_flags);
cb3732d0
LD		 268
269 if (list_entry(ch->list.next, struct tegra_dma_req, node) == _req)
270 stop = 1;
271
4de3a8fa
CC		 272 list_for_each_entry(req, &ch->list, node) {
273 if (req == _req) {
274 list_del(&req->node);
275 found = 1;
276 break;
277 }
278 }
279 if (!found) {
280 spin_unlock_irqrestore(&ch->lock, irq_flags);
281 return 0;
282 }
283
cb3732d0
LD		 284 if (!stop)
285 goto skip_stop_dma;
4de3a8fa 286
cb3732d0
LD		 287 status = get_channel_status(ch, req, true);
288 req->bytes_transferred = dma_active_count(ch, req, status);
4de3a8fa 289
4de3a8fa
CC		 290 if (!list_empty(&ch->list)) {
291 /* if the list is not empty, queue the next request */
292 struct tegra_dma_req *next_req;
293 next_req = list_entry(ch->list.next,
294 typeof(*next_req), node);
295 tegra_dma_update_hw(ch, next_req);
296 }
cb3732d0
LD		 297
298skip_stop_dma:
4de3a8fa
CC		 299 req->status = -TEGRA_DMA_REQ_ERROR_ABORTED;
300
301 spin_unlock_irqrestore(&ch->lock, irq_flags);
302
303 /* Callback should be called without any lock */
304 req->complete(req);
305 return 0;
306}
307EXPORT_SYMBOL(tegra_dma_dequeue_req);
308
309bool tegra_dma_is_empty(struct tegra_dma_channel *ch)
310{
311 unsigned long irq_flags;
312 bool is_empty;
313
314 spin_lock_irqsave(&ch->lock, irq_flags);
315 if (list_empty(&ch->list))
316 is_empty = true;
317 else
318 is_empty = false;
319 spin_unlock_irqrestore(&ch->lock, irq_flags);
320 return is_empty;
321}
322EXPORT_SYMBOL(tegra_dma_is_empty);
323
324bool tegra_dma_is_req_inflight(struct tegra_dma_channel *ch,
325 struct tegra_dma_req *_req)
326{
327 unsigned long irq_flags;
328 struct tegra_dma_req *req;
329
330 spin_lock_irqsave(&ch->lock, irq_flags);
331 list_for_each_entry(req, &ch->list, node) {
332 if (req == _req) {
333 spin_unlock_irqrestore(&ch->lock, irq_flags);
334 return true;
335 }
336 }
337 spin_unlock_irqrestore(&ch->lock, irq_flags);
338 return false;
339}
340EXPORT_SYMBOL(tegra_dma_is_req_inflight);
341
342int tegra_dma_enqueue_req(struct tegra_dma_channel *ch,
343 struct tegra_dma_req *req)
344{
345 unsigned long irq_flags;
499ef7a5 346 struct tegra_dma_req *_req;
4de3a8fa
CC		 347 int start_dma = 0;
348
349 if (req->size > NV_DMA_MAX_TRASFER_SIZE ||
350 req->source_addr & 0x3 || req->dest_addr & 0x3) {
351 pr_err("Invalid DMA request for channel %d\n", ch->id);
352 return -EINVAL;
353 }
354
355 spin_lock_irqsave(&ch->lock, irq_flags);
356
499ef7a5
SW		 357 list_for_each_entry(_req, &ch->list, node) {
358 if (req == _req) {
359 spin_unlock_irqrestore(&ch->lock, irq_flags);
360 return -EEXIST;
361 }
362 }
363
4de3a8fa
CC		 364 req->bytes_transferred = 0;
365 req->status = 0;
366 req->buffer_status = 0;
367 if (list_empty(&ch->list))
368 start_dma = 1;
369
370 list_add_tail(&req->node, &ch->list);
371
372 if (start_dma)
373 tegra_dma_update_hw(ch, req);
374
375 spin_unlock_irqrestore(&ch->lock, irq_flags);
376
377 return 0;
378}
379EXPORT_SYMBOL(tegra_dma_enqueue_req);
380
381struct tegra_dma_channel *tegra_dma_allocate_channel(int mode)
382{
383 int channel;
5789fee9
CC		 384 struct tegra_dma_channel *ch = NULL;
385
13ae3d5b 386 if (!tegra_dma_initialized)
ccac0515
SW		 387 return NULL;
388
5789fee9 389 mutex_lock(&tegra_dma_lock);
4de3a8fa
CC		 390
391 /* first channel is the shared channel */
392 if (mode & TEGRA_DMA_SHARED) {
393 channel = TEGRA_SYSTEM_DMA_CH_MIN;
394 } else {
395 channel = find_first_zero_bit(channel_usage,
396 ARRAY_SIZE(dma_channels));
397 if (channel >= ARRAY_SIZE(dma_channels))
5789fee9 398 goto out;
4de3a8fa
CC		 399 }
400 __set_bit(channel, channel_usage);
401 ch = &dma_channels[channel];
402 ch->mode = mode;
5789fee9
CC		 403
404out:
405 mutex_unlock(&tegra_dma_lock);
4de3a8fa
CC		 406 return ch;
407}
408EXPORT_SYMBOL(tegra_dma_allocate_channel);
409
410void tegra_dma_free_channel(struct tegra_dma_channel *ch)
411{
412 if (ch->mode & TEGRA_DMA_SHARED)
413 return;
414 tegra_dma_cancel(ch);
5789fee9 415 mutex_lock(&tegra_dma_lock);
4de3a8fa 416 __clear_bit(ch->id, channel_usage);
5789fee9 417 mutex_unlock(&tegra_dma_lock);
4de3a8fa
CC		 418}
419EXPORT_SYMBOL(tegra_dma_free_channel);
420
421static void tegra_dma_update_hw_partial(struct tegra_dma_channel *ch,
422 struct tegra_dma_req *req)
423{
5789fee9
CC		 424 u32 apb_ptr;
425 u32 ahb_ptr;
426
4de3a8fa 427 if (req->to_memory) {
5789fee9
CC		 428 apb_ptr = req->source_addr;
429 ahb_ptr = req->dest_addr;
4de3a8fa 430 } else {
5789fee9
CC		 431 apb_ptr = req->dest_addr;
432 ahb_ptr = req->source_addr;
4de3a8fa 433 }
5789fee9
CC		 434 writel(apb_ptr, ch->addr + APB_DMA_CHAN_APB_PTR);
435 writel(ahb_ptr, ch->addr + APB_DMA_CHAN_AHB_PTR);
4de3a8fa
CC		 436
437 req->status = TEGRA_DMA_REQ_INFLIGHT;
438 return;
439}
440
441static void tegra_dma_update_hw(struct tegra_dma_channel *ch,
442 struct tegra_dma_req *req)
443{
444 int ahb_addr_wrap;
445 int apb_addr_wrap;
446 int ahb_bus_width;
447 int apb_bus_width;
448 int index;
4de3a8fa 449
5789fee9
CC		 450 u32 ahb_seq;
451 u32 apb_seq;
452 u32 ahb_ptr;
453 u32 apb_ptr;
454 u32 csr;
455
456 csr = CSR_IE_EOC | CSR_FLOW;
457 ahb_seq = AHB_SEQ_INTR_ENB | AHB_SEQ_BURST_1;
458 apb_seq = 0;
4de3a8fa 459
5789fee9 460 csr |= req->req_sel << CSR_REQ_SEL_SHIFT;
4de3a8fa
CC		 461
462 /* One shot mode is always single buffered,
463 * continuous mode is always double buffered
464 * */
465 if (ch->mode & TEGRA_DMA_MODE_ONESHOT) {
5789fee9
CC		 466 csr |= CSR_ONCE;
467 ch->req_transfer_count = (req->size >> 2) - 1;
4de3a8fa 468 } else {
5789fee9 469 ahb_seq |= AHB_SEQ_DBL_BUF;
4de3a8fa
CC		 470
471 /* In double buffered mode, we set the size to half the
472 * requested size and interrupt when half the buffer
473 * is full */
5789fee9 474 ch->req_transfer_count = (req->size >> 3) - 1;
4de3a8fa
CC		 475 }
476
5789fee9
CC		 477 csr |= ch->req_transfer_count << CSR_WCOUNT_SHIFT;
478
4de3a8fa 479 if (req->to_memory) {
5789fee9
CC		 480 apb_ptr = req->source_addr;
481 ahb_ptr = req->dest_addr;
4de3a8fa
CC		 482
483 apb_addr_wrap = req->source_wrap;
484 ahb_addr_wrap = req->dest_wrap;
485 apb_bus_width = req->source_bus_width;
486 ahb_bus_width = req->dest_bus_width;
487
488 } else {
5789fee9
CC		 489 csr |= CSR_DIR;
490 apb_ptr = req->dest_addr;
491 ahb_ptr = req->source_addr;
4de3a8fa
CC		 492
493 apb_addr_wrap = req->dest_wrap;
494 ahb_addr_wrap = req->source_wrap;
495 apb_bus_width = req->dest_bus_width;
496 ahb_bus_width = req->source_bus_width;
497 }
498
499 apb_addr_wrap >>= 2;
500 ahb_addr_wrap >>= 2;
501
502 /* set address wrap for APB size */
503 index = 0;
504 do {
505 if (apb_addr_wrap_table[index] == apb_addr_wrap)
506 break;
507 index++;
508 } while (index < ARRAY_SIZE(apb_addr_wrap_table));
509 BUG_ON(index == ARRAY_SIZE(apb_addr_wrap_table));
5789fee9 510 apb_seq |= index << APB_SEQ_WRAP_SHIFT;
4de3a8fa
CC		 511
512 /* set address wrap for AHB size */
513 index = 0;
514 do {
515 if (ahb_addr_wrap_table[index] == ahb_addr_wrap)
516 break;
517 index++;
518 } while (index < ARRAY_SIZE(ahb_addr_wrap_table));
519 BUG_ON(index == ARRAY_SIZE(ahb_addr_wrap_table));
5789fee9 520 ahb_seq |= index << AHB_SEQ_WRAP_SHIFT;
4de3a8fa
CC		 521
522 for (index = 0; index < ARRAY_SIZE(bus_width_table); index++) {
523 if (bus_width_table[index] == ahb_bus_width)
524 break;
525 }
526 BUG_ON(index == ARRAY_SIZE(bus_width_table));
5789fee9 527 ahb_seq |= index << AHB_SEQ_BUS_WIDTH_SHIFT;
4de3a8fa
CC		 528
529 for (index = 0; index < ARRAY_SIZE(bus_width_table); index++) {
530 if (bus_width_table[index] == apb_bus_width)
531 break;
532 }
533 BUG_ON(index == ARRAY_SIZE(bus_width_table));
5789fee9 534 apb_seq |= index << APB_SEQ_BUS_WIDTH_SHIFT;
4de3a8fa 535
5789fee9
CC		 536 writel(csr, ch->addr + APB_DMA_CHAN_CSR);
537 writel(apb_seq, ch->addr + APB_DMA_CHAN_APB_SEQ);
538 writel(apb_ptr, ch->addr + APB_DMA_CHAN_APB_PTR);
539 writel(ahb_seq, ch->addr + APB_DMA_CHAN_AHB_SEQ);
540 writel(ahb_ptr, ch->addr + APB_DMA_CHAN_AHB_PTR);
4de3a8fa 541
5789fee9 542 csr |= CSR_ENB;
4de3a8fa
CC		 543 writel(csr, ch->addr + APB_DMA_CHAN_CSR);
544
545 req->status = TEGRA_DMA_REQ_INFLIGHT;
546}
547
4de3a8fa
CC		 548static void handle_oneshot_dma(struct tegra_dma_channel *ch)
549{
550 struct tegra_dma_req *req;
5789fee9 551 unsigned long irq_flags;
4de3a8fa 552
5789fee9 553 spin_lock_irqsave(&ch->lock, irq_flags);
4de3a8fa 554 if (list_empty(&ch->list)) {
5789fee9 555 spin_unlock_irqrestore(&ch->lock, irq_flags);
4de3a8fa
CC		 556 return;
557 }
558
559 req = list_entry(ch->list.next, typeof(*req), node);
560 if (req) {
561 int bytes_transferred;
562
5789fee9 563 bytes_transferred = ch->req_transfer_count;
4de3a8fa
CC		 564 bytes_transferred += 1;
565 bytes_transferred <<= 2;
566
567 list_del(&req->node);
568 req->bytes_transferred = bytes_transferred;
569 req->status = TEGRA_DMA_REQ_SUCCESS;
570
5789fee9 571 spin_unlock_irqrestore(&ch->lock, irq_flags);
4de3a8fa
CC		 572 /* Callback should be called without any lock */
573 pr_debug("%s: transferred %d bytes\n", __func__,
574 req->bytes_transferred);
575 req->complete(req);
5789fee9 576 spin_lock_irqsave(&ch->lock, irq_flags);
4de3a8fa
CC		 577 }
578
579 if (!list_empty(&ch->list)) {
580 req = list_entry(ch->list.next, typeof(*req), node);
581 /* the complete function we just called may have enqueued
582 another req, in which case dma has already started */
583 if (req->status != TEGRA_DMA_REQ_INFLIGHT)
584 tegra_dma_update_hw(ch, req);
585 }
5789fee9 586 spin_unlock_irqrestore(&ch->lock, irq_flags);
4de3a8fa
CC		 587}
588
589static void handle_continuous_dma(struct tegra_dma_channel *ch)
590{
591 struct tegra_dma_req *req;
5789fee9 592 unsigned long irq_flags;
4de3a8fa 593
5789fee9 594 spin_lock_irqsave(&ch->lock, irq_flags);
4de3a8fa 595 if (list_empty(&ch->list)) {
5789fee9 596 spin_unlock_irqrestore(&ch->lock, irq_flags);
4de3a8fa
CC		 597 return;
598 }
599
600 req = list_entry(ch->list.next, typeof(*req), node);
601 if (req) {
602 if (req->buffer_status == TEGRA_DMA_REQ_BUF_STATUS_EMPTY) {
5789fee9
CC		 603 bool is_dma_ping_complete;
604 is_dma_ping_complete = (readl(ch->addr + APB_DMA_CHAN_STA)
605 & STA_PING_PONG) ? true : false;
606 if (req->to_memory)
607 is_dma_ping_complete = !is_dma_ping_complete;
608 /* Out of sync - Release current buffer */
609 if (!is_dma_ping_complete) {
610 int bytes_transferred;
611
612 bytes_transferred = ch->req_transfer_count;
613 bytes_transferred += 1;
614 bytes_transferred <<= 3;
615 req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_FULL;
616 req->bytes_transferred = bytes_transferred;
617 req->status = TEGRA_DMA_REQ_SUCCESS;
618 tegra_dma_stop(ch);
619
620 if (!list_is_last(&req->node, &ch->list)) {
621 struct tegra_dma_req *next_req;
622
623 next_req = list_entry(req->node.next,
624 typeof(*next_req), node);
625 tegra_dma_update_hw(ch, next_req);
626 }
627
628 list_del(&req->node);
629
630 /* DMA lock is NOT held when callbak is called */
631 spin_unlock_irqrestore(&ch->lock, irq_flags);
632 req->complete(req);
633 return;
634 }
4de3a8fa
CC		 635 /* Load the next request into the hardware, if available
636 * */
637 if (!list_is_last(&req->node, &ch->list)) {
638 struct tegra_dma_req *next_req;
639
640 next_req = list_entry(req->node.next,
641 typeof(*next_req), node);
642 tegra_dma_update_hw_partial(ch, next_req);
643 }
644 req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL;
645 req->status = TEGRA_DMA_REQ_SUCCESS;
646 /* DMA lock is NOT held when callback is called */
5789fee9 647 spin_unlock_irqrestore(&ch->lock, irq_flags);
4de3a8fa
CC		 648 if (likely(req->threshold))
649 req->threshold(req);
650 return;
651
652 } else if (req->buffer_status ==
653 TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL) {
654 /* Callback when the buffer is completely full (i.e on
655 * the second interrupt */
656 int bytes_transferred;
657
5789fee9 658 bytes_transferred = ch->req_transfer_count;
4de3a8fa
CC		 659 bytes_transferred += 1;
660 bytes_transferred <<= 3;
661
662 req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_FULL;
663 req->bytes_transferred = bytes_transferred;
664 req->status = TEGRA_DMA_REQ_SUCCESS;
665 list_del(&req->node);
666
667 /* DMA lock is NOT held when callbak is called */
5789fee9 668 spin_unlock_irqrestore(&ch->lock, irq_flags);
4de3a8fa
CC		 669 req->complete(req);
670 return;
671
672 } else {
673 BUG();
674 }
675 }
5789fee9 676 spin_unlock_irqrestore(&ch->lock, irq_flags);
4de3a8fa
CC		 677}
678
679static irqreturn_t dma_isr(int irq, void *data)
680{
681 struct tegra_dma_channel *ch = data;
682 unsigned long status;
683
684 status = readl(ch->addr + APB_DMA_CHAN_STA);
685 if (status & STA_ISE_EOC)
686 writel(status, ch->addr + APB_DMA_CHAN_STA);
687 else {
688 pr_warning("Got a spurious ISR for DMA channel %d\n", ch->id);
689 return IRQ_HANDLED;
690 }
691 return IRQ_WAKE_THREAD;
692}
693
694static irqreturn_t dma_thread_fn(int irq, void *data)
695{
696 struct tegra_dma_channel *ch = data;
697
698 if (ch->mode & TEGRA_DMA_MODE_ONESHOT)
699 handle_oneshot_dma(ch);
700 else
701 handle_continuous_dma(ch);
702
703
704 return IRQ_HANDLED;
705}
706
707int __init tegra_dma_init(void)
708{
709 int ret = 0;
710 int i;
711 unsigned int irq;
712 void __iomem *addr;
1ca00347
SW		 713 struct clk *c;
714
ccac0515
SW		 715 bitmap_fill(channel_usage, NV_DMA_MAX_CHANNELS);
716
1ca00347
SW		 717 c = clk_get_sys("tegra-dma", NULL);
718 if (IS_ERR(c)) {
719 pr_err("Unable to get clock for APB DMA\n");
720 ret = PTR_ERR(c);
721 goto fail;
722 }
6a5278d0 723 ret = clk_prepare_enable(c);
1ca00347
SW		 724 if (ret != 0) {
725 pr_err("Unable to enable clock for APB DMA\n");
726 goto fail;
727 }
4de3a8fa
CC		 728
729 addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
730 writel(GEN_ENABLE, addr + APB_DMA_GEN);
731 writel(0, addr + APB_DMA_CNTRL);
732 writel(0xFFFFFFFFul >> (31 - TEGRA_SYSTEM_DMA_CH_MAX),
733 addr + APB_DMA_IRQ_MASK_SET);
734
4de3a8fa
CC		 735 for (i = TEGRA_SYSTEM_DMA_CH_MIN; i <= TEGRA_SYSTEM_DMA_CH_MAX; i++) {
736 struct tegra_dma_channel *ch = &dma_channels[i];
737
4de3a8fa
CC		 738 ch->id = i;
739 snprintf(ch->name, TEGRA_DMA_NAME_SIZE, "dma_channel_%d", i);
740
741 ch->addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE +
742 TEGRA_APB_DMA_CH0_SIZE * i);
743
744 spin_lock_init(&ch->lock);
745 INIT_LIST_HEAD(&ch->list);
4de3a8fa
CC		 746
747 irq = INT_APB_DMA_CH0 + i;
748 ret = request_threaded_irq(irq, dma_isr, dma_thread_fn, 0,
749 dma_channels[i].name, ch);
750 if (ret) {
751 pr_err("Failed to register IRQ %d for DMA %d\n",
752 irq, i);
753 goto fail;
754 }
755 ch->irq = irq;
ccac0515
SW		 756
757 __clear_bit(i, channel_usage);
4de3a8fa
CC		 758 }
759 /* mark the shared channel allocated */
760 __set_bit(TEGRA_SYSTEM_DMA_CH_MIN, channel_usage);
761
ccac0515 762 tegra_dma_initialized = true;
4de3a8fa 763
ccac0515 764 return 0;
4de3a8fa
CC		 765fail:
766 writel(0, addr + APB_DMA_GEN);
767 for (i = TEGRA_SYSTEM_DMA_CH_MIN; i <= TEGRA_SYSTEM_DMA_CH_MAX; i++) {
768 struct tegra_dma_channel *ch = &dma_channels[i];
769 if (ch->irq)
770 free_irq(ch->irq, ch);
771 }
772 return ret;
773}
dc54c23b 774postcore_initcall(tegra_dma_init);
4de3a8fa
CC		 775
776#ifdef CONFIG_PM
777static u32 apb_dma[5*TEGRA_SYSTEM_DMA_CH_NR + 3];
778
779void tegra_dma_suspend(void)
780{
781 void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
782 u32 *ctx = apb_dma;
783 int i;
784
785 *ctx++ = readl(addr + APB_DMA_GEN);
786 *ctx++ = readl(addr + APB_DMA_CNTRL);
787 *ctx++ = readl(addr + APB_DMA_IRQ_MASK);
788
789 for (i = 0; i < TEGRA_SYSTEM_DMA_CH_NR; i++) {
790 addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE +
791 TEGRA_APB_DMA_CH0_SIZE * i);
792
793 *ctx++ = readl(addr + APB_DMA_CHAN_CSR);
794 *ctx++ = readl(addr + APB_DMA_CHAN_AHB_PTR);
795 *ctx++ = readl(addr + APB_DMA_CHAN_AHB_SEQ);
796 *ctx++ = readl(addr + APB_DMA_CHAN_APB_PTR);
797 *ctx++ = readl(addr + APB_DMA_CHAN_APB_SEQ);
798 }
799}
800
801void tegra_dma_resume(void)
802{
803 void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
804 u32 *ctx = apb_dma;
805 int i;
806
807 writel(*ctx++, addr + APB_DMA_GEN);
808 writel(*ctx++, addr + APB_DMA_CNTRL);
809 writel(*ctx++, addr + APB_DMA_IRQ_MASK);
810
811 for (i = 0; i < TEGRA_SYSTEM_DMA_CH_NR; i++) {
812 addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE +
813 TEGRA_APB_DMA_CH0_SIZE * i);
814
815 writel(*ctx++, addr + APB_DMA_CHAN_CSR);
816 writel(*ctx++, addr + APB_DMA_CHAN_AHB_PTR);
817 writel(*ctx++, addr + APB_DMA_CHAN_AHB_SEQ);
818 writel(*ctx++, addr + APB_DMA_CHAN_APB_PTR);
819 writel(*ctx++, addr + APB_DMA_CHAN_APB_SEQ);
820 }
821}
822
823#endif
Linux 6.x block layer and io_uring tree(s)