projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
treewide: Use array_size() in vzalloc()
[linux-block.git] / drivers / dma / mic_x100_dma.c
CommitLineData
95b4ecbf
SY		 1/*
2 * Intel MIC Platform Software Stack (MPSS)
3 *
4 * Copyright(c) 2014 Intel Corporation.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License, version 2, as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
14 *
15 * The full GNU General Public License is included in this distribution in
16 * the file called "COPYING".
17 *
18 * Intel MIC X100 DMA Driver.
19 *
20 * Adapted from IOAT dma driver.
21 */
22#include <linux/module.h>
23#include <linux/io.h>
24#include <linux/seq_file.h>
d6472302 25#include <linux/vmalloc.h>
95b4ecbf
SY		 26
27#include "mic_x100_dma.h"
28
29#define MIC_DMA_MAX_XFER_SIZE_CARD (1 * 1024 * 1024 -\
30 MIC_DMA_ALIGN_BYTES)
31#define MIC_DMA_MAX_XFER_SIZE_HOST (1 * 1024 * 1024 >> 1)
32#define MIC_DMA_DESC_TYPE_SHIFT 60
33#define MIC_DMA_MEMCPY_LEN_SHIFT 46
34#define MIC_DMA_STAT_INTR_SHIFT 59
35
36/* high-water mark for pushing dma descriptors */
37static int mic_dma_pending_level = 4;
38
39/* Status descriptor is used to write a 64 bit value to a memory location */
40enum mic_dma_desc_format_type {
41 MIC_DMA_MEMCPY = 1,
42 MIC_DMA_STATUS,
43};
44
45static inline u32 mic_dma_hw_ring_inc(u32 val)
46{
47 return (val + 1) % MIC_DMA_DESC_RX_SIZE;
48}
49
50static inline u32 mic_dma_hw_ring_dec(u32 val)
51{
52 return val ? val - 1 : MIC_DMA_DESC_RX_SIZE - 1;
53}
54
55static inline void mic_dma_hw_ring_inc_head(struct mic_dma_chan *ch)
56{
57 ch->head = mic_dma_hw_ring_inc(ch->head);
58}
59
60/* Prepare a memcpy desc */
61static inline void mic_dma_memcpy_desc(struct mic_dma_desc *desc,
62 dma_addr_t src_phys, dma_addr_t dst_phys, u64 size)
63{
64 u64 qw0, qw1;
65
66 qw0 = src_phys;
67 qw0 |= (size >> MIC_DMA_ALIGN_SHIFT) << MIC_DMA_MEMCPY_LEN_SHIFT;
68 qw1 = MIC_DMA_MEMCPY;
69 qw1 <<= MIC_DMA_DESC_TYPE_SHIFT;
70 qw1 |= dst_phys;
71 desc->qw0 = qw0;
72 desc->qw1 = qw1;
73}
74
75/* Prepare a status desc. with @data to be written at @dst_phys */
76static inline void mic_dma_prep_status_desc(struct mic_dma_desc *desc, u64 data,
77 dma_addr_t dst_phys, bool generate_intr)
78{
79 u64 qw0, qw1;
80
81 qw0 = data;
82 qw1 = (u64) MIC_DMA_STATUS << MIC_DMA_DESC_TYPE_SHIFT | dst_phys;
83 if (generate_intr)
84 qw1 |= (1ULL << MIC_DMA_STAT_INTR_SHIFT);
85 desc->qw0 = qw0;
86 desc->qw1 = qw1;
87}
88
89static void mic_dma_cleanup(struct mic_dma_chan *ch)
90{
91 struct dma_async_tx_descriptor *tx;
92 u32 tail;
93 u32 last_tail;
94
95 spin_lock(&ch->cleanup_lock);
96 tail = mic_dma_read_cmp_cnt(ch);
97 /*
98 * This is the barrier pair for smp_wmb() in fn.
99 * mic_dma_tx_submit_unlock. It's required so that we read the
100 * updated cookie value from tx->cookie.
101 */
102 smp_rmb();
103 for (last_tail = ch->last_tail; tail != last_tail;) {
104 tx = &ch->tx_array[last_tail];
105 if (tx->cookie) {
106 dma_cookie_complete(tx);
7a883acd
DJ		 107 dmaengine_desc_get_callback_invoke(tx, NULL);
108 tx->callback = NULL;
95b4ecbf
SY		 109 }
110 last_tail = mic_dma_hw_ring_inc(last_tail);
111 }
112 /* finish all completion callbacks before incrementing tail */
113 smp_mb();
114 ch->last_tail = last_tail;
115 spin_unlock(&ch->cleanup_lock);
116}
117
118static u32 mic_dma_ring_count(u32 head, u32 tail)
119{
120 u32 count;
121
122 if (head >= tail)
123 count = (tail - 0) + (MIC_DMA_DESC_RX_SIZE - head);
124 else
125 count = tail - head;
126 return count - 1;
127}
128
129/* Returns the num. of free descriptors on success, -ENOMEM on failure */
130static int mic_dma_avail_desc_ring_space(struct mic_dma_chan *ch, int required)
131{
132 struct device *dev = mic_dma_ch_to_device(ch);
133 u32 count;
134
135 count = mic_dma_ring_count(ch->head, ch->last_tail);
136 if (count < required) {
137 mic_dma_cleanup(ch);
138 count = mic_dma_ring_count(ch->head, ch->last_tail);
139 }
140
141 if (count < required) {
142 dev_dbg(dev, "Not enough desc space");
143 dev_dbg(dev, "%s %d required=%u, avail=%u\n",
144 __func__, __LINE__, required, count);
145 return -ENOMEM;
146 } else {
147 return count;
148 }
149}
150
151/* Program memcpy descriptors into the descriptor ring and update s/w head ptr*/
152static int mic_dma_prog_memcpy_desc(struct mic_dma_chan *ch, dma_addr_t src,
153 dma_addr_t dst, size_t len)
154{
155 size_t current_transfer_len;
156 size_t max_xfer_size = to_mic_dma_dev(ch)->max_xfer_size;
157 /* 3 is added to make sure we have enough space for status desc */
158 int num_desc = len / max_xfer_size + 3;
159 int ret;
160
161 if (len % max_xfer_size)
162 num_desc++;
163
164 ret = mic_dma_avail_desc_ring_space(ch, num_desc);
165 if (ret < 0)
166 return ret;
167 do {
168 current_transfer_len = min(len, max_xfer_size);
169 mic_dma_memcpy_desc(&ch->desc_ring[ch->head],
170 src, dst, current_transfer_len);
171 mic_dma_hw_ring_inc_head(ch);
172 len -= current_transfer_len;
173 dst = dst + current_transfer_len;
174 src = src + current_transfer_len;
175 } while (len > 0);
176 return 0;
177}
178
179/* It's a h/w quirk and h/w needs 2 status descriptors for every status desc */
180static void mic_dma_prog_intr(struct mic_dma_chan *ch)
181{
182 mic_dma_prep_status_desc(&ch->desc_ring[ch->head], 0,
183 ch->status_dest_micpa, false);
184 mic_dma_hw_ring_inc_head(ch);
185 mic_dma_prep_status_desc(&ch->desc_ring[ch->head], 0,
186 ch->status_dest_micpa, true);
187 mic_dma_hw_ring_inc_head(ch);
188}
189
190/* Wrapper function to program memcpy descriptors/status descriptors */
191static int mic_dma_do_dma(struct mic_dma_chan *ch, int flags, dma_addr_t src,
192 dma_addr_t dst, size_t len)
193{
ff39988a 194 if (len && -ENOMEM == mic_dma_prog_memcpy_desc(ch, src, dst, len)) {
95b4ecbf 195 return -ENOMEM;
ff39988a
SY		 196 } else {
197 /* 3 is the maximum number of status descriptors */
198 int ret = mic_dma_avail_desc_ring_space(ch, 3);
199
200 if (ret < 0)
201 return ret;
202 }
203
95b4ecbf
SY		 204 /* Above mic_dma_prog_memcpy_desc() makes sure we have enough space */
205 if (flags & DMA_PREP_FENCE) {
206 mic_dma_prep_status_desc(&ch->desc_ring[ch->head], 0,
207 ch->status_dest_micpa, false);
208 mic_dma_hw_ring_inc_head(ch);
209 }
210
211 if (flags & DMA_PREP_INTERRUPT)
212 mic_dma_prog_intr(ch);
213
214 return 0;
215}
216
217static inline void mic_dma_issue_pending(struct dma_chan *ch)
218{
219 struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
220
221 spin_lock(&mic_ch->issue_lock);
222 /*
223 * Write to head triggers h/w to act on the descriptors.
224 * On MIC, writing the same head value twice causes
225 * a h/w error. On second write, h/w assumes we filled
226 * the entire ring & overwrote some of the descriptors.
227 */
228 if (mic_ch->issued == mic_ch->submitted)
229 goto out;
230 mic_ch->issued = mic_ch->submitted;
231 /*
232 * make descriptor updates visible before advancing head,
233 * this is purposefully not smp_wmb() since we are also
234 * publishing the descriptor updates to a dma device
235 */
236 wmb();
237 mic_dma_write_reg(mic_ch, MIC_DMA_REG_DHPR, mic_ch->issued);
238out:
239 spin_unlock(&mic_ch->issue_lock);
240}
241
242static inline void mic_dma_update_pending(struct mic_dma_chan *ch)
243{
244 if (mic_dma_ring_count(ch->issued, ch->submitted)
245 > mic_dma_pending_level)
246 mic_dma_issue_pending(&ch->api_ch);
247}
248
249static dma_cookie_t mic_dma_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
250{
251 struct mic_dma_chan *mic_ch = to_mic_dma_chan(tx->chan);
252 dma_cookie_t cookie;
253
254 dma_cookie_assign(tx);
255 cookie = tx->cookie;
256 /*
257 * We need an smp write barrier here because another CPU might see
258 * an update to submitted and update h/w head even before we
259 * assigned a cookie to this tx.
260 */
261 smp_wmb();
262 mic_ch->submitted = mic_ch->head;
263 spin_unlock(&mic_ch->prep_lock);
264 mic_dma_update_pending(mic_ch);
265 return cookie;
266}
267
268static inline struct dma_async_tx_descriptor *
269allocate_tx(struct mic_dma_chan *ch)
270{
271 u32 idx = mic_dma_hw_ring_dec(ch->head);
272 struct dma_async_tx_descriptor *tx = &ch->tx_array[idx];
273
274 dma_async_tx_descriptor_init(tx, &ch->api_ch);
275 tx->tx_submit = mic_dma_tx_submit_unlock;
276 return tx;
277}
278
ff39988a
SY		 279/* Program a status descriptor with dst as address and value to be written */
280static struct dma_async_tx_descriptor *
281mic_dma_prep_status_lock(struct dma_chan *ch, dma_addr_t dst, u64 src_val,
282 unsigned long flags)
283{
284 struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
285 int result;
286
287 spin_lock(&mic_ch->prep_lock);
288 result = mic_dma_avail_desc_ring_space(mic_ch, 4);
289 if (result < 0)
290 goto error;
291 mic_dma_prep_status_desc(&mic_ch->desc_ring[mic_ch->head], src_val, dst,
292 false);
293 mic_dma_hw_ring_inc_head(mic_ch);
294 result = mic_dma_do_dma(mic_ch, flags, 0, 0, 0);
295 if (result < 0)
296 goto error;
297
298 return allocate_tx(mic_ch);
299error:
300 dev_err(mic_dma_ch_to_device(mic_ch),
301 "Error enqueueing dma status descriptor, error=%d\n", result);
302 spin_unlock(&mic_ch->prep_lock);
303 return NULL;
304}
305
95b4ecbf
SY		 306/*
307 * Prepare a memcpy descriptor to be added to the ring.
308 * Note that the temporary descriptor adds an extra overhead of copying the
309 * descriptor to ring. So, we copy directly to the descriptor ring
310 */
311static struct dma_async_tx_descriptor *
312mic_dma_prep_memcpy_lock(struct dma_chan *ch, dma_addr_t dma_dest,
313 dma_addr_t dma_src, size_t len, unsigned long flags)
314{
315 struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
316 struct device *dev = mic_dma_ch_to_device(mic_ch);
317 int result;
318
319 if (!len && !flags)
320 return NULL;
321
322 spin_lock(&mic_ch->prep_lock);
323 result = mic_dma_do_dma(mic_ch, flags, dma_src, dma_dest, len);
324 if (result >= 0)
16605e8d
AD		 325 return allocate_tx(mic_ch);
326 dev_err(dev, "Error enqueueing dma, error=%d\n", result);
95b4ecbf 327 spin_unlock(&mic_ch->prep_lock);
16605e8d 328 return NULL;
95b4ecbf
SY		 329}
330
331static struct dma_async_tx_descriptor *
332mic_dma_prep_interrupt_lock(struct dma_chan *ch, unsigned long flags)
333{
334 struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
335 int ret;
336
337 spin_lock(&mic_ch->prep_lock);
338 ret = mic_dma_do_dma(mic_ch, flags, 0, 0, 0);
339 if (!ret)
16605e8d 340 return allocate_tx(mic_ch);
95b4ecbf 341 spin_unlock(&mic_ch->prep_lock);
16605e8d 342 return NULL;
95b4ecbf
SY		 343}
344
345/* Return the status of the transaction */
346static enum dma_status
347mic_dma_tx_status(struct dma_chan *ch, dma_cookie_t cookie,
348 struct dma_tx_state *txstate)
349{
350 struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
351
352 if (DMA_COMPLETE != dma_cookie_status(ch, cookie, txstate))
353 mic_dma_cleanup(mic_ch);
354
355 return dma_cookie_status(ch, cookie, txstate);
356}
357
358static irqreturn_t mic_dma_thread_fn(int irq, void *data)
359{
360 mic_dma_cleanup((struct mic_dma_chan *)data);
361 return IRQ_HANDLED;
362}
363
364static irqreturn_t mic_dma_intr_handler(int irq, void *data)
365{
366 struct mic_dma_chan *ch = ((struct mic_dma_chan *)data);
367
368 mic_dma_ack_interrupt(ch);
369 return IRQ_WAKE_THREAD;
370}
371
372static int mic_dma_alloc_desc_ring(struct mic_dma_chan *ch)
373{
374 u64 desc_ring_size = MIC_DMA_DESC_RX_SIZE * sizeof(*ch->desc_ring);
375 struct device *dev = &to_mbus_device(ch)->dev;
376
377 desc_ring_size = ALIGN(desc_ring_size, MIC_DMA_ALIGN_BYTES);
378 ch->desc_ring = kzalloc(desc_ring_size, GFP_KERNEL);
379
380 if (!ch->desc_ring)
381 return -ENOMEM;
382
383 ch->desc_ring_micpa = dma_map_single(dev, ch->desc_ring,
384 desc_ring_size, DMA_BIDIRECTIONAL);
385 if (dma_mapping_error(dev, ch->desc_ring_micpa))
386 goto map_error;
387
fad953ce
KC		 388 ch->tx_array = vzalloc(array_size(MIC_DMA_DESC_RX_SIZE,
389 sizeof(*ch->tx_array)));
95b4ecbf
SY		 390 if (!ch->tx_array)
391 goto tx_error;
392 return 0;
393tx_error:
394 dma_unmap_single(dev, ch->desc_ring_micpa, desc_ring_size,
395 DMA_BIDIRECTIONAL);
396map_error:
397 kfree(ch->desc_ring);
398 return -ENOMEM;
399}
400
401static void mic_dma_free_desc_ring(struct mic_dma_chan *ch)
402{
403 u64 desc_ring_size = MIC_DMA_DESC_RX_SIZE * sizeof(*ch->desc_ring);
404
405 vfree(ch->tx_array);
406 desc_ring_size = ALIGN(desc_ring_size, MIC_DMA_ALIGN_BYTES);
407 dma_unmap_single(&to_mbus_device(ch)->dev, ch->desc_ring_micpa,
408 desc_ring_size, DMA_BIDIRECTIONAL);
409 kfree(ch->desc_ring);
410 ch->desc_ring = NULL;
411}
412
413static void mic_dma_free_status_dest(struct mic_dma_chan *ch)
414{
415 dma_unmap_single(&to_mbus_device(ch)->dev, ch->status_dest_micpa,
416 L1_CACHE_BYTES, DMA_BIDIRECTIONAL);
417 kfree(ch->status_dest);
418}
419
420static int mic_dma_alloc_status_dest(struct mic_dma_chan *ch)
421{
422 struct device *dev = &to_mbus_device(ch)->dev;
423
424 ch->status_dest = kzalloc(L1_CACHE_BYTES, GFP_KERNEL);
425 if (!ch->status_dest)
426 return -ENOMEM;
427 ch->status_dest_micpa = dma_map_single(dev, ch->status_dest,
428 L1_CACHE_BYTES, DMA_BIDIRECTIONAL);
429 if (dma_mapping_error(dev, ch->status_dest_micpa)) {
430 kfree(ch->status_dest);
431 ch->status_dest = NULL;
432 return -ENOMEM;
433 }
434 return 0;
435}
436
437static int mic_dma_check_chan(struct mic_dma_chan *ch)
438{
439 if (mic_dma_read_reg(ch, MIC_DMA_REG_DCHERR) ||
440 mic_dma_read_reg(ch, MIC_DMA_REG_DSTAT) & MIC_DMA_CHAN_QUIESCE) {
441 mic_dma_disable_chan(ch);
442 mic_dma_chan_mask_intr(ch);
443 dev_err(mic_dma_ch_to_device(ch),
444 "%s %d error setting up mic dma chan %d\n",
445 __func__, __LINE__, ch->ch_num);
446 return -EBUSY;
447 }
448 return 0;
449}
450
451static int mic_dma_chan_setup(struct mic_dma_chan *ch)
452{
453 if (MIC_DMA_CHAN_MIC == ch->owner)
454 mic_dma_chan_set_owner(ch);
455 mic_dma_disable_chan(ch);
456 mic_dma_chan_mask_intr(ch);
457 mic_dma_write_reg(ch, MIC_DMA_REG_DCHERRMSK, 0);
458 mic_dma_chan_set_desc_ring(ch);
459 ch->last_tail = mic_dma_read_reg(ch, MIC_DMA_REG_DTPR);
460 ch->head = ch->last_tail;
461 ch->issued = 0;
462 mic_dma_chan_unmask_intr(ch);
463 mic_dma_enable_chan(ch);
464 return mic_dma_check_chan(ch);
465}
466
467static void mic_dma_chan_destroy(struct mic_dma_chan *ch)
468{
469 mic_dma_disable_chan(ch);
470 mic_dma_chan_mask_intr(ch);
471}
472
473static void mic_dma_unregister_dma_device(struct mic_dma_device *mic_dma_dev)
474{
475 dma_async_device_unregister(&mic_dma_dev->dma_dev);
476}
477
478static int mic_dma_setup_irq(struct mic_dma_chan *ch)
479{
480 ch->cookie =
481 to_mbus_hw_ops(ch)->request_threaded_irq(to_mbus_device(ch),
482 mic_dma_intr_handler, mic_dma_thread_fn,
483 "mic dma_channel", ch, ch->ch_num);
a8ffa34f 484 return PTR_ERR_OR_ZERO(ch->cookie);
95b4ecbf
SY		 485}
486
487static inline void mic_dma_free_irq(struct mic_dma_chan *ch)
488{
489 to_mbus_hw_ops(ch)->free_irq(to_mbus_device(ch), ch->cookie, ch);
490}
491
492static int mic_dma_chan_init(struct mic_dma_chan *ch)
493{
494 int ret = mic_dma_alloc_desc_ring(ch);
495
496 if (ret)
497 goto ring_error;
498 ret = mic_dma_alloc_status_dest(ch);
499 if (ret)
500 goto status_error;
501 ret = mic_dma_chan_setup(ch);
502 if (ret)
503 goto chan_error;
504 return ret;
505chan_error:
506 mic_dma_free_status_dest(ch);
507status_error:
508 mic_dma_free_desc_ring(ch);
509ring_error:
510 return ret;
511}
512
513static int mic_dma_drain_chan(struct mic_dma_chan *ch)
514{
515 struct dma_async_tx_descriptor *tx;
516 int err = 0;
517 dma_cookie_t cookie;
518
519 tx = mic_dma_prep_memcpy_lock(&ch->api_ch, 0, 0, 0, DMA_PREP_FENCE);
520 if (!tx) {
521 err = -ENOMEM;
522 goto error;
523 }
524
525 cookie = tx->tx_submit(tx);
526 if (dma_submit_error(cookie))
527 err = -ENOMEM;
528 else
529 err = dma_sync_wait(&ch->api_ch, cookie);
530 if (err) {
531 dev_err(mic_dma_ch_to_device(ch), "%s %d TO chan 0x%x\n",
532 __func__, __LINE__, ch->ch_num);
533 err = -EIO;
534 }
535error:
536 mic_dma_cleanup(ch);
537 return err;
538}
539
540static inline void mic_dma_chan_uninit(struct mic_dma_chan *ch)
541{
542 mic_dma_chan_destroy(ch);
543 mic_dma_cleanup(ch);
544 mic_dma_free_status_dest(ch);
545 mic_dma_free_desc_ring(ch);
546}
547
548static int mic_dma_init(struct mic_dma_device *mic_dma_dev,
549 enum mic_dma_chan_owner owner)
550{
551 int i, first_chan = mic_dma_dev->start_ch;
552 struct mic_dma_chan *ch;
553 int ret;
554
555 for (i = first_chan; i < first_chan + MIC_DMA_NUM_CHAN; i++) {
95b4ecbf 556 ch = &mic_dma_dev->mic_ch[i];
95b4ecbf
SY		 557 ch->ch_num = i;
558 ch->owner = owner;
559 spin_lock_init(&ch->cleanup_lock);
560 spin_lock_init(&ch->prep_lock);
561 spin_lock_init(&ch->issue_lock);
562 ret = mic_dma_setup_irq(ch);
563 if (ret)
564 goto error;
565 }
566 return 0;
567error:
568 for (i = i - 1; i >= first_chan; i--)
569 mic_dma_free_irq(ch);
570 return ret;
571}
572
573static void mic_dma_uninit(struct mic_dma_device *mic_dma_dev)
574{
575 int i, first_chan = mic_dma_dev->start_ch;
576 struct mic_dma_chan *ch;
577
578 for (i = first_chan; i < first_chan + MIC_DMA_NUM_CHAN; i++) {
579 ch = &mic_dma_dev->mic_ch[i];
580 mic_dma_free_irq(ch);
581 }
582}
583
584static int mic_dma_alloc_chan_resources(struct dma_chan *ch)
585{
586 int ret = mic_dma_chan_init(to_mic_dma_chan(ch));
587 if (ret)
588 return ret;
589 return MIC_DMA_DESC_RX_SIZE;
590}
591
592static void mic_dma_free_chan_resources(struct dma_chan *ch)
593{
594 struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
595 mic_dma_drain_chan(mic_ch);
596 mic_dma_chan_uninit(mic_ch);
597}
598
599/* Set the fn. handlers and register the dma device with dma api */
600static int mic_dma_register_dma_device(struct mic_dma_device *mic_dma_dev,
601 enum mic_dma_chan_owner owner)
602{
603 int i, first_chan = mic_dma_dev->start_ch;
604
605 dma_cap_zero(mic_dma_dev->dma_dev.cap_mask);
606 /*
607 * This dma engine is not capable of host memory to host memory
608 * transfers
609 */
610 dma_cap_set(DMA_MEMCPY, mic_dma_dev->dma_dev.cap_mask);
611
612 if (MIC_DMA_CHAN_HOST == owner)
613 dma_cap_set(DMA_PRIVATE, mic_dma_dev->dma_dev.cap_mask);
614 mic_dma_dev->dma_dev.device_alloc_chan_resources =
615 mic_dma_alloc_chan_resources;
616 mic_dma_dev->dma_dev.device_free_chan_resources =
617 mic_dma_free_chan_resources;
618 mic_dma_dev->dma_dev.device_tx_status = mic_dma_tx_status;
619 mic_dma_dev->dma_dev.device_prep_dma_memcpy = mic_dma_prep_memcpy_lock;
ff39988a
SY		 620 mic_dma_dev->dma_dev.device_prep_dma_imm_data =
621 mic_dma_prep_status_lock;
95b4ecbf
SY		 622 mic_dma_dev->dma_dev.device_prep_dma_interrupt =
623 mic_dma_prep_interrupt_lock;
624 mic_dma_dev->dma_dev.device_issue_pending = mic_dma_issue_pending;
625 mic_dma_dev->dma_dev.copy_align = MIC_DMA_ALIGN_SHIFT;
626 INIT_LIST_HEAD(&mic_dma_dev->dma_dev.channels);
627 for (i = first_chan; i < first_chan + MIC_DMA_NUM_CHAN; i++) {
628 mic_dma_dev->mic_ch[i].api_ch.device = &mic_dma_dev->dma_dev;
629 dma_cookie_init(&mic_dma_dev->mic_ch[i].api_ch);
630 list_add_tail(&mic_dma_dev->mic_ch[i].api_ch.device_node,
631 &mic_dma_dev->dma_dev.channels);
632 }
633 return dma_async_device_register(&mic_dma_dev->dma_dev);
634}
635
636/*
637 * Initializes dma channels and registers the dma device with the
638 * dma engine api.
639 */
640static struct mic_dma_device *mic_dma_dev_reg(struct mbus_device *mbdev,
641 enum mic_dma_chan_owner owner)
642{
643 struct mic_dma_device *mic_dma_dev;
644 int ret;
645 struct device *dev = &mbdev->dev;
646
647 mic_dma_dev = kzalloc(sizeof(*mic_dma_dev), GFP_KERNEL);
648 if (!mic_dma_dev) {
649 ret = -ENOMEM;
650 goto alloc_error;
651 }
652 mic_dma_dev->mbdev = mbdev;
653 mic_dma_dev->dma_dev.dev = dev;
654 mic_dma_dev->mmio = mbdev->mmio_va;
655 if (MIC_DMA_CHAN_HOST == owner) {
656 mic_dma_dev->start_ch = 0;
657 mic_dma_dev->max_xfer_size = MIC_DMA_MAX_XFER_SIZE_HOST;
658 } else {
659 mic_dma_dev->start_ch = 4;
660 mic_dma_dev->max_xfer_size = MIC_DMA_MAX_XFER_SIZE_CARD;
661 }
662 ret = mic_dma_init(mic_dma_dev, owner);
663 if (ret)
664 goto init_error;
665 ret = mic_dma_register_dma_device(mic_dma_dev, owner);
666 if (ret)
667 goto reg_error;
668 return mic_dma_dev;
669reg_error:
670 mic_dma_uninit(mic_dma_dev);
671init_error:
672 kfree(mic_dma_dev);
673 mic_dma_dev = NULL;
674alloc_error:
675 dev_err(dev, "Error at %s %d ret=%d\n", __func__, __LINE__, ret);
676 return mic_dma_dev;
677}
678
679static void mic_dma_dev_unreg(struct mic_dma_device *mic_dma_dev)
680{
681 mic_dma_unregister_dma_device(mic_dma_dev);
682 mic_dma_uninit(mic_dma_dev);
683 kfree(mic_dma_dev);
684}
685
686/* DEBUGFS CODE */
687static int mic_dma_reg_seq_show(struct seq_file *s, void *pos)
688{
689 struct mic_dma_device *mic_dma_dev = s->private;
690 int i, chan_num, first_chan = mic_dma_dev->start_ch;
691 struct mic_dma_chan *ch;
692
693 seq_printf(s, "SBOX_DCR: %#x\n",
694 mic_dma_mmio_read(&mic_dma_dev->mic_ch[first_chan],
695 MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR));
696 seq_puts(s, "DMA Channel Registers\n");
697 seq_printf(s, "%-10s| %-10s %-10s %-10s %-10s %-10s",
698 "Channel", "DCAR", "DTPR", "DHPR", "DRAR_HI", "DRAR_LO");
699 seq_printf(s, " %-11s %-14s %-10s\n", "DCHERR", "DCHERRMSK", "DSTAT");
700 for (i = first_chan; i < first_chan + MIC_DMA_NUM_CHAN; i++) {
701 ch = &mic_dma_dev->mic_ch[i];
702 chan_num = ch->ch_num;
703 seq_printf(s, "%-10i| %-#10x %-#10x %-#10x %-#10x",
704 chan_num,
705 mic_dma_read_reg(ch, MIC_DMA_REG_DCAR),
706 mic_dma_read_reg(ch, MIC_DMA_REG_DTPR),
707 mic_dma_read_reg(ch, MIC_DMA_REG_DHPR),
708 mic_dma_read_reg(ch, MIC_DMA_REG_DRAR_HI));
709 seq_printf(s, " %-#10x %-#10x %-#14x %-#10x\n",
710 mic_dma_read_reg(ch, MIC_DMA_REG_DRAR_LO),
711 mic_dma_read_reg(ch, MIC_DMA_REG_DCHERR),
712 mic_dma_read_reg(ch, MIC_DMA_REG_DCHERRMSK),
713 mic_dma_read_reg(ch, MIC_DMA_REG_DSTAT));
714 }
715 return 0;
716}
717
718static int mic_dma_reg_debug_open(struct inode *inode, struct file *file)
719{
720 return single_open(file, mic_dma_reg_seq_show, inode->i_private);
721}
722
723static int mic_dma_reg_debug_release(struct inode *inode, struct file *file)
724{
725 return single_release(inode, file);
726}
727
728static const struct file_operations mic_dma_reg_ops = {
729 .owner = THIS_MODULE,
730 .open = mic_dma_reg_debug_open,
731 .read = seq_read,
732 .llseek = seq_lseek,
733 .release = mic_dma_reg_debug_release
734};
735
736/* Debugfs parent dir */
737static struct dentry *mic_dma_dbg;
738
739static int mic_dma_driver_probe(struct mbus_device *mbdev)
740{
741 struct mic_dma_device *mic_dma_dev;
742 enum mic_dma_chan_owner owner;
743
744 if (MBUS_DEV_DMA_MIC == mbdev->id.device)
745 owner = MIC_DMA_CHAN_MIC;
746 else
747 owner = MIC_DMA_CHAN_HOST;
748
749 mic_dma_dev = mic_dma_dev_reg(mbdev, owner);
750 dev_set_drvdata(&mbdev->dev, mic_dma_dev);
751
752 if (mic_dma_dbg) {
753 mic_dma_dev->dbg_dir = debugfs_create_dir(dev_name(&mbdev->dev),
754 mic_dma_dbg);
755 if (mic_dma_dev->dbg_dir)
756 debugfs_create_file("mic_dma_reg", 0444,
757 mic_dma_dev->dbg_dir, mic_dma_dev,
758 &mic_dma_reg_ops);
759 }
760 return 0;
761}
762
763static void mic_dma_driver_remove(struct mbus_device *mbdev)
764{
765 struct mic_dma_device *mic_dma_dev;
766
767 mic_dma_dev = dev_get_drvdata(&mbdev->dev);
768 debugfs_remove_recursive(mic_dma_dev->dbg_dir);
769 mic_dma_dev_unreg(mic_dma_dev);
770}
771
772static struct mbus_device_id id_table[] = {
773 {MBUS_DEV_DMA_MIC, MBUS_DEV_ANY_ID},
774 {MBUS_DEV_DMA_HOST, MBUS_DEV_ANY_ID},
775 {0},
776};
777
778static struct mbus_driver mic_dma_driver = {
779 .driver.name = KBUILD_MODNAME,
780 .driver.owner = THIS_MODULE,
781 .id_table = id_table,
782 .probe = mic_dma_driver_probe,
783 .remove = mic_dma_driver_remove,
784};
785
786static int __init mic_x100_dma_init(void)
787{
788 int rc = mbus_register_driver(&mic_dma_driver);
789 if (rc)
790 return rc;
791 mic_dma_dbg = debugfs_create_dir(KBUILD_MODNAME, NULL);
792 return 0;
793}
794
795static void __exit mic_x100_dma_exit(void)
796{
797 debugfs_remove_recursive(mic_dma_dbg);
798 mbus_unregister_driver(&mic_dma_driver);
799}
800
801module_init(mic_x100_dma_init);
802module_exit(mic_x100_dma_exit);
803
804MODULE_DEVICE_TABLE(mbus, id_table);
805MODULE_AUTHOR("Intel Corporation");
806MODULE_DESCRIPTION("Intel(R) MIC X100 DMA Driver");
807MODULE_LICENSE("GPL v2");
Linux 6.x block layer and io_uring tree(s)