projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'turbostat' of git://git.kernel.org/pub/scm/linux/kernel/git/lenb/linux
[linux-2.6-block.git] / drivers / dma / mic_x100_dma.c
CommitLineData
4e43d779 1// SPDX-License-Identifier: GPL-2.0-only
95b4ecbf
SY		 2/*
3 * Intel MIC Platform Software Stack (MPSS)
4 *
5 * Copyright(c) 2014 Intel Corporation.
6 *
95b4ecbf
SY		 7 * Intel MIC X100 DMA Driver.
8 *
9 * Adapted from IOAT dma driver.
10 */
11#include <linux/module.h>
12#include <linux/io.h>
13#include <linux/seq_file.h>
d6472302 14#include <linux/vmalloc.h>
95b4ecbf
SY		 15
16#include "mic_x100_dma.h"
17
18#define MIC_DMA_MAX_XFER_SIZE_CARD (1 * 1024 * 1024 -\
19 MIC_DMA_ALIGN_BYTES)
20#define MIC_DMA_MAX_XFER_SIZE_HOST (1 * 1024 * 1024 >> 1)
21#define MIC_DMA_DESC_TYPE_SHIFT 60
22#define MIC_DMA_MEMCPY_LEN_SHIFT 46
23#define MIC_DMA_STAT_INTR_SHIFT 59
24
25/* high-water mark for pushing dma descriptors */
26static int mic_dma_pending_level = 4;
27
28/* Status descriptor is used to write a 64 bit value to a memory location */
29enum mic_dma_desc_format_type {
30 MIC_DMA_MEMCPY = 1,
31 MIC_DMA_STATUS,
32};
33
34static inline u32 mic_dma_hw_ring_inc(u32 val)
35{
36 return (val + 1) % MIC_DMA_DESC_RX_SIZE;
37}
38
39static inline u32 mic_dma_hw_ring_dec(u32 val)
40{
41 return val ? val - 1 : MIC_DMA_DESC_RX_SIZE - 1;
42}
43
44static inline void mic_dma_hw_ring_inc_head(struct mic_dma_chan *ch)
45{
46 ch->head = mic_dma_hw_ring_inc(ch->head);
47}
48
49/* Prepare a memcpy desc */
50static inline void mic_dma_memcpy_desc(struct mic_dma_desc *desc,
51 dma_addr_t src_phys, dma_addr_t dst_phys, u64 size)
52{
53 u64 qw0, qw1;
54
55 qw0 = src_phys;
56 qw0 |= (size >> MIC_DMA_ALIGN_SHIFT) << MIC_DMA_MEMCPY_LEN_SHIFT;
57 qw1 = MIC_DMA_MEMCPY;
58 qw1 <<= MIC_DMA_DESC_TYPE_SHIFT;
59 qw1 |= dst_phys;
60 desc->qw0 = qw0;
61 desc->qw1 = qw1;
62}
63
64/* Prepare a status desc. with @data to be written at @dst_phys */
65static inline void mic_dma_prep_status_desc(struct mic_dma_desc *desc, u64 data,
66 dma_addr_t dst_phys, bool generate_intr)
67{
68 u64 qw0, qw1;
69
70 qw0 = data;
71 qw1 = (u64) MIC_DMA_STATUS << MIC_DMA_DESC_TYPE_SHIFT | dst_phys;
72 if (generate_intr)
73 qw1 |= (1ULL << MIC_DMA_STAT_INTR_SHIFT);
74 desc->qw0 = qw0;
75 desc->qw1 = qw1;
76}
77
78static void mic_dma_cleanup(struct mic_dma_chan *ch)
79{
80 struct dma_async_tx_descriptor *tx;
81 u32 tail;
82 u32 last_tail;
83
84 spin_lock(&ch->cleanup_lock);
85 tail = mic_dma_read_cmp_cnt(ch);
86 /*
87 * This is the barrier pair for smp_wmb() in fn.
88 * mic_dma_tx_submit_unlock. It's required so that we read the
89 * updated cookie value from tx->cookie.
90 */
91 smp_rmb();
92 for (last_tail = ch->last_tail; tail != last_tail;) {
93 tx = &ch->tx_array[last_tail];
94 if (tx->cookie) {
95 dma_cookie_complete(tx);
7a883acd
DJ		 96 dmaengine_desc_get_callback_invoke(tx, NULL);
97 tx->callback = NULL;
95b4ecbf
SY		 98 }
99 last_tail = mic_dma_hw_ring_inc(last_tail);
100 }
101 /* finish all completion callbacks before incrementing tail */
102 smp_mb();
103 ch->last_tail = last_tail;
104 spin_unlock(&ch->cleanup_lock);
105}
106
107static u32 mic_dma_ring_count(u32 head, u32 tail)
108{
109 u32 count;
110
111 if (head >= tail)
112 count = (tail - 0) + (MIC_DMA_DESC_RX_SIZE - head);
113 else
114 count = tail - head;
115 return count - 1;
116}
117
118/* Returns the num. of free descriptors on success, -ENOMEM on failure */
119static int mic_dma_avail_desc_ring_space(struct mic_dma_chan *ch, int required)
120{
121 struct device *dev = mic_dma_ch_to_device(ch);
122 u32 count;
123
124 count = mic_dma_ring_count(ch->head, ch->last_tail);
125 if (count < required) {
126 mic_dma_cleanup(ch);
127 count = mic_dma_ring_count(ch->head, ch->last_tail);
128 }
129
130 if (count < required) {
131 dev_dbg(dev, "Not enough desc space");
132 dev_dbg(dev, "%s %d required=%u, avail=%u\n",
133 __func__, __LINE__, required, count);
134 return -ENOMEM;
135 } else {
136 return count;
137 }
138}
139
140/* Program memcpy descriptors into the descriptor ring and update s/w head ptr*/
141static int mic_dma_prog_memcpy_desc(struct mic_dma_chan *ch, dma_addr_t src,
142 dma_addr_t dst, size_t len)
143{
144 size_t current_transfer_len;
145 size_t max_xfer_size = to_mic_dma_dev(ch)->max_xfer_size;
146 /* 3 is added to make sure we have enough space for status desc */
147 int num_desc = len / max_xfer_size + 3;
148 int ret;
149
150 if (len % max_xfer_size)
151 num_desc++;
152
153 ret = mic_dma_avail_desc_ring_space(ch, num_desc);
154 if (ret < 0)
155 return ret;
156 do {
157 current_transfer_len = min(len, max_xfer_size);
158 mic_dma_memcpy_desc(&ch->desc_ring[ch->head],
159 src, dst, current_transfer_len);
160 mic_dma_hw_ring_inc_head(ch);
161 len -= current_transfer_len;
162 dst = dst + current_transfer_len;
163 src = src + current_transfer_len;
164 } while (len > 0);
165 return 0;
166}
167
168/* It's a h/w quirk and h/w needs 2 status descriptors for every status desc */
169static void mic_dma_prog_intr(struct mic_dma_chan *ch)
170{
171 mic_dma_prep_status_desc(&ch->desc_ring[ch->head], 0,
172 ch->status_dest_micpa, false);
173 mic_dma_hw_ring_inc_head(ch);
174 mic_dma_prep_status_desc(&ch->desc_ring[ch->head], 0,
175 ch->status_dest_micpa, true);
176 mic_dma_hw_ring_inc_head(ch);
177}
178
179/* Wrapper function to program memcpy descriptors/status descriptors */
180static int mic_dma_do_dma(struct mic_dma_chan *ch, int flags, dma_addr_t src,
181 dma_addr_t dst, size_t len)
182{
ff39988a 183 if (len && -ENOMEM == mic_dma_prog_memcpy_desc(ch, src, dst, len)) {
95b4ecbf 184 return -ENOMEM;
ff39988a
SY		 185 } else {
186 /* 3 is the maximum number of status descriptors */
187 int ret = mic_dma_avail_desc_ring_space(ch, 3);
188
189 if (ret < 0)
190 return ret;
191 }
192
95b4ecbf
SY		 193 /* Above mic_dma_prog_memcpy_desc() makes sure we have enough space */
194 if (flags & DMA_PREP_FENCE) {
195 mic_dma_prep_status_desc(&ch->desc_ring[ch->head], 0,
196 ch->status_dest_micpa, false);
197 mic_dma_hw_ring_inc_head(ch);
198 }
199
200 if (flags & DMA_PREP_INTERRUPT)
201 mic_dma_prog_intr(ch);
202
203 return 0;
204}
205
206static inline void mic_dma_issue_pending(struct dma_chan *ch)
207{
208 struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
209
210 spin_lock(&mic_ch->issue_lock);
211 /*
212 * Write to head triggers h/w to act on the descriptors.
213 * On MIC, writing the same head value twice causes
214 * a h/w error. On second write, h/w assumes we filled
215 * the entire ring & overwrote some of the descriptors.
216 */
217 if (mic_ch->issued == mic_ch->submitted)
218 goto out;
219 mic_ch->issued = mic_ch->submitted;
220 /*
221 * make descriptor updates visible before advancing head,
222 * this is purposefully not smp_wmb() since we are also
223 * publishing the descriptor updates to a dma device
224 */
225 wmb();
226 mic_dma_write_reg(mic_ch, MIC_DMA_REG_DHPR, mic_ch->issued);
227out:
228 spin_unlock(&mic_ch->issue_lock);
229}
230
231static inline void mic_dma_update_pending(struct mic_dma_chan *ch)
232{
233 if (mic_dma_ring_count(ch->issued, ch->submitted)
234 > mic_dma_pending_level)
235 mic_dma_issue_pending(&ch->api_ch);
236}
237
238static dma_cookie_t mic_dma_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
239{
240 struct mic_dma_chan *mic_ch = to_mic_dma_chan(tx->chan);
241 dma_cookie_t cookie;
242
243 dma_cookie_assign(tx);
244 cookie = tx->cookie;
245 /*
246 * We need an smp write barrier here because another CPU might see
247 * an update to submitted and update h/w head even before we
248 * assigned a cookie to this tx.
249 */
250 smp_wmb();
251 mic_ch->submitted = mic_ch->head;
252 spin_unlock(&mic_ch->prep_lock);
253 mic_dma_update_pending(mic_ch);
254 return cookie;
255}
256
257static inline struct dma_async_tx_descriptor *
258allocate_tx(struct mic_dma_chan *ch)
259{
260 u32 idx = mic_dma_hw_ring_dec(ch->head);
261 struct dma_async_tx_descriptor *tx = &ch->tx_array[idx];
262
263 dma_async_tx_descriptor_init(tx, &ch->api_ch);
264 tx->tx_submit = mic_dma_tx_submit_unlock;
265 return tx;
266}
267
ff39988a
SY		 268/* Program a status descriptor with dst as address and value to be written */
269static struct dma_async_tx_descriptor *
270mic_dma_prep_status_lock(struct dma_chan *ch, dma_addr_t dst, u64 src_val,
271 unsigned long flags)
272{
273 struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
274 int result;
275
276 spin_lock(&mic_ch->prep_lock);
277 result = mic_dma_avail_desc_ring_space(mic_ch, 4);
278 if (result < 0)
279 goto error;
280 mic_dma_prep_status_desc(&mic_ch->desc_ring[mic_ch->head], src_val, dst,
281 false);
282 mic_dma_hw_ring_inc_head(mic_ch);
283 result = mic_dma_do_dma(mic_ch, flags, 0, 0, 0);
284 if (result < 0)
285 goto error;
286
287 return allocate_tx(mic_ch);
288error:
289 dev_err(mic_dma_ch_to_device(mic_ch),
290 "Error enqueueing dma status descriptor, error=%d\n", result);
291 spin_unlock(&mic_ch->prep_lock);
292 return NULL;
293}
294
95b4ecbf
SY		 295/*
296 * Prepare a memcpy descriptor to be added to the ring.
297 * Note that the temporary descriptor adds an extra overhead of copying the
298 * descriptor to ring. So, we copy directly to the descriptor ring
299 */
300static struct dma_async_tx_descriptor *
301mic_dma_prep_memcpy_lock(struct dma_chan *ch, dma_addr_t dma_dest,
302 dma_addr_t dma_src, size_t len, unsigned long flags)
303{
304 struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
305 struct device *dev = mic_dma_ch_to_device(mic_ch);
306 int result;
307
308 if (!len && !flags)
309 return NULL;
310
311 spin_lock(&mic_ch->prep_lock);
312 result = mic_dma_do_dma(mic_ch, flags, dma_src, dma_dest, len);
313 if (result >= 0)
16605e8d
AD		 314 return allocate_tx(mic_ch);
315 dev_err(dev, "Error enqueueing dma, error=%d\n", result);
95b4ecbf 316 spin_unlock(&mic_ch->prep_lock);
16605e8d 317 return NULL;
95b4ecbf
SY		 318}
319
320static struct dma_async_tx_descriptor *
321mic_dma_prep_interrupt_lock(struct dma_chan *ch, unsigned long flags)
322{
323 struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
324 int ret;
325
326 spin_lock(&mic_ch->prep_lock);
327 ret = mic_dma_do_dma(mic_ch, flags, 0, 0, 0);
328 if (!ret)
16605e8d 329 return allocate_tx(mic_ch);
95b4ecbf 330 spin_unlock(&mic_ch->prep_lock);
16605e8d 331 return NULL;
95b4ecbf
SY		 332}
333
334/* Return the status of the transaction */
335static enum dma_status
336mic_dma_tx_status(struct dma_chan *ch, dma_cookie_t cookie,
337 struct dma_tx_state *txstate)
338{
339 struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
340
341 if (DMA_COMPLETE != dma_cookie_status(ch, cookie, txstate))
342 mic_dma_cleanup(mic_ch);
343
344 return dma_cookie_status(ch, cookie, txstate);
345}
346
347static irqreturn_t mic_dma_thread_fn(int irq, void *data)
348{
349 mic_dma_cleanup((struct mic_dma_chan *)data);
350 return IRQ_HANDLED;
351}
352
353static irqreturn_t mic_dma_intr_handler(int irq, void *data)
354{
355 struct mic_dma_chan *ch = ((struct mic_dma_chan *)data);
356
357 mic_dma_ack_interrupt(ch);
358 return IRQ_WAKE_THREAD;
359}
360
361static int mic_dma_alloc_desc_ring(struct mic_dma_chan *ch)
362{
363 u64 desc_ring_size = MIC_DMA_DESC_RX_SIZE * sizeof(*ch->desc_ring);
364 struct device *dev = &to_mbus_device(ch)->dev;
365
366 desc_ring_size = ALIGN(desc_ring_size, MIC_DMA_ALIGN_BYTES);
367 ch->desc_ring = kzalloc(desc_ring_size, GFP_KERNEL);
368
369 if (!ch->desc_ring)
370 return -ENOMEM;
371
372 ch->desc_ring_micpa = dma_map_single(dev, ch->desc_ring,
373 desc_ring_size, DMA_BIDIRECTIONAL);
374 if (dma_mapping_error(dev, ch->desc_ring_micpa))
375 goto map_error;
376
fad953ce
KC		 377 ch->tx_array = vzalloc(array_size(MIC_DMA_DESC_RX_SIZE,
378 sizeof(*ch->tx_array)));
95b4ecbf
SY		 379 if (!ch->tx_array)
380 goto tx_error;
381 return 0;
382tx_error:
383 dma_unmap_single(dev, ch->desc_ring_micpa, desc_ring_size,
384 DMA_BIDIRECTIONAL);
385map_error:
386 kfree(ch->desc_ring);
387 return -ENOMEM;
388}
389
390static void mic_dma_free_desc_ring(struct mic_dma_chan *ch)
391{
392 u64 desc_ring_size = MIC_DMA_DESC_RX_SIZE * sizeof(*ch->desc_ring);
393
394 vfree(ch->tx_array);
395 desc_ring_size = ALIGN(desc_ring_size, MIC_DMA_ALIGN_BYTES);
396 dma_unmap_single(&to_mbus_device(ch)->dev, ch->desc_ring_micpa,
397 desc_ring_size, DMA_BIDIRECTIONAL);
398 kfree(ch->desc_ring);
399 ch->desc_ring = NULL;
400}
401
402static void mic_dma_free_status_dest(struct mic_dma_chan *ch)
403{
404 dma_unmap_single(&to_mbus_device(ch)->dev, ch->status_dest_micpa,
405 L1_CACHE_BYTES, DMA_BIDIRECTIONAL);
406 kfree(ch->status_dest);
407}
408
409static int mic_dma_alloc_status_dest(struct mic_dma_chan *ch)
410{
411 struct device *dev = &to_mbus_device(ch)->dev;
412
413 ch->status_dest = kzalloc(L1_CACHE_BYTES, GFP_KERNEL);
414 if (!ch->status_dest)
415 return -ENOMEM;
416 ch->status_dest_micpa = dma_map_single(dev, ch->status_dest,
417 L1_CACHE_BYTES, DMA_BIDIRECTIONAL);
418 if (dma_mapping_error(dev, ch->status_dest_micpa)) {
419 kfree(ch->status_dest);
420 ch->status_dest = NULL;
421 return -ENOMEM;
422 }
423 return 0;
424}
425
426static int mic_dma_check_chan(struct mic_dma_chan *ch)
427{
428 if (mic_dma_read_reg(ch, MIC_DMA_REG_DCHERR) ||
429 mic_dma_read_reg(ch, MIC_DMA_REG_DSTAT) & MIC_DMA_CHAN_QUIESCE) {
430 mic_dma_disable_chan(ch);
431 mic_dma_chan_mask_intr(ch);
432 dev_err(mic_dma_ch_to_device(ch),
433 "%s %d error setting up mic dma chan %d\n",
434 __func__, __LINE__, ch->ch_num);
435 return -EBUSY;
436 }
437 return 0;
438}
439
440static int mic_dma_chan_setup(struct mic_dma_chan *ch)
441{
442 if (MIC_DMA_CHAN_MIC == ch->owner)
443 mic_dma_chan_set_owner(ch);
444 mic_dma_disable_chan(ch);
445 mic_dma_chan_mask_intr(ch);
446 mic_dma_write_reg(ch, MIC_DMA_REG_DCHERRMSK, 0);
447 mic_dma_chan_set_desc_ring(ch);
448 ch->last_tail = mic_dma_read_reg(ch, MIC_DMA_REG_DTPR);
449 ch->head = ch->last_tail;
450 ch->issued = 0;
451 mic_dma_chan_unmask_intr(ch);
452 mic_dma_enable_chan(ch);
453 return mic_dma_check_chan(ch);
454}
455
456static void mic_dma_chan_destroy(struct mic_dma_chan *ch)
457{
458 mic_dma_disable_chan(ch);
459 mic_dma_chan_mask_intr(ch);
460}
461
95b4ecbf
SY		 462static int mic_dma_setup_irq(struct mic_dma_chan *ch)
463{
464 ch->cookie =
465 to_mbus_hw_ops(ch)->request_threaded_irq(to_mbus_device(ch),
466 mic_dma_intr_handler, mic_dma_thread_fn,
467 "mic dma_channel", ch, ch->ch_num);
a8ffa34f 468 return PTR_ERR_OR_ZERO(ch->cookie);
95b4ecbf
SY		 469}
470
471static inline void mic_dma_free_irq(struct mic_dma_chan *ch)
472{
473 to_mbus_hw_ops(ch)->free_irq(to_mbus_device(ch), ch->cookie, ch);
474}
475
476static int mic_dma_chan_init(struct mic_dma_chan *ch)
477{
478 int ret = mic_dma_alloc_desc_ring(ch);
479
480 if (ret)
481 goto ring_error;
482 ret = mic_dma_alloc_status_dest(ch);
483 if (ret)
484 goto status_error;
485 ret = mic_dma_chan_setup(ch);
486 if (ret)
487 goto chan_error;
488 return ret;
489chan_error:
490 mic_dma_free_status_dest(ch);
491status_error:
492 mic_dma_free_desc_ring(ch);
493ring_error:
494 return ret;
495}
496
497static int mic_dma_drain_chan(struct mic_dma_chan *ch)
498{
499 struct dma_async_tx_descriptor *tx;
500 int err = 0;
501 dma_cookie_t cookie;
502
503 tx = mic_dma_prep_memcpy_lock(&ch->api_ch, 0, 0, 0, DMA_PREP_FENCE);
504 if (!tx) {
505 err = -ENOMEM;
506 goto error;
507 }
508
509 cookie = tx->tx_submit(tx);
510 if (dma_submit_error(cookie))
511 err = -ENOMEM;
512 else
513 err = dma_sync_wait(&ch->api_ch, cookie);
514 if (err) {
515 dev_err(mic_dma_ch_to_device(ch), "%s %d TO chan 0x%x\n",
516 __func__, __LINE__, ch->ch_num);
517 err = -EIO;
518 }
519error:
520 mic_dma_cleanup(ch);
521 return err;
522}
523
524static inline void mic_dma_chan_uninit(struct mic_dma_chan *ch)
525{
526 mic_dma_chan_destroy(ch);
527 mic_dma_cleanup(ch);
528 mic_dma_free_status_dest(ch);
529 mic_dma_free_desc_ring(ch);
530}
531
532static int mic_dma_init(struct mic_dma_device *mic_dma_dev,
533 enum mic_dma_chan_owner owner)
534{
535 int i, first_chan = mic_dma_dev->start_ch;
536 struct mic_dma_chan *ch;
537 int ret;
538
539 for (i = first_chan; i < first_chan + MIC_DMA_NUM_CHAN; i++) {
95b4ecbf 540 ch = &mic_dma_dev->mic_ch[i];
95b4ecbf
SY		 541 ch->ch_num = i;
542 ch->owner = owner;
543 spin_lock_init(&ch->cleanup_lock);
544 spin_lock_init(&ch->prep_lock);
545 spin_lock_init(&ch->issue_lock);
546 ret = mic_dma_setup_irq(ch);
547 if (ret)
548 goto error;
549 }
550 return 0;
551error:
552 for (i = i - 1; i >= first_chan; i--)
553 mic_dma_free_irq(ch);
554 return ret;
555}
556
557static void mic_dma_uninit(struct mic_dma_device *mic_dma_dev)
558{
559 int i, first_chan = mic_dma_dev->start_ch;
560 struct mic_dma_chan *ch;
561
562 for (i = first_chan; i < first_chan + MIC_DMA_NUM_CHAN; i++) {
563 ch = &mic_dma_dev->mic_ch[i];
564 mic_dma_free_irq(ch);
565 }
566}
567
568static int mic_dma_alloc_chan_resources(struct dma_chan *ch)
569{
570 int ret = mic_dma_chan_init(to_mic_dma_chan(ch));
571 if (ret)
572 return ret;
573 return MIC_DMA_DESC_RX_SIZE;
574}
575
576static void mic_dma_free_chan_resources(struct dma_chan *ch)
577{
578 struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
579 mic_dma_drain_chan(mic_ch);
580 mic_dma_chan_uninit(mic_ch);
581}
582
583/* Set the fn. handlers and register the dma device with dma api */
584static int mic_dma_register_dma_device(struct mic_dma_device *mic_dma_dev,
585 enum mic_dma_chan_owner owner)
586{
587 int i, first_chan = mic_dma_dev->start_ch;
588
589 dma_cap_zero(mic_dma_dev->dma_dev.cap_mask);
590 /*
591 * This dma engine is not capable of host memory to host memory
592 * transfers
593 */
594 dma_cap_set(DMA_MEMCPY, mic_dma_dev->dma_dev.cap_mask);
595
596 if (MIC_DMA_CHAN_HOST == owner)
597 dma_cap_set(DMA_PRIVATE, mic_dma_dev->dma_dev.cap_mask);
598 mic_dma_dev->dma_dev.device_alloc_chan_resources =
599 mic_dma_alloc_chan_resources;
600 mic_dma_dev->dma_dev.device_free_chan_resources =
601 mic_dma_free_chan_resources;
602 mic_dma_dev->dma_dev.device_tx_status = mic_dma_tx_status;
603 mic_dma_dev->dma_dev.device_prep_dma_memcpy = mic_dma_prep_memcpy_lock;
ff39988a
SY		 604 mic_dma_dev->dma_dev.device_prep_dma_imm_data =
605 mic_dma_prep_status_lock;
95b4ecbf
SY		 606 mic_dma_dev->dma_dev.device_prep_dma_interrupt =
607 mic_dma_prep_interrupt_lock;
608 mic_dma_dev->dma_dev.device_issue_pending = mic_dma_issue_pending;
609 mic_dma_dev->dma_dev.copy_align = MIC_DMA_ALIGN_SHIFT;
610 INIT_LIST_HEAD(&mic_dma_dev->dma_dev.channels);
611 for (i = first_chan; i < first_chan + MIC_DMA_NUM_CHAN; i++) {
612 mic_dma_dev->mic_ch[i].api_ch.device = &mic_dma_dev->dma_dev;
613 dma_cookie_init(&mic_dma_dev->mic_ch[i].api_ch);
614 list_add_tail(&mic_dma_dev->mic_ch[i].api_ch.device_node,
615 &mic_dma_dev->dma_dev.channels);
616 }
f6206f00 617 return dmaenginem_async_device_register(&mic_dma_dev->dma_dev);
95b4ecbf
SY		 618}
619
620/*
621 * Initializes dma channels and registers the dma device with the
622 * dma engine api.
623 */
624static struct mic_dma_device *mic_dma_dev_reg(struct mbus_device *mbdev,
625 enum mic_dma_chan_owner owner)
626{
627 struct mic_dma_device *mic_dma_dev;
628 int ret;
629 struct device *dev = &mbdev->dev;
630
111b009f 631 mic_dma_dev = devm_kzalloc(dev, sizeof(*mic_dma_dev), GFP_KERNEL);
95b4ecbf
SY		 632 if (!mic_dma_dev) {
633 ret = -ENOMEM;
634 goto alloc_error;
635 }
636 mic_dma_dev->mbdev = mbdev;
637 mic_dma_dev->dma_dev.dev = dev;
638 mic_dma_dev->mmio = mbdev->mmio_va;
639 if (MIC_DMA_CHAN_HOST == owner) {
640 mic_dma_dev->start_ch = 0;
641 mic_dma_dev->max_xfer_size = MIC_DMA_MAX_XFER_SIZE_HOST;
642 } else {
643 mic_dma_dev->start_ch = 4;
644 mic_dma_dev->max_xfer_size = MIC_DMA_MAX_XFER_SIZE_CARD;
645 }
646 ret = mic_dma_init(mic_dma_dev, owner);
647 if (ret)
648 goto init_error;
649 ret = mic_dma_register_dma_device(mic_dma_dev, owner);
650 if (ret)
651 goto reg_error;
652 return mic_dma_dev;
653reg_error:
654 mic_dma_uninit(mic_dma_dev);
655init_error:
95b4ecbf
SY		 656 mic_dma_dev = NULL;
657alloc_error:
658 dev_err(dev, "Error at %s %d ret=%d\n", __func__, __LINE__, ret);
659 return mic_dma_dev;
660}
661
662static void mic_dma_dev_unreg(struct mic_dma_device *mic_dma_dev)
663{
95b4ecbf 664 mic_dma_uninit(mic_dma_dev);
95b4ecbf
SY		 665}
666
667/* DEBUGFS CODE */
0f7ab39a 668static int mic_dma_reg_show(struct seq_file *s, void *pos)
95b4ecbf
SY		 669{
670 struct mic_dma_device *mic_dma_dev = s->private;
671 int i, chan_num, first_chan = mic_dma_dev->start_ch;
672 struct mic_dma_chan *ch;
673
674 seq_printf(s, "SBOX_DCR: %#x\n",
675 mic_dma_mmio_read(&mic_dma_dev->mic_ch[first_chan],
676 MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR));
677 seq_puts(s, "DMA Channel Registers\n");
678 seq_printf(s, "%-10s| %-10s %-10s %-10s %-10s %-10s",
679 "Channel", "DCAR", "DTPR", "DHPR", "DRAR_HI", "DRAR_LO");
680 seq_printf(s, " %-11s %-14s %-10s\n", "DCHERR", "DCHERRMSK", "DSTAT");
681 for (i = first_chan; i < first_chan + MIC_DMA_NUM_CHAN; i++) {
682 ch = &mic_dma_dev->mic_ch[i];
683 chan_num = ch->ch_num;
684 seq_printf(s, "%-10i| %-#10x %-#10x %-#10x %-#10x",
685 chan_num,
686 mic_dma_read_reg(ch, MIC_DMA_REG_DCAR),
687 mic_dma_read_reg(ch, MIC_DMA_REG_DTPR),
688 mic_dma_read_reg(ch, MIC_DMA_REG_DHPR),
689 mic_dma_read_reg(ch, MIC_DMA_REG_DRAR_HI));
690 seq_printf(s, " %-#10x %-#10x %-#14x %-#10x\n",
691 mic_dma_read_reg(ch, MIC_DMA_REG_DRAR_LO),
692 mic_dma_read_reg(ch, MIC_DMA_REG_DCHERR),
693 mic_dma_read_reg(ch, MIC_DMA_REG_DCHERRMSK),
694 mic_dma_read_reg(ch, MIC_DMA_REG_DSTAT));
695 }
696 return 0;
697}
698
0f7ab39a 699DEFINE_SHOW_ATTRIBUTE(mic_dma_reg);
95b4ecbf
SY		 700
701/* Debugfs parent dir */
702static struct dentry *mic_dma_dbg;
703
704static int mic_dma_driver_probe(struct mbus_device *mbdev)
705{
706 struct mic_dma_device *mic_dma_dev;
707 enum mic_dma_chan_owner owner;
708
709 if (MBUS_DEV_DMA_MIC == mbdev->id.device)
710 owner = MIC_DMA_CHAN_MIC;
711 else
712 owner = MIC_DMA_CHAN_HOST;
713
714 mic_dma_dev = mic_dma_dev_reg(mbdev, owner);
715 dev_set_drvdata(&mbdev->dev, mic_dma_dev);
716
717 if (mic_dma_dbg) {
718 mic_dma_dev->dbg_dir = debugfs_create_dir(dev_name(&mbdev->dev),
719 mic_dma_dbg);
bea696c5
GKH		 720 debugfs_create_file("mic_dma_reg", 0444, mic_dma_dev->dbg_dir,
721 mic_dma_dev, &mic_dma_reg_fops);
95b4ecbf
SY		 722 }
723 return 0;
724}
725
726static void mic_dma_driver_remove(struct mbus_device *mbdev)
727{
728 struct mic_dma_device *mic_dma_dev;
729
730 mic_dma_dev = dev_get_drvdata(&mbdev->dev);
731 debugfs_remove_recursive(mic_dma_dev->dbg_dir);
732 mic_dma_dev_unreg(mic_dma_dev);
733}
734
735static struct mbus_device_id id_table[] = {
736 {MBUS_DEV_DMA_MIC, MBUS_DEV_ANY_ID},
737 {MBUS_DEV_DMA_HOST, MBUS_DEV_ANY_ID},
738 {0},
739};
740
741static struct mbus_driver mic_dma_driver = {
742 .driver.name = KBUILD_MODNAME,
743 .driver.owner = THIS_MODULE,
744 .id_table = id_table,
745 .probe = mic_dma_driver_probe,
746 .remove = mic_dma_driver_remove,
747};
748
749static int __init mic_x100_dma_init(void)
750{
751 int rc = mbus_register_driver(&mic_dma_driver);
752 if (rc)
753 return rc;
754 mic_dma_dbg = debugfs_create_dir(KBUILD_MODNAME, NULL);
755 return 0;
756}
757
758static void __exit mic_x100_dma_exit(void)
759{
760 debugfs_remove_recursive(mic_dma_dbg);
761 mbus_unregister_driver(&mic_dma_driver);
762}
763
764module_init(mic_x100_dma_init);
765module_exit(mic_x100_dma_exit);
766
767MODULE_DEVICE_TABLE(mbus, id_table);
768MODULE_AUTHOR("Intel Corporation");
769MODULE_DESCRIPTION("Intel(R) MIC X100 DMA Driver");
770MODULE_LICENSE("GPL v2");
Linux 6.x block layer and io_uring tree(s)