projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ioatdma: Remove the wrappers around read(bwl)/write(bwl) in ioatdma
[linux-2.6-block.git] / drivers / dma / ioatdma.c
CommitLineData
0bbd5f4e
CL		 1/*
2 * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the Free
6 * Software Foundation; either version 2 of the License, or (at your option)
7 * any later version.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc., 59
16 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
18 * The full GNU General Public License is included in this distribution in the
19 * file called COPYING.
20 */
21
22/*
23 * This driver supports an Intel I/OAT DMA engine, which does asynchronous
24 * copy operations.
25 */
26
27#include <linux/init.h>
28#include <linux/module.h>
29#include <linux/pci.h>
30#include <linux/interrupt.h>
31#include <linux/dmaengine.h>
32#include <linux/delay.h>
6b00c92c 33#include <linux/dma-mapping.h>
0bbd5f4e 34#include "ioatdma.h"
0bbd5f4e
CL		 35#include "ioatdma_registers.h"
36#include "ioatdma_hw.h"
37
38#define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common)
39#define to_ioat_device(dev) container_of(dev, struct ioat_device, common)
40#define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node)
41
42/* internal functions */
43static int __devinit ioat_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
44static void __devexit ioat_remove(struct pci_dev *pdev);
45
46static int enumerate_dma_channels(struct ioat_device *device)
47{
48 u8 xfercap_scale;
49 u32 xfercap;
50 int i;
51 struct ioat_dma_chan *ioat_chan;
52
e3828811
CL		 53 device->common.chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
54 xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
0bbd5f4e
CL		 55 xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
56
57 for (i = 0; i < device->common.chancnt; i++) {
58 ioat_chan = kzalloc(sizeof(*ioat_chan), GFP_KERNEL);
59 if (!ioat_chan) {
60 device->common.chancnt = i;
61 break;
62 }
63
64 ioat_chan->device = device;
65 ioat_chan->reg_base = device->reg_base + (0x80 * (i + 1));
66 ioat_chan->xfercap = xfercap;
67 spin_lock_init(&ioat_chan->cleanup_lock);
68 spin_lock_init(&ioat_chan->desc_lock);
69 INIT_LIST_HEAD(&ioat_chan->free_desc);
70 INIT_LIST_HEAD(&ioat_chan->used_desc);
71 /* This should be made common somewhere in dmaengine.c */
72 ioat_chan->common.device = &device->common;
73 ioat_chan->common.client = NULL;
74 list_add_tail(&ioat_chan->common.device_node,
75 &device->common.channels);
76 }
77 return device->common.chancnt;
78}
79
80static struct ioat_desc_sw *ioat_dma_alloc_descriptor(
81 struct ioat_dma_chan *ioat_chan,
47b16539 82 gfp_t flags)
0bbd5f4e
CL		 83{
84 struct ioat_dma_descriptor *desc;
85 struct ioat_desc_sw *desc_sw;
86 struct ioat_device *ioat_device;
87 dma_addr_t phys;
88
89 ioat_device = to_ioat_device(ioat_chan->common.device);
90 desc = pci_pool_alloc(ioat_device->dma_pool, flags, &phys);
91 if (unlikely(!desc))
92 return NULL;
93
94 desc_sw = kzalloc(sizeof(*desc_sw), flags);
95 if (unlikely(!desc_sw)) {
96 pci_pool_free(ioat_device->dma_pool, desc, phys);
97 return NULL;
98 }
99
100 memset(desc, 0, sizeof(*desc));
101 desc_sw->hw = desc;
102 desc_sw->phys = phys;
103
104 return desc_sw;
105}
106
107#define INITIAL_IOAT_DESC_COUNT 128
108
109static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan);
110
111/* returns the actual number of allocated descriptors */
112static int ioat_dma_alloc_chan_resources(struct dma_chan *chan)
113{
114 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
115 struct ioat_desc_sw *desc = NULL;
116 u16 chanctrl;
117 u32 chanerr;
118 int i;
119 LIST_HEAD(tmp_list);
120
121 /*
122 * In-use bit automatically set by reading chanctrl
123 * If 0, we got it, if 1, someone else did
124 */
e3828811 125 chanctrl = readw(ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
0bbd5f4e
CL		 126 if (chanctrl & IOAT_CHANCTRL_CHANNEL_IN_USE)
127 return -EBUSY;
128
129 /* Setup register to interrupt and write completion status on error */
130 chanctrl = IOAT_CHANCTRL_CHANNEL_IN_USE |
131 IOAT_CHANCTRL_ERR_INT_EN |
132 IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
133 IOAT_CHANCTRL_ERR_COMPLETION_EN;
e3828811 134 writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
0bbd5f4e 135
e3828811 136 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
0bbd5f4e
CL		 137 if (chanerr) {
138 printk("IOAT: CHANERR = %x, clearing\n", chanerr);
e3828811 139 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
0bbd5f4e
CL		 140 }
141
142 /* Allocate descriptors */
143 for (i = 0; i < INITIAL_IOAT_DESC_COUNT; i++) {
144 desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL);
145 if (!desc) {
146 printk(KERN_ERR "IOAT: Only %d initial descriptors\n", i);
147 break;
148 }
149 list_add_tail(&desc->node, &tmp_list);
150 }
151 spin_lock_bh(&ioat_chan->desc_lock);
152 list_splice(&tmp_list, &ioat_chan->free_desc);
153 spin_unlock_bh(&ioat_chan->desc_lock);
154
155 /* allocate a completion writeback area */
156 /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
157 ioat_chan->completion_virt =
158 pci_pool_alloc(ioat_chan->device->completion_pool,
159 GFP_KERNEL,
160 &ioat_chan->completion_addr);
161 memset(ioat_chan->completion_virt, 0,
162 sizeof(*ioat_chan->completion_virt));
e3828811
CL		 163 writel(((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF,
164 ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
165 writel(((u64) ioat_chan->completion_addr) >> 32,
166 ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
0bbd5f4e
CL		 167
168 ioat_start_null_desc(ioat_chan);
169 return i;
170}
171
172static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan);
173
174static void ioat_dma_free_chan_resources(struct dma_chan *chan)
175{
176 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
177 struct ioat_device *ioat_device = to_ioat_device(chan->device);
178 struct ioat_desc_sw *desc, *_desc;
179 u16 chanctrl;
180 int in_use_descs = 0;
181
182 ioat_dma_memcpy_cleanup(ioat_chan);
183
e3828811 184 writeb(IOAT_CHANCMD_RESET, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
0bbd5f4e
CL		 185
186 spin_lock_bh(&ioat_chan->desc_lock);
187 list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) {
188 in_use_descs++;
189 list_del(&desc->node);
190 pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys);
191 kfree(desc);
192 }
193 list_for_each_entry_safe(desc, _desc, &ioat_chan->free_desc, node) {
194 list_del(&desc->node);
195 pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys);
196 kfree(desc);
197 }
198 spin_unlock_bh(&ioat_chan->desc_lock);
199
200 pci_pool_free(ioat_device->completion_pool,
201 ioat_chan->completion_virt,
202 ioat_chan->completion_addr);
203
204 /* one is ok since we left it on there on purpose */
205 if (in_use_descs > 1)
206 printk(KERN_ERR "IOAT: Freeing %d in use descriptors!\n",
207 in_use_descs - 1);
208
209 ioat_chan->last_completion = ioat_chan->completion_addr = 0;
210
211 /* Tell hw the chan is free */
e3828811 212 chanctrl = readw(ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
0bbd5f4e 213 chanctrl &= ~IOAT_CHANCTRL_CHANNEL_IN_USE;
e3828811 214 writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
0bbd5f4e
CL		 215}
216
217/**
218 * do_ioat_dma_memcpy - actual function that initiates a IOAT DMA transaction
6508871e 219 * @ioat_chan: IOAT DMA channel handle
0bbd5f4e
CL		 220 * @dest: DMA destination address
221 * @src: DMA source address
222 * @len: transaction length in bytes
223 */
224
225static dma_cookie_t do_ioat_dma_memcpy(struct ioat_dma_chan *ioat_chan,
226 dma_addr_t dest,
227 dma_addr_t src,
228 size_t len)
229{
230 struct ioat_desc_sw *first;
231 struct ioat_desc_sw *prev;
232 struct ioat_desc_sw *new;
233 dma_cookie_t cookie;
234 LIST_HEAD(new_chain);
235 u32 copy;
236 size_t orig_len;
237 dma_addr_t orig_src, orig_dst;
238 unsigned int desc_count = 0;
239 unsigned int append = 0;
240
241 if (!ioat_chan || !dest || !src)
242 return -EFAULT;
243
244 if (!len)
245 return ioat_chan->common.cookie;
246
247 orig_len = len;
248 orig_src = src;
249 orig_dst = dest;
250
251 first = NULL;
252 prev = NULL;
253
254 spin_lock_bh(&ioat_chan->desc_lock);
255
256 while (len) {
257 if (!list_empty(&ioat_chan->free_desc)) {
258 new = to_ioat_desc(ioat_chan->free_desc.next);
259 list_del(&new->node);
260 } else {
261 /* try to get another desc */
262 new = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC);
263 /* will this ever happen? */
264 /* TODO add upper limit on these */
265 BUG_ON(!new);
266 }
267
268 copy = min((u32) len, ioat_chan->xfercap);
269
270 new->hw->size = copy;
271 new->hw->ctl = 0;
272 new->hw->src_addr = src;
273 new->hw->dst_addr = dest;
274 new->cookie = 0;
275
276 /* chain together the physical address list for the HW */
277 if (!first)
278 first = new;
279 else
280 prev->hw->next = (u64) new->phys;
281
282 prev = new;
283
284 len -= copy;
285 dest += copy;
286 src += copy;
287
288 list_add_tail(&new->node, &new_chain);
289 desc_count++;
290 }
291 new->hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
292 new->hw->next = 0;
293
294 /* cookie incr and addition to used_list must be atomic */
295
296 cookie = ioat_chan->common.cookie;
297 cookie++;
298 if (cookie < 0)
299 cookie = 1;
300 ioat_chan->common.cookie = new->cookie = cookie;
301
302 pci_unmap_addr_set(new, src, orig_src);
303 pci_unmap_addr_set(new, dst, orig_dst);
304 pci_unmap_len_set(new, src_len, orig_len);
305 pci_unmap_len_set(new, dst_len, orig_len);
306
307 /* write address into NextDescriptor field of last desc in chain */
308 to_ioat_desc(ioat_chan->used_desc.prev)->hw->next = first->phys;
309 list_splice_init(&new_chain, ioat_chan->used_desc.prev);
310
311 ioat_chan->pending += desc_count;
000725d5 312 if (ioat_chan->pending >= 4) {
0bbd5f4e
CL		 313 append = 1;
314 ioat_chan->pending = 0;
315 }
316
317 spin_unlock_bh(&ioat_chan->desc_lock);
318
319 if (append)
e3828811
CL		 320 writeb(IOAT_CHANCMD_APPEND,
321 ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
0bbd5f4e
CL		 322 return cookie;
323}
324
325/**
326 * ioat_dma_memcpy_buf_to_buf - wrapper that takes src & dest bufs
327 * @chan: IOAT DMA channel handle
328 * @dest: DMA destination address
329 * @src: DMA source address
330 * @len: transaction length in bytes
331 */
332
333static dma_cookie_t ioat_dma_memcpy_buf_to_buf(struct dma_chan *chan,
334 void *dest,
335 void *src,
336 size_t len)
337{
338 dma_addr_t dest_addr;
339 dma_addr_t src_addr;
340 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
341
342 dest_addr = pci_map_single(ioat_chan->device->pdev,
343 dest, len, PCI_DMA_FROMDEVICE);
344 src_addr = pci_map_single(ioat_chan->device->pdev,
345 src, len, PCI_DMA_TODEVICE);
346
347 return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
348}
349
350/**
351 * ioat_dma_memcpy_buf_to_pg - wrapper, copying from a buf to a page
352 * @chan: IOAT DMA channel handle
353 * @page: pointer to the page to copy to
354 * @offset: offset into that page
355 * @src: DMA source address
356 * @len: transaction length in bytes
357 */
358
359static dma_cookie_t ioat_dma_memcpy_buf_to_pg(struct dma_chan *chan,
360 struct page *page,
361 unsigned int offset,
362 void *src,
363 size_t len)
364{
365 dma_addr_t dest_addr;
366 dma_addr_t src_addr;
367 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
368
369 dest_addr = pci_map_page(ioat_chan->device->pdev,
370 page, offset, len, PCI_DMA_FROMDEVICE);
371 src_addr = pci_map_single(ioat_chan->device->pdev,
372 src, len, PCI_DMA_TODEVICE);
373
374 return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
375}
376
377/**
378 * ioat_dma_memcpy_pg_to_pg - wrapper, copying between two pages
379 * @chan: IOAT DMA channel handle
380 * @dest_pg: pointer to the page to copy to
381 * @dest_off: offset into that page
382 * @src_pg: pointer to the page to copy from
383 * @src_off: offset into that page
6508871e 384 * @len: transaction length in bytes. This is guaranteed not to make a copy
0bbd5f4e
CL		 385 * across a page boundary.
386 */
387
388static dma_cookie_t ioat_dma_memcpy_pg_to_pg(struct dma_chan *chan,
389 struct page *dest_pg,
390 unsigned int dest_off,
391 struct page *src_pg,
392 unsigned int src_off,
393 size_t len)
394{
395 dma_addr_t dest_addr;
396 dma_addr_t src_addr;
397 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
398
399 dest_addr = pci_map_page(ioat_chan->device->pdev,
400 dest_pg, dest_off, len, PCI_DMA_FROMDEVICE);
401 src_addr = pci_map_page(ioat_chan->device->pdev,
402 src_pg, src_off, len, PCI_DMA_TODEVICE);
403
404 return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
405}
406
407/**
6508871e 408 * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended descriptors to hw
0bbd5f4e
CL		 409 * @chan: DMA channel handle
410 */
411
412static void ioat_dma_memcpy_issue_pending(struct dma_chan *chan)
413{
414 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
415
416 if (ioat_chan->pending != 0) {
417 ioat_chan->pending = 0;
e3828811
CL		 418 writeb(IOAT_CHANCMD_APPEND,
419 ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
0bbd5f4e
CL		 420 }
421}
422
423static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan)
424{
425 unsigned long phys_complete;
426 struct ioat_desc_sw *desc, *_desc;
427 dma_cookie_t cookie = 0;
428
429 prefetch(chan->completion_virt);
430
431 if (!spin_trylock(&chan->cleanup_lock))
432 return;
433
434 /* The completion writeback can happen at any time,
435 so reads by the driver need to be atomic operations
436 The descriptor physical addresses are limited to 32-bits
437 when the CPU can only do a 32-bit mov */
438
439#if (BITS_PER_LONG == 64)
440 phys_complete =
441 chan->completion_virt->full & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
442#else
443 phys_complete = chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK;
444#endif
445
446 if ((chan->completion_virt->full & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
447 IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) {
448 printk("IOAT: Channel halted, chanerr = %x\n",
e3828811 449 readl(chan->reg_base + IOAT_CHANERR_OFFSET));
0bbd5f4e
CL		 450
451 /* TODO do something to salvage the situation */
452 }
453
454 if (phys_complete == chan->last_completion) {
455 spin_unlock(&chan->cleanup_lock);
456 return;
457 }
458
459 spin_lock_bh(&chan->desc_lock);
460 list_for_each_entry_safe(desc, _desc, &chan->used_desc, node) {
461
462 /*
463 * Incoming DMA requests may use multiple descriptors, due to
464 * exceeding xfercap, perhaps. If so, only the last one will
465 * have a cookie, and require unmapping.
466 */
467 if (desc->cookie) {
468 cookie = desc->cookie;
469
470 /* yes we are unmapping both _page and _single alloc'd
471 regions with unmap_page. Is this *really* that bad?
472 */
473 pci_unmap_page(chan->device->pdev,
474 pci_unmap_addr(desc, dst),
475 pci_unmap_len(desc, dst_len),
476 PCI_DMA_FROMDEVICE);
477 pci_unmap_page(chan->device->pdev,
478 pci_unmap_addr(desc, src),
479 pci_unmap_len(desc, src_len),
480 PCI_DMA_TODEVICE);
481 }
482
483 if (desc->phys != phys_complete) {
484 /* a completed entry, but not the last, so cleanup */
485 list_del(&desc->node);
486 list_add_tail(&desc->node, &chan->free_desc);
487 } else {
488 /* last used desc. Do not remove, so we can append from
489 it, but don't look at it next time, either */
490 desc->cookie = 0;
491
492 /* TODO check status bits? */
493 break;
494 }
495 }
496
497 spin_unlock_bh(&chan->desc_lock);
498
499 chan->last_completion = phys_complete;
500 if (cookie != 0)
501 chan->completed_cookie = cookie;
502
503 spin_unlock(&chan->cleanup_lock);
504}
505
506/**
507 * ioat_dma_is_complete - poll the status of a IOAT DMA transaction
508 * @chan: IOAT DMA channel handle
509 * @cookie: DMA transaction identifier
6508871e
RD		 510 * @done: if not %NULL, updated with last completed transaction
511 * @used: if not %NULL, updated with last used transaction
0bbd5f4e
CL		 512 */
513
514static enum dma_status ioat_dma_is_complete(struct dma_chan *chan,
515 dma_cookie_t cookie,
516 dma_cookie_t *done,
517 dma_cookie_t *used)
518{
519 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
520 dma_cookie_t last_used;
521 dma_cookie_t last_complete;
522 enum dma_status ret;
523
524 last_used = chan->cookie;
525 last_complete = ioat_chan->completed_cookie;
526
527 if (done)
528 *done= last_complete;
529 if (used)
530 *used = last_used;
531
532 ret = dma_async_is_complete(cookie, last_complete, last_used);
533 if (ret == DMA_SUCCESS)
534 return ret;
535
536 ioat_dma_memcpy_cleanup(ioat_chan);
537
538 last_used = chan->cookie;
539 last_complete = ioat_chan->completed_cookie;
540
541 if (done)
542 *done= last_complete;
543 if (used)
544 *used = last_used;
545
546 return dma_async_is_complete(cookie, last_complete, last_used);
547}
548
549/* PCI API */
550
551static struct pci_device_id ioat_pci_tbl[] = {
552 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT) },
553 { 0, }
554};
555
92504f79 556static struct pci_driver ioat_pci_driver = {
0bbd5f4e
CL		 557 .name = "ioatdma",
558 .id_table = ioat_pci_tbl,
559 .probe = ioat_probe,
560 .remove = __devexit_p(ioat_remove),
561};
562
7d12e780 563static irqreturn_t ioat_do_interrupt(int irq, void *data)
0bbd5f4e
CL		 564{
565 struct ioat_device *instance = data;
566 unsigned long attnstatus;
567 u8 intrctrl;
568
e3828811 569 intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
0bbd5f4e
CL		 570
571 if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
572 return IRQ_NONE;
573
574 if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
e3828811 575 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
0bbd5f4e
CL		 576 return IRQ_NONE;
577 }
578
e3828811 579 attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
0bbd5f4e
CL		 580
581 printk(KERN_ERR "ioatdma error: interrupt! status %lx\n", attnstatus);
582
e3828811 583 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
0bbd5f4e
CL		 584 return IRQ_HANDLED;
585}
586
587static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan)
588{
589 struct ioat_desc_sw *desc;
590
591 spin_lock_bh(&ioat_chan->desc_lock);
592
593 if (!list_empty(&ioat_chan->free_desc)) {
594 desc = to_ioat_desc(ioat_chan->free_desc.next);
595 list_del(&desc->node);
596 } else {
597 /* try to get another desc */
598 spin_unlock_bh(&ioat_chan->desc_lock);
599 desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL);
600 spin_lock_bh(&ioat_chan->desc_lock);
601 /* will this ever happen? */
602 BUG_ON(!desc);
603 }
604
605 desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL;
606 desc->hw->next = 0;
607
608 list_add_tail(&desc->node, &ioat_chan->used_desc);
609 spin_unlock_bh(&ioat_chan->desc_lock);
610
611#if (BITS_PER_LONG == 64)
e3828811 612 writeq(desc->phys, ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET);
0bbd5f4e 613#else
e3828811
CL		 614 writel((u32) desc->phys,
615 ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_LOW);
616 writel(0, ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_HIGH);
0bbd5f4e 617#endif
e3828811 618 writeb(IOAT_CHANCMD_START, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
0bbd5f4e
CL		 619}
620
621/*
622 * Perform a IOAT transaction to verify the HW works.
623 */
624#define IOAT_TEST_SIZE 2000
625
626static int ioat_self_test(struct ioat_device *device)
627{
628 int i;
629 u8 *src;
630 u8 *dest;
631 struct dma_chan *dma_chan;
632 dma_cookie_t cookie;
633 int err = 0;
634
e94b1766 635 src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
0bbd5f4e
CL		 636 if (!src)
637 return -ENOMEM;
e94b1766 638 dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
0bbd5f4e
CL		 639 if (!dest) {
640 kfree(src);
641 return -ENOMEM;
642 }
643
644 /* Fill in src buffer */
645 for (i = 0; i < IOAT_TEST_SIZE; i++)
646 src[i] = (u8)i;
647
648 /* Start copy, using first DMA channel */
649 dma_chan = container_of(device->common.channels.next,
650 struct dma_chan,
651 device_node);
652 if (ioat_dma_alloc_chan_resources(dma_chan) < 1) {
653 err = -ENODEV;
654 goto out;
655 }
656
657 cookie = ioat_dma_memcpy_buf_to_buf(dma_chan, dest, src, IOAT_TEST_SIZE);
658 ioat_dma_memcpy_issue_pending(dma_chan);
659 msleep(1);
660
661 if (ioat_dma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
662 printk(KERN_ERR "ioatdma: Self-test copy timed out, disabling\n");
663 err = -ENODEV;
664 goto free_resources;
665 }
666 if (memcmp(src, dest, IOAT_TEST_SIZE)) {
667 printk(KERN_ERR "ioatdma: Self-test copy failed compare, disabling\n");
668 err = -ENODEV;
669 goto free_resources;
670 }
671
672free_resources:
673 ioat_dma_free_chan_resources(dma_chan);
674out:
675 kfree(src);
676 kfree(dest);
677 return err;
678}
679
680static int __devinit ioat_probe(struct pci_dev *pdev,
681 const struct pci_device_id *ent)
682{
683 int err;
684 unsigned long mmio_start, mmio_len;
47b16539 685 void __iomem *reg_base;
0bbd5f4e
CL		 686 struct ioat_device *device;
687
688 err = pci_enable_device(pdev);
689 if (err)
690 goto err_enable_device;
691
692 err = pci_set_dma_mask(pdev, DMA_64BIT_MASK);
693 if (err)
694 err = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
695 if (err)
696 goto err_set_dma_mask;
697
92504f79 698 err = pci_request_regions(pdev, ioat_pci_driver.name);
0bbd5f4e
CL		 699 if (err)
700 goto err_request_regions;
701
702 mmio_start = pci_resource_start(pdev, 0);
703 mmio_len = pci_resource_len(pdev, 0);
704
705 reg_base = ioremap(mmio_start, mmio_len);
706 if (!reg_base) {
707 err = -ENOMEM;
708 goto err_ioremap;
709 }
710
711 device = kzalloc(sizeof(*device), GFP_KERNEL);
712 if (!device) {
713 err = -ENOMEM;
714 goto err_kzalloc;
715 }
716
717 /* DMA coherent memory pool for DMA descriptor allocations */
718 device->dma_pool = pci_pool_create("dma_desc_pool", pdev,
719 sizeof(struct ioat_dma_descriptor), 64, 0);
720 if (!device->dma_pool) {
721 err = -ENOMEM;
722 goto err_dma_pool;
723 }
724
725 device->completion_pool = pci_pool_create("completion_pool", pdev, sizeof(u64), SMP_CACHE_BYTES, SMP_CACHE_BYTES);
726 if (!device->completion_pool) {
727 err = -ENOMEM;
728 goto err_completion_pool;
729 }
730
731 device->pdev = pdev;
732 pci_set_drvdata(pdev, device);
733#ifdef CONFIG_PCI_MSI
734 if (pci_enable_msi(pdev) == 0) {
735 device->msi = 1;
736 } else {
737 device->msi = 0;
738 }
739#endif
dace1453 740 err = request_irq(pdev->irq, &ioat_do_interrupt, IRQF_SHARED, "ioat",
0bbd5f4e
CL		 741 device);
742 if (err)
743 goto err_irq;
744
745 device->reg_base = reg_base;
746
e3828811 747 writeb(IOAT_INTRCTRL_MASTER_INT_EN, device->reg_base + IOAT_INTRCTRL_OFFSET);
0bbd5f4e
CL		 748 pci_set_master(pdev);
749
750 INIT_LIST_HEAD(&device->common.channels);
751 enumerate_dma_channels(device);
752
753 device->common.device_alloc_chan_resources = ioat_dma_alloc_chan_resources;
754 device->common.device_free_chan_resources = ioat_dma_free_chan_resources;
755 device->common.device_memcpy_buf_to_buf = ioat_dma_memcpy_buf_to_buf;
756 device->common.device_memcpy_buf_to_pg = ioat_dma_memcpy_buf_to_pg;
757 device->common.device_memcpy_pg_to_pg = ioat_dma_memcpy_pg_to_pg;
758 device->common.device_memcpy_complete = ioat_dma_is_complete;
759 device->common.device_memcpy_issue_pending = ioat_dma_memcpy_issue_pending;
760 printk(KERN_INFO "Intel(R) I/OAT DMA Engine found, %d channels\n",
761 device->common.chancnt);
762
763 err = ioat_self_test(device);
764 if (err)
765 goto err_self_test;
766
767 dma_async_device_register(&device->common);
768
769 return 0;
770
771err_self_test:
772err_irq:
773 pci_pool_destroy(device->completion_pool);
774err_completion_pool:
775 pci_pool_destroy(device->dma_pool);
776err_dma_pool:
777 kfree(device);
778err_kzalloc:
779 iounmap(reg_base);
780err_ioremap:
781 pci_release_regions(pdev);
782err_request_regions:
783err_set_dma_mask:
784 pci_disable_device(pdev);
785err_enable_device:
786 return err;
787}
788
789static void __devexit ioat_remove(struct pci_dev *pdev)
790{
791 struct ioat_device *device;
792 struct dma_chan *chan, *_chan;
793 struct ioat_dma_chan *ioat_chan;
794
795 device = pci_get_drvdata(pdev);
796 dma_async_device_unregister(&device->common);
797
798 free_irq(device->pdev->irq, device);
799#ifdef CONFIG_PCI_MSI
800 if (device->msi)
801 pci_disable_msi(device->pdev);
802#endif
803 pci_pool_destroy(device->dma_pool);
804 pci_pool_destroy(device->completion_pool);
805 iounmap(device->reg_base);
806 pci_release_regions(pdev);
807 pci_disable_device(pdev);
808 list_for_each_entry_safe(chan, _chan, &device->common.channels, device_node) {
809 ioat_chan = to_ioat_chan(chan);
810 list_del(&chan->device_node);
811 kfree(ioat_chan);
812 }
813 kfree(device);
814}
815
816/* MODULE API */
000725d5 817MODULE_VERSION("1.9");
0bbd5f4e
CL		 818MODULE_LICENSE("GPL");
819MODULE_AUTHOR("Intel Corporation");
820
821static int __init ioat_init_module(void)
822{
823 /* it's currently unsafe to unload this module */
824 /* if forced, worst case is that rmmod hangs */
8070b2b1 825 __unsafe(THIS_MODULE);
0bbd5f4e 826
92504f79 827 return pci_register_driver(&ioat_pci_driver);
0bbd5f4e
CL		 828}
829
830module_init(ioat_init_module);
831
832static void __exit ioat_exit_module(void)
833{
92504f79 834 pci_unregister_driver(&ioat_pci_driver);
0bbd5f4e
CL		 835}
836
837module_exit(ioat_exit_module);
Linux 6.x block layer and io_uring tree(s)