Commit | Line | Data |
---|---|---|
0bbd5f4e CL |
1 | /* |
2 | * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify it | |
5 | * under the terms of the GNU General Public License as published by the Free | |
6 | * Software Foundation; either version 2 of the License, or (at your option) | |
7 | * any later version. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, but WITHOUT | |
10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
12 | * more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License along with | |
15 | * this program; if not, write to the Free Software Foundation, Inc., 59 | |
16 | * Temple Place - Suite 330, Boston, MA 02111-1307, USA. | |
17 | * | |
18 | * The full GNU General Public License is included in this distribution in the | |
19 | * file called COPYING. | |
20 | */ | |
21 | ||
22 | /* | |
23 | * This driver supports an Intel I/OAT DMA engine, which does asynchronous | |
24 | * copy operations. | |
25 | */ | |
26 | ||
27 | #include <linux/init.h> | |
28 | #include <linux/module.h> | |
29 | #include <linux/pci.h> | |
30 | #include <linux/interrupt.h> | |
31 | #include <linux/dmaengine.h> | |
32 | #include <linux/delay.h> | |
6b00c92c | 33 | #include <linux/dma-mapping.h> |
0bbd5f4e | 34 | #include "ioatdma.h" |
0bbd5f4e CL |
35 | #include "ioatdma_registers.h" |
36 | #include "ioatdma_hw.h" | |
37 | ||
38 | #define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common) | |
39 | #define to_ioat_device(dev) container_of(dev, struct ioat_device, common) | |
40 | #define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node) | |
41 | ||
42 | /* internal functions */ | |
43 | static int __devinit ioat_probe(struct pci_dev *pdev, const struct pci_device_id *ent); | |
44 | static void __devexit ioat_remove(struct pci_dev *pdev); | |
45 | ||
46 | static int enumerate_dma_channels(struct ioat_device *device) | |
47 | { | |
48 | u8 xfercap_scale; | |
49 | u32 xfercap; | |
50 | int i; | |
51 | struct ioat_dma_chan *ioat_chan; | |
52 | ||
e3828811 CL |
53 | device->common.chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET); |
54 | xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET); | |
0bbd5f4e CL |
55 | xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale)); |
56 | ||
57 | for (i = 0; i < device->common.chancnt; i++) { | |
58 | ioat_chan = kzalloc(sizeof(*ioat_chan), GFP_KERNEL); | |
59 | if (!ioat_chan) { | |
60 | device->common.chancnt = i; | |
61 | break; | |
62 | } | |
63 | ||
64 | ioat_chan->device = device; | |
65 | ioat_chan->reg_base = device->reg_base + (0x80 * (i + 1)); | |
66 | ioat_chan->xfercap = xfercap; | |
67 | spin_lock_init(&ioat_chan->cleanup_lock); | |
68 | spin_lock_init(&ioat_chan->desc_lock); | |
69 | INIT_LIST_HEAD(&ioat_chan->free_desc); | |
70 | INIT_LIST_HEAD(&ioat_chan->used_desc); | |
71 | /* This should be made common somewhere in dmaengine.c */ | |
72 | ioat_chan->common.device = &device->common; | |
73 | ioat_chan->common.client = NULL; | |
74 | list_add_tail(&ioat_chan->common.device_node, | |
75 | &device->common.channels); | |
76 | } | |
77 | return device->common.chancnt; | |
78 | } | |
79 | ||
80 | static struct ioat_desc_sw *ioat_dma_alloc_descriptor( | |
81 | struct ioat_dma_chan *ioat_chan, | |
47b16539 | 82 | gfp_t flags) |
0bbd5f4e CL |
83 | { |
84 | struct ioat_dma_descriptor *desc; | |
85 | struct ioat_desc_sw *desc_sw; | |
86 | struct ioat_device *ioat_device; | |
87 | dma_addr_t phys; | |
88 | ||
89 | ioat_device = to_ioat_device(ioat_chan->common.device); | |
90 | desc = pci_pool_alloc(ioat_device->dma_pool, flags, &phys); | |
91 | if (unlikely(!desc)) | |
92 | return NULL; | |
93 | ||
94 | desc_sw = kzalloc(sizeof(*desc_sw), flags); | |
95 | if (unlikely(!desc_sw)) { | |
96 | pci_pool_free(ioat_device->dma_pool, desc, phys); | |
97 | return NULL; | |
98 | } | |
99 | ||
100 | memset(desc, 0, sizeof(*desc)); | |
101 | desc_sw->hw = desc; | |
102 | desc_sw->phys = phys; | |
103 | ||
104 | return desc_sw; | |
105 | } | |
106 | ||
107 | #define INITIAL_IOAT_DESC_COUNT 128 | |
108 | ||
109 | static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan); | |
110 | ||
111 | /* returns the actual number of allocated descriptors */ | |
112 | static int ioat_dma_alloc_chan_resources(struct dma_chan *chan) | |
113 | { | |
114 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | |
115 | struct ioat_desc_sw *desc = NULL; | |
116 | u16 chanctrl; | |
117 | u32 chanerr; | |
118 | int i; | |
119 | LIST_HEAD(tmp_list); | |
120 | ||
121 | /* | |
122 | * In-use bit automatically set by reading chanctrl | |
123 | * If 0, we got it, if 1, someone else did | |
124 | */ | |
e3828811 | 125 | chanctrl = readw(ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET); |
0bbd5f4e CL |
126 | if (chanctrl & IOAT_CHANCTRL_CHANNEL_IN_USE) |
127 | return -EBUSY; | |
128 | ||
129 | /* Setup register to interrupt and write completion status on error */ | |
130 | chanctrl = IOAT_CHANCTRL_CHANNEL_IN_USE | | |
131 | IOAT_CHANCTRL_ERR_INT_EN | | |
132 | IOAT_CHANCTRL_ANY_ERR_ABORT_EN | | |
133 | IOAT_CHANCTRL_ERR_COMPLETION_EN; | |
e3828811 | 134 | writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET); |
0bbd5f4e | 135 | |
e3828811 | 136 | chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); |
0bbd5f4e CL |
137 | if (chanerr) { |
138 | printk("IOAT: CHANERR = %x, clearing\n", chanerr); | |
e3828811 | 139 | writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET); |
0bbd5f4e CL |
140 | } |
141 | ||
142 | /* Allocate descriptors */ | |
143 | for (i = 0; i < INITIAL_IOAT_DESC_COUNT; i++) { | |
144 | desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL); | |
145 | if (!desc) { | |
146 | printk(KERN_ERR "IOAT: Only %d initial descriptors\n", i); | |
147 | break; | |
148 | } | |
149 | list_add_tail(&desc->node, &tmp_list); | |
150 | } | |
151 | spin_lock_bh(&ioat_chan->desc_lock); | |
152 | list_splice(&tmp_list, &ioat_chan->free_desc); | |
153 | spin_unlock_bh(&ioat_chan->desc_lock); | |
154 | ||
155 | /* allocate a completion writeback area */ | |
156 | /* doing 2 32bit writes to mmio since 1 64b write doesn't work */ | |
157 | ioat_chan->completion_virt = | |
158 | pci_pool_alloc(ioat_chan->device->completion_pool, | |
159 | GFP_KERNEL, | |
160 | &ioat_chan->completion_addr); | |
161 | memset(ioat_chan->completion_virt, 0, | |
162 | sizeof(*ioat_chan->completion_virt)); | |
e3828811 CL |
163 | writel(((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF, |
164 | ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW); | |
165 | writel(((u64) ioat_chan->completion_addr) >> 32, | |
166 | ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH); | |
0bbd5f4e CL |
167 | |
168 | ioat_start_null_desc(ioat_chan); | |
169 | return i; | |
170 | } | |
171 | ||
172 | static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan); | |
173 | ||
174 | static void ioat_dma_free_chan_resources(struct dma_chan *chan) | |
175 | { | |
176 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | |
177 | struct ioat_device *ioat_device = to_ioat_device(chan->device); | |
178 | struct ioat_desc_sw *desc, *_desc; | |
179 | u16 chanctrl; | |
180 | int in_use_descs = 0; | |
181 | ||
182 | ioat_dma_memcpy_cleanup(ioat_chan); | |
183 | ||
e3828811 | 184 | writeb(IOAT_CHANCMD_RESET, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET); |
0bbd5f4e CL |
185 | |
186 | spin_lock_bh(&ioat_chan->desc_lock); | |
187 | list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) { | |
188 | in_use_descs++; | |
189 | list_del(&desc->node); | |
190 | pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys); | |
191 | kfree(desc); | |
192 | } | |
193 | list_for_each_entry_safe(desc, _desc, &ioat_chan->free_desc, node) { | |
194 | list_del(&desc->node); | |
195 | pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys); | |
196 | kfree(desc); | |
197 | } | |
198 | spin_unlock_bh(&ioat_chan->desc_lock); | |
199 | ||
200 | pci_pool_free(ioat_device->completion_pool, | |
201 | ioat_chan->completion_virt, | |
202 | ioat_chan->completion_addr); | |
203 | ||
204 | /* one is ok since we left it on there on purpose */ | |
205 | if (in_use_descs > 1) | |
206 | printk(KERN_ERR "IOAT: Freeing %d in use descriptors!\n", | |
207 | in_use_descs - 1); | |
208 | ||
209 | ioat_chan->last_completion = ioat_chan->completion_addr = 0; | |
210 | ||
211 | /* Tell hw the chan is free */ | |
e3828811 | 212 | chanctrl = readw(ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET); |
0bbd5f4e | 213 | chanctrl &= ~IOAT_CHANCTRL_CHANNEL_IN_USE; |
e3828811 | 214 | writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET); |
0bbd5f4e CL |
215 | } |
216 | ||
217 | /** | |
218 | * do_ioat_dma_memcpy - actual function that initiates a IOAT DMA transaction | |
6508871e | 219 | * @ioat_chan: IOAT DMA channel handle |
0bbd5f4e CL |
220 | * @dest: DMA destination address |
221 | * @src: DMA source address | |
222 | * @len: transaction length in bytes | |
223 | */ | |
224 | ||
225 | static dma_cookie_t do_ioat_dma_memcpy(struct ioat_dma_chan *ioat_chan, | |
226 | dma_addr_t dest, | |
227 | dma_addr_t src, | |
228 | size_t len) | |
229 | { | |
230 | struct ioat_desc_sw *first; | |
231 | struct ioat_desc_sw *prev; | |
232 | struct ioat_desc_sw *new; | |
233 | dma_cookie_t cookie; | |
234 | LIST_HEAD(new_chain); | |
235 | u32 copy; | |
236 | size_t orig_len; | |
237 | dma_addr_t orig_src, orig_dst; | |
238 | unsigned int desc_count = 0; | |
239 | unsigned int append = 0; | |
240 | ||
241 | if (!ioat_chan || !dest || !src) | |
242 | return -EFAULT; | |
243 | ||
244 | if (!len) | |
245 | return ioat_chan->common.cookie; | |
246 | ||
247 | orig_len = len; | |
248 | orig_src = src; | |
249 | orig_dst = dest; | |
250 | ||
251 | first = NULL; | |
252 | prev = NULL; | |
253 | ||
254 | spin_lock_bh(&ioat_chan->desc_lock); | |
255 | ||
256 | while (len) { | |
257 | if (!list_empty(&ioat_chan->free_desc)) { | |
258 | new = to_ioat_desc(ioat_chan->free_desc.next); | |
259 | list_del(&new->node); | |
260 | } else { | |
261 | /* try to get another desc */ | |
262 | new = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC); | |
263 | /* will this ever happen? */ | |
264 | /* TODO add upper limit on these */ | |
265 | BUG_ON(!new); | |
266 | } | |
267 | ||
268 | copy = min((u32) len, ioat_chan->xfercap); | |
269 | ||
270 | new->hw->size = copy; | |
271 | new->hw->ctl = 0; | |
272 | new->hw->src_addr = src; | |
273 | new->hw->dst_addr = dest; | |
274 | new->cookie = 0; | |
275 | ||
276 | /* chain together the physical address list for the HW */ | |
277 | if (!first) | |
278 | first = new; | |
279 | else | |
280 | prev->hw->next = (u64) new->phys; | |
281 | ||
282 | prev = new; | |
283 | ||
284 | len -= copy; | |
285 | dest += copy; | |
286 | src += copy; | |
287 | ||
288 | list_add_tail(&new->node, &new_chain); | |
289 | desc_count++; | |
290 | } | |
291 | new->hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS; | |
292 | new->hw->next = 0; | |
293 | ||
294 | /* cookie incr and addition to used_list must be atomic */ | |
295 | ||
296 | cookie = ioat_chan->common.cookie; | |
297 | cookie++; | |
298 | if (cookie < 0) | |
299 | cookie = 1; | |
300 | ioat_chan->common.cookie = new->cookie = cookie; | |
301 | ||
302 | pci_unmap_addr_set(new, src, orig_src); | |
303 | pci_unmap_addr_set(new, dst, orig_dst); | |
304 | pci_unmap_len_set(new, src_len, orig_len); | |
305 | pci_unmap_len_set(new, dst_len, orig_len); | |
306 | ||
307 | /* write address into NextDescriptor field of last desc in chain */ | |
308 | to_ioat_desc(ioat_chan->used_desc.prev)->hw->next = first->phys; | |
309 | list_splice_init(&new_chain, ioat_chan->used_desc.prev); | |
310 | ||
311 | ioat_chan->pending += desc_count; | |
000725d5 | 312 | if (ioat_chan->pending >= 4) { |
0bbd5f4e CL |
313 | append = 1; |
314 | ioat_chan->pending = 0; | |
315 | } | |
316 | ||
317 | spin_unlock_bh(&ioat_chan->desc_lock); | |
318 | ||
319 | if (append) | |
e3828811 CL |
320 | writeb(IOAT_CHANCMD_APPEND, |
321 | ioat_chan->reg_base + IOAT_CHANCMD_OFFSET); | |
0bbd5f4e CL |
322 | return cookie; |
323 | } | |
324 | ||
325 | /** | |
326 | * ioat_dma_memcpy_buf_to_buf - wrapper that takes src & dest bufs | |
327 | * @chan: IOAT DMA channel handle | |
328 | * @dest: DMA destination address | |
329 | * @src: DMA source address | |
330 | * @len: transaction length in bytes | |
331 | */ | |
332 | ||
333 | static dma_cookie_t ioat_dma_memcpy_buf_to_buf(struct dma_chan *chan, | |
334 | void *dest, | |
335 | void *src, | |
336 | size_t len) | |
337 | { | |
338 | dma_addr_t dest_addr; | |
339 | dma_addr_t src_addr; | |
340 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | |
341 | ||
342 | dest_addr = pci_map_single(ioat_chan->device->pdev, | |
343 | dest, len, PCI_DMA_FROMDEVICE); | |
344 | src_addr = pci_map_single(ioat_chan->device->pdev, | |
345 | src, len, PCI_DMA_TODEVICE); | |
346 | ||
347 | return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len); | |
348 | } | |
349 | ||
350 | /** | |
351 | * ioat_dma_memcpy_buf_to_pg - wrapper, copying from a buf to a page | |
352 | * @chan: IOAT DMA channel handle | |
353 | * @page: pointer to the page to copy to | |
354 | * @offset: offset into that page | |
355 | * @src: DMA source address | |
356 | * @len: transaction length in bytes | |
357 | */ | |
358 | ||
359 | static dma_cookie_t ioat_dma_memcpy_buf_to_pg(struct dma_chan *chan, | |
360 | struct page *page, | |
361 | unsigned int offset, | |
362 | void *src, | |
363 | size_t len) | |
364 | { | |
365 | dma_addr_t dest_addr; | |
366 | dma_addr_t src_addr; | |
367 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | |
368 | ||
369 | dest_addr = pci_map_page(ioat_chan->device->pdev, | |
370 | page, offset, len, PCI_DMA_FROMDEVICE); | |
371 | src_addr = pci_map_single(ioat_chan->device->pdev, | |
372 | src, len, PCI_DMA_TODEVICE); | |
373 | ||
374 | return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len); | |
375 | } | |
376 | ||
377 | /** | |
378 | * ioat_dma_memcpy_pg_to_pg - wrapper, copying between two pages | |
379 | * @chan: IOAT DMA channel handle | |
380 | * @dest_pg: pointer to the page to copy to | |
381 | * @dest_off: offset into that page | |
382 | * @src_pg: pointer to the page to copy from | |
383 | * @src_off: offset into that page | |
6508871e | 384 | * @len: transaction length in bytes. This is guaranteed not to make a copy |
0bbd5f4e CL |
385 | * across a page boundary. |
386 | */ | |
387 | ||
388 | static dma_cookie_t ioat_dma_memcpy_pg_to_pg(struct dma_chan *chan, | |
389 | struct page *dest_pg, | |
390 | unsigned int dest_off, | |
391 | struct page *src_pg, | |
392 | unsigned int src_off, | |
393 | size_t len) | |
394 | { | |
395 | dma_addr_t dest_addr; | |
396 | dma_addr_t src_addr; | |
397 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | |
398 | ||
399 | dest_addr = pci_map_page(ioat_chan->device->pdev, | |
400 | dest_pg, dest_off, len, PCI_DMA_FROMDEVICE); | |
401 | src_addr = pci_map_page(ioat_chan->device->pdev, | |
402 | src_pg, src_off, len, PCI_DMA_TODEVICE); | |
403 | ||
404 | return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len); | |
405 | } | |
406 | ||
407 | /** | |
6508871e | 408 | * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended descriptors to hw |
0bbd5f4e CL |
409 | * @chan: DMA channel handle |
410 | */ | |
411 | ||
412 | static void ioat_dma_memcpy_issue_pending(struct dma_chan *chan) | |
413 | { | |
414 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | |
415 | ||
416 | if (ioat_chan->pending != 0) { | |
417 | ioat_chan->pending = 0; | |
e3828811 CL |
418 | writeb(IOAT_CHANCMD_APPEND, |
419 | ioat_chan->reg_base + IOAT_CHANCMD_OFFSET); | |
0bbd5f4e CL |
420 | } |
421 | } | |
422 | ||
423 | static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan) | |
424 | { | |
425 | unsigned long phys_complete; | |
426 | struct ioat_desc_sw *desc, *_desc; | |
427 | dma_cookie_t cookie = 0; | |
428 | ||
429 | prefetch(chan->completion_virt); | |
430 | ||
431 | if (!spin_trylock(&chan->cleanup_lock)) | |
432 | return; | |
433 | ||
434 | /* The completion writeback can happen at any time, | |
435 | so reads by the driver need to be atomic operations | |
436 | The descriptor physical addresses are limited to 32-bits | |
437 | when the CPU can only do a 32-bit mov */ | |
438 | ||
439 | #if (BITS_PER_LONG == 64) | |
440 | phys_complete = | |
441 | chan->completion_virt->full & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR; | |
442 | #else | |
443 | phys_complete = chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK; | |
444 | #endif | |
445 | ||
446 | if ((chan->completion_virt->full & IOAT_CHANSTS_DMA_TRANSFER_STATUS) == | |
447 | IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) { | |
448 | printk("IOAT: Channel halted, chanerr = %x\n", | |
e3828811 | 449 | readl(chan->reg_base + IOAT_CHANERR_OFFSET)); |
0bbd5f4e CL |
450 | |
451 | /* TODO do something to salvage the situation */ | |
452 | } | |
453 | ||
454 | if (phys_complete == chan->last_completion) { | |
455 | spin_unlock(&chan->cleanup_lock); | |
456 | return; | |
457 | } | |
458 | ||
459 | spin_lock_bh(&chan->desc_lock); | |
460 | list_for_each_entry_safe(desc, _desc, &chan->used_desc, node) { | |
461 | ||
462 | /* | |
463 | * Incoming DMA requests may use multiple descriptors, due to | |
464 | * exceeding xfercap, perhaps. If so, only the last one will | |
465 | * have a cookie, and require unmapping. | |
466 | */ | |
467 | if (desc->cookie) { | |
468 | cookie = desc->cookie; | |
469 | ||
470 | /* yes we are unmapping both _page and _single alloc'd | |
471 | regions with unmap_page. Is this *really* that bad? | |
472 | */ | |
473 | pci_unmap_page(chan->device->pdev, | |
474 | pci_unmap_addr(desc, dst), | |
475 | pci_unmap_len(desc, dst_len), | |
476 | PCI_DMA_FROMDEVICE); | |
477 | pci_unmap_page(chan->device->pdev, | |
478 | pci_unmap_addr(desc, src), | |
479 | pci_unmap_len(desc, src_len), | |
480 | PCI_DMA_TODEVICE); | |
481 | } | |
482 | ||
483 | if (desc->phys != phys_complete) { | |
484 | /* a completed entry, but not the last, so cleanup */ | |
485 | list_del(&desc->node); | |
486 | list_add_tail(&desc->node, &chan->free_desc); | |
487 | } else { | |
488 | /* last used desc. Do not remove, so we can append from | |
489 | it, but don't look at it next time, either */ | |
490 | desc->cookie = 0; | |
491 | ||
492 | /* TODO check status bits? */ | |
493 | break; | |
494 | } | |
495 | } | |
496 | ||
497 | spin_unlock_bh(&chan->desc_lock); | |
498 | ||
499 | chan->last_completion = phys_complete; | |
500 | if (cookie != 0) | |
501 | chan->completed_cookie = cookie; | |
502 | ||
503 | spin_unlock(&chan->cleanup_lock); | |
504 | } | |
505 | ||
506 | /** | |
507 | * ioat_dma_is_complete - poll the status of a IOAT DMA transaction | |
508 | * @chan: IOAT DMA channel handle | |
509 | * @cookie: DMA transaction identifier | |
6508871e RD |
510 | * @done: if not %NULL, updated with last completed transaction |
511 | * @used: if not %NULL, updated with last used transaction | |
0bbd5f4e CL |
512 | */ |
513 | ||
514 | static enum dma_status ioat_dma_is_complete(struct dma_chan *chan, | |
515 | dma_cookie_t cookie, | |
516 | dma_cookie_t *done, | |
517 | dma_cookie_t *used) | |
518 | { | |
519 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | |
520 | dma_cookie_t last_used; | |
521 | dma_cookie_t last_complete; | |
522 | enum dma_status ret; | |
523 | ||
524 | last_used = chan->cookie; | |
525 | last_complete = ioat_chan->completed_cookie; | |
526 | ||
527 | if (done) | |
528 | *done= last_complete; | |
529 | if (used) | |
530 | *used = last_used; | |
531 | ||
532 | ret = dma_async_is_complete(cookie, last_complete, last_used); | |
533 | if (ret == DMA_SUCCESS) | |
534 | return ret; | |
535 | ||
536 | ioat_dma_memcpy_cleanup(ioat_chan); | |
537 | ||
538 | last_used = chan->cookie; | |
539 | last_complete = ioat_chan->completed_cookie; | |
540 | ||
541 | if (done) | |
542 | *done= last_complete; | |
543 | if (used) | |
544 | *used = last_used; | |
545 | ||
546 | return dma_async_is_complete(cookie, last_complete, last_used); | |
547 | } | |
548 | ||
549 | /* PCI API */ | |
550 | ||
551 | static struct pci_device_id ioat_pci_tbl[] = { | |
552 | { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT) }, | |
553 | { 0, } | |
554 | }; | |
555 | ||
92504f79 | 556 | static struct pci_driver ioat_pci_driver = { |
0bbd5f4e CL |
557 | .name = "ioatdma", |
558 | .id_table = ioat_pci_tbl, | |
559 | .probe = ioat_probe, | |
560 | .remove = __devexit_p(ioat_remove), | |
561 | }; | |
562 | ||
7d12e780 | 563 | static irqreturn_t ioat_do_interrupt(int irq, void *data) |
0bbd5f4e CL |
564 | { |
565 | struct ioat_device *instance = data; | |
566 | unsigned long attnstatus; | |
567 | u8 intrctrl; | |
568 | ||
e3828811 | 569 | intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET); |
0bbd5f4e CL |
570 | |
571 | if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN)) | |
572 | return IRQ_NONE; | |
573 | ||
574 | if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) { | |
e3828811 | 575 | writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET); |
0bbd5f4e CL |
576 | return IRQ_NONE; |
577 | } | |
578 | ||
e3828811 | 579 | attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET); |
0bbd5f4e CL |
580 | |
581 | printk(KERN_ERR "ioatdma error: interrupt! status %lx\n", attnstatus); | |
582 | ||
e3828811 | 583 | writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET); |
0bbd5f4e CL |
584 | return IRQ_HANDLED; |
585 | } | |
586 | ||
587 | static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan) | |
588 | { | |
589 | struct ioat_desc_sw *desc; | |
590 | ||
591 | spin_lock_bh(&ioat_chan->desc_lock); | |
592 | ||
593 | if (!list_empty(&ioat_chan->free_desc)) { | |
594 | desc = to_ioat_desc(ioat_chan->free_desc.next); | |
595 | list_del(&desc->node); | |
596 | } else { | |
597 | /* try to get another desc */ | |
598 | spin_unlock_bh(&ioat_chan->desc_lock); | |
599 | desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL); | |
600 | spin_lock_bh(&ioat_chan->desc_lock); | |
601 | /* will this ever happen? */ | |
602 | BUG_ON(!desc); | |
603 | } | |
604 | ||
605 | desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL; | |
606 | desc->hw->next = 0; | |
607 | ||
608 | list_add_tail(&desc->node, &ioat_chan->used_desc); | |
609 | spin_unlock_bh(&ioat_chan->desc_lock); | |
610 | ||
70774b47 | 611 | writel(((u64) desc->phys) & 0x00000000FFFFFFFF, |
e3828811 | 612 | ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_LOW); |
70774b47 CL |
613 | writel(((u64) desc->phys) >> 32, |
614 | ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_HIGH); | |
615 | ||
e3828811 | 616 | writeb(IOAT_CHANCMD_START, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET); |
0bbd5f4e CL |
617 | } |
618 | ||
619 | /* | |
620 | * Perform a IOAT transaction to verify the HW works. | |
621 | */ | |
622 | #define IOAT_TEST_SIZE 2000 | |
623 | ||
624 | static int ioat_self_test(struct ioat_device *device) | |
625 | { | |
626 | int i; | |
627 | u8 *src; | |
628 | u8 *dest; | |
629 | struct dma_chan *dma_chan; | |
630 | dma_cookie_t cookie; | |
631 | int err = 0; | |
632 | ||
e94b1766 | 633 | src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL); |
0bbd5f4e CL |
634 | if (!src) |
635 | return -ENOMEM; | |
e94b1766 | 636 | dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL); |
0bbd5f4e CL |
637 | if (!dest) { |
638 | kfree(src); | |
639 | return -ENOMEM; | |
640 | } | |
641 | ||
642 | /* Fill in src buffer */ | |
643 | for (i = 0; i < IOAT_TEST_SIZE; i++) | |
644 | src[i] = (u8)i; | |
645 | ||
646 | /* Start copy, using first DMA channel */ | |
647 | dma_chan = container_of(device->common.channels.next, | |
648 | struct dma_chan, | |
649 | device_node); | |
650 | if (ioat_dma_alloc_chan_resources(dma_chan) < 1) { | |
651 | err = -ENODEV; | |
652 | goto out; | |
653 | } | |
654 | ||
655 | cookie = ioat_dma_memcpy_buf_to_buf(dma_chan, dest, src, IOAT_TEST_SIZE); | |
656 | ioat_dma_memcpy_issue_pending(dma_chan); | |
657 | msleep(1); | |
658 | ||
659 | if (ioat_dma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) { | |
660 | printk(KERN_ERR "ioatdma: Self-test copy timed out, disabling\n"); | |
661 | err = -ENODEV; | |
662 | goto free_resources; | |
663 | } | |
664 | if (memcmp(src, dest, IOAT_TEST_SIZE)) { | |
665 | printk(KERN_ERR "ioatdma: Self-test copy failed compare, disabling\n"); | |
666 | err = -ENODEV; | |
667 | goto free_resources; | |
668 | } | |
669 | ||
670 | free_resources: | |
671 | ioat_dma_free_chan_resources(dma_chan); | |
672 | out: | |
673 | kfree(src); | |
674 | kfree(dest); | |
675 | return err; | |
676 | } | |
677 | ||
678 | static int __devinit ioat_probe(struct pci_dev *pdev, | |
679 | const struct pci_device_id *ent) | |
680 | { | |
681 | int err; | |
682 | unsigned long mmio_start, mmio_len; | |
47b16539 | 683 | void __iomem *reg_base; |
0bbd5f4e CL |
684 | struct ioat_device *device; |
685 | ||
686 | err = pci_enable_device(pdev); | |
687 | if (err) | |
688 | goto err_enable_device; | |
689 | ||
690 | err = pci_set_dma_mask(pdev, DMA_64BIT_MASK); | |
691 | if (err) | |
692 | err = pci_set_dma_mask(pdev, DMA_32BIT_MASK); | |
693 | if (err) | |
694 | goto err_set_dma_mask; | |
695 | ||
92504f79 | 696 | err = pci_request_regions(pdev, ioat_pci_driver.name); |
0bbd5f4e CL |
697 | if (err) |
698 | goto err_request_regions; | |
699 | ||
700 | mmio_start = pci_resource_start(pdev, 0); | |
701 | mmio_len = pci_resource_len(pdev, 0); | |
702 | ||
703 | reg_base = ioremap(mmio_start, mmio_len); | |
704 | if (!reg_base) { | |
705 | err = -ENOMEM; | |
706 | goto err_ioremap; | |
707 | } | |
708 | ||
709 | device = kzalloc(sizeof(*device), GFP_KERNEL); | |
710 | if (!device) { | |
711 | err = -ENOMEM; | |
712 | goto err_kzalloc; | |
713 | } | |
714 | ||
715 | /* DMA coherent memory pool for DMA descriptor allocations */ | |
716 | device->dma_pool = pci_pool_create("dma_desc_pool", pdev, | |
717 | sizeof(struct ioat_dma_descriptor), 64, 0); | |
718 | if (!device->dma_pool) { | |
719 | err = -ENOMEM; | |
720 | goto err_dma_pool; | |
721 | } | |
722 | ||
723 | device->completion_pool = pci_pool_create("completion_pool", pdev, sizeof(u64), SMP_CACHE_BYTES, SMP_CACHE_BYTES); | |
724 | if (!device->completion_pool) { | |
725 | err = -ENOMEM; | |
726 | goto err_completion_pool; | |
727 | } | |
728 | ||
729 | device->pdev = pdev; | |
730 | pci_set_drvdata(pdev, device); | |
731 | #ifdef CONFIG_PCI_MSI | |
732 | if (pci_enable_msi(pdev) == 0) { | |
733 | device->msi = 1; | |
734 | } else { | |
735 | device->msi = 0; | |
736 | } | |
737 | #endif | |
dace1453 | 738 | err = request_irq(pdev->irq, &ioat_do_interrupt, IRQF_SHARED, "ioat", |
0bbd5f4e CL |
739 | device); |
740 | if (err) | |
741 | goto err_irq; | |
742 | ||
743 | device->reg_base = reg_base; | |
744 | ||
e3828811 | 745 | writeb(IOAT_INTRCTRL_MASTER_INT_EN, device->reg_base + IOAT_INTRCTRL_OFFSET); |
0bbd5f4e CL |
746 | pci_set_master(pdev); |
747 | ||
748 | INIT_LIST_HEAD(&device->common.channels); | |
749 | enumerate_dma_channels(device); | |
750 | ||
751 | device->common.device_alloc_chan_resources = ioat_dma_alloc_chan_resources; | |
752 | device->common.device_free_chan_resources = ioat_dma_free_chan_resources; | |
753 | device->common.device_memcpy_buf_to_buf = ioat_dma_memcpy_buf_to_buf; | |
754 | device->common.device_memcpy_buf_to_pg = ioat_dma_memcpy_buf_to_pg; | |
755 | device->common.device_memcpy_pg_to_pg = ioat_dma_memcpy_pg_to_pg; | |
756 | device->common.device_memcpy_complete = ioat_dma_is_complete; | |
757 | device->common.device_memcpy_issue_pending = ioat_dma_memcpy_issue_pending; | |
758 | printk(KERN_INFO "Intel(R) I/OAT DMA Engine found, %d channels\n", | |
759 | device->common.chancnt); | |
760 | ||
761 | err = ioat_self_test(device); | |
762 | if (err) | |
763 | goto err_self_test; | |
764 | ||
765 | dma_async_device_register(&device->common); | |
766 | ||
767 | return 0; | |
768 | ||
769 | err_self_test: | |
770 | err_irq: | |
771 | pci_pool_destroy(device->completion_pool); | |
772 | err_completion_pool: | |
773 | pci_pool_destroy(device->dma_pool); | |
774 | err_dma_pool: | |
775 | kfree(device); | |
776 | err_kzalloc: | |
777 | iounmap(reg_base); | |
778 | err_ioremap: | |
779 | pci_release_regions(pdev); | |
780 | err_request_regions: | |
781 | err_set_dma_mask: | |
782 | pci_disable_device(pdev); | |
783 | err_enable_device: | |
784 | return err; | |
785 | } | |
786 | ||
787 | static void __devexit ioat_remove(struct pci_dev *pdev) | |
788 | { | |
789 | struct ioat_device *device; | |
790 | struct dma_chan *chan, *_chan; | |
791 | struct ioat_dma_chan *ioat_chan; | |
792 | ||
793 | device = pci_get_drvdata(pdev); | |
794 | dma_async_device_unregister(&device->common); | |
795 | ||
796 | free_irq(device->pdev->irq, device); | |
797 | #ifdef CONFIG_PCI_MSI | |
798 | if (device->msi) | |
799 | pci_disable_msi(device->pdev); | |
800 | #endif | |
801 | pci_pool_destroy(device->dma_pool); | |
802 | pci_pool_destroy(device->completion_pool); | |
803 | iounmap(device->reg_base); | |
804 | pci_release_regions(pdev); | |
805 | pci_disable_device(pdev); | |
806 | list_for_each_entry_safe(chan, _chan, &device->common.channels, device_node) { | |
807 | ioat_chan = to_ioat_chan(chan); | |
808 | list_del(&chan->device_node); | |
809 | kfree(ioat_chan); | |
810 | } | |
811 | kfree(device); | |
812 | } | |
813 | ||
814 | /* MODULE API */ | |
000725d5 | 815 | MODULE_VERSION("1.9"); |
0bbd5f4e CL |
816 | MODULE_LICENSE("GPL"); |
817 | MODULE_AUTHOR("Intel Corporation"); | |
818 | ||
819 | static int __init ioat_init_module(void) | |
820 | { | |
821 | /* it's currently unsafe to unload this module */ | |
822 | /* if forced, worst case is that rmmod hangs */ | |
8070b2b1 | 823 | __unsafe(THIS_MODULE); |
0bbd5f4e | 824 | |
92504f79 | 825 | return pci_register_driver(&ioat_pci_driver); |
0bbd5f4e CL |
826 | } |
827 | ||
828 | module_init(ioat_init_module); | |
829 | ||
830 | static void __exit ioat_exit_module(void) | |
831 | { | |
92504f79 | 832 | pci_unregister_driver(&ioat_pci_driver); |
0bbd5f4e CL |
833 | } |
834 | ||
835 | module_exit(ioat_exit_module); |