Commit | Line | Data |
---|---|---|
0bbd5f4e | 1 | /* |
43d6e369 | 2 | * Intel I/OAT DMA Linux driver |
211a22ce | 3 | * Copyright(c) 2004 - 2009 Intel Corporation. |
0bbd5f4e CL |
4 | * |
5 | * This program is free software; you can redistribute it and/or modify it | |
43d6e369 SN |
6 | * under the terms and conditions of the GNU General Public License, |
7 | * version 2, as published by the Free Software Foundation. | |
0bbd5f4e CL |
8 | * |
9 | * This program is distributed in the hope that it will be useful, but WITHOUT | |
10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
12 | * more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License along with | |
43d6e369 SN |
15 | * this program; if not, write to the Free Software Foundation, Inc., |
16 | * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. | |
17 | * | |
18 | * The full GNU General Public License is included in this distribution in | |
19 | * the file called "COPYING". | |
0bbd5f4e | 20 | * |
0bbd5f4e CL |
21 | */ |
22 | ||
23 | /* | |
24 | * This driver supports an Intel I/OAT DMA engine, which does asynchronous | |
25 | * copy operations. | |
26 | */ | |
27 | ||
28 | #include <linux/init.h> | |
29 | #include <linux/module.h> | |
30 | #include <linux/pci.h> | |
31 | #include <linux/interrupt.h> | |
32 | #include <linux/dmaengine.h> | |
33 | #include <linux/delay.h> | |
6b00c92c | 34 | #include <linux/dma-mapping.h> |
09177e85 | 35 | #include <linux/workqueue.h> |
3ad0b02e | 36 | #include <linux/i7300_idle.h> |
584ec227 DW |
37 | #include "dma.h" |
38 | #include "registers.h" | |
39 | #include "hw.h" | |
0bbd5f4e | 40 | |
7bb67c14 SN |
41 | static int ioat_pending_level = 4; |
42 | module_param(ioat_pending_level, int, 0644); | |
43 | MODULE_PARM_DESC(ioat_pending_level, | |
44 | "high-water mark for pushing ioat descriptors (default: 4)"); | |
45 | ||
09177e85 MS |
46 | static void ioat_dma_chan_reset_part2(struct work_struct *work); |
47 | static void ioat_dma_chan_watchdog(struct work_struct *work); | |
48 | ||
0bbd5f4e | 49 | /* internal functions */ |
dcbc853a DW |
50 | static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat); |
51 | static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat); | |
7bb67c14 SN |
52 | |
53 | static struct ioat_desc_sw * | |
dcbc853a | 54 | ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat); |
7f2b291f | 55 | static struct ioat_desc_sw * |
dcbc853a | 56 | ioat2_dma_get_next_descriptor(struct ioat_dma_chan *ioat); |
0bbd5f4e | 57 | |
dcbc853a | 58 | static inline struct ioat_chan_common * |
bc3c7025 | 59 | ioat_chan_by_index(struct ioatdma_device *device, int index) |
3e037454 SN |
60 | { |
61 | return device->idx[index]; | |
62 | } | |
63 | ||
64 | /** | |
65 | * ioat_dma_do_interrupt - handler used for single vector interrupt mode | |
66 | * @irq: interrupt id | |
67 | * @data: interrupt data | |
68 | */ | |
69 | static irqreturn_t ioat_dma_do_interrupt(int irq, void *data) | |
70 | { | |
71 | struct ioatdma_device *instance = data; | |
dcbc853a | 72 | struct ioat_chan_common *chan; |
3e037454 SN |
73 | unsigned long attnstatus; |
74 | int bit; | |
75 | u8 intrctrl; | |
76 | ||
77 | intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET); | |
78 | ||
79 | if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN)) | |
80 | return IRQ_NONE; | |
81 | ||
82 | if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) { | |
83 | writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET); | |
84 | return IRQ_NONE; | |
85 | } | |
86 | ||
87 | attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET); | |
88 | for_each_bit(bit, &attnstatus, BITS_PER_LONG) { | |
dcbc853a DW |
89 | chan = ioat_chan_by_index(instance, bit); |
90 | tasklet_schedule(&chan->cleanup_task); | |
3e037454 SN |
91 | } |
92 | ||
93 | writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET); | |
94 | return IRQ_HANDLED; | |
95 | } | |
96 | ||
97 | /** | |
98 | * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode | |
99 | * @irq: interrupt id | |
100 | * @data: interrupt data | |
101 | */ | |
102 | static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data) | |
103 | { | |
dcbc853a | 104 | struct ioat_chan_common *chan = data; |
3e037454 | 105 | |
dcbc853a | 106 | tasklet_schedule(&chan->cleanup_task); |
3e037454 SN |
107 | |
108 | return IRQ_HANDLED; | |
109 | } | |
110 | ||
111 | static void ioat_dma_cleanup_tasklet(unsigned long data); | |
112 | ||
113 | /** | |
114 | * ioat_dma_enumerate_channels - find and initialize the device's channels | |
115 | * @device: the device to be enumerated | |
116 | */ | |
8ab89567 | 117 | static int ioat_dma_enumerate_channels(struct ioatdma_device *device) |
0bbd5f4e CL |
118 | { |
119 | u8 xfercap_scale; | |
120 | u32 xfercap; | |
121 | int i; | |
dcbc853a DW |
122 | struct ioat_chan_common *chan; |
123 | struct ioat_dma_chan *ioat; | |
e6c0b69a | 124 | struct device *dev = &device->pdev->dev; |
f2427e27 | 125 | struct dma_device *dma = &device->common; |
0bbd5f4e | 126 | |
f2427e27 DW |
127 | INIT_LIST_HEAD(&dma->channels); |
128 | dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET); | |
e3828811 | 129 | xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET); |
0bbd5f4e CL |
130 | xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale)); |
131 | ||
f371be63 | 132 | #ifdef CONFIG_I7300_IDLE_IOAT_CHANNEL |
f2427e27 DW |
133 | if (i7300_idle_platform_probe(NULL, NULL, 1) == 0) |
134 | dma->chancnt--; | |
27471fdb | 135 | #endif |
f2427e27 | 136 | for (i = 0; i < dma->chancnt; i++) { |
dcbc853a DW |
137 | ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL); |
138 | if (!ioat) { | |
f2427e27 | 139 | dma->chancnt = i; |
0bbd5f4e CL |
140 | break; |
141 | } | |
142 | ||
dcbc853a DW |
143 | chan = &ioat->base; |
144 | chan->device = device; | |
145 | chan->reg_base = device->reg_base + (0x80 * (i + 1)); | |
146 | ioat->xfercap = xfercap; | |
147 | ioat->desccount = 0; | |
148 | INIT_DELAYED_WORK(&chan->work, ioat_dma_chan_reset_part2); | |
149 | spin_lock_init(&chan->cleanup_lock); | |
150 | spin_lock_init(&ioat->desc_lock); | |
151 | INIT_LIST_HEAD(&ioat->free_desc); | |
152 | INIT_LIST_HEAD(&ioat->used_desc); | |
0bbd5f4e | 153 | /* This should be made common somewhere in dmaengine.c */ |
dcbc853a DW |
154 | chan->common.device = &device->common; |
155 | list_add_tail(&chan->common.device_node, &dma->channels); | |
156 | device->idx[i] = chan; | |
157 | tasklet_init(&chan->cleanup_task, | |
3e037454 | 158 | ioat_dma_cleanup_tasklet, |
dcbc853a DW |
159 | (unsigned long) ioat); |
160 | tasklet_disable(&chan->cleanup_task); | |
0bbd5f4e | 161 | } |
f2427e27 | 162 | return dma->chancnt; |
0bbd5f4e CL |
163 | } |
164 | ||
711924b1 SN |
165 | /** |
166 | * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended | |
167 | * descriptors to hw | |
168 | * @chan: DMA channel handle | |
169 | */ | |
bc3c7025 | 170 | static inline void |
dcbc853a | 171 | __ioat1_dma_memcpy_issue_pending(struct ioat_dma_chan *ioat) |
711924b1 | 172 | { |
dcbc853a DW |
173 | void __iomem *reg_base = ioat->base.reg_base; |
174 | ||
175 | ioat->pending = 0; | |
176 | writeb(IOAT_CHANCMD_APPEND, reg_base + IOAT1_CHANCMD_OFFSET); | |
711924b1 SN |
177 | } |
178 | ||
179 | static void ioat1_dma_memcpy_issue_pending(struct dma_chan *chan) | |
180 | { | |
dcbc853a | 181 | struct ioat_dma_chan *ioat = to_ioat_chan(chan); |
711924b1 | 182 | |
dcbc853a DW |
183 | if (ioat->pending > 0) { |
184 | spin_lock_bh(&ioat->desc_lock); | |
185 | __ioat1_dma_memcpy_issue_pending(ioat); | |
186 | spin_unlock_bh(&ioat->desc_lock); | |
711924b1 SN |
187 | } |
188 | } | |
189 | ||
bc3c7025 | 190 | static inline void |
dcbc853a | 191 | __ioat2_dma_memcpy_issue_pending(struct ioat_dma_chan *ioat) |
711924b1 | 192 | { |
dcbc853a DW |
193 | void __iomem *reg_base = ioat->base.reg_base; |
194 | ||
195 | ioat->pending = 0; | |
196 | writew(ioat->dmacount, reg_base + IOAT_CHAN_DMACOUNT_OFFSET); | |
711924b1 SN |
197 | } |
198 | ||
199 | static void ioat2_dma_memcpy_issue_pending(struct dma_chan *chan) | |
200 | { | |
dcbc853a | 201 | struct ioat_dma_chan *ioat = to_ioat_chan(chan); |
711924b1 | 202 | |
dcbc853a DW |
203 | if (ioat->pending > 0) { |
204 | spin_lock_bh(&ioat->desc_lock); | |
205 | __ioat2_dma_memcpy_issue_pending(ioat); | |
206 | spin_unlock_bh(&ioat->desc_lock); | |
711924b1 SN |
207 | } |
208 | } | |
7bb67c14 | 209 | |
09177e85 MS |
210 | |
211 | /** | |
212 | * ioat_dma_chan_reset_part2 - reinit the channel after a reset | |
213 | */ | |
214 | static void ioat_dma_chan_reset_part2(struct work_struct *work) | |
215 | { | |
dcbc853a DW |
216 | struct ioat_chan_common *chan; |
217 | struct ioat_dma_chan *ioat; | |
09177e85 MS |
218 | struct ioat_desc_sw *desc; |
219 | ||
dcbc853a DW |
220 | chan = container_of(work, struct ioat_chan_common, work.work); |
221 | ioat = container_of(chan, struct ioat_dma_chan, base); | |
222 | spin_lock_bh(&chan->cleanup_lock); | |
223 | spin_lock_bh(&ioat->desc_lock); | |
09177e85 | 224 | |
dcbc853a DW |
225 | chan->completion_virt->low = 0; |
226 | chan->completion_virt->high = 0; | |
227 | ioat->pending = 0; | |
09177e85 MS |
228 | |
229 | /* | |
230 | * count the descriptors waiting, and be sure to do it | |
231 | * right for both the CB1 line and the CB2 ring | |
232 | */ | |
dcbc853a DW |
233 | ioat->dmacount = 0; |
234 | if (ioat->used_desc.prev) { | |
235 | desc = to_ioat_desc(ioat->used_desc.prev); | |
09177e85 | 236 | do { |
dcbc853a | 237 | ioat->dmacount++; |
09177e85 | 238 | desc = to_ioat_desc(desc->node.next); |
dcbc853a | 239 | } while (&desc->node != ioat->used_desc.next); |
09177e85 MS |
240 | } |
241 | ||
242 | /* | |
243 | * write the new starting descriptor address | |
244 | * this puts channel engine into ARMED state | |
245 | */ | |
dcbc853a DW |
246 | desc = to_ioat_desc(ioat->used_desc.prev); |
247 | switch (chan->device->version) { | |
09177e85 | 248 | case IOAT_VER_1_2: |
bc3c7025 | 249 | writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF, |
dcbc853a | 250 | chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW); |
bc3c7025 | 251 | writel(((u64) desc->txd.phys) >> 32, |
dcbc853a | 252 | chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH); |
09177e85 | 253 | |
dcbc853a DW |
254 | writeb(IOAT_CHANCMD_START, chan->reg_base |
255 | + IOAT_CHANCMD_OFFSET(chan->device->version)); | |
09177e85 MS |
256 | break; |
257 | case IOAT_VER_2_0: | |
bc3c7025 | 258 | writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF, |
dcbc853a | 259 | chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW); |
bc3c7025 | 260 | writel(((u64) desc->txd.phys) >> 32, |
dcbc853a | 261 | chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH); |
09177e85 MS |
262 | |
263 | /* tell the engine to go with what's left to be done */ | |
dcbc853a DW |
264 | writew(ioat->dmacount, |
265 | chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET); | |
09177e85 MS |
266 | |
267 | break; | |
268 | } | |
dcbc853a | 269 | dev_err(to_dev(chan), |
09177e85 | 270 | "chan%d reset - %d descs waiting, %d total desc\n", |
dcbc853a | 271 | chan_num(chan), ioat->dmacount, ioat->desccount); |
09177e85 | 272 | |
dcbc853a DW |
273 | spin_unlock_bh(&ioat->desc_lock); |
274 | spin_unlock_bh(&chan->cleanup_lock); | |
09177e85 MS |
275 | } |
276 | ||
277 | /** | |
278 | * ioat_dma_reset_channel - restart a channel | |
dcbc853a | 279 | * @ioat: IOAT DMA channel handle |
09177e85 | 280 | */ |
dcbc853a | 281 | static void ioat_dma_reset_channel(struct ioat_dma_chan *ioat) |
09177e85 | 282 | { |
dcbc853a DW |
283 | struct ioat_chan_common *chan = &ioat->base; |
284 | void __iomem *reg_base = chan->reg_base; | |
09177e85 MS |
285 | u32 chansts, chanerr; |
286 | ||
dcbc853a | 287 | if (!ioat->used_desc.prev) |
09177e85 MS |
288 | return; |
289 | ||
dcbc853a DW |
290 | chanerr = readl(reg_base + IOAT_CHANERR_OFFSET); |
291 | chansts = (chan->completion_virt->low | |
09177e85 MS |
292 | & IOAT_CHANSTS_DMA_TRANSFER_STATUS); |
293 | if (chanerr) { | |
dcbc853a | 294 | dev_err(to_dev(chan), |
09177e85 | 295 | "chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n", |
dcbc853a DW |
296 | chan_num(chan), chansts, chanerr); |
297 | writel(chanerr, reg_base + IOAT_CHANERR_OFFSET); | |
09177e85 MS |
298 | } |
299 | ||
300 | /* | |
301 | * whack it upside the head with a reset | |
302 | * and wait for things to settle out. | |
303 | * force the pending count to a really big negative | |
304 | * to make sure no one forces an issue_pending | |
305 | * while we're waiting. | |
306 | */ | |
307 | ||
dcbc853a DW |
308 | spin_lock_bh(&ioat->desc_lock); |
309 | ioat->pending = INT_MIN; | |
09177e85 | 310 | writeb(IOAT_CHANCMD_RESET, |
dcbc853a DW |
311 | reg_base + IOAT_CHANCMD_OFFSET(chan->device->version)); |
312 | spin_unlock_bh(&ioat->desc_lock); | |
09177e85 MS |
313 | |
314 | /* schedule the 2nd half instead of sleeping a long time */ | |
dcbc853a | 315 | schedule_delayed_work(&chan->work, RESET_DELAY); |
09177e85 MS |
316 | } |
317 | ||
318 | /** | |
319 | * ioat_dma_chan_watchdog - watch for stuck channels | |
320 | */ | |
321 | static void ioat_dma_chan_watchdog(struct work_struct *work) | |
322 | { | |
323 | struct ioatdma_device *device = | |
324 | container_of(work, struct ioatdma_device, work.work); | |
dcbc853a DW |
325 | struct ioat_dma_chan *ioat; |
326 | struct ioat_chan_common *chan; | |
09177e85 MS |
327 | int i; |
328 | ||
329 | union { | |
330 | u64 full; | |
331 | struct { | |
332 | u32 low; | |
333 | u32 high; | |
334 | }; | |
335 | } completion_hw; | |
336 | unsigned long compl_desc_addr_hw; | |
337 | ||
338 | for (i = 0; i < device->common.chancnt; i++) { | |
dcbc853a DW |
339 | chan = ioat_chan_by_index(device, i); |
340 | ioat = container_of(chan, struct ioat_dma_chan, base); | |
09177e85 | 341 | |
dcbc853a | 342 | if (chan->device->version == IOAT_VER_1_2 |
09177e85 | 343 | /* have we started processing anything yet */ |
dcbc853a | 344 | && chan->last_completion |
09177e85 | 345 | /* have we completed any since last watchdog cycle? */ |
dcbc853a | 346 | && (chan->last_completion == chan->watchdog_completion) |
09177e85 | 347 | /* has TCP stuck on one cookie since last watchdog? */ |
dcbc853a DW |
348 | && (chan->watchdog_tcp_cookie == chan->watchdog_last_tcp_cookie) |
349 | && (chan->watchdog_tcp_cookie != chan->completed_cookie) | |
09177e85 MS |
350 | /* is there something in the chain to be processed? */ |
351 | /* CB1 chain always has at least the last one processed */ | |
dcbc853a DW |
352 | && (ioat->used_desc.prev != ioat->used_desc.next) |
353 | && ioat->pending == 0) { | |
09177e85 MS |
354 | |
355 | /* | |
356 | * check CHANSTS register for completed | |
357 | * descriptor address. | |
358 | * if it is different than completion writeback, | |
359 | * it is not zero | |
360 | * and it has changed since the last watchdog | |
361 | * we can assume that channel | |
362 | * is still working correctly | |
363 | * and the problem is in completion writeback. | |
364 | * update completion writeback | |
365 | * with actual CHANSTS value | |
366 | * else | |
367 | * try resetting the channel | |
368 | */ | |
369 | ||
dcbc853a DW |
370 | completion_hw.low = readl(chan->reg_base + |
371 | IOAT_CHANSTS_OFFSET_LOW(chan->device->version)); | |
372 | completion_hw.high = readl(chan->reg_base + | |
373 | IOAT_CHANSTS_OFFSET_HIGH(chan->device->version)); | |
09177e85 MS |
374 | #if (BITS_PER_LONG == 64) |
375 | compl_desc_addr_hw = | |
376 | completion_hw.full | |
377 | & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR; | |
378 | #else | |
379 | compl_desc_addr_hw = | |
380 | completion_hw.low & IOAT_LOW_COMPLETION_MASK; | |
381 | #endif | |
382 | ||
383 | if ((compl_desc_addr_hw != 0) | |
dcbc853a DW |
384 | && (compl_desc_addr_hw != chan->watchdog_completion) |
385 | && (compl_desc_addr_hw != chan->last_compl_desc_addr_hw)) { | |
386 | chan->last_compl_desc_addr_hw = compl_desc_addr_hw; | |
387 | chan->completion_virt->low = completion_hw.low; | |
388 | chan->completion_virt->high = completion_hw.high; | |
09177e85 | 389 | } else { |
dcbc853a DW |
390 | ioat_dma_reset_channel(ioat); |
391 | chan->watchdog_completion = 0; | |
392 | chan->last_compl_desc_addr_hw = 0; | |
09177e85 MS |
393 | } |
394 | ||
395 | /* | |
396 | * for version 2.0 if there are descriptors yet to be processed | |
397 | * and the last completed hasn't changed since the last watchdog | |
398 | * if they haven't hit the pending level | |
399 | * issue the pending to push them through | |
400 | * else | |
401 | * try resetting the channel | |
402 | */ | |
dcbc853a DW |
403 | } else if (chan->device->version == IOAT_VER_2_0 |
404 | && ioat->used_desc.prev | |
405 | && chan->last_completion | |
406 | && chan->last_completion == chan->watchdog_completion) { | |
09177e85 | 407 | |
dcbc853a DW |
408 | if (ioat->pending < ioat_pending_level) |
409 | ioat2_dma_memcpy_issue_pending(&chan->common); | |
09177e85 | 410 | else { |
dcbc853a DW |
411 | ioat_dma_reset_channel(ioat); |
412 | chan->watchdog_completion = 0; | |
09177e85 MS |
413 | } |
414 | } else { | |
dcbc853a DW |
415 | chan->last_compl_desc_addr_hw = 0; |
416 | chan->watchdog_completion = chan->last_completion; | |
09177e85 | 417 | } |
dcbc853a | 418 | chan->watchdog_last_tcp_cookie = chan->watchdog_tcp_cookie; |
09177e85 MS |
419 | } |
420 | ||
421 | schedule_delayed_work(&device->work, WATCHDOG_DELAY); | |
422 | } | |
423 | ||
7bb67c14 | 424 | static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx) |
7405f74b | 425 | { |
dcbc853a DW |
426 | struct dma_chan *c = tx->chan; |
427 | struct ioat_dma_chan *ioat = to_ioat_chan(c); | |
a0587bcf DW |
428 | struct ioat_desc_sw *desc = tx_to_ioat_desc(tx); |
429 | struct ioat_desc_sw *first; | |
430 | struct ioat_desc_sw *chain_tail; | |
7405f74b | 431 | dma_cookie_t cookie; |
7405f74b | 432 | |
dcbc853a | 433 | spin_lock_bh(&ioat->desc_lock); |
7405f74b | 434 | /* cookie incr and addition to used_list must be atomic */ |
dcbc853a | 435 | cookie = c->cookie; |
7405f74b DW |
436 | cookie++; |
437 | if (cookie < 0) | |
438 | cookie = 1; | |
dcbc853a DW |
439 | c->cookie = cookie; |
440 | tx->cookie = cookie; | |
7405f74b DW |
441 | |
442 | /* write address into NextDescriptor field of last desc in chain */ | |
a0587bcf | 443 | first = to_ioat_desc(tx->tx_list.next); |
dcbc853a | 444 | chain_tail = to_ioat_desc(ioat->used_desc.prev); |
a0587bcf DW |
445 | /* make descriptor updates globally visible before chaining */ |
446 | wmb(); | |
447 | chain_tail->hw->next = first->txd.phys; | |
dcbc853a | 448 | list_splice_tail_init(&tx->tx_list, &ioat->used_desc); |
a0587bcf | 449 | |
dcbc853a DW |
450 | ioat->dmacount += desc->tx_cnt; |
451 | ioat->pending += desc->tx_cnt; | |
452 | if (ioat->pending >= ioat_pending_level) | |
453 | __ioat1_dma_memcpy_issue_pending(ioat); | |
454 | spin_unlock_bh(&ioat->desc_lock); | |
7405f74b | 455 | |
7bb67c14 SN |
456 | return cookie; |
457 | } | |
458 | ||
459 | static dma_cookie_t ioat2_tx_submit(struct dma_async_tx_descriptor *tx) | |
460 | { | |
dcbc853a | 461 | struct ioat_dma_chan *ioat = to_ioat_chan(tx->chan); |
7bb67c14 SN |
462 | struct ioat_desc_sw *first = tx_to_ioat_desc(tx); |
463 | struct ioat_desc_sw *new; | |
464 | struct ioat_dma_descriptor *hw; | |
465 | dma_cookie_t cookie; | |
466 | u32 copy; | |
467 | size_t len; | |
468 | dma_addr_t src, dst; | |
636bdeaa | 469 | unsigned long orig_flags; |
7bb67c14 SN |
470 | unsigned int desc_count = 0; |
471 | ||
472 | /* src and dest and len are stored in the initial descriptor */ | |
473 | len = first->len; | |
474 | src = first->src; | |
475 | dst = first->dst; | |
bc3c7025 | 476 | orig_flags = first->txd.flags; |
7bb67c14 SN |
477 | new = first; |
478 | ||
711924b1 | 479 | /* |
dcbc853a | 480 | * ioat->desc_lock is still in force in version 2 path |
711924b1 SN |
481 | * it gets unlocked at end of this function |
482 | */ | |
7bb67c14 | 483 | do { |
dcbc853a | 484 | copy = min_t(size_t, len, ioat->xfercap); |
7bb67c14 | 485 | |
bc3c7025 | 486 | async_tx_ack(&new->txd); |
7bb67c14 SN |
487 | |
488 | hw = new->hw; | |
489 | hw->size = copy; | |
490 | hw->ctl = 0; | |
491 | hw->src_addr = src; | |
492 | hw->dst_addr = dst; | |
493 | ||
494 | len -= copy; | |
495 | dst += copy; | |
496 | src += copy; | |
497 | desc_count++; | |
dcbc853a | 498 | } while (len && (new = ioat2_dma_get_next_descriptor(ioat))); |
7bb67c14 | 499 | |
7f1b358a | 500 | if (!new) { |
dcbc853a DW |
501 | dev_err(to_dev(&ioat->base), "tx submit failed\n"); |
502 | spin_unlock_bh(&ioat->desc_lock); | |
7f1b358a MS |
503 | return -ENOMEM; |
504 | } | |
505 | ||
c7984f4e | 506 | hw->ctl_f.compl_write = 1; |
bc3c7025 | 507 | if (first->txd.callback) { |
c7984f4e | 508 | hw->ctl_f.int_en = 1; |
7bb67c14 SN |
509 | if (first != new) { |
510 | /* move callback into to last desc */ | |
bc3c7025 DW |
511 | new->txd.callback = first->txd.callback; |
512 | new->txd.callback_param | |
513 | = first->txd.callback_param; | |
514 | first->txd.callback = NULL; | |
515 | first->txd.callback_param = NULL; | |
7bb67c14 SN |
516 | } |
517 | } | |
518 | ||
519 | new->tx_cnt = desc_count; | |
bc3c7025 | 520 | new->txd.flags = orig_flags; /* client is in control of this ack */ |
7bb67c14 SN |
521 | |
522 | /* store the original values for use in later cleanup */ | |
523 | if (new != first) { | |
524 | new->src = first->src; | |
525 | new->dst = first->dst; | |
526 | new->len = first->len; | |
527 | } | |
528 | ||
529 | /* cookie incr and addition to used_list must be atomic */ | |
dcbc853a | 530 | cookie = ioat->base.common.cookie; |
7bb67c14 SN |
531 | cookie++; |
532 | if (cookie < 0) | |
533 | cookie = 1; | |
dcbc853a | 534 | ioat->base.common.cookie = new->txd.cookie = cookie; |
7bb67c14 | 535 | |
dcbc853a DW |
536 | ioat->dmacount += desc_count; |
537 | ioat->pending += desc_count; | |
538 | if (ioat->pending >= ioat_pending_level) | |
539 | __ioat2_dma_memcpy_issue_pending(ioat); | |
540 | spin_unlock_bh(&ioat->desc_lock); | |
1fda5f4e | 541 | |
7405f74b DW |
542 | return cookie; |
543 | } | |
544 | ||
7bb67c14 SN |
545 | /** |
546 | * ioat_dma_alloc_descriptor - allocate and return a sw and hw descriptor pair | |
dcbc853a | 547 | * @ioat: the channel supplying the memory pool for the descriptors |
7bb67c14 SN |
548 | * @flags: allocation flags |
549 | */ | |
bc3c7025 | 550 | static struct ioat_desc_sw * |
dcbc853a | 551 | ioat_dma_alloc_descriptor(struct ioat_dma_chan *ioat, gfp_t flags) |
0bbd5f4e CL |
552 | { |
553 | struct ioat_dma_descriptor *desc; | |
554 | struct ioat_desc_sw *desc_sw; | |
8ab89567 | 555 | struct ioatdma_device *ioatdma_device; |
0bbd5f4e CL |
556 | dma_addr_t phys; |
557 | ||
dcbc853a | 558 | ioatdma_device = ioat->base.device; |
8ab89567 | 559 | desc = pci_pool_alloc(ioatdma_device->dma_pool, flags, &phys); |
0bbd5f4e CL |
560 | if (unlikely(!desc)) |
561 | return NULL; | |
562 | ||
563 | desc_sw = kzalloc(sizeof(*desc_sw), flags); | |
564 | if (unlikely(!desc_sw)) { | |
8ab89567 | 565 | pci_pool_free(ioatdma_device->dma_pool, desc, phys); |
0bbd5f4e CL |
566 | return NULL; |
567 | } | |
568 | ||
569 | memset(desc, 0, sizeof(*desc)); | |
dcbc853a DW |
570 | dma_async_tx_descriptor_init(&desc_sw->txd, &ioat->base.common); |
571 | switch (ioatdma_device->version) { | |
7bb67c14 | 572 | case IOAT_VER_1_2: |
bc3c7025 | 573 | desc_sw->txd.tx_submit = ioat1_tx_submit; |
7bb67c14 SN |
574 | break; |
575 | case IOAT_VER_2_0: | |
7f1b358a | 576 | case IOAT_VER_3_0: |
bc3c7025 | 577 | desc_sw->txd.tx_submit = ioat2_tx_submit; |
7bb67c14 SN |
578 | break; |
579 | } | |
7bb67c14 | 580 | |
0bbd5f4e | 581 | desc_sw->hw = desc; |
bc3c7025 | 582 | desc_sw->txd.phys = phys; |
0bbd5f4e CL |
583 | |
584 | return desc_sw; | |
585 | } | |
586 | ||
7bb67c14 SN |
587 | static int ioat_initial_desc_count = 256; |
588 | module_param(ioat_initial_desc_count, int, 0644); | |
589 | MODULE_PARM_DESC(ioat_initial_desc_count, | |
590 | "initial descriptors per channel (default: 256)"); | |
591 | ||
592 | /** | |
593 | * ioat2_dma_massage_chan_desc - link the descriptors into a circle | |
dcbc853a | 594 | * @ioat: the channel to be massaged |
7bb67c14 | 595 | */ |
dcbc853a | 596 | static void ioat2_dma_massage_chan_desc(struct ioat_dma_chan *ioat) |
7bb67c14 SN |
597 | { |
598 | struct ioat_desc_sw *desc, *_desc; | |
599 | ||
600 | /* setup used_desc */ | |
dcbc853a DW |
601 | ioat->used_desc.next = ioat->free_desc.next; |
602 | ioat->used_desc.prev = NULL; | |
7bb67c14 SN |
603 | |
604 | /* pull free_desc out of the circle so that every node is a hw | |
605 | * descriptor, but leave it pointing to the list | |
606 | */ | |
dcbc853a DW |
607 | ioat->free_desc.prev->next = ioat->free_desc.next; |
608 | ioat->free_desc.next->prev = ioat->free_desc.prev; | |
7bb67c14 SN |
609 | |
610 | /* circle link the hw descriptors */ | |
dcbc853a | 611 | desc = to_ioat_desc(ioat->free_desc.next); |
bc3c7025 | 612 | desc->hw->next = to_ioat_desc(desc->node.next)->txd.phys; |
dcbc853a | 613 | list_for_each_entry_safe(desc, _desc, ioat->free_desc.next, node) { |
bc3c7025 | 614 | desc->hw->next = to_ioat_desc(desc->node.next)->txd.phys; |
7bb67c14 SN |
615 | } |
616 | } | |
617 | ||
618 | /** | |
619 | * ioat_dma_alloc_chan_resources - returns the number of allocated descriptors | |
620 | * @chan: the channel to be filled out | |
621 | */ | |
dcbc853a | 622 | static int ioat_dma_alloc_chan_resources(struct dma_chan *c) |
0bbd5f4e | 623 | { |
dcbc853a DW |
624 | struct ioat_dma_chan *ioat = to_ioat_chan(c); |
625 | struct ioat_chan_common *chan = &ioat->base; | |
711924b1 | 626 | struct ioat_desc_sw *desc; |
0bbd5f4e CL |
627 | u16 chanctrl; |
628 | u32 chanerr; | |
629 | int i; | |
630 | LIST_HEAD(tmp_list); | |
631 | ||
e4223976 | 632 | /* have we already been set up? */ |
dcbc853a DW |
633 | if (!list_empty(&ioat->free_desc)) |
634 | return ioat->desccount; | |
0bbd5f4e | 635 | |
43d6e369 | 636 | /* Setup register to interrupt and write completion status on error */ |
e4223976 | 637 | chanctrl = IOAT_CHANCTRL_ERR_INT_EN | |
0bbd5f4e CL |
638 | IOAT_CHANCTRL_ANY_ERR_ABORT_EN | |
639 | IOAT_CHANCTRL_ERR_COMPLETION_EN; | |
dcbc853a | 640 | writew(chanctrl, chan->reg_base + IOAT_CHANCTRL_OFFSET); |
0bbd5f4e | 641 | |
dcbc853a | 642 | chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); |
0bbd5f4e | 643 | if (chanerr) { |
dcbc853a DW |
644 | dev_err(to_dev(chan), "CHANERR = %x, clearing\n", chanerr); |
645 | writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET); | |
0bbd5f4e CL |
646 | } |
647 | ||
648 | /* Allocate descriptors */ | |
7bb67c14 | 649 | for (i = 0; i < ioat_initial_desc_count; i++) { |
dcbc853a | 650 | desc = ioat_dma_alloc_descriptor(ioat, GFP_KERNEL); |
0bbd5f4e | 651 | if (!desc) { |
dcbc853a | 652 | dev_err(to_dev(chan), "Only %d initial descriptors\n", i); |
0bbd5f4e CL |
653 | break; |
654 | } | |
655 | list_add_tail(&desc->node, &tmp_list); | |
656 | } | |
dcbc853a DW |
657 | spin_lock_bh(&ioat->desc_lock); |
658 | ioat->desccount = i; | |
659 | list_splice(&tmp_list, &ioat->free_desc); | |
660 | if (chan->device->version != IOAT_VER_1_2) | |
661 | ioat2_dma_massage_chan_desc(ioat); | |
662 | spin_unlock_bh(&ioat->desc_lock); | |
0bbd5f4e CL |
663 | |
664 | /* allocate a completion writeback area */ | |
665 | /* doing 2 32bit writes to mmio since 1 64b write doesn't work */ | |
dcbc853a DW |
666 | chan->completion_virt = pci_pool_alloc(chan->device->completion_pool, |
667 | GFP_KERNEL, | |
668 | &chan->completion_addr); | |
669 | memset(chan->completion_virt, 0, | |
670 | sizeof(*chan->completion_virt)); | |
671 | writel(((u64) chan->completion_addr) & 0x00000000FFFFFFFF, | |
672 | chan->reg_base + IOAT_CHANCMP_OFFSET_LOW); | |
673 | writel(((u64) chan->completion_addr) >> 32, | |
674 | chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH); | |
675 | ||
676 | tasklet_enable(&chan->cleanup_task); | |
677 | ioat_dma_start_null_desc(ioat); /* give chain to dma device */ | |
678 | return ioat->desccount; | |
0bbd5f4e CL |
679 | } |
680 | ||
7bb67c14 SN |
681 | /** |
682 | * ioat_dma_free_chan_resources - release all the descriptors | |
683 | * @chan: the channel to be cleaned | |
684 | */ | |
dcbc853a | 685 | static void ioat_dma_free_chan_resources(struct dma_chan *c) |
0bbd5f4e | 686 | { |
dcbc853a DW |
687 | struct ioat_dma_chan *ioat = to_ioat_chan(c); |
688 | struct ioat_chan_common *chan = &ioat->base; | |
689 | struct ioatdma_device *ioatdma_device = chan->device; | |
0bbd5f4e | 690 | struct ioat_desc_sw *desc, *_desc; |
0bbd5f4e CL |
691 | int in_use_descs = 0; |
692 | ||
c3d4f44f MS |
693 | /* Before freeing channel resources first check |
694 | * if they have been previously allocated for this channel. | |
695 | */ | |
dcbc853a | 696 | if (ioat->desccount == 0) |
c3d4f44f MS |
697 | return; |
698 | ||
dcbc853a DW |
699 | tasklet_disable(&chan->cleanup_task); |
700 | ioat_dma_memcpy_cleanup(ioat); | |
0bbd5f4e | 701 | |
3e037454 SN |
702 | /* Delay 100ms after reset to allow internal DMA logic to quiesce |
703 | * before removing DMA descriptor resources. | |
704 | */ | |
7bb67c14 | 705 | writeb(IOAT_CHANCMD_RESET, |
dcbc853a | 706 | chan->reg_base + IOAT_CHANCMD_OFFSET(chan->device->version)); |
3e037454 | 707 | mdelay(100); |
0bbd5f4e | 708 | |
dcbc853a DW |
709 | spin_lock_bh(&ioat->desc_lock); |
710 | switch (chan->device->version) { | |
7bb67c14 SN |
711 | case IOAT_VER_1_2: |
712 | list_for_each_entry_safe(desc, _desc, | |
dcbc853a | 713 | &ioat->used_desc, node) { |
7bb67c14 SN |
714 | in_use_descs++; |
715 | list_del(&desc->node); | |
716 | pci_pool_free(ioatdma_device->dma_pool, desc->hw, | |
bc3c7025 | 717 | desc->txd.phys); |
7bb67c14 SN |
718 | kfree(desc); |
719 | } | |
720 | list_for_each_entry_safe(desc, _desc, | |
dcbc853a | 721 | &ioat->free_desc, node) { |
7bb67c14 SN |
722 | list_del(&desc->node); |
723 | pci_pool_free(ioatdma_device->dma_pool, desc->hw, | |
bc3c7025 | 724 | desc->txd.phys); |
7bb67c14 SN |
725 | kfree(desc); |
726 | } | |
727 | break; | |
728 | case IOAT_VER_2_0: | |
7f1b358a | 729 | case IOAT_VER_3_0: |
7bb67c14 | 730 | list_for_each_entry_safe(desc, _desc, |
dcbc853a | 731 | ioat->free_desc.next, node) { |
7bb67c14 SN |
732 | list_del(&desc->node); |
733 | pci_pool_free(ioatdma_device->dma_pool, desc->hw, | |
bc3c7025 | 734 | desc->txd.phys); |
7bb67c14 SN |
735 | kfree(desc); |
736 | } | |
dcbc853a | 737 | desc = to_ioat_desc(ioat->free_desc.next); |
8ab89567 | 738 | pci_pool_free(ioatdma_device->dma_pool, desc->hw, |
bc3c7025 | 739 | desc->txd.phys); |
0bbd5f4e | 740 | kfree(desc); |
dcbc853a DW |
741 | INIT_LIST_HEAD(&ioat->free_desc); |
742 | INIT_LIST_HEAD(&ioat->used_desc); | |
7bb67c14 | 743 | break; |
0bbd5f4e | 744 | } |
dcbc853a | 745 | spin_unlock_bh(&ioat->desc_lock); |
0bbd5f4e | 746 | |
8ab89567 | 747 | pci_pool_free(ioatdma_device->completion_pool, |
dcbc853a DW |
748 | chan->completion_virt, |
749 | chan->completion_addr); | |
0bbd5f4e CL |
750 | |
751 | /* one is ok since we left it on there on purpose */ | |
752 | if (in_use_descs > 1) | |
dcbc853a | 753 | dev_err(to_dev(chan), "Freeing %d in use descriptors!\n", |
0bbd5f4e CL |
754 | in_use_descs - 1); |
755 | ||
dcbc853a DW |
756 | chan->last_completion = chan->completion_addr = 0; |
757 | chan->watchdog_completion = 0; | |
758 | chan->last_compl_desc_addr_hw = 0; | |
759 | chan->watchdog_tcp_cookie = chan->watchdog_last_tcp_cookie = 0; | |
760 | ioat->pending = 0; | |
761 | ioat->dmacount = 0; | |
762 | ioat->desccount = 0; | |
3e037454 | 763 | } |
7f2b291f | 764 | |
3e037454 | 765 | /** |
dcbc853a DW |
766 | * ioat1_dma_get_next_descriptor - return the next available descriptor |
767 | * @ioat: IOAT DMA channel handle | |
3e037454 SN |
768 | * |
769 | * Gets the next descriptor from the chain, and must be called with the | |
770 | * channel's desc_lock held. Allocates more descriptors if the channel | |
771 | * has run out. | |
772 | */ | |
7f2b291f | 773 | static struct ioat_desc_sw * |
dcbc853a | 774 | ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat) |
3e037454 | 775 | { |
711924b1 | 776 | struct ioat_desc_sw *new; |
3e037454 | 777 | |
dcbc853a DW |
778 | if (!list_empty(&ioat->free_desc)) { |
779 | new = to_ioat_desc(ioat->free_desc.next); | |
3e037454 SN |
780 | list_del(&new->node); |
781 | } else { | |
782 | /* try to get another desc */ | |
dcbc853a | 783 | new = ioat_dma_alloc_descriptor(ioat, GFP_ATOMIC); |
711924b1 | 784 | if (!new) { |
dcbc853a | 785 | dev_err(to_dev(&ioat->base), "alloc failed\n"); |
711924b1 SN |
786 | return NULL; |
787 | } | |
3e037454 SN |
788 | } |
789 | ||
790 | prefetch(new->hw); | |
791 | return new; | |
0bbd5f4e CL |
792 | } |
793 | ||
7bb67c14 | 794 | static struct ioat_desc_sw * |
dcbc853a | 795 | ioat2_dma_get_next_descriptor(struct ioat_dma_chan *ioat) |
7bb67c14 | 796 | { |
711924b1 | 797 | struct ioat_desc_sw *new; |
7bb67c14 SN |
798 | |
799 | /* | |
800 | * used.prev points to where to start processing | |
801 | * used.next points to next free descriptor | |
802 | * if used.prev == NULL, there are none waiting to be processed | |
803 | * if used.next == used.prev.prev, there is only one free descriptor, | |
804 | * and we need to use it to as a noop descriptor before | |
805 | * linking in a new set of descriptors, since the device | |
806 | * has probably already read the pointer to it | |
807 | */ | |
dcbc853a DW |
808 | if (ioat->used_desc.prev && |
809 | ioat->used_desc.next == ioat->used_desc.prev->prev) { | |
7bb67c14 | 810 | |
711924b1 SN |
811 | struct ioat_desc_sw *desc; |
812 | struct ioat_desc_sw *noop_desc; | |
7bb67c14 SN |
813 | int i; |
814 | ||
815 | /* set up the noop descriptor */ | |
dcbc853a | 816 | noop_desc = to_ioat_desc(ioat->used_desc.next); |
7f1b358a MS |
817 | /* set size to non-zero value (channel returns error when size is 0) */ |
818 | noop_desc->hw->size = NULL_DESC_BUFFER_SIZE; | |
c7984f4e DW |
819 | noop_desc->hw->ctl = 0; |
820 | noop_desc->hw->ctl_f.null = 1; | |
7bb67c14 SN |
821 | noop_desc->hw->src_addr = 0; |
822 | noop_desc->hw->dst_addr = 0; | |
823 | ||
dcbc853a DW |
824 | ioat->used_desc.next = ioat->used_desc.next->next; |
825 | ioat->pending++; | |
826 | ioat->dmacount++; | |
7bb67c14 | 827 | |
711924b1 | 828 | /* try to get a few more descriptors */ |
7bb67c14 | 829 | for (i = 16; i; i--) { |
dcbc853a | 830 | desc = ioat_dma_alloc_descriptor(ioat, GFP_ATOMIC); |
711924b1 | 831 | if (!desc) { |
dcbc853a DW |
832 | dev_err(to_dev(&ioat->base), |
833 | "alloc failed\n"); | |
711924b1 SN |
834 | break; |
835 | } | |
dcbc853a | 836 | list_add_tail(&desc->node, ioat->used_desc.next); |
7bb67c14 SN |
837 | |
838 | desc->hw->next | |
bc3c7025 | 839 | = to_ioat_desc(desc->node.next)->txd.phys; |
7bb67c14 | 840 | to_ioat_desc(desc->node.prev)->hw->next |
bc3c7025 | 841 | = desc->txd.phys; |
dcbc853a | 842 | ioat->desccount++; |
7bb67c14 SN |
843 | } |
844 | ||
dcbc853a | 845 | ioat->used_desc.next = noop_desc->node.next; |
7bb67c14 | 846 | } |
dcbc853a | 847 | new = to_ioat_desc(ioat->used_desc.next); |
7bb67c14 | 848 | prefetch(new); |
dcbc853a | 849 | ioat->used_desc.next = new->node.next; |
7bb67c14 | 850 | |
dcbc853a DW |
851 | if (ioat->used_desc.prev == NULL) |
852 | ioat->used_desc.prev = &new->node; | |
7bb67c14 SN |
853 | |
854 | prefetch(new->hw); | |
855 | return new; | |
856 | } | |
857 | ||
bc3c7025 | 858 | static struct ioat_desc_sw * |
dcbc853a | 859 | ioat_dma_get_next_descriptor(struct ioat_dma_chan *ioat) |
7bb67c14 | 860 | { |
dcbc853a | 861 | if (!ioat) |
7bb67c14 SN |
862 | return NULL; |
863 | ||
dcbc853a | 864 | switch (ioat->base.device->version) { |
7bb67c14 | 865 | case IOAT_VER_1_2: |
dcbc853a | 866 | return ioat1_dma_get_next_descriptor(ioat); |
7bb67c14 | 867 | case IOAT_VER_2_0: |
7f1b358a | 868 | case IOAT_VER_3_0: |
dcbc853a | 869 | return ioat2_dma_get_next_descriptor(ioat); |
7bb67c14 SN |
870 | } |
871 | return NULL; | |
872 | } | |
873 | ||
bc3c7025 | 874 | static struct dma_async_tx_descriptor * |
dcbc853a | 875 | ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest, |
bc3c7025 | 876 | dma_addr_t dma_src, size_t len, unsigned long flags) |
0bbd5f4e | 877 | { |
dcbc853a | 878 | struct ioat_dma_chan *ioat = to_ioat_chan(c); |
a0587bcf DW |
879 | struct ioat_desc_sw *desc; |
880 | size_t copy; | |
881 | LIST_HEAD(chain); | |
882 | dma_addr_t src = dma_src; | |
883 | dma_addr_t dest = dma_dest; | |
884 | size_t total_len = len; | |
885 | struct ioat_dma_descriptor *hw = NULL; | |
886 | int tx_cnt = 0; | |
0bbd5f4e | 887 | |
dcbc853a DW |
888 | spin_lock_bh(&ioat->desc_lock); |
889 | desc = ioat_dma_get_next_descriptor(ioat); | |
a0587bcf DW |
890 | do { |
891 | if (!desc) | |
892 | break; | |
0bbd5f4e | 893 | |
a0587bcf | 894 | tx_cnt++; |
dcbc853a | 895 | copy = min_t(size_t, len, ioat->xfercap); |
a0587bcf DW |
896 | |
897 | hw = desc->hw; | |
898 | hw->size = copy; | |
899 | hw->ctl = 0; | |
900 | hw->src_addr = src; | |
901 | hw->dst_addr = dest; | |
902 | ||
903 | list_add_tail(&desc->node, &chain); | |
904 | ||
905 | len -= copy; | |
906 | dest += copy; | |
907 | src += copy; | |
908 | if (len) { | |
909 | struct ioat_desc_sw *next; | |
910 | ||
911 | async_tx_ack(&desc->txd); | |
dcbc853a | 912 | next = ioat_dma_get_next_descriptor(ioat); |
a0587bcf DW |
913 | hw->next = next ? next->txd.phys : 0; |
914 | desc = next; | |
915 | } else | |
916 | hw->next = 0; | |
917 | } while (len); | |
918 | ||
919 | if (!desc) { | |
dcbc853a DW |
920 | struct ioat_chan_common *chan = &ioat->base; |
921 | ||
922 | dev_err(to_dev(chan), | |
09177e85 | 923 | "chan%d - get_next_desc failed: %d descs waiting, %d total desc\n", |
dcbc853a DW |
924 | chan_num(chan), ioat->dmacount, ioat->desccount); |
925 | list_splice(&chain, &ioat->free_desc); | |
926 | spin_unlock_bh(&ioat->desc_lock); | |
711924b1 | 927 | return NULL; |
09177e85 | 928 | } |
dcbc853a | 929 | spin_unlock_bh(&ioat->desc_lock); |
a0587bcf DW |
930 | |
931 | desc->txd.flags = flags; | |
932 | desc->tx_cnt = tx_cnt; | |
933 | desc->src = dma_src; | |
934 | desc->dst = dma_dest; | |
935 | desc->len = total_len; | |
936 | list_splice(&chain, &desc->txd.tx_list); | |
937 | hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); | |
938 | hw->ctl_f.compl_write = 1; | |
939 | ||
940 | return &desc->txd; | |
0bbd5f4e CL |
941 | } |
942 | ||
bc3c7025 | 943 | static struct dma_async_tx_descriptor * |
dcbc853a | 944 | ioat2_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest, |
bc3c7025 | 945 | dma_addr_t dma_src, size_t len, unsigned long flags) |
7bb67c14 | 946 | { |
dcbc853a | 947 | struct ioat_dma_chan *ioat = to_ioat_chan(c); |
7bb67c14 SN |
948 | struct ioat_desc_sw *new; |
949 | ||
dcbc853a DW |
950 | spin_lock_bh(&ioat->desc_lock); |
951 | new = ioat2_dma_get_next_descriptor(ioat); | |
7bb67c14 | 952 | |
711924b1 | 953 | /* |
dcbc853a | 954 | * leave ioat->desc_lock set in ioat 2 path |
711924b1 SN |
955 | * it will get unlocked at end of tx_submit |
956 | */ | |
7bb67c14 | 957 | |
711924b1 SN |
958 | if (new) { |
959 | new->len = len; | |
0036731c DW |
960 | new->dst = dma_dest; |
961 | new->src = dma_src; | |
bc3c7025 DW |
962 | new->txd.flags = flags; |
963 | return &new->txd; | |
09177e85 | 964 | } else { |
dcbc853a DW |
965 | struct ioat_chan_common *chan = &ioat->base; |
966 | ||
967 | spin_unlock_bh(&ioat->desc_lock); | |
968 | dev_err(to_dev(chan), | |
09177e85 | 969 | "chan%d - get_next_desc failed: %d descs waiting, %d total desc\n", |
dcbc853a | 970 | chan_num(chan), ioat->dmacount, ioat->desccount); |
711924b1 | 971 | return NULL; |
09177e85 | 972 | } |
0bbd5f4e CL |
973 | } |
974 | ||
3e037454 SN |
975 | static void ioat_dma_cleanup_tasklet(unsigned long data) |
976 | { | |
977 | struct ioat_dma_chan *chan = (void *)data; | |
978 | ioat_dma_memcpy_cleanup(chan); | |
979 | writew(IOAT_CHANCTRL_INT_DISABLE, | |
dcbc853a | 980 | chan->base.reg_base + IOAT_CHANCTRL_OFFSET); |
3e037454 SN |
981 | } |
982 | ||
e1d181ef | 983 | static void |
dcbc853a | 984 | ioat_dma_unmap(struct ioat_chan_common *chan, struct ioat_desc_sw *desc) |
e1d181ef | 985 | { |
bc3c7025 DW |
986 | if (!(desc->txd.flags & DMA_COMPL_SKIP_DEST_UNMAP)) { |
987 | if (desc->txd.flags & DMA_COMPL_DEST_UNMAP_SINGLE) | |
dcbc853a | 988 | pci_unmap_single(chan->device->pdev, |
4f005dbe MS |
989 | pci_unmap_addr(desc, dst), |
990 | pci_unmap_len(desc, len), | |
991 | PCI_DMA_FROMDEVICE); | |
992 | else | |
dcbc853a | 993 | pci_unmap_page(chan->device->pdev, |
4f005dbe MS |
994 | pci_unmap_addr(desc, dst), |
995 | pci_unmap_len(desc, len), | |
996 | PCI_DMA_FROMDEVICE); | |
997 | } | |
998 | ||
bc3c7025 DW |
999 | if (!(desc->txd.flags & DMA_COMPL_SKIP_SRC_UNMAP)) { |
1000 | if (desc->txd.flags & DMA_COMPL_SRC_UNMAP_SINGLE) | |
dcbc853a | 1001 | pci_unmap_single(chan->device->pdev, |
4f005dbe MS |
1002 | pci_unmap_addr(desc, src), |
1003 | pci_unmap_len(desc, len), | |
1004 | PCI_DMA_TODEVICE); | |
1005 | else | |
dcbc853a | 1006 | pci_unmap_page(chan->device->pdev, |
4f005dbe MS |
1007 | pci_unmap_addr(desc, src), |
1008 | pci_unmap_len(desc, len), | |
1009 | PCI_DMA_TODEVICE); | |
1010 | } | |
e1d181ef DW |
1011 | } |
1012 | ||
7bb67c14 SN |
1013 | /** |
1014 | * ioat_dma_memcpy_cleanup - cleanup up finished descriptors | |
1015 | * @chan: ioat channel to be cleaned up | |
1016 | */ | |
dcbc853a | 1017 | static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat) |
0bbd5f4e | 1018 | { |
dcbc853a | 1019 | struct ioat_chan_common *chan = &ioat->base; |
0bbd5f4e CL |
1020 | unsigned long phys_complete; |
1021 | struct ioat_desc_sw *desc, *_desc; | |
1022 | dma_cookie_t cookie = 0; | |
7bb67c14 SN |
1023 | unsigned long desc_phys; |
1024 | struct ioat_desc_sw *latest_desc; | |
bc3c7025 | 1025 | struct dma_async_tx_descriptor *tx; |
0bbd5f4e | 1026 | |
dcbc853a | 1027 | prefetch(chan->completion_virt); |
0bbd5f4e | 1028 | |
dcbc853a | 1029 | if (!spin_trylock_bh(&chan->cleanup_lock)) |
0bbd5f4e CL |
1030 | return; |
1031 | ||
1032 | /* The completion writeback can happen at any time, | |
1033 | so reads by the driver need to be atomic operations | |
1034 | The descriptor physical addresses are limited to 32-bits | |
1035 | when the CPU can only do a 32-bit mov */ | |
1036 | ||
1037 | #if (BITS_PER_LONG == 64) | |
1038 | phys_complete = | |
dcbc853a | 1039 | chan->completion_virt->full |
7f2b291f | 1040 | & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR; |
0bbd5f4e | 1041 | #else |
dcbc853a | 1042 | phys_complete = chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK; |
0bbd5f4e CL |
1043 | #endif |
1044 | ||
dcbc853a | 1045 | if ((chan->completion_virt->full |
7f2b291f | 1046 | & IOAT_CHANSTS_DMA_TRANSFER_STATUS) == |
43d6e369 | 1047 | IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) { |
dcbc853a DW |
1048 | dev_err(to_dev(chan), "Channel halted, chanerr = %x\n", |
1049 | readl(chan->reg_base + IOAT_CHANERR_OFFSET)); | |
0bbd5f4e CL |
1050 | |
1051 | /* TODO do something to salvage the situation */ | |
1052 | } | |
1053 | ||
dcbc853a DW |
1054 | if (phys_complete == chan->last_completion) { |
1055 | spin_unlock_bh(&chan->cleanup_lock); | |
09177e85 MS |
1056 | /* |
1057 | * perhaps we're stuck so hard that the watchdog can't go off? | |
1058 | * try to catch it after 2 seconds | |
1059 | */ | |
dcbc853a | 1060 | if (chan->device->version != IOAT_VER_3_0) { |
7f1b358a | 1061 | if (time_after(jiffies, |
dcbc853a DW |
1062 | chan->last_completion_time + HZ*WATCHDOG_DELAY)) { |
1063 | ioat_dma_chan_watchdog(&(chan->device->work.work)); | |
1064 | chan->last_completion_time = jiffies; | |
7f1b358a | 1065 | } |
09177e85 | 1066 | } |
0bbd5f4e CL |
1067 | return; |
1068 | } | |
dcbc853a | 1069 | chan->last_completion_time = jiffies; |
0bbd5f4e | 1070 | |
3e037454 | 1071 | cookie = 0; |
dcbc853a DW |
1072 | if (!spin_trylock_bh(&ioat->desc_lock)) { |
1073 | spin_unlock_bh(&chan->cleanup_lock); | |
09177e85 MS |
1074 | return; |
1075 | } | |
1076 | ||
dcbc853a | 1077 | switch (chan->device->version) { |
7bb67c14 | 1078 | case IOAT_VER_1_2: |
dcbc853a | 1079 | list_for_each_entry_safe(desc, _desc, &ioat->used_desc, node) { |
bc3c7025 | 1080 | tx = &desc->txd; |
43d6e369 | 1081 | /* |
7bb67c14 SN |
1082 | * Incoming DMA requests may use multiple descriptors, |
1083 | * due to exceeding xfercap, perhaps. If so, only the | |
1084 | * last one will have a cookie, and require unmapping. | |
43d6e369 | 1085 | */ |
bc3c7025 DW |
1086 | if (tx->cookie) { |
1087 | cookie = tx->cookie; | |
dcbc853a | 1088 | ioat_dma_unmap(chan, desc); |
bc3c7025 DW |
1089 | if (tx->callback) { |
1090 | tx->callback(tx->callback_param); | |
1091 | tx->callback = NULL; | |
7bb67c14 | 1092 | } |
95218430 | 1093 | } |
0bbd5f4e | 1094 | |
bc3c7025 | 1095 | if (tx->phys != phys_complete) { |
7bb67c14 SN |
1096 | /* |
1097 | * a completed entry, but not the last, so clean | |
1098 | * up if the client is done with the descriptor | |
1099 | */ | |
bc3c7025 | 1100 | if (async_tx_test_ack(tx)) { |
aa2d0b8b | 1101 | list_move_tail(&desc->node, |
dcbc853a | 1102 | &ioat->free_desc); |
7bb67c14 | 1103 | } else |
bc3c7025 | 1104 | tx->cookie = 0; |
7bb67c14 SN |
1105 | } else { |
1106 | /* | |
1107 | * last used desc. Do not remove, so we can | |
1108 | * append from it, but don't look at it next | |
1109 | * time, either | |
1110 | */ | |
bc3c7025 | 1111 | tx->cookie = 0; |
0bbd5f4e | 1112 | |
7bb67c14 SN |
1113 | /* TODO check status bits? */ |
1114 | break; | |
1115 | } | |
1116 | } | |
1117 | break; | |
1118 | case IOAT_VER_2_0: | |
7f1b358a | 1119 | case IOAT_VER_3_0: |
7bb67c14 | 1120 | /* has some other thread has already cleaned up? */ |
dcbc853a | 1121 | if (ioat->used_desc.prev == NULL) |
0bbd5f4e | 1122 | break; |
7bb67c14 SN |
1123 | |
1124 | /* work backwards to find latest finished desc */ | |
dcbc853a | 1125 | desc = to_ioat_desc(ioat->used_desc.next); |
bc3c7025 | 1126 | tx = &desc->txd; |
7bb67c14 SN |
1127 | latest_desc = NULL; |
1128 | do { | |
1129 | desc = to_ioat_desc(desc->node.prev); | |
bc3c7025 | 1130 | desc_phys = (unsigned long)tx->phys |
7bb67c14 SN |
1131 | & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR; |
1132 | if (desc_phys == phys_complete) { | |
1133 | latest_desc = desc; | |
1134 | break; | |
1135 | } | |
dcbc853a | 1136 | } while (&desc->node != ioat->used_desc.prev); |
7bb67c14 SN |
1137 | |
1138 | if (latest_desc != NULL) { | |
7bb67c14 | 1139 | /* work forwards to clear finished descriptors */ |
dcbc853a | 1140 | for (desc = to_ioat_desc(ioat->used_desc.prev); |
7bb67c14 | 1141 | &desc->node != latest_desc->node.next && |
dcbc853a | 1142 | &desc->node != ioat->used_desc.next; |
7bb67c14 | 1143 | desc = to_ioat_desc(desc->node.next)) { |
bc3c7025 DW |
1144 | if (tx->cookie) { |
1145 | cookie = tx->cookie; | |
1146 | tx->cookie = 0; | |
dcbc853a | 1147 | ioat_dma_unmap(chan, desc); |
bc3c7025 DW |
1148 | if (tx->callback) { |
1149 | tx->callback(tx->callback_param); | |
1150 | tx->callback = NULL; | |
7bb67c14 SN |
1151 | } |
1152 | } | |
1153 | } | |
1154 | ||
1155 | /* move used.prev up beyond those that are finished */ | |
dcbc853a DW |
1156 | if (&desc->node == ioat->used_desc.next) |
1157 | ioat->used_desc.prev = NULL; | |
7bb67c14 | 1158 | else |
dcbc853a | 1159 | ioat->used_desc.prev = &desc->node; |
0bbd5f4e | 1160 | } |
7bb67c14 | 1161 | break; |
0bbd5f4e CL |
1162 | } |
1163 | ||
dcbc853a | 1164 | spin_unlock_bh(&ioat->desc_lock); |
0bbd5f4e | 1165 | |
dcbc853a | 1166 | chan->last_completion = phys_complete; |
0bbd5f4e | 1167 | if (cookie != 0) |
dcbc853a | 1168 | chan->completed_cookie = cookie; |
0bbd5f4e | 1169 | |
dcbc853a | 1170 | spin_unlock_bh(&chan->cleanup_lock); |
0bbd5f4e CL |
1171 | } |
1172 | ||
1173 | /** | |
1174 | * ioat_dma_is_complete - poll the status of a IOAT DMA transaction | |
1175 | * @chan: IOAT DMA channel handle | |
1176 | * @cookie: DMA transaction identifier | |
6508871e RD |
1177 | * @done: if not %NULL, updated with last completed transaction |
1178 | * @used: if not %NULL, updated with last used transaction | |
0bbd5f4e | 1179 | */ |
bc3c7025 | 1180 | static enum dma_status |
dcbc853a | 1181 | ioat_dma_is_complete(struct dma_chan *c, dma_cookie_t cookie, |
bc3c7025 | 1182 | dma_cookie_t *done, dma_cookie_t *used) |
0bbd5f4e | 1183 | { |
dcbc853a DW |
1184 | struct ioat_dma_chan *ioat = to_ioat_chan(c); |
1185 | struct ioat_chan_common *chan = &ioat->base; | |
0bbd5f4e CL |
1186 | dma_cookie_t last_used; |
1187 | dma_cookie_t last_complete; | |
1188 | enum dma_status ret; | |
1189 | ||
dcbc853a DW |
1190 | last_used = c->cookie; |
1191 | last_complete = chan->completed_cookie; | |
1192 | chan->watchdog_tcp_cookie = cookie; | |
0bbd5f4e CL |
1193 | |
1194 | if (done) | |
43d6e369 | 1195 | *done = last_complete; |
0bbd5f4e CL |
1196 | if (used) |
1197 | *used = last_used; | |
1198 | ||
1199 | ret = dma_async_is_complete(cookie, last_complete, last_used); | |
1200 | if (ret == DMA_SUCCESS) | |
1201 | return ret; | |
1202 | ||
dcbc853a | 1203 | ioat_dma_memcpy_cleanup(ioat); |
0bbd5f4e | 1204 | |
dcbc853a DW |
1205 | last_used = c->cookie; |
1206 | last_complete = chan->completed_cookie; | |
0bbd5f4e CL |
1207 | |
1208 | if (done) | |
43d6e369 | 1209 | *done = last_complete; |
0bbd5f4e CL |
1210 | if (used) |
1211 | *used = last_used; | |
1212 | ||
1213 | return dma_async_is_complete(cookie, last_complete, last_used); | |
1214 | } | |
1215 | ||
dcbc853a | 1216 | static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat) |
0bbd5f4e | 1217 | { |
dcbc853a | 1218 | struct ioat_chan_common *chan = &ioat->base; |
0bbd5f4e | 1219 | struct ioat_desc_sw *desc; |
c7984f4e | 1220 | struct ioat_dma_descriptor *hw; |
0bbd5f4e | 1221 | |
dcbc853a | 1222 | spin_lock_bh(&ioat->desc_lock); |
0bbd5f4e | 1223 | |
dcbc853a | 1224 | desc = ioat_dma_get_next_descriptor(ioat); |
7f1b358a MS |
1225 | |
1226 | if (!desc) { | |
dcbc853a | 1227 | dev_err(to_dev(chan), |
7f1b358a | 1228 | "Unable to start null desc - get next desc failed\n"); |
dcbc853a | 1229 | spin_unlock_bh(&ioat->desc_lock); |
7f1b358a MS |
1230 | return; |
1231 | } | |
1232 | ||
c7984f4e DW |
1233 | hw = desc->hw; |
1234 | hw->ctl = 0; | |
1235 | hw->ctl_f.null = 1; | |
1236 | hw->ctl_f.int_en = 1; | |
1237 | hw->ctl_f.compl_write = 1; | |
7f1b358a | 1238 | /* set size to non-zero value (channel returns error when size is 0) */ |
c7984f4e DW |
1239 | hw->size = NULL_DESC_BUFFER_SIZE; |
1240 | hw->src_addr = 0; | |
1241 | hw->dst_addr = 0; | |
bc3c7025 | 1242 | async_tx_ack(&desc->txd); |
dcbc853a | 1243 | switch (chan->device->version) { |
7bb67c14 | 1244 | case IOAT_VER_1_2: |
c7984f4e | 1245 | hw->next = 0; |
dcbc853a | 1246 | list_add_tail(&desc->node, &ioat->used_desc); |
7bb67c14 | 1247 | |
bc3c7025 | 1248 | writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF, |
dcbc853a | 1249 | chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW); |
bc3c7025 | 1250 | writel(((u64) desc->txd.phys) >> 32, |
dcbc853a | 1251 | chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH); |
7bb67c14 | 1252 | |
dcbc853a DW |
1253 | writeb(IOAT_CHANCMD_START, chan->reg_base |
1254 | + IOAT_CHANCMD_OFFSET(chan->device->version)); | |
7bb67c14 SN |
1255 | break; |
1256 | case IOAT_VER_2_0: | |
7f1b358a | 1257 | case IOAT_VER_3_0: |
bc3c7025 | 1258 | writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF, |
dcbc853a | 1259 | chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW); |
bc3c7025 | 1260 | writel(((u64) desc->txd.phys) >> 32, |
dcbc853a | 1261 | chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH); |
7bb67c14 | 1262 | |
dcbc853a DW |
1263 | ioat->dmacount++; |
1264 | __ioat2_dma_memcpy_issue_pending(ioat); | |
7bb67c14 SN |
1265 | break; |
1266 | } | |
dcbc853a | 1267 | spin_unlock_bh(&ioat->desc_lock); |
0bbd5f4e CL |
1268 | } |
1269 | ||
1270 | /* | |
1271 | * Perform a IOAT transaction to verify the HW works. | |
1272 | */ | |
1273 | #define IOAT_TEST_SIZE 2000 | |
1274 | ||
95218430 SN |
1275 | static void ioat_dma_test_callback(void *dma_async_param) |
1276 | { | |
b9bdcbba DW |
1277 | struct completion *cmp = dma_async_param; |
1278 | ||
1279 | complete(cmp); | |
95218430 SN |
1280 | } |
1281 | ||
3e037454 SN |
1282 | /** |
1283 | * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works. | |
1284 | * @device: device to be tested | |
1285 | */ | |
1286 | static int ioat_dma_self_test(struct ioatdma_device *device) | |
0bbd5f4e CL |
1287 | { |
1288 | int i; | |
1289 | u8 *src; | |
1290 | u8 *dest; | |
bc3c7025 DW |
1291 | struct dma_device *dma = &device->common; |
1292 | struct device *dev = &device->pdev->dev; | |
0bbd5f4e | 1293 | struct dma_chan *dma_chan; |
711924b1 | 1294 | struct dma_async_tx_descriptor *tx; |
0036731c | 1295 | dma_addr_t dma_dest, dma_src; |
0bbd5f4e CL |
1296 | dma_cookie_t cookie; |
1297 | int err = 0; | |
b9bdcbba | 1298 | struct completion cmp; |
0c33e1ca | 1299 | unsigned long tmo; |
4f005dbe | 1300 | unsigned long flags; |
0bbd5f4e | 1301 | |
e94b1766 | 1302 | src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL); |
0bbd5f4e CL |
1303 | if (!src) |
1304 | return -ENOMEM; | |
e94b1766 | 1305 | dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL); |
0bbd5f4e CL |
1306 | if (!dest) { |
1307 | kfree(src); | |
1308 | return -ENOMEM; | |
1309 | } | |
1310 | ||
1311 | /* Fill in src buffer */ | |
1312 | for (i = 0; i < IOAT_TEST_SIZE; i++) | |
1313 | src[i] = (u8)i; | |
1314 | ||
1315 | /* Start copy, using first DMA channel */ | |
bc3c7025 | 1316 | dma_chan = container_of(dma->channels.next, struct dma_chan, |
43d6e369 | 1317 | device_node); |
bc3c7025 DW |
1318 | if (dma->device_alloc_chan_resources(dma_chan) < 1) { |
1319 | dev_err(dev, "selftest cannot allocate chan resource\n"); | |
0bbd5f4e CL |
1320 | err = -ENODEV; |
1321 | goto out; | |
1322 | } | |
1323 | ||
bc3c7025 DW |
1324 | dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE); |
1325 | dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE); | |
a6a39ca1 DW |
1326 | flags = DMA_COMPL_SRC_UNMAP_SINGLE | DMA_COMPL_DEST_UNMAP_SINGLE | |
1327 | DMA_PREP_INTERRUPT; | |
0036731c | 1328 | tx = device->common.device_prep_dma_memcpy(dma_chan, dma_dest, dma_src, |
4f005dbe | 1329 | IOAT_TEST_SIZE, flags); |
5149fd01 | 1330 | if (!tx) { |
bc3c7025 | 1331 | dev_err(dev, "Self-test prep failed, disabling\n"); |
5149fd01 SN |
1332 | err = -ENODEV; |
1333 | goto free_resources; | |
1334 | } | |
1335 | ||
7405f74b | 1336 | async_tx_ack(tx); |
b9bdcbba | 1337 | init_completion(&cmp); |
95218430 | 1338 | tx->callback = ioat_dma_test_callback; |
b9bdcbba | 1339 | tx->callback_param = &cmp; |
7bb67c14 | 1340 | cookie = tx->tx_submit(tx); |
7f2b291f | 1341 | if (cookie < 0) { |
bc3c7025 | 1342 | dev_err(dev, "Self-test setup failed, disabling\n"); |
7f2b291f SN |
1343 | err = -ENODEV; |
1344 | goto free_resources; | |
1345 | } | |
bc3c7025 | 1346 | dma->device_issue_pending(dma_chan); |
532d3b1f | 1347 | |
0c33e1ca | 1348 | tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)); |
0bbd5f4e | 1349 | |
0c33e1ca | 1350 | if (tmo == 0 || |
bc3c7025 | 1351 | dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL) |
7bb67c14 | 1352 | != DMA_SUCCESS) { |
bc3c7025 | 1353 | dev_err(dev, "Self-test copy timed out, disabling\n"); |
0bbd5f4e CL |
1354 | err = -ENODEV; |
1355 | goto free_resources; | |
1356 | } | |
1357 | if (memcmp(src, dest, IOAT_TEST_SIZE)) { | |
bc3c7025 | 1358 | dev_err(dev, "Self-test copy failed compare, disabling\n"); |
0bbd5f4e CL |
1359 | err = -ENODEV; |
1360 | goto free_resources; | |
1361 | } | |
1362 | ||
1363 | free_resources: | |
bc3c7025 | 1364 | dma->device_free_chan_resources(dma_chan); |
0bbd5f4e CL |
1365 | out: |
1366 | kfree(src); | |
1367 | kfree(dest); | |
1368 | return err; | |
1369 | } | |
1370 | ||
3e037454 SN |
1371 | static char ioat_interrupt_style[32] = "msix"; |
1372 | module_param_string(ioat_interrupt_style, ioat_interrupt_style, | |
1373 | sizeof(ioat_interrupt_style), 0644); | |
1374 | MODULE_PARM_DESC(ioat_interrupt_style, | |
1375 | "set ioat interrupt style: msix (default), " | |
1376 | "msix-single-vector, msi, intx)"); | |
1377 | ||
1378 | /** | |
1379 | * ioat_dma_setup_interrupts - setup interrupt handler | |
1380 | * @device: ioat device | |
1381 | */ | |
1382 | static int ioat_dma_setup_interrupts(struct ioatdma_device *device) | |
1383 | { | |
dcbc853a | 1384 | struct ioat_chan_common *chan; |
e6c0b69a DW |
1385 | struct pci_dev *pdev = device->pdev; |
1386 | struct device *dev = &pdev->dev; | |
1387 | struct msix_entry *msix; | |
1388 | int i, j, msixcnt; | |
1389 | int err = -EINVAL; | |
3e037454 SN |
1390 | u8 intrctrl = 0; |
1391 | ||
1392 | if (!strcmp(ioat_interrupt_style, "msix")) | |
1393 | goto msix; | |
1394 | if (!strcmp(ioat_interrupt_style, "msix-single-vector")) | |
1395 | goto msix_single_vector; | |
1396 | if (!strcmp(ioat_interrupt_style, "msi")) | |
1397 | goto msi; | |
1398 | if (!strcmp(ioat_interrupt_style, "intx")) | |
1399 | goto intx; | |
e6c0b69a | 1400 | dev_err(dev, "invalid ioat_interrupt_style %s\n", ioat_interrupt_style); |
5149fd01 | 1401 | goto err_no_irq; |
3e037454 SN |
1402 | |
1403 | msix: | |
1404 | /* The number of MSI-X vectors should equal the number of channels */ | |
1405 | msixcnt = device->common.chancnt; | |
1406 | for (i = 0; i < msixcnt; i++) | |
1407 | device->msix_entries[i].entry = i; | |
1408 | ||
e6c0b69a | 1409 | err = pci_enable_msix(pdev, device->msix_entries, msixcnt); |
3e037454 SN |
1410 | if (err < 0) |
1411 | goto msi; | |
1412 | if (err > 0) | |
1413 | goto msix_single_vector; | |
1414 | ||
1415 | for (i = 0; i < msixcnt; i++) { | |
e6c0b69a | 1416 | msix = &device->msix_entries[i]; |
dcbc853a | 1417 | chan = ioat_chan_by_index(device, i); |
e6c0b69a DW |
1418 | err = devm_request_irq(dev, msix->vector, |
1419 | ioat_dma_do_interrupt_msix, 0, | |
dcbc853a | 1420 | "ioat-msix", chan); |
3e037454 SN |
1421 | if (err) { |
1422 | for (j = 0; j < i; j++) { | |
e6c0b69a | 1423 | msix = &device->msix_entries[j]; |
dcbc853a DW |
1424 | chan = ioat_chan_by_index(device, j); |
1425 | devm_free_irq(dev, msix->vector, chan); | |
3e037454 SN |
1426 | } |
1427 | goto msix_single_vector; | |
1428 | } | |
1429 | } | |
1430 | intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL; | |
3e037454 SN |
1431 | goto done; |
1432 | ||
1433 | msix_single_vector: | |
e6c0b69a DW |
1434 | msix = &device->msix_entries[0]; |
1435 | msix->entry = 0; | |
1436 | err = pci_enable_msix(pdev, device->msix_entries, 1); | |
3e037454 SN |
1437 | if (err) |
1438 | goto msi; | |
1439 | ||
e6c0b69a DW |
1440 | err = devm_request_irq(dev, msix->vector, ioat_dma_do_interrupt, 0, |
1441 | "ioat-msix", device); | |
3e037454 | 1442 | if (err) { |
e6c0b69a | 1443 | pci_disable_msix(pdev); |
3e037454 SN |
1444 | goto msi; |
1445 | } | |
3e037454 SN |
1446 | goto done; |
1447 | ||
1448 | msi: | |
e6c0b69a | 1449 | err = pci_enable_msi(pdev); |
3e037454 SN |
1450 | if (err) |
1451 | goto intx; | |
1452 | ||
e6c0b69a DW |
1453 | err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, 0, |
1454 | "ioat-msi", device); | |
3e037454 | 1455 | if (err) { |
e6c0b69a | 1456 | pci_disable_msi(pdev); |
3e037454 SN |
1457 | goto intx; |
1458 | } | |
3e037454 SN |
1459 | goto done; |
1460 | ||
1461 | intx: | |
e6c0b69a DW |
1462 | err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, |
1463 | IRQF_SHARED, "ioat-intx", device); | |
3e037454 SN |
1464 | if (err) |
1465 | goto err_no_irq; | |
3e037454 SN |
1466 | |
1467 | done: | |
f2427e27 DW |
1468 | if (device->intr_quirk) |
1469 | device->intr_quirk(device); | |
3e037454 SN |
1470 | intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN; |
1471 | writeb(intrctrl, device->reg_base + IOAT_INTRCTRL_OFFSET); | |
1472 | return 0; | |
1473 | ||
1474 | err_no_irq: | |
1475 | /* Disable all interrupt generation */ | |
1476 | writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET); | |
e6c0b69a DW |
1477 | dev_err(dev, "no usable interrupts\n"); |
1478 | return err; | |
3e037454 SN |
1479 | } |
1480 | ||
e6c0b69a | 1481 | static void ioat_disable_interrupts(struct ioatdma_device *device) |
3e037454 | 1482 | { |
3e037454 SN |
1483 | /* Disable all interrupt generation */ |
1484 | writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET); | |
3e037454 SN |
1485 | } |
1486 | ||
f2427e27 | 1487 | static int ioat_probe(struct ioatdma_device *device) |
0bbd5f4e | 1488 | { |
f2427e27 DW |
1489 | int err = -ENODEV; |
1490 | struct dma_device *dma = &device->common; | |
1491 | struct pci_dev *pdev = device->pdev; | |
e6c0b69a | 1492 | struct device *dev = &pdev->dev; |
0bbd5f4e CL |
1493 | |
1494 | /* DMA coherent memory pool for DMA descriptor allocations */ | |
1495 | device->dma_pool = pci_pool_create("dma_desc_pool", pdev, | |
8ab89567 SN |
1496 | sizeof(struct ioat_dma_descriptor), |
1497 | 64, 0); | |
0bbd5f4e CL |
1498 | if (!device->dma_pool) { |
1499 | err = -ENOMEM; | |
1500 | goto err_dma_pool; | |
1501 | } | |
1502 | ||
43d6e369 SN |
1503 | device->completion_pool = pci_pool_create("completion_pool", pdev, |
1504 | sizeof(u64), SMP_CACHE_BYTES, | |
1505 | SMP_CACHE_BYTES); | |
0bbd5f4e CL |
1506 | if (!device->completion_pool) { |
1507 | err = -ENOMEM; | |
1508 | goto err_completion_pool; | |
1509 | } | |
1510 | ||
43d6e369 | 1511 | ioat_dma_enumerate_channels(device); |
0bbd5f4e | 1512 | |
f2427e27 | 1513 | dma_cap_set(DMA_MEMCPY, dma->cap_mask); |
bc3c7025 DW |
1514 | dma->device_alloc_chan_resources = ioat_dma_alloc_chan_resources; |
1515 | dma->device_free_chan_resources = ioat_dma_free_chan_resources; | |
bc3c7025 | 1516 | dma->device_is_tx_complete = ioat_dma_is_complete; |
f2427e27 | 1517 | dma->dev = &pdev->dev; |
7bb67c14 | 1518 | |
e6c0b69a | 1519 | dev_err(dev, "Intel(R) I/OAT DMA Engine found," |
5149fd01 | 1520 | " %d channels, device version 0x%02x, driver version %s\n", |
bc3c7025 | 1521 | dma->chancnt, device->version, IOAT_DMA_VERSION); |
8ab89567 | 1522 | |
bc3c7025 | 1523 | if (!dma->chancnt) { |
e6c0b69a | 1524 | dev_err(dev, "Intel(R) I/OAT DMA Engine problem found: " |
8b794b14 MS |
1525 | "zero channels detected\n"); |
1526 | goto err_setup_interrupts; | |
1527 | } | |
1528 | ||
3e037454 | 1529 | err = ioat_dma_setup_interrupts(device); |
8ab89567 | 1530 | if (err) |
3e037454 | 1531 | goto err_setup_interrupts; |
0bbd5f4e | 1532 | |
3e037454 | 1533 | err = ioat_dma_self_test(device); |
0bbd5f4e CL |
1534 | if (err) |
1535 | goto err_self_test; | |
1536 | ||
f2427e27 | 1537 | return 0; |
0bbd5f4e CL |
1538 | |
1539 | err_self_test: | |
e6c0b69a | 1540 | ioat_disable_interrupts(device); |
3e037454 | 1541 | err_setup_interrupts: |
0bbd5f4e CL |
1542 | pci_pool_destroy(device->completion_pool); |
1543 | err_completion_pool: | |
1544 | pci_pool_destroy(device->dma_pool); | |
1545 | err_dma_pool: | |
f2427e27 DW |
1546 | return err; |
1547 | } | |
1548 | ||
1549 | static int ioat_register(struct ioatdma_device *device) | |
1550 | { | |
1551 | int err = dma_async_device_register(&device->common); | |
1552 | ||
1553 | if (err) { | |
1554 | ioat_disable_interrupts(device); | |
1555 | pci_pool_destroy(device->completion_pool); | |
1556 | pci_pool_destroy(device->dma_pool); | |
1557 | } | |
1558 | ||
1559 | return err; | |
1560 | } | |
1561 | ||
1562 | /* ioat1_intr_quirk - fix up dma ctrl register to enable / disable msi */ | |
1563 | static void ioat1_intr_quirk(struct ioatdma_device *device) | |
1564 | { | |
1565 | struct pci_dev *pdev = device->pdev; | |
1566 | u32 dmactrl; | |
1567 | ||
1568 | pci_read_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, &dmactrl); | |
1569 | if (pdev->msi_enabled) | |
1570 | dmactrl |= IOAT_PCI_DMACTRL_MSI_EN; | |
1571 | else | |
1572 | dmactrl &= ~IOAT_PCI_DMACTRL_MSI_EN; | |
1573 | pci_write_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, dmactrl); | |
1574 | } | |
1575 | ||
1576 | int ioat1_dma_probe(struct ioatdma_device *device, int dca) | |
1577 | { | |
1578 | struct pci_dev *pdev = device->pdev; | |
1579 | struct dma_device *dma; | |
1580 | int err; | |
1581 | ||
1582 | device->intr_quirk = ioat1_intr_quirk; | |
1583 | dma = &device->common; | |
1584 | dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy; | |
1585 | dma->device_issue_pending = ioat1_dma_memcpy_issue_pending; | |
1586 | ||
1587 | err = ioat_probe(device); | |
1588 | if (err) | |
1589 | return err; | |
1590 | ioat_set_tcp_copy_break(4096); | |
1591 | err = ioat_register(device); | |
1592 | if (err) | |
1593 | return err; | |
1594 | if (dca) | |
1595 | device->dca = ioat_dca_init(pdev, device->reg_base); | |
1596 | ||
1597 | INIT_DELAYED_WORK(&device->work, ioat_dma_chan_watchdog); | |
1598 | schedule_delayed_work(&device->work, WATCHDOG_DELAY); | |
1599 | ||
1600 | return err; | |
1601 | } | |
1602 | ||
1603 | int ioat2_dma_probe(struct ioatdma_device *device, int dca) | |
1604 | { | |
1605 | struct pci_dev *pdev = device->pdev; | |
1606 | struct dma_device *dma; | |
dcbc853a DW |
1607 | struct dma_chan *c; |
1608 | struct ioat_chan_common *chan; | |
f2427e27 DW |
1609 | int err; |
1610 | ||
1611 | dma = &device->common; | |
1612 | dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy; | |
1613 | dma->device_issue_pending = ioat2_dma_memcpy_issue_pending; | |
1614 | ||
1615 | err = ioat_probe(device); | |
1616 | if (err) | |
1617 | return err; | |
1618 | ioat_set_tcp_copy_break(2048); | |
1619 | ||
dcbc853a DW |
1620 | list_for_each_entry(c, &dma->channels, device_node) { |
1621 | chan = to_chan_common(c); | |
f2427e27 | 1622 | writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE | IOAT_DMA_DCA_ANY_CPU, |
dcbc853a | 1623 | chan->reg_base + IOAT_DCACTRL_OFFSET); |
f2427e27 DW |
1624 | } |
1625 | ||
1626 | err = ioat_register(device); | |
1627 | if (err) | |
1628 | return err; | |
1629 | if (dca) | |
1630 | device->dca = ioat2_dca_init(pdev, device->reg_base); | |
1631 | ||
1632 | INIT_DELAYED_WORK(&device->work, ioat_dma_chan_watchdog); | |
1633 | schedule_delayed_work(&device->work, WATCHDOG_DELAY); | |
1634 | ||
1635 | return err; | |
1636 | } | |
1637 | ||
1638 | int ioat3_dma_probe(struct ioatdma_device *device, int dca) | |
1639 | { | |
1640 | struct pci_dev *pdev = device->pdev; | |
1641 | struct dma_device *dma; | |
dcbc853a DW |
1642 | struct dma_chan *c; |
1643 | struct ioat_chan_common *chan; | |
f2427e27 DW |
1644 | int err; |
1645 | u16 dev_id; | |
1646 | ||
1647 | dma = &device->common; | |
1648 | dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy; | |
1649 | dma->device_issue_pending = ioat2_dma_memcpy_issue_pending; | |
1650 | ||
1651 | /* -= IOAT ver.3 workarounds =- */ | |
1652 | /* Write CHANERRMSK_INT with 3E07h to mask out the errors | |
1653 | * that can cause stability issues for IOAT ver.3 | |
1654 | */ | |
1655 | pci_write_config_dword(pdev, IOAT_PCI_CHANERRMASK_INT_OFFSET, 0x3e07); | |
1656 | ||
1657 | /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit | |
1658 | * (workaround for spurious config parity error after restart) | |
1659 | */ | |
1660 | pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id); | |
1661 | if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0) | |
1662 | pci_write_config_dword(pdev, IOAT_PCI_DMAUNCERRSTS_OFFSET, 0x10); | |
1663 | ||
1664 | err = ioat_probe(device); | |
1665 | if (err) | |
1666 | return err; | |
1667 | ioat_set_tcp_copy_break(262144); | |
1668 | ||
dcbc853a DW |
1669 | list_for_each_entry(c, &dma->channels, device_node) { |
1670 | chan = to_chan_common(c); | |
f2427e27 | 1671 | writel(IOAT_DMA_DCA_ANY_CPU, |
dcbc853a | 1672 | chan->reg_base + IOAT_DCACTRL_OFFSET); |
f2427e27 DW |
1673 | } |
1674 | ||
1675 | err = ioat_register(device); | |
1676 | if (err) | |
1677 | return err; | |
1678 | if (dca) | |
1679 | device->dca = ioat3_dca_init(pdev, device->reg_base); | |
1680 | ||
1681 | return err; | |
428ed602 DA |
1682 | } |
1683 | ||
8ab89567 | 1684 | void ioat_dma_remove(struct ioatdma_device *device) |
0bbd5f4e | 1685 | { |
bc3c7025 | 1686 | struct dma_device *dma = &device->common; |
0bbd5f4e | 1687 | |
2b8a6bf8 MS |
1688 | if (device->version != IOAT_VER_3_0) |
1689 | cancel_delayed_work(&device->work); | |
1690 | ||
e6c0b69a | 1691 | ioat_disable_interrupts(device); |
8ab89567 | 1692 | |
bc3c7025 | 1693 | dma_async_device_unregister(dma); |
dfe2299e | 1694 | |
0bbd5f4e CL |
1695 | pci_pool_destroy(device->dma_pool); |
1696 | pci_pool_destroy(device->completion_pool); | |
8ab89567 | 1697 | |
dcbc853a | 1698 | INIT_LIST_HEAD(&dma->channels); |
0bbd5f4e CL |
1699 | } |
1700 |