projects / linux-block.git / blame
| Commit | Line | Data |
|---|---|---|
| a61127c2 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
| 9bc89cd8 DW |
2 | /* |
| 3 | * copy offload engine support | |
| 4 | * | |
| 5 | * Copyright © 2006, Intel Corporation. | |
| 6 | * | |
| 7 | * Dan Williams <dan.j.williams@intel.com> | |
| 8 | * | |
| 9 | * with architecture considerations by: | |
| 10 | * Neil Brown <neilb@suse.de> | |
| 11 | * Jeff Garzik <jeff@garzik.org> | |
| 9bc89cd8 DW |
12 | */ |
| 13 | #include <linux/kernel.h> | |
| 14 | #include <linux/highmem.h> | |
| 4bb33cc8 | 15 | #include <linux/module.h> |
| 9bc89cd8 DW |
16 | #include <linux/mm.h> |
| 17 | #include <linux/dma-mapping.h> | |
| 18 | #include <linux/async_tx.h> | |
| 19 | ||
| 20 | /** | |
| 21 | * async_memcpy - attempt to copy memory with a dma engine. | |
| 22 | * @dest: destination page | |
| 23 | * @src: src page | |
| a08abd8c DW |
24 | * @dest_offset: offset into 'dest' to start transaction |
| 25 | * @src_offset: offset into 'src' to start transaction | |
| 9bc89cd8 | 26 | * @len: length in bytes |
| a08abd8c DW |
27 | * @submit: submission / completion modifiers |
| 28 | * | |
| 29 | * honored flags: ASYNC_TX_ACK | |
| 9bc89cd8 DW |
30 | */ |
| 31 | struct dma_async_tx_descriptor * | |
| 32 | async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset, | |
| a08abd8c DW |
33 | unsigned int src_offset, size_t len, |
| 34 | struct async_submit_ctl *submit) | |
| 9bc89cd8 | 35 | { |
| a08abd8c | 36 | struct dma_chan *chan = async_tx_find_channel(submit, DMA_MEMCPY, |
| 47437b2c | 37 | &dest, 1, &src, 1, len); |
| 9bc89cd8 | 38 | struct dma_device *device = chan ? chan->device : NULL; |
| 0036731c | 39 | struct dma_async_tx_descriptor *tx = NULL; |
| 89716462 | 40 | struct dmaengine_unmap_data *unmap = NULL; |
| 9bc89cd8 | 41 | |
| 89716462 | 42 | if (device) |
| b02bab6b | 43 | unmap = dmaengine_get_unmap_data(device->dev, 2, GFP_NOWAIT); |
| 89716462 DW |
44 | |
| 45 | if (unmap && is_dma_copy_aligned(device, src_offset, dest_offset, len)) { | |
| 0776ae7b | 46 | unsigned long dma_prep_flags = 0; |
| 9bc89cd8 | 47 | |
| 0403e382 DW |
48 | if (submit->cb_fn) |
| 49 | dma_prep_flags |= DMA_PREP_INTERRUPT; | |
| 50 | if (submit->flags & ASYNC_TX_FENCE) | |
| 51 | dma_prep_flags |= DMA_PREP_FENCE; | |
| 89716462 DW |
52 | |
| 53 | unmap->to_cnt = 1; | |
| 54 | unmap->addr[0] = dma_map_page(device->dev, src, src_offset, len, | |
| 55 | DMA_TO_DEVICE); | |
| 56 | unmap->from_cnt = 1; | |
| 57 | unmap->addr[1] = dma_map_page(device->dev, dest, dest_offset, len, | |
| 58 | DMA_FROM_DEVICE); | |
| 59 | unmap->len = len; | |
| 60 | ||
| 61 | tx = device->device_prep_dma_memcpy(chan, unmap->addr[1], | |
| 62 | unmap->addr[0], len, | |
| 63 | dma_prep_flags); | |
| 0036731c | 64 | } |
| 9bc89cd8 | 65 | |
| 0036731c | 66 | if (tx) { |
| 3280ab3e | 67 | pr_debug("%s: (async) len: %zu\n", __func__, len); |
| 89716462 DW |
68 | |
| 69 | dma_set_unmap(tx, unmap); | |
| a08abd8c | 70 | async_tx_submit(chan, tx, submit); |
| 0036731c | 71 | } else { |
| 9bc89cd8 | 72 | void *dest_buf, *src_buf; |
| 3280ab3e | 73 | pr_debug("%s: (sync) len: %zu\n", __func__, len); |
| 9bc89cd8 DW |
74 | |
| 75 | /* wait for any prerequisite operations */ | |
| a08abd8c | 76 | async_tx_quiesce(&submit->depend_tx); |
| 9bc89cd8 | 77 | |
| f0dfc0b0 CW |
78 | dest_buf = kmap_atomic(dest) + dest_offset; |
| 79 | src_buf = kmap_atomic(src) + src_offset; | |
| 9bc89cd8 DW |
80 | |
| 81 | memcpy(dest_buf, src_buf, len); | |
| 82 | ||
| f0dfc0b0 CW |
83 | kunmap_atomic(src_buf); |
| 84 | kunmap_atomic(dest_buf); | |
| 9bc89cd8 | 85 | |
| a08abd8c | 86 | async_tx_sync_epilog(submit); |
| 9bc89cd8 DW |
87 | } |
| 88 | ||
| 89716462 DW |
89 | dmaengine_unmap_put(unmap); |
| 90 | ||
| 9bc89cd8 DW |
91 | return tx; |
| 92 | } | |
| 93 | EXPORT_SYMBOL_GPL(async_memcpy); | |
| 94 | ||
| 9bc89cd8 DW |
95 | MODULE_AUTHOR("Intel Corporation"); |
| 96 | MODULE_DESCRIPTION("asynchronous memcpy api"); | |
| 97 | MODULE_LICENSE("GPL"); |