Commit | Line | Data |
---|---|---|
f0edfea8 CH |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
0c3b3171 | 3 | * Copyright (C) 2012 ARM Ltd. |
f0edfea8 CH |
4 | * Copyright (c) 2014 The Linux Foundation |
5 | */ | |
0c3b3171 CH |
6 | #include <linux/dma-direct.h> |
7 | #include <linux/dma-noncoherent.h> | |
8 | #include <linux/dma-contiguous.h> | |
9 | #include <linux/init.h> | |
10 | #include <linux/genalloc.h> | |
f0edfea8 CH |
11 | #include <linux/slab.h> |
12 | #include <linux/vmalloc.h> | |
13 | ||
5cf45379 CH |
14 | struct page **dma_common_find_pages(void *cpu_addr) |
15 | { | |
16 | struct vm_struct *area = find_vm_area(cpu_addr); | |
17 | ||
18 | if (!area || area->flags != VM_DMA_COHERENT) | |
19 | return NULL; | |
20 | return area->pages; | |
21 | } | |
22 | ||
f0edfea8 | 23 | static struct vm_struct *__dma_common_pages_remap(struct page **pages, |
51231740 | 24 | size_t size, pgprot_t prot, const void *caller) |
f0edfea8 CH |
25 | { |
26 | struct vm_struct *area; | |
27 | ||
51231740 | 28 | area = get_vm_area_caller(size, VM_DMA_COHERENT, caller); |
f0edfea8 CH |
29 | if (!area) |
30 | return NULL; | |
31 | ||
32 | if (map_vm_area(area, prot, pages)) { | |
33 | vunmap(area->addr); | |
34 | return NULL; | |
35 | } | |
36 | ||
37 | return area; | |
38 | } | |
39 | ||
40 | /* | |
41 | * Remaps an array of PAGE_SIZE pages into another vm_area. | |
42 | * Cannot be used in non-sleeping contexts | |
43 | */ | |
44 | void *dma_common_pages_remap(struct page **pages, size_t size, | |
51231740 | 45 | pgprot_t prot, const void *caller) |
f0edfea8 CH |
46 | { |
47 | struct vm_struct *area; | |
48 | ||
51231740 | 49 | area = __dma_common_pages_remap(pages, size, prot, caller); |
f0edfea8 CH |
50 | if (!area) |
51 | return NULL; | |
52 | ||
53 | area->pages = pages; | |
54 | ||
55 | return area->addr; | |
56 | } | |
57 | ||
58 | /* | |
59 | * Remaps an allocated contiguous region into another vm_area. | |
60 | * Cannot be used in non-sleeping contexts | |
61 | */ | |
62 | void *dma_common_contiguous_remap(struct page *page, size_t size, | |
f0edfea8 CH |
63 | pgprot_t prot, const void *caller) |
64 | { | |
65 | int i; | |
66 | struct page **pages; | |
67 | struct vm_struct *area; | |
68 | ||
69 | pages = kmalloc(sizeof(struct page *) << get_order(size), GFP_KERNEL); | |
70 | if (!pages) | |
71 | return NULL; | |
72 | ||
73 | for (i = 0; i < (size >> PAGE_SHIFT); i++) | |
74 | pages[i] = nth_page(page, i); | |
75 | ||
51231740 | 76 | area = __dma_common_pages_remap(pages, size, prot, caller); |
f0edfea8 CH |
77 | |
78 | kfree(pages); | |
79 | ||
80 | if (!area) | |
81 | return NULL; | |
82 | return area->addr; | |
83 | } | |
84 | ||
85 | /* | |
86 | * Unmaps a range previously mapped by dma_common_*_remap | |
87 | */ | |
51231740 | 88 | void dma_common_free_remap(void *cpu_addr, size_t size) |
f0edfea8 | 89 | { |
2cf2aa6a | 90 | struct vm_struct *area = find_vm_area(cpu_addr); |
f0edfea8 | 91 | |
2cf2aa6a | 92 | if (!area || area->flags != VM_DMA_COHERENT) { |
f0edfea8 CH |
93 | WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr); |
94 | return; | |
95 | } | |
96 | ||
97 | unmap_kernel_range((unsigned long)cpu_addr, PAGE_ALIGN(size)); | |
98 | vunmap(cpu_addr); | |
99 | } | |
0c3b3171 CH |
100 | |
101 | #ifdef CONFIG_DMA_DIRECT_REMAP | |
102 | static struct gen_pool *atomic_pool __ro_after_init; | |
103 | ||
104 | #define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K | |
105 | static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE; | |
106 | ||
107 | static int __init early_coherent_pool(char *p) | |
108 | { | |
109 | atomic_pool_size = memparse(p, &p); | |
110 | return 0; | |
111 | } | |
112 | early_param("coherent_pool", early_coherent_pool); | |
113 | ||
8e3a68fb CH |
114 | static gfp_t dma_atomic_pool_gfp(void) |
115 | { | |
116 | if (IS_ENABLED(CONFIG_ZONE_DMA)) | |
117 | return GFP_DMA; | |
118 | if (IS_ENABLED(CONFIG_ZONE_DMA32)) | |
119 | return GFP_DMA32; | |
120 | return GFP_KERNEL; | |
121 | } | |
122 | ||
123 | static int __init dma_atomic_pool_init(void) | |
0c3b3171 CH |
124 | { |
125 | unsigned int pool_size_order = get_order(atomic_pool_size); | |
126 | unsigned long nr_pages = atomic_pool_size >> PAGE_SHIFT; | |
127 | struct page *page; | |
128 | void *addr; | |
129 | int ret; | |
130 | ||
131 | if (dev_get_cma_area(NULL)) | |
132 | page = dma_alloc_from_contiguous(NULL, nr_pages, | |
133 | pool_size_order, false); | |
134 | else | |
8e3a68fb | 135 | page = alloc_pages(dma_atomic_pool_gfp(), pool_size_order); |
0c3b3171 CH |
136 | if (!page) |
137 | goto out; | |
138 | ||
0c3b3171 CH |
139 | arch_dma_prep_coherent(page, atomic_pool_size); |
140 | ||
141 | atomic_pool = gen_pool_create(PAGE_SHIFT, -1); | |
142 | if (!atomic_pool) | |
143 | goto free_page; | |
144 | ||
51231740 | 145 | addr = dma_common_contiguous_remap(page, atomic_pool_size, |
8e3a68fb CH |
146 | pgprot_dmacoherent(PAGE_KERNEL), |
147 | __builtin_return_address(0)); | |
0c3b3171 CH |
148 | if (!addr) |
149 | goto destroy_genpool; | |
150 | ||
151 | ret = gen_pool_add_virt(atomic_pool, (unsigned long)addr, | |
152 | page_to_phys(page), atomic_pool_size, -1); | |
153 | if (ret) | |
154 | goto remove_mapping; | |
155 | gen_pool_set_algo(atomic_pool, gen_pool_first_fit_order_align, NULL); | |
156 | ||
157 | pr_info("DMA: preallocated %zu KiB pool for atomic allocations\n", | |
158 | atomic_pool_size / 1024); | |
159 | return 0; | |
160 | ||
161 | remove_mapping: | |
51231740 | 162 | dma_common_free_remap(addr, atomic_pool_size); |
0c3b3171 CH |
163 | destroy_genpool: |
164 | gen_pool_destroy(atomic_pool); | |
165 | atomic_pool = NULL; | |
166 | free_page: | |
167 | if (!dma_release_from_contiguous(NULL, page, nr_pages)) | |
168 | __free_pages(page, pool_size_order); | |
169 | out: | |
170 | pr_err("DMA: failed to allocate %zu KiB pool for atomic coherent allocation\n", | |
171 | atomic_pool_size / 1024); | |
172 | return -ENOMEM; | |
173 | } | |
8e3a68fb | 174 | postcore_initcall(dma_atomic_pool_init); |
0c3b3171 CH |
175 | |
176 | bool dma_in_atomic_pool(void *start, size_t size) | |
177 | { | |
4b4b077c FF |
178 | if (unlikely(!atomic_pool)) |
179 | return false; | |
180 | ||
0c3b3171 CH |
181 | return addr_in_gen_pool(atomic_pool, (unsigned long)start, size); |
182 | } | |
183 | ||
184 | void *dma_alloc_from_pool(size_t size, struct page **ret_page, gfp_t flags) | |
185 | { | |
186 | unsigned long val; | |
187 | void *ptr = NULL; | |
188 | ||
189 | if (!atomic_pool) { | |
190 | WARN(1, "coherent pool not initialised!\n"); | |
191 | return NULL; | |
192 | } | |
193 | ||
194 | val = gen_pool_alloc(atomic_pool, size); | |
195 | if (val) { | |
196 | phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val); | |
197 | ||
198 | *ret_page = pfn_to_page(__phys_to_pfn(phys)); | |
199 | ptr = (void *)val; | |
200 | memset(ptr, 0, size); | |
201 | } | |
202 | ||
203 | return ptr; | |
204 | } | |
205 | ||
206 | bool dma_free_from_pool(void *start, size_t size) | |
207 | { | |
208 | if (!dma_in_atomic_pool(start, size)) | |
209 | return false; | |
210 | gen_pool_free(atomic_pool, (unsigned long)start, size); | |
211 | return true; | |
212 | } | |
213 | ||
214 | void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, | |
215 | gfp_t flags, unsigned long attrs) | |
216 | { | |
217 | struct page *page = NULL; | |
bfd56cd6 | 218 | void *ret; |
0c3b3171 CH |
219 | |
220 | size = PAGE_ALIGN(size); | |
221 | ||
d98849af | 222 | if (!gfpflags_allow_blocking(flags)) { |
0c3b3171 CH |
223 | ret = dma_alloc_from_pool(size, &page, flags); |
224 | if (!ret) | |
225 | return NULL; | |
8270f3a1 | 226 | goto done; |
0c3b3171 CH |
227 | } |
228 | ||
bfd56cd6 CH |
229 | page = __dma_direct_alloc_pages(dev, size, dma_handle, flags, attrs); |
230 | if (!page) | |
0c3b3171 | 231 | return NULL; |
0c3b3171 CH |
232 | |
233 | /* remove any dirty cache lines on the kernel alias */ | |
234 | arch_dma_prep_coherent(page, size); | |
235 | ||
236 | /* create a coherent mapping */ | |
51231740 | 237 | ret = dma_common_contiguous_remap(page, size, |
33dcb37c | 238 | dma_pgprot(dev, PAGE_KERNEL, attrs), |
0c3b3171 | 239 | __builtin_return_address(0)); |
a1da439c | 240 | if (!ret) { |
bfd56cd6 | 241 | __dma_direct_free_pages(dev, size, page); |
a1da439c MS |
242 | return ret; |
243 | } | |
244 | ||
a1da439c | 245 | memset(ret, 0, size); |
8270f3a1 CH |
246 | done: |
247 | *dma_handle = phys_to_dma(dev, page_to_phys(page)); | |
0c3b3171 CH |
248 | return ret; |
249 | } | |
250 | ||
251 | void arch_dma_free(struct device *dev, size_t size, void *vaddr, | |
252 | dma_addr_t dma_handle, unsigned long attrs) | |
253 | { | |
d98849af | 254 | if (!dma_free_from_pool(vaddr, PAGE_ALIGN(size))) { |
bfd56cd6 CH |
255 | phys_addr_t phys = dma_to_phys(dev, dma_handle); |
256 | struct page *page = pfn_to_page(__phys_to_pfn(phys)); | |
0c3b3171 CH |
257 | |
258 | vunmap(vaddr); | |
bfd56cd6 | 259 | __dma_direct_free_pages(dev, size, page); |
0c3b3171 CH |
260 | } |
261 | } | |
262 | ||
263 | long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr, | |
264 | dma_addr_t dma_addr) | |
265 | { | |
266 | return __phys_to_pfn(dma_to_phys(dev, dma_addr)); | |
267 | } | |
268 | #endif /* CONFIG_DMA_DIRECT_REMAP */ |