Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
a8463d4b | 2 | /* |
bc3ec75d CH |
3 | * Copyright (C) 2018 Christoph Hellwig. |
4 | * | |
5 | * DMA operations that map physical memory directly without using an IOMMU. | |
a8463d4b | 6 | */ |
57c8a661 | 7 | #include <linux/memblock.h> /* for max_pfn */ |
a8463d4b CB |
8 | #include <linux/export.h> |
9 | #include <linux/mm.h> | |
2e86a047 | 10 | #include <linux/dma-direct.h> |
a8463d4b | 11 | #include <linux/scatterlist.h> |
080321d3 | 12 | #include <linux/dma-contiguous.h> |
bc3ec75d | 13 | #include <linux/dma-noncoherent.h> |
25f1e188 | 14 | #include <linux/pfn.h> |
3acac065 | 15 | #include <linux/vmalloc.h> |
c10f07aa | 16 | #include <linux/set_memory.h> |
55897af6 | 17 | #include <linux/swiotlb.h> |
a8463d4b | 18 | |
c61e9637 CH |
19 | /* |
20 | * Most architectures use ZONE_DMA for the first 16 Megabytes, but | |
21 | * some use it for entirely different regions: | |
22 | */ | |
23 | #ifndef ARCH_ZONE_DMA_BITS | |
24 | #define ARCH_ZONE_DMA_BITS 24 | |
25 | #endif | |
26 | ||
58dfd4ac | 27 | static void report_addr(struct device *dev, dma_addr_t dma_addr, size_t size) |
27975969 | 28 | { |
58dfd4ac CH |
29 | if (!dev->dma_mask) { |
30 | dev_err_once(dev, "DMA map on device without dma_mask\n"); | |
31 | } else if (*dev->dma_mask >= DMA_BIT_MASK(32) || dev->bus_dma_mask) { | |
32 | dev_err_once(dev, | |
33 | "overflow %pad+%zu of DMA mask %llx bus mask %llx\n", | |
34 | &dma_addr, size, *dev->dma_mask, dev->bus_dma_mask); | |
27975969 | 35 | } |
58dfd4ac | 36 | WARN_ON_ONCE(1); |
27975969 CH |
37 | } |
38 | ||
a20bb058 CH |
39 | static inline dma_addr_t phys_to_dma_direct(struct device *dev, |
40 | phys_addr_t phys) | |
41 | { | |
9087c375 | 42 | if (force_dma_unencrypted(dev)) |
a20bb058 CH |
43 | return __phys_to_dma(dev, phys); |
44 | return phys_to_dma(dev, phys); | |
45 | } | |
46 | ||
34dc0ea6 CH |
47 | static inline struct page *dma_direct_to_page(struct device *dev, |
48 | dma_addr_t dma_addr) | |
49 | { | |
50 | return pfn_to_page(PHYS_PFN(dma_to_phys(dev, dma_addr))); | |
51 | } | |
52 | ||
a20bb058 CH |
53 | u64 dma_direct_get_required_mask(struct device *dev) |
54 | { | |
55 | u64 max_dma = phys_to_dma_direct(dev, (max_pfn - 1) << PAGE_SHIFT); | |
56 | ||
57 | return (1ULL << (fls64(max_dma) - 1)) * 2 - 1; | |
58 | } | |
59 | ||
7d21ee4c CH |
60 | static gfp_t __dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask, |
61 | u64 *phys_mask) | |
62 | { | |
b4ebe606 CH |
63 | if (dev->bus_dma_mask && dev->bus_dma_mask < dma_mask) |
64 | dma_mask = dev->bus_dma_mask; | |
65 | ||
9087c375 | 66 | if (force_dma_unencrypted(dev)) |
7d21ee4c CH |
67 | *phys_mask = __dma_to_phys(dev, dma_mask); |
68 | else | |
69 | *phys_mask = dma_to_phys(dev, dma_mask); | |
70 | ||
79ac32a4 CH |
71 | /* |
72 | * Optimistically try the zone that the physical address mask falls | |
73 | * into first. If that returns memory that isn't actually addressable | |
74 | * we will fallback to the next lower zone and try again. | |
75 | * | |
76 | * Note that GFP_DMA32 and GFP_DMA are no ops without the corresponding | |
77 | * zones. | |
78 | */ | |
7d21ee4c CH |
79 | if (*phys_mask <= DMA_BIT_MASK(ARCH_ZONE_DMA_BITS)) |
80 | return GFP_DMA; | |
81 | if (*phys_mask <= DMA_BIT_MASK(32)) | |
82 | return GFP_DMA32; | |
83 | return 0; | |
84 | } | |
85 | ||
95f18391 CH |
86 | static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size) |
87 | { | |
a20bb058 | 88 | return phys_to_dma_direct(dev, phys) + size - 1 <= |
b4ebe606 | 89 | min_not_zero(dev->coherent_dma_mask, dev->bus_dma_mask); |
95f18391 CH |
90 | } |
91 | ||
b18814e7 | 92 | struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, |
4e1003aa | 93 | gfp_t gfp, unsigned long attrs) |
a8463d4b | 94 | { |
90ae409f CH |
95 | size_t alloc_size = PAGE_ALIGN(size); |
96 | int node = dev_to_node(dev); | |
080321d3 | 97 | struct page *page = NULL; |
7d21ee4c | 98 | u64 phys_mask; |
a8463d4b | 99 | |
b9fd0426 CH |
100 | if (attrs & DMA_ATTR_NO_WARN) |
101 | gfp |= __GFP_NOWARN; | |
102 | ||
e89f5b37 CH |
103 | /* we always manually zero the memory once we are done: */ |
104 | gfp &= ~__GFP_ZERO; | |
7d21ee4c CH |
105 | gfp |= __dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask, |
106 | &phys_mask); | |
90ae409f CH |
107 | page = dma_alloc_contiguous(dev, alloc_size, gfp); |
108 | if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) { | |
109 | dma_free_contiguous(dev, page, alloc_size); | |
110 | page = NULL; | |
111 | } | |
95f18391 | 112 | again: |
90ae409f CH |
113 | if (!page) |
114 | page = alloc_pages_node(node, gfp, get_order(alloc_size)); | |
95f18391 | 115 | if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) { |
b1d2dc00 | 116 | dma_free_contiguous(dev, page, size); |
95f18391 CH |
117 | page = NULL; |
118 | ||
de7eab30 | 119 | if (IS_ENABLED(CONFIG_ZONE_DMA32) && |
7d21ee4c | 120 | phys_mask < DMA_BIT_MASK(64) && |
de7eab30 TI |
121 | !(gfp & (GFP_DMA32 | GFP_DMA))) { |
122 | gfp |= GFP_DMA32; | |
123 | goto again; | |
124 | } | |
125 | ||
fbce251b | 126 | if (IS_ENABLED(CONFIG_ZONE_DMA) && !(gfp & GFP_DMA)) { |
95f18391 CH |
127 | gfp = (gfp & ~GFP_DMA32) | GFP_DMA; |
128 | goto again; | |
129 | } | |
130 | } | |
131 | ||
b18814e7 CH |
132 | return page; |
133 | } | |
134 | ||
135 | void *dma_direct_alloc_pages(struct device *dev, size_t size, | |
136 | dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) | |
137 | { | |
138 | struct page *page; | |
139 | void *ret; | |
140 | ||
3acac065 CH |
141 | if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) && |
142 | dma_alloc_need_uncached(dev, attrs) && | |
143 | !gfpflags_allow_blocking(gfp)) { | |
144 | ret = dma_alloc_from_pool(PAGE_ALIGN(size), &page, gfp); | |
145 | if (!ret) | |
146 | return NULL; | |
147 | goto done; | |
148 | } | |
149 | ||
4e1003aa | 150 | page = __dma_direct_alloc_pages(dev, size, gfp, attrs); |
080321d3 CH |
151 | if (!page) |
152 | return NULL; | |
b18814e7 | 153 | |
cf14be0b CH |
154 | if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) && |
155 | !force_dma_unencrypted(dev)) { | |
d98849af CH |
156 | /* remove any dirty cache lines on the kernel alias */ |
157 | if (!PageHighMem(page)) | |
158 | arch_dma_prep_coherent(page, size); | |
159 | /* return the page pointer as the opaque cookie */ | |
3acac065 CH |
160 | ret = page; |
161 | goto done; | |
162 | } | |
163 | ||
164 | if ((IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) && | |
165 | dma_alloc_need_uncached(dev, attrs)) || | |
166 | (IS_ENABLED(CONFIG_DMA_REMAP) && PageHighMem(page))) { | |
167 | /* remove any dirty cache lines on the kernel alias */ | |
168 | arch_dma_prep_coherent(page, PAGE_ALIGN(size)); | |
169 | ||
170 | /* create a coherent mapping */ | |
171 | ret = dma_common_contiguous_remap(page, PAGE_ALIGN(size), | |
172 | dma_pgprot(dev, PAGE_KERNEL, attrs), | |
173 | __builtin_return_address(0)); | |
174 | if (!ret) { | |
175 | dma_free_contiguous(dev, page, size); | |
176 | return ret; | |
177 | } | |
178 | ||
179 | memset(ret, 0, size); | |
180 | goto done; | |
d98849af CH |
181 | } |
182 | ||
704f2c20 CH |
183 | if (PageHighMem(page)) { |
184 | /* | |
185 | * Depending on the cma= arguments and per-arch setup | |
b1d2dc00 | 186 | * dma_alloc_contiguous could return highmem pages. |
704f2c20 CH |
187 | * Without remapping there is no way to return them here, |
188 | * so log an error and fail. | |
189 | */ | |
190 | dev_info(dev, "Rejecting highmem page from CMA.\n"); | |
acaade1a | 191 | dma_free_contiguous(dev, page, size); |
704f2c20 CH |
192 | return NULL; |
193 | } | |
194 | ||
c10f07aa | 195 | ret = page_address(page); |
3acac065 | 196 | if (force_dma_unencrypted(dev)) |
b18814e7 | 197 | set_memory_decrypted((unsigned long)ret, 1 << get_order(size)); |
3acac065 | 198 | |
c10f07aa | 199 | memset(ret, 0, size); |
c30700db CH |
200 | |
201 | if (IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) && | |
4b85faed | 202 | dma_alloc_need_uncached(dev, attrs)) { |
c30700db CH |
203 | arch_dma_prep_coherent(page, size); |
204 | ret = uncached_kernel_address(ret); | |
205 | } | |
3acac065 CH |
206 | done: |
207 | if (force_dma_unencrypted(dev)) | |
208 | *dma_handle = __phys_to_dma(dev, page_to_phys(page)); | |
209 | else | |
210 | *dma_handle = phys_to_dma(dev, page_to_phys(page)); | |
c10f07aa | 211 | return ret; |
a8463d4b CB |
212 | } |
213 | ||
bc3ec75d | 214 | void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr, |
002e6745 | 215 | dma_addr_t dma_addr, unsigned long attrs) |
a8463d4b | 216 | { |
c10f07aa | 217 | unsigned int page_order = get_order(size); |
080321d3 | 218 | |
cf14be0b CH |
219 | if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) && |
220 | !force_dma_unencrypted(dev)) { | |
d98849af | 221 | /* cpu_addr is a struct page cookie, not a kernel address */ |
acaade1a | 222 | dma_free_contiguous(dev, cpu_addr, size); |
d98849af CH |
223 | return; |
224 | } | |
225 | ||
3acac065 CH |
226 | if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) && |
227 | dma_free_from_pool(cpu_addr, PAGE_ALIGN(size))) | |
228 | return; | |
229 | ||
9087c375 | 230 | if (force_dma_unencrypted(dev)) |
c10f07aa | 231 | set_memory_encrypted((unsigned long)cpu_addr, 1 << page_order); |
c30700db | 232 | |
3acac065 CH |
233 | if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr)) |
234 | vunmap(cpu_addr); | |
235 | ||
236 | dma_free_contiguous(dev, dma_direct_to_page(dev, dma_addr), size); | |
a8463d4b CB |
237 | } |
238 | ||
bc3ec75d CH |
239 | void *dma_direct_alloc(struct device *dev, size_t size, |
240 | dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) | |
241 | { | |
c30700db | 242 | if (!IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) && |
3acac065 | 243 | !IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) && |
c2f2124e | 244 | dma_alloc_need_uncached(dev, attrs)) |
bc3ec75d CH |
245 | return arch_dma_alloc(dev, size, dma_handle, gfp, attrs); |
246 | return dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs); | |
247 | } | |
248 | ||
249 | void dma_direct_free(struct device *dev, size_t size, | |
250 | void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs) | |
251 | { | |
c30700db | 252 | if (!IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) && |
3acac065 | 253 | !IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) && |
c2f2124e | 254 | dma_alloc_need_uncached(dev, attrs)) |
bc3ec75d CH |
255 | arch_dma_free(dev, size, cpu_addr, dma_addr, attrs); |
256 | else | |
257 | dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs); | |
258 | } | |
259 | ||
55897af6 CH |
260 | #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \ |
261 | defined(CONFIG_SWIOTLB) | |
262 | void dma_direct_sync_single_for_device(struct device *dev, | |
bc3ec75d CH |
263 | dma_addr_t addr, size_t size, enum dma_data_direction dir) |
264 | { | |
55897af6 CH |
265 | phys_addr_t paddr = dma_to_phys(dev, addr); |
266 | ||
267 | if (unlikely(is_swiotlb_buffer(paddr))) | |
268 | swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE); | |
269 | ||
270 | if (!dev_is_dma_coherent(dev)) | |
271 | arch_sync_dma_for_device(dev, paddr, size, dir); | |
bc3ec75d | 272 | } |
356da6d0 | 273 | EXPORT_SYMBOL(dma_direct_sync_single_for_device); |
bc3ec75d | 274 | |
55897af6 | 275 | void dma_direct_sync_sg_for_device(struct device *dev, |
bc3ec75d CH |
276 | struct scatterlist *sgl, int nents, enum dma_data_direction dir) |
277 | { | |
278 | struct scatterlist *sg; | |
279 | int i; | |
280 | ||
55897af6 | 281 | for_each_sg(sgl, sg, nents, i) { |
449fa54d FD |
282 | phys_addr_t paddr = dma_to_phys(dev, sg_dma_address(sg)); |
283 | ||
284 | if (unlikely(is_swiotlb_buffer(paddr))) | |
285 | swiotlb_tbl_sync_single(dev, paddr, sg->length, | |
55897af6 | 286 | dir, SYNC_FOR_DEVICE); |
bc3ec75d | 287 | |
55897af6 | 288 | if (!dev_is_dma_coherent(dev)) |
449fa54d | 289 | arch_sync_dma_for_device(dev, paddr, sg->length, |
55897af6 CH |
290 | dir); |
291 | } | |
bc3ec75d | 292 | } |
356da6d0 | 293 | EXPORT_SYMBOL(dma_direct_sync_sg_for_device); |
17ac5247 | 294 | #endif |
bc3ec75d CH |
295 | |
296 | #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \ | |
55897af6 CH |
297 | defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \ |
298 | defined(CONFIG_SWIOTLB) | |
299 | void dma_direct_sync_single_for_cpu(struct device *dev, | |
bc3ec75d CH |
300 | dma_addr_t addr, size_t size, enum dma_data_direction dir) |
301 | { | |
55897af6 CH |
302 | phys_addr_t paddr = dma_to_phys(dev, addr); |
303 | ||
304 | if (!dev_is_dma_coherent(dev)) { | |
305 | arch_sync_dma_for_cpu(dev, paddr, size, dir); | |
306 | arch_sync_dma_for_cpu_all(dev); | |
307 | } | |
308 | ||
309 | if (unlikely(is_swiotlb_buffer(paddr))) | |
310 | swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_CPU); | |
bc3ec75d | 311 | } |
356da6d0 | 312 | EXPORT_SYMBOL(dma_direct_sync_single_for_cpu); |
bc3ec75d | 313 | |
55897af6 | 314 | void dma_direct_sync_sg_for_cpu(struct device *dev, |
bc3ec75d CH |
315 | struct scatterlist *sgl, int nents, enum dma_data_direction dir) |
316 | { | |
317 | struct scatterlist *sg; | |
318 | int i; | |
319 | ||
55897af6 | 320 | for_each_sg(sgl, sg, nents, i) { |
449fa54d FD |
321 | phys_addr_t paddr = dma_to_phys(dev, sg_dma_address(sg)); |
322 | ||
55897af6 | 323 | if (!dev_is_dma_coherent(dev)) |
449fa54d FD |
324 | arch_sync_dma_for_cpu(dev, paddr, sg->length, dir); |
325 | ||
326 | if (unlikely(is_swiotlb_buffer(paddr))) | |
327 | swiotlb_tbl_sync_single(dev, paddr, sg->length, dir, | |
55897af6 CH |
328 | SYNC_FOR_CPU); |
329 | } | |
bc3ec75d | 330 | |
55897af6 CH |
331 | if (!dev_is_dma_coherent(dev)) |
332 | arch_sync_dma_for_cpu_all(dev); | |
bc3ec75d | 333 | } |
356da6d0 | 334 | EXPORT_SYMBOL(dma_direct_sync_sg_for_cpu); |
bc3ec75d | 335 | |
55897af6 | 336 | void dma_direct_unmap_page(struct device *dev, dma_addr_t addr, |
bc3ec75d CH |
337 | size_t size, enum dma_data_direction dir, unsigned long attrs) |
338 | { | |
55897af6 CH |
339 | phys_addr_t phys = dma_to_phys(dev, addr); |
340 | ||
bc3ec75d CH |
341 | if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) |
342 | dma_direct_sync_single_for_cpu(dev, addr, size, dir); | |
55897af6 CH |
343 | |
344 | if (unlikely(is_swiotlb_buffer(phys))) | |
3fc1ca00 | 345 | swiotlb_tbl_unmap_single(dev, phys, size, size, dir, attrs); |
bc3ec75d | 346 | } |
356da6d0 | 347 | EXPORT_SYMBOL(dma_direct_unmap_page); |
bc3ec75d | 348 | |
55897af6 | 349 | void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl, |
bc3ec75d CH |
350 | int nents, enum dma_data_direction dir, unsigned long attrs) |
351 | { | |
55897af6 CH |
352 | struct scatterlist *sg; |
353 | int i; | |
354 | ||
355 | for_each_sg(sgl, sg, nents, i) | |
356 | dma_direct_unmap_page(dev, sg->dma_address, sg_dma_len(sg), dir, | |
357 | attrs); | |
bc3ec75d | 358 | } |
356da6d0 | 359 | EXPORT_SYMBOL(dma_direct_unmap_sg); |
bc3ec75d CH |
360 | #endif |
361 | ||
55897af6 CH |
362 | static inline bool dma_direct_possible(struct device *dev, dma_addr_t dma_addr, |
363 | size_t size) | |
364 | { | |
365 | return swiotlb_force != SWIOTLB_FORCE && | |
d7e02a93 | 366 | dma_capable(dev, dma_addr, size); |
55897af6 CH |
367 | } |
368 | ||
782e6769 | 369 | dma_addr_t dma_direct_map_page(struct device *dev, struct page *page, |
002e6745 CH |
370 | unsigned long offset, size_t size, enum dma_data_direction dir, |
371 | unsigned long attrs) | |
a8463d4b | 372 | { |
bc3ec75d CH |
373 | phys_addr_t phys = page_to_phys(page) + offset; |
374 | dma_addr_t dma_addr = phys_to_dma(dev, phys); | |
27975969 | 375 | |
55897af6 CH |
376 | if (unlikely(!dma_direct_possible(dev, dma_addr, size)) && |
377 | !swiotlb_map(dev, &phys, &dma_addr, size, dir, attrs)) { | |
58dfd4ac | 378 | report_addr(dev, dma_addr, size); |
b0cbeae4 | 379 | return DMA_MAPPING_ERROR; |
58dfd4ac | 380 | } |
bc3ec75d | 381 | |
55897af6 CH |
382 | if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) |
383 | arch_sync_dma_for_device(dev, phys, size, dir); | |
27975969 | 384 | return dma_addr; |
a8463d4b | 385 | } |
356da6d0 | 386 | EXPORT_SYMBOL(dma_direct_map_page); |
a8463d4b | 387 | |
782e6769 CH |
388 | int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents, |
389 | enum dma_data_direction dir, unsigned long attrs) | |
a8463d4b CB |
390 | { |
391 | int i; | |
392 | struct scatterlist *sg; | |
393 | ||
394 | for_each_sg(sgl, sg, nents, i) { | |
17ac5247 CH |
395 | sg->dma_address = dma_direct_map_page(dev, sg_page(sg), |
396 | sg->offset, sg->length, dir, attrs); | |
397 | if (sg->dma_address == DMA_MAPPING_ERROR) | |
55897af6 | 398 | goto out_unmap; |
a8463d4b CB |
399 | sg_dma_len(sg) = sg->length; |
400 | } | |
401 | ||
402 | return nents; | |
55897af6 CH |
403 | |
404 | out_unmap: | |
405 | dma_direct_unmap_sg(dev, sgl, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC); | |
406 | return 0; | |
a8463d4b | 407 | } |
356da6d0 | 408 | EXPORT_SYMBOL(dma_direct_map_sg); |
a8463d4b | 409 | |
cfced786 CH |
410 | dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr, |
411 | size_t size, enum dma_data_direction dir, unsigned long attrs) | |
412 | { | |
413 | dma_addr_t dma_addr = paddr; | |
414 | ||
415 | if (unlikely(!dma_direct_possible(dev, dma_addr, size))) { | |
416 | report_addr(dev, dma_addr, size); | |
417 | return DMA_MAPPING_ERROR; | |
418 | } | |
419 | ||
420 | return dma_addr; | |
421 | } | |
422 | EXPORT_SYMBOL(dma_direct_map_resource); | |
423 | ||
34dc0ea6 CH |
424 | int dma_direct_get_sgtable(struct device *dev, struct sg_table *sgt, |
425 | void *cpu_addr, dma_addr_t dma_addr, size_t size, | |
426 | unsigned long attrs) | |
427 | { | |
428 | struct page *page = dma_direct_to_page(dev, dma_addr); | |
429 | int ret; | |
430 | ||
431 | ret = sg_alloc_table(sgt, 1, GFP_KERNEL); | |
432 | if (!ret) | |
433 | sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); | |
434 | return ret; | |
435 | } | |
436 | ||
437 | #ifdef CONFIG_MMU | |
438 | bool dma_direct_can_mmap(struct device *dev) | |
439 | { | |
440 | return dev_is_dma_coherent(dev) || | |
441 | IS_ENABLED(CONFIG_DMA_NONCOHERENT_MMAP); | |
442 | } | |
443 | ||
444 | int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma, | |
445 | void *cpu_addr, dma_addr_t dma_addr, size_t size, | |
446 | unsigned long attrs) | |
447 | { | |
448 | unsigned long user_count = vma_pages(vma); | |
449 | unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; | |
450 | unsigned long pfn = PHYS_PFN(dma_to_phys(dev, dma_addr)); | |
451 | int ret = -ENXIO; | |
452 | ||
453 | vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs); | |
454 | ||
455 | if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) | |
456 | return ret; | |
457 | ||
458 | if (vma->vm_pgoff >= count || user_count > count - vma->vm_pgoff) | |
459 | return -ENXIO; | |
460 | return remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff, | |
461 | user_count << PAGE_SHIFT, vma->vm_page_prot); | |
462 | } | |
463 | #else /* CONFIG_MMU */ | |
464 | bool dma_direct_can_mmap(struct device *dev) | |
465 | { | |
466 | return false; | |
467 | } | |
468 | ||
469 | int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma, | |
470 | void *cpu_addr, dma_addr_t dma_addr, size_t size, | |
471 | unsigned long attrs) | |
472 | { | |
473 | return -ENXIO; | |
474 | } | |
475 | #endif /* CONFIG_MMU */ | |
476 | ||
9d7a224b CH |
477 | /* |
478 | * Because 32-bit DMA masks are so common we expect every architecture to be | |
479 | * able to satisfy them - either by not supporting more physical memory, or by | |
480 | * providing a ZONE_DMA32. If neither is the case, the architecture needs to | |
481 | * use an IOMMU instead of the direct mapping. | |
482 | */ | |
1a9777a8 CH |
483 | int dma_direct_supported(struct device *dev, u64 mask) |
484 | { | |
9d7a224b CH |
485 | u64 min_mask; |
486 | ||
487 | if (IS_ENABLED(CONFIG_ZONE_DMA)) | |
488 | min_mask = DMA_BIT_MASK(ARCH_ZONE_DMA_BITS); | |
489 | else | |
490 | min_mask = DMA_BIT_MASK(32); | |
491 | ||
492 | min_mask = min_t(u64, min_mask, (max_pfn - 1) << PAGE_SHIFT); | |
493 | ||
c92a54cf LT |
494 | /* |
495 | * This check needs to be against the actual bit mask value, so | |
496 | * use __phys_to_dma() here so that the SME encryption mask isn't | |
497 | * part of the check. | |
498 | */ | |
499 | return mask >= __phys_to_dma(dev, min_mask); | |
1a9777a8 | 500 | } |
133d624b JR |
501 | |
502 | size_t dma_direct_max_mapping_size(struct device *dev) | |
503 | { | |
133d624b | 504 | /* If SWIOTLB is active, use its maximum mapping size */ |
a5008b59 CH |
505 | if (is_swiotlb_active() && |
506 | (dma_addressing_limited(dev) || swiotlb_force == SWIOTLB_FORCE)) | |
507 | return swiotlb_max_mapping_size(dev); | |
508 | return SIZE_MAX; | |
133d624b | 509 | } |