projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
dma-mapping: fix ia64 build, use PHYS_PFN
[linux-2.6-block.git] / include / linux / dma-mapping.h
CommitLineData
96532bab
RD		 1#ifndef _LINUX_DMA_MAPPING_H
2#define _LINUX_DMA_MAPPING_H
1da177e4 3
002edb6f 4#include <linux/sizes.h>
842fa69f 5#include <linux/string.h>
1da177e4
LT		 6#include <linux/device.h>
7#include <linux/err.h>
e1c7e324 8#include <linux/dma-debug.h>
b7f080cf 9#include <linux/dma-direction.h>
f0402a26 10#include <linux/scatterlist.h>
e1c7e324
CH		 11#include <linux/kmemcheck.h>
12#include <linux/bug.h>
1da177e4 13
00085f1e
KK		 14/**
15 * List of possible attributes associated with a DMA mapping. The semantics
16 * of each attribute should be defined in Documentation/DMA-attributes.txt.
17 *
18 * DMA_ATTR_WRITE_BARRIER: DMA to a memory region with this attribute
19 * forces all pending DMA writes to complete.
20 */
21#define DMA_ATTR_WRITE_BARRIER (1UL << 0)
22/*
23 * DMA_ATTR_WEAK_ORDERING: Specifies that reads and writes to the mapping
24 * may be weakly ordered, that is that reads and writes may pass each other.
25 */
26#define DMA_ATTR_WEAK_ORDERING (1UL << 1)
27/*
28 * DMA_ATTR_WRITE_COMBINE: Specifies that writes to the mapping may be
29 * buffered to improve performance.
30 */
31#define DMA_ATTR_WRITE_COMBINE (1UL << 2)
32/*
33 * DMA_ATTR_NON_CONSISTENT: Lets the platform to choose to return either
34 * consistent or non-consistent memory as it sees fit.
35 */
36#define DMA_ATTR_NON_CONSISTENT (1UL << 3)
37/*
38 * DMA_ATTR_NO_KERNEL_MAPPING: Lets the platform to avoid creating a kernel
39 * virtual mapping for the allocated buffer.
40 */
41#define DMA_ATTR_NO_KERNEL_MAPPING (1UL << 4)
42/*
43 * DMA_ATTR_SKIP_CPU_SYNC: Allows platform code to skip synchronization of
44 * the CPU cache for the given buffer assuming that it has been already
45 * transferred to 'device' domain.
46 */
47#define DMA_ATTR_SKIP_CPU_SYNC (1UL << 5)
48/*
49 * DMA_ATTR_FORCE_CONTIGUOUS: Forces contiguous allocation of the buffer
50 * in physical memory.
51 */
52#define DMA_ATTR_FORCE_CONTIGUOUS (1UL << 6)
53/*
54 * DMA_ATTR_ALLOC_SINGLE_PAGES: This is a hint to the DMA-mapping subsystem
55 * that it's probably not worth the time to try to allocate memory to in a way
56 * that gives better TLB efficiency.
57 */
58#define DMA_ATTR_ALLOC_SINGLE_PAGES (1UL << 7)
59
77f2ea2f
BH		 60/*
61 * A dma_addr_t can hold any valid DMA or bus address for the platform.
62 * It can be given to a device to use as a DMA source or target. A CPU cannot
63 * reference a dma_addr_t directly because there may be translation between
64 * its physical address space and the bus address space.
65 */
f0402a26 66struct dma_map_ops {
613c4578
MS		 67 void* (*alloc)(struct device *dev, size_t size,
68 dma_addr_t *dma_handle, gfp_t gfp,
00085f1e 69 unsigned long attrs);
613c4578
MS		 70 void (*free)(struct device *dev, size_t size,
71 void *vaddr, dma_addr_t dma_handle,
00085f1e 72 unsigned long attrs);
9adc5374 73 int (*mmap)(struct device *, struct vm_area_struct *,
00085f1e
KK		 74 void *, dma_addr_t, size_t,
75 unsigned long attrs);
9adc5374 76
d2b7428e 77 int (*get_sgtable)(struct device *dev, struct sg_table *sgt, void *,
00085f1e 78 dma_addr_t, size_t, unsigned long attrs);
d2b7428e 79
f0402a26
FT		 80 dma_addr_t (*map_page)(struct device *dev, struct page *page,
81 unsigned long offset, size_t size,
82 enum dma_data_direction dir,
00085f1e 83 unsigned long attrs);
f0402a26
FT		 84 void (*unmap_page)(struct device *dev, dma_addr_t dma_handle,
85 size_t size, enum dma_data_direction dir,
00085f1e 86 unsigned long attrs);
04abab69
RRD		 87 /*
88 * map_sg returns 0 on error and a value > 0 on success.
89 * It should never return a value < 0.
90 */
f0402a26
FT		 91 int (*map_sg)(struct device *dev, struct scatterlist *sg,
92 int nents, enum dma_data_direction dir,
00085f1e 93 unsigned long attrs);
f0402a26
FT		 94 void (*unmap_sg)(struct device *dev,
95 struct scatterlist *sg, int nents,
96 enum dma_data_direction dir,
00085f1e 97 unsigned long attrs);
ba409b31
NS		 98 dma_addr_t (*map_resource)(struct device *dev, phys_addr_t phys_addr,
99 size_t size, enum dma_data_direction dir,
100 unsigned long attrs);
101 void (*unmap_resource)(struct device *dev, dma_addr_t dma_handle,
102 size_t size, enum dma_data_direction dir,
103 unsigned long attrs);
f0402a26
FT		 104 void (*sync_single_for_cpu)(struct device *dev,
105 dma_addr_t dma_handle, size_t size,
106 enum dma_data_direction dir);
107 void (*sync_single_for_device)(struct device *dev,
108 dma_addr_t dma_handle, size_t size,
109 enum dma_data_direction dir);
f0402a26
FT		 110 void (*sync_sg_for_cpu)(struct device *dev,
111 struct scatterlist *sg, int nents,
112 enum dma_data_direction dir);
113 void (*sync_sg_for_device)(struct device *dev,
114 struct scatterlist *sg, int nents,
115 enum dma_data_direction dir);
116 int (*mapping_error)(struct device *dev, dma_addr_t dma_addr);
117 int (*dma_supported)(struct device *dev, u64 mask);
f726f30e 118 int (*set_dma_mask)(struct device *dev, u64 mask);
3a8f7558
MM		 119#ifdef ARCH_HAS_DMA_GET_REQUIRED_MASK
120 u64 (*get_required_mask)(struct device *dev);
121#endif
f0402a26
FT		 122 int is_phys;
123};
124
a8463d4b
CB		 125extern struct dma_map_ops dma_noop_ops;
126
8f286c33 127#define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL<<(n))-1))
34c65384 128
32e8f702
JB		 129#define DMA_MASK_NONE 0x0ULL
130
d6bd3a39
REB		 131static inline int valid_dma_direction(int dma_direction)
132{
133 return ((dma_direction == DMA_BIDIRECTIONAL) ||
134 (dma_direction == DMA_TO_DEVICE) ||
135 (dma_direction == DMA_FROM_DEVICE));
136}
137
32e8f702
JB		 138static inline int is_device_dma_capable(struct device *dev)
139{
140 return dev->dma_mask != NULL && *dev->dma_mask != DMA_MASK_NONE;
141}
142
20d666e4
CH		 143#ifdef CONFIG_HAVE_GENERIC_DMA_COHERENT
144/*
145 * These three functions are only for dma allocator.
146 * Don't use them in device drivers.
147 */
148int dma_alloc_from_coherent(struct device *dev, ssize_t size,
149 dma_addr_t *dma_handle, void **ret);
150int dma_release_from_coherent(struct device *dev, int order, void *vaddr);
151
152int dma_mmap_from_coherent(struct device *dev, struct vm_area_struct *vma,
153 void *cpu_addr, size_t size, int *ret);
154#else
155#define dma_alloc_from_coherent(dev, size, handle, ret) (0)
156#define dma_release_from_coherent(dev, order, vaddr) (0)
157#define dma_mmap_from_coherent(dev, vma, vaddr, order, ret) (0)
158#endif /* CONFIG_HAVE_GENERIC_DMA_COHERENT */
159
1b0fac45 160#ifdef CONFIG_HAS_DMA
1da177e4 161#include <asm/dma-mapping.h>
1b0fac45 162#else
e1c7e324
CH		 163/*
164 * Define the dma api to allow compilation but not linking of
165 * dma dependent code. Code that depends on the dma-mapping
166 * API needs to set 'depends on HAS_DMA' in its Kconfig
167 */
168extern struct dma_map_ops bad_dma_ops;
169static inline struct dma_map_ops *get_dma_ops(struct device *dev)
170{
171 return &bad_dma_ops;
172}
173#endif
174
175static inline dma_addr_t dma_map_single_attrs(struct device *dev, void *ptr,
176 size_t size,
177 enum dma_data_direction dir,
00085f1e 178 unsigned long attrs)
e1c7e324
CH		 179{
180 struct dma_map_ops *ops = get_dma_ops(dev);
181 dma_addr_t addr;
182
183 kmemcheck_mark_initialized(ptr, size);
184 BUG_ON(!valid_dma_direction(dir));
185 addr = ops->map_page(dev, virt_to_page(ptr),
8e99469a 186 offset_in_page(ptr), size,
e1c7e324
CH		 187 dir, attrs);
188 debug_dma_map_page(dev, virt_to_page(ptr),
8e99469a 189 offset_in_page(ptr), size,
e1c7e324
CH		 190 dir, addr, true);
191 return addr;
192}
193
194static inline void dma_unmap_single_attrs(struct device *dev, dma_addr_t addr,
195 size_t size,
196 enum dma_data_direction dir,
00085f1e 197 unsigned long attrs)
e1c7e324
CH		 198{
199 struct dma_map_ops *ops = get_dma_ops(dev);
200
201 BUG_ON(!valid_dma_direction(dir));
202 if (ops->unmap_page)
203 ops->unmap_page(dev, addr, size, dir, attrs);
204 debug_dma_unmap_page(dev, addr, size, dir, true);
205}
206
207/*
208 * dma_maps_sg_attrs returns 0 on error and > 0 on success.
209 * It should never return a value < 0.
210 */
211static inline int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
212 int nents, enum dma_data_direction dir,
00085f1e 213 unsigned long attrs)
e1c7e324
CH		 214{
215 struct dma_map_ops *ops = get_dma_ops(dev);
216 int i, ents;
217 struct scatterlist *s;
218
219 for_each_sg(sg, s, nents, i)
220 kmemcheck_mark_initialized(sg_virt(s), s->length);
221 BUG_ON(!valid_dma_direction(dir));
222 ents = ops->map_sg(dev, sg, nents, dir, attrs);
223 BUG_ON(ents < 0);
224 debug_dma_map_sg(dev, sg, nents, ents, dir);
225
226 return ents;
227}
228
229static inline void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
230 int nents, enum dma_data_direction dir,
00085f1e 231 unsigned long attrs)
e1c7e324
CH		 232{
233 struct dma_map_ops *ops = get_dma_ops(dev);
234
235 BUG_ON(!valid_dma_direction(dir));
236 debug_dma_unmap_sg(dev, sg, nents, dir);
237 if (ops->unmap_sg)
238 ops->unmap_sg(dev, sg, nents, dir, attrs);
239}
240
241static inline dma_addr_t dma_map_page(struct device *dev, struct page *page,
242 size_t offset, size_t size,
243 enum dma_data_direction dir)
244{
245 struct dma_map_ops *ops = get_dma_ops(dev);
246 dma_addr_t addr;
247
248 kmemcheck_mark_initialized(page_address(page) + offset, size);
249 BUG_ON(!valid_dma_direction(dir));
00085f1e 250 addr = ops->map_page(dev, page, offset, size, dir, 0);
e1c7e324
CH		 251 debug_dma_map_page(dev, page, offset, size, dir, addr, false);
252
253 return addr;
254}
255
256static inline void dma_unmap_page(struct device *dev, dma_addr_t addr,
257 size_t size, enum dma_data_direction dir)
258{
259 struct dma_map_ops *ops = get_dma_ops(dev);
260
261 BUG_ON(!valid_dma_direction(dir));
262 if (ops->unmap_page)
00085f1e 263 ops->unmap_page(dev, addr, size, dir, 0);
e1c7e324
CH		 264 debug_dma_unmap_page(dev, addr, size, dir, false);
265}
266
6f3d8796
NS		 267static inline dma_addr_t dma_map_resource(struct device *dev,
268 phys_addr_t phys_addr,
269 size_t size,
270 enum dma_data_direction dir,
271 unsigned long attrs)
272{
273 struct dma_map_ops *ops = get_dma_ops(dev);
2895e1f8 274 unsigned long pfn = PHYS_PFN(phys_addr);
6f3d8796
NS		 275 dma_addr_t addr;
276
277 BUG_ON(!valid_dma_direction(dir));
278
279 /* Don't allow RAM to be mapped */
280 BUG_ON(pfn_valid(pfn));
281
282 addr = phys_addr;
283 if (ops->map_resource)
284 addr = ops->map_resource(dev, phys_addr, size, dir, attrs);
285
286 debug_dma_map_resource(dev, phys_addr, size, dir, addr);
287
288 return addr;
289}
290
291static inline void dma_unmap_resource(struct device *dev, dma_addr_t addr,
292 size_t size, enum dma_data_direction dir,
293 unsigned long attrs)
294{
295 struct dma_map_ops *ops = get_dma_ops(dev);
296
297 BUG_ON(!valid_dma_direction(dir));
298 if (ops->unmap_resource)
299 ops->unmap_resource(dev, addr, size, dir, attrs);
300 debug_dma_unmap_resource(dev, addr, size, dir);
301}
302
e1c7e324
CH		 303static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
304 size_t size,
305 enum dma_data_direction dir)
306{
307 struct dma_map_ops *ops = get_dma_ops(dev);
308
309 BUG_ON(!valid_dma_direction(dir));
310 if (ops->sync_single_for_cpu)
311 ops->sync_single_for_cpu(dev, addr, size, dir);
312 debug_dma_sync_single_for_cpu(dev, addr, size, dir);
313}
314
315static inline void dma_sync_single_for_device(struct device *dev,
316 dma_addr_t addr, size_t size,
317 enum dma_data_direction dir)
318{
319 struct dma_map_ops *ops = get_dma_ops(dev);
320
321 BUG_ON(!valid_dma_direction(dir));
322 if (ops->sync_single_for_device)
323 ops->sync_single_for_device(dev, addr, size, dir);
324 debug_dma_sync_single_for_device(dev, addr, size, dir);
325}
326
327static inline void dma_sync_single_range_for_cpu(struct device *dev,
328 dma_addr_t addr,
329 unsigned long offset,
330 size_t size,
331 enum dma_data_direction dir)
332{
333 const struct dma_map_ops *ops = get_dma_ops(dev);
334
335 BUG_ON(!valid_dma_direction(dir));
336 if (ops->sync_single_for_cpu)
337 ops->sync_single_for_cpu(dev, addr + offset, size, dir);
338 debug_dma_sync_single_range_for_cpu(dev, addr, offset, size, dir);
339}
340
341static inline void dma_sync_single_range_for_device(struct device *dev,
342 dma_addr_t addr,
343 unsigned long offset,
344 size_t size,
345 enum dma_data_direction dir)
346{
347 const struct dma_map_ops *ops = get_dma_ops(dev);
348
349 BUG_ON(!valid_dma_direction(dir));
350 if (ops->sync_single_for_device)
351 ops->sync_single_for_device(dev, addr + offset, size, dir);
352 debug_dma_sync_single_range_for_device(dev, addr, offset, size, dir);
353}
354
355static inline void
356dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
357 int nelems, enum dma_data_direction dir)
358{
359 struct dma_map_ops *ops = get_dma_ops(dev);
360
361 BUG_ON(!valid_dma_direction(dir));
362 if (ops->sync_sg_for_cpu)
363 ops->sync_sg_for_cpu(dev, sg, nelems, dir);
364 debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir);
365}
366
367static inline void
368dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
369 int nelems, enum dma_data_direction dir)
370{
371 struct dma_map_ops *ops = get_dma_ops(dev);
372
373 BUG_ON(!valid_dma_direction(dir));
374 if (ops->sync_sg_for_device)
375 ops->sync_sg_for_device(dev, sg, nelems, dir);
376 debug_dma_sync_sg_for_device(dev, sg, nelems, dir);
377
378}
379
00085f1e
KK		 380#define dma_map_single(d, a, s, r) dma_map_single_attrs(d, a, s, r, 0)
381#define dma_unmap_single(d, a, s, r) dma_unmap_single_attrs(d, a, s, r, 0)
382#define dma_map_sg(d, s, n, r) dma_map_sg_attrs(d, s, n, r, 0)
383#define dma_unmap_sg(d, s, n, r) dma_unmap_sg_attrs(d, s, n, r, 0)
e1c7e324
CH		 384
385extern int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
386 void *cpu_addr, dma_addr_t dma_addr, size_t size);
387
388void *dma_common_contiguous_remap(struct page *page, size_t size,
389 unsigned long vm_flags,
390 pgprot_t prot, const void *caller);
391
392void *dma_common_pages_remap(struct page **pages, size_t size,
393 unsigned long vm_flags, pgprot_t prot,
394 const void *caller);
395void dma_common_free_remap(void *cpu_addr, size_t size, unsigned long vm_flags);
396
397/**
398 * dma_mmap_attrs - map a coherent DMA allocation into user space
399 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
400 * @vma: vm_area_struct describing requested user mapping
401 * @cpu_addr: kernel CPU-view address returned from dma_alloc_attrs
402 * @handle: device-view address returned from dma_alloc_attrs
403 * @size: size of memory originally requested in dma_alloc_attrs
404 * @attrs: attributes of mapping properties requested in dma_alloc_attrs
405 *
406 * Map a coherent DMA buffer previously allocated by dma_alloc_attrs
407 * into user space. The coherent DMA buffer must not be freed by the
408 * driver until the user space mapping has been released.
409 */
410static inline int
411dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma, void *cpu_addr,
00085f1e 412 dma_addr_t dma_addr, size_t size, unsigned long attrs)
e1c7e324
CH		 413{
414 struct dma_map_ops *ops = get_dma_ops(dev);
415 BUG_ON(!ops);
416 if (ops->mmap)
417 return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
418 return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
419}
420
00085f1e 421#define dma_mmap_coherent(d, v, c, h, s) dma_mmap_attrs(d, v, c, h, s, 0)
e1c7e324
CH		 422
423int
424dma_common_get_sgtable(struct device *dev, struct sg_table *sgt,
425 void *cpu_addr, dma_addr_t dma_addr, size_t size);
426
427static inline int
428dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, void *cpu_addr,
00085f1e
KK		 429 dma_addr_t dma_addr, size_t size,
430 unsigned long attrs)
e1c7e324
CH		 431{
432 struct dma_map_ops *ops = get_dma_ops(dev);
433 BUG_ON(!ops);
434 if (ops->get_sgtable)
435 return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size,
436 attrs);
437 return dma_common_get_sgtable(dev, sgt, cpu_addr, dma_addr, size);
438}
439
00085f1e 440#define dma_get_sgtable(d, t, v, h, s) dma_get_sgtable_attrs(d, t, v, h, s, 0)
e1c7e324
CH		 441
442#ifndef arch_dma_alloc_attrs
443#define arch_dma_alloc_attrs(dev, flag) (true)
444#endif
445
446static inline void *dma_alloc_attrs(struct device *dev, size_t size,
447 dma_addr_t *dma_handle, gfp_t flag,
00085f1e 448 unsigned long attrs)
e1c7e324
CH		 449{
450 struct dma_map_ops *ops = get_dma_ops(dev);
451 void *cpu_addr;
452
453 BUG_ON(!ops);
454
455 if (dma_alloc_from_coherent(dev, size, dma_handle, &cpu_addr))
456 return cpu_addr;
457
458 if (!arch_dma_alloc_attrs(&dev, &flag))
459 return NULL;
460 if (!ops->alloc)
461 return NULL;
462
463 cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs);
464 debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr);
465 return cpu_addr;
466}
467
468static inline void dma_free_attrs(struct device *dev, size_t size,
469 void *cpu_addr, dma_addr_t dma_handle,
00085f1e 470 unsigned long attrs)
e1c7e324
CH		 471{
472 struct dma_map_ops *ops = get_dma_ops(dev);
473
474 BUG_ON(!ops);
475 WARN_ON(irqs_disabled());
476
477 if (dma_release_from_coherent(dev, get_order(size), cpu_addr))
478 return;
479
d6b7eaeb 480 if (!ops->free || !cpu_addr)
e1c7e324
CH		 481 return;
482
483 debug_dma_free_coherent(dev, size, cpu_addr, dma_handle);
484 ops->free(dev, size, cpu_addr, dma_handle, attrs);
485}
486
487static inline void *dma_alloc_coherent(struct device *dev, size_t size,
488 dma_addr_t *dma_handle, gfp_t flag)
489{
00085f1e 490 return dma_alloc_attrs(dev, size, dma_handle, flag, 0);
e1c7e324
CH		 491}
492
493static inline void dma_free_coherent(struct device *dev, size_t size,
494 void *cpu_addr, dma_addr_t dma_handle)
495{
00085f1e 496 return dma_free_attrs(dev, size, cpu_addr, dma_handle, 0);
e1c7e324
CH		 497}
498
499static inline void *dma_alloc_noncoherent(struct device *dev, size_t size,
500 dma_addr_t *dma_handle, gfp_t gfp)
501{
00085f1e
KK		 502 return dma_alloc_attrs(dev, size, dma_handle, gfp,
503 DMA_ATTR_NON_CONSISTENT);
e1c7e324
CH		 504}
505
506static inline void dma_free_noncoherent(struct device *dev, size_t size,
507 void *cpu_addr, dma_addr_t dma_handle)
508{
00085f1e
KK		 509 dma_free_attrs(dev, size, cpu_addr, dma_handle,
510 DMA_ATTR_NON_CONSISTENT);
e1c7e324
CH		 511}
512
513static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
514{
515 debug_dma_mapping_error(dev, dma_addr);
516
517 if (get_dma_ops(dev)->mapping_error)
518 return get_dma_ops(dev)->mapping_error(dev, dma_addr);
519
520#ifdef DMA_ERROR_CODE
521 return dma_addr == DMA_ERROR_CODE;
522#else
523 return 0;
524#endif
525}
526
527#ifndef HAVE_ARCH_DMA_SUPPORTED
528static inline int dma_supported(struct device *dev, u64 mask)
529{
530 struct dma_map_ops *ops = get_dma_ops(dev);
531
532 if (!ops)
533 return 0;
534 if (!ops->dma_supported)
535 return 1;
536 return ops->dma_supported(dev, mask);
537}
538#endif
539
540#ifndef HAVE_ARCH_DMA_SET_MASK
541static inline int dma_set_mask(struct device *dev, u64 mask)
542{
543 struct dma_map_ops *ops = get_dma_ops(dev);
544
545 if (ops->set_dma_mask)
546 return ops->set_dma_mask(dev, mask);
547
548 if (!dev->dma_mask || !dma_supported(dev, mask))
549 return -EIO;
550 *dev->dma_mask = mask;
551 return 0;
552}
1b0fac45 553#endif
1da177e4 554
589fc9a6
FT		 555static inline u64 dma_get_mask(struct device *dev)
556{
07a2c01a 557 if (dev && dev->dma_mask && *dev->dma_mask)
589fc9a6 558 return *dev->dma_mask;
284901a9 559 return DMA_BIT_MASK(32);
589fc9a6
FT		 560}
561
58af4a24 562#ifdef CONFIG_ARCH_HAS_DMA_SET_COHERENT_MASK
710224fa
FT		 563int dma_set_coherent_mask(struct device *dev, u64 mask);
564#else
6a1961f4
FT		 565static inline int dma_set_coherent_mask(struct device *dev, u64 mask)
566{
567 if (!dma_supported(dev, mask))
568 return -EIO;
569 dev->coherent_dma_mask = mask;
570 return 0;
571}
710224fa 572#endif
6a1961f4 573
4aa806b7
RK		 574/*
575 * Set both the DMA mask and the coherent DMA mask to the same thing.
576 * Note that we don't check the return value from dma_set_coherent_mask()
577 * as the DMA API guarantees that the coherent DMA mask can be set to
578 * the same or smaller than the streaming DMA mask.
579 */
580static inline int dma_set_mask_and_coherent(struct device *dev, u64 mask)
581{
582 int rc = dma_set_mask(dev, mask);
583 if (rc == 0)
584 dma_set_coherent_mask(dev, mask);
585 return rc;
586}
587
fa6a8d6d
RK		 588/*
589 * Similar to the above, except it deals with the case where the device
590 * does not have dev->dma_mask appropriately setup.
591 */
592static inline int dma_coerce_mask_and_coherent(struct device *dev, u64 mask)
593{
594 dev->dma_mask = &dev->coherent_dma_mask;
595 return dma_set_mask_and_coherent(dev, mask);
596}
597
1da177e4
LT		 598extern u64 dma_get_required_mask(struct device *dev);
599
a3a60f81 600#ifndef arch_setup_dma_ops
97890ba9 601static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base,
53c92d79 602 u64 size, const struct iommu_ops *iommu,
97890ba9
WD		 603 bool coherent) { }
604#endif
605
606#ifndef arch_teardown_dma_ops
607static inline void arch_teardown_dma_ops(struct device *dev) { }
591c1ee4
SS		 608#endif
609
6b7b6510
FT		 610static inline unsigned int dma_get_max_seg_size(struct device *dev)
611{
002edb6f
RM		 612 if (dev->dma_parms && dev->dma_parms->max_segment_size)
613 return dev->dma_parms->max_segment_size;
614 return SZ_64K;
6b7b6510
FT		 615}
616
617static inline unsigned int dma_set_max_seg_size(struct device *dev,
618 unsigned int size)
619{
620 if (dev->dma_parms) {
621 dev->dma_parms->max_segment_size = size;
622 return 0;
002edb6f
RM		 623 }
624 return -EIO;
6b7b6510
FT		 625}
626
d22a6966
FT		 627static inline unsigned long dma_get_seg_boundary(struct device *dev)
628{
002edb6f
RM		 629 if (dev->dma_parms && dev->dma_parms->segment_boundary_mask)
630 return dev->dma_parms->segment_boundary_mask;
631 return DMA_BIT_MASK(32);
d22a6966
FT		 632}
633
634static inline int dma_set_seg_boundary(struct device *dev, unsigned long mask)
635{
636 if (dev->dma_parms) {
637 dev->dma_parms->segment_boundary_mask = mask;
638 return 0;
002edb6f
RM		 639 }
640 return -EIO;
d22a6966
FT		 641}
642
00c8f162
SS		 643#ifndef dma_max_pfn
644static inline unsigned long dma_max_pfn(struct device *dev)
645{
646 return *dev->dma_mask >> PAGE_SHIFT;
647}
648#endif
649
842fa69f
AM		 650static inline void *dma_zalloc_coherent(struct device *dev, size_t size,
651 dma_addr_t *dma_handle, gfp_t flag)
652{
ede23fa8
JP		 653 void *ret = dma_alloc_coherent(dev, size, dma_handle,
654 flag | __GFP_ZERO);
842fa69f
AM		 655 return ret;
656}
657
e259f191 658#ifdef CONFIG_HAS_DMA
4565f017
FT		 659static inline int dma_get_cache_alignment(void)
660{
661#ifdef ARCH_DMA_MINALIGN
662 return ARCH_DMA_MINALIGN;
663#endif
664 return 1;
665}
e259f191 666#endif
4565f017 667
1da177e4
LT		 668/* flags for the coherent memory api */
669#define DMA_MEMORY_MAP 0x01
670#define DMA_MEMORY_IO 0x02
671#define DMA_MEMORY_INCLUDES_CHILDREN 0x04
672#define DMA_MEMORY_EXCLUSIVE 0x08
673
20d666e4
CH		 674#ifdef CONFIG_HAVE_GENERIC_DMA_COHERENT
675int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
676 dma_addr_t device_addr, size_t size, int flags);
677void dma_release_declared_memory(struct device *dev);
678void *dma_mark_declared_memory_occupied(struct device *dev,
679 dma_addr_t device_addr, size_t size);
680#else
1da177e4 681static inline int
88a984ba 682dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
1da177e4
LT		 683 dma_addr_t device_addr, size_t size, int flags)
684{
685 return 0;
686}
687
688static inline void
689dma_release_declared_memory(struct device *dev)
690{
691}
692
693static inline void *
694dma_mark_declared_memory_occupied(struct device *dev,
695 dma_addr_t device_addr, size_t size)
696{
697 return ERR_PTR(-EBUSY);
698}
20d666e4 699#endif /* CONFIG_HAVE_GENERIC_DMA_COHERENT */
1da177e4 700
9ac7849e
TH		 701/*
702 * Managed DMA API
703 */
704extern void *dmam_alloc_coherent(struct device *dev, size_t size,
705 dma_addr_t *dma_handle, gfp_t gfp);
706extern void dmam_free_coherent(struct device *dev, size_t size, void *vaddr,
707 dma_addr_t dma_handle);
708extern void *dmam_alloc_noncoherent(struct device *dev, size_t size,
709 dma_addr_t *dma_handle, gfp_t gfp);
710extern void dmam_free_noncoherent(struct device *dev, size_t size, void *vaddr,
711 dma_addr_t dma_handle);
20d666e4 712#ifdef CONFIG_HAVE_GENERIC_DMA_COHERENT
88a984ba
BH		 713extern int dmam_declare_coherent_memory(struct device *dev,
714 phys_addr_t phys_addr,
9ac7849e
TH		 715 dma_addr_t device_addr, size_t size,
716 int flags);
717extern void dmam_release_declared_memory(struct device *dev);
20d666e4 718#else /* CONFIG_HAVE_GENERIC_DMA_COHERENT */
9ac7849e 719static inline int dmam_declare_coherent_memory(struct device *dev,
88a984ba 720 phys_addr_t phys_addr, dma_addr_t device_addr,
9ac7849e
TH		 721 size_t size, gfp_t gfp)
722{
723 return 0;
724}
1da177e4 725
9ac7849e
TH		 726static inline void dmam_release_declared_memory(struct device *dev)
727{
728}
20d666e4 729#endif /* CONFIG_HAVE_GENERIC_DMA_COHERENT */
1da177e4 730
f6e45661
LR		 731static inline void *dma_alloc_wc(struct device *dev, size_t size,
732 dma_addr_t *dma_addr, gfp_t gfp)
b4bbb107 733{
00085f1e
KK		 734 return dma_alloc_attrs(dev, size, dma_addr, gfp,
735 DMA_ATTR_WRITE_COMBINE);
b4bbb107 736}
f6e45661
LR		 737#ifndef dma_alloc_writecombine
738#define dma_alloc_writecombine dma_alloc_wc
739#endif
b4bbb107 740
f6e45661
LR		 741static inline void dma_free_wc(struct device *dev, size_t size,
742 void *cpu_addr, dma_addr_t dma_addr)
b4bbb107 743{
00085f1e
KK		 744 return dma_free_attrs(dev, size, cpu_addr, dma_addr,
745 DMA_ATTR_WRITE_COMBINE);
b4bbb107 746}
f6e45661
LR		 747#ifndef dma_free_writecombine
748#define dma_free_writecombine dma_free_wc
749#endif
b4bbb107 750
f6e45661
LR		 751static inline int dma_mmap_wc(struct device *dev,
752 struct vm_area_struct *vma,
753 void *cpu_addr, dma_addr_t dma_addr,
754 size_t size)
b4bbb107 755{
00085f1e
KK		 756 return dma_mmap_attrs(dev, vma, cpu_addr, dma_addr, size,
757 DMA_ATTR_WRITE_COMBINE);
b4bbb107 758}
f6e45661
LR		 759#ifndef dma_mmap_writecombine
760#define dma_mmap_writecombine dma_mmap_wc
761#endif
74bc7cee 762
0acedc12
FT		 763#ifdef CONFIG_NEED_DMA_MAP_STATE
764#define DEFINE_DMA_UNMAP_ADDR(ADDR_NAME) dma_addr_t ADDR_NAME
765#define DEFINE_DMA_UNMAP_LEN(LEN_NAME) __u32 LEN_NAME
766#define dma_unmap_addr(PTR, ADDR_NAME) ((PTR)->ADDR_NAME)
767#define dma_unmap_addr_set(PTR, ADDR_NAME, VAL) (((PTR)->ADDR_NAME) = (VAL))
768#define dma_unmap_len(PTR, LEN_NAME) ((PTR)->LEN_NAME)
769#define dma_unmap_len_set(PTR, LEN_NAME, VAL) (((PTR)->LEN_NAME) = (VAL))
770#else
771#define DEFINE_DMA_UNMAP_ADDR(ADDR_NAME)
772#define DEFINE_DMA_UNMAP_LEN(LEN_NAME)
773#define dma_unmap_addr(PTR, ADDR_NAME) (0)
774#define dma_unmap_addr_set(PTR, ADDR_NAME, VAL) do { } while (0)
775#define dma_unmap_len(PTR, LEN_NAME) (0)
776#define dma_unmap_len_set(PTR, LEN_NAME, VAL) do { } while (0)
777#endif
778
9ac7849e 779#endif
Linux 6.x block layer and io_uring tree(s)