Commit | Line | Data |
---|---|---|
1da177e4 | 1 | /* |
0ddbccd1 | 2 | * linux/arch/arm/mm/dma-mapping.c |
1da177e4 LT |
3 | * |
4 | * Copyright (C) 2000-2004 Russell King | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify | |
7 | * it under the terms of the GNU General Public License version 2 as | |
8 | * published by the Free Software Foundation. | |
9 | * | |
10 | * DMA uncached mapping support. | |
11 | */ | |
12 | #include <linux/module.h> | |
13 | #include <linux/mm.h> | |
5a0e3ad6 | 14 | #include <linux/gfp.h> |
1da177e4 LT |
15 | #include <linux/errno.h> |
16 | #include <linux/list.h> | |
17 | #include <linux/init.h> | |
18 | #include <linux/device.h> | |
19 | #include <linux/dma-mapping.h> | |
c7909509 | 20 | #include <linux/dma-contiguous.h> |
39af22a7 | 21 | #include <linux/highmem.h> |
c7909509 | 22 | #include <linux/memblock.h> |
99d1717d | 23 | #include <linux/slab.h> |
4ce63fcd MS |
24 | #include <linux/iommu.h> |
25 | #include <linux/vmalloc.h> | |
1da177e4 | 26 | |
23759dc6 | 27 | #include <asm/memory.h> |
43377453 | 28 | #include <asm/highmem.h> |
1da177e4 | 29 | #include <asm/cacheflush.h> |
1da177e4 | 30 | #include <asm/tlbflush.h> |
37134cd5 | 31 | #include <asm/sizes.h> |
99d1717d | 32 | #include <asm/mach/arch.h> |
4ce63fcd | 33 | #include <asm/dma-iommu.h> |
c7909509 MS |
34 | #include <asm/mach/map.h> |
35 | #include <asm/system_info.h> | |
36 | #include <asm/dma-contiguous.h> | |
37134cd5 | 37 | |
022ae537 RK |
38 | #include "mm.h" |
39 | ||
15237e1f MS |
40 | /* |
41 | * The DMA API is built upon the notion of "buffer ownership". A buffer | |
42 | * is either exclusively owned by the CPU (and therefore may be accessed | |
43 | * by it) or exclusively owned by the DMA device. These helper functions | |
44 | * represent the transitions between these two ownership states. | |
45 | * | |
46 | * Note, however, that on later ARMs, this notion does not work due to | |
47 | * speculative prefetches. We model our approach on the assumption that | |
48 | * the CPU does do speculative prefetches, which means we clean caches | |
49 | * before transfers and delay cache invalidation until transfer completion. | |
50 | * | |
15237e1f | 51 | */ |
51fde349 | 52 | static void __dma_page_cpu_to_dev(struct page *, unsigned long, |
15237e1f | 53 | size_t, enum dma_data_direction); |
51fde349 | 54 | static void __dma_page_dev_to_cpu(struct page *, unsigned long, |
15237e1f MS |
55 | size_t, enum dma_data_direction); |
56 | ||
2dc6a016 MS |
57 | /** |
58 | * arm_dma_map_page - map a portion of a page for streaming DMA | |
59 | * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices | |
60 | * @page: page that buffer resides in | |
61 | * @offset: offset into page for start of buffer | |
62 | * @size: size of buffer to map | |
63 | * @dir: DMA transfer direction | |
64 | * | |
65 | * Ensure that any data held in the cache is appropriately discarded | |
66 | * or written back. | |
67 | * | |
68 | * The device owns this memory once this call has completed. The CPU | |
69 | * can regain ownership by calling dma_unmap_page(). | |
70 | */ | |
51fde349 | 71 | static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page, |
2dc6a016 MS |
72 | unsigned long offset, size_t size, enum dma_data_direction dir, |
73 | struct dma_attrs *attrs) | |
74 | { | |
51fde349 MS |
75 | if (!arch_is_coherent()) |
76 | __dma_page_cpu_to_dev(page, offset, size, dir); | |
77 | return pfn_to_dma(dev, page_to_pfn(page)) + offset; | |
2dc6a016 MS |
78 | } |
79 | ||
80 | /** | |
81 | * arm_dma_unmap_page - unmap a buffer previously mapped through dma_map_page() | |
82 | * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices | |
83 | * @handle: DMA address of buffer | |
84 | * @size: size of buffer (same as passed to dma_map_page) | |
85 | * @dir: DMA transfer direction (same as passed to dma_map_page) | |
86 | * | |
87 | * Unmap a page streaming mode DMA translation. The handle and size | |
88 | * must match what was provided in the previous dma_map_page() call. | |
89 | * All other usages are undefined. | |
90 | * | |
91 | * After this call, reads by the CPU to the buffer are guaranteed to see | |
92 | * whatever the device wrote there. | |
93 | */ | |
51fde349 | 94 | static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle, |
2dc6a016 MS |
95 | size_t size, enum dma_data_direction dir, |
96 | struct dma_attrs *attrs) | |
97 | { | |
51fde349 MS |
98 | if (!arch_is_coherent()) |
99 | __dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)), | |
100 | handle & ~PAGE_MASK, size, dir); | |
2dc6a016 MS |
101 | } |
102 | ||
51fde349 | 103 | static void arm_dma_sync_single_for_cpu(struct device *dev, |
2dc6a016 MS |
104 | dma_addr_t handle, size_t size, enum dma_data_direction dir) |
105 | { | |
106 | unsigned int offset = handle & (PAGE_SIZE - 1); | |
107 | struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset)); | |
51fde349 MS |
108 | if (!arch_is_coherent()) |
109 | __dma_page_dev_to_cpu(page, offset, size, dir); | |
2dc6a016 MS |
110 | } |
111 | ||
51fde349 | 112 | static void arm_dma_sync_single_for_device(struct device *dev, |
2dc6a016 MS |
113 | dma_addr_t handle, size_t size, enum dma_data_direction dir) |
114 | { | |
115 | unsigned int offset = handle & (PAGE_SIZE - 1); | |
116 | struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset)); | |
51fde349 MS |
117 | if (!arch_is_coherent()) |
118 | __dma_page_cpu_to_dev(page, offset, size, dir); | |
2dc6a016 MS |
119 | } |
120 | ||
121 | static int arm_dma_set_mask(struct device *dev, u64 dma_mask); | |
122 | ||
123 | struct dma_map_ops arm_dma_ops = { | |
f99d6034 MS |
124 | .alloc = arm_dma_alloc, |
125 | .free = arm_dma_free, | |
126 | .mmap = arm_dma_mmap, | |
2dc6a016 MS |
127 | .map_page = arm_dma_map_page, |
128 | .unmap_page = arm_dma_unmap_page, | |
129 | .map_sg = arm_dma_map_sg, | |
130 | .unmap_sg = arm_dma_unmap_sg, | |
131 | .sync_single_for_cpu = arm_dma_sync_single_for_cpu, | |
132 | .sync_single_for_device = arm_dma_sync_single_for_device, | |
133 | .sync_sg_for_cpu = arm_dma_sync_sg_for_cpu, | |
134 | .sync_sg_for_device = arm_dma_sync_sg_for_device, | |
135 | .set_dma_mask = arm_dma_set_mask, | |
136 | }; | |
137 | EXPORT_SYMBOL(arm_dma_ops); | |
138 | ||
ab6494f0 CM |
139 | static u64 get_coherent_dma_mask(struct device *dev) |
140 | { | |
022ae537 | 141 | u64 mask = (u64)arm_dma_limit; |
ab6494f0 CM |
142 | |
143 | if (dev) { | |
144 | mask = dev->coherent_dma_mask; | |
145 | ||
146 | /* | |
147 | * Sanity check the DMA mask - it must be non-zero, and | |
148 | * must be able to be satisfied by a DMA allocation. | |
149 | */ | |
150 | if (mask == 0) { | |
151 | dev_warn(dev, "coherent DMA mask is unset\n"); | |
152 | return 0; | |
153 | } | |
154 | ||
022ae537 | 155 | if ((~mask) & (u64)arm_dma_limit) { |
ab6494f0 CM |
156 | dev_warn(dev, "coherent DMA mask %#llx is smaller " |
157 | "than system GFP_DMA mask %#llx\n", | |
022ae537 | 158 | mask, (u64)arm_dma_limit); |
ab6494f0 CM |
159 | return 0; |
160 | } | |
161 | } | |
1da177e4 | 162 | |
ab6494f0 CM |
163 | return mask; |
164 | } | |
165 | ||
c7909509 MS |
166 | static void __dma_clear_buffer(struct page *page, size_t size) |
167 | { | |
168 | void *ptr; | |
169 | /* | |
170 | * Ensure that the allocated pages are zeroed, and that any data | |
171 | * lurking in the kernel direct-mapped region is invalidated. | |
172 | */ | |
173 | ptr = page_address(page); | |
4ce63fcd MS |
174 | if (ptr) { |
175 | memset(ptr, 0, size); | |
176 | dmac_flush_range(ptr, ptr + size); | |
177 | outer_flush_range(__pa(ptr), __pa(ptr) + size); | |
178 | } | |
c7909509 MS |
179 | } |
180 | ||
7a9a32a9 RK |
181 | /* |
182 | * Allocate a DMA buffer for 'dev' of size 'size' using the | |
183 | * specified gfp mask. Note that 'size' must be page aligned. | |
184 | */ | |
185 | static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gfp) | |
186 | { | |
187 | unsigned long order = get_order(size); | |
188 | struct page *page, *p, *e; | |
7a9a32a9 RK |
189 | |
190 | page = alloc_pages(gfp, order); | |
191 | if (!page) | |
192 | return NULL; | |
193 | ||
194 | /* | |
195 | * Now split the huge page and free the excess pages | |
196 | */ | |
197 | split_page(page, order); | |
198 | for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++) | |
199 | __free_page(p); | |
200 | ||
c7909509 | 201 | __dma_clear_buffer(page, size); |
7a9a32a9 RK |
202 | |
203 | return page; | |
204 | } | |
205 | ||
206 | /* | |
207 | * Free a DMA buffer. 'size' must be page aligned. | |
208 | */ | |
209 | static void __dma_free_buffer(struct page *page, size_t size) | |
210 | { | |
211 | struct page *e = page + (size >> PAGE_SHIFT); | |
212 | ||
213 | while (page < e) { | |
214 | __free_page(page); | |
215 | page++; | |
216 | } | |
217 | } | |
218 | ||
ab6494f0 | 219 | #ifdef CONFIG_MMU |
a5e9d38b | 220 | |
99d1717d | 221 | #define CONSISTENT_OFFSET(x) (((unsigned long)(x) - consistent_base) >> PAGE_SHIFT) |
1fdb24e9 | 222 | #define CONSISTENT_PTE_INDEX(x) (((unsigned long)(x) - consistent_base) >> PMD_SHIFT) |
a5e9d38b | 223 | |
1da177e4 | 224 | /* |
37134cd5 | 225 | * These are the page tables (2MB each) covering uncached, DMA consistent allocations |
1da177e4 | 226 | */ |
99d1717d JM |
227 | static pte_t **consistent_pte; |
228 | ||
99d1717d | 229 | #define DEFAULT_CONSISTENT_DMA_SIZE SZ_2M |
99d1717d JM |
230 | |
231 | unsigned long consistent_base = CONSISTENT_END - DEFAULT_CONSISTENT_DMA_SIZE; | |
232 | ||
233 | void __init init_consistent_dma_size(unsigned long size) | |
234 | { | |
235 | unsigned long base = CONSISTENT_END - ALIGN(size, SZ_2M); | |
236 | ||
237 | BUG_ON(consistent_pte); /* Check we're called before DMA region init */ | |
238 | BUG_ON(base < VMALLOC_END); | |
239 | ||
240 | /* Grow region to accommodate specified size */ | |
241 | if (base < consistent_base) | |
242 | consistent_base = base; | |
243 | } | |
1da177e4 | 244 | |
13ccf3ad | 245 | #include "vmregion.h" |
1da177e4 | 246 | |
13ccf3ad RK |
247 | static struct arm_vmregion_head consistent_head = { |
248 | .vm_lock = __SPIN_LOCK_UNLOCKED(&consistent_head.vm_lock), | |
1da177e4 | 249 | .vm_list = LIST_HEAD_INIT(consistent_head.vm_list), |
1da177e4 LT |
250 | .vm_end = CONSISTENT_END, |
251 | }; | |
252 | ||
1da177e4 LT |
253 | #ifdef CONFIG_HUGETLB_PAGE |
254 | #error ARM Coherent DMA allocator does not (yet) support huge TLB | |
255 | #endif | |
256 | ||
88c58f3b RK |
257 | /* |
258 | * Initialise the consistent memory allocation. | |
259 | */ | |
260 | static int __init consistent_init(void) | |
261 | { | |
262 | int ret = 0; | |
263 | pgd_t *pgd; | |
516295e5 | 264 | pud_t *pud; |
88c58f3b RK |
265 | pmd_t *pmd; |
266 | pte_t *pte; | |
267 | int i = 0; | |
99d1717d | 268 | unsigned long base = consistent_base; |
53cbcbcf | 269 | unsigned long num_ptes = (CONSISTENT_END - base) >> PMD_SHIFT; |
99d1717d | 270 | |
0f51596b | 271 | #ifndef CONFIG_ARM_DMA_USE_IOMMU |
c7909509 MS |
272 | if (cpu_architecture() >= CPU_ARCH_ARMv6) |
273 | return 0; | |
0f51596b | 274 | #endif |
c7909509 | 275 | |
99d1717d JM |
276 | consistent_pte = kmalloc(num_ptes * sizeof(pte_t), GFP_KERNEL); |
277 | if (!consistent_pte) { | |
278 | pr_err("%s: no memory\n", __func__); | |
279 | return -ENOMEM; | |
280 | } | |
281 | ||
282 | pr_debug("DMA memory: 0x%08lx - 0x%08lx:\n", base, CONSISTENT_END); | |
283 | consistent_head.vm_start = base; | |
88c58f3b RK |
284 | |
285 | do { | |
286 | pgd = pgd_offset(&init_mm, base); | |
516295e5 RK |
287 | |
288 | pud = pud_alloc(&init_mm, pgd, base); | |
289 | if (!pud) { | |
6b6f770b | 290 | pr_err("%s: no pud tables\n", __func__); |
516295e5 RK |
291 | ret = -ENOMEM; |
292 | break; | |
293 | } | |
294 | ||
295 | pmd = pmd_alloc(&init_mm, pud, base); | |
88c58f3b | 296 | if (!pmd) { |
6b6f770b | 297 | pr_err("%s: no pmd tables\n", __func__); |
88c58f3b RK |
298 | ret = -ENOMEM; |
299 | break; | |
300 | } | |
301 | WARN_ON(!pmd_none(*pmd)); | |
302 | ||
303 | pte = pte_alloc_kernel(pmd, base); | |
304 | if (!pte) { | |
6b6f770b | 305 | pr_err("%s: no pte tables\n", __func__); |
88c58f3b RK |
306 | ret = -ENOMEM; |
307 | break; | |
308 | } | |
309 | ||
310 | consistent_pte[i++] = pte; | |
e73fc88e | 311 | base += PMD_SIZE; |
88c58f3b RK |
312 | } while (base < CONSISTENT_END); |
313 | ||
314 | return ret; | |
315 | } | |
88c58f3b RK |
316 | core_initcall(consistent_init); |
317 | ||
c7909509 MS |
318 | static void *__alloc_from_contiguous(struct device *dev, size_t size, |
319 | pgprot_t prot, struct page **ret_page); | |
320 | ||
321 | static struct arm_vmregion_head coherent_head = { | |
322 | .vm_lock = __SPIN_LOCK_UNLOCKED(&coherent_head.vm_lock), | |
323 | .vm_list = LIST_HEAD_INIT(coherent_head.vm_list), | |
324 | }; | |
325 | ||
326 | size_t coherent_pool_size = DEFAULT_CONSISTENT_DMA_SIZE / 8; | |
327 | ||
328 | static int __init early_coherent_pool(char *p) | |
329 | { | |
330 | coherent_pool_size = memparse(p, &p); | |
331 | return 0; | |
332 | } | |
333 | early_param("coherent_pool", early_coherent_pool); | |
334 | ||
335 | /* | |
336 | * Initialise the coherent pool for atomic allocations. | |
337 | */ | |
338 | static int __init coherent_init(void) | |
339 | { | |
340 | pgprot_t prot = pgprot_dmacoherent(pgprot_kernel); | |
341 | size_t size = coherent_pool_size; | |
342 | struct page *page; | |
343 | void *ptr; | |
344 | ||
345 | if (cpu_architecture() < CPU_ARCH_ARMv6) | |
346 | return 0; | |
347 | ||
348 | ptr = __alloc_from_contiguous(NULL, size, prot, &page); | |
349 | if (ptr) { | |
350 | coherent_head.vm_start = (unsigned long) ptr; | |
351 | coherent_head.vm_end = (unsigned long) ptr + size; | |
352 | printk(KERN_INFO "DMA: preallocated %u KiB pool for atomic coherent allocations\n", | |
353 | (unsigned)size / 1024); | |
354 | return 0; | |
355 | } | |
356 | printk(KERN_ERR "DMA: failed to allocate %u KiB pool for atomic coherent allocation\n", | |
357 | (unsigned)size / 1024); | |
358 | return -ENOMEM; | |
359 | } | |
360 | /* | |
361 | * CMA is activated by core_initcall, so we must be called after it. | |
362 | */ | |
363 | postcore_initcall(coherent_init); | |
364 | ||
365 | struct dma_contig_early_reserve { | |
366 | phys_addr_t base; | |
367 | unsigned long size; | |
368 | }; | |
369 | ||
370 | static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata; | |
371 | ||
372 | static int dma_mmu_remap_num __initdata; | |
373 | ||
374 | void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size) | |
375 | { | |
376 | dma_mmu_remap[dma_mmu_remap_num].base = base; | |
377 | dma_mmu_remap[dma_mmu_remap_num].size = size; | |
378 | dma_mmu_remap_num++; | |
379 | } | |
380 | ||
381 | void __init dma_contiguous_remap(void) | |
382 | { | |
383 | int i; | |
384 | for (i = 0; i < dma_mmu_remap_num; i++) { | |
385 | phys_addr_t start = dma_mmu_remap[i].base; | |
386 | phys_addr_t end = start + dma_mmu_remap[i].size; | |
387 | struct map_desc map; | |
388 | unsigned long addr; | |
389 | ||
390 | if (end > arm_lowmem_limit) | |
391 | end = arm_lowmem_limit; | |
392 | if (start >= end) | |
393 | return; | |
394 | ||
395 | map.pfn = __phys_to_pfn(start); | |
396 | map.virtual = __phys_to_virt(start); | |
397 | map.length = end - start; | |
398 | map.type = MT_MEMORY_DMA_READY; | |
399 | ||
400 | /* | |
401 | * Clear previous low-memory mapping | |
402 | */ | |
403 | for (addr = __phys_to_virt(start); addr < __phys_to_virt(end); | |
61f6c7a4 | 404 | addr += PMD_SIZE) |
c7909509 MS |
405 | pmd_clear(pmd_off_k(addr)); |
406 | ||
407 | iotable_init(&map, 1); | |
408 | } | |
409 | } | |
410 | ||
1da177e4 | 411 | static void * |
45cd5290 RK |
412 | __dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot, |
413 | const void *caller) | |
1da177e4 | 414 | { |
13ccf3ad | 415 | struct arm_vmregion *c; |
5bc23d32 RK |
416 | size_t align; |
417 | int bit; | |
1da177e4 | 418 | |
99d1717d | 419 | if (!consistent_pte) { |
6b6f770b | 420 | pr_err("%s: not initialised\n", __func__); |
ebd7a845 | 421 | dump_stack(); |
ebd7a845 RK |
422 | return NULL; |
423 | } | |
424 | ||
5bc23d32 RK |
425 | /* |
426 | * Align the virtual region allocation - maximum alignment is | |
427 | * a section size, minimum is a page size. This helps reduce | |
428 | * fragmentation of the DMA space, and also prevents allocations | |
429 | * smaller than a section from crossing a section boundary. | |
430 | */ | |
c947f69f | 431 | bit = fls(size - 1); |
5bc23d32 RK |
432 | if (bit > SECTION_SHIFT) |
433 | bit = SECTION_SHIFT; | |
434 | align = 1 << bit; | |
435 | ||
1da177e4 LT |
436 | /* |
437 | * Allocate a virtual address in the consistent mapping region. | |
438 | */ | |
5bc23d32 | 439 | c = arm_vmregion_alloc(&consistent_head, align, size, |
45cd5290 | 440 | gfp & ~(__GFP_DMA | __GFP_HIGHMEM), caller); |
1da177e4 | 441 | if (c) { |
37134cd5 | 442 | pte_t *pte; |
37134cd5 KH |
443 | int idx = CONSISTENT_PTE_INDEX(c->vm_start); |
444 | u32 off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1); | |
1da177e4 | 445 | |
37134cd5 | 446 | pte = consistent_pte[idx] + off; |
4ce63fcd | 447 | c->priv = page; |
1da177e4 | 448 | |
1da177e4 LT |
449 | do { |
450 | BUG_ON(!pte_none(*pte)); | |
451 | ||
ad1ae2fe | 452 | set_pte_ext(pte, mk_pte(page, prot), 0); |
1da177e4 LT |
453 | page++; |
454 | pte++; | |
37134cd5 KH |
455 | off++; |
456 | if (off >= PTRS_PER_PTE) { | |
457 | off = 0; | |
458 | pte = consistent_pte[++idx]; | |
459 | } | |
1da177e4 LT |
460 | } while (size -= PAGE_SIZE); |
461 | ||
2be23c47 RK |
462 | dsb(); |
463 | ||
1da177e4 LT |
464 | return (void *)c->vm_start; |
465 | } | |
1da177e4 LT |
466 | return NULL; |
467 | } | |
695ae0af RK |
468 | |
469 | static void __dma_free_remap(void *cpu_addr, size_t size) | |
470 | { | |
471 | struct arm_vmregion *c; | |
472 | unsigned long addr; | |
473 | pte_t *ptep; | |
474 | int idx; | |
475 | u32 off; | |
476 | ||
477 | c = arm_vmregion_find_remove(&consistent_head, (unsigned long)cpu_addr); | |
478 | if (!c) { | |
6b6f770b | 479 | pr_err("%s: trying to free invalid coherent area: %p\n", |
695ae0af RK |
480 | __func__, cpu_addr); |
481 | dump_stack(); | |
482 | return; | |
483 | } | |
484 | ||
485 | if ((c->vm_end - c->vm_start) != size) { | |
6b6f770b | 486 | pr_err("%s: freeing wrong coherent size (%ld != %d)\n", |
695ae0af RK |
487 | __func__, c->vm_end - c->vm_start, size); |
488 | dump_stack(); | |
489 | size = c->vm_end - c->vm_start; | |
490 | } | |
491 | ||
492 | idx = CONSISTENT_PTE_INDEX(c->vm_start); | |
493 | off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1); | |
494 | ptep = consistent_pte[idx] + off; | |
495 | addr = c->vm_start; | |
496 | do { | |
497 | pte_t pte = ptep_get_and_clear(&init_mm, addr, ptep); | |
695ae0af RK |
498 | |
499 | ptep++; | |
500 | addr += PAGE_SIZE; | |
501 | off++; | |
502 | if (off >= PTRS_PER_PTE) { | |
503 | off = 0; | |
504 | ptep = consistent_pte[++idx]; | |
505 | } | |
506 | ||
acaac256 | 507 | if (pte_none(pte) || !pte_present(pte)) |
6b6f770b MS |
508 | pr_crit("%s: bad page in kernel page table\n", |
509 | __func__); | |
695ae0af RK |
510 | } while (size -= PAGE_SIZE); |
511 | ||
512 | flush_tlb_kernel_range(c->vm_start, c->vm_end); | |
513 | ||
514 | arm_vmregion_free(&consistent_head, c); | |
515 | } | |
516 | ||
c7909509 MS |
517 | static int __dma_update_pte(pte_t *pte, pgtable_t token, unsigned long addr, |
518 | void *data) | |
519 | { | |
520 | struct page *page = virt_to_page(addr); | |
521 | pgprot_t prot = *(pgprot_t *)data; | |
522 | ||
523 | set_pte_ext(pte, mk_pte(page, prot), 0); | |
524 | return 0; | |
525 | } | |
526 | ||
527 | static void __dma_remap(struct page *page, size_t size, pgprot_t prot) | |
528 | { | |
529 | unsigned long start = (unsigned long) page_address(page); | |
530 | unsigned end = start + size; | |
531 | ||
532 | apply_to_page_range(&init_mm, start, size, __dma_update_pte, &prot); | |
533 | dsb(); | |
534 | flush_tlb_kernel_range(start, end); | |
535 | } | |
536 | ||
537 | static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp, | |
538 | pgprot_t prot, struct page **ret_page, | |
539 | const void *caller) | |
540 | { | |
541 | struct page *page; | |
542 | void *ptr; | |
543 | page = __dma_alloc_buffer(dev, size, gfp); | |
544 | if (!page) | |
545 | return NULL; | |
546 | ||
547 | ptr = __dma_alloc_remap(page, size, gfp, prot, caller); | |
548 | if (!ptr) { | |
549 | __dma_free_buffer(page, size); | |
550 | return NULL; | |
551 | } | |
552 | ||
553 | *ret_page = page; | |
554 | return ptr; | |
555 | } | |
556 | ||
557 | static void *__alloc_from_pool(struct device *dev, size_t size, | |
558 | struct page **ret_page, const void *caller) | |
559 | { | |
560 | struct arm_vmregion *c; | |
561 | size_t align; | |
562 | ||
563 | if (!coherent_head.vm_start) { | |
564 | printk(KERN_ERR "%s: coherent pool not initialised!\n", | |
565 | __func__); | |
566 | dump_stack(); | |
567 | return NULL; | |
568 | } | |
569 | ||
570 | /* | |
571 | * Align the region allocation - allocations from pool are rather | |
572 | * small, so align them to their order in pages, minimum is a page | |
573 | * size. This helps reduce fragmentation of the DMA space. | |
574 | */ | |
575 | align = PAGE_SIZE << get_order(size); | |
576 | c = arm_vmregion_alloc(&coherent_head, align, size, 0, caller); | |
577 | if (c) { | |
578 | void *ptr = (void *)c->vm_start; | |
579 | struct page *page = virt_to_page(ptr); | |
580 | *ret_page = page; | |
581 | return ptr; | |
582 | } | |
583 | return NULL; | |
584 | } | |
585 | ||
586 | static int __free_from_pool(void *cpu_addr, size_t size) | |
587 | { | |
588 | unsigned long start = (unsigned long)cpu_addr; | |
589 | unsigned long end = start + size; | |
590 | struct arm_vmregion *c; | |
591 | ||
592 | if (start < coherent_head.vm_start || end > coherent_head.vm_end) | |
593 | return 0; | |
594 | ||
595 | c = arm_vmregion_find_remove(&coherent_head, (unsigned long)start); | |
596 | ||
597 | if ((c->vm_end - c->vm_start) != size) { | |
598 | printk(KERN_ERR "%s: freeing wrong coherent size (%ld != %d)\n", | |
599 | __func__, c->vm_end - c->vm_start, size); | |
600 | dump_stack(); | |
601 | size = c->vm_end - c->vm_start; | |
602 | } | |
603 | ||
604 | arm_vmregion_free(&coherent_head, c); | |
605 | return 1; | |
606 | } | |
607 | ||
608 | static void *__alloc_from_contiguous(struct device *dev, size_t size, | |
609 | pgprot_t prot, struct page **ret_page) | |
610 | { | |
611 | unsigned long order = get_order(size); | |
612 | size_t count = size >> PAGE_SHIFT; | |
613 | struct page *page; | |
614 | ||
615 | page = dma_alloc_from_contiguous(dev, count, order); | |
616 | if (!page) | |
617 | return NULL; | |
618 | ||
619 | __dma_clear_buffer(page, size); | |
620 | __dma_remap(page, size, prot); | |
621 | ||
622 | *ret_page = page; | |
623 | return page_address(page); | |
624 | } | |
625 | ||
626 | static void __free_from_contiguous(struct device *dev, struct page *page, | |
627 | size_t size) | |
628 | { | |
629 | __dma_remap(page, size, pgprot_kernel); | |
630 | dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); | |
631 | } | |
632 | ||
f99d6034 MS |
633 | static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot) |
634 | { | |
635 | prot = dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs) ? | |
636 | pgprot_writecombine(prot) : | |
637 | pgprot_dmacoherent(prot); | |
638 | return prot; | |
639 | } | |
640 | ||
c7909509 MS |
641 | #define nommu() 0 |
642 | ||
ab6494f0 | 643 | #else /* !CONFIG_MMU */ |
695ae0af | 644 | |
c7909509 MS |
645 | #define nommu() 1 |
646 | ||
f99d6034 | 647 | #define __get_dma_pgprot(attrs, prot) __pgprot(0) |
c7909509 MS |
648 | #define __alloc_remap_buffer(dev, size, gfp, prot, ret, c) NULL |
649 | #define __alloc_from_pool(dev, size, ret_page, c) NULL | |
650 | #define __alloc_from_contiguous(dev, size, prot, ret) NULL | |
651 | #define __free_from_pool(cpu_addr, size) 0 | |
652 | #define __free_from_contiguous(dev, page, size) do { } while (0) | |
653 | #define __dma_free_remap(cpu_addr, size) do { } while (0) | |
31ebf944 RK |
654 | |
655 | #endif /* CONFIG_MMU */ | |
656 | ||
c7909509 MS |
657 | static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp, |
658 | struct page **ret_page) | |
ab6494f0 | 659 | { |
c7909509 MS |
660 | struct page *page; |
661 | page = __dma_alloc_buffer(dev, size, gfp); | |
662 | if (!page) | |
663 | return NULL; | |
664 | ||
665 | *ret_page = page; | |
666 | return page_address(page); | |
667 | } | |
668 | ||
669 | ||
670 | ||
671 | static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, | |
672 | gfp_t gfp, pgprot_t prot, const void *caller) | |
673 | { | |
674 | u64 mask = get_coherent_dma_mask(dev); | |
04da5694 | 675 | struct page *page; |
31ebf944 | 676 | void *addr; |
ab6494f0 | 677 | |
c7909509 MS |
678 | #ifdef CONFIG_DMA_API_DEBUG |
679 | u64 limit = (mask + 1) & ~mask; | |
680 | if (limit && size >= limit) { | |
681 | dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n", | |
682 | size, mask); | |
683 | return NULL; | |
684 | } | |
685 | #endif | |
686 | ||
687 | if (!mask) | |
688 | return NULL; | |
689 | ||
690 | if (mask < 0xffffffffULL) | |
691 | gfp |= GFP_DMA; | |
692 | ||
ea2e7057 SB |
693 | /* |
694 | * Following is a work-around (a.k.a. hack) to prevent pages | |
695 | * with __GFP_COMP being passed to split_page() which cannot | |
696 | * handle them. The real problem is that this flag probably | |
697 | * should be 0 on ARM as it is not supported on this | |
698 | * platform; see CONFIG_HUGETLBFS. | |
699 | */ | |
700 | gfp &= ~(__GFP_COMP); | |
701 | ||
553ac788 | 702 | *handle = DMA_ERROR_CODE; |
04da5694 | 703 | size = PAGE_ALIGN(size); |
ab6494f0 | 704 | |
c7909509 MS |
705 | if (arch_is_coherent() || nommu()) |
706 | addr = __alloc_simple_buffer(dev, size, gfp, &page); | |
707 | else if (cpu_architecture() < CPU_ARCH_ARMv6) | |
708 | addr = __alloc_remap_buffer(dev, size, gfp, prot, &page, caller); | |
709 | else if (gfp & GFP_ATOMIC) | |
710 | addr = __alloc_from_pool(dev, size, &page, caller); | |
31ebf944 | 711 | else |
c7909509 | 712 | addr = __alloc_from_contiguous(dev, size, prot, &page); |
695ae0af | 713 | |
31ebf944 | 714 | if (addr) |
9eedd963 | 715 | *handle = pfn_to_dma(dev, page_to_pfn(page)); |
695ae0af | 716 | |
31ebf944 RK |
717 | return addr; |
718 | } | |
1da177e4 LT |
719 | |
720 | /* | |
721 | * Allocate DMA-coherent memory space and return both the kernel remapped | |
722 | * virtual and bus address for that space. | |
723 | */ | |
f99d6034 MS |
724 | void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, |
725 | gfp_t gfp, struct dma_attrs *attrs) | |
1da177e4 | 726 | { |
f99d6034 | 727 | pgprot_t prot = __get_dma_pgprot(attrs, pgprot_kernel); |
1fe53268 DB |
728 | void *memory; |
729 | ||
730 | if (dma_alloc_from_coherent(dev, size, handle, &memory)) | |
731 | return memory; | |
732 | ||
f99d6034 | 733 | return __dma_alloc(dev, size, handle, gfp, prot, |
45cd5290 | 734 | __builtin_return_address(0)); |
1da177e4 | 735 | } |
1da177e4 LT |
736 | |
737 | /* | |
f99d6034 | 738 | * Create userspace mapping for the DMA-coherent memory. |
1da177e4 | 739 | */ |
f99d6034 MS |
740 | int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma, |
741 | void *cpu_addr, dma_addr_t dma_addr, size_t size, | |
742 | struct dma_attrs *attrs) | |
1da177e4 | 743 | { |
ab6494f0 CM |
744 | int ret = -ENXIO; |
745 | #ifdef CONFIG_MMU | |
c7909509 | 746 | unsigned long pfn = dma_to_pfn(dev, dma_addr); |
f99d6034 MS |
747 | vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot); |
748 | ||
47142f07 MS |
749 | if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret)) |
750 | return ret; | |
751 | ||
c7909509 MS |
752 | ret = remap_pfn_range(vma, vma->vm_start, |
753 | pfn + vma->vm_pgoff, | |
754 | vma->vm_end - vma->vm_start, | |
755 | vma->vm_page_prot); | |
ab6494f0 | 756 | #endif /* CONFIG_MMU */ |
1da177e4 LT |
757 | |
758 | return ret; | |
759 | } | |
760 | ||
1da177e4 | 761 | /* |
c7909509 | 762 | * Free a buffer as defined by the above mapping. |
1da177e4 | 763 | */ |
f99d6034 MS |
764 | void arm_dma_free(struct device *dev, size_t size, void *cpu_addr, |
765 | dma_addr_t handle, struct dma_attrs *attrs) | |
1da177e4 | 766 | { |
c7909509 | 767 | struct page *page = pfn_to_page(dma_to_pfn(dev, handle)); |
5edf71ae | 768 | |
1fe53268 DB |
769 | if (dma_release_from_coherent(dev, get_order(size), cpu_addr)) |
770 | return; | |
771 | ||
3e82d012 RK |
772 | size = PAGE_ALIGN(size); |
773 | ||
c7909509 MS |
774 | if (arch_is_coherent() || nommu()) { |
775 | __dma_free_buffer(page, size); | |
776 | } else if (cpu_architecture() < CPU_ARCH_ARMv6) { | |
695ae0af | 777 | __dma_free_remap(cpu_addr, size); |
c7909509 MS |
778 | __dma_free_buffer(page, size); |
779 | } else { | |
780 | if (__free_from_pool(cpu_addr, size)) | |
781 | return; | |
782 | /* | |
783 | * Non-atomic allocations cannot be freed with IRQs disabled | |
784 | */ | |
785 | WARN_ON(irqs_disabled()); | |
786 | __free_from_contiguous(dev, page, size); | |
787 | } | |
1da177e4 | 788 | } |
afd1a321 | 789 | |
4ea0d737 | 790 | static void dma_cache_maint_page(struct page *page, unsigned long offset, |
a9c9147e RK |
791 | size_t size, enum dma_data_direction dir, |
792 | void (*op)(const void *, size_t, int)) | |
43377453 NP |
793 | { |
794 | /* | |
795 | * A single sg entry may refer to multiple physically contiguous | |
796 | * pages. But we still need to process highmem pages individually. | |
797 | * If highmem is not configured then the bulk of this loop gets | |
798 | * optimized out. | |
799 | */ | |
800 | size_t left = size; | |
801 | do { | |
802 | size_t len = left; | |
93f1d629 RK |
803 | void *vaddr; |
804 | ||
805 | if (PageHighMem(page)) { | |
806 | if (len + offset > PAGE_SIZE) { | |
807 | if (offset >= PAGE_SIZE) { | |
808 | page += offset / PAGE_SIZE; | |
809 | offset %= PAGE_SIZE; | |
810 | } | |
811 | len = PAGE_SIZE - offset; | |
812 | } | |
813 | vaddr = kmap_high_get(page); | |
814 | if (vaddr) { | |
815 | vaddr += offset; | |
a9c9147e | 816 | op(vaddr, len, dir); |
93f1d629 | 817 | kunmap_high(page); |
7e5a69e8 | 818 | } else if (cache_is_vipt()) { |
39af22a7 NP |
819 | /* unmapped pages might still be cached */ |
820 | vaddr = kmap_atomic(page); | |
7e5a69e8 | 821 | op(vaddr + offset, len, dir); |
39af22a7 | 822 | kunmap_atomic(vaddr); |
43377453 | 823 | } |
93f1d629 RK |
824 | } else { |
825 | vaddr = page_address(page) + offset; | |
a9c9147e | 826 | op(vaddr, len, dir); |
43377453 | 827 | } |
43377453 NP |
828 | offset = 0; |
829 | page++; | |
830 | left -= len; | |
831 | } while (left); | |
832 | } | |
4ea0d737 | 833 | |
51fde349 MS |
834 | /* |
835 | * Make an area consistent for devices. | |
836 | * Note: Drivers should NOT use this function directly, as it will break | |
837 | * platforms with CONFIG_DMABOUNCE. | |
838 | * Use the driver DMA support - see dma-mapping.h (dma_sync_*) | |
839 | */ | |
840 | static void __dma_page_cpu_to_dev(struct page *page, unsigned long off, | |
4ea0d737 RK |
841 | size_t size, enum dma_data_direction dir) |
842 | { | |
65af191a | 843 | unsigned long paddr; |
65af191a | 844 | |
a9c9147e | 845 | dma_cache_maint_page(page, off, size, dir, dmac_map_area); |
65af191a RK |
846 | |
847 | paddr = page_to_phys(page) + off; | |
2ffe2da3 RK |
848 | if (dir == DMA_FROM_DEVICE) { |
849 | outer_inv_range(paddr, paddr + size); | |
850 | } else { | |
851 | outer_clean_range(paddr, paddr + size); | |
852 | } | |
853 | /* FIXME: non-speculating: flush on bidirectional mappings? */ | |
4ea0d737 | 854 | } |
4ea0d737 | 855 | |
51fde349 | 856 | static void __dma_page_dev_to_cpu(struct page *page, unsigned long off, |
4ea0d737 RK |
857 | size_t size, enum dma_data_direction dir) |
858 | { | |
2ffe2da3 RK |
859 | unsigned long paddr = page_to_phys(page) + off; |
860 | ||
861 | /* FIXME: non-speculating: not required */ | |
862 | /* don't bother invalidating if DMA to device */ | |
863 | if (dir != DMA_TO_DEVICE) | |
864 | outer_inv_range(paddr, paddr + size); | |
865 | ||
a9c9147e | 866 | dma_cache_maint_page(page, off, size, dir, dmac_unmap_area); |
c0177800 CM |
867 | |
868 | /* | |
869 | * Mark the D-cache clean for this page to avoid extra flushing. | |
870 | */ | |
871 | if (dir != DMA_TO_DEVICE && off == 0 && size >= PAGE_SIZE) | |
872 | set_bit(PG_dcache_clean, &page->flags); | |
4ea0d737 | 873 | } |
43377453 | 874 | |
afd1a321 | 875 | /** |
2a550e73 | 876 | * arm_dma_map_sg - map a set of SG buffers for streaming mode DMA |
afd1a321 RK |
877 | * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices |
878 | * @sg: list of buffers | |
879 | * @nents: number of buffers to map | |
880 | * @dir: DMA transfer direction | |
881 | * | |
882 | * Map a set of buffers described by scatterlist in streaming mode for DMA. | |
883 | * This is the scatter-gather version of the dma_map_single interface. | |
884 | * Here the scatter gather list elements are each tagged with the | |
885 | * appropriate dma address and length. They are obtained via | |
886 | * sg_dma_{address,length}. | |
887 | * | |
888 | * Device ownership issues as mentioned for dma_map_single are the same | |
889 | * here. | |
890 | */ | |
2dc6a016 MS |
891 | int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, |
892 | enum dma_data_direction dir, struct dma_attrs *attrs) | |
afd1a321 | 893 | { |
2a550e73 | 894 | struct dma_map_ops *ops = get_dma_ops(dev); |
afd1a321 | 895 | struct scatterlist *s; |
01135d92 | 896 | int i, j; |
afd1a321 RK |
897 | |
898 | for_each_sg(sg, s, nents, i) { | |
4ce63fcd MS |
899 | #ifdef CONFIG_NEED_SG_DMA_LENGTH |
900 | s->dma_length = s->length; | |
901 | #endif | |
2a550e73 MS |
902 | s->dma_address = ops->map_page(dev, sg_page(s), s->offset, |
903 | s->length, dir, attrs); | |
01135d92 RK |
904 | if (dma_mapping_error(dev, s->dma_address)) |
905 | goto bad_mapping; | |
afd1a321 | 906 | } |
afd1a321 | 907 | return nents; |
01135d92 RK |
908 | |
909 | bad_mapping: | |
910 | for_each_sg(sg, s, i, j) | |
2a550e73 | 911 | ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs); |
01135d92 | 912 | return 0; |
afd1a321 | 913 | } |
afd1a321 RK |
914 | |
915 | /** | |
2a550e73 | 916 | * arm_dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg |
afd1a321 RK |
917 | * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices |
918 | * @sg: list of buffers | |
0adfca6f | 919 | * @nents: number of buffers to unmap (same as was passed to dma_map_sg) |
afd1a321 RK |
920 | * @dir: DMA transfer direction (same as was passed to dma_map_sg) |
921 | * | |
922 | * Unmap a set of streaming mode DMA translations. Again, CPU access | |
923 | * rules concerning calls here are the same as for dma_unmap_single(). | |
924 | */ | |
2dc6a016 MS |
925 | void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, |
926 | enum dma_data_direction dir, struct dma_attrs *attrs) | |
afd1a321 | 927 | { |
2a550e73 | 928 | struct dma_map_ops *ops = get_dma_ops(dev); |
01135d92 | 929 | struct scatterlist *s; |
01135d92 | 930 | |
01135d92 | 931 | int i; |
24056f52 | 932 | |
01135d92 | 933 | for_each_sg(sg, s, nents, i) |
2a550e73 | 934 | ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs); |
afd1a321 | 935 | } |
afd1a321 RK |
936 | |
937 | /** | |
2a550e73 | 938 | * arm_dma_sync_sg_for_cpu |
afd1a321 RK |
939 | * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices |
940 | * @sg: list of buffers | |
941 | * @nents: number of buffers to map (returned from dma_map_sg) | |
942 | * @dir: DMA transfer direction (same as was passed to dma_map_sg) | |
943 | */ | |
2dc6a016 | 944 | void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, |
afd1a321 RK |
945 | int nents, enum dma_data_direction dir) |
946 | { | |
2a550e73 | 947 | struct dma_map_ops *ops = get_dma_ops(dev); |
afd1a321 RK |
948 | struct scatterlist *s; |
949 | int i; | |
950 | ||
2a550e73 MS |
951 | for_each_sg(sg, s, nents, i) |
952 | ops->sync_single_for_cpu(dev, sg_dma_address(s), s->length, | |
953 | dir); | |
afd1a321 | 954 | } |
afd1a321 RK |
955 | |
956 | /** | |
2a550e73 | 957 | * arm_dma_sync_sg_for_device |
afd1a321 RK |
958 | * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices |
959 | * @sg: list of buffers | |
960 | * @nents: number of buffers to map (returned from dma_map_sg) | |
961 | * @dir: DMA transfer direction (same as was passed to dma_map_sg) | |
962 | */ | |
2dc6a016 | 963 | void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, |
afd1a321 RK |
964 | int nents, enum dma_data_direction dir) |
965 | { | |
2a550e73 | 966 | struct dma_map_ops *ops = get_dma_ops(dev); |
afd1a321 RK |
967 | struct scatterlist *s; |
968 | int i; | |
969 | ||
2a550e73 MS |
970 | for_each_sg(sg, s, nents, i) |
971 | ops->sync_single_for_device(dev, sg_dma_address(s), s->length, | |
972 | dir); | |
afd1a321 | 973 | } |
24056f52 | 974 | |
022ae537 RK |
975 | /* |
976 | * Return whether the given device DMA address mask can be supported | |
977 | * properly. For example, if your device can only drive the low 24-bits | |
978 | * during bus mastering, then you would pass 0x00ffffff as the mask | |
979 | * to this function. | |
980 | */ | |
981 | int dma_supported(struct device *dev, u64 mask) | |
982 | { | |
983 | if (mask < (u64)arm_dma_limit) | |
984 | return 0; | |
985 | return 1; | |
986 | } | |
987 | EXPORT_SYMBOL(dma_supported); | |
988 | ||
2dc6a016 | 989 | static int arm_dma_set_mask(struct device *dev, u64 dma_mask) |
022ae537 RK |
990 | { |
991 | if (!dev->dma_mask || !dma_supported(dev, dma_mask)) | |
992 | return -EIO; | |
993 | ||
022ae537 | 994 | *dev->dma_mask = dma_mask; |
022ae537 RK |
995 | |
996 | return 0; | |
997 | } | |
022ae537 | 998 | |
24056f52 RK |
999 | #define PREALLOC_DMA_DEBUG_ENTRIES 4096 |
1000 | ||
1001 | static int __init dma_debug_do_init(void) | |
1002 | { | |
45cd5290 RK |
1003 | #ifdef CONFIG_MMU |
1004 | arm_vmregion_create_proc("dma-mappings", &consistent_head); | |
1005 | #endif | |
24056f52 RK |
1006 | dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES); |
1007 | return 0; | |
1008 | } | |
1009 | fs_initcall(dma_debug_do_init); | |
4ce63fcd MS |
1010 | |
1011 | #ifdef CONFIG_ARM_DMA_USE_IOMMU | |
1012 | ||
1013 | /* IOMMU */ | |
1014 | ||
1015 | static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping, | |
1016 | size_t size) | |
1017 | { | |
1018 | unsigned int order = get_order(size); | |
1019 | unsigned int align = 0; | |
1020 | unsigned int count, start; | |
1021 | unsigned long flags; | |
1022 | ||
1023 | count = ((PAGE_ALIGN(size) >> PAGE_SHIFT) + | |
1024 | (1 << mapping->order) - 1) >> mapping->order; | |
1025 | ||
1026 | if (order > mapping->order) | |
1027 | align = (1 << (order - mapping->order)) - 1; | |
1028 | ||
1029 | spin_lock_irqsave(&mapping->lock, flags); | |
1030 | start = bitmap_find_next_zero_area(mapping->bitmap, mapping->bits, 0, | |
1031 | count, align); | |
1032 | if (start > mapping->bits) { | |
1033 | spin_unlock_irqrestore(&mapping->lock, flags); | |
1034 | return DMA_ERROR_CODE; | |
1035 | } | |
1036 | ||
1037 | bitmap_set(mapping->bitmap, start, count); | |
1038 | spin_unlock_irqrestore(&mapping->lock, flags); | |
1039 | ||
1040 | return mapping->base + (start << (mapping->order + PAGE_SHIFT)); | |
1041 | } | |
1042 | ||
1043 | static inline void __free_iova(struct dma_iommu_mapping *mapping, | |
1044 | dma_addr_t addr, size_t size) | |
1045 | { | |
1046 | unsigned int start = (addr - mapping->base) >> | |
1047 | (mapping->order + PAGE_SHIFT); | |
1048 | unsigned int count = ((size >> PAGE_SHIFT) + | |
1049 | (1 << mapping->order) - 1) >> mapping->order; | |
1050 | unsigned long flags; | |
1051 | ||
1052 | spin_lock_irqsave(&mapping->lock, flags); | |
1053 | bitmap_clear(mapping->bitmap, start, count); | |
1054 | spin_unlock_irqrestore(&mapping->lock, flags); | |
1055 | } | |
1056 | ||
1057 | static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, gfp_t gfp) | |
1058 | { | |
1059 | struct page **pages; | |
1060 | int count = size >> PAGE_SHIFT; | |
1061 | int array_size = count * sizeof(struct page *); | |
1062 | int i = 0; | |
1063 | ||
1064 | if (array_size <= PAGE_SIZE) | |
1065 | pages = kzalloc(array_size, gfp); | |
1066 | else | |
1067 | pages = vzalloc(array_size); | |
1068 | if (!pages) | |
1069 | return NULL; | |
1070 | ||
1071 | while (count) { | |
1072 | int j, order = __ffs(count); | |
1073 | ||
1074 | pages[i] = alloc_pages(gfp | __GFP_NOWARN, order); | |
1075 | while (!pages[i] && order) | |
1076 | pages[i] = alloc_pages(gfp | __GFP_NOWARN, --order); | |
1077 | if (!pages[i]) | |
1078 | goto error; | |
1079 | ||
1080 | if (order) | |
1081 | split_page(pages[i], order); | |
1082 | j = 1 << order; | |
1083 | while (--j) | |
1084 | pages[i + j] = pages[i] + j; | |
1085 | ||
1086 | __dma_clear_buffer(pages[i], PAGE_SIZE << order); | |
1087 | i += 1 << order; | |
1088 | count -= 1 << order; | |
1089 | } | |
1090 | ||
1091 | return pages; | |
1092 | error: | |
1093 | while (--i) | |
1094 | if (pages[i]) | |
1095 | __free_pages(pages[i], 0); | |
1096 | if (array_size < PAGE_SIZE) | |
1097 | kfree(pages); | |
1098 | else | |
1099 | vfree(pages); | |
1100 | return NULL; | |
1101 | } | |
1102 | ||
1103 | static int __iommu_free_buffer(struct device *dev, struct page **pages, size_t size) | |
1104 | { | |
1105 | int count = size >> PAGE_SHIFT; | |
1106 | int array_size = count * sizeof(struct page *); | |
1107 | int i; | |
1108 | for (i = 0; i < count; i++) | |
1109 | if (pages[i]) | |
1110 | __free_pages(pages[i], 0); | |
1111 | if (array_size < PAGE_SIZE) | |
1112 | kfree(pages); | |
1113 | else | |
1114 | vfree(pages); | |
1115 | return 0; | |
1116 | } | |
1117 | ||
1118 | /* | |
1119 | * Create a CPU mapping for a specified pages | |
1120 | */ | |
1121 | static void * | |
1122 | __iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot) | |
1123 | { | |
1124 | struct arm_vmregion *c; | |
1125 | size_t align; | |
1126 | size_t count = size >> PAGE_SHIFT; | |
1127 | int bit; | |
1128 | ||
1129 | if (!consistent_pte[0]) { | |
1130 | pr_err("%s: not initialised\n", __func__); | |
1131 | dump_stack(); | |
1132 | return NULL; | |
1133 | } | |
1134 | ||
1135 | /* | |
1136 | * Align the virtual region allocation - maximum alignment is | |
1137 | * a section size, minimum is a page size. This helps reduce | |
1138 | * fragmentation of the DMA space, and also prevents allocations | |
1139 | * smaller than a section from crossing a section boundary. | |
1140 | */ | |
1141 | bit = fls(size - 1); | |
1142 | if (bit > SECTION_SHIFT) | |
1143 | bit = SECTION_SHIFT; | |
1144 | align = 1 << bit; | |
1145 | ||
1146 | /* | |
1147 | * Allocate a virtual address in the consistent mapping region. | |
1148 | */ | |
1149 | c = arm_vmregion_alloc(&consistent_head, align, size, | |
1150 | gfp & ~(__GFP_DMA | __GFP_HIGHMEM), NULL); | |
1151 | if (c) { | |
1152 | pte_t *pte; | |
1153 | int idx = CONSISTENT_PTE_INDEX(c->vm_start); | |
1154 | int i = 0; | |
1155 | u32 off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1); | |
1156 | ||
1157 | pte = consistent_pte[idx] + off; | |
1158 | c->priv = pages; | |
1159 | ||
1160 | do { | |
1161 | BUG_ON(!pte_none(*pte)); | |
1162 | ||
1163 | set_pte_ext(pte, mk_pte(pages[i], prot), 0); | |
1164 | pte++; | |
1165 | off++; | |
1166 | i++; | |
1167 | if (off >= PTRS_PER_PTE) { | |
1168 | off = 0; | |
1169 | pte = consistent_pte[++idx]; | |
1170 | } | |
1171 | } while (i < count); | |
1172 | ||
1173 | dsb(); | |
1174 | ||
1175 | return (void *)c->vm_start; | |
1176 | } | |
1177 | return NULL; | |
1178 | } | |
1179 | ||
1180 | /* | |
1181 | * Create a mapping in device IO address space for specified pages | |
1182 | */ | |
1183 | static dma_addr_t | |
1184 | __iommu_create_mapping(struct device *dev, struct page **pages, size_t size) | |
1185 | { | |
1186 | struct dma_iommu_mapping *mapping = dev->archdata.mapping; | |
1187 | unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; | |
1188 | dma_addr_t dma_addr, iova; | |
1189 | int i, ret = DMA_ERROR_CODE; | |
1190 | ||
1191 | dma_addr = __alloc_iova(mapping, size); | |
1192 | if (dma_addr == DMA_ERROR_CODE) | |
1193 | return dma_addr; | |
1194 | ||
1195 | iova = dma_addr; | |
1196 | for (i = 0; i < count; ) { | |
1197 | unsigned int next_pfn = page_to_pfn(pages[i]) + 1; | |
1198 | phys_addr_t phys = page_to_phys(pages[i]); | |
1199 | unsigned int len, j; | |
1200 | ||
1201 | for (j = i + 1; j < count; j++, next_pfn++) | |
1202 | if (page_to_pfn(pages[j]) != next_pfn) | |
1203 | break; | |
1204 | ||
1205 | len = (j - i) << PAGE_SHIFT; | |
1206 | ret = iommu_map(mapping->domain, iova, phys, len, 0); | |
1207 | if (ret < 0) | |
1208 | goto fail; | |
1209 | iova += len; | |
1210 | i = j; | |
1211 | } | |
1212 | return dma_addr; | |
1213 | fail: | |
1214 | iommu_unmap(mapping->domain, dma_addr, iova-dma_addr); | |
1215 | __free_iova(mapping, dma_addr, size); | |
1216 | return DMA_ERROR_CODE; | |
1217 | } | |
1218 | ||
1219 | static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size) | |
1220 | { | |
1221 | struct dma_iommu_mapping *mapping = dev->archdata.mapping; | |
1222 | ||
1223 | /* | |
1224 | * add optional in-page offset from iova to size and align | |
1225 | * result to page size | |
1226 | */ | |
1227 | size = PAGE_ALIGN((iova & ~PAGE_MASK) + size); | |
1228 | iova &= PAGE_MASK; | |
1229 | ||
1230 | iommu_unmap(mapping->domain, iova, size); | |
1231 | __free_iova(mapping, iova, size); | |
1232 | return 0; | |
1233 | } | |
1234 | ||
1235 | static void *arm_iommu_alloc_attrs(struct device *dev, size_t size, | |
1236 | dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs) | |
1237 | { | |
1238 | pgprot_t prot = __get_dma_pgprot(attrs, pgprot_kernel); | |
1239 | struct page **pages; | |
1240 | void *addr = NULL; | |
1241 | ||
1242 | *handle = DMA_ERROR_CODE; | |
1243 | size = PAGE_ALIGN(size); | |
1244 | ||
1245 | pages = __iommu_alloc_buffer(dev, size, gfp); | |
1246 | if (!pages) | |
1247 | return NULL; | |
1248 | ||
1249 | *handle = __iommu_create_mapping(dev, pages, size); | |
1250 | if (*handle == DMA_ERROR_CODE) | |
1251 | goto err_buffer; | |
1252 | ||
1253 | addr = __iommu_alloc_remap(pages, size, gfp, prot); | |
1254 | if (!addr) | |
1255 | goto err_mapping; | |
1256 | ||
1257 | return addr; | |
1258 | ||
1259 | err_mapping: | |
1260 | __iommu_remove_mapping(dev, *handle, size); | |
1261 | err_buffer: | |
1262 | __iommu_free_buffer(dev, pages, size); | |
1263 | return NULL; | |
1264 | } | |
1265 | ||
1266 | static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma, | |
1267 | void *cpu_addr, dma_addr_t dma_addr, size_t size, | |
1268 | struct dma_attrs *attrs) | |
1269 | { | |
1270 | struct arm_vmregion *c; | |
1271 | ||
1272 | vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot); | |
1273 | c = arm_vmregion_find(&consistent_head, (unsigned long)cpu_addr); | |
1274 | ||
1275 | if (c) { | |
1276 | struct page **pages = c->priv; | |
1277 | ||
1278 | unsigned long uaddr = vma->vm_start; | |
1279 | unsigned long usize = vma->vm_end - vma->vm_start; | |
1280 | int i = 0; | |
1281 | ||
1282 | do { | |
1283 | int ret; | |
1284 | ||
1285 | ret = vm_insert_page(vma, uaddr, pages[i++]); | |
1286 | if (ret) { | |
1287 | pr_err("Remapping memory, error: %d\n", ret); | |
1288 | return ret; | |
1289 | } | |
1290 | ||
1291 | uaddr += PAGE_SIZE; | |
1292 | usize -= PAGE_SIZE; | |
1293 | } while (usize > 0); | |
1294 | } | |
1295 | return 0; | |
1296 | } | |
1297 | ||
1298 | /* | |
1299 | * free a page as defined by the above mapping. | |
1300 | * Must not be called with IRQs disabled. | |
1301 | */ | |
1302 | void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr, | |
1303 | dma_addr_t handle, struct dma_attrs *attrs) | |
1304 | { | |
1305 | struct arm_vmregion *c; | |
1306 | size = PAGE_ALIGN(size); | |
1307 | ||
1308 | c = arm_vmregion_find(&consistent_head, (unsigned long)cpu_addr); | |
1309 | if (c) { | |
1310 | struct page **pages = c->priv; | |
1311 | __dma_free_remap(cpu_addr, size); | |
1312 | __iommu_remove_mapping(dev, handle, size); | |
1313 | __iommu_free_buffer(dev, pages, size); | |
1314 | } | |
1315 | } | |
1316 | ||
1317 | /* | |
1318 | * Map a part of the scatter-gather list into contiguous io address space | |
1319 | */ | |
1320 | static int __map_sg_chunk(struct device *dev, struct scatterlist *sg, | |
1321 | size_t size, dma_addr_t *handle, | |
1322 | enum dma_data_direction dir) | |
1323 | { | |
1324 | struct dma_iommu_mapping *mapping = dev->archdata.mapping; | |
1325 | dma_addr_t iova, iova_base; | |
1326 | int ret = 0; | |
1327 | unsigned int count; | |
1328 | struct scatterlist *s; | |
1329 | ||
1330 | size = PAGE_ALIGN(size); | |
1331 | *handle = DMA_ERROR_CODE; | |
1332 | ||
1333 | iova_base = iova = __alloc_iova(mapping, size); | |
1334 | if (iova == DMA_ERROR_CODE) | |
1335 | return -ENOMEM; | |
1336 | ||
1337 | for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) { | |
1338 | phys_addr_t phys = page_to_phys(sg_page(s)); | |
1339 | unsigned int len = PAGE_ALIGN(s->offset + s->length); | |
1340 | ||
1341 | if (!arch_is_coherent()) | |
1342 | __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); | |
1343 | ||
1344 | ret = iommu_map(mapping->domain, iova, phys, len, 0); | |
1345 | if (ret < 0) | |
1346 | goto fail; | |
1347 | count += len >> PAGE_SHIFT; | |
1348 | iova += len; | |
1349 | } | |
1350 | *handle = iova_base; | |
1351 | ||
1352 | return 0; | |
1353 | fail: | |
1354 | iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE); | |
1355 | __free_iova(mapping, iova_base, size); | |
1356 | return ret; | |
1357 | } | |
1358 | ||
1359 | /** | |
1360 | * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA | |
1361 | * @dev: valid struct device pointer | |
1362 | * @sg: list of buffers | |
1363 | * @nents: number of buffers to map | |
1364 | * @dir: DMA transfer direction | |
1365 | * | |
1366 | * Map a set of buffers described by scatterlist in streaming mode for DMA. | |
1367 | * The scatter gather list elements are merged together (if possible) and | |
1368 | * tagged with the appropriate dma address and length. They are obtained via | |
1369 | * sg_dma_{address,length}. | |
1370 | */ | |
1371 | int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents, | |
1372 | enum dma_data_direction dir, struct dma_attrs *attrs) | |
1373 | { | |
1374 | struct scatterlist *s = sg, *dma = sg, *start = sg; | |
1375 | int i, count = 0; | |
1376 | unsigned int offset = s->offset; | |
1377 | unsigned int size = s->offset + s->length; | |
1378 | unsigned int max = dma_get_max_seg_size(dev); | |
1379 | ||
1380 | for (i = 1; i < nents; i++) { | |
1381 | s = sg_next(s); | |
1382 | ||
1383 | s->dma_address = DMA_ERROR_CODE; | |
1384 | s->dma_length = 0; | |
1385 | ||
1386 | if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) { | |
1387 | if (__map_sg_chunk(dev, start, size, &dma->dma_address, | |
1388 | dir) < 0) | |
1389 | goto bad_mapping; | |
1390 | ||
1391 | dma->dma_address += offset; | |
1392 | dma->dma_length = size - offset; | |
1393 | ||
1394 | size = offset = s->offset; | |
1395 | start = s; | |
1396 | dma = sg_next(dma); | |
1397 | count += 1; | |
1398 | } | |
1399 | size += s->length; | |
1400 | } | |
1401 | if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir) < 0) | |
1402 | goto bad_mapping; | |
1403 | ||
1404 | dma->dma_address += offset; | |
1405 | dma->dma_length = size - offset; | |
1406 | ||
1407 | return count+1; | |
1408 | ||
1409 | bad_mapping: | |
1410 | for_each_sg(sg, s, count, i) | |
1411 | __iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s)); | |
1412 | return 0; | |
1413 | } | |
1414 | ||
1415 | /** | |
1416 | * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg | |
1417 | * @dev: valid struct device pointer | |
1418 | * @sg: list of buffers | |
1419 | * @nents: number of buffers to unmap (same as was passed to dma_map_sg) | |
1420 | * @dir: DMA transfer direction (same as was passed to dma_map_sg) | |
1421 | * | |
1422 | * Unmap a set of streaming mode DMA translations. Again, CPU access | |
1423 | * rules concerning calls here are the same as for dma_unmap_single(). | |
1424 | */ | |
1425 | void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, | |
1426 | enum dma_data_direction dir, struct dma_attrs *attrs) | |
1427 | { | |
1428 | struct scatterlist *s; | |
1429 | int i; | |
1430 | ||
1431 | for_each_sg(sg, s, nents, i) { | |
1432 | if (sg_dma_len(s)) | |
1433 | __iommu_remove_mapping(dev, sg_dma_address(s), | |
1434 | sg_dma_len(s)); | |
1435 | if (!arch_is_coherent()) | |
1436 | __dma_page_dev_to_cpu(sg_page(s), s->offset, | |
1437 | s->length, dir); | |
1438 | } | |
1439 | } | |
1440 | ||
1441 | /** | |
1442 | * arm_iommu_sync_sg_for_cpu | |
1443 | * @dev: valid struct device pointer | |
1444 | * @sg: list of buffers | |
1445 | * @nents: number of buffers to map (returned from dma_map_sg) | |
1446 | * @dir: DMA transfer direction (same as was passed to dma_map_sg) | |
1447 | */ | |
1448 | void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, | |
1449 | int nents, enum dma_data_direction dir) | |
1450 | { | |
1451 | struct scatterlist *s; | |
1452 | int i; | |
1453 | ||
1454 | for_each_sg(sg, s, nents, i) | |
1455 | if (!arch_is_coherent()) | |
1456 | __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir); | |
1457 | ||
1458 | } | |
1459 | ||
1460 | /** | |
1461 | * arm_iommu_sync_sg_for_device | |
1462 | * @dev: valid struct device pointer | |
1463 | * @sg: list of buffers | |
1464 | * @nents: number of buffers to map (returned from dma_map_sg) | |
1465 | * @dir: DMA transfer direction (same as was passed to dma_map_sg) | |
1466 | */ | |
1467 | void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg, | |
1468 | int nents, enum dma_data_direction dir) | |
1469 | { | |
1470 | struct scatterlist *s; | |
1471 | int i; | |
1472 | ||
1473 | for_each_sg(sg, s, nents, i) | |
1474 | if (!arch_is_coherent()) | |
1475 | __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); | |
1476 | } | |
1477 | ||
1478 | ||
1479 | /** | |
1480 | * arm_iommu_map_page | |
1481 | * @dev: valid struct device pointer | |
1482 | * @page: page that buffer resides in | |
1483 | * @offset: offset into page for start of buffer | |
1484 | * @size: size of buffer to map | |
1485 | * @dir: DMA transfer direction | |
1486 | * | |
1487 | * IOMMU aware version of arm_dma_map_page() | |
1488 | */ | |
1489 | static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page, | |
1490 | unsigned long offset, size_t size, enum dma_data_direction dir, | |
1491 | struct dma_attrs *attrs) | |
1492 | { | |
1493 | struct dma_iommu_mapping *mapping = dev->archdata.mapping; | |
1494 | dma_addr_t dma_addr; | |
1495 | int ret, len = PAGE_ALIGN(size + offset); | |
1496 | ||
1497 | if (!arch_is_coherent()) | |
1498 | __dma_page_cpu_to_dev(page, offset, size, dir); | |
1499 | ||
1500 | dma_addr = __alloc_iova(mapping, len); | |
1501 | if (dma_addr == DMA_ERROR_CODE) | |
1502 | return dma_addr; | |
1503 | ||
1504 | ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, 0); | |
1505 | if (ret < 0) | |
1506 | goto fail; | |
1507 | ||
1508 | return dma_addr + offset; | |
1509 | fail: | |
1510 | __free_iova(mapping, dma_addr, len); | |
1511 | return DMA_ERROR_CODE; | |
1512 | } | |
1513 | ||
1514 | /** | |
1515 | * arm_iommu_unmap_page | |
1516 | * @dev: valid struct device pointer | |
1517 | * @handle: DMA address of buffer | |
1518 | * @size: size of buffer (same as passed to dma_map_page) | |
1519 | * @dir: DMA transfer direction (same as passed to dma_map_page) | |
1520 | * | |
1521 | * IOMMU aware version of arm_dma_unmap_page() | |
1522 | */ | |
1523 | static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle, | |
1524 | size_t size, enum dma_data_direction dir, | |
1525 | struct dma_attrs *attrs) | |
1526 | { | |
1527 | struct dma_iommu_mapping *mapping = dev->archdata.mapping; | |
1528 | dma_addr_t iova = handle & PAGE_MASK; | |
1529 | struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); | |
1530 | int offset = handle & ~PAGE_MASK; | |
1531 | int len = PAGE_ALIGN(size + offset); | |
1532 | ||
1533 | if (!iova) | |
1534 | return; | |
1535 | ||
1536 | if (!arch_is_coherent()) | |
1537 | __dma_page_dev_to_cpu(page, offset, size, dir); | |
1538 | ||
1539 | iommu_unmap(mapping->domain, iova, len); | |
1540 | __free_iova(mapping, iova, len); | |
1541 | } | |
1542 | ||
1543 | static void arm_iommu_sync_single_for_cpu(struct device *dev, | |
1544 | dma_addr_t handle, size_t size, enum dma_data_direction dir) | |
1545 | { | |
1546 | struct dma_iommu_mapping *mapping = dev->archdata.mapping; | |
1547 | dma_addr_t iova = handle & PAGE_MASK; | |
1548 | struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); | |
1549 | unsigned int offset = handle & ~PAGE_MASK; | |
1550 | ||
1551 | if (!iova) | |
1552 | return; | |
1553 | ||
1554 | if (!arch_is_coherent()) | |
1555 | __dma_page_dev_to_cpu(page, offset, size, dir); | |
1556 | } | |
1557 | ||
1558 | static void arm_iommu_sync_single_for_device(struct device *dev, | |
1559 | dma_addr_t handle, size_t size, enum dma_data_direction dir) | |
1560 | { | |
1561 | struct dma_iommu_mapping *mapping = dev->archdata.mapping; | |
1562 | dma_addr_t iova = handle & PAGE_MASK; | |
1563 | struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); | |
1564 | unsigned int offset = handle & ~PAGE_MASK; | |
1565 | ||
1566 | if (!iova) | |
1567 | return; | |
1568 | ||
1569 | __dma_page_cpu_to_dev(page, offset, size, dir); | |
1570 | } | |
1571 | ||
1572 | struct dma_map_ops iommu_ops = { | |
1573 | .alloc = arm_iommu_alloc_attrs, | |
1574 | .free = arm_iommu_free_attrs, | |
1575 | .mmap = arm_iommu_mmap_attrs, | |
1576 | ||
1577 | .map_page = arm_iommu_map_page, | |
1578 | .unmap_page = arm_iommu_unmap_page, | |
1579 | .sync_single_for_cpu = arm_iommu_sync_single_for_cpu, | |
1580 | .sync_single_for_device = arm_iommu_sync_single_for_device, | |
1581 | ||
1582 | .map_sg = arm_iommu_map_sg, | |
1583 | .unmap_sg = arm_iommu_unmap_sg, | |
1584 | .sync_sg_for_cpu = arm_iommu_sync_sg_for_cpu, | |
1585 | .sync_sg_for_device = arm_iommu_sync_sg_for_device, | |
1586 | }; | |
1587 | ||
1588 | /** | |
1589 | * arm_iommu_create_mapping | |
1590 | * @bus: pointer to the bus holding the client device (for IOMMU calls) | |
1591 | * @base: start address of the valid IO address space | |
1592 | * @size: size of the valid IO address space | |
1593 | * @order: accuracy of the IO addresses allocations | |
1594 | * | |
1595 | * Creates a mapping structure which holds information about used/unused | |
1596 | * IO address ranges, which is required to perform memory allocation and | |
1597 | * mapping with IOMMU aware functions. | |
1598 | * | |
1599 | * The client device need to be attached to the mapping with | |
1600 | * arm_iommu_attach_device function. | |
1601 | */ | |
1602 | struct dma_iommu_mapping * | |
1603 | arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, size_t size, | |
1604 | int order) | |
1605 | { | |
1606 | unsigned int count = size >> (PAGE_SHIFT + order); | |
1607 | unsigned int bitmap_size = BITS_TO_LONGS(count) * sizeof(long); | |
1608 | struct dma_iommu_mapping *mapping; | |
1609 | int err = -ENOMEM; | |
1610 | ||
1611 | if (!count) | |
1612 | return ERR_PTR(-EINVAL); | |
1613 | ||
1614 | mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL); | |
1615 | if (!mapping) | |
1616 | goto err; | |
1617 | ||
1618 | mapping->bitmap = kzalloc(bitmap_size, GFP_KERNEL); | |
1619 | if (!mapping->bitmap) | |
1620 | goto err2; | |
1621 | ||
1622 | mapping->base = base; | |
1623 | mapping->bits = BITS_PER_BYTE * bitmap_size; | |
1624 | mapping->order = order; | |
1625 | spin_lock_init(&mapping->lock); | |
1626 | ||
1627 | mapping->domain = iommu_domain_alloc(bus); | |
1628 | if (!mapping->domain) | |
1629 | goto err3; | |
1630 | ||
1631 | kref_init(&mapping->kref); | |
1632 | return mapping; | |
1633 | err3: | |
1634 | kfree(mapping->bitmap); | |
1635 | err2: | |
1636 | kfree(mapping); | |
1637 | err: | |
1638 | return ERR_PTR(err); | |
1639 | } | |
1640 | ||
1641 | static void release_iommu_mapping(struct kref *kref) | |
1642 | { | |
1643 | struct dma_iommu_mapping *mapping = | |
1644 | container_of(kref, struct dma_iommu_mapping, kref); | |
1645 | ||
1646 | iommu_domain_free(mapping->domain); | |
1647 | kfree(mapping->bitmap); | |
1648 | kfree(mapping); | |
1649 | } | |
1650 | ||
1651 | void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping) | |
1652 | { | |
1653 | if (mapping) | |
1654 | kref_put(&mapping->kref, release_iommu_mapping); | |
1655 | } | |
1656 | ||
1657 | /** | |
1658 | * arm_iommu_attach_device | |
1659 | * @dev: valid struct device pointer | |
1660 | * @mapping: io address space mapping structure (returned from | |
1661 | * arm_iommu_create_mapping) | |
1662 | * | |
1663 | * Attaches specified io address space mapping to the provided device, | |
1664 | * this replaces the dma operations (dma_map_ops pointer) with the | |
1665 | * IOMMU aware version. More than one client might be attached to | |
1666 | * the same io address space mapping. | |
1667 | */ | |
1668 | int arm_iommu_attach_device(struct device *dev, | |
1669 | struct dma_iommu_mapping *mapping) | |
1670 | { | |
1671 | int err; | |
1672 | ||
1673 | err = iommu_attach_device(mapping->domain, dev); | |
1674 | if (err) | |
1675 | return err; | |
1676 | ||
1677 | kref_get(&mapping->kref); | |
1678 | dev->archdata.mapping = mapping; | |
1679 | set_dma_ops(dev, &iommu_ops); | |
1680 | ||
1681 | pr_info("Attached IOMMU controller to %s device.\n", dev_name(dev)); | |
1682 | return 0; | |
1683 | } | |
1684 | ||
1685 | #endif |