1 // SPDX-License-Identifier: GPL-2.0-only
3 * Dynamic DMA mapping support.
5 * This implementation is a fallback for platforms that do not support
6 * I/O TLBs (aka DMA address translation hardware).
7 * Copyright (C) 2000 Asit Mallick <Asit.K.Mallick@intel.com>
8 * Copyright (C) 2000 Goutham Rao <goutham.rao@intel.com>
9 * Copyright (C) 2000, 2003 Hewlett-Packard Co
10 * David Mosberger-Tang <davidm@hpl.hp.com>
12 * 03/05/07 davidm Switch from PCI-DMA to generic device DMA API.
13 * 00/12/13 davidm Rename to swiotlb.c and add mark_clean() to avoid
14 * unnecessary i-cache flushing.
15 * 04/07/.. ak Better overflow handling. Assorted fixes.
16 * 05/09/10 linville Add support for syncing ranges, support syncing for
17 * DMA_BIDIRECTIONAL mappings, miscellaneous cleanup.
18 * 08/12/11 beckyb Add highmem support
21 #define pr_fmt(fmt) "software IO TLB: " fmt
23 #include <linux/cache.h>
24 #include <linux/cc_platform.h>
25 #include <linux/ctype.h>
26 #include <linux/debugfs.h>
27 #include <linux/dma-direct.h>
28 #include <linux/dma-map-ops.h>
29 #include <linux/export.h>
30 #include <linux/gfp.h>
31 #include <linux/highmem.h>
33 #include <linux/iommu-helper.h>
34 #include <linux/init.h>
35 #include <linux/memblock.h>
37 #include <linux/pfn.h>
38 #include <linux/rculist.h>
39 #include <linux/scatterlist.h>
40 #include <linux/set_memory.h>
41 #include <linux/spinlock.h>
42 #include <linux/string.h>
43 #include <linux/swiotlb.h>
44 #include <linux/types.h>
45 #ifdef CONFIG_DMA_RESTRICTED_POOL
47 #include <linux/of_fdt.h>
48 #include <linux/of_reserved_mem.h>
49 #include <linux/slab.h>
52 #define CREATE_TRACE_POINTS
53 #include <trace/events/swiotlb.h>
55 #define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
58 * Minimum IO TLB size to bother booting with. Systems with mainly
59 * 64bit capable cards will only lightly use the swiotlb. If we can't
60 * allocate a contiguous 1MB, we're probably in trouble anyway.
62 #define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
64 #define INVALID_PHYS_ADDR (~(phys_addr_t)0)
67 * struct io_tlb_slot - IO TLB slot descriptor
68 * @orig_addr: The original address corresponding to a mapped entry.
69 * @alloc_size: Size of the allocated buffer.
70 * @list: The free list describing the number of free entries available
72 * @pad_slots: Number of preceding padding slots. Valid only in the first
73 * allocated non-padding slot.
76 phys_addr_t orig_addr;
79 unsigned short pad_slots;
82 static bool swiotlb_force_bounce;
83 static bool swiotlb_force_disable;
85 #ifdef CONFIG_SWIOTLB_DYNAMIC
87 static void swiotlb_dyn_alloc(struct work_struct *work);
89 static struct io_tlb_mem io_tlb_default_mem = {
90 .lock = __SPIN_LOCK_UNLOCKED(io_tlb_default_mem.lock),
91 .pools = LIST_HEAD_INIT(io_tlb_default_mem.pools),
92 .dyn_alloc = __WORK_INITIALIZER(io_tlb_default_mem.dyn_alloc,
96 #else /* !CONFIG_SWIOTLB_DYNAMIC */
98 static struct io_tlb_mem io_tlb_default_mem;
100 #endif /* CONFIG_SWIOTLB_DYNAMIC */
102 static unsigned long default_nslabs = IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT;
103 static unsigned long default_nareas;
106 * struct io_tlb_area - IO TLB memory area descriptor
108 * This is a single area with a single lock.
110 * @used: The number of used IO TLB block.
111 * @index: The slot index to start searching in this area for next round.
112 * @lock: The lock to protect the above data structures in the map and
122 * Round up number of slabs to the next power of 2. The last area is going
123 * be smaller than the rest if default_nslabs is not power of two.
124 * The number of slot in an area should be a multiple of IO_TLB_SEGSIZE,
125 * otherwise a segment may span two or more areas. It conflicts with free
126 * contiguous slots tracking: free slots are treated contiguous no matter
127 * whether they cross an area boundary.
129 * Return true if default_nslabs is rounded up.
131 static bool round_up_default_nslabs(void)
136 if (default_nslabs < IO_TLB_SEGSIZE * default_nareas)
137 default_nslabs = IO_TLB_SEGSIZE * default_nareas;
138 else if (is_power_of_2(default_nslabs))
140 default_nslabs = roundup_pow_of_two(default_nslabs);
145 * swiotlb_adjust_nareas() - adjust the number of areas and slots
146 * @nareas: Desired number of areas. Zero is treated as 1.
148 * Adjust the default number of areas in a memory pool.
149 * The default size of the memory pool may also change to meet minimum area
152 static void swiotlb_adjust_nareas(unsigned int nareas)
156 else if (!is_power_of_2(nareas))
157 nareas = roundup_pow_of_two(nareas);
159 default_nareas = nareas;
161 pr_info("area num %d.\n", nareas);
162 if (round_up_default_nslabs())
163 pr_info("SWIOTLB bounce buffer size roundup to %luMB",
164 (default_nslabs << IO_TLB_SHIFT) >> 20);
168 * limit_nareas() - get the maximum number of areas for a given memory pool size
169 * @nareas: Desired number of areas.
170 * @nslots: Total number of slots in the memory pool.
172 * Limit the number of areas to the maximum possible number of areas in
173 * a memory pool of the given size.
175 * Return: Maximum possible number of areas.
177 static unsigned int limit_nareas(unsigned int nareas, unsigned long nslots)
179 if (nslots < nareas * IO_TLB_SEGSIZE)
180 return nslots / IO_TLB_SEGSIZE;
185 setup_io_tlb_npages(char *str)
188 /* avoid tail segment of size < IO_TLB_SEGSIZE */
190 ALIGN(simple_strtoul(str, &str, 0), IO_TLB_SEGSIZE);
195 swiotlb_adjust_nareas(simple_strtoul(str, &str, 0));
198 if (!strcmp(str, "force"))
199 swiotlb_force_bounce = true;
200 else if (!strcmp(str, "noforce"))
201 swiotlb_force_disable = true;
205 early_param("swiotlb", setup_io_tlb_npages);
207 unsigned long swiotlb_size_or_default(void)
209 return default_nslabs << IO_TLB_SHIFT;
212 void __init swiotlb_adjust_size(unsigned long size)
215 * If swiotlb parameter has not been specified, give a chance to
216 * architectures such as those supporting memory encryption to
217 * adjust/expand SWIOTLB size for their use.
219 if (default_nslabs != IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT)
222 size = ALIGN(size, IO_TLB_SIZE);
223 default_nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
224 if (round_up_default_nslabs())
225 size = default_nslabs << IO_TLB_SHIFT;
226 pr_info("SWIOTLB bounce buffer size adjusted to %luMB", size >> 20);
229 void swiotlb_print_info(void)
231 struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
234 pr_warn("No low mem\n");
238 pr_info("mapped [mem %pa-%pa] (%luMB)\n", &mem->start, &mem->end,
239 (mem->nslabs << IO_TLB_SHIFT) >> 20);
242 static inline unsigned long io_tlb_offset(unsigned long val)
244 return val & (IO_TLB_SEGSIZE - 1);
247 static inline unsigned long nr_slots(u64 val)
249 return DIV_ROUND_UP(val, IO_TLB_SIZE);
253 * Early SWIOTLB allocation may be too early to allow an architecture to
254 * perform the desired operations. This function allows the architecture to
255 * call SWIOTLB when the operations are possible. It needs to be called
256 * before the SWIOTLB memory is used.
258 void __init swiotlb_update_mem_attributes(void)
260 struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
263 if (!mem->nslabs || mem->late_alloc)
265 bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT);
266 set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT);
269 static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start,
270 unsigned long nslabs, bool late_alloc, unsigned int nareas)
272 void *vaddr = phys_to_virt(start);
273 unsigned long bytes = nslabs << IO_TLB_SHIFT, i;
275 mem->nslabs = nslabs;
277 mem->end = mem->start + bytes;
278 mem->late_alloc = late_alloc;
279 mem->nareas = nareas;
280 mem->area_nslabs = nslabs / mem->nareas;
282 for (i = 0; i < mem->nareas; i++) {
283 spin_lock_init(&mem->areas[i].lock);
284 mem->areas[i].index = 0;
285 mem->areas[i].used = 0;
288 for (i = 0; i < mem->nslabs; i++) {
289 mem->slots[i].list = min(IO_TLB_SEGSIZE - io_tlb_offset(i),
291 mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
292 mem->slots[i].alloc_size = 0;
293 mem->slots[i].pad_slots = 0;
296 memset(vaddr, 0, bytes);
302 * add_mem_pool() - add a memory pool to the allocator
303 * @mem: Software IO TLB allocator.
304 * @pool: Memory pool to be added.
306 static void add_mem_pool(struct io_tlb_mem *mem, struct io_tlb_pool *pool)
308 #ifdef CONFIG_SWIOTLB_DYNAMIC
309 spin_lock(&mem->lock);
310 list_add_rcu(&pool->node, &mem->pools);
311 mem->nslabs += pool->nslabs;
312 spin_unlock(&mem->lock);
314 mem->nslabs = pool->nslabs;
318 static void __init *swiotlb_memblock_alloc(unsigned long nslabs,
320 int (*remap)(void *tlb, unsigned long nslabs))
322 size_t bytes = PAGE_ALIGN(nslabs << IO_TLB_SHIFT);
326 * By default allocate the bounce buffer memory from low memory, but
327 * allow to pick a location everywhere for hypervisors with guest
330 if (flags & SWIOTLB_ANY)
331 tlb = memblock_alloc(bytes, PAGE_SIZE);
333 tlb = memblock_alloc_low(bytes, PAGE_SIZE);
336 pr_warn("%s: Failed to allocate %zu bytes tlb structure\n",
341 if (remap && remap(tlb, nslabs) < 0) {
342 memblock_free(tlb, PAGE_ALIGN(bytes));
343 pr_warn("%s: Failed to remap %zu bytes\n", __func__, bytes);
351 * Statically reserve bounce buffer space and initialize bounce buffer data
352 * structures for the software IO TLB used to implement the DMA API.
354 void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,
355 int (*remap)(void *tlb, unsigned long nslabs))
357 struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
358 unsigned long nslabs;
363 if (!addressing_limit && !swiotlb_force_bounce)
365 if (swiotlb_force_disable)
368 io_tlb_default_mem.force_bounce =
369 swiotlb_force_bounce || (flags & SWIOTLB_FORCE);
371 #ifdef CONFIG_SWIOTLB_DYNAMIC
373 io_tlb_default_mem.can_grow = true;
374 if (flags & SWIOTLB_ANY)
375 io_tlb_default_mem.phys_limit = virt_to_phys(high_memory - 1);
377 io_tlb_default_mem.phys_limit = ARCH_LOW_ADDRESS_LIMIT;
381 swiotlb_adjust_nareas(num_possible_cpus());
383 nslabs = default_nslabs;
384 nareas = limit_nareas(default_nareas, nslabs);
385 while ((tlb = swiotlb_memblock_alloc(nslabs, flags, remap)) == NULL) {
386 if (nslabs <= IO_TLB_MIN_SLABS)
388 nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
389 nareas = limit_nareas(nareas, nslabs);
392 if (default_nslabs != nslabs) {
393 pr_info("SWIOTLB bounce buffer size adjusted %lu -> %lu slabs",
394 default_nslabs, nslabs);
395 default_nslabs = nslabs;
398 alloc_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), nslabs));
399 mem->slots = memblock_alloc(alloc_size, PAGE_SIZE);
401 pr_warn("%s: Failed to allocate %zu bytes align=0x%lx\n",
402 __func__, alloc_size, PAGE_SIZE);
406 mem->areas = memblock_alloc(array_size(sizeof(struct io_tlb_area),
407 nareas), SMP_CACHE_BYTES);
409 pr_warn("%s: Failed to allocate mem->areas.\n", __func__);
413 swiotlb_init_io_tlb_pool(mem, __pa(tlb), nslabs, false, nareas);
414 add_mem_pool(&io_tlb_default_mem, mem);
416 if (flags & SWIOTLB_VERBOSE)
417 swiotlb_print_info();
420 void __init swiotlb_init(bool addressing_limit, unsigned int flags)
422 swiotlb_init_remap(addressing_limit, flags, NULL);
426 * Systems with larger DMA zones (those that don't support ISA) can
427 * initialize the swiotlb later using the slab allocator if needed.
428 * This should be just like above, but with some error catching.
430 int swiotlb_init_late(size_t size, gfp_t gfp_mask,
431 int (*remap)(void *tlb, unsigned long nslabs))
433 struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
434 unsigned long nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
436 unsigned char *vstart = NULL;
437 unsigned int order, area_order;
438 bool retried = false;
441 if (io_tlb_default_mem.nslabs)
444 if (swiotlb_force_disable)
447 io_tlb_default_mem.force_bounce = swiotlb_force_bounce;
449 #ifdef CONFIG_SWIOTLB_DYNAMIC
451 io_tlb_default_mem.can_grow = true;
452 if (IS_ENABLED(CONFIG_ZONE_DMA) && (gfp_mask & __GFP_DMA))
453 io_tlb_default_mem.phys_limit = zone_dma_limit;
454 else if (IS_ENABLED(CONFIG_ZONE_DMA32) && (gfp_mask & __GFP_DMA32))
455 io_tlb_default_mem.phys_limit = max(DMA_BIT_MASK(32), zone_dma_limit);
457 io_tlb_default_mem.phys_limit = virt_to_phys(high_memory - 1);
461 swiotlb_adjust_nareas(num_possible_cpus());
464 order = get_order(nslabs << IO_TLB_SHIFT);
465 nslabs = SLABS_PER_PAGE << order;
467 while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
468 vstart = (void *)__get_free_pages(gfp_mask | __GFP_NOWARN,
473 nslabs = SLABS_PER_PAGE << order;
481 rc = remap(vstart, nslabs);
483 free_pages((unsigned long)vstart, order);
485 nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
486 if (nslabs < IO_TLB_MIN_SLABS)
493 pr_warn("only able to allocate %ld MB\n",
494 (PAGE_SIZE << order) >> 20);
497 nareas = limit_nareas(default_nareas, nslabs);
498 area_order = get_order(array_size(sizeof(*mem->areas), nareas));
499 mem->areas = (struct io_tlb_area *)
500 __get_free_pages(GFP_KERNEL | __GFP_ZERO, area_order);
504 mem->slots = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
505 get_order(array_size(sizeof(*mem->slots), nslabs)));
509 set_memory_decrypted((unsigned long)vstart,
510 (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT);
511 swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), nslabs, true,
513 add_mem_pool(&io_tlb_default_mem, mem);
515 swiotlb_print_info();
519 free_pages((unsigned long)mem->areas, area_order);
521 free_pages((unsigned long)vstart, order);
525 void __init swiotlb_exit(void)
527 struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
528 unsigned long tbl_vaddr;
529 size_t tbl_size, slots_size;
530 unsigned int area_order;
532 if (swiotlb_force_bounce)
538 pr_info("tearing down default memory pool\n");
539 tbl_vaddr = (unsigned long)phys_to_virt(mem->start);
540 tbl_size = PAGE_ALIGN(mem->end - mem->start);
541 slots_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), mem->nslabs));
543 set_memory_encrypted(tbl_vaddr, tbl_size >> PAGE_SHIFT);
544 if (mem->late_alloc) {
545 area_order = get_order(array_size(sizeof(*mem->areas),
547 free_pages((unsigned long)mem->areas, area_order);
548 free_pages(tbl_vaddr, get_order(tbl_size));
549 free_pages((unsigned long)mem->slots, get_order(slots_size));
551 memblock_free_late(__pa(mem->areas),
552 array_size(sizeof(*mem->areas), mem->nareas));
553 memblock_free_late(mem->start, tbl_size);
554 memblock_free_late(__pa(mem->slots), slots_size);
557 memset(mem, 0, sizeof(*mem));
560 #ifdef CONFIG_SWIOTLB_DYNAMIC
563 * alloc_dma_pages() - allocate pages to be used for DMA
564 * @gfp: GFP flags for the allocation.
565 * @bytes: Size of the buffer.
566 * @phys_limit: Maximum allowed physical address of the buffer.
568 * Allocate pages from the buddy allocator. If successful, make the allocated
569 * pages decrypted that they can be used for DMA.
571 * Return: Decrypted pages, %NULL on allocation failure, or ERR_PTR(-EAGAIN)
572 * if the allocated physical address was above @phys_limit.
574 static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limit)
576 unsigned int order = get_order(bytes);
581 page = alloc_pages(gfp, order);
585 paddr = page_to_phys(page);
586 if (paddr + bytes - 1 > phys_limit) {
587 __free_pages(page, order);
588 return ERR_PTR(-EAGAIN);
591 vaddr = phys_to_virt(paddr);
592 if (set_memory_decrypted((unsigned long)vaddr, PFN_UP(bytes)))
597 /* Intentional leak if pages cannot be encrypted again. */
598 if (!set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
599 __free_pages(page, order);
604 * swiotlb_alloc_tlb() - allocate a dynamic IO TLB buffer
605 * @dev: Device for which a memory pool is allocated.
606 * @bytes: Size of the buffer.
607 * @phys_limit: Maximum allowed physical address of the buffer.
608 * @gfp: GFP flags for the allocation.
610 * Return: Allocated pages, or %NULL on allocation failure.
612 static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
613 u64 phys_limit, gfp_t gfp)
618 * Allocate from the atomic pools if memory is encrypted and
619 * the allocation is atomic, because decrypting may block.
621 if (!gfpflags_allow_blocking(gfp) && dev && force_dma_unencrypted(dev)) {
624 if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))
627 return dma_alloc_from_pool(dev, bytes, &vaddr, gfp,
631 gfp &= ~GFP_ZONEMASK;
632 if (phys_limit <= zone_dma_limit)
634 else if (phys_limit <= DMA_BIT_MASK(32))
637 while (IS_ERR(page = alloc_dma_pages(gfp, bytes, phys_limit))) {
638 if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
639 phys_limit < DMA_BIT_MASK(64) &&
640 !(gfp & (__GFP_DMA32 | __GFP_DMA)))
642 else if (IS_ENABLED(CONFIG_ZONE_DMA) &&
644 gfp = (gfp & ~__GFP_DMA32) | __GFP_DMA;
653 * swiotlb_free_tlb() - free a dynamically allocated IO TLB buffer
654 * @vaddr: Virtual address of the buffer.
655 * @bytes: Size of the buffer.
657 static void swiotlb_free_tlb(void *vaddr, size_t bytes)
659 if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
660 dma_free_from_pool(NULL, vaddr, bytes))
663 /* Intentional leak if pages cannot be encrypted again. */
664 if (!set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
665 __free_pages(virt_to_page(vaddr), get_order(bytes));
669 * swiotlb_alloc_pool() - allocate a new IO TLB memory pool
670 * @dev: Device for which a memory pool is allocated.
671 * @minslabs: Minimum number of slabs.
672 * @nslabs: Desired (maximum) number of slabs.
673 * @nareas: Number of areas.
674 * @phys_limit: Maximum DMA buffer physical address.
675 * @gfp: GFP flags for the allocations.
677 * Allocate and initialize a new IO TLB memory pool. The actual number of
678 * slabs may be reduced if allocation of @nslabs fails. If even
679 * @minslabs cannot be allocated, this function fails.
681 * Return: New memory pool, or %NULL on allocation failure.
683 static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
684 unsigned long minslabs, unsigned long nslabs,
685 unsigned int nareas, u64 phys_limit, gfp_t gfp)
687 struct io_tlb_pool *pool;
688 unsigned int slot_order;
693 if (nslabs > SLABS_PER_PAGE << MAX_PAGE_ORDER) {
694 nslabs = SLABS_PER_PAGE << MAX_PAGE_ORDER;
695 nareas = limit_nareas(nareas, nslabs);
698 pool_size = sizeof(*pool) + array_size(sizeof(*pool->areas), nareas);
699 pool = kzalloc(pool_size, gfp);
702 pool->areas = (void *)pool + sizeof(*pool);
704 tlb_size = nslabs << IO_TLB_SHIFT;
705 while (!(tlb = swiotlb_alloc_tlb(dev, tlb_size, phys_limit, gfp))) {
706 if (nslabs <= minslabs)
708 nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
709 nareas = limit_nareas(nareas, nslabs);
710 tlb_size = nslabs << IO_TLB_SHIFT;
713 slot_order = get_order(array_size(sizeof(*pool->slots), nslabs));
714 pool->slots = (struct io_tlb_slot *)
715 __get_free_pages(gfp, slot_order);
719 swiotlb_init_io_tlb_pool(pool, page_to_phys(tlb), nslabs, true, nareas);
723 swiotlb_free_tlb(page_address(tlb), tlb_size);
731 * swiotlb_dyn_alloc() - dynamic memory pool allocation worker
732 * @work: Pointer to dyn_alloc in struct io_tlb_mem.
734 static void swiotlb_dyn_alloc(struct work_struct *work)
736 struct io_tlb_mem *mem =
737 container_of(work, struct io_tlb_mem, dyn_alloc);
738 struct io_tlb_pool *pool;
740 pool = swiotlb_alloc_pool(NULL, IO_TLB_MIN_SLABS, default_nslabs,
741 default_nareas, mem->phys_limit, GFP_KERNEL);
743 pr_warn_ratelimited("Failed to allocate new pool");
747 add_mem_pool(mem, pool);
751 * swiotlb_dyn_free() - RCU callback to free a memory pool
752 * @rcu: RCU head in the corresponding struct io_tlb_pool.
754 static void swiotlb_dyn_free(struct rcu_head *rcu)
756 struct io_tlb_pool *pool = container_of(rcu, struct io_tlb_pool, rcu);
757 size_t slots_size = array_size(sizeof(*pool->slots), pool->nslabs);
758 size_t tlb_size = pool->end - pool->start;
760 free_pages((unsigned long)pool->slots, get_order(slots_size));
761 swiotlb_free_tlb(pool->vaddr, tlb_size);
766 * __swiotlb_find_pool() - find the IO TLB pool for a physical address
767 * @dev: Device which has mapped the DMA buffer.
768 * @paddr: Physical address within the DMA buffer.
770 * Find the IO TLB memory pool descriptor which contains the given physical
771 * address, if any. This function is for use only when the dev is known to
772 * be using swiotlb. Use swiotlb_find_pool() for the more general case
773 * when this condition is not met.
775 * Return: Memory pool which contains @paddr, or %NULL if none.
777 struct io_tlb_pool *__swiotlb_find_pool(struct device *dev, phys_addr_t paddr)
779 struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
780 struct io_tlb_pool *pool;
783 list_for_each_entry_rcu(pool, &mem->pools, node) {
784 if (paddr >= pool->start && paddr < pool->end)
788 list_for_each_entry_rcu(pool, &dev->dma_io_tlb_pools, node) {
789 if (paddr >= pool->start && paddr < pool->end)
799 * swiotlb_del_pool() - remove an IO TLB pool from a device
800 * @dev: Owning device.
801 * @pool: Memory pool to be removed.
803 static void swiotlb_del_pool(struct device *dev, struct io_tlb_pool *pool)
807 spin_lock_irqsave(&dev->dma_io_tlb_lock, flags);
808 list_del_rcu(&pool->node);
809 spin_unlock_irqrestore(&dev->dma_io_tlb_lock, flags);
811 call_rcu(&pool->rcu, swiotlb_dyn_free);
814 #endif /* CONFIG_SWIOTLB_DYNAMIC */
817 * swiotlb_dev_init() - initialize swiotlb fields in &struct device
818 * @dev: Device to be initialized.
820 void swiotlb_dev_init(struct device *dev)
822 dev->dma_io_tlb_mem = &io_tlb_default_mem;
823 #ifdef CONFIG_SWIOTLB_DYNAMIC
824 INIT_LIST_HEAD(&dev->dma_io_tlb_pools);
825 spin_lock_init(&dev->dma_io_tlb_lock);
826 dev->dma_uses_io_tlb = false;
831 * swiotlb_align_offset() - Get required offset into an IO TLB allocation.
832 * @dev: Owning device.
833 * @align_mask: Allocation alignment mask.
834 * @addr: DMA address.
836 * Return the minimum offset from the start of an IO TLB allocation which is
837 * required for a given buffer address and allocation alignment to keep the
840 * First, the address bits covered by min_align_mask must be identical in the
841 * original address and the bounce buffer address. High bits are preserved by
842 * choosing a suitable IO TLB slot, but bits below IO_TLB_SHIFT require extra
843 * padding bytes before the bounce buffer.
845 * Second, @align_mask specifies which bits of the first allocated slot must
846 * be zero. This may require allocating additional padding slots, and then the
847 * offset (in bytes) from the first such padding slot is returned.
849 static unsigned int swiotlb_align_offset(struct device *dev,
850 unsigned int align_mask, u64 addr)
852 return addr & dma_get_min_align_mask(dev) &
853 (align_mask | (IO_TLB_SIZE - 1));
857 * Bounce: copy the swiotlb buffer from or back to the original dma location
859 static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size,
860 enum dma_data_direction dir, struct io_tlb_pool *mem)
862 int index = (tlb_addr - mem->start) >> IO_TLB_SHIFT;
863 phys_addr_t orig_addr = mem->slots[index].orig_addr;
864 size_t alloc_size = mem->slots[index].alloc_size;
865 unsigned long pfn = PFN_DOWN(orig_addr);
866 unsigned char *vaddr = mem->vaddr + tlb_addr - mem->start;
869 if (orig_addr == INVALID_PHYS_ADDR)
873 * It's valid for tlb_offset to be negative. This can happen when the
874 * "offset" returned by swiotlb_align_offset() is non-zero, and the
875 * tlb_addr is pointing within the first "offset" bytes of the second
876 * or subsequent slots of the allocated swiotlb area. While it's not
877 * valid for tlb_addr to be pointing within the first "offset" bytes
878 * of the first slot, there's no way to check for such an error since
879 * this function can't distinguish the first slot from the second and
882 tlb_offset = (tlb_addr & (IO_TLB_SIZE - 1)) -
883 swiotlb_align_offset(dev, 0, orig_addr);
885 orig_addr += tlb_offset;
886 alloc_size -= tlb_offset;
888 if (size > alloc_size) {
889 dev_WARN_ONCE(dev, 1,
890 "Buffer overflow detected. Allocation size: %zu. Mapping size: %zu.\n",
895 if (PageHighMem(pfn_to_page(pfn))) {
896 unsigned int offset = orig_addr & ~PAGE_MASK;
902 sz = min_t(size_t, PAGE_SIZE - offset, size);
904 local_irq_save(flags);
905 page = pfn_to_page(pfn);
906 if (dir == DMA_TO_DEVICE)
907 memcpy_from_page(vaddr, page, offset, sz);
909 memcpy_to_page(page, offset, vaddr, sz);
910 local_irq_restore(flags);
917 } else if (dir == DMA_TO_DEVICE) {
918 memcpy(vaddr, phys_to_virt(orig_addr), size);
920 memcpy(phys_to_virt(orig_addr), vaddr, size);
924 static inline phys_addr_t slot_addr(phys_addr_t start, phys_addr_t idx)
926 return start + (idx << IO_TLB_SHIFT);
930 * Carefully handle integer overflow which can occur when boundary_mask == ~0UL.
932 static inline unsigned long get_max_slots(unsigned long boundary_mask)
934 return (boundary_mask >> IO_TLB_SHIFT) + 1;
937 static unsigned int wrap_area_index(struct io_tlb_pool *mem, unsigned int index)
939 if (index >= mem->area_nslabs)
945 * Track the total used slots with a global atomic value in order to have
946 * correct information to determine the high water mark. The mem_used()
947 * function gives imprecise results because there's no locking across
950 #ifdef CONFIG_DEBUG_FS
951 static void inc_used_and_hiwater(struct io_tlb_mem *mem, unsigned int nslots)
953 unsigned long old_hiwater, new_used;
955 new_used = atomic_long_add_return(nslots, &mem->total_used);
956 old_hiwater = atomic_long_read(&mem->used_hiwater);
958 if (new_used <= old_hiwater)
960 } while (!atomic_long_try_cmpxchg(&mem->used_hiwater,
961 &old_hiwater, new_used));
964 static void dec_used(struct io_tlb_mem *mem, unsigned int nslots)
966 atomic_long_sub(nslots, &mem->total_used);
969 #else /* !CONFIG_DEBUG_FS */
970 static void inc_used_and_hiwater(struct io_tlb_mem *mem, unsigned int nslots)
973 static void dec_used(struct io_tlb_mem *mem, unsigned int nslots)
976 #endif /* CONFIG_DEBUG_FS */
978 #ifdef CONFIG_SWIOTLB_DYNAMIC
979 #ifdef CONFIG_DEBUG_FS
980 static void inc_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
982 atomic_long_add(nslots, &mem->transient_nslabs);
985 static void dec_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
987 atomic_long_sub(nslots, &mem->transient_nslabs);
990 #else /* !CONFIG_DEBUG_FS */
991 static void inc_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
994 static void dec_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
997 #endif /* CONFIG_DEBUG_FS */
998 #endif /* CONFIG_SWIOTLB_DYNAMIC */
1001 * swiotlb_search_pool_area() - search one memory area in one pool
1002 * @dev: Device which maps the buffer.
1003 * @pool: Memory pool to be searched.
1004 * @area_index: Index of the IO TLB memory area to be searched.
1005 * @orig_addr: Original (non-bounced) IO buffer address.
1006 * @alloc_size: Total requested size of the bounce buffer,
1007 * including initial alignment padding.
1008 * @alloc_align_mask: Required alignment of the allocated buffer.
1010 * Find a suitable sequence of IO TLB entries for the request and allocate
1011 * a buffer from the given IO TLB memory area.
1012 * This function takes care of locking.
1014 * Return: Index of the first allocated slot, or -1 on error.
1016 static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool,
1017 int area_index, phys_addr_t orig_addr, size_t alloc_size,
1018 unsigned int alloc_align_mask)
1020 struct io_tlb_area *area = pool->areas + area_index;
1021 unsigned long boundary_mask = dma_get_seg_boundary(dev);
1022 dma_addr_t tbl_dma_addr =
1023 phys_to_dma_unencrypted(dev, pool->start) & boundary_mask;
1024 unsigned long max_slots = get_max_slots(boundary_mask);
1025 unsigned int iotlb_align_mask = dma_get_min_align_mask(dev);
1026 unsigned int nslots = nr_slots(alloc_size), stride;
1027 unsigned int offset = swiotlb_align_offset(dev, 0, orig_addr);
1028 unsigned int index, slots_checked, count = 0, i;
1029 unsigned long flags;
1030 unsigned int slot_base;
1031 unsigned int slot_index;
1034 BUG_ON(area_index >= pool->nareas);
1037 * Historically, swiotlb allocations >= PAGE_SIZE were guaranteed to be
1038 * page-aligned in the absence of any other alignment requirements.
1039 * 'alloc_align_mask' was later introduced to specify the alignment
1040 * explicitly, however this is passed as zero for streaming mappings
1041 * and so we preserve the old behaviour there in case any drivers are
1044 if (!alloc_align_mask && !iotlb_align_mask && alloc_size >= PAGE_SIZE)
1045 alloc_align_mask = PAGE_SIZE - 1;
1048 * Ensure that the allocation is at least slot-aligned and update
1049 * 'iotlb_align_mask' to ignore bits that will be preserved when
1050 * offsetting into the allocation.
1052 alloc_align_mask |= (IO_TLB_SIZE - 1);
1053 iotlb_align_mask &= ~alloc_align_mask;
1056 * For mappings with an alignment requirement don't bother looping to
1057 * unaligned slots once we found an aligned one.
1059 stride = get_max_slots(max(alloc_align_mask, iotlb_align_mask));
1061 spin_lock_irqsave(&area->lock, flags);
1062 if (unlikely(nslots > pool->area_nslabs - area->used))
1065 slot_base = area_index * pool->area_nslabs;
1066 index = area->index;
1068 for (slots_checked = 0; slots_checked < pool->area_nslabs; ) {
1069 phys_addr_t tlb_addr;
1071 slot_index = slot_base + index;
1072 tlb_addr = slot_addr(tbl_dma_addr, slot_index);
1074 if ((tlb_addr & alloc_align_mask) ||
1075 (orig_addr && (tlb_addr & iotlb_align_mask) !=
1076 (orig_addr & iotlb_align_mask))) {
1077 index = wrap_area_index(pool, index + 1);
1082 if (!iommu_is_span_boundary(slot_index, nslots,
1083 nr_slots(tbl_dma_addr),
1085 if (pool->slots[slot_index].list >= nslots)
1088 index = wrap_area_index(pool, index + stride);
1089 slots_checked += stride;
1093 spin_unlock_irqrestore(&area->lock, flags);
1098 * If we find a slot that indicates we have 'nslots' number of
1099 * contiguous buffers, we allocate the buffers from that slot onwards
1100 * and set the list of free entries to '0' indicating unavailable.
1102 for (i = slot_index; i < slot_index + nslots; i++) {
1103 pool->slots[i].list = 0;
1104 pool->slots[i].alloc_size = alloc_size - (offset +
1105 ((i - slot_index) << IO_TLB_SHIFT));
1107 for (i = slot_index - 1;
1108 io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 &&
1109 pool->slots[i].list; i--)
1110 pool->slots[i].list = ++count;
1113 * Update the indices to avoid searching in the next round.
1115 area->index = wrap_area_index(pool, index + nslots);
1116 area->used += nslots;
1117 spin_unlock_irqrestore(&area->lock, flags);
1119 inc_used_and_hiwater(dev->dma_io_tlb_mem, nslots);
1123 #ifdef CONFIG_SWIOTLB_DYNAMIC
1126 * swiotlb_search_area() - search one memory area in all pools
1127 * @dev: Device which maps the buffer.
1128 * @start_cpu: Start CPU number.
1129 * @cpu_offset: Offset from @start_cpu.
1130 * @orig_addr: Original (non-bounced) IO buffer address.
1131 * @alloc_size: Total requested size of the bounce buffer,
1132 * including initial alignment padding.
1133 * @alloc_align_mask: Required alignment of the allocated buffer.
1134 * @retpool: Used memory pool, updated on return.
1136 * Search one memory area in all pools for a sequence of slots that match the
1137 * allocation constraints.
1139 * Return: Index of the first allocated slot, or -1 on error.
1141 static int swiotlb_search_area(struct device *dev, int start_cpu,
1142 int cpu_offset, phys_addr_t orig_addr, size_t alloc_size,
1143 unsigned int alloc_align_mask, struct io_tlb_pool **retpool)
1145 struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1146 struct io_tlb_pool *pool;
1151 list_for_each_entry_rcu(pool, &mem->pools, node) {
1152 if (cpu_offset >= pool->nareas)
1154 area_index = (start_cpu + cpu_offset) & (pool->nareas - 1);
1155 index = swiotlb_search_pool_area(dev, pool, area_index,
1156 orig_addr, alloc_size,
1168 * swiotlb_find_slots() - search for slots in the whole swiotlb
1169 * @dev: Device which maps the buffer.
1170 * @orig_addr: Original (non-bounced) IO buffer address.
1171 * @alloc_size: Total requested size of the bounce buffer,
1172 * including initial alignment padding.
1173 * @alloc_align_mask: Required alignment of the allocated buffer.
1174 * @retpool: Used memory pool, updated on return.
1176 * Search through the whole software IO TLB to find a sequence of slots that
1177 * match the allocation constraints.
1179 * Return: Index of the first allocated slot, or -1 on error.
1181 static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
1182 size_t alloc_size, unsigned int alloc_align_mask,
1183 struct io_tlb_pool **retpool)
1185 struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1186 struct io_tlb_pool *pool;
1187 unsigned long nslabs;
1188 unsigned long flags;
1193 if (alloc_size > IO_TLB_SEGSIZE * IO_TLB_SIZE)
1196 cpu = raw_smp_processor_id();
1197 for (i = 0; i < default_nareas; ++i) {
1198 index = swiotlb_search_area(dev, cpu, i, orig_addr, alloc_size,
1199 alloc_align_mask, &pool);
1207 schedule_work(&mem->dyn_alloc);
1209 nslabs = nr_slots(alloc_size);
1210 phys_limit = min_not_zero(*dev->dma_mask, dev->bus_dma_limit);
1211 pool = swiotlb_alloc_pool(dev, nslabs, nslabs, 1, phys_limit,
1212 GFP_NOWAIT | __GFP_NOWARN);
1216 index = swiotlb_search_pool_area(dev, pool, 0, orig_addr,
1217 alloc_size, alloc_align_mask);
1219 swiotlb_dyn_free(&pool->rcu);
1223 pool->transient = true;
1224 spin_lock_irqsave(&dev->dma_io_tlb_lock, flags);
1225 list_add_rcu(&pool->node, &dev->dma_io_tlb_pools);
1226 spin_unlock_irqrestore(&dev->dma_io_tlb_lock, flags);
1227 inc_transient_used(mem, pool->nslabs);
1230 WRITE_ONCE(dev->dma_uses_io_tlb, true);
1233 * The general barrier orders reads and writes against a presumed store
1234 * of the SWIOTLB buffer address by a device driver (to a driver private
1235 * data structure). It serves two purposes.
1237 * First, the store to dev->dma_uses_io_tlb must be ordered before the
1238 * presumed store. This guarantees that the returned buffer address
1239 * cannot be passed to another CPU before updating dev->dma_uses_io_tlb.
1241 * Second, the load from mem->pools must be ordered before the same
1242 * presumed store. This guarantees that the returned buffer address
1243 * cannot be observed by another CPU before an update of the RCU list
1244 * that was made by swiotlb_dyn_alloc() on a third CPU (cf. multicopy
1247 * See also the comment in swiotlb_find_pool().
1255 #else /* !CONFIG_SWIOTLB_DYNAMIC */
1257 static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
1258 size_t alloc_size, unsigned int alloc_align_mask,
1259 struct io_tlb_pool **retpool)
1261 struct io_tlb_pool *pool;
1265 *retpool = pool = &dev->dma_io_tlb_mem->defpool;
1266 i = start = raw_smp_processor_id() & (pool->nareas - 1);
1268 index = swiotlb_search_pool_area(dev, pool, i, orig_addr,
1269 alloc_size, alloc_align_mask);
1272 if (++i >= pool->nareas)
1274 } while (i != start);
1278 #endif /* CONFIG_SWIOTLB_DYNAMIC */
1280 #ifdef CONFIG_DEBUG_FS
1283 * mem_used() - get number of used slots in an allocator
1284 * @mem: Software IO TLB allocator.
1286 * The result is accurate in this version of the function, because an atomic
1287 * counter is available if CONFIG_DEBUG_FS is set.
1289 * Return: Number of used slots.
1291 static unsigned long mem_used(struct io_tlb_mem *mem)
1293 return atomic_long_read(&mem->total_used);
1296 #else /* !CONFIG_DEBUG_FS */
1299 * mem_pool_used() - get number of used slots in a memory pool
1300 * @pool: Software IO TLB memory pool.
1302 * The result is not accurate, see mem_used().
1304 * Return: Approximate number of used slots.
1306 static unsigned long mem_pool_used(struct io_tlb_pool *pool)
1309 unsigned long used = 0;
1311 for (i = 0; i < pool->nareas; i++)
1312 used += pool->areas[i].used;
1317 * mem_used() - get number of used slots in an allocator
1318 * @mem: Software IO TLB allocator.
1320 * The result is not accurate, because there is no locking of individual
1323 * Return: Approximate number of used slots.
1325 static unsigned long mem_used(struct io_tlb_mem *mem)
1327 #ifdef CONFIG_SWIOTLB_DYNAMIC
1328 struct io_tlb_pool *pool;
1329 unsigned long used = 0;
1332 list_for_each_entry_rcu(pool, &mem->pools, node)
1333 used += mem_pool_used(pool);
1338 return mem_pool_used(&mem->defpool);
1342 #endif /* CONFIG_DEBUG_FS */
1345 * swiotlb_tbl_map_single() - bounce buffer map a single contiguous physical area
1346 * @dev: Device which maps the buffer.
1347 * @orig_addr: Original (non-bounced) physical IO buffer address
1348 * @mapping_size: Requested size of the actual bounce buffer, excluding
1349 * any pre- or post-padding for alignment
1350 * @alloc_align_mask: Required start and end alignment of the allocated buffer
1351 * @dir: DMA direction
1352 * @attrs: Optional DMA attributes for the map operation
1354 * Find and allocate a suitable sequence of IO TLB slots for the request.
1355 * The allocated space starts at an alignment specified by alloc_align_mask,
1356 * and the size of the allocated space is rounded up so that the total amount
1357 * of allocated space is a multiple of (alloc_align_mask + 1). If
1358 * alloc_align_mask is zero, the allocated space may be at any alignment and
1359 * the size is not rounded up.
1361 * The returned address is within the allocated space and matches the bits
1362 * of orig_addr that are specified in the DMA min_align_mask for the device. As
1363 * such, this returned address may be offset from the beginning of the allocated
1364 * space. The bounce buffer space starting at the returned address for
1365 * mapping_size bytes is initialized to the contents of the original IO buffer
1366 * area. Any pre-padding (due to an offset) and any post-padding (due to
1367 * rounding-up the size) is not initialized.
1369 phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
1370 size_t mapping_size, unsigned int alloc_align_mask,
1371 enum dma_data_direction dir, unsigned long attrs)
1373 struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1374 unsigned int offset;
1375 struct io_tlb_pool *pool;
1379 phys_addr_t tlb_addr;
1380 unsigned short pad_slots;
1382 if (!mem || !mem->nslabs) {
1383 dev_warn_ratelimited(dev,
1384 "Can not allocate SWIOTLB buffer earlier and can't now provide you with the DMA bounce buffer");
1385 return (phys_addr_t)DMA_MAPPING_ERROR;
1388 if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
1389 pr_warn_once("Memory encryption is active and system is using DMA bounce buffers\n");
1392 * The default swiotlb memory pool is allocated with PAGE_SIZE
1393 * alignment. If a mapping is requested with larger alignment,
1394 * the mapping may be unable to use the initial slot(s) in all
1395 * sets of IO_TLB_SEGSIZE slots. In such case, a mapping request
1396 * of or near the maximum mapping size would always fail.
1398 dev_WARN_ONCE(dev, alloc_align_mask > ~PAGE_MASK,
1399 "Alloc alignment may prevent fulfilling requests with max mapping_size\n");
1401 offset = swiotlb_align_offset(dev, alloc_align_mask, orig_addr);
1402 size = ALIGN(mapping_size + offset, alloc_align_mask + 1);
1403 index = swiotlb_find_slots(dev, orig_addr, size, alloc_align_mask, &pool);
1405 if (!(attrs & DMA_ATTR_NO_WARN))
1406 dev_warn_ratelimited(dev,
1407 "swiotlb buffer is full (sz: %zd bytes), total %lu (slots), used %lu (slots)\n",
1408 size, mem->nslabs, mem_used(mem));
1409 return (phys_addr_t)DMA_MAPPING_ERROR;
1413 * If dma_skip_sync was set, reset it on first SWIOTLB buffer
1414 * mapping to always sync SWIOTLB buffers.
1416 dma_reset_need_sync(dev);
1419 * Save away the mapping from the original address to the DMA address.
1420 * This is needed when we sync the memory. Then we sync the buffer if
1423 pad_slots = offset >> IO_TLB_SHIFT;
1424 offset &= (IO_TLB_SIZE - 1);
1426 pool->slots[index].pad_slots = pad_slots;
1427 for (i = 0; i < (nr_slots(size) - pad_slots); i++)
1428 pool->slots[index + i].orig_addr = slot_addr(orig_addr, i);
1429 tlb_addr = slot_addr(pool->start, index) + offset;
1431 * When the device is writing memory, i.e. dir == DMA_FROM_DEVICE, copy
1432 * the original buffer to the TLB buffer before initiating DMA in order
1433 * to preserve the original's data if the device does a partial write,
1434 * i.e. if the device doesn't overwrite the entire buffer. Preserving
1435 * the original data, even if it's garbage, is necessary to match
1436 * hardware behavior. Use of swiotlb is supposed to be transparent,
1437 * i.e. swiotlb must not corrupt memory by clobbering unwritten bytes.
1439 swiotlb_bounce(dev, tlb_addr, mapping_size, DMA_TO_DEVICE, pool);
1443 static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr,
1444 struct io_tlb_pool *mem)
1446 unsigned long flags;
1447 unsigned int offset = swiotlb_align_offset(dev, 0, tlb_addr);
1448 int index, nslots, aindex;
1449 struct io_tlb_area *area;
1452 index = (tlb_addr - offset - mem->start) >> IO_TLB_SHIFT;
1453 index -= mem->slots[index].pad_slots;
1454 nslots = nr_slots(mem->slots[index].alloc_size + offset);
1455 aindex = index / mem->area_nslabs;
1456 area = &mem->areas[aindex];
1459 * Return the buffer to the free list by setting the corresponding
1460 * entries to indicate the number of contiguous entries available.
1461 * While returning the entries to the free list, we merge the entries
1462 * with slots below and above the pool being returned.
1464 BUG_ON(aindex >= mem->nareas);
1466 spin_lock_irqsave(&area->lock, flags);
1467 if (index + nslots < ALIGN(index + 1, IO_TLB_SEGSIZE))
1468 count = mem->slots[index + nslots].list;
1473 * Step 1: return the slots to the free list, merging the slots with
1474 * superceeding slots
1476 for (i = index + nslots - 1; i >= index; i--) {
1477 mem->slots[i].list = ++count;
1478 mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
1479 mem->slots[i].alloc_size = 0;
1480 mem->slots[i].pad_slots = 0;
1484 * Step 2: merge the returned slots with the preceding slots, if
1485 * available (non zero)
1488 io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 && mem->slots[i].list;
1490 mem->slots[i].list = ++count;
1491 area->used -= nslots;
1492 spin_unlock_irqrestore(&area->lock, flags);
1494 dec_used(dev->dma_io_tlb_mem, nslots);
1497 #ifdef CONFIG_SWIOTLB_DYNAMIC
1500 * swiotlb_del_transient() - delete a transient memory pool
1501 * @dev: Device which mapped the buffer.
1502 * @tlb_addr: Physical address within a bounce buffer.
1503 * @pool: Pointer to the transient memory pool to be checked and deleted.
1505 * Check whether the address belongs to a transient SWIOTLB memory pool.
1506 * If yes, then delete the pool.
1508 * Return: %true if @tlb_addr belonged to a transient pool that was released.
1510 static bool swiotlb_del_transient(struct device *dev, phys_addr_t tlb_addr,
1511 struct io_tlb_pool *pool)
1513 if (!pool->transient)
1516 dec_used(dev->dma_io_tlb_mem, pool->nslabs);
1517 swiotlb_del_pool(dev, pool);
1518 dec_transient_used(dev->dma_io_tlb_mem, pool->nslabs);
1522 #else /* !CONFIG_SWIOTLB_DYNAMIC */
1524 static inline bool swiotlb_del_transient(struct device *dev,
1525 phys_addr_t tlb_addr, struct io_tlb_pool *pool)
1530 #endif /* CONFIG_SWIOTLB_DYNAMIC */
1533 * tlb_addr is the physical address of the bounce buffer to unmap.
1535 void __swiotlb_tbl_unmap_single(struct device *dev, phys_addr_t tlb_addr,
1536 size_t mapping_size, enum dma_data_direction dir,
1537 unsigned long attrs, struct io_tlb_pool *pool)
1540 * First, sync the memory before unmapping the entry
1542 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
1543 (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
1544 swiotlb_bounce(dev, tlb_addr, mapping_size,
1545 DMA_FROM_DEVICE, pool);
1547 if (swiotlb_del_transient(dev, tlb_addr, pool))
1549 swiotlb_release_slots(dev, tlb_addr, pool);
1552 void __swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr,
1553 size_t size, enum dma_data_direction dir,
1554 struct io_tlb_pool *pool)
1556 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
1557 swiotlb_bounce(dev, tlb_addr, size, DMA_TO_DEVICE, pool);
1559 BUG_ON(dir != DMA_FROM_DEVICE);
1562 void __swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr,
1563 size_t size, enum dma_data_direction dir,
1564 struct io_tlb_pool *pool)
1566 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
1567 swiotlb_bounce(dev, tlb_addr, size, DMA_FROM_DEVICE, pool);
1569 BUG_ON(dir != DMA_TO_DEVICE);
1573 * Create a swiotlb mapping for the buffer at @paddr, and in case of DMAing
1574 * to the device copy the data into it as well.
1576 dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size,
1577 enum dma_data_direction dir, unsigned long attrs)
1579 phys_addr_t swiotlb_addr;
1580 dma_addr_t dma_addr;
1582 trace_swiotlb_bounced(dev, phys_to_dma(dev, paddr), size);
1584 swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, 0, dir, attrs);
1585 if (swiotlb_addr == (phys_addr_t)DMA_MAPPING_ERROR)
1586 return DMA_MAPPING_ERROR;
1588 /* Ensure that the address returned is DMA'ble */
1589 dma_addr = phys_to_dma_unencrypted(dev, swiotlb_addr);
1590 if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
1591 __swiotlb_tbl_unmap_single(dev, swiotlb_addr, size, dir,
1592 attrs | DMA_ATTR_SKIP_CPU_SYNC,
1593 swiotlb_find_pool(dev, swiotlb_addr));
1594 dev_WARN_ONCE(dev, 1,
1595 "swiotlb addr %pad+%zu overflow (mask %llx, bus limit %llx).\n",
1596 &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);
1597 return DMA_MAPPING_ERROR;
1600 if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
1601 arch_sync_dma_for_device(swiotlb_addr, size, dir);
1605 size_t swiotlb_max_mapping_size(struct device *dev)
1607 int min_align_mask = dma_get_min_align_mask(dev);
1611 * swiotlb_find_slots() skips slots according to
1612 * min align mask. This affects max mapping size.
1613 * Take it into acount here.
1616 min_align = roundup(min_align_mask, IO_TLB_SIZE);
1618 return ((size_t)IO_TLB_SIZE) * IO_TLB_SEGSIZE - min_align;
1622 * is_swiotlb_allocated() - check if the default software IO TLB is initialized
1624 bool is_swiotlb_allocated(void)
1626 return io_tlb_default_mem.nslabs;
1629 bool is_swiotlb_active(struct device *dev)
1631 struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1633 return mem && mem->nslabs;
1637 * default_swiotlb_base() - get the base address of the default SWIOTLB
1639 * Get the lowest physical address used by the default software IO TLB pool.
1641 phys_addr_t default_swiotlb_base(void)
1643 #ifdef CONFIG_SWIOTLB_DYNAMIC
1644 io_tlb_default_mem.can_grow = false;
1646 return io_tlb_default_mem.defpool.start;
1650 * default_swiotlb_limit() - get the address limit of the default SWIOTLB
1652 * Get the highest physical address used by the default software IO TLB pool.
1654 phys_addr_t default_swiotlb_limit(void)
1656 #ifdef CONFIG_SWIOTLB_DYNAMIC
1657 return io_tlb_default_mem.phys_limit;
1659 return io_tlb_default_mem.defpool.end - 1;
1663 #ifdef CONFIG_DEBUG_FS
1664 #ifdef CONFIG_SWIOTLB_DYNAMIC
1665 static unsigned long mem_transient_used(struct io_tlb_mem *mem)
1667 return atomic_long_read(&mem->transient_nslabs);
1670 static int io_tlb_transient_used_get(void *data, u64 *val)
1672 struct io_tlb_mem *mem = data;
1674 *val = mem_transient_used(mem);
1678 DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_transient_used, io_tlb_transient_used_get,
1680 #endif /* CONFIG_SWIOTLB_DYNAMIC */
1682 static int io_tlb_used_get(void *data, u64 *val)
1684 struct io_tlb_mem *mem = data;
1686 *val = mem_used(mem);
1690 static int io_tlb_hiwater_get(void *data, u64 *val)
1692 struct io_tlb_mem *mem = data;
1694 *val = atomic_long_read(&mem->used_hiwater);
1698 static int io_tlb_hiwater_set(void *data, u64 val)
1700 struct io_tlb_mem *mem = data;
1702 /* Only allow setting to zero */
1706 atomic_long_set(&mem->used_hiwater, val);
1710 DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_used, io_tlb_used_get, NULL, "%llu\n");
1711 DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_hiwater, io_tlb_hiwater_get,
1712 io_tlb_hiwater_set, "%llu\n");
1714 static void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
1715 const char *dirname)
1717 mem->debugfs = debugfs_create_dir(dirname, io_tlb_default_mem.debugfs);
1721 debugfs_create_ulong("io_tlb_nslabs", 0400, mem->debugfs, &mem->nslabs);
1722 debugfs_create_file("io_tlb_used", 0400, mem->debugfs, mem,
1724 debugfs_create_file("io_tlb_used_hiwater", 0600, mem->debugfs, mem,
1725 &fops_io_tlb_hiwater);
1726 #ifdef CONFIG_SWIOTLB_DYNAMIC
1727 debugfs_create_file("io_tlb_transient_nslabs", 0400, mem->debugfs,
1728 mem, &fops_io_tlb_transient_used);
1732 static int __init swiotlb_create_default_debugfs(void)
1734 swiotlb_create_debugfs_files(&io_tlb_default_mem, "swiotlb");
1738 late_initcall(swiotlb_create_default_debugfs);
1740 #else /* !CONFIG_DEBUG_FS */
1742 static inline void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
1743 const char *dirname)
1747 #endif /* CONFIG_DEBUG_FS */
1749 #ifdef CONFIG_DMA_RESTRICTED_POOL
1751 struct page *swiotlb_alloc(struct device *dev, size_t size)
1753 struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1754 struct io_tlb_pool *pool;
1755 phys_addr_t tlb_addr;
1762 align = (1 << (get_order(size) + PAGE_SHIFT)) - 1;
1763 index = swiotlb_find_slots(dev, 0, size, align, &pool);
1767 tlb_addr = slot_addr(pool->start, index);
1768 if (unlikely(!PAGE_ALIGNED(tlb_addr))) {
1769 dev_WARN_ONCE(dev, 1, "Cannot allocate pages from non page-aligned swiotlb addr 0x%pa.\n",
1771 swiotlb_release_slots(dev, tlb_addr, pool);
1775 return pfn_to_page(PFN_DOWN(tlb_addr));
1778 bool swiotlb_free(struct device *dev, struct page *page, size_t size)
1780 phys_addr_t tlb_addr = page_to_phys(page);
1781 struct io_tlb_pool *pool;
1783 pool = swiotlb_find_pool(dev, tlb_addr);
1787 swiotlb_release_slots(dev, tlb_addr, pool);
1792 static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
1795 struct io_tlb_mem *mem = rmem->priv;
1796 unsigned long nslabs = rmem->size >> IO_TLB_SHIFT;
1798 /* Set Per-device io tlb area to one */
1799 unsigned int nareas = 1;
1801 if (PageHighMem(pfn_to_page(PHYS_PFN(rmem->base)))) {
1802 dev_err(dev, "Restricted DMA pool must be accessible within the linear mapping.");
1807 * Since multiple devices can share the same pool, the private data,
1808 * io_tlb_mem struct, will be initialized by the first device attached
1812 struct io_tlb_pool *pool;
1814 mem = kzalloc(sizeof(*mem), GFP_KERNEL);
1817 pool = &mem->defpool;
1819 pool->slots = kcalloc(nslabs, sizeof(*pool->slots), GFP_KERNEL);
1825 pool->areas = kcalloc(nareas, sizeof(*pool->areas),
1833 set_memory_decrypted((unsigned long)phys_to_virt(rmem->base),
1834 rmem->size >> PAGE_SHIFT);
1835 swiotlb_init_io_tlb_pool(pool, rmem->base, nslabs,
1837 mem->force_bounce = true;
1838 mem->for_alloc = true;
1839 #ifdef CONFIG_SWIOTLB_DYNAMIC
1840 spin_lock_init(&mem->lock);
1841 INIT_LIST_HEAD_RCU(&mem->pools);
1843 add_mem_pool(mem, pool);
1847 swiotlb_create_debugfs_files(mem, rmem->name);
1850 dev->dma_io_tlb_mem = mem;
1855 static void rmem_swiotlb_device_release(struct reserved_mem *rmem,
1858 dev->dma_io_tlb_mem = &io_tlb_default_mem;
1861 static const struct reserved_mem_ops rmem_swiotlb_ops = {
1862 .device_init = rmem_swiotlb_device_init,
1863 .device_release = rmem_swiotlb_device_release,
1866 static int __init rmem_swiotlb_setup(struct reserved_mem *rmem)
1868 unsigned long node = rmem->fdt_node;
1870 if (of_get_flat_dt_prop(node, "reusable", NULL) ||
1871 of_get_flat_dt_prop(node, "linux,cma-default", NULL) ||
1872 of_get_flat_dt_prop(node, "linux,dma-default", NULL) ||
1873 of_get_flat_dt_prop(node, "no-map", NULL))
1876 rmem->ops = &rmem_swiotlb_ops;
1877 pr_info("Reserved memory: created restricted DMA pool at %pa, size %ld MiB\n",
1878 &rmem->base, (unsigned long)rmem->size / SZ_1M);
1882 RESERVEDMEM_OF_DECLARE(dma, "restricted-dma-pool", rmem_swiotlb_setup);
1883 #endif /* CONFIG_DMA_RESTRICTED_POOL */