Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
1648993f JR |
2 | #ifndef __LINUX_SWIOTLB_H |
3 | #define __LINUX_SWIOTLB_H | |
4 | ||
7fd856aa | 5 | #include <linux/device.h> |
38674442 TR |
6 | #include <linux/dma-direction.h> |
7 | #include <linux/init.h> | |
1648993f | 8 | #include <linux/types.h> |
f51778db | 9 | #include <linux/limits.h> |
73f62095 | 10 | #include <linux/spinlock.h> |
1aaa7368 | 11 | #include <linux/workqueue.h> |
1648993f JR |
12 | |
13 | struct device; | |
38674442 | 14 | struct page; |
1648993f JR |
15 | struct scatterlist; |
16 | ||
c6af2aa9 CH |
17 | #define SWIOTLB_VERBOSE (1 << 0) /* verbose initialization */ |
18 | #define SWIOTLB_FORCE (1 << 1) /* force bounce buffering */ | |
8ba2ed1b | 19 | #define SWIOTLB_ANY (1 << 2) /* allow any memory for the buffer */ |
ae7871be | 20 | |
0016fdee IC |
21 | /* |
22 | * Maximum allowable number of contiguous slabs to map, | |
23 | * must be a power of 2. What is the appropriate value ? | |
24 | * The complexity of {map,unmap}_single is linearly dependent on this value. | |
25 | */ | |
26 | #define IO_TLB_SEGSIZE 128 | |
27 | ||
0016fdee IC |
28 | /* |
29 | * log of the size of each IO TLB slab. The number of slabs is command line | |
30 | * controllable. | |
31 | */ | |
32 | #define IO_TLB_SHIFT 11 | |
b5d7ccb7 | 33 | #define IO_TLB_SIZE (1 << IO_TLB_SHIFT) |
0016fdee | 34 | |
e998879d AK |
35 | /* default to 64MB */ |
36 | #define IO_TLB_DEFAULT_SIZE (64UL<<20) | |
37 | ||
c729de8f | 38 | unsigned long swiotlb_size_or_default(void); |
7374153d CH |
39 | void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags, |
40 | int (*remap)(void *tlb, unsigned long nslabs)); | |
41 | int swiotlb_init_late(size_t size, gfp_t gfp_mask, | |
42 | int (*remap)(void *tlb, unsigned long nslabs)); | |
c7753208 | 43 | extern void __init swiotlb_update_mem_attributes(void); |
1648993f | 44 | |
5740afdb | 45 | #ifdef CONFIG_SWIOTLB |
73f62095 CC |
46 | |
47 | /** | |
158dbe9c | 48 | * struct io_tlb_pool - IO TLB memory pool descriptor |
73f62095 CC |
49 | * @start: The start address of the swiotlb memory pool. Used to do a quick |
50 | * range check to see if the memory was in fact allocated by this | |
51 | * API. | |
52 | * @end: The end address of the swiotlb memory pool. Used to do a quick | |
53 | * range check to see if the memory was in fact allocated by this | |
54 | * API. | |
1a5e91d8 TL |
55 | * @vaddr: The vaddr of the swiotlb memory pool. The swiotlb memory pool |
56 | * may be remapped in the memory encrypted case and store virtual | |
57 | * address for bounce buffer operation. | |
158dbe9c PT |
58 | * @nslabs: The number of IO TLB slots between @start and @end. For the |
59 | * default swiotlb, this can be adjusted with a boot parameter, | |
60 | * see setup_io_tlb_npages(). | |
61 | * @late_alloc: %true if allocated using the page allocator. | |
62 | * @nareas: Number of areas in the pool. | |
63 | * @area_nslabs: Number of slots in each area. | |
64 | * @areas: Array of memory area descriptors. | |
65 | * @slots: Array of slot descriptors. | |
79636caa PT |
66 | * @node: Member of the IO TLB memory pool list. |
67 | * @rcu: RCU head for swiotlb_dyn_free(). | |
68 | * @transient: %true if transient memory pool. | |
158dbe9c PT |
69 | */ |
70 | struct io_tlb_pool { | |
71 | phys_addr_t start; | |
72 | phys_addr_t end; | |
73 | void *vaddr; | |
74 | unsigned long nslabs; | |
75 | bool late_alloc; | |
76 | unsigned int nareas; | |
77 | unsigned int area_nslabs; | |
78 | struct io_tlb_area *areas; | |
79 | struct io_tlb_slot *slots; | |
79636caa PT |
80 | #ifdef CONFIG_SWIOTLB_DYNAMIC |
81 | struct list_head node; | |
82 | struct rcu_head rcu; | |
83 | bool transient; | |
84 | #endif | |
158dbe9c PT |
85 | }; |
86 | ||
87 | /** | |
88 | * struct io_tlb_mem - Software IO TLB allocator | |
89 | * @defpool: Default (initial) IO TLB memory pool descriptor. | |
1aaa7368 | 90 | * @pool: IO TLB memory pool descriptor (if not dynamic). |
158dbe9c | 91 | * @nslabs: Total number of IO TLB slabs in all pools. |
73f62095 | 92 | * @debugfs: The dentry to debugfs. |
903cd0f3 | 93 | * @force_bounce: %true if swiotlb bouncing is forced |
f4111e39 | 94 | * @for_alloc: %true if the pool is used for memory allocation |
62708b2b | 95 | * @can_grow: %true if more pools can be allocated dynamically. |
ad96ce32 | 96 | * @phys_limit: Maximum allowed physical address. |
1aaa7368 PT |
97 | * @lock: Lock to synchronize changes to the list. |
98 | * @pools: List of IO TLB memory pool descriptors (if dynamic). | |
99 | * @dyn_alloc: Dynamic IO TLB pool allocation work. | |
8b0977ec MK |
100 | * @total_used: The total number of slots in the pool that are currently used |
101 | * across all areas. Used only for calculating used_hiwater in | |
102 | * debugfs. | |
103 | * @used_hiwater: The high water mark for total_used. Used only for reporting | |
104 | * in debugfs. | |
02e76569 Z |
105 | * @transient_nslabs: The total number of slots in all transient pools that |
106 | * are currently used across all areas. | |
73f62095 CC |
107 | */ |
108 | struct io_tlb_mem { | |
158dbe9c | 109 | struct io_tlb_pool defpool; |
73f62095 | 110 | unsigned long nslabs; |
73f62095 | 111 | struct dentry *debugfs; |
903cd0f3 | 112 | bool force_bounce; |
f4111e39 | 113 | bool for_alloc; |
62708b2b PT |
114 | #ifdef CONFIG_SWIOTLB_DYNAMIC |
115 | bool can_grow; | |
ad96ce32 | 116 | u64 phys_limit; |
1aaa7368 PT |
117 | spinlock_t lock; |
118 | struct list_head pools; | |
119 | struct work_struct dyn_alloc; | |
62708b2b | 120 | #endif |
ec274aff | 121 | #ifdef CONFIG_DEBUG_FS |
8b0977ec MK |
122 | atomic_long_t total_used; |
123 | atomic_long_t used_hiwater; | |
02e76569 | 124 | atomic_long_t transient_nslabs; |
ec274aff | 125 | #endif |
73f62095 | 126 | }; |
55897af6 | 127 | |
7296f230 | 128 | struct io_tlb_pool *__swiotlb_find_pool(struct device *dev, phys_addr_t paddr); |
79636caa | 129 | |
fea18777 | 130 | /** |
7296f230 | 131 | * swiotlb_find_pool() - find swiotlb pool to which a physical address belongs |
fea18777 PT |
132 | * @dev: Device which has mapped the buffer. |
133 | * @paddr: Physical address within the DMA buffer. | |
134 | * | |
7296f230 | 135 | * Find the swiotlb pool that @paddr points into. |
fea18777 PT |
136 | * |
137 | * Return: | |
7296f230 MK |
138 | * * pool address if @paddr points into a bounce buffer |
139 | * * NULL if @paddr does not point into a bounce buffer. As such, this function | |
140 | * can be used to determine if @paddr denotes a swiotlb bounce buffer. | |
fea18777 | 141 | */ |
7296f230 MK |
142 | static inline struct io_tlb_pool *swiotlb_find_pool(struct device *dev, |
143 | phys_addr_t paddr) | |
55897af6 | 144 | { |
7fd856aa | 145 | struct io_tlb_mem *mem = dev->dma_io_tlb_mem; |
73f62095 | 146 | |
79636caa | 147 | if (!mem) |
7296f230 | 148 | return NULL; |
79636caa | 149 | |
2d5780bb PT |
150 | #ifdef CONFIG_SWIOTLB_DYNAMIC |
151 | /* | |
152 | * All SWIOTLB buffer addresses must have been returned by | |
153 | * swiotlb_tbl_map_single() and passed to a device driver. | |
154 | * If a SWIOTLB address is checked on another CPU, then it was | |
155 | * presumably loaded by the device driver from an unspecified private | |
156 | * data structure. Make sure that this load is ordered before reading | |
7296f230 | 157 | * dev->dma_uses_io_tlb here and mem->pools in __swiotlb_find_pool(). |
2d5780bb PT |
158 | * |
159 | * This barrier pairs with smp_mb() in swiotlb_find_slots(). | |
160 | */ | |
161 | smp_rmb(); | |
7296f230 MK |
162 | if (READ_ONCE(dev->dma_uses_io_tlb)) |
163 | return __swiotlb_find_pool(dev, paddr); | |
2d5780bb | 164 | #else |
7296f230 MK |
165 | if (paddr >= mem->defpool.start && paddr < mem->defpool.end) |
166 | return &mem->defpool; | |
2d5780bb | 167 | #endif |
7296f230 MK |
168 | |
169 | return NULL; | |
55897af6 CH |
170 | } |
171 | ||
903cd0f3 CC |
172 | static inline bool is_swiotlb_force_bounce(struct device *dev) |
173 | { | |
174 | struct io_tlb_mem *mem = dev->dma_io_tlb_mem; | |
175 | ||
176 | return mem && mem->force_bounce; | |
177 | } | |
178 | ||
c6af2aa9 | 179 | void swiotlb_init(bool addressing_limited, unsigned int flags); |
55897af6 | 180 | void __init swiotlb_exit(void); |
05ee7741 | 181 | void swiotlb_dev_init(struct device *dev); |
abe420bf | 182 | size_t swiotlb_max_mapping_size(struct device *dev); |
05ee7741 | 183 | bool is_swiotlb_allocated(void); |
6f2beb26 | 184 | bool is_swiotlb_active(struct device *dev); |
2d29960a | 185 | void __init swiotlb_adjust_size(unsigned long size); |
05ee7741 PT |
186 | phys_addr_t default_swiotlb_base(void); |
187 | phys_addr_t default_swiotlb_limit(void); | |
5740afdb | 188 | #else |
c6af2aa9 CH |
189 | static inline void swiotlb_init(bool addressing_limited, unsigned int flags) |
190 | { | |
191 | } | |
05ee7741 PT |
192 | |
193 | static inline void swiotlb_dev_init(struct device *dev) | |
194 | { | |
195 | } | |
196 | ||
7296f230 MK |
197 | static inline struct io_tlb_pool *swiotlb_find_pool(struct device *dev, |
198 | phys_addr_t paddr) | |
55897af6 | 199 | { |
7296f230 | 200 | return NULL; |
55897af6 | 201 | } |
903cd0f3 CC |
202 | static inline bool is_swiotlb_force_bounce(struct device *dev) |
203 | { | |
204 | return false; | |
205 | } | |
55897af6 CH |
206 | static inline void swiotlb_exit(void) |
207 | { | |
208 | } | |
abe420bf JR |
209 | static inline size_t swiotlb_max_mapping_size(struct device *dev) |
210 | { | |
211 | return SIZE_MAX; | |
212 | } | |
492366f7 | 213 | |
05ee7741 PT |
214 | static inline bool is_swiotlb_allocated(void) |
215 | { | |
216 | return false; | |
217 | } | |
218 | ||
6f2beb26 | 219 | static inline bool is_swiotlb_active(struct device *dev) |
492366f7 JR |
220 | { |
221 | return false; | |
222 | } | |
e998879d | 223 | |
2d29960a | 224 | static inline void swiotlb_adjust_size(unsigned long size) |
e998879d AK |
225 | { |
226 | } | |
05ee7741 PT |
227 | |
228 | static inline phys_addr_t default_swiotlb_base(void) | |
229 | { | |
230 | return 0; | |
231 | } | |
232 | ||
233 | static inline phys_addr_t default_swiotlb_limit(void) | |
234 | { | |
235 | return 0; | |
236 | } | |
55897af6 | 237 | #endif /* CONFIG_SWIOTLB */ |
5740afdb | 238 | |
7296f230 MK |
239 | phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys, |
240 | size_t mapping_size, unsigned int alloc_aligned_mask, | |
241 | enum dma_data_direction dir, unsigned long attrs); | |
242 | dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys, | |
243 | size_t size, enum dma_data_direction dir, unsigned long attrs); | |
244 | ||
245 | void __swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr, | |
246 | size_t mapping_size, enum dma_data_direction dir, | |
247 | unsigned long attrs, struct io_tlb_pool *pool); | |
248 | static inline void swiotlb_tbl_unmap_single(struct device *dev, | |
249 | phys_addr_t addr, size_t size, enum dma_data_direction dir, | |
250 | unsigned long attrs) | |
251 | { | |
252 | struct io_tlb_pool *pool = swiotlb_find_pool(dev, addr); | |
253 | ||
254 | if (unlikely(pool)) | |
255 | __swiotlb_tbl_unmap_single(dev, addr, size, dir, attrs, pool); | |
256 | } | |
257 | ||
258 | void __swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr, | |
259 | size_t size, enum dma_data_direction dir, | |
260 | struct io_tlb_pool *pool); | |
261 | static inline void swiotlb_sync_single_for_device(struct device *dev, | |
262 | phys_addr_t addr, size_t size, enum dma_data_direction dir) | |
263 | { | |
264 | struct io_tlb_pool *pool = swiotlb_find_pool(dev, addr); | |
265 | ||
266 | if (unlikely(pool)) | |
267 | __swiotlb_sync_single_for_device(dev, addr, size, dir, pool); | |
268 | } | |
269 | ||
270 | void __swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr, | |
271 | size_t size, enum dma_data_direction dir, | |
272 | struct io_tlb_pool *pool); | |
273 | static inline void swiotlb_sync_single_for_cpu(struct device *dev, | |
274 | phys_addr_t addr, size_t size, enum dma_data_direction dir) | |
275 | { | |
276 | struct io_tlb_pool *pool = swiotlb_find_pool(dev, addr); | |
277 | ||
278 | if (unlikely(pool)) | |
279 | __swiotlb_sync_single_for_cpu(dev, addr, size, dir, pool); | |
280 | } | |
281 | ||
ad32e8cb | 282 | extern void swiotlb_print_info(void); |
9c5a3621 | 283 | |
f4111e39 CC |
284 | #ifdef CONFIG_DMA_RESTRICTED_POOL |
285 | struct page *swiotlb_alloc(struct device *dev, size_t size); | |
286 | bool swiotlb_free(struct device *dev, struct page *page, size_t size); | |
287 | ||
288 | static inline bool is_swiotlb_for_alloc(struct device *dev) | |
289 | { | |
290 | return dev->dma_io_tlb_mem->for_alloc; | |
291 | } | |
292 | #else | |
293 | static inline struct page *swiotlb_alloc(struct device *dev, size_t size) | |
294 | { | |
295 | return NULL; | |
296 | } | |
297 | static inline bool swiotlb_free(struct device *dev, struct page *page, | |
298 | size_t size) | |
299 | { | |
300 | return false; | |
301 | } | |
302 | static inline bool is_swiotlb_for_alloc(struct device *dev) | |
303 | { | |
304 | return false; | |
305 | } | |
306 | #endif /* CONFIG_DMA_RESTRICTED_POOL */ | |
307 | ||
1648993f | 308 | #endif /* __LINUX_SWIOTLB_H */ |