Commit | Line | Data |
---|---|---|
caab277b | 1 | // SPDX-License-Identifier: GPL-2.0-only |
0db2e5d1 RM |
2 | /* |
3 | * A fairly generic DMA-API to IOMMU-API glue layer. | |
4 | * | |
5 | * Copyright (C) 2014-2015 ARM Ltd. | |
6 | * | |
7 | * based in part on arch/arm/mm/dma-mapping.c: | |
8 | * Copyright (C) 2000-2004 Russell King | |
0db2e5d1 RM |
9 | */ |
10 | ||
f51dc892 | 11 | #include <linux/acpi_iort.h> |
0db2e5d1 | 12 | #include <linux/device.h> |
0a0f0d8b | 13 | #include <linux/dma-map-ops.h> |
0db2e5d1 | 14 | #include <linux/dma-iommu.h> |
5b11e9cd | 15 | #include <linux/gfp.h> |
0db2e5d1 RM |
16 | #include <linux/huge_mm.h> |
17 | #include <linux/iommu.h> | |
18 | #include <linux/iova.h> | |
44bb7e24 | 19 | #include <linux/irq.h> |
0db2e5d1 | 20 | #include <linux/mm.h> |
c1864790 | 21 | #include <linux/mutex.h> |
fade1ec0 | 22 | #include <linux/pci.h> |
82612d66 | 23 | #include <linux/swiotlb.h> |
5b11e9cd RM |
24 | #include <linux/scatterlist.h> |
25 | #include <linux/vmalloc.h> | |
795bbbb9 | 26 | #include <linux/crash_dump.h> |
82612d66 | 27 | #include <linux/dma-direct.h> |
0db2e5d1 | 28 | |
44bb7e24 RM |
29 | struct iommu_dma_msi_page { |
30 | struct list_head list; | |
31 | dma_addr_t iova; | |
32 | phys_addr_t phys; | |
33 | }; | |
34 | ||
fdbe574e RM |
35 | enum iommu_dma_cookie_type { |
36 | IOMMU_DMA_IOVA_COOKIE, | |
37 | IOMMU_DMA_MSI_COOKIE, | |
38 | }; | |
39 | ||
44bb7e24 | 40 | struct iommu_dma_cookie { |
fdbe574e RM |
41 | enum iommu_dma_cookie_type type; |
42 | union { | |
43 | /* Full allocator for IOMMU_DMA_IOVA_COOKIE */ | |
44 | struct iova_domain iovad; | |
45 | /* Trivial linear page allocator for IOMMU_DMA_MSI_COOKIE */ | |
46 | dma_addr_t msi_iova; | |
47 | }; | |
48 | struct list_head msi_page_list; | |
2da274cd ZL |
49 | |
50 | /* Domain for flush queue callback; NULL if flush queue not in use */ | |
51 | struct iommu_domain *fq_domain; | |
44bb7e24 RM |
52 | }; |
53 | ||
a8e8af35 | 54 | static DEFINE_STATIC_KEY_FALSE(iommu_deferred_attach_enabled); |
3542dcb1 RM |
55 | bool iommu_dma_forcedac __read_mostly; |
56 | ||
57 | static int __init iommu_dma_forcedac_setup(char *str) | |
58 | { | |
59 | int ret = kstrtobool(str, &iommu_dma_forcedac); | |
60 | ||
61 | if (!ret && iommu_dma_forcedac) | |
62 | pr_info("Forcing DAC for PCI devices\n"); | |
63 | return ret; | |
64 | } | |
65 | early_param("iommu.forcedac", iommu_dma_forcedac_setup); | |
a8e8af35 | 66 | |
2a2b8eaa TM |
67 | static void iommu_dma_entry_dtor(unsigned long data) |
68 | { | |
69 | struct page *freelist = (struct page *)data; | |
70 | ||
71 | while (freelist) { | |
72 | unsigned long p = (unsigned long)page_address(freelist); | |
73 | ||
74 | freelist = freelist->freelist; | |
75 | free_page(p); | |
76 | } | |
77 | } | |
78 | ||
fdbe574e RM |
79 | static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie) |
80 | { | |
81 | if (cookie->type == IOMMU_DMA_IOVA_COOKIE) | |
82 | return cookie->iovad.granule; | |
83 | return PAGE_SIZE; | |
84 | } | |
85 | ||
fdbe574e RM |
86 | static struct iommu_dma_cookie *cookie_alloc(enum iommu_dma_cookie_type type) |
87 | { | |
88 | struct iommu_dma_cookie *cookie; | |
89 | ||
90 | cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); | |
91 | if (cookie) { | |
fdbe574e RM |
92 | INIT_LIST_HEAD(&cookie->msi_page_list); |
93 | cookie->type = type; | |
94 | } | |
95 | return cookie; | |
44bb7e24 RM |
96 | } |
97 | ||
0db2e5d1 RM |
98 | /** |
99 | * iommu_get_dma_cookie - Acquire DMA-API resources for a domain | |
100 | * @domain: IOMMU domain to prepare for DMA-API usage | |
101 | * | |
102 | * IOMMU drivers should normally call this from their domain_alloc | |
103 | * callback when domain->type == IOMMU_DOMAIN_DMA. | |
104 | */ | |
105 | int iommu_get_dma_cookie(struct iommu_domain *domain) | |
fdbe574e RM |
106 | { |
107 | if (domain->iova_cookie) | |
108 | return -EEXIST; | |
109 | ||
110 | domain->iova_cookie = cookie_alloc(IOMMU_DMA_IOVA_COOKIE); | |
111 | if (!domain->iova_cookie) | |
112 | return -ENOMEM; | |
113 | ||
114 | return 0; | |
115 | } | |
116 | EXPORT_SYMBOL(iommu_get_dma_cookie); | |
117 | ||
118 | /** | |
119 | * iommu_get_msi_cookie - Acquire just MSI remapping resources | |
120 | * @domain: IOMMU domain to prepare | |
121 | * @base: Start address of IOVA region for MSI mappings | |
122 | * | |
123 | * Users who manage their own IOVA allocation and do not want DMA API support, | |
124 | * but would still like to take advantage of automatic MSI remapping, can use | |
125 | * this to initialise their own domain appropriately. Users should reserve a | |
126 | * contiguous IOVA region, starting at @base, large enough to accommodate the | |
127 | * number of PAGE_SIZE mappings necessary to cover every MSI doorbell address | |
128 | * used by the devices attached to @domain. | |
129 | */ | |
130 | int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base) | |
0db2e5d1 | 131 | { |
44bb7e24 | 132 | struct iommu_dma_cookie *cookie; |
0db2e5d1 | 133 | |
fdbe574e RM |
134 | if (domain->type != IOMMU_DOMAIN_UNMANAGED) |
135 | return -EINVAL; | |
136 | ||
0db2e5d1 RM |
137 | if (domain->iova_cookie) |
138 | return -EEXIST; | |
139 | ||
fdbe574e | 140 | cookie = cookie_alloc(IOMMU_DMA_MSI_COOKIE); |
44bb7e24 RM |
141 | if (!cookie) |
142 | return -ENOMEM; | |
0db2e5d1 | 143 | |
fdbe574e | 144 | cookie->msi_iova = base; |
44bb7e24 RM |
145 | domain->iova_cookie = cookie; |
146 | return 0; | |
0db2e5d1 | 147 | } |
fdbe574e | 148 | EXPORT_SYMBOL(iommu_get_msi_cookie); |
0db2e5d1 RM |
149 | |
150 | /** | |
151 | * iommu_put_dma_cookie - Release a domain's DMA mapping resources | |
fdbe574e RM |
152 | * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie() or |
153 | * iommu_get_msi_cookie() | |
0db2e5d1 RM |
154 | * |
155 | * IOMMU drivers should normally call this from their domain_free callback. | |
156 | */ | |
157 | void iommu_put_dma_cookie(struct iommu_domain *domain) | |
158 | { | |
44bb7e24 RM |
159 | struct iommu_dma_cookie *cookie = domain->iova_cookie; |
160 | struct iommu_dma_msi_page *msi, *tmp; | |
0db2e5d1 | 161 | |
44bb7e24 | 162 | if (!cookie) |
0db2e5d1 RM |
163 | return; |
164 | ||
fdbe574e | 165 | if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule) |
44bb7e24 RM |
166 | put_iova_domain(&cookie->iovad); |
167 | ||
168 | list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list) { | |
169 | list_del(&msi->list); | |
170 | kfree(msi); | |
171 | } | |
172 | kfree(cookie); | |
0db2e5d1 RM |
173 | domain->iova_cookie = NULL; |
174 | } | |
175 | EXPORT_SYMBOL(iommu_put_dma_cookie); | |
176 | ||
273df963 RM |
177 | /** |
178 | * iommu_dma_get_resv_regions - Reserved region driver helper | |
179 | * @dev: Device from iommu_get_resv_regions() | |
180 | * @list: Reserved region list from iommu_get_resv_regions() | |
181 | * | |
182 | * IOMMU drivers can use this to implement their .get_resv_regions callback | |
cd2c9fcf SK |
183 | * for general non-IOMMU-specific reservations. Currently, this covers GICv3 |
184 | * ITS region reservation on ACPI based ARM platforms that may require HW MSI | |
185 | * reservation. | |
273df963 RM |
186 | */ |
187 | void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list) | |
fade1ec0 | 188 | { |
fade1ec0 | 189 | |
98cc4f71 | 190 | if (!is_of_node(dev_iommu_fwspec_get(dev)->iommu_fwnode)) |
cd2c9fcf | 191 | iort_iommu_msi_get_resv_regions(dev, list); |
273df963 | 192 | |
fade1ec0 | 193 | } |
273df963 | 194 | EXPORT_SYMBOL(iommu_dma_get_resv_regions); |
fade1ec0 | 195 | |
7c1b058c RM |
196 | static int cookie_init_hw_msi_region(struct iommu_dma_cookie *cookie, |
197 | phys_addr_t start, phys_addr_t end) | |
198 | { | |
199 | struct iova_domain *iovad = &cookie->iovad; | |
200 | struct iommu_dma_msi_page *msi_page; | |
201 | int i, num_pages; | |
202 | ||
203 | start -= iova_offset(iovad, start); | |
204 | num_pages = iova_align(iovad, end - start) >> iova_shift(iovad); | |
205 | ||
7c1b058c | 206 | for (i = 0; i < num_pages; i++) { |
65ac74f1 MZ |
207 | msi_page = kmalloc(sizeof(*msi_page), GFP_KERNEL); |
208 | if (!msi_page) | |
209 | return -ENOMEM; | |
210 | ||
211 | msi_page->phys = start; | |
212 | msi_page->iova = start; | |
213 | INIT_LIST_HEAD(&msi_page->list); | |
214 | list_add(&msi_page->list, &cookie->msi_page_list); | |
7c1b058c RM |
215 | start += iovad->granule; |
216 | } | |
217 | ||
218 | return 0; | |
219 | } | |
220 | ||
aadad097 | 221 | static int iova_reserve_pci_windows(struct pci_dev *dev, |
cd2c9fcf SK |
222 | struct iova_domain *iovad) |
223 | { | |
224 | struct pci_host_bridge *bridge = pci_find_host_bridge(dev->bus); | |
225 | struct resource_entry *window; | |
226 | unsigned long lo, hi; | |
aadad097 | 227 | phys_addr_t start = 0, end; |
cd2c9fcf SK |
228 | |
229 | resource_list_for_each_entry(window, &bridge->windows) { | |
230 | if (resource_type(window->res) != IORESOURCE_MEM) | |
231 | continue; | |
232 | ||
233 | lo = iova_pfn(iovad, window->res->start - window->offset); | |
234 | hi = iova_pfn(iovad, window->res->end - window->offset); | |
235 | reserve_iova(iovad, lo, hi); | |
236 | } | |
aadad097 SM |
237 | |
238 | /* Get reserved DMA windows from host bridge */ | |
239 | resource_list_for_each_entry(window, &bridge->dma_ranges) { | |
240 | end = window->res->start - window->offset; | |
241 | resv_iova: | |
242 | if (end > start) { | |
243 | lo = iova_pfn(iovad, start); | |
244 | hi = iova_pfn(iovad, end); | |
245 | reserve_iova(iovad, lo, hi); | |
571f3160 | 246 | } else if (end < start) { |
aadad097 | 247 | /* dma_ranges list should be sorted */ |
571f3160 | 248 | dev_err(&dev->dev, |
7154cbd3 JR |
249 | "Failed to reserve IOVA [%pa-%pa]\n", |
250 | &start, &end); | |
aadad097 SM |
251 | return -EINVAL; |
252 | } | |
253 | ||
254 | start = window->res->end - window->offset + 1; | |
255 | /* If window is last entry */ | |
256 | if (window->node.next == &bridge->dma_ranges && | |
29fcea8c AB |
257 | end != ~(phys_addr_t)0) { |
258 | end = ~(phys_addr_t)0; | |
aadad097 SM |
259 | goto resv_iova; |
260 | } | |
261 | } | |
262 | ||
263 | return 0; | |
cd2c9fcf SK |
264 | } |
265 | ||
7c1b058c RM |
266 | static int iova_reserve_iommu_regions(struct device *dev, |
267 | struct iommu_domain *domain) | |
268 | { | |
269 | struct iommu_dma_cookie *cookie = domain->iova_cookie; | |
270 | struct iova_domain *iovad = &cookie->iovad; | |
271 | struct iommu_resv_region *region; | |
272 | LIST_HEAD(resv_regions); | |
273 | int ret = 0; | |
274 | ||
aadad097 SM |
275 | if (dev_is_pci(dev)) { |
276 | ret = iova_reserve_pci_windows(to_pci_dev(dev), iovad); | |
277 | if (ret) | |
278 | return ret; | |
279 | } | |
cd2c9fcf | 280 | |
7c1b058c RM |
281 | iommu_get_resv_regions(dev, &resv_regions); |
282 | list_for_each_entry(region, &resv_regions, list) { | |
283 | unsigned long lo, hi; | |
284 | ||
285 | /* We ARE the software that manages these! */ | |
286 | if (region->type == IOMMU_RESV_SW_MSI) | |
287 | continue; | |
288 | ||
289 | lo = iova_pfn(iovad, region->start); | |
290 | hi = iova_pfn(iovad, region->start + region->length - 1); | |
291 | reserve_iova(iovad, lo, hi); | |
292 | ||
293 | if (region->type == IOMMU_RESV_MSI) | |
294 | ret = cookie_init_hw_msi_region(cookie, region->start, | |
295 | region->start + region->length); | |
296 | if (ret) | |
297 | break; | |
298 | } | |
299 | iommu_put_resv_regions(dev, &resv_regions); | |
300 | ||
301 | return ret; | |
302 | } | |
303 | ||
2da274cd ZL |
304 | static void iommu_dma_flush_iotlb_all(struct iova_domain *iovad) |
305 | { | |
306 | struct iommu_dma_cookie *cookie; | |
307 | struct iommu_domain *domain; | |
308 | ||
309 | cookie = container_of(iovad, struct iommu_dma_cookie, iovad); | |
310 | domain = cookie->fq_domain; | |
a250c23f | 311 | |
2da274cd ZL |
312 | domain->ops->flush_iotlb_all(domain); |
313 | } | |
314 | ||
82c3cefb LB |
315 | static bool dev_is_untrusted(struct device *dev) |
316 | { | |
317 | return dev_is_pci(dev) && to_pci_dev(dev)->untrusted; | |
318 | } | |
319 | ||
0db2e5d1 RM |
320 | /** |
321 | * iommu_dma_init_domain - Initialise a DMA mapping domain | |
322 | * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie() | |
323 | * @base: IOVA at which the mappable address space starts | |
ac6d7046 | 324 | * @limit: Last address of the IOVA space |
fade1ec0 | 325 | * @dev: Device the domain is being initialised for |
0db2e5d1 | 326 | * |
ac6d7046 | 327 | * @base and @limit + 1 should be exact multiples of IOMMU page granularity to |
0db2e5d1 RM |
328 | * avoid rounding surprises. If necessary, we reserve the page at address 0 |
329 | * to ensure it is an invalid IOVA. It is safe to reinitialise a domain, but | |
330 | * any change which could make prior IOVAs invalid will fail. | |
331 | */ | |
06d60728 | 332 | static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, |
ac6d7046 | 333 | dma_addr_t limit, struct device *dev) |
0db2e5d1 | 334 | { |
fdbe574e | 335 | struct iommu_dma_cookie *cookie = domain->iova_cookie; |
c61a4633 | 336 | unsigned long order, base_pfn; |
6b0c54e7 | 337 | struct iova_domain *iovad; |
0db2e5d1 | 338 | |
fdbe574e RM |
339 | if (!cookie || cookie->type != IOMMU_DMA_IOVA_COOKIE) |
340 | return -EINVAL; | |
0db2e5d1 | 341 | |
6b0c54e7 YL |
342 | iovad = &cookie->iovad; |
343 | ||
0db2e5d1 | 344 | /* Use the smallest supported page size for IOVA granularity */ |
d16e0faa | 345 | order = __ffs(domain->pgsize_bitmap); |
0db2e5d1 | 346 | base_pfn = max_t(unsigned long, 1, base >> order); |
0db2e5d1 RM |
347 | |
348 | /* Check the domain allows at least some access to the device... */ | |
349 | if (domain->geometry.force_aperture) { | |
350 | if (base > domain->geometry.aperture_end || | |
ac6d7046 | 351 | limit < domain->geometry.aperture_start) { |
0db2e5d1 RM |
352 | pr_warn("specified DMA range outside IOMMU capability\n"); |
353 | return -EFAULT; | |
354 | } | |
355 | /* ...then finally give it a kicking to make sure it fits */ | |
356 | base_pfn = max_t(unsigned long, base_pfn, | |
357 | domain->geometry.aperture_start >> order); | |
0db2e5d1 RM |
358 | } |
359 | ||
f51d7bb7 | 360 | /* start_pfn is always nonzero for an already-initialised domain */ |
0db2e5d1 RM |
361 | if (iovad->start_pfn) { |
362 | if (1UL << order != iovad->granule || | |
f51d7bb7 | 363 | base_pfn != iovad->start_pfn) { |
0db2e5d1 RM |
364 | pr_warn("Incompatible range for DMA domain\n"); |
365 | return -EFAULT; | |
366 | } | |
7c1b058c RM |
367 | |
368 | return 0; | |
0db2e5d1 | 369 | } |
7c1b058c | 370 | |
aa3ac946 | 371 | init_iova_domain(iovad, 1UL << order, base_pfn); |
2da274cd | 372 | |
8d971243 | 373 | if (!cookie->fq_domain && !dev_is_untrusted(dev) && |
a250c23f | 374 | domain->ops->flush_iotlb_all && !iommu_get_dma_strict(domain)) { |
b34e9b0d | 375 | if (init_iova_flush_queue(iovad, iommu_dma_flush_iotlb_all, |
2a2b8eaa | 376 | iommu_dma_entry_dtor)) |
b34e9b0d TM |
377 | pr_warn("iova flush queue initialization failed\n"); |
378 | else | |
379 | cookie->fq_domain = domain; | |
2da274cd ZL |
380 | } |
381 | ||
7c1b058c | 382 | return iova_reserve_iommu_regions(dev, domain); |
0db2e5d1 | 383 | } |
0db2e5d1 RM |
384 | |
385 | /** | |
737c85ca MH |
386 | * dma_info_to_prot - Translate DMA API directions and attributes to IOMMU API |
387 | * page flags. | |
0db2e5d1 RM |
388 | * @dir: Direction of DMA transfer |
389 | * @coherent: Is the DMA master cache-coherent? | |
737c85ca | 390 | * @attrs: DMA attributes for the mapping |
0db2e5d1 RM |
391 | * |
392 | * Return: corresponding IOMMU API page protection flags | |
393 | */ | |
06d60728 | 394 | static int dma_info_to_prot(enum dma_data_direction dir, bool coherent, |
737c85ca | 395 | unsigned long attrs) |
0db2e5d1 RM |
396 | { |
397 | int prot = coherent ? IOMMU_CACHE : 0; | |
398 | ||
737c85ca MH |
399 | if (attrs & DMA_ATTR_PRIVILEGED) |
400 | prot |= IOMMU_PRIV; | |
401 | ||
0db2e5d1 RM |
402 | switch (dir) { |
403 | case DMA_BIDIRECTIONAL: | |
404 | return prot | IOMMU_READ | IOMMU_WRITE; | |
405 | case DMA_TO_DEVICE: | |
406 | return prot | IOMMU_READ; | |
407 | case DMA_FROM_DEVICE: | |
408 | return prot | IOMMU_WRITE; | |
409 | default: | |
410 | return 0; | |
411 | } | |
412 | } | |
413 | ||
842fe519 | 414 | static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain, |
bd036d2f | 415 | size_t size, u64 dma_limit, struct device *dev) |
0db2e5d1 | 416 | { |
a44e6657 RM |
417 | struct iommu_dma_cookie *cookie = domain->iova_cookie; |
418 | struct iova_domain *iovad = &cookie->iovad; | |
bb65a64c | 419 | unsigned long shift, iova_len, iova = 0; |
0db2e5d1 | 420 | |
a44e6657 RM |
421 | if (cookie->type == IOMMU_DMA_MSI_COOKIE) { |
422 | cookie->msi_iova += size; | |
423 | return cookie->msi_iova - size; | |
424 | } | |
425 | ||
426 | shift = iova_shift(iovad); | |
427 | iova_len = size >> shift; | |
bb65a64c RM |
428 | /* |
429 | * Freeing non-power-of-two-sized allocations back into the IOVA caches | |
430 | * will come back to bite us badly, so we have to waste a bit of space | |
431 | * rounding up anything cacheable to make sure that can't happen. The | |
432 | * order of the unadjusted size will still match upon freeing. | |
433 | */ | |
434 | if (iova_len < (1 << (IOVA_RANGE_CACHE_MAX_SIZE - 1))) | |
435 | iova_len = roundup_pow_of_two(iova_len); | |
a44e6657 | 436 | |
a7ba70f1 | 437 | dma_limit = min_not_zero(dma_limit, dev->bus_dma_limit); |
03bfdc31 | 438 | |
c987ff0d | 439 | if (domain->geometry.force_aperture) |
bd036d2f | 440 | dma_limit = min(dma_limit, (u64)domain->geometry.aperture_end); |
122fac03 RM |
441 | |
442 | /* Try to get PCI devices a SAC address */ | |
3542dcb1 | 443 | if (dma_limit > DMA_BIT_MASK(32) && !iommu_dma_forcedac && dev_is_pci(dev)) |
538d5b33 TN |
444 | iova = alloc_iova_fast(iovad, iova_len, |
445 | DMA_BIT_MASK(32) >> shift, false); | |
bb65a64c | 446 | |
122fac03 | 447 | if (!iova) |
538d5b33 TN |
448 | iova = alloc_iova_fast(iovad, iova_len, dma_limit >> shift, |
449 | true); | |
122fac03 | 450 | |
bb65a64c | 451 | return (dma_addr_t)iova << shift; |
0db2e5d1 RM |
452 | } |
453 | ||
842fe519 | 454 | static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie, |
2a2b8eaa | 455 | dma_addr_t iova, size_t size, struct page *freelist) |
0db2e5d1 | 456 | { |
842fe519 | 457 | struct iova_domain *iovad = &cookie->iovad; |
0db2e5d1 | 458 | |
a44e6657 | 459 | /* The MSI case is only ever cleaning up its most recent allocation */ |
bb65a64c | 460 | if (cookie->type == IOMMU_DMA_MSI_COOKIE) |
a44e6657 | 461 | cookie->msi_iova -= size; |
2da274cd ZL |
462 | else if (cookie->fq_domain) /* non-strict mode */ |
463 | queue_iova(iovad, iova_pfn(iovad, iova), | |
2a2b8eaa TM |
464 | size >> iova_shift(iovad), |
465 | (unsigned long)freelist); | |
bb65a64c | 466 | else |
1cc896ed RM |
467 | free_iova_fast(iovad, iova_pfn(iovad, iova), |
468 | size >> iova_shift(iovad)); | |
842fe519 RM |
469 | } |
470 | ||
b61d271e | 471 | static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr, |
842fe519 RM |
472 | size_t size) |
473 | { | |
b61d271e | 474 | struct iommu_domain *domain = iommu_get_dma_domain(dev); |
a44e6657 RM |
475 | struct iommu_dma_cookie *cookie = domain->iova_cookie; |
476 | struct iova_domain *iovad = &cookie->iovad; | |
842fe519 | 477 | size_t iova_off = iova_offset(iovad, dma_addr); |
a7d20dc1 WD |
478 | struct iommu_iotlb_gather iotlb_gather; |
479 | size_t unmapped; | |
842fe519 RM |
480 | |
481 | dma_addr -= iova_off; | |
482 | size = iova_align(iovad, size + iova_off); | |
a7d20dc1 | 483 | iommu_iotlb_gather_init(&iotlb_gather); |
7a7c5bad | 484 | iotlb_gather.queued = cookie->fq_domain; |
a7d20dc1 WD |
485 | |
486 | unmapped = iommu_unmap_fast(domain, dma_addr, size, &iotlb_gather); | |
487 | WARN_ON(unmapped != size); | |
842fe519 | 488 | |
2da274cd | 489 | if (!cookie->fq_domain) |
aae4c8e2 | 490 | iommu_iotlb_sync(domain, &iotlb_gather); |
2a2b8eaa | 491 | iommu_dma_free_iova(cookie, dma_addr, size, iotlb_gather.freelist); |
0db2e5d1 RM |
492 | } |
493 | ||
82612d66 TM |
494 | static void __iommu_dma_unmap_swiotlb(struct device *dev, dma_addr_t dma_addr, |
495 | size_t size, enum dma_data_direction dir, | |
496 | unsigned long attrs) | |
497 | { | |
498 | struct iommu_domain *domain = iommu_get_dma_domain(dev); | |
82612d66 TM |
499 | phys_addr_t phys; |
500 | ||
501 | phys = iommu_iova_to_phys(domain, dma_addr); | |
502 | if (WARN_ON(!phys)) | |
503 | return; | |
504 | ||
505 | __iommu_dma_unmap(dev, dma_addr, size); | |
506 | ||
507 | if (unlikely(is_swiotlb_buffer(phys))) | |
2973073a | 508 | swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs); |
82612d66 TM |
509 | } |
510 | ||
92aec09c | 511 | static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys, |
bd036d2f | 512 | size_t size, int prot, u64 dma_mask) |
92aec09c | 513 | { |
b61d271e | 514 | struct iommu_domain *domain = iommu_get_dma_domain(dev); |
92aec09c | 515 | struct iommu_dma_cookie *cookie = domain->iova_cookie; |
8af23fad RM |
516 | struct iova_domain *iovad = &cookie->iovad; |
517 | size_t iova_off = iova_offset(iovad, phys); | |
92aec09c CH |
518 | dma_addr_t iova; |
519 | ||
a8e8af35 | 520 | if (static_branch_unlikely(&iommu_deferred_attach_enabled) && |
3ab65729 | 521 | iommu_deferred_attach(dev, domain)) |
795bbbb9 TM |
522 | return DMA_MAPPING_ERROR; |
523 | ||
8af23fad | 524 | size = iova_align(iovad, size + iova_off); |
92aec09c | 525 | |
6e235020 | 526 | iova = iommu_dma_alloc_iova(domain, size, dma_mask, dev); |
92aec09c CH |
527 | if (!iova) |
528 | return DMA_MAPPING_ERROR; | |
529 | ||
781ca2de | 530 | if (iommu_map_atomic(domain, iova, phys - iova_off, size, prot)) { |
2a2b8eaa | 531 | iommu_dma_free_iova(cookie, iova, size, NULL); |
92aec09c CH |
532 | return DMA_MAPPING_ERROR; |
533 | } | |
534 | return iova + iova_off; | |
535 | } | |
536 | ||
82612d66 TM |
537 | static dma_addr_t __iommu_dma_map_swiotlb(struct device *dev, phys_addr_t phys, |
538 | size_t org_size, dma_addr_t dma_mask, bool coherent, | |
539 | enum dma_data_direction dir, unsigned long attrs) | |
540 | { | |
541 | int prot = dma_info_to_prot(dir, coherent, attrs); | |
542 | struct iommu_domain *domain = iommu_get_dma_domain(dev); | |
543 | struct iommu_dma_cookie *cookie = domain->iova_cookie; | |
544 | struct iova_domain *iovad = &cookie->iovad; | |
545 | size_t aligned_size = org_size; | |
546 | void *padding_start; | |
547 | size_t padding_size; | |
548 | dma_addr_t iova; | |
549 | ||
550 | /* | |
551 | * If both the physical buffer start address and size are | |
552 | * page aligned, we don't need to use a bounce page. | |
553 | */ | |
554 | if (IS_ENABLED(CONFIG_SWIOTLB) && dev_is_untrusted(dev) && | |
555 | iova_offset(iovad, phys | org_size)) { | |
556 | aligned_size = iova_align(iovad, org_size); | |
557 | phys = swiotlb_tbl_map_single(dev, phys, org_size, | |
558 | aligned_size, dir, attrs); | |
559 | ||
560 | if (phys == DMA_MAPPING_ERROR) | |
561 | return DMA_MAPPING_ERROR; | |
562 | ||
563 | /* Cleanup the padding area. */ | |
564 | padding_start = phys_to_virt(phys); | |
565 | padding_size = aligned_size; | |
566 | ||
567 | if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) && | |
568 | (dir == DMA_TO_DEVICE || | |
569 | dir == DMA_BIDIRECTIONAL)) { | |
570 | padding_start += org_size; | |
571 | padding_size -= org_size; | |
572 | } | |
573 | ||
574 | memset(padding_start, 0, padding_size); | |
575 | } | |
576 | ||
577 | iova = __iommu_dma_map(dev, phys, aligned_size, prot, dma_mask); | |
2973073a CH |
578 | if (iova == DMA_MAPPING_ERROR && is_swiotlb_buffer(phys)) |
579 | swiotlb_tbl_unmap_single(dev, phys, org_size, dir, attrs); | |
82612d66 TM |
580 | return iova; |
581 | } | |
582 | ||
0db2e5d1 RM |
583 | static void __iommu_dma_free_pages(struct page **pages, int count) |
584 | { | |
585 | while (count--) | |
586 | __free_page(pages[count]); | |
587 | kvfree(pages); | |
588 | } | |
589 | ||
c4b17afb GK |
590 | static struct page **__iommu_dma_alloc_pages(struct device *dev, |
591 | unsigned int count, unsigned long order_mask, gfp_t gfp) | |
0db2e5d1 RM |
592 | { |
593 | struct page **pages; | |
c4b17afb | 594 | unsigned int i = 0, nid = dev_to_node(dev); |
3b6b7e19 RM |
595 | |
596 | order_mask &= (2U << MAX_ORDER) - 1; | |
597 | if (!order_mask) | |
598 | return NULL; | |
0db2e5d1 | 599 | |
c4b17afb | 600 | pages = kvzalloc(count * sizeof(*pages), GFP_KERNEL); |
0db2e5d1 RM |
601 | if (!pages) |
602 | return NULL; | |
603 | ||
604 | /* IOMMU can map any pages, so himem can also be used here */ | |
605 | gfp |= __GFP_NOWARN | __GFP_HIGHMEM; | |
606 | ||
4604393c RM |
607 | /* It makes no sense to muck about with huge pages */ |
608 | gfp &= ~__GFP_COMP; | |
609 | ||
0db2e5d1 RM |
610 | while (count) { |
611 | struct page *page = NULL; | |
3b6b7e19 | 612 | unsigned int order_size; |
0db2e5d1 RM |
613 | |
614 | /* | |
615 | * Higher-order allocations are a convenience rather | |
616 | * than a necessity, hence using __GFP_NORETRY until | |
3b6b7e19 | 617 | * falling back to minimum-order allocations. |
0db2e5d1 | 618 | */ |
3b6b7e19 RM |
619 | for (order_mask &= (2U << __fls(count)) - 1; |
620 | order_mask; order_mask &= ~order_size) { | |
621 | unsigned int order = __fls(order_mask); | |
c4b17afb | 622 | gfp_t alloc_flags = gfp; |
3b6b7e19 RM |
623 | |
624 | order_size = 1U << order; | |
c4b17afb GK |
625 | if (order_mask > order_size) |
626 | alloc_flags |= __GFP_NORETRY; | |
627 | page = alloc_pages_node(nid, alloc_flags, order); | |
0db2e5d1 RM |
628 | if (!page) |
629 | continue; | |
4604393c | 630 | if (order) |
0db2e5d1 | 631 | split_page(page, order); |
4604393c | 632 | break; |
0db2e5d1 | 633 | } |
0db2e5d1 RM |
634 | if (!page) { |
635 | __iommu_dma_free_pages(pages, i); | |
636 | return NULL; | |
637 | } | |
3b6b7e19 RM |
638 | count -= order_size; |
639 | while (order_size--) | |
0db2e5d1 RM |
640 | pages[i++] = page++; |
641 | } | |
642 | return pages; | |
643 | } | |
644 | ||
8230ce9a CH |
645 | /* |
646 | * If size is less than PAGE_SIZE, then a full CPU page will be allocated, | |
0db2e5d1 | 647 | * but an IOMMU which supports smaller pages might not map the whole thing. |
0db2e5d1 | 648 | */ |
8230ce9a CH |
649 | static struct page **__iommu_dma_alloc_noncontiguous(struct device *dev, |
650 | size_t size, struct sg_table *sgt, gfp_t gfp, pgprot_t prot, | |
e8d39a90 | 651 | unsigned long attrs) |
0db2e5d1 | 652 | { |
43c5bf11 | 653 | struct iommu_domain *domain = iommu_get_dma_domain(dev); |
842fe519 RM |
654 | struct iommu_dma_cookie *cookie = domain->iova_cookie; |
655 | struct iova_domain *iovad = &cookie->iovad; | |
21b95aaf CH |
656 | bool coherent = dev_is_dma_coherent(dev); |
657 | int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs); | |
21b95aaf | 658 | unsigned int count, min_size, alloc_sizes = domain->pgsize_bitmap; |
0db2e5d1 | 659 | struct page **pages; |
842fe519 | 660 | dma_addr_t iova; |
0db2e5d1 | 661 | |
a8e8af35 | 662 | if (static_branch_unlikely(&iommu_deferred_attach_enabled) && |
3ab65729 | 663 | iommu_deferred_attach(dev, domain)) |
795bbbb9 TM |
664 | return NULL; |
665 | ||
3b6b7e19 RM |
666 | min_size = alloc_sizes & -alloc_sizes; |
667 | if (min_size < PAGE_SIZE) { | |
668 | min_size = PAGE_SIZE; | |
669 | alloc_sizes |= PAGE_SIZE; | |
670 | } else { | |
671 | size = ALIGN(size, min_size); | |
672 | } | |
00085f1e | 673 | if (attrs & DMA_ATTR_ALLOC_SINGLE_PAGES) |
3b6b7e19 RM |
674 | alloc_sizes = min_size; |
675 | ||
676 | count = PAGE_ALIGN(size) >> PAGE_SHIFT; | |
c4b17afb GK |
677 | pages = __iommu_dma_alloc_pages(dev, count, alloc_sizes >> PAGE_SHIFT, |
678 | gfp); | |
0db2e5d1 RM |
679 | if (!pages) |
680 | return NULL; | |
681 | ||
842fe519 RM |
682 | size = iova_align(iovad, size); |
683 | iova = iommu_dma_alloc_iova(domain, size, dev->coherent_dma_mask, dev); | |
0db2e5d1 RM |
684 | if (!iova) |
685 | goto out_free_pages; | |
686 | ||
8230ce9a | 687 | if (sg_alloc_table_from_pages(sgt, pages, count, 0, size, GFP_KERNEL)) |
0db2e5d1 RM |
688 | goto out_free_iova; |
689 | ||
21b95aaf | 690 | if (!(ioprot & IOMMU_CACHE)) { |
23f88e0a CH |
691 | struct scatterlist *sg; |
692 | int i; | |
693 | ||
8230ce9a | 694 | for_each_sg(sgt->sgl, sg, sgt->orig_nents, i) |
23f88e0a | 695 | arch_dma_prep_coherent(sg_page(sg), sg->length); |
0db2e5d1 RM |
696 | } |
697 | ||
8230ce9a | 698 | if (iommu_map_sg_atomic(domain, iova, sgt->sgl, sgt->orig_nents, ioprot) |
0db2e5d1 RM |
699 | < size) |
700 | goto out_free_sg; | |
701 | ||
8230ce9a | 702 | sgt->sgl->dma_address = iova; |
e817ee5f | 703 | sgt->sgl->dma_length = size; |
8230ce9a CH |
704 | return pages; |
705 | ||
706 | out_free_sg: | |
707 | sg_free_table(sgt); | |
708 | out_free_iova: | |
709 | iommu_dma_free_iova(cookie, iova, size, NULL); | |
710 | out_free_pages: | |
711 | __iommu_dma_free_pages(pages, count); | |
712 | return NULL; | |
713 | } | |
714 | ||
715 | static void *iommu_dma_alloc_remap(struct device *dev, size_t size, | |
716 | dma_addr_t *dma_handle, gfp_t gfp, pgprot_t prot, | |
717 | unsigned long attrs) | |
718 | { | |
719 | struct page **pages; | |
720 | struct sg_table sgt; | |
721 | void *vaddr; | |
722 | ||
723 | pages = __iommu_dma_alloc_noncontiguous(dev, size, &sgt, gfp, prot, | |
724 | attrs); | |
725 | if (!pages) | |
726 | return NULL; | |
727 | *dma_handle = sgt.sgl->dma_address; | |
728 | sg_free_table(&sgt); | |
51231740 | 729 | vaddr = dma_common_pages_remap(pages, size, prot, |
21b95aaf CH |
730 | __builtin_return_address(0)); |
731 | if (!vaddr) | |
732 | goto out_unmap; | |
21b95aaf | 733 | return vaddr; |
0db2e5d1 | 734 | |
21b95aaf | 735 | out_unmap: |
8230ce9a CH |
736 | __iommu_dma_unmap(dev, *dma_handle, size); |
737 | __iommu_dma_free_pages(pages, PAGE_ALIGN(size) >> PAGE_SHIFT); | |
0db2e5d1 RM |
738 | return NULL; |
739 | } | |
740 | ||
e817ee5f CH |
741 | #ifdef CONFIG_DMA_REMAP |
742 | static struct sg_table *iommu_dma_alloc_noncontiguous(struct device *dev, | |
743 | size_t size, enum dma_data_direction dir, gfp_t gfp, | |
744 | unsigned long attrs) | |
745 | { | |
746 | struct dma_sgt_handle *sh; | |
747 | ||
748 | sh = kmalloc(sizeof(*sh), gfp); | |
749 | if (!sh) | |
750 | return NULL; | |
751 | ||
752 | sh->pages = __iommu_dma_alloc_noncontiguous(dev, size, &sh->sgt, gfp, | |
753 | PAGE_KERNEL, attrs); | |
754 | if (!sh->pages) { | |
755 | kfree(sh); | |
756 | return NULL; | |
757 | } | |
758 | return &sh->sgt; | |
759 | } | |
760 | ||
761 | static void iommu_dma_free_noncontiguous(struct device *dev, size_t size, | |
762 | struct sg_table *sgt, enum dma_data_direction dir) | |
763 | { | |
764 | struct dma_sgt_handle *sh = sgt_handle(sgt); | |
765 | ||
766 | __iommu_dma_unmap(dev, sgt->sgl->dma_address, size); | |
767 | __iommu_dma_free_pages(sh->pages, PAGE_ALIGN(size) >> PAGE_SHIFT); | |
768 | sg_free_table(&sh->sgt); | |
769 | } | |
770 | #endif /* CONFIG_DMA_REMAP */ | |
771 | ||
06d60728 CH |
772 | static void iommu_dma_sync_single_for_cpu(struct device *dev, |
773 | dma_addr_t dma_handle, size_t size, enum dma_data_direction dir) | |
0db2e5d1 | 774 | { |
06d60728 | 775 | phys_addr_t phys; |
0db2e5d1 | 776 | |
82612d66 | 777 | if (dev_is_dma_coherent(dev) && !dev_is_untrusted(dev)) |
06d60728 | 778 | return; |
1cc896ed | 779 | |
06d60728 | 780 | phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle); |
82612d66 TM |
781 | if (!dev_is_dma_coherent(dev)) |
782 | arch_sync_dma_for_cpu(phys, size, dir); | |
783 | ||
784 | if (is_swiotlb_buffer(phys)) | |
80808d27 | 785 | swiotlb_sync_single_for_cpu(dev, phys, size, dir); |
0db2e5d1 | 786 | } |
0db2e5d1 | 787 | |
06d60728 CH |
788 | static void iommu_dma_sync_single_for_device(struct device *dev, |
789 | dma_addr_t dma_handle, size_t size, enum dma_data_direction dir) | |
0db2e5d1 | 790 | { |
06d60728 | 791 | phys_addr_t phys; |
0db2e5d1 | 792 | |
82612d66 | 793 | if (dev_is_dma_coherent(dev) && !dev_is_untrusted(dev)) |
06d60728 | 794 | return; |
1cc896ed | 795 | |
06d60728 | 796 | phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle); |
82612d66 | 797 | if (is_swiotlb_buffer(phys)) |
80808d27 | 798 | swiotlb_sync_single_for_device(dev, phys, size, dir); |
82612d66 TM |
799 | |
800 | if (!dev_is_dma_coherent(dev)) | |
801 | arch_sync_dma_for_device(phys, size, dir); | |
06d60728 | 802 | } |
0db2e5d1 | 803 | |
06d60728 CH |
804 | static void iommu_dma_sync_sg_for_cpu(struct device *dev, |
805 | struct scatterlist *sgl, int nelems, | |
806 | enum dma_data_direction dir) | |
807 | { | |
808 | struct scatterlist *sg; | |
809 | int i; | |
810 | ||
82612d66 | 811 | if (dev_is_dma_coherent(dev) && !dev_is_untrusted(dev)) |
06d60728 CH |
812 | return; |
813 | ||
82612d66 TM |
814 | for_each_sg(sgl, sg, nelems, i) { |
815 | if (!dev_is_dma_coherent(dev)) | |
816 | arch_sync_dma_for_cpu(sg_phys(sg), sg->length, dir); | |
817 | ||
818 | if (is_swiotlb_buffer(sg_phys(sg))) | |
80808d27 CH |
819 | swiotlb_sync_single_for_cpu(dev, sg_phys(sg), |
820 | sg->length, dir); | |
82612d66 | 821 | } |
06d60728 CH |
822 | } |
823 | ||
824 | static void iommu_dma_sync_sg_for_device(struct device *dev, | |
825 | struct scatterlist *sgl, int nelems, | |
826 | enum dma_data_direction dir) | |
827 | { | |
828 | struct scatterlist *sg; | |
829 | int i; | |
830 | ||
82612d66 | 831 | if (dev_is_dma_coherent(dev) && !dev_is_untrusted(dev)) |
06d60728 CH |
832 | return; |
833 | ||
82612d66 TM |
834 | for_each_sg(sgl, sg, nelems, i) { |
835 | if (is_swiotlb_buffer(sg_phys(sg))) | |
80808d27 CH |
836 | swiotlb_sync_single_for_device(dev, sg_phys(sg), |
837 | sg->length, dir); | |
82612d66 TM |
838 | |
839 | if (!dev_is_dma_coherent(dev)) | |
840 | arch_sync_dma_for_device(sg_phys(sg), sg->length, dir); | |
841 | } | |
0db2e5d1 RM |
842 | } |
843 | ||
06d60728 CH |
844 | static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, |
845 | unsigned long offset, size_t size, enum dma_data_direction dir, | |
846 | unsigned long attrs) | |
51f8cc9e | 847 | { |
06d60728 CH |
848 | phys_addr_t phys = page_to_phys(page) + offset; |
849 | bool coherent = dev_is_dma_coherent(dev); | |
850 | dma_addr_t dma_handle; | |
851 | ||
82612d66 TM |
852 | dma_handle = __iommu_dma_map_swiotlb(dev, phys, size, dma_get_mask(dev), |
853 | coherent, dir, attrs); | |
06d60728 CH |
854 | if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC) && |
855 | dma_handle != DMA_MAPPING_ERROR) | |
56e35f9c | 856 | arch_sync_dma_for_device(phys, size, dir); |
06d60728 | 857 | return dma_handle; |
51f8cc9e RM |
858 | } |
859 | ||
06d60728 CH |
860 | static void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle, |
861 | size_t size, enum dma_data_direction dir, unsigned long attrs) | |
0db2e5d1 | 862 | { |
06d60728 CH |
863 | if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) |
864 | iommu_dma_sync_single_for_cpu(dev, dma_handle, size, dir); | |
82612d66 | 865 | __iommu_dma_unmap_swiotlb(dev, dma_handle, size, dir, attrs); |
0db2e5d1 RM |
866 | } |
867 | ||
868 | /* | |
869 | * Prepare a successfully-mapped scatterlist to give back to the caller. | |
809eac54 RM |
870 | * |
871 | * At this point the segments are already laid out by iommu_dma_map_sg() to | |
872 | * avoid individually crossing any boundaries, so we merely need to check a | |
873 | * segment's start address to avoid concatenating across one. | |
0db2e5d1 RM |
874 | */ |
875 | static int __finalise_sg(struct device *dev, struct scatterlist *sg, int nents, | |
876 | dma_addr_t dma_addr) | |
877 | { | |
809eac54 RM |
878 | struct scatterlist *s, *cur = sg; |
879 | unsigned long seg_mask = dma_get_seg_boundary(dev); | |
880 | unsigned int cur_len = 0, max_len = dma_get_max_seg_size(dev); | |
881 | int i, count = 0; | |
0db2e5d1 RM |
882 | |
883 | for_each_sg(sg, s, nents, i) { | |
809eac54 RM |
884 | /* Restore this segment's original unaligned fields first */ |
885 | unsigned int s_iova_off = sg_dma_address(s); | |
0db2e5d1 | 886 | unsigned int s_length = sg_dma_len(s); |
809eac54 | 887 | unsigned int s_iova_len = s->length; |
0db2e5d1 | 888 | |
809eac54 | 889 | s->offset += s_iova_off; |
0db2e5d1 | 890 | s->length = s_length; |
cad34be7 | 891 | sg_dma_address(s) = DMA_MAPPING_ERROR; |
809eac54 RM |
892 | sg_dma_len(s) = 0; |
893 | ||
894 | /* | |
895 | * Now fill in the real DMA data. If... | |
896 | * - there is a valid output segment to append to | |
897 | * - and this segment starts on an IOVA page boundary | |
898 | * - but doesn't fall at a segment boundary | |
899 | * - and wouldn't make the resulting output segment too long | |
900 | */ | |
901 | if (cur_len && !s_iova_off && (dma_addr & seg_mask) && | |
ab2cbeb0 | 902 | (max_len - cur_len >= s_length)) { |
809eac54 RM |
903 | /* ...then concatenate it with the previous one */ |
904 | cur_len += s_length; | |
905 | } else { | |
906 | /* Otherwise start the next output segment */ | |
907 | if (i > 0) | |
908 | cur = sg_next(cur); | |
909 | cur_len = s_length; | |
910 | count++; | |
911 | ||
912 | sg_dma_address(cur) = dma_addr + s_iova_off; | |
913 | } | |
914 | ||
915 | sg_dma_len(cur) = cur_len; | |
916 | dma_addr += s_iova_len; | |
917 | ||
918 | if (s_length + s_iova_off < s_iova_len) | |
919 | cur_len = 0; | |
0db2e5d1 | 920 | } |
809eac54 | 921 | return count; |
0db2e5d1 RM |
922 | } |
923 | ||
924 | /* | |
925 | * If mapping failed, then just restore the original list, | |
926 | * but making sure the DMA fields are invalidated. | |
927 | */ | |
928 | static void __invalidate_sg(struct scatterlist *sg, int nents) | |
929 | { | |
930 | struct scatterlist *s; | |
931 | int i; | |
932 | ||
933 | for_each_sg(sg, s, nents, i) { | |
cad34be7 | 934 | if (sg_dma_address(s) != DMA_MAPPING_ERROR) |
07b48ac4 | 935 | s->offset += sg_dma_address(s); |
0db2e5d1 RM |
936 | if (sg_dma_len(s)) |
937 | s->length = sg_dma_len(s); | |
cad34be7 | 938 | sg_dma_address(s) = DMA_MAPPING_ERROR; |
0db2e5d1 RM |
939 | sg_dma_len(s) = 0; |
940 | } | |
941 | } | |
942 | ||
82612d66 TM |
943 | static void iommu_dma_unmap_sg_swiotlb(struct device *dev, struct scatterlist *sg, |
944 | int nents, enum dma_data_direction dir, unsigned long attrs) | |
945 | { | |
946 | struct scatterlist *s; | |
947 | int i; | |
948 | ||
949 | for_each_sg(sg, s, nents, i) | |
950 | __iommu_dma_unmap_swiotlb(dev, sg_dma_address(s), | |
951 | sg_dma_len(s), dir, attrs); | |
952 | } | |
953 | ||
954 | static int iommu_dma_map_sg_swiotlb(struct device *dev, struct scatterlist *sg, | |
955 | int nents, enum dma_data_direction dir, unsigned long attrs) | |
956 | { | |
957 | struct scatterlist *s; | |
958 | int i; | |
959 | ||
960 | for_each_sg(sg, s, nents, i) { | |
961 | sg_dma_address(s) = __iommu_dma_map_swiotlb(dev, sg_phys(s), | |
962 | s->length, dma_get_mask(dev), | |
963 | dev_is_dma_coherent(dev), dir, attrs); | |
964 | if (sg_dma_address(s) == DMA_MAPPING_ERROR) | |
965 | goto out_unmap; | |
966 | sg_dma_len(s) = s->length; | |
967 | } | |
968 | ||
969 | return nents; | |
970 | ||
971 | out_unmap: | |
972 | iommu_dma_unmap_sg_swiotlb(dev, sg, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC); | |
973 | return 0; | |
974 | } | |
975 | ||
0db2e5d1 RM |
976 | /* |
977 | * The DMA API client is passing in a scatterlist which could describe | |
978 | * any old buffer layout, but the IOMMU API requires everything to be | |
979 | * aligned to IOMMU pages. Hence the need for this complicated bit of | |
980 | * impedance-matching, to be able to hand off a suitably-aligned list, | |
981 | * but still preserve the original offsets and sizes for the caller. | |
982 | */ | |
06d60728 CH |
983 | static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, |
984 | int nents, enum dma_data_direction dir, unsigned long attrs) | |
0db2e5d1 | 985 | { |
43c5bf11 | 986 | struct iommu_domain *domain = iommu_get_dma_domain(dev); |
842fe519 RM |
987 | struct iommu_dma_cookie *cookie = domain->iova_cookie; |
988 | struct iova_domain *iovad = &cookie->iovad; | |
0db2e5d1 | 989 | struct scatterlist *s, *prev = NULL; |
06d60728 | 990 | int prot = dma_info_to_prot(dir, dev_is_dma_coherent(dev), attrs); |
842fe519 | 991 | dma_addr_t iova; |
0db2e5d1 | 992 | size_t iova_len = 0; |
809eac54 | 993 | unsigned long mask = dma_get_seg_boundary(dev); |
0db2e5d1 RM |
994 | int i; |
995 | ||
a8e8af35 | 996 | if (static_branch_unlikely(&iommu_deferred_attach_enabled) && |
3ab65729 | 997 | iommu_deferred_attach(dev, domain)) |
795bbbb9 TM |
998 | return 0; |
999 | ||
06d60728 CH |
1000 | if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) |
1001 | iommu_dma_sync_sg_for_device(dev, sg, nents, dir); | |
1002 | ||
82612d66 TM |
1003 | if (dev_is_untrusted(dev)) |
1004 | return iommu_dma_map_sg_swiotlb(dev, sg, nents, dir, attrs); | |
1005 | ||
0db2e5d1 RM |
1006 | /* |
1007 | * Work out how much IOVA space we need, and align the segments to | |
1008 | * IOVA granules for the IOMMU driver to handle. With some clever | |
1009 | * trickery we can modify the list in-place, but reversibly, by | |
809eac54 | 1010 | * stashing the unaligned parts in the as-yet-unused DMA fields. |
0db2e5d1 RM |
1011 | */ |
1012 | for_each_sg(sg, s, nents, i) { | |
809eac54 | 1013 | size_t s_iova_off = iova_offset(iovad, s->offset); |
0db2e5d1 | 1014 | size_t s_length = s->length; |
809eac54 | 1015 | size_t pad_len = (mask - iova_len + 1) & mask; |
0db2e5d1 | 1016 | |
809eac54 | 1017 | sg_dma_address(s) = s_iova_off; |
0db2e5d1 | 1018 | sg_dma_len(s) = s_length; |
809eac54 RM |
1019 | s->offset -= s_iova_off; |
1020 | s_length = iova_align(iovad, s_length + s_iova_off); | |
0db2e5d1 RM |
1021 | s->length = s_length; |
1022 | ||
1023 | /* | |
809eac54 RM |
1024 | * Due to the alignment of our single IOVA allocation, we can |
1025 | * depend on these assumptions about the segment boundary mask: | |
1026 | * - If mask size >= IOVA size, then the IOVA range cannot | |
1027 | * possibly fall across a boundary, so we don't care. | |
1028 | * - If mask size < IOVA size, then the IOVA range must start | |
1029 | * exactly on a boundary, therefore we can lay things out | |
1030 | * based purely on segment lengths without needing to know | |
1031 | * the actual addresses beforehand. | |
1032 | * - The mask must be a power of 2, so pad_len == 0 if | |
1033 | * iova_len == 0, thus we cannot dereference prev the first | |
1034 | * time through here (i.e. before it has a meaningful value). | |
0db2e5d1 | 1035 | */ |
809eac54 | 1036 | if (pad_len && pad_len < s_length - 1) { |
0db2e5d1 RM |
1037 | prev->length += pad_len; |
1038 | iova_len += pad_len; | |
1039 | } | |
1040 | ||
1041 | iova_len += s_length; | |
1042 | prev = s; | |
1043 | } | |
1044 | ||
842fe519 | 1045 | iova = iommu_dma_alloc_iova(domain, iova_len, dma_get_mask(dev), dev); |
0db2e5d1 RM |
1046 | if (!iova) |
1047 | goto out_restore_sg; | |
1048 | ||
1049 | /* | |
1050 | * We'll leave any physical concatenation to the IOMMU driver's | |
1051 | * implementation - it knows better than we do. | |
1052 | */ | |
781ca2de | 1053 | if (iommu_map_sg_atomic(domain, iova, sg, nents, prot) < iova_len) |
0db2e5d1 RM |
1054 | goto out_free_iova; |
1055 | ||
842fe519 | 1056 | return __finalise_sg(dev, sg, nents, iova); |
0db2e5d1 RM |
1057 | |
1058 | out_free_iova: | |
2a2b8eaa | 1059 | iommu_dma_free_iova(cookie, iova, iova_len, NULL); |
0db2e5d1 RM |
1060 | out_restore_sg: |
1061 | __invalidate_sg(sg, nents); | |
1062 | return 0; | |
1063 | } | |
1064 | ||
06d60728 CH |
1065 | static void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, |
1066 | int nents, enum dma_data_direction dir, unsigned long attrs) | |
0db2e5d1 | 1067 | { |
842fe519 RM |
1068 | dma_addr_t start, end; |
1069 | struct scatterlist *tmp; | |
1070 | int i; | |
06d60728 | 1071 | |
1b961423 | 1072 | if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) |
06d60728 CH |
1073 | iommu_dma_sync_sg_for_cpu(dev, sg, nents, dir); |
1074 | ||
82612d66 TM |
1075 | if (dev_is_untrusted(dev)) { |
1076 | iommu_dma_unmap_sg_swiotlb(dev, sg, nents, dir, attrs); | |
1077 | return; | |
1078 | } | |
1079 | ||
0db2e5d1 RM |
1080 | /* |
1081 | * The scatterlist segments are mapped into a single | |
1082 | * contiguous IOVA allocation, so this is incredibly easy. | |
1083 | */ | |
842fe519 RM |
1084 | start = sg_dma_address(sg); |
1085 | for_each_sg(sg_next(sg), tmp, nents - 1, i) { | |
1086 | if (sg_dma_len(tmp) == 0) | |
1087 | break; | |
1088 | sg = tmp; | |
1089 | } | |
1090 | end = sg_dma_address(sg) + sg_dma_len(sg); | |
b61d271e | 1091 | __iommu_dma_unmap(dev, start, end - start); |
0db2e5d1 RM |
1092 | } |
1093 | ||
06d60728 | 1094 | static dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys, |
51f8cc9e RM |
1095 | size_t size, enum dma_data_direction dir, unsigned long attrs) |
1096 | { | |
1097 | return __iommu_dma_map(dev, phys, size, | |
6e235020 TM |
1098 | dma_info_to_prot(dir, false, attrs) | IOMMU_MMIO, |
1099 | dma_get_mask(dev)); | |
51f8cc9e RM |
1100 | } |
1101 | ||
06d60728 | 1102 | static void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle, |
51f8cc9e RM |
1103 | size_t size, enum dma_data_direction dir, unsigned long attrs) |
1104 | { | |
b61d271e | 1105 | __iommu_dma_unmap(dev, handle, size); |
51f8cc9e RM |
1106 | } |
1107 | ||
8553f6e6 | 1108 | static void __iommu_dma_free(struct device *dev, size_t size, void *cpu_addr) |
bcf4b9c4 RM |
1109 | { |
1110 | size_t alloc_size = PAGE_ALIGN(size); | |
1111 | int count = alloc_size >> PAGE_SHIFT; | |
1112 | struct page *page = NULL, **pages = NULL; | |
1113 | ||
bcf4b9c4 | 1114 | /* Non-coherent atomic allocation? Easy */ |
e6475eb0 | 1115 | if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) && |
c84dc6e6 | 1116 | dma_free_from_pool(dev, cpu_addr, alloc_size)) |
bcf4b9c4 RM |
1117 | return; |
1118 | ||
e6475eb0 | 1119 | if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr)) { |
bcf4b9c4 RM |
1120 | /* |
1121 | * If it the address is remapped, then it's either non-coherent | |
1122 | * or highmem CMA, or an iommu_dma_alloc_remap() construction. | |
1123 | */ | |
5cf45379 | 1124 | pages = dma_common_find_pages(cpu_addr); |
bcf4b9c4 RM |
1125 | if (!pages) |
1126 | page = vmalloc_to_page(cpu_addr); | |
51231740 | 1127 | dma_common_free_remap(cpu_addr, alloc_size); |
bcf4b9c4 RM |
1128 | } else { |
1129 | /* Lowmem means a coherent atomic or CMA allocation */ | |
1130 | page = virt_to_page(cpu_addr); | |
1131 | } | |
1132 | ||
1133 | if (pages) | |
1134 | __iommu_dma_free_pages(pages, count); | |
591fcf3b NC |
1135 | if (page) |
1136 | dma_free_contiguous(dev, page, alloc_size); | |
bcf4b9c4 RM |
1137 | } |
1138 | ||
8553f6e6 RM |
1139 | static void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr, |
1140 | dma_addr_t handle, unsigned long attrs) | |
1141 | { | |
1142 | __iommu_dma_unmap(dev, handle, size); | |
1143 | __iommu_dma_free(dev, size, cpu_addr); | |
1144 | } | |
1145 | ||
ee1ef05d CH |
1146 | static void *iommu_dma_alloc_pages(struct device *dev, size_t size, |
1147 | struct page **pagep, gfp_t gfp, unsigned long attrs) | |
06d60728 CH |
1148 | { |
1149 | bool coherent = dev_is_dma_coherent(dev); | |
9ad5d6ed | 1150 | size_t alloc_size = PAGE_ALIGN(size); |
90ae409f | 1151 | int node = dev_to_node(dev); |
9a4ab94a | 1152 | struct page *page = NULL; |
9ad5d6ed | 1153 | void *cpu_addr; |
06d60728 | 1154 | |
591fcf3b | 1155 | page = dma_alloc_contiguous(dev, alloc_size, gfp); |
90ae409f CH |
1156 | if (!page) |
1157 | page = alloc_pages_node(node, gfp, get_order(alloc_size)); | |
072bebc0 RM |
1158 | if (!page) |
1159 | return NULL; | |
1160 | ||
e6475eb0 | 1161 | if (IS_ENABLED(CONFIG_DMA_REMAP) && (!coherent || PageHighMem(page))) { |
33dcb37c | 1162 | pgprot_t prot = dma_pgprot(dev, PAGE_KERNEL, attrs); |
072bebc0 | 1163 | |
9ad5d6ed | 1164 | cpu_addr = dma_common_contiguous_remap(page, alloc_size, |
51231740 | 1165 | prot, __builtin_return_address(0)); |
9ad5d6ed | 1166 | if (!cpu_addr) |
ee1ef05d | 1167 | goto out_free_pages; |
8680aa5a RM |
1168 | |
1169 | if (!coherent) | |
9ad5d6ed | 1170 | arch_dma_prep_coherent(page, size); |
8680aa5a | 1171 | } else { |
9ad5d6ed | 1172 | cpu_addr = page_address(page); |
8680aa5a | 1173 | } |
ee1ef05d CH |
1174 | |
1175 | *pagep = page; | |
9ad5d6ed RM |
1176 | memset(cpu_addr, 0, alloc_size); |
1177 | return cpu_addr; | |
072bebc0 | 1178 | out_free_pages: |
591fcf3b | 1179 | dma_free_contiguous(dev, page, alloc_size); |
072bebc0 | 1180 | return NULL; |
06d60728 CH |
1181 | } |
1182 | ||
ee1ef05d CH |
1183 | static void *iommu_dma_alloc(struct device *dev, size_t size, |
1184 | dma_addr_t *handle, gfp_t gfp, unsigned long attrs) | |
1185 | { | |
1186 | bool coherent = dev_is_dma_coherent(dev); | |
1187 | int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs); | |
1188 | struct page *page = NULL; | |
1189 | void *cpu_addr; | |
1190 | ||
1191 | gfp |= __GFP_ZERO; | |
1192 | ||
e6475eb0 | 1193 | if (IS_ENABLED(CONFIG_DMA_REMAP) && gfpflags_allow_blocking(gfp) && |
e8d39a90 CH |
1194 | !(attrs & DMA_ATTR_FORCE_CONTIGUOUS)) { |
1195 | return iommu_dma_alloc_remap(dev, size, handle, gfp, | |
1196 | dma_pgprot(dev, PAGE_KERNEL, attrs), attrs); | |
1197 | } | |
ee1ef05d | 1198 | |
e6475eb0 CH |
1199 | if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) && |
1200 | !gfpflags_allow_blocking(gfp) && !coherent) | |
9420139f CH |
1201 | page = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &cpu_addr, |
1202 | gfp, NULL); | |
ee1ef05d CH |
1203 | else |
1204 | cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs); | |
1205 | if (!cpu_addr) | |
1206 | return NULL; | |
1207 | ||
6e235020 TM |
1208 | *handle = __iommu_dma_map(dev, page_to_phys(page), size, ioprot, |
1209 | dev->coherent_dma_mask); | |
ee1ef05d CH |
1210 | if (*handle == DMA_MAPPING_ERROR) { |
1211 | __iommu_dma_free(dev, size, cpu_addr); | |
1212 | return NULL; | |
1213 | } | |
1214 | ||
1215 | return cpu_addr; | |
1216 | } | |
1217 | ||
06d60728 CH |
1218 | static int iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma, |
1219 | void *cpu_addr, dma_addr_t dma_addr, size_t size, | |
1220 | unsigned long attrs) | |
1221 | { | |
1222 | unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; | |
efd9f10b | 1223 | unsigned long pfn, off = vma->vm_pgoff; |
06d60728 CH |
1224 | int ret; |
1225 | ||
33dcb37c | 1226 | vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs); |
06d60728 CH |
1227 | |
1228 | if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) | |
1229 | return ret; | |
1230 | ||
1231 | if (off >= nr_pages || vma_pages(vma) > nr_pages - off) | |
1232 | return -ENXIO; | |
1233 | ||
e6475eb0 | 1234 | if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr)) { |
5cf45379 | 1235 | struct page **pages = dma_common_find_pages(cpu_addr); |
06d60728 | 1236 | |
efd9f10b | 1237 | if (pages) |
71fe89ce | 1238 | return vm_map_pages(vma, pages, nr_pages); |
efd9f10b CH |
1239 | pfn = vmalloc_to_pfn(cpu_addr); |
1240 | } else { | |
1241 | pfn = page_to_pfn(virt_to_page(cpu_addr)); | |
06d60728 CH |
1242 | } |
1243 | ||
efd9f10b CH |
1244 | return remap_pfn_range(vma, vma->vm_start, pfn + off, |
1245 | vma->vm_end - vma->vm_start, | |
1246 | vma->vm_page_prot); | |
06d60728 CH |
1247 | } |
1248 | ||
06d60728 CH |
1249 | static int iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt, |
1250 | void *cpu_addr, dma_addr_t dma_addr, size_t size, | |
1251 | unsigned long attrs) | |
1252 | { | |
3fb3378b CH |
1253 | struct page *page; |
1254 | int ret; | |
06d60728 | 1255 | |
e6475eb0 | 1256 | if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr)) { |
5cf45379 | 1257 | struct page **pages = dma_common_find_pages(cpu_addr); |
06d60728 | 1258 | |
3fb3378b CH |
1259 | if (pages) { |
1260 | return sg_alloc_table_from_pages(sgt, pages, | |
1261 | PAGE_ALIGN(size) >> PAGE_SHIFT, | |
1262 | 0, size, GFP_KERNEL); | |
1263 | } | |
1264 | ||
1265 | page = vmalloc_to_page(cpu_addr); | |
1266 | } else { | |
1267 | page = virt_to_page(cpu_addr); | |
06d60728 CH |
1268 | } |
1269 | ||
3fb3378b CH |
1270 | ret = sg_alloc_table(sgt, 1, GFP_KERNEL); |
1271 | if (!ret) | |
1272 | sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); | |
1273 | return ret; | |
06d60728 CH |
1274 | } |
1275 | ||
158a6d3c YS |
1276 | static unsigned long iommu_dma_get_merge_boundary(struct device *dev) |
1277 | { | |
1278 | struct iommu_domain *domain = iommu_get_dma_domain(dev); | |
1279 | ||
1280 | return (1UL << __ffs(domain->pgsize_bitmap)) - 1; | |
1281 | } | |
1282 | ||
06d60728 CH |
1283 | static const struct dma_map_ops iommu_dma_ops = { |
1284 | .alloc = iommu_dma_alloc, | |
1285 | .free = iommu_dma_free, | |
efa70f2f CH |
1286 | .alloc_pages = dma_common_alloc_pages, |
1287 | .free_pages = dma_common_free_pages, | |
e817ee5f CH |
1288 | #ifdef CONFIG_DMA_REMAP |
1289 | .alloc_noncontiguous = iommu_dma_alloc_noncontiguous, | |
1290 | .free_noncontiguous = iommu_dma_free_noncontiguous, | |
1291 | #endif | |
06d60728 CH |
1292 | .mmap = iommu_dma_mmap, |
1293 | .get_sgtable = iommu_dma_get_sgtable, | |
1294 | .map_page = iommu_dma_map_page, | |
1295 | .unmap_page = iommu_dma_unmap_page, | |
1296 | .map_sg = iommu_dma_map_sg, | |
1297 | .unmap_sg = iommu_dma_unmap_sg, | |
1298 | .sync_single_for_cpu = iommu_dma_sync_single_for_cpu, | |
1299 | .sync_single_for_device = iommu_dma_sync_single_for_device, | |
1300 | .sync_sg_for_cpu = iommu_dma_sync_sg_for_cpu, | |
1301 | .sync_sg_for_device = iommu_dma_sync_sg_for_device, | |
1302 | .map_resource = iommu_dma_map_resource, | |
1303 | .unmap_resource = iommu_dma_unmap_resource, | |
158a6d3c | 1304 | .get_merge_boundary = iommu_dma_get_merge_boundary, |
06d60728 CH |
1305 | }; |
1306 | ||
1307 | /* | |
1308 | * The IOMMU core code allocates the default DMA domain, which the underlying | |
1309 | * IOMMU driver needs to support via the dma-iommu layer. | |
1310 | */ | |
ac6d7046 | 1311 | void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 dma_limit) |
06d60728 CH |
1312 | { |
1313 | struct iommu_domain *domain = iommu_get_domain_for_dev(dev); | |
1314 | ||
1315 | if (!domain) | |
1316 | goto out_err; | |
1317 | ||
1318 | /* | |
1319 | * The IOMMU core code allocates the default DMA domain, which the | |
1320 | * underlying IOMMU driver needs to support via the dma-iommu layer. | |
1321 | */ | |
1322 | if (domain->type == IOMMU_DOMAIN_DMA) { | |
ac6d7046 | 1323 | if (iommu_dma_init_domain(domain, dma_base, dma_limit, dev)) |
06d60728 CH |
1324 | goto out_err; |
1325 | dev->dma_ops = &iommu_dma_ops; | |
1326 | } | |
1327 | ||
1328 | return; | |
1329 | out_err: | |
1330 | pr_warn("Failed to set up IOMMU for device %s; retaining platform DMA ops\n", | |
1331 | dev_name(dev)); | |
51f8cc9e | 1332 | } |
8ce4904b | 1333 | EXPORT_SYMBOL_GPL(iommu_setup_dma_ops); |
51f8cc9e | 1334 | |
44bb7e24 RM |
1335 | static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev, |
1336 | phys_addr_t msi_addr, struct iommu_domain *domain) | |
1337 | { | |
1338 | struct iommu_dma_cookie *cookie = domain->iova_cookie; | |
1339 | struct iommu_dma_msi_page *msi_page; | |
842fe519 | 1340 | dma_addr_t iova; |
44bb7e24 | 1341 | int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; |
fdbe574e | 1342 | size_t size = cookie_msi_granule(cookie); |
44bb7e24 | 1343 | |
fdbe574e | 1344 | msi_addr &= ~(phys_addr_t)(size - 1); |
44bb7e24 RM |
1345 | list_for_each_entry(msi_page, &cookie->msi_page_list, list) |
1346 | if (msi_page->phys == msi_addr) | |
1347 | return msi_page; | |
1348 | ||
c1864790 | 1349 | msi_page = kzalloc(sizeof(*msi_page), GFP_KERNEL); |
44bb7e24 RM |
1350 | if (!msi_page) |
1351 | return NULL; | |
1352 | ||
8af23fad RM |
1353 | iova = iommu_dma_alloc_iova(domain, size, dma_get_mask(dev), dev); |
1354 | if (!iova) | |
a44e6657 | 1355 | goto out_free_page; |
44bb7e24 | 1356 | |
8af23fad RM |
1357 | if (iommu_map(domain, iova, msi_addr, size, prot)) |
1358 | goto out_free_iova; | |
1359 | ||
44bb7e24 | 1360 | INIT_LIST_HEAD(&msi_page->list); |
a44e6657 RM |
1361 | msi_page->phys = msi_addr; |
1362 | msi_page->iova = iova; | |
44bb7e24 RM |
1363 | list_add(&msi_page->list, &cookie->msi_page_list); |
1364 | return msi_page; | |
1365 | ||
8af23fad | 1366 | out_free_iova: |
2a2b8eaa | 1367 | iommu_dma_free_iova(cookie, iova, size, NULL); |
44bb7e24 RM |
1368 | out_free_page: |
1369 | kfree(msi_page); | |
1370 | return NULL; | |
1371 | } | |
1372 | ||
ece6e6f0 | 1373 | int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr) |
44bb7e24 | 1374 | { |
ece6e6f0 | 1375 | struct device *dev = msi_desc_to_dev(desc); |
44bb7e24 | 1376 | struct iommu_domain *domain = iommu_get_domain_for_dev(dev); |
44bb7e24 | 1377 | struct iommu_dma_msi_page *msi_page; |
c1864790 | 1378 | static DEFINE_MUTEX(msi_prepare_lock); /* see below */ |
44bb7e24 | 1379 | |
ece6e6f0 JG |
1380 | if (!domain || !domain->iova_cookie) { |
1381 | desc->iommu_cookie = NULL; | |
1382 | return 0; | |
1383 | } | |
44bb7e24 | 1384 | |
44bb7e24 | 1385 | /* |
c1864790 RM |
1386 | * In fact the whole prepare operation should already be serialised by |
1387 | * irq_domain_mutex further up the callchain, but that's pretty subtle | |
1388 | * on its own, so consider this locking as failsafe documentation... | |
44bb7e24 | 1389 | */ |
c1864790 | 1390 | mutex_lock(&msi_prepare_lock); |
44bb7e24 | 1391 | msi_page = iommu_dma_get_msi_page(dev, msi_addr, domain); |
c1864790 | 1392 | mutex_unlock(&msi_prepare_lock); |
44bb7e24 | 1393 | |
ece6e6f0 JG |
1394 | msi_desc_set_iommu_cookie(desc, msi_page); |
1395 | ||
1396 | if (!msi_page) | |
1397 | return -ENOMEM; | |
1398 | return 0; | |
1399 | } | |
1400 | ||
1401 | void iommu_dma_compose_msi_msg(struct msi_desc *desc, | |
1402 | struct msi_msg *msg) | |
1403 | { | |
1404 | struct device *dev = msi_desc_to_dev(desc); | |
1405 | const struct iommu_domain *domain = iommu_get_domain_for_dev(dev); | |
1406 | const struct iommu_dma_msi_page *msi_page; | |
1407 | ||
1408 | msi_page = msi_desc_get_iommu_cookie(desc); | |
1409 | ||
1410 | if (!domain || !domain->iova_cookie || WARN_ON(!msi_page)) | |
1411 | return; | |
1412 | ||
1413 | msg->address_hi = upper_32_bits(msi_page->iova); | |
1414 | msg->address_lo &= cookie_msi_granule(domain->iova_cookie) - 1; | |
1415 | msg->address_lo += lower_32_bits(msi_page->iova); | |
44bb7e24 | 1416 | } |
06d60728 CH |
1417 | |
1418 | static int iommu_dma_init(void) | |
1419 | { | |
a8e8af35 LJ |
1420 | if (is_kdump_kernel()) |
1421 | static_branch_enable(&iommu_deferred_attach_enabled); | |
1422 | ||
06d60728 | 1423 | return iova_cache_get(); |
44bb7e24 | 1424 | } |
06d60728 | 1425 | arch_initcall(iommu_dma_init); |