1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 // Copyright (C) 2016-2018, Allwinner Technology CO., LTD.
3 // Copyright (C) 2019-2020, Cerno
5 #include <linux/bitfield.h>
8 #include <linux/device.h>
9 #include <linux/dma-direction.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/err.h>
12 #include <linux/errno.h>
13 #include <linux/interrupt.h>
14 #include <linux/iommu.h>
15 #include <linux/iopoll.h>
16 #include <linux/ioport.h>
17 #include <linux/log2.h>
18 #include <linux/module.h>
19 #include <linux/of_platform.h>
20 #include <linux/platform_device.h>
22 #include <linux/pm_runtime.h>
23 #include <linux/reset.h>
24 #include <linux/sizes.h>
25 #include <linux/slab.h>
26 #include <linux/spinlock.h>
27 #include <linux/types.h>
29 #define IOMMU_RESET_REG 0x010
30 #define IOMMU_RESET_RELEASE_ALL 0xffffffff
31 #define IOMMU_ENABLE_REG 0x020
32 #define IOMMU_ENABLE_ENABLE BIT(0)
34 #define IOMMU_BYPASS_REG 0x030
35 #define IOMMU_AUTO_GATING_REG 0x040
36 #define IOMMU_AUTO_GATING_ENABLE BIT(0)
38 #define IOMMU_WBUF_CTRL_REG 0x044
39 #define IOMMU_OOO_CTRL_REG 0x048
40 #define IOMMU_4KB_BDY_PRT_CTRL_REG 0x04c
41 #define IOMMU_TTB_REG 0x050
42 #define IOMMU_TLB_ENABLE_REG 0x060
43 #define IOMMU_TLB_PREFETCH_REG 0x070
44 #define IOMMU_TLB_PREFETCH_MASTER_ENABLE(m) BIT(m)
46 #define IOMMU_TLB_FLUSH_REG 0x080
47 #define IOMMU_TLB_FLUSH_PTW_CACHE BIT(17)
48 #define IOMMU_TLB_FLUSH_MACRO_TLB BIT(16)
49 #define IOMMU_TLB_FLUSH_MICRO_TLB(i) (BIT(i) & GENMASK(5, 0))
51 #define IOMMU_TLB_IVLD_ADDR_REG 0x090
52 #define IOMMU_TLB_IVLD_ADDR_MASK_REG 0x094
53 #define IOMMU_TLB_IVLD_ENABLE_REG 0x098
54 #define IOMMU_TLB_IVLD_ENABLE_ENABLE BIT(0)
56 #define IOMMU_PC_IVLD_ADDR_REG 0x0a0
57 #define IOMMU_PC_IVLD_ENABLE_REG 0x0a8
58 #define IOMMU_PC_IVLD_ENABLE_ENABLE BIT(0)
60 #define IOMMU_DM_AUT_CTRL_REG(d) (0x0b0 + ((d) / 2) * 4)
61 #define IOMMU_DM_AUT_CTRL_RD_UNAVAIL(d, m) (1 << (((d & 1) * 16) + ((m) * 2)))
62 #define IOMMU_DM_AUT_CTRL_WR_UNAVAIL(d, m) (1 << (((d & 1) * 16) + ((m) * 2) + 1))
64 #define IOMMU_DM_AUT_OVWT_REG 0x0d0
65 #define IOMMU_INT_ENABLE_REG 0x100
66 #define IOMMU_INT_CLR_REG 0x104
67 #define IOMMU_INT_STA_REG 0x108
68 #define IOMMU_INT_ERR_ADDR_REG(i) (0x110 + (i) * 4)
69 #define IOMMU_INT_ERR_ADDR_L1_REG 0x130
70 #define IOMMU_INT_ERR_ADDR_L2_REG 0x134
71 #define IOMMU_INT_ERR_DATA_REG(i) (0x150 + (i) * 4)
72 #define IOMMU_L1PG_INT_REG 0x0180
73 #define IOMMU_L2PG_INT_REG 0x0184
75 #define IOMMU_INT_INVALID_L2PG BIT(17)
76 #define IOMMU_INT_INVALID_L1PG BIT(16)
77 #define IOMMU_INT_MASTER_PERMISSION(m) BIT(m)
78 #define IOMMU_INT_MASTER_MASK (IOMMU_INT_MASTER_PERMISSION(0) | \
79 IOMMU_INT_MASTER_PERMISSION(1) | \
80 IOMMU_INT_MASTER_PERMISSION(2) | \
81 IOMMU_INT_MASTER_PERMISSION(3) | \
82 IOMMU_INT_MASTER_PERMISSION(4) | \
83 IOMMU_INT_MASTER_PERMISSION(5))
84 #define IOMMU_INT_MASK (IOMMU_INT_INVALID_L1PG | \
85 IOMMU_INT_INVALID_L2PG | \
86 IOMMU_INT_MASTER_MASK)
88 #define PT_ENTRY_SIZE sizeof(u32)
90 #define NUM_DT_ENTRIES 4096
91 #define DT_SIZE (NUM_DT_ENTRIES * PT_ENTRY_SIZE)
93 #define NUM_PT_ENTRIES 256
94 #define PT_SIZE (NUM_PT_ENTRIES * PT_ENTRY_SIZE)
96 #define SPAGE_SIZE 4096
99 struct iommu_device iommu;
101 /* Lock to modify the IOMMU registers */
102 spinlock_t iommu_lock;
106 struct reset_control *reset;
109 struct iommu_domain *domain;
110 struct kmem_cache *pt_pool;
113 struct sun50i_iommu_domain {
114 struct iommu_domain domain;
116 /* Number of devices attached to the domain */
123 struct sun50i_iommu *iommu;
126 static struct sun50i_iommu_domain *to_sun50i_domain(struct iommu_domain *domain)
128 return container_of(domain, struct sun50i_iommu_domain, domain);
131 static struct sun50i_iommu *sun50i_iommu_from_dev(struct device *dev)
133 return dev_iommu_priv_get(dev);
136 static u32 iommu_read(struct sun50i_iommu *iommu, u32 offset)
138 return readl(iommu->base + offset);
141 static void iommu_write(struct sun50i_iommu *iommu, u32 offset, u32 value)
143 writel(value, iommu->base + offset);
147 * The Allwinner H6 IOMMU uses a 2-level page table.
149 * The first level is the usual Directory Table (DT), that consists of
150 * 4096 4-bytes Directory Table Entries (DTE), each pointing to a Page
153 * Each PT consits of 256 4-bytes Page Table Entries (PTE), each
154 * pointing to a 4kB page of physical memory.
156 * The IOMMU supports a single DT, pointed by the IOMMU_TTB_REG
157 * register that contains its physical address.
160 #define SUN50I_IOVA_DTE_MASK GENMASK(31, 20)
161 #define SUN50I_IOVA_PTE_MASK GENMASK(19, 12)
162 #define SUN50I_IOVA_PAGE_MASK GENMASK(11, 0)
164 static u32 sun50i_iova_get_dte_index(dma_addr_t iova)
166 return FIELD_GET(SUN50I_IOVA_DTE_MASK, iova);
169 static u32 sun50i_iova_get_pte_index(dma_addr_t iova)
171 return FIELD_GET(SUN50I_IOVA_PTE_MASK, iova);
174 static u32 sun50i_iova_get_page_offset(dma_addr_t iova)
176 return FIELD_GET(SUN50I_IOVA_PAGE_MASK, iova);
180 * Each Directory Table Entry has a Page Table address and a valid
183 * +---------------------+-----------+-+
184 * | PT address | Reserved |V|
185 * +---------------------+-----------+-+
186 * 31:10 - Page Table address
188 * 1:0 - 1 if the entry is valid
191 #define SUN50I_DTE_PT_ADDRESS_MASK GENMASK(31, 10)
192 #define SUN50I_DTE_PT_ATTRS GENMASK(1, 0)
193 #define SUN50I_DTE_PT_VALID 1
195 static phys_addr_t sun50i_dte_get_pt_address(u32 dte)
197 return (phys_addr_t)dte & SUN50I_DTE_PT_ADDRESS_MASK;
200 static bool sun50i_dte_is_pt_valid(u32 dte)
202 return (dte & SUN50I_DTE_PT_ATTRS) == SUN50I_DTE_PT_VALID;
205 static u32 sun50i_mk_dte(dma_addr_t pt_dma)
207 return (pt_dma & SUN50I_DTE_PT_ADDRESS_MASK) | SUN50I_DTE_PT_VALID;
211 * Each PTE has a Page address, an authority index and a valid bit:
213 * +----------------+-----+-----+-----+---+-----+
214 * | Page address | Rsv | ACI | Rsv | V | Rsv |
215 * +----------------+-----+-----+-----+---+-----+
216 * 31:12 - Page address
218 * 7:4 - Authority Control Index
220 * 1 - 1 if the entry is valid
223 * The way permissions work is that the IOMMU has 16 "domains" that
224 * can be configured to give each masters either read or write
225 * permissions through the IOMMU_DM_AUT_CTRL_REG registers. The domain
226 * 0 seems like the default domain, and its permissions in the
227 * IOMMU_DM_AUT_CTRL_REG are only read-only, so it's not really
228 * useful to enforce any particular permission.
230 * Each page entry will then have a reference to the domain they are
231 * affected to, so that we can actually enforce them on a per-page
234 * In order to make it work with the IOMMU framework, we will be using
235 * 4 different domains, starting at 1: RD_WR, RD, WR and NONE
236 * depending on the permission we want to enforce. Each domain will
237 * have each master setup in the same way, since the IOMMU framework
238 * doesn't seem to restrict page access on a per-device basis. And
239 * then we will use the relevant domain index when generating the page
240 * table entry depending on the permissions we want to be enforced.
243 enum sun50i_iommu_aci {
244 SUN50I_IOMMU_ACI_DO_NOT_USE = 0,
245 SUN50I_IOMMU_ACI_NONE,
248 SUN50I_IOMMU_ACI_RD_WR,
251 #define SUN50I_PTE_PAGE_ADDRESS_MASK GENMASK(31, 12)
252 #define SUN50I_PTE_ACI_MASK GENMASK(7, 4)
253 #define SUN50I_PTE_PAGE_VALID BIT(1)
255 static phys_addr_t sun50i_pte_get_page_address(u32 pte)
257 return (phys_addr_t)pte & SUN50I_PTE_PAGE_ADDRESS_MASK;
260 static enum sun50i_iommu_aci sun50i_get_pte_aci(u32 pte)
262 return FIELD_GET(SUN50I_PTE_ACI_MASK, pte);
265 static bool sun50i_pte_is_page_valid(u32 pte)
267 return pte & SUN50I_PTE_PAGE_VALID;
270 static u32 sun50i_mk_pte(phys_addr_t page, int prot)
272 enum sun50i_iommu_aci aci;
275 if ((prot & (IOMMU_READ | IOMMU_WRITE)) == (IOMMU_READ | IOMMU_WRITE))
276 aci = SUN50I_IOMMU_ACI_RD_WR;
277 else if (prot & IOMMU_READ)
278 aci = SUN50I_IOMMU_ACI_RD;
279 else if (prot & IOMMU_WRITE)
280 aci = SUN50I_IOMMU_ACI_WR;
282 aci = SUN50I_IOMMU_ACI_NONE;
284 flags |= FIELD_PREP(SUN50I_PTE_ACI_MASK, aci);
285 page &= SUN50I_PTE_PAGE_ADDRESS_MASK;
286 return page | flags | SUN50I_PTE_PAGE_VALID;
289 static void sun50i_table_flush(struct sun50i_iommu_domain *sun50i_domain,
290 void *vaddr, unsigned int count)
292 struct sun50i_iommu *iommu = sun50i_domain->iommu;
293 dma_addr_t dma = virt_to_phys(vaddr);
294 size_t size = count * PT_ENTRY_SIZE;
296 dma_sync_single_for_device(iommu->dev, dma, size, DMA_TO_DEVICE);
299 static void sun50i_iommu_zap_iova(struct sun50i_iommu *iommu,
305 iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_REG, iova);
306 iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_MASK_REG, GENMASK(31, 12));
307 iommu_write(iommu, IOMMU_TLB_IVLD_ENABLE_REG,
308 IOMMU_TLB_IVLD_ENABLE_ENABLE);
310 ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_IVLD_ENABLE_REG,
313 dev_warn(iommu->dev, "TLB invalidation timed out!\n");
316 static void sun50i_iommu_zap_ptw_cache(struct sun50i_iommu *iommu,
322 iommu_write(iommu, IOMMU_PC_IVLD_ADDR_REG, iova);
323 iommu_write(iommu, IOMMU_PC_IVLD_ENABLE_REG,
324 IOMMU_PC_IVLD_ENABLE_ENABLE);
326 ret = readl_poll_timeout_atomic(iommu->base + IOMMU_PC_IVLD_ENABLE_REG,
329 dev_warn(iommu->dev, "PTW cache invalidation timed out!\n");
332 static void sun50i_iommu_zap_range(struct sun50i_iommu *iommu,
333 unsigned long iova, size_t size)
335 assert_spin_locked(&iommu->iommu_lock);
337 iommu_write(iommu, IOMMU_AUTO_GATING_REG, 0);
339 sun50i_iommu_zap_iova(iommu, iova);
340 sun50i_iommu_zap_iova(iommu, iova + SPAGE_SIZE);
341 if (size > SPAGE_SIZE) {
342 sun50i_iommu_zap_iova(iommu, iova + size);
343 sun50i_iommu_zap_iova(iommu, iova + size + SPAGE_SIZE);
345 sun50i_iommu_zap_ptw_cache(iommu, iova);
346 sun50i_iommu_zap_ptw_cache(iommu, iova + SZ_1M);
348 sun50i_iommu_zap_ptw_cache(iommu, iova + size);
349 sun50i_iommu_zap_ptw_cache(iommu, iova + size + SZ_1M);
352 iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE);
355 static int sun50i_iommu_flush_all_tlb(struct sun50i_iommu *iommu)
360 assert_spin_locked(&iommu->iommu_lock);
364 IOMMU_TLB_FLUSH_PTW_CACHE |
365 IOMMU_TLB_FLUSH_MACRO_TLB |
366 IOMMU_TLB_FLUSH_MICRO_TLB(5) |
367 IOMMU_TLB_FLUSH_MICRO_TLB(4) |
368 IOMMU_TLB_FLUSH_MICRO_TLB(3) |
369 IOMMU_TLB_FLUSH_MICRO_TLB(2) |
370 IOMMU_TLB_FLUSH_MICRO_TLB(1) |
371 IOMMU_TLB_FLUSH_MICRO_TLB(0));
373 ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_FLUSH_REG,
377 dev_warn(iommu->dev, "TLB Flush timed out!\n");
382 static void sun50i_iommu_flush_iotlb_all(struct iommu_domain *domain)
384 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
385 struct sun50i_iommu *iommu = sun50i_domain->iommu;
389 * At boot, we'll have a first call into .flush_iotlb_all right after
390 * .probe_device, and since we link our (single) domain to our iommu in
391 * the .attach_device callback, we don't have that pointer set.
393 * It shouldn't really be any trouble to ignore it though since we flush
394 * all caches as part of the device powerup.
399 spin_lock_irqsave(&iommu->iommu_lock, flags);
400 sun50i_iommu_flush_all_tlb(iommu);
401 spin_unlock_irqrestore(&iommu->iommu_lock, flags);
404 static int sun50i_iommu_iotlb_sync_map(struct iommu_domain *domain,
405 unsigned long iova, size_t size)
407 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
408 struct sun50i_iommu *iommu = sun50i_domain->iommu;
411 spin_lock_irqsave(&iommu->iommu_lock, flags);
412 sun50i_iommu_zap_range(iommu, iova, size);
413 spin_unlock_irqrestore(&iommu->iommu_lock, flags);
418 static void sun50i_iommu_iotlb_sync(struct iommu_domain *domain,
419 struct iommu_iotlb_gather *gather)
421 sun50i_iommu_flush_iotlb_all(domain);
424 static int sun50i_iommu_enable(struct sun50i_iommu *iommu)
426 struct sun50i_iommu_domain *sun50i_domain;
433 sun50i_domain = to_sun50i_domain(iommu->domain);
435 ret = reset_control_deassert(iommu->reset);
439 ret = clk_prepare_enable(iommu->clk);
441 goto err_reset_assert;
443 spin_lock_irqsave(&iommu->iommu_lock, flags);
445 iommu_write(iommu, IOMMU_TTB_REG, sun50i_domain->dt_dma);
446 iommu_write(iommu, IOMMU_TLB_PREFETCH_REG,
447 IOMMU_TLB_PREFETCH_MASTER_ENABLE(0) |
448 IOMMU_TLB_PREFETCH_MASTER_ENABLE(1) |
449 IOMMU_TLB_PREFETCH_MASTER_ENABLE(2) |
450 IOMMU_TLB_PREFETCH_MASTER_ENABLE(3) |
451 IOMMU_TLB_PREFETCH_MASTER_ENABLE(4) |
452 IOMMU_TLB_PREFETCH_MASTER_ENABLE(5));
453 iommu_write(iommu, IOMMU_INT_ENABLE_REG, IOMMU_INT_MASK);
454 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_NONE),
455 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) |
456 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) |
457 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) |
458 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) |
459 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) |
460 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) |
461 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) |
462 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) |
463 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) |
464 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) |
465 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5) |
466 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5));
468 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_RD),
469 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 0) |
470 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 1) |
471 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 2) |
472 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 3) |
473 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 4) |
474 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 5));
476 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_WR),
477 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 0) |
478 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 1) |
479 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 2) |
480 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 3) |
481 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 4) |
482 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 5));
484 ret = sun50i_iommu_flush_all_tlb(iommu);
486 spin_unlock_irqrestore(&iommu->iommu_lock, flags);
487 goto err_clk_disable;
490 iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE);
491 iommu_write(iommu, IOMMU_ENABLE_REG, IOMMU_ENABLE_ENABLE);
493 spin_unlock_irqrestore(&iommu->iommu_lock, flags);
498 clk_disable_unprepare(iommu->clk);
501 reset_control_assert(iommu->reset);
506 static void sun50i_iommu_disable(struct sun50i_iommu *iommu)
510 spin_lock_irqsave(&iommu->iommu_lock, flags);
512 iommu_write(iommu, IOMMU_ENABLE_REG, 0);
513 iommu_write(iommu, IOMMU_TTB_REG, 0);
515 spin_unlock_irqrestore(&iommu->iommu_lock, flags);
517 clk_disable_unprepare(iommu->clk);
518 reset_control_assert(iommu->reset);
521 static void *sun50i_iommu_alloc_page_table(struct sun50i_iommu *iommu,
527 page_table = kmem_cache_zalloc(iommu->pt_pool, gfp);
529 return ERR_PTR(-ENOMEM);
531 pt_dma = dma_map_single(iommu->dev, page_table, PT_SIZE, DMA_TO_DEVICE);
532 if (dma_mapping_error(iommu->dev, pt_dma)) {
533 dev_err(iommu->dev, "Couldn't map L2 Page Table\n");
534 kmem_cache_free(iommu->pt_pool, page_table);
535 return ERR_PTR(-ENOMEM);
538 /* We rely on the physical address and DMA address being the same */
539 WARN_ON(pt_dma != virt_to_phys(page_table));
544 static void sun50i_iommu_free_page_table(struct sun50i_iommu *iommu,
547 phys_addr_t pt_phys = virt_to_phys(page_table);
549 dma_unmap_single(iommu->dev, pt_phys, PT_SIZE, DMA_TO_DEVICE);
550 kmem_cache_free(iommu->pt_pool, page_table);
553 static u32 *sun50i_dte_get_page_table(struct sun50i_iommu_domain *sun50i_domain,
554 dma_addr_t iova, gfp_t gfp)
556 struct sun50i_iommu *iommu = sun50i_domain->iommu;
562 dte_addr = &sun50i_domain->dt[sun50i_iova_get_dte_index(iova)];
564 if (sun50i_dte_is_pt_valid(dte)) {
565 phys_addr_t pt_phys = sun50i_dte_get_pt_address(dte);
566 return (u32 *)phys_to_virt(pt_phys);
569 page_table = sun50i_iommu_alloc_page_table(iommu, gfp);
570 if (IS_ERR(page_table))
573 dte = sun50i_mk_dte(virt_to_phys(page_table));
574 old_dte = cmpxchg(dte_addr, 0, dte);
576 phys_addr_t installed_pt_phys =
577 sun50i_dte_get_pt_address(old_dte);
578 u32 *installed_pt = phys_to_virt(installed_pt_phys);
579 u32 *drop_pt = page_table;
581 page_table = installed_pt;
583 sun50i_iommu_free_page_table(iommu, drop_pt);
586 sun50i_table_flush(sun50i_domain, page_table, NUM_PT_ENTRIES);
587 sun50i_table_flush(sun50i_domain, dte_addr, 1);
592 static int sun50i_iommu_map(struct iommu_domain *domain, unsigned long iova,
593 phys_addr_t paddr, size_t size, size_t count,
594 int prot, gfp_t gfp, size_t *mapped)
596 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
597 struct sun50i_iommu *iommu = sun50i_domain->iommu;
599 u32 *page_table, *pte_addr;
602 page_table = sun50i_dte_get_page_table(sun50i_domain, iova, gfp);
603 if (IS_ERR(page_table)) {
604 ret = PTR_ERR(page_table);
608 pte_index = sun50i_iova_get_pte_index(iova);
609 pte_addr = &page_table[pte_index];
610 if (unlikely(sun50i_pte_is_page_valid(*pte_addr))) {
611 phys_addr_t page_phys = sun50i_pte_get_page_address(*pte_addr);
613 "iova %pad already mapped to %pa cannot remap to %pa prot: %#x\n",
614 &iova, &page_phys, &paddr, prot);
619 *pte_addr = sun50i_mk_pte(paddr, prot);
620 sun50i_table_flush(sun50i_domain, pte_addr, 1);
627 static size_t sun50i_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
628 size_t size, size_t count, struct iommu_iotlb_gather *gather)
630 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
635 dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)];
636 if (!sun50i_dte_is_pt_valid(dte))
639 pt_phys = sun50i_dte_get_pt_address(dte);
640 pte_addr = (u32 *)phys_to_virt(pt_phys) + sun50i_iova_get_pte_index(iova);
642 if (!sun50i_pte_is_page_valid(*pte_addr))
645 memset(pte_addr, 0, sizeof(*pte_addr));
646 sun50i_table_flush(sun50i_domain, pte_addr, 1);
651 static phys_addr_t sun50i_iommu_iova_to_phys(struct iommu_domain *domain,
654 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
659 dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)];
660 if (!sun50i_dte_is_pt_valid(dte))
663 pt_phys = sun50i_dte_get_pt_address(dte);
664 page_table = (u32 *)phys_to_virt(pt_phys);
665 pte = page_table[sun50i_iova_get_pte_index(iova)];
666 if (!sun50i_pte_is_page_valid(pte))
669 return sun50i_pte_get_page_address(pte) +
670 sun50i_iova_get_page_offset(iova);
673 static struct iommu_domain *
674 sun50i_iommu_domain_alloc_paging(struct device *dev)
676 struct sun50i_iommu_domain *sun50i_domain;
678 sun50i_domain = kzalloc(sizeof(*sun50i_domain), GFP_KERNEL);
682 sun50i_domain->dt = (u32 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
684 if (!sun50i_domain->dt)
685 goto err_free_domain;
687 refcount_set(&sun50i_domain->refcnt, 1);
689 sun50i_domain->domain.geometry.aperture_start = 0;
690 sun50i_domain->domain.geometry.aperture_end = DMA_BIT_MASK(32);
691 sun50i_domain->domain.geometry.force_aperture = true;
693 return &sun50i_domain->domain;
696 kfree(sun50i_domain);
701 static void sun50i_iommu_domain_free(struct iommu_domain *domain)
703 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
705 free_pages((unsigned long)sun50i_domain->dt, get_order(DT_SIZE));
706 sun50i_domain->dt = NULL;
708 kfree(sun50i_domain);
711 static int sun50i_iommu_attach_domain(struct sun50i_iommu *iommu,
712 struct sun50i_iommu_domain *sun50i_domain)
714 iommu->domain = &sun50i_domain->domain;
715 sun50i_domain->iommu = iommu;
717 sun50i_domain->dt_dma = dma_map_single(iommu->dev, sun50i_domain->dt,
718 DT_SIZE, DMA_TO_DEVICE);
719 if (dma_mapping_error(iommu->dev, sun50i_domain->dt_dma)) {
720 dev_err(iommu->dev, "Couldn't map L1 Page Table\n");
724 return sun50i_iommu_enable(iommu);
727 static void sun50i_iommu_detach_domain(struct sun50i_iommu *iommu,
728 struct sun50i_iommu_domain *sun50i_domain)
732 for (i = 0; i < NUM_DT_ENTRIES; i++) {
738 dte_addr = &sun50i_domain->dt[i];
740 if (!sun50i_dte_is_pt_valid(dte))
743 memset(dte_addr, 0, sizeof(*dte_addr));
744 sun50i_table_flush(sun50i_domain, dte_addr, 1);
746 pt_phys = sun50i_dte_get_pt_address(dte);
747 page_table = phys_to_virt(pt_phys);
748 sun50i_iommu_free_page_table(iommu, page_table);
752 sun50i_iommu_disable(iommu);
754 dma_unmap_single(iommu->dev, virt_to_phys(sun50i_domain->dt),
755 DT_SIZE, DMA_TO_DEVICE);
757 iommu->domain = NULL;
760 static int sun50i_iommu_identity_attach(struct iommu_domain *identity_domain,
763 struct sun50i_iommu *iommu = dev_iommu_priv_get(dev);
764 struct sun50i_iommu_domain *sun50i_domain;
766 dev_dbg(dev, "Detaching from IOMMU domain\n");
768 if (iommu->domain == identity_domain)
771 sun50i_domain = to_sun50i_domain(iommu->domain);
772 if (refcount_dec_and_test(&sun50i_domain->refcnt))
773 sun50i_iommu_detach_domain(iommu, sun50i_domain);
777 static struct iommu_domain_ops sun50i_iommu_identity_ops = {
778 .attach_dev = sun50i_iommu_identity_attach,
781 static struct iommu_domain sun50i_iommu_identity_domain = {
782 .type = IOMMU_DOMAIN_IDENTITY,
783 .ops = &sun50i_iommu_identity_ops,
786 static int sun50i_iommu_attach_device(struct iommu_domain *domain,
789 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
790 struct sun50i_iommu *iommu;
792 iommu = sun50i_iommu_from_dev(dev);
796 dev_dbg(dev, "Attaching to IOMMU domain\n");
798 refcount_inc(&sun50i_domain->refcnt);
800 if (iommu->domain == domain)
803 sun50i_iommu_identity_attach(&sun50i_iommu_identity_domain, dev);
805 sun50i_iommu_attach_domain(iommu, sun50i_domain);
810 static struct iommu_device *sun50i_iommu_probe_device(struct device *dev)
812 struct sun50i_iommu *iommu;
814 iommu = sun50i_iommu_from_dev(dev);
816 return ERR_PTR(-ENODEV);
818 return &iommu->iommu;
821 static int sun50i_iommu_of_xlate(struct device *dev,
822 struct of_phandle_args *args)
824 struct platform_device *iommu_pdev = of_find_device_by_node(args->np);
825 unsigned id = args->args[0];
827 dev_iommu_priv_set(dev, platform_get_drvdata(iommu_pdev));
829 return iommu_fwspec_add_ids(dev, &id, 1);
832 static const struct iommu_ops sun50i_iommu_ops = {
833 .identity_domain = &sun50i_iommu_identity_domain,
834 .pgsize_bitmap = SZ_4K,
835 .device_group = generic_single_device_group,
836 .domain_alloc_paging = sun50i_iommu_domain_alloc_paging,
837 .of_xlate = sun50i_iommu_of_xlate,
838 .probe_device = sun50i_iommu_probe_device,
839 .default_domain_ops = &(const struct iommu_domain_ops) {
840 .attach_dev = sun50i_iommu_attach_device,
841 .flush_iotlb_all = sun50i_iommu_flush_iotlb_all,
842 .iotlb_sync_map = sun50i_iommu_iotlb_sync_map,
843 .iotlb_sync = sun50i_iommu_iotlb_sync,
844 .iova_to_phys = sun50i_iommu_iova_to_phys,
845 .map_pages = sun50i_iommu_map,
846 .unmap_pages = sun50i_iommu_unmap,
847 .free = sun50i_iommu_domain_free,
851 static void sun50i_iommu_report_fault(struct sun50i_iommu *iommu,
852 unsigned master, phys_addr_t iova,
855 dev_err(iommu->dev, "Page fault for %pad (master %d, dir %s)\n",
856 &iova, master, (prot == IOMMU_FAULT_WRITE) ? "wr" : "rd");
859 report_iommu_fault(iommu->domain, iommu->dev, iova, prot);
861 dev_err(iommu->dev, "Page fault while iommu not attached to any domain?\n");
863 sun50i_iommu_zap_range(iommu, iova, SPAGE_SIZE);
866 static phys_addr_t sun50i_iommu_handle_pt_irq(struct sun50i_iommu *iommu,
874 assert_spin_locked(&iommu->iommu_lock);
876 iova = iommu_read(iommu, addr_reg);
877 blame = iommu_read(iommu, blame_reg);
878 master = ilog2(blame & IOMMU_INT_MASTER_MASK);
881 * If the address is not in the page table, we can't get what
882 * operation triggered the fault. Assume it's a read
885 sun50i_iommu_report_fault(iommu, master, iova, IOMMU_FAULT_READ);
890 static phys_addr_t sun50i_iommu_handle_perm_irq(struct sun50i_iommu *iommu)
892 enum sun50i_iommu_aci aci;
898 assert_spin_locked(&iommu->iommu_lock);
900 blame = iommu_read(iommu, IOMMU_INT_STA_REG);
901 master = ilog2(blame & IOMMU_INT_MASTER_MASK);
902 iova = iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG(master));
903 aci = sun50i_get_pte_aci(iommu_read(iommu,
904 IOMMU_INT_ERR_DATA_REG(master)));
908 * If we are in the read-only domain, then it means we
911 case SUN50I_IOMMU_ACI_RD:
912 dir = IOMMU_FAULT_WRITE;
916 * If we are in the write-only domain, then it means
919 case SUN50I_IOMMU_ACI_WR:
922 * If we are in the domain without any permission, we
923 * can't really tell. Let's default to a read
926 case SUN50I_IOMMU_ACI_NONE:
929 case SUN50I_IOMMU_ACI_RD_WR:
931 dir = IOMMU_FAULT_READ;
936 * If the address is not in the page table, we can't get what
937 * operation triggered the fault. Assume it's a read
940 sun50i_iommu_report_fault(iommu, master, iova, dir);
945 static irqreturn_t sun50i_iommu_irq(int irq, void *dev_id)
947 u32 status, l1_status, l2_status, resets;
948 struct sun50i_iommu *iommu = dev_id;
950 spin_lock(&iommu->iommu_lock);
952 status = iommu_read(iommu, IOMMU_INT_STA_REG);
953 if (!(status & IOMMU_INT_MASK)) {
954 spin_unlock(&iommu->iommu_lock);
958 l1_status = iommu_read(iommu, IOMMU_L1PG_INT_REG);
959 l2_status = iommu_read(iommu, IOMMU_L2PG_INT_REG);
961 if (status & IOMMU_INT_INVALID_L2PG)
962 sun50i_iommu_handle_pt_irq(iommu,
963 IOMMU_INT_ERR_ADDR_L2_REG,
965 else if (status & IOMMU_INT_INVALID_L1PG)
966 sun50i_iommu_handle_pt_irq(iommu,
967 IOMMU_INT_ERR_ADDR_L1_REG,
970 sun50i_iommu_handle_perm_irq(iommu);
972 iommu_write(iommu, IOMMU_INT_CLR_REG, status);
974 resets = (status | l1_status | l2_status) & IOMMU_INT_MASTER_MASK;
975 iommu_write(iommu, IOMMU_RESET_REG, ~resets);
976 iommu_write(iommu, IOMMU_RESET_REG, IOMMU_RESET_RELEASE_ALL);
978 spin_unlock(&iommu->iommu_lock);
983 static int sun50i_iommu_probe(struct platform_device *pdev)
985 struct sun50i_iommu *iommu;
988 iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL);
991 spin_lock_init(&iommu->iommu_lock);
992 iommu->domain = &sun50i_iommu_identity_domain;
993 platform_set_drvdata(pdev, iommu);
994 iommu->dev = &pdev->dev;
996 iommu->pt_pool = kmem_cache_create(dev_name(&pdev->dev),
1000 if (!iommu->pt_pool)
1003 iommu->base = devm_platform_ioremap_resource(pdev, 0);
1004 if (IS_ERR(iommu->base)) {
1005 ret = PTR_ERR(iommu->base);
1006 goto err_free_cache;
1009 irq = platform_get_irq(pdev, 0);
1012 goto err_free_cache;
1015 iommu->clk = devm_clk_get(&pdev->dev, NULL);
1016 if (IS_ERR(iommu->clk)) {
1017 dev_err(&pdev->dev, "Couldn't get our clock.\n");
1018 ret = PTR_ERR(iommu->clk);
1019 goto err_free_cache;
1022 iommu->reset = devm_reset_control_get(&pdev->dev, NULL);
1023 if (IS_ERR(iommu->reset)) {
1024 dev_err(&pdev->dev, "Couldn't get our reset line.\n");
1025 ret = PTR_ERR(iommu->reset);
1026 goto err_free_cache;
1029 ret = iommu_device_sysfs_add(&iommu->iommu, &pdev->dev,
1030 NULL, dev_name(&pdev->dev));
1032 goto err_free_cache;
1034 ret = iommu_device_register(&iommu->iommu, &sun50i_iommu_ops, &pdev->dev);
1036 goto err_remove_sysfs;
1038 ret = devm_request_irq(&pdev->dev, irq, sun50i_iommu_irq, 0,
1039 dev_name(&pdev->dev), iommu);
1041 goto err_unregister;
1046 iommu_device_unregister(&iommu->iommu);
1049 iommu_device_sysfs_remove(&iommu->iommu);
1052 kmem_cache_destroy(iommu->pt_pool);
1057 static const struct of_device_id sun50i_iommu_dt[] = {
1058 { .compatible = "allwinner,sun50i-h6-iommu", },
1061 MODULE_DEVICE_TABLE(of, sun50i_iommu_dt);
1063 static struct platform_driver sun50i_iommu_driver = {
1065 .name = "sun50i-iommu",
1066 .of_match_table = sun50i_iommu_dt,
1067 .suppress_bind_attrs = true,
1070 builtin_platform_driver_probe(sun50i_iommu_driver, sun50i_iommu_probe);
1072 MODULE_DESCRIPTION("Allwinner H6 IOMMU driver");
1073 MODULE_AUTHOR("Maxime Ripard <maxime@cerno.tech>");
1074 MODULE_AUTHOR("zhuxianbin <zhuxianbin@allwinnertech.com>");