Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
fdb1d7be WD |
2 | #ifndef __IO_PGTABLE_H |
3 | #define __IO_PGTABLE_H | |
e5fc9753 | 4 | #include <linux/bitops.h> |
fdb1d7be WD |
5 | |
6 | /* | |
7 | * Public API for use by IOMMU drivers | |
8 | */ | |
9 | enum io_pgtable_fmt { | |
e1d3c0fd WD |
10 | ARM_32_LPAE_S1, |
11 | ARM_32_LPAE_S2, | |
12 | ARM_64_LPAE_S1, | |
13 | ARM_64_LPAE_S2, | |
e5fc9753 | 14 | ARM_V7S, |
fdb1d7be WD |
15 | IO_PGTABLE_NUM_FMTS, |
16 | }; | |
17 | ||
18 | /** | |
19 | * struct iommu_gather_ops - IOMMU callbacks for TLB and page table management. | |
20 | * | |
21 | * @tlb_flush_all: Synchronously invalidate the entire TLB context. | |
22 | * @tlb_add_flush: Queue up a TLB invalidation for a virtual address range. | |
87a91b15 RM |
23 | * @tlb_sync: Ensure any queued TLB invalidation has taken effect, and |
24 | * any corresponding page table updates are visible to the | |
25 | * IOMMU. | |
fdb1d7be WD |
26 | * |
27 | * Note that these can all be called in atomic context and must therefore | |
28 | * not block. | |
29 | */ | |
30 | struct iommu_gather_ops { | |
31 | void (*tlb_flush_all)(void *cookie); | |
06c610e8 RM |
32 | void (*tlb_add_flush)(unsigned long iova, size_t size, size_t granule, |
33 | bool leaf, void *cookie); | |
fdb1d7be | 34 | void (*tlb_sync)(void *cookie); |
fdb1d7be WD |
35 | }; |
36 | ||
37 | /** | |
38 | * struct io_pgtable_cfg - Configuration data for a set of page tables. | |
39 | * | |
40 | * @quirks: A bitmap of hardware quirks that require some special | |
41 | * action by the low-level page table allocator. | |
42 | * @pgsize_bitmap: A bitmap of page sizes supported by this set of page | |
43 | * tables. | |
44 | * @ias: Input address (iova) size, in bits. | |
45 | * @oas: Output address (paddr) size, in bits. | |
46 | * @tlb: TLB management callbacks for this set of tables. | |
f8d54961 RM |
47 | * @iommu_dev: The device representing the DMA configuration for the |
48 | * page table walker. | |
fdb1d7be WD |
49 | */ |
50 | struct io_pgtable_cfg { | |
3850db49 RM |
51 | /* |
52 | * IO_PGTABLE_QUIRK_ARM_NS: (ARM formats) Set NS and NSTABLE bits in | |
53 | * stage 1 PTEs, for hardware which insists on validating them | |
54 | * even in non-secure state where they should normally be ignored. | |
55 | * | |
56 | * IO_PGTABLE_QUIRK_NO_PERMS: Ignore the IOMMU_READ, IOMMU_WRITE and | |
57 | * IOMMU_NOEXEC flags and map everything with full access, for | |
58 | * hardware which does not implement the permissions of a given | |
59 | * format, and/or requires some format-specific default value. | |
60 | * | |
61 | * IO_PGTABLE_QUIRK_TLBI_ON_MAP: If the format forbids caching invalid | |
62 | * (unmapped) entries but the hardware might do so anyway, perform | |
63 | * TLB maintenance when mapping as well as when unmapping. | |
1afe2319 YW |
64 | * |
65 | * IO_PGTABLE_QUIRK_ARM_MTK_4GB: (ARM v7s format) Set bit 9 in all | |
66 | * PTEs, for Mediatek IOMMUs which treat it as a 33rd address bit | |
67 | * when the SoC is in "4GB mode" and they can only access the high | |
68 | * remap of DRAM (0x1_00000000 to 0x1_ffffffff). | |
81b3c252 RM |
69 | * |
70 | * IO_PGTABLE_QUIRK_NO_DMA: Guarantees that the tables will only ever | |
71 | * be accessed by a fully cache-coherent IOMMU or CPU (e.g. for a | |
72 | * software-emulated IOMMU), such that pagetable updates need not | |
73 | * be treated as explicit DMA data. | |
3850db49 RM |
74 | */ |
75 | #define IO_PGTABLE_QUIRK_ARM_NS BIT(0) | |
76 | #define IO_PGTABLE_QUIRK_NO_PERMS BIT(1) | |
77 | #define IO_PGTABLE_QUIRK_TLBI_ON_MAP BIT(2) | |
1afe2319 | 78 | #define IO_PGTABLE_QUIRK_ARM_MTK_4GB BIT(3) |
81b3c252 | 79 | #define IO_PGTABLE_QUIRK_NO_DMA BIT(4) |
3850db49 | 80 | unsigned long quirks; |
fdb1d7be WD |
81 | unsigned long pgsize_bitmap; |
82 | unsigned int ias; | |
83 | unsigned int oas; | |
84 | const struct iommu_gather_ops *tlb; | |
f8d54961 | 85 | struct device *iommu_dev; |
fdb1d7be WD |
86 | |
87 | /* Low-level data specific to the table format */ | |
88 | union { | |
e1d3c0fd WD |
89 | struct { |
90 | u64 ttbr[2]; | |
91 | u64 tcr; | |
92 | u64 mair[2]; | |
93 | } arm_lpae_s1_cfg; | |
94 | ||
95 | struct { | |
96 | u64 vttbr; | |
97 | u64 vtcr; | |
98 | } arm_lpae_s2_cfg; | |
e5fc9753 RM |
99 | |
100 | struct { | |
101 | u32 ttbr[2]; | |
102 | u32 tcr; | |
103 | u32 nmrr; | |
104 | u32 prrr; | |
105 | } arm_v7s_cfg; | |
fdb1d7be WD |
106 | }; |
107 | }; | |
108 | ||
109 | /** | |
110 | * struct io_pgtable_ops - Page table manipulation API for IOMMU drivers. | |
111 | * | |
112 | * @map: Map a physically contiguous memory region. | |
113 | * @unmap: Unmap a physically contiguous memory region. | |
114 | * @iova_to_phys: Translate iova to physical address. | |
115 | * | |
116 | * These functions map directly onto the iommu_ops member functions with | |
117 | * the same names. | |
118 | */ | |
119 | struct io_pgtable_ops { | |
120 | int (*map)(struct io_pgtable_ops *ops, unsigned long iova, | |
121 | phys_addr_t paddr, size_t size, int prot); | |
122 | int (*unmap)(struct io_pgtable_ops *ops, unsigned long iova, | |
123 | size_t size); | |
124 | phys_addr_t (*iova_to_phys)(struct io_pgtable_ops *ops, | |
125 | unsigned long iova); | |
126 | }; | |
127 | ||
128 | /** | |
129 | * alloc_io_pgtable_ops() - Allocate a page table allocator for use by an IOMMU. | |
130 | * | |
131 | * @fmt: The page table format. | |
132 | * @cfg: The page table configuration. This will be modified to represent | |
133 | * the configuration actually provided by the allocator (e.g. the | |
134 | * pgsize_bitmap may be restricted). | |
135 | * @cookie: An opaque token provided by the IOMMU driver and passed back to | |
136 | * the callback routines in cfg->tlb. | |
137 | */ | |
138 | struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt, | |
139 | struct io_pgtable_cfg *cfg, | |
140 | void *cookie); | |
141 | ||
142 | /** | |
143 | * free_io_pgtable_ops() - Free an io_pgtable_ops structure. The caller | |
144 | * *must* ensure that the page table is no longer | |
145 | * live, but the TLB can be dirty. | |
146 | * | |
147 | * @ops: The ops returned from alloc_io_pgtable_ops. | |
148 | */ | |
149 | void free_io_pgtable_ops(struct io_pgtable_ops *ops); | |
150 | ||
151 | ||
152 | /* | |
153 | * Internal structures for page table allocator implementations. | |
154 | */ | |
155 | ||
156 | /** | |
157 | * struct io_pgtable - Internal structure describing a set of page tables. | |
158 | * | |
159 | * @fmt: The page table format. | |
160 | * @cookie: An opaque token provided by the IOMMU driver and passed back to | |
161 | * any callback routines. | |
162 | * @cfg: A copy of the page table configuration. | |
163 | * @ops: The page table operations in use for this set of page tables. | |
164 | */ | |
165 | struct io_pgtable { | |
166 | enum io_pgtable_fmt fmt; | |
167 | void *cookie; | |
168 | struct io_pgtable_cfg cfg; | |
169 | struct io_pgtable_ops ops; | |
170 | }; | |
171 | ||
fdc38967 RM |
172 | #define io_pgtable_ops_to_pgtable(x) container_of((x), struct io_pgtable, ops) |
173 | ||
507e4c9d RM |
174 | static inline void io_pgtable_tlb_flush_all(struct io_pgtable *iop) |
175 | { | |
176 | iop->cfg.tlb->tlb_flush_all(iop->cookie); | |
177 | } | |
178 | ||
179 | static inline void io_pgtable_tlb_add_flush(struct io_pgtable *iop, | |
180 | unsigned long iova, size_t size, size_t granule, bool leaf) | |
181 | { | |
182 | iop->cfg.tlb->tlb_add_flush(iova, size, granule, leaf, iop->cookie); | |
183 | } | |
184 | ||
185 | static inline void io_pgtable_tlb_sync(struct io_pgtable *iop) | |
186 | { | |
2984f7f3 | 187 | iop->cfg.tlb->tlb_sync(iop->cookie); |
507e4c9d RM |
188 | } |
189 | ||
fdb1d7be WD |
190 | /** |
191 | * struct io_pgtable_init_fns - Alloc/free a set of page tables for a | |
192 | * particular format. | |
193 | * | |
194 | * @alloc: Allocate a set of page tables described by cfg. | |
195 | * @free: Free the page tables associated with iop. | |
196 | */ | |
197 | struct io_pgtable_init_fns { | |
198 | struct io_pgtable *(*alloc)(struct io_pgtable_cfg *cfg, void *cookie); | |
199 | void (*free)(struct io_pgtable *iop); | |
200 | }; | |
201 | ||
2e169bb3 JR |
202 | extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns; |
203 | extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns; | |
204 | extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns; | |
205 | extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns; | |
e5fc9753 | 206 | extern struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns; |
2e169bb3 | 207 | |
fdb1d7be | 208 | #endif /* __IO_PGTABLE_H */ |