Commit | Line | Data |
---|---|---|
57d3f11c | 1 | // SPDX-License-Identifier: GPL-2.0 |
d25a2a16 | 2 | /* |
8128ac3b PG |
3 | * IOMMU API for Renesas VMSA-compatible IPMMU |
4 | * Author: Laurent Pinchart <laurent.pinchart@ideasonboard.com> | |
d25a2a16 LP |
5 | * |
6 | * Copyright (C) 2014 Renesas Electronics Corporation | |
d25a2a16 LP |
7 | */ |
8 | ||
dbb70692 | 9 | #include <linux/bitmap.h> |
d25a2a16 | 10 | #include <linux/delay.h> |
3ae47292 | 11 | #include <linux/dma-iommu.h> |
d25a2a16 LP |
12 | #include <linux/dma-mapping.h> |
13 | #include <linux/err.h> | |
14 | #include <linux/export.h> | |
8128ac3b | 15 | #include <linux/init.h> |
d25a2a16 LP |
16 | #include <linux/interrupt.h> |
17 | #include <linux/io.h> | |
b77cf11f | 18 | #include <linux/io-pgtable.h> |
d25a2a16 | 19 | #include <linux/iommu.h> |
275f5053 | 20 | #include <linux/of.h> |
33f3ac9b | 21 | #include <linux/of_device.h> |
cda52fcd | 22 | #include <linux/of_iommu.h> |
7b2d5961 | 23 | #include <linux/of_platform.h> |
d25a2a16 LP |
24 | #include <linux/platform_device.h> |
25 | #include <linux/sizes.h> | |
26 | #include <linux/slab.h> | |
58b8e8bf | 27 | #include <linux/sys_soc.h> |
d25a2a16 | 28 | |
3ae47292 | 29 | #if defined(CONFIG_ARM) && !defined(CONFIG_IOMMU_DMA) |
d25a2a16 LP |
30 | #include <asm/dma-iommu.h> |
31 | #include <asm/pgalloc.h> | |
49c875f0 RM |
32 | #else |
33 | #define arm_iommu_create_mapping(...) NULL | |
34 | #define arm_iommu_attach_device(...) -ENODEV | |
35 | #define arm_iommu_release_mapping(...) do {} while (0) | |
36 | #define arm_iommu_detach_device(...) do {} while (0) | |
3ae47292 | 37 | #endif |
d25a2a16 | 38 | |
da38e9ec GU |
39 | #define IPMMU_CTX_MAX 8U |
40 | #define IPMMU_CTX_INVALID -1 | |
41 | ||
42 | #define IPMMU_UTLB_MAX 48U | |
dbb70692 | 43 | |
33f3ac9b MD |
44 | struct ipmmu_features { |
45 | bool use_ns_alias_offset; | |
fd5140e2 | 46 | bool has_cache_leaf_nodes; |
5fd16341 | 47 | unsigned int number_of_contexts; |
b7f3f047 | 48 | unsigned int num_utlbs; |
f5c85891 | 49 | bool setup_imbuscr; |
c295f504 | 50 | bool twobit_imttbcr_sl0; |
2ae86955 | 51 | bool reserved_context; |
3623002f | 52 | bool cache_snoop; |
33f3ac9b MD |
53 | }; |
54 | ||
d25a2a16 LP |
55 | struct ipmmu_vmsa_device { |
56 | struct device *dev; | |
57 | void __iomem *base; | |
01da21e5 | 58 | struct iommu_device iommu; |
fd5140e2 | 59 | struct ipmmu_vmsa_device *root; |
33f3ac9b | 60 | const struct ipmmu_features *features; |
5fd16341 | 61 | unsigned int num_ctx; |
dbb70692 MD |
62 | spinlock_t lock; /* Protects ctx and domains[] */ |
63 | DECLARE_BITMAP(ctx, IPMMU_CTX_MAX); | |
64 | struct ipmmu_vmsa_domain *domains[IPMMU_CTX_MAX]; | |
da38e9ec | 65 | s8 utlb_ctx[IPMMU_UTLB_MAX]; |
d25a2a16 | 66 | |
b354c73e | 67 | struct iommu_group *group; |
d25a2a16 LP |
68 | struct dma_iommu_mapping *mapping; |
69 | }; | |
70 | ||
71 | struct ipmmu_vmsa_domain { | |
72 | struct ipmmu_vmsa_device *mmu; | |
5914c5fd | 73 | struct iommu_domain io_domain; |
d25a2a16 | 74 | |
f20ed39f LP |
75 | struct io_pgtable_cfg cfg; |
76 | struct io_pgtable_ops *iop; | |
77 | ||
d25a2a16 | 78 | unsigned int context_id; |
46583e8c | 79 | struct mutex mutex; /* Protects mappings */ |
d25a2a16 LP |
80 | }; |
81 | ||
5914c5fd JR |
82 | static struct ipmmu_vmsa_domain *to_vmsa_domain(struct iommu_domain *dom) |
83 | { | |
84 | return container_of(dom, struct ipmmu_vmsa_domain, io_domain); | |
85 | } | |
86 | ||
e4efe4a9 | 87 | static struct ipmmu_vmsa_device *to_ipmmu(struct device *dev) |
0fbc8b04 | 88 | { |
df903655 JR |
89 | struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); |
90 | ||
91 | return fwspec ? fwspec->iommu_priv : NULL; | |
0fbc8b04 MD |
92 | } |
93 | ||
d25a2a16 LP |
94 | #define TLB_LOOP_TIMEOUT 100 /* 100us */ |
95 | ||
96 | /* ----------------------------------------------------------------------------- | |
97 | * Registers Definition | |
98 | */ | |
99 | ||
275f5053 LP |
100 | #define IM_NS_ALIAS_OFFSET 0x800 |
101 | ||
d25a2a16 LP |
102 | #define IM_CTX_SIZE 0x40 |
103 | ||
104 | #define IMCTR 0x0000 | |
105 | #define IMCTR_TRE (1 << 17) | |
106 | #define IMCTR_AFE (1 << 16) | |
107 | #define IMCTR_RTSEL_MASK (3 << 4) | |
108 | #define IMCTR_RTSEL_SHIFT 4 | |
109 | #define IMCTR_TREN (1 << 3) | |
110 | #define IMCTR_INTEN (1 << 2) | |
111 | #define IMCTR_FLUSH (1 << 1) | |
112 | #define IMCTR_MMUEN (1 << 0) | |
113 | ||
114 | #define IMCAAR 0x0004 | |
115 | ||
116 | #define IMTTBCR 0x0008 | |
117 | #define IMTTBCR_EAE (1 << 31) | |
118 | #define IMTTBCR_PMB (1 << 30) | |
3623002f HNP |
119 | #define IMTTBCR_SH1_NON_SHAREABLE (0 << 28) /* R-Car Gen2 only */ |
120 | #define IMTTBCR_SH1_OUTER_SHAREABLE (2 << 28) /* R-Car Gen2 only */ | |
121 | #define IMTTBCR_SH1_INNER_SHAREABLE (3 << 28) /* R-Car Gen2 only */ | |
122 | #define IMTTBCR_SH1_MASK (3 << 28) /* R-Car Gen2 only */ | |
123 | #define IMTTBCR_ORGN1_NC (0 << 26) /* R-Car Gen2 only */ | |
124 | #define IMTTBCR_ORGN1_WB_WA (1 << 26) /* R-Car Gen2 only */ | |
125 | #define IMTTBCR_ORGN1_WT (2 << 26) /* R-Car Gen2 only */ | |
126 | #define IMTTBCR_ORGN1_WB (3 << 26) /* R-Car Gen2 only */ | |
127 | #define IMTTBCR_ORGN1_MASK (3 << 26) /* R-Car Gen2 only */ | |
128 | #define IMTTBCR_IRGN1_NC (0 << 24) /* R-Car Gen2 only */ | |
129 | #define IMTTBCR_IRGN1_WB_WA (1 << 24) /* R-Car Gen2 only */ | |
130 | #define IMTTBCR_IRGN1_WT (2 << 24) /* R-Car Gen2 only */ | |
131 | #define IMTTBCR_IRGN1_WB (3 << 24) /* R-Car Gen2 only */ | |
132 | #define IMTTBCR_IRGN1_MASK (3 << 24) /* R-Car Gen2 only */ | |
d25a2a16 LP |
133 | #define IMTTBCR_TSZ1_MASK (7 << 16) |
134 | #define IMTTBCR_TSZ1_SHIFT 16 | |
3623002f HNP |
135 | #define IMTTBCR_SH0_NON_SHAREABLE (0 << 12) /* R-Car Gen2 only */ |
136 | #define IMTTBCR_SH0_OUTER_SHAREABLE (2 << 12) /* R-Car Gen2 only */ | |
137 | #define IMTTBCR_SH0_INNER_SHAREABLE (3 << 12) /* R-Car Gen2 only */ | |
138 | #define IMTTBCR_SH0_MASK (3 << 12) /* R-Car Gen2 only */ | |
139 | #define IMTTBCR_ORGN0_NC (0 << 10) /* R-Car Gen2 only */ | |
140 | #define IMTTBCR_ORGN0_WB_WA (1 << 10) /* R-Car Gen2 only */ | |
141 | #define IMTTBCR_ORGN0_WT (2 << 10) /* R-Car Gen2 only */ | |
142 | #define IMTTBCR_ORGN0_WB (3 << 10) /* R-Car Gen2 only */ | |
143 | #define IMTTBCR_ORGN0_MASK (3 << 10) /* R-Car Gen2 only */ | |
144 | #define IMTTBCR_IRGN0_NC (0 << 8) /* R-Car Gen2 only */ | |
145 | #define IMTTBCR_IRGN0_WB_WA (1 << 8) /* R-Car Gen2 only */ | |
146 | #define IMTTBCR_IRGN0_WT (2 << 8) /* R-Car Gen2 only */ | |
147 | #define IMTTBCR_IRGN0_WB (3 << 8) /* R-Car Gen2 only */ | |
148 | #define IMTTBCR_IRGN0_MASK (3 << 8) /* R-Car Gen2 only */ | |
5ca54fdc GU |
149 | #define IMTTBCR_SL0_TWOBIT_LVL_3 (0 << 6) /* R-Car Gen3 only */ |
150 | #define IMTTBCR_SL0_TWOBIT_LVL_2 (1 << 6) /* R-Car Gen3 only */ | |
151 | #define IMTTBCR_SL0_TWOBIT_LVL_1 (2 << 6) /* R-Car Gen3 only */ | |
d25a2a16 LP |
152 | #define IMTTBCR_SL0_LVL_2 (0 << 4) |
153 | #define IMTTBCR_SL0_LVL_1 (1 << 4) | |
154 | #define IMTTBCR_TSZ0_MASK (7 << 0) | |
155 | #define IMTTBCR_TSZ0_SHIFT O | |
156 | ||
157 | #define IMBUSCR 0x000c | |
158 | #define IMBUSCR_DVM (1 << 2) | |
159 | #define IMBUSCR_BUSSEL_SYS (0 << 0) | |
160 | #define IMBUSCR_BUSSEL_CCI (1 << 0) | |
161 | #define IMBUSCR_BUSSEL_IMCAAR (2 << 0) | |
162 | #define IMBUSCR_BUSSEL_CCI_IMCAAR (3 << 0) | |
163 | #define IMBUSCR_BUSSEL_MASK (3 << 0) | |
164 | ||
165 | #define IMTTLBR0 0x0010 | |
166 | #define IMTTUBR0 0x0014 | |
167 | #define IMTTLBR1 0x0018 | |
168 | #define IMTTUBR1 0x001c | |
169 | ||
170 | #define IMSTR 0x0020 | |
171 | #define IMSTR_ERRLVL_MASK (3 << 12) | |
172 | #define IMSTR_ERRLVL_SHIFT 12 | |
173 | #define IMSTR_ERRCODE_TLB_FORMAT (1 << 8) | |
174 | #define IMSTR_ERRCODE_ACCESS_PERM (4 << 8) | |
175 | #define IMSTR_ERRCODE_SECURE_ACCESS (5 << 8) | |
176 | #define IMSTR_ERRCODE_MASK (7 << 8) | |
177 | #define IMSTR_MHIT (1 << 4) | |
178 | #define IMSTR_ABORT (1 << 2) | |
179 | #define IMSTR_PF (1 << 1) | |
180 | #define IMSTR_TF (1 << 0) | |
181 | ||
182 | #define IMMAIR0 0x0028 | |
183 | #define IMMAIR1 0x002c | |
184 | #define IMMAIR_ATTR_MASK 0xff | |
185 | #define IMMAIR_ATTR_DEVICE 0x04 | |
186 | #define IMMAIR_ATTR_NC 0x44 | |
187 | #define IMMAIR_ATTR_WBRWA 0xff | |
188 | #define IMMAIR_ATTR_SHIFT(n) ((n) << 3) | |
189 | #define IMMAIR_ATTR_IDX_NC 0 | |
190 | #define IMMAIR_ATTR_IDX_WBRWA 1 | |
191 | #define IMMAIR_ATTR_IDX_DEV 2 | |
192 | ||
82576aa8 GU |
193 | #define IMELAR 0x0030 /* IMEAR on R-Car Gen2 */ |
194 | #define IMEUAR 0x0034 /* R-Car Gen3 only */ | |
d25a2a16 LP |
195 | |
196 | #define IMPCTR 0x0200 | |
197 | #define IMPSTR 0x0208 | |
198 | #define IMPEAR 0x020c | |
199 | #define IMPMBA(n) (0x0280 + ((n) * 4)) | |
200 | #define IMPMBD(n) (0x02c0 + ((n) * 4)) | |
201 | ||
ddbbddd7 MD |
202 | #define IMUCTR(n) ((n) < 32 ? IMUCTR0(n) : IMUCTR32(n)) |
203 | #define IMUCTR0(n) (0x0300 + ((n) * 16)) | |
204 | #define IMUCTR32(n) (0x0600 + (((n) - 32) * 16)) | |
d25a2a16 LP |
205 | #define IMUCTR_FIXADDEN (1 << 31) |
206 | #define IMUCTR_FIXADD_MASK (0xff << 16) | |
207 | #define IMUCTR_FIXADD_SHIFT 16 | |
208 | #define IMUCTR_TTSEL_MMU(n) ((n) << 4) | |
209 | #define IMUCTR_TTSEL_PMB (8 << 4) | |
210 | #define IMUCTR_TTSEL_MASK (15 << 4) | |
211 | #define IMUCTR_FLUSH (1 << 1) | |
212 | #define IMUCTR_MMUEN (1 << 0) | |
213 | ||
ddbbddd7 MD |
214 | #define IMUASID(n) ((n) < 32 ? IMUASID0(n) : IMUASID32(n)) |
215 | #define IMUASID0(n) (0x0308 + ((n) * 16)) | |
216 | #define IMUASID32(n) (0x0608 + (((n) - 32) * 16)) | |
d25a2a16 LP |
217 | #define IMUASID_ASID8_MASK (0xff << 8) |
218 | #define IMUASID_ASID8_SHIFT 8 | |
219 | #define IMUASID_ASID0_MASK (0xff << 0) | |
220 | #define IMUASID_ASID0_SHIFT 0 | |
221 | ||
fd5140e2 MD |
222 | /* ----------------------------------------------------------------------------- |
223 | * Root device handling | |
224 | */ | |
225 | ||
226 | static struct platform_driver ipmmu_driver; | |
227 | ||
228 | static bool ipmmu_is_root(struct ipmmu_vmsa_device *mmu) | |
229 | { | |
230 | return mmu->root == mmu; | |
231 | } | |
232 | ||
233 | static int __ipmmu_check_device(struct device *dev, void *data) | |
234 | { | |
235 | struct ipmmu_vmsa_device *mmu = dev_get_drvdata(dev); | |
236 | struct ipmmu_vmsa_device **rootp = data; | |
237 | ||
238 | if (ipmmu_is_root(mmu)) | |
239 | *rootp = mmu; | |
240 | ||
241 | return 0; | |
242 | } | |
243 | ||
244 | static struct ipmmu_vmsa_device *ipmmu_find_root(void) | |
245 | { | |
246 | struct ipmmu_vmsa_device *root = NULL; | |
247 | ||
248 | return driver_for_each_device(&ipmmu_driver.driver, NULL, &root, | |
249 | __ipmmu_check_device) == 0 ? root : NULL; | |
250 | } | |
251 | ||
d25a2a16 LP |
252 | /* ----------------------------------------------------------------------------- |
253 | * Read/Write Access | |
254 | */ | |
255 | ||
256 | static u32 ipmmu_read(struct ipmmu_vmsa_device *mmu, unsigned int offset) | |
257 | { | |
258 | return ioread32(mmu->base + offset); | |
259 | } | |
260 | ||
261 | static void ipmmu_write(struct ipmmu_vmsa_device *mmu, unsigned int offset, | |
262 | u32 data) | |
263 | { | |
264 | iowrite32(data, mmu->base + offset); | |
265 | } | |
266 | ||
d574893a MD |
267 | static u32 ipmmu_ctx_read_root(struct ipmmu_vmsa_domain *domain, |
268 | unsigned int reg) | |
d25a2a16 | 269 | { |
fd5140e2 MD |
270 | return ipmmu_read(domain->mmu->root, |
271 | domain->context_id * IM_CTX_SIZE + reg); | |
d25a2a16 LP |
272 | } |
273 | ||
d574893a MD |
274 | static void ipmmu_ctx_write_root(struct ipmmu_vmsa_domain *domain, |
275 | unsigned int reg, u32 data) | |
d25a2a16 | 276 | { |
fd5140e2 MD |
277 | ipmmu_write(domain->mmu->root, |
278 | domain->context_id * IM_CTX_SIZE + reg, data); | |
d25a2a16 LP |
279 | } |
280 | ||
d574893a MD |
281 | static void ipmmu_ctx_write_all(struct ipmmu_vmsa_domain *domain, |
282 | unsigned int reg, u32 data) | |
283 | { | |
284 | if (domain->mmu != domain->mmu->root) | |
285 | ipmmu_write(domain->mmu, | |
286 | domain->context_id * IM_CTX_SIZE + reg, data); | |
287 | ||
288 | ipmmu_write(domain->mmu->root, | |
289 | domain->context_id * IM_CTX_SIZE + reg, data); | |
290 | } | |
291 | ||
d25a2a16 LP |
292 | /* ----------------------------------------------------------------------------- |
293 | * TLB and microTLB Management | |
294 | */ | |
295 | ||
296 | /* Wait for any pending TLB invalidations to complete */ | |
297 | static void ipmmu_tlb_sync(struct ipmmu_vmsa_domain *domain) | |
298 | { | |
299 | unsigned int count = 0; | |
300 | ||
d574893a | 301 | while (ipmmu_ctx_read_root(domain, IMCTR) & IMCTR_FLUSH) { |
d25a2a16 LP |
302 | cpu_relax(); |
303 | if (++count == TLB_LOOP_TIMEOUT) { | |
304 | dev_err_ratelimited(domain->mmu->dev, | |
305 | "TLB sync timed out -- MMU may be deadlocked\n"); | |
306 | return; | |
307 | } | |
308 | udelay(1); | |
309 | } | |
310 | } | |
311 | ||
312 | static void ipmmu_tlb_invalidate(struct ipmmu_vmsa_domain *domain) | |
313 | { | |
314 | u32 reg; | |
315 | ||
d574893a | 316 | reg = ipmmu_ctx_read_root(domain, IMCTR); |
d25a2a16 | 317 | reg |= IMCTR_FLUSH; |
d574893a | 318 | ipmmu_ctx_write_all(domain, IMCTR, reg); |
d25a2a16 LP |
319 | |
320 | ipmmu_tlb_sync(domain); | |
321 | } | |
322 | ||
323 | /* | |
324 | * Enable MMU translation for the microTLB. | |
325 | */ | |
326 | static void ipmmu_utlb_enable(struct ipmmu_vmsa_domain *domain, | |
192d2045 | 327 | unsigned int utlb) |
d25a2a16 LP |
328 | { |
329 | struct ipmmu_vmsa_device *mmu = domain->mmu; | |
330 | ||
192d2045 LP |
331 | /* |
332 | * TODO: Reference-count the microTLB as several bus masters can be | |
333 | * connected to the same microTLB. | |
334 | */ | |
335 | ||
d25a2a16 | 336 | /* TODO: What should we set the ASID to ? */ |
192d2045 | 337 | ipmmu_write(mmu, IMUASID(utlb), 0); |
d25a2a16 | 338 | /* TODO: Do we need to flush the microTLB ? */ |
192d2045 | 339 | ipmmu_write(mmu, IMUCTR(utlb), |
d25a2a16 LP |
340 | IMUCTR_TTSEL_MMU(domain->context_id) | IMUCTR_FLUSH | |
341 | IMUCTR_MMUEN); | |
da38e9ec | 342 | mmu->utlb_ctx[utlb] = domain->context_id; |
d25a2a16 LP |
343 | } |
344 | ||
345 | /* | |
346 | * Disable MMU translation for the microTLB. | |
347 | */ | |
348 | static void ipmmu_utlb_disable(struct ipmmu_vmsa_domain *domain, | |
192d2045 | 349 | unsigned int utlb) |
d25a2a16 LP |
350 | { |
351 | struct ipmmu_vmsa_device *mmu = domain->mmu; | |
352 | ||
192d2045 | 353 | ipmmu_write(mmu, IMUCTR(utlb), 0); |
da38e9ec | 354 | mmu->utlb_ctx[utlb] = IPMMU_CTX_INVALID; |
d25a2a16 LP |
355 | } |
356 | ||
f20ed39f | 357 | static void ipmmu_tlb_flush_all(void *cookie) |
d25a2a16 | 358 | { |
f20ed39f LP |
359 | struct ipmmu_vmsa_domain *domain = cookie; |
360 | ||
361 | ipmmu_tlb_invalidate(domain); | |
362 | } | |
363 | ||
05aed941 WD |
364 | static void ipmmu_tlb_flush(unsigned long iova, size_t size, |
365 | size_t granule, void *cookie) | |
f20ed39f | 366 | { |
05aed941 | 367 | ipmmu_tlb_flush_all(cookie); |
f20ed39f LP |
368 | } |
369 | ||
298f7889 | 370 | static const struct iommu_flush_ops ipmmu_flush_ops = { |
f20ed39f | 371 | .tlb_flush_all = ipmmu_tlb_flush_all, |
05aed941 WD |
372 | .tlb_flush_walk = ipmmu_tlb_flush, |
373 | .tlb_flush_leaf = ipmmu_tlb_flush, | |
f20ed39f LP |
374 | }; |
375 | ||
d25a2a16 LP |
376 | /* ----------------------------------------------------------------------------- |
377 | * Domain/Context Management | |
378 | */ | |
379 | ||
dbb70692 MD |
380 | static int ipmmu_domain_allocate_context(struct ipmmu_vmsa_device *mmu, |
381 | struct ipmmu_vmsa_domain *domain) | |
382 | { | |
383 | unsigned long flags; | |
384 | int ret; | |
385 | ||
386 | spin_lock_irqsave(&mmu->lock, flags); | |
387 | ||
5fd16341 MD |
388 | ret = find_first_zero_bit(mmu->ctx, mmu->num_ctx); |
389 | if (ret != mmu->num_ctx) { | |
dbb70692 MD |
390 | mmu->domains[ret] = domain; |
391 | set_bit(ret, mmu->ctx); | |
5fd16341 MD |
392 | } else |
393 | ret = -EBUSY; | |
dbb70692 MD |
394 | |
395 | spin_unlock_irqrestore(&mmu->lock, flags); | |
396 | ||
397 | return ret; | |
398 | } | |
399 | ||
a175a67d OT |
400 | static void ipmmu_domain_free_context(struct ipmmu_vmsa_device *mmu, |
401 | unsigned int context_id) | |
402 | { | |
403 | unsigned long flags; | |
404 | ||
405 | spin_lock_irqsave(&mmu->lock, flags); | |
406 | ||
407 | clear_bit(context_id, mmu->ctx); | |
408 | mmu->domains[context_id] = NULL; | |
409 | ||
410 | spin_unlock_irqrestore(&mmu->lock, flags); | |
411 | } | |
412 | ||
892db541 | 413 | static void ipmmu_domain_setup_context(struct ipmmu_vmsa_domain *domain) |
d25a2a16 | 414 | { |
f64232ee | 415 | u64 ttbr; |
c295f504 | 416 | u32 tmp; |
a175a67d | 417 | |
d25a2a16 | 418 | /* TTBR0 */ |
f20ed39f | 419 | ttbr = domain->cfg.arm_lpae_s1_cfg.ttbr[0]; |
d574893a MD |
420 | ipmmu_ctx_write_root(domain, IMTTLBR0, ttbr); |
421 | ipmmu_ctx_write_root(domain, IMTTUBR0, ttbr >> 32); | |
d25a2a16 LP |
422 | |
423 | /* | |
424 | * TTBCR | |
3623002f HNP |
425 | * We use long descriptors and allocate the whole 32-bit VA space to |
426 | * TTBR0. | |
d25a2a16 | 427 | */ |
c295f504 MD |
428 | if (domain->mmu->features->twobit_imttbcr_sl0) |
429 | tmp = IMTTBCR_SL0_TWOBIT_LVL_1; | |
430 | else | |
431 | tmp = IMTTBCR_SL0_LVL_1; | |
432 | ||
3623002f HNP |
433 | if (domain->mmu->features->cache_snoop) |
434 | tmp |= IMTTBCR_SH0_INNER_SHAREABLE | IMTTBCR_ORGN0_WB_WA | | |
435 | IMTTBCR_IRGN0_WB_WA; | |
436 | ||
437 | ipmmu_ctx_write_root(domain, IMTTBCR, IMTTBCR_EAE | tmp); | |
d25a2a16 | 438 | |
f20ed39f | 439 | /* MAIR0 */ |
d574893a MD |
440 | ipmmu_ctx_write_root(domain, IMMAIR0, |
441 | domain->cfg.arm_lpae_s1_cfg.mair[0]); | |
d25a2a16 LP |
442 | |
443 | /* IMBUSCR */ | |
f5c85891 MD |
444 | if (domain->mmu->features->setup_imbuscr) |
445 | ipmmu_ctx_write_root(domain, IMBUSCR, | |
446 | ipmmu_ctx_read_root(domain, IMBUSCR) & | |
447 | ~(IMBUSCR_DVM | IMBUSCR_BUSSEL_MASK)); | |
d25a2a16 LP |
448 | |
449 | /* | |
450 | * IMSTR | |
451 | * Clear all interrupt flags. | |
452 | */ | |
d574893a | 453 | ipmmu_ctx_write_root(domain, IMSTR, ipmmu_ctx_read_root(domain, IMSTR)); |
d25a2a16 LP |
454 | |
455 | /* | |
456 | * IMCTR | |
457 | * Enable the MMU and interrupt generation. The long-descriptor | |
458 | * translation table format doesn't use TEX remapping. Don't enable AF | |
459 | * software management as we have no use for it. Flush the TLB as | |
460 | * required when modifying the context registers. | |
461 | */ | |
d574893a MD |
462 | ipmmu_ctx_write_all(domain, IMCTR, |
463 | IMCTR_INTEN | IMCTR_FLUSH | IMCTR_MMUEN); | |
892db541 GU |
464 | } |
465 | ||
466 | static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain) | |
467 | { | |
468 | int ret; | |
469 | ||
470 | /* | |
471 | * Allocate the page table operations. | |
472 | * | |
473 | * VMSA states in section B3.6.3 "Control of Secure or Non-secure memory | |
474 | * access, Long-descriptor format" that the NStable bit being set in a | |
475 | * table descriptor will result in the NStable and NS bits of all child | |
476 | * entries being ignored and considered as being set. The IPMMU seems | |
477 | * not to comply with this, as it generates a secure access page fault | |
478 | * if any of the NStable and NS bits isn't set when running in | |
479 | * non-secure mode. | |
480 | */ | |
481 | domain->cfg.quirks = IO_PGTABLE_QUIRK_ARM_NS; | |
482 | domain->cfg.pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K; | |
483 | domain->cfg.ias = 32; | |
484 | domain->cfg.oas = 40; | |
298f7889 | 485 | domain->cfg.tlb = &ipmmu_flush_ops; |
892db541 GU |
486 | domain->io_domain.geometry.aperture_end = DMA_BIT_MASK(32); |
487 | domain->io_domain.geometry.force_aperture = true; | |
488 | /* | |
489 | * TODO: Add support for coherent walk through CCI with DVM and remove | |
490 | * cache handling. For now, delegate it to the io-pgtable code. | |
491 | */ | |
3430abd6 | 492 | domain->cfg.coherent_walk = false; |
892db541 GU |
493 | domain->cfg.iommu_dev = domain->mmu->root->dev; |
494 | ||
495 | /* | |
496 | * Find an unused context. | |
497 | */ | |
498 | ret = ipmmu_domain_allocate_context(domain->mmu->root, domain); | |
499 | if (ret < 0) | |
500 | return ret; | |
501 | ||
502 | domain->context_id = ret; | |
503 | ||
504 | domain->iop = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &domain->cfg, | |
505 | domain); | |
506 | if (!domain->iop) { | |
507 | ipmmu_domain_free_context(domain->mmu->root, | |
508 | domain->context_id); | |
509 | return -EINVAL; | |
510 | } | |
d25a2a16 | 511 | |
892db541 | 512 | ipmmu_domain_setup_context(domain); |
d25a2a16 LP |
513 | return 0; |
514 | } | |
515 | ||
516 | static void ipmmu_domain_destroy_context(struct ipmmu_vmsa_domain *domain) | |
517 | { | |
e5b78f2e GU |
518 | if (!domain->mmu) |
519 | return; | |
520 | ||
d25a2a16 LP |
521 | /* |
522 | * Disable the context. Flush the TLB as required when modifying the | |
523 | * context registers. | |
524 | * | |
525 | * TODO: Is TLB flush really needed ? | |
526 | */ | |
d574893a | 527 | ipmmu_ctx_write_all(domain, IMCTR, IMCTR_FLUSH); |
d25a2a16 | 528 | ipmmu_tlb_sync(domain); |
fd5140e2 | 529 | ipmmu_domain_free_context(domain->mmu->root, domain->context_id); |
d25a2a16 LP |
530 | } |
531 | ||
532 | /* ----------------------------------------------------------------------------- | |
533 | * Fault Handling | |
534 | */ | |
535 | ||
536 | static irqreturn_t ipmmu_domain_irq(struct ipmmu_vmsa_domain *domain) | |
537 | { | |
538 | const u32 err_mask = IMSTR_MHIT | IMSTR_ABORT | IMSTR_PF | IMSTR_TF; | |
539 | struct ipmmu_vmsa_device *mmu = domain->mmu; | |
82576aa8 | 540 | unsigned long iova; |
d25a2a16 | 541 | u32 status; |
d25a2a16 | 542 | |
d574893a | 543 | status = ipmmu_ctx_read_root(domain, IMSTR); |
d25a2a16 LP |
544 | if (!(status & err_mask)) |
545 | return IRQ_NONE; | |
546 | ||
82576aa8 GU |
547 | iova = ipmmu_ctx_read_root(domain, IMELAR); |
548 | if (IS_ENABLED(CONFIG_64BIT)) | |
549 | iova |= (u64)ipmmu_ctx_read_root(domain, IMEUAR) << 32; | |
d25a2a16 LP |
550 | |
551 | /* | |
552 | * Clear the error status flags. Unlike traditional interrupt flag | |
553 | * registers that must be cleared by writing 1, this status register | |
554 | * seems to require 0. The error address register must be read before, | |
555 | * otherwise its value will be 0. | |
556 | */ | |
d574893a | 557 | ipmmu_ctx_write_root(domain, IMSTR, 0); |
d25a2a16 LP |
558 | |
559 | /* Log fatal errors. */ | |
560 | if (status & IMSTR_MHIT) | |
82576aa8 | 561 | dev_err_ratelimited(mmu->dev, "Multiple TLB hits @0x%lx\n", |
d25a2a16 LP |
562 | iova); |
563 | if (status & IMSTR_ABORT) | |
82576aa8 | 564 | dev_err_ratelimited(mmu->dev, "Page Table Walk Abort @0x%lx\n", |
d25a2a16 LP |
565 | iova); |
566 | ||
567 | if (!(status & (IMSTR_PF | IMSTR_TF))) | |
568 | return IRQ_NONE; | |
569 | ||
570 | /* | |
571 | * Try to handle page faults and translation faults. | |
572 | * | |
573 | * TODO: We need to look up the faulty device based on the I/O VA. Use | |
574 | * the IOMMU device for now. | |
575 | */ | |
5914c5fd | 576 | if (!report_iommu_fault(&domain->io_domain, mmu->dev, iova, 0)) |
d25a2a16 LP |
577 | return IRQ_HANDLED; |
578 | ||
579 | dev_err_ratelimited(mmu->dev, | |
82576aa8 | 580 | "Unhandled fault: status 0x%08x iova 0x%lx\n", |
d25a2a16 LP |
581 | status, iova); |
582 | ||
583 | return IRQ_HANDLED; | |
584 | } | |
585 | ||
586 | static irqreturn_t ipmmu_irq(int irq, void *dev) | |
587 | { | |
588 | struct ipmmu_vmsa_device *mmu = dev; | |
dbb70692 MD |
589 | irqreturn_t status = IRQ_NONE; |
590 | unsigned int i; | |
591 | unsigned long flags; | |
d25a2a16 | 592 | |
dbb70692 MD |
593 | spin_lock_irqsave(&mmu->lock, flags); |
594 | ||
595 | /* | |
596 | * Check interrupts for all active contexts. | |
597 | */ | |
5fd16341 | 598 | for (i = 0; i < mmu->num_ctx; i++) { |
dbb70692 MD |
599 | if (!mmu->domains[i]) |
600 | continue; | |
601 | if (ipmmu_domain_irq(mmu->domains[i]) == IRQ_HANDLED) | |
602 | status = IRQ_HANDLED; | |
603 | } | |
d25a2a16 | 604 | |
dbb70692 | 605 | spin_unlock_irqrestore(&mmu->lock, flags); |
d25a2a16 | 606 | |
dbb70692 | 607 | return status; |
d25a2a16 LP |
608 | } |
609 | ||
d25a2a16 LP |
610 | /* ----------------------------------------------------------------------------- |
611 | * IOMMU Operations | |
612 | */ | |
613 | ||
8e73bf65 | 614 | static struct iommu_domain *__ipmmu_domain_alloc(unsigned type) |
d25a2a16 LP |
615 | { |
616 | struct ipmmu_vmsa_domain *domain; | |
617 | ||
618 | domain = kzalloc(sizeof(*domain), GFP_KERNEL); | |
619 | if (!domain) | |
5914c5fd | 620 | return NULL; |
d25a2a16 | 621 | |
46583e8c | 622 | mutex_init(&domain->mutex); |
d25a2a16 | 623 | |
5914c5fd | 624 | return &domain->io_domain; |
d25a2a16 LP |
625 | } |
626 | ||
1c7e7c02 RM |
627 | static struct iommu_domain *ipmmu_domain_alloc(unsigned type) |
628 | { | |
629 | struct iommu_domain *io_domain = NULL; | |
630 | ||
631 | switch (type) { | |
632 | case IOMMU_DOMAIN_UNMANAGED: | |
633 | io_domain = __ipmmu_domain_alloc(type); | |
634 | break; | |
635 | ||
636 | case IOMMU_DOMAIN_DMA: | |
637 | io_domain = __ipmmu_domain_alloc(type); | |
638 | if (io_domain && iommu_get_dma_cookie(io_domain)) { | |
639 | kfree(io_domain); | |
640 | io_domain = NULL; | |
641 | } | |
642 | break; | |
643 | } | |
644 | ||
645 | return io_domain; | |
646 | } | |
647 | ||
5914c5fd | 648 | static void ipmmu_domain_free(struct iommu_domain *io_domain) |
d25a2a16 | 649 | { |
5914c5fd | 650 | struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); |
d25a2a16 LP |
651 | |
652 | /* | |
653 | * Free the domain resources. We assume that all devices have already | |
654 | * been detached. | |
655 | */ | |
1c7e7c02 | 656 | iommu_put_dma_cookie(io_domain); |
d25a2a16 | 657 | ipmmu_domain_destroy_context(domain); |
f20ed39f | 658 | free_io_pgtable_ops(domain->iop); |
d25a2a16 LP |
659 | kfree(domain); |
660 | } | |
661 | ||
662 | static int ipmmu_attach_device(struct iommu_domain *io_domain, | |
663 | struct device *dev) | |
664 | { | |
df903655 | 665 | struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); |
e4efe4a9 | 666 | struct ipmmu_vmsa_device *mmu = to_ipmmu(dev); |
5914c5fd | 667 | struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); |
a166d31e | 668 | unsigned int i; |
d25a2a16 LP |
669 | int ret = 0; |
670 | ||
e4efe4a9 | 671 | if (!mmu) { |
d25a2a16 LP |
672 | dev_err(dev, "Cannot attach to IPMMU\n"); |
673 | return -ENXIO; | |
674 | } | |
675 | ||
46583e8c | 676 | mutex_lock(&domain->mutex); |
d25a2a16 LP |
677 | |
678 | if (!domain->mmu) { | |
679 | /* The domain hasn't been used yet, initialize it. */ | |
680 | domain->mmu = mmu; | |
681 | ret = ipmmu_domain_init_context(domain); | |
5fd16341 MD |
682 | if (ret < 0) { |
683 | dev_err(dev, "Unable to initialize IPMMU context\n"); | |
684 | domain->mmu = NULL; | |
685 | } else { | |
686 | dev_info(dev, "Using IPMMU context %u\n", | |
687 | domain->context_id); | |
688 | } | |
d25a2a16 LP |
689 | } else if (domain->mmu != mmu) { |
690 | /* | |
691 | * Something is wrong, we can't attach two devices using | |
692 | * different IOMMUs to the same domain. | |
693 | */ | |
694 | dev_err(dev, "Can't attach IPMMU %s to domain on IPMMU %s\n", | |
695 | dev_name(mmu->dev), dev_name(domain->mmu->dev)); | |
696 | ret = -EINVAL; | |
3ae47292 MD |
697 | } else |
698 | dev_info(dev, "Reusing IPMMU context %u\n", domain->context_id); | |
d25a2a16 | 699 | |
46583e8c | 700 | mutex_unlock(&domain->mutex); |
d25a2a16 LP |
701 | |
702 | if (ret < 0) | |
703 | return ret; | |
704 | ||
7b2d5961 MD |
705 | for (i = 0; i < fwspec->num_ids; ++i) |
706 | ipmmu_utlb_enable(domain, fwspec->ids[i]); | |
d25a2a16 LP |
707 | |
708 | return 0; | |
709 | } | |
710 | ||
711 | static void ipmmu_detach_device(struct iommu_domain *io_domain, | |
712 | struct device *dev) | |
713 | { | |
df903655 | 714 | struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); |
5914c5fd | 715 | struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); |
a166d31e | 716 | unsigned int i; |
d25a2a16 | 717 | |
7b2d5961 MD |
718 | for (i = 0; i < fwspec->num_ids; ++i) |
719 | ipmmu_utlb_disable(domain, fwspec->ids[i]); | |
d25a2a16 LP |
720 | |
721 | /* | |
722 | * TODO: Optimize by disabling the context when no device is attached. | |
723 | */ | |
724 | } | |
725 | ||
726 | static int ipmmu_map(struct iommu_domain *io_domain, unsigned long iova, | |
727 | phys_addr_t paddr, size_t size, int prot) | |
728 | { | |
5914c5fd | 729 | struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); |
d25a2a16 LP |
730 | |
731 | if (!domain) | |
732 | return -ENODEV; | |
733 | ||
f20ed39f | 734 | return domain->iop->map(domain->iop, iova, paddr, size, prot); |
d25a2a16 LP |
735 | } |
736 | ||
737 | static size_t ipmmu_unmap(struct iommu_domain *io_domain, unsigned long iova, | |
56f8af5e | 738 | size_t size, struct iommu_iotlb_gather *gather) |
d25a2a16 | 739 | { |
5914c5fd | 740 | struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); |
d25a2a16 | 741 | |
a2d3a382 | 742 | return domain->iop->unmap(domain->iop, iova, size, gather); |
d25a2a16 LP |
743 | } |
744 | ||
56f8af5e | 745 | static void ipmmu_flush_iotlb_all(struct iommu_domain *io_domain) |
32b12449 RM |
746 | { |
747 | struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); | |
748 | ||
749 | if (domain->mmu) | |
750 | ipmmu_tlb_flush_all(domain); | |
751 | } | |
752 | ||
56f8af5e WD |
753 | static void ipmmu_iotlb_sync(struct iommu_domain *io_domain, |
754 | struct iommu_iotlb_gather *gather) | |
755 | { | |
756 | ipmmu_flush_iotlb_all(io_domain); | |
757 | } | |
758 | ||
d25a2a16 LP |
759 | static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain, |
760 | dma_addr_t iova) | |
761 | { | |
5914c5fd | 762 | struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); |
d25a2a16 LP |
763 | |
764 | /* TODO: Is locking needed ? */ | |
765 | ||
f20ed39f | 766 | return domain->iop->iova_to_phys(domain->iop, iova); |
d25a2a16 LP |
767 | } |
768 | ||
7b2d5961 MD |
769 | static int ipmmu_init_platform_device(struct device *dev, |
770 | struct of_phandle_args *args) | |
d25a2a16 | 771 | { |
df903655 | 772 | struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); |
7b2d5961 | 773 | struct platform_device *ipmmu_pdev; |
bb590c90 | 774 | |
7b2d5961 MD |
775 | ipmmu_pdev = of_find_device_by_node(args->np); |
776 | if (!ipmmu_pdev) | |
bb590c90 LP |
777 | return -ENODEV; |
778 | ||
df903655 | 779 | fwspec->iommu_priv = platform_get_drvdata(ipmmu_pdev); |
383fef5f | 780 | |
383fef5f | 781 | return 0; |
58b8e8bf MD |
782 | } |
783 | ||
0b8ac140 | 784 | static const struct soc_device_attribute soc_rcar_gen3[] = { |
60fb0083 | 785 | { .soc_id = "r8a774a1", }, |
b6d39cd8 | 786 | { .soc_id = "r8a774c0", }, |
58b8e8bf | 787 | { .soc_id = "r8a7795", }, |
0b8ac140 | 788 | { .soc_id = "r8a7796", }, |
98dbffd3 | 789 | { .soc_id = "r8a77965", }, |
3701c123 | 790 | { .soc_id = "r8a77970", }, |
b0c32912 | 791 | { .soc_id = "r8a77990", }, |
3701c123 | 792 | { .soc_id = "r8a77995", }, |
58b8e8bf MD |
793 | { /* sentinel */ } |
794 | }; | |
795 | ||
b7ee92c6 | 796 | static const struct soc_device_attribute soc_rcar_gen3_whitelist[] = { |
b6d39cd8 | 797 | { .soc_id = "r8a774c0", }, |
b7ee92c6 YS |
798 | { .soc_id = "r8a7795", .revision = "ES3.*" }, |
799 | { .soc_id = "r8a77965", }, | |
800 | { .soc_id = "r8a77990", }, | |
3701c123 | 801 | { .soc_id = "r8a77995", }, |
58b8e8bf MD |
802 | { /* sentinel */ } |
803 | }; | |
804 | ||
80759649 YS |
805 | static const char * const rcar_gen3_slave_whitelist[] = { |
806 | }; | |
807 | ||
b7ee92c6 YS |
808 | static bool ipmmu_slave_whitelist(struct device *dev) |
809 | { | |
80759649 YS |
810 | unsigned int i; |
811 | ||
b7ee92c6 YS |
812 | /* |
813 | * For R-Car Gen3 use a white list to opt-in slave devices. | |
814 | * For Other SoCs, this returns true anyway. | |
815 | */ | |
816 | if (!soc_device_match(soc_rcar_gen3)) | |
817 | return true; | |
818 | ||
819 | /* Check whether this R-Car Gen3 can use the IPMMU correctly or not */ | |
820 | if (!soc_device_match(soc_rcar_gen3_whitelist)) | |
821 | return false; | |
822 | ||
80759649 YS |
823 | /* Check whether this slave device can work with the IPMMU */ |
824 | for (i = 0; i < ARRAY_SIZE(rcar_gen3_slave_whitelist); i++) { | |
825 | if (!strcmp(dev_name(dev), rcar_gen3_slave_whitelist[i])) | |
826 | return true; | |
827 | } | |
828 | ||
829 | /* Otherwise, do not allow use of IPMMU */ | |
b7ee92c6 YS |
830 | return false; |
831 | } | |
832 | ||
49558da0 MD |
833 | static int ipmmu_of_xlate(struct device *dev, |
834 | struct of_phandle_args *spec) | |
835 | { | |
b7ee92c6 | 836 | if (!ipmmu_slave_whitelist(dev)) |
58b8e8bf MD |
837 | return -ENODEV; |
838 | ||
7b2d5961 MD |
839 | iommu_fwspec_add_ids(dev, spec->args, 1); |
840 | ||
49558da0 | 841 | /* Initialize once - xlate() will call multiple times */ |
e4efe4a9 | 842 | if (to_ipmmu(dev)) |
49558da0 MD |
843 | return 0; |
844 | ||
7b2d5961 | 845 | return ipmmu_init_platform_device(dev, spec); |
49558da0 MD |
846 | } |
847 | ||
49c875f0 | 848 | static int ipmmu_init_arm_mapping(struct device *dev) |
383fef5f | 849 | { |
e4efe4a9 | 850 | struct ipmmu_vmsa_device *mmu = to_ipmmu(dev); |
383fef5f MD |
851 | struct iommu_group *group; |
852 | int ret; | |
853 | ||
d25a2a16 LP |
854 | /* Create a device group and add the device to it. */ |
855 | group = iommu_group_alloc(); | |
856 | if (IS_ERR(group)) { | |
857 | dev_err(dev, "Failed to allocate IOMMU group\n"); | |
49c875f0 | 858 | return PTR_ERR(group); |
d25a2a16 LP |
859 | } |
860 | ||
861 | ret = iommu_group_add_device(group, dev); | |
862 | iommu_group_put(group); | |
863 | ||
864 | if (ret < 0) { | |
865 | dev_err(dev, "Failed to add device to IPMMU group\n"); | |
49c875f0 | 866 | return ret; |
d25a2a16 LP |
867 | } |
868 | ||
d25a2a16 LP |
869 | /* |
870 | * Create the ARM mapping, used by the ARM DMA mapping core to allocate | |
871 | * VAs. This will allocate a corresponding IOMMU domain. | |
872 | * | |
873 | * TODO: | |
874 | * - Create one mapping per context (TLB). | |
875 | * - Make the mapping size configurable ? We currently use a 2GB mapping | |
876 | * at a 1GB offset to ensure that NULL VAs will fault. | |
877 | */ | |
878 | if (!mmu->mapping) { | |
879 | struct dma_iommu_mapping *mapping; | |
880 | ||
881 | mapping = arm_iommu_create_mapping(&platform_bus_type, | |
720b0cef | 882 | SZ_1G, SZ_2G); |
d25a2a16 LP |
883 | if (IS_ERR(mapping)) { |
884 | dev_err(mmu->dev, "failed to create ARM IOMMU mapping\n"); | |
b8f80bff LP |
885 | ret = PTR_ERR(mapping); |
886 | goto error; | |
d25a2a16 LP |
887 | } |
888 | ||
889 | mmu->mapping = mapping; | |
890 | } | |
891 | ||
892 | /* Attach the ARM VA mapping to the device. */ | |
893 | ret = arm_iommu_attach_device(dev, mmu->mapping); | |
894 | if (ret < 0) { | |
895 | dev_err(dev, "Failed to attach device to VA mapping\n"); | |
896 | goto error; | |
897 | } | |
898 | ||
899 | return 0; | |
900 | ||
901 | error: | |
49c875f0 RM |
902 | iommu_group_remove_device(dev); |
903 | if (mmu->mapping) | |
383fef5f | 904 | arm_iommu_release_mapping(mmu->mapping); |
a166d31e | 905 | |
d25a2a16 LP |
906 | return ret; |
907 | } | |
908 | ||
49c875f0 | 909 | static int ipmmu_add_device(struct device *dev) |
3ae47292 | 910 | { |
80eaa9f5 | 911 | struct ipmmu_vmsa_device *mmu = to_ipmmu(dev); |
3ae47292 | 912 | struct iommu_group *group; |
80eaa9f5 | 913 | int ret; |
3ae47292 | 914 | |
0fbc8b04 MD |
915 | /* |
916 | * Only let through devices that have been verified in xlate() | |
0fbc8b04 | 917 | */ |
80eaa9f5 | 918 | if (!mmu) |
3ae47292 MD |
919 | return -ENODEV; |
920 | ||
80eaa9f5 GU |
921 | if (IS_ENABLED(CONFIG_ARM) && !IS_ENABLED(CONFIG_IOMMU_DMA)) { |
922 | ret = ipmmu_init_arm_mapping(dev); | |
923 | if (ret) | |
924 | return ret; | |
925 | } else { | |
926 | group = iommu_group_get_for_dev(dev); | |
927 | if (IS_ERR(group)) | |
928 | return PTR_ERR(group); | |
49c875f0 | 929 | |
80eaa9f5 GU |
930 | iommu_group_put(group); |
931 | } | |
3ae47292 | 932 | |
80eaa9f5 | 933 | iommu_device_link(&mmu->iommu, dev); |
3ae47292 MD |
934 | return 0; |
935 | } | |
936 | ||
49c875f0 | 937 | static void ipmmu_remove_device(struct device *dev) |
3ae47292 | 938 | { |
80eaa9f5 GU |
939 | struct ipmmu_vmsa_device *mmu = to_ipmmu(dev); |
940 | ||
941 | iommu_device_unlink(&mmu->iommu, dev); | |
49c875f0 | 942 | arm_iommu_detach_device(dev); |
3ae47292 MD |
943 | iommu_group_remove_device(dev); |
944 | } | |
945 | ||
b354c73e | 946 | static struct iommu_group *ipmmu_find_group(struct device *dev) |
3ae47292 | 947 | { |
e4efe4a9 | 948 | struct ipmmu_vmsa_device *mmu = to_ipmmu(dev); |
3ae47292 | 949 | struct iommu_group *group; |
3ae47292 | 950 | |
e4efe4a9 RM |
951 | if (mmu->group) |
952 | return iommu_group_ref_get(mmu->group); | |
b354c73e RM |
953 | |
954 | group = iommu_group_alloc(); | |
955 | if (!IS_ERR(group)) | |
e4efe4a9 | 956 | mmu->group = group; |
3ae47292 MD |
957 | |
958 | return group; | |
959 | } | |
960 | ||
3ae47292 | 961 | static const struct iommu_ops ipmmu_ops = { |
1c7e7c02 RM |
962 | .domain_alloc = ipmmu_domain_alloc, |
963 | .domain_free = ipmmu_domain_free, | |
3ae47292 MD |
964 | .attach_dev = ipmmu_attach_device, |
965 | .detach_dev = ipmmu_detach_device, | |
966 | .map = ipmmu_map, | |
967 | .unmap = ipmmu_unmap, | |
56f8af5e | 968 | .flush_iotlb_all = ipmmu_flush_iotlb_all, |
32b12449 | 969 | .iotlb_sync = ipmmu_iotlb_sync, |
3ae47292 | 970 | .iova_to_phys = ipmmu_iova_to_phys, |
49c875f0 RM |
971 | .add_device = ipmmu_add_device, |
972 | .remove_device = ipmmu_remove_device, | |
b354c73e | 973 | .device_group = ipmmu_find_group, |
3ae47292 | 974 | .pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K, |
49558da0 | 975 | .of_xlate = ipmmu_of_xlate, |
3ae47292 MD |
976 | }; |
977 | ||
d25a2a16 LP |
978 | /* ----------------------------------------------------------------------------- |
979 | * Probe/remove and init | |
980 | */ | |
981 | ||
982 | static void ipmmu_device_reset(struct ipmmu_vmsa_device *mmu) | |
983 | { | |
984 | unsigned int i; | |
985 | ||
986 | /* Disable all contexts. */ | |
5fd16341 | 987 | for (i = 0; i < mmu->num_ctx; ++i) |
d25a2a16 LP |
988 | ipmmu_write(mmu, i * IM_CTX_SIZE + IMCTR, 0); |
989 | } | |
990 | ||
33f3ac9b MD |
991 | static const struct ipmmu_features ipmmu_features_default = { |
992 | .use_ns_alias_offset = true, | |
fd5140e2 | 993 | .has_cache_leaf_nodes = false, |
5fd16341 | 994 | .number_of_contexts = 1, /* software only tested with one context */ |
b7f3f047 | 995 | .num_utlbs = 32, |
f5c85891 | 996 | .setup_imbuscr = true, |
c295f504 | 997 | .twobit_imttbcr_sl0 = false, |
2ae86955 | 998 | .reserved_context = false, |
3623002f | 999 | .cache_snoop = true, |
33f3ac9b MD |
1000 | }; |
1001 | ||
0b8ac140 | 1002 | static const struct ipmmu_features ipmmu_features_rcar_gen3 = { |
58b8e8bf MD |
1003 | .use_ns_alias_offset = false, |
1004 | .has_cache_leaf_nodes = true, | |
1005 | .number_of_contexts = 8, | |
b7f3f047 | 1006 | .num_utlbs = 48, |
58b8e8bf MD |
1007 | .setup_imbuscr = false, |
1008 | .twobit_imttbcr_sl0 = true, | |
2ae86955 | 1009 | .reserved_context = true, |
3623002f | 1010 | .cache_snoop = false, |
58b8e8bf MD |
1011 | }; |
1012 | ||
33f3ac9b MD |
1013 | static const struct of_device_id ipmmu_of_ids[] = { |
1014 | { | |
1015 | .compatible = "renesas,ipmmu-vmsa", | |
1016 | .data = &ipmmu_features_default, | |
60fb0083 FC |
1017 | }, { |
1018 | .compatible = "renesas,ipmmu-r8a774a1", | |
1019 | .data = &ipmmu_features_rcar_gen3, | |
b6d39cd8 FC |
1020 | }, { |
1021 | .compatible = "renesas,ipmmu-r8a774c0", | |
1022 | .data = &ipmmu_features_rcar_gen3, | |
58b8e8bf MD |
1023 | }, { |
1024 | .compatible = "renesas,ipmmu-r8a7795", | |
0b8ac140 MD |
1025 | .data = &ipmmu_features_rcar_gen3, |
1026 | }, { | |
1027 | .compatible = "renesas,ipmmu-r8a7796", | |
1028 | .data = &ipmmu_features_rcar_gen3, | |
98dbffd3 JM |
1029 | }, { |
1030 | .compatible = "renesas,ipmmu-r8a77965", | |
1031 | .data = &ipmmu_features_rcar_gen3, | |
3701c123 SH |
1032 | }, { |
1033 | .compatible = "renesas,ipmmu-r8a77970", | |
1034 | .data = &ipmmu_features_rcar_gen3, | |
b0c32912 HNP |
1035 | }, { |
1036 | .compatible = "renesas,ipmmu-r8a77990", | |
1037 | .data = &ipmmu_features_rcar_gen3, | |
3701c123 SH |
1038 | }, { |
1039 | .compatible = "renesas,ipmmu-r8a77995", | |
1040 | .data = &ipmmu_features_rcar_gen3, | |
33f3ac9b MD |
1041 | }, { |
1042 | /* Terminator */ | |
1043 | }, | |
1044 | }; | |
1045 | ||
d25a2a16 LP |
1046 | static int ipmmu_probe(struct platform_device *pdev) |
1047 | { | |
1048 | struct ipmmu_vmsa_device *mmu; | |
1049 | struct resource *res; | |
1050 | int irq; | |
1051 | int ret; | |
1052 | ||
d25a2a16 LP |
1053 | mmu = devm_kzalloc(&pdev->dev, sizeof(*mmu), GFP_KERNEL); |
1054 | if (!mmu) { | |
1055 | dev_err(&pdev->dev, "cannot allocate device data\n"); | |
1056 | return -ENOMEM; | |
1057 | } | |
1058 | ||
1059 | mmu->dev = &pdev->dev; | |
dbb70692 MD |
1060 | spin_lock_init(&mmu->lock); |
1061 | bitmap_zero(mmu->ctx, IPMMU_CTX_MAX); | |
33f3ac9b | 1062 | mmu->features = of_device_get_match_data(&pdev->dev); |
da38e9ec | 1063 | memset(mmu->utlb_ctx, IPMMU_CTX_INVALID, mmu->features->num_utlbs); |
1c894225 | 1064 | dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)); |
d25a2a16 LP |
1065 | |
1066 | /* Map I/O memory and request IRQ. */ | |
1067 | res = platform_get_resource(pdev, IORESOURCE_MEM, 0); | |
1068 | mmu->base = devm_ioremap_resource(&pdev->dev, res); | |
1069 | if (IS_ERR(mmu->base)) | |
1070 | return PTR_ERR(mmu->base); | |
1071 | ||
275f5053 LP |
1072 | /* |
1073 | * The IPMMU has two register banks, for secure and non-secure modes. | |
1074 | * The bank mapped at the beginning of the IPMMU address space | |
1075 | * corresponds to the running mode of the CPU. When running in secure | |
1076 | * mode the non-secure register bank is also available at an offset. | |
1077 | * | |
1078 | * Secure mode operation isn't clearly documented and is thus currently | |
1079 | * not implemented in the driver. Furthermore, preliminary tests of | |
1080 | * non-secure operation with the main register bank were not successful. | |
1081 | * Offset the registers base unconditionally to point to the non-secure | |
1082 | * alias space for now. | |
1083 | */ | |
33f3ac9b MD |
1084 | if (mmu->features->use_ns_alias_offset) |
1085 | mmu->base += IM_NS_ALIAS_OFFSET; | |
275f5053 | 1086 | |
b43e0d8a | 1087 | mmu->num_ctx = min(IPMMU_CTX_MAX, mmu->features->number_of_contexts); |
5fd16341 | 1088 | |
d25a2a16 | 1089 | irq = platform_get_irq(pdev, 0); |
d25a2a16 | 1090 | |
fd5140e2 MD |
1091 | /* |
1092 | * Determine if this IPMMU instance is a root device by checking for | |
1093 | * the lack of has_cache_leaf_nodes flag or renesas,ipmmu-main property. | |
1094 | */ | |
1095 | if (!mmu->features->has_cache_leaf_nodes || | |
1096 | !of_find_property(pdev->dev.of_node, "renesas,ipmmu-main", NULL)) | |
1097 | mmu->root = mmu; | |
1098 | else | |
1099 | mmu->root = ipmmu_find_root(); | |
d25a2a16 | 1100 | |
fd5140e2 MD |
1101 | /* |
1102 | * Wait until the root device has been registered for sure. | |
1103 | */ | |
1104 | if (!mmu->root) | |
1105 | return -EPROBE_DEFER; | |
1106 | ||
1107 | /* Root devices have mandatory IRQs */ | |
1108 | if (ipmmu_is_root(mmu)) { | |
1109 | if (irq < 0) { | |
1110 | dev_err(&pdev->dev, "no IRQ found\n"); | |
1111 | return irq; | |
1112 | } | |
1113 | ||
1114 | ret = devm_request_irq(&pdev->dev, irq, ipmmu_irq, 0, | |
1115 | dev_name(&pdev->dev), mmu); | |
1116 | if (ret < 0) { | |
1117 | dev_err(&pdev->dev, "failed to request IRQ %d\n", irq); | |
1118 | return ret; | |
1119 | } | |
1120 | ||
1121 | ipmmu_device_reset(mmu); | |
2ae86955 YS |
1122 | |
1123 | if (mmu->features->reserved_context) { | |
1124 | dev_info(&pdev->dev, "IPMMU context 0 is reserved\n"); | |
1125 | set_bit(0, mmu->ctx); | |
1126 | } | |
fd5140e2 | 1127 | } |
d25a2a16 | 1128 | |
cda52fcd MD |
1129 | /* |
1130 | * Register the IPMMU to the IOMMU subsystem in the following cases: | |
1131 | * - R-Car Gen2 IPMMU (all devices registered) | |
1132 | * - R-Car Gen3 IPMMU (leaf devices only - skip root IPMMU-MM device) | |
1133 | */ | |
1134 | if (!mmu->features->has_cache_leaf_nodes || !ipmmu_is_root(mmu)) { | |
1135 | ret = iommu_device_sysfs_add(&mmu->iommu, &pdev->dev, NULL, | |
1136 | dev_name(&pdev->dev)); | |
1137 | if (ret) | |
1138 | return ret; | |
7af9a5fd | 1139 | |
cda52fcd MD |
1140 | iommu_device_set_ops(&mmu->iommu, &ipmmu_ops); |
1141 | iommu_device_set_fwnode(&mmu->iommu, | |
1142 | &pdev->dev.of_node->fwnode); | |
01da21e5 | 1143 | |
cda52fcd MD |
1144 | ret = iommu_device_register(&mmu->iommu); |
1145 | if (ret) | |
1146 | return ret; | |
1147 | ||
1148 | #if defined(CONFIG_IOMMU_DMA) | |
1149 | if (!iommu_present(&platform_bus_type)) | |
1150 | bus_set_iommu(&platform_bus_type, &ipmmu_ops); | |
1151 | #endif | |
1152 | } | |
01da21e5 | 1153 | |
d25a2a16 LP |
1154 | /* |
1155 | * We can't create the ARM mapping here as it requires the bus to have | |
1156 | * an IOMMU, which only happens when bus_set_iommu() is called in | |
1157 | * ipmmu_init() after the probe function returns. | |
1158 | */ | |
1159 | ||
d25a2a16 LP |
1160 | platform_set_drvdata(pdev, mmu); |
1161 | ||
1162 | return 0; | |
1163 | } | |
1164 | ||
1165 | static int ipmmu_remove(struct platform_device *pdev) | |
1166 | { | |
1167 | struct ipmmu_vmsa_device *mmu = platform_get_drvdata(pdev); | |
1168 | ||
7af9a5fd | 1169 | iommu_device_sysfs_remove(&mmu->iommu); |
01da21e5 MD |
1170 | iommu_device_unregister(&mmu->iommu); |
1171 | ||
d25a2a16 LP |
1172 | arm_iommu_release_mapping(mmu->mapping); |
1173 | ||
1174 | ipmmu_device_reset(mmu); | |
1175 | ||
1176 | return 0; | |
1177 | } | |
1178 | ||
da38e9ec GU |
1179 | #ifdef CONFIG_PM_SLEEP |
1180 | static int ipmmu_resume_noirq(struct device *dev) | |
1181 | { | |
1182 | struct ipmmu_vmsa_device *mmu = dev_get_drvdata(dev); | |
1183 | unsigned int i; | |
1184 | ||
1185 | /* Reset root MMU and restore contexts */ | |
1186 | if (ipmmu_is_root(mmu)) { | |
1187 | ipmmu_device_reset(mmu); | |
1188 | ||
1189 | for (i = 0; i < mmu->num_ctx; i++) { | |
1190 | if (!mmu->domains[i]) | |
1191 | continue; | |
1192 | ||
1193 | ipmmu_domain_setup_context(mmu->domains[i]); | |
1194 | } | |
1195 | } | |
1196 | ||
1197 | /* Re-enable active micro-TLBs */ | |
1198 | for (i = 0; i < mmu->features->num_utlbs; i++) { | |
1199 | if (mmu->utlb_ctx[i] == IPMMU_CTX_INVALID) | |
1200 | continue; | |
1201 | ||
1202 | ipmmu_utlb_enable(mmu->root->domains[mmu->utlb_ctx[i]], i); | |
1203 | } | |
1204 | ||
1205 | return 0; | |
1206 | } | |
1207 | ||
1208 | static const struct dev_pm_ops ipmmu_pm = { | |
1209 | SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(NULL, ipmmu_resume_noirq) | |
1210 | }; | |
1211 | #define DEV_PM_OPS &ipmmu_pm | |
1212 | #else | |
1213 | #define DEV_PM_OPS NULL | |
1214 | #endif /* CONFIG_PM_SLEEP */ | |
1215 | ||
d25a2a16 LP |
1216 | static struct platform_driver ipmmu_driver = { |
1217 | .driver = { | |
d25a2a16 | 1218 | .name = "ipmmu-vmsa", |
275f5053 | 1219 | .of_match_table = of_match_ptr(ipmmu_of_ids), |
da38e9ec | 1220 | .pm = DEV_PM_OPS, |
d25a2a16 LP |
1221 | }, |
1222 | .probe = ipmmu_probe, | |
1223 | .remove = ipmmu_remove, | |
1224 | }; | |
1225 | ||
1226 | static int __init ipmmu_init(void) | |
1227 | { | |
5c5c8741 | 1228 | struct device_node *np; |
cda52fcd | 1229 | static bool setup_done; |
d25a2a16 LP |
1230 | int ret; |
1231 | ||
cda52fcd MD |
1232 | if (setup_done) |
1233 | return 0; | |
1234 | ||
5c5c8741 DO |
1235 | np = of_find_matching_node(NULL, ipmmu_of_ids); |
1236 | if (!np) | |
1237 | return 0; | |
1238 | ||
1239 | of_node_put(np); | |
1240 | ||
d25a2a16 LP |
1241 | ret = platform_driver_register(&ipmmu_driver); |
1242 | if (ret < 0) | |
1243 | return ret; | |
1244 | ||
cda52fcd | 1245 | #if defined(CONFIG_ARM) && !defined(CONFIG_IOMMU_DMA) |
d25a2a16 LP |
1246 | if (!iommu_present(&platform_bus_type)) |
1247 | bus_set_iommu(&platform_bus_type, &ipmmu_ops); | |
cda52fcd | 1248 | #endif |
d25a2a16 | 1249 | |
cda52fcd | 1250 | setup_done = true; |
d25a2a16 LP |
1251 | return 0; |
1252 | } | |
d25a2a16 | 1253 | subsys_initcall(ipmmu_init); |