Commit | Line | Data |
---|---|---|
57d3f11c | 1 | // SPDX-License-Identifier: GPL-2.0 |
d25a2a16 | 2 | /* |
8128ac3b PG |
3 | * IOMMU API for Renesas VMSA-compatible IPMMU |
4 | * Author: Laurent Pinchart <laurent.pinchart@ideasonboard.com> | |
d25a2a16 | 5 | * |
17fe1618 | 6 | * Copyright (C) 2014-2020 Renesas Electronics Corporation |
d25a2a16 LP |
7 | */ |
8 | ||
dbb70692 | 9 | #include <linux/bitmap.h> |
d25a2a16 LP |
10 | #include <linux/delay.h> |
11 | #include <linux/dma-mapping.h> | |
12 | #include <linux/err.h> | |
13 | #include <linux/export.h> | |
8128ac3b | 14 | #include <linux/init.h> |
d25a2a16 LP |
15 | #include <linux/interrupt.h> |
16 | #include <linux/io.h> | |
b77cf11f | 17 | #include <linux/io-pgtable.h> |
d25a2a16 | 18 | #include <linux/iommu.h> |
275f5053 | 19 | #include <linux/of.h> |
33f3ac9b | 20 | #include <linux/of_device.h> |
7b2d5961 | 21 | #include <linux/of_platform.h> |
d25a2a16 LP |
22 | #include <linux/platform_device.h> |
23 | #include <linux/sizes.h> | |
24 | #include <linux/slab.h> | |
58b8e8bf | 25 | #include <linux/sys_soc.h> |
d25a2a16 | 26 | |
3ae47292 | 27 | #if defined(CONFIG_ARM) && !defined(CONFIG_IOMMU_DMA) |
d25a2a16 | 28 | #include <asm/dma-iommu.h> |
49c875f0 RM |
29 | #else |
30 | #define arm_iommu_create_mapping(...) NULL | |
31 | #define arm_iommu_attach_device(...) -ENODEV | |
32 | #define arm_iommu_release_mapping(...) do {} while (0) | |
33 | #define arm_iommu_detach_device(...) do {} while (0) | |
3ae47292 | 34 | #endif |
d25a2a16 | 35 | |
7a62ced8 | 36 | #define IPMMU_CTX_MAX 16U |
da38e9ec GU |
37 | #define IPMMU_CTX_INVALID -1 |
38 | ||
7a62ced8 | 39 | #define IPMMU_UTLB_MAX 64U |
dbb70692 | 40 | |
33f3ac9b MD |
41 | struct ipmmu_features { |
42 | bool use_ns_alias_offset; | |
fd5140e2 | 43 | bool has_cache_leaf_nodes; |
5fd16341 | 44 | unsigned int number_of_contexts; |
b7f3f047 | 45 | unsigned int num_utlbs; |
f5c85891 | 46 | bool setup_imbuscr; |
c295f504 | 47 | bool twobit_imttbcr_sl0; |
2ae86955 | 48 | bool reserved_context; |
3623002f | 49 | bool cache_snoop; |
3dc28d9f YS |
50 | unsigned int ctx_offset_base; |
51 | unsigned int ctx_offset_stride; | |
1289f7f1 | 52 | unsigned int utlb_offset_base; |
33f3ac9b MD |
53 | }; |
54 | ||
d25a2a16 LP |
55 | struct ipmmu_vmsa_device { |
56 | struct device *dev; | |
57 | void __iomem *base; | |
01da21e5 | 58 | struct iommu_device iommu; |
fd5140e2 | 59 | struct ipmmu_vmsa_device *root; |
33f3ac9b | 60 | const struct ipmmu_features *features; |
5fd16341 | 61 | unsigned int num_ctx; |
dbb70692 MD |
62 | spinlock_t lock; /* Protects ctx and domains[] */ |
63 | DECLARE_BITMAP(ctx, IPMMU_CTX_MAX); | |
64 | struct ipmmu_vmsa_domain *domains[IPMMU_CTX_MAX]; | |
da38e9ec | 65 | s8 utlb_ctx[IPMMU_UTLB_MAX]; |
d25a2a16 | 66 | |
b354c73e | 67 | struct iommu_group *group; |
d25a2a16 LP |
68 | struct dma_iommu_mapping *mapping; |
69 | }; | |
70 | ||
71 | struct ipmmu_vmsa_domain { | |
72 | struct ipmmu_vmsa_device *mmu; | |
5914c5fd | 73 | struct iommu_domain io_domain; |
d25a2a16 | 74 | |
f20ed39f LP |
75 | struct io_pgtable_cfg cfg; |
76 | struct io_pgtable_ops *iop; | |
77 | ||
d25a2a16 | 78 | unsigned int context_id; |
46583e8c | 79 | struct mutex mutex; /* Protects mappings */ |
d25a2a16 LP |
80 | }; |
81 | ||
5914c5fd JR |
82 | static struct ipmmu_vmsa_domain *to_vmsa_domain(struct iommu_domain *dom) |
83 | { | |
84 | return container_of(dom, struct ipmmu_vmsa_domain, io_domain); | |
85 | } | |
86 | ||
e4efe4a9 | 87 | static struct ipmmu_vmsa_device *to_ipmmu(struct device *dev) |
0fbc8b04 | 88 | { |
be568d6d | 89 | return dev_iommu_priv_get(dev); |
0fbc8b04 MD |
90 | } |
91 | ||
d25a2a16 LP |
92 | #define TLB_LOOP_TIMEOUT 100 /* 100us */ |
93 | ||
94 | /* ----------------------------------------------------------------------------- | |
95 | * Registers Definition | |
96 | */ | |
97 | ||
275f5053 LP |
98 | #define IM_NS_ALIAS_OFFSET 0x800 |
99 | ||
df9828aa YS |
100 | /* MMU "context" registers */ |
101 | #define IMCTR 0x0000 /* R-Car Gen2/3 */ | |
102 | #define IMCTR_INTEN (1 << 2) /* R-Car Gen2/3 */ | |
103 | #define IMCTR_FLUSH (1 << 1) /* R-Car Gen2/3 */ | |
104 | #define IMCTR_MMUEN (1 << 0) /* R-Car Gen2/3 */ | |
105 | ||
106 | #define IMTTBCR 0x0008 /* R-Car Gen2/3 */ | |
107 | #define IMTTBCR_EAE (1 << 31) /* R-Car Gen2/3 */ | |
3623002f | 108 | #define IMTTBCR_SH0_INNER_SHAREABLE (3 << 12) /* R-Car Gen2 only */ |
3623002f | 109 | #define IMTTBCR_ORGN0_WB_WA (1 << 10) /* R-Car Gen2 only */ |
3623002f | 110 | #define IMTTBCR_IRGN0_WB_WA (1 << 8) /* R-Car Gen2 only */ |
5ca54fdc | 111 | #define IMTTBCR_SL0_TWOBIT_LVL_1 (2 << 6) /* R-Car Gen3 only */ |
df9828aa | 112 | #define IMTTBCR_SL0_LVL_1 (1 << 4) /* R-Car Gen2 only */ |
d25a2a16 | 113 | |
df9828aa YS |
114 | #define IMBUSCR 0x000c /* R-Car Gen2 only */ |
115 | #define IMBUSCR_DVM (1 << 2) /* R-Car Gen2 only */ | |
116 | #define IMBUSCR_BUSSEL_MASK (3 << 0) /* R-Car Gen2 only */ | |
d25a2a16 | 117 | |
df9828aa YS |
118 | #define IMTTLBR0 0x0010 /* R-Car Gen2/3 */ |
119 | #define IMTTUBR0 0x0014 /* R-Car Gen2/3 */ | |
d25a2a16 | 120 | |
df9828aa YS |
121 | #define IMSTR 0x0020 /* R-Car Gen2/3 */ |
122 | #define IMSTR_MHIT (1 << 4) /* R-Car Gen2/3 */ | |
123 | #define IMSTR_ABORT (1 << 2) /* R-Car Gen2/3 */ | |
124 | #define IMSTR_PF (1 << 1) /* R-Car Gen2/3 */ | |
125 | #define IMSTR_TF (1 << 0) /* R-Car Gen2/3 */ | |
d25a2a16 | 126 | |
df9828aa | 127 | #define IMMAIR0 0x0028 /* R-Car Gen2/3 */ |
d25a2a16 | 128 | |
df9828aa YS |
129 | #define IMELAR 0x0030 /* R-Car Gen2/3, IMEAR on R-Car Gen2 */ |
130 | #define IMEUAR 0x0034 /* R-Car Gen3 only */ | |
d25a2a16 | 131 | |
df9828aa | 132 | /* uTLB registers */ |
ddbbddd7 | 133 | #define IMUCTR(n) ((n) < 32 ? IMUCTR0(n) : IMUCTR32(n)) |
df9828aa YS |
134 | #define IMUCTR0(n) (0x0300 + ((n) * 16)) /* R-Car Gen2/3 */ |
135 | #define IMUCTR32(n) (0x0600 + (((n) - 32) * 16)) /* R-Car Gen3 only */ | |
136 | #define IMUCTR_TTSEL_MMU(n) ((n) << 4) /* R-Car Gen2/3 */ | |
137 | #define IMUCTR_FLUSH (1 << 1) /* R-Car Gen2/3 */ | |
138 | #define IMUCTR_MMUEN (1 << 0) /* R-Car Gen2/3 */ | |
d25a2a16 | 139 | |
ddbbddd7 | 140 | #define IMUASID(n) ((n) < 32 ? IMUASID0(n) : IMUASID32(n)) |
df9828aa YS |
141 | #define IMUASID0(n) (0x0308 + ((n) * 16)) /* R-Car Gen2/3 */ |
142 | #define IMUASID32(n) (0x0608 + (((n) - 32) * 16)) /* R-Car Gen3 only */ | |
d25a2a16 | 143 | |
fd5140e2 MD |
144 | /* ----------------------------------------------------------------------------- |
145 | * Root device handling | |
146 | */ | |
147 | ||
148 | static struct platform_driver ipmmu_driver; | |
149 | ||
150 | static bool ipmmu_is_root(struct ipmmu_vmsa_device *mmu) | |
151 | { | |
152 | return mmu->root == mmu; | |
153 | } | |
154 | ||
155 | static int __ipmmu_check_device(struct device *dev, void *data) | |
156 | { | |
157 | struct ipmmu_vmsa_device *mmu = dev_get_drvdata(dev); | |
158 | struct ipmmu_vmsa_device **rootp = data; | |
159 | ||
160 | if (ipmmu_is_root(mmu)) | |
161 | *rootp = mmu; | |
162 | ||
163 | return 0; | |
164 | } | |
165 | ||
166 | static struct ipmmu_vmsa_device *ipmmu_find_root(void) | |
167 | { | |
168 | struct ipmmu_vmsa_device *root = NULL; | |
169 | ||
170 | return driver_for_each_device(&ipmmu_driver.driver, NULL, &root, | |
171 | __ipmmu_check_device) == 0 ? root : NULL; | |
172 | } | |
173 | ||
d25a2a16 LP |
174 | /* ----------------------------------------------------------------------------- |
175 | * Read/Write Access | |
176 | */ | |
177 | ||
178 | static u32 ipmmu_read(struct ipmmu_vmsa_device *mmu, unsigned int offset) | |
179 | { | |
180 | return ioread32(mmu->base + offset); | |
181 | } | |
182 | ||
183 | static void ipmmu_write(struct ipmmu_vmsa_device *mmu, unsigned int offset, | |
184 | u32 data) | |
185 | { | |
186 | iowrite32(data, mmu->base + offset); | |
187 | } | |
188 | ||
16d9454f YS |
189 | static unsigned int ipmmu_ctx_reg(struct ipmmu_vmsa_device *mmu, |
190 | unsigned int context_id, unsigned int reg) | |
191 | { | |
7a62ced8 YS |
192 | unsigned int base = mmu->features->ctx_offset_base; |
193 | ||
194 | if (context_id > 7) | |
195 | base += 0x800 - 8 * 0x40; | |
196 | ||
197 | return base + context_id * mmu->features->ctx_offset_stride + reg; | |
16d9454f YS |
198 | } |
199 | ||
200 | static u32 ipmmu_ctx_read(struct ipmmu_vmsa_device *mmu, | |
201 | unsigned int context_id, unsigned int reg) | |
202 | { | |
203 | return ipmmu_read(mmu, ipmmu_ctx_reg(mmu, context_id, reg)); | |
204 | } | |
205 | ||
206 | static void ipmmu_ctx_write(struct ipmmu_vmsa_device *mmu, | |
207 | unsigned int context_id, unsigned int reg, u32 data) | |
208 | { | |
209 | ipmmu_write(mmu, ipmmu_ctx_reg(mmu, context_id, reg), data); | |
210 | } | |
211 | ||
d574893a MD |
212 | static u32 ipmmu_ctx_read_root(struct ipmmu_vmsa_domain *domain, |
213 | unsigned int reg) | |
d25a2a16 | 214 | { |
16d9454f | 215 | return ipmmu_ctx_read(domain->mmu->root, domain->context_id, reg); |
d25a2a16 LP |
216 | } |
217 | ||
d574893a MD |
218 | static void ipmmu_ctx_write_root(struct ipmmu_vmsa_domain *domain, |
219 | unsigned int reg, u32 data) | |
d25a2a16 | 220 | { |
16d9454f | 221 | ipmmu_ctx_write(domain->mmu->root, domain->context_id, reg, data); |
d25a2a16 LP |
222 | } |
223 | ||
d574893a MD |
224 | static void ipmmu_ctx_write_all(struct ipmmu_vmsa_domain *domain, |
225 | unsigned int reg, u32 data) | |
226 | { | |
227 | if (domain->mmu != domain->mmu->root) | |
16d9454f | 228 | ipmmu_ctx_write(domain->mmu, domain->context_id, reg, data); |
d574893a | 229 | |
16d9454f | 230 | ipmmu_ctx_write(domain->mmu->root, domain->context_id, reg, data); |
d574893a MD |
231 | } |
232 | ||
3667c997 YS |
233 | static u32 ipmmu_utlb_reg(struct ipmmu_vmsa_device *mmu, unsigned int reg) |
234 | { | |
1289f7f1 | 235 | return mmu->features->utlb_offset_base + reg; |
3667c997 YS |
236 | } |
237 | ||
238 | static void ipmmu_imuasid_write(struct ipmmu_vmsa_device *mmu, | |
239 | unsigned int utlb, u32 data) | |
240 | { | |
241 | ipmmu_write(mmu, ipmmu_utlb_reg(mmu, IMUASID(utlb)), data); | |
242 | } | |
d574893a | 243 | |
3667c997 YS |
244 | static void ipmmu_imuctr_write(struct ipmmu_vmsa_device *mmu, |
245 | unsigned int utlb, u32 data) | |
246 | { | |
247 | ipmmu_write(mmu, ipmmu_utlb_reg(mmu, IMUCTR(utlb)), data); | |
d574893a MD |
248 | } |
249 | ||
d25a2a16 LP |
250 | /* ----------------------------------------------------------------------------- |
251 | * TLB and microTLB Management | |
252 | */ | |
253 | ||
254 | /* Wait for any pending TLB invalidations to complete */ | |
255 | static void ipmmu_tlb_sync(struct ipmmu_vmsa_domain *domain) | |
256 | { | |
257 | unsigned int count = 0; | |
258 | ||
d574893a | 259 | while (ipmmu_ctx_read_root(domain, IMCTR) & IMCTR_FLUSH) { |
d25a2a16 LP |
260 | cpu_relax(); |
261 | if (++count == TLB_LOOP_TIMEOUT) { | |
262 | dev_err_ratelimited(domain->mmu->dev, | |
263 | "TLB sync timed out -- MMU may be deadlocked\n"); | |
264 | return; | |
265 | } | |
266 | udelay(1); | |
267 | } | |
268 | } | |
269 | ||
270 | static void ipmmu_tlb_invalidate(struct ipmmu_vmsa_domain *domain) | |
271 | { | |
272 | u32 reg; | |
273 | ||
d574893a | 274 | reg = ipmmu_ctx_read_root(domain, IMCTR); |
d25a2a16 | 275 | reg |= IMCTR_FLUSH; |
d574893a | 276 | ipmmu_ctx_write_all(domain, IMCTR, reg); |
d25a2a16 LP |
277 | |
278 | ipmmu_tlb_sync(domain); | |
279 | } | |
280 | ||
281 | /* | |
282 | * Enable MMU translation for the microTLB. | |
283 | */ | |
284 | static void ipmmu_utlb_enable(struct ipmmu_vmsa_domain *domain, | |
192d2045 | 285 | unsigned int utlb) |
d25a2a16 LP |
286 | { |
287 | struct ipmmu_vmsa_device *mmu = domain->mmu; | |
288 | ||
192d2045 LP |
289 | /* |
290 | * TODO: Reference-count the microTLB as several bus masters can be | |
291 | * connected to the same microTLB. | |
292 | */ | |
293 | ||
d25a2a16 | 294 | /* TODO: What should we set the ASID to ? */ |
3667c997 | 295 | ipmmu_imuasid_write(mmu, utlb, 0); |
d25a2a16 | 296 | /* TODO: Do we need to flush the microTLB ? */ |
3667c997 YS |
297 | ipmmu_imuctr_write(mmu, utlb, IMUCTR_TTSEL_MMU(domain->context_id) | |
298 | IMUCTR_FLUSH | IMUCTR_MMUEN); | |
da38e9ec | 299 | mmu->utlb_ctx[utlb] = domain->context_id; |
d25a2a16 LP |
300 | } |
301 | ||
302 | /* | |
303 | * Disable MMU translation for the microTLB. | |
304 | */ | |
305 | static void ipmmu_utlb_disable(struct ipmmu_vmsa_domain *domain, | |
192d2045 | 306 | unsigned int utlb) |
d25a2a16 LP |
307 | { |
308 | struct ipmmu_vmsa_device *mmu = domain->mmu; | |
309 | ||
3667c997 | 310 | ipmmu_imuctr_write(mmu, utlb, 0); |
da38e9ec | 311 | mmu->utlb_ctx[utlb] = IPMMU_CTX_INVALID; |
d25a2a16 LP |
312 | } |
313 | ||
f20ed39f | 314 | static void ipmmu_tlb_flush_all(void *cookie) |
d25a2a16 | 315 | { |
f20ed39f LP |
316 | struct ipmmu_vmsa_domain *domain = cookie; |
317 | ||
318 | ipmmu_tlb_invalidate(domain); | |
319 | } | |
320 | ||
05aed941 WD |
321 | static void ipmmu_tlb_flush(unsigned long iova, size_t size, |
322 | size_t granule, void *cookie) | |
f20ed39f | 323 | { |
05aed941 | 324 | ipmmu_tlb_flush_all(cookie); |
f20ed39f LP |
325 | } |
326 | ||
298f7889 | 327 | static const struct iommu_flush_ops ipmmu_flush_ops = { |
f20ed39f | 328 | .tlb_flush_all = ipmmu_tlb_flush_all, |
05aed941 | 329 | .tlb_flush_walk = ipmmu_tlb_flush, |
f20ed39f LP |
330 | }; |
331 | ||
d25a2a16 LP |
332 | /* ----------------------------------------------------------------------------- |
333 | * Domain/Context Management | |
334 | */ | |
335 | ||
dbb70692 MD |
336 | static int ipmmu_domain_allocate_context(struct ipmmu_vmsa_device *mmu, |
337 | struct ipmmu_vmsa_domain *domain) | |
338 | { | |
339 | unsigned long flags; | |
340 | int ret; | |
341 | ||
342 | spin_lock_irqsave(&mmu->lock, flags); | |
343 | ||
5fd16341 MD |
344 | ret = find_first_zero_bit(mmu->ctx, mmu->num_ctx); |
345 | if (ret != mmu->num_ctx) { | |
dbb70692 MD |
346 | mmu->domains[ret] = domain; |
347 | set_bit(ret, mmu->ctx); | |
5fd16341 MD |
348 | } else |
349 | ret = -EBUSY; | |
dbb70692 MD |
350 | |
351 | spin_unlock_irqrestore(&mmu->lock, flags); | |
352 | ||
353 | return ret; | |
354 | } | |
355 | ||
a175a67d OT |
356 | static void ipmmu_domain_free_context(struct ipmmu_vmsa_device *mmu, |
357 | unsigned int context_id) | |
358 | { | |
359 | unsigned long flags; | |
360 | ||
361 | spin_lock_irqsave(&mmu->lock, flags); | |
362 | ||
363 | clear_bit(context_id, mmu->ctx); | |
364 | mmu->domains[context_id] = NULL; | |
365 | ||
366 | spin_unlock_irqrestore(&mmu->lock, flags); | |
367 | } | |
368 | ||
892db541 | 369 | static void ipmmu_domain_setup_context(struct ipmmu_vmsa_domain *domain) |
d25a2a16 | 370 | { |
f64232ee | 371 | u64 ttbr; |
c295f504 | 372 | u32 tmp; |
a175a67d | 373 | |
d25a2a16 | 374 | /* TTBR0 */ |
d1e5f26f | 375 | ttbr = domain->cfg.arm_lpae_s1_cfg.ttbr; |
d574893a MD |
376 | ipmmu_ctx_write_root(domain, IMTTLBR0, ttbr); |
377 | ipmmu_ctx_write_root(domain, IMTTUBR0, ttbr >> 32); | |
d25a2a16 LP |
378 | |
379 | /* | |
380 | * TTBCR | |
3623002f HNP |
381 | * We use long descriptors and allocate the whole 32-bit VA space to |
382 | * TTBR0. | |
d25a2a16 | 383 | */ |
c295f504 MD |
384 | if (domain->mmu->features->twobit_imttbcr_sl0) |
385 | tmp = IMTTBCR_SL0_TWOBIT_LVL_1; | |
386 | else | |
387 | tmp = IMTTBCR_SL0_LVL_1; | |
388 | ||
3623002f HNP |
389 | if (domain->mmu->features->cache_snoop) |
390 | tmp |= IMTTBCR_SH0_INNER_SHAREABLE | IMTTBCR_ORGN0_WB_WA | | |
391 | IMTTBCR_IRGN0_WB_WA; | |
392 | ||
393 | ipmmu_ctx_write_root(domain, IMTTBCR, IMTTBCR_EAE | tmp); | |
d25a2a16 | 394 | |
f20ed39f | 395 | /* MAIR0 */ |
d574893a | 396 | ipmmu_ctx_write_root(domain, IMMAIR0, |
205577ab | 397 | domain->cfg.arm_lpae_s1_cfg.mair); |
d25a2a16 LP |
398 | |
399 | /* IMBUSCR */ | |
f5c85891 MD |
400 | if (domain->mmu->features->setup_imbuscr) |
401 | ipmmu_ctx_write_root(domain, IMBUSCR, | |
402 | ipmmu_ctx_read_root(domain, IMBUSCR) & | |
403 | ~(IMBUSCR_DVM | IMBUSCR_BUSSEL_MASK)); | |
d25a2a16 LP |
404 | |
405 | /* | |
406 | * IMSTR | |
407 | * Clear all interrupt flags. | |
408 | */ | |
d574893a | 409 | ipmmu_ctx_write_root(domain, IMSTR, ipmmu_ctx_read_root(domain, IMSTR)); |
d25a2a16 LP |
410 | |
411 | /* | |
412 | * IMCTR | |
413 | * Enable the MMU and interrupt generation. The long-descriptor | |
414 | * translation table format doesn't use TEX remapping. Don't enable AF | |
415 | * software management as we have no use for it. Flush the TLB as | |
416 | * required when modifying the context registers. | |
417 | */ | |
d574893a MD |
418 | ipmmu_ctx_write_all(domain, IMCTR, |
419 | IMCTR_INTEN | IMCTR_FLUSH | IMCTR_MMUEN); | |
892db541 GU |
420 | } |
421 | ||
422 | static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain) | |
423 | { | |
424 | int ret; | |
425 | ||
426 | /* | |
427 | * Allocate the page table operations. | |
428 | * | |
429 | * VMSA states in section B3.6.3 "Control of Secure or Non-secure memory | |
430 | * access, Long-descriptor format" that the NStable bit being set in a | |
431 | * table descriptor will result in the NStable and NS bits of all child | |
432 | * entries being ignored and considered as being set. The IPMMU seems | |
433 | * not to comply with this, as it generates a secure access page fault | |
434 | * if any of the NStable and NS bits isn't set when running in | |
435 | * non-secure mode. | |
436 | */ | |
437 | domain->cfg.quirks = IO_PGTABLE_QUIRK_ARM_NS; | |
438 | domain->cfg.pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K; | |
439 | domain->cfg.ias = 32; | |
440 | domain->cfg.oas = 40; | |
298f7889 | 441 | domain->cfg.tlb = &ipmmu_flush_ops; |
892db541 GU |
442 | domain->io_domain.geometry.aperture_end = DMA_BIT_MASK(32); |
443 | domain->io_domain.geometry.force_aperture = true; | |
444 | /* | |
445 | * TODO: Add support for coherent walk through CCI with DVM and remove | |
446 | * cache handling. For now, delegate it to the io-pgtable code. | |
447 | */ | |
3430abd6 | 448 | domain->cfg.coherent_walk = false; |
892db541 GU |
449 | domain->cfg.iommu_dev = domain->mmu->root->dev; |
450 | ||
451 | /* | |
452 | * Find an unused context. | |
453 | */ | |
454 | ret = ipmmu_domain_allocate_context(domain->mmu->root, domain); | |
455 | if (ret < 0) | |
456 | return ret; | |
457 | ||
458 | domain->context_id = ret; | |
459 | ||
460 | domain->iop = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &domain->cfg, | |
461 | domain); | |
462 | if (!domain->iop) { | |
463 | ipmmu_domain_free_context(domain->mmu->root, | |
464 | domain->context_id); | |
465 | return -EINVAL; | |
466 | } | |
d25a2a16 | 467 | |
892db541 | 468 | ipmmu_domain_setup_context(domain); |
d25a2a16 LP |
469 | return 0; |
470 | } | |
471 | ||
472 | static void ipmmu_domain_destroy_context(struct ipmmu_vmsa_domain *domain) | |
473 | { | |
e5b78f2e GU |
474 | if (!domain->mmu) |
475 | return; | |
476 | ||
d25a2a16 LP |
477 | /* |
478 | * Disable the context. Flush the TLB as required when modifying the | |
479 | * context registers. | |
480 | * | |
481 | * TODO: Is TLB flush really needed ? | |
482 | */ | |
d574893a | 483 | ipmmu_ctx_write_all(domain, IMCTR, IMCTR_FLUSH); |
d25a2a16 | 484 | ipmmu_tlb_sync(domain); |
fd5140e2 | 485 | ipmmu_domain_free_context(domain->mmu->root, domain->context_id); |
d25a2a16 LP |
486 | } |
487 | ||
488 | /* ----------------------------------------------------------------------------- | |
489 | * Fault Handling | |
490 | */ | |
491 | ||
492 | static irqreturn_t ipmmu_domain_irq(struct ipmmu_vmsa_domain *domain) | |
493 | { | |
494 | const u32 err_mask = IMSTR_MHIT | IMSTR_ABORT | IMSTR_PF | IMSTR_TF; | |
495 | struct ipmmu_vmsa_device *mmu = domain->mmu; | |
82576aa8 | 496 | unsigned long iova; |
d25a2a16 | 497 | u32 status; |
d25a2a16 | 498 | |
d574893a | 499 | status = ipmmu_ctx_read_root(domain, IMSTR); |
d25a2a16 LP |
500 | if (!(status & err_mask)) |
501 | return IRQ_NONE; | |
502 | ||
82576aa8 GU |
503 | iova = ipmmu_ctx_read_root(domain, IMELAR); |
504 | if (IS_ENABLED(CONFIG_64BIT)) | |
505 | iova |= (u64)ipmmu_ctx_read_root(domain, IMEUAR) << 32; | |
d25a2a16 LP |
506 | |
507 | /* | |
508 | * Clear the error status flags. Unlike traditional interrupt flag | |
509 | * registers that must be cleared by writing 1, this status register | |
510 | * seems to require 0. The error address register must be read before, | |
511 | * otherwise its value will be 0. | |
512 | */ | |
d574893a | 513 | ipmmu_ctx_write_root(domain, IMSTR, 0); |
d25a2a16 LP |
514 | |
515 | /* Log fatal errors. */ | |
516 | if (status & IMSTR_MHIT) | |
82576aa8 | 517 | dev_err_ratelimited(mmu->dev, "Multiple TLB hits @0x%lx\n", |
d25a2a16 LP |
518 | iova); |
519 | if (status & IMSTR_ABORT) | |
82576aa8 | 520 | dev_err_ratelimited(mmu->dev, "Page Table Walk Abort @0x%lx\n", |
d25a2a16 LP |
521 | iova); |
522 | ||
523 | if (!(status & (IMSTR_PF | IMSTR_TF))) | |
524 | return IRQ_NONE; | |
525 | ||
526 | /* | |
527 | * Try to handle page faults and translation faults. | |
528 | * | |
529 | * TODO: We need to look up the faulty device based on the I/O VA. Use | |
530 | * the IOMMU device for now. | |
531 | */ | |
5914c5fd | 532 | if (!report_iommu_fault(&domain->io_domain, mmu->dev, iova, 0)) |
d25a2a16 LP |
533 | return IRQ_HANDLED; |
534 | ||
535 | dev_err_ratelimited(mmu->dev, | |
82576aa8 | 536 | "Unhandled fault: status 0x%08x iova 0x%lx\n", |
d25a2a16 LP |
537 | status, iova); |
538 | ||
539 | return IRQ_HANDLED; | |
540 | } | |
541 | ||
542 | static irqreturn_t ipmmu_irq(int irq, void *dev) | |
543 | { | |
544 | struct ipmmu_vmsa_device *mmu = dev; | |
dbb70692 MD |
545 | irqreturn_t status = IRQ_NONE; |
546 | unsigned int i; | |
547 | unsigned long flags; | |
d25a2a16 | 548 | |
dbb70692 MD |
549 | spin_lock_irqsave(&mmu->lock, flags); |
550 | ||
551 | /* | |
552 | * Check interrupts for all active contexts. | |
553 | */ | |
5fd16341 | 554 | for (i = 0; i < mmu->num_ctx; i++) { |
dbb70692 MD |
555 | if (!mmu->domains[i]) |
556 | continue; | |
557 | if (ipmmu_domain_irq(mmu->domains[i]) == IRQ_HANDLED) | |
558 | status = IRQ_HANDLED; | |
559 | } | |
d25a2a16 | 560 | |
dbb70692 | 561 | spin_unlock_irqrestore(&mmu->lock, flags); |
d25a2a16 | 562 | |
dbb70692 | 563 | return status; |
d25a2a16 LP |
564 | } |
565 | ||
d25a2a16 LP |
566 | /* ----------------------------------------------------------------------------- |
567 | * IOMMU Operations | |
568 | */ | |
569 | ||
5d894182 | 570 | static struct iommu_domain *ipmmu_domain_alloc(unsigned type) |
d25a2a16 LP |
571 | { |
572 | struct ipmmu_vmsa_domain *domain; | |
573 | ||
5d894182 RM |
574 | if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA) |
575 | return NULL; | |
576 | ||
d25a2a16 LP |
577 | domain = kzalloc(sizeof(*domain), GFP_KERNEL); |
578 | if (!domain) | |
5914c5fd | 579 | return NULL; |
d25a2a16 | 580 | |
46583e8c | 581 | mutex_init(&domain->mutex); |
d25a2a16 | 582 | |
5914c5fd | 583 | return &domain->io_domain; |
d25a2a16 LP |
584 | } |
585 | ||
5914c5fd | 586 | static void ipmmu_domain_free(struct iommu_domain *io_domain) |
d25a2a16 | 587 | { |
5914c5fd | 588 | struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); |
d25a2a16 LP |
589 | |
590 | /* | |
591 | * Free the domain resources. We assume that all devices have already | |
592 | * been detached. | |
593 | */ | |
594 | ipmmu_domain_destroy_context(domain); | |
f20ed39f | 595 | free_io_pgtable_ops(domain->iop); |
d25a2a16 LP |
596 | kfree(domain); |
597 | } | |
598 | ||
599 | static int ipmmu_attach_device(struct iommu_domain *io_domain, | |
600 | struct device *dev) | |
601 | { | |
df903655 | 602 | struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); |
e4efe4a9 | 603 | struct ipmmu_vmsa_device *mmu = to_ipmmu(dev); |
5914c5fd | 604 | struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); |
a166d31e | 605 | unsigned int i; |
d25a2a16 LP |
606 | int ret = 0; |
607 | ||
e4efe4a9 | 608 | if (!mmu) { |
d25a2a16 LP |
609 | dev_err(dev, "Cannot attach to IPMMU\n"); |
610 | return -ENXIO; | |
611 | } | |
612 | ||
46583e8c | 613 | mutex_lock(&domain->mutex); |
d25a2a16 LP |
614 | |
615 | if (!domain->mmu) { | |
616 | /* The domain hasn't been used yet, initialize it. */ | |
617 | domain->mmu = mmu; | |
618 | ret = ipmmu_domain_init_context(domain); | |
5fd16341 MD |
619 | if (ret < 0) { |
620 | dev_err(dev, "Unable to initialize IPMMU context\n"); | |
621 | domain->mmu = NULL; | |
622 | } else { | |
623 | dev_info(dev, "Using IPMMU context %u\n", | |
624 | domain->context_id); | |
625 | } | |
d25a2a16 LP |
626 | } else if (domain->mmu != mmu) { |
627 | /* | |
628 | * Something is wrong, we can't attach two devices using | |
629 | * different IOMMUs to the same domain. | |
630 | */ | |
631 | dev_err(dev, "Can't attach IPMMU %s to domain on IPMMU %s\n", | |
632 | dev_name(mmu->dev), dev_name(domain->mmu->dev)); | |
633 | ret = -EINVAL; | |
3ae47292 MD |
634 | } else |
635 | dev_info(dev, "Reusing IPMMU context %u\n", domain->context_id); | |
d25a2a16 | 636 | |
46583e8c | 637 | mutex_unlock(&domain->mutex); |
d25a2a16 LP |
638 | |
639 | if (ret < 0) | |
640 | return ret; | |
641 | ||
7b2d5961 MD |
642 | for (i = 0; i < fwspec->num_ids; ++i) |
643 | ipmmu_utlb_enable(domain, fwspec->ids[i]); | |
d25a2a16 LP |
644 | |
645 | return 0; | |
646 | } | |
647 | ||
648 | static void ipmmu_detach_device(struct iommu_domain *io_domain, | |
649 | struct device *dev) | |
650 | { | |
df903655 | 651 | struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); |
5914c5fd | 652 | struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); |
a166d31e | 653 | unsigned int i; |
d25a2a16 | 654 | |
7b2d5961 MD |
655 | for (i = 0; i < fwspec->num_ids; ++i) |
656 | ipmmu_utlb_disable(domain, fwspec->ids[i]); | |
d25a2a16 LP |
657 | |
658 | /* | |
659 | * TODO: Optimize by disabling the context when no device is attached. | |
660 | */ | |
661 | } | |
662 | ||
663 | static int ipmmu_map(struct iommu_domain *io_domain, unsigned long iova, | |
781ca2de | 664 | phys_addr_t paddr, size_t size, int prot, gfp_t gfp) |
d25a2a16 | 665 | { |
5914c5fd | 666 | struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); |
d25a2a16 LP |
667 | |
668 | if (!domain) | |
669 | return -ENODEV; | |
670 | ||
f34ce7a7 | 671 | return domain->iop->map(domain->iop, iova, paddr, size, prot, gfp); |
d25a2a16 LP |
672 | } |
673 | ||
674 | static size_t ipmmu_unmap(struct iommu_domain *io_domain, unsigned long iova, | |
56f8af5e | 675 | size_t size, struct iommu_iotlb_gather *gather) |
d25a2a16 | 676 | { |
5914c5fd | 677 | struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); |
d25a2a16 | 678 | |
a2d3a382 | 679 | return domain->iop->unmap(domain->iop, iova, size, gather); |
d25a2a16 LP |
680 | } |
681 | ||
56f8af5e | 682 | static void ipmmu_flush_iotlb_all(struct iommu_domain *io_domain) |
32b12449 RM |
683 | { |
684 | struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); | |
685 | ||
686 | if (domain->mmu) | |
687 | ipmmu_tlb_flush_all(domain); | |
688 | } | |
689 | ||
56f8af5e WD |
690 | static void ipmmu_iotlb_sync(struct iommu_domain *io_domain, |
691 | struct iommu_iotlb_gather *gather) | |
692 | { | |
693 | ipmmu_flush_iotlb_all(io_domain); | |
694 | } | |
695 | ||
d25a2a16 LP |
696 | static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain, |
697 | dma_addr_t iova) | |
698 | { | |
5914c5fd | 699 | struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); |
d25a2a16 LP |
700 | |
701 | /* TODO: Is locking needed ? */ | |
702 | ||
f20ed39f | 703 | return domain->iop->iova_to_phys(domain->iop, iova); |
d25a2a16 LP |
704 | } |
705 | ||
7b2d5961 MD |
706 | static int ipmmu_init_platform_device(struct device *dev, |
707 | struct of_phandle_args *args) | |
d25a2a16 | 708 | { |
7b2d5961 | 709 | struct platform_device *ipmmu_pdev; |
bb590c90 | 710 | |
7b2d5961 MD |
711 | ipmmu_pdev = of_find_device_by_node(args->np); |
712 | if (!ipmmu_pdev) | |
bb590c90 LP |
713 | return -ENODEV; |
714 | ||
be568d6d | 715 | dev_iommu_priv_set(dev, platform_get_drvdata(ipmmu_pdev)); |
383fef5f | 716 | |
383fef5f | 717 | return 0; |
58b8e8bf MD |
718 | } |
719 | ||
815cdd86 YS |
720 | static const struct soc_device_attribute soc_needs_opt_in[] = { |
721 | { .family = "R-Car Gen3", }, | |
ae684caf | 722 | { .family = "R-Car Gen4", }, |
815cdd86 | 723 | { .family = "RZ/G2", }, |
58b8e8bf MD |
724 | { /* sentinel */ } |
725 | }; | |
726 | ||
815cdd86 YS |
727 | static const struct soc_device_attribute soc_denylist[] = { |
728 | { .soc_id = "r8a774a1", }, | |
729 | { .soc_id = "r8a7795", .revision = "ES1.*" }, | |
730 | { .soc_id = "r8a7795", .revision = "ES2.*" }, | |
731 | { .soc_id = "r8a7796", }, | |
58b8e8bf MD |
732 | { /* sentinel */ } |
733 | }; | |
734 | ||
815cdd86 | 735 | static const char * const devices_allowlist[] = { |
cec0813d YS |
736 | "ee100000.mmc", |
737 | "ee120000.mmc", | |
738 | "ee140000.mmc", | |
739 | "ee160000.mmc" | |
80759649 YS |
740 | }; |
741 | ||
815cdd86 | 742 | static bool ipmmu_device_is_allowed(struct device *dev) |
b7ee92c6 | 743 | { |
80759649 YS |
744 | unsigned int i; |
745 | ||
b7ee92c6 | 746 | /* |
ae684caf | 747 | * R-Car Gen3/4 and RZ/G2 use the allow list to opt-in devices. |
b7ee92c6 YS |
748 | * For Other SoCs, this returns true anyway. |
749 | */ | |
815cdd86 | 750 | if (!soc_device_match(soc_needs_opt_in)) |
b7ee92c6 YS |
751 | return true; |
752 | ||
815cdd86 YS |
753 | /* Check whether this SoC can use the IPMMU correctly or not */ |
754 | if (soc_device_match(soc_denylist)) | |
b7ee92c6 YS |
755 | return false; |
756 | ||
815cdd86 YS |
757 | /* Check whether this device can work with the IPMMU */ |
758 | for (i = 0; i < ARRAY_SIZE(devices_allowlist); i++) { | |
759 | if (!strcmp(dev_name(dev), devices_allowlist[i])) | |
80759649 YS |
760 | return true; |
761 | } | |
762 | ||
763 | /* Otherwise, do not allow use of IPMMU */ | |
b7ee92c6 YS |
764 | return false; |
765 | } | |
766 | ||
49558da0 MD |
767 | static int ipmmu_of_xlate(struct device *dev, |
768 | struct of_phandle_args *spec) | |
769 | { | |
815cdd86 | 770 | if (!ipmmu_device_is_allowed(dev)) |
58b8e8bf MD |
771 | return -ENODEV; |
772 | ||
7b2d5961 MD |
773 | iommu_fwspec_add_ids(dev, spec->args, 1); |
774 | ||
49558da0 | 775 | /* Initialize once - xlate() will call multiple times */ |
e4efe4a9 | 776 | if (to_ipmmu(dev)) |
49558da0 MD |
777 | return 0; |
778 | ||
7b2d5961 | 779 | return ipmmu_init_platform_device(dev, spec); |
49558da0 MD |
780 | } |
781 | ||
49c875f0 | 782 | static int ipmmu_init_arm_mapping(struct device *dev) |
383fef5f | 783 | { |
e4efe4a9 | 784 | struct ipmmu_vmsa_device *mmu = to_ipmmu(dev); |
383fef5f MD |
785 | int ret; |
786 | ||
d25a2a16 LP |
787 | /* |
788 | * Create the ARM mapping, used by the ARM DMA mapping core to allocate | |
789 | * VAs. This will allocate a corresponding IOMMU domain. | |
790 | * | |
791 | * TODO: | |
792 | * - Create one mapping per context (TLB). | |
793 | * - Make the mapping size configurable ? We currently use a 2GB mapping | |
794 | * at a 1GB offset to ensure that NULL VAs will fault. | |
795 | */ | |
796 | if (!mmu->mapping) { | |
797 | struct dma_iommu_mapping *mapping; | |
798 | ||
799 | mapping = arm_iommu_create_mapping(&platform_bus_type, | |
720b0cef | 800 | SZ_1G, SZ_2G); |
d25a2a16 LP |
801 | if (IS_ERR(mapping)) { |
802 | dev_err(mmu->dev, "failed to create ARM IOMMU mapping\n"); | |
b8f80bff LP |
803 | ret = PTR_ERR(mapping); |
804 | goto error; | |
d25a2a16 LP |
805 | } |
806 | ||
807 | mmu->mapping = mapping; | |
808 | } | |
809 | ||
810 | /* Attach the ARM VA mapping to the device. */ | |
811 | ret = arm_iommu_attach_device(dev, mmu->mapping); | |
812 | if (ret < 0) { | |
813 | dev_err(dev, "Failed to attach device to VA mapping\n"); | |
814 | goto error; | |
815 | } | |
816 | ||
817 | return 0; | |
818 | ||
819 | error: | |
49c875f0 | 820 | if (mmu->mapping) |
383fef5f | 821 | arm_iommu_release_mapping(mmu->mapping); |
a166d31e | 822 | |
d25a2a16 LP |
823 | return ret; |
824 | } | |
825 | ||
6580c8a7 | 826 | static struct iommu_device *ipmmu_probe_device(struct device *dev) |
3ae47292 | 827 | { |
80eaa9f5 | 828 | struct ipmmu_vmsa_device *mmu = to_ipmmu(dev); |
3ae47292 | 829 | |
0fbc8b04 MD |
830 | /* |
831 | * Only let through devices that have been verified in xlate() | |
0fbc8b04 | 832 | */ |
80eaa9f5 | 833 | if (!mmu) |
6580c8a7 | 834 | return ERR_PTR(-ENODEV); |
3ae47292 | 835 | |
6580c8a7 JR |
836 | return &mmu->iommu; |
837 | } | |
838 | ||
839 | static void ipmmu_probe_finalize(struct device *dev) | |
840 | { | |
841 | int ret = 0; | |
842 | ||
843 | if (IS_ENABLED(CONFIG_ARM) && !IS_ENABLED(CONFIG_IOMMU_DMA)) | |
80eaa9f5 | 844 | ret = ipmmu_init_arm_mapping(dev); |
3ae47292 | 845 | |
6580c8a7 JR |
846 | if (ret) |
847 | dev_err(dev, "Can't create IOMMU mapping - DMA-OPS will not work\n"); | |
3ae47292 MD |
848 | } |
849 | ||
6580c8a7 | 850 | static void ipmmu_release_device(struct device *dev) |
3ae47292 | 851 | { |
49c875f0 | 852 | arm_iommu_detach_device(dev); |
3ae47292 MD |
853 | } |
854 | ||
b354c73e | 855 | static struct iommu_group *ipmmu_find_group(struct device *dev) |
3ae47292 | 856 | { |
e4efe4a9 | 857 | struct ipmmu_vmsa_device *mmu = to_ipmmu(dev); |
3ae47292 | 858 | struct iommu_group *group; |
3ae47292 | 859 | |
e4efe4a9 RM |
860 | if (mmu->group) |
861 | return iommu_group_ref_get(mmu->group); | |
b354c73e RM |
862 | |
863 | group = iommu_group_alloc(); | |
864 | if (!IS_ERR(group)) | |
e4efe4a9 | 865 | mmu->group = group; |
3ae47292 MD |
866 | |
867 | return group; | |
868 | } | |
869 | ||
3ae47292 | 870 | static const struct iommu_ops ipmmu_ops = { |
1c7e7c02 | 871 | .domain_alloc = ipmmu_domain_alloc, |
6580c8a7 JR |
872 | .probe_device = ipmmu_probe_device, |
873 | .release_device = ipmmu_release_device, | |
874 | .probe_finalize = ipmmu_probe_finalize, | |
2ba20b5a AB |
875 | .device_group = IS_ENABLED(CONFIG_ARM) && !IS_ENABLED(CONFIG_IOMMU_DMA) |
876 | ? generic_device_group : ipmmu_find_group, | |
3ae47292 | 877 | .pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K, |
49558da0 | 878 | .of_xlate = ipmmu_of_xlate, |
9a630a4b LB |
879 | .default_domain_ops = &(const struct iommu_domain_ops) { |
880 | .attach_dev = ipmmu_attach_device, | |
881 | .detach_dev = ipmmu_detach_device, | |
882 | .map = ipmmu_map, | |
883 | .unmap = ipmmu_unmap, | |
884 | .flush_iotlb_all = ipmmu_flush_iotlb_all, | |
885 | .iotlb_sync = ipmmu_iotlb_sync, | |
886 | .iova_to_phys = ipmmu_iova_to_phys, | |
887 | .free = ipmmu_domain_free, | |
888 | } | |
3ae47292 MD |
889 | }; |
890 | ||
d25a2a16 LP |
891 | /* ----------------------------------------------------------------------------- |
892 | * Probe/remove and init | |
893 | */ | |
894 | ||
895 | static void ipmmu_device_reset(struct ipmmu_vmsa_device *mmu) | |
896 | { | |
897 | unsigned int i; | |
898 | ||
899 | /* Disable all contexts. */ | |
5fd16341 | 900 | for (i = 0; i < mmu->num_ctx; ++i) |
16d9454f | 901 | ipmmu_ctx_write(mmu, i, IMCTR, 0); |
d25a2a16 LP |
902 | } |
903 | ||
33f3ac9b MD |
904 | static const struct ipmmu_features ipmmu_features_default = { |
905 | .use_ns_alias_offset = true, | |
fd5140e2 | 906 | .has_cache_leaf_nodes = false, |
5fd16341 | 907 | .number_of_contexts = 1, /* software only tested with one context */ |
b7f3f047 | 908 | .num_utlbs = 32, |
f5c85891 | 909 | .setup_imbuscr = true, |
c295f504 | 910 | .twobit_imttbcr_sl0 = false, |
2ae86955 | 911 | .reserved_context = false, |
3623002f | 912 | .cache_snoop = true, |
3dc28d9f YS |
913 | .ctx_offset_base = 0, |
914 | .ctx_offset_stride = 0x40, | |
1289f7f1 | 915 | .utlb_offset_base = 0, |
33f3ac9b MD |
916 | }; |
917 | ||
0b8ac140 | 918 | static const struct ipmmu_features ipmmu_features_rcar_gen3 = { |
58b8e8bf MD |
919 | .use_ns_alias_offset = false, |
920 | .has_cache_leaf_nodes = true, | |
921 | .number_of_contexts = 8, | |
b7f3f047 | 922 | .num_utlbs = 48, |
58b8e8bf MD |
923 | .setup_imbuscr = false, |
924 | .twobit_imttbcr_sl0 = true, | |
2ae86955 | 925 | .reserved_context = true, |
3623002f | 926 | .cache_snoop = false, |
3dc28d9f YS |
927 | .ctx_offset_base = 0, |
928 | .ctx_offset_stride = 0x40, | |
1289f7f1 | 929 | .utlb_offset_base = 0, |
58b8e8bf MD |
930 | }; |
931 | ||
ae684caf | 932 | static const struct ipmmu_features ipmmu_features_rcar_gen4 = { |
7a62ced8 YS |
933 | .use_ns_alias_offset = false, |
934 | .has_cache_leaf_nodes = true, | |
935 | .number_of_contexts = 16, | |
936 | .num_utlbs = 64, | |
937 | .setup_imbuscr = false, | |
938 | .twobit_imttbcr_sl0 = true, | |
939 | .reserved_context = true, | |
940 | .cache_snoop = false, | |
941 | .ctx_offset_base = 0x10000, | |
942 | .ctx_offset_stride = 0x1040, | |
943 | .utlb_offset_base = 0x3000, | |
944 | }; | |
945 | ||
33f3ac9b MD |
946 | static const struct of_device_id ipmmu_of_ids[] = { |
947 | { | |
948 | .compatible = "renesas,ipmmu-vmsa", | |
949 | .data = &ipmmu_features_default, | |
60fb0083 FC |
950 | }, { |
951 | .compatible = "renesas,ipmmu-r8a774a1", | |
952 | .data = &ipmmu_features_rcar_gen3, | |
757f26a3 BD |
953 | }, { |
954 | .compatible = "renesas,ipmmu-r8a774b1", | |
955 | .data = &ipmmu_features_rcar_gen3, | |
b6d39cd8 FC |
956 | }, { |
957 | .compatible = "renesas,ipmmu-r8a774c0", | |
958 | .data = &ipmmu_features_rcar_gen3, | |
4b2aa7a6 MCR |
959 | }, { |
960 | .compatible = "renesas,ipmmu-r8a774e1", | |
961 | .data = &ipmmu_features_rcar_gen3, | |
58b8e8bf MD |
962 | }, { |
963 | .compatible = "renesas,ipmmu-r8a7795", | |
0b8ac140 MD |
964 | .data = &ipmmu_features_rcar_gen3, |
965 | }, { | |
966 | .compatible = "renesas,ipmmu-r8a7796", | |
967 | .data = &ipmmu_features_rcar_gen3, | |
17fe1618 YS |
968 | }, { |
969 | .compatible = "renesas,ipmmu-r8a77961", | |
970 | .data = &ipmmu_features_rcar_gen3, | |
98dbffd3 JM |
971 | }, { |
972 | .compatible = "renesas,ipmmu-r8a77965", | |
973 | .data = &ipmmu_features_rcar_gen3, | |
3701c123 SH |
974 | }, { |
975 | .compatible = "renesas,ipmmu-r8a77970", | |
976 | .data = &ipmmu_features_rcar_gen3, | |
1cdeb52e NY |
977 | }, { |
978 | .compatible = "renesas,ipmmu-r8a77980", | |
979 | .data = &ipmmu_features_rcar_gen3, | |
b0c32912 HNP |
980 | }, { |
981 | .compatible = "renesas,ipmmu-r8a77990", | |
982 | .data = &ipmmu_features_rcar_gen3, | |
3701c123 SH |
983 | }, { |
984 | .compatible = "renesas,ipmmu-r8a77995", | |
985 | .data = &ipmmu_features_rcar_gen3, | |
7a62ced8 YS |
986 | }, { |
987 | .compatible = "renesas,ipmmu-r8a779a0", | |
ae684caf YS |
988 | .data = &ipmmu_features_rcar_gen4, |
989 | }, { | |
9f7d09fe | 990 | .compatible = "renesas,rcar-gen4-ipmmu-vmsa", |
ae684caf | 991 | .data = &ipmmu_features_rcar_gen4, |
33f3ac9b MD |
992 | }, { |
993 | /* Terminator */ | |
994 | }, | |
995 | }; | |
996 | ||
d25a2a16 LP |
997 | static int ipmmu_probe(struct platform_device *pdev) |
998 | { | |
999 | struct ipmmu_vmsa_device *mmu; | |
1000 | struct resource *res; | |
1001 | int irq; | |
1002 | int ret; | |
1003 | ||
d25a2a16 LP |
1004 | mmu = devm_kzalloc(&pdev->dev, sizeof(*mmu), GFP_KERNEL); |
1005 | if (!mmu) { | |
1006 | dev_err(&pdev->dev, "cannot allocate device data\n"); | |
1007 | return -ENOMEM; | |
1008 | } | |
1009 | ||
1010 | mmu->dev = &pdev->dev; | |
dbb70692 MD |
1011 | spin_lock_init(&mmu->lock); |
1012 | bitmap_zero(mmu->ctx, IPMMU_CTX_MAX); | |
33f3ac9b | 1013 | mmu->features = of_device_get_match_data(&pdev->dev); |
da38e9ec | 1014 | memset(mmu->utlb_ctx, IPMMU_CTX_INVALID, mmu->features->num_utlbs); |
1fdbbfd5 JJ |
1015 | ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)); |
1016 | if (ret) | |
1017 | return ret; | |
d25a2a16 LP |
1018 | |
1019 | /* Map I/O memory and request IRQ. */ | |
1020 | res = platform_get_resource(pdev, IORESOURCE_MEM, 0); | |
1021 | mmu->base = devm_ioremap_resource(&pdev->dev, res); | |
1022 | if (IS_ERR(mmu->base)) | |
1023 | return PTR_ERR(mmu->base); | |
1024 | ||
275f5053 LP |
1025 | /* |
1026 | * The IPMMU has two register banks, for secure and non-secure modes. | |
1027 | * The bank mapped at the beginning of the IPMMU address space | |
1028 | * corresponds to the running mode of the CPU. When running in secure | |
1029 | * mode the non-secure register bank is also available at an offset. | |
1030 | * | |
1031 | * Secure mode operation isn't clearly documented and is thus currently | |
1032 | * not implemented in the driver. Furthermore, preliminary tests of | |
1033 | * non-secure operation with the main register bank were not successful. | |
1034 | * Offset the registers base unconditionally to point to the non-secure | |
1035 | * alias space for now. | |
1036 | */ | |
33f3ac9b MD |
1037 | if (mmu->features->use_ns_alias_offset) |
1038 | mmu->base += IM_NS_ALIAS_OFFSET; | |
275f5053 | 1039 | |
b43e0d8a | 1040 | mmu->num_ctx = min(IPMMU_CTX_MAX, mmu->features->number_of_contexts); |
5fd16341 | 1041 | |
fd5140e2 MD |
1042 | /* |
1043 | * Determine if this IPMMU instance is a root device by checking for | |
1044 | * the lack of has_cache_leaf_nodes flag or renesas,ipmmu-main property. | |
1045 | */ | |
1046 | if (!mmu->features->has_cache_leaf_nodes || | |
1047 | !of_find_property(pdev->dev.of_node, "renesas,ipmmu-main", NULL)) | |
1048 | mmu->root = mmu; | |
1049 | else | |
1050 | mmu->root = ipmmu_find_root(); | |
d25a2a16 | 1051 | |
fd5140e2 MD |
1052 | /* |
1053 | * Wait until the root device has been registered for sure. | |
1054 | */ | |
1055 | if (!mmu->root) | |
1056 | return -EPROBE_DEFER; | |
1057 | ||
1058 | /* Root devices have mandatory IRQs */ | |
1059 | if (ipmmu_is_root(mmu)) { | |
ec37d4e9 | 1060 | irq = platform_get_irq(pdev, 0); |
565d4542 | 1061 | if (irq < 0) |
fd5140e2 | 1062 | return irq; |
fd5140e2 MD |
1063 | |
1064 | ret = devm_request_irq(&pdev->dev, irq, ipmmu_irq, 0, | |
1065 | dev_name(&pdev->dev), mmu); | |
1066 | if (ret < 0) { | |
1067 | dev_err(&pdev->dev, "failed to request IRQ %d\n", irq); | |
1068 | return ret; | |
1069 | } | |
1070 | ||
1071 | ipmmu_device_reset(mmu); | |
2ae86955 YS |
1072 | |
1073 | if (mmu->features->reserved_context) { | |
1074 | dev_info(&pdev->dev, "IPMMU context 0 is reserved\n"); | |
1075 | set_bit(0, mmu->ctx); | |
1076 | } | |
fd5140e2 | 1077 | } |
d25a2a16 | 1078 | |
cda52fcd MD |
1079 | /* |
1080 | * Register the IPMMU to the IOMMU subsystem in the following cases: | |
1081 | * - R-Car Gen2 IPMMU (all devices registered) | |
1082 | * - R-Car Gen3 IPMMU (leaf devices only - skip root IPMMU-MM device) | |
1083 | */ | |
1084 | if (!mmu->features->has_cache_leaf_nodes || !ipmmu_is_root(mmu)) { | |
1085 | ret = iommu_device_sysfs_add(&mmu->iommu, &pdev->dev, NULL, | |
1086 | dev_name(&pdev->dev)); | |
1087 | if (ret) | |
1088 | return ret; | |
7af9a5fd | 1089 | |
2d471b20 | 1090 | ret = iommu_device_register(&mmu->iommu, &ipmmu_ops, &pdev->dev); |
cda52fcd MD |
1091 | if (ret) |
1092 | return ret; | |
1093 | ||
1094 | #if defined(CONFIG_IOMMU_DMA) | |
1095 | if (!iommu_present(&platform_bus_type)) | |
1096 | bus_set_iommu(&platform_bus_type, &ipmmu_ops); | |
1097 | #endif | |
1098 | } | |
01da21e5 | 1099 | |
d25a2a16 LP |
1100 | /* |
1101 | * We can't create the ARM mapping here as it requires the bus to have | |
1102 | * an IOMMU, which only happens when bus_set_iommu() is called in | |
1103 | * ipmmu_init() after the probe function returns. | |
1104 | */ | |
1105 | ||
d25a2a16 LP |
1106 | platform_set_drvdata(pdev, mmu); |
1107 | ||
1108 | return 0; | |
1109 | } | |
1110 | ||
1111 | static int ipmmu_remove(struct platform_device *pdev) | |
1112 | { | |
1113 | struct ipmmu_vmsa_device *mmu = platform_get_drvdata(pdev); | |
1114 | ||
7af9a5fd | 1115 | iommu_device_sysfs_remove(&mmu->iommu); |
01da21e5 MD |
1116 | iommu_device_unregister(&mmu->iommu); |
1117 | ||
d25a2a16 LP |
1118 | arm_iommu_release_mapping(mmu->mapping); |
1119 | ||
1120 | ipmmu_device_reset(mmu); | |
1121 | ||
1122 | return 0; | |
1123 | } | |
1124 | ||
da38e9ec GU |
1125 | #ifdef CONFIG_PM_SLEEP |
1126 | static int ipmmu_resume_noirq(struct device *dev) | |
1127 | { | |
1128 | struct ipmmu_vmsa_device *mmu = dev_get_drvdata(dev); | |
1129 | unsigned int i; | |
1130 | ||
1131 | /* Reset root MMU and restore contexts */ | |
1132 | if (ipmmu_is_root(mmu)) { | |
1133 | ipmmu_device_reset(mmu); | |
1134 | ||
1135 | for (i = 0; i < mmu->num_ctx; i++) { | |
1136 | if (!mmu->domains[i]) | |
1137 | continue; | |
1138 | ||
1139 | ipmmu_domain_setup_context(mmu->domains[i]); | |
1140 | } | |
1141 | } | |
1142 | ||
1143 | /* Re-enable active micro-TLBs */ | |
1144 | for (i = 0; i < mmu->features->num_utlbs; i++) { | |
1145 | if (mmu->utlb_ctx[i] == IPMMU_CTX_INVALID) | |
1146 | continue; | |
1147 | ||
1148 | ipmmu_utlb_enable(mmu->root->domains[mmu->utlb_ctx[i]], i); | |
1149 | } | |
1150 | ||
1151 | return 0; | |
1152 | } | |
1153 | ||
1154 | static const struct dev_pm_ops ipmmu_pm = { | |
1155 | SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(NULL, ipmmu_resume_noirq) | |
1156 | }; | |
1157 | #define DEV_PM_OPS &ipmmu_pm | |
1158 | #else | |
1159 | #define DEV_PM_OPS NULL | |
1160 | #endif /* CONFIG_PM_SLEEP */ | |
1161 | ||
d25a2a16 LP |
1162 | static struct platform_driver ipmmu_driver = { |
1163 | .driver = { | |
d25a2a16 | 1164 | .name = "ipmmu-vmsa", |
275f5053 | 1165 | .of_match_table = of_match_ptr(ipmmu_of_ids), |
da38e9ec | 1166 | .pm = DEV_PM_OPS, |
d25a2a16 LP |
1167 | }, |
1168 | .probe = ipmmu_probe, | |
1169 | .remove = ipmmu_remove, | |
1170 | }; | |
1171 | ||
1172 | static int __init ipmmu_init(void) | |
1173 | { | |
5c5c8741 | 1174 | struct device_node *np; |
cda52fcd | 1175 | static bool setup_done; |
d25a2a16 LP |
1176 | int ret; |
1177 | ||
cda52fcd MD |
1178 | if (setup_done) |
1179 | return 0; | |
1180 | ||
5c5c8741 DO |
1181 | np = of_find_matching_node(NULL, ipmmu_of_ids); |
1182 | if (!np) | |
1183 | return 0; | |
1184 | ||
1185 | of_node_put(np); | |
1186 | ||
d25a2a16 LP |
1187 | ret = platform_driver_register(&ipmmu_driver); |
1188 | if (ret < 0) | |
1189 | return ret; | |
1190 | ||
cda52fcd | 1191 | #if defined(CONFIG_ARM) && !defined(CONFIG_IOMMU_DMA) |
d25a2a16 LP |
1192 | if (!iommu_present(&platform_bus_type)) |
1193 | bus_set_iommu(&platform_bus_type, &ipmmu_ops); | |
cda52fcd | 1194 | #endif |
d25a2a16 | 1195 | |
cda52fcd | 1196 | setup_done = true; |
d25a2a16 LP |
1197 | return 0; |
1198 | } | |
d25a2a16 | 1199 | subsys_initcall(ipmmu_init); |