Commit | Line | Data |
---|---|---|
b17336c5 HZ |
1 | /* |
2 | * Copyright (c) 2015-2016 MediaTek Inc. | |
3 | * Author: Honghui Zhang <honghui.zhang@mediatek.com> | |
4 | * | |
5 | * Based on driver/iommu/mtk_iommu.c | |
6 | * | |
7 | * This program is free software; you can redistribute it and/or modify | |
8 | * it under the terms of the GNU General Public License version 2 as | |
9 | * published by the Free Software Foundation. | |
10 | * | |
11 | * This program is distributed in the hope that it will be useful, | |
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
14 | * GNU General Public License for more details. | |
15 | */ | |
16 | #include <linux/bootmem.h> | |
17 | #include <linux/bug.h> | |
18 | #include <linux/clk.h> | |
19 | #include <linux/component.h> | |
20 | #include <linux/device.h> | |
745b6e74 | 21 | #include <linux/dma-mapping.h> |
b17336c5 HZ |
22 | #include <linux/dma-iommu.h> |
23 | #include <linux/err.h> | |
24 | #include <linux/interrupt.h> | |
25 | #include <linux/io.h> | |
26 | #include <linux/iommu.h> | |
27 | #include <linux/iopoll.h> | |
28 | #include <linux/kmemleak.h> | |
29 | #include <linux/list.h> | |
30 | #include <linux/of_address.h> | |
31 | #include <linux/of_iommu.h> | |
32 | #include <linux/of_irq.h> | |
33 | #include <linux/of_platform.h> | |
34 | #include <linux/platform_device.h> | |
35 | #include <linux/slab.h> | |
36 | #include <linux/spinlock.h> | |
37 | #include <asm/barrier.h> | |
38 | #include <asm/dma-iommu.h> | |
39 | #include <linux/module.h> | |
40 | #include <dt-bindings/memory/mt2701-larb-port.h> | |
41 | #include <soc/mediatek/smi.h> | |
42 | #include "mtk_iommu.h" | |
43 | ||
44 | #define REG_MMU_PT_BASE_ADDR 0x000 | |
45 | ||
46 | #define F_ALL_INVLD 0x2 | |
47 | #define F_MMU_INV_RANGE 0x1 | |
48 | #define F_INVLD_EN0 BIT(0) | |
49 | #define F_INVLD_EN1 BIT(1) | |
50 | ||
51 | #define F_MMU_FAULT_VA_MSK 0xfffff000 | |
52 | #define MTK_PROTECT_PA_ALIGN 128 | |
53 | ||
54 | #define REG_MMU_CTRL_REG 0x210 | |
55 | #define F_MMU_CTRL_COHERENT_EN BIT(8) | |
56 | #define REG_MMU_IVRP_PADDR 0x214 | |
57 | #define REG_MMU_INT_CONTROL 0x220 | |
58 | #define F_INT_TRANSLATION_FAULT BIT(0) | |
59 | #define F_INT_MAIN_MULTI_HIT_FAULT BIT(1) | |
60 | #define F_INT_INVALID_PA_FAULT BIT(2) | |
61 | #define F_INT_ENTRY_REPLACEMENT_FAULT BIT(3) | |
62 | #define F_INT_TABLE_WALK_FAULT BIT(4) | |
63 | #define F_INT_TLB_MISS_FAULT BIT(5) | |
64 | #define F_INT_PFH_DMA_FIFO_OVERFLOW BIT(6) | |
65 | #define F_INT_MISS_DMA_FIFO_OVERFLOW BIT(7) | |
66 | ||
67 | #define F_MMU_TF_PROTECT_SEL(prot) (((prot) & 0x3) << 5) | |
68 | #define F_INT_CLR_BIT BIT(12) | |
69 | ||
70 | #define REG_MMU_FAULT_ST 0x224 | |
71 | #define REG_MMU_FAULT_VA 0x228 | |
72 | #define REG_MMU_INVLD_PA 0x22C | |
73 | #define REG_MMU_INT_ID 0x388 | |
74 | #define REG_MMU_INVALIDATE 0x5c0 | |
75 | #define REG_MMU_INVLD_START_A 0x5c4 | |
76 | #define REG_MMU_INVLD_END_A 0x5c8 | |
77 | ||
78 | #define REG_MMU_INV_SEL 0x5d8 | |
79 | #define REG_MMU_STANDARD_AXI_MODE 0x5e8 | |
80 | ||
81 | #define REG_MMU_DCM 0x5f0 | |
82 | #define F_MMU_DCM_ON BIT(1) | |
83 | #define REG_MMU_CPE_DONE 0x60c | |
84 | #define F_DESC_VALID 0x2 | |
85 | #define F_DESC_NONSEC BIT(3) | |
86 | #define MT2701_M4U_TF_LARB(TF) (6 - (((TF) >> 13) & 0x7)) | |
87 | #define MT2701_M4U_TF_PORT(TF) (((TF) >> 8) & 0xF) | |
88 | /* MTK generation one iommu HW only support 4K size mapping */ | |
89 | #define MT2701_IOMMU_PAGE_SHIFT 12 | |
90 | #define MT2701_IOMMU_PAGE_SIZE (1UL << MT2701_IOMMU_PAGE_SHIFT) | |
91 | ||
92 | /* | |
93 | * MTK m4u support 4GB iova address space, and only support 4K page | |
94 | * mapping. So the pagetable size should be exactly as 4M. | |
95 | */ | |
96 | #define M2701_IOMMU_PGT_SIZE SZ_4M | |
97 | ||
98 | struct mtk_iommu_domain { | |
99 | spinlock_t pgtlock; /* lock for page table */ | |
100 | struct iommu_domain domain; | |
101 | u32 *pgt_va; | |
102 | dma_addr_t pgt_pa; | |
103 | struct mtk_iommu_data *data; | |
104 | }; | |
105 | ||
106 | static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom) | |
107 | { | |
108 | return container_of(dom, struct mtk_iommu_domain, domain); | |
109 | } | |
110 | ||
111 | static const int mt2701_m4u_in_larb[] = { | |
112 | LARB0_PORT_OFFSET, LARB1_PORT_OFFSET, | |
113 | LARB2_PORT_OFFSET, LARB3_PORT_OFFSET | |
114 | }; | |
115 | ||
116 | static inline int mt2701_m4u_to_larb(int id) | |
117 | { | |
118 | int i; | |
119 | ||
120 | for (i = ARRAY_SIZE(mt2701_m4u_in_larb) - 1; i >= 0; i--) | |
121 | if ((id) >= mt2701_m4u_in_larb[i]) | |
122 | return i; | |
123 | ||
124 | return 0; | |
125 | } | |
126 | ||
127 | static inline int mt2701_m4u_to_port(int id) | |
128 | { | |
129 | int larb = mt2701_m4u_to_larb(id); | |
130 | ||
131 | return id - mt2701_m4u_in_larb[larb]; | |
132 | } | |
133 | ||
134 | static void mtk_iommu_tlb_flush_all(struct mtk_iommu_data *data) | |
135 | { | |
136 | writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, | |
137 | data->base + REG_MMU_INV_SEL); | |
138 | writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE); | |
139 | wmb(); /* Make sure the tlb flush all done */ | |
140 | } | |
141 | ||
142 | static void mtk_iommu_tlb_flush_range(struct mtk_iommu_data *data, | |
143 | unsigned long iova, size_t size) | |
144 | { | |
145 | int ret; | |
146 | u32 tmp; | |
147 | ||
148 | writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, | |
149 | data->base + REG_MMU_INV_SEL); | |
150 | writel_relaxed(iova & F_MMU_FAULT_VA_MSK, | |
151 | data->base + REG_MMU_INVLD_START_A); | |
152 | writel_relaxed((iova + size - 1) & F_MMU_FAULT_VA_MSK, | |
153 | data->base + REG_MMU_INVLD_END_A); | |
154 | writel_relaxed(F_MMU_INV_RANGE, data->base + REG_MMU_INVALIDATE); | |
155 | ||
156 | ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE, | |
157 | tmp, tmp != 0, 10, 100000); | |
158 | if (ret) { | |
159 | dev_warn(data->dev, | |
160 | "Partial TLB flush timed out, falling back to full flush\n"); | |
161 | mtk_iommu_tlb_flush_all(data); | |
162 | } | |
163 | /* Clear the CPE status */ | |
164 | writel_relaxed(0, data->base + REG_MMU_CPE_DONE); | |
165 | } | |
166 | ||
167 | static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) | |
168 | { | |
169 | struct mtk_iommu_data *data = dev_id; | |
170 | struct mtk_iommu_domain *dom = data->m4u_dom; | |
171 | u32 int_state, regval, fault_iova, fault_pa; | |
172 | unsigned int fault_larb, fault_port; | |
173 | ||
174 | /* Read error information from registers */ | |
175 | int_state = readl_relaxed(data->base + REG_MMU_FAULT_ST); | |
176 | fault_iova = readl_relaxed(data->base + REG_MMU_FAULT_VA); | |
177 | ||
178 | fault_iova &= F_MMU_FAULT_VA_MSK; | |
179 | fault_pa = readl_relaxed(data->base + REG_MMU_INVLD_PA); | |
180 | regval = readl_relaxed(data->base + REG_MMU_INT_ID); | |
181 | fault_larb = MT2701_M4U_TF_LARB(regval); | |
182 | fault_port = MT2701_M4U_TF_PORT(regval); | |
183 | ||
184 | /* | |
185 | * MTK v1 iommu HW could not determine whether the fault is read or | |
186 | * write fault, report as read fault. | |
187 | */ | |
188 | if (report_iommu_fault(&dom->domain, data->dev, fault_iova, | |
189 | IOMMU_FAULT_READ)) | |
190 | dev_err_ratelimited(data->dev, | |
191 | "fault type=0x%x iova=0x%x pa=0x%x larb=%d port=%d\n", | |
192 | int_state, fault_iova, fault_pa, | |
193 | fault_larb, fault_port); | |
194 | ||
195 | /* Interrupt clear */ | |
196 | regval = readl_relaxed(data->base + REG_MMU_INT_CONTROL); | |
197 | regval |= F_INT_CLR_BIT; | |
198 | writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL); | |
199 | ||
200 | mtk_iommu_tlb_flush_all(data); | |
201 | ||
202 | return IRQ_HANDLED; | |
203 | } | |
204 | ||
205 | static void mtk_iommu_config(struct mtk_iommu_data *data, | |
206 | struct device *dev, bool enable) | |
207 | { | |
b17336c5 HZ |
208 | struct mtk_smi_larb_iommu *larb_mmu; |
209 | unsigned int larbid, portid; | |
84672f19 RM |
210 | struct iommu_fwspec *fwspec = dev->iommu_fwspec; |
211 | int i; | |
b17336c5 | 212 | |
84672f19 RM |
213 | for (i = 0; i < fwspec->num_ids; ++i) { |
214 | larbid = mt2701_m4u_to_larb(fwspec->ids[i]); | |
215 | portid = mt2701_m4u_to_port(fwspec->ids[i]); | |
b17336c5 HZ |
216 | larb_mmu = &data->smi_imu.larb_imu[larbid]; |
217 | ||
218 | dev_dbg(dev, "%s iommu port: %d\n", | |
219 | enable ? "enable" : "disable", portid); | |
220 | ||
221 | if (enable) | |
222 | larb_mmu->mmu |= MTK_SMI_MMU_EN(portid); | |
223 | else | |
224 | larb_mmu->mmu &= ~MTK_SMI_MMU_EN(portid); | |
225 | } | |
226 | } | |
227 | ||
228 | static int mtk_iommu_domain_finalise(struct mtk_iommu_data *data) | |
229 | { | |
230 | struct mtk_iommu_domain *dom = data->m4u_dom; | |
231 | ||
232 | spin_lock_init(&dom->pgtlock); | |
233 | ||
234 | dom->pgt_va = dma_zalloc_coherent(data->dev, | |
235 | M2701_IOMMU_PGT_SIZE, | |
236 | &dom->pgt_pa, GFP_KERNEL); | |
237 | if (!dom->pgt_va) | |
238 | return -ENOMEM; | |
239 | ||
240 | writel(dom->pgt_pa, data->base + REG_MMU_PT_BASE_ADDR); | |
241 | ||
242 | dom->data = data; | |
243 | ||
244 | return 0; | |
245 | } | |
246 | ||
247 | static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type) | |
248 | { | |
249 | struct mtk_iommu_domain *dom; | |
250 | ||
251 | if (type != IOMMU_DOMAIN_UNMANAGED) | |
252 | return NULL; | |
253 | ||
254 | dom = kzalloc(sizeof(*dom), GFP_KERNEL); | |
255 | if (!dom) | |
256 | return NULL; | |
257 | ||
258 | return &dom->domain; | |
259 | } | |
260 | ||
261 | static void mtk_iommu_domain_free(struct iommu_domain *domain) | |
262 | { | |
263 | struct mtk_iommu_domain *dom = to_mtk_domain(domain); | |
264 | struct mtk_iommu_data *data = dom->data; | |
265 | ||
266 | dma_free_coherent(data->dev, M2701_IOMMU_PGT_SIZE, | |
267 | dom->pgt_va, dom->pgt_pa); | |
268 | kfree(to_mtk_domain(domain)); | |
269 | } | |
270 | ||
271 | static int mtk_iommu_attach_device(struct iommu_domain *domain, | |
272 | struct device *dev) | |
273 | { | |
274 | struct mtk_iommu_domain *dom = to_mtk_domain(domain); | |
84672f19 | 275 | struct mtk_iommu_data *data = dev->iommu_fwspec->iommu_priv; |
b17336c5 HZ |
276 | int ret; |
277 | ||
84672f19 | 278 | if (!data) |
b17336c5 HZ |
279 | return -ENODEV; |
280 | ||
b17336c5 HZ |
281 | if (!data->m4u_dom) { |
282 | data->m4u_dom = dom; | |
283 | ret = mtk_iommu_domain_finalise(data); | |
284 | if (ret) { | |
285 | data->m4u_dom = NULL; | |
286 | return ret; | |
287 | } | |
288 | } | |
289 | ||
290 | mtk_iommu_config(data, dev, true); | |
291 | return 0; | |
292 | } | |
293 | ||
294 | static void mtk_iommu_detach_device(struct iommu_domain *domain, | |
295 | struct device *dev) | |
296 | { | |
84672f19 | 297 | struct mtk_iommu_data *data = dev->iommu_fwspec->iommu_priv; |
b17336c5 | 298 | |
84672f19 | 299 | if (!data) |
b17336c5 HZ |
300 | return; |
301 | ||
b17336c5 HZ |
302 | mtk_iommu_config(data, dev, false); |
303 | } | |
304 | ||
305 | static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova, | |
306 | phys_addr_t paddr, size_t size, int prot) | |
307 | { | |
308 | struct mtk_iommu_domain *dom = to_mtk_domain(domain); | |
309 | unsigned int page_num = size >> MT2701_IOMMU_PAGE_SHIFT; | |
310 | unsigned long flags; | |
311 | unsigned int i; | |
312 | u32 *pgt_base_iova = dom->pgt_va + (iova >> MT2701_IOMMU_PAGE_SHIFT); | |
313 | u32 pabase = (u32)paddr; | |
314 | int map_size = 0; | |
315 | ||
316 | spin_lock_irqsave(&dom->pgtlock, flags); | |
317 | for (i = 0; i < page_num; i++) { | |
318 | if (pgt_base_iova[i]) { | |
319 | memset(pgt_base_iova, 0, i * sizeof(u32)); | |
320 | break; | |
321 | } | |
322 | pgt_base_iova[i] = pabase | F_DESC_VALID | F_DESC_NONSEC; | |
323 | pabase += MT2701_IOMMU_PAGE_SIZE; | |
324 | map_size += MT2701_IOMMU_PAGE_SIZE; | |
325 | } | |
326 | ||
327 | spin_unlock_irqrestore(&dom->pgtlock, flags); | |
328 | ||
329 | mtk_iommu_tlb_flush_range(dom->data, iova, size); | |
330 | ||
331 | return map_size == size ? 0 : -EEXIST; | |
332 | } | |
333 | ||
334 | static size_t mtk_iommu_unmap(struct iommu_domain *domain, | |
335 | unsigned long iova, size_t size) | |
336 | { | |
337 | struct mtk_iommu_domain *dom = to_mtk_domain(domain); | |
338 | unsigned long flags; | |
339 | u32 *pgt_base_iova = dom->pgt_va + (iova >> MT2701_IOMMU_PAGE_SHIFT); | |
340 | unsigned int page_num = size >> MT2701_IOMMU_PAGE_SHIFT; | |
341 | ||
342 | spin_lock_irqsave(&dom->pgtlock, flags); | |
343 | memset(pgt_base_iova, 0, page_num * sizeof(u32)); | |
344 | spin_unlock_irqrestore(&dom->pgtlock, flags); | |
345 | ||
346 | mtk_iommu_tlb_flush_range(dom->data, iova, size); | |
347 | ||
348 | return size; | |
349 | } | |
350 | ||
351 | static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, | |
352 | dma_addr_t iova) | |
353 | { | |
354 | struct mtk_iommu_domain *dom = to_mtk_domain(domain); | |
355 | unsigned long flags; | |
356 | phys_addr_t pa; | |
357 | ||
358 | spin_lock_irqsave(&dom->pgtlock, flags); | |
359 | pa = *(dom->pgt_va + (iova >> MT2701_IOMMU_PAGE_SHIFT)); | |
360 | pa = pa & (~(MT2701_IOMMU_PAGE_SIZE - 1)); | |
361 | spin_unlock_irqrestore(&dom->pgtlock, flags); | |
362 | ||
363 | return pa; | |
364 | } | |
365 | ||
84672f19 RM |
366 | static struct iommu_ops mtk_iommu_ops; |
367 | ||
b17336c5 HZ |
368 | /* |
369 | * MTK generation one iommu HW only support one iommu domain, and all the client | |
370 | * sharing the same iova address space. | |
371 | */ | |
372 | static int mtk_iommu_create_mapping(struct device *dev, | |
373 | struct of_phandle_args *args) | |
374 | { | |
84672f19 | 375 | struct mtk_iommu_data *data; |
b17336c5 HZ |
376 | struct platform_device *m4updev; |
377 | struct dma_iommu_mapping *mtk_mapping; | |
378 | struct device *m4udev; | |
379 | int ret; | |
380 | ||
381 | if (args->args_count != 1) { | |
382 | dev_err(dev, "invalid #iommu-cells(%d) property for IOMMU\n", | |
383 | args->args_count); | |
384 | return -EINVAL; | |
385 | } | |
386 | ||
84672f19 RM |
387 | if (!dev->iommu_fwspec) { |
388 | ret = iommu_fwspec_init(dev, &args->np->fwnode, &mtk_iommu_ops); | |
389 | if (ret) | |
390 | return ret; | |
391 | } else if (dev->iommu_fwspec->ops != &mtk_iommu_ops) { | |
392 | return -EINVAL; | |
393 | } | |
394 | ||
395 | if (!dev->iommu_fwspec->iommu_priv) { | |
b17336c5 HZ |
396 | /* Get the m4u device */ |
397 | m4updev = of_find_device_by_node(args->np); | |
398 | if (WARN_ON(!m4updev)) | |
399 | return -EINVAL; | |
400 | ||
84672f19 | 401 | dev->iommu_fwspec->iommu_priv = platform_get_drvdata(m4updev); |
b17336c5 HZ |
402 | } |
403 | ||
84672f19 RM |
404 | ret = iommu_fwspec_add_ids(dev, args->args, 1); |
405 | if (ret) | |
406 | return ret; | |
b17336c5 | 407 | |
84672f19 RM |
408 | data = dev->iommu_fwspec->iommu_priv; |
409 | m4udev = data->dev; | |
b17336c5 HZ |
410 | mtk_mapping = m4udev->archdata.iommu; |
411 | if (!mtk_mapping) { | |
412 | /* MTK iommu support 4GB iova address space. */ | |
413 | mtk_mapping = arm_iommu_create_mapping(&platform_bus_type, | |
414 | 0, 1ULL << 32); | |
84672f19 RM |
415 | if (IS_ERR(mtk_mapping)) |
416 | return PTR_ERR(mtk_mapping); | |
417 | ||
b17336c5 HZ |
418 | m4udev->archdata.iommu = mtk_mapping; |
419 | } | |
420 | ||
421 | ret = arm_iommu_attach_device(dev, mtk_mapping); | |
422 | if (ret) | |
423 | goto err_release_mapping; | |
424 | ||
425 | return 0; | |
426 | ||
427 | err_release_mapping: | |
428 | arm_iommu_release_mapping(mtk_mapping); | |
429 | m4udev->archdata.iommu = NULL; | |
b17336c5 HZ |
430 | return ret; |
431 | } | |
432 | ||
433 | static int mtk_iommu_add_device(struct device *dev) | |
434 | { | |
b17336c5 HZ |
435 | struct of_phandle_args iommu_spec; |
436 | struct of_phandle_iterator it; | |
6f66ea09 JR |
437 | struct mtk_iommu_data *data; |
438 | struct iommu_group *group; | |
b17336c5 HZ |
439 | int err; |
440 | ||
441 | of_for_each_phandle(&it, err, dev->of_node, "iommus", | |
442 | "#iommu-cells", 0) { | |
443 | int count = of_phandle_iterator_args(&it, iommu_spec.args, | |
444 | MAX_PHANDLE_ARGS); | |
445 | iommu_spec.np = of_node_get(it.node); | |
446 | iommu_spec.args_count = count; | |
447 | ||
448 | mtk_iommu_create_mapping(dev, &iommu_spec); | |
449 | of_node_put(iommu_spec.np); | |
450 | } | |
451 | ||
84672f19 RM |
452 | if (!dev->iommu_fwspec || dev->iommu_fwspec->ops != &mtk_iommu_ops) |
453 | return -ENODEV; /* Not a iommu client device */ | |
b17336c5 | 454 | |
6f66ea09 JR |
455 | data = dev->iommu_fwspec->iommu_priv; |
456 | iommu_device_link(&data->iommu, dev); | |
457 | ||
b17336c5 HZ |
458 | group = iommu_group_get_for_dev(dev); |
459 | if (IS_ERR(group)) | |
460 | return PTR_ERR(group); | |
461 | ||
462 | iommu_group_put(group); | |
463 | return 0; | |
464 | } | |
465 | ||
466 | static void mtk_iommu_remove_device(struct device *dev) | |
467 | { | |
6f66ea09 JR |
468 | struct mtk_iommu_data *data; |
469 | ||
84672f19 | 470 | if (!dev->iommu_fwspec || dev->iommu_fwspec->ops != &mtk_iommu_ops) |
b17336c5 HZ |
471 | return; |
472 | ||
6f66ea09 JR |
473 | data = dev->iommu_fwspec->iommu_priv; |
474 | iommu_device_unlink(&data->iommu, dev); | |
475 | ||
b17336c5 | 476 | iommu_group_remove_device(dev); |
84672f19 | 477 | iommu_fwspec_free(dev); |
b17336c5 HZ |
478 | } |
479 | ||
480 | static struct iommu_group *mtk_iommu_device_group(struct device *dev) | |
481 | { | |
84672f19 | 482 | struct mtk_iommu_data *data = dev->iommu_fwspec->iommu_priv; |
b17336c5 | 483 | |
84672f19 | 484 | if (!data) |
b17336c5 HZ |
485 | return ERR_PTR(-ENODEV); |
486 | ||
487 | /* All the client devices are in the same m4u iommu-group */ | |
b17336c5 HZ |
488 | if (!data->m4u_group) { |
489 | data->m4u_group = iommu_group_alloc(); | |
490 | if (IS_ERR(data->m4u_group)) | |
491 | dev_err(dev, "Failed to allocate M4U IOMMU group\n"); | |
5d1d43b0 RM |
492 | } else { |
493 | iommu_group_ref_get(data->m4u_group); | |
b17336c5 HZ |
494 | } |
495 | return data->m4u_group; | |
496 | } | |
497 | ||
498 | static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) | |
499 | { | |
500 | u32 regval; | |
501 | int ret; | |
502 | ||
503 | ret = clk_prepare_enable(data->bclk); | |
504 | if (ret) { | |
505 | dev_err(data->dev, "Failed to enable iommu bclk(%d)\n", ret); | |
506 | return ret; | |
507 | } | |
508 | ||
509 | regval = F_MMU_CTRL_COHERENT_EN | F_MMU_TF_PROTECT_SEL(2); | |
510 | writel_relaxed(regval, data->base + REG_MMU_CTRL_REG); | |
511 | ||
512 | regval = F_INT_TRANSLATION_FAULT | | |
513 | F_INT_MAIN_MULTI_HIT_FAULT | | |
514 | F_INT_INVALID_PA_FAULT | | |
515 | F_INT_ENTRY_REPLACEMENT_FAULT | | |
516 | F_INT_TABLE_WALK_FAULT | | |
517 | F_INT_TLB_MISS_FAULT | | |
518 | F_INT_PFH_DMA_FIFO_OVERFLOW | | |
519 | F_INT_MISS_DMA_FIFO_OVERFLOW; | |
520 | writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL); | |
521 | ||
522 | /* protect memory,hw will write here while translation fault */ | |
523 | writel_relaxed(data->protect_base, | |
524 | data->base + REG_MMU_IVRP_PADDR); | |
525 | ||
526 | writel_relaxed(F_MMU_DCM_ON, data->base + REG_MMU_DCM); | |
527 | ||
528 | if (devm_request_irq(data->dev, data->irq, mtk_iommu_isr, 0, | |
529 | dev_name(data->dev), (void *)data)) { | |
530 | writel_relaxed(0, data->base + REG_MMU_PT_BASE_ADDR); | |
531 | clk_disable_unprepare(data->bclk); | |
532 | dev_err(data->dev, "Failed @ IRQ-%d Request\n", data->irq); | |
533 | return -ENODEV; | |
534 | } | |
535 | ||
536 | return 0; | |
537 | } | |
538 | ||
539 | static struct iommu_ops mtk_iommu_ops = { | |
540 | .domain_alloc = mtk_iommu_domain_alloc, | |
541 | .domain_free = mtk_iommu_domain_free, | |
542 | .attach_dev = mtk_iommu_attach_device, | |
543 | .detach_dev = mtk_iommu_detach_device, | |
544 | .map = mtk_iommu_map, | |
545 | .unmap = mtk_iommu_unmap, | |
546 | .map_sg = default_iommu_map_sg, | |
547 | .iova_to_phys = mtk_iommu_iova_to_phys, | |
548 | .add_device = mtk_iommu_add_device, | |
549 | .remove_device = mtk_iommu_remove_device, | |
550 | .device_group = mtk_iommu_device_group, | |
551 | .pgsize_bitmap = ~0UL << MT2701_IOMMU_PAGE_SHIFT, | |
552 | }; | |
553 | ||
554 | static const struct of_device_id mtk_iommu_of_ids[] = { | |
555 | { .compatible = "mediatek,mt2701-m4u", }, | |
556 | {} | |
557 | }; | |
558 | ||
559 | static const struct component_master_ops mtk_iommu_com_ops = { | |
560 | .bind = mtk_iommu_bind, | |
561 | .unbind = mtk_iommu_unbind, | |
562 | }; | |
563 | ||
564 | static int mtk_iommu_probe(struct platform_device *pdev) | |
565 | { | |
566 | struct mtk_iommu_data *data; | |
567 | struct device *dev = &pdev->dev; | |
568 | struct resource *res; | |
569 | struct component_match *match = NULL; | |
570 | struct of_phandle_args larb_spec; | |
571 | struct of_phandle_iterator it; | |
572 | void *protect; | |
573 | int larb_nr, ret, err; | |
574 | ||
575 | data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); | |
576 | if (!data) | |
577 | return -ENOMEM; | |
578 | ||
579 | data->dev = dev; | |
580 | ||
581 | /* Protect memory. HW will access here while translation fault.*/ | |
582 | protect = devm_kzalloc(dev, MTK_PROTECT_PA_ALIGN * 2, | |
583 | GFP_KERNEL | GFP_DMA); | |
584 | if (!protect) | |
585 | return -ENOMEM; | |
586 | data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN); | |
587 | ||
588 | res = platform_get_resource(pdev, IORESOURCE_MEM, 0); | |
589 | data->base = devm_ioremap_resource(dev, res); | |
590 | if (IS_ERR(data->base)) | |
591 | return PTR_ERR(data->base); | |
592 | ||
593 | data->irq = platform_get_irq(pdev, 0); | |
594 | if (data->irq < 0) | |
595 | return data->irq; | |
596 | ||
597 | data->bclk = devm_clk_get(dev, "bclk"); | |
598 | if (IS_ERR(data->bclk)) | |
599 | return PTR_ERR(data->bclk); | |
600 | ||
601 | larb_nr = 0; | |
602 | of_for_each_phandle(&it, err, dev->of_node, | |
603 | "mediatek,larbs", NULL, 0) { | |
604 | struct platform_device *plarbdev; | |
605 | int count = of_phandle_iterator_args(&it, larb_spec.args, | |
606 | MAX_PHANDLE_ARGS); | |
607 | ||
608 | if (count) | |
609 | continue; | |
610 | ||
611 | larb_spec.np = of_node_get(it.node); | |
612 | if (!of_device_is_available(larb_spec.np)) | |
613 | continue; | |
614 | ||
615 | plarbdev = of_find_device_by_node(larb_spec.np); | |
b17336c5 HZ |
616 | if (!plarbdev) { |
617 | plarbdev = of_platform_device_create( | |
618 | larb_spec.np, NULL, | |
619 | platform_bus_type.dev_root); | |
00c7c81f RK |
620 | if (!plarbdev) { |
621 | of_node_put(larb_spec.np); | |
b17336c5 | 622 | return -EPROBE_DEFER; |
00c7c81f | 623 | } |
b17336c5 HZ |
624 | } |
625 | ||
626 | data->smi_imu.larb_imu[larb_nr].dev = &plarbdev->dev; | |
00c7c81f RK |
627 | component_match_add_release(dev, &match, release_of, |
628 | compare_of, larb_spec.np); | |
b17336c5 HZ |
629 | larb_nr++; |
630 | } | |
631 | ||
632 | data->smi_imu.larb_nr = larb_nr; | |
633 | ||
634 | platform_set_drvdata(pdev, data); | |
635 | ||
636 | ret = mtk_iommu_hw_init(data); | |
637 | if (ret) | |
638 | return ret; | |
639 | ||
6f66ea09 JR |
640 | ret = iommu_device_sysfs_add(&data->iommu, &pdev->dev, NULL, |
641 | dev_name(&pdev->dev)); | |
642 | if (ret) | |
643 | return ret; | |
644 | ||
645 | iommu_device_set_ops(&data->iommu, &mtk_iommu_ops); | |
646 | ||
647 | ret = iommu_device_register(&data->iommu); | |
648 | if (ret) | |
649 | return ret; | |
650 | ||
b17336c5 HZ |
651 | if (!iommu_present(&platform_bus_type)) |
652 | bus_set_iommu(&platform_bus_type, &mtk_iommu_ops); | |
653 | ||
654 | return component_master_add_with_match(dev, &mtk_iommu_com_ops, match); | |
655 | } | |
656 | ||
657 | static int mtk_iommu_remove(struct platform_device *pdev) | |
658 | { | |
659 | struct mtk_iommu_data *data = platform_get_drvdata(pdev); | |
660 | ||
6f66ea09 JR |
661 | iommu_device_sysfs_remove(&data->iommu); |
662 | iommu_device_unregister(&data->iommu); | |
663 | ||
b17336c5 HZ |
664 | if (iommu_present(&platform_bus_type)) |
665 | bus_set_iommu(&platform_bus_type, NULL); | |
666 | ||
667 | clk_disable_unprepare(data->bclk); | |
668 | devm_free_irq(&pdev->dev, data->irq, data); | |
669 | component_master_del(&pdev->dev, &mtk_iommu_com_ops); | |
670 | return 0; | |
671 | } | |
672 | ||
673 | static int __maybe_unused mtk_iommu_suspend(struct device *dev) | |
674 | { | |
675 | struct mtk_iommu_data *data = dev_get_drvdata(dev); | |
676 | struct mtk_iommu_suspend_reg *reg = &data->reg; | |
677 | void __iomem *base = data->base; | |
678 | ||
679 | reg->standard_axi_mode = readl_relaxed(base + | |
680 | REG_MMU_STANDARD_AXI_MODE); | |
681 | reg->dcm_dis = readl_relaxed(base + REG_MMU_DCM); | |
682 | reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG); | |
683 | reg->int_control0 = readl_relaxed(base + REG_MMU_INT_CONTROL); | |
684 | return 0; | |
685 | } | |
686 | ||
687 | static int __maybe_unused mtk_iommu_resume(struct device *dev) | |
688 | { | |
689 | struct mtk_iommu_data *data = dev_get_drvdata(dev); | |
690 | struct mtk_iommu_suspend_reg *reg = &data->reg; | |
691 | void __iomem *base = data->base; | |
692 | ||
693 | writel_relaxed(data->m4u_dom->pgt_pa, base + REG_MMU_PT_BASE_ADDR); | |
694 | writel_relaxed(reg->standard_axi_mode, | |
695 | base + REG_MMU_STANDARD_AXI_MODE); | |
696 | writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM); | |
697 | writel_relaxed(reg->ctrl_reg, base + REG_MMU_CTRL_REG); | |
698 | writel_relaxed(reg->int_control0, base + REG_MMU_INT_CONTROL); | |
699 | writel_relaxed(data->protect_base, base + REG_MMU_IVRP_PADDR); | |
700 | return 0; | |
701 | } | |
702 | ||
131bc8eb | 703 | static const struct dev_pm_ops mtk_iommu_pm_ops = { |
b17336c5 HZ |
704 | SET_SYSTEM_SLEEP_PM_OPS(mtk_iommu_suspend, mtk_iommu_resume) |
705 | }; | |
706 | ||
707 | static struct platform_driver mtk_iommu_driver = { | |
708 | .probe = mtk_iommu_probe, | |
709 | .remove = mtk_iommu_remove, | |
710 | .driver = { | |
711 | .name = "mtk-iommu", | |
712 | .of_match_table = mtk_iommu_of_ids, | |
713 | .pm = &mtk_iommu_pm_ops, | |
714 | } | |
715 | }; | |
716 | ||
717 | static int __init m4u_init(void) | |
718 | { | |
719 | return platform_driver_register(&mtk_iommu_driver); | |
720 | } | |
721 | ||
722 | static void __exit m4u_exit(void) | |
723 | { | |
724 | return platform_driver_unregister(&mtk_iommu_driver); | |
725 | } | |
726 | ||
727 | subsys_initcall(m4u_init); | |
728 | module_exit(m4u_exit); | |
729 | ||
730 | MODULE_DESCRIPTION("IOMMU API for MTK architected m4u v1 implementations"); | |
731 | MODULE_AUTHOR("Honghui Zhang <honghui.zhang@mediatek.com>"); | |
732 | MODULE_LICENSE("GPL v2"); |