Commit | Line | Data |
---|---|---|
46d1fb07 SP |
1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* | |
3 | * Apple DART (Device Address Resolution Table) IOMMU driver | |
4 | * | |
5 | * Copyright (C) 2021 The Asahi Linux Contributors | |
6 | * | |
7 | * Based on arm/arm-smmu/arm-ssmu.c and arm/arm-smmu-v3/arm-smmu-v3.c | |
8 | * Copyright (C) 2013 ARM Limited | |
9 | * Copyright (C) 2015 ARM Limited | |
10 | * and on exynos-iommu.c | |
11 | * Copyright (c) 2011,2016 Samsung Electronics Co., Ltd. | |
12 | */ | |
13 | ||
14 | #include <linux/atomic.h> | |
15 | #include <linux/bitfield.h> | |
16 | #include <linux/clk.h> | |
17 | #include <linux/dev_printk.h> | |
46d1fb07 SP |
18 | #include <linux/dma-mapping.h> |
19 | #include <linux/err.h> | |
20 | #include <linux/interrupt.h> | |
21 | #include <linux/io-pgtable.h> | |
22 | #include <linux/iommu.h> | |
23 | #include <linux/iopoll.h> | |
24 | #include <linux/module.h> | |
25 | #include <linux/of.h> | |
26 | #include <linux/of_address.h> | |
27 | #include <linux/of_iommu.h> | |
28 | #include <linux/of_platform.h> | |
29 | #include <linux/pci.h> | |
30 | #include <linux/platform_device.h> | |
31 | #include <linux/slab.h> | |
32 | #include <linux/swab.h> | |
33 | #include <linux/types.h> | |
34 | ||
f2042ed2 RM |
35 | #include "dma-iommu.h" |
36 | ||
510d4072 | 37 | #define DART_MAX_STREAMS 256 |
46d1fb07 SP |
38 | #define DART_MAX_TTBR 4 |
39 | #define MAX_DARTS_PER_DEVICE 2 | |
40 | ||
b76c68fc | 41 | /* Common registers */ |
46d1fb07 SP |
42 | |
43 | #define DART_PARAMS1 0x00 | |
a772a02c | 44 | #define DART_PARAMS1_PAGE_SHIFT GENMASK(27, 24) |
46d1fb07 SP |
45 | |
46 | #define DART_PARAMS2 0x04 | |
a772a02c | 47 | #define DART_PARAMS2_BYPASS_SUPPORT BIT(0) |
46d1fb07 | 48 | |
b76c68fc | 49 | /* T8020/T6000 registers */ |
46d1fb07 | 50 | |
b76c68fc HM |
51 | #define DART_T8020_STREAM_COMMAND 0x20 |
52 | #define DART_T8020_STREAM_COMMAND_BUSY BIT(2) | |
53 | #define DART_T8020_STREAM_COMMAND_INVALIDATE BIT(20) | |
46d1fb07 | 54 | |
b76c68fc | 55 | #define DART_T8020_STREAM_SELECT 0x34 |
46d1fb07 | 56 | |
b76c68fc HM |
57 | #define DART_T8020_ERROR 0x40 |
58 | #define DART_T8020_ERROR_STREAM GENMASK(27, 24) | |
59 | #define DART_T8020_ERROR_CODE GENMASK(11, 0) | |
60 | #define DART_T8020_ERROR_FLAG BIT(31) | |
46d1fb07 | 61 | |
b76c68fc HM |
62 | #define DART_T8020_ERROR_READ_FAULT BIT(4) |
63 | #define DART_T8020_ERROR_WRITE_FAULT BIT(3) | |
64 | #define DART_T8020_ERROR_NO_PTE BIT(2) | |
65 | #define DART_T8020_ERROR_NO_PMD BIT(1) | |
66 | #define DART_T8020_ERROR_NO_TTBR BIT(0) | |
46d1fb07 | 67 | |
b76c68fc HM |
68 | #define DART_T8020_CONFIG 0x60 |
69 | #define DART_T8020_CONFIG_LOCK BIT(15) | |
5a009fc1 | 70 | |
46d1fb07 | 71 | #define DART_STREAM_COMMAND_BUSY_TIMEOUT 100 |
46d1fb07 | 72 | |
b76c68fc HM |
73 | #define DART_T8020_ERROR_ADDR_HI 0x54 |
74 | #define DART_T8020_ERROR_ADDR_LO 0x50 | |
75 | ||
76 | #define DART_T8020_STREAMS_ENABLE 0xfc | |
46d1fb07 | 77 | |
b76c68fc HM |
78 | #define DART_T8020_TCR 0x100 |
79 | #define DART_T8020_TCR_TRANSLATE_ENABLE BIT(7) | |
80 | #define DART_T8020_TCR_BYPASS_DART BIT(8) | |
81 | #define DART_T8020_TCR_BYPASS_DAPF BIT(12) | |
5a009fc1 | 82 | |
b76c68fc HM |
83 | #define DART_T8020_TTBR 0x200 |
84 | #define DART_T8020_TTBR_VALID BIT(31) | |
85 | #define DART_T8020_TTBR_ADDR_FIELD_SHIFT 0 | |
86 | #define DART_T8020_TTBR_SHIFT 12 | |
46d1fb07 | 87 | |
d8bcc870 HM |
88 | /* T8110 registers */ |
89 | ||
90 | #define DART_T8110_PARAMS3 0x08 | |
91 | #define DART_T8110_PARAMS3_PA_WIDTH GENMASK(29, 24) | |
92 | #define DART_T8110_PARAMS3_VA_WIDTH GENMASK(21, 16) | |
93 | #define DART_T8110_PARAMS3_VER_MAJ GENMASK(15, 8) | |
94 | #define DART_T8110_PARAMS3_VER_MIN GENMASK(7, 0) | |
95 | ||
96 | #define DART_T8110_PARAMS4 0x0c | |
97 | #define DART_T8110_PARAMS4_NUM_CLIENTS GENMASK(24, 16) | |
98 | #define DART_T8110_PARAMS4_NUM_SIDS GENMASK(8, 0) | |
99 | ||
100 | #define DART_T8110_TLB_CMD 0x80 | |
101 | #define DART_T8110_TLB_CMD_BUSY BIT(31) | |
102 | #define DART_T8110_TLB_CMD_OP GENMASK(10, 8) | |
103 | #define DART_T8110_TLB_CMD_OP_FLUSH_ALL 0 | |
104 | #define DART_T8110_TLB_CMD_OP_FLUSH_SID 1 | |
105 | #define DART_T8110_TLB_CMD_STREAM GENMASK(7, 0) | |
106 | ||
107 | #define DART_T8110_ERROR 0x100 | |
108 | #define DART_T8110_ERROR_STREAM GENMASK(27, 20) | |
109 | #define DART_T8110_ERROR_CODE GENMASK(14, 0) | |
110 | #define DART_T8110_ERROR_FLAG BIT(31) | |
111 | ||
112 | #define DART_T8110_ERROR_MASK 0x104 | |
113 | ||
9e6a1825 EC |
114 | #define DART_T8110_ERROR_READ_FAULT BIT(5) |
115 | #define DART_T8110_ERROR_WRITE_FAULT BIT(4) | |
d8bcc870 HM |
116 | #define DART_T8110_ERROR_NO_PTE BIT(3) |
117 | #define DART_T8110_ERROR_NO_PMD BIT(2) | |
118 | #define DART_T8110_ERROR_NO_PGD BIT(1) | |
119 | #define DART_T8110_ERROR_NO_TTBR BIT(0) | |
120 | ||
121 | #define DART_T8110_ERROR_ADDR_LO 0x170 | |
122 | #define DART_T8110_ERROR_ADDR_HI 0x174 | |
123 | ||
124 | #define DART_T8110_PROTECT 0x200 | |
125 | #define DART_T8110_UNPROTECT 0x204 | |
126 | #define DART_T8110_PROTECT_LOCK 0x208 | |
127 | #define DART_T8110_PROTECT_TTBR_TCR BIT(0) | |
128 | ||
129 | #define DART_T8110_ENABLE_STREAMS 0xc00 | |
130 | #define DART_T8110_DISABLE_STREAMS 0xc20 | |
131 | ||
132 | #define DART_T8110_TCR 0x1000 | |
133 | #define DART_T8110_TCR_REMAP GENMASK(11, 8) | |
134 | #define DART_T8110_TCR_REMAP_EN BIT(7) | |
135 | #define DART_T8110_TCR_BYPASS_DAPF BIT(2) | |
136 | #define DART_T8110_TCR_BYPASS_DART BIT(1) | |
137 | #define DART_T8110_TCR_TRANSLATE_ENABLE BIT(0) | |
138 | ||
139 | #define DART_T8110_TTBR 0x1400 | |
140 | #define DART_T8110_TTBR_VALID BIT(0) | |
141 | #define DART_T8110_TTBR_ADDR_FIELD_SHIFT 2 | |
142 | #define DART_T8110_TTBR_SHIFT 14 | |
143 | ||
b76c68fc | 144 | #define DART_TCR(dart, sid) ((dart)->hw->tcr + ((sid) << 2)) |
46d1fb07 | 145 | |
b76c68fc | 146 | #define DART_TTBR(dart, sid, idx) ((dart)->hw->ttbr + \ |
0b459bcd HM |
147 | (((dart)->hw->ttbr_count * (sid)) << 2) + \ |
148 | ((idx) << 2)) | |
149 | ||
b76c68fc | 150 | struct apple_dart_stream_map; |
0b459bcd | 151 | |
d8bcc870 HM |
152 | enum dart_type { |
153 | DART_T8020, | |
154 | DART_T6000, | |
155 | DART_T8110, | |
156 | }; | |
46d1fb07 | 157 | |
a380b8dc | 158 | struct apple_dart_hw { |
d8bcc870 | 159 | enum dart_type type; |
b76c68fc HM |
160 | irqreturn_t (*irq_handler)(int irq, void *dev); |
161 | int (*invalidate_tlb)(struct apple_dart_stream_map *stream_map); | |
162 | ||
a380b8dc SP |
163 | u32 oas; |
164 | enum io_pgtable_fmt fmt; | |
510d4072 HM |
165 | |
166 | int max_sid_count; | |
0b459bcd | 167 | |
b76c68fc HM |
168 | u64 lock; |
169 | u64 lock_bit; | |
170 | ||
171 | u64 error; | |
172 | ||
173 | u64 enable_streams; | |
174 | ||
175 | u64 tcr; | |
176 | u64 tcr_enabled; | |
177 | u64 tcr_disabled; | |
178 | u64 tcr_bypass; | |
179 | ||
180 | u64 ttbr; | |
181 | u64 ttbr_valid; | |
182 | u64 ttbr_addr_field_shift; | |
183 | u64 ttbr_shift; | |
0b459bcd | 184 | int ttbr_count; |
a380b8dc SP |
185 | }; |
186 | ||
46d1fb07 SP |
187 | /* |
188 | * Private structure associated with each DART device. | |
189 | * | |
190 | * @dev: device struct | |
a380b8dc | 191 | * @hw: SoC-specific hardware data |
46d1fb07 SP |
192 | * @regs: mapped MMIO region |
193 | * @irq: interrupt number, can be shared with other DARTs | |
194 | * @clks: clocks associated with this DART | |
195 | * @num_clks: number of @clks | |
196 | * @lock: lock for hardware operations involving this dart | |
197 | * @pgsize: pagesize supported by this DART | |
198 | * @supports_bypass: indicates if this DART supports bypass mode | |
199 | * @force_bypass: force bypass mode due to pagesize mismatch? | |
200 | * @sid2group: maps stream ids to iommu_groups | |
201 | * @iommu: iommu core device | |
202 | */ | |
203 | struct apple_dart { | |
204 | struct device *dev; | |
a380b8dc | 205 | const struct apple_dart_hw *hw; |
46d1fb07 SP |
206 | |
207 | void __iomem *regs; | |
208 | ||
209 | int irq; | |
210 | struct clk_bulk_data *clks; | |
211 | int num_clks; | |
212 | ||
213 | spinlock_t lock; | |
214 | ||
d8bcc870 HM |
215 | u32 ias; |
216 | u32 oas; | |
46d1fb07 | 217 | u32 pgsize; |
510d4072 | 218 | u32 num_streams; |
46d1fb07 SP |
219 | u32 supports_bypass : 1; |
220 | u32 force_bypass : 1; | |
221 | ||
222 | struct iommu_group *sid2group[DART_MAX_STREAMS]; | |
223 | struct iommu_device iommu; | |
3d68bbb8 HM |
224 | |
225 | u32 save_tcr[DART_MAX_STREAMS]; | |
226 | u32 save_ttbr[DART_MAX_STREAMS][DART_MAX_TTBR]; | |
46d1fb07 SP |
227 | }; |
228 | ||
229 | /* | |
230 | * Convenience struct to identify streams. | |
231 | * | |
232 | * The normal variant is used inside apple_dart_master_cfg which isn't written | |
233 | * to concurrently. | |
234 | * The atomic variant is used inside apple_dart_domain where we have to guard | |
235 | * against races from potential parallel calls to attach/detach_device. | |
236 | * Note that even inside the atomic variant the apple_dart pointer is not | |
237 | * protected: This pointer is initialized once under the domain init mutex | |
238 | * and never changed again afterwards. Devices with different dart pointers | |
239 | * cannot be attached to the same domain. | |
240 | * | |
241 | * @dart dart pointer | |
242 | * @sid stream id bitmap | |
243 | */ | |
244 | struct apple_dart_stream_map { | |
245 | struct apple_dart *dart; | |
510d4072 | 246 | DECLARE_BITMAP(sidmap, DART_MAX_STREAMS); |
46d1fb07 SP |
247 | }; |
248 | struct apple_dart_atomic_stream_map { | |
249 | struct apple_dart *dart; | |
510d4072 | 250 | atomic_long_t sidmap[BITS_TO_LONGS(DART_MAX_STREAMS)]; |
46d1fb07 SP |
251 | }; |
252 | ||
253 | /* | |
254 | * This structure is attached to each iommu domain handled by a DART. | |
255 | * | |
256 | * @pgtbl_ops: pagetable ops allocated by io-pgtable | |
257 | * @finalized: true if the domain has been completely initialized | |
258 | * @init_lock: protects domain initialization | |
259 | * @stream_maps: streams attached to this domain (valid for DMA/UNMANAGED only) | |
260 | * @domain: core iommu domain pointer | |
261 | */ | |
262 | struct apple_dart_domain { | |
263 | struct io_pgtable_ops *pgtbl_ops; | |
264 | ||
265 | bool finalized; | |
266 | struct mutex init_lock; | |
267 | struct apple_dart_atomic_stream_map stream_maps[MAX_DARTS_PER_DEVICE]; | |
268 | ||
269 | struct iommu_domain domain; | |
270 | }; | |
271 | ||
272 | /* | |
273 | * This structure is attached to devices with dev_iommu_priv_set() on of_xlate | |
274 | * and contains a list of streams bound to this device. | |
275 | * So far the worst case seen is a single device with two streams | |
276 | * from different darts, such that this simple static array is enough. | |
277 | * | |
278 | * @streams: streams for this device | |
279 | */ | |
280 | struct apple_dart_master_cfg { | |
281 | struct apple_dart_stream_map stream_maps[MAX_DARTS_PER_DEVICE]; | |
282 | }; | |
283 | ||
284 | /* | |
285 | * Helper macro to iterate over apple_dart_master_cfg.stream_maps and | |
286 | * apple_dart_domain.stream_maps | |
287 | * | |
288 | * @i int used as loop variable | |
289 | * @base pointer to base struct (apple_dart_master_cfg or apple_dart_domain) | |
290 | * @stream pointer to the apple_dart_streams struct for each loop iteration | |
291 | */ | |
292 | #define for_each_stream_map(i, base, stream_map) \ | |
293 | for (i = 0, stream_map = &(base)->stream_maps[0]; \ | |
294 | i < MAX_DARTS_PER_DEVICE && stream_map->dart; \ | |
295 | stream_map = &(base)->stream_maps[++i]) | |
296 | ||
297 | static struct platform_driver apple_dart_driver; | |
298 | static const struct iommu_ops apple_dart_iommu_ops; | |
46d1fb07 SP |
299 | |
300 | static struct apple_dart_domain *to_dart_domain(struct iommu_domain *dom) | |
301 | { | |
302 | return container_of(dom, struct apple_dart_domain, domain); | |
303 | } | |
304 | ||
305 | static void | |
306 | apple_dart_hw_enable_translation(struct apple_dart_stream_map *stream_map) | |
307 | { | |
510d4072 | 308 | struct apple_dart *dart = stream_map->dart; |
46d1fb07 SP |
309 | int sid; |
310 | ||
510d4072 | 311 | for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) |
b76c68fc | 312 | writel(dart->hw->tcr_enabled, dart->regs + DART_TCR(dart, sid)); |
46d1fb07 SP |
313 | } |
314 | ||
315 | static void apple_dart_hw_disable_dma(struct apple_dart_stream_map *stream_map) | |
316 | { | |
510d4072 | 317 | struct apple_dart *dart = stream_map->dart; |
46d1fb07 SP |
318 | int sid; |
319 | ||
510d4072 | 320 | for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) |
b76c68fc | 321 | writel(dart->hw->tcr_disabled, dart->regs + DART_TCR(dart, sid)); |
46d1fb07 SP |
322 | } |
323 | ||
324 | static void | |
325 | apple_dart_hw_enable_bypass(struct apple_dart_stream_map *stream_map) | |
326 | { | |
510d4072 | 327 | struct apple_dart *dart = stream_map->dart; |
46d1fb07 SP |
328 | int sid; |
329 | ||
330 | WARN_ON(!stream_map->dart->supports_bypass); | |
510d4072 | 331 | for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) |
b76c68fc HM |
332 | writel(dart->hw->tcr_bypass, |
333 | dart->regs + DART_TCR(dart, sid)); | |
46d1fb07 SP |
334 | } |
335 | ||
336 | static void apple_dart_hw_set_ttbr(struct apple_dart_stream_map *stream_map, | |
337 | u8 idx, phys_addr_t paddr) | |
338 | { | |
510d4072 | 339 | struct apple_dart *dart = stream_map->dart; |
46d1fb07 SP |
340 | int sid; |
341 | ||
b76c68fc | 342 | WARN_ON(paddr & ((1 << dart->hw->ttbr_shift) - 1)); |
510d4072 | 343 | for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) |
b76c68fc HM |
344 | writel(dart->hw->ttbr_valid | |
345 | (paddr >> dart->hw->ttbr_shift) << dart->hw->ttbr_addr_field_shift, | |
0b459bcd | 346 | dart->regs + DART_TTBR(dart, sid, idx)); |
46d1fb07 SP |
347 | } |
348 | ||
349 | static void apple_dart_hw_clear_ttbr(struct apple_dart_stream_map *stream_map, | |
350 | u8 idx) | |
351 | { | |
510d4072 | 352 | struct apple_dart *dart = stream_map->dart; |
46d1fb07 SP |
353 | int sid; |
354 | ||
510d4072 | 355 | for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) |
0b459bcd | 356 | writel(0, dart->regs + DART_TTBR(dart, sid, idx)); |
46d1fb07 SP |
357 | } |
358 | ||
359 | static void | |
360 | apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map *stream_map) | |
361 | { | |
362 | int i; | |
363 | ||
0b459bcd | 364 | for (i = 0; i < stream_map->dart->hw->ttbr_count; ++i) |
46d1fb07 SP |
365 | apple_dart_hw_clear_ttbr(stream_map, i); |
366 | } | |
367 | ||
368 | static int | |
b76c68fc | 369 | apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map *stream_map, |
46d1fb07 SP |
370 | u32 command) |
371 | { | |
372 | unsigned long flags; | |
373 | int ret; | |
374 | u32 command_reg; | |
375 | ||
376 | spin_lock_irqsave(&stream_map->dart->lock, flags); | |
377 | ||
b76c68fc HM |
378 | writel(stream_map->sidmap[0], stream_map->dart->regs + DART_T8020_STREAM_SELECT); |
379 | writel(command, stream_map->dart->regs + DART_T8020_STREAM_COMMAND); | |
46d1fb07 SP |
380 | |
381 | ret = readl_poll_timeout_atomic( | |
b76c68fc HM |
382 | stream_map->dart->regs + DART_T8020_STREAM_COMMAND, command_reg, |
383 | !(command_reg & DART_T8020_STREAM_COMMAND_BUSY), 1, | |
46d1fb07 SP |
384 | DART_STREAM_COMMAND_BUSY_TIMEOUT); |
385 | ||
386 | spin_unlock_irqrestore(&stream_map->dart->lock, flags); | |
387 | ||
388 | if (ret) { | |
389 | dev_err(stream_map->dart->dev, | |
390 | "busy bit did not clear after command %x for streams %lx\n", | |
510d4072 | 391 | command, stream_map->sidmap[0]); |
46d1fb07 SP |
392 | return ret; |
393 | } | |
394 | ||
395 | return 0; | |
396 | } | |
397 | ||
d8bcc870 HM |
398 | static int |
399 | apple_dart_t8110_hw_tlb_command(struct apple_dart_stream_map *stream_map, | |
400 | u32 command) | |
401 | { | |
402 | struct apple_dart *dart = stream_map->dart; | |
403 | unsigned long flags; | |
404 | int ret = 0; | |
405 | int sid; | |
406 | ||
407 | spin_lock_irqsave(&dart->lock, flags); | |
408 | ||
409 | for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { | |
410 | u32 val = FIELD_PREP(DART_T8110_TLB_CMD_OP, command) | | |
411 | FIELD_PREP(DART_T8110_TLB_CMD_STREAM, sid); | |
412 | writel(val, dart->regs + DART_T8110_TLB_CMD); | |
413 | ||
414 | ret = readl_poll_timeout_atomic( | |
415 | dart->regs + DART_T8110_TLB_CMD, val, | |
416 | !(val & DART_T8110_TLB_CMD_BUSY), 1, | |
417 | DART_STREAM_COMMAND_BUSY_TIMEOUT); | |
418 | ||
419 | if (ret) | |
420 | break; | |
421 | ||
422 | } | |
423 | ||
424 | spin_unlock_irqrestore(&dart->lock, flags); | |
425 | ||
426 | if (ret) { | |
427 | dev_err(stream_map->dart->dev, | |
428 | "busy bit did not clear after command %x for stream %d\n", | |
429 | command, sid); | |
46d1fb07 SP |
430 | return ret; |
431 | } | |
432 | ||
433 | return 0; | |
434 | } | |
435 | ||
436 | static int | |
b76c68fc | 437 | apple_dart_t8020_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) |
46d1fb07 | 438 | { |
b76c68fc HM |
439 | return apple_dart_t8020_hw_stream_command( |
440 | stream_map, DART_T8020_STREAM_COMMAND_INVALIDATE); | |
46d1fb07 SP |
441 | } |
442 | ||
d8bcc870 HM |
443 | static int |
444 | apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) | |
46d1fb07 | 445 | { |
d8bcc870 HM |
446 | return apple_dart_t8110_hw_tlb_command( |
447 | stream_map, DART_T8110_TLB_CMD_OP_FLUSH_SID); | |
46d1fb07 SP |
448 | } |
449 | ||
450 | static int apple_dart_hw_reset(struct apple_dart *dart) | |
451 | { | |
452 | u32 config; | |
453 | struct apple_dart_stream_map stream_map; | |
510d4072 | 454 | int i; |
46d1fb07 | 455 | |
b76c68fc HM |
456 | config = readl(dart->regs + dart->hw->lock); |
457 | if (config & dart->hw->lock_bit) { | |
46d1fb07 SP |
458 | dev_err(dart->dev, "DART is locked down until reboot: %08x\n", |
459 | config); | |
460 | return -EINVAL; | |
461 | } | |
462 | ||
463 | stream_map.dart = dart; | |
510d4072 HM |
464 | bitmap_zero(stream_map.sidmap, DART_MAX_STREAMS); |
465 | bitmap_set(stream_map.sidmap, 0, dart->num_streams); | |
46d1fb07 SP |
466 | apple_dart_hw_disable_dma(&stream_map); |
467 | apple_dart_hw_clear_all_ttbrs(&stream_map); | |
468 | ||
5a009fc1 | 469 | /* enable all streams globally since TCR is used to control isolation */ |
510d4072 | 470 | for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) |
b76c68fc | 471 | writel(U32_MAX, dart->regs + dart->hw->enable_streams + 4 * i); |
5a009fc1 | 472 | |
46d1fb07 | 473 | /* clear any pending errors before the interrupt is unmasked */ |
b76c68fc | 474 | writel(readl(dart->regs + dart->hw->error), dart->regs + dart->hw->error); |
46d1fb07 | 475 | |
d8bcc870 HM |
476 | if (dart->hw->type == DART_T8110) |
477 | writel(0, dart->regs + DART_T8110_ERROR_MASK); | |
46d1fb07 | 478 | |
b76c68fc | 479 | return dart->hw->invalidate_tlb(&stream_map); |
46d1fb07 SP |
480 | } |
481 | ||
482 | static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain) | |
483 | { | |
510d4072 | 484 | int i, j; |
46d1fb07 SP |
485 | struct apple_dart_atomic_stream_map *domain_stream_map; |
486 | struct apple_dart_stream_map stream_map; | |
487 | ||
488 | for_each_stream_map(i, domain, domain_stream_map) { | |
489 | stream_map.dart = domain_stream_map->dart; | |
510d4072 HM |
490 | |
491 | for (j = 0; j < BITS_TO_LONGS(stream_map.dart->num_streams); j++) | |
492 | stream_map.sidmap[j] = atomic_long_read(&domain_stream_map->sidmap[j]); | |
493 | ||
b76c68fc | 494 | stream_map.dart->hw->invalidate_tlb(&stream_map); |
46d1fb07 SP |
495 | } |
496 | } | |
497 | ||
498 | static void apple_dart_flush_iotlb_all(struct iommu_domain *domain) | |
499 | { | |
500 | apple_dart_domain_flush_tlb(to_dart_domain(domain)); | |
501 | } | |
502 | ||
503 | static void apple_dart_iotlb_sync(struct iommu_domain *domain, | |
504 | struct iommu_iotlb_gather *gather) | |
505 | { | |
506 | apple_dart_domain_flush_tlb(to_dart_domain(domain)); | |
507 | } | |
508 | ||
509 | static void apple_dart_iotlb_sync_map(struct iommu_domain *domain, | |
510 | unsigned long iova, size_t size) | |
511 | { | |
512 | apple_dart_domain_flush_tlb(to_dart_domain(domain)); | |
513 | } | |
514 | ||
46d1fb07 SP |
515 | static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, |
516 | dma_addr_t iova) | |
517 | { | |
518 | struct apple_dart_domain *dart_domain = to_dart_domain(domain); | |
519 | struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; | |
520 | ||
521 | if (!ops) | |
522 | return 0; | |
523 | ||
524 | return ops->iova_to_phys(ops, iova); | |
525 | } | |
526 | ||
527 | static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, | |
528 | phys_addr_t paddr, size_t pgsize, | |
529 | size_t pgcount, int prot, gfp_t gfp, | |
530 | size_t *mapped) | |
531 | { | |
532 | struct apple_dart_domain *dart_domain = to_dart_domain(domain); | |
533 | struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; | |
534 | ||
535 | if (!ops) | |
536 | return -ENODEV; | |
537 | ||
538 | return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, | |
539 | mapped); | |
540 | } | |
541 | ||
542 | static size_t apple_dart_unmap_pages(struct iommu_domain *domain, | |
543 | unsigned long iova, size_t pgsize, | |
544 | size_t pgcount, | |
545 | struct iommu_iotlb_gather *gather) | |
546 | { | |
547 | struct apple_dart_domain *dart_domain = to_dart_domain(domain); | |
548 | struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; | |
549 | ||
550 | return ops->unmap_pages(ops, iova, pgsize, pgcount, gather); | |
551 | } | |
552 | ||
553 | static void | |
554 | apple_dart_setup_translation(struct apple_dart_domain *domain, | |
555 | struct apple_dart_stream_map *stream_map) | |
556 | { | |
557 | int i; | |
558 | struct io_pgtable_cfg *pgtbl_cfg = | |
559 | &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg; | |
560 | ||
561 | for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i) | |
562 | apple_dart_hw_set_ttbr(stream_map, i, | |
563 | pgtbl_cfg->apple_dart_cfg.ttbr[i]); | |
0b459bcd | 564 | for (; i < stream_map->dart->hw->ttbr_count; ++i) |
46d1fb07 SP |
565 | apple_dart_hw_clear_ttbr(stream_map, i); |
566 | ||
567 | apple_dart_hw_enable_translation(stream_map); | |
b76c68fc | 568 | stream_map->dart->hw->invalidate_tlb(stream_map); |
46d1fb07 SP |
569 | } |
570 | ||
571 | static int apple_dart_finalize_domain(struct iommu_domain *domain, | |
572 | struct apple_dart_master_cfg *cfg) | |
573 | { | |
574 | struct apple_dart_domain *dart_domain = to_dart_domain(domain); | |
575 | struct apple_dart *dart = cfg->stream_maps[0].dart; | |
576 | struct io_pgtable_cfg pgtbl_cfg; | |
577 | int ret = 0; | |
510d4072 | 578 | int i, j; |
46d1fb07 SP |
579 | |
580 | mutex_lock(&dart_domain->init_lock); | |
581 | ||
582 | if (dart_domain->finalized) | |
583 | goto done; | |
584 | ||
585 | for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { | |
586 | dart_domain->stream_maps[i].dart = cfg->stream_maps[i].dart; | |
510d4072 HM |
587 | for (j = 0; j < BITS_TO_LONGS(dart->num_streams); j++) |
588 | atomic_long_set(&dart_domain->stream_maps[i].sidmap[j], | |
589 | cfg->stream_maps[i].sidmap[j]); | |
46d1fb07 SP |
590 | } |
591 | ||
592 | pgtbl_cfg = (struct io_pgtable_cfg){ | |
593 | .pgsize_bitmap = dart->pgsize, | |
d8bcc870 HM |
594 | .ias = dart->ias, |
595 | .oas = dart->oas, | |
46d1fb07 | 596 | .coherent_walk = 1, |
46d1fb07 SP |
597 | .iommu_dev = dart->dev, |
598 | }; | |
599 | ||
600 | dart_domain->pgtbl_ops = | |
a380b8dc | 601 | alloc_io_pgtable_ops(dart->hw->fmt, &pgtbl_cfg, domain); |
46d1fb07 SP |
602 | if (!dart_domain->pgtbl_ops) { |
603 | ret = -ENOMEM; | |
604 | goto done; | |
605 | } | |
606 | ||
607 | domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; | |
608 | domain->geometry.aperture_start = 0; | |
d8bcc870 | 609 | domain->geometry.aperture_end = (dma_addr_t)DMA_BIT_MASK(dart->ias); |
46d1fb07 SP |
610 | domain->geometry.force_aperture = true; |
611 | ||
612 | dart_domain->finalized = true; | |
613 | ||
614 | done: | |
615 | mutex_unlock(&dart_domain->init_lock); | |
616 | return ret; | |
617 | } | |
618 | ||
619 | static int | |
620 | apple_dart_mod_streams(struct apple_dart_atomic_stream_map *domain_maps, | |
621 | struct apple_dart_stream_map *master_maps, | |
622 | bool add_streams) | |
623 | { | |
510d4072 | 624 | int i, j; |
46d1fb07 SP |
625 | |
626 | for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { | |
627 | if (domain_maps[i].dart != master_maps[i].dart) | |
628 | return -EINVAL; | |
629 | } | |
630 | ||
631 | for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { | |
632 | if (!domain_maps[i].dart) | |
633 | break; | |
510d4072 HM |
634 | for (j = 0; j < BITS_TO_LONGS(domain_maps[i].dart->num_streams); j++) { |
635 | if (add_streams) | |
636 | atomic_long_or(master_maps[i].sidmap[j], | |
637 | &domain_maps[i].sidmap[j]); | |
638 | else | |
639 | atomic_long_and(~master_maps[i].sidmap[j], | |
640 | &domain_maps[i].sidmap[j]); | |
641 | } | |
46d1fb07 SP |
642 | } |
643 | ||
644 | return 0; | |
645 | } | |
646 | ||
647 | static int apple_dart_domain_add_streams(struct apple_dart_domain *domain, | |
648 | struct apple_dart_master_cfg *cfg) | |
649 | { | |
650 | return apple_dart_mod_streams(domain->stream_maps, cfg->stream_maps, | |
651 | true); | |
652 | } | |
653 | ||
46d1fb07 SP |
654 | static int apple_dart_attach_dev(struct iommu_domain *domain, |
655 | struct device *dev) | |
656 | { | |
657 | int ret, i; | |
658 | struct apple_dart_stream_map *stream_map; | |
659 | struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); | |
660 | struct apple_dart_domain *dart_domain = to_dart_domain(domain); | |
661 | ||
662 | if (cfg->stream_maps[0].dart->force_bypass && | |
663 | domain->type != IOMMU_DOMAIN_IDENTITY) | |
664 | return -EINVAL; | |
665 | if (!cfg->stream_maps[0].dart->supports_bypass && | |
666 | domain->type == IOMMU_DOMAIN_IDENTITY) | |
667 | return -EINVAL; | |
668 | ||
669 | ret = apple_dart_finalize_domain(domain, cfg); | |
670 | if (ret) | |
671 | return ret; | |
672 | ||
673 | switch (domain->type) { | |
674 | case IOMMU_DOMAIN_DMA: | |
675 | case IOMMU_DOMAIN_UNMANAGED: | |
676 | ret = apple_dart_domain_add_streams(dart_domain, cfg); | |
677 | if (ret) | |
678 | return ret; | |
679 | ||
680 | for_each_stream_map(i, cfg, stream_map) | |
681 | apple_dart_setup_translation(dart_domain, stream_map); | |
682 | break; | |
683 | case IOMMU_DOMAIN_BLOCKED: | |
684 | for_each_stream_map(i, cfg, stream_map) | |
685 | apple_dart_hw_disable_dma(stream_map); | |
686 | break; | |
687 | case IOMMU_DOMAIN_IDENTITY: | |
688 | for_each_stream_map(i, cfg, stream_map) | |
689 | apple_dart_hw_enable_bypass(stream_map); | |
690 | break; | |
691 | } | |
692 | ||
693 | return ret; | |
694 | } | |
695 | ||
46d1fb07 SP |
696 | static struct iommu_device *apple_dart_probe_device(struct device *dev) |
697 | { | |
698 | struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); | |
699 | struct apple_dart_stream_map *stream_map; | |
700 | int i; | |
701 | ||
702 | if (!cfg) | |
703 | return ERR_PTR(-ENODEV); | |
704 | ||
705 | for_each_stream_map(i, cfg, stream_map) | |
706 | device_link_add( | |
707 | dev, stream_map->dart->dev, | |
708 | DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER); | |
709 | ||
710 | return &cfg->stream_maps[0].dart->iommu; | |
711 | } | |
712 | ||
713 | static void apple_dart_release_device(struct device *dev) | |
714 | { | |
715 | struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); | |
716 | ||
46d1fb07 SP |
717 | dev_iommu_priv_set(dev, NULL); |
718 | kfree(cfg); | |
719 | } | |
720 | ||
721 | static struct iommu_domain *apple_dart_domain_alloc(unsigned int type) | |
722 | { | |
723 | struct apple_dart_domain *dart_domain; | |
724 | ||
725 | if (type != IOMMU_DOMAIN_DMA && type != IOMMU_DOMAIN_UNMANAGED && | |
726 | type != IOMMU_DOMAIN_IDENTITY && type != IOMMU_DOMAIN_BLOCKED) | |
727 | return NULL; | |
728 | ||
729 | dart_domain = kzalloc(sizeof(*dart_domain), GFP_KERNEL); | |
730 | if (!dart_domain) | |
731 | return NULL; | |
732 | ||
46d1fb07 SP |
733 | mutex_init(&dart_domain->init_lock); |
734 | ||
735 | /* no need to allocate pgtbl_ops or do any other finalization steps */ | |
736 | if (type == IOMMU_DOMAIN_IDENTITY || type == IOMMU_DOMAIN_BLOCKED) | |
737 | dart_domain->finalized = true; | |
738 | ||
739 | return &dart_domain->domain; | |
740 | } | |
741 | ||
742 | static void apple_dart_domain_free(struct iommu_domain *domain) | |
743 | { | |
744 | struct apple_dart_domain *dart_domain = to_dart_domain(domain); | |
745 | ||
746 | if (dart_domain->pgtbl_ops) | |
747 | free_io_pgtable_ops(dart_domain->pgtbl_ops); | |
748 | ||
749 | kfree(dart_domain); | |
750 | } | |
751 | ||
752 | static int apple_dart_of_xlate(struct device *dev, struct of_phandle_args *args) | |
753 | { | |
754 | struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); | |
755 | struct platform_device *iommu_pdev = of_find_device_by_node(args->np); | |
756 | struct apple_dart *dart = platform_get_drvdata(iommu_pdev); | |
757 | struct apple_dart *cfg_dart; | |
758 | int i, sid; | |
759 | ||
760 | if (args->args_count != 1) | |
761 | return -EINVAL; | |
762 | sid = args->args[0]; | |
763 | ||
764 | if (!cfg) | |
765 | cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); | |
766 | if (!cfg) | |
767 | return -ENOMEM; | |
768 | dev_iommu_priv_set(dev, cfg); | |
769 | ||
770 | cfg_dart = cfg->stream_maps[0].dart; | |
771 | if (cfg_dart) { | |
772 | if (cfg_dart->supports_bypass != dart->supports_bypass) | |
773 | return -EINVAL; | |
774 | if (cfg_dart->force_bypass != dart->force_bypass) | |
775 | return -EINVAL; | |
776 | if (cfg_dart->pgsize != dart->pgsize) | |
777 | return -EINVAL; | |
778 | } | |
779 | ||
780 | for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { | |
781 | if (cfg->stream_maps[i].dart == dart) { | |
510d4072 | 782 | set_bit(sid, cfg->stream_maps[i].sidmap); |
46d1fb07 SP |
783 | return 0; |
784 | } | |
785 | } | |
786 | for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { | |
787 | if (!cfg->stream_maps[i].dart) { | |
788 | cfg->stream_maps[i].dart = dart; | |
510d4072 | 789 | set_bit(sid, cfg->stream_maps[i].sidmap); |
46d1fb07 SP |
790 | return 0; |
791 | } | |
792 | } | |
793 | ||
794 | return -EINVAL; | |
795 | } | |
796 | ||
f0b63680 SP |
797 | static DEFINE_MUTEX(apple_dart_groups_lock); |
798 | ||
799 | static void apple_dart_release_group(void *iommu_data) | |
800 | { | |
801 | int i, sid; | |
802 | struct apple_dart_stream_map *stream_map; | |
803 | struct apple_dart_master_cfg *group_master_cfg = iommu_data; | |
804 | ||
805 | mutex_lock(&apple_dart_groups_lock); | |
806 | ||
807 | for_each_stream_map(i, group_master_cfg, stream_map) | |
510d4072 | 808 | for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) |
f0b63680 SP |
809 | stream_map->dart->sid2group[sid] = NULL; |
810 | ||
811 | kfree(iommu_data); | |
812 | mutex_unlock(&apple_dart_groups_lock); | |
813 | } | |
814 | ||
cf5c1c87 SP |
815 | static int apple_dart_merge_master_cfg(struct apple_dart_master_cfg *dst, |
816 | struct apple_dart_master_cfg *src) | |
817 | { | |
818 | /* | |
819 | * We know that this function is only called for groups returned from | |
820 | * pci_device_group and that all Apple Silicon platforms never spread | |
821 | * PCIe devices from the same bus across multiple DARTs such that we can | |
822 | * just assume that both src and dst only have the same single DART. | |
823 | */ | |
824 | if (src->stream_maps[1].dart) | |
825 | return -EINVAL; | |
826 | if (dst->stream_maps[1].dart) | |
827 | return -EINVAL; | |
828 | if (src->stream_maps[0].dart != dst->stream_maps[0].dart) | |
829 | return -EINVAL; | |
830 | ||
831 | bitmap_or(dst->stream_maps[0].sidmap, | |
832 | dst->stream_maps[0].sidmap, | |
833 | src->stream_maps[0].sidmap, | |
834 | dst->stream_maps[0].dart->num_streams); | |
835 | return 0; | |
836 | } | |
837 | ||
46d1fb07 SP |
838 | static struct iommu_group *apple_dart_device_group(struct device *dev) |
839 | { | |
46d1fb07 SP |
840 | int i, sid; |
841 | struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); | |
842 | struct apple_dart_stream_map *stream_map; | |
f0b63680 | 843 | struct apple_dart_master_cfg *group_master_cfg; |
46d1fb07 SP |
844 | struct iommu_group *group = NULL; |
845 | struct iommu_group *res = ERR_PTR(-EINVAL); | |
846 | ||
f0b63680 | 847 | mutex_lock(&apple_dart_groups_lock); |
46d1fb07 SP |
848 | |
849 | for_each_stream_map(i, cfg, stream_map) { | |
510d4072 | 850 | for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) { |
46d1fb07 SP |
851 | struct iommu_group *stream_group = |
852 | stream_map->dart->sid2group[sid]; | |
853 | ||
854 | if (group && group != stream_group) { | |
855 | res = ERR_PTR(-EINVAL); | |
856 | goto out; | |
857 | } | |
858 | ||
859 | group = stream_group; | |
860 | } | |
861 | } | |
862 | ||
863 | if (group) { | |
864 | res = iommu_group_ref_get(group); | |
865 | goto out; | |
866 | } | |
867 | ||
868 | #ifdef CONFIG_PCI | |
869 | if (dev_is_pci(dev)) | |
870 | group = pci_device_group(dev); | |
871 | else | |
872 | #endif | |
873 | group = generic_device_group(dev); | |
874 | ||
f0b63680 SP |
875 | res = ERR_PTR(-ENOMEM); |
876 | if (!group) | |
877 | goto out; | |
878 | ||
cf5c1c87 SP |
879 | group_master_cfg = iommu_group_get_iommudata(group); |
880 | if (group_master_cfg) { | |
881 | int ret; | |
882 | ||
883 | ret = apple_dart_merge_master_cfg(group_master_cfg, cfg); | |
884 | if (ret) { | |
885 | dev_err(dev, "Failed to merge DART IOMMU grups.\n"); | |
886 | iommu_group_put(group); | |
887 | res = ERR_PTR(ret); | |
888 | goto out; | |
889 | } | |
890 | } else { | |
891 | group_master_cfg = kmemdup(cfg, sizeof(*group_master_cfg), | |
892 | GFP_KERNEL); | |
893 | if (!group_master_cfg) { | |
894 | iommu_group_put(group); | |
895 | goto out; | |
896 | } | |
f0b63680 | 897 | |
cf5c1c87 SP |
898 | iommu_group_set_iommudata(group, group_master_cfg, |
899 | apple_dart_release_group); | |
900 | } | |
f0b63680 | 901 | |
46d1fb07 | 902 | for_each_stream_map(i, cfg, stream_map) |
510d4072 | 903 | for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) |
46d1fb07 SP |
904 | stream_map->dart->sid2group[sid] = group; |
905 | ||
906 | res = group; | |
907 | ||
908 | out: | |
f0b63680 | 909 | mutex_unlock(&apple_dart_groups_lock); |
46d1fb07 SP |
910 | return res; |
911 | } | |
912 | ||
913 | static int apple_dart_def_domain_type(struct device *dev) | |
914 | { | |
915 | struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); | |
916 | ||
917 | if (cfg->stream_maps[0].dart->force_bypass) | |
918 | return IOMMU_DOMAIN_IDENTITY; | |
919 | if (!cfg->stream_maps[0].dart->supports_bypass) | |
920 | return IOMMU_DOMAIN_DMA; | |
921 | ||
922 | return 0; | |
923 | } | |
924 | ||
946d619f MZ |
925 | #ifndef CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR |
926 | /* Keep things compiling when CONFIG_PCI_APPLE isn't selected */ | |
927 | #define CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR 0 | |
928 | #endif | |
929 | #define DOORBELL_ADDR (CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR & PAGE_MASK) | |
930 | ||
931 | static void apple_dart_get_resv_regions(struct device *dev, | |
932 | struct list_head *head) | |
933 | { | |
934 | if (IS_ENABLED(CONFIG_PCIE_APPLE) && dev_is_pci(dev)) { | |
935 | struct iommu_resv_region *region; | |
936 | int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; | |
937 | ||
938 | region = iommu_alloc_resv_region(DOORBELL_ADDR, | |
939 | PAGE_SIZE, prot, | |
0251d010 | 940 | IOMMU_RESV_MSI, GFP_KERNEL); |
946d619f MZ |
941 | if (!region) |
942 | return; | |
943 | ||
944 | list_add_tail(®ion->list, head); | |
945 | } | |
946 | ||
947 | iommu_dma_get_resv_regions(dev, head); | |
948 | } | |
949 | ||
46d1fb07 SP |
950 | static const struct iommu_ops apple_dart_iommu_ops = { |
951 | .domain_alloc = apple_dart_domain_alloc, | |
46d1fb07 SP |
952 | .probe_device = apple_dart_probe_device, |
953 | .release_device = apple_dart_release_device, | |
954 | .device_group = apple_dart_device_group, | |
955 | .of_xlate = apple_dart_of_xlate, | |
956 | .def_domain_type = apple_dart_def_domain_type, | |
946d619f | 957 | .get_resv_regions = apple_dart_get_resv_regions, |
46d1fb07 | 958 | .pgsize_bitmap = -1UL, /* Restricted during dart probe */ |
2ac2fab5 | 959 | .owner = THIS_MODULE, |
9a630a4b LB |
960 | .default_domain_ops = &(const struct iommu_domain_ops) { |
961 | .attach_dev = apple_dart_attach_dev, | |
9a630a4b LB |
962 | .map_pages = apple_dart_map_pages, |
963 | .unmap_pages = apple_dart_unmap_pages, | |
964 | .flush_iotlb_all = apple_dart_flush_iotlb_all, | |
965 | .iotlb_sync = apple_dart_iotlb_sync, | |
966 | .iotlb_sync_map = apple_dart_iotlb_sync_map, | |
967 | .iova_to_phys = apple_dart_iova_to_phys, | |
968 | .free = apple_dart_domain_free, | |
969 | } | |
46d1fb07 SP |
970 | }; |
971 | ||
b76c68fc | 972 | static irqreturn_t apple_dart_t8020_irq(int irq, void *dev) |
46d1fb07 SP |
973 | { |
974 | struct apple_dart *dart = dev; | |
975 | const char *fault_name = NULL; | |
b76c68fc HM |
976 | u32 error = readl(dart->regs + DART_T8020_ERROR); |
977 | u32 error_code = FIELD_GET(DART_T8020_ERROR_CODE, error); | |
978 | u32 addr_lo = readl(dart->regs + DART_T8020_ERROR_ADDR_LO); | |
979 | u32 addr_hi = readl(dart->regs + DART_T8020_ERROR_ADDR_HI); | |
46d1fb07 | 980 | u64 addr = addr_lo | (((u64)addr_hi) << 32); |
b76c68fc | 981 | u8 stream_idx = FIELD_GET(DART_T8020_ERROR_STREAM, error); |
46d1fb07 | 982 | |
b76c68fc | 983 | if (!(error & DART_T8020_ERROR_FLAG)) |
46d1fb07 SP |
984 | return IRQ_NONE; |
985 | ||
986 | /* there should only be a single bit set but let's use == to be sure */ | |
b76c68fc | 987 | if (error_code == DART_T8020_ERROR_READ_FAULT) |
46d1fb07 | 988 | fault_name = "READ FAULT"; |
b76c68fc | 989 | else if (error_code == DART_T8020_ERROR_WRITE_FAULT) |
46d1fb07 | 990 | fault_name = "WRITE FAULT"; |
b76c68fc | 991 | else if (error_code == DART_T8020_ERROR_NO_PTE) |
46d1fb07 | 992 | fault_name = "NO PTE FOR IOVA"; |
b76c68fc | 993 | else if (error_code == DART_T8020_ERROR_NO_PMD) |
46d1fb07 | 994 | fault_name = "NO PMD FOR IOVA"; |
b76c68fc | 995 | else if (error_code == DART_T8020_ERROR_NO_TTBR) |
46d1fb07 SP |
996 | fault_name = "NO TTBR FOR IOVA"; |
997 | else | |
998 | fault_name = "unknown"; | |
999 | ||
1000 | dev_err_ratelimited( | |
1001 | dart->dev, | |
1002 | "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx", | |
1003 | error, stream_idx, error_code, fault_name, addr); | |
1004 | ||
b76c68fc | 1005 | writel(error, dart->regs + DART_T8020_ERROR); |
46d1fb07 SP |
1006 | return IRQ_HANDLED; |
1007 | } | |
1008 | ||
d8bcc870 | 1009 | static irqreturn_t apple_dart_t8110_irq(int irq, void *dev) |
46d1fb07 SP |
1010 | { |
1011 | struct apple_dart *dart = dev; | |
1012 | const char *fault_name = NULL; | |
d8bcc870 HM |
1013 | u32 error = readl(dart->regs + DART_T8110_ERROR); |
1014 | u32 error_code = FIELD_GET(DART_T8110_ERROR_CODE, error); | |
1015 | u32 addr_lo = readl(dart->regs + DART_T8110_ERROR_ADDR_LO); | |
1016 | u32 addr_hi = readl(dart->regs + DART_T8110_ERROR_ADDR_HI); | |
46d1fb07 | 1017 | u64 addr = addr_lo | (((u64)addr_hi) << 32); |
d8bcc870 | 1018 | u8 stream_idx = FIELD_GET(DART_T8110_ERROR_STREAM, error); |
46d1fb07 | 1019 | |
d8bcc870 | 1020 | if (!(error & DART_T8110_ERROR_FLAG)) |
46d1fb07 SP |
1021 | return IRQ_NONE; |
1022 | ||
1023 | /* there should only be a single bit set but let's use == to be sure */ | |
d8bcc870 | 1024 | if (error_code == DART_T8110_ERROR_READ_FAULT) |
46d1fb07 | 1025 | fault_name = "READ FAULT"; |
d8bcc870 | 1026 | else if (error_code == DART_T8110_ERROR_WRITE_FAULT) |
46d1fb07 | 1027 | fault_name = "WRITE FAULT"; |
d8bcc870 | 1028 | else if (error_code == DART_T8110_ERROR_NO_PTE) |
46d1fb07 | 1029 | fault_name = "NO PTE FOR IOVA"; |
d8bcc870 | 1030 | else if (error_code == DART_T8110_ERROR_NO_PMD) |
46d1fb07 | 1031 | fault_name = "NO PMD FOR IOVA"; |
d8bcc870 HM |
1032 | else if (error_code == DART_T8110_ERROR_NO_PGD) |
1033 | fault_name = "NO PGD FOR IOVA"; | |
1034 | else if (error_code == DART_T8110_ERROR_NO_TTBR) | |
46d1fb07 SP |
1035 | fault_name = "NO TTBR FOR IOVA"; |
1036 | else | |
1037 | fault_name = "unknown"; | |
1038 | ||
1039 | dev_err_ratelimited( | |
1040 | dart->dev, | |
1041 | "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx", | |
1042 | error, stream_idx, error_code, fault_name, addr); | |
1043 | ||
d8bcc870 | 1044 | writel(error, dart->regs + DART_T8110_ERROR); |
46d1fb07 SP |
1045 | return IRQ_HANDLED; |
1046 | } | |
1047 | ||
46d1fb07 SP |
1048 | static int apple_dart_probe(struct platform_device *pdev) |
1049 | { | |
1050 | int ret; | |
d8bcc870 | 1051 | u32 dart_params[4]; |
46d1fb07 SP |
1052 | struct resource *res; |
1053 | struct apple_dart *dart; | |
1054 | struct device *dev = &pdev->dev; | |
1055 | ||
1056 | dart = devm_kzalloc(dev, sizeof(*dart), GFP_KERNEL); | |
1057 | if (!dart) | |
1058 | return -ENOMEM; | |
1059 | ||
1060 | dart->dev = dev; | |
a380b8dc | 1061 | dart->hw = of_device_get_match_data(dev); |
46d1fb07 SP |
1062 | spin_lock_init(&dart->lock); |
1063 | ||
a15932f4 YY |
1064 | dart->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &res); |
1065 | if (IS_ERR(dart->regs)) | |
1066 | return PTR_ERR(dart->regs); | |
1067 | ||
46d1fb07 SP |
1068 | if (resource_size(res) < 0x4000) { |
1069 | dev_err(dev, "MMIO region too small (%pr)\n", res); | |
1070 | return -EINVAL; | |
1071 | } | |
1072 | ||
46d1fb07 SP |
1073 | dart->irq = platform_get_irq(pdev, 0); |
1074 | if (dart->irq < 0) | |
1075 | return -ENODEV; | |
1076 | ||
1077 | ret = devm_clk_bulk_get_all(dev, &dart->clks); | |
1078 | if (ret < 0) | |
1079 | return ret; | |
1080 | dart->num_clks = ret; | |
1081 | ||
1082 | ret = clk_bulk_prepare_enable(dart->num_clks, dart->clks); | |
1083 | if (ret) | |
1084 | return ret; | |
1085 | ||
46d1fb07 SP |
1086 | dart_params[0] = readl(dart->regs + DART_PARAMS1); |
1087 | dart_params[1] = readl(dart->regs + DART_PARAMS2); | |
a772a02c HM |
1088 | dart->pgsize = 1 << FIELD_GET(DART_PARAMS1_PAGE_SHIFT, dart_params[0]); |
1089 | dart->supports_bypass = dart_params[1] & DART_PARAMS2_BYPASS_SUPPORT; | |
510d4072 | 1090 | |
d8bcc870 HM |
1091 | switch (dart->hw->type) { |
1092 | case DART_T8020: | |
1093 | case DART_T6000: | |
1094 | dart->ias = 32; | |
1095 | dart->oas = dart->hw->oas; | |
1096 | dart->num_streams = dart->hw->max_sid_count; | |
1097 | break; | |
1098 | ||
1099 | case DART_T8110: | |
1100 | dart_params[2] = readl(dart->regs + DART_T8110_PARAMS3); | |
1101 | dart_params[3] = readl(dart->regs + DART_T8110_PARAMS4); | |
1102 | dart->ias = FIELD_GET(DART_T8110_PARAMS3_VA_WIDTH, dart_params[2]); | |
1103 | dart->oas = FIELD_GET(DART_T8110_PARAMS3_PA_WIDTH, dart_params[2]); | |
1104 | dart->num_streams = FIELD_GET(DART_T8110_PARAMS4_NUM_SIDS, dart_params[3]); | |
1105 | break; | |
1106 | } | |
510d4072 HM |
1107 | |
1108 | if (dart->num_streams > DART_MAX_STREAMS) { | |
1109 | dev_err(&pdev->dev, "Too many streams (%d > %d)\n", | |
1110 | dart->num_streams, DART_MAX_STREAMS); | |
1111 | ret = -EINVAL; | |
1112 | goto err_clk_disable; | |
1113 | } | |
1114 | ||
46d1fb07 SP |
1115 | dart->force_bypass = dart->pgsize > PAGE_SIZE; |
1116 | ||
510d4072 HM |
1117 | ret = apple_dart_hw_reset(dart); |
1118 | if (ret) | |
1119 | goto err_clk_disable; | |
1120 | ||
b76c68fc | 1121 | ret = request_irq(dart->irq, dart->hw->irq_handler, IRQF_SHARED, |
46d1fb07 SP |
1122 | "apple-dart fault handler", dart); |
1123 | if (ret) | |
1124 | goto err_clk_disable; | |
1125 | ||
1126 | platform_set_drvdata(pdev, dart); | |
1127 | ||
46d1fb07 SP |
1128 | ret = iommu_device_sysfs_add(&dart->iommu, dev, NULL, "apple-dart.%s", |
1129 | dev_name(&pdev->dev)); | |
1130 | if (ret) | |
006abbe3 | 1131 | goto err_free_irq; |
46d1fb07 SP |
1132 | |
1133 | ret = iommu_device_register(&dart->iommu, &apple_dart_iommu_ops, dev); | |
1134 | if (ret) | |
1135 | goto err_sysfs_remove; | |
1136 | ||
1137 | dev_info( | |
1138 | &pdev->dev, | |
510d4072 HM |
1139 | "DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d] initialized\n", |
1140 | dart->pgsize, dart->num_streams, dart->supports_bypass, dart->force_bypass); | |
46d1fb07 SP |
1141 | return 0; |
1142 | ||
1143 | err_sysfs_remove: | |
1144 | iommu_device_sysfs_remove(&dart->iommu); | |
46d1fb07 SP |
1145 | err_free_irq: |
1146 | free_irq(dart->irq, dart); | |
1147 | err_clk_disable: | |
1148 | clk_bulk_disable_unprepare(dart->num_clks, dart->clks); | |
1149 | ||
1150 | return ret; | |
1151 | } | |
1152 | ||
f8047318 | 1153 | static void apple_dart_remove(struct platform_device *pdev) |
46d1fb07 SP |
1154 | { |
1155 | struct apple_dart *dart = platform_get_drvdata(pdev); | |
1156 | ||
1157 | apple_dart_hw_reset(dart); | |
1158 | free_irq(dart->irq, dart); | |
46d1fb07 SP |
1159 | |
1160 | iommu_device_unregister(&dart->iommu); | |
1161 | iommu_device_sysfs_remove(&dart->iommu); | |
1162 | ||
1163 | clk_bulk_disable_unprepare(dart->num_clks, dart->clks); | |
46d1fb07 SP |
1164 | } |
1165 | ||
a380b8dc | 1166 | static const struct apple_dart_hw apple_dart_hw_t8103 = { |
d8bcc870 | 1167 | .type = DART_T8020, |
b76c68fc HM |
1168 | .irq_handler = apple_dart_t8020_irq, |
1169 | .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb, | |
a380b8dc SP |
1170 | .oas = 36, |
1171 | .fmt = APPLE_DART, | |
510d4072 | 1172 | .max_sid_count = 16, |
0b459bcd | 1173 | |
b76c68fc HM |
1174 | .enable_streams = DART_T8020_STREAMS_ENABLE, |
1175 | .lock = DART_T8020_CONFIG, | |
1176 | .lock_bit = DART_T8020_CONFIG_LOCK, | |
1177 | ||
1178 | .error = DART_T8020_ERROR, | |
1179 | ||
1180 | .tcr = DART_T8020_TCR, | |
1181 | .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE, | |
1182 | .tcr_disabled = 0, | |
1183 | .tcr_bypass = DART_T8020_TCR_BYPASS_DAPF | DART_T8020_TCR_BYPASS_DART, | |
1184 | ||
1185 | .ttbr = DART_T8020_TTBR, | |
1186 | .ttbr_valid = DART_T8020_TTBR_VALID, | |
1187 | .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT, | |
1188 | .ttbr_shift = DART_T8020_TTBR_SHIFT, | |
0b459bcd | 1189 | .ttbr_count = 4, |
a380b8dc SP |
1190 | }; |
1191 | static const struct apple_dart_hw apple_dart_hw_t6000 = { | |
d8bcc870 | 1192 | .type = DART_T6000, |
b76c68fc HM |
1193 | .irq_handler = apple_dart_t8020_irq, |
1194 | .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb, | |
a380b8dc SP |
1195 | .oas = 42, |
1196 | .fmt = APPLE_DART2, | |
510d4072 | 1197 | .max_sid_count = 16, |
0b459bcd | 1198 | |
b76c68fc HM |
1199 | .enable_streams = DART_T8020_STREAMS_ENABLE, |
1200 | .lock = DART_T8020_CONFIG, | |
1201 | .lock_bit = DART_T8020_CONFIG_LOCK, | |
1202 | ||
1203 | .error = DART_T8020_ERROR, | |
1204 | ||
1205 | .tcr = DART_T8020_TCR, | |
1206 | .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE, | |
1207 | .tcr_disabled = 0, | |
1208 | .tcr_bypass = DART_T8020_TCR_BYPASS_DAPF | DART_T8020_TCR_BYPASS_DART, | |
1209 | ||
1210 | .ttbr = DART_T8020_TTBR, | |
1211 | .ttbr_valid = DART_T8020_TTBR_VALID, | |
1212 | .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT, | |
1213 | .ttbr_shift = DART_T8020_TTBR_SHIFT, | |
0b459bcd | 1214 | .ttbr_count = 4, |
a380b8dc SP |
1215 | }; |
1216 | ||
d8bcc870 HM |
1217 | static const struct apple_dart_hw apple_dart_hw_t8110 = { |
1218 | .type = DART_T8110, | |
1219 | .irq_handler = apple_dart_t8110_irq, | |
1220 | .invalidate_tlb = apple_dart_t8110_hw_invalidate_tlb, | |
1221 | .fmt = APPLE_DART2, | |
1222 | .max_sid_count = 256, | |
1223 | ||
1224 | .enable_streams = DART_T8110_ENABLE_STREAMS, | |
1225 | .lock = DART_T8110_PROTECT, | |
1226 | .lock_bit = DART_T8110_PROTECT_TTBR_TCR, | |
1227 | ||
1228 | .error = DART_T8110_ERROR, | |
1229 | ||
1230 | .tcr = DART_T8110_TCR, | |
1231 | .tcr_enabled = DART_T8110_TCR_TRANSLATE_ENABLE, | |
1232 | .tcr_disabled = 0, | |
1233 | .tcr_bypass = DART_T8110_TCR_BYPASS_DAPF | DART_T8110_TCR_BYPASS_DART, | |
1234 | ||
1235 | .ttbr = DART_T8110_TTBR, | |
1236 | .ttbr_valid = DART_T8110_TTBR_VALID, | |
1237 | .ttbr_addr_field_shift = DART_T8110_TTBR_ADDR_FIELD_SHIFT, | |
1238 | .ttbr_shift = DART_T8110_TTBR_SHIFT, | |
1239 | .ttbr_count = 1, | |
1240 | }; | |
1241 | ||
3d68bbb8 HM |
1242 | static __maybe_unused int apple_dart_suspend(struct device *dev) |
1243 | { | |
1244 | struct apple_dart *dart = dev_get_drvdata(dev); | |
1245 | unsigned int sid, idx; | |
1246 | ||
510d4072 | 1247 | for (sid = 0; sid < dart->num_streams; sid++) { |
b76c68fc | 1248 | dart->save_tcr[sid] = readl_relaxed(dart->regs + DART_TCR(dart, sid)); |
0b459bcd | 1249 | for (idx = 0; idx < dart->hw->ttbr_count; idx++) |
3d68bbb8 | 1250 | dart->save_ttbr[sid][idx] = |
0b459bcd | 1251 | readl(dart->regs + DART_TTBR(dart, sid, idx)); |
3d68bbb8 HM |
1252 | } |
1253 | ||
1254 | return 0; | |
1255 | } | |
1256 | ||
1257 | static __maybe_unused int apple_dart_resume(struct device *dev) | |
1258 | { | |
1259 | struct apple_dart *dart = dev_get_drvdata(dev); | |
1260 | unsigned int sid, idx; | |
1261 | int ret; | |
1262 | ||
1263 | ret = apple_dart_hw_reset(dart); | |
1264 | if (ret) { | |
1265 | dev_err(dev, "Failed to reset DART on resume\n"); | |
1266 | return ret; | |
1267 | } | |
1268 | ||
510d4072 | 1269 | for (sid = 0; sid < dart->num_streams; sid++) { |
0b459bcd | 1270 | for (idx = 0; idx < dart->hw->ttbr_count; idx++) |
3d68bbb8 | 1271 | writel(dart->save_ttbr[sid][idx], |
0b459bcd | 1272 | dart->regs + DART_TTBR(dart, sid, idx)); |
b76c68fc | 1273 | writel(dart->save_tcr[sid], dart->regs + DART_TCR(dart, sid)); |
3d68bbb8 HM |
1274 | } |
1275 | ||
1276 | return 0; | |
1277 | } | |
1278 | ||
1279 | DEFINE_SIMPLE_DEV_PM_OPS(apple_dart_pm_ops, apple_dart_suspend, apple_dart_resume); | |
1280 | ||
46d1fb07 | 1281 | static const struct of_device_id apple_dart_of_match[] = { |
a380b8dc | 1282 | { .compatible = "apple,t8103-dart", .data = &apple_dart_hw_t8103 }, |
d8bcc870 | 1283 | { .compatible = "apple,t8110-dart", .data = &apple_dart_hw_t8110 }, |
a380b8dc | 1284 | { .compatible = "apple,t6000-dart", .data = &apple_dart_hw_t6000 }, |
46d1fb07 SP |
1285 | {}, |
1286 | }; | |
1287 | MODULE_DEVICE_TABLE(of, apple_dart_of_match); | |
1288 | ||
1289 | static struct platform_driver apple_dart_driver = { | |
1290 | .driver = { | |
1291 | .name = "apple-dart", | |
1292 | .of_match_table = apple_dart_of_match, | |
1293 | .suppress_bind_attrs = true, | |
3d68bbb8 | 1294 | .pm = pm_sleep_ptr(&apple_dart_pm_ops), |
46d1fb07 SP |
1295 | }, |
1296 | .probe = apple_dart_probe, | |
f8047318 | 1297 | .remove_new = apple_dart_remove, |
46d1fb07 SP |
1298 | }; |
1299 | ||
1300 | module_platform_driver(apple_dart_driver); | |
1301 | ||
1302 | MODULE_DESCRIPTION("IOMMU API for Apple's DART"); | |
1303 | MODULE_AUTHOR("Sven Peter <sven@svenpeter.dev>"); | |
1304 | MODULE_LICENSE("GPL v2"); |