Commit | Line | Data |
---|---|---|
d2912cb1 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
89184651 TR |
2 | /* |
3 | * Copyright (C) 2014 NVIDIA CORPORATION. All rights reserved. | |
89184651 TR |
4 | */ |
5 | ||
6 | #include <linux/clk.h> | |
20e92462 | 7 | #include <linux/delay.h> |
c4c21f22 | 8 | #include <linux/dma-mapping.h> |
0c56eda8 | 9 | #include <linux/export.h> |
89184651 TR |
10 | #include <linux/interrupt.h> |
11 | #include <linux/kernel.h> | |
12 | #include <linux/module.h> | |
13 | #include <linux/of.h> | |
59cd046f | 14 | #include <linux/of_device.h> |
89184651 TR |
15 | #include <linux/platform_device.h> |
16 | #include <linux/slab.h> | |
3d9dd6fd | 17 | #include <linux/sort.h> |
9a38cb27 | 18 | #include <linux/tegra-icc.h> |
3d9dd6fd MP |
19 | |
20 | #include <soc/tegra/fuse.h> | |
89184651 TR |
21 | |
22 | #include "mc.h" | |
23 | ||
89184651 | 24 | static const struct of_device_id tegra_mc_of_match[] = { |
a8d502fd | 25 | #ifdef CONFIG_ARCH_TEGRA_2x_SOC |
96efa118 | 26 | { .compatible = "nvidia,tegra20-mc-gart", .data = &tegra20_mc_soc }, |
a8d502fd | 27 | #endif |
89184651 TR |
28 | #ifdef CONFIG_ARCH_TEGRA_3x_SOC |
29 | { .compatible = "nvidia,tegra30-mc", .data = &tegra30_mc_soc }, | |
30 | #endif | |
31 | #ifdef CONFIG_ARCH_TEGRA_114_SOC | |
32 | { .compatible = "nvidia,tegra114-mc", .data = &tegra114_mc_soc }, | |
33 | #endif | |
34 | #ifdef CONFIG_ARCH_TEGRA_124_SOC | |
35 | { .compatible = "nvidia,tegra124-mc", .data = &tegra124_mc_soc }, | |
242b1d71 TR |
36 | #endif |
37 | #ifdef CONFIG_ARCH_TEGRA_132_SOC | |
38 | { .compatible = "nvidia,tegra132-mc", .data = &tegra132_mc_soc }, | |
588c43a7 TR |
39 | #endif |
40 | #ifdef CONFIG_ARCH_TEGRA_210_SOC | |
41 | { .compatible = "nvidia,tegra210-mc", .data = &tegra210_mc_soc }, | |
89184651 | 42 | #endif |
7355c7b9 TR |
43 | #ifdef CONFIG_ARCH_TEGRA_186_SOC |
44 | { .compatible = "nvidia,tegra186-mc", .data = &tegra186_mc_soc }, | |
45 | #endif | |
46 | #ifdef CONFIG_ARCH_TEGRA_194_SOC | |
47 | { .compatible = "nvidia,tegra194-mc", .data = &tegra194_mc_soc }, | |
72c81bb6 TR |
48 | #endif |
49 | #ifdef CONFIG_ARCH_TEGRA_234_SOC | |
50 | { .compatible = "nvidia,tegra234-mc", .data = &tegra234_mc_soc }, | |
7355c7b9 TR |
51 | #endif |
52 | { /* sentinel */ } | |
89184651 TR |
53 | }; |
54 | MODULE_DEVICE_TABLE(of, tegra_mc_of_match); | |
55 | ||
6c6bd207 DO |
56 | static void tegra_mc_devm_action_put_device(void *data) |
57 | { | |
58 | struct tegra_mc *mc = data; | |
59 | ||
60 | put_device(mc->dev); | |
61 | } | |
62 | ||
63 | /** | |
64 | * devm_tegra_memory_controller_get() - get Tegra Memory Controller handle | |
65 | * @dev: device pointer for the consumer device | |
66 | * | |
67 | * This function will search for the Memory Controller node in a device-tree | |
68 | * and retrieve the Memory Controller handle. | |
69 | * | |
70 | * Return: ERR_PTR() on error or a valid pointer to a struct tegra_mc. | |
71 | */ | |
72 | struct tegra_mc *devm_tegra_memory_controller_get(struct device *dev) | |
73 | { | |
74 | struct platform_device *pdev; | |
75 | struct device_node *np; | |
76 | struct tegra_mc *mc; | |
77 | int err; | |
78 | ||
79 | np = of_parse_phandle(dev->of_node, "nvidia,memory-controller", 0); | |
80 | if (!np) | |
81 | return ERR_PTR(-ENOENT); | |
82 | ||
83 | pdev = of_find_device_by_node(np); | |
84 | of_node_put(np); | |
85 | if (!pdev) | |
86 | return ERR_PTR(-ENODEV); | |
87 | ||
88 | mc = platform_get_drvdata(pdev); | |
89 | if (!mc) { | |
90 | put_device(&pdev->dev); | |
91 | return ERR_PTR(-EPROBE_DEFER); | |
92 | } | |
93 | ||
1d8e0223 CH |
94 | err = devm_add_action_or_reset(dev, tegra_mc_devm_action_put_device, mc); |
95 | if (err) | |
6c6bd207 | 96 | return ERR_PTR(err); |
6c6bd207 DO |
97 | |
98 | return mc; | |
99 | } | |
100 | EXPORT_SYMBOL_GPL(devm_tegra_memory_controller_get); | |
101 | ||
393d66fd TR |
102 | int tegra_mc_probe_device(struct tegra_mc *mc, struct device *dev) |
103 | { | |
104 | if (mc->soc->ops && mc->soc->ops->probe_device) | |
105 | return mc->soc->ops->probe_device(mc, dev); | |
106 | ||
107 | return 0; | |
108 | } | |
109 | EXPORT_SYMBOL_GPL(tegra_mc_probe_device); | |
110 | ||
7946920d MP |
111 | int tegra_mc_get_carveout_info(struct tegra_mc *mc, unsigned int id, |
112 | phys_addr_t *base, u64 *size) | |
113 | { | |
114 | u32 offset; | |
115 | ||
116 | if (id < 1 || id >= mc->soc->num_carveouts) | |
117 | return -EINVAL; | |
118 | ||
119 | if (id < 6) | |
120 | offset = 0xc0c + 0x50 * (id - 1); | |
121 | else | |
122 | offset = 0x2004 + 0x50 * (id - 6); | |
123 | ||
124 | *base = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, offset + 0x0); | |
125 | #ifdef CONFIG_PHYS_ADDR_T_64BIT | |
126 | *base |= (phys_addr_t)mc_ch_readl(mc, MC_BROADCAST_CHANNEL, offset + 0x4) << 32; | |
127 | #endif | |
128 | ||
129 | if (size) | |
130 | *size = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, offset + 0x8) << 17; | |
131 | ||
132 | return 0; | |
133 | } | |
134 | EXPORT_SYMBOL_GPL(tegra_mc_get_carveout_info); | |
135 | ||
cb2b5839 | 136 | static int tegra_mc_block_dma_common(struct tegra_mc *mc, |
20e92462 DO |
137 | const struct tegra_mc_reset *rst) |
138 | { | |
139 | unsigned long flags; | |
140 | u32 value; | |
141 | ||
142 | spin_lock_irqsave(&mc->lock, flags); | |
143 | ||
144 | value = mc_readl(mc, rst->control) | BIT(rst->bit); | |
145 | mc_writel(mc, value, rst->control); | |
146 | ||
147 | spin_unlock_irqrestore(&mc->lock, flags); | |
148 | ||
149 | return 0; | |
150 | } | |
151 | ||
cb2b5839 | 152 | static bool tegra_mc_dma_idling_common(struct tegra_mc *mc, |
20e92462 DO |
153 | const struct tegra_mc_reset *rst) |
154 | { | |
155 | return (mc_readl(mc, rst->status) & BIT(rst->bit)) != 0; | |
156 | } | |
157 | ||
cb2b5839 | 158 | static int tegra_mc_unblock_dma_common(struct tegra_mc *mc, |
20e92462 DO |
159 | const struct tegra_mc_reset *rst) |
160 | { | |
161 | unsigned long flags; | |
162 | u32 value; | |
163 | ||
164 | spin_lock_irqsave(&mc->lock, flags); | |
165 | ||
166 | value = mc_readl(mc, rst->control) & ~BIT(rst->bit); | |
167 | mc_writel(mc, value, rst->control); | |
168 | ||
169 | spin_unlock_irqrestore(&mc->lock, flags); | |
170 | ||
171 | return 0; | |
172 | } | |
173 | ||
cb2b5839 | 174 | static int tegra_mc_reset_status_common(struct tegra_mc *mc, |
20e92462 DO |
175 | const struct tegra_mc_reset *rst) |
176 | { | |
177 | return (mc_readl(mc, rst->control) & BIT(rst->bit)) != 0; | |
178 | } | |
179 | ||
cb2b5839 TR |
180 | const struct tegra_mc_reset_ops tegra_mc_reset_ops_common = { |
181 | .block_dma = tegra_mc_block_dma_common, | |
182 | .dma_idling = tegra_mc_dma_idling_common, | |
183 | .unblock_dma = tegra_mc_unblock_dma_common, | |
184 | .reset_status = tegra_mc_reset_status_common, | |
20e92462 DO |
185 | }; |
186 | ||
187 | static inline struct tegra_mc *reset_to_mc(struct reset_controller_dev *rcdev) | |
188 | { | |
189 | return container_of(rcdev, struct tegra_mc, reset); | |
190 | } | |
191 | ||
192 | static const struct tegra_mc_reset *tegra_mc_reset_find(struct tegra_mc *mc, | |
193 | unsigned long id) | |
194 | { | |
195 | unsigned int i; | |
196 | ||
197 | for (i = 0; i < mc->soc->num_resets; i++) | |
198 | if (mc->soc->resets[i].id == id) | |
199 | return &mc->soc->resets[i]; | |
200 | ||
201 | return NULL; | |
202 | } | |
203 | ||
204 | static int tegra_mc_hotreset_assert(struct reset_controller_dev *rcdev, | |
205 | unsigned long id) | |
206 | { | |
207 | struct tegra_mc *mc = reset_to_mc(rcdev); | |
208 | const struct tegra_mc_reset_ops *rst_ops; | |
209 | const struct tegra_mc_reset *rst; | |
210 | int retries = 500; | |
211 | int err; | |
212 | ||
213 | rst = tegra_mc_reset_find(mc, id); | |
214 | if (!rst) | |
215 | return -ENODEV; | |
216 | ||
217 | rst_ops = mc->soc->reset_ops; | |
218 | if (!rst_ops) | |
219 | return -ENODEV; | |
220 | ||
6ce84ab6 DO |
221 | /* DMA flushing will fail if reset is already asserted */ |
222 | if (rst_ops->reset_status) { | |
223 | /* check whether reset is asserted */ | |
224 | if (rst_ops->reset_status(mc, rst)) | |
225 | return 0; | |
226 | } | |
227 | ||
20e92462 DO |
228 | if (rst_ops->block_dma) { |
229 | /* block clients DMA requests */ | |
230 | err = rst_ops->block_dma(mc, rst); | |
231 | if (err) { | |
f2dcded1 | 232 | dev_err(mc->dev, "failed to block %s DMA: %d\n", |
20e92462 DO |
233 | rst->name, err); |
234 | return err; | |
235 | } | |
236 | } | |
237 | ||
238 | if (rst_ops->dma_idling) { | |
239 | /* wait for completion of the outstanding DMA requests */ | |
240 | while (!rst_ops->dma_idling(mc, rst)) { | |
241 | if (!retries--) { | |
f2dcded1 | 242 | dev_err(mc->dev, "failed to flush %s DMA\n", |
20e92462 DO |
243 | rst->name); |
244 | return -EBUSY; | |
245 | } | |
246 | ||
247 | usleep_range(10, 100); | |
248 | } | |
249 | } | |
250 | ||
251 | if (rst_ops->hotreset_assert) { | |
252 | /* clear clients DMA requests sitting before arbitration */ | |
253 | err = rst_ops->hotreset_assert(mc, rst); | |
254 | if (err) { | |
f2dcded1 | 255 | dev_err(mc->dev, "failed to hot reset %s: %d\n", |
20e92462 DO |
256 | rst->name, err); |
257 | return err; | |
258 | } | |
259 | } | |
260 | ||
261 | return 0; | |
262 | } | |
263 | ||
264 | static int tegra_mc_hotreset_deassert(struct reset_controller_dev *rcdev, | |
265 | unsigned long id) | |
266 | { | |
267 | struct tegra_mc *mc = reset_to_mc(rcdev); | |
268 | const struct tegra_mc_reset_ops *rst_ops; | |
269 | const struct tegra_mc_reset *rst; | |
270 | int err; | |
271 | ||
272 | rst = tegra_mc_reset_find(mc, id); | |
273 | if (!rst) | |
274 | return -ENODEV; | |
275 | ||
276 | rst_ops = mc->soc->reset_ops; | |
277 | if (!rst_ops) | |
278 | return -ENODEV; | |
279 | ||
280 | if (rst_ops->hotreset_deassert) { | |
281 | /* take out client from hot reset */ | |
282 | err = rst_ops->hotreset_deassert(mc, rst); | |
283 | if (err) { | |
f2dcded1 | 284 | dev_err(mc->dev, "failed to deassert hot reset %s: %d\n", |
20e92462 DO |
285 | rst->name, err); |
286 | return err; | |
287 | } | |
288 | } | |
289 | ||
290 | if (rst_ops->unblock_dma) { | |
291 | /* allow new DMA requests to proceed to arbitration */ | |
292 | err = rst_ops->unblock_dma(mc, rst); | |
293 | if (err) { | |
f2dcded1 | 294 | dev_err(mc->dev, "failed to unblock %s DMA : %d\n", |
20e92462 DO |
295 | rst->name, err); |
296 | return err; | |
297 | } | |
298 | } | |
299 | ||
300 | return 0; | |
301 | } | |
302 | ||
303 | static int tegra_mc_hotreset_status(struct reset_controller_dev *rcdev, | |
304 | unsigned long id) | |
305 | { | |
306 | struct tegra_mc *mc = reset_to_mc(rcdev); | |
307 | const struct tegra_mc_reset_ops *rst_ops; | |
308 | const struct tegra_mc_reset *rst; | |
309 | ||
310 | rst = tegra_mc_reset_find(mc, id); | |
311 | if (!rst) | |
312 | return -ENODEV; | |
313 | ||
314 | rst_ops = mc->soc->reset_ops; | |
315 | if (!rst_ops) | |
316 | return -ENODEV; | |
317 | ||
318 | return rst_ops->reset_status(mc, rst); | |
319 | } | |
320 | ||
321 | static const struct reset_control_ops tegra_mc_reset_ops = { | |
322 | .assert = tegra_mc_hotreset_assert, | |
323 | .deassert = tegra_mc_hotreset_deassert, | |
324 | .status = tegra_mc_hotreset_status, | |
325 | }; | |
326 | ||
327 | static int tegra_mc_reset_setup(struct tegra_mc *mc) | |
328 | { | |
329 | int err; | |
330 | ||
331 | mc->reset.ops = &tegra_mc_reset_ops; | |
332 | mc->reset.owner = THIS_MODULE; | |
333 | mc->reset.of_node = mc->dev->of_node; | |
334 | mc->reset.of_reset_n_cells = 1; | |
335 | mc->reset.nr_resets = mc->soc->num_resets; | |
336 | ||
337 | err = reset_controller_register(&mc->reset); | |
338 | if (err < 0) | |
339 | return err; | |
340 | ||
341 | return 0; | |
342 | } | |
343 | ||
e34212c7 | 344 | int tegra_mc_write_emem_configuration(struct tegra_mc *mc, unsigned long rate) |
3d9dd6fd MP |
345 | { |
346 | unsigned int i; | |
347 | struct tegra_mc_timing *timing = NULL; | |
348 | ||
349 | for (i = 0; i < mc->num_timings; i++) { | |
350 | if (mc->timings[i].rate == rate) { | |
351 | timing = &mc->timings[i]; | |
352 | break; | |
353 | } | |
354 | } | |
355 | ||
356 | if (!timing) { | |
357 | dev_err(mc->dev, "no memory timing registered for rate %lu\n", | |
358 | rate); | |
e34212c7 | 359 | return -EINVAL; |
3d9dd6fd MP |
360 | } |
361 | ||
362 | for (i = 0; i < mc->soc->num_emem_regs; ++i) | |
363 | mc_writel(mc, timing->emem_data[i], mc->soc->emem_regs[i]); | |
e34212c7 DO |
364 | |
365 | return 0; | |
3d9dd6fd | 366 | } |
0c56eda8 | 367 | EXPORT_SYMBOL_GPL(tegra_mc_write_emem_configuration); |
3d9dd6fd MP |
368 | |
369 | unsigned int tegra_mc_get_emem_device_count(struct tegra_mc *mc) | |
370 | { | |
371 | u8 dram_count; | |
372 | ||
373 | dram_count = mc_readl(mc, MC_EMEM_ADR_CFG); | |
374 | dram_count &= MC_EMEM_ADR_CFG_EMEM_NUMDEV; | |
375 | dram_count++; | |
376 | ||
377 | return dram_count; | |
378 | } | |
0c56eda8 | 379 | EXPORT_SYMBOL_GPL(tegra_mc_get_emem_device_count); |
3d9dd6fd | 380 | |
ddeceab0 TR |
381 | #if defined(CONFIG_ARCH_TEGRA_3x_SOC) || \ |
382 | defined(CONFIG_ARCH_TEGRA_114_SOC) || \ | |
383 | defined(CONFIG_ARCH_TEGRA_124_SOC) || \ | |
384 | defined(CONFIG_ARCH_TEGRA_132_SOC) || \ | |
385 | defined(CONFIG_ARCH_TEGRA_210_SOC) | |
386 | static int tegra_mc_setup_latency_allowance(struct tegra_mc *mc) | |
387 | { | |
388 | unsigned long long tick; | |
389 | unsigned int i; | |
390 | u32 value; | |
391 | ||
392 | /* compute the number of MC clock cycles per tick */ | |
393 | tick = (unsigned long long)mc->tick * clk_get_rate(mc->clk); | |
394 | do_div(tick, NSEC_PER_SEC); | |
395 | ||
396 | value = mc_readl(mc, MC_EMEM_ARB_CFG); | |
397 | value &= ~MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK; | |
398 | value |= MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(tick); | |
399 | mc_writel(mc, value, MC_EMEM_ARB_CFG); | |
400 | ||
401 | /* write latency allowance defaults */ | |
402 | for (i = 0; i < mc->soc->num_clients; i++) { | |
403 | const struct tegra_mc_client *client = &mc->soc->clients[i]; | |
404 | u32 value; | |
405 | ||
406 | value = mc_readl(mc, client->regs.la.reg); | |
407 | value &= ~(client->regs.la.mask << client->regs.la.shift); | |
408 | value |= (client->regs.la.def & client->regs.la.mask) << client->regs.la.shift; | |
409 | mc_writel(mc, value, client->regs.la.reg); | |
410 | } | |
411 | ||
412 | /* latch new values */ | |
413 | mc_writel(mc, MC_TIMING_UPDATE, MC_TIMING_CONTROL); | |
414 | ||
415 | return 0; | |
416 | } | |
417 | ||
3d9dd6fd MP |
418 | static int load_one_timing(struct tegra_mc *mc, |
419 | struct tegra_mc_timing *timing, | |
420 | struct device_node *node) | |
421 | { | |
422 | int err; | |
423 | u32 tmp; | |
424 | ||
425 | err = of_property_read_u32(node, "clock-frequency", &tmp); | |
426 | if (err) { | |
427 | dev_err(mc->dev, | |
c86f9854 | 428 | "timing %pOFn: failed to read rate\n", node); |
3d9dd6fd MP |
429 | return err; |
430 | } | |
431 | ||
432 | timing->rate = tmp; | |
433 | timing->emem_data = devm_kcalloc(mc->dev, mc->soc->num_emem_regs, | |
434 | sizeof(u32), GFP_KERNEL); | |
435 | if (!timing->emem_data) | |
436 | return -ENOMEM; | |
437 | ||
438 | err = of_property_read_u32_array(node, "nvidia,emem-configuration", | |
439 | timing->emem_data, | |
440 | mc->soc->num_emem_regs); | |
441 | if (err) { | |
442 | dev_err(mc->dev, | |
c86f9854 RH |
443 | "timing %pOFn: failed to read EMEM configuration\n", |
444 | node); | |
3d9dd6fd MP |
445 | return err; |
446 | } | |
447 | ||
448 | return 0; | |
449 | } | |
450 | ||
451 | static int load_timings(struct tegra_mc *mc, struct device_node *node) | |
452 | { | |
453 | struct device_node *child; | |
454 | struct tegra_mc_timing *timing; | |
455 | int child_count = of_get_child_count(node); | |
456 | int i = 0, err; | |
457 | ||
458 | mc->timings = devm_kcalloc(mc->dev, child_count, sizeof(*timing), | |
459 | GFP_KERNEL); | |
460 | if (!mc->timings) | |
461 | return -ENOMEM; | |
462 | ||
463 | mc->num_timings = child_count; | |
464 | ||
465 | for_each_child_of_node(node, child) { | |
466 | timing = &mc->timings[i++]; | |
467 | ||
468 | err = load_one_timing(mc, timing, child); | |
55bb1d83 AKC |
469 | if (err) { |
470 | of_node_put(child); | |
3d9dd6fd | 471 | return err; |
55bb1d83 | 472 | } |
3d9dd6fd MP |
473 | } |
474 | ||
475 | return 0; | |
476 | } | |
477 | ||
478 | static int tegra_mc_setup_timings(struct tegra_mc *mc) | |
479 | { | |
480 | struct device_node *node; | |
481 | u32 ram_code, node_ram_code; | |
482 | int err; | |
483 | ||
484 | ram_code = tegra_read_ram_code(); | |
485 | ||
486 | mc->num_timings = 0; | |
487 | ||
488 | for_each_child_of_node(mc->dev->of_node, node) { | |
489 | err = of_property_read_u32(node, "nvidia,ram-code", | |
490 | &node_ram_code); | |
d1122e4b | 491 | if (err || (node_ram_code != ram_code)) |
3d9dd6fd | 492 | continue; |
3d9dd6fd MP |
493 | |
494 | err = load_timings(mc, node); | |
55bb1d83 | 495 | of_node_put(node); |
3d9dd6fd MP |
496 | if (err) |
497 | return err; | |
3d9dd6fd MP |
498 | break; |
499 | } | |
500 | ||
501 | if (mc->num_timings == 0) | |
502 | dev_warn(mc->dev, | |
503 | "no memory timings for RAM code %u registered\n", | |
504 | ram_code); | |
505 | ||
506 | return 0; | |
507 | } | |
508 | ||
ddeceab0 TR |
509 | int tegra30_mc_probe(struct tegra_mc *mc) |
510 | { | |
511 | int err; | |
512 | ||
513 | mc->clk = devm_clk_get_optional(mc->dev, "mc"); | |
514 | if (IS_ERR(mc->clk)) { | |
515 | dev_err(mc->dev, "failed to get MC clock: %ld\n", PTR_ERR(mc->clk)); | |
516 | return PTR_ERR(mc->clk); | |
517 | } | |
518 | ||
519 | /* ensure that debug features are disabled */ | |
520 | mc_writel(mc, 0x00000000, MC_TIMING_CONTROL_DBG); | |
521 | ||
522 | err = tegra_mc_setup_latency_allowance(mc); | |
523 | if (err < 0) { | |
524 | dev_err(mc->dev, "failed to setup latency allowance: %d\n", err); | |
525 | return err; | |
526 | } | |
527 | ||
528 | err = tegra_mc_setup_timings(mc); | |
529 | if (err < 0) { | |
530 | dev_err(mc->dev, "failed to setup timings: %d\n", err); | |
531 | return err; | |
532 | } | |
533 | ||
534 | return 0; | |
535 | } | |
536 | ||
54a85e09 AM |
537 | const struct tegra_mc_ops tegra30_mc_ops = { |
538 | .probe = tegra30_mc_probe, | |
539 | .handle_irq = tegra30_mc_handle_irq, | |
540 | }; | |
541 | #endif | |
542 | ||
543 | static int mc_global_intstatus_to_channel(const struct tegra_mc *mc, u32 status, | |
544 | unsigned int *mc_channel) | |
545 | { | |
546 | if ((status & mc->soc->ch_intmask) == 0) | |
547 | return -EINVAL; | |
548 | ||
549 | *mc_channel = __ffs((status & mc->soc->ch_intmask) >> | |
550 | mc->soc->global_intstatus_channel_shift); | |
551 | ||
552 | return 0; | |
553 | } | |
554 | ||
555 | static u32 mc_channel_to_global_intstatus(const struct tegra_mc *mc, | |
556 | unsigned int channel) | |
557 | { | |
558 | return BIT(channel) << mc->soc->global_intstatus_channel_shift; | |
559 | } | |
560 | ||
561 | irqreturn_t tegra30_mc_handle_irq(int irq, void *data) | |
89184651 TR |
562 | { |
563 | struct tegra_mc *mc = data; | |
54a85e09 | 564 | unsigned int bit, channel; |
1c74d5c0 | 565 | unsigned long status; |
89184651 | 566 | |
54a85e09 AM |
567 | if (mc->soc->num_channels) { |
568 | u32 global_status; | |
569 | int err; | |
570 | ||
571 | global_status = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, MC_GLOBAL_INTSTATUS); | |
572 | err = mc_global_intstatus_to_channel(mc, global_status, &channel); | |
573 | if (err < 0) { | |
574 | dev_err_ratelimited(mc->dev, "unknown interrupt channel 0x%08x\n", | |
575 | global_status); | |
576 | return IRQ_NONE; | |
577 | } | |
578 | ||
579 | /* mask all interrupts to avoid flooding */ | |
580 | status = mc_ch_readl(mc, channel, MC_INTSTATUS) & mc->soc->intmask; | |
581 | } else { | |
582 | status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask; | |
583 | } | |
584 | ||
bf3fbdfb DO |
585 | if (!status) |
586 | return IRQ_NONE; | |
89184651 TR |
587 | |
588 | for_each_set_bit(bit, &status, 32) { | |
1079a66b | 589 | const char *error = tegra_mc_status_names[bit] ?: "unknown"; |
89184651 TR |
590 | const char *client = "unknown", *desc; |
591 | const char *direction, *secure; | |
54a85e09 AM |
592 | u32 status_reg, addr_reg; |
593 | u32 intmask = BIT(bit); | |
89184651 | 594 | phys_addr_t addr = 0; |
54a85e09 AM |
595 | #ifdef CONFIG_PHYS_ADDR_T_64BIT |
596 | u32 addr_hi_reg = 0; | |
597 | #endif | |
89184651 TR |
598 | unsigned int i; |
599 | char perm[7]; | |
600 | u8 id, type; | |
601 | u32 value; | |
602 | ||
54a85e09 AM |
603 | switch (intmask) { |
604 | case MC_INT_DECERR_VPR: | |
605 | status_reg = MC_ERR_VPR_STATUS; | |
606 | addr_reg = MC_ERR_VPR_ADR; | |
607 | break; | |
608 | ||
609 | case MC_INT_SECERR_SEC: | |
610 | status_reg = MC_ERR_SEC_STATUS; | |
611 | addr_reg = MC_ERR_SEC_ADR; | |
612 | break; | |
613 | ||
614 | case MC_INT_DECERR_MTS: | |
615 | status_reg = MC_ERR_MTS_STATUS; | |
616 | addr_reg = MC_ERR_MTS_ADR; | |
617 | break; | |
618 | ||
619 | case MC_INT_DECERR_GENERALIZED_CARVEOUT: | |
620 | status_reg = MC_ERR_GENERALIZED_CARVEOUT_STATUS; | |
621 | addr_reg = MC_ERR_GENERALIZED_CARVEOUT_ADR; | |
622 | break; | |
623 | ||
624 | case MC_INT_DECERR_ROUTE_SANITY: | |
625 | status_reg = MC_ERR_ROUTE_SANITY_STATUS; | |
626 | addr_reg = MC_ERR_ROUTE_SANITY_ADR; | |
627 | break; | |
628 | ||
629 | default: | |
630 | status_reg = MC_ERR_STATUS; | |
631 | addr_reg = MC_ERR_ADR; | |
632 | ||
633 | #ifdef CONFIG_PHYS_ADDR_T_64BIT | |
634 | if (mc->soc->has_addr_hi_reg) | |
635 | addr_hi_reg = MC_ERR_ADR_HI; | |
636 | #endif | |
637 | break; | |
638 | } | |
639 | ||
640 | if (mc->soc->num_channels) | |
641 | value = mc_ch_readl(mc, channel, status_reg); | |
642 | else | |
643 | value = mc_readl(mc, status_reg); | |
89184651 TR |
644 | |
645 | #ifdef CONFIG_PHYS_ADDR_T_64BIT | |
646 | if (mc->soc->num_address_bits > 32) { | |
54a85e09 AM |
647 | if (addr_hi_reg) { |
648 | if (mc->soc->num_channels) | |
649 | addr = mc_ch_readl(mc, channel, addr_hi_reg); | |
650 | else | |
651 | addr = mc_readl(mc, addr_hi_reg); | |
652 | } else { | |
653 | addr = ((value >> MC_ERR_STATUS_ADR_HI_SHIFT) & | |
654 | MC_ERR_STATUS_ADR_HI_MASK); | |
655 | } | |
89184651 TR |
656 | addr <<= 32; |
657 | } | |
658 | #endif | |
659 | ||
660 | if (value & MC_ERR_STATUS_RW) | |
661 | direction = "write"; | |
662 | else | |
663 | direction = "read"; | |
664 | ||
665 | if (value & MC_ERR_STATUS_SECURITY) | |
666 | secure = "secure "; | |
667 | else | |
668 | secure = ""; | |
669 | ||
3c01cf3b | 670 | id = value & mc->soc->client_id_mask; |
89184651 TR |
671 | |
672 | for (i = 0; i < mc->soc->num_clients; i++) { | |
673 | if (mc->soc->clients[i].id == id) { | |
674 | client = mc->soc->clients[i].name; | |
675 | break; | |
676 | } | |
677 | } | |
678 | ||
679 | type = (value & MC_ERR_STATUS_TYPE_MASK) >> | |
680 | MC_ERR_STATUS_TYPE_SHIFT; | |
1079a66b | 681 | desc = tegra_mc_error_names[type]; |
89184651 TR |
682 | |
683 | switch (value & MC_ERR_STATUS_TYPE_MASK) { | |
684 | case MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE: | |
685 | perm[0] = ' '; | |
686 | perm[1] = '['; | |
687 | ||
688 | if (value & MC_ERR_STATUS_READABLE) | |
689 | perm[2] = 'R'; | |
690 | else | |
691 | perm[2] = '-'; | |
692 | ||
693 | if (value & MC_ERR_STATUS_WRITABLE) | |
694 | perm[3] = 'W'; | |
695 | else | |
696 | perm[3] = '-'; | |
697 | ||
698 | if (value & MC_ERR_STATUS_NONSECURE) | |
699 | perm[4] = '-'; | |
700 | else | |
701 | perm[4] = 'S'; | |
702 | ||
703 | perm[5] = ']'; | |
704 | perm[6] = '\0'; | |
705 | break; | |
706 | ||
707 | default: | |
708 | perm[0] = '\0'; | |
709 | break; | |
710 | } | |
711 | ||
54a85e09 AM |
712 | if (mc->soc->num_channels) |
713 | value = mc_ch_readl(mc, channel, addr_reg); | |
714 | else | |
715 | value = mc_readl(mc, addr_reg); | |
89184651 TR |
716 | addr |= value; |
717 | ||
718 | dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s%s)\n", | |
719 | client, secure, direction, &addr, error, | |
720 | desc, perm); | |
721 | } | |
722 | ||
723 | /* clear interrupts */ | |
54a85e09 AM |
724 | if (mc->soc->num_channels) { |
725 | mc_ch_writel(mc, channel, status, MC_INTSTATUS); | |
726 | mc_ch_writel(mc, MC_BROADCAST_CHANNEL, | |
727 | mc_channel_to_global_intstatus(mc, channel), | |
728 | MC_GLOBAL_INTSTATUS); | |
729 | } else { | |
730 | mc_writel(mc, status, MC_INTSTATUS); | |
731 | } | |
89184651 TR |
732 | |
733 | return IRQ_HANDLED; | |
734 | } | |
735 | ||
1079a66b TR |
736 | const char *const tegra_mc_status_names[32] = { |
737 | [ 1] = "External interrupt", | |
738 | [ 6] = "EMEM address decode error", | |
739 | [ 7] = "GART page fault", | |
740 | [ 8] = "Security violation", | |
741 | [ 9] = "EMEM arbitration error", | |
742 | [10] = "Page fault", | |
743 | [11] = "Invalid APB ASID update", | |
744 | [12] = "VPR violation", | |
745 | [13] = "Secure carveout violation", | |
746 | [16] = "MTS carveout violation", | |
54a85e09 AM |
747 | [17] = "Generalized carveout violation", |
748 | [20] = "Route Sanity error", | |
1079a66b | 749 | }; |
a8d502fd | 750 | |
1079a66b TR |
751 | const char *const tegra_mc_error_names[8] = { |
752 | [2] = "EMEM decode error", | |
753 | [3] = "TrustZone violation", | |
754 | [4] = "Carveout violation", | |
755 | [6] = "SMMU translation error", | |
756 | }; | |
a8d502fd | 757 | |
d1478aea TR |
758 | struct icc_node *tegra_mc_icc_xlate(struct of_phandle_args *spec, void *data) |
759 | { | |
760 | struct tegra_mc *mc = icc_provider_to_tegra_mc(data); | |
761 | struct icc_node *node; | |
762 | ||
763 | list_for_each_entry(node, &mc->provider.nodes, node_list) { | |
764 | if (node->id == spec->args[0]) | |
765 | return node; | |
766 | } | |
767 | ||
768 | /* | |
769 | * If a client driver calls devm_of_icc_get() before the MC driver | |
770 | * is probed, then return EPROBE_DEFER to the client driver. | |
771 | */ | |
772 | return ERR_PTR(-EPROBE_DEFER); | |
773 | } | |
774 | ||
775 | static int tegra_mc_icc_get(struct icc_node *node, u32 *average, u32 *peak) | |
776 | { | |
777 | *average = 0; | |
778 | *peak = 0; | |
779 | ||
780 | return 0; | |
781 | } | |
782 | ||
783 | static int tegra_mc_icc_set(struct icc_node *src, struct icc_node *dst) | |
784 | { | |
785 | return 0; | |
786 | } | |
787 | ||
788 | const struct tegra_mc_icc_ops tegra_mc_icc_ops = { | |
789 | .xlate = tegra_mc_icc_xlate, | |
790 | .aggregate = icc_std_aggregate, | |
791 | .get_bw = tegra_mc_icc_get, | |
792 | .set = tegra_mc_icc_set, | |
793 | }; | |
794 | ||
06f07981 DO |
795 | /* |
796 | * Memory Controller (MC) has few Memory Clients that are issuing memory | |
797 | * bandwidth allocation requests to the MC interconnect provider. The MC | |
798 | * provider aggregates the requests and then sends the aggregated request | |
799 | * up to the External Memory Controller (EMC) interconnect provider which | |
800 | * re-configures hardware interface to External Memory (EMEM) in accordance | |
801 | * to the required bandwidth. Each MC interconnect node represents an | |
802 | * individual Memory Client. | |
803 | * | |
804 | * Memory interconnect topology: | |
805 | * | |
806 | * +----+ | |
807 | * +--------+ | | | |
808 | * | TEXSRD +--->+ | | |
809 | * +--------+ | | | |
810 | * | | +-----+ +------+ | |
811 | * ... | MC +--->+ EMC +--->+ EMEM | | |
812 | * | | +-----+ +------+ | |
813 | * +--------+ | | | |
814 | * | DISP.. +--->+ | | |
815 | * +--------+ | | | |
816 | * +----+ | |
817 | */ | |
818 | static int tegra_mc_interconnect_setup(struct tegra_mc *mc) | |
819 | { | |
820 | struct icc_node *node; | |
821 | unsigned int i; | |
822 | int err; | |
823 | ||
824 | /* older device-trees don't have interconnect properties */ | |
825 | if (!device_property_present(mc->dev, "#interconnect-cells") || | |
826 | !mc->soc->icc_ops) | |
827 | return 0; | |
828 | ||
829 | mc->provider.dev = mc->dev; | |
830 | mc->provider.data = &mc->provider; | |
831 | mc->provider.set = mc->soc->icc_ops->set; | |
832 | mc->provider.aggregate = mc->soc->icc_ops->aggregate; | |
9a38cb27 SG |
833 | mc->provider.get_bw = mc->soc->icc_ops->get_bw; |
834 | mc->provider.xlate = mc->soc->icc_ops->xlate; | |
06f07981 DO |
835 | mc->provider.xlate_extended = mc->soc->icc_ops->xlate_extended; |
836 | ||
5553055c | 837 | icc_provider_init(&mc->provider); |
06f07981 DO |
838 | |
839 | /* create Memory Controller node */ | |
840 | node = icc_node_create(TEGRA_ICC_MC); | |
5553055c JH |
841 | if (IS_ERR(node)) |
842 | return PTR_ERR(node); | |
06f07981 DO |
843 | |
844 | node->name = "Memory Controller"; | |
845 | icc_node_add(node, &mc->provider); | |
846 | ||
847 | /* link Memory Controller to External Memory Controller */ | |
848 | err = icc_link_create(node, TEGRA_ICC_EMC); | |
849 | if (err) | |
850 | goto remove_nodes; | |
851 | ||
852 | for (i = 0; i < mc->soc->num_clients; i++) { | |
853 | /* create MC client node */ | |
854 | node = icc_node_create(mc->soc->clients[i].id); | |
855 | if (IS_ERR(node)) { | |
856 | err = PTR_ERR(node); | |
857 | goto remove_nodes; | |
858 | } | |
859 | ||
860 | node->name = mc->soc->clients[i].name; | |
861 | icc_node_add(node, &mc->provider); | |
862 | ||
863 | /* link Memory Client to Memory Controller */ | |
864 | err = icc_link_create(node, TEGRA_ICC_MC); | |
865 | if (err) | |
866 | goto remove_nodes; | |
9a38cb27 SG |
867 | |
868 | node->data = (struct tegra_mc_client *)&(mc->soc->clients[i]); | |
06f07981 DO |
869 | } |
870 | ||
5553055c JH |
871 | err = icc_provider_register(&mc->provider); |
872 | if (err) | |
873 | goto remove_nodes; | |
874 | ||
06f07981 DO |
875 | return 0; |
876 | ||
877 | remove_nodes: | |
878 | icc_nodes_remove(&mc->provider); | |
06f07981 DO |
879 | |
880 | return err; | |
881 | } | |
882 | ||
e852af72 SG |
883 | static void tegra_mc_num_channel_enabled(struct tegra_mc *mc) |
884 | { | |
885 | unsigned int i; | |
886 | u32 value; | |
887 | ||
888 | value = mc_ch_readl(mc, 0, MC_EMEM_ADR_CFG_CHANNEL_ENABLE); | |
889 | if (value <= 0) { | |
890 | mc->num_channels = mc->soc->num_channels; | |
891 | return; | |
892 | } | |
893 | ||
894 | for (i = 0; i < 32; i++) { | |
895 | if (value & BIT(i)) | |
896 | mc->num_channels++; | |
897 | } | |
898 | } | |
899 | ||
89184651 TR |
900 | static int tegra_mc_probe(struct platform_device *pdev) |
901 | { | |
89184651 | 902 | struct tegra_mc *mc; |
c4c21f22 | 903 | u64 mask; |
89184651 TR |
904 | int err; |
905 | ||
89184651 TR |
906 | mc = devm_kzalloc(&pdev->dev, sizeof(*mc), GFP_KERNEL); |
907 | if (!mc) | |
908 | return -ENOMEM; | |
909 | ||
910 | platform_set_drvdata(pdev, mc); | |
20e92462 | 911 | spin_lock_init(&mc->lock); |
59cd046f | 912 | mc->soc = of_device_get_match_data(&pdev->dev); |
89184651 TR |
913 | mc->dev = &pdev->dev; |
914 | ||
c4c21f22 TR |
915 | mask = DMA_BIT_MASK(mc->soc->num_address_bits); |
916 | ||
917 | err = dma_coerce_mask_and_coherent(&pdev->dev, mask); | |
918 | if (err < 0) { | |
919 | dev_err(&pdev->dev, "failed to set DMA mask: %d\n", err); | |
920 | return err; | |
921 | } | |
922 | ||
89184651 TR |
923 | /* length of MC tick in nanoseconds */ |
924 | mc->tick = 30; | |
925 | ||
dab022f2 | 926 | mc->regs = devm_platform_ioremap_resource(pdev, 0); |
89184651 TR |
927 | if (IS_ERR(mc->regs)) |
928 | return PTR_ERR(mc->regs); | |
929 | ||
c64738e9 TR |
930 | mc->debugfs.root = debugfs_create_dir("mc", NULL); |
931 | ||
932 | if (mc->soc->ops && mc->soc->ops->probe) { | |
933 | err = mc->soc->ops->probe(mc); | |
934 | if (err < 0) | |
935 | return err; | |
936 | } | |
937 | ||
e852af72 SG |
938 | tegra_mc_num_channel_enabled(mc); |
939 | ||
e474b3a1 TR |
940 | if (mc->soc->ops && mc->soc->ops->handle_irq) { |
941 | mc->irq = platform_get_irq(pdev, 0); | |
942 | if (mc->irq < 0) | |
943 | return mc->irq; | |
89184651 | 944 | |
e474b3a1 | 945 | WARN(!mc->soc->client_id_mask, "missing client ID mask for this SoC\n"); |
3c01cf3b | 946 | |
54a85e09 AM |
947 | if (mc->soc->num_channels) |
948 | mc_ch_writel(mc, MC_BROADCAST_CHANNEL, mc->soc->intmask, | |
949 | MC_INTMASK); | |
950 | else | |
951 | mc_writel(mc, mc->soc->intmask, MC_INTMASK); | |
89184651 | 952 | |
e474b3a1 TR |
953 | err = devm_request_irq(&pdev->dev, mc->irq, mc->soc->ops->handle_irq, 0, |
954 | dev_name(&pdev->dev), mc); | |
955 | if (err < 0) { | |
956 | dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", mc->irq, | |
957 | err); | |
958 | return err; | |
959 | } | |
db4a9c19 DO |
960 | } |
961 | ||
0de93c69 TR |
962 | if (mc->soc->reset_ops) { |
963 | err = tegra_mc_reset_setup(mc); | |
964 | if (err < 0) | |
965 | dev_err(&pdev->dev, "failed to register reset controller: %d\n", err); | |
966 | } | |
1662dd64 | 967 | |
06f07981 DO |
968 | err = tegra_mc_interconnect_setup(mc); |
969 | if (err < 0) | |
970 | dev_err(&pdev->dev, "failed to initialize interconnect: %d\n", | |
971 | err); | |
972 | ||
568ece5b | 973 | if (IS_ENABLED(CONFIG_TEGRA_IOMMU_SMMU) && mc->soc->smmu) { |
45a81df0 | 974 | mc->smmu = tegra_smmu_probe(&pdev->dev, mc->soc->smmu, mc); |
568ece5b | 975 | if (IS_ERR(mc->smmu)) { |
45a81df0 DO |
976 | dev_err(&pdev->dev, "failed to probe SMMU: %ld\n", |
977 | PTR_ERR(mc->smmu)); | |
568ece5b DO |
978 | mc->smmu = NULL; |
979 | } | |
45a81df0 DO |
980 | } |
981 | ||
ce2785a7 DO |
982 | if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && !mc->soc->smmu) { |
983 | mc->gart = tegra_gart_probe(&pdev->dev, mc); | |
984 | if (IS_ERR(mc->gart)) { | |
985 | dev_err(&pdev->dev, "failed to probe GART: %ld\n", | |
986 | PTR_ERR(mc->gart)); | |
987 | mc->gart = NULL; | |
988 | } | |
989 | } | |
990 | ||
991 | return 0; | |
992 | } | |
993 | ||
5c9016f0 | 994 | static int __maybe_unused tegra_mc_suspend(struct device *dev) |
ce2785a7 DO |
995 | { |
996 | struct tegra_mc *mc = dev_get_drvdata(dev); | |
ce2785a7 | 997 | |
5c9016f0 TR |
998 | if (mc->soc->ops && mc->soc->ops->suspend) |
999 | return mc->soc->ops->suspend(mc); | |
ce2785a7 | 1000 | |
89184651 TR |
1001 | return 0; |
1002 | } | |
1003 | ||
5c9016f0 | 1004 | static int __maybe_unused tegra_mc_resume(struct device *dev) |
ce2785a7 DO |
1005 | { |
1006 | struct tegra_mc *mc = dev_get_drvdata(dev); | |
ce2785a7 | 1007 | |
5c9016f0 TR |
1008 | if (mc->soc->ops && mc->soc->ops->resume) |
1009 | return mc->soc->ops->resume(mc); | |
ce2785a7 DO |
1010 | |
1011 | return 0; | |
1012 | } | |
1013 | ||
77b14c9d DO |
1014 | static void tegra_mc_sync_state(struct device *dev) |
1015 | { | |
1016 | struct tegra_mc *mc = dev_get_drvdata(dev); | |
1017 | ||
1018 | /* check whether ICC provider is registered */ | |
1019 | if (mc->provider.dev == dev) | |
1020 | icc_sync_state(dev); | |
1021 | } | |
1022 | ||
ce2785a7 | 1023 | static const struct dev_pm_ops tegra_mc_pm_ops = { |
5c9016f0 | 1024 | SET_SYSTEM_SLEEP_PM_OPS(tegra_mc_suspend, tegra_mc_resume) |
ce2785a7 DO |
1025 | }; |
1026 | ||
89184651 TR |
1027 | static struct platform_driver tegra_mc_driver = { |
1028 | .driver = { | |
1029 | .name = "tegra-mc", | |
1030 | .of_match_table = tegra_mc_of_match, | |
ce2785a7 | 1031 | .pm = &tegra_mc_pm_ops, |
89184651 | 1032 | .suppress_bind_attrs = true, |
77b14c9d | 1033 | .sync_state = tegra_mc_sync_state, |
89184651 TR |
1034 | }, |
1035 | .prevent_deferred_probe = true, | |
1036 | .probe = tegra_mc_probe, | |
1037 | }; | |
1038 | ||
1039 | static int tegra_mc_init(void) | |
1040 | { | |
1041 | return platform_driver_register(&tegra_mc_driver); | |
1042 | } | |
1043 | arch_initcall(tegra_mc_init); | |
1044 | ||
1045 | MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>"); | |
1046 | MODULE_DESCRIPTION("NVIDIA Tegra Memory Controller driver"); |