Commit | Line | Data |
---|---|---|
860fbde4 DO |
1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* | |
3 | * CPU idle driver for Tegra CPUs | |
4 | * | |
5 | * Copyright (c) 2010-2013, NVIDIA Corporation. | |
6 | * Copyright (c) 2011 Google, Inc. | |
7 | * Author: Colin Cross <ccross@android.com> | |
8 | * Gary King <gking@nvidia.com> | |
9 | * | |
10 | * Rework for 3.3 by Peter De Schrijver <pdeschrijver@nvidia.com> | |
11 | * | |
12 | * Tegra20/124 driver unification by Dmitry Osipenko <digetx@gmail.com> | |
13 | */ | |
14 | ||
15 | #define pr_fmt(fmt) "tegra-cpuidle: " fmt | |
16 | ||
17 | #include <linux/atomic.h> | |
18 | #include <linux/cpuidle.h> | |
19 | #include <linux/cpumask.h> | |
20 | #include <linux/cpu_pm.h> | |
21 | #include <linux/delay.h> | |
22 | #include <linux/errno.h> | |
23 | #include <linux/platform_device.h> | |
24 | #include <linux/types.h> | |
25 | ||
26 | #include <linux/clk/tegra.h> | |
14e086ba | 27 | #include <linux/firmware/trusted_foundations.h> |
860fbde4 DO |
28 | |
29 | #include <soc/tegra/cpuidle.h> | |
30 | #include <soc/tegra/flowctrl.h> | |
31 | #include <soc/tegra/fuse.h> | |
32 | #include <soc/tegra/irq.h> | |
33 | #include <soc/tegra/pm.h> | |
382ac8e2 | 34 | #include <soc/tegra/pmc.h> |
860fbde4 DO |
35 | |
36 | #include <asm/cpuidle.h> | |
14e086ba | 37 | #include <asm/firmware.h> |
860fbde4 DO |
38 | #include <asm/smp_plat.h> |
39 | #include <asm/suspend.h> | |
40 | ||
41 | enum tegra_state { | |
42 | TEGRA_C1, | |
19461a49 | 43 | TEGRA_C7, |
860fbde4 DO |
44 | TEGRA_CC6, |
45 | TEGRA_STATE_COUNT, | |
46 | }; | |
47 | ||
48 | static atomic_t tegra_idle_barrier; | |
49 | static atomic_t tegra_abort_flag; | |
50 | ||
14e086ba DO |
51 | static inline bool tegra_cpuidle_using_firmware(void) |
52 | { | |
53 | return firmware_ops->prepare_idle && firmware_ops->do_idle; | |
54 | } | |
55 | ||
860fbde4 DO |
56 | static void tegra_cpuidle_report_cpus_state(void) |
57 | { | |
58 | unsigned long cpu, lcpu, csr; | |
59 | ||
60 | for_each_cpu(lcpu, cpu_possible_mask) { | |
61 | cpu = cpu_logical_map(lcpu); | |
62 | csr = flowctrl_read_cpu_csr(cpu); | |
63 | ||
64 | pr_err("cpu%lu: online=%d flowctrl_csr=0x%08lx\n", | |
65 | cpu, cpu_online(lcpu), csr); | |
66 | } | |
67 | } | |
68 | ||
69 | static int tegra_cpuidle_wait_for_secondary_cpus_parking(void) | |
70 | { | |
71 | unsigned int retries = 3; | |
72 | ||
73 | while (retries--) { | |
74 | unsigned int delay_us = 10; | |
75 | unsigned int timeout_us = 500 * 1000 / delay_us; | |
76 | ||
77 | /* | |
78 | * The primary CPU0 core shall wait for the secondaries | |
79 | * shutdown in order to power-off CPU's cluster safely. | |
80 | * The timeout value depends on the current CPU frequency, | |
81 | * it takes about 40-150us in average and over 1000us in | |
82 | * a worst case scenario. | |
83 | */ | |
84 | do { | |
85 | if (tegra_cpu_rail_off_ready()) | |
86 | return 0; | |
87 | ||
88 | udelay(delay_us); | |
89 | ||
90 | } while (timeout_us--); | |
91 | ||
92 | pr_err("secondary CPU taking too long to park\n"); | |
93 | ||
94 | tegra_cpuidle_report_cpus_state(); | |
95 | } | |
96 | ||
97 | pr_err("timed out waiting secondaries to park\n"); | |
98 | ||
99 | return -ETIMEDOUT; | |
100 | } | |
101 | ||
102 | static void tegra_cpuidle_unpark_secondary_cpus(void) | |
103 | { | |
104 | unsigned int cpu, lcpu; | |
105 | ||
106 | for_each_cpu(lcpu, cpu_online_mask) { | |
107 | cpu = cpu_logical_map(lcpu); | |
108 | ||
109 | if (cpu > 0) { | |
110 | tegra_enable_cpu_clock(cpu); | |
111 | tegra_cpu_out_of_reset(cpu); | |
112 | flowctrl_write_cpu_halt(cpu, 0); | |
113 | } | |
114 | } | |
115 | } | |
116 | ||
117 | static int tegra_cpuidle_cc6_enter(unsigned int cpu) | |
118 | { | |
119 | int ret; | |
120 | ||
121 | if (cpu > 0) { | |
122 | ret = cpu_suspend(cpu, tegra_pm_park_secondary_cpu); | |
123 | } else { | |
124 | ret = tegra_cpuidle_wait_for_secondary_cpus_parking(); | |
125 | if (!ret) | |
126 | ret = tegra_pm_enter_lp2(); | |
127 | ||
128 | tegra_cpuidle_unpark_secondary_cpus(); | |
129 | } | |
130 | ||
131 | return ret; | |
132 | } | |
133 | ||
19461a49 DO |
134 | static int tegra_cpuidle_c7_enter(void) |
135 | { | |
14e086ba DO |
136 | int err; |
137 | ||
138 | if (tegra_cpuidle_using_firmware()) { | |
139 | err = call_firmware_op(prepare_idle, TF_PM_MODE_LP2_NOFLUSH_L2); | |
140 | if (err) | |
141 | return err; | |
142 | ||
143 | return call_firmware_op(do_idle, 0); | |
144 | } | |
145 | ||
19461a49 DO |
146 | return cpu_suspend(0, tegra30_pm_secondary_cpu_suspend); |
147 | } | |
148 | ||
860fbde4 DO |
149 | static int tegra_cpuidle_coupled_barrier(struct cpuidle_device *dev) |
150 | { | |
151 | if (tegra_pending_sgi()) { | |
152 | /* | |
153 | * CPU got local interrupt that will be lost after GIC's | |
154 | * shutdown because GIC driver doesn't save/restore the | |
155 | * pending SGI state across CPU cluster PM. Abort and retry | |
156 | * next time. | |
157 | */ | |
158 | atomic_set(&tegra_abort_flag, 1); | |
159 | } | |
160 | ||
161 | cpuidle_coupled_parallel_barrier(dev, &tegra_idle_barrier); | |
162 | ||
163 | if (atomic_read(&tegra_abort_flag)) { | |
164 | cpuidle_coupled_parallel_barrier(dev, &tegra_idle_barrier); | |
165 | atomic_set(&tegra_abort_flag, 0); | |
166 | return -EINTR; | |
167 | } | |
168 | ||
169 | return 0; | |
170 | } | |
171 | ||
172 | static int tegra_cpuidle_state_enter(struct cpuidle_device *dev, | |
173 | int index, unsigned int cpu) | |
174 | { | |
1170433e | 175 | int err; |
860fbde4 DO |
176 | |
177 | /* | |
178 | * CC6 state is the "CPU cluster power-off" state. In order to | |
179 | * enter this state, at first the secondary CPU cores need to be | |
180 | * parked into offline mode, then the last CPU should clean out | |
181 | * remaining dirty cache lines into DRAM and trigger Flow Controller | |
182 | * logic that turns off the cluster's power domain (which includes | |
183 | * CPU cores, GIC and L2 cache). | |
184 | */ | |
185 | if (index == TEGRA_CC6) { | |
1170433e DO |
186 | err = tegra_cpuidle_coupled_barrier(dev); |
187 | if (err) | |
188 | return err; | |
860fbde4 DO |
189 | } |
190 | ||
191 | local_fiq_disable(); | |
c39de538 | 192 | RCU_NONIDLE(tegra_pm_set_cpu_in_lp2()); |
860fbde4 DO |
193 | cpu_pm_enter(); |
194 | ||
195 | switch (index) { | |
19461a49 | 196 | case TEGRA_C7: |
1170433e | 197 | err = tegra_cpuidle_c7_enter(); |
19461a49 DO |
198 | break; |
199 | ||
860fbde4 | 200 | case TEGRA_CC6: |
1170433e | 201 | err = tegra_cpuidle_cc6_enter(cpu); |
860fbde4 DO |
202 | break; |
203 | ||
204 | default: | |
1170433e | 205 | err = -EINVAL; |
860fbde4 DO |
206 | break; |
207 | } | |
208 | ||
209 | cpu_pm_exit(); | |
c39de538 | 210 | RCU_NONIDLE(tegra_pm_clear_cpu_in_lp2()); |
860fbde4 DO |
211 | local_fiq_enable(); |
212 | ||
1170433e | 213 | return err ?: index; |
860fbde4 DO |
214 | } |
215 | ||
19461a49 DO |
216 | static int tegra_cpuidle_adjust_state_index(int index, unsigned int cpu) |
217 | { | |
218 | /* | |
219 | * On Tegra30 CPU0 can't be power-gated separately from secondary | |
220 | * cores because it gates the whole CPU cluster. | |
221 | */ | |
222 | if (cpu > 0 || index != TEGRA_C7 || tegra_get_chip_id() != TEGRA30) | |
223 | return index; | |
224 | ||
225 | /* put CPU0 into C1 if C7 is requested and secondaries are online */ | |
226 | if (!IS_ENABLED(CONFIG_PM_SLEEP) || num_online_cpus() > 1) | |
227 | index = TEGRA_C1; | |
228 | else | |
229 | index = TEGRA_CC6; | |
230 | ||
231 | return index; | |
232 | } | |
233 | ||
860fbde4 DO |
234 | static int tegra_cpuidle_enter(struct cpuidle_device *dev, |
235 | struct cpuidle_driver *drv, | |
236 | int index) | |
237 | { | |
238 | unsigned int cpu = cpu_logical_map(dev->cpu); | |
1170433e | 239 | int ret; |
860fbde4 | 240 | |
19461a49 DO |
241 | index = tegra_cpuidle_adjust_state_index(index, cpu); |
242 | if (dev->states_usage[index].disable) | |
243 | return -1; | |
244 | ||
245 | if (index == TEGRA_C1) | |
1170433e | 246 | ret = arm_cpuidle_simple_enter(dev, drv, index); |
19461a49 | 247 | else |
1170433e | 248 | ret = tegra_cpuidle_state_enter(dev, index, cpu); |
19461a49 | 249 | |
1170433e DO |
250 | if (ret < 0) { |
251 | if (ret != -EINTR || index != TEGRA_CC6) | |
252 | pr_err_once("failed to enter state %d err: %d\n", | |
253 | index, ret); | |
254 | index = -1; | |
255 | } else { | |
256 | index = ret; | |
257 | } | |
860fbde4 | 258 | |
1170433e | 259 | return index; |
860fbde4 DO |
260 | } |
261 | ||
efe97112 NL |
262 | static int tegra114_enter_s2idle(struct cpuidle_device *dev, |
263 | struct cpuidle_driver *drv, | |
264 | int index) | |
14e086ba DO |
265 | { |
266 | tegra_cpuidle_enter(dev, drv, index); | |
efe97112 NL |
267 | |
268 | return 0; | |
14e086ba DO |
269 | } |
270 | ||
860fbde4 DO |
271 | /* |
272 | * The previous versions of Tegra CPUIDLE driver used a different "legacy" | |
273 | * terminology for naming of the idling states, while this driver uses the | |
274 | * new terminology. | |
275 | * | |
276 | * Mapping of the old terms into the new ones: | |
277 | * | |
278 | * Old | New | |
279 | * --------- | |
280 | * LP3 | C1 (CPU core clock gating) | |
281 | * LP2 | C7 (CPU core power gating) | |
282 | * LP2 | CC6 (CPU cluster power gating) | |
283 | * | |
284 | * Note that that the older CPUIDLE driver versions didn't explicitly | |
285 | * differentiate the LP2 states because these states either used the same | |
286 | * code path or because CC6 wasn't supported. | |
287 | */ | |
288 | static struct cpuidle_driver tegra_idle_driver = { | |
289 | .name = "tegra_idle", | |
290 | .states = { | |
291 | [TEGRA_C1] = ARM_CPUIDLE_WFI_STATE_PWR(600), | |
19461a49 DO |
292 | [TEGRA_C7] = { |
293 | .enter = tegra_cpuidle_enter, | |
294 | .exit_latency = 2000, | |
295 | .target_residency = 2200, | |
296 | .power_usage = 100, | |
297 | .flags = CPUIDLE_FLAG_TIMER_STOP, | |
298 | .name = "C7", | |
299 | .desc = "CPU core powered off", | |
300 | }, | |
860fbde4 DO |
301 | [TEGRA_CC6] = { |
302 | .enter = tegra_cpuidle_enter, | |
303 | .exit_latency = 5000, | |
304 | .target_residency = 10000, | |
305 | .power_usage = 0, | |
306 | .flags = CPUIDLE_FLAG_TIMER_STOP | | |
307 | CPUIDLE_FLAG_COUPLED, | |
308 | .name = "CC6", | |
309 | .desc = "CPU cluster powered off", | |
310 | }, | |
311 | }, | |
312 | .state_count = TEGRA_STATE_COUNT, | |
313 | .safe_state_index = TEGRA_C1, | |
314 | }; | |
315 | ||
316 | static inline void tegra_cpuidle_disable_state(enum tegra_state state) | |
317 | { | |
318 | cpuidle_driver_state_disabled(&tegra_idle_driver, state, true); | |
319 | } | |
320 | ||
321 | /* | |
322 | * Tegra20 HW appears to have a bug such that PCIe device interrupts, whether | |
323 | * they are legacy IRQs or MSI, are lost when CC6 is enabled. To work around | |
324 | * this, simply disable CC6 if the PCI driver and DT node are both enabled. | |
325 | */ | |
326 | void tegra_cpuidle_pcie_irqs_in_use(void) | |
327 | { | |
328 | struct cpuidle_state *state_cc6 = &tegra_idle_driver.states[TEGRA_CC6]; | |
329 | ||
330 | if ((state_cc6->flags & CPUIDLE_FLAG_UNUSABLE) || | |
331 | tegra_get_chip_id() != TEGRA20) | |
332 | return; | |
333 | ||
334 | pr_info("disabling CC6 state, since PCIe IRQs are in use\n"); | |
335 | tegra_cpuidle_disable_state(TEGRA_CC6); | |
336 | } | |
337 | ||
14e086ba DO |
338 | static void tegra_cpuidle_setup_tegra114_c7_state(void) |
339 | { | |
340 | struct cpuidle_state *s = &tegra_idle_driver.states[TEGRA_C7]; | |
341 | ||
342 | s->enter_s2idle = tegra114_enter_s2idle; | |
343 | s->target_residency = 1000; | |
344 | s->exit_latency = 500; | |
345 | } | |
346 | ||
860fbde4 DO |
347 | static int tegra_cpuidle_probe(struct platform_device *pdev) |
348 | { | |
382ac8e2 DO |
349 | /* LP2 could be disabled in device-tree */ |
350 | if (tegra_pmc_get_suspend_mode() < TEGRA_SUSPEND_LP2) | |
351 | tegra_cpuidle_disable_state(TEGRA_CC6); | |
352 | ||
860fbde4 DO |
353 | /* |
354 | * Required suspend-resume functionality, which is provided by the | |
355 | * Tegra-arch core and PMC driver, is unavailable if PM-sleep option | |
356 | * is disabled. | |
357 | */ | |
19461a49 | 358 | if (!IS_ENABLED(CONFIG_PM_SLEEP)) { |
14e086ba DO |
359 | if (!tegra_cpuidle_using_firmware()) |
360 | tegra_cpuidle_disable_state(TEGRA_C7); | |
361 | ||
860fbde4 | 362 | tegra_cpuidle_disable_state(TEGRA_CC6); |
19461a49 DO |
363 | } |
364 | ||
365 | /* | |
366 | * Generic WFI state (also known as C1 or LP3) and the coupled CPU | |
367 | * cluster power-off (CC6 or LP2) states are common for all Tegra SoCs. | |
368 | */ | |
369 | switch (tegra_get_chip_id()) { | |
370 | case TEGRA20: | |
371 | /* Tegra20 isn't capable to power-off individual CPU cores */ | |
372 | tegra_cpuidle_disable_state(TEGRA_C7); | |
373 | break; | |
374 | ||
375 | case TEGRA30: | |
19461a49 DO |
376 | break; |
377 | ||
14e086ba DO |
378 | case TEGRA114: |
379 | case TEGRA124: | |
380 | tegra_cpuidle_setup_tegra114_c7_state(); | |
381 | ||
382 | /* coupled CC6 (LP2) state isn't implemented yet */ | |
383 | tegra_cpuidle_disable_state(TEGRA_CC6); | |
384 | break; | |
385 | ||
19461a49 DO |
386 | default: |
387 | return -EINVAL; | |
388 | } | |
860fbde4 DO |
389 | |
390 | return cpuidle_register(&tegra_idle_driver, cpu_possible_mask); | |
391 | } | |
392 | ||
393 | static struct platform_driver tegra_cpuidle_driver = { | |
394 | .probe = tegra_cpuidle_probe, | |
395 | .driver = { | |
396 | .name = "tegra-cpuidle", | |
397 | }, | |
398 | }; | |
399 | builtin_platform_driver(tegra_cpuidle_driver); |