Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
de818bd4 | 2 | #include <linux/ftrace.h> |
fb4a9602 | 3 | #include <linux/percpu.h> |
95322526 | 4 | #include <linux/slab.h> |
e1281f56 | 5 | #include <linux/uaccess.h> |
65fddcfc | 6 | #include <linux/pgtable.h> |
19235e47 | 7 | #include <linux/cpuidle.h> |
d0854412 | 8 | #include <asm/alternative.h> |
95322526 | 9 | #include <asm/cacheflush.h> |
d0854412 | 10 | #include <asm/cpufeature.h> |
77345ef7 | 11 | #include <asm/cpuidle.h> |
0fbeb318 | 12 | #include <asm/daifflags.h> |
95322526 | 13 | #include <asm/debug-monitors.h> |
d0854412 | 14 | #include <asm/exec.h> |
95338648 | 15 | #include <asm/fpsimd.h> |
39d08e83 | 16 | #include <asm/mte.h> |
95322526 | 17 | #include <asm/memory.h> |
f43c2718 | 18 | #include <asm/mmu_context.h> |
95322526 LP |
19 | #include <asm/smp_plat.h> |
20 | #include <asm/suspend.h> | |
95322526 | 21 | |
95322526 | 22 | /* |
cabe1c81 JM |
23 | * This is allocated by cpu_suspend_init(), and used to store a pointer to |
24 | * the 'struct sleep_stack_data' the contains a particular CPUs state. | |
95322526 | 25 | */ |
cabe1c81 | 26 | unsigned long *sleep_save_stash; |
95322526 | 27 | |
65c021bb LP |
28 | /* |
29 | * This hook is provided so that cpu_suspend code can restore HW | |
30 | * breakpoints as early as possible in the resume path, before reenabling | |
31 | * debug exceptions. Code cannot be run from a CPU PM notifier since by the | |
32 | * time the notifier runs debug exceptions might have been enabled already, | |
33 | * with HW breakpoints registers content still in an unknown state. | |
34 | */ | |
d7a83d12 WD |
35 | static int (*hw_breakpoint_restore)(unsigned int); |
36 | void __init cpu_suspend_set_dbg_restorer(int (*hw_bp_restore)(unsigned int)) | |
65c021bb LP |
37 | { |
38 | /* Prevent multiple restore hook initializations */ | |
39 | if (WARN_ON(hw_breakpoint_restore)) | |
40 | return; | |
41 | hw_breakpoint_restore = hw_bp_restore; | |
42 | } | |
43 | ||
adc9b2df JM |
44 | void notrace __cpu_suspend_exit(void) |
45 | { | |
d7a83d12 WD |
46 | unsigned int cpu = smp_processor_id(); |
47 | ||
973b9e37 PC |
48 | mte_suspend_exit(); |
49 | ||
adc9b2df JM |
50 | /* |
51 | * We are resuming from reset with the idmap active in TTBR0_EL1. | |
52 | * We must uninstall the idmap and restore the expected MMU | |
53 | * state before we can possibly return to userspace. | |
54 | */ | |
55 | cpu_uninstall_idmap(); | |
56 | ||
5ffdfaed VM |
57 | /* Restore CnP bit in TTBR1_EL1 */ |
58 | if (system_supports_cnp()) | |
54c8818a | 59 | cpu_enable_swapper_cnp(); |
5ffdfaed | 60 | |
d0854412 JM |
61 | /* |
62 | * PSTATE was not saved over suspend/resume, re-enable any detected | |
63 | * features that might not have been set correctly. | |
64 | */ | |
25693f17 | 65 | if (alternative_has_cap_unlikely(ARM64_HAS_DIT)) |
01ab991f | 66 | set_pstate_dit(1); |
e1281f56 | 67 | __uaccess_enable_hw_pan(); |
d0854412 | 68 | |
adc9b2df JM |
69 | /* |
70 | * Restore HW breakpoint registers to sane values | |
71 | * before debug exceptions are possibly reenabled | |
0fbeb318 | 72 | * by cpu_suspend()s local_daif_restore() call. |
adc9b2df JM |
73 | */ |
74 | if (hw_breakpoint_restore) | |
d7a83d12 | 75 | hw_breakpoint_restore(cpu); |
647d0519 MZ |
76 | |
77 | /* | |
78 | * On resume, firmware implementing dynamic mitigation will | |
79 | * have turned the mitigation on. If the user has forcefully | |
80 | * disabled it, make sure their wishes are obeyed. | |
81 | */ | |
c2876207 | 82 | spectre_v4_enable_mitigation(NULL); |
39d08e83 | 83 | |
95338648 MB |
84 | sme_suspend_exit(); |
85 | ||
b90e4839 | 86 | /* Restore additional feature-specific configuration */ |
b90e4839 | 87 | ptrauth_suspend_exit(); |
adc9b2df JM |
88 | } |
89 | ||
714f5992 | 90 | /* |
af391b15 | 91 | * cpu_suspend |
714f5992 LP |
92 | * |
93 | * arg: argument to pass to the finisher function | |
94 | * fn: finisher function pointer | |
95 | * | |
96 | */ | |
af391b15 | 97 | int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) |
714f5992 | 98 | { |
adc9b2df | 99 | int ret = 0; |
714f5992 | 100 | unsigned long flags; |
adc9b2df | 101 | struct sleep_stack_data state; |
77345ef7 | 102 | struct arm_cpuidle_irq_context context; |
95322526 | 103 | |
25693f17 MR |
104 | /* |
105 | * Some portions of CPU state (e.g. PSTATE.{PAN,DIT}) are initialized | |
106 | * before alternatives are patched, but are only restored by | |
107 | * __cpu_suspend_exit() after alternatives are patched. To avoid | |
108 | * accidentally losing these bits we must not attempt to suspend until | |
109 | * after alternatives have been patched. | |
110 | */ | |
111 | WARN_ON(!system_capabilities_finalized()); | |
112 | ||
eab0e6e1 VF |
113 | /* Report any MTE async fault before going to suspend */ |
114 | mte_suspend_enter(); | |
115 | ||
95322526 LP |
116 | /* |
117 | * From this point debug exceptions are disabled to prevent | |
118 | * updates to mdscr register (saved and restored along with | |
119 | * general purpose registers) from kernel debuggers. | |
19235e47 PZ |
120 | * |
121 | * Strictly speaking the trace_hardirqs_off() here is superfluous, | |
122 | * hardirqs should be firmly off by now. This really ought to use | |
123 | * something like raw_local_daif_save(). | |
95322526 | 124 | */ |
0fbeb318 | 125 | flags = local_daif_save(); |
95322526 | 126 | |
de818bd4 | 127 | /* |
77345ef7 | 128 | * Function graph tracer state gets inconsistent when the kernel |
de818bd4 LP |
129 | * calls functions that never return (aka suspend finishers) hence |
130 | * disable graph tracing during their execution. | |
131 | */ | |
132 | pause_graph_tracing(); | |
133 | ||
77345ef7 MZ |
134 | /* |
135 | * Switch to using DAIF.IF instead of PMR in order to reliably | |
136 | * resume if we're using pseudo-NMIs. | |
137 | */ | |
138 | arm_cpuidle_save_irq_context(&context); | |
139 | ||
19235e47 PZ |
140 | ct_cpuidle_enter(); |
141 | ||
adc9b2df JM |
142 | if (__cpu_suspend_enter(&state)) { |
143 | /* Call the suspend finisher */ | |
144 | ret = fn(arg); | |
fb4a9602 | 145 | |
65c021bb | 146 | /* |
adc9b2df JM |
147 | * Never gets here, unless the suspend finisher fails. |
148 | * Successful cpu_suspend() should return from cpu_resume(), | |
149 | * returning through this code path is considered an error | |
150 | * If the return value is set to 0 force ret = -EOPNOTSUPP | |
151 | * to make sure a proper error condition is propagated | |
65c021bb | 152 | */ |
adc9b2df JM |
153 | if (!ret) |
154 | ret = -EOPNOTSUPP; | |
19235e47 PZ |
155 | |
156 | ct_cpuidle_exit(); | |
adc9b2df | 157 | } else { |
19235e47 PZ |
158 | ct_cpuidle_exit(); |
159 | __cpu_suspend_exit(); | |
95322526 LP |
160 | } |
161 | ||
77345ef7 MZ |
162 | arm_cpuidle_restore_irq_context(&context); |
163 | ||
de818bd4 LP |
164 | unpause_graph_tracing(); |
165 | ||
95322526 LP |
166 | /* |
167 | * Restore pstate flags. OS lock and mdscr have been already | |
168 | * restored, so from this point onwards, debugging is fully | |
dd671f16 | 169 | * reenabled if it was enabled when core started shutdown. |
95322526 | 170 | */ |
0fbeb318 | 171 | local_daif_restore(flags); |
95322526 LP |
172 | |
173 | return ret; | |
174 | } | |
175 | ||
18ab7db6 | 176 | static int __init cpu_suspend_init(void) |
95322526 | 177 | { |
95322526 | 178 | /* ctx_ptr is an array of physical addresses */ |
cabe1c81 JM |
179 | sleep_save_stash = kcalloc(mpidr_hash_size(), sizeof(*sleep_save_stash), |
180 | GFP_KERNEL); | |
95322526 | 181 | |
cabe1c81 | 182 | if (WARN_ON(!sleep_save_stash)) |
95322526 LP |
183 | return -ENOMEM; |
184 | ||
95322526 LP |
185 | return 0; |
186 | } | |
187 | early_initcall(cpu_suspend_init); |