Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
2104180a NP |
2 | /* |
3 | * Watchdog support on powerpc systems. | |
4 | * | |
5 | * Copyright 2017, IBM Corporation. | |
6 | * | |
7 | * This uses code from arch/sparc/kernel/nmi.c and kernel/watchdog.c | |
8 | */ | |
9 | #include <linux/kernel.h> | |
10 | #include <linux/param.h> | |
11 | #include <linux/init.h> | |
12 | #include <linux/percpu.h> | |
13 | #include <linux/cpu.h> | |
14 | #include <linux/nmi.h> | |
15 | #include <linux/module.h> | |
16 | #include <linux/export.h> | |
17 | #include <linux/kprobes.h> | |
18 | #include <linux/hardirq.h> | |
19 | #include <linux/reboot.h> | |
20 | #include <linux/slab.h> | |
21 | #include <linux/kdebug.h> | |
22 | #include <linux/sched/debug.h> | |
23 | #include <linux/delay.h> | |
24 | #include <linux/smp.h> | |
25 | ||
26 | #include <asm/paca.h> | |
27 | ||
28 | /* | |
29 | * The watchdog has a simple timer that runs on each CPU, once per timer | |
30 | * period. This is the heartbeat. | |
31 | * | |
32 | * Then there are checks to see if the heartbeat has not triggered on a CPU | |
33 | * for the panic timeout period. Currently the watchdog only supports an | |
34 | * SMP check, so the heartbeat only turns on when we have 2 or more CPUs. | |
35 | * | |
36 | * This is not an NMI watchdog, but Linux uses that name for a generic | |
37 | * watchdog in some cases, so NMI gets used in some places. | |
38 | */ | |
39 | ||
40 | static cpumask_t wd_cpus_enabled __read_mostly; | |
41 | ||
42 | static u64 wd_panic_timeout_tb __read_mostly; /* timebase ticks until panic */ | |
43 | static u64 wd_smp_panic_timeout_tb __read_mostly; /* panic other CPUs */ | |
44 | ||
45 | static u64 wd_timer_period_ms __read_mostly; /* interval between heartbeat */ | |
46 | ||
47 | static DEFINE_PER_CPU(struct timer_list, wd_timer); | |
48 | static DEFINE_PER_CPU(u64, wd_timer_tb); | |
49 | ||
50 | /* | |
51 | * These are for the SMP checker. CPUs clear their pending bit in their | |
52 | * heartbeat. If the bitmask becomes empty, the time is noted and the | |
53 | * bitmask is refilled. | |
54 | * | |
55 | * All CPUs clear their bit in the pending mask every timer period. | |
56 | * Once all have cleared, the time is noted and the bits are reset. | |
57 | * If the time since all clear was greater than the panic timeout, | |
58 | * we can panic with the list of stuck CPUs. | |
59 | * | |
60 | * This will work best with NMI IPIs for crash code so the stuck CPUs | |
61 | * can be pulled out to get their backtraces. | |
62 | */ | |
63 | static unsigned long __wd_smp_lock; | |
64 | static cpumask_t wd_smp_cpus_pending; | |
65 | static cpumask_t wd_smp_cpus_stuck; | |
66 | static u64 wd_smp_last_reset_tb; | |
67 | ||
68 | static inline void wd_smp_lock(unsigned long *flags) | |
69 | { | |
70 | /* | |
71 | * Avoid locking layers if possible. | |
72 | * This may be called from low level interrupt handlers at some | |
73 | * point in future. | |
74 | */ | |
d8e2a405 NP |
75 | raw_local_irq_save(*flags); |
76 | hard_irq_disable(); /* Make it soft-NMI safe */ | |
77 | while (unlikely(test_and_set_bit_lock(0, &__wd_smp_lock))) { | |
78 | raw_local_irq_restore(*flags); | |
79 | spin_until_cond(!test_bit(0, &__wd_smp_lock)); | |
80 | raw_local_irq_save(*flags); | |
81 | hard_irq_disable(); | |
82 | } | |
2104180a NP |
83 | } |
84 | ||
85 | static inline void wd_smp_unlock(unsigned long *flags) | |
86 | { | |
87 | clear_bit_unlock(0, &__wd_smp_lock); | |
d8e2a405 | 88 | raw_local_irq_restore(*flags); |
2104180a NP |
89 | } |
90 | ||
91 | static void wd_lockup_ipi(struct pt_regs *regs) | |
92 | { | |
93 | pr_emerg("Watchdog CPU:%d Hard LOCKUP\n", raw_smp_processor_id()); | |
94 | print_modules(); | |
95 | print_irqtrace_events(current); | |
96 | if (regs) | |
97 | show_regs(regs); | |
98 | else | |
99 | dump_stack(); | |
100 | ||
101 | if (hardlockup_panic) | |
102 | nmi_panic(regs, "Hard LOCKUP"); | |
103 | } | |
104 | ||
87607a30 | 105 | static void set_cpumask_stuck(const struct cpumask *cpumask, u64 tb) |
2104180a | 106 | { |
87607a30 NP |
107 | cpumask_or(&wd_smp_cpus_stuck, &wd_smp_cpus_stuck, cpumask); |
108 | cpumask_andnot(&wd_smp_cpus_pending, &wd_smp_cpus_pending, cpumask); | |
2104180a NP |
109 | if (cpumask_empty(&wd_smp_cpus_pending)) { |
110 | wd_smp_last_reset_tb = tb; | |
111 | cpumask_andnot(&wd_smp_cpus_pending, | |
112 | &wd_cpus_enabled, | |
113 | &wd_smp_cpus_stuck); | |
114 | } | |
115 | } | |
87607a30 NP |
116 | static void set_cpu_stuck(int cpu, u64 tb) |
117 | { | |
118 | set_cpumask_stuck(cpumask_of(cpu), tb); | |
119 | } | |
2104180a NP |
120 | |
121 | static void watchdog_smp_panic(int cpu, u64 tb) | |
122 | { | |
123 | unsigned long flags; | |
124 | int c; | |
125 | ||
126 | wd_smp_lock(&flags); | |
127 | /* Double check some things under lock */ | |
128 | if ((s64)(tb - wd_smp_last_reset_tb) < (s64)wd_smp_panic_timeout_tb) | |
129 | goto out; | |
130 | if (cpumask_test_cpu(cpu, &wd_smp_cpus_pending)) | |
131 | goto out; | |
132 | if (cpumask_weight(&wd_smp_cpus_pending) == 0) | |
133 | goto out; | |
134 | ||
135 | pr_emerg("Watchdog CPU:%d detected Hard LOCKUP other CPUS:%*pbl\n", | |
136 | cpu, cpumask_pr_args(&wd_smp_cpus_pending)); | |
137 | ||
138 | /* | |
139 | * Try to trigger the stuck CPUs. | |
140 | */ | |
141 | for_each_cpu(c, &wd_smp_cpus_pending) { | |
142 | if (c == cpu) | |
143 | continue; | |
144 | smp_send_nmi_ipi(c, wd_lockup_ipi, 1000000); | |
145 | } | |
146 | smp_flush_nmi_ipi(1000000); | |
147 | ||
87607a30 NP |
148 | /* Take the stuck CPUs out of the watch group */ |
149 | set_cpumask_stuck(&wd_smp_cpus_pending, tb); | |
2104180a | 150 | |
2104180a NP |
151 | wd_smp_unlock(&flags); |
152 | ||
153 | printk_safe_flush(); | |
154 | /* | |
155 | * printk_safe_flush() seems to require another print | |
156 | * before anything actually goes out to console. | |
157 | */ | |
158 | if (sysctl_hardlockup_all_cpu_backtrace) | |
159 | trigger_allbutself_cpu_backtrace(); | |
160 | ||
161 | if (hardlockup_panic) | |
162 | nmi_panic(NULL, "Hard LOCKUP"); | |
8e236921 NP |
163 | |
164 | return; | |
165 | ||
166 | out: | |
167 | wd_smp_unlock(&flags); | |
2104180a NP |
168 | } |
169 | ||
170 | static void wd_smp_clear_cpu_pending(int cpu, u64 tb) | |
171 | { | |
172 | if (!cpumask_test_cpu(cpu, &wd_smp_cpus_pending)) { | |
173 | if (unlikely(cpumask_test_cpu(cpu, &wd_smp_cpus_stuck))) { | |
174 | unsigned long flags; | |
175 | ||
176 | pr_emerg("Watchdog CPU:%d became unstuck\n", cpu); | |
177 | wd_smp_lock(&flags); | |
178 | cpumask_clear_cpu(cpu, &wd_smp_cpus_stuck); | |
179 | wd_smp_unlock(&flags); | |
180 | } | |
181 | return; | |
182 | } | |
183 | cpumask_clear_cpu(cpu, &wd_smp_cpus_pending); | |
184 | if (cpumask_empty(&wd_smp_cpus_pending)) { | |
185 | unsigned long flags; | |
186 | ||
187 | wd_smp_lock(&flags); | |
188 | if (cpumask_empty(&wd_smp_cpus_pending)) { | |
189 | wd_smp_last_reset_tb = tb; | |
190 | cpumask_andnot(&wd_smp_cpus_pending, | |
191 | &wd_cpus_enabled, | |
192 | &wd_smp_cpus_stuck); | |
193 | } | |
194 | wd_smp_unlock(&flags); | |
195 | } | |
196 | } | |
197 | ||
198 | static void watchdog_timer_interrupt(int cpu) | |
199 | { | |
200 | u64 tb = get_tb(); | |
201 | ||
202 | per_cpu(wd_timer_tb, cpu) = tb; | |
203 | ||
204 | wd_smp_clear_cpu_pending(cpu, tb); | |
205 | ||
206 | if ((s64)(tb - wd_smp_last_reset_tb) >= (s64)wd_smp_panic_timeout_tb) | |
207 | watchdog_smp_panic(cpu, tb); | |
208 | } | |
209 | ||
210 | void soft_nmi_interrupt(struct pt_regs *regs) | |
211 | { | |
212 | unsigned long flags; | |
213 | int cpu = raw_smp_processor_id(); | |
214 | u64 tb; | |
215 | ||
216 | if (!cpumask_test_cpu(cpu, &wd_cpus_enabled)) | |
217 | return; | |
218 | ||
219 | nmi_enter(); | |
04019bf8 NP |
220 | |
221 | __this_cpu_inc(irq_stat.soft_nmi_irqs); | |
222 | ||
2104180a NP |
223 | tb = get_tb(); |
224 | if (tb - per_cpu(wd_timer_tb, cpu) >= wd_panic_timeout_tb) { | |
225 | per_cpu(wd_timer_tb, cpu) = tb; | |
226 | ||
227 | wd_smp_lock(&flags); | |
228 | if (cpumask_test_cpu(cpu, &wd_smp_cpus_stuck)) { | |
229 | wd_smp_unlock(&flags); | |
230 | goto out; | |
231 | } | |
232 | set_cpu_stuck(cpu, tb); | |
233 | ||
234 | pr_emerg("Watchdog CPU:%d Hard LOCKUP\n", cpu); | |
235 | print_modules(); | |
236 | print_irqtrace_events(current); | |
237 | if (regs) | |
238 | show_regs(regs); | |
239 | else | |
240 | dump_stack(); | |
241 | ||
242 | wd_smp_unlock(&flags); | |
243 | ||
244 | if (sysctl_hardlockup_all_cpu_backtrace) | |
245 | trigger_allbutself_cpu_backtrace(); | |
246 | ||
247 | if (hardlockup_panic) | |
248 | nmi_panic(regs, "Hard LOCKUP"); | |
249 | } | |
250 | if (wd_panic_timeout_tb < 0x7fffffff) | |
251 | mtspr(SPRN_DEC, wd_panic_timeout_tb); | |
252 | ||
253 | out: | |
254 | nmi_exit(); | |
255 | } | |
256 | ||
257 | static void wd_timer_reset(unsigned int cpu, struct timer_list *t) | |
258 | { | |
259 | t->expires = jiffies + msecs_to_jiffies(wd_timer_period_ms); | |
260 | if (wd_timer_period_ms > 1000) | |
261 | t->expires = __round_jiffies_up(t->expires, cpu); | |
262 | add_timer_on(t, cpu); | |
263 | } | |
264 | ||
265 | static void wd_timer_fn(unsigned long data) | |
266 | { | |
267 | struct timer_list *t = this_cpu_ptr(&wd_timer); | |
268 | int cpu = smp_processor_id(); | |
269 | ||
270 | watchdog_timer_interrupt(cpu); | |
271 | ||
272 | wd_timer_reset(cpu, t); | |
273 | } | |
274 | ||
275 | void arch_touch_nmi_watchdog(void) | |
276 | { | |
26c5c6e1 | 277 | unsigned long ticks = tb_ticks_per_usec * wd_timer_period_ms * 1000; |
2104180a NP |
278 | int cpu = smp_processor_id(); |
279 | ||
26c5c6e1 NP |
280 | if (get_tb() - per_cpu(wd_timer_tb, cpu) >= ticks) |
281 | watchdog_timer_interrupt(cpu); | |
2104180a NP |
282 | } |
283 | EXPORT_SYMBOL(arch_touch_nmi_watchdog); | |
284 | ||
285 | static void start_watchdog_timer_on(unsigned int cpu) | |
286 | { | |
287 | struct timer_list *t = per_cpu_ptr(&wd_timer, cpu); | |
288 | ||
289 | per_cpu(wd_timer_tb, cpu) = get_tb(); | |
290 | ||
291 | setup_pinned_timer(t, wd_timer_fn, 0); | |
292 | wd_timer_reset(cpu, t); | |
293 | } | |
294 | ||
295 | static void stop_watchdog_timer_on(unsigned int cpu) | |
296 | { | |
297 | struct timer_list *t = per_cpu_ptr(&wd_timer, cpu); | |
298 | ||
299 | del_timer_sync(t); | |
300 | } | |
301 | ||
302 | static int start_wd_on_cpu(unsigned int cpu) | |
303 | { | |
96ea91e7 NP |
304 | unsigned long flags; |
305 | ||
2104180a NP |
306 | if (cpumask_test_cpu(cpu, &wd_cpus_enabled)) { |
307 | WARN_ON(1); | |
308 | return 0; | |
309 | } | |
310 | ||
311 | if (!(watchdog_enabled & NMI_WATCHDOG_ENABLED)) | |
312 | return 0; | |
313 | ||
2104180a NP |
314 | if (!cpumask_test_cpu(cpu, &watchdog_cpumask)) |
315 | return 0; | |
316 | ||
96ea91e7 | 317 | wd_smp_lock(&flags); |
2104180a NP |
318 | cpumask_set_cpu(cpu, &wd_cpus_enabled); |
319 | if (cpumask_weight(&wd_cpus_enabled) == 1) { | |
320 | cpumask_set_cpu(cpu, &wd_smp_cpus_pending); | |
321 | wd_smp_last_reset_tb = get_tb(); | |
322 | } | |
96ea91e7 NP |
323 | wd_smp_unlock(&flags); |
324 | ||
2104180a NP |
325 | start_watchdog_timer_on(cpu); |
326 | ||
327 | return 0; | |
328 | } | |
329 | ||
330 | static int stop_wd_on_cpu(unsigned int cpu) | |
331 | { | |
96ea91e7 NP |
332 | unsigned long flags; |
333 | ||
2104180a NP |
334 | if (!cpumask_test_cpu(cpu, &wd_cpus_enabled)) |
335 | return 0; /* Can happen in CPU unplug case */ | |
336 | ||
337 | stop_watchdog_timer_on(cpu); | |
338 | ||
96ea91e7 | 339 | wd_smp_lock(&flags); |
2104180a | 340 | cpumask_clear_cpu(cpu, &wd_cpus_enabled); |
96ea91e7 NP |
341 | wd_smp_unlock(&flags); |
342 | ||
2104180a NP |
343 | wd_smp_clear_cpu_pending(cpu, get_tb()); |
344 | ||
345 | return 0; | |
346 | } | |
347 | ||
348 | static void watchdog_calc_timeouts(void) | |
349 | { | |
350 | wd_panic_timeout_tb = watchdog_thresh * ppc_tb_freq; | |
351 | ||
352 | /* Have the SMP detector trigger a bit later */ | |
353 | wd_smp_panic_timeout_tb = wd_panic_timeout_tb * 3 / 2; | |
354 | ||
355 | /* 2/5 is the factor that the perf based detector uses */ | |
356 | wd_timer_period_ms = watchdog_thresh * 1000 * 2 / 5; | |
357 | } | |
358 | ||
6b9dc480 | 359 | void watchdog_nmi_stop(void) |
2104180a NP |
360 | { |
361 | int cpu; | |
362 | ||
6b9dc480 TG |
363 | for_each_cpu(cpu, &wd_cpus_enabled) |
364 | stop_wd_on_cpu(cpu); | |
6b9dc480 TG |
365 | } |
366 | ||
367 | void watchdog_nmi_start(void) | |
368 | { | |
369 | int cpu; | |
370 | ||
6b9dc480 TG |
371 | watchdog_calc_timeouts(); |
372 | for_each_cpu_and(cpu, cpu_online_mask, &watchdog_cpumask) | |
373 | start_wd_on_cpu(cpu); | |
2104180a NP |
374 | } |
375 | ||
376 | /* | |
34ddaa3e | 377 | * Invoked from core watchdog init. |
2104180a | 378 | */ |
34ddaa3e | 379 | int __init watchdog_nmi_probe(void) |
2104180a NP |
380 | { |
381 | int err; | |
382 | ||
34ddaa3e TG |
383 | err = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, |
384 | "powerpc/watchdog:online", | |
385 | start_wd_on_cpu, stop_wd_on_cpu); | |
386 | if (err < 0) { | |
2104180a | 387 | pr_warn("Watchdog could not be initialized"); |
34ddaa3e TG |
388 | return err; |
389 | } | |
2104180a NP |
390 | return 0; |
391 | } | |
2104180a NP |
392 | |
393 | static void handle_backtrace_ipi(struct pt_regs *regs) | |
394 | { | |
395 | nmi_cpu_backtrace(regs); | |
396 | } | |
397 | ||
398 | static void raise_backtrace_ipi(cpumask_t *mask) | |
399 | { | |
400 | unsigned int cpu; | |
401 | ||
402 | for_each_cpu(cpu, mask) { | |
403 | if (cpu == smp_processor_id()) | |
404 | handle_backtrace_ipi(NULL); | |
405 | else | |
406 | smp_send_nmi_ipi(cpu, handle_backtrace_ipi, 1000000); | |
407 | } | |
408 | } | |
409 | ||
410 | void arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self) | |
411 | { | |
412 | nmi_trigger_cpumask_backtrace(mask, exclude_self, raise_backtrace_ipi); | |
413 | } |