Commit | Line | Data |
---|---|---|
8f8e76c0 | 1 | // SPDX-License-Identifier: GPL-2.0+ |
51b1130e PM |
2 | /* |
3 | * Common functions for in-kernel torture tests. | |
4 | * | |
51b1130e PM |
5 | * Copyright (C) IBM Corporation, 2014 |
6 | * | |
8f8e76c0 | 7 | * Author: Paul E. McKenney <paulmck@linux.ibm.com> |
51b1130e PM |
8 | * Based on kernel/rcu/torture.c. |
9 | */ | |
60500037 PM |
10 | |
11 | #define pr_fmt(fmt) fmt | |
12 | ||
51b1130e PM |
13 | #include <linux/types.h> |
14 | #include <linux/kernel.h> | |
15 | #include <linux/init.h> | |
16 | #include <linux/module.h> | |
17 | #include <linux/kthread.h> | |
18 | #include <linux/err.h> | |
19 | #include <linux/spinlock.h> | |
20 | #include <linux/smp.h> | |
21 | #include <linux/interrupt.h> | |
22 | #include <linux/sched.h> | |
e6017571 | 23 | #include <linux/sched/clock.h> |
51b1130e PM |
24 | #include <linux/atomic.h> |
25 | #include <linux/bitops.h> | |
26 | #include <linux/completion.h> | |
27 | #include <linux/moduleparam.h> | |
28 | #include <linux/percpu.h> | |
29 | #include <linux/notifier.h> | |
30 | #include <linux/reboot.h> | |
31 | #include <linux/freezer.h> | |
32 | #include <linux/cpu.h> | |
33 | #include <linux/delay.h> | |
34 | #include <linux/stat.h> | |
35 | #include <linux/slab.h> | |
36 | #include <linux/trace_clock.h> | |
31257c3c | 37 | #include <linux/ktime.h> |
51b1130e PM |
38 | #include <asm/byteorder.h> |
39 | #include <linux/torture.h> | |
dac95906 | 40 | #include "rcu/rcu.h" |
51b1130e PM |
41 | |
42 | MODULE_LICENSE("GPL"); | |
8f8e76c0 | 43 | MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>"); |
51b1130e | 44 | |
8171d3e0 PM |
45 | static bool disable_onoff_at_boot; |
46 | module_param(disable_onoff_at_boot, bool, 0444); | |
47 | ||
2102ad29 PM |
48 | static bool ftrace_dump_at_shutdown; |
49 | module_param(ftrace_dump_at_shutdown, bool, 0444); | |
50 | ||
8a67a20b PM |
51 | static int verbose_sleep_frequency; |
52 | module_param(verbose_sleep_frequency, int, 0444); | |
53 | ||
54 | static int verbose_sleep_duration = 1; | |
55 | module_param(verbose_sleep_duration, int, 0444); | |
56 | ||
b5daa8f3 | 57 | static char *torture_type; |
90127d60 | 58 | static int verbose; |
b5daa8f3 | 59 | |
36970bb9 PM |
60 | /* Mediate rmmod and system shutdown. Concurrent rmmod & shutdown illegal! */ |
61 | #define FULLSTOP_DONTSTOP 0 /* Normal operation. */ | |
62 | #define FULLSTOP_SHUTDOWN 1 /* System shutdown with torture running. */ | |
63 | #define FULLSTOP_RMMOD 2 /* Normal rmmod of torture. */ | |
64 | static int fullstop = FULLSTOP_RMMOD; | |
4622b487 | 65 | static DEFINE_MUTEX(fullstop_mutex); |
f67a3356 | 66 | |
8a67a20b PM |
67 | static atomic_t verbose_sleep_counter; |
68 | ||
69 | /* | |
70 | * Sleep if needed from VERBOSE_TOROUT*(). | |
71 | */ | |
72 | void verbose_torout_sleep(void) | |
73 | { | |
74 | if (verbose_sleep_frequency > 0 && | |
75 | verbose_sleep_duration > 0 && | |
76 | !(atomic_inc_return(&verbose_sleep_counter) % verbose_sleep_frequency)) | |
77 | schedule_timeout_uninterruptible(verbose_sleep_duration); | |
78 | } | |
79 | EXPORT_SYMBOL_GPL(verbose_torout_sleep); | |
80 | ||
ae19aaaf PM |
81 | /* |
82 | * Schedule a high-resolution-timer sleep in nanoseconds, with a 32-bit | |
83 | * nanosecond random fuzz. This function and its friends desynchronize | |
84 | * testing from the timer wheel. | |
85 | */ | |
86 | int torture_hrtimeout_ns(ktime_t baset_ns, u32 fuzzt_ns, struct torture_random_state *trsp) | |
87 | { | |
88 | ktime_t hto = baset_ns; | |
89 | ||
90 | if (trsp) | |
91 | hto += (torture_random(trsp) >> 3) % fuzzt_ns; | |
92 | set_current_state(TASK_UNINTERRUPTIBLE); | |
93 | return schedule_hrtimeout(&hto, HRTIMER_MODE_REL); | |
94 | } | |
95 | EXPORT_SYMBOL_GPL(torture_hrtimeout_ns); | |
96 | ||
97 | /* | |
98 | * Schedule a high-resolution-timer sleep in microseconds, with a 32-bit | |
99 | * nanosecond (not microsecond!) random fuzz. | |
100 | */ | |
101 | int torture_hrtimeout_us(u32 baset_us, u32 fuzzt_ns, struct torture_random_state *trsp) | |
102 | { | |
103 | ktime_t baset_ns = baset_us * NSEC_PER_USEC; | |
104 | ||
105 | return torture_hrtimeout_ns(baset_ns, fuzzt_ns, trsp); | |
106 | } | |
107 | EXPORT_SYMBOL_GPL(torture_hrtimeout_us); | |
108 | ||
109 | /* | |
110 | * Schedule a high-resolution-timer sleep in milliseconds, with a 32-bit | |
111 | * microsecond (not millisecond!) random fuzz. | |
112 | */ | |
113 | int torture_hrtimeout_ms(u32 baset_ms, u32 fuzzt_us, struct torture_random_state *trsp) | |
114 | { | |
115 | ktime_t baset_ns = baset_ms * NSEC_PER_MSEC; | |
116 | u32 fuzzt_ns; | |
117 | ||
118 | if ((u32)~0U / NSEC_PER_USEC < fuzzt_us) | |
119 | fuzzt_ns = (u32)~0U; | |
120 | else | |
121 | fuzzt_ns = fuzzt_us * NSEC_PER_USEC; | |
122 | return torture_hrtimeout_ns(baset_ns, fuzzt_ns, trsp); | |
123 | } | |
124 | EXPORT_SYMBOL_GPL(torture_hrtimeout_ms); | |
125 | ||
126 | /* | |
127 | * Schedule a high-resolution-timer sleep in jiffies, with an | |
128 | * implied one-jiffy random fuzz. This is intended to replace calls to | |
129 | * schedule_timeout_interruptible() and friends. | |
130 | */ | |
131 | int torture_hrtimeout_jiffies(u32 baset_j, struct torture_random_state *trsp) | |
132 | { | |
133 | ktime_t baset_ns = jiffies_to_nsecs(baset_j); | |
134 | ||
135 | return torture_hrtimeout_ns(baset_ns, jiffies_to_nsecs(1), trsp); | |
136 | } | |
137 | EXPORT_SYMBOL_GPL(torture_hrtimeout_jiffies); | |
138 | ||
139 | /* | |
140 | * Schedule a high-resolution-timer sleep in milliseconds, with a 32-bit | |
141 | * millisecond (not second!) random fuzz. | |
142 | */ | |
143 | int torture_hrtimeout_s(u32 baset_s, u32 fuzzt_ms, struct torture_random_state *trsp) | |
144 | { | |
145 | ktime_t baset_ns = baset_s * NSEC_PER_SEC; | |
146 | u32 fuzzt_ns; | |
147 | ||
148 | if ((u32)~0U / NSEC_PER_MSEC < fuzzt_ms) | |
149 | fuzzt_ns = (u32)~0U; | |
150 | else | |
151 | fuzzt_ns = fuzzt_ms * NSEC_PER_MSEC; | |
152 | return torture_hrtimeout_ns(baset_ns, fuzzt_ns, trsp); | |
153 | } | |
154 | EXPORT_SYMBOL_GPL(torture_hrtimeout_s); | |
155 | ||
2e9e8081 PM |
156 | #ifdef CONFIG_HOTPLUG_CPU |
157 | ||
158 | /* | |
159 | * Variables for online-offline handling. Only present if CPU hotplug | |
160 | * is enabled, otherwise does nothing. | |
161 | */ | |
162 | ||
163 | static struct task_struct *onoff_task; | |
164 | static long onoff_holdoff; | |
165 | static long onoff_interval; | |
3a6cb58f | 166 | static torture_ofl_func *onoff_f; |
2e9e8081 PM |
167 | static long n_offline_attempts; |
168 | static long n_offline_successes; | |
169 | static unsigned long sum_offline; | |
170 | static int min_offline = -1; | |
171 | static int max_offline; | |
172 | static long n_online_attempts; | |
173 | static long n_online_successes; | |
174 | static unsigned long sum_online; | |
175 | static int min_online = -1; | |
176 | static int max_online; | |
177 | ||
1afb95fe PM |
178 | static int torture_online_cpus = NR_CPUS; |
179 | ||
180 | /* | |
181 | * Some torture testing leverages confusion as to the number of online | |
182 | * CPUs. This function returns the torture-testing view of this number, | |
183 | * which allows torture tests to load-balance appropriately. | |
184 | */ | |
185 | int torture_num_online_cpus(void) | |
186 | { | |
187 | return READ_ONCE(torture_online_cpus); | |
188 | } | |
189 | EXPORT_SYMBOL_GPL(torture_num_online_cpus); | |
190 | ||
d95f5ba9 PM |
191 | /* |
192 | * Attempt to take a CPU offline. Return false if the CPU is already | |
193 | * offline or if it is not subject to CPU-hotplug operations. The | |
194 | * caller can detect other failures by looking at the statistics. | |
195 | */ | |
196 | bool torture_offline(int cpu, long *n_offl_attempts, long *n_offl_successes, | |
197 | unsigned long *sum_offl, int *min_offl, int *max_offl) | |
198 | { | |
199 | unsigned long delta; | |
200 | int ret; | |
a59ee765 | 201 | char *s; |
d95f5ba9 PM |
202 | unsigned long starttime; |
203 | ||
204 | if (!cpu_online(cpu) || !cpu_is_hotpluggable(cpu)) | |
205 | return false; | |
24aca4ae PM |
206 | if (num_online_cpus() <= 1) |
207 | return false; /* Can't offline the last CPU. */ | |
d95f5ba9 | 208 | |
90127d60 | 209 | if (verbose > 1) |
d95f5ba9 PM |
210 | pr_alert("%s" TORTURE_FLAG |
211 | "torture_onoff task: offlining %d\n", | |
212 | torture_type, cpu); | |
213 | starttime = jiffies; | |
214 | (*n_offl_attempts)++; | |
457bc8ed | 215 | ret = remove_cpu(cpu); |
d95f5ba9 | 216 | if (ret) { |
a59ee765 PM |
217 | s = ""; |
218 | if (!rcu_inkernel_boot_has_ended() && ret == -EBUSY) { | |
219 | // PCI probe frequently disables hotplug during boot. | |
220 | (*n_offl_attempts)--; | |
221 | s = " (-EBUSY forgiven during boot)"; | |
222 | } | |
d95f5ba9 PM |
223 | if (verbose) |
224 | pr_alert("%s" TORTURE_FLAG | |
a59ee765 PM |
225 | "torture_onoff task: offline %d failed%s: errno %d\n", |
226 | torture_type, cpu, s, ret); | |
d95f5ba9 | 227 | } else { |
90127d60 | 228 | if (verbose > 1) |
d95f5ba9 PM |
229 | pr_alert("%s" TORTURE_FLAG |
230 | "torture_onoff task: offlined %d\n", | |
231 | torture_type, cpu); | |
3a6cb58f PM |
232 | if (onoff_f) |
233 | onoff_f(); | |
d95f5ba9 PM |
234 | (*n_offl_successes)++; |
235 | delta = jiffies - starttime; | |
a2b2df20 | 236 | *sum_offl += delta; |
d95f5ba9 PM |
237 | if (*min_offl < 0) { |
238 | *min_offl = delta; | |
239 | *max_offl = delta; | |
240 | } | |
241 | if (*min_offl > delta) | |
242 | *min_offl = delta; | |
243 | if (*max_offl < delta) | |
244 | *max_offl = delta; | |
1afb95fe PM |
245 | WRITE_ONCE(torture_online_cpus, torture_online_cpus - 1); |
246 | WARN_ON_ONCE(torture_online_cpus <= 0); | |
d95f5ba9 PM |
247 | } |
248 | ||
249 | return true; | |
250 | } | |
251 | EXPORT_SYMBOL_GPL(torture_offline); | |
252 | ||
253 | /* | |
254 | * Attempt to bring a CPU online. Return false if the CPU is already | |
255 | * online or if it is not subject to CPU-hotplug operations. The | |
256 | * caller can detect other failures by looking at the statistics. | |
257 | */ | |
258 | bool torture_online(int cpu, long *n_onl_attempts, long *n_onl_successes, | |
259 | unsigned long *sum_onl, int *min_onl, int *max_onl) | |
260 | { | |
261 | unsigned long delta; | |
262 | int ret; | |
a59ee765 | 263 | char *s; |
d95f5ba9 PM |
264 | unsigned long starttime; |
265 | ||
266 | if (cpu_online(cpu) || !cpu_is_hotpluggable(cpu)) | |
267 | return false; | |
268 | ||
90127d60 | 269 | if (verbose > 1) |
d95f5ba9 PM |
270 | pr_alert("%s" TORTURE_FLAG |
271 | "torture_onoff task: onlining %d\n", | |
272 | torture_type, cpu); | |
273 | starttime = jiffies; | |
274 | (*n_onl_attempts)++; | |
457bc8ed | 275 | ret = add_cpu(cpu); |
d95f5ba9 | 276 | if (ret) { |
a59ee765 PM |
277 | s = ""; |
278 | if (!rcu_inkernel_boot_has_ended() && ret == -EBUSY) { | |
279 | // PCI probe frequently disables hotplug during boot. | |
280 | (*n_onl_attempts)--; | |
281 | s = " (-EBUSY forgiven during boot)"; | |
282 | } | |
d95f5ba9 PM |
283 | if (verbose) |
284 | pr_alert("%s" TORTURE_FLAG | |
a59ee765 PM |
285 | "torture_onoff task: online %d failed%s: errno %d\n", |
286 | torture_type, cpu, s, ret); | |
d95f5ba9 | 287 | } else { |
90127d60 | 288 | if (verbose > 1) |
d95f5ba9 PM |
289 | pr_alert("%s" TORTURE_FLAG |
290 | "torture_onoff task: onlined %d\n", | |
291 | torture_type, cpu); | |
292 | (*n_onl_successes)++; | |
293 | delta = jiffies - starttime; | |
294 | *sum_onl += delta; | |
295 | if (*min_onl < 0) { | |
296 | *min_onl = delta; | |
297 | *max_onl = delta; | |
298 | } | |
299 | if (*min_onl > delta) | |
300 | *min_onl = delta; | |
301 | if (*max_onl < delta) | |
302 | *max_onl = delta; | |
1afb95fe | 303 | WRITE_ONCE(torture_online_cpus, torture_online_cpus + 1); |
d95f5ba9 PM |
304 | } |
305 | ||
306 | return true; | |
307 | } | |
308 | EXPORT_SYMBOL_GPL(torture_online); | |
309 | ||
0b962c8f PM |
310 | /* |
311 | * Get everything online at the beginning and ends of tests. | |
312 | */ | |
313 | static void torture_online_all(char *phase) | |
314 | { | |
315 | int cpu; | |
316 | int ret; | |
317 | ||
318 | for_each_possible_cpu(cpu) { | |
319 | if (cpu_online(cpu)) | |
320 | continue; | |
321 | ret = add_cpu(cpu); | |
322 | if (ret && verbose) { | |
323 | pr_alert("%s" TORTURE_FLAG | |
324 | "%s: %s online %d: errno %d\n", | |
325 | __func__, phase, torture_type, cpu, ret); | |
326 | } | |
327 | } | |
328 | } | |
329 | ||
2e9e8081 PM |
330 | /* |
331 | * Execute random CPU-hotplug operations at the interval specified | |
332 | * by the onoff_interval. | |
333 | */ | |
334 | static int | |
335 | torture_onoff(void *arg) | |
336 | { | |
337 | int cpu; | |
2e9e8081 PM |
338 | int maxcpu = -1; |
339 | DEFINE_TORTURE_RANDOM(rand); | |
2e9e8081 PM |
340 | |
341 | VERBOSE_TOROUT_STRING("torture_onoff task started"); | |
342 | for_each_online_cpu(cpu) | |
343 | maxcpu = cpu; | |
344 | WARN_ON(maxcpu < 0); | |
0b962c8f | 345 | torture_online_all("Initial"); |
750db0f5 BF |
346 | if (maxcpu == 0) { |
347 | VERBOSE_TOROUT_STRING("Only one CPU, so CPU-hotplug testing is disabled"); | |
348 | goto stop; | |
349 | } | |
350 | ||
2e9e8081 PM |
351 | if (onoff_holdoff > 0) { |
352 | VERBOSE_TOROUT_STRING("torture_onoff begin holdoff"); | |
353 | schedule_timeout_interruptible(onoff_holdoff); | |
354 | VERBOSE_TOROUT_STRING("torture_onoff end holdoff"); | |
355 | } | |
356 | while (!torture_must_stop()) { | |
8171d3e0 PM |
357 | if (disable_onoff_at_boot && !rcu_inkernel_boot_has_ended()) { |
358 | schedule_timeout_interruptible(HZ / 10); | |
359 | continue; | |
360 | } | |
2e9e8081 | 361 | cpu = (torture_random(&rand) >> 4) % (maxcpu + 1); |
d95f5ba9 PM |
362 | if (!torture_offline(cpu, |
363 | &n_offline_attempts, &n_offline_successes, | |
364 | &sum_offline, &min_offline, &max_offline)) | |
365 | torture_online(cpu, | |
366 | &n_online_attempts, &n_online_successes, | |
367 | &sum_online, &min_online, &max_online); | |
2e9e8081 PM |
368 | schedule_timeout_interruptible(onoff_interval); |
369 | } | |
750db0f5 BF |
370 | |
371 | stop: | |
7fafaac5 | 372 | torture_kthread_stopping("torture_onoff"); |
0b962c8f | 373 | torture_online_all("Final"); |
2e9e8081 PM |
374 | return 0; |
375 | } | |
376 | ||
377 | #endif /* #ifdef CONFIG_HOTPLUG_CPU */ | |
378 | ||
379 | /* | |
380 | * Initiate online-offline handling. | |
381 | */ | |
3a6cb58f | 382 | int torture_onoff_init(long ooholdoff, long oointerval, torture_ofl_func *f) |
2e9e8081 | 383 | { |
47cf29b9 | 384 | #ifdef CONFIG_HOTPLUG_CPU |
2e9e8081 PM |
385 | onoff_holdoff = ooholdoff; |
386 | onoff_interval = oointerval; | |
3a6cb58f | 387 | onoff_f = f; |
2e9e8081 PM |
388 | if (onoff_interval <= 0) |
389 | return 0; | |
2a7d9688 PG |
390 | return torture_create_kthread(torture_onoff, NULL, onoff_task); |
391 | #else /* #ifdef CONFIG_HOTPLUG_CPU */ | |
392 | return 0; | |
393 | #endif /* #else #ifdef CONFIG_HOTPLUG_CPU */ | |
2e9e8081 PM |
394 | } |
395 | EXPORT_SYMBOL_GPL(torture_onoff_init); | |
396 | ||
397 | /* | |
398 | * Clean up after online/offline testing. | |
399 | */ | |
cc47ae08 | 400 | static void torture_onoff_cleanup(void) |
2e9e8081 PM |
401 | { |
402 | #ifdef CONFIG_HOTPLUG_CPU | |
403 | if (onoff_task == NULL) | |
404 | return; | |
405 | VERBOSE_TOROUT_STRING("Stopping torture_onoff task"); | |
406 | kthread_stop(onoff_task); | |
407 | onoff_task = NULL; | |
408 | #endif /* #ifdef CONFIG_HOTPLUG_CPU */ | |
409 | } | |
2e9e8081 PM |
410 | |
411 | /* | |
412 | * Print online/offline testing statistics. | |
413 | */ | |
eea203fe | 414 | void torture_onoff_stats(void) |
2e9e8081 PM |
415 | { |
416 | #ifdef CONFIG_HOTPLUG_CPU | |
eea203fe JP |
417 | pr_cont("onoff: %ld/%ld:%ld/%ld %d,%d:%d,%d %lu:%lu (HZ=%d) ", |
418 | n_online_successes, n_online_attempts, | |
419 | n_offline_successes, n_offline_attempts, | |
420 | min_online, max_online, | |
421 | min_offline, max_offline, | |
422 | sum_online, sum_offline, HZ); | |
2e9e8081 | 423 | #endif /* #ifdef CONFIG_HOTPLUG_CPU */ |
2e9e8081 PM |
424 | } |
425 | EXPORT_SYMBOL_GPL(torture_onoff_stats); | |
426 | ||
427 | /* | |
428 | * Were all the online/offline operations successful? | |
429 | */ | |
430 | bool torture_onoff_failures(void) | |
431 | { | |
432 | #ifdef CONFIG_HOTPLUG_CPU | |
433 | return n_online_successes != n_online_attempts || | |
434 | n_offline_successes != n_offline_attempts; | |
435 | #else /* #ifdef CONFIG_HOTPLUG_CPU */ | |
436 | return false; | |
437 | #endif /* #else #ifdef CONFIG_HOTPLUG_CPU */ | |
438 | } | |
439 | EXPORT_SYMBOL_GPL(torture_onoff_failures); | |
440 | ||
51b1130e PM |
441 | #define TORTURE_RANDOM_MULT 39916801 /* prime */ |
442 | #define TORTURE_RANDOM_ADD 479001701 /* prime */ | |
443 | #define TORTURE_RANDOM_REFRESH 10000 | |
444 | ||
445 | /* | |
446 | * Crude but fast random-number generator. Uses a linear congruential | |
447 | * generator, with occasional help from cpu_clock(). | |
448 | */ | |
449 | unsigned long | |
450 | torture_random(struct torture_random_state *trsp) | |
451 | { | |
452 | if (--trsp->trs_count < 0) { | |
453 | trsp->trs_state += (unsigned long)local_clock(); | |
454 | trsp->trs_count = TORTURE_RANDOM_REFRESH; | |
455 | } | |
456 | trsp->trs_state = trsp->trs_state * TORTURE_RANDOM_MULT + | |
457 | TORTURE_RANDOM_ADD; | |
458 | return swahw32(trsp->trs_state); | |
459 | } | |
460 | EXPORT_SYMBOL_GPL(torture_random); | |
f67a3356 | 461 | |
3808dc9f PM |
462 | /* |
463 | * Variables for shuffling. The idea is to ensure that each CPU stays | |
464 | * idle for an extended period to test interactions with dyntick idle, | |
b564d62e | 465 | * as well as interactions with any per-CPU variables. |
3808dc9f PM |
466 | */ |
467 | struct shuffle_task { | |
468 | struct list_head st_l; | |
469 | struct task_struct *st_t; | |
470 | }; | |
471 | ||
472 | static long shuffle_interval; /* In jiffies. */ | |
473 | static struct task_struct *shuffler_task; | |
474 | static cpumask_var_t shuffle_tmp_mask; | |
475 | static int shuffle_idle_cpu; /* Force all torture tasks off this CPU */ | |
476 | static struct list_head shuffle_task_list = LIST_HEAD_INIT(shuffle_task_list); | |
477 | static DEFINE_MUTEX(shuffle_task_mutex); | |
478 | ||
479 | /* | |
480 | * Register a task to be shuffled. If there is no memory, just splat | |
481 | * and don't bother registering. | |
482 | */ | |
483 | void torture_shuffle_task_register(struct task_struct *tp) | |
484 | { | |
485 | struct shuffle_task *stp; | |
486 | ||
487 | if (WARN_ON_ONCE(tp == NULL)) | |
488 | return; | |
489 | stp = kmalloc(sizeof(*stp), GFP_KERNEL); | |
490 | if (WARN_ON_ONCE(stp == NULL)) | |
491 | return; | |
492 | stp->st_t = tp; | |
493 | mutex_lock(&shuffle_task_mutex); | |
494 | list_add(&stp->st_l, &shuffle_task_list); | |
495 | mutex_unlock(&shuffle_task_mutex); | |
496 | } | |
497 | EXPORT_SYMBOL_GPL(torture_shuffle_task_register); | |
498 | ||
499 | /* | |
500 | * Unregister all tasks, for example, at the end of the torture run. | |
501 | */ | |
502 | static void torture_shuffle_task_unregister_all(void) | |
503 | { | |
504 | struct shuffle_task *stp; | |
505 | struct shuffle_task *p; | |
506 | ||
507 | mutex_lock(&shuffle_task_mutex); | |
508 | list_for_each_entry_safe(stp, p, &shuffle_task_list, st_l) { | |
509 | list_del(&stp->st_l); | |
510 | kfree(stp); | |
511 | } | |
512 | mutex_unlock(&shuffle_task_mutex); | |
513 | } | |
514 | ||
515 | /* Shuffle tasks such that we allow shuffle_idle_cpu to become idle. | |
516 | * A special case is when shuffle_idle_cpu = -1, in which case we allow | |
517 | * the tasks to run on all CPUs. | |
518 | */ | |
519 | static void torture_shuffle_tasks(void) | |
520 | { | |
521 | struct shuffle_task *stp; | |
522 | ||
523 | cpumask_setall(shuffle_tmp_mask); | |
ed4fa244 | 524 | cpus_read_lock(); |
3808dc9f PM |
525 | |
526 | /* No point in shuffling if there is only one online CPU (ex: UP) */ | |
527 | if (num_online_cpus() == 1) { | |
ed4fa244 | 528 | cpus_read_unlock(); |
3808dc9f PM |
529 | return; |
530 | } | |
531 | ||
532 | /* Advance to the next CPU. Upon overflow, don't idle any CPUs. */ | |
533 | shuffle_idle_cpu = cpumask_next(shuffle_idle_cpu, shuffle_tmp_mask); | |
534 | if (shuffle_idle_cpu >= nr_cpu_ids) | |
535 | shuffle_idle_cpu = -1; | |
5ed63b19 | 536 | else |
3808dc9f | 537 | cpumask_clear_cpu(shuffle_idle_cpu, shuffle_tmp_mask); |
3808dc9f PM |
538 | |
539 | mutex_lock(&shuffle_task_mutex); | |
540 | list_for_each_entry(stp, &shuffle_task_list, st_l) | |
541 | set_cpus_allowed_ptr(stp->st_t, shuffle_tmp_mask); | |
542 | mutex_unlock(&shuffle_task_mutex); | |
543 | ||
ed4fa244 | 544 | cpus_read_unlock(); |
3808dc9f PM |
545 | } |
546 | ||
547 | /* Shuffle tasks across CPUs, with the intent of allowing each CPU in the | |
548 | * system to become idle at a time and cut off its timer ticks. This is meant | |
549 | * to test the support for such tickless idle CPU in RCU. | |
550 | */ | |
551 | static int torture_shuffle(void *arg) | |
552 | { | |
553 | VERBOSE_TOROUT_STRING("torture_shuffle task started"); | |
554 | do { | |
555 | schedule_timeout_interruptible(shuffle_interval); | |
556 | torture_shuffle_tasks(); | |
557 | torture_shutdown_absorb("torture_shuffle"); | |
558 | } while (!torture_must_stop()); | |
7fafaac5 | 559 | torture_kthread_stopping("torture_shuffle"); |
3808dc9f PM |
560 | return 0; |
561 | } | |
562 | ||
563 | /* | |
564 | * Start the shuffler, with shuffint in jiffies. | |
565 | */ | |
566 | int torture_shuffle_init(long shuffint) | |
567 | { | |
3808dc9f PM |
568 | shuffle_interval = shuffint; |
569 | ||
570 | shuffle_idle_cpu = -1; | |
571 | ||
572 | if (!alloc_cpumask_var(&shuffle_tmp_mask, GFP_KERNEL)) { | |
81faa4f6 | 573 | TOROUT_ERRSTRING("Failed to alloc mask"); |
3808dc9f PM |
574 | return -ENOMEM; |
575 | } | |
576 | ||
577 | /* Create the shuffler thread */ | |
47cf29b9 | 578 | return torture_create_kthread(torture_shuffle, NULL, shuffler_task); |
3808dc9f PM |
579 | } |
580 | EXPORT_SYMBOL_GPL(torture_shuffle_init); | |
581 | ||
582 | /* | |
583 | * Stop the shuffling. | |
584 | */ | |
cc47ae08 | 585 | static void torture_shuffle_cleanup(void) |
3808dc9f PM |
586 | { |
587 | torture_shuffle_task_unregister_all(); | |
588 | if (shuffler_task) { | |
589 | VERBOSE_TOROUT_STRING("Stopping torture_shuffle task"); | |
590 | kthread_stop(shuffler_task); | |
591 | free_cpumask_var(shuffle_tmp_mask); | |
592 | } | |
593 | shuffler_task = NULL; | |
594 | } | |
3808dc9f | 595 | |
e991dbc0 PM |
596 | /* |
597 | * Variables for auto-shutdown. This allows "lights out" torture runs | |
598 | * to be fully scripted. | |
599 | */ | |
e991dbc0 | 600 | static struct task_struct *shutdown_task; |
31257c3c | 601 | static ktime_t shutdown_time; /* time to system shutdown. */ |
e991dbc0 PM |
602 | static void (*torture_shutdown_hook)(void); |
603 | ||
f67a3356 PM |
604 | /* |
605 | * Absorb kthreads into a kernel function that won't return, so that | |
606 | * they won't ever access module text or data again. | |
607 | */ | |
608 | void torture_shutdown_absorb(const char *title) | |
609 | { | |
7d0ae808 | 610 | while (READ_ONCE(fullstop) == FULLSTOP_SHUTDOWN) { |
4622b487 PM |
611 | pr_notice("torture thread %s parking due to system shutdown\n", |
612 | title); | |
f67a3356 PM |
613 | schedule_timeout_uninterruptible(MAX_SCHEDULE_TIMEOUT); |
614 | } | |
615 | } | |
616 | EXPORT_SYMBOL_GPL(torture_shutdown_absorb); | |
b5daa8f3 | 617 | |
e991dbc0 PM |
618 | /* |
619 | * Cause the torture test to shutdown the system after the test has | |
620 | * run for the time specified by the shutdown_secs parameter. | |
621 | */ | |
622 | static int torture_shutdown(void *arg) | |
623 | { | |
31257c3c | 624 | ktime_t ktime_snap; |
e991dbc0 PM |
625 | |
626 | VERBOSE_TOROUT_STRING("torture_shutdown task started"); | |
31257c3c PM |
627 | ktime_snap = ktime_get(); |
628 | while (ktime_before(ktime_snap, shutdown_time) && | |
e991dbc0 | 629 | !torture_must_stop()) { |
e991dbc0 PM |
630 | if (verbose) |
631 | pr_alert("%s" TORTURE_FLAG | |
31257c3c PM |
632 | "torture_shutdown task: %llu ms remaining\n", |
633 | torture_type, | |
634 | ktime_ms_delta(shutdown_time, ktime_snap)); | |
635 | set_current_state(TASK_INTERRUPTIBLE); | |
636 | schedule_hrtimeout(&shutdown_time, HRTIMER_MODE_ABS); | |
637 | ktime_snap = ktime_get(); | |
e991dbc0 PM |
638 | } |
639 | if (torture_must_stop()) { | |
7fafaac5 | 640 | torture_kthread_stopping("torture_shutdown"); |
e991dbc0 PM |
641 | return 0; |
642 | } | |
643 | ||
644 | /* OK, shut down the system. */ | |
645 | ||
646 | VERBOSE_TOROUT_STRING("torture_shutdown task shutting down system"); | |
647 | shutdown_task = NULL; /* Avoid self-kill deadlock. */ | |
f881825a PM |
648 | if (torture_shutdown_hook) |
649 | torture_shutdown_hook(); | |
650 | else | |
651 | VERBOSE_TOROUT_STRING("No torture_shutdown_hook(), skipping."); | |
2102ad29 PM |
652 | if (ftrace_dump_at_shutdown) |
653 | rcu_ftrace_dump(DUMP_ALL); | |
e991dbc0 PM |
654 | kernel_power_off(); /* Shut down the system. */ |
655 | return 0; | |
656 | } | |
657 | ||
658 | /* | |
659 | * Start up the shutdown task. | |
660 | */ | |
661 | int torture_shutdown_init(int ssecs, void (*cleanup)(void)) | |
662 | { | |
e991dbc0 | 663 | torture_shutdown_hook = cleanup; |
31257c3c PM |
664 | if (ssecs > 0) { |
665 | shutdown_time = ktime_add(ktime_get(), ktime_set(ssecs, 0)); | |
2a7d9688 | 666 | return torture_create_kthread(torture_shutdown, NULL, |
47cf29b9 | 667 | shutdown_task); |
e991dbc0 | 668 | } |
2a7d9688 | 669 | return 0; |
e991dbc0 PM |
670 | } |
671 | EXPORT_SYMBOL_GPL(torture_shutdown_init); | |
672 | ||
4622b487 PM |
673 | /* |
674 | * Detect and respond to a system shutdown. | |
675 | */ | |
676 | static int torture_shutdown_notify(struct notifier_block *unused1, | |
677 | unsigned long unused2, void *unused3) | |
678 | { | |
679 | mutex_lock(&fullstop_mutex); | |
7d0ae808 | 680 | if (READ_ONCE(fullstop) == FULLSTOP_DONTSTOP) { |
fac480ef | 681 | VERBOSE_TOROUT_STRING("Unscheduled system shutdown detected"); |
7d0ae808 | 682 | WRITE_ONCE(fullstop, FULLSTOP_SHUTDOWN); |
fac480ef | 683 | } else { |
4622b487 | 684 | pr_warn("Concurrent rmmod and shutdown illegal!\n"); |
fac480ef | 685 | } |
4622b487 PM |
686 | mutex_unlock(&fullstop_mutex); |
687 | return NOTIFY_DONE; | |
688 | } | |
689 | ||
690 | static struct notifier_block torture_shutdown_nb = { | |
691 | .notifier_call = torture_shutdown_notify, | |
692 | }; | |
693 | ||
bfefc73a PM |
694 | /* |
695 | * Shut down the shutdown task. Say what??? Heh! This can happen if | |
696 | * the torture module gets an rmmod before the shutdown time arrives. ;-) | |
697 | */ | |
698 | static void torture_shutdown_cleanup(void) | |
699 | { | |
700 | unregister_reboot_notifier(&torture_shutdown_nb); | |
701 | if (shutdown_task != NULL) { | |
702 | VERBOSE_TOROUT_STRING("Stopping torture_shutdown task"); | |
703 | kthread_stop(shutdown_task); | |
704 | } | |
705 | shutdown_task = NULL; | |
706 | } | |
707 | ||
628edaa5 PM |
708 | /* |
709 | * Variables for stuttering, which means to periodically pause and | |
710 | * restart testing in order to catch bugs that appear when load is | |
711 | * suddenly applied to or removed from the system. | |
712 | */ | |
713 | static struct task_struct *stutter_task; | |
714 | static int stutter_pause_test; | |
715 | static int stutter; | |
ff3bf92d | 716 | static int stutter_gap; |
628edaa5 PM |
717 | |
718 | /* | |
719 | * Block until the stutter interval ends. This must be called periodically | |
720 | * by all running kthreads that need to be subject to stuttering. | |
721 | */ | |
474e59b4 | 722 | bool stutter_wait(const char *title) |
628edaa5 | 723 | { |
19012b78 | 724 | unsigned int i = 0; |
e8516c64 | 725 | bool ret = false; |
19012b78 | 726 | int spt; |
4ced3314 | 727 | |
cee43939 | 728 | cond_resched_tasks_rcu_qs(); |
4ced3314 | 729 | spt = READ_ONCE(stutter_pause_test); |
29d39390 | 730 | for (; spt; spt = READ_ONCE(stutter_pause_test)) { |
ab1b7880 PM |
731 | if (!ret) { |
732 | sched_set_normal(current, MAX_NICE); | |
733 | ret = true; | |
734 | } | |
4ced3314 PM |
735 | if (spt == 1) { |
736 | schedule_timeout_interruptible(1); | |
737 | } else if (spt == 2) { | |
19012b78 | 738 | while (READ_ONCE(stutter_pause_test)) { |
ed24affa PM |
739 | if (!(i++ & 0xffff)) |
740 | torture_hrtimeout_us(10, 0, NULL); | |
4ced3314 | 741 | cond_resched(); |
19012b78 | 742 | } |
4ced3314 | 743 | } else { |
628edaa5 | 744 | schedule_timeout_interruptible(round_jiffies_relative(HZ)); |
4ced3314 | 745 | } |
628edaa5 PM |
746 | torture_shutdown_absorb(title); |
747 | } | |
e8516c64 | 748 | return ret; |
628edaa5 PM |
749 | } |
750 | EXPORT_SYMBOL_GPL(stutter_wait); | |
751 | ||
752 | /* | |
753 | * Cause the torture test to "stutter", starting and stopping all | |
754 | * threads periodically. | |
755 | */ | |
756 | static int torture_stutter(void *arg) | |
757 | { | |
fda5ba9e | 758 | DEFINE_TORTURE_RANDOM(rand); |
e8516c64 PM |
759 | int wtime; |
760 | ||
628edaa5 PM |
761 | VERBOSE_TOROUT_STRING("torture_stutter task started"); |
762 | do { | |
4ced3314 | 763 | if (!torture_must_stop() && stutter > 1) { |
e8516c64 | 764 | wtime = stutter; |
fda5ba9e | 765 | if (stutter > 2) { |
e8516c64 | 766 | WRITE_ONCE(stutter_pause_test, 1); |
fda5ba9e | 767 | wtime = stutter - 3; |
ed24affa | 768 | torture_hrtimeout_jiffies(wtime, &rand); |
fda5ba9e | 769 | wtime = 2; |
e8516c64 | 770 | } |
4ced3314 | 771 | WRITE_ONCE(stutter_pause_test, 2); |
ed24affa | 772 | torture_hrtimeout_jiffies(wtime, NULL); |
628edaa5 | 773 | } |
4ced3314 | 774 | WRITE_ONCE(stutter_pause_test, 0); |
628edaa5 | 775 | if (!torture_must_stop()) |
ed24affa | 776 | torture_hrtimeout_jiffies(stutter_gap, NULL); |
628edaa5 PM |
777 | torture_shutdown_absorb("torture_stutter"); |
778 | } while (!torture_must_stop()); | |
7fafaac5 | 779 | torture_kthread_stopping("torture_stutter"); |
628edaa5 PM |
780 | return 0; |
781 | } | |
782 | ||
783 | /* | |
784 | * Initialize and kick off the torture_stutter kthread. | |
785 | */ | |
ff3bf92d | 786 | int torture_stutter_init(const int s, const int sgap) |
628edaa5 | 787 | { |
628edaa5 | 788 | stutter = s; |
ff3bf92d | 789 | stutter_gap = sgap; |
2a7d9688 | 790 | return torture_create_kthread(torture_stutter, NULL, stutter_task); |
628edaa5 PM |
791 | } |
792 | EXPORT_SYMBOL_GPL(torture_stutter_init); | |
793 | ||
794 | /* | |
795 | * Cleanup after the torture_stutter kthread. | |
796 | */ | |
bfefc73a | 797 | static void torture_stutter_cleanup(void) |
628edaa5 PM |
798 | { |
799 | if (!stutter_task) | |
800 | return; | |
801 | VERBOSE_TOROUT_STRING("Stopping torture_stutter task"); | |
802 | kthread_stop(stutter_task); | |
803 | stutter_task = NULL; | |
804 | } | |
628edaa5 | 805 | |
b5daa8f3 PM |
806 | /* |
807 | * Initialize torture module. Please note that this is -not- invoked via | |
808 | * the usual module_init() mechanism, but rather by an explicit call from | |
809 | * the client torture module. This call must be paired with a later | |
810 | * torture_init_end(). | |
628edaa5 PM |
811 | * |
812 | * The runnable parameter points to a flag that controls whether or not | |
813 | * the test is currently runnable. If there is no such flag, pass in NULL. | |
b5daa8f3 | 814 | */ |
90127d60 | 815 | bool torture_init_begin(char *ttype, int v) |
b5daa8f3 PM |
816 | { |
817 | mutex_lock(&fullstop_mutex); | |
5228084e | 818 | if (torture_type != NULL) { |
4ac9de07 SZ |
819 | pr_alert("%s: Refusing %s init: %s running.\n", |
820 | __func__, ttype, torture_type); | |
821 | pr_alert("%s: One torture test at a time!\n", __func__); | |
5228084e PM |
822 | mutex_unlock(&fullstop_mutex); |
823 | return false; | |
824 | } | |
b5daa8f3 PM |
825 | torture_type = ttype; |
826 | verbose = v; | |
36970bb9 | 827 | fullstop = FULLSTOP_DONTSTOP; |
5228084e | 828 | return true; |
b5daa8f3 PM |
829 | } |
830 | EXPORT_SYMBOL_GPL(torture_init_begin); | |
831 | ||
832 | /* | |
833 | * Tell the torture module that initialization is complete. | |
834 | */ | |
2b3f8ffe | 835 | void torture_init_end(void) |
b5daa8f3 PM |
836 | { |
837 | mutex_unlock(&fullstop_mutex); | |
4622b487 | 838 | register_reboot_notifier(&torture_shutdown_nb); |
b5daa8f3 PM |
839 | } |
840 | EXPORT_SYMBOL_GPL(torture_init_end); | |
cc47ae08 PM |
841 | |
842 | /* | |
843 | * Clean up torture module. Please note that this is -not- invoked via | |
844 | * the usual module_exit() mechanism, but rather by an explicit call from | |
845 | * the client torture module. Returns true if a race with system shutdown | |
bfefc73a PM |
846 | * is detected, otherwise, all kthreads started by functions in this file |
847 | * will be shut down. | |
cc47ae08 PM |
848 | * |
849 | * This must be called before the caller starts shutting down its own | |
850 | * kthreads. | |
d36a7a0d DB |
851 | * |
852 | * Both torture_cleanup_begin() and torture_cleanup_end() must be paired, | |
853 | * in order to correctly perform the cleanup. They are separated because | |
854 | * threads can still need to reference the torture_type type, thus nullify | |
855 | * only after completing all other relevant calls. | |
cc47ae08 | 856 | */ |
d36a7a0d | 857 | bool torture_cleanup_begin(void) |
cc47ae08 PM |
858 | { |
859 | mutex_lock(&fullstop_mutex); | |
7d0ae808 | 860 | if (READ_ONCE(fullstop) == FULLSTOP_SHUTDOWN) { |
cc47ae08 PM |
861 | pr_warn("Concurrent rmmod and shutdown illegal!\n"); |
862 | mutex_unlock(&fullstop_mutex); | |
863 | schedule_timeout_uninterruptible(10); | |
864 | return true; | |
865 | } | |
7d0ae808 | 866 | WRITE_ONCE(fullstop, FULLSTOP_RMMOD); |
cc47ae08 | 867 | mutex_unlock(&fullstop_mutex); |
bfefc73a | 868 | torture_shutdown_cleanup(); |
cc47ae08 | 869 | torture_shuffle_cleanup(); |
bfefc73a | 870 | torture_stutter_cleanup(); |
cc47ae08 | 871 | torture_onoff_cleanup(); |
d36a7a0d DB |
872 | return false; |
873 | } | |
874 | EXPORT_SYMBOL_GPL(torture_cleanup_begin); | |
875 | ||
876 | void torture_cleanup_end(void) | |
877 | { | |
5228084e PM |
878 | mutex_lock(&fullstop_mutex); |
879 | torture_type = NULL; | |
880 | mutex_unlock(&fullstop_mutex); | |
cc47ae08 | 881 | } |
d36a7a0d | 882 | EXPORT_SYMBOL_GPL(torture_cleanup_end); |
36970bb9 PM |
883 | |
884 | /* | |
885 | * Is it time for the current torture test to stop? | |
886 | */ | |
887 | bool torture_must_stop(void) | |
888 | { | |
889 | return torture_must_stop_irq() || kthread_should_stop(); | |
890 | } | |
891 | EXPORT_SYMBOL_GPL(torture_must_stop); | |
892 | ||
893 | /* | |
894 | * Is it time for the current torture test to stop? This is the irq-safe | |
895 | * version, hence no check for kthread_should_stop(). | |
896 | */ | |
897 | bool torture_must_stop_irq(void) | |
898 | { | |
7d0ae808 | 899 | return READ_ONCE(fullstop) != FULLSTOP_DONTSTOP; |
36970bb9 PM |
900 | } |
901 | EXPORT_SYMBOL_GPL(torture_must_stop_irq); | |
7fafaac5 PM |
902 | |
903 | /* | |
904 | * Each kthread must wait for kthread_should_stop() before returning from | |
905 | * its top-level function, otherwise segfaults ensue. This function | |
906 | * prints a "stopping" message and waits for kthread_should_stop(), and | |
907 | * should be called from all torture kthreads immediately prior to | |
908 | * returning. | |
909 | */ | |
910 | void torture_kthread_stopping(char *title) | |
911 | { | |
0d6821d5 PM |
912 | char buf[128]; |
913 | ||
2b4a7f20 | 914 | snprintf(buf, sizeof(buf), "%s is stopping", title); |
0d6821d5 | 915 | VERBOSE_TOROUT_STRING(buf); |
7fafaac5 PM |
916 | while (!kthread_should_stop()) { |
917 | torture_shutdown_absorb(title); | |
918 | schedule_timeout_uninterruptible(1); | |
919 | } | |
920 | } | |
921 | EXPORT_SYMBOL_GPL(torture_kthread_stopping); | |
47cf29b9 PM |
922 | |
923 | /* | |
924 | * Create a generic torture kthread that is immediately runnable. If you | |
925 | * need the kthread to be stopped so that you can do something to it before | |
926 | * it starts, you will need to open-code your own. | |
927 | */ | |
928 | int _torture_create_kthread(int (*fn)(void *arg), void *arg, char *s, char *m, | |
929 | char *f, struct task_struct **tp) | |
930 | { | |
931 | int ret = 0; | |
932 | ||
933 | VERBOSE_TOROUT_STRING(m); | |
6b8646a9 | 934 | *tp = kthread_create(fn, arg, "%s", s); |
47cf29b9 PM |
935 | if (IS_ERR(*tp)) { |
936 | ret = PTR_ERR(*tp); | |
81faa4f6 | 937 | TOROUT_ERRSTRING(f); |
47cf29b9 | 938 | *tp = NULL; |
6b8646a9 | 939 | return ret; |
47cf29b9 | 940 | } |
6b8646a9 | 941 | wake_up_process(*tp); // Process is sleeping, so ordering provided. |
47cf29b9 PM |
942 | torture_shuffle_task_register(*tp); |
943 | return ret; | |
944 | } | |
945 | EXPORT_SYMBOL_GPL(_torture_create_kthread); | |
9c029b86 PM |
946 | |
947 | /* | |
948 | * Stop a generic kthread, emitting a message. | |
949 | */ | |
950 | void _torture_stop_kthread(char *m, struct task_struct **tp) | |
951 | { | |
952 | if (*tp == NULL) | |
953 | return; | |
954 | VERBOSE_TOROUT_STRING(m); | |
955 | kthread_stop(*tp); | |
956 | *tp = NULL; | |
957 | } | |
958 | EXPORT_SYMBOL_GPL(_torture_stop_kthread); |