kernel/locking/locktorture.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Module-based torture test facility for locking
   4  *
   5  * Copyright (C) IBM Corporation, 2014
   6  *
   7  * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
   8  *          Davidlohr Bueso <dave@stgolabs.net>
   9  *      Based on kernel/rcu/torture.c.
  10  */
  11
  12 #define pr_fmt(fmt) fmt
  13
  14 #include <linux/kernel.h>
  15 #include <linux/module.h>
  16 #include <linux/kthread.h>
  17 #include <linux/sched/rt.h>
  18 #include <linux/spinlock.h>
  19 #include <linux/mutex.h>
  20 #include <linux/rwsem.h>
  21 #include <linux/smp.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/sched.h>
  24 #include <uapi/linux/sched/types.h>
  25 #include <linux/rtmutex.h>
  26 #include <linux/atomic.h>
  27 #include <linux/moduleparam.h>
  28 #include <linux/delay.h>
  29 #include <linux/slab.h>
  30 #include <linux/torture.h>
  31 #include <linux/reboot.h>
  32
  33 MODULE_LICENSE("GPL");
  34 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>");
  35
  36 torture_param(int, nwriters_stress, -1,
  37              "Number of write-locking stress-test threads");
  38 torture_param(int, nreaders_stress, -1,
  39              "Number of read-locking stress-test threads");
  40 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
  41 torture_param(int, onoff_interval, 0,
  42              "Time between CPU hotplugs (s), 0=disable");
  43 torture_param(int, shuffle_interval, 3,
  44              "Number of jiffies between shuffles, 0=disable");
  45 torture_param(int, shutdown_secs, 0, "Shutdown time (j), <= zero to disable.");
  46 torture_param(int, stat_interval, 60,
  47              "Number of seconds between stats printk()s");
  48 torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable");
  49 torture_param(int, rt_boost, 2,
  50                 "Do periodic rt-boost. 0=Disable, 1=Only for rt_mutex, 2=For all lock types.");
  51 torture_param(int, rt_boost_factor, 50, "A factor determining how often rt-boost happens.");
  52 torture_param(int, verbose, 1,
  53              "Enable verbose debugging printk()s");
  54 torture_param(int, nested_locks, 0, "Number of nested locks (max = 8)");
  55 /* Going much higher trips "BUG: MAX_LOCKDEP_CHAIN_HLOCKS too low!" errors */
  56 #define MAX_NESTED_LOCKS 8
  57
  58 static char *torture_type = IS_ENABLED(CONFIG_PREEMPT_RT) ? "raw_spin_lock" : "spin_lock";
  59 module_param(torture_type, charp, 0444);
  60 MODULE_PARM_DESC(torture_type,
  61                  "Type of lock to torture (spin_lock, spin_lock_irq, mutex_lock, ...)");
  62
  63 static struct task_struct *stats_task;
  64 static struct task_struct **writer_tasks;
  65 static struct task_struct **reader_tasks;
  66
  67 static bool lock_is_write_held;
  68 static atomic_t lock_is_read_held;
  69 static unsigned long last_lock_release;
  70
  71 struct lock_stress_stats {
  72         long n_lock_fail;
  73         long n_lock_acquired;
  74 };
  75
  76 /* Forward reference. */
  77 static void lock_torture_cleanup(void);
  78
  79 /*
  80  * Operations vector for selecting different types of tests.
  81  */
  82 struct lock_torture_ops {
  83         void (*init)(void);
  84         void (*exit)(void);
  85         int (*nested_lock)(int tid, u32 lockset);
  86         int (*writelock)(int tid);
  87         void (*write_delay)(struct torture_random_state *trsp);
  88         void (*task_boost)(struct torture_random_state *trsp);
  89         void (*writeunlock)(int tid);
  90         void (*nested_unlock)(int tid, u32 lockset);
  91         int (*readlock)(int tid);
  92         void (*read_delay)(struct torture_random_state *trsp);
  93         void (*readunlock)(int tid);
  94
  95         unsigned long flags; /* for irq spinlocks */
  96         const char *name;
  97 };
  98
  99 struct lock_torture_cxt {
 100         int nrealwriters_stress;
 101         int nrealreaders_stress;
 102         bool debug_lock;
 103         bool init_called;
 104         atomic_t n_lock_torture_errors;
 105         struct lock_torture_ops *cur_ops;
 106         struct lock_stress_stats *lwsa; /* writer statistics */
 107         struct lock_stress_stats *lrsa; /* reader statistics */
 108 };
 109 static struct lock_torture_cxt cxt = { 0, 0, false, false,
 110                                        ATOMIC_INIT(0),
 111                                        NULL, NULL};
 112 /*
 113  * Definitions for lock torture testing.
 114  */
 115
 116 static int torture_lock_busted_write_lock(int tid __maybe_unused)
 117 {
 118         return 0;  /* BUGGY, do not use in real life!!! */
 119 }
 120
 121 static void torture_lock_busted_write_delay(struct torture_random_state *trsp)
 122 {
 123         const unsigned long longdelay_ms = 100;
 124
 125         /* We want a long delay occasionally to force massive contention.  */
 126         if (!(torture_random(trsp) %
 127               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 128                 mdelay(longdelay_ms);
 129         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 130                 torture_preempt_schedule();  /* Allow test to be preempted. */
 131 }
 132
 133 static void torture_lock_busted_write_unlock(int tid __maybe_unused)
 134 {
 135           /* BUGGY, do not use in real life!!! */
 136 }
 137
 138 static void __torture_rt_boost(struct torture_random_state *trsp)
 139 {
 140         const unsigned int factor = rt_boost_factor;
 141
 142         if (!rt_task(current)) {
 143                 /*
 144                  * Boost priority once every rt_boost_factor operations. When
 145                  * the task tries to take the lock, the rtmutex it will account
 146                  * for the new priority, and do any corresponding pi-dance.
 147                  */
 148                 if (trsp && !(torture_random(trsp) %
 149                               (cxt.nrealwriters_stress * factor))) {
 150                         sched_set_fifo(current);
 151                 } else /* common case, do nothing */
 152                         return;
 153         } else {
 154                 /*
 155                  * The task will remain boosted for another 10 * rt_boost_factor
 156                  * operations, then restored back to its original prio, and so
 157                  * forth.
 158                  *
 159                  * When @trsp is nil, we want to force-reset the task for
 160                  * stopping the kthread.
 161                  */
 162                 if (!trsp || !(torture_random(trsp) %
 163                                (cxt.nrealwriters_stress * factor * 2))) {
 164                         sched_set_normal(current, 0);
 165                 } else /* common case, do nothing */
 166                         return;
 167         }
 168 }
 169
 170 static void torture_rt_boost(struct torture_random_state *trsp)
 171 {
 172         if (rt_boost != 2)
 173                 return;
 174
 175         __torture_rt_boost(trsp);
 176 }
 177
 178 static struct lock_torture_ops lock_busted_ops = {
 179         .writelock      = torture_lock_busted_write_lock,
 180         .write_delay    = torture_lock_busted_write_delay,
 181         .task_boost     = torture_rt_boost,
 182         .writeunlock    = torture_lock_busted_write_unlock,
 183         .readlock       = NULL,
 184         .read_delay     = NULL,
 185         .readunlock     = NULL,
 186         .name           = "lock_busted"
 187 };
 188
 189 static DEFINE_SPINLOCK(torture_spinlock);
 190
 191 static int torture_spin_lock_write_lock(int tid __maybe_unused)
 192 __acquires(torture_spinlock)
 193 {
 194         spin_lock(&torture_spinlock);
 195         return 0;
 196 }
 197
 198 static void torture_spin_lock_write_delay(struct torture_random_state *trsp)
 199 {
 200         const unsigned long shortdelay_us = 2;
 201         const unsigned long longdelay_ms = 100;
 202
 203         /* We want a short delay mostly to emulate likely code, and
 204          * we want a long delay occasionally to force massive contention.
 205          */
 206         if (!(torture_random(trsp) %
 207               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 208                 mdelay(longdelay_ms);
 209         if (!(torture_random(trsp) %
 210               (cxt.nrealwriters_stress * 2 * shortdelay_us)))
 211                 udelay(shortdelay_us);
 212         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 213                 torture_preempt_schedule();  /* Allow test to be preempted. */
 214 }
 215
 216 static void torture_spin_lock_write_unlock(int tid __maybe_unused)
 217 __releases(torture_spinlock)
 218 {
 219         spin_unlock(&torture_spinlock);
 220 }
 221
 222 static struct lock_torture_ops spin_lock_ops = {
 223         .writelock      = torture_spin_lock_write_lock,
 224         .write_delay    = torture_spin_lock_write_delay,
 225         .task_boost     = torture_rt_boost,
 226         .writeunlock    = torture_spin_lock_write_unlock,
 227         .readlock       = NULL,
 228         .read_delay     = NULL,
 229         .readunlock     = NULL,
 230         .name           = "spin_lock"
 231 };
 232
 233 static int torture_spin_lock_write_lock_irq(int tid __maybe_unused)
 234 __acquires(torture_spinlock)
 235 {
 236         unsigned long flags;
 237
 238         spin_lock_irqsave(&torture_spinlock, flags);
 239         cxt.cur_ops->flags = flags;
 240         return 0;
 241 }
 242
 243 static void torture_lock_spin_write_unlock_irq(int tid __maybe_unused)
 244 __releases(torture_spinlock)
 245 {
 246         spin_unlock_irqrestore(&torture_spinlock, cxt.cur_ops->flags);
 247 }
 248
 249 static struct lock_torture_ops spin_lock_irq_ops = {
 250         .writelock      = torture_spin_lock_write_lock_irq,
 251         .write_delay    = torture_spin_lock_write_delay,
 252         .task_boost     = torture_rt_boost,
 253         .writeunlock    = torture_lock_spin_write_unlock_irq,
 254         .readlock       = NULL,
 255         .read_delay     = NULL,
 256         .readunlock     = NULL,
 257         .name           = "spin_lock_irq"
 258 };
 259
 260 static DEFINE_RAW_SPINLOCK(torture_raw_spinlock);
 261
 262 static int torture_raw_spin_lock_write_lock(int tid __maybe_unused)
 263 __acquires(torture_raw_spinlock)
 264 {
 265         raw_spin_lock(&torture_raw_spinlock);
 266         return 0;
 267 }
 268
 269 static void torture_raw_spin_lock_write_unlock(int tid __maybe_unused)
 270 __releases(torture_raw_spinlock)
 271 {
 272         raw_spin_unlock(&torture_raw_spinlock);
 273 }
 274
 275 static struct lock_torture_ops raw_spin_lock_ops = {
 276         .writelock      = torture_raw_spin_lock_write_lock,
 277         .write_delay    = torture_spin_lock_write_delay,
 278         .task_boost     = torture_rt_boost,
 279         .writeunlock    = torture_raw_spin_lock_write_unlock,
 280         .readlock       = NULL,
 281         .read_delay     = NULL,
 282         .readunlock     = NULL,
 283         .name           = "raw_spin_lock"
 284 };
 285
 286 static int torture_raw_spin_lock_write_lock_irq(int tid __maybe_unused)
 287 __acquires(torture_raw_spinlock)
 288 {
 289         unsigned long flags;
 290
 291         raw_spin_lock_irqsave(&torture_raw_spinlock, flags);
 292         cxt.cur_ops->flags = flags;
 293         return 0;
 294 }
 295
 296 static void torture_raw_spin_lock_write_unlock_irq(int tid __maybe_unused)
 297 __releases(torture_raw_spinlock)
 298 {
 299         raw_spin_unlock_irqrestore(&torture_raw_spinlock, cxt.cur_ops->flags);
 300 }
 301
 302 static struct lock_torture_ops raw_spin_lock_irq_ops = {
 303         .writelock      = torture_raw_spin_lock_write_lock_irq,
 304         .write_delay    = torture_spin_lock_write_delay,
 305         .task_boost     = torture_rt_boost,
 306         .writeunlock    = torture_raw_spin_lock_write_unlock_irq,
 307         .readlock       = NULL,
 308         .read_delay     = NULL,
 309         .readunlock     = NULL,
 310         .name           = "raw_spin_lock_irq"
 311 };
 312
 313 static DEFINE_RWLOCK(torture_rwlock);
 314
 315 static int torture_rwlock_write_lock(int tid __maybe_unused)
 316 __acquires(torture_rwlock)
 317 {
 318         write_lock(&torture_rwlock);
 319         return 0;
 320 }
 321
 322 static void torture_rwlock_write_delay(struct torture_random_state *trsp)
 323 {
 324         const unsigned long shortdelay_us = 2;
 325         const unsigned long longdelay_ms = 100;
 326
 327         /* We want a short delay mostly to emulate likely code, and
 328          * we want a long delay occasionally to force massive contention.
 329          */
 330         if (!(torture_random(trsp) %
 331               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 332                 mdelay(longdelay_ms);
 333         else
 334                 udelay(shortdelay_us);
 335 }
 336
 337 static void torture_rwlock_write_unlock(int tid __maybe_unused)
 338 __releases(torture_rwlock)
 339 {
 340         write_unlock(&torture_rwlock);
 341 }
 342
 343 static int torture_rwlock_read_lock(int tid __maybe_unused)
 344 __acquires(torture_rwlock)
 345 {
 346         read_lock(&torture_rwlock);
 347         return 0;
 348 }
 349
 350 static void torture_rwlock_read_delay(struct torture_random_state *trsp)
 351 {
 352         const unsigned long shortdelay_us = 10;
 353         const unsigned long longdelay_ms = 100;
 354
 355         /* We want a short delay mostly to emulate likely code, and
 356          * we want a long delay occasionally to force massive contention.
 357          */
 358         if (!(torture_random(trsp) %
 359               (cxt.nrealreaders_stress * 2000 * longdelay_ms)))
 360                 mdelay(longdelay_ms);
 361         else
 362                 udelay(shortdelay_us);
 363 }
 364
 365 static void torture_rwlock_read_unlock(int tid __maybe_unused)
 366 __releases(torture_rwlock)
 367 {
 368         read_unlock(&torture_rwlock);
 369 }
 370
 371 static struct lock_torture_ops rw_lock_ops = {
 372         .writelock      = torture_rwlock_write_lock,
 373         .write_delay    = torture_rwlock_write_delay,
 374         .task_boost     = torture_rt_boost,
 375         .writeunlock    = torture_rwlock_write_unlock,
 376         .readlock       = torture_rwlock_read_lock,
 377         .read_delay     = torture_rwlock_read_delay,
 378         .readunlock     = torture_rwlock_read_unlock,
 379         .name           = "rw_lock"
 380 };
 381
 382 static int torture_rwlock_write_lock_irq(int tid __maybe_unused)
 383 __acquires(torture_rwlock)
 384 {
 385         unsigned long flags;
 386
 387         write_lock_irqsave(&torture_rwlock, flags);
 388         cxt.cur_ops->flags = flags;
 389         return 0;
 390 }
 391
 392 static void torture_rwlock_write_unlock_irq(int tid __maybe_unused)
 393 __releases(torture_rwlock)
 394 {
 395         write_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags);
 396 }
 397
 398 static int torture_rwlock_read_lock_irq(int tid __maybe_unused)
 399 __acquires(torture_rwlock)
 400 {
 401         unsigned long flags;
 402
 403         read_lock_irqsave(&torture_rwlock, flags);
 404         cxt.cur_ops->flags = flags;
 405         return 0;
 406 }
 407
 408 static void torture_rwlock_read_unlock_irq(int tid __maybe_unused)
 409 __releases(torture_rwlock)
 410 {
 411         read_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags);
 412 }
 413
 414 static struct lock_torture_ops rw_lock_irq_ops = {
 415         .writelock      = torture_rwlock_write_lock_irq,
 416         .write_delay    = torture_rwlock_write_delay,
 417         .task_boost     = torture_rt_boost,
 418         .writeunlock    = torture_rwlock_write_unlock_irq,
 419         .readlock       = torture_rwlock_read_lock_irq,
 420         .read_delay     = torture_rwlock_read_delay,
 421         .readunlock     = torture_rwlock_read_unlock_irq,
 422         .name           = "rw_lock_irq"
 423 };
 424
 425 static DEFINE_MUTEX(torture_mutex);
 426 static struct mutex torture_nested_mutexes[MAX_NESTED_LOCKS];
 427 static struct lock_class_key nested_mutex_keys[MAX_NESTED_LOCKS];
 428
 429 static void torture_mutex_init(void)
 430 {
 431         int i;
 432
 433         for (i = 0; i < MAX_NESTED_LOCKS; i++)
 434                 __mutex_init(&torture_nested_mutexes[i], __func__,
 435                              &nested_mutex_keys[i]);
 436 }
 437
 438 static int torture_mutex_nested_lock(int tid __maybe_unused,
 439                                      u32 lockset)
 440 {
 441         int i;
 442
 443         for (i = 0; i < nested_locks; i++)
 444                 if (lockset & (1 << i))
 445                         mutex_lock(&torture_nested_mutexes[i]);
 446         return 0;
 447 }
 448
 449 static int torture_mutex_lock(int tid __maybe_unused)
 450 __acquires(torture_mutex)
 451 {
 452         mutex_lock(&torture_mutex);
 453         return 0;
 454 }
 455
 456 static void torture_mutex_delay(struct torture_random_state *trsp)
 457 {
 458         const unsigned long longdelay_ms = 100;
 459
 460         /* We want a long delay occasionally to force massive contention.  */
 461         if (!(torture_random(trsp) %
 462               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 463                 mdelay(longdelay_ms * 5);
 464         else
 465                 mdelay(longdelay_ms / 5);
 466         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 467                 torture_preempt_schedule();  /* Allow test to be preempted. */
 468 }
 469
 470 static void torture_mutex_unlock(int tid __maybe_unused)
 471 __releases(torture_mutex)
 472 {
 473         mutex_unlock(&torture_mutex);
 474 }
 475
 476 static void torture_mutex_nested_unlock(int tid __maybe_unused,
 477                                         u32 lockset)
 478 {
 479         int i;
 480
 481         for (i = nested_locks - 1; i >= 0; i--)
 482                 if (lockset & (1 << i))
 483                         mutex_unlock(&torture_nested_mutexes[i]);
 484 }
 485
 486 static struct lock_torture_ops mutex_lock_ops = {
 487         .init           = torture_mutex_init,
 488         .nested_lock    = torture_mutex_nested_lock,
 489         .writelock      = torture_mutex_lock,
 490         .write_delay    = torture_mutex_delay,
 491         .task_boost     = torture_rt_boost,
 492         .writeunlock    = torture_mutex_unlock,
 493         .nested_unlock  = torture_mutex_nested_unlock,
 494         .readlock       = NULL,
 495         .read_delay     = NULL,
 496         .readunlock     = NULL,
 497         .name           = "mutex_lock"
 498 };
 499
 500 #include <linux/ww_mutex.h>
 501 /*
 502  * The torture ww_mutexes should belong to the same lock class as
 503  * torture_ww_class to avoid lockdep problem. The ww_mutex_init()
 504  * function is called for initialization to ensure that.
 505  */
 506 static DEFINE_WD_CLASS(torture_ww_class);
 507 static struct ww_mutex torture_ww_mutex_0, torture_ww_mutex_1, torture_ww_mutex_2;
 508 static struct ww_acquire_ctx *ww_acquire_ctxs;
 509
 510 static void torture_ww_mutex_init(void)
 511 {
 512         ww_mutex_init(&torture_ww_mutex_0, &torture_ww_class);
 513         ww_mutex_init(&torture_ww_mutex_1, &torture_ww_class);
 514         ww_mutex_init(&torture_ww_mutex_2, &torture_ww_class);
 515
 516         ww_acquire_ctxs = kmalloc_array(cxt.nrealwriters_stress,
 517                                         sizeof(*ww_acquire_ctxs),
 518                                         GFP_KERNEL);
 519         if (!ww_acquire_ctxs)
 520                 VERBOSE_TOROUT_STRING("ww_acquire_ctx: Out of memory");
 521 }
 522
 523 static void torture_ww_mutex_exit(void)
 524 {
 525         kfree(ww_acquire_ctxs);
 526 }
 527
 528 static int torture_ww_mutex_lock(int tid)
 529 __acquires(torture_ww_mutex_0)
 530 __acquires(torture_ww_mutex_1)
 531 __acquires(torture_ww_mutex_2)
 532 {
 533         LIST_HEAD(list);
 534         struct reorder_lock {
 535                 struct list_head link;
 536                 struct ww_mutex *lock;
 537         } locks[3], *ll, *ln;
 538         struct ww_acquire_ctx *ctx = &ww_acquire_ctxs[tid];
 539
 540         locks[0].lock = &torture_ww_mutex_0;
 541         list_add(&locks[0].link, &list);
 542
 543         locks[1].lock = &torture_ww_mutex_1;
 544         list_add(&locks[1].link, &list);
 545
 546         locks[2].lock = &torture_ww_mutex_2;
 547         list_add(&locks[2].link, &list);
 548
 549         ww_acquire_init(ctx, &torture_ww_class);
 550
 551         list_for_each_entry(ll, &list, link) {
 552                 int err;
 553
 554                 err = ww_mutex_lock(ll->lock, ctx);
 555                 if (!err)
 556                         continue;
 557
 558                 ln = ll;
 559                 list_for_each_entry_continue_reverse(ln, &list, link)
 560                         ww_mutex_unlock(ln->lock);
 561
 562                 if (err != -EDEADLK)
 563                         return err;
 564
 565                 ww_mutex_lock_slow(ll->lock, ctx);
 566                 list_move(&ll->link, &list);
 567         }
 568
 569         return 0;
 570 }
 571
 572 static void torture_ww_mutex_unlock(int tid)
 573 __releases(torture_ww_mutex_0)
 574 __releases(torture_ww_mutex_1)
 575 __releases(torture_ww_mutex_2)
 576 {
 577         struct ww_acquire_ctx *ctx = &ww_acquire_ctxs[tid];
 578
 579         ww_mutex_unlock(&torture_ww_mutex_0);
 580         ww_mutex_unlock(&torture_ww_mutex_1);
 581         ww_mutex_unlock(&torture_ww_mutex_2);
 582         ww_acquire_fini(ctx);
 583 }
 584
 585 static struct lock_torture_ops ww_mutex_lock_ops = {
 586         .init           = torture_ww_mutex_init,
 587         .exit           = torture_ww_mutex_exit,
 588         .writelock      = torture_ww_mutex_lock,
 589         .write_delay    = torture_mutex_delay,
 590         .task_boost     = torture_rt_boost,
 591         .writeunlock    = torture_ww_mutex_unlock,
 592         .readlock       = NULL,
 593         .read_delay     = NULL,
 594         .readunlock     = NULL,
 595         .name           = "ww_mutex_lock"
 596 };
 597
 598 #ifdef CONFIG_RT_MUTEXES
 599 static DEFINE_RT_MUTEX(torture_rtmutex);
 600 static struct rt_mutex torture_nested_rtmutexes[MAX_NESTED_LOCKS];
 601 static struct lock_class_key nested_rtmutex_keys[MAX_NESTED_LOCKS];
 602
 603 static void torture_rtmutex_init(void)
 604 {
 605         int i;
 606
 607         for (i = 0; i < MAX_NESTED_LOCKS; i++)
 608                 __rt_mutex_init(&torture_nested_rtmutexes[i], __func__,
 609                                 &nested_rtmutex_keys[i]);
 610 }
 611
 612 static int torture_rtmutex_nested_lock(int tid __maybe_unused,
 613                                        u32 lockset)
 614 {
 615         int i;
 616
 617         for (i = 0; i < nested_locks; i++)
 618                 if (lockset & (1 << i))
 619                         rt_mutex_lock(&torture_nested_rtmutexes[i]);
 620         return 0;
 621 }
 622
 623 static int torture_rtmutex_lock(int tid __maybe_unused)
 624 __acquires(torture_rtmutex)
 625 {
 626         rt_mutex_lock(&torture_rtmutex);
 627         return 0;
 628 }
 629
 630 static void torture_rtmutex_delay(struct torture_random_state *trsp)
 631 {
 632         const unsigned long shortdelay_us = 2;
 633         const unsigned long longdelay_ms = 100;
 634
 635         /*
 636          * We want a short delay mostly to emulate likely code, and
 637          * we want a long delay occasionally to force massive contention.
 638          */
 639         if (!(torture_random(trsp) %
 640               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 641                 mdelay(longdelay_ms);
 642         if (!(torture_random(trsp) %
 643               (cxt.nrealwriters_stress * 2 * shortdelay_us)))
 644                 udelay(shortdelay_us);
 645         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 646                 torture_preempt_schedule();  /* Allow test to be preempted. */
 647 }
 648
 649 static void torture_rtmutex_unlock(int tid __maybe_unused)
 650 __releases(torture_rtmutex)
 651 {
 652         rt_mutex_unlock(&torture_rtmutex);
 653 }
 654
 655 static void torture_rt_boost_rtmutex(struct torture_random_state *trsp)
 656 {
 657         if (!rt_boost)
 658                 return;
 659
 660         __torture_rt_boost(trsp);
 661 }
 662
 663 static void torture_rtmutex_nested_unlock(int tid __maybe_unused,
 664                                           u32 lockset)
 665 {
 666         int i;
 667
 668         for (i = nested_locks - 1; i >= 0; i--)
 669                 if (lockset & (1 << i))
 670                         rt_mutex_unlock(&torture_nested_rtmutexes[i]);
 671 }
 672
 673 static struct lock_torture_ops rtmutex_lock_ops = {
 674         .init           = torture_rtmutex_init,
 675         .nested_lock    = torture_rtmutex_nested_lock,
 676         .writelock      = torture_rtmutex_lock,
 677         .write_delay    = torture_rtmutex_delay,
 678         .task_boost     = torture_rt_boost_rtmutex,
 679         .writeunlock    = torture_rtmutex_unlock,
 680         .nested_unlock  = torture_rtmutex_nested_unlock,
 681         .readlock       = NULL,
 682         .read_delay     = NULL,
 683         .readunlock     = NULL,
 684         .name           = "rtmutex_lock"
 685 };
 686 #endif
 687
 688 static DECLARE_RWSEM(torture_rwsem);
 689 static int torture_rwsem_down_write(int tid __maybe_unused)
 690 __acquires(torture_rwsem)
 691 {
 692         down_write(&torture_rwsem);
 693         return 0;
 694 }
 695
 696 static void torture_rwsem_write_delay(struct torture_random_state *trsp)
 697 {
 698         const unsigned long longdelay_ms = 100;
 699
 700         /* We want a long delay occasionally to force massive contention.  */
 701         if (!(torture_random(trsp) %
 702               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 703                 mdelay(longdelay_ms * 10);
 704         else
 705                 mdelay(longdelay_ms / 10);
 706         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 707                 torture_preempt_schedule();  /* Allow test to be preempted. */
 708 }
 709
 710 static void torture_rwsem_up_write(int tid __maybe_unused)
 711 __releases(torture_rwsem)
 712 {
 713         up_write(&torture_rwsem);
 714 }
 715
 716 static int torture_rwsem_down_read(int tid __maybe_unused)
 717 __acquires(torture_rwsem)
 718 {
 719         down_read(&torture_rwsem);
 720         return 0;
 721 }
 722
 723 static void torture_rwsem_read_delay(struct torture_random_state *trsp)
 724 {
 725         const unsigned long longdelay_ms = 100;
 726
 727         /* We want a long delay occasionally to force massive contention.  */
 728         if (!(torture_random(trsp) %
 729               (cxt.nrealreaders_stress * 2000 * longdelay_ms)))
 730                 mdelay(longdelay_ms * 2);
 731         else
 732                 mdelay(longdelay_ms / 2);
 733         if (!(torture_random(trsp) % (cxt.nrealreaders_stress * 20000)))
 734                 torture_preempt_schedule();  /* Allow test to be preempted. */
 735 }
 736
 737 static void torture_rwsem_up_read(int tid __maybe_unused)
 738 __releases(torture_rwsem)
 739 {
 740         up_read(&torture_rwsem);
 741 }
 742
 743 static struct lock_torture_ops rwsem_lock_ops = {
 744         .writelock      = torture_rwsem_down_write,
 745         .write_delay    = torture_rwsem_write_delay,
 746         .task_boost     = torture_rt_boost,
 747         .writeunlock    = torture_rwsem_up_write,
 748         .readlock       = torture_rwsem_down_read,
 749         .read_delay     = torture_rwsem_read_delay,
 750         .readunlock     = torture_rwsem_up_read,
 751         .name           = "rwsem_lock"
 752 };
 753
 754 #include <linux/percpu-rwsem.h>
 755 static struct percpu_rw_semaphore pcpu_rwsem;
 756
 757 static void torture_percpu_rwsem_init(void)
 758 {
 759         BUG_ON(percpu_init_rwsem(&pcpu_rwsem));
 760 }
 761
 762 static void torture_percpu_rwsem_exit(void)
 763 {
 764         percpu_free_rwsem(&pcpu_rwsem);
 765 }
 766
 767 static int torture_percpu_rwsem_down_write(int tid __maybe_unused)
 768 __acquires(pcpu_rwsem)
 769 {
 770         percpu_down_write(&pcpu_rwsem);
 771         return 0;
 772 }
 773
 774 static void torture_percpu_rwsem_up_write(int tid __maybe_unused)
 775 __releases(pcpu_rwsem)
 776 {
 777         percpu_up_write(&pcpu_rwsem);
 778 }
 779
 780 static int torture_percpu_rwsem_down_read(int tid __maybe_unused)
 781 __acquires(pcpu_rwsem)
 782 {
 783         percpu_down_read(&pcpu_rwsem);
 784         return 0;
 785 }
 786
 787 static void torture_percpu_rwsem_up_read(int tid __maybe_unused)
 788 __releases(pcpu_rwsem)
 789 {
 790         percpu_up_read(&pcpu_rwsem);
 791 }
 792
 793 static struct lock_torture_ops percpu_rwsem_lock_ops = {
 794         .init           = torture_percpu_rwsem_init,
 795         .exit           = torture_percpu_rwsem_exit,
 796         .writelock      = torture_percpu_rwsem_down_write,
 797         .write_delay    = torture_rwsem_write_delay,
 798         .task_boost     = torture_rt_boost,
 799         .writeunlock    = torture_percpu_rwsem_up_write,
 800         .readlock       = torture_percpu_rwsem_down_read,
 801         .read_delay     = torture_rwsem_read_delay,
 802         .readunlock     = torture_percpu_rwsem_up_read,
 803         .name           = "percpu_rwsem_lock"
 804 };
 805
 806 /*
 807  * Lock torture writer kthread.  Repeatedly acquires and releases
 808  * the lock, checking for duplicate acquisitions.
 809  */
 810 static int lock_torture_writer(void *arg)
 811 {
 812         struct lock_stress_stats *lwsp = arg;
 813         int tid = lwsp - cxt.lwsa;
 814         DEFINE_TORTURE_RANDOM(rand);
 815         u32 lockset_mask;
 816         bool skip_main_lock;
 817
 818         VERBOSE_TOROUT_STRING("lock_torture_writer task started");
 819         set_user_nice(current, MAX_NICE);
 820
 821         do {
 822                 if ((torture_random(&rand) & 0xfffff) == 0)
 823                         schedule_timeout_uninterruptible(1);
 824
 825                 lockset_mask = torture_random(&rand);
 826                 /*
 827                  * When using nested_locks, we want to occasionally
 828                  * skip the main lock so we can avoid always serializing
 829                  * the lock chains on that central lock. By skipping the
 830                  * main lock occasionally, we can create different
 831                  * contention patterns (allowing for multiple disjoint
 832                  * blocked trees)
 833                  */
 834                 skip_main_lock = (nested_locks &&
 835                                  !(torture_random(&rand) % 100));
 836
 837                 cxt.cur_ops->task_boost(&rand);
 838                 if (cxt.cur_ops->nested_lock)
 839                         cxt.cur_ops->nested_lock(tid, lockset_mask);
 840
 841                 if (!skip_main_lock) {
 842                         cxt.cur_ops->writelock(tid);
 843                         if (WARN_ON_ONCE(lock_is_write_held))
 844                                 lwsp->n_lock_fail++;
 845                         lock_is_write_held = true;
 846                         if (WARN_ON_ONCE(atomic_read(&lock_is_read_held)))
 847                                 lwsp->n_lock_fail++; /* rare, but... */
 848
 849                         lwsp->n_lock_acquired++;
 850                 }
 851                 cxt.cur_ops->write_delay(&rand);
 852                 if (!skip_main_lock) {
 853                         lock_is_write_held = false;
 854                         WRITE_ONCE(last_lock_release, jiffies);
 855                         cxt.cur_ops->writeunlock(tid);
 856                 }
 857                 if (cxt.cur_ops->nested_unlock)
 858                         cxt.cur_ops->nested_unlock(tid, lockset_mask);
 859
 860                 stutter_wait("lock_torture_writer");
 861         } while (!torture_must_stop());
 862
 863         cxt.cur_ops->task_boost(NULL); /* reset prio */
 864         torture_kthread_stopping("lock_torture_writer");
 865         return 0;
 866 }
 867
 868 /*
 869  * Lock torture reader kthread.  Repeatedly acquires and releases
 870  * the reader lock.
 871  */
 872 static int lock_torture_reader(void *arg)
 873 {
 874         struct lock_stress_stats *lrsp = arg;
 875         int tid = lrsp - cxt.lrsa;
 876         DEFINE_TORTURE_RANDOM(rand);
 877
 878         VERBOSE_TOROUT_STRING("lock_torture_reader task started");
 879         set_user_nice(current, MAX_NICE);
 880
 881         do {
 882                 if ((torture_random(&rand) & 0xfffff) == 0)
 883                         schedule_timeout_uninterruptible(1);
 884
 885                 cxt.cur_ops->readlock(tid);
 886                 atomic_inc(&lock_is_read_held);
 887                 if (WARN_ON_ONCE(lock_is_write_held))
 888                         lrsp->n_lock_fail++; /* rare, but... */
 889
 890                 lrsp->n_lock_acquired++;
 891                 cxt.cur_ops->read_delay(&rand);
 892                 atomic_dec(&lock_is_read_held);
 893                 cxt.cur_ops->readunlock(tid);
 894
 895                 stutter_wait("lock_torture_reader");
 896         } while (!torture_must_stop());
 897         torture_kthread_stopping("lock_torture_reader");
 898         return 0;
 899 }
 900
 901 /*
 902  * Create an lock-torture-statistics message in the specified buffer.
 903  */
 904 static void __torture_print_stats(char *page,
 905                                   struct lock_stress_stats *statp, bool write)
 906 {
 907         long cur;
 908         bool fail = false;
 909         int i, n_stress;
 910         long max = 0, min = statp ? data_race(statp[0].n_lock_acquired) : 0;
 911         long long sum = 0;
 912
 913         n_stress = write ? cxt.nrealwriters_stress : cxt.nrealreaders_stress;
 914         for (i = 0; i < n_stress; i++) {
 915                 if (data_race(statp[i].n_lock_fail))
 916                         fail = true;
 917                 cur = data_race(statp[i].n_lock_acquired);
 918                 sum += cur;
 919                 if (max < cur)
 920                         max = cur;
 921                 if (min > cur)
 922                         min = cur;
 923         }
 924         page += sprintf(page,
 925                         "%s:  Total: %lld  Max/Min: %ld/%ld %s  Fail: %d %s\n",
 926                         write ? "Writes" : "Reads ",
 927                         sum, max, min,
 928                         !onoff_interval && max / 2 > min ? "???" : "",
 929                         fail, fail ? "!!!" : "");
 930         if (fail)
 931                 atomic_inc(&cxt.n_lock_torture_errors);
 932 }
 933
 934 /*
 935  * Print torture statistics.  Caller must ensure that there is only one
 936  * call to this function at a given time!!!  This is normally accomplished
 937  * by relying on the module system to only have one copy of the module
 938  * loaded, and then by giving the lock_torture_stats kthread full control
 939  * (or the init/cleanup functions when lock_torture_stats thread is not
 940  * running).
 941  */
 942 static void lock_torture_stats_print(void)
 943 {
 944         int size = cxt.nrealwriters_stress * 200 + 8192;
 945         char *buf;
 946
 947         if (cxt.cur_ops->readlock)
 948                 size += cxt.nrealreaders_stress * 200 + 8192;
 949
 950         buf = kmalloc(size, GFP_KERNEL);
 951         if (!buf) {
 952                 pr_err("lock_torture_stats_print: Out of memory, need: %d",
 953                        size);
 954                 return;
 955         }
 956
 957         __torture_print_stats(buf, cxt.lwsa, true);
 958         pr_alert("%s", buf);
 959         kfree(buf);
 960
 961         if (cxt.cur_ops->readlock) {
 962                 buf = kmalloc(size, GFP_KERNEL);
 963                 if (!buf) {
 964                         pr_err("lock_torture_stats_print: Out of memory, need: %d",
 965                                size);
 966                         return;
 967                 }
 968
 969                 __torture_print_stats(buf, cxt.lrsa, false);
 970                 pr_alert("%s", buf);
 971                 kfree(buf);
 972         }
 973 }
 974
 975 /*
 976  * Periodically prints torture statistics, if periodic statistics printing
 977  * was specified via the stat_interval module parameter.
 978  *
 979  * No need to worry about fullstop here, since this one doesn't reference
 980  * volatile state or register callbacks.
 981  */
 982 static int lock_torture_stats(void *arg)
 983 {
 984         VERBOSE_TOROUT_STRING("lock_torture_stats task started");
 985         do {
 986                 schedule_timeout_interruptible(stat_interval * HZ);
 987                 lock_torture_stats_print();
 988                 torture_shutdown_absorb("lock_torture_stats");
 989         } while (!torture_must_stop());
 990         torture_kthread_stopping("lock_torture_stats");
 991         return 0;
 992 }
 993
 994 static inline void
 995 lock_torture_print_module_parms(struct lock_torture_ops *cur_ops,
 996                                 const char *tag)
 997 {
 998         pr_alert("%s" TORTURE_FLAG
 999                  "--- %s%s: nwriters_stress=%d nreaders_stress=%d nested_locks=%d stat_interval=%d verbose=%d shuffle_interval=%d stutter=%d shutdown_secs=%d onoff_interval=%d onoff_holdoff=%d\n",
1000                  torture_type, tag, cxt.debug_lock ? " [debug]": "",
1001                  cxt.nrealwriters_stress, cxt.nrealreaders_stress,
1002                  nested_locks, stat_interval, verbose, shuffle_interval,
1003                  stutter, shutdown_secs, onoff_interval, onoff_holdoff);
1004 }
1005
1006 static void lock_torture_cleanup(void)
1007 {
1008         int i;
1009
1010         if (torture_cleanup_begin())
1011                 return;
1012
1013         /*
1014          * Indicates early cleanup, meaning that the test has not run,
1015          * such as when passing bogus args when loading the module.
1016          * However cxt->cur_ops.init() may have been invoked, so beside
1017          * perform the underlying torture-specific cleanups, cur_ops.exit()
1018          * will be invoked if needed.
1019          */
1020         if (!cxt.lwsa && !cxt.lrsa)
1021                 goto end;
1022
1023         if (writer_tasks) {
1024                 for (i = 0; i < cxt.nrealwriters_stress; i++)
1025                         torture_stop_kthread(lock_torture_writer,
1026                                              writer_tasks[i]);
1027                 kfree(writer_tasks);
1028                 writer_tasks = NULL;
1029         }
1030
1031         if (reader_tasks) {
1032                 for (i = 0; i < cxt.nrealreaders_stress; i++)
1033                         torture_stop_kthread(lock_torture_reader,
1034                                              reader_tasks[i]);
1035                 kfree(reader_tasks);
1036                 reader_tasks = NULL;
1037         }
1038
1039         torture_stop_kthread(lock_torture_stats, stats_task);
1040         lock_torture_stats_print();  /* -After- the stats thread is stopped! */
1041
1042         if (atomic_read(&cxt.n_lock_torture_errors))
1043                 lock_torture_print_module_parms(cxt.cur_ops,
1044                                                 "End of test: FAILURE");
1045         else if (torture_onoff_failures())
1046                 lock_torture_print_module_parms(cxt.cur_ops,
1047                                                 "End of test: LOCK_HOTPLUG");
1048         else
1049                 lock_torture_print_module_parms(cxt.cur_ops,
1050                                                 "End of test: SUCCESS");
1051
1052         kfree(cxt.lwsa);
1053         cxt.lwsa = NULL;
1054         kfree(cxt.lrsa);
1055         cxt.lrsa = NULL;
1056
1057 end:
1058         if (cxt.init_called) {
1059                 if (cxt.cur_ops->exit)
1060                         cxt.cur_ops->exit();
1061                 cxt.init_called = false;
1062         }
1063         torture_cleanup_end();
1064 }
1065
1066 static int __init lock_torture_init(void)
1067 {
1068         int i, j;
1069         int firsterr = 0;
1070         static struct lock_torture_ops *torture_ops[] = {
1071                 &lock_busted_ops,
1072                 &spin_lock_ops, &spin_lock_irq_ops,
1073                 &raw_spin_lock_ops, &raw_spin_lock_irq_ops,
1074                 &rw_lock_ops, &rw_lock_irq_ops,
1075                 &mutex_lock_ops,
1076                 &ww_mutex_lock_ops,
1077 #ifdef CONFIG_RT_MUTEXES
1078                 &rtmutex_lock_ops,
1079 #endif
1080                 &rwsem_lock_ops,
1081                 &percpu_rwsem_lock_ops,
1082         };
1083
1084         if (!torture_init_begin(torture_type, verbose))
1085                 return -EBUSY;
1086
1087         /* Process args and tell the world that the torturer is on the job. */
1088         for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
1089                 cxt.cur_ops = torture_ops[i];
1090                 if (strcmp(torture_type, cxt.cur_ops->name) == 0)
1091                         break;
1092         }
1093         if (i == ARRAY_SIZE(torture_ops)) {
1094                 pr_alert("lock-torture: invalid torture type: \"%s\"\n",
1095                          torture_type);
1096                 pr_alert("lock-torture types:");
1097                 for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
1098                         pr_alert(" %s", torture_ops[i]->name);
1099                 pr_alert("\n");
1100                 firsterr = -EINVAL;
1101                 goto unwind;
1102         }
1103
1104         if (nwriters_stress == 0 &&
1105             (!cxt.cur_ops->readlock || nreaders_stress == 0)) {
1106                 pr_alert("lock-torture: must run at least one locking thread\n");
1107                 firsterr = -EINVAL;
1108                 goto unwind;
1109         }
1110
1111         if (nwriters_stress >= 0)
1112                 cxt.nrealwriters_stress = nwriters_stress;
1113         else
1114                 cxt.nrealwriters_stress = 2 * num_online_cpus();
1115
1116         if (cxt.cur_ops->init) {
1117                 cxt.cur_ops->init();
1118                 cxt.init_called = true;
1119         }
1120
1121 #ifdef CONFIG_DEBUG_MUTEXES
1122         if (str_has_prefix(torture_type, "mutex"))
1123                 cxt.debug_lock = true;
1124 #endif
1125 #ifdef CONFIG_DEBUG_RT_MUTEXES
1126         if (str_has_prefix(torture_type, "rtmutex"))
1127                 cxt.debug_lock = true;
1128 #endif
1129 #ifdef CONFIG_DEBUG_SPINLOCK
1130         if ((str_has_prefix(torture_type, "spin")) ||
1131             (str_has_prefix(torture_type, "rw_lock")))
1132                 cxt.debug_lock = true;
1133 #endif
1134
1135         /* Initialize the statistics so that each run gets its own numbers. */
1136         if (nwriters_stress) {
1137                 lock_is_write_held = false;
1138                 cxt.lwsa = kmalloc_array(cxt.nrealwriters_stress,
1139                                          sizeof(*cxt.lwsa),
1140                                          GFP_KERNEL);
1141                 if (cxt.lwsa == NULL) {
1142                         VERBOSE_TOROUT_STRING("cxt.lwsa: Out of memory");
1143                         firsterr = -ENOMEM;
1144                         goto unwind;
1145                 }
1146
1147                 for (i = 0; i < cxt.nrealwriters_stress; i++) {
1148                         cxt.lwsa[i].n_lock_fail = 0;
1149                         cxt.lwsa[i].n_lock_acquired = 0;
1150                 }
1151         }
1152
1153         if (cxt.cur_ops->readlock) {
1154                 if (nreaders_stress >= 0)
1155                         cxt.nrealreaders_stress = nreaders_stress;
1156                 else {
1157                         /*
1158                          * By default distribute evenly the number of
1159                          * readers and writers. We still run the same number
1160                          * of threads as the writer-only locks default.
1161                          */
1162                         if (nwriters_stress < 0) /* user doesn't care */
1163                                 cxt.nrealwriters_stress = num_online_cpus();
1164                         cxt.nrealreaders_stress = cxt.nrealwriters_stress;
1165                 }
1166
1167                 if (nreaders_stress) {
1168                         cxt.lrsa = kmalloc_array(cxt.nrealreaders_stress,
1169                                                  sizeof(*cxt.lrsa),
1170                                                  GFP_KERNEL);
1171                         if (cxt.lrsa == NULL) {
1172                                 VERBOSE_TOROUT_STRING("cxt.lrsa: Out of memory");
1173                                 firsterr = -ENOMEM;
1174                                 kfree(cxt.lwsa);
1175                                 cxt.lwsa = NULL;
1176                                 goto unwind;
1177                         }
1178
1179                         for (i = 0; i < cxt.nrealreaders_stress; i++) {
1180                                 cxt.lrsa[i].n_lock_fail = 0;
1181                                 cxt.lrsa[i].n_lock_acquired = 0;
1182                         }
1183                 }
1184         }
1185
1186         lock_torture_print_module_parms(cxt.cur_ops, "Start of test");
1187
1188         /* Prepare torture context. */
1189         if (onoff_interval > 0) {
1190                 firsterr = torture_onoff_init(onoff_holdoff * HZ,
1191                                               onoff_interval * HZ, NULL);
1192                 if (torture_init_error(firsterr))
1193                         goto unwind;
1194         }
1195         if (shuffle_interval > 0) {
1196                 firsterr = torture_shuffle_init(shuffle_interval);
1197                 if (torture_init_error(firsterr))
1198                         goto unwind;
1199         }
1200         if (shutdown_secs > 0) {
1201                 firsterr = torture_shutdown_init(shutdown_secs,
1202                                                  lock_torture_cleanup);
1203                 if (torture_init_error(firsterr))
1204                         goto unwind;
1205         }
1206         if (stutter > 0) {
1207                 firsterr = torture_stutter_init(stutter, stutter);
1208                 if (torture_init_error(firsterr))
1209                         goto unwind;
1210         }
1211
1212         if (nwriters_stress) {
1213                 writer_tasks = kcalloc(cxt.nrealwriters_stress,
1214                                        sizeof(writer_tasks[0]),
1215                                        GFP_KERNEL);
1216                 if (writer_tasks == NULL) {
1217                         TOROUT_ERRSTRING("writer_tasks: Out of memory");
1218                         firsterr = -ENOMEM;
1219                         goto unwind;
1220                 }
1221         }
1222
1223         /* cap nested_locks to MAX_NESTED_LOCKS */
1224         if (nested_locks > MAX_NESTED_LOCKS)
1225                 nested_locks = MAX_NESTED_LOCKS;
1226
1227         if (cxt.cur_ops->readlock) {
1228                 reader_tasks = kcalloc(cxt.nrealreaders_stress,
1229                                        sizeof(reader_tasks[0]),
1230                                        GFP_KERNEL);
1231                 if (reader_tasks == NULL) {
1232                         TOROUT_ERRSTRING("reader_tasks: Out of memory");
1233                         kfree(writer_tasks);
1234                         writer_tasks = NULL;
1235                         firsterr = -ENOMEM;
1236                         goto unwind;
1237                 }
1238         }
1239
1240         /*
1241          * Create the kthreads and start torturing (oh, those poor little locks).
1242          *
1243          * TODO: Note that we interleave writers with readers, giving writers a
1244          * slight advantage, by creating its kthread first. This can be modified
1245          * for very specific needs, or even let the user choose the policy, if
1246          * ever wanted.
1247          */
1248         for (i = 0, j = 0; i < cxt.nrealwriters_stress ||
1249                     j < cxt.nrealreaders_stress; i++, j++) {
1250                 if (i >= cxt.nrealwriters_stress)
1251                         goto create_reader;
1252
1253                 /* Create writer. */
1254                 firsterr = torture_create_kthread(lock_torture_writer, &cxt.lwsa[i],
1255                                                   writer_tasks[i]);
1256                 if (torture_init_error(firsterr))
1257                         goto unwind;
1258
1259         create_reader:
1260                 if (cxt.cur_ops->readlock == NULL || (j >= cxt.nrealreaders_stress))
1261                         continue;
1262                 /* Create reader. */
1263                 firsterr = torture_create_kthread(lock_torture_reader, &cxt.lrsa[j],
1264                                                   reader_tasks[j]);
1265                 if (torture_init_error(firsterr))
1266                         goto unwind;
1267         }
1268         if (stat_interval > 0) {
1269                 firsterr = torture_create_kthread(lock_torture_stats, NULL,
1270                                                   stats_task);
1271                 if (torture_init_error(firsterr))
1272                         goto unwind;
1273         }
1274         torture_init_end();
1275         return 0;
1276
1277 unwind:
1278         torture_init_end();
1279         lock_torture_cleanup();
1280         if (shutdown_secs) {
1281                 WARN_ON(!IS_MODULE(CONFIG_LOCK_TORTURE_TEST));
1282                 kernel_power_off();
1283         }
1284         return firsterr;
1285 }
1286
1287 module_init(lock_torture_init);
1288 module_exit(lock_torture_cleanup);