tools/testing/selftests/rcutorture/bin/torture.sh

   1 #!/bin/bash
   2 # SPDX-License-Identifier: GPL-2.0+
   3 #
   4 # Run a series of torture tests, intended for overnight or
   5 # longer timeframes, and also for large systems.
   6 #
   7 # Usage: torture.sh [ options ]
   8 #
   9 # Copyright (C) 2020 Facebook, Inc.
  10 #
  11 # Authors: Paul E. McKenney <paulmck@kernel.org>
  12
  13 scriptname=$0
  14 args="$*"
  15
  16 RCUTORTURE="`pwd`/tools/testing/selftests/rcutorture"; export RCUTORTURE
  17 PATH=${RCUTORTURE}/bin:$PATH; export PATH
  18 . functions.sh
  19
  20 TORTURE_ALLOTED_CPUS="`identify_qemu_vcpus`"
  21 MAKE_ALLOTED_CPUS=$((TORTURE_ALLOTED_CPUS*2))
  22 HALF_ALLOTED_CPUS=$((TORTURE_ALLOTED_CPUS/2))
  23 if test "$HALF_ALLOTED_CPUS" -lt 1
  24 then
  25         HALF_ALLOTED_CPUS=1
  26 fi
  27 VERBOSE_BATCH_CPUS=$((TORTURE_ALLOTED_CPUS/16))
  28 if test "$VERBOSE_BATCH_CPUS" -lt 2
  29 then
  30         VERBOSE_BATCH_CPUS=0
  31 fi
  32
  33 # Configurations/scenarios.
  34 configs_rcutorture=
  35 configs_locktorture=
  36 configs_scftorture=
  37 kcsan_kmake_args=
  38
  39 # Default compression, duration, and apportionment.
  40 compress_concurrency="`identify_qemu_vcpus`"
  41 duration_base=10
  42 duration_rcutorture_frac=7
  43 duration_locktorture_frac=1
  44 duration_scftorture_frac=2
  45
  46 # "yes" or "no" parameters
  47 do_allmodconfig=yes
  48 do_rcutorture=yes
  49 do_locktorture=yes
  50 do_scftorture=yes
  51 do_rcuscale=yes
  52 do_refscale=yes
  53 do_kvfree=yes
  54 do_kasan=yes
  55 do_kcsan=no
  56 do_clocksourcewd=yes
  57 do_rt=yes
  58 do_rcutasksflavors=yes
  59 do_srcu_lockdep=yes
  60
  61 # doyesno - Helper function for yes/no arguments
  62 function doyesno () {
  63         if test "$1" = "$2"
  64         then
  65                 echo yes
  66         else
  67                 echo no
  68         fi
  69 }
  70
  71 usage () {
  72         echo "Usage: $scriptname optional arguments:"
  73         echo "       --compress-concurrency concurrency"
  74         echo "       --configs-rcutorture \"config-file list w/ repeat factor (3*TINY01)\""
  75         echo "       --configs-locktorture \"config-file list w/ repeat factor (10*LOCK01)\""
  76         echo "       --configs-scftorture \"config-file list w/ repeat factor (2*CFLIST)\""
  77         echo "       --do-all"
  78         echo "       --do-allmodconfig / --do-no-allmodconfig / --no-allmodconfig"
  79         echo "       --do-clocksourcewd / --do-no-clocksourcewd / --no-clocksourcewd"
  80         echo "       --do-kasan / --do-no-kasan / --no-kasan"
  81         echo "       --do-kcsan / --do-no-kcsan / --no-kcsan"
  82         echo "       --do-kvfree / --do-no-kvfree / --no-kvfree"
  83         echo "       --do-locktorture / --do-no-locktorture / --no-locktorture"
  84         echo "       --do-none"
  85         echo "       --do-rcuscale / --do-no-rcuscale / --no-rcuscale"
  86         echo "       --do-rcutasksflavors / --do-no-rcutasksflavors / --no-rcutasksflavors"
  87         echo "       --do-rcutorture / --do-no-rcutorture / --no-rcutorture"
  88         echo "       --do-refscale / --do-no-refscale / --no-refscale"
  89         echo "       --do-rt / --do-no-rt / --no-rt"
  90         echo "       --do-scftorture / --do-no-scftorture / --no-scftorture"
  91         echo "       --do-srcu-lockdep / --do-no-srcu-lockdep / --no-srcu-lockdep"
  92         echo "       --duration [ <minutes> | <hours>h | <days>d ]"
  93         echo "       --kcsan-kmake-arg kernel-make-arguments"
  94         exit 1
  95 }
  96
  97 while test $# -gt 0
  98 do
  99         case "$1" in
 100         --compress-concurrency)
 101                 checkarg --compress-concurrency "(concurrency level)" $# "$2" '^[0-9][0-9]*$' '^error'
 102                 compress_concurrency=$2
 103                 shift
 104                 ;;
 105         --config-rcutorture|--configs-rcutorture)
 106                 checkarg --configs-rcutorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--'
 107                 configs_rcutorture="$configs_rcutorture $2"
 108                 shift
 109                 ;;
 110         --config-locktorture|--configs-locktorture)
 111                 checkarg --configs-locktorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--'
 112                 configs_locktorture="$configs_locktorture $2"
 113                 shift
 114                 ;;
 115         --config-scftorture|--configs-scftorture)
 116                 checkarg --configs-scftorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--'
 117                 configs_scftorture="$configs_scftorture $2"
 118                 shift
 119                 ;;
 120         --do-all|--doall)
 121                 do_allmodconfig=yes
 122                 do_rcutasksflavor=yes
 123                 do_rcutorture=yes
 124                 do_locktorture=yes
 125                 do_scftorture=yes
 126                 do_rcuscale=yes
 127                 do_refscale=yes
 128                 do_rt=yes
 129                 do_kvfree=yes
 130                 do_kasan=yes
 131                 do_kcsan=yes
 132                 do_clocksourcewd=yes
 133                 do_srcu_lockdep=yes
 134                 ;;
 135         --do-allmodconfig|--do-no-allmodconfig|--no-allmodconfig)
 136                 do_allmodconfig=`doyesno "$1" --do-allmodconfig`
 137                 ;;
 138         --do-clocksourcewd|--do-no-clocksourcewd|--no-clocksourcewd)
 139                 do_clocksourcewd=`doyesno "$1" --do-clocksourcewd`
 140                 ;;
 141         --do-kasan|--do-no-kasan|--no-kasan)
 142                 do_kasan=`doyesno "$1" --do-kasan`
 143                 ;;
 144         --do-kcsan|--do-no-kcsan|--no-kcsan)
 145                 do_kcsan=`doyesno "$1" --do-kcsan`
 146                 ;;
 147         --do-kvfree|--do-no-kvfree|--no-kvfree)
 148                 do_kvfree=`doyesno "$1" --do-kvfree`
 149                 ;;
 150         --do-locktorture|--do-no-locktorture|--no-locktorture)
 151                 do_locktorture=`doyesno "$1" --do-locktorture`
 152                 ;;
 153         --do-none|--donone)
 154                 do_allmodconfig=no
 155                 do_rcutasksflavors=no
 156                 do_rcutorture=no
 157                 do_locktorture=no
 158                 do_scftorture=no
 159                 do_rcuscale=no
 160                 do_refscale=no
 161                 do_rt=no
 162                 do_kvfree=no
 163                 do_kasan=no
 164                 do_kcsan=no
 165                 do_clocksourcewd=no
 166                 do_srcu_lockdep=no
 167                 ;;
 168         --do-rcuscale|--do-no-rcuscale|--no-rcuscale)
 169                 do_rcuscale=`doyesno "$1" --do-rcuscale`
 170                 ;;
 171         --do-rcutasksflavors|--do-no-rcutasksflavors|--no-rcutasksflavors)
 172                 do_rcutasksflavors=`doyesno "$1" --do-rcutasksflavors`
 173                 ;;
 174         --do-rcutorture|--do-no-rcutorture|--no-rcutorture)
 175                 do_rcutorture=`doyesno "$1" --do-rcutorture`
 176                 ;;
 177         --do-refscale|--do-no-refscale|--no-refscale)
 178                 do_refscale=`doyesno "$1" --do-refscale`
 179                 ;;
 180         --do-rt|--do-no-rt|--no-rt)
 181                 do_rt=`doyesno "$1" --do-rt`
 182                 ;;
 183         --do-scftorture|--do-no-scftorture|--no-scftorture)
 184                 do_scftorture=`doyesno "$1" --do-scftorture`
 185                 ;;
 186         --do-srcu-lockdep|--do-no-srcu-lockdep|--no-srcu-lockdep)
 187                 do_srcu_lockdep=`doyesno "$1" --do-srcu-lockdep`
 188                 ;;
 189         --duration)
 190                 checkarg --duration "(minutes)" $# "$2" '^[0-9][0-9]*\(m\|h\|d\|\)$' '^error'
 191                 mult=1
 192                 if echo "$2" | grep -q 'm$'
 193                 then
 194                         mult=1
 195                 elif echo "$2" | grep -q 'h$'
 196                 then
 197                         mult=60
 198                 elif echo "$2" | grep -q 'd$'
 199                 then
 200                         mult=1440
 201                 fi
 202                 ts=`echo $2 | sed -e 's/[smhd]$//'`
 203                 duration_base=$(($ts*mult))
 204                 shift
 205                 ;;
 206         --kcsan-kmake-arg|--kcsan-kmake-args)
 207                 checkarg --kcsan-kmake-arg "(kernel make arguments)" $# "$2" '.*' '^error$'
 208                 kcsan_kmake_args="`echo "$kcsan_kmake_args $2" | sed -e 's/^ *//' -e 's/ *$//'`"
 209                 shift
 210                 ;;
 211         *)
 212                 echo Unknown argument $1
 213                 usage
 214                 ;;
 215         esac
 216         shift
 217 done
 218
 219 ds="`date +%Y.%m.%d-%H.%M.%S`-torture"
 220 startdate="`date`"
 221 starttime="`get_starttime`"
 222
 223 T="`mktemp -d ${TMPDIR-/tmp}/torture.sh.XXXXXX`"
 224 trap 'rm -rf $T' 0 2
 225
 226 echo " --- " $scriptname $args | tee -a $T/log
 227 echo " --- Results directory: " $ds | tee -a $T/log
 228
 229 # Calculate rcutorture defaults and apportion time
 230 if test -z "$configs_rcutorture"
 231 then
 232         configs_rcutorture=CFLIST
 233 fi
 234 duration_rcutorture=$((duration_base*duration_rcutorture_frac/10))
 235 if test "$duration_rcutorture" -eq 0
 236 then
 237         echo " --- Zero time for rcutorture, disabling" | tee -a $T/log
 238         do_rcutorture=no
 239 fi
 240
 241 # Calculate locktorture defaults and apportion time
 242 if test -z "$configs_locktorture"
 243 then
 244         configs_locktorture=CFLIST
 245 fi
 246 duration_locktorture=$((duration_base*duration_locktorture_frac/10))
 247 if test "$duration_locktorture" -eq 0
 248 then
 249         echo " --- Zero time for locktorture, disabling" | tee -a $T/log
 250         do_locktorture=no
 251 fi
 252
 253 # Calculate scftorture defaults and apportion time
 254 if test -z "$configs_scftorture"
 255 then
 256         configs_scftorture=CFLIST
 257 fi
 258 duration_scftorture=$((duration_base*duration_scftorture_frac/10))
 259 if test "$duration_scftorture" -eq 0
 260 then
 261         echo " --- Zero time for scftorture, disabling" | tee -a $T/log
 262         do_scftorture=no
 263 fi
 264
 265 touch $T/failures
 266 touch $T/successes
 267
 268 # torture_one - Does a single kvm.sh run.
 269 #
 270 # Usage:
 271 #       torture_bootargs="[ kernel boot arguments ]"
 272 #       torture_one flavor [ kvm.sh arguments ]
 273 #
 274 # Note that "flavor" is an arbitrary string.  Supply --torture if needed.
 275 # Note that quoting is problematic.  So on the command line, pass multiple
 276 # values with multiple kvm.sh argument instances.
 277 function torture_one {
 278         local cur_bootargs=
 279         local boottag=
 280
 281         echo " --- $curflavor:" Start `date` | tee -a $T/log
 282         if test -n "$torture_bootargs"
 283         then
 284                 boottag="--bootargs"
 285                 cur_bootargs="$torture_bootargs"
 286         fi
 287         "$@" $boottag "$cur_bootargs" --datestamp "$ds/results-$curflavor" > $T/$curflavor.out 2>&1
 288         retcode=$?
 289         resdir="`grep '^Results directory: ' $T/$curflavor.out | tail -1 | sed -e 's/^Results directory: //'`"
 290         if test -z "$resdir"
 291         then
 292                 cat $T/$curflavor.out | tee -a $T/log
 293                 echo retcode=$retcode | tee -a $T/log
 294         else
 295                 echo $resdir > $T/last-resdir
 296         fi
 297         if test "$retcode" == 0
 298         then
 299                 echo "$curflavor($retcode)" $resdir >> $T/successes
 300         else
 301                 echo "$curflavor($retcode)" $resdir >> $T/failures
 302         fi
 303 }
 304
 305 # torture_set - Does a set of tortures with and without KASAN and KCSAN.
 306 #
 307 # Usage:
 308 #       torture_bootargs="[ kernel boot arguments ]"
 309 #       torture_set flavor [ kvm.sh arguments ]
 310 #
 311 # Note that "flavor" is an arbitrary string that does not affect kvm.sh
 312 # in any way.  So also supply --torture if you need something other than
 313 # the default.
 314 function torture_set {
 315         local cur_kcsan_kmake_args=
 316         local kcsan_kmake_tag=
 317         local flavor=$1
 318         shift
 319         curflavor=$flavor
 320         torture_one "$@"
 321         mv $T/last-resdir $T/last-resdir-nodebug || :
 322         if test "$do_kasan" = "yes"
 323         then
 324                 curflavor=${flavor}-kasan
 325                 torture_one "$@" --kasan
 326                 mv $T/last-resdir $T/last-resdir-kasan || :
 327         fi
 328         if test "$do_kcsan" = "yes"
 329         then
 330                 curflavor=${flavor}-kcsan
 331                 if test -n "$kcsan_kmake_args"
 332                 then
 333                         kcsan_kmake_tag="--kmake-args"
 334                         cur_kcsan_kmake_args="$kcsan_kmake_args"
 335                 fi
 336                 torture_one "$@" --kconfig "CONFIG_DEBUG_LOCK_ALLOC=y CONFIG_PROVE_LOCKING=y" $kcsan_kmake_tag $cur_kcsan_kmake_args --kcsan
 337                 mv $T/last-resdir $T/last-resdir-kcsan || :
 338         fi
 339 }
 340
 341 # make allmodconfig
 342 if test "$do_allmodconfig" = "yes"
 343 then
 344         echo " --- allmodconfig:" Start `date` | tee -a $T/log
 345         amcdir="tools/testing/selftests/rcutorture/res/$ds/allmodconfig"
 346         mkdir -p "$amcdir"
 347         echo " --- make clean" | tee $amcdir/log > "$amcdir/Make.out" 2>&1
 348         make -j$MAKE_ALLOTED_CPUS clean >> "$amcdir/Make.out" 2>&1
 349         retcode=$?
 350         buildphase='"make clean"'
 351         if test "$retcode" -eq 0
 352         then
 353                 echo " --- make allmodconfig" | tee -a $amcdir/log >> "$amcdir/Make.out" 2>&1
 354                 cp .config $amcdir
 355                 make -j$MAKE_ALLOTED_CPUS allmodconfig >> "$amcdir/Make.out" 2>&1
 356                 retcode=$?
 357                 buildphase='"make allmodconfig"'
 358         fi
 359         if test "$retcode" -eq 0
 360         then
 361                 echo " --- make " | tee -a $amcdir/log >> "$amcdir/Make.out" 2>&1
 362                 make -j$MAKE_ALLOTED_CPUS >> "$amcdir/Make.out" 2>&1
 363                 retcode="$?"
 364                 echo $retcode > "$amcdir/Make.exitcode"
 365                 buildphase='"make"'
 366         fi
 367         if test "$retcode" -eq 0
 368         then
 369                 echo "allmodconfig($retcode)" $amcdir >> $T/successes
 370                 echo Success >> $amcdir/log
 371         else
 372                 echo "allmodconfig($retcode)" $amcdir >> $T/failures
 373                 echo " --- allmodconfig Test summary:" >> $amcdir/log
 374                 echo " --- Summary: Exit code $retcode from $buildphase, see Make.out" >> $amcdir/log
 375         fi
 376 fi
 377
 378 # Test building RCU Tasks flavors in isolation, both SMP and !SMP
 379 if test "$do_rcutasksflavors" = "yes"
 380 then
 381         echo " --- rcutasksflavors:" Start `date` | tee -a $T/log
 382         rtfdir="tools/testing/selftests/rcutorture/res/$ds/results-rcutasksflavors"
 383         mkdir -p "$rtfdir"
 384         cat > $T/rcutasksflavors << __EOF__
 385 #CHECK#CONFIG_TASKS_RCU=n
 386 #CHECK#CONFIG_TASKS_RUDE_RCU=n
 387 #CHECK#CONFIG_TASKS_TRACE_RCU=n
 388 __EOF__
 389         for flavor in CONFIG_TASKS_RCU CONFIG_TASKS_RUDE_RCU CONFIG_TASKS_TRACE_RCU
 390         do
 391                 forceflavor="`echo $flavor | sed -e 's/^CONFIG/CONFIG_FORCE/'`"
 392                 deselectedflavors="`grep -v $flavor $T/rcutasksflavors | tr '\012' ' ' | tr -s ' ' | sed -e 's/ *$//'`"
 393                 echo " --- Running RCU Tasks Trace flavor $flavor `date`" >> $rtfdir/log
 394                 tools/testing/selftests/rcutorture/bin/kvm.sh --datestamp "$ds/results-rcutasksflavors/$flavor" --buildonly --configs "TINY01 TREE04" --kconfig "CONFIG_RCU_EXPERT=y CONFIG_RCU_SCALE_TEST=y $forceflavor=y $deselectedflavors" --trust-make > $T/$flavor.out 2>&1
 395                 retcode=$?
 396                 if test "$retcode" -ne 0
 397                 then
 398                         break
 399                 fi
 400         done
 401         if test "$retcode" -eq 0
 402         then
 403                 echo "rcutasksflavors($retcode)" $rtfdir >> $T/successes
 404                 echo Success >> $rtfdir/log
 405         else
 406                 echo "rcutasksflavors($retcode)" $rtfdir >> $T/failures
 407                 echo " --- rcutasksflavors Test summary:" >> $rtfdir/log
 408                 echo " --- Summary: Exit code $retcode from $flavor, see Make.out" >> $rtfdir/log
 409         fi
 410 fi
 411
 412 # --torture rcu
 413 if test "$do_rcutorture" = "yes"
 414 then
 415         torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000"
 416         torture_set "rcutorture" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration "$duration_rcutorture" --configs "$configs_rcutorture" --trust-make
 417 fi
 418
 419 if test "$do_locktorture" = "yes"
 420 then
 421         torture_bootargs="torture.disable_onoff_at_boot"
 422         torture_set "locktorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture lock --allcpus --duration "$duration_locktorture" --configs "$configs_locktorture" --trust-make
 423 fi
 424
 425 if test "$do_scftorture" = "yes"
 426 then
 427         # Scale memory based on the number of CPUs.
 428         scfmem=$((2+HALF_ALLOTED_CPUS/16))
 429         torture_bootargs="scftorture.nthreads=$HALF_ALLOTED_CPUS torture.disable_onoff_at_boot csdlock_debug=1"
 430         torture_set "scftorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture scf --allcpus --duration "$duration_scftorture" --configs "$configs_scftorture" --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --memory ${scfmem}G --trust-make
 431 fi
 432
 433 if test "$do_rt" = "yes"
 434 then
 435         # With all post-boot grace periods forced to normal.
 436         torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 rcupdate.rcu_normal=1"
 437         torture_set "rcurttorture" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration "$duration_rcutorture" --configs "TREE03" --trust-make
 438
 439         # With all post-boot grace periods forced to expedited.
 440         torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 rcupdate.rcu_expedited=1"
 441         torture_set "rcurttorture-exp" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration "$duration_rcutorture" --configs "TREE03" --trust-make
 442 fi
 443
 444 if test "$do_srcu_lockdep" = "yes"
 445 then
 446         echo " --- do-srcu-lockdep:" Start `date` | tee -a $T/log
 447         tools/testing/selftests/rcutorture/bin/srcu_lockdep.sh --datestamp "$ds/results-srcu-lockdep" > $T/srcu_lockdep.sh.out 2>&1
 448         retcode=$?
 449         cp $T/srcu_lockdep.sh.out "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep/log"
 450         if test "$retcode" -eq 0
 451         then
 452                 echo "srcu_lockdep($retcode)" "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep" >> $T/successes
 453                 echo Success >> "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep/log"
 454         else
 455                 echo "srcu_lockdep($retcode)" "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep" >> $T/failures
 456                 echo " --- srcu_lockdep Test Summary:" >> "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep/log"
 457                 echo " --- Summary: Exit code $retcode from srcu_lockdep.sh, see ds/results-srcu-lockdep" >> "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep/log"
 458         fi
 459 fi
 460
 461 if test "$do_refscale" = yes
 462 then
 463         primlist="`grep '\.name[        ]*=' kernel/rcu/refscale.c | sed -e 's/^[^"]*"//' -e 's/".*$//'`"
 464 else
 465         primlist=
 466 fi
 467 firsttime=1
 468 do_kasan_save="$do_kasan"
 469 do_kcsan_save="$do_kcsan"
 470 for prim in $primlist
 471 do
 472         if test -n "$firsttime"
 473         then
 474                 torture_bootargs="refscale.scale_type="$prim" refscale.nreaders=$HALF_ALLOTED_CPUS refscale.loops=10000 refscale.holdoff=20 torture.disable_onoff_at_boot"
 475                 torture_set "refscale-$prim" tools/testing/selftests/rcutorture/bin/kvm.sh --torture refscale --allcpus --duration 5 --kconfig "CONFIG_TASKS_TRACE_RCU=y CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --bootargs "refscale.verbose_batched=$VERBOSE_BATCH_CPUS torture.verbose_sleep_frequency=8 torture.verbose_sleep_duration=$VERBOSE_BATCH_CPUS" --trust-make
 476                 mv $T/last-resdir-nodebug $T/first-resdir-nodebug || :
 477                 if test -f "$T/last-resdir-kasan"
 478                 then
 479                         mv $T/last-resdir-kasan $T/first-resdir-kasan || :
 480                 fi
 481                 if test -f "$T/last-resdir-kcsan"
 482                 then
 483                         mv $T/last-resdir-kcsan $T/first-resdir-kcsan || :
 484                 fi
 485                 firsttime=
 486                 do_kasan=
 487                 do_kcsan=
 488         else
 489                 torture_bootargs=
 490                 for i in $T/first-resdir-*
 491                 do
 492                         case "$i" in
 493                         *-nodebug)
 494                                 torture_suffix=
 495                                 ;;
 496                         *-kasan)
 497                                 torture_suffix="-kasan"
 498                                 ;;
 499                         *-kcsan)
 500                                 torture_suffix="-kcsan"
 501                                 ;;
 502                         esac
 503                         torture_set "refscale-$prim$torture_suffix" tools/testing/selftests/rcutorture/bin/kvm-again.sh "`cat "$i"`" --duration 5 --bootargs "refscale.scale_type=$prim"
 504                 done
 505         fi
 506 done
 507 do_kasan="$do_kasan_save"
 508 do_kcsan="$do_kcsan_save"
 509
 510 if test "$do_rcuscale" = yes
 511 then
 512         primlist="`grep '\.name[        ]*=' kernel/rcu/rcuscale.c | sed -e 's/^[^"]*"//' -e 's/".*$//'`"
 513 else
 514         primlist=
 515 fi
 516 firsttime=1
 517 do_kasan_save="$do_kasan"
 518 do_kcsan_save="$do_kcsan"
 519 for prim in $primlist
 520 do
 521         if test -n "$firsttime"
 522         then
 523                 torture_bootargs="rcuscale.scale_type="$prim" rcuscale.nwriters=$HALF_ALLOTED_CPUS rcuscale.holdoff=20 torture.disable_onoff_at_boot"
 524                 torture_set "rcuscale-$prim" tools/testing/selftests/rcutorture/bin/kvm.sh --torture rcuscale --allcpus --duration 5 --kconfig "CONFIG_TASKS_TRACE_RCU=y CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --trust-make
 525                 mv $T/last-resdir-nodebug $T/first-resdir-nodebug || :
 526                 if test -f "$T/last-resdir-kasan"
 527                 then
 528                         mv $T/last-resdir-kasan $T/first-resdir-kasan || :
 529                 fi
 530                 if test -f "$T/last-resdir-kcsan"
 531                 then
 532                         mv $T/last-resdir-kcsan $T/first-resdir-kcsan || :
 533                 fi
 534                 firsttime=
 535                 do_kasan=
 536                 do_kcsan=
 537         else
 538                 torture_bootargs=
 539                 for i in $T/first-resdir-*
 540                 do
 541                         case "$i" in
 542                         *-nodebug)
 543                                 torture_suffix=
 544                                 ;;
 545                         *-kasan)
 546                                 torture_suffix="-kasan"
 547                                 ;;
 548                         *-kcsan)
 549                                 torture_suffix="-kcsan"
 550                                 ;;
 551                         esac
 552                         torture_set "rcuscale-$prim$torture_suffix" tools/testing/selftests/rcutorture/bin/kvm-again.sh "`cat "$i"`" --duration 5 --bootargs "rcuscale.scale_type=$prim"
 553                 done
 554         fi
 555 done
 556 do_kasan="$do_kasan_save"
 557 do_kcsan="$do_kcsan_save"
 558
 559 if test "$do_kvfree" = "yes"
 560 then
 561         torture_bootargs="rcuscale.kfree_rcu_test=1 rcuscale.kfree_nthreads=16 rcuscale.holdoff=20 rcuscale.kfree_loops=10000 torture.disable_onoff_at_boot"
 562         torture_set "rcuscale-kvfree" tools/testing/selftests/rcutorture/bin/kvm.sh --torture rcuscale --allcpus --duration 10 --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --memory 2G --trust-make
 563 fi
 564
 565 if test "$do_clocksourcewd" = "yes"
 566 then
 567         torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 tsc=watchdog"
 568         torture_set "clocksourcewd-1" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --kconfig "CONFIG_TEST_CLOCKSOURCE_WATCHDOG=y" --trust-make
 569
 570         torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 clocksource.max_cswd_read_retries=1 tsc=watchdog"
 571         torture_set "clocksourcewd-2" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --kconfig "CONFIG_TEST_CLOCKSOURCE_WATCHDOG=y" --trust-make
 572
 573         # In case our work is already done...
 574         if test "$do_rcutorture" != "yes"
 575         then
 576                 torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 tsc=watchdog"
 577                 torture_set "clocksourcewd-3" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --trust-make
 578         fi
 579 fi
 580
 581 echo " --- " $scriptname $args
 582 echo " --- " Done `date` | tee -a $T/log
 583 ret=0
 584 nsuccesses=0
 585 echo SUCCESSES: | tee -a $T/log
 586 if test -s "$T/successes"
 587 then
 588         cat "$T/successes" | tee -a $T/log
 589         nsuccesses="`wc -l "$T/successes" | awk '{ print $1 }'`"
 590 fi
 591 nfailures=0
 592 echo FAILURES: | tee -a $T/log
 593 if test -s "$T/failures"
 594 then
 595         awk < "$T/failures" -v sq="'" '{ print "echo " sq $0 sq; print "sed -e " sq "1,/^ --- .* Test summary:$/d" sq " " $2 "/log | grep Summary: | sed -e " sq "s/^[^S]*/  /" sq; }' | sh | tee -a $T/log | tee "$T/failuresum"
 596         nfailures="`wc -l "$T/failures" | awk '{ print $1 }'`"
 597         grep "^  Summary: " "$T/failuresum" |
 598                 grep -v '^  Summary: Bugs: [0-9]* (all bugs kcsan)$' > "$T/nonkcsan"
 599         if test -s "$T/nonkcsan"
 600         then
 601                 nonkcsanbug="yes"
 602         fi
 603         ret=2
 604 fi
 605 if test "$do_kcsan" = "yes"
 606 then
 607         TORTURE_KCONFIG_KCSAN_ARG=1 tools/testing/selftests/rcutorture/bin/kcsan-collapse.sh tools/testing/selftests/rcutorture/res/$ds > tools/testing/selftests/rcutorture/res/$ds/kcsan.sum
 608 fi
 609 echo Started at $startdate, ended at `date`, duration `get_starttime_duration $starttime`. | tee -a $T/log
 610 echo Summary: Successes: $nsuccesses Failures: $nfailures. | tee -a $T/log
 611 tdir="`cat $T/successes $T/failures | head -1 | awk '{ print $NF }' | sed -e 's,/[^/]\+/*$,,'`"
 612 find "$tdir" -name 'ConfigFragment.diags' -print > $T/configerrors
 613 find "$tdir" -name 'Make.out.diags' -print > $T/builderrors
 614 if test -s "$T/configerrors"
 615 then
 616         echo "  Scenarios with .config errors: `wc -l "$T/configerrors" | awk '{ print $1 }'`"
 617         nonkcsanbug="yes"
 618 fi
 619 if test -s "$T/builderrors"
 620 then
 621         echo "  Scenarios with build errors: `wc -l "$T/builderrors" | awk '{ print $1 }'`"
 622         nonkcsanbug="yes"
 623 fi
 624 if test -z "$nonkcsanbug" && test -s "$T/failuresum"
 625 then
 626         echo "  All bugs were KCSAN failures."
 627 fi
 628 if test -n "$tdir" && test $compress_concurrency -gt 0
 629 then
 630         # KASAN vmlinux files can approach 1GB in size, so compress them.
 631         echo Looking for K[AC]SAN files to compress: `date` > "$tdir/log-xz" 2>&1
 632         find "$tdir" -type d -name '*-k[ac]san' -print > $T/xz-todo-all
 633         find "$tdir" -type f -name 're-run' -print | sed -e 's,/re-run,,' |
 634                 grep -e '-k[ac]san$' > $T/xz-todo-copy
 635         sort $T/xz-todo-all $T/xz-todo-copy | uniq -u > $T/xz-todo
 636         ncompresses=0
 637         batchno=1
 638         if test -s $T/xz-todo
 639         then
 640                 for i in `cat $T/xz-todo`
 641                 do
 642                         find $i -name 'vmlinux*' -print
 643                 done | wc -l | awk '{ print $1 }' > $T/xz-todo-count
 644                 n2compress="`cat $T/xz-todo-count`"
 645                 echo Size before compressing $n2compress files: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log
 646                 for i in `cat $T/xz-todo`
 647                 do
 648                         echo Compressing vmlinux files in ${i}: `date` >> "$tdir/log-xz" 2>&1
 649                         for j in $i/*/vmlinux
 650                         do
 651                                 xz "$j" >> "$tdir/log-xz" 2>&1 &
 652                                 ncompresses=$((ncompresses+1))
 653                                 if test $ncompresses -ge $compress_concurrency
 654                                 then
 655                                         echo Waiting for batch $batchno of $ncompresses compressions `date` | tee -a "$tdir/log-xz" | tee -a $T/log
 656                                         wait
 657                                         ncompresses=0
 658                                         batchno=$((batchno+1))
 659                                 fi
 660                         done
 661                 done
 662                 if test $ncompresses -gt 0
 663                 then
 664                         echo Waiting for final batch $batchno of $ncompresses compressions `date` | tee -a "$tdir/log-xz" | tee -a $T/log
 665                 fi
 666                 wait
 667                 if test -s $T/xz-todo-copy
 668                 then
 669                         # The trick here is that we need corresponding
 670                         # vmlinux files from corresponding scenarios.
 671                         echo Linking vmlinux.xz files to re-use scenarios `date` | tee -a "$tdir/log-xz" | tee -a $T/log
 672                         dirstash="`pwd`"
 673                         for i in `cat $T/xz-todo-copy`
 674                         do
 675                                 cd $i
 676                                 find . -name vmlinux -print > $T/xz-todo-copy-vmlinux
 677                                 for v in `cat $T/xz-todo-copy-vmlinux`
 678                                 do
 679                                         rm -f "$v"
 680                                         cp -l `cat $i/re-run`/"$i/$v".xz "`dirname "$v"`"
 681                                 done
 682                                 cd "$dirstash"
 683                         done
 684                 fi
 685                 echo Size after compressing $n2compress files: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log
 686                 echo Total duration `get_starttime_duration $starttime`. | tee -a $T/log
 687         else
 688                 echo No compression needed: `date` >> "$tdir/log-xz" 2>&1
 689         fi
 690 fi
 691 if test -n "$tdir"
 692 then
 693         cp $T/log "$tdir"
 694 fi
 695 exit $ret