| 1 | #!/bin/bash |
| 2 | # |
| 3 | # Copyright (C) 2018 Western Digital Corporation or its affiliates. |
| 4 | # |
| 5 | # This file is released under the GPL. |
| 6 | |
| 7 | usage() { |
| 8 | echo "Usage: $(basename "$0") [OPTIONS] <test target device file>" |
| 9 | echo "Options:" |
| 10 | echo -e "\t-d Run fio with valgrind using DRD tool" |
| 11 | echo -e "\t-e Run fio with valgrind using helgrind tool" |
| 12 | echo -e "\t-v Run fio with valgrind --read-var-info option" |
| 13 | echo -e "\t-l Test with libzbc ioengine" |
| 14 | echo -e "\t-r Reset all zones before test start" |
| 15 | echo -e "\t-w Reset all zones before executing each write test case" |
| 16 | echo -e "\t-o <max_open_zones> Run fio with max_open_zones limit" |
| 17 | echo -e "\t-t <test #> Run only a single test case with specified number" |
| 18 | echo -e "\t-q Quit the test run after any failed test" |
| 19 | echo -e "\t-z Run fio with debug=zbd option" |
| 20 | } |
| 21 | |
| 22 | max() { |
| 23 | if [ "$1" -gt "$2" ]; then |
| 24 | echo "$1" |
| 25 | else |
| 26 | echo "$2" |
| 27 | fi |
| 28 | } |
| 29 | |
| 30 | min() { |
| 31 | if [ "$1" -lt "$2" ]; then |
| 32 | echo "$1" |
| 33 | else |
| 34 | echo "$2" |
| 35 | fi |
| 36 | } |
| 37 | |
| 38 | ioengine() { |
| 39 | if [ -n "$use_libzbc" ]; then |
| 40 | echo -n "--ioengine=libzbc" |
| 41 | else |
| 42 | echo -n "--ioengine=$1" |
| 43 | fi |
| 44 | } |
| 45 | |
| 46 | set_io_scheduler() { |
| 47 | local dev=$1 sched=$2 |
| 48 | |
| 49 | [ -e "/sys/block/$dev" ] || return $? |
| 50 | if [ -e "/sys/block/$dev/mq" ]; then |
| 51 | case "$sched" in |
| 52 | noop) sched=none;; |
| 53 | deadline) sched=mq-deadline;; |
| 54 | esac |
| 55 | else |
| 56 | case "$sched" in |
| 57 | none) sched=noop;; |
| 58 | mq-deadline) sched=deadline;; |
| 59 | esac |
| 60 | fi |
| 61 | |
| 62 | echo "$sched" >"/sys/block/$dev/queue/scheduler" |
| 63 | } |
| 64 | |
| 65 | check_read() { |
| 66 | local read |
| 67 | |
| 68 | read=$(fio_read <"${logfile}.${test_number}") |
| 69 | echo "read: $read <> $1" >> "${logfile}.${test_number}" |
| 70 | [ "$read" = "$1" ] |
| 71 | } |
| 72 | |
| 73 | check_written() { |
| 74 | local written |
| 75 | |
| 76 | written=$(fio_written <"${logfile}.${test_number}") |
| 77 | echo "written: $written <> $1" >> "${logfile}.${test_number}" |
| 78 | [ "$written" = "$1" ] |
| 79 | } |
| 80 | |
| 81 | # Compare the reset count from the log file with reset count $2 using operator |
| 82 | # $1 (=, -ge, -gt, -le, -lt). |
| 83 | check_reset_count() { |
| 84 | local reset_count |
| 85 | |
| 86 | reset_count=$(fio_reset_count <"${logfile}.${test_number}") |
| 87 | echo "reset_count: test $reset_count $1 $2" >> "${logfile}.${test_number}" |
| 88 | eval "[ '$reset_count' '$1' '$2' ]" |
| 89 | } |
| 90 | |
| 91 | # Check log for failed assertions and crashes. Without these checks, |
| 92 | # a test can succeed even when these events happen, but it must fail. |
| 93 | check_log() { |
| 94 | [ ! -f "${logfile}.${1}" ] && return 0 |
| 95 | ! grep -q -e "Assertion " -e "Aborted " "${logfile}.${1}" |
| 96 | } |
| 97 | |
| 98 | # Whether or not $1 (/dev/...) is a SCSI device. |
| 99 | is_scsi_device() { |
| 100 | local d f |
| 101 | |
| 102 | d=$(basename "$dev") |
| 103 | for f in /sys/class/scsi_device/*/device/block/"$d"; do |
| 104 | [ -e "$f" ] && return 0 |
| 105 | done |
| 106 | return 1 |
| 107 | } |
| 108 | |
| 109 | job_var_opts_exclude() { |
| 110 | local o |
| 111 | local ex_key="${1}" |
| 112 | |
| 113 | for o in "${job_var_opts[@]}"; do |
| 114 | if [[ ${o} =~ "${ex_key}" ]]; then |
| 115 | continue |
| 116 | fi |
| 117 | echo -n "${o}" |
| 118 | done |
| 119 | } |
| 120 | |
| 121 | has_max_open_zones() { |
| 122 | while (($# > 1)); do |
| 123 | if [[ ${1} =~ "--max_open_zones" ]]; then |
| 124 | return 0 |
| 125 | fi |
| 126 | shift |
| 127 | done |
| 128 | return 1 |
| 129 | } |
| 130 | |
| 131 | run_fio() { |
| 132 | local fio opts |
| 133 | |
| 134 | fio=$(dirname "$0")/../../fio |
| 135 | |
| 136 | opts=(${global_var_opts[@]}) |
| 137 | opts+=("--max-jobs=16" "--aux-path=/tmp" "--allow_file_create=0" \ |
| 138 | "--significant_figures=10" "$@") |
| 139 | # When max_open_zones option is specified to this test script, add |
| 140 | # max_open_zones option to fio command unless the test case already add it. |
| 141 | if [[ -n ${max_open_zones_opt} ]] && ! has_max_open_zones "${opts[@]}"; then |
| 142 | opts+=("--max_open_zones=${max_open_zones_opt}") |
| 143 | fi |
| 144 | { echo; echo "fio ${opts[*]}"; echo; } >>"${logfile}.${test_number}" |
| 145 | |
| 146 | "${dynamic_analyzer[@]}" "$fio" "${opts[@]}" |
| 147 | } |
| 148 | |
| 149 | run_one_fio_job() { |
| 150 | local r |
| 151 | |
| 152 | r=$(((RANDOM << 16) | RANDOM)) |
| 153 | run_fio --name="$dev" --filename="$dev" "$@" --randseed="$r" \ |
| 154 | --thread=1 --direct=1 |
| 155 | } |
| 156 | |
| 157 | write_and_run_one_fio_job() { |
| 158 | local r |
| 159 | local write_offset="${1}" |
| 160 | local write_size="${2}" |
| 161 | local -a write_opts |
| 162 | |
| 163 | shift 2 |
| 164 | r=$(((RANDOM << 16) | RANDOM)) |
| 165 | write_opts=(--name="write_job" --rw=write "$(ioengine "psync")" \ |
| 166 | --bs="${logical_block_size}" --zonemode=zbd \ |
| 167 | --zonesize="${zone_size}" --thread=1 --direct=1 \ |
| 168 | --offset="${write_offset}" --size="${write_size}") |
| 169 | write_opts+=("${job_var_opts[@]}") |
| 170 | run_fio --filename="$dev" --randseed="$r" "${write_opts[@]}" \ |
| 171 | --name="$dev" --wait_for="write_job" "$@" --thread=1 --direct=1 |
| 172 | } |
| 173 | |
| 174 | # Run fio on the first four sequential zones of the disk. |
| 175 | run_fio_on_seq() { |
| 176 | local opts=() |
| 177 | |
| 178 | opts+=("--offset=$((first_sequential_zone_sector * 512))") |
| 179 | opts+=("--size=$((4 * zone_size))" "--zonemode=zbd") |
| 180 | if [ -z "$is_zbd" ]; then |
| 181 | opts+=("--zonesize=${zone_size}") |
| 182 | fi |
| 183 | run_one_fio_job "${opts[@]}" "$@" |
| 184 | } |
| 185 | |
| 186 | # Prepare for write test by resetting zones. When reset_before_write or |
| 187 | # max_open_zones option is specified, reset all zones of the test target to |
| 188 | # ensure that zones out of the test target range do not have open zones. This |
| 189 | # allows the write test to the target range to be able to open zones up to |
| 190 | # max_open_zones limit specified as the option or obtained from sysfs. |
| 191 | prep_write() { |
| 192 | [[ -n "${reset_before_write}" || -n "${max_open_zones_opt}" ]] && |
| 193 | [[ -n "${is_zbd}" ]] && reset_zone "${dev}" -1 |
| 194 | } |
| 195 | |
| 196 | SKIP_TESTCASE=255 |
| 197 | |
| 198 | require_scsi_dev() { |
| 199 | if ! is_scsi_device "$dev"; then |
| 200 | SKIP_REASON="$dev is not a SCSI device" |
| 201 | return 1 |
| 202 | fi |
| 203 | return 0 |
| 204 | } |
| 205 | |
| 206 | require_conv_zone_bytes() { |
| 207 | local req_bytes=${1} |
| 208 | |
| 209 | if ((req_bytes > first_sequential_zone_sector * 512)); then |
| 210 | SKIP_REASON="$dev does not have enough conventional zones" |
| 211 | return 1 |
| 212 | fi |
| 213 | return 0 |
| 214 | } |
| 215 | |
| 216 | require_zbd() { |
| 217 | if [[ -z ${is_zbd} ]]; then |
| 218 | SKIP_REASON="$dev is not a zoned block device" |
| 219 | return 1 |
| 220 | fi |
| 221 | return 0 |
| 222 | } |
| 223 | |
| 224 | require_regular_block_dev() { |
| 225 | if [[ -n ${is_zbd} ]]; then |
| 226 | SKIP_REASON="$dev is not a regular block device" |
| 227 | return 1 |
| 228 | fi |
| 229 | return 0 |
| 230 | } |
| 231 | |
| 232 | require_seq_zones() { |
| 233 | local req_seq_zones=${1} |
| 234 | local seq_bytes=$((disk_size - first_sequential_zone_sector * 512)) |
| 235 | |
| 236 | if ((req_seq_zones > seq_bytes / zone_size)); then |
| 237 | SKIP_REASON="$dev does not have $req_seq_zones sequential zones" |
| 238 | return 1 |
| 239 | fi |
| 240 | return 0 |
| 241 | } |
| 242 | |
| 243 | require_conv_zones() { |
| 244 | local req_c_zones=${1} |
| 245 | local conv_bytes=$((first_sequential_zone_sector * 512)) |
| 246 | |
| 247 | if ((req_c_zones > conv_bytes / zone_size)); then |
| 248 | SKIP_REASON="$dev does not have $req_c_zones conventional zones" |
| 249 | return 1 |
| 250 | fi |
| 251 | return 0 |
| 252 | } |
| 253 | |
| 254 | # Check whether buffered writes are refused. |
| 255 | test1() { |
| 256 | run_fio --name=job1 --filename="$dev" --rw=write --direct=0 --bs=4K \ |
| 257 | "$(ioengine "psync")" --size="${zone_size}" --thread=1 \ |
| 258 | --zonemode=zbd --zonesize="${zone_size}" 2>&1 | |
| 259 | tee -a "${logfile}.${test_number}" | |
| 260 | grep -q 'Using direct I/O is mandatory for writing to ZBD drives' |
| 261 | local fio_rc=${PIPESTATUS[0]} grep_rc=${PIPESTATUS[2]} |
| 262 | case "$fio_rc" in |
| 263 | 0|1) ;; |
| 264 | *) return "$fio_rc" |
| 265 | esac |
| 266 | if [ -n "$is_zbd" ]; then |
| 267 | [ "$grep_rc" = 0 ] |
| 268 | else |
| 269 | [ "$grep_rc" != 0 ] |
| 270 | fi |
| 271 | } |
| 272 | |
| 273 | # Block size exceeds zone size. |
| 274 | test2() { |
| 275 | local bs off opts=() rc |
| 276 | |
| 277 | off=$(((first_sequential_zone_sector + 2 * sectors_per_zone) * 512)) |
| 278 | bs=$((2 * zone_size)) |
| 279 | opts+=("$(ioengine "psync")") |
| 280 | opts+=("--name=job1" "--filename=$dev" "--rw=write" "--direct=1") |
| 281 | opts+=("--zonemode=zbd" "--offset=$off" "--bs=$bs" "--size=$bs") |
| 282 | if [ -z "$is_zbd" ]; then |
| 283 | opts+=("--zonesize=${zone_size}") |
| 284 | fi |
| 285 | run_fio "${opts[@]}" >> "${logfile}.${test_number}" 2>&1 && return 1 |
| 286 | grep -q 'buflen exceeds zone size' "${logfile}.${test_number}" |
| 287 | } |
| 288 | |
| 289 | # Run fio against an empty zone. This causes fio to report "No I/O performed". |
| 290 | test3() { |
| 291 | local off opts=() rc |
| 292 | |
| 293 | require_seq_zones 129 || return $SKIP_TESTCASE |
| 294 | off=$((first_sequential_zone_sector * 512 + 128 * zone_size)) |
| 295 | size=$((zone_size)) |
| 296 | [ -n "$is_zbd" ] && reset_zone "$dev" $((off / 512)) |
| 297 | opts+=("--name=$dev" "--filename=$dev" "--offset=$off" "--bs=4K") |
| 298 | opts+=("--size=$size" "--zonemode=zbd") |
| 299 | opts+=("$(ioengine "psync")" "--rw=read" "--direct=1" "--thread=1") |
| 300 | if [ -z "$is_zbd" ]; then |
| 301 | opts+=("--zonesize=${zone_size}") |
| 302 | fi |
| 303 | run_fio "${opts[@]}" >> "${logfile}.${test_number}" 2>&1 || return $? |
| 304 | ! grep -q 'READ:' "${logfile}.${test_number}" |
| 305 | } |
| 306 | |
| 307 | # Run fio with --read_beyond_wp=1 against an empty zone. |
| 308 | test4() { |
| 309 | local off opts=() |
| 310 | |
| 311 | require_seq_zones 130 || return $SKIP_TESTCASE |
| 312 | off=$((first_sequential_zone_sector * 512 + 129 * zone_size)) |
| 313 | size=$((zone_size)) |
| 314 | [ -n "$is_zbd" ] && reset_zone "$dev" $((off / 512)) |
| 315 | opts+=("--name=$dev" "--filename=$dev" "--offset=$off") |
| 316 | opts+=(--bs="$(min $((logical_block_size * 256)) $size)") |
| 317 | opts+=("--size=$size" "--thread=1" "--read_beyond_wp=1") |
| 318 | opts+=("$(ioengine "psync")" "--rw=read" "--direct=1" "--disable_lat=1") |
| 319 | opts+=("--zonemode=zbd" "--zonesize=${zone_size}") |
| 320 | run_fio "${opts[@]}" >> "${logfile}.${test_number}" 2>&1 || return $? |
| 321 | check_read $size || return $? |
| 322 | } |
| 323 | |
| 324 | # Sequential write to sequential zones. |
| 325 | test5() { |
| 326 | local size off capacity bs |
| 327 | |
| 328 | prep_write |
| 329 | off=$((first_sequential_zone_sector * 512)) |
| 330 | capacity=$(total_zone_capacity 4 $off $dev) |
| 331 | size=$((4 * zone_size)) |
| 332 | bs=$(min "$(max $((zone_size / 64)) "$logical_block_size")" "$zone_cap_bs") |
| 333 | run_fio_on_seq "$(ioengine "psync")" --iodepth=1 --rw=write \ |
| 334 | --bs="$bs" --do_verify=1 --verify=md5 \ |
| 335 | >>"${logfile}.${test_number}" 2>&1 || return $? |
| 336 | check_written $capacity || return $? |
| 337 | check_read $capacity || return $? |
| 338 | } |
| 339 | |
| 340 | # Sequential read from sequential zones. |
| 341 | test6() { |
| 342 | local size off capacity bs |
| 343 | |
| 344 | prep_write |
| 345 | off=$((first_sequential_zone_sector * 512)) |
| 346 | capacity=$(total_zone_capacity 4 $off $dev) |
| 347 | size=$((4 * zone_size)) |
| 348 | bs=$(min "$(max $((zone_size / 64)) "$logical_block_size")" "$zone_cap_bs") |
| 349 | write_and_run_one_fio_job \ |
| 350 | $((first_sequential_zone_sector * 512)) "${size}" \ |
| 351 | --offset="${off}" \ |
| 352 | --size="${size}" --zonemode=zbd --zonesize="${zone_size}" \ |
| 353 | "$(ioengine "psync")" --iodepth=1 --rw=read --bs="$bs" \ |
| 354 | >>"${logfile}.${test_number}" 2>&1 || return $? |
| 355 | check_read $capacity || return $? |
| 356 | } |
| 357 | |
| 358 | # Random write to sequential zones, libaio, queue depth 1. |
| 359 | test7() { |
| 360 | local size=$((zone_size)) |
| 361 | local off capacity |
| 362 | |
| 363 | prep_write |
| 364 | off=$((first_sequential_zone_sector * 512)) |
| 365 | capacity=$(total_zone_capacity 1 $off $dev) |
| 366 | run_fio_on_seq "$(ioengine "libaio")" --iodepth=1 --rw=randwrite \ |
| 367 | --bs="$(min 16384 "${zone_size}")" \ |
| 368 | --do_verify=1 --verify=md5 --size="$size" \ |
| 369 | >>"${logfile}.${test_number}" 2>&1 || return $? |
| 370 | check_written $capacity || return $? |
| 371 | check_read $capacity || return $? |
| 372 | } |
| 373 | |
| 374 | # Random write to sequential zones, libaio, queue depth 64. |
| 375 | test8() { |
| 376 | local size off capacity |
| 377 | |
| 378 | prep_write |
| 379 | size=$((4 * zone_size)) |
| 380 | off=$((first_sequential_zone_sector * 512)) |
| 381 | capacity=$(total_zone_capacity 4 $off $dev) |
| 382 | run_fio_on_seq "$(ioengine "libaio")" --iodepth=64 --rw=randwrite \ |
| 383 | --bs="$(min 16384 "${zone_size}")" \ |
| 384 | --do_verify=1 --verify=md5 \ |
| 385 | >>"${logfile}.${test_number}" 2>&1 || return $? |
| 386 | check_written $capacity || return $? |
| 387 | check_read $capacity || return $? |
| 388 | } |
| 389 | |
| 390 | # Random write to sequential zones, sg, queue depth 1. |
| 391 | test9() { |
| 392 | local size |
| 393 | |
| 394 | require_scsi_dev || return $SKIP_TESTCASE |
| 395 | |
| 396 | prep_write |
| 397 | size=$((4 * zone_size)) |
| 398 | run_fio_on_seq --ioengine=sg \ |
| 399 | --iodepth=1 --rw=randwrite --bs=16K \ |
| 400 | --do_verify=1 --verify=md5 \ |
| 401 | >>"${logfile}.${test_number}" 2>&1 || return $? |
| 402 | check_written $size || return $? |
| 403 | check_read $size || return $? |
| 404 | } |
| 405 | |
| 406 | # Random write to sequential zones, sg, queue depth 64. |
| 407 | test10() { |
| 408 | local size |
| 409 | |
| 410 | require_scsi_dev || return $SKIP_TESTCASE |
| 411 | |
| 412 | prep_write |
| 413 | size=$((4 * zone_size)) |
| 414 | run_fio_on_seq --ioengine=sg \ |
| 415 | --iodepth=64 --rw=randwrite --bs=16K \ |
| 416 | --do_verify=1 --verify=md5 \ |
| 417 | >>"${logfile}.${test_number}" 2>&1 || return $? |
| 418 | check_written $size || return $? |
| 419 | check_read $size || return $? |
| 420 | } |
| 421 | |
| 422 | # Random write to sequential zones, libaio, queue depth 64, random block size. |
| 423 | test11() { |
| 424 | local size off capacity |
| 425 | |
| 426 | prep_write |
| 427 | size=$((4 * zone_size)) |
| 428 | off=$((first_sequential_zone_sector * 512)) |
| 429 | capacity=$(total_zone_capacity 4 $off $dev) |
| 430 | run_fio_on_seq "$(ioengine "libaio")" --iodepth=64 --rw=randwrite \ |
| 431 | --bsrange=4K-64K --do_verify=1 --verify=md5 \ |
| 432 | --debug=zbd >>"${logfile}.${test_number}" 2>&1 || return $? |
| 433 | check_written $capacity || return $? |
| 434 | check_read $capacity || return $? |
| 435 | } |
| 436 | |
| 437 | # Random write to sequential zones, libaio, queue depth 64, max 1 open zone. |
| 438 | test12() { |
| 439 | local size off capacity |
| 440 | |
| 441 | prep_write |
| 442 | size=$((8 * zone_size)) |
| 443 | off=$((first_sequential_zone_sector * 512)) |
| 444 | capacity=$(total_zone_capacity 8 $off $dev) |
| 445 | run_fio_on_seq "$(ioengine "libaio")" --iodepth=64 --rw=randwrite --bs=16K \ |
| 446 | --max_open_zones=1 --size=$size --do_verify=1 --verify=md5 \ |
| 447 | --debug=zbd >>"${logfile}.${test_number}" 2>&1 || return $? |
| 448 | check_written $capacity || return $? |
| 449 | check_read $capacity || return $? |
| 450 | } |
| 451 | |
| 452 | # Random write to sequential zones, libaio, queue depth 64, max 4 open zones. |
| 453 | test13() { |
| 454 | local size off capacity |
| 455 | |
| 456 | prep_write |
| 457 | size=$((8 * zone_size)) |
| 458 | off=$((first_sequential_zone_sector * 512)) |
| 459 | capacity=$(total_zone_capacity 8 $off $dev) |
| 460 | run_fio_on_seq "$(ioengine "libaio")" --iodepth=64 --rw=randwrite --bs=16K \ |
| 461 | --max_open_zones=4 --size=$size --do_verify=1 --verify=md5 \ |
| 462 | --debug=zbd \ |
| 463 | >>"${logfile}.${test_number}" 2>&1 || return $? |
| 464 | check_written $capacity || return $? |
| 465 | check_read $capacity || return $? |
| 466 | } |
| 467 | |
| 468 | # Random write to conventional zones. |
| 469 | test14() { |
| 470 | local off size |
| 471 | |
| 472 | if ! result=($(first_online_zone "$dev")); then |
| 473 | echo "Failed to determine first online zone" |
| 474 | exit 1 |
| 475 | fi |
| 476 | off=${result[0]} |
| 477 | prep_write |
| 478 | size=$((16 * 2**20)) # 20 MB |
| 479 | require_conv_zone_bytes "${size}" || return $SKIP_TESTCASE |
| 480 | |
| 481 | run_one_fio_job "$(ioengine "libaio")" --iodepth=64 --rw=randwrite --bs=16K \ |
| 482 | --zonemode=zbd --zonesize="${zone_size}" --do_verify=1 \ |
| 483 | --verify=md5 --offset=$off --size=$size\ |
| 484 | >>"${logfile}.${test_number}" 2>&1 || return $? |
| 485 | check_written $((size)) || return $? |
| 486 | check_read $((size)) || return $? |
| 487 | } |
| 488 | |
| 489 | # Sequential read on a mix of empty and full zones. |
| 490 | test15() { |
| 491 | local i off size bs |
| 492 | local w_off w_size w_capacity |
| 493 | |
| 494 | for ((i=0;i<4;i++)); do |
| 495 | [ -n "$is_zbd" ] && |
| 496 | reset_zone "$dev" $((first_sequential_zone_sector + |
| 497 | i*sectors_per_zone)) |
| 498 | done |
| 499 | prep_write |
| 500 | w_off=$(((first_sequential_zone_sector + 2 * sectors_per_zone) * 512)) |
| 501 | w_size=$((2 * zone_size)) |
| 502 | w_capacity=$(total_zone_capacity 2 $w_off $dev) |
| 503 | off=$((first_sequential_zone_sector * 512)) |
| 504 | size=$((4 * zone_size)) |
| 505 | bs=$(min $((zone_size / 16)) "$zone_cap_bs") |
| 506 | write_and_run_one_fio_job "${w_off}" "${w_size}" \ |
| 507 | "$(ioengine "psync")" --rw=read --bs="$bs" \ |
| 508 | --zonemode=zbd --zonesize="${zone_size}" --offset=$off \ |
| 509 | --size=$((size)) >>"${logfile}.${test_number}" 2>&1 || |
| 510 | return $? |
| 511 | check_written $((w_capacity)) || return $? |
| 512 | check_read $((w_capacity)) |
| 513 | } |
| 514 | |
| 515 | # Random read on a mix of empty and full zones. |
| 516 | test16() { |
| 517 | local off size |
| 518 | local i w_off w_size w_capacity |
| 519 | |
| 520 | for ((i=0;i<4;i++)); do |
| 521 | [ -n "$is_zbd" ] && |
| 522 | reset_zone "$dev" $((first_sequential_zone_sector + |
| 523 | i*sectors_per_zone)) |
| 524 | done |
| 525 | prep_write |
| 526 | w_off=$(((first_sequential_zone_sector + 2 * sectors_per_zone) * 512)) |
| 527 | w_size=$((2 * zone_size)) |
| 528 | w_capacity=$(total_zone_capacity 2 $w_off $dev) |
| 529 | off=$((first_sequential_zone_sector * 512)) |
| 530 | size=$((4 * zone_size)) |
| 531 | write_and_run_one_fio_job "${w_off}" "${w_size}" \ |
| 532 | "$(ioengine "libaio")" --iodepth=64 --rw=randread --bs=16K \ |
| 533 | --zonemode=zbd --zonesize="${zone_size}" --offset=$off \ |
| 534 | --size=$size >>"${logfile}.${test_number}" 2>&1 || return $? |
| 535 | check_written $w_capacity || return $? |
| 536 | check_read $size || return $? |
| 537 | } |
| 538 | |
| 539 | # Random reads and writes in the last zone. |
| 540 | test17() { |
| 541 | local io off last read size written |
| 542 | |
| 543 | off=$(((disk_size / zone_size - 1) * zone_size)) |
| 544 | size=$((disk_size - off)) |
| 545 | if ! last=($(last_online_zone "$dev")); then |
| 546 | echo "Failed to determine last online zone" |
| 547 | exit 1 |
| 548 | fi |
| 549 | if [[ "$((last * 512))" -lt "$off" ]]; then |
| 550 | off=$((last * 512)) |
| 551 | size=$zone_size |
| 552 | fi |
| 553 | if [ -n "$is_zbd" ]; then |
| 554 | reset_zone "$dev" $((off / 512)) || return $? |
| 555 | fi |
| 556 | prep_write |
| 557 | run_one_fio_job "$(ioengine "libaio")" --iodepth=8 --rw=randrw --bs=4K \ |
| 558 | --zonemode=zbd --zonesize="${zone_size}" \ |
| 559 | --offset=$off --loops=2 --norandommap=1 \ |
| 560 | --size="$size"\ |
| 561 | >>"${logfile}.${test_number}" 2>&1 || return $? |
| 562 | written=$(fio_written <"${logfile}.${test_number}") |
| 563 | read=$(fio_read <"${logfile}.${test_number}") |
| 564 | io=$((written + read)) |
| 565 | echo "Total number of bytes read and written: $io <> $size" \ |
| 566 | >>"${logfile}.${test_number}" |
| 567 | [ $io = $((size * 2)) ]; |
| 568 | } |
| 569 | |
| 570 | # Out-of-range zone reset threshold and frequency parameters. |
| 571 | test18() { |
| 572 | run_fio_on_seq --zone_reset_threshold=-1 |& |
| 573 | tee -a "${logfile}.${test_number}" | |
| 574 | grep -q 'value out of range' || return $? |
| 575 | } |
| 576 | |
| 577 | test19() { |
| 578 | run_fio_on_seq --zone_reset_threshold=2 |& |
| 579 | tee -a "${logfile}.${test_number}" | |
| 580 | grep -q 'value out of range' || return $? |
| 581 | } |
| 582 | |
| 583 | test20() { |
| 584 | run_fio_on_seq --zone_reset_threshold=.4:.6 |& |
| 585 | tee -a "${logfile}.${test_number}" | |
| 586 | grep -q 'the list exceeding max length' || return $? |
| 587 | } |
| 588 | |
| 589 | test21() { |
| 590 | run_fio_on_seq --zone_reset_frequency=-1 |& |
| 591 | tee -a "${logfile}.${test_number}" | |
| 592 | grep -q 'value out of range' || return $? |
| 593 | } |
| 594 | |
| 595 | test22() { |
| 596 | run_fio_on_seq --zone_reset_frequency=2 |& |
| 597 | tee -a "${logfile}.${test_number}" | |
| 598 | grep -q 'value out of range' || return $? |
| 599 | } |
| 600 | |
| 601 | test23() { |
| 602 | run_fio_on_seq --zone_reset_frequency=.4:.6 |& |
| 603 | tee -a "${logfile}.${test_number}" | |
| 604 | grep -q 'the list exceeding max length' || return $? |
| 605 | } |
| 606 | |
| 607 | test24() { |
| 608 | local bs loops=9 size=$((zone_size)) |
| 609 | local off capacity |
| 610 | |
| 611 | prep_write |
| 612 | off=$((first_sequential_zone_sector * 512)) |
| 613 | capacity=$(total_zone_capacity 1 $off $dev) |
| 614 | |
| 615 | bs=$(min $((256*1024)) "$zone_size") |
| 616 | run_fio_on_seq "$(ioengine "psync")" --rw=write --bs="$bs" \ |
| 617 | --size=$size --loops=$loops \ |
| 618 | --zone_reset_frequency=.01 --zone_reset_threshold=.90 \ |
| 619 | >> "${logfile}.${test_number}" 2>&1 || return $? |
| 620 | check_written $((capacity * loops)) || return $? |
| 621 | check_reset_count -eq 8 || |
| 622 | check_reset_count -eq 9 || |
| 623 | check_reset_count -eq 10 || return $? |
| 624 | } |
| 625 | |
| 626 | # Multiple non-overlapping sequential write jobs for the same drive. |
| 627 | test25() { |
| 628 | local i opts=() |
| 629 | |
| 630 | for ((i=0;i<16;i++)); do |
| 631 | [ -n "$is_zbd" ] && |
| 632 | reset_zone "$dev" $((first_sequential_zone_sector + i*sectors_per_zone)) |
| 633 | done |
| 634 | prep_write |
| 635 | for ((i=0;i<16;i++)); do |
| 636 | opts+=("--name=job$i" "--filename=$dev" "--thread=1" "--direct=1") |
| 637 | opts+=("--offset=$((first_sequential_zone_sector*512 + zone_size*i))") |
| 638 | opts+=("--size=$zone_size" "$(ioengine "psync")" "--rw=write" "--bs=16K") |
| 639 | opts+=("--zonemode=zbd" "--zonesize=${zone_size}" "--group_reporting=1") |
| 640 | opts+=(${job_var_opts[@]}) |
| 641 | done |
| 642 | run_fio "${opts[@]}" >> "${logfile}.${test_number}" 2>&1 || return $? |
| 643 | } |
| 644 | |
| 645 | write_to_first_seq_zone() { |
| 646 | local loops=4 r |
| 647 | local off capacity |
| 648 | |
| 649 | prep_write |
| 650 | off=$((first_sequential_zone_sector * 512)) |
| 651 | capacity=$(total_zone_capacity 1 $off $dev) |
| 652 | |
| 653 | r=$(((RANDOM << 16) | RANDOM)) |
| 654 | run_fio --name="$dev" --filename="$dev" "$(ioengine "psync")" --rw="$1" \ |
| 655 | --thread=1 --do_verify=1 --verify=md5 --direct=1 --bs=4K \ |
| 656 | --offset=$off \ |
| 657 | --size=$zone_size --loops=$loops --randseed="$r" \ |
| 658 | --zonemode=zbd --zonesize="${zone_size}" --group_reporting=1 \ |
| 659 | --gtod_reduce=1 >> "${logfile}.${test_number}" 2>&1 || return $? |
| 660 | check_written $((loops * capacity)) || return $? |
| 661 | } |
| 662 | |
| 663 | # Overwrite the first sequential zone four times sequentially. |
| 664 | test26() { |
| 665 | write_to_first_seq_zone write |
| 666 | } |
| 667 | |
| 668 | # Overwrite the first sequential zone four times using random writes. |
| 669 | test27() { |
| 670 | write_to_first_seq_zone randwrite |
| 671 | } |
| 672 | |
| 673 | # Multiple overlapping random write jobs for the same drive. |
| 674 | test28() { |
| 675 | local i jobs=16 off opts |
| 676 | |
| 677 | require_seq_zones 65 || return $SKIP_TESTCASE |
| 678 | off=$((first_sequential_zone_sector * 512 + 64 * zone_size)) |
| 679 | [ -n "$is_zbd" ] && reset_zone "$dev" $((off / 512)) |
| 680 | prep_write |
| 681 | opts=("--debug=zbd") |
| 682 | capacity=$(total_zone_capacity 1 $off $dev) |
| 683 | for ((i=0;i<jobs;i++)); do |
| 684 | opts+=("--name=job$i" "--filename=$dev" "--offset=$off" "--bs=16K") |
| 685 | opts+=("--size=$zone_size" "--io_size=$capacity" "$(ioengine "psync")" "--rw=randwrite") |
| 686 | opts+=("--thread=1" "--direct=1" "--zonemode=zbd") |
| 687 | opts+=("--zonesize=${zone_size}" "--group_reporting=1") |
| 688 | opts+=(${job_var_opts[@]}) |
| 689 | done |
| 690 | run_fio "${opts[@]}" >> "${logfile}.${test_number}" 2>&1 || return $? |
| 691 | check_written $((jobs * $capacity)) || return $? |
| 692 | check_reset_count -eq $jobs || |
| 693 | check_reset_count -eq $((jobs - 1)) || |
| 694 | return $? |
| 695 | } |
| 696 | |
| 697 | # Multiple overlapping random write jobs for the same drive and with a limited |
| 698 | # number of open zones. |
| 699 | test29() { |
| 700 | local i jobs=16 off opts=() |
| 701 | |
| 702 | require_seq_zones 80 || return $SKIP_TESTCASE |
| 703 | off=$((first_sequential_zone_sector * 512 + 64 * zone_size)) |
| 704 | size=$((16*zone_size)) |
| 705 | prep_write |
| 706 | opts=("--debug=zbd") |
| 707 | for ((i=0;i<jobs;i++)); do |
| 708 | opts+=("--name=job$i" "--filename=$dev" "--offset=$off" "--bs=16K") |
| 709 | opts+=("--size=$size" "--io_size=$zone_size" "--thread=1") |
| 710 | opts+=("$(ioengine "psync")" "--rw=randwrite" "--direct=1") |
| 711 | opts+=("--max_open_zones=4" "--group_reporting=1") |
| 712 | opts+=("--zonemode=zbd" "--zonesize=${zone_size}") |
| 713 | # max_open_zones is already specified |
| 714 | opts+=($(job_var_opts_exclude "--max_open_zones")) |
| 715 | done |
| 716 | run_fio "${opts[@]}" >> "${logfile}.${test_number}" 2>&1 || return $? |
| 717 | check_written $((jobs * zone_size)) || return $? |
| 718 | } |
| 719 | |
| 720 | # Random reads and writes across the entire disk for 30s. |
| 721 | test30() { |
| 722 | local off |
| 723 | |
| 724 | prep_write |
| 725 | off=$((first_sequential_zone_sector * 512)) |
| 726 | run_one_fio_job "$(ioengine "libaio")" --iodepth=8 --rw=randrw \ |
| 727 | --bs="$(max $((zone_size / 128)) "$logical_block_size")"\ |
| 728 | --zonemode=zbd --zonesize="${zone_size}" --offset=$off\ |
| 729 | --loops=2 --time_based --runtime=30s --norandommap=1\ |
| 730 | >>"${logfile}.${test_number}" 2>&1 |
| 731 | } |
| 732 | |
| 733 | # Random reads across all sequential zones for 30s. This is not only a fio |
| 734 | # test but also allows to verify the performance of a drive. |
| 735 | test31() { |
| 736 | local bs inc nz off opts size |
| 737 | |
| 738 | [ -n "$is_zbd" ] && reset_zone "$dev" -1 |
| 739 | |
| 740 | # As preparation, write 128 KB to sequential write required zones. Limit |
| 741 | # write target zones up to max_open_zones to keep test time reasonable. |
| 742 | # To distribute the write target zones evenly, skip certain zones for every |
| 743 | # write. Utilize zonemode strided for such write patterns. |
| 744 | bs=$((128 * 1024)) |
| 745 | nz=$((max_open_zones)) |
| 746 | if [[ $nz -eq 0 ]]; then |
| 747 | nz=128 |
| 748 | fi |
| 749 | off=$((first_sequential_zone_sector * 512)) |
| 750 | size=$((disk_size - off)) |
| 751 | inc=$(((size / nz / zone_size) * zone_size)) |
| 752 | opts=("--name=$dev" "--filename=$dev" "--rw=write" "--bs=${bs}") |
| 753 | opts+=("--offset=$off" "--size=$((inc * nz))" "--io_size=$((bs * nz))") |
| 754 | opts+=("--zonemode=strided" "--zonesize=${bs}" "--zonerange=${inc}") |
| 755 | opts+=("--direct=1") |
| 756 | echo "fio ${opts[@]}" >> "${logfile}.${test_number}" |
| 757 | "$(dirname "$0")/../../fio" "${opts[@]}" >> "${logfile}.${test_number}" 2>&1 |
| 758 | |
| 759 | # Next, run the test. |
| 760 | opts=("--name=$dev" "--filename=$dev" "--offset=$off" "--size=$size") |
| 761 | opts+=("--bs=$bs" "$(ioengine "psync")" "--rw=randread" "--direct=1") |
| 762 | opts+=("--thread=1" "--time_based" "--runtime=30" "--zonemode=zbd") |
| 763 | opts+=("--zonesize=${zone_size}") |
| 764 | opts+=(${job_var_opts[@]}) |
| 765 | run_fio "${opts[@]}" >> "${logfile}.${test_number}" 2>&1 || return $? |
| 766 | } |
| 767 | |
| 768 | # Random writes across all sequential zones. This is not only a fio test but |
| 769 | # also allows to verify the performance of a drive. |
| 770 | test32() { |
| 771 | local off opts=() size |
| 772 | |
| 773 | require_zbd || return $SKIP_TESTCASE |
| 774 | |
| 775 | prep_write |
| 776 | off=$((first_sequential_zone_sector * 512)) |
| 777 | size=$((disk_size - off)) |
| 778 | opts+=("--name=$dev" "--filename=$dev" "--offset=$off" "--size=$size") |
| 779 | opts+=("--bs=128K" "$(ioengine "psync")" "--rw=randwrite" "--direct=1") |
| 780 | opts+=("--thread=1" "--time_based" "--runtime=30") |
| 781 | opts+=("--max_open_zones=$max_open_zones" "--zonemode=zbd") |
| 782 | opts+=("--zonesize=${zone_size}") |
| 783 | run_fio "${opts[@]}" >> "${logfile}.${test_number}" 2>&1 || return $? |
| 784 | } |
| 785 | |
| 786 | # Write to sequential zones with a block size that is not a divisor of the |
| 787 | # zone size. |
| 788 | test33() { |
| 789 | local bs io_size size |
| 790 | local off capacity=0; |
| 791 | |
| 792 | prep_write |
| 793 | off=$((first_sequential_zone_sector * 512)) |
| 794 | capacity=$(total_zone_capacity 1 $off $dev) |
| 795 | size=$((2 * zone_size)) |
| 796 | io_size=$((5 * capacity)) |
| 797 | bs=$((3 * capacity / 4)) |
| 798 | run_fio_on_seq "$(ioengine "psync")" --iodepth=1 --rw=write \ |
| 799 | --size=$size --io_size=$io_size --bs=$bs \ |
| 800 | >> "${logfile}.${test_number}" 2>&1 || return $? |
| 801 | check_written $(((io_size + bs - 1) / bs * bs)) || return $? |
| 802 | } |
| 803 | |
| 804 | # Write to sequential zones with a block size that is not a divisor of the |
| 805 | # zone size and with data verification enabled. |
| 806 | test34() { |
| 807 | local size |
| 808 | |
| 809 | prep_write |
| 810 | size=$((2 * zone_size)) |
| 811 | run_fio_on_seq "$(ioengine "psync")" --iodepth=1 --rw=write --size=$size \ |
| 812 | --do_verify=1 --verify=md5 --bs=$((3 * zone_size / 4)) \ |
| 813 | >> "${logfile}.${test_number}" 2>&1 && return 1 |
| 814 | grep -q 'not a divisor of' "${logfile}.${test_number}" |
| 815 | } |
| 816 | |
| 817 | # Test 1/4 for the I/O boundary rounding code: $size < $zone_size. |
| 818 | test35() { |
| 819 | local bs off io_size size |
| 820 | |
| 821 | prep_write |
| 822 | off=$(((first_sequential_zone_sector + 1) * 512)) |
| 823 | size=$((zone_size - 2 * 512)) |
| 824 | bs=$((zone_size / 4)) |
| 825 | run_one_fio_job --offset=$off --size=$size "$(ioengine "psync")" \ |
| 826 | --iodepth=1 --rw=write --do_verify=1 --verify=md5 \ |
| 827 | --bs=$bs --zonemode=zbd --zonesize="${zone_size}" \ |
| 828 | >> "${logfile}.${test_number}" 2>&1 && return 1 |
| 829 | grep -q 'io_size must be at least one zone' "${logfile}.${test_number}" |
| 830 | } |
| 831 | |
| 832 | # Test 2/4 for the I/O boundary rounding code: $size < $zone_size. |
| 833 | test36() { |
| 834 | local bs off io_size size |
| 835 | |
| 836 | prep_write |
| 837 | off=$(((first_sequential_zone_sector) * 512)) |
| 838 | size=$((zone_size - 512)) |
| 839 | bs=$((zone_size / 4)) |
| 840 | run_one_fio_job --offset=$off --size=$size "$(ioengine "psync")" \ |
| 841 | --iodepth=1 --rw=write --do_verify=1 --verify=md5 \ |
| 842 | --bs=$bs --zonemode=zbd --zonesize="${zone_size}" \ |
| 843 | >> "${logfile}.${test_number}" 2>&1 && return 1 |
| 844 | grep -q 'io_size must be at least one zone' "${logfile}.${test_number}" |
| 845 | } |
| 846 | |
| 847 | # Test 3/4 for the I/O boundary rounding code: $size > $zone_size. |
| 848 | test37() { |
| 849 | local bs off size capacity |
| 850 | |
| 851 | prep_write |
| 852 | capacity=$(total_zone_capacity 1 $((first_sequential_zone_sector*512)) $dev) |
| 853 | if [ "$first_sequential_zone_sector" = 0 ]; then |
| 854 | off=0 |
| 855 | else |
| 856 | off=$(((first_sequential_zone_sector - 1) * 512)) |
| 857 | fi |
| 858 | size=$((zone_size + 2 * 512)) |
| 859 | bs=$(min $((zone_size / 4)) "$zone_cap_bs") |
| 860 | run_one_fio_job --offset=$off --size=$size "$(ioengine "psync")" \ |
| 861 | --iodepth=1 --rw=write --do_verify=1 --verify=md5 \ |
| 862 | --bs=$bs --zonemode=zbd --zonesize="${zone_size}" \ |
| 863 | >> "${logfile}.${test_number}" 2>&1 |
| 864 | check_written $capacity || return $? |
| 865 | } |
| 866 | |
| 867 | # Test 4/4 for the I/O boundary rounding code: $offset > $disk_size - $zone_size |
| 868 | test38() { |
| 869 | local bs off size |
| 870 | |
| 871 | prep_write |
| 872 | size=$((logical_block_size)) |
| 873 | off=$((disk_size - logical_block_size)) |
| 874 | bs=$((logical_block_size)) |
| 875 | run_one_fio_job --offset=$off --size=$size "$(ioengine "psync")" \ |
| 876 | --iodepth=1 --rw=write --do_verify=1 --verify=md5 \ |
| 877 | --bs=$bs --zonemode=zbd --zonesize="${zone_size}" \ |
| 878 | >> "${logfile}.${test_number}" 2>&1 && return 1 |
| 879 | grep -q 'io_size must be at least one zone' "${logfile}.${test_number}" |
| 880 | } |
| 881 | |
| 882 | # Read one block from a block device. |
| 883 | read_one_block() { |
| 884 | local off |
| 885 | local bs |
| 886 | |
| 887 | if ! result=($(first_online_zone "$dev")); then |
| 888 | echo "Failed to determine first online zone" |
| 889 | exit 1 |
| 890 | fi |
| 891 | off=${result[0]} |
| 892 | bs=$((logical_block_size)) |
| 893 | run_one_fio_job --rw=read "$(ioengine "psync")" --offset=$off --bs=$bs \ |
| 894 | --size=$bs "$@" 2>&1 | |
| 895 | tee -a "${logfile}.${test_number}" |
| 896 | } |
| 897 | |
| 898 | # Check whether fio accepts --zonemode=none for zoned block devices. |
| 899 | test39() { |
| 900 | require_zbd || return $SKIP_TESTCASE |
| 901 | read_one_block --zonemode=none >/dev/null || return $? |
| 902 | check_read $((logical_block_size)) || return $? |
| 903 | } |
| 904 | |
| 905 | # Check whether fio accepts --zonemode=strided for zoned block devices. |
| 906 | test40() { |
| 907 | local bs |
| 908 | |
| 909 | bs=$((logical_block_size)) |
| 910 | require_zbd || return $SKIP_TESTCASE |
| 911 | read_one_block --zonemode=strided | |
| 912 | grep -q 'fio: --zonesize must be specified when using --zonemode=strided' || |
| 913 | return $? |
| 914 | read_one_block --zonemode=strided --zonesize=$bs >/dev/null || return $? |
| 915 | check_read $bs || return $? |
| 916 | } |
| 917 | |
| 918 | # Check whether fio checks the zone size for zoned block devices. |
| 919 | test41() { |
| 920 | require_zbd || return $SKIP_TESTCASE |
| 921 | read_one_block --zonemode=zbd --zonesize=$((2 * zone_size)) | |
| 922 | grep -q 'job parameter zonesize.*does not match disk zone size' |
| 923 | } |
| 924 | |
| 925 | # Check whether fio handles --zonesize=0 correctly for regular block devices. |
| 926 | test42() { |
| 927 | require_regular_block_dev || return $SKIP_TESTCASE |
| 928 | read_one_block --zonemode=zbd --zonesize=0 | |
| 929 | grep -q 'Specifying the zone size is mandatory for regular file/block device with --zonemode=zbd' |
| 930 | } |
| 931 | |
| 932 | # Check whether fio handles --zonesize=1 correctly for regular block devices. |
| 933 | test43() { |
| 934 | require_regular_block_dev || return $SKIP_TESTCASE |
| 935 | read_one_block --zonemode=zbd --zonesize=1 | |
| 936 | grep -q 'zone size must be at least 512 bytes for --zonemode=zbd' |
| 937 | } |
| 938 | |
| 939 | # Check whether fio handles --zonemode=none --zonesize=1 correctly. |
| 940 | test44() { |
| 941 | read_one_block --zonemode=none --zonesize=1 | |
| 942 | grep -q 'fio: --zonemode=none and --zonesize are not compatible' |
| 943 | } |
| 944 | |
| 945 | test45() { |
| 946 | local bs i |
| 947 | |
| 948 | require_zbd || return $SKIP_TESTCASE |
| 949 | prep_write |
| 950 | bs=$((logical_block_size)) |
| 951 | run_one_fio_job "$(ioengine "psync")" --iodepth=1 --rw=randwrite --bs=$bs\ |
| 952 | --offset=$((first_sequential_zone_sector * 512)) \ |
| 953 | --size="$zone_size" --do_verify=1 --verify=md5 2>&1 | |
| 954 | tee -a "${logfile}.${test_number}" | |
| 955 | grep -q "fio: first I/O failed. If .* is a zoned block device, consider --zonemode=zbd" |
| 956 | } |
| 957 | |
| 958 | # Random write to sequential zones, libaio, 8 jobs, queue depth 64 per job |
| 959 | test46() { |
| 960 | local size |
| 961 | |
| 962 | prep_write |
| 963 | size=$((4 * zone_size)) |
| 964 | run_fio_on_seq "$(ioengine "libaio")" --iodepth=64 --rw=randwrite --bs=4K \ |
| 965 | --group_reporting=1 --numjobs=8 \ |
| 966 | >> "${logfile}.${test_number}" 2>&1 || return $? |
| 967 | check_written $((size * 8)) || return $? |
| 968 | } |
| 969 | |
| 970 | # Check whether fio handles --zonemode=zbd --zoneskip=1 correctly. |
| 971 | test47() { |
| 972 | local bs |
| 973 | |
| 974 | prep_write |
| 975 | bs=$((logical_block_size)) |
| 976 | run_fio_on_seq "$(ioengine "psync")" --rw=write --bs=$bs --zoneskip=1 \ |
| 977 | >> "${logfile}.${test_number}" 2>&1 && return 1 |
| 978 | grep -q 'zoneskip 1 is not a multiple of the device zone size' "${logfile}.${test_number}" |
| 979 | } |
| 980 | |
| 981 | # Multiple overlapping random write jobs for the same drive and with a |
| 982 | # limited number of open zones. This is similar to test29, but uses libaio |
| 983 | # to stress test zone locking. |
| 984 | test48() { |
| 985 | local i jobs=16 off opts=() |
| 986 | |
| 987 | require_zbd || return $SKIP_TESTCASE |
| 988 | require_seq_zones 80 || return $SKIP_TESTCASE |
| 989 | |
| 990 | off=$((first_sequential_zone_sector * 512 + 64 * zone_size)) |
| 991 | size=$((16*zone_size)) |
| 992 | prep_write |
| 993 | opts=("--aux-path=/tmp" "--allow_file_create=0" "--significant_figures=10") |
| 994 | opts+=("--debug=zbd") |
| 995 | opts+=("$(ioengine "libaio")" "--rw=randwrite" "--direct=1") |
| 996 | opts+=("--time_based" "--runtime=30") |
| 997 | opts+=("--zonemode=zbd" "--zonesize=${zone_size}") |
| 998 | opts+=("--max_open_zones=4") |
| 999 | for ((i=0;i<jobs;i++)); do |
| 1000 | opts+=("--name=job$i" "--filename=$dev" "--offset=$off" "--bs=16K") |
| 1001 | opts+=("--io_size=$zone_size" "--iodepth=256" "--thread=1") |
| 1002 | opts+=("--size=$size" "--group_reporting=1") |
| 1003 | # max_open_zones is already specified |
| 1004 | opts+=($(job_var_opts_exclude "--max_open_zones")) |
| 1005 | done |
| 1006 | |
| 1007 | fio=$(dirname "$0")/../../fio |
| 1008 | |
| 1009 | { echo; echo "fio ${opts[*]}"; echo; } >>"${logfile}.${test_number}" |
| 1010 | |
| 1011 | timeout -v -s KILL 180s \ |
| 1012 | "${dynamic_analyzer[@]}" "$fio" "${opts[@]}" \ |
| 1013 | >> "${logfile}.${test_number}" 2>&1 || return $? |
| 1014 | } |
| 1015 | |
| 1016 | # Check if fio handles --zonecapacity on a normal block device correctly |
| 1017 | test49() { |
| 1018 | |
| 1019 | require_regular_block_dev || return $SKIP_TESTCASE |
| 1020 | |
| 1021 | size=$((2 * zone_size)) |
| 1022 | capacity=$((zone_size * 3 / 4)) |
| 1023 | |
| 1024 | run_one_fio_job "$(ioengine "psync")" --rw=write \ |
| 1025 | --zonemode=zbd --zonesize="${zone_size}" \ |
| 1026 | --zonecapacity=${capacity} \ |
| 1027 | --verify=md5 --size=${size} >>"${logfile}.${test_number}" 2>&1 || |
| 1028 | return $? |
| 1029 | check_written $((capacity * 2)) || return $? |
| 1030 | check_read $((capacity * 2)) || return $? |
| 1031 | } |
| 1032 | |
| 1033 | # Verify that conv zones are not locked and only seq zones are locked during |
| 1034 | # random read on conv-seq mixed zones. |
| 1035 | test50() { |
| 1036 | local off |
| 1037 | |
| 1038 | require_zbd || return $SKIP_TESTCASE |
| 1039 | require_conv_zones 8 || return $SKIP_TESTCASE |
| 1040 | require_seq_zones 8 || return $SKIP_TESTCASE |
| 1041 | |
| 1042 | reset_zone "${dev}" -1 |
| 1043 | |
| 1044 | off=$((first_sequential_zone_sector * 512 - 8 * zone_size)) |
| 1045 | run_fio --name=job --filename=${dev} --offset=${off} --bs=64K \ |
| 1046 | --size=$((16 * zone_size)) "$(ioengine "libaio")" --rw=randread\ |
| 1047 | --time_based --runtime=3 --zonemode=zbd --zonesize=${zone_size}\ |
| 1048 | --direct=1 --group_reporting=1 ${job_var_opts[@]} \ |
| 1049 | >> "${logfile}.${test_number}" 2>&1 || return $? |
| 1050 | } |
| 1051 | |
| 1052 | # Verify that conv zones are neither locked nor opened during random write on |
| 1053 | # conv-seq mixed zones. Zone lock and zone open shall happen only on seq zones. |
| 1054 | test51() { |
| 1055 | local off jobs=16 |
| 1056 | local -a opts |
| 1057 | |
| 1058 | require_zbd || return $SKIP_TESTCASE |
| 1059 | require_conv_zones 8 || return $SKIP_TESTCASE |
| 1060 | require_seq_zones 8 || return $SKIP_TESTCASE |
| 1061 | |
| 1062 | prep_write |
| 1063 | |
| 1064 | off=$((first_sequential_zone_sector * 512 - 8 * zone_size)) |
| 1065 | opts+=("--size=$((16 * zone_size))" "$(ioengine "libaio")") |
| 1066 | opts+=("--zonemode=zbd" "--direct=1" "--zonesize=${zone_size}") |
| 1067 | opts+=("--max_open_zones=2" "--offset=$off") |
| 1068 | opts+=("--thread=1" "--group_reporting=1") |
| 1069 | opts+=("--time_based" "--runtime=30" "--rw=randwrite") |
| 1070 | for ((i=0;i<jobs;i++)); do |
| 1071 | opts+=("--name=job${i}" "--filename=$dev") |
| 1072 | opts+=("--bs=$(((i+1)*16))K") |
| 1073 | opts+=($(job_var_opts_exclude "--max_open_zones")) |
| 1074 | done |
| 1075 | run_fio "${opts[@]}" >> "${logfile}.${test_number}" 2>&1 || return $? |
| 1076 | } |
| 1077 | |
| 1078 | # Verify that zone_reset_threshold only takes logical blocks from seq |
| 1079 | # zones into account, and logical blocks of conv zones are not counted. |
| 1080 | test52() { |
| 1081 | local off io_size |
| 1082 | |
| 1083 | require_zbd || return $SKIP_TESTCASE |
| 1084 | require_conv_zones 8 || return $SKIP_TESTCASE |
| 1085 | require_seq_zones 8 || return $SKIP_TESTCASE |
| 1086 | |
| 1087 | reset_zone "${dev}" -1 |
| 1088 | |
| 1089 | # Total I/O size is 1/8 = 0.125 of the I/O range of cont + seq zones. |
| 1090 | # Set zone_reset_threshold as 0.1. The threshold size is less than |
| 1091 | # 0.125, then, reset count zero is expected. |
| 1092 | # On the other hand, half of the I/O range is covered by conv zones. |
| 1093 | # If fio would count the conv zones for zone_reset_threshold, the ratio |
| 1094 | # were more than 0.5 and would trigger zone resets. |
| 1095 | |
| 1096 | off=$((first_sequential_zone_sector * 512 - 8 * zone_size)) |
| 1097 | io_size=$((zone_size * 16 / 8)) |
| 1098 | run_fio --name=job --filename=$dev --rw=randwrite --bs=$((zone_size/16))\ |
| 1099 | --size=$((zone_size * 16)) --softrandommap=1 \ |
| 1100 | --io_size=$((io_size)) "$(ioengine "psync")" --offset=$off \ |
| 1101 | --zonemode=zbd --direct=1 --zonesize=${zone_size} \ |
| 1102 | --zone_reset_threshold=.1 --zone_reset_frequency=1.0 \ |
| 1103 | ${job_var_opts[@]} --debug=zbd \ |
| 1104 | >> "${logfile}.${test_number}" 2>&1 || return $? |
| 1105 | |
| 1106 | check_written ${io_size} || return $? |
| 1107 | check_reset_count -eq 0 || return $? |
| 1108 | } |
| 1109 | |
| 1110 | # Check both reads and writes are executed by random I/O to conventional zones. |
| 1111 | test53() { |
| 1112 | local off capacity io read_b=0 written_b=0 |
| 1113 | |
| 1114 | require_zbd || return $SKIP_TESTCASE |
| 1115 | require_conv_zones 4 || return $SKIP_TESTCASE |
| 1116 | |
| 1117 | off=$((first_sequential_zone_sector * 512 - 4 * zone_size)) |
| 1118 | capacity=$(total_zone_capacity 4 $off $dev) |
| 1119 | run_fio --name=job --filename=${dev} --rw=randrw --bs=64K \ |
| 1120 | --size=$((4 * zone_size)) "$(ioengine "psync")" --offset=${off}\ |
| 1121 | --zonemode=zbd --direct=1 --zonesize=${zone_size} \ |
| 1122 | ${job_var_opts[@]} \ |
| 1123 | >> "${logfile}.${test_number}" 2>&1 || return $? |
| 1124 | |
| 1125 | written_b=$(fio_written <"${logfile}.${test_number}") |
| 1126 | read_b=$(fio_read <"${logfile}.${test_number}") |
| 1127 | io=$((written_b + read_b)) |
| 1128 | echo "Number of bytes read: $read_b" >>"${logfile}.${test_number}" |
| 1129 | echo "Number of bytes written: $written_b" >>"${logfile}.${test_number}" |
| 1130 | echo "Total number of bytes read and written: $io <> $capacity" \ |
| 1131 | >>"${logfile}.${test_number}" |
| 1132 | if ((io==capacity && written_b != 0 && read_b != 0)); then |
| 1133 | return 0 |
| 1134 | fi |
| 1135 | return 1 |
| 1136 | } |
| 1137 | |
| 1138 | # Test read/write mix with verify. |
| 1139 | test54() { |
| 1140 | require_zbd || return $SKIP_TESTCASE |
| 1141 | require_seq_zones 8 || return $SKIP_TESTCASE |
| 1142 | |
| 1143 | run_fio --name=job --filename=${dev} "$(ioengine "libaio")" \ |
| 1144 | --time_based=1 --runtime=30s --continue_on_error=0 \ |
| 1145 | --offset=$((first_sequential_zone_sector * 512)) \ |
| 1146 | --size=$((8*zone_size)) --direct=1 --iodepth=1 \ |
| 1147 | --rw=randrw:2 --rwmixwrite=25 --bsrange=4k-${zone_size} \ |
| 1148 | --zonemode=zbd --zonesize=${zone_size} \ |
| 1149 | --verify=crc32c --do_verify=1 --verify_backlog=2 \ |
| 1150 | --experimental_verify=1 \ |
| 1151 | --alloc-size=65536 --random_generator=tausworthe64 \ |
| 1152 | ${job_var_opts[@]} --debug=zbd \ |
| 1153 | >> "${logfile}.${test_number}" 2>&1 || return $? |
| 1154 | } |
| 1155 | |
| 1156 | # test 'z' suffix parsing only |
| 1157 | test55() { |
| 1158 | local bs |
| 1159 | bs=$((logical_block_size)) |
| 1160 | |
| 1161 | require_zbd || return $SKIP_TESTCASE |
| 1162 | # offset=1z + offset_increment=10z + size=2z |
| 1163 | require_seq_zones 13 || return $SKIP_TESTCASE |
| 1164 | |
| 1165 | run_fio --name=j \ |
| 1166 | --filename=${dev} \ |
| 1167 | --direct=1 \ |
| 1168 | "$(ioengine "psync")" \ |
| 1169 | --zonemode=zbd \ |
| 1170 | --zonesize=${zone_size} \ |
| 1171 | --rw=write \ |
| 1172 | --bs=${bs} \ |
| 1173 | --numjobs=2 \ |
| 1174 | --offset_increment=10z \ |
| 1175 | --offset=1z \ |
| 1176 | --size=2z \ |
| 1177 | --io_size=3z \ |
| 1178 | ${job_var_opts[@]} --debug=zbd \ |
| 1179 | >> "${logfile}.${test_number}" 2>&1 || return $? |
| 1180 | } |
| 1181 | |
| 1182 | # test 'z' suffix parsing only |
| 1183 | test56() { |
| 1184 | local bs |
| 1185 | bs=$((logical_block_size)) |
| 1186 | |
| 1187 | require_regular_block_dev || return $SKIP_TESTCASE |
| 1188 | require_seq_zones 10 || return $SKIP_TESTCASE |
| 1189 | |
| 1190 | run_fio --name=j \ |
| 1191 | --filename=${dev} \ |
| 1192 | --direct=1 \ |
| 1193 | "$(ioengine "psync")" \ |
| 1194 | --zonemode=strided \ |
| 1195 | --zonesize=${zone_size} \ |
| 1196 | --rw=write \ |
| 1197 | --bs=${bs} \ |
| 1198 | --size=10z \ |
| 1199 | --zoneskip=2z \ |
| 1200 | ${job_var_opts[@]} --debug=zbd \ |
| 1201 | >> "${logfile}.${test_number}" 2>&1 || return $? |
| 1202 | } |
| 1203 | |
| 1204 | # Test that repeated async write job does not cause zone reset during writes |
| 1205 | # in-flight, when the block size is not a divisor of the zone size. |
| 1206 | test57() { |
| 1207 | local bs off |
| 1208 | |
| 1209 | require_zbd || return $SKIP_TESTCASE |
| 1210 | |
| 1211 | bs=$((4096 * 7)) |
| 1212 | off=$((first_sequential_zone_sector * 512)) |
| 1213 | |
| 1214 | run_fio --name=job --filename="${dev}" --rw=randwrite --bs="${bs}" \ |
| 1215 | --offset="${off}" --size=$((4 * zone_size)) --iodepth=256 \ |
| 1216 | "$(ioengine "libaio")" --time_based=1 --runtime=30s \ |
| 1217 | --zonemode=zbd --direct=1 --zonesize="${zone_size}" \ |
| 1218 | ${job_var_opts[@]} \ |
| 1219 | >> "${logfile}.${test_number}" 2>&1 || return $? |
| 1220 | } |
| 1221 | |
| 1222 | # Random writes and random trims to sequential write required zones for 30s. |
| 1223 | test58() { |
| 1224 | local off size bs |
| 1225 | |
| 1226 | require_seq_zones 128 || return $SKIP_TESTCASE |
| 1227 | |
| 1228 | size=$((zone_size * 128)) |
| 1229 | bs="$(max $((zone_size / 128)) "$logical_block_size")" |
| 1230 | prep_write |
| 1231 | off=$((first_sequential_zone_sector * 512)) |
| 1232 | run_fio --zonemode=zbd --direct=1 --zonesize="${zone_size}" --thread=1 \ |
| 1233 | --filename="${dev}" --norandommap=1 \ |
| 1234 | --name="precondition" --rw=write "$(ioengine "psync")" \ |
| 1235 | --offset="${off}" --size=$((zone_size * 16)) --bs="${bs}" \ |
| 1236 | "${job_var_opts[@]}" \ |
| 1237 | --name=wjob --wait_for="precondition" --rw=randwrite \ |
| 1238 | "$(ioengine "libaio")" --iodepth=8 \ |
| 1239 | --offset="${off}" --size="${size}" --bs="${bs}" \ |
| 1240 | --time_based --runtime=30s --flow=128 "${job_var_opts[@]}" \ |
| 1241 | --name=trimjob --wait_for="precondition" --rw=randtrim \ |
| 1242 | "$(ioengine "psync")" \ |
| 1243 | --offset="${off}" --size="${size}" --bs="${zone_size}" \ |
| 1244 | --time_based --runtime=30s --flow=1 "${job_var_opts[@]}" \ |
| 1245 | >>"${logfile}.${test_number}" 2>&1 |
| 1246 | } |
| 1247 | |
| 1248 | SECONDS=0 |
| 1249 | tests=() |
| 1250 | dynamic_analyzer=() |
| 1251 | reset_all_zones= |
| 1252 | reset_before_write= |
| 1253 | use_libzbc= |
| 1254 | zbd_debug= |
| 1255 | max_open_zones_opt= |
| 1256 | quit_on_err= |
| 1257 | |
| 1258 | while [ "${1#-}" != "$1" ]; do |
| 1259 | case "$1" in |
| 1260 | -d) dynamic_analyzer=(valgrind "--read-var-info=yes" "--tool=drd" |
| 1261 | "--show-confl-seg=no"); |
| 1262 | shift;; |
| 1263 | -e) dynamic_analyzer=(valgrind "--read-var-info=yes" "--tool=helgrind"); |
| 1264 | shift;; |
| 1265 | -l) use_libzbc=1; shift;; |
| 1266 | -r) reset_all_zones=1; shift;; |
| 1267 | -w) reset_before_write=1; shift;; |
| 1268 | -t) tests+=("$2"); shift; shift;; |
| 1269 | -o) max_open_zones_opt="${2}"; shift; shift;; |
| 1270 | -v) dynamic_analyzer=(valgrind "--read-var-info=yes"); |
| 1271 | shift;; |
| 1272 | -q) quit_on_err=1; shift;; |
| 1273 | -z) zbd_debug=1; shift;; |
| 1274 | --) shift; break;; |
| 1275 | *) usage; exit 1;; |
| 1276 | esac |
| 1277 | done |
| 1278 | |
| 1279 | if [ $# != 1 ]; then |
| 1280 | usage |
| 1281 | exit 1 |
| 1282 | fi |
| 1283 | |
| 1284 | # shellcheck source=functions |
| 1285 | source "$(dirname "$0")/functions" || exit $? |
| 1286 | |
| 1287 | global_var_opts=() |
| 1288 | job_var_opts=() |
| 1289 | if [ -n "$zbd_debug" ]; then |
| 1290 | global_var_opts+=("--debug=zbd") |
| 1291 | fi |
| 1292 | dev=$1 |
| 1293 | realdev=$(readlink -f "$dev") |
| 1294 | basename=$(basename "$realdev") |
| 1295 | |
| 1296 | if [[ -b "$realdev" ]]; then |
| 1297 | major=$((0x$(stat -L -c '%t' "$realdev"))) || exit $? |
| 1298 | minor=$((0x$(stat -L -c '%T' "$realdev"))) || exit $? |
| 1299 | disk_size=$(($(<"/sys/dev/block/$major:$minor/size")*512)) |
| 1300 | |
| 1301 | # When the target is a partition device, get basename of its |
| 1302 | # holder device to access sysfs path of the holder device |
| 1303 | if [[ -r "/sys/dev/block/$major:$minor/partition" ]]; then |
| 1304 | realsysfs=$(readlink "/sys/dev/block/$major:$minor") |
| 1305 | basename=$(basename "${realsysfs%/*}") |
| 1306 | fi |
| 1307 | logical_block_size=$(<"/sys/block/$basename/queue/logical_block_size") |
| 1308 | case "$(<"/sys/class/block/$basename/queue/zoned")" in |
| 1309 | host-managed|host-aware) |
| 1310 | is_zbd=true |
| 1311 | if ! check_blkzone "${dev}"; then |
| 1312 | exit 1 |
| 1313 | fi |
| 1314 | if ! result=($(first_sequential_zone "$dev")); then |
| 1315 | echo "Failed to determine first sequential zone" |
| 1316 | exit 1 |
| 1317 | fi |
| 1318 | first_sequential_zone_sector=${result[0]} |
| 1319 | sectors_per_zone=${result[1]} |
| 1320 | zone_size=$((sectors_per_zone * 512)) |
| 1321 | if ! max_open_zones=$(max_open_zones "$dev"); then |
| 1322 | echo "Failed to determine maximum number of open zones" |
| 1323 | exit 1 |
| 1324 | fi |
| 1325 | set_io_scheduler "$basename" deadline || exit $? |
| 1326 | if [ -n "$reset_all_zones" ]; then |
| 1327 | reset_zone "$dev" -1 |
| 1328 | fi |
| 1329 | ;; |
| 1330 | *) |
| 1331 | first_sequential_zone_sector=$(((disk_size / 2) & |
| 1332 | (logical_block_size - 1))) |
| 1333 | zone_size=$(max 65536 "$logical_block_size") |
| 1334 | sectors_per_zone=$((zone_size / 512)) |
| 1335 | max_open_zones=128 |
| 1336 | set_io_scheduler "$basename" none || exit $? |
| 1337 | ;; |
| 1338 | esac |
| 1339 | elif [[ -c "$realdev" ]]; then |
| 1340 | # For an SG node, we must have libzbc option specified |
| 1341 | if [[ ! -n "$use_libzbc" ]]; then |
| 1342 | echo "Character device files can only be used with -l (libzbc) option" |
| 1343 | exit 1 |
| 1344 | fi |
| 1345 | |
| 1346 | if ! $(is_zbc "$dev"); then |
| 1347 | echo "Device is not a ZBC disk" |
| 1348 | exit 1 |
| 1349 | fi |
| 1350 | is_zbd=true |
| 1351 | |
| 1352 | if ! disk_size=($(( $(zbc_disk_sectors "$dev") * 512))); then |
| 1353 | echo "Failed to determine disk size" |
| 1354 | exit 1 |
| 1355 | fi |
| 1356 | if ! logical_block_size=($(zbc_logical_block_size "$dev")); then |
| 1357 | echo "Failed to determine logical block size" |
| 1358 | exit 1 |
| 1359 | fi |
| 1360 | if ! result=($(first_sequential_zone "$dev")); then |
| 1361 | echo "Failed to determine first sequential zone" |
| 1362 | exit 1 |
| 1363 | fi |
| 1364 | first_sequential_zone_sector=${result[0]} |
| 1365 | sectors_per_zone=${result[1]} |
| 1366 | zone_size=$((sectors_per_zone * 512)) |
| 1367 | if ! max_open_zones=$(max_open_zones "$dev"); then |
| 1368 | echo "Failed to determine maximum number of open zones" |
| 1369 | exit 1 |
| 1370 | fi |
| 1371 | if [ -n "$reset_all_zones" ]; then |
| 1372 | reset_zone "$dev" -1 |
| 1373 | fi |
| 1374 | fi |
| 1375 | |
| 1376 | if [[ -n ${max_open_zones_opt} ]]; then |
| 1377 | # Override max_open_zones with the script option value |
| 1378 | max_open_zones="${max_open_zones_opt}" |
| 1379 | global_var_opts+=("--ignore_zone_limits=1") |
| 1380 | job_var_opts+=("--max_open_zones=${max_open_zones_opt}") |
| 1381 | fi |
| 1382 | |
| 1383 | echo -n "First sequential zone starts at sector $first_sequential_zone_sector;" |
| 1384 | echo " zone size: $((zone_size >> 20)) MB" |
| 1385 | |
| 1386 | zone_cap_bs=$(zone_cap_bs "$dev" "$zone_size") |
| 1387 | |
| 1388 | if [ "${#tests[@]}" = 0 ]; then |
| 1389 | readarray -t tests < <(declare -F | grep "test[0-9]*" | \ |
| 1390 | tr -c -d "[:digit:]\n" | sort -n) |
| 1391 | fi |
| 1392 | |
| 1393 | logfile=$0.log |
| 1394 | |
| 1395 | passed=0 |
| 1396 | skipped=0 |
| 1397 | failed=0 |
| 1398 | if [ -t 1 ]; then |
| 1399 | red="\e[1;31m" |
| 1400 | green="\e[1;32m" |
| 1401 | cyan="\e[1;36m" |
| 1402 | end="\e[m" |
| 1403 | else |
| 1404 | red="" |
| 1405 | green="" |
| 1406 | end="" |
| 1407 | fi |
| 1408 | rc=0 |
| 1409 | |
| 1410 | intr=0 |
| 1411 | trap 'intr=1' SIGINT |
| 1412 | ret=0 |
| 1413 | |
| 1414 | for test_number in "${tests[@]}"; do |
| 1415 | rm -f "${logfile}.${test_number}" |
| 1416 | unset SKIP_REASON |
| 1417 | echo -n "Running test $(printf "%02d" $test_number) ... " |
| 1418 | eval "test$test_number" |
| 1419 | ret=$? |
| 1420 | if ((!ret)) && check_log $test_number; then |
| 1421 | status="PASS" |
| 1422 | cc_status="${green}${status}${end}" |
| 1423 | ((passed++)) |
| 1424 | elif ((ret==SKIP_TESTCASE)); then |
| 1425 | status="SKIP" |
| 1426 | echo "${SKIP_REASON}" >> "${logfile}.${test_number}" |
| 1427 | cc_status="${cyan}${status}${end} ${SKIP_REASON}" |
| 1428 | ((skipped++)) |
| 1429 | else |
| 1430 | status="FAIL" |
| 1431 | cc_status="${red}${status}${end}" |
| 1432 | ((failed++)) |
| 1433 | rc=1 |
| 1434 | fi |
| 1435 | echo -e "$cc_status" |
| 1436 | echo "$status" >> "${logfile}.${test_number}" |
| 1437 | [ $intr -ne 0 ] && exit 1 |
| 1438 | [ -n "$quit_on_err" -a "$rc" -ne 0 ] && exit 1 |
| 1439 | done |
| 1440 | |
| 1441 | echo "$passed tests passed" |
| 1442 | if [ $skipped -gt 0 ]; then |
| 1443 | echo " $skipped tests skipped" |
| 1444 | fi |
| 1445 | if [ $failed -gt 0 ]; then |
| 1446 | echo " $failed tests failed" |
| 1447 | fi |
| 1448 | echo "Run time: $(TZ=UTC0 printf "%(%H:%M:%S)T\n" $(( SECONDS )) )" |
| 1449 | exit $rc |