t/run-fio-tests.py

   1 #!/usr/bin/env python3
   2 # SPDX-License-Identifier: GPL-2.0-only
   3 #
   4 # Copyright (c) 2019 Western Digital Corporation or its affiliates.
   5 #
   6 """
   7 # run-fio-tests.py
   8 #
   9 # Automate running of fio tests
  10 #
  11 # USAGE
  12 # python3 run-fio-tests.py [-r fio-root] [-f fio-path] [-a artifact-root]
  13 #                           [--skip # # #...] [--run-only # # #...]
  14 #
  15 #
  16 # EXAMPLE
  17 # # git clone git://git.kernel.dk/fio.git
  18 # # cd fio
  19 # # make -j
  20 # # python3 t/run-fio-tests.py
  21 #
  22 #
  23 # REQUIREMENTS
  24 # - Python 3.5 (subprocess.run)
  25 # - Linux (libaio ioengine, zbd tests, etc)
  26 # - The artifact directory must be on a file system that accepts 512-byte IO
  27 #   (t0002, t0003, t0004).
  28 # - The artifact directory needs to be on an SSD. Otherwise tests that carry
  29 #   out file-based IO will trigger a timeout (t0006).
  30 # - 4 CPUs (t0009)
  31 # - SciPy (steadystate_tests.py)
  32 # - libzbc (zbd tests)
  33 # - root privileges (zbd test)
  34 # - kernel 4.19 or later for zoned null block devices (zbd tests)
  35 # - CUnit support (unittests)
  36 #
  37 """
  38
  39 #
  40 # TODO  run multiple tests simultaneously
  41 # TODO  Add sgunmap tests (requires SAS SSD)
  42 #
  43
  44 import os
  45 import sys
  46 import json
  47 import time
  48 import shutil
  49 import logging
  50 import argparse
  51 import platform
  52 import subprocess
  53 import multiprocessing
  54 from pathlib import Path
  55
  56
  57 class FioTest(object):
  58     """Base for all fio tests."""
  59
  60     def __init__(self, exe_path, parameters, success):
  61         self.exe_path = exe_path
  62         self.parameters = parameters
  63         self.success = success
  64         self.output = {}
  65         self.artifact_root = None
  66         self.testnum = None
  67         self.test_dir = None
  68         self.passed = True
  69         self.failure_reason = ''
  70
  71     def setup(self, artifact_root, testnum):
  72         self.artifact_root = artifact_root
  73         self.testnum = testnum
  74         self.test_dir = os.path.join(artifact_root, "{:04d}".format(testnum))
  75         if not os.path.exists(self.test_dir):
  76             os.mkdir(self.test_dir)
  77
  78         self.command_file = os.path.join(
  79                 self.test_dir,
  80                 "{0}.command".format(os.path.basename(self.exe_path)))
  81         self.stdout_file = os.path.join(
  82                 self.test_dir,
  83                 "{0}.stdout".format(os.path.basename(self.exe_path)))
  84         self.stderr_file = os.path.join(
  85                 self.test_dir,
  86                 "{0}.stderr".format(os.path.basename(self.exe_path)))
  87         self.exticode_file = os.path.join(
  88                 self.test_dir,
  89                 "{0}.exitcode".format(os.path.basename(self.exe_path)))
  90
  91     def run(self):
  92         raise NotImplementedError()
  93
  94     def check_result(self):
  95         raise NotImplementedError()
  96
  97
  98 class FioExeTest(FioTest):
  99     """Test consists of an executable binary or script"""
 100
 101     def __init__(self, exe_path, parameters, success):
 102         """Construct a FioExeTest which is a FioTest consisting of an
 103         executable binary or script.
 104
 105         exe_path:       location of executable binary or script
 106         parameters:     list of parameters for executable
 107         success:        Definition of test success
 108         """
 109
 110         FioTest.__init__(self, exe_path, parameters, success)
 111
 112     def setup(self, artifact_root, testnum):
 113         super(FioExeTest, self).setup(artifact_root, testnum)
 114
 115     def run(self):
 116         if self.parameters:
 117             command = [self.exe_path] + self.parameters
 118         else:
 119             command = [self.exe_path]
 120         command_file = open(self.command_file, "w+")
 121         command_file.write("%s\n" % command)
 122         command_file.close()
 123
 124         stdout_file = open(self.stdout_file, "w+")
 125         stderr_file = open(self.stderr_file, "w+")
 126         exticode_file = open(self.exticode_file, "w+")
 127         try:
 128             proc = None
 129             # Avoid using subprocess.run() here because when a timeout occurs,
 130             # fio will be stopped with SIGKILL. This does not give fio a
 131             # chance to clean up and means that child processes may continue
 132             # running and submitting IO.
 133             proc = subprocess.Popen(command,
 134                                     stdout=stdout_file,
 135                                     stderr=stderr_file,
 136                                     cwd=self.test_dir,
 137                                     universal_newlines=True)
 138             proc.communicate(timeout=self.success['timeout'])
 139             exticode_file.write('{0}\n'.format(proc.returncode))
 140             logging.debug("return code: %d" % proc.returncode)
 141             self.output['proc'] = proc
 142         except subprocess.TimeoutExpired:
 143             proc.terminate()
 144             proc.communicate()
 145             assert proc.poll()
 146             self.output['failure'] = 'timeout'
 147         except Exception:
 148             if proc:
 149                 if not proc.poll():
 150                     proc.terminate()
 151                     proc.communicate()
 152             self.output['failure'] = 'exception'
 153             self.output['exc_info'] = sys.exc_info()
 154         finally:
 155             stdout_file.close()
 156             stderr_file.close()
 157             exticode_file.close()
 158
 159     def check_result(self):
 160         if 'proc' not in self.output:
 161             if self.output['failure'] == 'timeout':
 162                 self.failure_reason = "{0} timeout,".format(self.failure_reason)
 163             else:
 164                 assert self.output['failure'] == 'exception'
 165                 self.failure_reason = '{0} exception: {1}, {2}'.format(
 166                         self.failure_reason, self.output['exc_info'][0],
 167                         self.output['exc_info'][1])
 168
 169             self.passed = False
 170             return
 171
 172         if 'zero_return' in self.success:
 173             if self.success['zero_return']:
 174                 if self.output['proc'].returncode != 0:
 175                     self.passed = False
 176                     self.failure_reason = "{0} non-zero return code,".format(self.failure_reason)
 177             else:
 178                 if self.output['proc'].returncode == 0:
 179                     self.failure_reason = "{0} zero return code,".format(self.failure_reason)
 180                     self.passed = False
 181
 182         stderr_size = os.path.getsize(self.stderr_file)
 183         if 'stderr_empty' in self.success:
 184             if self.success['stderr_empty']:
 185                 if stderr_size != 0:
 186                     self.failure_reason = "{0} stderr not empty,".format(self.failure_reason)
 187                     self.passed = False
 188             else:
 189                 if stderr_size == 0:
 190                     self.failure_reason = "{0} stderr empty,".format(self.failure_reason)
 191                     self.passed = False
 192
 193
 194 class FioJobTest(FioExeTest):
 195     """Test consists of a fio job"""
 196
 197     def __init__(self, fio_path, fio_job, success, fio_pre_job=None,
 198                  fio_pre_success=None, output_format="normal"):
 199         """Construct a FioJobTest which is a FioExeTest consisting of a
 200         single fio job file with an optional setup step.
 201
 202         fio_path:           location of fio executable
 203         fio_job:            location of fio job file
 204         success:            Definition of test success
 205         fio_pre_job:        fio job for preconditioning
 206         fio_pre_success:    Definition of test success for fio precon job
 207         output_format:      normal (default), json, jsonplus, or terse
 208         """
 209
 210         self.fio_job = fio_job
 211         self.fio_pre_job = fio_pre_job
 212         self.fio_pre_success = fio_pre_success if fio_pre_success else success
 213         self.output_format = output_format
 214         self.precon_failed = False
 215         self.json_data = None
 216         self.fio_output = "{0}.output".format(os.path.basename(self.fio_job))
 217         self.fio_args = [
 218             "--output-format={0}".format(self.output_format),
 219             "--output={0}".format(self.fio_output),
 220             self.fio_job,
 221             ]
 222         FioExeTest.__init__(self, fio_path, self.fio_args, success)
 223
 224     def setup(self, artifact_root, testnum):
 225         super(FioJobTest, self).setup(artifact_root, testnum)
 226
 227         self.command_file = os.path.join(
 228                 self.test_dir,
 229                 "{0}.command".format(os.path.basename(self.fio_job)))
 230         self.stdout_file = os.path.join(
 231                 self.test_dir,
 232                 "{0}.stdout".format(os.path.basename(self.fio_job)))
 233         self.stderr_file = os.path.join(
 234                 self.test_dir,
 235                 "{0}.stderr".format(os.path.basename(self.fio_job)))
 236         self.exticode_file = os.path.join(
 237                 self.test_dir,
 238                 "{0}.exitcode".format(os.path.basename(self.fio_job)))
 239
 240     def run_pre_job(self):
 241         precon = FioJobTest(self.exe_path, self.fio_pre_job,
 242                             self.fio_pre_success,
 243                             output_format=self.output_format)
 244         precon.setup(self.artifact_root, self.testnum)
 245         precon.run()
 246         precon.check_result()
 247         self.precon_failed = not precon.passed
 248         self.failure_reason = precon.failure_reason
 249
 250     def run(self):
 251         if self.fio_pre_job:
 252             self.run_pre_job()
 253
 254         if not self.precon_failed:
 255             super(FioJobTest, self).run()
 256         else:
 257             logging.debug("precondition step failed")
 258
 259     def check_result(self):
 260         if self.precon_failed:
 261             self.passed = False
 262             self.failure_reason = "{0} precondition step failed,".format(self.failure_reason)
 263             return
 264
 265         super(FioJobTest, self).check_result()
 266
 267         if not self.passed:
 268             return
 269
 270         if 'json' in self.output_format:
 271             try:
 272                 with open(os.path.join(self.test_dir, self.fio_output), "r") as output_file:
 273                     file_data = output_file.read()
 274             except EnvironmentError:
 275                 self.failure_reason = "{0} unable to open output file,".format(self.failure_reason)
 276                 self.passed = False
 277             else:
 278                 try:
 279                     self.json_data = json.loads(file_data)
 280                 except json.JSONDecodeError:
 281                     self.failure_reason = "{0} unable to decode JSON data,".format(self.failure_reason)
 282                     self.passed = False
 283
 284
 285 class FioJobTest_t0005(FioJobTest):
 286     """Test consists of fio test job t0005
 287     Confirm that read['io_kbytes'] == write['io_kbytes'] == 102400"""
 288
 289     def check_result(self):
 290         super(FioJobTest_t0005, self).check_result()
 291
 292         if not self.passed:
 293             return
 294
 295         if self.json_data['jobs'][0]['read']['io_kbytes'] != 102400:
 296             self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
 297             self.passed = False
 298         if self.json_data['jobs'][0]['write']['io_kbytes'] != 102400:
 299             self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
 300             self.passed = False
 301
 302
 303 class FioJobTest_t0006(FioJobTest):
 304     """Test consists of fio test job t0006
 305     Confirm that read['io_kbytes'] ~ 2*write['io_kbytes']"""
 306
 307     def check_result(self):
 308         super(FioJobTest_t0006, self).check_result()
 309
 310         if not self.passed:
 311             return
 312
 313         ratio = self.json_data['jobs'][0]['read']['io_kbytes'] \
 314             / self.json_data['jobs'][0]['write']['io_kbytes']
 315         logging.debug("ratio: %f" % ratio)
 316         if ratio < 1.99 or ratio > 2.01:
 317             self.failure_reason = "{0} read/write ratio mismatch,".format(self.failure_reason)
 318             self.passed = False
 319
 320
 321 class FioJobTest_t0007(FioJobTest):
 322     """Test consists of fio test job t0007
 323     Confirm that read['io_kbytes'] = 87040"""
 324
 325     def check_result(self):
 326         super(FioJobTest_t0007, self).check_result()
 327
 328         if not self.passed:
 329             return
 330
 331         if self.json_data['jobs'][0]['read']['io_kbytes'] != 87040:
 332             self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
 333             self.passed = False
 334
 335
 336 class FioJobTest_t0008(FioJobTest):
 337     """Test consists of fio test job t0008
 338     Confirm that read['io_kbytes'] = 32768 and that
 339                 write['io_kbytes'] ~ 16568
 340
 341     I did runs with fio-ae2fafc8 and saw write['io_kbytes'] values of
 342     16585, 16588. With two runs of fio-3.16 I obtained 16568"""
 343
 344     def check_result(self):
 345         super(FioJobTest_t0008, self).check_result()
 346
 347         if not self.passed:
 348             return
 349
 350         ratio = self.json_data['jobs'][0]['write']['io_kbytes'] / 16568
 351         logging.debug("ratio: %f" % ratio)
 352
 353         if ratio < 0.99 or ratio > 1.01:
 354             self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
 355             self.passed = False
 356         if self.json_data['jobs'][0]['read']['io_kbytes'] != 32768:
 357             self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
 358             self.passed = False
 359
 360
 361 class FioJobTest_t0009(FioJobTest):
 362     """Test consists of fio test job t0009
 363     Confirm that runtime >= 60s"""
 364
 365     def check_result(self):
 366         super(FioJobTest_t0009, self).check_result()
 367
 368         if not self.passed:
 369             return
 370
 371         logging.debug('elapsed: %d' % self.json_data['jobs'][0]['elapsed'])
 372
 373         if self.json_data['jobs'][0]['elapsed'] < 60:
 374             self.failure_reason = "{0} elapsed time mismatch,".format(self.failure_reason)
 375             self.passed = False
 376
 377
 378 class FioJobTest_t0011(FioJobTest):
 379     """Test consists of fio test job t0009
 380     Confirm that job0 iops == 1000
 381     and that job1_iops / job0_iops ~ 8
 382     With two runs of fio-3.16 I observed a ratio of 8.3"""
 383
 384     def check_result(self):
 385         super(FioJobTest_t0011, self).check_result()
 386
 387         if not self.passed:
 388             return
 389
 390         iops1 = self.json_data['jobs'][0]['read']['iops']
 391         iops2 = self.json_data['jobs'][1]['read']['iops']
 392         ratio = iops2 / iops1
 393         logging.debug("ratio: %f" % ratio)
 394
 395         if iops1 < 999 or iops1 > 1001:
 396             self.failure_reason = "{0} iops value mismatch,".format(self.failure_reason)
 397             self.passed = False
 398
 399         if ratio < 7 or ratio > 9:
 400             self.failure_reason = "{0} iops ratio mismatch,".format(self.failure_reason)
 401             self.passed = False
 402
 403
 404 class Requirements(object):
 405     """Requirements consists of multiple run environment characteristics.
 406     These are to determine if a particular test can be run"""
 407
 408     _linux = False
 409     _libaio = False
 410     _zbd = False
 411     _root = False
 412     _zoned_nullb = False
 413     _not_macos = False
 414     _unittests = False
 415     _cpucount4 = False
 416
 417     def __init__(self, fio_root):
 418         Requirements._not_macos = platform.system() != "Darwin"
 419         Requirements._linux = platform.system() == "Linux"
 420
 421         if Requirements._linux:
 422             try:
 423                 config_file = os.path.join(fio_root, "config-host.h")
 424                 with open(config_file, "r") as config:
 425                     contents = config.read()
 426             except Exception:
 427                 print("Unable to open {0} to check requirements".format(config_file))
 428                 Requirements._zbd = True
 429             else:
 430                 Requirements._zbd = "CONFIG_LINUX_BLKZONED" in contents
 431                 Requirements._libaio = "CONFIG_LIBAIO" in contents
 432
 433             Requirements._root = (os.geteuid() == 0)
 434             if Requirements._zbd and Requirements._root:
 435                     subprocess.run(["modprobe", "null_blk"],
 436                                    stdout=subprocess.PIPE,
 437                                    stderr=subprocess.PIPE)
 438                     if os.path.exists("/sys/module/null_blk/parameters/zoned"):
 439                         Requirements._zoned_nullb = True
 440
 441         unittest_path = os.path.join(fio_root, "unittests", "unittest")
 442         Requirements._unittests = os.path.exists(unittest_path)
 443
 444         Requirements._cpucount4 = multiprocessing.cpu_count() >= 4
 445
 446         req_list = [Requirements.linux,
 447                     Requirements.libaio,
 448                     Requirements.zbd,
 449                     Requirements.root,
 450                     Requirements.zoned_nullb,
 451                     Requirements.not_macos,
 452                     Requirements.unittests,
 453                     Requirements.cpucount4]
 454         for req in req_list:
 455             value, desc = req()
 456             logging.debug("Requirement '%s' met? %s" % (desc, value))
 457
 458     def linux():
 459         return Requirements._linux, "Linux required"
 460
 461     def libaio():
 462         return Requirements._libaio, "libaio required"
 463
 464     def zbd():
 465         return Requirements._zbd, "Zoned block device support required"
 466
 467     def root():
 468         return Requirements._root, "root required"
 469
 470     def zoned_nullb():
 471         return Requirements._zoned_nullb, "Zoned null block device support required"
 472
 473     def not_macos():
 474         return Requirements._not_macos, "platform other than macOS required"
 475
 476     def unittests():
 477         return Requirements._unittests, "Unittests support required"
 478
 479     def cpucount4():
 480         return Requirements._cpucount4, "4+ CPUs required"
 481
 482
 483 SUCCESS_DEFAULT = {
 484         'zero_return': True,
 485         'stderr_empty': True,
 486         'timeout': 300,
 487         }
 488 SUCCESS_NONZERO = {
 489         'zero_return': False,
 490         'stderr_empty': False,
 491         'timeout': 300,
 492         }
 493 SUCCESS_STDERR = {
 494         'zero_return': True,
 495         'stderr_empty': False,
 496         'timeout': 300,
 497         }
 498 TEST_LIST = [
 499         {
 500             'test_id':          1,
 501             'test_class':       FioJobTest,
 502             'job':              't0001-52c58027.fio',
 503             'success':          SUCCESS_DEFAULT,
 504             'pre_job':          None,
 505             'pre_success':      None,
 506             'requirements':     [],
 507         },
 508         {
 509             'test_id':          2,
 510             'test_class':       FioJobTest,
 511             'job':              't0002-13af05ae-post.fio',
 512             'success':          SUCCESS_DEFAULT,
 513             'pre_job':          't0002-13af05ae-pre.fio',
 514             'pre_success':      None,
 515             'requirements':     [Requirements.linux, Requirements.libaio],
 516         },
 517         {
 518             'test_id':          3,
 519             'test_class':       FioJobTest,
 520             'job':              't0003-0ae2c6e1-post.fio',
 521             'success':          SUCCESS_NONZERO,
 522             'pre_job':          't0003-0ae2c6e1-pre.fio',
 523             'pre_success':      SUCCESS_DEFAULT,
 524             'requirements':     [Requirements.linux, Requirements.libaio],
 525         },
 526         {
 527             'test_id':          4,
 528             'test_class':       FioJobTest,
 529             'job':              't0004-8a99fdf6.fio',
 530             'success':          SUCCESS_DEFAULT,
 531             'pre_job':          None,
 532             'pre_success':      None,
 533             'requirements':     [Requirements.linux, Requirements.libaio],
 534         },
 535         {
 536             'test_id':          5,
 537             'test_class':       FioJobTest_t0005,
 538             'job':              't0005-f7078f7b.fio',
 539             'success':          SUCCESS_DEFAULT,
 540             'pre_job':          None,
 541             'pre_success':      None,
 542             'output_format':    'json',
 543             'requirements':     [],
 544         },
 545         {
 546             'test_id':          6,
 547             'test_class':       FioJobTest_t0006,
 548             'job':              't0006-82af2a7c.fio',
 549             'success':          SUCCESS_DEFAULT,
 550             'pre_job':          None,
 551             'pre_success':      None,
 552             'output_format':    'json',
 553             'requirements':     [Requirements.linux, Requirements.libaio],
 554         },
 555         {
 556             'test_id':          7,
 557             'test_class':       FioJobTest_t0007,
 558             'job':              't0007-37cf9e3c.fio',
 559             'success':          SUCCESS_DEFAULT,
 560             'pre_job':          None,
 561             'pre_success':      None,
 562             'output_format':    'json',
 563             'requirements':     [],
 564         },
 565         {
 566             'test_id':          8,
 567             'test_class':       FioJobTest_t0008,
 568             'job':              't0008-ae2fafc8.fio',
 569             'success':          SUCCESS_DEFAULT,
 570             'pre_job':          None,
 571             'pre_success':      None,
 572             'output_format':    'json',
 573             'requirements':     [],
 574         },
 575         {
 576             'test_id':          9,
 577             'test_class':       FioJobTest_t0009,
 578             'job':              't0009-f8b0bd10.fio',
 579             'success':          SUCCESS_DEFAULT,
 580             'pre_job':          None,
 581             'pre_success':      None,
 582             'output_format':    'json',
 583             'requirements':     [Requirements.not_macos,
 584                                  Requirements.cpucount4],
 585                                 # mac os does not support CPU affinity
 586         },
 587         {
 588             'test_id':          10,
 589             'test_class':       FioJobTest,
 590             'job':              't0010-b7aae4ba.fio',
 591             'success':          SUCCESS_DEFAULT,
 592             'pre_job':          None,
 593             'pre_success':      None,
 594             'requirements':     [],
 595         },
 596         {
 597             'test_id':          11,
 598             'test_class':       FioJobTest_t0011,
 599             'job':              't0011-5d2788d5.fio',
 600             'success':          SUCCESS_DEFAULT,
 601             'pre_job':          None,
 602             'pre_success':      None,
 603             'output_format':    'json',
 604             'requirements':     [],
 605         },
 606         {
 607             'test_id':          1000,
 608             'test_class':       FioExeTest,
 609             'exe':              't/axmap',
 610             'parameters':       None,
 611             'success':          SUCCESS_DEFAULT,
 612             'requirements':     [],
 613         },
 614         {
 615             'test_id':          1001,
 616             'test_class':       FioExeTest,
 617             'exe':              't/ieee754',
 618             'parameters':       None,
 619             'success':          SUCCESS_DEFAULT,
 620             'requirements':     [],
 621         },
 622         {
 623             'test_id':          1002,
 624             'test_class':       FioExeTest,
 625             'exe':              't/lfsr-test',
 626             'parameters':       ['0xFFFFFF', '0', '0', 'verify'],
 627             'success':          SUCCESS_STDERR,
 628             'requirements':     [],
 629         },
 630         {
 631             'test_id':          1003,
 632             'test_class':       FioExeTest,
 633             'exe':              't/readonly.py',
 634             'parameters':       ['-f', '{fio_path}'],
 635             'success':          SUCCESS_DEFAULT,
 636             'requirements':     [],
 637         },
 638         {
 639             'test_id':          1004,
 640             'test_class':       FioExeTest,
 641             'exe':              't/steadystate_tests.py',
 642             'parameters':       ['{fio_path}'],
 643             'success':          SUCCESS_DEFAULT,
 644             'requirements':     [],
 645         },
 646         {
 647             'test_id':          1005,
 648             'test_class':       FioExeTest,
 649             'exe':              't/stest',
 650             'parameters':       None,
 651             'success':          SUCCESS_STDERR,
 652             'requirements':     [],
 653         },
 654         {
 655             'test_id':          1006,
 656             'test_class':       FioExeTest,
 657             'exe':              't/strided.py',
 658             'parameters':       ['{fio_path}'],
 659             'success':          SUCCESS_DEFAULT,
 660             'requirements':     [],
 661         },
 662         {
 663             'test_id':          1007,
 664             'test_class':       FioExeTest,
 665             'exe':              't/zbd/run-tests-against-regular-nullb',
 666             'parameters':       None,
 667             'success':          SUCCESS_DEFAULT,
 668             'requirements':     [Requirements.linux, Requirements.zbd,
 669                                  Requirements.root],
 670         },
 671         {
 672             'test_id':          1008,
 673             'test_class':       FioExeTest,
 674             'exe':              't/zbd/run-tests-against-zoned-nullb',
 675             'parameters':       None,
 676             'success':          SUCCESS_DEFAULT,
 677             'requirements':     [Requirements.linux, Requirements.zbd,
 678                                  Requirements.root, Requirements.zoned_nullb],
 679         },
 680         {
 681             'test_id':          1009,
 682             'test_class':       FioExeTest,
 683             'exe':              'unittests/unittest',
 684             'parameters':       None,
 685             'success':          SUCCESS_DEFAULT,
 686             'requirements':     [Requirements.unittests],
 687         },
 688 ]
 689
 690
 691 def parse_args():
 692     parser = argparse.ArgumentParser()
 693     parser.add_argument('-r', '--fio-root',
 694                         help='fio root path')
 695     parser.add_argument('-f', '--fio',
 696                         help='path to fio executable (e.g., ./fio)')
 697     parser.add_argument('-a', '--artifact-root',
 698                         help='artifact root directory')
 699     parser.add_argument('-s', '--skip', nargs='+', type=int,
 700                         help='list of test(s) to skip')
 701     parser.add_argument('-o', '--run-only', nargs='+', type=int,
 702                         help='list of test(s) to run, skipping all others')
 703     parser.add_argument('-d', '--debug', action='store_true',
 704                         help='provide debug output')
 705     parser.add_argument('-k', '--skip-req', action='store_true',
 706                         help='skip requirements checking')
 707     args = parser.parse_args()
 708
 709     return args
 710
 711
 712 def main():
 713     args = parse_args()
 714     if args.debug:
 715         logging.basicConfig(level=logging.DEBUG)
 716     else:
 717         logging.basicConfig(level=logging.INFO)
 718
 719     if args.fio_root:
 720         fio_root = args.fio_root
 721     else:
 722         fio_root = str(Path(__file__).absolute().parent.parent)
 723     print("fio root is %s" % fio_root)
 724
 725     if args.fio:
 726         fio_path = args.fio
 727     else:
 728         fio_path = os.path.join(fio_root, "fio")
 729     print("fio path is %s" % fio_path)
 730     if not shutil.which(fio_path):
 731         print("Warning: fio executable not found")
 732
 733     artifact_root = args.artifact_root if args.artifact_root else \
 734         "fio-test-{0}".format(time.strftime("%Y%m%d-%H%M%S"))
 735     os.mkdir(artifact_root)
 736     print("Artifact directory is %s" % artifact_root)
 737
 738     if not args.skip_req:
 739         req = Requirements(fio_root)
 740
 741     passed = 0
 742     failed = 0
 743     skipped = 0
 744
 745     for config in TEST_LIST:
 746         if (args.skip and config['test_id'] in args.skip) or \
 747            (args.run_only and config['test_id'] not in args.run_only):
 748             skipped = skipped + 1
 749             print("Test {0} SKIPPED (User request)".format(config['test_id']))
 750             continue
 751
 752         if issubclass(config['test_class'], FioJobTest):
 753             if config['pre_job']:
 754                 fio_pre_job = os.path.join(fio_root, 't', 'jobs',
 755                                            config['pre_job'])
 756             else:
 757                 fio_pre_job = None
 758             if config['pre_success']:
 759                 fio_pre_success = config['pre_success']
 760             else:
 761                 fio_pre_success = None
 762             if 'output_format' in config:
 763                 output_format = config['output_format']
 764             else:
 765                 output_format = 'normal'
 766             test = config['test_class'](
 767                 fio_path,
 768                 os.path.join(fio_root, 't', 'jobs', config['job']),
 769                 config['success'],
 770                 fio_pre_job=fio_pre_job,
 771                 fio_pre_success=fio_pre_success,
 772                 output_format=output_format)
 773         elif issubclass(config['test_class'], FioExeTest):
 774             exe_path = os.path.join(fio_root, config['exe'])
 775             if config['parameters']:
 776                 parameters = [p.format(fio_path=fio_path) for p in config['parameters']]
 777             else:
 778                 parameters = None
 779             test = config['test_class'](exe_path, parameters,
 780                                         config['success'])
 781         else:
 782             print("Test {0} FAILED: unable to process test config".format(config['test_id']))
 783             failed = failed + 1
 784             continue
 785
 786         if not args.skip_req:
 787             skip = False
 788             for req in config['requirements']:
 789                 ok, reason = req()
 790                 skip = not ok
 791                 logging.debug("Requirement '%s' met? %s" % (reason, ok))
 792                 if skip:
 793                     break
 794             if skip:
 795                 print("Test {0} SKIPPED ({1})".format(config['test_id'], reason))
 796                 skipped = skipped + 1
 797                 continue
 798
 799         test.setup(artifact_root, config['test_id'])
 800         test.run()
 801         test.check_result()
 802         if test.passed:
 803             result = "PASSED"
 804             passed = passed + 1
 805         else:
 806             result = "FAILED: {0}".format(test.failure_reason)
 807             failed = failed + 1
 808             with open(test.stderr_file, "r") as stderr_file:
 809                 logging.debug("stderr:\n%s" % stderr_file.read())
 810             with open(test.stdout_file, "r") as stdout_file:
 811                 logging.debug("stdout:\n%s" % stdout_file.read())
 812         print("Test {0} {1}".format(config['test_id'], result))
 813
 814     print("{0} test(s) passed, {1} failed, {2} skipped".format(passed, failed, skipped))
 815
 816     sys.exit(failed)
 817
 818
 819 if __name__ == '__main__':
 820     main()