t/fiotestlib.py

   1 #!/usr/bin/env python3
   2 """
   3 fiotestlib.py
   4
   5 This library contains FioTest objects that provide convenient means to run
   6 different sorts of fio tests.
   7
   8 It also contains a test runner that runs an array of dictionary objects
   9 describing fio tests.
  10 """
  11
  12 import os
  13 import sys
  14 import json
  15 import locale
  16 import logging
  17 import platform
  18 import traceback
  19 import subprocess
  20 from pathlib import Path
  21 from fiotestcommon import get_file, SUCCESS_DEFAULT
  22
  23
  24 class FioTest():
  25     """Base for all fio tests."""
  26
  27     def __init__(self, exe_path, success, testnum, artifact_root):
  28         self.success = success
  29         self.testnum = testnum
  30         self.output = {}
  31         self.passed = True
  32         self.failure_reason = ''
  33         self.parameters = None
  34         self.paths = {
  35                         'exe': exe_path,
  36                         'artifacts': artifact_root,
  37                         'test_dir': os.path.join(artifact_root, \
  38                                 f"{testnum:04d}"),
  39                         }
  40         self.filenames = {
  41                             'cmd': os.path.join(self.paths['test_dir'], \
  42                                     f"{os.path.basename(self.paths['exe'])}.command"),
  43                             'stdout': os.path.join(self.paths['test_dir'], \
  44                                     f"{os.path.basename(self.paths['exe'])}.stdout"),
  45                             'stderr': os.path.join(self.paths['test_dir'], \
  46                                     f"{os.path.basename(self.paths['exe'])}.stderr"),
  47                             'exitcode': os.path.join(self.paths['test_dir'], \
  48                                     f"{os.path.basename(self.paths['exe'])}.exitcode"),
  49                             }
  50
  51     def setup(self, parameters):
  52         """Setup instance variables for test."""
  53
  54         self.parameters = parameters
  55         if not os.path.exists(self.paths['test_dir']):
  56             os.mkdir(self.paths['test_dir'])
  57
  58     def run(self):
  59         """Run the test."""
  60
  61         raise NotImplementedError()
  62
  63     def check_result(self):
  64         """Check test results."""
  65
  66         raise NotImplementedError()
  67
  68
  69 class FioExeTest(FioTest):
  70     """Test consists of an executable binary or script"""
  71
  72     def run(self):
  73         """Execute the binary or script described by this instance."""
  74
  75         command = [self.paths['exe']] + self.parameters
  76         with open(self.filenames['cmd'], "w+",
  77                   encoding=locale.getpreferredencoding()) as command_file:
  78             command_file.write(" \\\n ".join(command))
  79
  80         try:
  81             with open(self.filenames['stdout'], "w+",
  82                       encoding=locale.getpreferredencoding()) as stdout_file, \
  83                 open(self.filenames['stderr'], "w+",
  84                      encoding=locale.getpreferredencoding()) as stderr_file, \
  85                 open(self.filenames['exitcode'], "w+",
  86                      encoding=locale.getpreferredencoding()) as exitcode_file:
  87                 proc = None
  88                 # Avoid using subprocess.run() here because when a timeout occurs,
  89                 # fio will be stopped with SIGKILL. This does not give fio a
  90                 # chance to clean up and means that child processes may continue
  91                 # running and submitting IO.
  92                 proc = subprocess.Popen(command,
  93                                         stdout=stdout_file,
  94                                         stderr=stderr_file,
  95                                         cwd=self.paths['test_dir'],
  96                                         universal_newlines=True)
  97                 proc.communicate(timeout=self.success['timeout'])
  98                 exitcode_file.write(f'{proc.returncode}\n')
  99                 logging.debug("Test %d: return code: %d", self.testnum, proc.returncode)
 100                 self.output['proc'] = proc
 101         except subprocess.TimeoutExpired:
 102             proc.terminate()
 103             proc.communicate()
 104             assert proc.poll()
 105             self.output['failure'] = 'timeout'
 106         except Exception:
 107             if proc:
 108                 if not proc.poll():
 109                     proc.terminate()
 110                     proc.communicate()
 111             self.output['failure'] = 'exception'
 112             self.output['exc_info'] = sys.exc_info()
 113
 114     def check_result(self):
 115         """Check results of test run."""
 116
 117         if 'proc' not in self.output:
 118             if self.output['failure'] == 'timeout':
 119                 self.failure_reason = f"{self.failure_reason} timeout,"
 120             else:
 121                 assert self.output['failure'] == 'exception'
 122                 self.failure_reason = f'{self.failure_reason} exception: ' + \
 123                 f'{self.output["exc_info"][0]}, {self.output["exc_info"][1]}'
 124
 125             self.passed = False
 126             return
 127
 128         if 'zero_return' in self.success:
 129             if self.success['zero_return']:
 130                 if self.output['proc'].returncode != 0:
 131                     self.passed = False
 132                     self.failure_reason = f"{self.failure_reason} non-zero return code,"
 133             else:
 134                 if self.output['proc'].returncode == 0:
 135                     self.failure_reason = f"{self.failure_reason} zero return code,"
 136                     self.passed = False
 137
 138         stderr_size = os.path.getsize(self.filenames['stderr'])
 139         if 'stderr_empty' in self.success:
 140             if self.success['stderr_empty']:
 141                 if stderr_size != 0:
 142                     self.failure_reason = f"{self.failure_reason} stderr not empty,"
 143                     self.passed = False
 144             else:
 145                 if stderr_size == 0:
 146                     self.failure_reason = f"{self.failure_reason} stderr empty,"
 147                     self.passed = False
 148
 149
 150 class FioJobFileTest(FioExeTest):
 151     """Test consists of a fio job with options in a job file."""
 152
 153     def __init__(self, fio_path, fio_job, success, testnum, artifact_root,
 154                  fio_pre_job=None, fio_pre_success=None,
 155                  output_format="normal"):
 156         """Construct a FioJobFileTest which is a FioExeTest consisting of a
 157         single fio job file with an optional setup step.
 158
 159         fio_path:           location of fio executable
 160         fio_job:            location of fio job file
 161         success:            Definition of test success
 162         testnum:            test ID
 163         artifact_root:      root directory for artifacts
 164         fio_pre_job:        fio job for preconditioning
 165         fio_pre_success:    Definition of test success for fio precon job
 166         output_format:      normal (default), json, jsonplus, or terse
 167         """
 168
 169         self.fio_job = fio_job
 170         self.fio_pre_job = fio_pre_job
 171         self.fio_pre_success = fio_pre_success if fio_pre_success else success
 172         self.output_format = output_format
 173         self.precon_failed = False
 174         self.json_data = None
 175
 176         super().__init__(fio_path, success, testnum, artifact_root)
 177
 178     def setup(self, parameters):
 179         """Setup instance variables for fio job test."""
 180
 181         self.filenames['fio_output'] = f"{os.path.basename(self.fio_job)}.output"
 182         fio_args = [
 183             "--max-jobs=16",
 184             f"--output-format={self.output_format}",
 185             f"--output={self.filenames['fio_output']}",
 186             self.fio_job,
 187             ]
 188         if parameters:
 189             fio_args += parameters
 190
 191         super().setup(fio_args)
 192
 193         # Update the filenames from the default
 194         self.filenames['cmd'] = os.path.join(self.paths['test_dir'],
 195                                              f"{os.path.basename(self.fio_job)}.command")
 196         self.filenames['stdout'] = os.path.join(self.paths['test_dir'],
 197                                                 f"{os.path.basename(self.fio_job)}.stdout")
 198         self.filenames['stderr'] = os.path.join(self.paths['test_dir'],
 199                                                 f"{os.path.basename(self.fio_job)}.stderr")
 200         self.filenames['exitcode'] = os.path.join(self.paths['test_dir'],
 201                                                   f"{os.path.basename(self.fio_job)}.exitcode")
 202
 203     def run_pre_job(self):
 204         """Run fio job precondition step."""
 205
 206         precon = FioJobFileTest(self.paths['exe'], self.fio_pre_job,
 207                             self.fio_pre_success,
 208                             self.testnum,
 209                             self.paths['artifacts'],
 210                             output_format=self.output_format)
 211         precon.setup(None)
 212         precon.run()
 213         precon.check_result()
 214         self.precon_failed = not precon.passed
 215         self.failure_reason = precon.failure_reason
 216
 217     def run(self):
 218         """Run fio job test."""
 219
 220         if self.fio_pre_job:
 221             self.run_pre_job()
 222
 223         if not self.precon_failed:
 224             super().run()
 225         else:
 226             logging.debug("Test %d: precondition step failed", self.testnum)
 227
 228     def get_file_fail(self, filename):
 229         """Safely read a file and fail the test upon error."""
 230         file_data = None
 231
 232         try:
 233             with open(filename, "r", encoding=locale.getpreferredencoding()) as output_file:
 234                 file_data = output_file.read()
 235         except OSError:
 236             self.failure_reason += f" unable to read file {filename}"
 237             self.passed = False
 238
 239         return file_data
 240
 241     def check_result(self):
 242         """Check fio job results."""
 243
 244         if self.precon_failed:
 245             self.passed = False
 246             self.failure_reason = f"{self.failure_reason} precondition step failed,"
 247             return
 248
 249         super().check_result()
 250
 251         if not self.passed:
 252             return
 253
 254         if 'json' not in self.output_format:
 255             return
 256
 257         file_data = self.get_file_fail(os.path.join(self.paths['test_dir'],
 258                                                     self.filenames['fio_output']))
 259         if not file_data:
 260             return
 261
 262         #
 263         # Sometimes fio informational messages are included at the top of the
 264         # JSON output, especially under Windows. Try to decode output as JSON
 265         # data, skipping everything until the first {
 266         #
 267         lines = file_data.splitlines()
 268         file_data = '\n'.join(lines[lines.index("{"):])
 269         try:
 270             self.json_data = json.loads(file_data)
 271         except json.JSONDecodeError:
 272             self.failure_reason = f"{self.failure_reason} unable to decode JSON data,"
 273             self.passed = False
 274
 275
 276 class FioJobCmdTest(FioExeTest):
 277     """This runs a fio job with options specified on the command line."""
 278
 279     def __init__(self, fio_path, success, testnum, artifact_root, fio_opts, basename=None):
 280
 281         self.basename = basename if basename else os.path.basename(fio_path)
 282         self.fio_opts = fio_opts
 283         self.json_data = None
 284         self.iops_log_lines = None
 285
 286         super().__init__(fio_path, success, testnum, artifact_root)
 287
 288         filename_stub = os.path.join(self.paths['test_dir'], f"{self.basename}{self.testnum:03d}")
 289         self.filenames['cmd'] = f"{filename_stub}.command"
 290         self.filenames['stdout'] = f"{filename_stub}.stdout"
 291         self.filenames['stderr'] = f"{filename_stub}.stderr"
 292         self.filenames['output'] = os.path.abspath(f"{filename_stub}.output")
 293         self.filenames['exitcode'] = f"{filename_stub}.exitcode"
 294         self.filenames['iopslog'] = os.path.abspath(f"{filename_stub}")
 295
 296     def run(self):
 297         super().run()
 298
 299         if 'output-format' in self.fio_opts and 'json' in \
 300                 self.fio_opts['output-format']:
 301             if not self.get_json():
 302                 print('Unable to decode JSON data')
 303                 self.passed = False
 304
 305         if any('--write_iops_log=' in param for param in self.parameters):
 306             self.get_iops_log()
 307
 308     def get_iops_log(self):
 309         """Read IOPS log from the first job."""
 310
 311         log_filename = self.filenames['iopslog'] + "_iops.1.log"
 312         with open(log_filename, 'r', encoding=locale.getpreferredencoding()) as iops_file:
 313             self.iops_log_lines = iops_file.read()
 314
 315     def get_json(self):
 316         """Convert fio JSON output into a python JSON object"""
 317
 318         filename = self.filenames['output']
 319         with open(filename, 'r', encoding=locale.getpreferredencoding()) as file:
 320             file_data = file.read()
 321
 322         #
 323         # Sometimes fio informational messages are included at the top of the
 324         # JSON output, especially under Windows. Try to decode output as JSON
 325         # data, lopping off up to the first four lines
 326         #
 327         lines = file_data.splitlines()
 328         for i in range(5):
 329             file_data = '\n'.join(lines[i:])
 330             try:
 331                 self.json_data = json.loads(file_data)
 332             except json.JSONDecodeError:
 333                 continue
 334             else:
 335                 return True
 336
 337         return False
 338
 339     @staticmethod
 340     def check_empty(job):
 341         """
 342         Make sure JSON data is empty.
 343
 344         Some data structures should be empty. This function makes sure that they are.
 345
 346         job         JSON object that we need to check for emptiness
 347         """
 348
 349         return job['total_ios'] == 0 and \
 350                 job['slat_ns']['N'] == 0 and \
 351                 job['clat_ns']['N'] == 0 and \
 352                 job['lat_ns']['N'] == 0
 353
 354     def check_all_ddirs(self, ddir_nonzero, job):
 355         """
 356         Iterate over the data directions and check whether each is
 357         appropriately empty or not.
 358         """
 359
 360         retval = True
 361         ddirlist = ['read', 'write', 'trim']
 362
 363         for ddir in ddirlist:
 364             if ddir in ddir_nonzero:
 365                 if self.check_empty(job[ddir]):
 366                     print(f"Unexpected zero {ddir} data found in output")
 367                     retval = False
 368             else:
 369                 if not self.check_empty(job[ddir]):
 370                     print(f"Unexpected {ddir} data found in output")
 371                     retval = False
 372
 373         return retval
 374
 375
 376 def run_fio_tests(test_list, test_env, args):
 377     """
 378     Run tests as specified in test_list.
 379     """
 380
 381     passed = 0
 382     failed = 0
 383     skipped = 0
 384
 385     for config in test_list:
 386         if (args.skip and config['test_id'] in args.skip) or \
 387            (args.run_only and config['test_id'] not in args.run_only) or \
 388            ('force_skip' in config and config['force_skip']):
 389             skipped = skipped + 1
 390             print(f"Test {config['test_id']} SKIPPED (User request or override)")
 391             continue
 392
 393         if issubclass(config['test_class'], FioJobFileTest):
 394             if config['pre_job']:
 395                 fio_pre_job = os.path.join(test_env['fio_root'], 't', 'jobs',
 396                                            config['pre_job'])
 397             else:
 398                 fio_pre_job = None
 399             if config['pre_success']:
 400                 fio_pre_success = config['pre_success']
 401             else:
 402                 fio_pre_success = None
 403             if 'output_format' in config:
 404                 output_format = config['output_format']
 405             else:
 406                 output_format = 'normal'
 407             test = config['test_class'](
 408                 test_env['fio_path'],
 409                 os.path.join(test_env['fio_root'], 't', 'jobs', config['job']),
 410                 config['success'],
 411                 config['test_id'],
 412                 test_env['artifact_root'],
 413                 fio_pre_job=fio_pre_job,
 414                 fio_pre_success=fio_pre_success,
 415                 output_format=output_format)
 416             desc = config['job']
 417             parameters = config['parameters'] if 'parameters' in config else None
 418         elif issubclass(config['test_class'], FioJobCmdTest):
 419             if not 'success' in config:
 420                 config['success'] = SUCCESS_DEFAULT
 421             test = config['test_class'](test_env['fio_path'],
 422                                         config['success'],
 423                                         config['test_id'],
 424                                         test_env['artifact_root'],
 425                                         config['fio_opts'],
 426                                         test_env['basename'])
 427             desc = config['test_id']
 428             parameters = config
 429         elif issubclass(config['test_class'], FioExeTest):
 430             exe_path = os.path.join(test_env['fio_root'], config['exe'])
 431             parameters = []
 432             if config['parameters']:
 433                 parameters = [p.format(fio_path=test_env['fio_path'], nvmecdev=args.nvmecdev)
 434                               for p in config['parameters']]
 435             if Path(exe_path).suffix == '.py' and platform.system() == "Windows":
 436                 parameters.insert(0, exe_path)
 437                 exe_path = "python.exe"
 438             if config['test_id'] in test_env['pass_through']:
 439                 parameters += test_env['pass_through'][config['test_id']].split()
 440             test = config['test_class'](
 441                     exe_path,
 442                     config['success'],
 443                     config['test_id'],
 444                     test_env['artifact_root'])
 445             desc = config['exe']
 446         else:
 447             print(f"Test {config['test_id']} FAILED: unable to process test config")
 448             failed = failed + 1
 449             continue
 450
 451         if 'requirements' in config and not args.skip_req:
 452             reqs_met = True
 453             for req in config['requirements']:
 454                 reqs_met, reason = req()
 455                 logging.debug("Test %d: Requirement '%s' met? %s", config['test_id'], reason,
 456                               reqs_met)
 457                 if not reqs_met:
 458                     break
 459             if not reqs_met:
 460                 print(f"Test {config['test_id']} SKIPPED ({reason}) {desc}")
 461                 skipped = skipped + 1
 462                 continue
 463
 464         try:
 465             test.setup(parameters)
 466             test.run()
 467             test.check_result()
 468         except KeyboardInterrupt:
 469             break
 470         except Exception as e:
 471             test.passed = False
 472             test.failure_reason += str(e)
 473             logging.debug("Test %d exception:\n%s\n", config['test_id'], traceback.format_exc())
 474         if test.passed:
 475             result = "PASSED"
 476             passed = passed + 1
 477         else:
 478             result = f"FAILED: {test.failure_reason}"
 479             failed = failed + 1
 480             contents, _ = get_file(test.filenames['stderr'])
 481             logging.debug("Test %d: stderr:\n%s", config['test_id'], contents)
 482             contents, _ = get_file(test.filenames['stdout'])
 483             logging.debug("Test %d: stdout:\n%s", config['test_id'], contents)
 484         print(f"Test {config['test_id']} {result} {desc}")
 485
 486     print(f"{passed} test(s) passed, {failed} failed, {skipped} skipped")
 487
 488     return passed, failed, skipped