t/fiotestlib.py

   1 #!/usr/bin/env python3
   2 """
   3 fiotestlib.py
   4
   5 This library contains FioTest objects that provide convenient means to run
   6 different sorts of fio tests.
   7
   8 It also contains a test runner that runs an array of dictionary objects
   9 describing fio tests.
  10 """
  11
  12 import os
  13 import sys
  14 import json
  15 import locale
  16 import logging
  17 import platform
  18 import traceback
  19 import subprocess
  20 from pathlib import Path
  21 from fiotestcommon import get_file, SUCCESS_DEFAULT
  22
  23
  24 class FioTest():
  25     """Base for all fio tests."""
  26
  27     def __init__(self, exe_path, success, testnum, artifact_root):
  28         self.success = success
  29         self.testnum = testnum
  30         self.output = {}
  31         self.passed = True
  32         self.failure_reason = ''
  33         self.parameters = None
  34         self.paths = {
  35                         'exe': exe_path,
  36                         'artifacts': artifact_root,
  37                         'test_dir': os.path.join(artifact_root, \
  38                                 f"{testnum:04d}"),
  39                         }
  40         self.filenames = {
  41                             'cmd': os.path.join(self.paths['test_dir'], \
  42                                     f"{os.path.basename(self.paths['exe'])}.command"),
  43                             'stdout': os.path.join(self.paths['test_dir'], \
  44                                     f"{os.path.basename(self.paths['exe'])}.stdout"),
  45                             'stderr': os.path.join(self.paths['test_dir'], \
  46                                     f"{os.path.basename(self.paths['exe'])}.stderr"),
  47                             'exitcode': os.path.join(self.paths['test_dir'], \
  48                                     f"{os.path.basename(self.paths['exe'])}.exitcode"),
  49                             }
  50
  51     def setup(self, parameters):
  52         """Setup instance variables for test."""
  53
  54         self.parameters = parameters
  55         if not os.path.exists(self.paths['test_dir']):
  56             os.mkdir(self.paths['test_dir'])
  57
  58     def run(self):
  59         """Run the test."""
  60
  61         raise NotImplementedError()
  62
  63     def check_result(self):
  64         """Check test results."""
  65
  66         raise NotImplementedError()
  67
  68
  69 class FioExeTest(FioTest):
  70     """Test consists of an executable binary or script"""
  71
  72     def run(self):
  73         """Execute the binary or script described by this instance."""
  74
  75         command = [self.paths['exe']] + self.parameters
  76         with open(self.filenames['cmd'], "w+",
  77                   encoding=locale.getpreferredencoding()) as command_file:
  78             command_file.write(" \\\n ".join(command))
  79
  80         try:
  81             with open(self.filenames['stdout'], "w+",
  82                       encoding=locale.getpreferredencoding()) as stdout_file, \
  83                 open(self.filenames['stderr'], "w+",
  84                      encoding=locale.getpreferredencoding()) as stderr_file, \
  85                 open(self.filenames['exitcode'], "w+",
  86                      encoding=locale.getpreferredencoding()) as exitcode_file:
  87                 proc = None
  88                 # Avoid using subprocess.run() here because when a timeout occurs,
  89                 # fio will be stopped with SIGKILL. This does not give fio a
  90                 # chance to clean up and means that child processes may continue
  91                 # running and submitting IO.
  92                 proc = subprocess.Popen(command,
  93                                         stdout=stdout_file,
  94                                         stderr=stderr_file,
  95                                         cwd=self.paths['test_dir'],
  96                                         universal_newlines=True)
  97                 proc.communicate(timeout=self.success['timeout'])
  98                 exitcode_file.write(f'{proc.returncode}\n')
  99                 logging.debug("Test %d: return code: %d", self.testnum, proc.returncode)
 100                 self.output['proc'] = proc
 101         except subprocess.TimeoutExpired:
 102             proc.terminate()
 103             proc.communicate()
 104             assert proc.poll()
 105             self.output['failure'] = 'timeout'
 106         except Exception:
 107             if proc:
 108                 if not proc.poll():
 109                     proc.terminate()
 110                     proc.communicate()
 111             self.output['failure'] = 'exception'
 112             self.output['exc_info'] = sys.exc_info()
 113
 114     def check_result(self):
 115         """Check results of test run."""
 116
 117         if 'proc' not in self.output:
 118             if self.output['failure'] == 'timeout':
 119                 self.failure_reason = f"{self.failure_reason} timeout,"
 120             else:
 121                 assert self.output['failure'] == 'exception'
 122                 self.failure_reason = f'{self.failure_reason} exception: ' + \
 123                 f'{self.output["exc_info"][0]}, {self.output["exc_info"][1]}'
 124
 125             self.passed = False
 126             return
 127
 128         if 'zero_return' in self.success:
 129             if self.success['zero_return']:
 130                 if self.output['proc'].returncode != 0:
 131                     self.passed = False
 132                     self.failure_reason = f"{self.failure_reason} non-zero return code,"
 133             else:
 134                 if self.output['proc'].returncode == 0:
 135                     self.failure_reason = f"{self.failure_reason} zero return code,"
 136                     self.passed = False
 137
 138         stderr_size = os.path.getsize(self.filenames['stderr'])
 139         if 'stderr_empty' in self.success:
 140             if self.success['stderr_empty']:
 141                 if stderr_size != 0:
 142                     self.failure_reason = f"{self.failure_reason} stderr not empty,"
 143                     self.passed = False
 144             else:
 145                 if stderr_size == 0:
 146                     self.failure_reason = f"{self.failure_reason} stderr empty,"
 147                     self.passed = False
 148
 149
 150 class FioJobFileTest(FioExeTest):
 151     """Test consists of a fio job with options in a job file."""
 152
 153     def __init__(self, fio_path, fio_job, success, testnum, artifact_root,
 154                  fio_pre_job=None, fio_pre_success=None,
 155                  output_format="normal"):
 156         """Construct a FioJobFileTest which is a FioExeTest consisting of a
 157         single fio job file with an optional setup step.
 158
 159         fio_path:           location of fio executable
 160         fio_job:            location of fio job file
 161         success:            Definition of test success
 162         testnum:            test ID
 163         artifact_root:      root directory for artifacts
 164         fio_pre_job:        fio job for preconditioning
 165         fio_pre_success:    Definition of test success for fio precon job
 166         output_format:      normal (default), json, jsonplus, or terse
 167         """
 168
 169         self.fio_job = fio_job
 170         self.fio_pre_job = fio_pre_job
 171         self.fio_pre_success = fio_pre_success if fio_pre_success else success
 172         self.output_format = output_format
 173         self.precon_failed = False
 174         self.json_data = None
 175
 176         super().__init__(fio_path, success, testnum, artifact_root)
 177
 178     def setup(self, parameters=None):
 179         """Setup instance variables for fio job test."""
 180
 181         self.filenames['fio_output'] = f"{os.path.basename(self.fio_job)}.output"
 182         fio_args = [
 183             "--max-jobs=16",
 184             f"--output-format={self.output_format}",
 185             f"--output={self.filenames['fio_output']}",
 186             self.fio_job,
 187             ]
 188
 189         super().setup(fio_args)
 190
 191         # Update the filenames from the default
 192         self.filenames['cmd'] = os.path.join(self.paths['test_dir'],
 193                                              f"{os.path.basename(self.fio_job)}.command")
 194         self.filenames['stdout'] = os.path.join(self.paths['test_dir'],
 195                                                 f"{os.path.basename(self.fio_job)}.stdout")
 196         self.filenames['stderr'] = os.path.join(self.paths['test_dir'],
 197                                                 f"{os.path.basename(self.fio_job)}.stderr")
 198         self.filenames['exitcode'] = os.path.join(self.paths['test_dir'],
 199                                                   f"{os.path.basename(self.fio_job)}.exitcode")
 200
 201     def run_pre_job(self):
 202         """Run fio job precondition step."""
 203
 204         precon = FioJobFileTest(self.paths['exe'], self.fio_pre_job,
 205                             self.fio_pre_success,
 206                             self.testnum,
 207                             self.paths['artifacts'],
 208                             output_format=self.output_format)
 209         precon.setup()
 210         precon.run()
 211         precon.check_result()
 212         self.precon_failed = not precon.passed
 213         self.failure_reason = precon.failure_reason
 214
 215     def run(self):
 216         """Run fio job test."""
 217
 218         if self.fio_pre_job:
 219             self.run_pre_job()
 220
 221         if not self.precon_failed:
 222             super().run()
 223         else:
 224             logging.debug("Test %d: precondition step failed", self.testnum)
 225
 226     def get_file_fail(self, filename):
 227         """Safely read a file and fail the test upon error."""
 228         file_data = None
 229
 230         try:
 231             with open(filename, "r", encoding=locale.getpreferredencoding()) as output_file:
 232                 file_data = output_file.read()
 233         except OSError:
 234             self.failure_reason += f" unable to read file {filename}"
 235             self.passed = False
 236
 237         return file_data
 238
 239     def check_result(self):
 240         """Check fio job results."""
 241
 242         if self.precon_failed:
 243             self.passed = False
 244             self.failure_reason = f"{self.failure_reason} precondition step failed,"
 245             return
 246
 247         super().check_result()
 248
 249         if not self.passed:
 250             return
 251
 252         if 'json' not in self.output_format:
 253             return
 254
 255         file_data = self.get_file_fail(os.path.join(self.paths['test_dir'],
 256                                                     self.filenames['fio_output']))
 257         if not file_data:
 258             return
 259
 260         #
 261         # Sometimes fio informational messages are included at the top of the
 262         # JSON output, especially under Windows. Try to decode output as JSON
 263         # data, skipping everything until the first {
 264         #
 265         lines = file_data.splitlines()
 266         file_data = '\n'.join(lines[lines.index("{"):])
 267         try:
 268             self.json_data = json.loads(file_data)
 269         except json.JSONDecodeError:
 270             self.failure_reason = f"{self.failure_reason} unable to decode JSON data,"
 271             self.passed = False
 272
 273
 274 class FioJobCmdTest(FioExeTest):
 275     """This runs a fio job with options specified on the command line."""
 276
 277     def __init__(self, fio_path, success, testnum, artifact_root, fio_opts, basename=None):
 278
 279         self.basename = basename if basename else os.path.basename(fio_path)
 280         self.fio_opts = fio_opts
 281         self.json_data = None
 282         self.iops_log_lines = None
 283
 284         super().__init__(fio_path, success, testnum, artifact_root)
 285
 286         filename_stub = os.path.join(self.paths['test_dir'], f"{self.basename}{self.testnum:03d}")
 287         self.filenames['cmd'] = f"{filename_stub}.command"
 288         self.filenames['stdout'] = f"{filename_stub}.stdout"
 289         self.filenames['stderr'] = f"{filename_stub}.stderr"
 290         self.filenames['output'] = os.path.abspath(f"{filename_stub}.output")
 291         self.filenames['exitcode'] = f"{filename_stub}.exitcode"
 292         self.filenames['iopslog'] = os.path.abspath(f"{filename_stub}")
 293
 294     def run(self):
 295         super().run()
 296
 297         if 'output-format' in self.fio_opts and 'json' in \
 298                 self.fio_opts['output-format']:
 299             if not self.get_json():
 300                 print('Unable to decode JSON data')
 301                 self.passed = False
 302
 303         if any('--write_iops_log=' in param for param in self.parameters):
 304             self.get_iops_log()
 305
 306     def get_iops_log(self):
 307         """Read IOPS log from the first job."""
 308
 309         log_filename = self.filenames['iopslog'] + "_iops.1.log"
 310         with open(log_filename, 'r', encoding=locale.getpreferredencoding()) as iops_file:
 311             self.iops_log_lines = iops_file.read()
 312
 313     def get_json(self):
 314         """Convert fio JSON output into a python JSON object"""
 315
 316         filename = self.filenames['output']
 317         with open(filename, 'r', encoding=locale.getpreferredencoding()) as file:
 318             file_data = file.read()
 319
 320         #
 321         # Sometimes fio informational messages are included at the top of the
 322         # JSON output, especially under Windows. Try to decode output as JSON
 323         # data, lopping off up to the first four lines
 324         #
 325         lines = file_data.splitlines()
 326         for i in range(5):
 327             file_data = '\n'.join(lines[i:])
 328             try:
 329                 self.json_data = json.loads(file_data)
 330             except json.JSONDecodeError:
 331                 continue
 332             else:
 333                 return True
 334
 335         return False
 336
 337     @staticmethod
 338     def check_empty(job):
 339         """
 340         Make sure JSON data is empty.
 341
 342         Some data structures should be empty. This function makes sure that they are.
 343
 344         job         JSON object that we need to check for emptiness
 345         """
 346
 347         return job['total_ios'] == 0 and \
 348                 job['slat_ns']['N'] == 0 and \
 349                 job['clat_ns']['N'] == 0 and \
 350                 job['lat_ns']['N'] == 0
 351
 352     def check_all_ddirs(self, ddir_nonzero, job):
 353         """
 354         Iterate over the data directions and check whether each is
 355         appropriately empty or not.
 356         """
 357
 358         retval = True
 359         ddirlist = ['read', 'write', 'trim']
 360
 361         for ddir in ddirlist:
 362             if ddir in ddir_nonzero:
 363                 if self.check_empty(job[ddir]):
 364                     print(f"Unexpected zero {ddir} data found in output")
 365                     retval = False
 366             else:
 367                 if not self.check_empty(job[ddir]):
 368                     print(f"Unexpected {ddir} data found in output")
 369                     retval = False
 370
 371         return retval
 372
 373
 374 def run_fio_tests(test_list, test_env, args):
 375     """
 376     Run tests as specified in test_list.
 377     """
 378
 379     passed = 0
 380     failed = 0
 381     skipped = 0
 382
 383     for config in test_list:
 384         if (args.skip and config['test_id'] in args.skip) or \
 385            (args.run_only and config['test_id'] not in args.run_only) or \
 386            ('force_skip' in config and config['force_skip']):
 387             skipped = skipped + 1
 388             print(f"Test {config['test_id']} SKIPPED (User request or override)")
 389             continue
 390
 391         if issubclass(config['test_class'], FioJobFileTest):
 392             if config['pre_job']:
 393                 fio_pre_job = os.path.join(test_env['fio_root'], 't', 'jobs',
 394                                            config['pre_job'])
 395             else:
 396                 fio_pre_job = None
 397             if config['pre_success']:
 398                 fio_pre_success = config['pre_success']
 399             else:
 400                 fio_pre_success = None
 401             if 'output_format' in config:
 402                 output_format = config['output_format']
 403             else:
 404                 output_format = 'normal'
 405             test = config['test_class'](
 406                 test_env['fio_path'],
 407                 os.path.join(test_env['fio_root'], 't', 'jobs', config['job']),
 408                 config['success'],
 409                 config['test_id'],
 410                 test_env['artifact_root'],
 411                 fio_pre_job=fio_pre_job,
 412                 fio_pre_success=fio_pre_success,
 413                 output_format=output_format)
 414             desc = config['job']
 415             parameters = []
 416         elif issubclass(config['test_class'], FioJobCmdTest):
 417             if not 'success' in config:
 418                 config['success'] = SUCCESS_DEFAULT
 419             test = config['test_class'](test_env['fio_path'],
 420                                         config['success'],
 421                                         config['test_id'],
 422                                         test_env['artifact_root'],
 423                                         config['fio_opts'],
 424                                         test_env['basename'])
 425             desc = config['test_id']
 426             parameters = config
 427         elif issubclass(config['test_class'], FioExeTest):
 428             exe_path = os.path.join(test_env['fio_root'], config['exe'])
 429             parameters = []
 430             if config['parameters']:
 431                 parameters = [p.format(fio_path=test_env['fio_path'], nvmecdev=args.nvmecdev)
 432                               for p in config['parameters']]
 433             if Path(exe_path).suffix == '.py' and platform.system() == "Windows":
 434                 parameters.insert(0, exe_path)
 435                 exe_path = "python.exe"
 436             if config['test_id'] in test_env['pass_through']:
 437                 parameters += test_env['pass_through'][config['test_id']].split()
 438             test = config['test_class'](
 439                     exe_path,
 440                     config['success'],
 441                     config['test_id'],
 442                     test_env['artifact_root'])
 443             desc = config['exe']
 444         else:
 445             print(f"Test {config['test_id']} FAILED: unable to process test config")
 446             failed = failed + 1
 447             continue
 448
 449         if 'requirements' in config and not args.skip_req:
 450             reqs_met = True
 451             for req in config['requirements']:
 452                 reqs_met, reason = req()
 453                 logging.debug("Test %d: Requirement '%s' met? %s", config['test_id'], reason,
 454                               reqs_met)
 455                 if not reqs_met:
 456                     break
 457             if not reqs_met:
 458                 print(f"Test {config['test_id']} SKIPPED ({reason}) {desc}")
 459                 skipped = skipped + 1
 460                 continue
 461
 462         try:
 463             test.setup(parameters)
 464             test.run()
 465             test.check_result()
 466         except KeyboardInterrupt:
 467             break
 468         except Exception as e:
 469             test.passed = False
 470             test.failure_reason += str(e)
 471             logging.debug("Test %d exception:\n%s\n", config['test_id'], traceback.format_exc())
 472         if test.passed:
 473             result = "PASSED"
 474             passed = passed + 1
 475         else:
 476             result = f"FAILED: {test.failure_reason}"
 477             failed = failed + 1
 478             contents, _ = get_file(test.filenames['stderr'])
 479             logging.debug("Test %d: stderr:\n%s", config['test_id'], contents)
 480             contents, _ = get_file(test.filenames['stdout'])
 481             logging.debug("Test %d: stdout:\n%s", config['test_id'], contents)
 482         print(f"Test {config['test_id']} {result} {desc}")
 483
 484     print(f"{passed} test(s) passed, {failed} failed, {skipped} skipped")
 485
 486     return passed, failed, skipped