t/fiotestlib.py

   1 #!/usr/bin/env python3
   2 """
   3 fiotestlib.py
   4
   5 This library contains FioTest objects that provide convenient means to run
   6 different sorts of fio tests.
   7
   8 It also contains a test runner that runs an array of dictionary objects
   9 describing fio tests.
  10 """
  11
  12 import os
  13 import sys
  14 import json
  15 import locale
  16 import logging
  17 import platform
  18 import traceback
  19 import subprocess
  20 from pathlib import Path
  21 from fiotestcommon import get_file, SUCCESS_DEFAULT
  22
  23
  24 class FioTest():
  25     """Base for all fio tests."""
  26
  27     def __init__(self, exe_path, success, testnum, artifact_root):
  28         self.success = success
  29         self.testnum = testnum
  30         self.output = {}
  31         self.passed = True
  32         self.failure_reason = ''
  33         self.parameters = None
  34         self.paths = {
  35                         'exe': exe_path,
  36                         'artifacts': artifact_root,
  37                         'test_dir': os.path.join(artifact_root, \
  38                                 f"{testnum:04d}"),
  39                         }
  40         self.filenames = {
  41                             'cmd': os.path.join(self.paths['test_dir'], \
  42                                     f"{os.path.basename(self.paths['exe'])}.command"),
  43                             'stdout': os.path.join(self.paths['test_dir'], \
  44                                     f"{os.path.basename(self.paths['exe'])}.stdout"),
  45                             'stderr': os.path.join(self.paths['test_dir'], \
  46                                     f"{os.path.basename(self.paths['exe'])}.stderr"),
  47                             'exitcode': os.path.join(self.paths['test_dir'], \
  48                                     f"{os.path.basename(self.paths['exe'])}.exitcode"),
  49                             }
  50
  51     def setup(self, parameters):
  52         """Setup instance variables for test."""
  53
  54         self.parameters = parameters
  55         if not os.path.exists(self.paths['test_dir']):
  56             os.mkdir(self.paths['test_dir'])
  57
  58     def run(self):
  59         """Run the test."""
  60
  61         raise NotImplementedError()
  62
  63     def check_result(self):
  64         """Check test results."""
  65
  66         raise NotImplementedError()
  67
  68
  69 class FioExeTest(FioTest):
  70     """Test consists of an executable binary or script"""
  71
  72     def run(self):
  73         """Execute the binary or script described by this instance."""
  74
  75         command = [self.paths['exe']] + self.parameters
  76         with open(self.filenames['cmd'], "w+",
  77                   encoding=locale.getpreferredencoding()) as command_file:
  78             command_file.write(" ".join(command))
  79
  80         try:
  81             with open(self.filenames['stdout'], "w+",
  82                       encoding=locale.getpreferredencoding()) as stdout_file, \
  83                 open(self.filenames['stderr'], "w+",
  84                      encoding=locale.getpreferredencoding()) as stderr_file, \
  85                 open(self.filenames['exitcode'], "w+",
  86                      encoding=locale.getpreferredencoding()) as exitcode_file:
  87                 proc = None
  88                 # Avoid using subprocess.run() here because when a timeout occurs,
  89                 # fio will be stopped with SIGKILL. This does not give fio a
  90                 # chance to clean up and means that child processes may continue
  91                 # running and submitting IO.
  92                 proc = subprocess.Popen(command,
  93                                         stdout=stdout_file,
  94                                         stderr=stderr_file,
  95                                         cwd=self.paths['test_dir'],
  96                                         universal_newlines=True)
  97                 proc.communicate(timeout=self.success['timeout'])
  98                 exitcode_file.write(f'{proc.returncode}\n')
  99                 logging.debug("Test %d: return code: %d", self.testnum, proc.returncode)
 100                 self.output['proc'] = proc
 101         except subprocess.TimeoutExpired:
 102             proc.terminate()
 103             proc.communicate()
 104             assert proc.poll()
 105             self.output['failure'] = 'timeout'
 106         except Exception:
 107             if proc:
 108                 if not proc.poll():
 109                     proc.terminate()
 110                     proc.communicate()
 111             self.output['failure'] = 'exception'
 112             self.output['exc_info'] = sys.exc_info()
 113
 114     def check_result(self):
 115         """Check results of test run."""
 116
 117         if 'proc' not in self.output:
 118             if self.output['failure'] == 'timeout':
 119                 self.failure_reason = f"{self.failure_reason} timeout,"
 120             else:
 121                 assert self.output['failure'] == 'exception'
 122                 self.failure_reason = f'{self.failure_reason} exception: ' + \
 123                 f'{self.output["exc_info"][0]}, {self.output["exc_info"][1]}'
 124
 125             self.passed = False
 126             return
 127
 128         if 'zero_return' in self.success:
 129             if self.success['zero_return']:
 130                 if self.output['proc'].returncode != 0:
 131                     self.passed = False
 132                     self.failure_reason = f"{self.failure_reason} non-zero return code,"
 133             else:
 134                 if self.output['proc'].returncode == 0:
 135                     self.failure_reason = f"{self.failure_reason} zero return code,"
 136                     self.passed = False
 137
 138         stderr_size = os.path.getsize(self.filenames['stderr'])
 139         if 'stderr_empty' in self.success:
 140             if self.success['stderr_empty']:
 141                 if stderr_size != 0:
 142                     self.failure_reason = f"{self.failure_reason} stderr not empty,"
 143                     self.passed = False
 144             else:
 145                 if stderr_size == 0:
 146                     self.failure_reason = f"{self.failure_reason} stderr empty,"
 147                     self.passed = False
 148
 149
 150 class FioJobFileTest(FioExeTest):
 151     """Test consists of a fio job with options in a job file."""
 152
 153     def __init__(self, fio_path, fio_job, success, testnum, artifact_root,
 154                  fio_pre_job=None, fio_pre_success=None,
 155                  output_format="normal"):
 156         """Construct a FioJobFileTest which is a FioExeTest consisting of a
 157         single fio job file with an optional setup step.
 158
 159         fio_path:           location of fio executable
 160         fio_job:            location of fio job file
 161         success:            Definition of test success
 162         testnum:            test ID
 163         artifact_root:      root directory for artifacts
 164         fio_pre_job:        fio job for preconditioning
 165         fio_pre_success:    Definition of test success for fio precon job
 166         output_format:      normal (default), json, jsonplus, or terse
 167         """
 168
 169         self.fio_job = fio_job
 170         self.fio_pre_job = fio_pre_job
 171         self.fio_pre_success = fio_pre_success if fio_pre_success else success
 172         self.output_format = output_format
 173         self.precon_failed = False
 174         self.json_data = None
 175
 176         super().__init__(fio_path, success, testnum, artifact_root)
 177
 178     def setup(self, parameters=None):
 179         """Setup instance variables for fio job test."""
 180
 181         self.filenames['fio_output'] = f"{os.path.basename(self.fio_job)}.output"
 182         fio_args = [
 183             "--max-jobs=16",
 184             f"--output-format={self.output_format}",
 185             f"--output={self.filenames['fio_output']}",
 186             self.fio_job,
 187             ]
 188
 189         super().setup(fio_args)
 190
 191         # Update the filenames from the default
 192         self.filenames['cmd'] = os.path.join(self.paths['test_dir'],
 193                                              f"{os.path.basename(self.fio_job)}.command")
 194         self.filenames['stdout'] = os.path.join(self.paths['test_dir'],
 195                                                 f"{os.path.basename(self.fio_job)}.stdout")
 196         self.filenames['stderr'] = os.path.join(self.paths['test_dir'],
 197                                                 f"{os.path.basename(self.fio_job)}.stderr")
 198         self.filenames['exitcode'] = os.path.join(self.paths['test_dir'],
 199                                                   f"{os.path.basename(self.fio_job)}.exitcode")
 200
 201     def run_pre_job(self):
 202         """Run fio job precondition step."""
 203
 204         precon = FioJobFileTest(self.paths['exe'], self.fio_pre_job,
 205                             self.fio_pre_success,
 206                             self.testnum,
 207                             self.paths['artifacts'],
 208                             output_format=self.output_format)
 209         precon.setup()
 210         precon.run()
 211         precon.check_result()
 212         self.precon_failed = not precon.passed
 213         self.failure_reason = precon.failure_reason
 214
 215     def run(self):
 216         """Run fio job test."""
 217
 218         if self.fio_pre_job:
 219             self.run_pre_job()
 220
 221         if not self.precon_failed:
 222             super().run()
 223         else:
 224             logging.debug("Test %d: precondition step failed", self.testnum)
 225
 226     def get_file_fail(self, filename):
 227         """Safely read a file and fail the test upon error."""
 228         file_data = None
 229
 230         try:
 231             with open(filename, "r", encoding=locale.getpreferredencoding()) as output_file:
 232                 file_data = output_file.read()
 233         except OSError:
 234             self.failure_reason += f" unable to read file {filename}"
 235             self.passed = False
 236
 237         return file_data
 238
 239     def check_result(self):
 240         """Check fio job results."""
 241
 242         if self.precon_failed:
 243             self.passed = False
 244             self.failure_reason = f"{self.failure_reason} precondition step failed,"
 245             return
 246
 247         super().check_result()
 248
 249         if not self.passed:
 250             return
 251
 252         if 'json' not in self.output_format:
 253             return
 254
 255         file_data = self.get_file_fail(os.path.join(self.paths['test_dir'],
 256                                                     self.filenames['fio_output']))
 257         if not file_data:
 258             return
 259
 260         #
 261         # Sometimes fio informational messages are included at the top of the
 262         # JSON output, especially under Windows. Try to decode output as JSON
 263         # data, skipping everything until the first {
 264         #
 265         lines = file_data.splitlines()
 266         file_data = '\n'.join(lines[lines.index("{"):])
 267         try:
 268             self.json_data = json.loads(file_data)
 269         except json.JSONDecodeError:
 270             self.failure_reason = f"{self.failure_reason} unable to decode JSON data,"
 271             self.passed = False
 272
 273
 274 class FioJobCmdTest(FioExeTest):
 275     """This runs a fio job with options specified on the command line."""
 276
 277     def __init__(self, fio_path, success, testnum, artifact_root, fio_opts, basename=None):
 278
 279         self.basename = basename if basename else os.path.basename(fio_path)
 280         self.fio_opts = fio_opts
 281         self.json_data = None
 282         self.iops_log_lines = None
 283
 284         super().__init__(fio_path, success, testnum, artifact_root)
 285
 286         filename_stub = os.path.join(self.paths['test_dir'], f"{self.basename}{self.testnum:03d}")
 287         self.filenames['cmd'] = f"{filename_stub}.command"
 288         self.filenames['stdout'] = f"{filename_stub}.stdout"
 289         self.filenames['stderr'] = f"{filename_stub}.stderr"
 290         self.filenames['output'] = os.path.abspath(f"{filename_stub}.output")
 291         self.filenames['exitcode'] = f"{filename_stub}.exitcode"
 292         self.filenames['iopslog'] = os.path.abspath(f"{filename_stub}")
 293
 294     def run(self):
 295         super().run()
 296
 297         if 'output-format' in self.fio_opts and 'json' in \
 298                 self.fio_opts['output-format']:
 299             if not self.get_json():
 300                 print('Unable to decode JSON data')
 301                 self.passed = False
 302
 303         if any('--write_iops_log=' in param for param in self.parameters):
 304             self.get_iops_log()
 305
 306     def get_iops_log(self):
 307         """Read IOPS log from the first job."""
 308
 309         log_filename = self.filenames['iopslog'] + "_iops.1.log"
 310         with open(log_filename, 'r', encoding=locale.getpreferredencoding()) as iops_file:
 311             self.iops_log_lines = iops_file.read()
 312
 313     def get_json(self):
 314         """Convert fio JSON output into a python JSON object"""
 315
 316         filename = self.filenames['output']
 317         with open(filename, 'r', encoding=locale.getpreferredencoding()) as file:
 318             file_data = file.read()
 319
 320         #
 321         # Sometimes fio informational messages are included at the top of the
 322         # JSON output, especially under Windows. Try to decode output as JSON
 323         # data, lopping off up to the first four lines
 324         #
 325         lines = file_data.splitlines()
 326         for i in range(5):
 327             file_data = '\n'.join(lines[i:])
 328             try:
 329                 self.json_data = json.loads(file_data)
 330             except json.JSONDecodeError:
 331                 continue
 332             else:
 333                 return True
 334
 335         return False
 336
 337     @staticmethod
 338     def check_empty(job):
 339         """
 340         Make sure JSON data is empty.
 341
 342         Some data structures should be empty. This function makes sure that they are.
 343
 344         job         JSON object that we need to check for emptiness
 345         """
 346
 347         return job['total_ios'] == 0 and \
 348                 job['slat_ns']['N'] == 0 and \
 349                 job['clat_ns']['N'] == 0 and \
 350                 job['lat_ns']['N'] == 0
 351
 352     def check_all_ddirs(self, ddir_nonzero, job):
 353         """
 354         Iterate over the data directions and check whether each is
 355         appropriately empty or not.
 356         """
 357
 358         retval = True
 359         ddirlist = ['read', 'write', 'trim']
 360
 361         for ddir in ddirlist:
 362             if ddir in ddir_nonzero:
 363                 if self.check_empty(job[ddir]):
 364                     print(f"Unexpected zero {ddir} data found in output")
 365                     retval = False
 366             else:
 367                 if not self.check_empty(job[ddir]):
 368                     print(f"Unexpected {ddir} data found in output")
 369                     retval = False
 370
 371         return retval
 372
 373
 374 def run_fio_tests(test_list, test_env, args):
 375     """
 376     Run tests as specified in test_list.
 377     """
 378
 379     passed = 0
 380     failed = 0
 381     skipped = 0
 382
 383     for config in test_list:
 384         if (args.skip and config['test_id'] in args.skip) or \
 385            (args.run_only and config['test_id'] not in args.run_only):
 386             skipped = skipped + 1
 387             print(f"Test {config['test_id']} SKIPPED (User request)")
 388             continue
 389
 390         if issubclass(config['test_class'], FioJobFileTest):
 391             if config['pre_job']:
 392                 fio_pre_job = os.path.join(test_env['fio_root'], 't', 'jobs',
 393                                            config['pre_job'])
 394             else:
 395                 fio_pre_job = None
 396             if config['pre_success']:
 397                 fio_pre_success = config['pre_success']
 398             else:
 399                 fio_pre_success = None
 400             if 'output_format' in config:
 401                 output_format = config['output_format']
 402             else:
 403                 output_format = 'normal'
 404             test = config['test_class'](
 405                 test_env['fio_path'],
 406                 os.path.join(test_env['fio_root'], 't', 'jobs', config['job']),
 407                 config['success'],
 408                 config['test_id'],
 409                 test_env['artifact_root'],
 410                 fio_pre_job=fio_pre_job,
 411                 fio_pre_success=fio_pre_success,
 412                 output_format=output_format)
 413             desc = config['job']
 414             parameters = []
 415         elif issubclass(config['test_class'], FioJobCmdTest):
 416             if not 'success' in config:
 417                 config['success'] = SUCCESS_DEFAULT
 418             test = config['test_class'](test_env['fio_path'],
 419                                         config['success'],
 420                                         config['test_id'],
 421                                         test_env['artifact_root'],
 422                                         config['fio_opts'],
 423                                         test_env['basename'])
 424             desc = config['test_id']
 425             parameters = config
 426         elif issubclass(config['test_class'], FioExeTest):
 427             exe_path = os.path.join(test_env['fio_root'], config['exe'])
 428             parameters = []
 429             if config['parameters']:
 430                 parameters = [p.format(fio_path=test_env['fio_path'], nvmecdev=args.nvmecdev)
 431                               for p in config['parameters']]
 432             if Path(exe_path).suffix == '.py' and platform.system() == "Windows":
 433                 parameters.insert(0, exe_path)
 434                 exe_path = "python.exe"
 435             if config['test_id'] in test_env['pass_through']:
 436                 parameters += test_env['pass_through'][config['test_id']].split()
 437             test = config['test_class'](
 438                     exe_path,
 439                     config['success'],
 440                     config['test_id'],
 441                     test_env['artifact_root'])
 442             desc = config['exe']
 443         else:
 444             print(f"Test {config['test_id']} FAILED: unable to process test config")
 445             failed = failed + 1
 446             continue
 447
 448         if 'requirements' in config and not args.skip_req:
 449             reqs_met = True
 450             for req in config['requirements']:
 451                 reqs_met, reason = req()
 452                 logging.debug("Test %d: Requirement '%s' met? %s", config['test_id'], reason,
 453                               reqs_met)
 454                 if not reqs_met:
 455                     break
 456             if not reqs_met:
 457                 print(f"Test {config['test_id']} SKIPPED ({reason}) {desc}")
 458                 skipped = skipped + 1
 459                 continue
 460
 461         try:
 462             test.setup(parameters)
 463             test.run()
 464             test.check_result()
 465         except KeyboardInterrupt:
 466             break
 467         except Exception as e:
 468             test.passed = False
 469             test.failure_reason += str(e)
 470             logging.debug("Test %d exception:\n%s\n", config['test_id'], traceback.format_exc())
 471         if test.passed:
 472             result = "PASSED"
 473             passed = passed + 1
 474         else:
 475             result = f"FAILED: {test.failure_reason}"
 476             failed = failed + 1
 477             contents, _ = get_file(test.filenames['stderr'])
 478             logging.debug("Test %d: stderr:\n%s", config['test_id'], contents)
 479             contents, _ = get_file(test.filenames['stdout'])
 480             logging.debug("Test %d: stdout:\n%s", config['test_id'], contents)
 481         print(f"Test {config['test_id']} {result} {desc}")
 482
 483     print(f"{passed} test(s) passed, {failed} failed, {skipped} skipped")
 484
 485     return passed, failed, skipped