t/fiotestlib.py

   1 #!/usr/bin/env python3
   2 """
   3 fiotestlib.py
   4
   5 This library contains FioTest objects that provide convenient means to run
   6 different sorts of fio tests.
   7
   8 It also contains a test runner that runs an array of dictionary objects
   9 describing fio tests.
  10 """
  11
  12 import os
  13 import sys
  14 import json
  15 import locale
  16 import logging
  17 import platform
  18 import traceback
  19 import subprocess
  20 from pathlib import Path
  21 from fiotestcommon import get_file, SUCCESS_DEFAULT
  22
  23
  24 class FioTest():
  25     """Base for all fio tests."""
  26
  27     def __init__(self, exe_path, success, testnum, artifact_root):
  28         self.success = success
  29         self.testnum = testnum
  30         self.output = {}
  31         self.passed = True
  32         self.failure_reason = ''
  33         self.parameters = None
  34         self.paths = {
  35                         'exe': exe_path,
  36                         'artifacts': artifact_root,
  37                         'test_dir': os.path.join(artifact_root, \
  38                                 f"{testnum:04d}"),
  39                         }
  40         self.filenames = {
  41                             'cmd': os.path.join(self.paths['test_dir'], \
  42                                     f"{os.path.basename(self.paths['exe'])}.command"),
  43                             'stdout': os.path.join(self.paths['test_dir'], \
  44                                     f"{os.path.basename(self.paths['exe'])}.stdout"),
  45                             'stderr': os.path.join(self.paths['test_dir'], \
  46                                     f"{os.path.basename(self.paths['exe'])}.stderr"),
  47                             'exitcode': os.path.join(self.paths['test_dir'], \
  48                                     f"{os.path.basename(self.paths['exe'])}.exitcode"),
  49                             }
  50
  51     def setup(self, parameters):
  52         """Setup instance variables for test."""
  53
  54         self.parameters = parameters
  55         if not os.path.exists(self.paths['test_dir']):
  56             os.mkdir(self.paths['test_dir'])
  57
  58     def run(self):
  59         """Run the test."""
  60
  61         raise NotImplementedError()
  62
  63     def check_result(self):
  64         """Check test results."""
  65
  66         raise NotImplementedError()
  67
  68
  69 class FioExeTest(FioTest):
  70     """Test consists of an executable binary or script"""
  71
  72     def run(self):
  73         """Execute the binary or script described by this instance."""
  74
  75         command = [self.paths['exe']] + self.parameters
  76         with open(self.filenames['cmd'], "w+",
  77                   encoding=locale.getpreferredencoding()) as command_file:
  78             command_file.write(" \\\n ".join(command))
  79
  80         try:
  81             with open(self.filenames['stdout'], "w+",
  82                       encoding=locale.getpreferredencoding()) as stdout_file, \
  83                 open(self.filenames['stderr'], "w+",
  84                      encoding=locale.getpreferredencoding()) as stderr_file, \
  85                 open(self.filenames['exitcode'], "w+",
  86                      encoding=locale.getpreferredencoding()) as exitcode_file:
  87                 proc = None
  88                 # Avoid using subprocess.run() here because when a timeout occurs,
  89                 # fio will be stopped with SIGKILL. This does not give fio a
  90                 # chance to clean up and means that child processes may continue
  91                 # running and submitting IO.
  92                 proc = subprocess.Popen(command,
  93                                         stdout=stdout_file,
  94                                         stderr=stderr_file,
  95                                         cwd=self.paths['test_dir'],
  96                                         universal_newlines=True)
  97                 proc.communicate(timeout=self.success['timeout'])
  98                 exitcode_file.write(f'{proc.returncode}\n')
  99                 logging.debug("Test %d: return code: %d", self.testnum, proc.returncode)
 100                 self.output['proc'] = proc
 101         except subprocess.TimeoutExpired:
 102             proc.terminate()
 103             proc.communicate()
 104             assert proc.poll()
 105             self.output['failure'] = 'timeout'
 106         except Exception:
 107             if proc:
 108                 if not proc.poll():
 109                     proc.terminate()
 110                     proc.communicate()
 111             self.output['failure'] = 'exception'
 112             self.output['exc_info'] = sys.exc_info()
 113
 114     def check_result(self):
 115         """Check results of test run."""
 116
 117         if 'proc' not in self.output:
 118             if self.output['failure'] == 'timeout':
 119                 self.failure_reason = f"{self.failure_reason} timeout,"
 120             else:
 121                 assert self.output['failure'] == 'exception'
 122                 self.failure_reason = f'{self.failure_reason} exception: ' + \
 123                 f'{self.output["exc_info"][0]}, {self.output["exc_info"][1]}'
 124
 125             self.passed = False
 126             return
 127
 128         if 'zero_return' in self.success:
 129             if self.success['zero_return']:
 130                 if self.output['proc'].returncode != 0:
 131                     self.passed = False
 132                     self.failure_reason = f"{self.failure_reason} non-zero return code,"
 133             else:
 134                 if self.output['proc'].returncode == 0:
 135                     self.failure_reason = f"{self.failure_reason} zero return code,"
 136                     self.passed = False
 137
 138         stderr_size = os.path.getsize(self.filenames['stderr'])
 139         if 'stderr_empty' in self.success:
 140             if self.success['stderr_empty']:
 141                 if stderr_size != 0:
 142                     self.failure_reason = f"{self.failure_reason} stderr not empty size {stderr_size},"
 143                     self.passed = False
 144             else:
 145                 if stderr_size == 0:
 146                     self.failure_reason = f"{self.failure_reason} stderr empty,"
 147                     self.passed = False
 148
 149
 150 class FioJobFileTest(FioExeTest):
 151     """Test consists of a fio job with options in a job file."""
 152
 153     def __init__(self, fio_path, fio_job, success, testnum, artifact_root,
 154                  fio_pre_job=None, fio_pre_success=None,
 155                  output_format="normal"):
 156         """Construct a FioJobFileTest which is a FioExeTest consisting of a
 157         single fio job file with an optional setup step.
 158
 159         fio_path:           location of fio executable
 160         fio_job:            location of fio job file
 161         success:            Definition of test success
 162         testnum:            test ID
 163         artifact_root:      root directory for artifacts
 164         fio_pre_job:        fio job for preconditioning
 165         fio_pre_success:    Definition of test success for fio precon job
 166         output_format:      normal (default), json, jsonplus, or terse
 167         """
 168
 169         self.fio_job = fio_job
 170         self.fio_pre_job = fio_pre_job
 171         self.fio_pre_success = fio_pre_success if fio_pre_success else success
 172         self.output_format = output_format
 173         self.precon_failed = False
 174         self.json_data = None
 175
 176         super().__init__(fio_path, success, testnum, artifact_root)
 177
 178     def setup(self, parameters):
 179         """Setup instance variables for fio job test."""
 180
 181         self.filenames['fio_output'] = f"{os.path.basename(self.fio_job)}.output"
 182         fio_args = [
 183             "--max-jobs=16",
 184             f"--output-format={self.output_format}",
 185             f"--output={self.filenames['fio_output']}",
 186             self.fio_job,
 187             ]
 188         if parameters:
 189             fio_args += parameters
 190
 191         super().setup(fio_args)
 192
 193         # Update the filenames from the default
 194         self.filenames['cmd'] = os.path.join(self.paths['test_dir'],
 195                                              f"{os.path.basename(self.fio_job)}.command")
 196         self.filenames['stdout'] = os.path.join(self.paths['test_dir'],
 197                                                 f"{os.path.basename(self.fio_job)}.stdout")
 198         self.filenames['stderr'] = os.path.join(self.paths['test_dir'],
 199                                                 f"{os.path.basename(self.fio_job)}.stderr")
 200         self.filenames['exitcode'] = os.path.join(self.paths['test_dir'],
 201                                                   f"{os.path.basename(self.fio_job)}.exitcode")
 202
 203     def run_pre_job(self):
 204         """Run fio job precondition step."""
 205
 206         precon = FioJobFileTest(self.paths['exe'], self.fio_pre_job,
 207                             self.fio_pre_success,
 208                             self.testnum,
 209                             self.paths['artifacts'],
 210                             output_format=self.output_format)
 211         precon.setup(None)
 212         precon.run()
 213         precon.check_result()
 214         self.precon_failed = not precon.passed
 215         self.failure_reason = precon.failure_reason
 216
 217     def run(self):
 218         """Run fio job test."""
 219
 220         if self.fio_pre_job:
 221             self.run_pre_job()
 222
 223         if not self.precon_failed:
 224             super().run()
 225         else:
 226             logging.debug("Test %d: precondition step failed", self.testnum)
 227
 228     def get_file_fail(self, filename):
 229         """Safely read a file and fail the test upon error."""
 230         file_data = None
 231
 232         try:
 233             with open(filename, "r", encoding=locale.getpreferredencoding()) as output_file:
 234                 file_data = output_file.read()
 235         except OSError:
 236             self.failure_reason += f" unable to read file {filename}"
 237             self.passed = False
 238
 239         return file_data
 240
 241     def check_result(self):
 242         """Check fio job results."""
 243
 244         if self.precon_failed:
 245             self.passed = False
 246             self.failure_reason = f"{self.failure_reason} precondition step failed,"
 247             return
 248
 249         super().check_result()
 250
 251         if not self.passed:
 252             return
 253
 254         if 'json' not in self.output_format:
 255             return
 256
 257         file_data = self.get_file_fail(os.path.join(self.paths['test_dir'],
 258                                                     self.filenames['fio_output']))
 259         if not file_data:
 260             return
 261
 262         #
 263         # Sometimes fio informational messages are included outside the JSON
 264         # output, especially under Windows. Try to decode output as JSON data,
 265         # skipping outside the first { and last }
 266         #
 267         lines = file_data.splitlines()
 268         last = len(lines) - lines[::-1].index("}")
 269         file_data = '\n'.join(lines[lines.index("{"):last])
 270         try:
 271             self.json_data = json.loads(file_data)
 272         except json.JSONDecodeError:
 273             self.failure_reason = f"{self.failure_reason} unable to decode JSON data,"
 274             self.passed = False
 275
 276
 277 class FioJobCmdTest(FioExeTest):
 278     """This runs a fio job with options specified on the command line."""
 279
 280     def __init__(self, fio_path, success, testnum, artifact_root, fio_opts, basename=None):
 281
 282         self.basename = basename if basename else os.path.basename(fio_path)
 283         self.fio_opts = fio_opts
 284         self.json_data = None
 285         self.iops_log_lines = None
 286
 287         super().__init__(fio_path, success, testnum, artifact_root)
 288
 289         filename_stub = os.path.join(self.paths['test_dir'], f"{self.basename}{self.testnum:03d}")
 290         self.filenames['cmd'] = f"{filename_stub}.command"
 291         self.filenames['stdout'] = f"{filename_stub}.stdout"
 292         self.filenames['stderr'] = f"{filename_stub}.stderr"
 293         self.filenames['output'] = os.path.abspath(f"{filename_stub}.output")
 294         self.filenames['exitcode'] = f"{filename_stub}.exitcode"
 295         self.filenames['iopslog'] = os.path.abspath(f"{filename_stub}")
 296
 297     def run(self):
 298         super().run()
 299
 300         if 'output-format' in self.fio_opts and 'json' in \
 301                 self.fio_opts['output-format']:
 302             if not self.get_json():
 303                 print('Unable to decode JSON data')
 304                 self.passed = False
 305
 306         if any('--write_iops_log=' in param for param in self.parameters):
 307             self.get_iops_log()
 308
 309     def get_iops_log(self):
 310         """Read IOPS log from the first job."""
 311
 312         log_filename = self.filenames['iopslog'] + "_iops.1.log"
 313         with open(log_filename, 'r', encoding=locale.getpreferredencoding()) as iops_file:
 314             self.iops_log_lines = iops_file.read()
 315
 316     def get_json(self):
 317         """Convert fio JSON output into a python JSON object"""
 318
 319         filename = self.filenames['output']
 320         with open(filename, 'r', encoding=locale.getpreferredencoding()) as file:
 321             file_data = file.read()
 322
 323         #
 324         # Sometimes fio informational messages are included outside the JSON
 325         # output, especially under Windows. Try to decode output as JSON data,
 326         # skipping outside the first { and last }
 327         #
 328         lines = file_data.splitlines()
 329         last = len(lines) - lines[::-1].index("}")
 330         file_data = '\n'.join(lines[lines.index("{"):last])
 331         try:
 332             self.json_data = json.loads(file_data)
 333         except json.JSONDecodeError:
 334             return False
 335
 336         return True
 337
 338     @staticmethod
 339     def check_empty(job):
 340         """
 341         Make sure JSON data is empty.
 342
 343         Some data structures should be empty. This function makes sure that they are.
 344
 345         job         JSON object that we need to check for emptiness
 346         """
 347
 348         return job['total_ios'] == 0 and \
 349                 job['slat_ns']['N'] == 0 and \
 350                 job['clat_ns']['N'] == 0 and \
 351                 job['lat_ns']['N'] == 0
 352
 353     def check_all_ddirs(self, ddir_nonzero, job):
 354         """
 355         Iterate over the data directions and check whether each is
 356         appropriately empty or not.
 357         """
 358
 359         retval = True
 360         ddirlist = ['read', 'write', 'trim']
 361
 362         for ddir in ddirlist:
 363             if ddir in ddir_nonzero:
 364                 if self.check_empty(job[ddir]):
 365                     print(f"Unexpected zero {ddir} data found in output")
 366                     retval = False
 367             else:
 368                 if not self.check_empty(job[ddir]):
 369                     print(f"Unexpected {ddir} data found in output")
 370                     retval = False
 371
 372         return retval
 373
 374
 375 def run_fio_tests(test_list, test_env, args):
 376     """
 377     Run tests as specified in test_list.
 378     """
 379
 380     passed = 0
 381     failed = 0
 382     skipped = 0
 383
 384     for config in test_list:
 385         if (args.skip and config['test_id'] in args.skip) or \
 386            (args.run_only and config['test_id'] not in args.run_only) or \
 387            ('force_skip' in config and config['force_skip']):
 388             skipped = skipped + 1
 389             print(f"Test {config['test_id']} SKIPPED (User request or override)")
 390             continue
 391
 392         if issubclass(config['test_class'], FioJobFileTest):
 393             if config['pre_job']:
 394                 fio_pre_job = os.path.join(test_env['fio_root'], 't', 'jobs',
 395                                            config['pre_job'])
 396             else:
 397                 fio_pre_job = None
 398             if config['pre_success']:
 399                 fio_pre_success = config['pre_success']
 400             else:
 401                 fio_pre_success = None
 402             if 'output_format' in config:
 403                 output_format = config['output_format']
 404             else:
 405                 output_format = 'normal'
 406             test = config['test_class'](
 407                 test_env['fio_path'],
 408                 os.path.join(test_env['fio_root'], 't', 'jobs', config['job']),
 409                 config['success'],
 410                 config['test_id'],
 411                 test_env['artifact_root'],
 412                 fio_pre_job=fio_pre_job,
 413                 fio_pre_success=fio_pre_success,
 414                 output_format=output_format)
 415             desc = config['job']
 416             parameters = config['parameters'] if 'parameters' in config else None
 417         elif issubclass(config['test_class'], FioJobCmdTest):
 418             if not 'success' in config:
 419                 config['success'] = SUCCESS_DEFAULT
 420             test = config['test_class'](test_env['fio_path'],
 421                                         config['success'],
 422                                         config['test_id'],
 423                                         test_env['artifact_root'],
 424                                         config['fio_opts'],
 425                                         test_env['basename'])
 426             desc = config['test_id']
 427             parameters = config
 428         elif issubclass(config['test_class'], FioExeTest):
 429             exe_path = os.path.join(test_env['fio_root'], config['exe'])
 430             parameters = []
 431             if config['parameters']:
 432                 parameters = [p.format(fio_path=test_env['fio_path'], nvmecdev=args.nvmecdev)
 433                               for p in config['parameters']]
 434             if Path(exe_path).suffix == '.py' and platform.system() == "Windows":
 435                 parameters.insert(0, exe_path)
 436                 exe_path = "python.exe"
 437             if config['test_id'] in test_env['pass_through']:
 438                 parameters += test_env['pass_through'][config['test_id']].split()
 439             test = config['test_class'](
 440                     exe_path,
 441                     config['success'],
 442                     config['test_id'],
 443                     test_env['artifact_root'])
 444             desc = config['exe']
 445         else:
 446             print(f"Test {config['test_id']} FAILED: unable to process test config")
 447             failed = failed + 1
 448             continue
 449
 450         if 'requirements' in config and not args.skip_req:
 451             reqs_met = True
 452             for req in config['requirements']:
 453                 reqs_met, reason = req()
 454                 logging.debug("Test %d: Requirement '%s' met? %s", config['test_id'], reason,
 455                               reqs_met)
 456                 if not reqs_met:
 457                     break
 458             if not reqs_met:
 459                 print(f"Test {config['test_id']} SKIPPED ({reason}) {desc}")
 460                 skipped = skipped + 1
 461                 continue
 462
 463         try:
 464             test.setup(parameters)
 465             test.run()
 466             test.check_result()
 467         except KeyboardInterrupt:
 468             break
 469         except Exception as e:
 470             test.passed = False
 471             test.failure_reason += str(e)
 472             logging.debug("Test %d exception:\n%s\n", config['test_id'], traceback.format_exc())
 473         if test.passed:
 474             result = "PASSED"
 475             passed = passed + 1
 476         else:
 477             result = f"FAILED: {test.failure_reason}"
 478             failed = failed + 1
 479             contents, _ = get_file(test.filenames['stderr'])
 480             logging.debug("Test %d: stderr:\n%s", config['test_id'], contents)
 481             contents, _ = get_file(test.filenames['stdout'])
 482             logging.debug("Test %d: stdout:\n%s", config['test_id'], contents)
 483         print(f"Test {config['test_id']} {result} {desc}")
 484
 485     print(f"{passed} test(s) passed, {failed} failed, {skipped} skipped")
 486
 487     return passed, failed, skipped