t/fiotestlib.py

   1 #!/usr/bin/env python3
   2 """
   3 fiotestlib.py
   4
   5 This library contains FioTest objects that provide convenient means to run
   6 different sorts of fio tests.
   7
   8 It also contains a test runner that runs an array of dictionary objects
   9 describing fio tests.
  10 """
  11
  12 import os
  13 import sys
  14 import json
  15 import locale
  16 import logging
  17 import platform
  18 import traceback
  19 import subprocess
  20 from pathlib import Path
  21 from fiotestcommon import get_file
  22
  23
  24 class FioTest():
  25     """Base for all fio tests."""
  26
  27     def __init__(self, exe_path, success, testnum, artifact_root):
  28         self.success = success
  29         self.testnum = testnum
  30         self.output = {}
  31         self.passed = True
  32         self.failure_reason = ''
  33         self.parameters = None
  34         self.paths = {
  35                         'exe': exe_path,
  36                         'artifacts': artifact_root,
  37                         'test_dir': os.path.join(artifact_root, \
  38                                 f"{testnum:04d}"),
  39                         }
  40         self.filenames = {
  41                             'cmd': os.path.join(self.paths['test_dir'], \
  42                                     f"{os.path.basename(self.paths['exe'])}.command"),
  43                             'stdout': os.path.join(self.paths['test_dir'], \
  44                                     f"{os.path.basename(self.paths['exe'])}.stdout"),
  45                             'stderr': os.path.join(self.paths['test_dir'], \
  46                                     f"{os.path.basename(self.paths['exe'])}.stderr"),
  47                             'exitcode': os.path.join(self.paths['test_dir'], \
  48                                     f"{os.path.basename(self.paths['exe'])}.exitcode"),
  49                             }
  50
  51     def setup(self, parameters):
  52         """Setup instance variables for test."""
  53
  54         self.parameters = parameters
  55         if not os.path.exists(self.paths['test_dir']):
  56             os.mkdir(self.paths['test_dir'])
  57
  58     def run(self):
  59         """Run the test."""
  60
  61         raise NotImplementedError()
  62
  63     def check_result(self):
  64         """Check test results."""
  65
  66         raise NotImplementedError()
  67
  68
  69 class FioExeTest(FioTest):
  70     """Test consists of an executable binary or script"""
  71
  72     def run(self):
  73         """Execute the binary or script described by this instance."""
  74
  75         command = [self.paths['exe']] + self.parameters
  76         with open(self.filenames['cmd'], "w+",
  77                   encoding=locale.getpreferredencoding()) as command_file:
  78             command_file.write(" ".join(command))
  79
  80         try:
  81             with open(self.filenames['stdout'], "w+",
  82                       encoding=locale.getpreferredencoding()) as stdout_file, \
  83                 open(self.filenames['stderr'], "w+",
  84                      encoding=locale.getpreferredencoding()) as stderr_file, \
  85                 open(self.filenames['exitcode'], "w+",
  86                      encoding=locale.getpreferredencoding()) as exitcode_file:
  87                 proc = None
  88                 # Avoid using subprocess.run() here because when a timeout occurs,
  89                 # fio will be stopped with SIGKILL. This does not give fio a
  90                 # chance to clean up and means that child processes may continue
  91                 # running and submitting IO.
  92                 proc = subprocess.Popen(command,
  93                                         stdout=stdout_file,
  94                                         stderr=stderr_file,
  95                                         cwd=self.paths['test_dir'],
  96                                         universal_newlines=True)
  97                 proc.communicate(timeout=self.success['timeout'])
  98                 exitcode_file.write(f'{proc.returncode}\n')
  99                 logging.debug("Test %d: return code: %d", self.testnum, proc.returncode)
 100                 self.output['proc'] = proc
 101         except subprocess.TimeoutExpired:
 102             proc.terminate()
 103             proc.communicate()
 104             assert proc.poll()
 105             self.output['failure'] = 'timeout'
 106         except Exception:
 107             if proc:
 108                 if not proc.poll():
 109                     proc.terminate()
 110                     proc.communicate()
 111             self.output['failure'] = 'exception'
 112             self.output['exc_info'] = sys.exc_info()
 113
 114     def check_result(self):
 115         """Check results of test run."""
 116
 117         if 'proc' not in self.output:
 118             if self.output['failure'] == 'timeout':
 119                 self.failure_reason = f"{self.failure_reason} timeout,"
 120             else:
 121                 assert self.output['failure'] == 'exception'
 122                 self.failure_reason = f'{self.failure_reason} exception: ' + \
 123                 f'{self.output["exc_info"][0]}, {self.output["exc_info"][1]}'
 124
 125             self.passed = False
 126             return
 127
 128         if 'zero_return' in self.success:
 129             if self.success['zero_return']:
 130                 if self.output['proc'].returncode != 0:
 131                     self.passed = False
 132                     self.failure_reason = f"{self.failure_reason} non-zero return code,"
 133             else:
 134                 if self.output['proc'].returncode == 0:
 135                     self.failure_reason = f"{self.failure_reason} zero return code,"
 136                     self.passed = False
 137
 138         stderr_size = os.path.getsize(self.filenames['stderr'])
 139         if 'stderr_empty' in self.success:
 140             if self.success['stderr_empty']:
 141                 if stderr_size != 0:
 142                     self.failure_reason = f"{self.failure_reason} stderr not empty,"
 143                     self.passed = False
 144             else:
 145                 if stderr_size == 0:
 146                     self.failure_reason = f"{self.failure_reason} stderr empty,"
 147                     self.passed = False
 148
 149
 150 class FioJobFileTest(FioExeTest):
 151     """Test consists of a fio job with options in a job file."""
 152
 153     def __init__(self, fio_path, fio_job, success, testnum, artifact_root,
 154                  fio_pre_job=None, fio_pre_success=None,
 155                  output_format="normal"):
 156         """Construct a FioJobFileTest which is a FioExeTest consisting of a
 157         single fio job file with an optional setup step.
 158
 159         fio_path:           location of fio executable
 160         fio_job:            location of fio job file
 161         success:            Definition of test success
 162         testnum:            test ID
 163         artifact_root:      root directory for artifacts
 164         fio_pre_job:        fio job for preconditioning
 165         fio_pre_success:    Definition of test success for fio precon job
 166         output_format:      normal (default), json, jsonplus, or terse
 167         """
 168
 169         self.fio_job = fio_job
 170         self.fio_pre_job = fio_pre_job
 171         self.fio_pre_success = fio_pre_success if fio_pre_success else success
 172         self.output_format = output_format
 173         self.precon_failed = False
 174         self.json_data = None
 175
 176         super().__init__(fio_path, success, testnum, artifact_root)
 177
 178     def setup(self, parameters=None):
 179         """Setup instance variables for fio job test."""
 180
 181         self.filenames['fio_output'] = f"{os.path.basename(self.fio_job)}.output"
 182         fio_args = [
 183             "--max-jobs=16",
 184             f"--output-format={self.output_format}",
 185             f"--output={self.filenames['fio_output']}",
 186             self.fio_job,
 187             ]
 188
 189         super().setup(fio_args)
 190
 191         # Update the filenames from the default
 192         self.filenames['cmd'] = os.path.join(self.paths['test_dir'],
 193                                              f"{os.path.basename(self.fio_job)}.command")
 194         self.filenames['stdout'] = os.path.join(self.paths['test_dir'],
 195                                                 f"{os.path.basename(self.fio_job)}.stdout")
 196         self.filenames['stderr'] = os.path.join(self.paths['test_dir'],
 197                                                 f"{os.path.basename(self.fio_job)}.stderr")
 198         self.filenames['exitcode'] = os.path.join(self.paths['test_dir'],
 199                                                   f"{os.path.basename(self.fio_job)}.exitcode")
 200
 201     def run_pre_job(self):
 202         """Run fio job precondition step."""
 203
 204         precon = FioJobFileTest(self.paths['exe'], self.fio_pre_job,
 205                             self.fio_pre_success,
 206                             self.testnum,
 207                             self.paths['artifacts'],
 208                             output_format=self.output_format)
 209         precon.setup()
 210         precon.run()
 211         precon.check_result()
 212         self.precon_failed = not precon.passed
 213         self.failure_reason = precon.failure_reason
 214
 215     def run(self):
 216         """Run fio job test."""
 217
 218         if self.fio_pre_job:
 219             self.run_pre_job()
 220
 221         if not self.precon_failed:
 222             super().run()
 223         else:
 224             logging.debug("Test %d: precondition step failed", self.testnum)
 225
 226     def get_file_fail(self, filename):
 227         """Safely read a file and fail the test upon error."""
 228         file_data = None
 229
 230         try:
 231             with open(filename, "r", encoding=locale.getpreferredencoding()) as output_file:
 232                 file_data = output_file.read()
 233         except OSError:
 234             self.failure_reason += f" unable to read file {filename}"
 235             self.passed = False
 236
 237         return file_data
 238
 239     def check_result(self):
 240         """Check fio job results."""
 241
 242         if self.precon_failed:
 243             self.passed = False
 244             self.failure_reason = f"{self.failure_reason} precondition step failed,"
 245             return
 246
 247         super().check_result()
 248
 249         if not self.passed:
 250             return
 251
 252         if 'json' not in self.output_format:
 253             return
 254
 255         file_data = self.get_file_fail(os.path.join(self.paths['test_dir'],
 256                                                     self.filenames['fio_output']))
 257         if not file_data:
 258             return
 259
 260         #
 261         # Sometimes fio informational messages are included at the top of the
 262         # JSON output, especially under Windows. Try to decode output as JSON
 263         # data, skipping everything until the first {
 264         #
 265         lines = file_data.splitlines()
 266         file_data = '\n'.join(lines[lines.index("{"):])
 267         try:
 268             self.json_data = json.loads(file_data)
 269         except json.JSONDecodeError:
 270             self.failure_reason = f"{self.failure_reason} unable to decode JSON data,"
 271             self.passed = False
 272
 273
 274 def run_fio_tests(test_list, test_env, args):
 275     """
 276     Run tests as specified in test_list.
 277     """
 278
 279     passed = 0
 280     failed = 0
 281     skipped = 0
 282
 283     for config in test_list:
 284         if (args.skip and config['test_id'] in args.skip) or \
 285            (args.run_only and config['test_id'] not in args.run_only):
 286             skipped = skipped + 1
 287             print(f"Test {config['test_id']} SKIPPED (User request)")
 288             continue
 289
 290         if issubclass(config['test_class'], FioJobFileTest):
 291             if config['pre_job']:
 292                 fio_pre_job = os.path.join(test_env['fio_root'], 't', 'jobs',
 293                                            config['pre_job'])
 294             else:
 295                 fio_pre_job = None
 296             if config['pre_success']:
 297                 fio_pre_success = config['pre_success']
 298             else:
 299                 fio_pre_success = None
 300             if 'output_format' in config:
 301                 output_format = config['output_format']
 302             else:
 303                 output_format = 'normal'
 304             test = config['test_class'](
 305                 test_env['fio_path'],
 306                 os.path.join(test_env['fio_root'], 't', 'jobs', config['job']),
 307                 config['success'],
 308                 config['test_id'],
 309                 test_env['artifact_root'],
 310                 fio_pre_job=fio_pre_job,
 311                 fio_pre_success=fio_pre_success,
 312                 output_format=output_format)
 313             desc = config['job']
 314             parameters = []
 315         elif issubclass(config['test_class'], FioExeTest):
 316             exe_path = os.path.join(test_env['fio_root'], config['exe'])
 317             parameters = []
 318             if config['parameters']:
 319                 parameters = [p.format(fio_path=test_env['fio_path'], nvmecdev=args.nvmecdev)
 320                               for p in config['parameters']]
 321             if Path(exe_path).suffix == '.py' and platform.system() == "Windows":
 322                 parameters.insert(0, exe_path)
 323                 exe_path = "python.exe"
 324             if config['test_id'] in test_env['pass_through']:
 325                 parameters += test_env['pass_through'][config['test_id']].split()
 326             test = config['test_class'](
 327                     exe_path,
 328                     config['success'],
 329                     config['test_id'],
 330                     test_env['artifact_root'])
 331             desc = config['exe']
 332         else:
 333             print(f"Test {config['test_id']} FAILED: unable to process test config")
 334             failed = failed + 1
 335             continue
 336
 337         if not args.skip_req:
 338             reqs_met = True
 339             for req in config['requirements']:
 340                 reqs_met, reason = req()
 341                 logging.debug("Test %d: Requirement '%s' met? %s", config['test_id'], reason,
 342                               reqs_met)
 343                 if not reqs_met:
 344                     break
 345             if not reqs_met:
 346                 print(f"Test {config['test_id']} SKIPPED ({reason}) {desc}")
 347                 skipped = skipped + 1
 348                 continue
 349
 350         try:
 351             test.setup(parameters)
 352             test.run()
 353             test.check_result()
 354         except KeyboardInterrupt:
 355             break
 356         except Exception as e:
 357             test.passed = False
 358             test.failure_reason += str(e)
 359             logging.debug("Test %d exception:\n%s\n", config['test_id'], traceback.format_exc())
 360         if test.passed:
 361             result = "PASSED"
 362             passed = passed + 1
 363         else:
 364             result = f"FAILED: {test.failure_reason}"
 365             failed = failed + 1
 366             contents, _ = get_file(test.filenames['stderr'])
 367             logging.debug("Test %d: stderr:\n%s", config['test_id'], contents)
 368             contents, _ = get_file(test.filenames['stdout'])
 369             logging.debug("Test %d: stdout:\n%s", config['test_id'], contents)
 370         print(f"Test {config['test_id']} {result} {desc}")
 371
 372     print(f"{passed} test(s) passed, {failed} failed, {skipped} skipped")
 373
 374     return passed, failed, skipped