t/fiotestlib.py

   1 #!/usr/bin/env python3
   2 """
   3 fiotestlib.py
   4
   5 This library contains FioTest objects that provide convenient means to run
   6 different sorts of fio tests.
   7
   8 It also contains a test runner that runs an array of dictionary objects
   9 describing fio tests.
  10 """
  11
  12 import os
  13 import sys
  14 import json
  15 import locale
  16 import logging
  17 import platform
  18 import traceback
  19 import subprocess
  20 from pathlib import Path
  21 from fiotestcommon import get_file
  22
  23
  24 class FioTest():
  25     """Base for all fio tests."""
  26
  27     def __init__(self, exe_path, success):
  28         self.paths = {'exe': exe_path}
  29         self.success = success
  30         self.output = {}
  31         self.testnum = None
  32         self.passed = True
  33         self.failure_reason = ''
  34         self.filenames = {}
  35         self.parameters = None
  36
  37     def setup(self, artifact_root, testnum, parameters):
  38         """Setup instance variables for test."""
  39
  40         self.testnum = testnum
  41         self.parameters = parameters
  42         self.paths['artifacts'] = artifact_root
  43         self.paths['test_dir'] = os.path.join(artifact_root, f"{testnum:04d}")
  44         if not os.path.exists(self.paths['test_dir']):
  45             os.mkdir(self.paths['test_dir'])
  46
  47         self.filenames['cmd'] = os.path.join(self.paths['test_dir'],
  48                                              f"{os.path.basename(self.paths['exe'])}.command")
  49         self.filenames['stdout'] = os.path.join(self.paths['test_dir'],
  50                                                 f"{os.path.basename(self.paths['exe'])}.stdout")
  51         self.filenames['stderr'] = os.path.join(self.paths['test_dir'],
  52                                                 f"{os.path.basename(self.paths['exe'])}.stderr")
  53         self.filenames['exitcode'] = os.path.join(self.paths['test_dir'],
  54                                                   f"{os.path.basename(self.paths['exe'])}.exitcode")
  55
  56     def run(self):
  57         """Run the test."""
  58
  59         raise NotImplementedError()
  60
  61     def check_result(self):
  62         """Check test results."""
  63
  64         raise NotImplementedError()
  65
  66
  67 class FioExeTest(FioTest):
  68     """Test consists of an executable binary or script"""
  69
  70     def __init__(self, exe_path, success):
  71         """Construct a FioExeTest which is a FioTest consisting of an
  72         executable binary or script.
  73
  74         exe_path:       location of executable binary or script
  75         success:        Definition of test success
  76         """
  77
  78         FioTest.__init__(self, exe_path, success)
  79
  80     def run(self):
  81         """Execute the binary or script described by this instance."""
  82
  83         command = [self.paths['exe']] + self.parameters
  84         with open(self.filenames['cmd'], "w+",
  85                   encoding=locale.getpreferredencoding()) as command_file:
  86             command_file.write(f"{command}\n")
  87
  88         try:
  89             with open(self.filenames['stdout'], "w+",
  90                       encoding=locale.getpreferredencoding()) as stdout_file, \
  91                 open(self.filenames['stderr'], "w+",
  92                      encoding=locale.getpreferredencoding()) as stderr_file, \
  93                 open(self.filenames['exitcode'], "w+",
  94                      encoding=locale.getpreferredencoding()) as exitcode_file:
  95                 proc = None
  96                 # Avoid using subprocess.run() here because when a timeout occurs,
  97                 # fio will be stopped with SIGKILL. This does not give fio a
  98                 # chance to clean up and means that child processes may continue
  99                 # running and submitting IO.
 100                 proc = subprocess.Popen(command,
 101                                         stdout=stdout_file,
 102                                         stderr=stderr_file,
 103                                         cwd=self.paths['test_dir'],
 104                                         universal_newlines=True)
 105                 proc.communicate(timeout=self.success['timeout'])
 106                 exitcode_file.write(f'{proc.returncode}\n')
 107                 logging.debug("Test %d: return code: %d", self.testnum, proc.returncode)
 108                 self.output['proc'] = proc
 109         except subprocess.TimeoutExpired:
 110             proc.terminate()
 111             proc.communicate()
 112             assert proc.poll()
 113             self.output['failure'] = 'timeout'
 114         except Exception:
 115             if proc:
 116                 if not proc.poll():
 117                     proc.terminate()
 118                     proc.communicate()
 119             self.output['failure'] = 'exception'
 120             self.output['exc_info'] = sys.exc_info()
 121
 122     def check_result(self):
 123         """Check results of test run."""
 124
 125         if 'proc' not in self.output:
 126             if self.output['failure'] == 'timeout':
 127                 self.failure_reason = f"{self.failure_reason} timeout,"
 128             else:
 129                 assert self.output['failure'] == 'exception'
 130                 self.failure_reason = f'{self.failure_reason} exception: ' + \
 131                 f'{self.output["exc_info"][0]}, {self.output["exc_info"][1]}'
 132
 133             self.passed = False
 134             return
 135
 136         if 'zero_return' in self.success:
 137             if self.success['zero_return']:
 138                 if self.output['proc'].returncode != 0:
 139                     self.passed = False
 140                     self.failure_reason = f"{self.failure_reason} non-zero return code,"
 141             else:
 142                 if self.output['proc'].returncode == 0:
 143                     self.failure_reason = f"{self.failure_reason} zero return code,"
 144                     self.passed = False
 145
 146         stderr_size = os.path.getsize(self.filenames['stderr'])
 147         if 'stderr_empty' in self.success:
 148             if self.success['stderr_empty']:
 149                 if stderr_size != 0:
 150                     self.failure_reason = f"{self.failure_reason} stderr not empty,"
 151                     self.passed = False
 152             else:
 153                 if stderr_size == 0:
 154                     self.failure_reason = f"{self.failure_reason} stderr empty,"
 155                     self.passed = False
 156
 157
 158 class FioJobFileTest(FioExeTest):
 159     """Test consists of a fio job with options in a job file."""
 160
 161     def __init__(self, fio_path, fio_job, success, fio_pre_job=None,
 162                  fio_pre_success=None, output_format="normal"):
 163         """Construct a FioJobFileTest which is a FioExeTest consisting of a
 164         single fio job file with an optional setup step.
 165
 166         fio_path:           location of fio executable
 167         fio_job:            location of fio job file
 168         success:            Definition of test success
 169         fio_pre_job:        fio job for preconditioning
 170         fio_pre_success:    Definition of test success for fio precon job
 171         output_format:      normal (default), json, jsonplus, or terse
 172         """
 173
 174         self.fio_job = fio_job
 175         self.fio_pre_job = fio_pre_job
 176         self.fio_pre_success = fio_pre_success if fio_pre_success else success
 177         self.output_format = output_format
 178         self.precon_failed = False
 179         self.json_data = None
 180
 181         FioExeTest.__init__(self, fio_path, success)
 182
 183     def setup(self, artifact_root, testnum, parameters=None):
 184         """Setup instance variables for fio job test."""
 185
 186         self.filenames['fio_output'] = f"{os.path.basename(self.fio_job)}.output"
 187         fio_args = [
 188             "--max-jobs=16",
 189             f"--output-format={self.output_format}",
 190             f"--output={self.filenames['fio_output']}",
 191             self.fio_job,
 192             ]
 193
 194         super().setup(artifact_root, testnum, fio_args)
 195
 196         # Update the filenames from the default
 197         self.filenames['cmd'] = os.path.join(self.paths['test_dir'],
 198                                              f"{os.path.basename(self.fio_job)}.command")
 199         self.filenames['stdout'] = os.path.join(self.paths['test_dir'],
 200                                                 f"{os.path.basename(self.fio_job)}.stdout")
 201         self.filenames['stderr'] = os.path.join(self.paths['test_dir'],
 202                                                 f"{os.path.basename(self.fio_job)}.stderr")
 203         self.filenames['exitcode'] = os.path.join(self.paths['test_dir'],
 204                                                   f"{os.path.basename(self.fio_job)}.exitcode")
 205
 206     def run_pre_job(self):
 207         """Run fio job precondition step."""
 208
 209         precon = FioJobFileTest(self.paths['exe'], self.fio_pre_job,
 210                             self.fio_pre_success,
 211                             output_format=self.output_format)
 212         precon.setup(self.paths['artifacts'], self.testnum)
 213         precon.run()
 214         precon.check_result()
 215         self.precon_failed = not precon.passed
 216         self.failure_reason = precon.failure_reason
 217
 218     def run(self):
 219         """Run fio job test."""
 220
 221         if self.fio_pre_job:
 222             self.run_pre_job()
 223
 224         if not self.precon_failed:
 225             super().run()
 226         else:
 227             logging.debug("Test %d: precondition step failed", self.testnum)
 228
 229     def get_file_fail(self, filename):
 230         """Safely read a file and fail the test upon error."""
 231         file_data = None
 232
 233         try:
 234             with open(filename, "r", encoding=locale.getpreferredencoding()) as output_file:
 235                 file_data = output_file.read()
 236         except OSError:
 237             self.failure_reason += f" unable to read file {filename}"
 238             self.passed = False
 239
 240         return file_data
 241
 242     def check_result(self):
 243         """Check fio job results."""
 244
 245         if self.precon_failed:
 246             self.passed = False
 247             self.failure_reason = f"{self.failure_reason} precondition step failed,"
 248             return
 249
 250         super().check_result()
 251
 252         if not self.passed:
 253             return
 254
 255         if 'json' not in self.output_format:
 256             return
 257
 258         file_data = self.get_file_fail(os.path.join(self.paths['test_dir'],
 259                                                     self.filenames['fio_output']))
 260         if not file_data:
 261             return
 262
 263         #
 264         # Sometimes fio informational messages are included at the top of the
 265         # JSON output, especially under Windows. Try to decode output as JSON
 266         # data, skipping everything until the first {
 267         #
 268         lines = file_data.splitlines()
 269         file_data = '\n'.join(lines[lines.index("{"):])
 270         try:
 271             self.json_data = json.loads(file_data)
 272         except json.JSONDecodeError:
 273             self.failure_reason = f"{self.failure_reason} unable to decode JSON data,"
 274             self.passed = False
 275
 276
 277 def run_fio_tests(test_list, test_env, args):
 278     """
 279     Run tests as specified in test_list.
 280     """
 281
 282     passed = 0
 283     failed = 0
 284     skipped = 0
 285
 286     for config in test_list:
 287         if (args.skip and config['test_id'] in args.skip) or \
 288            (args.run_only and config['test_id'] not in args.run_only):
 289             skipped = skipped + 1
 290             print(f"Test {config['test_id']} SKIPPED (User request)")
 291             continue
 292
 293         if issubclass(config['test_class'], FioJobFileTest):
 294             if config['pre_job']:
 295                 fio_pre_job = os.path.join(test_env['fio_root'], 't', 'jobs',
 296                                            config['pre_job'])
 297             else:
 298                 fio_pre_job = None
 299             if config['pre_success']:
 300                 fio_pre_success = config['pre_success']
 301             else:
 302                 fio_pre_success = None
 303             if 'output_format' in config:
 304                 output_format = config['output_format']
 305             else:
 306                 output_format = 'normal'
 307             test = config['test_class'](
 308                 test_env['fio_path'],
 309                 os.path.join(test_env['fio_root'], 't', 'jobs', config['job']),
 310                 config['success'],
 311                 fio_pre_job=fio_pre_job,
 312                 fio_pre_success=fio_pre_success,
 313                 output_format=output_format)
 314             desc = config['job']
 315             parameters = []
 316         elif issubclass(config['test_class'], FioExeTest):
 317             exe_path = os.path.join(test_env['fio_root'], config['exe'])
 318             parameters = []
 319             if config['parameters']:
 320                 parameters = [p.format(fio_path=test_env['fio_path'], nvmecdev=args.nvmecdev)
 321                               for p in config['parameters']]
 322             if Path(exe_path).suffix == '.py' and platform.system() == "Windows":
 323                 parameters.insert(0, exe_path)
 324                 exe_path = "python.exe"
 325             if config['test_id'] in test_env['pass_through']:
 326                 parameters += test_env['pass_through'][config['test_id']].split()
 327             test = config['test_class'](exe_path, config['success'])
 328             desc = config['exe']
 329         else:
 330             print(f"Test {config['test_id']} FAILED: unable to process test config")
 331             failed = failed + 1
 332             continue
 333
 334         if not args.skip_req:
 335             reqs_met = True
 336             for req in config['requirements']:
 337                 reqs_met, reason = req()
 338                 logging.debug("Test %d: Requirement '%s' met? %s", config['test_id'], reason,
 339                               reqs_met)
 340                 if not reqs_met:
 341                     break
 342             if not reqs_met:
 343                 print(f"Test {config['test_id']} SKIPPED ({reason}) {desc}")
 344                 skipped = skipped + 1
 345                 continue
 346
 347         try:
 348             test.setup(test_env['artifact_root'], config['test_id'], parameters)
 349             test.run()
 350             test.check_result()
 351         except KeyboardInterrupt:
 352             break
 353         except Exception as e:
 354             test.passed = False
 355             test.failure_reason += str(e)
 356             logging.debug("Test %d exception:\n%s\n", config['test_id'], traceback.format_exc())
 357         if test.passed:
 358             result = "PASSED"
 359             passed = passed + 1
 360         else:
 361             result = f"FAILED: {test.failure_reason}"
 362             failed = failed + 1
 363             contents, _ = get_file(test.filenames['stderr'])
 364             logging.debug("Test %d: stderr:\n%s", config['test_id'], contents)
 365             contents, _ = get_file(test.filenames['stdout'])
 366             logging.debug("Test %d: stdout:\n%s", config['test_id'], contents)
 367         print(f"Test {config['test_id']} {result} {desc}")
 368
 369     print(f"{passed} test(s) passed, {failed} failed, {skipped} skipped")
 370
 371     return passed, failed, skipped