5 This library contains FioTest objects that provide convenient means to run
6 different sorts of fio tests.
8 It also contains a test runner that runs an array of dictionary objects
20 from pathlib import Path
21 from fiotestcommon import get_file, SUCCESS_DEFAULT
25 """Base for all fio tests."""
27 def __init__(self, exe_path, success, testnum, artifact_root):
28 self.success = success
29 self.testnum = testnum
32 self.failure_reason = ''
33 self.parameters = None
36 'artifacts': artifact_root,
37 'test_dir': os.path.join(artifact_root, \
41 'cmd': os.path.join(self.paths['test_dir'], \
42 f"{os.path.basename(self.paths['exe'])}.command"),
43 'stdout': os.path.join(self.paths['test_dir'], \
44 f"{os.path.basename(self.paths['exe'])}.stdout"),
45 'stderr': os.path.join(self.paths['test_dir'], \
46 f"{os.path.basename(self.paths['exe'])}.stderr"),
47 'exitcode': os.path.join(self.paths['test_dir'], \
48 f"{os.path.basename(self.paths['exe'])}.exitcode"),
51 def setup(self, parameters):
52 """Setup instance variables for test."""
54 self.parameters = parameters
55 if not os.path.exists(self.paths['test_dir']):
56 os.mkdir(self.paths['test_dir'])
61 raise NotImplementedError()
63 def check_result(self):
64 """Check test results."""
66 raise NotImplementedError()
69 class FioExeTest(FioTest):
70 """Test consists of an executable binary or script"""
73 """Execute the binary or script described by this instance."""
75 command = [self.paths['exe']] + self.parameters
76 with open(self.filenames['cmd'], "w+",
77 encoding=locale.getpreferredencoding()) as command_file:
78 command_file.write(" \\\n ".join(command))
81 with open(self.filenames['stdout'], "w+",
82 encoding=locale.getpreferredencoding()) as stdout_file, \
83 open(self.filenames['stderr'], "w+",
84 encoding=locale.getpreferredencoding()) as stderr_file, \
85 open(self.filenames['exitcode'], "w+",
86 encoding=locale.getpreferredencoding()) as exitcode_file:
88 # Avoid using subprocess.run() here because when a timeout occurs,
89 # fio will be stopped with SIGKILL. This does not give fio a
90 # chance to clean up and means that child processes may continue
91 # running and submitting IO.
92 proc = subprocess.Popen(command,
95 cwd=self.paths['test_dir'],
96 universal_newlines=True)
97 proc.communicate(timeout=self.success['timeout'])
98 exitcode_file.write(f'{proc.returncode}\n')
99 logging.debug("Test %d: return code: %d", self.testnum, proc.returncode)
100 self.output['proc'] = proc
101 except subprocess.TimeoutExpired:
105 self.output['failure'] = 'timeout'
111 self.output['failure'] = 'exception'
112 self.output['exc_info'] = sys.exc_info()
114 def check_result(self):
115 """Check results of test run."""
117 if 'proc' not in self.output:
118 if self.output['failure'] == 'timeout':
119 self.failure_reason = f"{self.failure_reason} timeout,"
121 assert self.output['failure'] == 'exception'
122 self.failure_reason = f'{self.failure_reason} exception: ' + \
123 f'{self.output["exc_info"][0]}, {self.output["exc_info"][1]}'
128 if 'zero_return' in self.success:
129 if self.success['zero_return']:
130 if self.output['proc'].returncode != 0:
132 self.failure_reason = f"{self.failure_reason} non-zero return code,"
134 if self.output['proc'].returncode == 0:
135 self.failure_reason = f"{self.failure_reason} zero return code,"
138 stderr_size = os.path.getsize(self.filenames['stderr'])
139 if 'stderr_empty' in self.success:
140 if self.success['stderr_empty']:
142 self.failure_reason = f"{self.failure_reason} stderr not empty,"
146 self.failure_reason = f"{self.failure_reason} stderr empty,"
150 class FioJobFileTest(FioExeTest):
151 """Test consists of a fio job with options in a job file."""
153 def __init__(self, fio_path, fio_job, success, testnum, artifact_root,
154 fio_pre_job=None, fio_pre_success=None,
155 output_format="normal"):
156 """Construct a FioJobFileTest which is a FioExeTest consisting of a
157 single fio job file with an optional setup step.
159 fio_path: location of fio executable
160 fio_job: location of fio job file
161 success: Definition of test success
163 artifact_root: root directory for artifacts
164 fio_pre_job: fio job for preconditioning
165 fio_pre_success: Definition of test success for fio precon job
166 output_format: normal (default), json, jsonplus, or terse
169 self.fio_job = fio_job
170 self.fio_pre_job = fio_pre_job
171 self.fio_pre_success = fio_pre_success if fio_pre_success else success
172 self.output_format = output_format
173 self.precon_failed = False
174 self.json_data = None
176 super().__init__(fio_path, success, testnum, artifact_root)
178 def setup(self, parameters):
179 """Setup instance variables for fio job test."""
181 self.filenames['fio_output'] = f"{os.path.basename(self.fio_job)}.output"
184 f"--output-format={self.output_format}",
185 f"--output={self.filenames['fio_output']}",
189 fio_args += parameters
191 super().setup(fio_args)
193 # Update the filenames from the default
194 self.filenames['cmd'] = os.path.join(self.paths['test_dir'],
195 f"{os.path.basename(self.fio_job)}.command")
196 self.filenames['stdout'] = os.path.join(self.paths['test_dir'],
197 f"{os.path.basename(self.fio_job)}.stdout")
198 self.filenames['stderr'] = os.path.join(self.paths['test_dir'],
199 f"{os.path.basename(self.fio_job)}.stderr")
200 self.filenames['exitcode'] = os.path.join(self.paths['test_dir'],
201 f"{os.path.basename(self.fio_job)}.exitcode")
203 def run_pre_job(self):
204 """Run fio job precondition step."""
206 precon = FioJobFileTest(self.paths['exe'], self.fio_pre_job,
207 self.fio_pre_success,
209 self.paths['artifacts'],
210 output_format=self.output_format)
213 precon.check_result()
214 self.precon_failed = not precon.passed
215 self.failure_reason = precon.failure_reason
218 """Run fio job test."""
223 if not self.precon_failed:
226 logging.debug("Test %d: precondition step failed", self.testnum)
228 def get_file_fail(self, filename):
229 """Safely read a file and fail the test upon error."""
233 with open(filename, "r", encoding=locale.getpreferredencoding()) as output_file:
234 file_data = output_file.read()
236 self.failure_reason += f" unable to read file {filename}"
241 def check_result(self):
242 """Check fio job results."""
244 if self.precon_failed:
246 self.failure_reason = f"{self.failure_reason} precondition step failed,"
249 super().check_result()
254 if 'json' not in self.output_format:
257 file_data = self.get_file_fail(os.path.join(self.paths['test_dir'],
258 self.filenames['fio_output']))
263 # Sometimes fio informational messages are included at the top of the
264 # JSON output, especially under Windows. Try to decode output as JSON
265 # data, skipping everything until the first {
267 lines = file_data.splitlines()
268 file_data = '\n'.join(lines[lines.index("{"):])
270 self.json_data = json.loads(file_data)
271 except json.JSONDecodeError:
272 self.failure_reason = f"{self.failure_reason} unable to decode JSON data,"
276 class FioJobCmdTest(FioExeTest):
277 """This runs a fio job with options specified on the command line."""
279 def __init__(self, fio_path, success, testnum, artifact_root, fio_opts, basename=None):
281 self.basename = basename if basename else os.path.basename(fio_path)
282 self.fio_opts = fio_opts
283 self.json_data = None
284 self.iops_log_lines = None
286 super().__init__(fio_path, success, testnum, artifact_root)
288 filename_stub = os.path.join(self.paths['test_dir'], f"{self.basename}{self.testnum:03d}")
289 self.filenames['cmd'] = f"{filename_stub}.command"
290 self.filenames['stdout'] = f"{filename_stub}.stdout"
291 self.filenames['stderr'] = f"{filename_stub}.stderr"
292 self.filenames['output'] = os.path.abspath(f"{filename_stub}.output")
293 self.filenames['exitcode'] = f"{filename_stub}.exitcode"
294 self.filenames['iopslog'] = os.path.abspath(f"{filename_stub}")
299 if 'output-format' in self.fio_opts and 'json' in \
300 self.fio_opts['output-format']:
301 if not self.get_json():
302 print('Unable to decode JSON data')
305 if any('--write_iops_log=' in param for param in self.parameters):
308 def get_iops_log(self):
309 """Read IOPS log from the first job."""
311 log_filename = self.filenames['iopslog'] + "_iops.1.log"
312 with open(log_filename, 'r', encoding=locale.getpreferredencoding()) as iops_file:
313 self.iops_log_lines = iops_file.read()
316 """Convert fio JSON output into a python JSON object"""
318 filename = self.filenames['output']
319 with open(filename, 'r', encoding=locale.getpreferredencoding()) as file:
320 file_data = file.read()
323 # Sometimes fio informational messages are included at the top of the
324 # JSON output, especially under Windows. Try to decode output as JSON
325 # data, lopping off up to the first four lines
327 lines = file_data.splitlines()
329 file_data = '\n'.join(lines[i:])
331 self.json_data = json.loads(file_data)
332 except json.JSONDecodeError:
340 def check_empty(job):
342 Make sure JSON data is empty.
344 Some data structures should be empty. This function makes sure that they are.
346 job JSON object that we need to check for emptiness
349 return job['total_ios'] == 0 and \
350 job['slat_ns']['N'] == 0 and \
351 job['clat_ns']['N'] == 0 and \
352 job['lat_ns']['N'] == 0
354 def check_all_ddirs(self, ddir_nonzero, job):
356 Iterate over the data directions and check whether each is
357 appropriately empty or not.
361 ddirlist = ['read', 'write', 'trim']
363 for ddir in ddirlist:
364 if ddir in ddir_nonzero:
365 if self.check_empty(job[ddir]):
366 print(f"Unexpected zero {ddir} data found in output")
369 if not self.check_empty(job[ddir]):
370 print(f"Unexpected {ddir} data found in output")
376 def run_fio_tests(test_list, test_env, args):
378 Run tests as specified in test_list.
385 for config in test_list:
386 if (args.skip and config['test_id'] in args.skip) or \
387 (args.run_only and config['test_id'] not in args.run_only) or \
388 ('force_skip' in config and config['force_skip']):
389 skipped = skipped + 1
390 print(f"Test {config['test_id']} SKIPPED (User request or override)")
393 if issubclass(config['test_class'], FioJobFileTest):
394 if config['pre_job']:
395 fio_pre_job = os.path.join(test_env['fio_root'], 't', 'jobs',
399 if config['pre_success']:
400 fio_pre_success = config['pre_success']
402 fio_pre_success = None
403 if 'output_format' in config:
404 output_format = config['output_format']
406 output_format = 'normal'
407 test = config['test_class'](
408 test_env['fio_path'],
409 os.path.join(test_env['fio_root'], 't', 'jobs', config['job']),
412 test_env['artifact_root'],
413 fio_pre_job=fio_pre_job,
414 fio_pre_success=fio_pre_success,
415 output_format=output_format)
417 parameters = config['parameters'] if 'parameters' in config else None
418 elif issubclass(config['test_class'], FioJobCmdTest):
419 if not 'success' in config:
420 config['success'] = SUCCESS_DEFAULT
421 test = config['test_class'](test_env['fio_path'],
424 test_env['artifact_root'],
426 test_env['basename'])
427 desc = config['test_id']
429 elif issubclass(config['test_class'], FioExeTest):
430 exe_path = os.path.join(test_env['fio_root'], config['exe'])
432 if config['parameters']:
433 parameters = [p.format(fio_path=test_env['fio_path'], nvmecdev=args.nvmecdev)
434 for p in config['parameters']]
435 if Path(exe_path).suffix == '.py' and platform.system() == "Windows":
436 parameters.insert(0, exe_path)
437 exe_path = "python.exe"
438 if config['test_id'] in test_env['pass_through']:
439 parameters += test_env['pass_through'][config['test_id']].split()
440 test = config['test_class'](
444 test_env['artifact_root'])
447 print(f"Test {config['test_id']} FAILED: unable to process test config")
451 if 'requirements' in config and not args.skip_req:
453 for req in config['requirements']:
454 reqs_met, reason = req()
455 logging.debug("Test %d: Requirement '%s' met? %s", config['test_id'], reason,
460 print(f"Test {config['test_id']} SKIPPED ({reason}) {desc}")
461 skipped = skipped + 1
465 test.setup(parameters)
468 except KeyboardInterrupt:
470 except Exception as e:
472 test.failure_reason += str(e)
473 logging.debug("Test %d exception:\n%s\n", config['test_id'], traceback.format_exc())
478 result = f"FAILED: {test.failure_reason}"
480 contents, _ = get_file(test.filenames['stderr'])
481 logging.debug("Test %d: stderr:\n%s", config['test_id'], contents)
482 contents, _ = get_file(test.filenames['stdout'])
483 logging.debug("Test %d: stdout:\n%s", config['test_id'], contents)
484 print(f"Test {config['test_id']} {result} {desc}")
486 print(f"{passed} test(s) passed, {failed} failed, {skipped} skipped")
488 return passed, failed, skipped