5 This library contains FioTest objects that provide convenient means to run
6 different sorts of fio tests.
8 It also contains a test runner that runs an array of dictionary objects
20 from pathlib import Path
21 from fiotestcommon import get_file
25 """Base for all fio tests."""
27 def __init__(self, exe_path, success, testnum, artifact_root):
28 self.success = success
29 self.testnum = testnum
32 self.failure_reason = ''
33 self.parameters = None
36 'artifacts': artifact_root,
37 'test_dir': os.path.join(artifact_root, \
41 'cmd': os.path.join(self.paths['test_dir'], \
42 f"{os.path.basename(self.paths['exe'])}.command"),
43 'stdout': os.path.join(self.paths['test_dir'], \
44 f"{os.path.basename(self.paths['exe'])}.stdout"),
45 'stderr': os.path.join(self.paths['test_dir'], \
46 f"{os.path.basename(self.paths['exe'])}.stderr"),
47 'exitcode': os.path.join(self.paths['test_dir'], \
48 f"{os.path.basename(self.paths['exe'])}.exitcode"),
51 def setup(self, parameters):
52 """Setup instance variables for test."""
54 self.parameters = parameters
55 if not os.path.exists(self.paths['test_dir']):
56 os.mkdir(self.paths['test_dir'])
61 raise NotImplementedError()
63 def check_result(self):
64 """Check test results."""
66 raise NotImplementedError()
69 class FioExeTest(FioTest):
70 """Test consists of an executable binary or script"""
73 """Execute the binary or script described by this instance."""
75 command = [self.paths['exe']] + self.parameters
76 with open(self.filenames['cmd'], "w+",
77 encoding=locale.getpreferredencoding()) as command_file:
78 command_file.write(" ".join(command))
81 with open(self.filenames['stdout'], "w+",
82 encoding=locale.getpreferredencoding()) as stdout_file, \
83 open(self.filenames['stderr'], "w+",
84 encoding=locale.getpreferredencoding()) as stderr_file, \
85 open(self.filenames['exitcode'], "w+",
86 encoding=locale.getpreferredencoding()) as exitcode_file:
88 # Avoid using subprocess.run() here because when a timeout occurs,
89 # fio will be stopped with SIGKILL. This does not give fio a
90 # chance to clean up and means that child processes may continue
91 # running and submitting IO.
92 proc = subprocess.Popen(command,
95 cwd=self.paths['test_dir'],
96 universal_newlines=True)
97 proc.communicate(timeout=self.success['timeout'])
98 exitcode_file.write(f'{proc.returncode}\n')
99 logging.debug("Test %d: return code: %d", self.testnum, proc.returncode)
100 self.output['proc'] = proc
101 except subprocess.TimeoutExpired:
105 self.output['failure'] = 'timeout'
111 self.output['failure'] = 'exception'
112 self.output['exc_info'] = sys.exc_info()
114 def check_result(self):
115 """Check results of test run."""
117 if 'proc' not in self.output:
118 if self.output['failure'] == 'timeout':
119 self.failure_reason = f"{self.failure_reason} timeout,"
121 assert self.output['failure'] == 'exception'
122 self.failure_reason = f'{self.failure_reason} exception: ' + \
123 f'{self.output["exc_info"][0]}, {self.output["exc_info"][1]}'
128 if 'zero_return' in self.success:
129 if self.success['zero_return']:
130 if self.output['proc'].returncode != 0:
132 self.failure_reason = f"{self.failure_reason} non-zero return code,"
134 if self.output['proc'].returncode == 0:
135 self.failure_reason = f"{self.failure_reason} zero return code,"
138 stderr_size = os.path.getsize(self.filenames['stderr'])
139 if 'stderr_empty' in self.success:
140 if self.success['stderr_empty']:
142 self.failure_reason = f"{self.failure_reason} stderr not empty,"
146 self.failure_reason = f"{self.failure_reason} stderr empty,"
150 class FioJobFileTest(FioExeTest):
151 """Test consists of a fio job with options in a job file."""
153 def __init__(self, fio_path, fio_job, success, testnum, artifact_root,
154 fio_pre_job=None, fio_pre_success=None,
155 output_format="normal"):
156 """Construct a FioJobFileTest which is a FioExeTest consisting of a
157 single fio job file with an optional setup step.
159 fio_path: location of fio executable
160 fio_job: location of fio job file
161 success: Definition of test success
163 artifact_root: root directory for artifacts
164 fio_pre_job: fio job for preconditioning
165 fio_pre_success: Definition of test success for fio precon job
166 output_format: normal (default), json, jsonplus, or terse
169 self.fio_job = fio_job
170 self.fio_pre_job = fio_pre_job
171 self.fio_pre_success = fio_pre_success if fio_pre_success else success
172 self.output_format = output_format
173 self.precon_failed = False
174 self.json_data = None
176 super().__init__(fio_path, success, testnum, artifact_root)
178 def setup(self, parameters=None):
179 """Setup instance variables for fio job test."""
181 self.filenames['fio_output'] = f"{os.path.basename(self.fio_job)}.output"
184 f"--output-format={self.output_format}",
185 f"--output={self.filenames['fio_output']}",
189 super().setup(fio_args)
191 # Update the filenames from the default
192 self.filenames['cmd'] = os.path.join(self.paths['test_dir'],
193 f"{os.path.basename(self.fio_job)}.command")
194 self.filenames['stdout'] = os.path.join(self.paths['test_dir'],
195 f"{os.path.basename(self.fio_job)}.stdout")
196 self.filenames['stderr'] = os.path.join(self.paths['test_dir'],
197 f"{os.path.basename(self.fio_job)}.stderr")
198 self.filenames['exitcode'] = os.path.join(self.paths['test_dir'],
199 f"{os.path.basename(self.fio_job)}.exitcode")
201 def run_pre_job(self):
202 """Run fio job precondition step."""
204 precon = FioJobFileTest(self.paths['exe'], self.fio_pre_job,
205 self.fio_pre_success,
207 self.paths['artifacts'],
208 output_format=self.output_format)
211 precon.check_result()
212 self.precon_failed = not precon.passed
213 self.failure_reason = precon.failure_reason
216 """Run fio job test."""
221 if not self.precon_failed:
224 logging.debug("Test %d: precondition step failed", self.testnum)
226 def get_file_fail(self, filename):
227 """Safely read a file and fail the test upon error."""
231 with open(filename, "r", encoding=locale.getpreferredencoding()) as output_file:
232 file_data = output_file.read()
234 self.failure_reason += f" unable to read file {filename}"
239 def check_result(self):
240 """Check fio job results."""
242 if self.precon_failed:
244 self.failure_reason = f"{self.failure_reason} precondition step failed,"
247 super().check_result()
252 if 'json' not in self.output_format:
255 file_data = self.get_file_fail(os.path.join(self.paths['test_dir'],
256 self.filenames['fio_output']))
261 # Sometimes fio informational messages are included at the top of the
262 # JSON output, especially under Windows. Try to decode output as JSON
263 # data, skipping everything until the first {
265 lines = file_data.splitlines()
266 file_data = '\n'.join(lines[lines.index("{"):])
268 self.json_data = json.loads(file_data)
269 except json.JSONDecodeError:
270 self.failure_reason = f"{self.failure_reason} unable to decode JSON data,"
274 def run_fio_tests(test_list, test_env, args):
276 Run tests as specified in test_list.
283 for config in test_list:
284 if (args.skip and config['test_id'] in args.skip) or \
285 (args.run_only and config['test_id'] not in args.run_only):
286 skipped = skipped + 1
287 print(f"Test {config['test_id']} SKIPPED (User request)")
290 if issubclass(config['test_class'], FioJobFileTest):
291 if config['pre_job']:
292 fio_pre_job = os.path.join(test_env['fio_root'], 't', 'jobs',
296 if config['pre_success']:
297 fio_pre_success = config['pre_success']
299 fio_pre_success = None
300 if 'output_format' in config:
301 output_format = config['output_format']
303 output_format = 'normal'
304 test = config['test_class'](
305 test_env['fio_path'],
306 os.path.join(test_env['fio_root'], 't', 'jobs', config['job']),
309 test_env['artifact_root'],
310 fio_pre_job=fio_pre_job,
311 fio_pre_success=fio_pre_success,
312 output_format=output_format)
315 elif issubclass(config['test_class'], FioExeTest):
316 exe_path = os.path.join(test_env['fio_root'], config['exe'])
318 if config['parameters']:
319 parameters = [p.format(fio_path=test_env['fio_path'], nvmecdev=args.nvmecdev)
320 for p in config['parameters']]
321 if Path(exe_path).suffix == '.py' and platform.system() == "Windows":
322 parameters.insert(0, exe_path)
323 exe_path = "python.exe"
324 if config['test_id'] in test_env['pass_through']:
325 parameters += test_env['pass_through'][config['test_id']].split()
326 test = config['test_class'](
330 test_env['artifact_root'])
333 print(f"Test {config['test_id']} FAILED: unable to process test config")
337 if not args.skip_req:
339 for req in config['requirements']:
340 reqs_met, reason = req()
341 logging.debug("Test %d: Requirement '%s' met? %s", config['test_id'], reason,
346 print(f"Test {config['test_id']} SKIPPED ({reason}) {desc}")
347 skipped = skipped + 1
351 test.setup(parameters)
354 except KeyboardInterrupt:
356 except Exception as e:
358 test.failure_reason += str(e)
359 logging.debug("Test %d exception:\n%s\n", config['test_id'], traceback.format_exc())
364 result = f"FAILED: {test.failure_reason}"
366 contents, _ = get_file(test.filenames['stderr'])
367 logging.debug("Test %d: stderr:\n%s", config['test_id'], contents)
368 contents, _ = get_file(test.filenames['stdout'])
369 logging.debug("Test %d: stdout:\n%s", config['test_id'], contents)
370 print(f"Test {config['test_id']} {result} {desc}")
372 print(f"{passed} test(s) passed, {failed} failed, {skipped} skipped")
374 return passed, failed, skipped