Commit | Line | Data |
---|---|---|
fb551941 VF |
1 | #!/usr/bin/env python3 |
2 | """ | |
3 | fiotestlib.py | |
4 | ||
5 | This library contains FioTest objects that provide convenient means to run | |
6 | different sorts of fio tests. | |
7 | ||
8 | It also contains a test runner that runs an array of dictionary objects | |
9 | describing fio tests. | |
10 | """ | |
11 | ||
12 | import os | |
13 | import sys | |
14 | import json | |
15 | import locale | |
16 | import logging | |
17 | import platform | |
18 | import traceback | |
19 | import subprocess | |
20 | from pathlib import Path | |
5dc5c6f6 | 21 | from fiotestcommon import get_file, SUCCESS_DEFAULT |
fb551941 VF |
22 | |
23 | ||
24 | class FioTest(): | |
25 | """Base for all fio tests.""" | |
26 | ||
111fa98e | 27 | def __init__(self, exe_path, success, testnum, artifact_root): |
fb551941 | 28 | self.success = success |
111fa98e | 29 | self.testnum = testnum |
fb551941 | 30 | self.output = {} |
fb551941 VF |
31 | self.passed = True |
32 | self.failure_reason = '' | |
68b3a741 | 33 | self.parameters = None |
111fa98e VF |
34 | self.paths = { |
35 | 'exe': exe_path, | |
36 | 'artifacts': artifact_root, | |
37 | 'test_dir': os.path.join(artifact_root, \ | |
38 | f"{testnum:04d}"), | |
39 | } | |
40 | self.filenames = { | |
41 | 'cmd': os.path.join(self.paths['test_dir'], \ | |
42 | f"{os.path.basename(self.paths['exe'])}.command"), | |
43 | 'stdout': os.path.join(self.paths['test_dir'], \ | |
44 | f"{os.path.basename(self.paths['exe'])}.stdout"), | |
45 | 'stderr': os.path.join(self.paths['test_dir'], \ | |
46 | f"{os.path.basename(self.paths['exe'])}.stderr"), | |
47 | 'exitcode': os.path.join(self.paths['test_dir'], \ | |
48 | f"{os.path.basename(self.paths['exe'])}.exitcode"), | |
49 | } | |
50 | ||
51 | def setup(self, parameters): | |
fb551941 VF |
52 | """Setup instance variables for test.""" |
53 | ||
68b3a741 | 54 | self.parameters = parameters |
68b3a741 VF |
55 | if not os.path.exists(self.paths['test_dir']): |
56 | os.mkdir(self.paths['test_dir']) | |
57 | ||
fb551941 VF |
58 | def run(self): |
59 | """Run the test.""" | |
60 | ||
61 | raise NotImplementedError() | |
62 | ||
63 | def check_result(self): | |
64 | """Check test results.""" | |
65 | ||
66 | raise NotImplementedError() | |
67 | ||
68 | ||
69 | class FioExeTest(FioTest): | |
70 | """Test consists of an executable binary or script""" | |
71 | ||
fb551941 VF |
72 | def run(self): |
73 | """Execute the binary or script described by this instance.""" | |
74 | ||
68b3a741 | 75 | command = [self.paths['exe']] + self.parameters |
1cf0ba9d VF |
76 | with open(self.filenames['cmd'], "w+", |
77 | encoding=locale.getpreferredencoding()) as command_file: | |
7b570114 | 78 | command_file.write(" \\\n ".join(command)) |
1cf0ba9d | 79 | |
fb551941 | 80 | try: |
1cf0ba9d VF |
81 | with open(self.filenames['stdout'], "w+", |
82 | encoding=locale.getpreferredencoding()) as stdout_file, \ | |
83 | open(self.filenames['stderr'], "w+", | |
84 | encoding=locale.getpreferredencoding()) as stderr_file, \ | |
85 | open(self.filenames['exitcode'], "w+", | |
86 | encoding=locale.getpreferredencoding()) as exitcode_file: | |
87 | proc = None | |
88 | # Avoid using subprocess.run() here because when a timeout occurs, | |
89 | # fio will be stopped with SIGKILL. This does not give fio a | |
90 | # chance to clean up and means that child processes may continue | |
91 | # running and submitting IO. | |
92 | proc = subprocess.Popen(command, | |
93 | stdout=stdout_file, | |
94 | stderr=stderr_file, | |
95 | cwd=self.paths['test_dir'], | |
96 | universal_newlines=True) | |
97 | proc.communicate(timeout=self.success['timeout']) | |
98 | exitcode_file.write(f'{proc.returncode}\n') | |
99 | logging.debug("Test %d: return code: %d", self.testnum, proc.returncode) | |
100 | self.output['proc'] = proc | |
fb551941 VF |
101 | except subprocess.TimeoutExpired: |
102 | proc.terminate() | |
103 | proc.communicate() | |
104 | assert proc.poll() | |
105 | self.output['failure'] = 'timeout' | |
106 | except Exception: | |
107 | if proc: | |
108 | if not proc.poll(): | |
109 | proc.terminate() | |
110 | proc.communicate() | |
111 | self.output['failure'] = 'exception' | |
112 | self.output['exc_info'] = sys.exc_info() | |
fb551941 VF |
113 | |
114 | def check_result(self): | |
115 | """Check results of test run.""" | |
116 | ||
117 | if 'proc' not in self.output: | |
118 | if self.output['failure'] == 'timeout': | |
119 | self.failure_reason = f"{self.failure_reason} timeout," | |
120 | else: | |
121 | assert self.output['failure'] == 'exception' | |
6544f1ae VF |
122 | self.failure_reason = f'{self.failure_reason} exception: ' + \ |
123 | f'{self.output["exc_info"][0]}, {self.output["exc_info"][1]}' | |
fb551941 VF |
124 | |
125 | self.passed = False | |
126 | return | |
127 | ||
128 | if 'zero_return' in self.success: | |
129 | if self.success['zero_return']: | |
130 | if self.output['proc'].returncode != 0: | |
131 | self.passed = False | |
132 | self.failure_reason = f"{self.failure_reason} non-zero return code," | |
133 | else: | |
134 | if self.output['proc'].returncode == 0: | |
135 | self.failure_reason = f"{self.failure_reason} zero return code," | |
136 | self.passed = False | |
137 | ||
68b3a741 | 138 | stderr_size = os.path.getsize(self.filenames['stderr']) |
fb551941 VF |
139 | if 'stderr_empty' in self.success: |
140 | if self.success['stderr_empty']: | |
141 | if stderr_size != 0: | |
142 | self.failure_reason = f"{self.failure_reason} stderr not empty," | |
143 | self.passed = False | |
144 | else: | |
145 | if stderr_size == 0: | |
146 | self.failure_reason = f"{self.failure_reason} stderr empty," | |
147 | self.passed = False | |
148 | ||
149 | ||
0dc6e911 | 150 | class FioJobFileTest(FioExeTest): |
68b3a741 | 151 | """Test consists of a fio job with options in a job file.""" |
fb551941 | 152 | |
111fa98e VF |
153 | def __init__(self, fio_path, fio_job, success, testnum, artifact_root, |
154 | fio_pre_job=None, fio_pre_success=None, | |
155 | output_format="normal"): | |
0dc6e911 | 156 | """Construct a FioJobFileTest which is a FioExeTest consisting of a |
fb551941 VF |
157 | single fio job file with an optional setup step. |
158 | ||
159 | fio_path: location of fio executable | |
160 | fio_job: location of fio job file | |
161 | success: Definition of test success | |
111fa98e VF |
162 | testnum: test ID |
163 | artifact_root: root directory for artifacts | |
fb551941 VF |
164 | fio_pre_job: fio job for preconditioning |
165 | fio_pre_success: Definition of test success for fio precon job | |
166 | output_format: normal (default), json, jsonplus, or terse | |
167 | """ | |
168 | ||
169 | self.fio_job = fio_job | |
170 | self.fio_pre_job = fio_pre_job | |
171 | self.fio_pre_success = fio_pre_success if fio_pre_success else success | |
172 | self.output_format = output_format | |
173 | self.precon_failed = False | |
174 | self.json_data = None | |
68b3a741 | 175 | |
111fa98e | 176 | super().__init__(fio_path, success, testnum, artifact_root) |
68b3a741 | 177 | |
2e976800 | 178 | def setup(self, parameters): |
68b3a741 VF |
179 | """Setup instance variables for fio job test.""" |
180 | ||
181 | self.filenames['fio_output'] = f"{os.path.basename(self.fio_job)}.output" | |
182 | fio_args = [ | |
fb551941 VF |
183 | "--max-jobs=16", |
184 | f"--output-format={self.output_format}", | |
68b3a741 | 185 | f"--output={self.filenames['fio_output']}", |
fb551941 VF |
186 | self.fio_job, |
187 | ] | |
2e976800 VF |
188 | if parameters: |
189 | fio_args += parameters | |
fb551941 | 190 | |
111fa98e | 191 | super().setup(fio_args) |
fb551941 | 192 | |
68b3a741 VF |
193 | # Update the filenames from the default |
194 | self.filenames['cmd'] = os.path.join(self.paths['test_dir'], | |
195 | f"{os.path.basename(self.fio_job)}.command") | |
196 | self.filenames['stdout'] = os.path.join(self.paths['test_dir'], | |
197 | f"{os.path.basename(self.fio_job)}.stdout") | |
198 | self.filenames['stderr'] = os.path.join(self.paths['test_dir'], | |
199 | f"{os.path.basename(self.fio_job)}.stderr") | |
200 | self.filenames['exitcode'] = os.path.join(self.paths['test_dir'], | |
201 | f"{os.path.basename(self.fio_job)}.exitcode") | |
fb551941 VF |
202 | |
203 | def run_pre_job(self): | |
204 | """Run fio job precondition step.""" | |
205 | ||
68b3a741 | 206 | precon = FioJobFileTest(self.paths['exe'], self.fio_pre_job, |
fb551941 | 207 | self.fio_pre_success, |
111fa98e VF |
208 | self.testnum, |
209 | self.paths['artifacts'], | |
fb551941 | 210 | output_format=self.output_format) |
2e976800 | 211 | precon.setup(None) |
fb551941 VF |
212 | precon.run() |
213 | precon.check_result() | |
214 | self.precon_failed = not precon.passed | |
215 | self.failure_reason = precon.failure_reason | |
216 | ||
217 | def run(self): | |
218 | """Run fio job test.""" | |
219 | ||
220 | if self.fio_pre_job: | |
221 | self.run_pre_job() | |
222 | ||
223 | if not self.precon_failed: | |
224 | super().run() | |
225 | else: | |
226 | logging.debug("Test %d: precondition step failed", self.testnum) | |
227 | ||
fb551941 VF |
228 | def get_file_fail(self, filename): |
229 | """Safely read a file and fail the test upon error.""" | |
230 | file_data = None | |
231 | ||
232 | try: | |
233 | with open(filename, "r", encoding=locale.getpreferredencoding()) as output_file: | |
234 | file_data = output_file.read() | |
235 | except OSError: | |
236 | self.failure_reason += f" unable to read file {filename}" | |
237 | self.passed = False | |
238 | ||
239 | return file_data | |
240 | ||
241 | def check_result(self): | |
242 | """Check fio job results.""" | |
243 | ||
244 | if self.precon_failed: | |
245 | self.passed = False | |
246 | self.failure_reason = f"{self.failure_reason} precondition step failed," | |
247 | return | |
248 | ||
249 | super().check_result() | |
250 | ||
251 | if not self.passed: | |
252 | return | |
253 | ||
254 | if 'json' not in self.output_format: | |
255 | return | |
256 | ||
68b3a741 VF |
257 | file_data = self.get_file_fail(os.path.join(self.paths['test_dir'], |
258 | self.filenames['fio_output'])) | |
fb551941 VF |
259 | if not file_data: |
260 | return | |
261 | ||
262 | # | |
263 | # Sometimes fio informational messages are included at the top of the | |
264 | # JSON output, especially under Windows. Try to decode output as JSON | |
265 | # data, skipping everything until the first { | |
266 | # | |
267 | lines = file_data.splitlines() | |
268 | file_data = '\n'.join(lines[lines.index("{"):]) | |
269 | try: | |
270 | self.json_data = json.loads(file_data) | |
271 | except json.JSONDecodeError: | |
272 | self.failure_reason = f"{self.failure_reason} unable to decode JSON data," | |
273 | self.passed = False | |
274 | ||
275 | ||
5dc5c6f6 VF |
276 | class FioJobCmdTest(FioExeTest): |
277 | """This runs a fio job with options specified on the command line.""" | |
278 | ||
279 | def __init__(self, fio_path, success, testnum, artifact_root, fio_opts, basename=None): | |
280 | ||
281 | self.basename = basename if basename else os.path.basename(fio_path) | |
282 | self.fio_opts = fio_opts | |
283 | self.json_data = None | |
885e170a | 284 | self.iops_log_lines = None |
5dc5c6f6 VF |
285 | |
286 | super().__init__(fio_path, success, testnum, artifact_root) | |
287 | ||
885e170a VF |
288 | filename_stub = os.path.join(self.paths['test_dir'], f"{self.basename}{self.testnum:03d}") |
289 | self.filenames['cmd'] = f"{filename_stub}.command" | |
290 | self.filenames['stdout'] = f"{filename_stub}.stdout" | |
291 | self.filenames['stderr'] = f"{filename_stub}.stderr" | |
292 | self.filenames['output'] = os.path.abspath(f"{filename_stub}.output") | |
293 | self.filenames['exitcode'] = f"{filename_stub}.exitcode" | |
294 | self.filenames['iopslog'] = os.path.abspath(f"{filename_stub}") | |
5dc5c6f6 VF |
295 | |
296 | def run(self): | |
297 | super().run() | |
298 | ||
299 | if 'output-format' in self.fio_opts and 'json' in \ | |
300 | self.fio_opts['output-format']: | |
301 | if not self.get_json(): | |
302 | print('Unable to decode JSON data') | |
303 | self.passed = False | |
304 | ||
885e170a VF |
305 | if any('--write_iops_log=' in param for param in self.parameters): |
306 | self.get_iops_log() | |
307 | ||
308 | def get_iops_log(self): | |
309 | """Read IOPS log from the first job.""" | |
310 | ||
311 | log_filename = self.filenames['iopslog'] + "_iops.1.log" | |
312 | with open(log_filename, 'r', encoding=locale.getpreferredencoding()) as iops_file: | |
313 | self.iops_log_lines = iops_file.read() | |
314 | ||
5dc5c6f6 VF |
315 | def get_json(self): |
316 | """Convert fio JSON output into a python JSON object""" | |
317 | ||
318 | filename = self.filenames['output'] | |
319 | with open(filename, 'r', encoding=locale.getpreferredencoding()) as file: | |
320 | file_data = file.read() | |
321 | ||
322 | # | |
323 | # Sometimes fio informational messages are included at the top of the | |
324 | # JSON output, especially under Windows. Try to decode output as JSON | |
325 | # data, lopping off up to the first four lines | |
326 | # | |
327 | lines = file_data.splitlines() | |
328 | for i in range(5): | |
329 | file_data = '\n'.join(lines[i:]) | |
330 | try: | |
331 | self.json_data = json.loads(file_data) | |
332 | except json.JSONDecodeError: | |
333 | continue | |
334 | else: | |
335 | return True | |
336 | ||
337 | return False | |
338 | ||
339 | @staticmethod | |
340 | def check_empty(job): | |
341 | """ | |
342 | Make sure JSON data is empty. | |
343 | ||
344 | Some data structures should be empty. This function makes sure that they are. | |
345 | ||
346 | job JSON object that we need to check for emptiness | |
347 | """ | |
348 | ||
349 | return job['total_ios'] == 0 and \ | |
350 | job['slat_ns']['N'] == 0 and \ | |
351 | job['clat_ns']['N'] == 0 and \ | |
352 | job['lat_ns']['N'] == 0 | |
353 | ||
354 | def check_all_ddirs(self, ddir_nonzero, job): | |
355 | """ | |
356 | Iterate over the data directions and check whether each is | |
357 | appropriately empty or not. | |
358 | """ | |
359 | ||
360 | retval = True | |
361 | ddirlist = ['read', 'write', 'trim'] | |
362 | ||
363 | for ddir in ddirlist: | |
364 | if ddir in ddir_nonzero: | |
365 | if self.check_empty(job[ddir]): | |
366 | print(f"Unexpected zero {ddir} data found in output") | |
367 | retval = False | |
368 | else: | |
369 | if not self.check_empty(job[ddir]): | |
370 | print(f"Unexpected {ddir} data found in output") | |
371 | retval = False | |
372 | ||
373 | return retval | |
374 | ||
375 | ||
fb551941 VF |
376 | def run_fio_tests(test_list, test_env, args): |
377 | """ | |
378 | Run tests as specified in test_list. | |
379 | """ | |
380 | ||
381 | passed = 0 | |
382 | failed = 0 | |
383 | skipped = 0 | |
384 | ||
385 | for config in test_list: | |
386 | if (args.skip and config['test_id'] in args.skip) or \ | |
140a30e3 VF |
387 | (args.run_only and config['test_id'] not in args.run_only) or \ |
388 | ('force_skip' in config and config['force_skip']): | |
fb551941 | 389 | skipped = skipped + 1 |
140a30e3 | 390 | print(f"Test {config['test_id']} SKIPPED (User request or override)") |
fb551941 VF |
391 | continue |
392 | ||
0dc6e911 | 393 | if issubclass(config['test_class'], FioJobFileTest): |
fb551941 VF |
394 | if config['pre_job']: |
395 | fio_pre_job = os.path.join(test_env['fio_root'], 't', 'jobs', | |
396 | config['pre_job']) | |
397 | else: | |
398 | fio_pre_job = None | |
399 | if config['pre_success']: | |
400 | fio_pre_success = config['pre_success'] | |
401 | else: | |
402 | fio_pre_success = None | |
403 | if 'output_format' in config: | |
404 | output_format = config['output_format'] | |
405 | else: | |
406 | output_format = 'normal' | |
407 | test = config['test_class']( | |
408 | test_env['fio_path'], | |
409 | os.path.join(test_env['fio_root'], 't', 'jobs', config['job']), | |
410 | config['success'], | |
111fa98e VF |
411 | config['test_id'], |
412 | test_env['artifact_root'], | |
fb551941 VF |
413 | fio_pre_job=fio_pre_job, |
414 | fio_pre_success=fio_pre_success, | |
415 | output_format=output_format) | |
416 | desc = config['job'] | |
2e976800 | 417 | parameters = config['parameters'] if 'parameters' in config else None |
5dc5c6f6 VF |
418 | elif issubclass(config['test_class'], FioJobCmdTest): |
419 | if not 'success' in config: | |
420 | config['success'] = SUCCESS_DEFAULT | |
421 | test = config['test_class'](test_env['fio_path'], | |
422 | config['success'], | |
423 | config['test_id'], | |
424 | test_env['artifact_root'], | |
425 | config['fio_opts'], | |
426 | test_env['basename']) | |
427 | desc = config['test_id'] | |
428 | parameters = config | |
fb551941 VF |
429 | elif issubclass(config['test_class'], FioExeTest): |
430 | exe_path = os.path.join(test_env['fio_root'], config['exe']) | |
68b3a741 | 431 | parameters = [] |
fb551941 VF |
432 | if config['parameters']: |
433 | parameters = [p.format(fio_path=test_env['fio_path'], nvmecdev=args.nvmecdev) | |
434 | for p in config['parameters']] | |
fb551941 VF |
435 | if Path(exe_path).suffix == '.py' and platform.system() == "Windows": |
436 | parameters.insert(0, exe_path) | |
437 | exe_path = "python.exe" | |
438 | if config['test_id'] in test_env['pass_through']: | |
439 | parameters += test_env['pass_through'][config['test_id']].split() | |
111fa98e VF |
440 | test = config['test_class']( |
441 | exe_path, | |
442 | config['success'], | |
443 | config['test_id'], | |
444 | test_env['artifact_root']) | |
fb551941 VF |
445 | desc = config['exe'] |
446 | else: | |
447 | print(f"Test {config['test_id']} FAILED: unable to process test config") | |
448 | failed = failed + 1 | |
449 | continue | |
450 | ||
5dc5c6f6 | 451 | if 'requirements' in config and not args.skip_req: |
fb551941 VF |
452 | reqs_met = True |
453 | for req in config['requirements']: | |
454 | reqs_met, reason = req() | |
455 | logging.debug("Test %d: Requirement '%s' met? %s", config['test_id'], reason, | |
456 | reqs_met) | |
457 | if not reqs_met: | |
458 | break | |
459 | if not reqs_met: | |
460 | print(f"Test {config['test_id']} SKIPPED ({reason}) {desc}") | |
461 | skipped = skipped + 1 | |
462 | continue | |
463 | ||
464 | try: | |
111fa98e | 465 | test.setup(parameters) |
fb551941 VF |
466 | test.run() |
467 | test.check_result() | |
468 | except KeyboardInterrupt: | |
469 | break | |
470 | except Exception as e: | |
471 | test.passed = False | |
472 | test.failure_reason += str(e) | |
473 | logging.debug("Test %d exception:\n%s\n", config['test_id'], traceback.format_exc()) | |
474 | if test.passed: | |
475 | result = "PASSED" | |
476 | passed = passed + 1 | |
477 | else: | |
478 | result = f"FAILED: {test.failure_reason}" | |
479 | failed = failed + 1 | |
68b3a741 | 480 | contents, _ = get_file(test.filenames['stderr']) |
fb551941 | 481 | logging.debug("Test %d: stderr:\n%s", config['test_id'], contents) |
68b3a741 | 482 | contents, _ = get_file(test.filenames['stdout']) |
fb551941 VF |
483 | logging.debug("Test %d: stdout:\n%s", config['test_id'], contents) |
484 | print(f"Test {config['test_id']} {result} {desc}") | |
485 | ||
486 | print(f"{passed} test(s) passed, {failed} failed, {skipped} skipped") | |
487 | ||
488 | return passed, failed, skipped |