Commit | Line | Data |
---|---|---|
fb551941 VF |
1 | #!/usr/bin/env python3 |
2 | """ | |
3 | fiotestlib.py | |
4 | ||
5 | This library contains FioTest objects that provide convenient means to run | |
6 | different sorts of fio tests. | |
7 | ||
8 | It also contains a test runner that runs an array of dictionary objects | |
9 | describing fio tests. | |
10 | """ | |
11 | ||
12 | import os | |
13 | import sys | |
14 | import json | |
15 | import locale | |
16 | import logging | |
17 | import platform | |
18 | import traceback | |
19 | import subprocess | |
20 | from pathlib import Path | |
5dc5c6f6 | 21 | from fiotestcommon import get_file, SUCCESS_DEFAULT |
fb551941 VF |
22 | |
23 | ||
24 | class FioTest(): | |
25 | """Base for all fio tests.""" | |
26 | ||
111fa98e | 27 | def __init__(self, exe_path, success, testnum, artifact_root): |
fb551941 | 28 | self.success = success |
111fa98e | 29 | self.testnum = testnum |
fb551941 | 30 | self.output = {} |
fb551941 VF |
31 | self.passed = True |
32 | self.failure_reason = '' | |
68b3a741 | 33 | self.parameters = None |
111fa98e VF |
34 | self.paths = { |
35 | 'exe': exe_path, | |
36 | 'artifacts': artifact_root, | |
37 | 'test_dir': os.path.join(artifact_root, \ | |
38 | f"{testnum:04d}"), | |
39 | } | |
40 | self.filenames = { | |
41 | 'cmd': os.path.join(self.paths['test_dir'], \ | |
42 | f"{os.path.basename(self.paths['exe'])}.command"), | |
43 | 'stdout': os.path.join(self.paths['test_dir'], \ | |
44 | f"{os.path.basename(self.paths['exe'])}.stdout"), | |
45 | 'stderr': os.path.join(self.paths['test_dir'], \ | |
46 | f"{os.path.basename(self.paths['exe'])}.stderr"), | |
47 | 'exitcode': os.path.join(self.paths['test_dir'], \ | |
48 | f"{os.path.basename(self.paths['exe'])}.exitcode"), | |
49 | } | |
50 | ||
51 | def setup(self, parameters): | |
fb551941 VF |
52 | """Setup instance variables for test.""" |
53 | ||
68b3a741 | 54 | self.parameters = parameters |
68b3a741 VF |
55 | if not os.path.exists(self.paths['test_dir']): |
56 | os.mkdir(self.paths['test_dir']) | |
57 | ||
fb551941 VF |
58 | def run(self): |
59 | """Run the test.""" | |
60 | ||
61 | raise NotImplementedError() | |
62 | ||
63 | def check_result(self): | |
64 | """Check test results.""" | |
65 | ||
66 | raise NotImplementedError() | |
67 | ||
68 | ||
69 | class FioExeTest(FioTest): | |
70 | """Test consists of an executable binary or script""" | |
71 | ||
fb551941 VF |
72 | def run(self): |
73 | """Execute the binary or script described by this instance.""" | |
74 | ||
68b3a741 | 75 | command = [self.paths['exe']] + self.parameters |
1cf0ba9d VF |
76 | with open(self.filenames['cmd'], "w+", |
77 | encoding=locale.getpreferredencoding()) as command_file: | |
7b570114 | 78 | command_file.write(" \\\n ".join(command)) |
1cf0ba9d | 79 | |
fb551941 | 80 | try: |
1cf0ba9d VF |
81 | with open(self.filenames['stdout'], "w+", |
82 | encoding=locale.getpreferredencoding()) as stdout_file, \ | |
83 | open(self.filenames['stderr'], "w+", | |
84 | encoding=locale.getpreferredencoding()) as stderr_file, \ | |
85 | open(self.filenames['exitcode'], "w+", | |
86 | encoding=locale.getpreferredencoding()) as exitcode_file: | |
87 | proc = None | |
88 | # Avoid using subprocess.run() here because when a timeout occurs, | |
89 | # fio will be stopped with SIGKILL. This does not give fio a | |
90 | # chance to clean up and means that child processes may continue | |
91 | # running and submitting IO. | |
92 | proc = subprocess.Popen(command, | |
93 | stdout=stdout_file, | |
94 | stderr=stderr_file, | |
95 | cwd=self.paths['test_dir'], | |
96 | universal_newlines=True) | |
97 | proc.communicate(timeout=self.success['timeout']) | |
98 | exitcode_file.write(f'{proc.returncode}\n') | |
99 | logging.debug("Test %d: return code: %d", self.testnum, proc.returncode) | |
100 | self.output['proc'] = proc | |
fb551941 VF |
101 | except subprocess.TimeoutExpired: |
102 | proc.terminate() | |
103 | proc.communicate() | |
104 | assert proc.poll() | |
105 | self.output['failure'] = 'timeout' | |
106 | except Exception: | |
107 | if proc: | |
108 | if not proc.poll(): | |
109 | proc.terminate() | |
110 | proc.communicate() | |
111 | self.output['failure'] = 'exception' | |
112 | self.output['exc_info'] = sys.exc_info() | |
fb551941 VF |
113 | |
114 | def check_result(self): | |
115 | """Check results of test run.""" | |
116 | ||
117 | if 'proc' not in self.output: | |
118 | if self.output['failure'] == 'timeout': | |
119 | self.failure_reason = f"{self.failure_reason} timeout," | |
120 | else: | |
121 | assert self.output['failure'] == 'exception' | |
6544f1ae VF |
122 | self.failure_reason = f'{self.failure_reason} exception: ' + \ |
123 | f'{self.output["exc_info"][0]}, {self.output["exc_info"][1]}' | |
fb551941 VF |
124 | |
125 | self.passed = False | |
126 | return | |
127 | ||
128 | if 'zero_return' in self.success: | |
129 | if self.success['zero_return']: | |
130 | if self.output['proc'].returncode != 0: | |
131 | self.passed = False | |
132 | self.failure_reason = f"{self.failure_reason} non-zero return code," | |
133 | else: | |
134 | if self.output['proc'].returncode == 0: | |
135 | self.failure_reason = f"{self.failure_reason} zero return code," | |
136 | self.passed = False | |
137 | ||
68b3a741 | 138 | stderr_size = os.path.getsize(self.filenames['stderr']) |
fb551941 VF |
139 | if 'stderr_empty' in self.success: |
140 | if self.success['stderr_empty']: | |
141 | if stderr_size != 0: | |
142 | self.failure_reason = f"{self.failure_reason} stderr not empty," | |
143 | self.passed = False | |
144 | else: | |
145 | if stderr_size == 0: | |
146 | self.failure_reason = f"{self.failure_reason} stderr empty," | |
147 | self.passed = False | |
148 | ||
149 | ||
0dc6e911 | 150 | class FioJobFileTest(FioExeTest): |
68b3a741 | 151 | """Test consists of a fio job with options in a job file.""" |
fb551941 | 152 | |
111fa98e VF |
153 | def __init__(self, fio_path, fio_job, success, testnum, artifact_root, |
154 | fio_pre_job=None, fio_pre_success=None, | |
155 | output_format="normal"): | |
0dc6e911 | 156 | """Construct a FioJobFileTest which is a FioExeTest consisting of a |
fb551941 VF |
157 | single fio job file with an optional setup step. |
158 | ||
159 | fio_path: location of fio executable | |
160 | fio_job: location of fio job file | |
161 | success: Definition of test success | |
111fa98e VF |
162 | testnum: test ID |
163 | artifact_root: root directory for artifacts | |
fb551941 VF |
164 | fio_pre_job: fio job for preconditioning |
165 | fio_pre_success: Definition of test success for fio precon job | |
166 | output_format: normal (default), json, jsonplus, or terse | |
167 | """ | |
168 | ||
169 | self.fio_job = fio_job | |
170 | self.fio_pre_job = fio_pre_job | |
171 | self.fio_pre_success = fio_pre_success if fio_pre_success else success | |
172 | self.output_format = output_format | |
173 | self.precon_failed = False | |
174 | self.json_data = None | |
68b3a741 | 175 | |
111fa98e | 176 | super().__init__(fio_path, success, testnum, artifact_root) |
68b3a741 | 177 | |
111fa98e | 178 | def setup(self, parameters=None): |
68b3a741 VF |
179 | """Setup instance variables for fio job test.""" |
180 | ||
181 | self.filenames['fio_output'] = f"{os.path.basename(self.fio_job)}.output" | |
182 | fio_args = [ | |
fb551941 VF |
183 | "--max-jobs=16", |
184 | f"--output-format={self.output_format}", | |
68b3a741 | 185 | f"--output={self.filenames['fio_output']}", |
fb551941 VF |
186 | self.fio_job, |
187 | ] | |
fb551941 | 188 | |
111fa98e | 189 | super().setup(fio_args) |
fb551941 | 190 | |
68b3a741 VF |
191 | # Update the filenames from the default |
192 | self.filenames['cmd'] = os.path.join(self.paths['test_dir'], | |
193 | f"{os.path.basename(self.fio_job)}.command") | |
194 | self.filenames['stdout'] = os.path.join(self.paths['test_dir'], | |
195 | f"{os.path.basename(self.fio_job)}.stdout") | |
196 | self.filenames['stderr'] = os.path.join(self.paths['test_dir'], | |
197 | f"{os.path.basename(self.fio_job)}.stderr") | |
198 | self.filenames['exitcode'] = os.path.join(self.paths['test_dir'], | |
199 | f"{os.path.basename(self.fio_job)}.exitcode") | |
fb551941 VF |
200 | |
201 | def run_pre_job(self): | |
202 | """Run fio job precondition step.""" | |
203 | ||
68b3a741 | 204 | precon = FioJobFileTest(self.paths['exe'], self.fio_pre_job, |
fb551941 | 205 | self.fio_pre_success, |
111fa98e VF |
206 | self.testnum, |
207 | self.paths['artifacts'], | |
fb551941 | 208 | output_format=self.output_format) |
111fa98e | 209 | precon.setup() |
fb551941 VF |
210 | precon.run() |
211 | precon.check_result() | |
212 | self.precon_failed = not precon.passed | |
213 | self.failure_reason = precon.failure_reason | |
214 | ||
215 | def run(self): | |
216 | """Run fio job test.""" | |
217 | ||
218 | if self.fio_pre_job: | |
219 | self.run_pre_job() | |
220 | ||
221 | if not self.precon_failed: | |
222 | super().run() | |
223 | else: | |
224 | logging.debug("Test %d: precondition step failed", self.testnum) | |
225 | ||
fb551941 VF |
226 | def get_file_fail(self, filename): |
227 | """Safely read a file and fail the test upon error.""" | |
228 | file_data = None | |
229 | ||
230 | try: | |
231 | with open(filename, "r", encoding=locale.getpreferredencoding()) as output_file: | |
232 | file_data = output_file.read() | |
233 | except OSError: | |
234 | self.failure_reason += f" unable to read file {filename}" | |
235 | self.passed = False | |
236 | ||
237 | return file_data | |
238 | ||
239 | def check_result(self): | |
240 | """Check fio job results.""" | |
241 | ||
242 | if self.precon_failed: | |
243 | self.passed = False | |
244 | self.failure_reason = f"{self.failure_reason} precondition step failed," | |
245 | return | |
246 | ||
247 | super().check_result() | |
248 | ||
249 | if not self.passed: | |
250 | return | |
251 | ||
252 | if 'json' not in self.output_format: | |
253 | return | |
254 | ||
68b3a741 VF |
255 | file_data = self.get_file_fail(os.path.join(self.paths['test_dir'], |
256 | self.filenames['fio_output'])) | |
fb551941 VF |
257 | if not file_data: |
258 | return | |
259 | ||
260 | # | |
261 | # Sometimes fio informational messages are included at the top of the | |
262 | # JSON output, especially under Windows. Try to decode output as JSON | |
263 | # data, skipping everything until the first { | |
264 | # | |
265 | lines = file_data.splitlines() | |
266 | file_data = '\n'.join(lines[lines.index("{"):]) | |
267 | try: | |
268 | self.json_data = json.loads(file_data) | |
269 | except json.JSONDecodeError: | |
270 | self.failure_reason = f"{self.failure_reason} unable to decode JSON data," | |
271 | self.passed = False | |
272 | ||
273 | ||
5dc5c6f6 VF |
274 | class FioJobCmdTest(FioExeTest): |
275 | """This runs a fio job with options specified on the command line.""" | |
276 | ||
277 | def __init__(self, fio_path, success, testnum, artifact_root, fio_opts, basename=None): | |
278 | ||
279 | self.basename = basename if basename else os.path.basename(fio_path) | |
280 | self.fio_opts = fio_opts | |
281 | self.json_data = None | |
885e170a | 282 | self.iops_log_lines = None |
5dc5c6f6 VF |
283 | |
284 | super().__init__(fio_path, success, testnum, artifact_root) | |
285 | ||
885e170a VF |
286 | filename_stub = os.path.join(self.paths['test_dir'], f"{self.basename}{self.testnum:03d}") |
287 | self.filenames['cmd'] = f"{filename_stub}.command" | |
288 | self.filenames['stdout'] = f"{filename_stub}.stdout" | |
289 | self.filenames['stderr'] = f"{filename_stub}.stderr" | |
290 | self.filenames['output'] = os.path.abspath(f"{filename_stub}.output") | |
291 | self.filenames['exitcode'] = f"{filename_stub}.exitcode" | |
292 | self.filenames['iopslog'] = os.path.abspath(f"{filename_stub}") | |
5dc5c6f6 VF |
293 | |
294 | def run(self): | |
295 | super().run() | |
296 | ||
297 | if 'output-format' in self.fio_opts and 'json' in \ | |
298 | self.fio_opts['output-format']: | |
299 | if not self.get_json(): | |
300 | print('Unable to decode JSON data') | |
301 | self.passed = False | |
302 | ||
885e170a VF |
303 | if any('--write_iops_log=' in param for param in self.parameters): |
304 | self.get_iops_log() | |
305 | ||
306 | def get_iops_log(self): | |
307 | """Read IOPS log from the first job.""" | |
308 | ||
309 | log_filename = self.filenames['iopslog'] + "_iops.1.log" | |
310 | with open(log_filename, 'r', encoding=locale.getpreferredencoding()) as iops_file: | |
311 | self.iops_log_lines = iops_file.read() | |
312 | ||
5dc5c6f6 VF |
313 | def get_json(self): |
314 | """Convert fio JSON output into a python JSON object""" | |
315 | ||
316 | filename = self.filenames['output'] | |
317 | with open(filename, 'r', encoding=locale.getpreferredencoding()) as file: | |
318 | file_data = file.read() | |
319 | ||
320 | # | |
321 | # Sometimes fio informational messages are included at the top of the | |
322 | # JSON output, especially under Windows. Try to decode output as JSON | |
323 | # data, lopping off up to the first four lines | |
324 | # | |
325 | lines = file_data.splitlines() | |
326 | for i in range(5): | |
327 | file_data = '\n'.join(lines[i:]) | |
328 | try: | |
329 | self.json_data = json.loads(file_data) | |
330 | except json.JSONDecodeError: | |
331 | continue | |
332 | else: | |
333 | return True | |
334 | ||
335 | return False | |
336 | ||
337 | @staticmethod | |
338 | def check_empty(job): | |
339 | """ | |
340 | Make sure JSON data is empty. | |
341 | ||
342 | Some data structures should be empty. This function makes sure that they are. | |
343 | ||
344 | job JSON object that we need to check for emptiness | |
345 | """ | |
346 | ||
347 | return job['total_ios'] == 0 and \ | |
348 | job['slat_ns']['N'] == 0 and \ | |
349 | job['clat_ns']['N'] == 0 and \ | |
350 | job['lat_ns']['N'] == 0 | |
351 | ||
352 | def check_all_ddirs(self, ddir_nonzero, job): | |
353 | """ | |
354 | Iterate over the data directions and check whether each is | |
355 | appropriately empty or not. | |
356 | """ | |
357 | ||
358 | retval = True | |
359 | ddirlist = ['read', 'write', 'trim'] | |
360 | ||
361 | for ddir in ddirlist: | |
362 | if ddir in ddir_nonzero: | |
363 | if self.check_empty(job[ddir]): | |
364 | print(f"Unexpected zero {ddir} data found in output") | |
365 | retval = False | |
366 | else: | |
367 | if not self.check_empty(job[ddir]): | |
368 | print(f"Unexpected {ddir} data found in output") | |
369 | retval = False | |
370 | ||
371 | return retval | |
372 | ||
373 | ||
fb551941 VF |
374 | def run_fio_tests(test_list, test_env, args): |
375 | """ | |
376 | Run tests as specified in test_list. | |
377 | """ | |
378 | ||
379 | passed = 0 | |
380 | failed = 0 | |
381 | skipped = 0 | |
382 | ||
383 | for config in test_list: | |
384 | if (args.skip and config['test_id'] in args.skip) or \ | |
140a30e3 VF |
385 | (args.run_only and config['test_id'] not in args.run_only) or \ |
386 | ('force_skip' in config and config['force_skip']): | |
fb551941 | 387 | skipped = skipped + 1 |
140a30e3 | 388 | print(f"Test {config['test_id']} SKIPPED (User request or override)") |
fb551941 VF |
389 | continue |
390 | ||
0dc6e911 | 391 | if issubclass(config['test_class'], FioJobFileTest): |
fb551941 VF |
392 | if config['pre_job']: |
393 | fio_pre_job = os.path.join(test_env['fio_root'], 't', 'jobs', | |
394 | config['pre_job']) | |
395 | else: | |
396 | fio_pre_job = None | |
397 | if config['pre_success']: | |
398 | fio_pre_success = config['pre_success'] | |
399 | else: | |
400 | fio_pre_success = None | |
401 | if 'output_format' in config: | |
402 | output_format = config['output_format'] | |
403 | else: | |
404 | output_format = 'normal' | |
405 | test = config['test_class']( | |
406 | test_env['fio_path'], | |
407 | os.path.join(test_env['fio_root'], 't', 'jobs', config['job']), | |
408 | config['success'], | |
111fa98e VF |
409 | config['test_id'], |
410 | test_env['artifact_root'], | |
fb551941 VF |
411 | fio_pre_job=fio_pre_job, |
412 | fio_pre_success=fio_pre_success, | |
413 | output_format=output_format) | |
414 | desc = config['job'] | |
68b3a741 | 415 | parameters = [] |
5dc5c6f6 VF |
416 | elif issubclass(config['test_class'], FioJobCmdTest): |
417 | if not 'success' in config: | |
418 | config['success'] = SUCCESS_DEFAULT | |
419 | test = config['test_class'](test_env['fio_path'], | |
420 | config['success'], | |
421 | config['test_id'], | |
422 | test_env['artifact_root'], | |
423 | config['fio_opts'], | |
424 | test_env['basename']) | |
425 | desc = config['test_id'] | |
426 | parameters = config | |
fb551941 VF |
427 | elif issubclass(config['test_class'], FioExeTest): |
428 | exe_path = os.path.join(test_env['fio_root'], config['exe']) | |
68b3a741 | 429 | parameters = [] |
fb551941 VF |
430 | if config['parameters']: |
431 | parameters = [p.format(fio_path=test_env['fio_path'], nvmecdev=args.nvmecdev) | |
432 | for p in config['parameters']] | |
fb551941 VF |
433 | if Path(exe_path).suffix == '.py' and platform.system() == "Windows": |
434 | parameters.insert(0, exe_path) | |
435 | exe_path = "python.exe" | |
436 | if config['test_id'] in test_env['pass_through']: | |
437 | parameters += test_env['pass_through'][config['test_id']].split() | |
111fa98e VF |
438 | test = config['test_class']( |
439 | exe_path, | |
440 | config['success'], | |
441 | config['test_id'], | |
442 | test_env['artifact_root']) | |
fb551941 VF |
443 | desc = config['exe'] |
444 | else: | |
445 | print(f"Test {config['test_id']} FAILED: unable to process test config") | |
446 | failed = failed + 1 | |
447 | continue | |
448 | ||
5dc5c6f6 | 449 | if 'requirements' in config and not args.skip_req: |
fb551941 VF |
450 | reqs_met = True |
451 | for req in config['requirements']: | |
452 | reqs_met, reason = req() | |
453 | logging.debug("Test %d: Requirement '%s' met? %s", config['test_id'], reason, | |
454 | reqs_met) | |
455 | if not reqs_met: | |
456 | break | |
457 | if not reqs_met: | |
458 | print(f"Test {config['test_id']} SKIPPED ({reason}) {desc}") | |
459 | skipped = skipped + 1 | |
460 | continue | |
461 | ||
462 | try: | |
111fa98e | 463 | test.setup(parameters) |
fb551941 VF |
464 | test.run() |
465 | test.check_result() | |
466 | except KeyboardInterrupt: | |
467 | break | |
468 | except Exception as e: | |
469 | test.passed = False | |
470 | test.failure_reason += str(e) | |
471 | logging.debug("Test %d exception:\n%s\n", config['test_id'], traceback.format_exc()) | |
472 | if test.passed: | |
473 | result = "PASSED" | |
474 | passed = passed + 1 | |
475 | else: | |
476 | result = f"FAILED: {test.failure_reason}" | |
477 | failed = failed + 1 | |
68b3a741 | 478 | contents, _ = get_file(test.filenames['stderr']) |
fb551941 | 479 | logging.debug("Test %d: stderr:\n%s", config['test_id'], contents) |
68b3a741 | 480 | contents, _ = get_file(test.filenames['stdout']) |
fb551941 VF |
481 | logging.debug("Test %d: stdout:\n%s", config['test_id'], contents) |
482 | print(f"Test {config['test_id']} {result} {desc}") | |
483 | ||
484 | print(f"{passed} test(s) passed, {failed} failed, {skipped} skipped") | |
485 | ||
486 | return passed, failed, skipped |