2 # SPDX-License-Identifier: GPL-2.0-only
4 # Copyright (c) 2019 Western Digital Corporation or its affiliates.
9 # Automate running of fio tests
12 # python3 run-fio-tests.py [-r fio-root] [-f fio-path] [-a artifact-root]
13 # [--skip # # #...] [--run-only # # #...]
17 # # git clone git://git.kernel.dk/fio.git
20 # # python3 t/run-fio-tests.py
24 # - Python 3.5 (subprocess.run)
25 # - Linux (libaio ioengine, zbd tests, etc)
26 # - The artifact directory must be on a file system that accepts 512-byte IO
27 # (t0002, t0003, t0004).
28 # - The artifact directory needs to be on an SSD. Otherwise tests that carry
29 # out file-based IO will trigger a timeout (t0006).
31 # - SciPy (steadystate_tests.py)
32 # - libzbc (zbd tests)
33 # - root privileges (zbd test)
34 # - kernel 4.19 or later for zoned null block devices (zbd tests)
35 # - CUnit support (unittests)
40 # TODO run multiple tests simultaneously
41 # TODO Add sgunmap tests (requires SAS SSD)
53 import multiprocessing
54 from pathlib import Path
57 class FioTest(object):
58 """Base for all fio tests."""
60 def __init__(self, exe_path, parameters, success):
61 self.exe_path = exe_path
62 self.parameters = parameters
63 self.success = success
65 self.artifact_root = None
69 self.failure_reason = ''
71 def setup(self, artifact_root, testnum):
72 self.artifact_root = artifact_root
73 self.testnum = testnum
74 self.test_dir = os.path.join(artifact_root, "{:04d}".format(testnum))
75 if not os.path.exists(self.test_dir):
76 os.mkdir(self.test_dir)
78 self.command_file = os.path.join(
80 "{0}.command".format(os.path.basename(self.exe_path)))
81 self.stdout_file = os.path.join(
83 "{0}.stdout".format(os.path.basename(self.exe_path)))
84 self.stderr_file = os.path.join(
86 "{0}.stderr".format(os.path.basename(self.exe_path)))
87 self.exticode_file = os.path.join(
89 "{0}.exitcode".format(os.path.basename(self.exe_path)))
92 raise NotImplementedError()
94 def check_result(self):
95 raise NotImplementedError()
98 class FioExeTest(FioTest):
99 """Test consists of an executable binary or script"""
101 def __init__(self, exe_path, parameters, success):
102 """Construct a FioExeTest which is a FioTest consisting of an
103 executable binary or script.
105 exe_path: location of executable binary or script
106 parameters: list of parameters for executable
107 success: Definition of test success
110 FioTest.__init__(self, exe_path, parameters, success)
112 def setup(self, artifact_root, testnum):
113 super(FioExeTest, self).setup(artifact_root, testnum)
117 command = [self.exe_path] + self.parameters
119 command = [self.exe_path]
120 command_file = open(self.command_file, "w+")
121 command_file.write("%s\n" % command)
124 stdout_file = open(self.stdout_file, "w+")
125 stderr_file = open(self.stderr_file, "w+")
126 exticode_file = open(self.exticode_file, "w+")
129 # Avoid using subprocess.run() here because when a timeout occurs,
130 # fio will be stopped with SIGKILL. This does not give fio a
131 # chance to clean up and means that child processes may continue
132 # running and submitting IO.
133 proc = subprocess.Popen(command,
137 universal_newlines=True)
138 proc.communicate(timeout=self.success['timeout'])
139 exticode_file.write('{0}\n'.format(proc.returncode))
140 logging.debug("Test %d: return code: %d" % (self.testnum, proc.returncode))
141 self.output['proc'] = proc
142 except subprocess.TimeoutExpired:
146 self.output['failure'] = 'timeout'
152 self.output['failure'] = 'exception'
153 self.output['exc_info'] = sys.exc_info()
157 exticode_file.close()
159 def check_result(self):
160 if 'proc' not in self.output:
161 if self.output['failure'] == 'timeout':
162 self.failure_reason = "{0} timeout,".format(self.failure_reason)
164 assert self.output['failure'] == 'exception'
165 self.failure_reason = '{0} exception: {1}, {2}'.format(
166 self.failure_reason, self.output['exc_info'][0],
167 self.output['exc_info'][1])
172 if 'zero_return' in self.success:
173 if self.success['zero_return']:
174 if self.output['proc'].returncode != 0:
176 self.failure_reason = "{0} non-zero return code,".format(self.failure_reason)
178 if self.output['proc'].returncode == 0:
179 self.failure_reason = "{0} zero return code,".format(self.failure_reason)
182 stderr_size = os.path.getsize(self.stderr_file)
183 if 'stderr_empty' in self.success:
184 if self.success['stderr_empty']:
186 self.failure_reason = "{0} stderr not empty,".format(self.failure_reason)
190 self.failure_reason = "{0} stderr empty,".format(self.failure_reason)
194 class FioJobTest(FioExeTest):
195 """Test consists of a fio job"""
197 def __init__(self, fio_path, fio_job, success, fio_pre_job=None,
198 fio_pre_success=None, output_format="normal"):
199 """Construct a FioJobTest which is a FioExeTest consisting of a
200 single fio job file with an optional setup step.
202 fio_path: location of fio executable
203 fio_job: location of fio job file
204 success: Definition of test success
205 fio_pre_job: fio job for preconditioning
206 fio_pre_success: Definition of test success for fio precon job
207 output_format: normal (default), json, jsonplus, or terse
210 self.fio_job = fio_job
211 self.fio_pre_job = fio_pre_job
212 self.fio_pre_success = fio_pre_success if fio_pre_success else success
213 self.output_format = output_format
214 self.precon_failed = False
215 self.json_data = None
216 self.fio_output = "{0}.output".format(os.path.basename(self.fio_job))
218 "--output-format={0}".format(self.output_format),
219 "--output={0}".format(self.fio_output),
222 FioExeTest.__init__(self, fio_path, self.fio_args, success)
224 def setup(self, artifact_root, testnum):
225 super(FioJobTest, self).setup(artifact_root, testnum)
227 self.command_file = os.path.join(
229 "{0}.command".format(os.path.basename(self.fio_job)))
230 self.stdout_file = os.path.join(
232 "{0}.stdout".format(os.path.basename(self.fio_job)))
233 self.stderr_file = os.path.join(
235 "{0}.stderr".format(os.path.basename(self.fio_job)))
236 self.exticode_file = os.path.join(
238 "{0}.exitcode".format(os.path.basename(self.fio_job)))
240 def run_pre_job(self):
241 precon = FioJobTest(self.exe_path, self.fio_pre_job,
242 self.fio_pre_success,
243 output_format=self.output_format)
244 precon.setup(self.artifact_root, self.testnum)
246 precon.check_result()
247 self.precon_failed = not precon.passed
248 self.failure_reason = precon.failure_reason
254 if not self.precon_failed:
255 super(FioJobTest, self).run()
257 logging.debug("Test %d: precondition step failed" % self.testnum)
259 def check_result(self):
260 if self.precon_failed:
262 self.failure_reason = "{0} precondition step failed,".format(self.failure_reason)
265 super(FioJobTest, self).check_result()
270 if not 'json' in self.output_format:
274 with open(os.path.join(self.test_dir, self.fio_output), "r") as output_file:
275 file_data = output_file.read()
276 except EnvironmentError:
277 self.failure_reason = "{0} unable to open output file,".format(self.failure_reason)
282 # Sometimes fio informational messages are included at the top of the
283 # JSON output, especially under Windows. Try to decode output as JSON
284 # data, lopping off up to the first four lines
286 lines = file_data.splitlines()
288 file_data = '\n'.join(lines[i:])
290 self.json_data = json.loads(file_data)
291 except json.JSONDecodeError:
294 logging.debug("Test %d: skipped %d lines decoding JSON data" % (self.testnum, i))
297 self.failure_reason = "{0} unable to decode JSON data,".format(self.failure_reason)
301 class FioJobTest_t0005(FioJobTest):
302 """Test consists of fio test job t0005
303 Confirm that read['io_kbytes'] == write['io_kbytes'] == 102400"""
305 def check_result(self):
306 super(FioJobTest_t0005, self).check_result()
311 if self.json_data['jobs'][0]['read']['io_kbytes'] != 102400:
312 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
314 if self.json_data['jobs'][0]['write']['io_kbytes'] != 102400:
315 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
319 class FioJobTest_t0006(FioJobTest):
320 """Test consists of fio test job t0006
321 Confirm that read['io_kbytes'] ~ 2*write['io_kbytes']"""
323 def check_result(self):
324 super(FioJobTest_t0006, self).check_result()
329 ratio = self.json_data['jobs'][0]['read']['io_kbytes'] \
330 / self.json_data['jobs'][0]['write']['io_kbytes']
331 logging.debug("Test %d: ratio: %f" % (self.testnum, ratio))
332 if ratio < 1.99 or ratio > 2.01:
333 self.failure_reason = "{0} read/write ratio mismatch,".format(self.failure_reason)
337 class FioJobTest_t0007(FioJobTest):
338 """Test consists of fio test job t0007
339 Confirm that read['io_kbytes'] = 87040"""
341 def check_result(self):
342 super(FioJobTest_t0007, self).check_result()
347 if self.json_data['jobs'][0]['read']['io_kbytes'] != 87040:
348 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
352 class FioJobTest_t0008(FioJobTest):
353 """Test consists of fio test job t0008
354 Confirm that read['io_kbytes'] = 32768 and that
355 write['io_kbytes'] ~ 16568
357 I did runs with fio-ae2fafc8 and saw write['io_kbytes'] values of
358 16585, 16588. With two runs of fio-3.16 I obtained 16568"""
360 def check_result(self):
361 super(FioJobTest_t0008, self).check_result()
366 ratio = self.json_data['jobs'][0]['write']['io_kbytes'] / 16568
367 logging.debug("Test %d: ratio: %f" % (self.testnum, ratio))
369 if ratio < 0.99 or ratio > 1.01:
370 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
372 if self.json_data['jobs'][0]['read']['io_kbytes'] != 32768:
373 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
377 class FioJobTest_t0009(FioJobTest):
378 """Test consists of fio test job t0009
379 Confirm that runtime >= 60s"""
381 def check_result(self):
382 super(FioJobTest_t0009, self).check_result()
387 logging.debug('Test %d: elapsed: %d' % (self.testnum, self.json_data['jobs'][0]['elapsed']))
389 if self.json_data['jobs'][0]['elapsed'] < 60:
390 self.failure_reason = "{0} elapsed time mismatch,".format(self.failure_reason)
394 class FioJobTest_t0011(FioJobTest):
395 """Test consists of fio test job t0009
396 Confirm that job0 iops == 1000
397 and that job1_iops / job0_iops ~ 8
398 With two runs of fio-3.16 I observed a ratio of 8.3"""
400 def check_result(self):
401 super(FioJobTest_t0011, self).check_result()
406 iops1 = self.json_data['jobs'][0]['read']['iops']
407 iops2 = self.json_data['jobs'][1]['read']['iops']
408 ratio = iops2 / iops1
409 logging.debug("Test %d: ratio: %f" % (self.testnum, ratio))
411 if iops1 < 999 or iops1 > 1001:
412 self.failure_reason = "{0} iops value mismatch,".format(self.failure_reason)
415 if ratio < 7 or ratio > 9:
416 self.failure_reason = "{0} iops ratio mismatch,".format(self.failure_reason)
420 class Requirements(object):
421 """Requirements consists of multiple run environment characteristics.
422 These are to determine if a particular test can be run"""
433 def __init__(self, fio_root):
434 Requirements._not_macos = platform.system() != "Darwin"
435 Requirements._linux = platform.system() == "Linux"
437 if Requirements._linux:
439 config_file = os.path.join(fio_root, "config-host.h")
440 with open(config_file, "r") as config:
441 contents = config.read()
443 print("Unable to open {0} to check requirements".format(config_file))
444 Requirements._zbd = True
446 Requirements._zbd = "CONFIG_LINUX_BLKZONED" in contents
447 Requirements._libaio = "CONFIG_LIBAIO" in contents
449 Requirements._root = (os.geteuid() == 0)
450 if Requirements._zbd and Requirements._root:
451 subprocess.run(["modprobe", "null_blk"],
452 stdout=subprocess.PIPE,
453 stderr=subprocess.PIPE)
454 if os.path.exists("/sys/module/null_blk/parameters/zoned"):
455 Requirements._zoned_nullb = True
457 if platform.system() == "Windows":
458 utest_exe = "unittest.exe"
460 utest_exe = "unittest"
461 unittest_path = os.path.join(fio_root, "unittests", utest_exe)
462 Requirements._unittests = os.path.exists(unittest_path)
464 Requirements._cpucount4 = multiprocessing.cpu_count() >= 4
466 req_list = [Requirements.linux,
470 Requirements.zoned_nullb,
471 Requirements.not_macos,
472 Requirements.unittests,
473 Requirements.cpucount4]
476 logging.debug("Requirements: Requirement '%s' met? %s" % (desc, value))
479 return Requirements._linux, "Linux required"
482 return Requirements._libaio, "libaio required"
485 return Requirements._zbd, "Zoned block device support required"
488 return Requirements._root, "root required"
491 return Requirements._zoned_nullb, "Zoned null block device support required"
494 return Requirements._not_macos, "platform other than macOS required"
497 return Requirements._unittests, "Unittests support required"
500 return Requirements._cpucount4, "4+ CPUs required"
505 'stderr_empty': True,
509 'zero_return': False,
510 'stderr_empty': False,
515 'stderr_empty': False,
521 'test_class': FioJobTest,
522 'job': 't0001-52c58027.fio',
523 'success': SUCCESS_DEFAULT,
530 'test_class': FioJobTest,
531 'job': 't0002-13af05ae-post.fio',
532 'success': SUCCESS_DEFAULT,
533 'pre_job': 't0002-13af05ae-pre.fio',
535 'requirements': [Requirements.linux, Requirements.libaio],
539 'test_class': FioJobTest,
540 'job': 't0003-0ae2c6e1-post.fio',
541 'success': SUCCESS_NONZERO,
542 'pre_job': 't0003-0ae2c6e1-pre.fio',
543 'pre_success': SUCCESS_DEFAULT,
544 'requirements': [Requirements.linux, Requirements.libaio],
548 'test_class': FioJobTest,
549 'job': 't0004-8a99fdf6.fio',
550 'success': SUCCESS_DEFAULT,
553 'requirements': [Requirements.linux, Requirements.libaio],
557 'test_class': FioJobTest_t0005,
558 'job': 't0005-f7078f7b.fio',
559 'success': SUCCESS_DEFAULT,
562 'output_format': 'json',
567 'test_class': FioJobTest_t0006,
568 'job': 't0006-82af2a7c.fio',
569 'success': SUCCESS_DEFAULT,
572 'output_format': 'json',
573 'requirements': [Requirements.linux, Requirements.libaio],
577 'test_class': FioJobTest_t0007,
578 'job': 't0007-37cf9e3c.fio',
579 'success': SUCCESS_DEFAULT,
582 'output_format': 'json',
587 'test_class': FioJobTest_t0008,
588 'job': 't0008-ae2fafc8.fio',
589 'success': SUCCESS_DEFAULT,
592 'output_format': 'json',
597 'test_class': FioJobTest_t0009,
598 'job': 't0009-f8b0bd10.fio',
599 'success': SUCCESS_DEFAULT,
602 'output_format': 'json',
603 'requirements': [Requirements.not_macos,
604 Requirements.cpucount4],
605 # mac os does not support CPU affinity
609 'test_class': FioJobTest,
610 'job': 't0010-b7aae4ba.fio',
611 'success': SUCCESS_DEFAULT,
618 'test_class': FioJobTest_t0011,
619 'job': 't0011-5d2788d5.fio',
620 'success': SUCCESS_DEFAULT,
623 'output_format': 'json',
628 'test_class': FioExeTest,
631 'success': SUCCESS_DEFAULT,
636 'test_class': FioExeTest,
639 'success': SUCCESS_DEFAULT,
644 'test_class': FioExeTest,
645 'exe': 't/lfsr-test',
646 'parameters': ['0xFFFFFF', '0', '0', 'verify'],
647 'success': SUCCESS_STDERR,
652 'test_class': FioExeTest,
653 'exe': 't/readonly.py',
654 'parameters': ['-f', '{fio_path}'],
655 'success': SUCCESS_DEFAULT,
660 'test_class': FioExeTest,
661 'exe': 't/steadystate_tests.py',
662 'parameters': ['{fio_path}'],
663 'success': SUCCESS_DEFAULT,
668 'test_class': FioExeTest,
671 'success': SUCCESS_STDERR,
676 'test_class': FioExeTest,
677 'exe': 't/strided.py',
678 'parameters': ['{fio_path}'],
679 'success': SUCCESS_DEFAULT,
684 'test_class': FioExeTest,
685 'exe': 't/zbd/run-tests-against-regular-nullb',
687 'success': SUCCESS_DEFAULT,
688 'requirements': [Requirements.linux, Requirements.zbd,
693 'test_class': FioExeTest,
694 'exe': 't/zbd/run-tests-against-zoned-nullb',
696 'success': SUCCESS_DEFAULT,
697 'requirements': [Requirements.linux, Requirements.zbd,
698 Requirements.root, Requirements.zoned_nullb],
702 'test_class': FioExeTest,
703 'exe': 'unittests/unittest',
705 'success': SUCCESS_DEFAULT,
706 'requirements': [Requirements.unittests],
712 parser = argparse.ArgumentParser()
713 parser.add_argument('-r', '--fio-root',
714 help='fio root path')
715 parser.add_argument('-f', '--fio',
716 help='path to fio executable (e.g., ./fio)')
717 parser.add_argument('-a', '--artifact-root',
718 help='artifact root directory')
719 parser.add_argument('-s', '--skip', nargs='+', type=int,
720 help='list of test(s) to skip')
721 parser.add_argument('-o', '--run-only', nargs='+', type=int,
722 help='list of test(s) to run, skipping all others')
723 parser.add_argument('-d', '--debug', action='store_true',
724 help='provide debug output')
725 parser.add_argument('-k', '--skip-req', action='store_true',
726 help='skip requirements checking')
727 args = parser.parse_args()
735 logging.basicConfig(level=logging.DEBUG)
737 logging.basicConfig(level=logging.INFO)
740 fio_root = args.fio_root
742 fio_root = str(Path(__file__).absolute().parent.parent)
743 print("fio root is %s" % fio_root)
748 if platform.system() == "Windows":
752 fio_path = os.path.join(fio_root, fio_exe)
753 print("fio path is %s" % fio_path)
754 if not shutil.which(fio_path):
755 print("Warning: fio executable not found")
757 artifact_root = args.artifact_root if args.artifact_root else \
758 "fio-test-{0}".format(time.strftime("%Y%m%d-%H%M%S"))
759 os.mkdir(artifact_root)
760 print("Artifact directory is %s" % artifact_root)
762 if not args.skip_req:
763 req = Requirements(fio_root)
769 for config in TEST_LIST:
770 if (args.skip and config['test_id'] in args.skip) or \
771 (args.run_only and config['test_id'] not in args.run_only):
772 skipped = skipped + 1
773 print("Test {0} SKIPPED (User request)".format(config['test_id']))
776 if issubclass(config['test_class'], FioJobTest):
777 if config['pre_job']:
778 fio_pre_job = os.path.join(fio_root, 't', 'jobs',
782 if config['pre_success']:
783 fio_pre_success = config['pre_success']
785 fio_pre_success = None
786 if 'output_format' in config:
787 output_format = config['output_format']
789 output_format = 'normal'
790 test = config['test_class'](
792 os.path.join(fio_root, 't', 'jobs', config['job']),
794 fio_pre_job=fio_pre_job,
795 fio_pre_success=fio_pre_success,
796 output_format=output_format)
797 elif issubclass(config['test_class'], FioExeTest):
798 exe_path = os.path.join(fio_root, config['exe'])
799 if config['parameters']:
800 parameters = [p.format(fio_path=fio_path) for p in config['parameters']]
803 if Path(exe_path).suffix == '.py' and platform.system() == "Windows":
805 parameters.insert(0, exe_path)
807 parameters = [exe_path]
808 exe_path = "python.exe"
809 test = config['test_class'](exe_path, parameters,
812 print("Test {0} FAILED: unable to process test config".format(config['test_id']))
816 if not args.skip_req:
818 for req in config['requirements']:
821 logging.debug("Test %d: Requirement '%s' met? %s" % (config['test_id'], reason, ok))
825 print("Test {0} SKIPPED ({1})".format(config['test_id'], reason))
826 skipped = skipped + 1
829 test.setup(artifact_root, config['test_id'])
836 result = "FAILED: {0}".format(test.failure_reason)
838 with open(test.stderr_file, "r") as stderr_file:
839 logging.debug("Test %d: stderr:\n%s" % (config['test_id'], stderr_file.read()))
840 with open(test.stdout_file, "r") as stdout_file:
841 logging.debug("Test %d: stdout:\n%s" % (config['test_id'], stdout_file.read()))
842 print("Test {0} {1}".format(config['test_id'], result))
844 print("{0} test(s) passed, {1} failed, {2} skipped".format(passed, failed, skipped))
849 if __name__ == '__main__':