2 # SPDX-License-Identifier: GPL-2.0-only
4 # Copyright (c) 2019 Western Digital Corporation or its affiliates.
9 # Automate running of fio tests
12 # python3 run-fio-tests.py [-r fio-root] [-f fio-path] [-a artifact-root]
13 # [--skip # # #...] [--run-only # # #...]
17 # # git clone git://git.kernel.dk/fio.git
20 # # python3 t/run-fio-tests.py
24 # - Python 3.5 (subprocess.run)
25 # - Linux (libaio ioengine, zbd tests, etc)
26 # - The artifact directory must be on a file system that accepts 512-byte IO
27 # (t0002, t0003, t0004).
28 # - The artifact directory needs to be on an SSD. Otherwise tests that carry
29 # out file-based IO will trigger a timeout (t0006).
31 # - SciPy (steadystate_tests.py)
32 # - libzbc (zbd tests)
33 # - root privileges (zbd test)
34 # - kernel 4.19 or later for zoned null block devices (zbd tests)
35 # - CUnit support (unittests)
40 # TODO run multiple tests simultaneously
41 # TODO Add sgunmap tests (requires SAS SSD)
53 import multiprocessing
54 from pathlib import Path
57 class FioTest(object):
58 """Base for all fio tests."""
60 def __init__(self, exe_path, parameters, success):
61 self.exe_path = exe_path
62 self.parameters = parameters
63 self.success = success
65 self.artifact_root = None
69 self.failure_reason = ''
70 self.command_file = None
71 self.stdout_file = None
72 self.stderr_file = None
73 self.exitcode_file = None
75 def setup(self, artifact_root, testnum):
76 """Setup instance variables for test."""
78 self.artifact_root = artifact_root
79 self.testnum = testnum
80 self.test_dir = os.path.join(artifact_root, "{:04d}".format(testnum))
81 if not os.path.exists(self.test_dir):
82 os.mkdir(self.test_dir)
84 self.command_file = os.path.join(
86 "{0}.command".format(os.path.basename(self.exe_path)))
87 self.stdout_file = os.path.join(
89 "{0}.stdout".format(os.path.basename(self.exe_path)))
90 self.stderr_file = os.path.join(
92 "{0}.stderr".format(os.path.basename(self.exe_path)))
93 self.exitcode_file = os.path.join(
95 "{0}.exitcode".format(os.path.basename(self.exe_path)))
100 raise NotImplementedError()
102 def check_result(self):
103 """Check test results."""
105 raise NotImplementedError()
108 class FioExeTest(FioTest):
109 """Test consists of an executable binary or script"""
111 def __init__(self, exe_path, parameters, success):
112 """Construct a FioExeTest which is a FioTest consisting of an
113 executable binary or script.
115 exe_path: location of executable binary or script
116 parameters: list of parameters for executable
117 success: Definition of test success
120 FioTest.__init__(self, exe_path, parameters, success)
123 """Execute the binary or script described by this instance."""
126 command = [self.exe_path] + self.parameters
128 command = [self.exe_path]
129 command_file = open(self.command_file, "w+")
130 command_file.write("%s\n" % command)
133 stdout_file = open(self.stdout_file, "w+")
134 stderr_file = open(self.stderr_file, "w+")
135 exitcode_file = open(self.exitcode_file, "w+")
138 # Avoid using subprocess.run() here because when a timeout occurs,
139 # fio will be stopped with SIGKILL. This does not give fio a
140 # chance to clean up and means that child processes may continue
141 # running and submitting IO.
142 proc = subprocess.Popen(command,
146 universal_newlines=True)
147 proc.communicate(timeout=self.success['timeout'])
148 exitcode_file.write('{0}\n'.format(proc.returncode))
149 logging.debug("Test %d: return code: %d", self.testnum, proc.returncode)
150 self.output['proc'] = proc
151 except subprocess.TimeoutExpired:
155 self.output['failure'] = 'timeout'
161 self.output['failure'] = 'exception'
162 self.output['exc_info'] = sys.exc_info()
166 exitcode_file.close()
168 def check_result(self):
169 """Check results of test run."""
171 if 'proc' not in self.output:
172 if self.output['failure'] == 'timeout':
173 self.failure_reason = "{0} timeout,".format(self.failure_reason)
175 assert self.output['failure'] == 'exception'
176 self.failure_reason = '{0} exception: {1}, {2}'.format(
177 self.failure_reason, self.output['exc_info'][0],
178 self.output['exc_info'][1])
183 if 'zero_return' in self.success:
184 if self.success['zero_return']:
185 if self.output['proc'].returncode != 0:
187 self.failure_reason = "{0} non-zero return code,".format(self.failure_reason)
189 if self.output['proc'].returncode == 0:
190 self.failure_reason = "{0} zero return code,".format(self.failure_reason)
193 stderr_size = os.path.getsize(self.stderr_file)
194 if 'stderr_empty' in self.success:
195 if self.success['stderr_empty']:
197 self.failure_reason = "{0} stderr not empty,".format(self.failure_reason)
201 self.failure_reason = "{0} stderr empty,".format(self.failure_reason)
205 class FioJobTest(FioExeTest):
206 """Test consists of a fio job"""
208 def __init__(self, fio_path, fio_job, success, fio_pre_job=None,
209 fio_pre_success=None, output_format="normal"):
210 """Construct a FioJobTest which is a FioExeTest consisting of a
211 single fio job file with an optional setup step.
213 fio_path: location of fio executable
214 fio_job: location of fio job file
215 success: Definition of test success
216 fio_pre_job: fio job for preconditioning
217 fio_pre_success: Definition of test success for fio precon job
218 output_format: normal (default), json, jsonplus, or terse
221 self.fio_job = fio_job
222 self.fio_pre_job = fio_pre_job
223 self.fio_pre_success = fio_pre_success if fio_pre_success else success
224 self.output_format = output_format
225 self.precon_failed = False
226 self.json_data = None
227 self.fio_output = "{0}.output".format(os.path.basename(self.fio_job))
230 "--output-format={0}".format(self.output_format),
231 "--output={0}".format(self.fio_output),
234 FioExeTest.__init__(self, fio_path, self.fio_args, success)
236 def setup(self, artifact_root, testnum):
237 """Setup instance variables for fio job test."""
239 super(FioJobTest, self).setup(artifact_root, testnum)
241 self.command_file = os.path.join(
243 "{0}.command".format(os.path.basename(self.fio_job)))
244 self.stdout_file = os.path.join(
246 "{0}.stdout".format(os.path.basename(self.fio_job)))
247 self.stderr_file = os.path.join(
249 "{0}.stderr".format(os.path.basename(self.fio_job)))
250 self.exitcode_file = os.path.join(
252 "{0}.exitcode".format(os.path.basename(self.fio_job)))
254 def run_pre_job(self):
255 """Run fio job precondition step."""
257 precon = FioJobTest(self.exe_path, self.fio_pre_job,
258 self.fio_pre_success,
259 output_format=self.output_format)
260 precon.setup(self.artifact_root, self.testnum)
262 precon.check_result()
263 self.precon_failed = not precon.passed
264 self.failure_reason = precon.failure_reason
267 """Run fio job test."""
272 if not self.precon_failed:
273 super(FioJobTest, self).run()
275 logging.debug("Test %d: precondition step failed", self.testnum)
278 def get_file(cls, filename):
279 """Safely read a file."""
284 with open(filename, "r") as output_file:
285 file_data = output_file.read()
289 return file_data, success
291 def check_result(self):
292 """Check fio job results."""
294 if self.precon_failed:
296 self.failure_reason = "{0} precondition step failed,".format(self.failure_reason)
299 super(FioJobTest, self).check_result()
304 if 'json' not in self.output_format:
307 file_data, success = self.get_file(os.path.join(self.test_dir, self.fio_output))
309 self.failure_reason = "{0} unable to open output file,".format(self.failure_reason)
314 # Sometimes fio informational messages are included at the top of the
315 # JSON output, especially under Windows. Try to decode output as JSON
316 # data, lopping off up to the first four lines
318 lines = file_data.splitlines()
320 file_data = '\n'.join(lines[i:])
322 self.json_data = json.loads(file_data)
323 except json.JSONDecodeError:
326 logging.debug("Test %d: skipped %d lines decoding JSON data", self.testnum, i)
329 self.failure_reason = "{0} unable to decode JSON data,".format(self.failure_reason)
333 class FioJobTest_t0005(FioJobTest):
334 """Test consists of fio test job t0005
335 Confirm that read['io_kbytes'] == write['io_kbytes'] == 102400"""
337 def check_result(self):
338 super(FioJobTest_t0005, self).check_result()
343 if self.json_data['jobs'][0]['read']['io_kbytes'] != 102400:
344 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
346 if self.json_data['jobs'][0]['write']['io_kbytes'] != 102400:
347 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
351 class FioJobTest_t0006(FioJobTest):
352 """Test consists of fio test job t0006
353 Confirm that read['io_kbytes'] ~ 2*write['io_kbytes']"""
355 def check_result(self):
356 super(FioJobTest_t0006, self).check_result()
361 ratio = self.json_data['jobs'][0]['read']['io_kbytes'] \
362 / self.json_data['jobs'][0]['write']['io_kbytes']
363 logging.debug("Test %d: ratio: %f", self.testnum, ratio)
364 if ratio < 1.99 or ratio > 2.01:
365 self.failure_reason = "{0} read/write ratio mismatch,".format(self.failure_reason)
369 class FioJobTest_t0007(FioJobTest):
370 """Test consists of fio test job t0007
371 Confirm that read['io_kbytes'] = 87040"""
373 def check_result(self):
374 super(FioJobTest_t0007, self).check_result()
379 if self.json_data['jobs'][0]['read']['io_kbytes'] != 87040:
380 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
384 class FioJobTest_t0008(FioJobTest):
385 """Test consists of fio test job t0008
386 Confirm that read['io_kbytes'] = 32768 and that
387 write['io_kbytes'] ~ 16568
389 I did runs with fio-ae2fafc8 and saw write['io_kbytes'] values of
390 16585, 16588. With two runs of fio-3.16 I obtained 16568"""
392 def check_result(self):
393 super(FioJobTest_t0008, self).check_result()
398 ratio = self.json_data['jobs'][0]['write']['io_kbytes'] / 16568
399 logging.debug("Test %d: ratio: %f", self.testnum, ratio)
401 if ratio < 0.99 or ratio > 1.01:
402 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
404 if self.json_data['jobs'][0]['read']['io_kbytes'] != 32768:
405 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
409 class FioJobTest_t0009(FioJobTest):
410 """Test consists of fio test job t0009
411 Confirm that runtime >= 60s"""
413 def check_result(self):
414 super(FioJobTest_t0009, self).check_result()
419 logging.debug('Test %d: elapsed: %d', self.testnum, self.json_data['jobs'][0]['elapsed'])
421 if self.json_data['jobs'][0]['elapsed'] < 60:
422 self.failure_reason = "{0} elapsed time mismatch,".format(self.failure_reason)
426 class FioJobTest_t0011(FioJobTest):
427 """Test consists of fio test job t0009
428 Confirm that job0 iops == 1000
429 and that job1_iops / job0_iops ~ 8
430 With two runs of fio-3.16 I observed a ratio of 8.3"""
432 def check_result(self):
433 super(FioJobTest_t0011, self).check_result()
438 iops1 = self.json_data['jobs'][0]['read']['iops']
439 iops2 = self.json_data['jobs'][1]['read']['iops']
440 ratio = iops2 / iops1
441 logging.debug("Test %d: iops1: %f", self.testnum, iops1)
442 logging.debug("Test %d: ratio: %f", self.testnum, ratio)
444 if iops1 < 998 or iops1 > 1002:
445 self.failure_reason = "{0} iops value mismatch,".format(self.failure_reason)
448 if ratio < 7 or ratio > 9:
449 self.failure_reason = "{0} iops ratio mismatch,".format(self.failure_reason)
453 class Requirements(object):
454 """Requirements consists of multiple run environment characteristics.
455 These are to determine if a particular test can be run"""
467 def __init__(self, fio_root):
468 Requirements._not_macos = platform.system() != "Darwin"
469 Requirements._not_windows = platform.system() != "Windows"
470 Requirements._linux = platform.system() == "Linux"
472 if Requirements._linux:
473 config_file = os.path.join(fio_root, "config-host.h")
474 contents, success = FioJobTest.get_file(config_file)
476 print("Unable to open {0} to check requirements".format(config_file))
477 Requirements._zbd = True
479 Requirements._zbd = "CONFIG_HAS_BLKZONED" in contents
480 Requirements._libaio = "CONFIG_LIBAIO" in contents
482 Requirements._root = (os.geteuid() == 0)
483 if Requirements._zbd and Requirements._root:
484 subprocess.run(["modprobe", "null_blk"],
485 stdout=subprocess.PIPE,
486 stderr=subprocess.PIPE)
487 if os.path.exists("/sys/module/null_blk/parameters/zoned"):
488 Requirements._zoned_nullb = True
490 if platform.system() == "Windows":
491 utest_exe = "unittest.exe"
493 utest_exe = "unittest"
494 unittest_path = os.path.join(fio_root, "unittests", utest_exe)
495 Requirements._unittests = os.path.exists(unittest_path)
497 Requirements._cpucount4 = multiprocessing.cpu_count() >= 4
499 req_list = [Requirements.linux,
503 Requirements.zoned_nullb,
504 Requirements.not_macos,
505 Requirements.not_windows,
506 Requirements.unittests,
507 Requirements.cpucount4]
510 logging.debug("Requirements: Requirement '%s' met? %s", desc, value)
514 """Are we running on Linux?"""
515 return Requirements._linux, "Linux required"
519 """Is libaio available?"""
520 return Requirements._libaio, "libaio required"
524 """Is ZBD support available?"""
525 return Requirements._zbd, "Zoned block device support required"
529 """Are we running as root?"""
530 return Requirements._root, "root required"
533 def zoned_nullb(cls):
534 """Are zoned null block devices available?"""
535 return Requirements._zoned_nullb, "Zoned null block device support required"
539 """Are we running on a platform other than macOS?"""
540 return Requirements._not_macos, "platform other than macOS required"
543 def not_windows(cls):
544 """Are we running on a platform other than Windws?"""
545 return Requirements._not_windows, "platform other than Windows required"
549 """Were unittests built?"""
550 return Requirements._unittests, "Unittests support required"
554 """Do we have at least 4 CPUs?"""
555 return Requirements._cpucount4, "4+ CPUs required"
560 'stderr_empty': True,
564 'zero_return': False,
565 'stderr_empty': False,
570 'stderr_empty': False,
576 'test_class': FioJobTest,
577 'job': 't0001-52c58027.fio',
578 'success': SUCCESS_DEFAULT,
585 'test_class': FioJobTest,
586 'job': 't0002-13af05ae-post.fio',
587 'success': SUCCESS_DEFAULT,
588 'pre_job': 't0002-13af05ae-pre.fio',
590 'requirements': [Requirements.linux, Requirements.libaio],
594 'test_class': FioJobTest,
595 'job': 't0003-0ae2c6e1-post.fio',
596 'success': SUCCESS_NONZERO,
597 'pre_job': 't0003-0ae2c6e1-pre.fio',
598 'pre_success': SUCCESS_DEFAULT,
599 'requirements': [Requirements.linux, Requirements.libaio],
603 'test_class': FioJobTest,
604 'job': 't0004-8a99fdf6.fio',
605 'success': SUCCESS_DEFAULT,
608 'requirements': [Requirements.linux, Requirements.libaio],
612 'test_class': FioJobTest_t0005,
613 'job': 't0005-f7078f7b.fio',
614 'success': SUCCESS_DEFAULT,
617 'output_format': 'json',
618 'requirements': [Requirements.not_windows],
622 'test_class': FioJobTest_t0006,
623 'job': 't0006-82af2a7c.fio',
624 'success': SUCCESS_DEFAULT,
627 'output_format': 'json',
628 'requirements': [Requirements.linux, Requirements.libaio],
632 'test_class': FioJobTest_t0007,
633 'job': 't0007-37cf9e3c.fio',
634 'success': SUCCESS_DEFAULT,
637 'output_format': 'json',
642 'test_class': FioJobTest_t0008,
643 'job': 't0008-ae2fafc8.fio',
644 'success': SUCCESS_DEFAULT,
647 'output_format': 'json',
652 'test_class': FioJobTest_t0009,
653 'job': 't0009-f8b0bd10.fio',
654 'success': SUCCESS_DEFAULT,
657 'output_format': 'json',
658 'requirements': [Requirements.not_macos,
659 Requirements.cpucount4],
660 # mac os does not support CPU affinity
664 'test_class': FioJobTest,
665 'job': 't0010-b7aae4ba.fio',
666 'success': SUCCESS_DEFAULT,
673 'test_class': FioJobTest_t0011,
674 'job': 't0011-5d2788d5.fio',
675 'success': SUCCESS_DEFAULT,
678 'output_format': 'json',
683 'test_class': FioExeTest,
686 'success': SUCCESS_DEFAULT,
691 'test_class': FioExeTest,
694 'success': SUCCESS_DEFAULT,
699 'test_class': FioExeTest,
700 'exe': 't/lfsr-test',
701 'parameters': ['0xFFFFFF', '0', '0', 'verify'],
702 'success': SUCCESS_STDERR,
707 'test_class': FioExeTest,
708 'exe': 't/readonly.py',
709 'parameters': ['-f', '{fio_path}'],
710 'success': SUCCESS_DEFAULT,
715 'test_class': FioExeTest,
716 'exe': 't/steadystate_tests.py',
717 'parameters': ['{fio_path}'],
718 'success': SUCCESS_DEFAULT,
723 'test_class': FioExeTest,
726 'success': SUCCESS_STDERR,
731 'test_class': FioExeTest,
732 'exe': 't/strided.py',
733 'parameters': ['{fio_path}'],
734 'success': SUCCESS_DEFAULT,
739 'test_class': FioExeTest,
740 'exe': 't/zbd/run-tests-against-regular-nullb',
742 'success': SUCCESS_DEFAULT,
743 'requirements': [Requirements.linux, Requirements.zbd,
748 'test_class': FioExeTest,
749 'exe': 't/zbd/run-tests-against-zoned-nullb',
751 'success': SUCCESS_DEFAULT,
752 'requirements': [Requirements.linux, Requirements.zbd,
753 Requirements.root, Requirements.zoned_nullb],
757 'test_class': FioExeTest,
758 'exe': 'unittests/unittest',
760 'success': SUCCESS_DEFAULT,
761 'requirements': [Requirements.unittests],
765 'test_class': FioExeTest,
766 'exe': 't/latency_percentiles.py',
767 'parameters': ['-f', '{fio_path}'],
768 'success': SUCCESS_DEFAULT,
773 'test_class': FioExeTest,
774 'exe': 't/jsonplus2csv_test.py',
775 'parameters': ['-f', '{fio_path}'],
776 'success': SUCCESS_DEFAULT,
783 """Parse command-line arguments."""
785 parser = argparse.ArgumentParser()
786 parser.add_argument('-r', '--fio-root',
787 help='fio root path')
788 parser.add_argument('-f', '--fio',
789 help='path to fio executable (e.g., ./fio)')
790 parser.add_argument('-a', '--artifact-root',
791 help='artifact root directory')
792 parser.add_argument('-s', '--skip', nargs='+', type=int,
793 help='list of test(s) to skip')
794 parser.add_argument('-o', '--run-only', nargs='+', type=int,
795 help='list of test(s) to run, skipping all others')
796 parser.add_argument('-d', '--debug', action='store_true',
797 help='provide debug output')
798 parser.add_argument('-k', '--skip-req', action='store_true',
799 help='skip requirements checking')
800 args = parser.parse_args()
810 logging.basicConfig(level=logging.DEBUG)
812 logging.basicConfig(level=logging.INFO)
815 fio_root = args.fio_root
817 fio_root = str(Path(__file__).absolute().parent.parent)
818 print("fio root is %s" % fio_root)
823 if platform.system() == "Windows":
827 fio_path = os.path.join(fio_root, fio_exe)
828 print("fio path is %s" % fio_path)
829 if not shutil.which(fio_path):
830 print("Warning: fio executable not found")
832 artifact_root = args.artifact_root if args.artifact_root else \
833 "fio-test-{0}".format(time.strftime("%Y%m%d-%H%M%S"))
834 os.mkdir(artifact_root)
835 print("Artifact directory is %s" % artifact_root)
837 if not args.skip_req:
838 req = Requirements(fio_root)
844 for config in TEST_LIST:
845 if (args.skip and config['test_id'] in args.skip) or \
846 (args.run_only and config['test_id'] not in args.run_only):
847 skipped = skipped + 1
848 print("Test {0} SKIPPED (User request)".format(config['test_id']))
851 if issubclass(config['test_class'], FioJobTest):
852 if config['pre_job']:
853 fio_pre_job = os.path.join(fio_root, 't', 'jobs',
857 if config['pre_success']:
858 fio_pre_success = config['pre_success']
860 fio_pre_success = None
861 if 'output_format' in config:
862 output_format = config['output_format']
864 output_format = 'normal'
865 test = config['test_class'](
867 os.path.join(fio_root, 't', 'jobs', config['job']),
869 fio_pre_job=fio_pre_job,
870 fio_pre_success=fio_pre_success,
871 output_format=output_format)
872 elif issubclass(config['test_class'], FioExeTest):
873 exe_path = os.path.join(fio_root, config['exe'])
874 if config['parameters']:
875 parameters = [p.format(fio_path=fio_path) for p in config['parameters']]
878 if Path(exe_path).suffix == '.py' and platform.system() == "Windows":
880 parameters.insert(0, exe_path)
882 parameters = [exe_path]
883 exe_path = "python.exe"
884 test = config['test_class'](exe_path, parameters,
887 print("Test {0} FAILED: unable to process test config".format(config['test_id']))
891 if not args.skip_req:
893 for req in config['requirements']:
894 reqs_met, reason = req()
895 logging.debug("Test %d: Requirement '%s' met? %s", config['test_id'], reason,
900 print("Test {0} SKIPPED ({1})".format(config['test_id'], reason))
901 skipped = skipped + 1
904 test.setup(artifact_root, config['test_id'])
911 result = "FAILED: {0}".format(test.failure_reason)
913 contents, _ = FioJobTest.get_file(test.stderr_file)
914 logging.debug("Test %d: stderr:\n%s", config['test_id'], contents)
915 contents, _ = FioJobTest.get_file(test.stdout_file)
916 logging.debug("Test %d: stdout:\n%s", config['test_id'], contents)
917 print("Test {0} {1}".format(config['test_id'], result))
919 print("{0} test(s) passed, {1} failed, {2} skipped".format(passed, failed, skipped))
924 if __name__ == '__main__':