2 # SPDX-License-Identifier: GPL-2.0-only
4 # Copyright (c) 2019 Western Digital Corporation or its affiliates.
9 # Automate running of fio tests
12 # python3 run-fio-tests.py [-r fio-root] [-f fio-path] [-a artifact-root]
13 # [--skip # # #...] [--run-only # # #...]
17 # # git clone git://git.kernel.dk/fio.git
20 # # python3 t/run-fio-tests.py
24 # - Python 3.5 (subprocess.run)
25 # - Linux (libaio ioengine, zbd tests, etc)
26 # - The artifact directory must be on a file system that accepts 512-byte IO
27 # (t0002, t0003, t0004).
28 # - The artifact directory needs to be on an SSD. Otherwise tests that carry
29 # out file-based IO will trigger a timeout (t0006).
31 # - SciPy (steadystate_tests.py)
32 # - libzbc (zbd tests)
33 # - root privileges (zbd test)
34 # - kernel 4.19 or later for zoned null block devices (zbd tests)
35 # - CUnit support (unittests)
40 # TODO run multiple tests simultaneously
41 # TODO Add sgunmap tests (requires SAS SSD)
53 import multiprocessing
54 from pathlib import Path
57 class FioTest(object):
58 """Base for all fio tests."""
60 def __init__(self, exe_path, parameters, success):
61 self.exe_path = exe_path
62 self.parameters = parameters
63 self.success = success
65 self.artifact_root = None
69 self.failure_reason = ''
71 def setup(self, artifact_root, testnum):
72 self.artifact_root = artifact_root
73 self.testnum = testnum
74 self.test_dir = os.path.join(artifact_root, "{:04d}".format(testnum))
75 if not os.path.exists(self.test_dir):
76 os.mkdir(self.test_dir)
78 self.command_file = os.path.join(
80 "{0}.command".format(os.path.basename(self.exe_path)))
81 self.stdout_file = os.path.join(
83 "{0}.stdout".format(os.path.basename(self.exe_path)))
84 self.stderr_file = os.path.join(
86 "{0}.stderr".format(os.path.basename(self.exe_path)))
87 self.exticode_file = os.path.join(
89 "{0}.exitcode".format(os.path.basename(self.exe_path)))
92 raise NotImplementedError()
94 def check_result(self):
95 raise NotImplementedError()
98 class FioExeTest(FioTest):
99 """Test consists of an executable binary or script"""
101 def __init__(self, exe_path, parameters, success):
102 """Construct a FioExeTest which is a FioTest consisting of an
103 executable binary or script.
105 exe_path: location of executable binary or script
106 parameters: list of parameters for executable
107 success: Definition of test success
110 FioTest.__init__(self, exe_path, parameters, success)
112 def setup(self, artifact_root, testnum):
113 super(FioExeTest, self).setup(artifact_root, testnum)
117 command = [self.exe_path] + self.parameters
119 command = [self.exe_path]
120 command_file = open(self.command_file, "w+")
121 command_file.write("%s\n" % command)
124 stdout_file = open(self.stdout_file, "w+")
125 stderr_file = open(self.stderr_file, "w+")
126 exticode_file = open(self.exticode_file, "w+")
129 # Avoid using subprocess.run() here because when a timeout occurs,
130 # fio will be stopped with SIGKILL. This does not give fio a
131 # chance to clean up and means that child processes may continue
132 # running and submitting IO.
133 proc = subprocess.Popen(command,
137 universal_newlines=True)
138 proc.communicate(timeout=self.success['timeout'])
139 exticode_file.write('{0}\n'.format(proc.returncode))
140 logging.debug("Test %d: return code: %d" % (self.testnum, proc.returncode))
141 self.output['proc'] = proc
142 except subprocess.TimeoutExpired:
146 self.output['failure'] = 'timeout'
152 self.output['failure'] = 'exception'
153 self.output['exc_info'] = sys.exc_info()
157 exticode_file.close()
159 def check_result(self):
160 if 'proc' not in self.output:
161 if self.output['failure'] == 'timeout':
162 self.failure_reason = "{0} timeout,".format(self.failure_reason)
164 assert self.output['failure'] == 'exception'
165 self.failure_reason = '{0} exception: {1}, {2}'.format(
166 self.failure_reason, self.output['exc_info'][0],
167 self.output['exc_info'][1])
172 if 'zero_return' in self.success:
173 if self.success['zero_return']:
174 if self.output['proc'].returncode != 0:
176 self.failure_reason = "{0} non-zero return code,".format(self.failure_reason)
178 if self.output['proc'].returncode == 0:
179 self.failure_reason = "{0} zero return code,".format(self.failure_reason)
182 stderr_size = os.path.getsize(self.stderr_file)
183 if 'stderr_empty' in self.success:
184 if self.success['stderr_empty']:
186 self.failure_reason = "{0} stderr not empty,".format(self.failure_reason)
190 self.failure_reason = "{0} stderr empty,".format(self.failure_reason)
194 class FioJobTest(FioExeTest):
195 """Test consists of a fio job"""
197 def __init__(self, fio_path, fio_job, success, fio_pre_job=None,
198 fio_pre_success=None, output_format="normal"):
199 """Construct a FioJobTest which is a FioExeTest consisting of a
200 single fio job file with an optional setup step.
202 fio_path: location of fio executable
203 fio_job: location of fio job file
204 success: Definition of test success
205 fio_pre_job: fio job for preconditioning
206 fio_pre_success: Definition of test success for fio precon job
207 output_format: normal (default), json, jsonplus, or terse
210 self.fio_job = fio_job
211 self.fio_pre_job = fio_pre_job
212 self.fio_pre_success = fio_pre_success if fio_pre_success else success
213 self.output_format = output_format
214 self.precon_failed = False
215 self.json_data = None
216 self.fio_output = "{0}.output".format(os.path.basename(self.fio_job))
218 "--output-format={0}".format(self.output_format),
219 "--output={0}".format(self.fio_output),
222 FioExeTest.__init__(self, fio_path, self.fio_args, success)
224 def setup(self, artifact_root, testnum):
225 super(FioJobTest, self).setup(artifact_root, testnum)
227 self.command_file = os.path.join(
229 "{0}.command".format(os.path.basename(self.fio_job)))
230 self.stdout_file = os.path.join(
232 "{0}.stdout".format(os.path.basename(self.fio_job)))
233 self.stderr_file = os.path.join(
235 "{0}.stderr".format(os.path.basename(self.fio_job)))
236 self.exticode_file = os.path.join(
238 "{0}.exitcode".format(os.path.basename(self.fio_job)))
240 def run_pre_job(self):
241 precon = FioJobTest(self.exe_path, self.fio_pre_job,
242 self.fio_pre_success,
243 output_format=self.output_format)
244 precon.setup(self.artifact_root, self.testnum)
246 precon.check_result()
247 self.precon_failed = not precon.passed
248 self.failure_reason = precon.failure_reason
254 if not self.precon_failed:
255 super(FioJobTest, self).run()
257 logging.debug("Test %d: precondition step failed" % self.testnum)
259 def check_result(self):
260 if self.precon_failed:
262 self.failure_reason = "{0} precondition step failed,".format(self.failure_reason)
265 super(FioJobTest, self).check_result()
270 if not 'json' in self.output_format:
274 with open(os.path.join(self.test_dir, self.fio_output), "r") as output_file:
275 file_data = output_file.read()
276 except EnvironmentError:
277 self.failure_reason = "{0} unable to open output file,".format(self.failure_reason)
282 # Sometimes fio informational messages are included at the top of the
283 # JSON output, especially under Windows. Try to decode output as JSON
284 # data, lopping off up to the first four lines
286 lines = file_data.splitlines()
288 file_data = '\n'.join(lines[i:])
290 self.json_data = json.loads(file_data)
291 except json.JSONDecodeError:
294 logging.debug("Test %d: skipped %d lines decoding JSON data" % (self.testnum, i))
297 self.failure_reason = "{0} unable to decode JSON data,".format(self.failure_reason)
301 class FioJobTest_t0005(FioJobTest):
302 """Test consists of fio test job t0005
303 Confirm that read['io_kbytes'] == write['io_kbytes'] == 102400"""
305 def check_result(self):
306 super(FioJobTest_t0005, self).check_result()
311 if self.json_data['jobs'][0]['read']['io_kbytes'] != 102400:
312 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
314 if self.json_data['jobs'][0]['write']['io_kbytes'] != 102400:
315 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
319 class FioJobTest_t0006(FioJobTest):
320 """Test consists of fio test job t0006
321 Confirm that read['io_kbytes'] ~ 2*write['io_kbytes']"""
323 def check_result(self):
324 super(FioJobTest_t0006, self).check_result()
329 ratio = self.json_data['jobs'][0]['read']['io_kbytes'] \
330 / self.json_data['jobs'][0]['write']['io_kbytes']
331 logging.debug("Test %d: ratio: %f" % (self.testnum, ratio))
332 if ratio < 1.99 or ratio > 2.01:
333 self.failure_reason = "{0} read/write ratio mismatch,".format(self.failure_reason)
337 class FioJobTest_t0007(FioJobTest):
338 """Test consists of fio test job t0007
339 Confirm that read['io_kbytes'] = 87040"""
341 def check_result(self):
342 super(FioJobTest_t0007, self).check_result()
347 if self.json_data['jobs'][0]['read']['io_kbytes'] != 87040:
348 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
352 class FioJobTest_t0008(FioJobTest):
353 """Test consists of fio test job t0008
354 Confirm that read['io_kbytes'] = 32768 and that
355 write['io_kbytes'] ~ 16568
357 I did runs with fio-ae2fafc8 and saw write['io_kbytes'] values of
358 16585, 16588. With two runs of fio-3.16 I obtained 16568"""
360 def check_result(self):
361 super(FioJobTest_t0008, self).check_result()
366 ratio = self.json_data['jobs'][0]['write']['io_kbytes'] / 16568
367 logging.debug("Test %d: ratio: %f" % (self.testnum, ratio))
369 if ratio < 0.99 or ratio > 1.01:
370 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
372 if self.json_data['jobs'][0]['read']['io_kbytes'] != 32768:
373 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
377 class FioJobTest_t0009(FioJobTest):
378 """Test consists of fio test job t0009
379 Confirm that runtime >= 60s"""
381 def check_result(self):
382 super(FioJobTest_t0009, self).check_result()
387 logging.debug('Test %d: elapsed: %d' % (self.testnum, self.json_data['jobs'][0]['elapsed']))
389 if self.json_data['jobs'][0]['elapsed'] < 60:
390 self.failure_reason = "{0} elapsed time mismatch,".format(self.failure_reason)
394 class FioJobTest_t0011(FioJobTest):
395 """Test consists of fio test job t0009
396 Confirm that job0 iops == 1000
397 and that job1_iops / job0_iops ~ 8
398 With two runs of fio-3.16 I observed a ratio of 8.3"""
400 def check_result(self):
401 super(FioJobTest_t0011, self).check_result()
406 iops1 = self.json_data['jobs'][0]['read']['iops']
407 iops2 = self.json_data['jobs'][1]['read']['iops']
408 ratio = iops2 / iops1
409 logging.debug("Test %d: iops1: %f" % (self.testnum, iops1))
410 logging.debug("Test %d: ratio: %f" % (self.testnum, ratio))
412 if iops1 < 998 or iops1 > 1002:
413 self.failure_reason = "{0} iops value mismatch,".format(self.failure_reason)
416 if ratio < 7 or ratio > 9:
417 self.failure_reason = "{0} iops ratio mismatch,".format(self.failure_reason)
421 class Requirements(object):
422 """Requirements consists of multiple run environment characteristics.
423 These are to determine if a particular test can be run"""
434 def __init__(self, fio_root):
435 Requirements._not_macos = platform.system() != "Darwin"
436 Requirements._linux = platform.system() == "Linux"
438 if Requirements._linux:
440 config_file = os.path.join(fio_root, "config-host.h")
441 with open(config_file, "r") as config:
442 contents = config.read()
444 print("Unable to open {0} to check requirements".format(config_file))
445 Requirements._zbd = True
447 Requirements._zbd = "CONFIG_LINUX_BLKZONED" in contents
448 Requirements._libaio = "CONFIG_LIBAIO" in contents
450 Requirements._root = (os.geteuid() == 0)
451 if Requirements._zbd and Requirements._root:
452 subprocess.run(["modprobe", "null_blk"],
453 stdout=subprocess.PIPE,
454 stderr=subprocess.PIPE)
455 if os.path.exists("/sys/module/null_blk/parameters/zoned"):
456 Requirements._zoned_nullb = True
458 if platform.system() == "Windows":
459 utest_exe = "unittest.exe"
461 utest_exe = "unittest"
462 unittest_path = os.path.join(fio_root, "unittests", utest_exe)
463 Requirements._unittests = os.path.exists(unittest_path)
465 Requirements._cpucount4 = multiprocessing.cpu_count() >= 4
467 req_list = [Requirements.linux,
471 Requirements.zoned_nullb,
472 Requirements.not_macos,
473 Requirements.unittests,
474 Requirements.cpucount4]
477 logging.debug("Requirements: Requirement '%s' met? %s" % (desc, value))
480 return Requirements._linux, "Linux required"
483 return Requirements._libaio, "libaio required"
486 return Requirements._zbd, "Zoned block device support required"
489 return Requirements._root, "root required"
492 return Requirements._zoned_nullb, "Zoned null block device support required"
495 return Requirements._not_macos, "platform other than macOS required"
498 return Requirements._unittests, "Unittests support required"
501 return Requirements._cpucount4, "4+ CPUs required"
506 'stderr_empty': True,
510 'zero_return': False,
511 'stderr_empty': False,
516 'stderr_empty': False,
522 'test_class': FioJobTest,
523 'job': 't0001-52c58027.fio',
524 'success': SUCCESS_DEFAULT,
531 'test_class': FioJobTest,
532 'job': 't0002-13af05ae-post.fio',
533 'success': SUCCESS_DEFAULT,
534 'pre_job': 't0002-13af05ae-pre.fio',
536 'requirements': [Requirements.linux, Requirements.libaio],
540 'test_class': FioJobTest,
541 'job': 't0003-0ae2c6e1-post.fio',
542 'success': SUCCESS_NONZERO,
543 'pre_job': 't0003-0ae2c6e1-pre.fio',
544 'pre_success': SUCCESS_DEFAULT,
545 'requirements': [Requirements.linux, Requirements.libaio],
549 'test_class': FioJobTest,
550 'job': 't0004-8a99fdf6.fio',
551 'success': SUCCESS_DEFAULT,
554 'requirements': [Requirements.linux, Requirements.libaio],
558 'test_class': FioJobTest_t0005,
559 'job': 't0005-f7078f7b.fio',
560 'success': SUCCESS_DEFAULT,
563 'output_format': 'json',
568 'test_class': FioJobTest_t0006,
569 'job': 't0006-82af2a7c.fio',
570 'success': SUCCESS_DEFAULT,
573 'output_format': 'json',
574 'requirements': [Requirements.linux, Requirements.libaio],
578 'test_class': FioJobTest_t0007,
579 'job': 't0007-37cf9e3c.fio',
580 'success': SUCCESS_DEFAULT,
583 'output_format': 'json',
588 'test_class': FioJobTest_t0008,
589 'job': 't0008-ae2fafc8.fio',
590 'success': SUCCESS_DEFAULT,
593 'output_format': 'json',
598 'test_class': FioJobTest_t0009,
599 'job': 't0009-f8b0bd10.fio',
600 'success': SUCCESS_DEFAULT,
603 'output_format': 'json',
604 'requirements': [Requirements.not_macos,
605 Requirements.cpucount4],
606 # mac os does not support CPU affinity
610 'test_class': FioJobTest,
611 'job': 't0010-b7aae4ba.fio',
612 'success': SUCCESS_DEFAULT,
619 'test_class': FioJobTest_t0011,
620 'job': 't0011-5d2788d5.fio',
621 'success': SUCCESS_DEFAULT,
624 'output_format': 'json',
629 'test_class': FioExeTest,
632 'success': SUCCESS_DEFAULT,
637 'test_class': FioExeTest,
640 'success': SUCCESS_DEFAULT,
645 'test_class': FioExeTest,
646 'exe': 't/lfsr-test',
647 'parameters': ['0xFFFFFF', '0', '0', 'verify'],
648 'success': SUCCESS_STDERR,
653 'test_class': FioExeTest,
654 'exe': 't/readonly.py',
655 'parameters': ['-f', '{fio_path}'],
656 'success': SUCCESS_DEFAULT,
661 'test_class': FioExeTest,
662 'exe': 't/steadystate_tests.py',
663 'parameters': ['{fio_path}'],
664 'success': SUCCESS_DEFAULT,
669 'test_class': FioExeTest,
672 'success': SUCCESS_STDERR,
677 'test_class': FioExeTest,
678 'exe': 't/strided.py',
679 'parameters': ['{fio_path}'],
680 'success': SUCCESS_DEFAULT,
685 'test_class': FioExeTest,
686 'exe': 't/zbd/run-tests-against-regular-nullb',
688 'success': SUCCESS_DEFAULT,
689 'requirements': [Requirements.linux, Requirements.zbd,
694 'test_class': FioExeTest,
695 'exe': 't/zbd/run-tests-against-zoned-nullb',
697 'success': SUCCESS_DEFAULT,
698 'requirements': [Requirements.linux, Requirements.zbd,
699 Requirements.root, Requirements.zoned_nullb],
703 'test_class': FioExeTest,
704 'exe': 'unittests/unittest',
706 'success': SUCCESS_DEFAULT,
707 'requirements': [Requirements.unittests],
713 parser = argparse.ArgumentParser()
714 parser.add_argument('-r', '--fio-root',
715 help='fio root path')
716 parser.add_argument('-f', '--fio',
717 help='path to fio executable (e.g., ./fio)')
718 parser.add_argument('-a', '--artifact-root',
719 help='artifact root directory')
720 parser.add_argument('-s', '--skip', nargs='+', type=int,
721 help='list of test(s) to skip')
722 parser.add_argument('-o', '--run-only', nargs='+', type=int,
723 help='list of test(s) to run, skipping all others')
724 parser.add_argument('-d', '--debug', action='store_true',
725 help='provide debug output')
726 parser.add_argument('-k', '--skip-req', action='store_true',
727 help='skip requirements checking')
728 args = parser.parse_args()
736 logging.basicConfig(level=logging.DEBUG)
738 logging.basicConfig(level=logging.INFO)
741 fio_root = args.fio_root
743 fio_root = str(Path(__file__).absolute().parent.parent)
744 print("fio root is %s" % fio_root)
749 if platform.system() == "Windows":
753 fio_path = os.path.join(fio_root, fio_exe)
754 print("fio path is %s" % fio_path)
755 if not shutil.which(fio_path):
756 print("Warning: fio executable not found")
758 artifact_root = args.artifact_root if args.artifact_root else \
759 "fio-test-{0}".format(time.strftime("%Y%m%d-%H%M%S"))
760 os.mkdir(artifact_root)
761 print("Artifact directory is %s" % artifact_root)
763 if not args.skip_req:
764 req = Requirements(fio_root)
770 for config in TEST_LIST:
771 if (args.skip and config['test_id'] in args.skip) or \
772 (args.run_only and config['test_id'] not in args.run_only):
773 skipped = skipped + 1
774 print("Test {0} SKIPPED (User request)".format(config['test_id']))
777 if issubclass(config['test_class'], FioJobTest):
778 if config['pre_job']:
779 fio_pre_job = os.path.join(fio_root, 't', 'jobs',
783 if config['pre_success']:
784 fio_pre_success = config['pre_success']
786 fio_pre_success = None
787 if 'output_format' in config:
788 output_format = config['output_format']
790 output_format = 'normal'
791 test = config['test_class'](
793 os.path.join(fio_root, 't', 'jobs', config['job']),
795 fio_pre_job=fio_pre_job,
796 fio_pre_success=fio_pre_success,
797 output_format=output_format)
798 elif issubclass(config['test_class'], FioExeTest):
799 exe_path = os.path.join(fio_root, config['exe'])
800 if config['parameters']:
801 parameters = [p.format(fio_path=fio_path) for p in config['parameters']]
804 if Path(exe_path).suffix == '.py' and platform.system() == "Windows":
806 parameters.insert(0, exe_path)
808 parameters = [exe_path]
809 exe_path = "python.exe"
810 test = config['test_class'](exe_path, parameters,
813 print("Test {0} FAILED: unable to process test config".format(config['test_id']))
817 if not args.skip_req:
819 for req in config['requirements']:
822 logging.debug("Test %d: Requirement '%s' met? %s" % (config['test_id'], reason, ok))
826 print("Test {0} SKIPPED ({1})".format(config['test_id'], reason))
827 skipped = skipped + 1
830 test.setup(artifact_root, config['test_id'])
837 result = "FAILED: {0}".format(test.failure_reason)
839 with open(test.stderr_file, "r") as stderr_file:
840 logging.debug("Test %d: stderr:\n%s" % (config['test_id'], stderr_file.read()))
841 with open(test.stdout_file, "r") as stdout_file:
842 logging.debug("Test %d: stdout:\n%s" % (config['test_id'], stdout_file.read()))
843 print("Test {0} {1}".format(config['test_id'], result))
845 print("{0} test(s) passed, {1} failed, {2} skipped".format(passed, failed, skipped))
850 if __name__ == '__main__':