2 # SPDX-License-Identifier: GPL-2.0-only
4 # Copyright (c) 2019 Western Digital Corporation or its affiliates.
9 # Automate running of fio tests
12 # python3 run-fio-tests.py [-r fio-root] [-f fio-path] [-a artifact-root]
13 # [--skip # # #...] [--run-only # # #...]
17 # # git clone git://git.kernel.dk/fio.git
20 # # python3 t/run-fio-tests.py
24 # - Python 3.5 (subprocess.run)
25 # - Linux (libaio ioengine, zbd tests, etc)
26 # - The artifact directory must be on a file system that accepts 512-byte IO
27 # (t0002, t0003, t0004).
28 # - The artifact directory needs to be on an SSD. Otherwise tests that carry
29 # out file-based IO will trigger a timeout (t0006).
31 # - SciPy (steadystate_tests.py)
32 # - libzbc (zbd tests)
33 # - root privileges (zbd test)
34 # - kernel 4.19 or later for zoned null block devices (zbd tests)
35 # - CUnit support (unittests)
40 # TODO run multiple tests simultaneously
41 # TODO Add sgunmap tests (requires SAS SSD)
53 import multiprocessing
54 from pathlib import Path
57 class FioTest(object):
58 """Base for all fio tests."""
60 def __init__(self, exe_path, parameters, success):
61 self.exe_path = exe_path
62 self.parameters = parameters
63 self.success = success
65 self.artifact_root = None
69 self.failure_reason = ''
71 def setup(self, artifact_root, testnum):
72 self.artifact_root = artifact_root
73 self.testnum = testnum
74 self.test_dir = os.path.join(artifact_root, "{:04d}".format(testnum))
75 if not os.path.exists(self.test_dir):
76 os.mkdir(self.test_dir)
78 self.command_file = os.path.join(
80 "{0}.command".format(os.path.basename(self.exe_path)))
81 self.stdout_file = os.path.join(
83 "{0}.stdout".format(os.path.basename(self.exe_path)))
84 self.stderr_file = os.path.join(
86 "{0}.stderr".format(os.path.basename(self.exe_path)))
87 self.exticode_file = os.path.join(
89 "{0}.exitcode".format(os.path.basename(self.exe_path)))
92 raise NotImplementedError()
94 def check_result(self):
95 raise NotImplementedError()
98 class FioExeTest(FioTest):
99 """Test consists of an executable binary or script"""
101 def __init__(self, exe_path, parameters, success):
102 """Construct a FioExeTest which is a FioTest consisting of an
103 executable binary or script.
105 exe_path: location of executable binary or script
106 parameters: list of parameters for executable
107 success: Definition of test success
110 FioTest.__init__(self, exe_path, parameters, success)
112 def setup(self, artifact_root, testnum):
113 super(FioExeTest, self).setup(artifact_root, testnum)
117 command = [self.exe_path] + self.parameters
119 command = [self.exe_path]
120 command_file = open(self.command_file, "w+")
121 command_file.write("%s\n" % command)
124 stdout_file = open(self.stdout_file, "w+")
125 stderr_file = open(self.stderr_file, "w+")
126 exticode_file = open(self.exticode_file, "w+")
129 # Avoid using subprocess.run() here because when a timeout occurs,
130 # fio will be stopped with SIGKILL. This does not give fio a
131 # chance to clean up and means that child processes may continue
132 # running and submitting IO.
133 proc = subprocess.Popen(command,
137 universal_newlines=True)
138 proc.communicate(timeout=self.success['timeout'])
139 exticode_file.write('{0}\n'.format(proc.returncode))
140 logging.debug("Test %d: return code: %d" % (self.testnum, proc.returncode))
141 self.output['proc'] = proc
142 except subprocess.TimeoutExpired:
146 self.output['failure'] = 'timeout'
152 self.output['failure'] = 'exception'
153 self.output['exc_info'] = sys.exc_info()
157 exticode_file.close()
159 def check_result(self):
160 if 'proc' not in self.output:
161 if self.output['failure'] == 'timeout':
162 self.failure_reason = "{0} timeout,".format(self.failure_reason)
164 assert self.output['failure'] == 'exception'
165 self.failure_reason = '{0} exception: {1}, {2}'.format(
166 self.failure_reason, self.output['exc_info'][0],
167 self.output['exc_info'][1])
172 if 'zero_return' in self.success:
173 if self.success['zero_return']:
174 if self.output['proc'].returncode != 0:
176 self.failure_reason = "{0} non-zero return code,".format(self.failure_reason)
178 if self.output['proc'].returncode == 0:
179 self.failure_reason = "{0} zero return code,".format(self.failure_reason)
182 stderr_size = os.path.getsize(self.stderr_file)
183 if 'stderr_empty' in self.success:
184 if self.success['stderr_empty']:
186 self.failure_reason = "{0} stderr not empty,".format(self.failure_reason)
190 self.failure_reason = "{0} stderr empty,".format(self.failure_reason)
194 class FioJobTest(FioExeTest):
195 """Test consists of a fio job"""
197 def __init__(self, fio_path, fio_job, success, fio_pre_job=None,
198 fio_pre_success=None, output_format="normal"):
199 """Construct a FioJobTest which is a FioExeTest consisting of a
200 single fio job file with an optional setup step.
202 fio_path: location of fio executable
203 fio_job: location of fio job file
204 success: Definition of test success
205 fio_pre_job: fio job for preconditioning
206 fio_pre_success: Definition of test success for fio precon job
207 output_format: normal (default), json, jsonplus, or terse
210 self.fio_job = fio_job
211 self.fio_pre_job = fio_pre_job
212 self.fio_pre_success = fio_pre_success if fio_pre_success else success
213 self.output_format = output_format
214 self.precon_failed = False
215 self.json_data = None
216 self.fio_output = "{0}.output".format(os.path.basename(self.fio_job))
218 "--output-format={0}".format(self.output_format),
219 "--output={0}".format(self.fio_output),
222 FioExeTest.__init__(self, fio_path, self.fio_args, success)
224 def setup(self, artifact_root, testnum):
225 super(FioJobTest, self).setup(artifact_root, testnum)
227 self.command_file = os.path.join(
229 "{0}.command".format(os.path.basename(self.fio_job)))
230 self.stdout_file = os.path.join(
232 "{0}.stdout".format(os.path.basename(self.fio_job)))
233 self.stderr_file = os.path.join(
235 "{0}.stderr".format(os.path.basename(self.fio_job)))
236 self.exticode_file = os.path.join(
238 "{0}.exitcode".format(os.path.basename(self.fio_job)))
240 def run_pre_job(self):
241 precon = FioJobTest(self.exe_path, self.fio_pre_job,
242 self.fio_pre_success,
243 output_format=self.output_format)
244 precon.setup(self.artifact_root, self.testnum)
246 precon.check_result()
247 self.precon_failed = not precon.passed
248 self.failure_reason = precon.failure_reason
254 if not self.precon_failed:
255 super(FioJobTest, self).run()
257 logging.debug("Test %d: precondition step failed" % self.testnum)
259 def check_result(self):
260 if self.precon_failed:
262 self.failure_reason = "{0} precondition step failed,".format(self.failure_reason)
265 super(FioJobTest, self).check_result()
270 if not 'json' in self.output_format:
274 with open(os.path.join(self.test_dir, self.fio_output), "r") as output_file:
275 file_data = output_file.read()
276 except EnvironmentError:
277 self.failure_reason = "{0} unable to open output file,".format(self.failure_reason)
282 # Sometimes fio informational messages are included at the top of the
283 # JSON output, especially under Windows. Try to decode output as JSON
284 # data, lopping off up to the first four lines
286 lines = file_data.splitlines()
288 file_data = '\n'.join(lines[i:])
290 self.json_data = json.loads(file_data)
291 except json.JSONDecodeError:
294 logging.debug("Test %d: skipped %d lines decoding JSON data" % (self.testnum, i))
297 self.failure_reason = "{0} unable to decode JSON data,".format(self.failure_reason)
301 class FioJobTest_t0005(FioJobTest):
302 """Test consists of fio test job t0005
303 Confirm that read['io_kbytes'] == write['io_kbytes'] == 102400"""
305 def check_result(self):
306 super(FioJobTest_t0005, self).check_result()
311 if self.json_data['jobs'][0]['read']['io_kbytes'] != 102400:
312 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
314 if self.json_data['jobs'][0]['write']['io_kbytes'] != 102400:
315 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
319 class FioJobTest_t0006(FioJobTest):
320 """Test consists of fio test job t0006
321 Confirm that read['io_kbytes'] ~ 2*write['io_kbytes']"""
323 def check_result(self):
324 super(FioJobTest_t0006, self).check_result()
329 ratio = self.json_data['jobs'][0]['read']['io_kbytes'] \
330 / self.json_data['jobs'][0]['write']['io_kbytes']
331 logging.debug("Test %d: ratio: %f" % (self.testnum, ratio))
332 if ratio < 1.99 or ratio > 2.01:
333 self.failure_reason = "{0} read/write ratio mismatch,".format(self.failure_reason)
337 class FioJobTest_t0007(FioJobTest):
338 """Test consists of fio test job t0007
339 Confirm that read['io_kbytes'] = 87040"""
341 def check_result(self):
342 super(FioJobTest_t0007, self).check_result()
347 if self.json_data['jobs'][0]['read']['io_kbytes'] != 87040:
348 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
352 class FioJobTest_t0008(FioJobTest):
353 """Test consists of fio test job t0008
354 Confirm that read['io_kbytes'] = 32768 and that
355 write['io_kbytes'] ~ 16568
357 I did runs with fio-ae2fafc8 and saw write['io_kbytes'] values of
358 16585, 16588. With two runs of fio-3.16 I obtained 16568"""
360 def check_result(self):
361 super(FioJobTest_t0008, self).check_result()
366 ratio = self.json_data['jobs'][0]['write']['io_kbytes'] / 16568
367 logging.debug("Test %d: ratio: %f" % (self.testnum, ratio))
369 if ratio < 0.99 or ratio > 1.01:
370 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
372 if self.json_data['jobs'][0]['read']['io_kbytes'] != 32768:
373 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
377 class FioJobTest_t0009(FioJobTest):
378 """Test consists of fio test job t0009
379 Confirm that runtime >= 60s"""
381 def check_result(self):
382 super(FioJobTest_t0009, self).check_result()
387 logging.debug('Test %d: elapsed: %d' % (self.testnum, self.json_data['jobs'][0]['elapsed']))
389 if self.json_data['jobs'][0]['elapsed'] < 60:
390 self.failure_reason = "{0} elapsed time mismatch,".format(self.failure_reason)
394 class FioJobTest_t0011(FioJobTest):
395 """Test consists of fio test job t0009
396 Confirm that job0 iops == 1000
397 and that job1_iops / job0_iops ~ 8
398 With two runs of fio-3.16 I observed a ratio of 8.3"""
400 def check_result(self):
401 super(FioJobTest_t0011, self).check_result()
406 iops1 = self.json_data['jobs'][0]['read']['iops']
407 iops2 = self.json_data['jobs'][1]['read']['iops']
408 ratio = iops2 / iops1
409 logging.debug("Test %d: iops1: %f" % (self.testnum, iops1))
410 logging.debug("Test %d: ratio: %f" % (self.testnum, ratio))
412 if iops1 < 998 or iops1 > 1002:
413 self.failure_reason = "{0} iops value mismatch,".format(self.failure_reason)
416 if ratio < 7 or ratio > 9:
417 self.failure_reason = "{0} iops ratio mismatch,".format(self.failure_reason)
421 class Requirements(object):
422 """Requirements consists of multiple run environment characteristics.
423 These are to determine if a particular test can be run"""
435 def __init__(self, fio_root):
436 Requirements._not_macos = platform.system() != "Darwin"
437 Requirements._not_windows = platform.system() != "Windows"
438 Requirements._linux = platform.system() == "Linux"
440 if Requirements._linux:
442 config_file = os.path.join(fio_root, "config-host.h")
443 with open(config_file, "r") as config:
444 contents = config.read()
446 print("Unable to open {0} to check requirements".format(config_file))
447 Requirements._zbd = True
449 Requirements._zbd = "CONFIG_LINUX_BLKZONED" in contents
450 Requirements._libaio = "CONFIG_LIBAIO" in contents
452 Requirements._root = (os.geteuid() == 0)
453 if Requirements._zbd and Requirements._root:
454 subprocess.run(["modprobe", "null_blk"],
455 stdout=subprocess.PIPE,
456 stderr=subprocess.PIPE)
457 if os.path.exists("/sys/module/null_blk/parameters/zoned"):
458 Requirements._zoned_nullb = True
460 if platform.system() == "Windows":
461 utest_exe = "unittest.exe"
463 utest_exe = "unittest"
464 unittest_path = os.path.join(fio_root, "unittests", utest_exe)
465 Requirements._unittests = os.path.exists(unittest_path)
467 Requirements._cpucount4 = multiprocessing.cpu_count() >= 4
469 req_list = [Requirements.linux,
473 Requirements.zoned_nullb,
474 Requirements.not_macos,
475 Requirements.not_windows,
476 Requirements.unittests,
477 Requirements.cpucount4]
480 logging.debug("Requirements: Requirement '%s' met? %s" % (desc, value))
483 return Requirements._linux, "Linux required"
486 return Requirements._libaio, "libaio required"
489 return Requirements._zbd, "Zoned block device support required"
492 return Requirements._root, "root required"
495 return Requirements._zoned_nullb, "Zoned null block device support required"
498 return Requirements._not_macos, "platform other than macOS required"
501 return Requirements._not_windows, "platform other than Windows required"
504 return Requirements._unittests, "Unittests support required"
507 return Requirements._cpucount4, "4+ CPUs required"
512 'stderr_empty': True,
516 'zero_return': False,
517 'stderr_empty': False,
522 'stderr_empty': False,
528 'test_class': FioJobTest,
529 'job': 't0001-52c58027.fio',
530 'success': SUCCESS_DEFAULT,
537 'test_class': FioJobTest,
538 'job': 't0002-13af05ae-post.fio',
539 'success': SUCCESS_DEFAULT,
540 'pre_job': 't0002-13af05ae-pre.fio',
542 'requirements': [Requirements.linux, Requirements.libaio],
546 'test_class': FioJobTest,
547 'job': 't0003-0ae2c6e1-post.fio',
548 'success': SUCCESS_NONZERO,
549 'pre_job': 't0003-0ae2c6e1-pre.fio',
550 'pre_success': SUCCESS_DEFAULT,
551 'requirements': [Requirements.linux, Requirements.libaio],
555 'test_class': FioJobTest,
556 'job': 't0004-8a99fdf6.fio',
557 'success': SUCCESS_DEFAULT,
560 'requirements': [Requirements.linux, Requirements.libaio],
564 'test_class': FioJobTest_t0005,
565 'job': 't0005-f7078f7b.fio',
566 'success': SUCCESS_DEFAULT,
569 'output_format': 'json',
570 'requirements': [Requirements.not_windows],
574 'test_class': FioJobTest_t0006,
575 'job': 't0006-82af2a7c.fio',
576 'success': SUCCESS_DEFAULT,
579 'output_format': 'json',
580 'requirements': [Requirements.linux, Requirements.libaio],
584 'test_class': FioJobTest_t0007,
585 'job': 't0007-37cf9e3c.fio',
586 'success': SUCCESS_DEFAULT,
589 'output_format': 'json',
594 'test_class': FioJobTest_t0008,
595 'job': 't0008-ae2fafc8.fio',
596 'success': SUCCESS_DEFAULT,
599 'output_format': 'json',
604 'test_class': FioJobTest_t0009,
605 'job': 't0009-f8b0bd10.fio',
606 'success': SUCCESS_DEFAULT,
609 'output_format': 'json',
610 'requirements': [Requirements.not_macos,
611 Requirements.cpucount4],
612 # mac os does not support CPU affinity
616 'test_class': FioJobTest,
617 'job': 't0010-b7aae4ba.fio',
618 'success': SUCCESS_DEFAULT,
625 'test_class': FioJobTest_t0011,
626 'job': 't0011-5d2788d5.fio',
627 'success': SUCCESS_DEFAULT,
630 'output_format': 'json',
635 'test_class': FioExeTest,
638 'success': SUCCESS_DEFAULT,
643 'test_class': FioExeTest,
646 'success': SUCCESS_DEFAULT,
651 'test_class': FioExeTest,
652 'exe': 't/lfsr-test',
653 'parameters': ['0xFFFFFF', '0', '0', 'verify'],
654 'success': SUCCESS_STDERR,
659 'test_class': FioExeTest,
660 'exe': 't/readonly.py',
661 'parameters': ['-f', '{fio_path}'],
662 'success': SUCCESS_DEFAULT,
667 'test_class': FioExeTest,
668 'exe': 't/steadystate_tests.py',
669 'parameters': ['{fio_path}'],
670 'success': SUCCESS_DEFAULT,
675 'test_class': FioExeTest,
678 'success': SUCCESS_STDERR,
683 'test_class': FioExeTest,
684 'exe': 't/strided.py',
685 'parameters': ['{fio_path}'],
686 'success': SUCCESS_DEFAULT,
691 'test_class': FioExeTest,
692 'exe': 't/zbd/run-tests-against-regular-nullb',
694 'success': SUCCESS_DEFAULT,
695 'requirements': [Requirements.linux, Requirements.zbd,
700 'test_class': FioExeTest,
701 'exe': 't/zbd/run-tests-against-zoned-nullb',
703 'success': SUCCESS_DEFAULT,
704 'requirements': [Requirements.linux, Requirements.zbd,
705 Requirements.root, Requirements.zoned_nullb],
709 'test_class': FioExeTest,
710 'exe': 'unittests/unittest',
712 'success': SUCCESS_DEFAULT,
713 'requirements': [Requirements.unittests],
717 'test_class': FioExeTest,
718 'exe': 't/latency_percentiles.py',
719 'parameters': ['-f', '{fio_path}'],
720 'success': SUCCESS_DEFAULT,
727 parser = argparse.ArgumentParser()
728 parser.add_argument('-r', '--fio-root',
729 help='fio root path')
730 parser.add_argument('-f', '--fio',
731 help='path to fio executable (e.g., ./fio)')
732 parser.add_argument('-a', '--artifact-root',
733 help='artifact root directory')
734 parser.add_argument('-s', '--skip', nargs='+', type=int,
735 help='list of test(s) to skip')
736 parser.add_argument('-o', '--run-only', nargs='+', type=int,
737 help='list of test(s) to run, skipping all others')
738 parser.add_argument('-d', '--debug', action='store_true',
739 help='provide debug output')
740 parser.add_argument('-k', '--skip-req', action='store_true',
741 help='skip requirements checking')
742 args = parser.parse_args()
750 logging.basicConfig(level=logging.DEBUG)
752 logging.basicConfig(level=logging.INFO)
755 fio_root = args.fio_root
757 fio_root = str(Path(__file__).absolute().parent.parent)
758 print("fio root is %s" % fio_root)
763 if platform.system() == "Windows":
767 fio_path = os.path.join(fio_root, fio_exe)
768 print("fio path is %s" % fio_path)
769 if not shutil.which(fio_path):
770 print("Warning: fio executable not found")
772 artifact_root = args.artifact_root if args.artifact_root else \
773 "fio-test-{0}".format(time.strftime("%Y%m%d-%H%M%S"))
774 os.mkdir(artifact_root)
775 print("Artifact directory is %s" % artifact_root)
777 if not args.skip_req:
778 req = Requirements(fio_root)
784 for config in TEST_LIST:
785 if (args.skip and config['test_id'] in args.skip) or \
786 (args.run_only and config['test_id'] not in args.run_only):
787 skipped = skipped + 1
788 print("Test {0} SKIPPED (User request)".format(config['test_id']))
791 if issubclass(config['test_class'], FioJobTest):
792 if config['pre_job']:
793 fio_pre_job = os.path.join(fio_root, 't', 'jobs',
797 if config['pre_success']:
798 fio_pre_success = config['pre_success']
800 fio_pre_success = None
801 if 'output_format' in config:
802 output_format = config['output_format']
804 output_format = 'normal'
805 test = config['test_class'](
807 os.path.join(fio_root, 't', 'jobs', config['job']),
809 fio_pre_job=fio_pre_job,
810 fio_pre_success=fio_pre_success,
811 output_format=output_format)
812 elif issubclass(config['test_class'], FioExeTest):
813 exe_path = os.path.join(fio_root, config['exe'])
814 if config['parameters']:
815 parameters = [p.format(fio_path=fio_path) for p in config['parameters']]
818 if Path(exe_path).suffix == '.py' and platform.system() == "Windows":
820 parameters.insert(0, exe_path)
822 parameters = [exe_path]
823 exe_path = "python.exe"
824 test = config['test_class'](exe_path, parameters,
827 print("Test {0} FAILED: unable to process test config".format(config['test_id']))
831 if not args.skip_req:
833 for req in config['requirements']:
836 logging.debug("Test %d: Requirement '%s' met? %s" % (config['test_id'], reason, ok))
840 print("Test {0} SKIPPED ({1})".format(config['test_id'], reason))
841 skipped = skipped + 1
844 test.setup(artifact_root, config['test_id'])
851 result = "FAILED: {0}".format(test.failure_reason)
853 with open(test.stderr_file, "r") as stderr_file:
854 logging.debug("Test %d: stderr:\n%s" % (config['test_id'], stderr_file.read()))
855 with open(test.stdout_file, "r") as stdout_file:
856 logging.debug("Test %d: stdout:\n%s" % (config['test_id'], stdout_file.read()))
857 print("Test {0} {1}".format(config['test_id'], result))
859 print("{0} test(s) passed, {1} failed, {2} skipped".format(passed, failed, skipped))
864 if __name__ == '__main__':