2 # SPDX-License-Identifier: GPL-2.0-only
4 # Copyright (c) 2019 Western Digital Corporation or its affiliates.
9 # Automate running of fio tests
12 # python3 run-fio-tests.py [-r fio-root] [-f fio-path] [-a artifact-root]
13 # [--skip # # #...] [--run-only # # #...]
17 # # git clone git://git.kernel.dk/fio.git
20 # # python3 t/run-fio-tests.py
24 # - Python 3.5 (subprocess.run)
25 # - Linux (libaio ioengine, zbd tests, etc)
26 # - The artifact directory must be on a file system that accepts 512-byte IO
27 # (t0002, t0003, t0004).
28 # - The artifact directory needs to be on an SSD. Otherwise tests that carry
29 # out file-based IO will trigger a timeout (t0006).
31 # - SciPy (steadystate_tests.py)
32 # - libzbc (zbd tests)
33 # - root privileges (zbd test)
34 # - kernel 4.19 or later for zoned null block devices (zbd tests)
35 # - CUnit support (unittests)
40 # TODO run multiple tests simultaneously
41 # TODO Add sgunmap tests (requires SAS SSD)
53 import multiprocessing
54 from pathlib import Path
57 class FioTest(object):
58 """Base for all fio tests."""
60 def __init__(self, exe_path, parameters, success):
61 self.exe_path = exe_path
62 self.parameters = parameters
63 self.success = success
65 self.artifact_root = None
69 self.failure_reason = ''
70 self.command_file = None
71 self.stdout_file = None
72 self.stderr_file = None
73 self.exitcode_file = None
75 def setup(self, artifact_root, testnum):
76 """Setup instance variables for test."""
78 self.artifact_root = artifact_root
79 self.testnum = testnum
80 self.test_dir = os.path.join(artifact_root, "{:04d}".format(testnum))
81 if not os.path.exists(self.test_dir):
82 os.mkdir(self.test_dir)
84 self.command_file = os.path.join(
86 "{0}.command".format(os.path.basename(self.exe_path)))
87 self.stdout_file = os.path.join(
89 "{0}.stdout".format(os.path.basename(self.exe_path)))
90 self.stderr_file = os.path.join(
92 "{0}.stderr".format(os.path.basename(self.exe_path)))
93 self.exitcode_file = os.path.join(
95 "{0}.exitcode".format(os.path.basename(self.exe_path)))
100 raise NotImplementedError()
102 def check_result(self):
103 """Check test results."""
105 raise NotImplementedError()
108 class FioExeTest(FioTest):
109 """Test consists of an executable binary or script"""
111 def __init__(self, exe_path, parameters, success):
112 """Construct a FioExeTest which is a FioTest consisting of an
113 executable binary or script.
115 exe_path: location of executable binary or script
116 parameters: list of parameters for executable
117 success: Definition of test success
120 FioTest.__init__(self, exe_path, parameters, success)
123 """Execute the binary or script described by this instance."""
125 command = [self.exe_path] + self.parameters
126 command_file = open(self.command_file, "w+")
127 command_file.write("%s\n" % command)
130 stdout_file = open(self.stdout_file, "w+")
131 stderr_file = open(self.stderr_file, "w+")
132 exitcode_file = open(self.exitcode_file, "w+")
135 # Avoid using subprocess.run() here because when a timeout occurs,
136 # fio will be stopped with SIGKILL. This does not give fio a
137 # chance to clean up and means that child processes may continue
138 # running and submitting IO.
139 proc = subprocess.Popen(command,
143 universal_newlines=True)
144 proc.communicate(timeout=self.success['timeout'])
145 exitcode_file.write('{0}\n'.format(proc.returncode))
146 logging.debug("Test %d: return code: %d", self.testnum, proc.returncode)
147 self.output['proc'] = proc
148 except subprocess.TimeoutExpired:
152 self.output['failure'] = 'timeout'
158 self.output['failure'] = 'exception'
159 self.output['exc_info'] = sys.exc_info()
163 exitcode_file.close()
165 def check_result(self):
166 """Check results of test run."""
168 if 'proc' not in self.output:
169 if self.output['failure'] == 'timeout':
170 self.failure_reason = "{0} timeout,".format(self.failure_reason)
172 assert self.output['failure'] == 'exception'
173 self.failure_reason = '{0} exception: {1}, {2}'.format(
174 self.failure_reason, self.output['exc_info'][0],
175 self.output['exc_info'][1])
180 if 'zero_return' in self.success:
181 if self.success['zero_return']:
182 if self.output['proc'].returncode != 0:
184 self.failure_reason = "{0} non-zero return code,".format(self.failure_reason)
186 if self.output['proc'].returncode == 0:
187 self.failure_reason = "{0} zero return code,".format(self.failure_reason)
190 stderr_size = os.path.getsize(self.stderr_file)
191 if 'stderr_empty' in self.success:
192 if self.success['stderr_empty']:
194 self.failure_reason = "{0} stderr not empty,".format(self.failure_reason)
198 self.failure_reason = "{0} stderr empty,".format(self.failure_reason)
202 class FioJobTest(FioExeTest):
203 """Test consists of a fio job"""
205 def __init__(self, fio_path, fio_job, success, fio_pre_job=None,
206 fio_pre_success=None, output_format="normal"):
207 """Construct a FioJobTest which is a FioExeTest consisting of a
208 single fio job file with an optional setup step.
210 fio_path: location of fio executable
211 fio_job: location of fio job file
212 success: Definition of test success
213 fio_pre_job: fio job for preconditioning
214 fio_pre_success: Definition of test success for fio precon job
215 output_format: normal (default), json, jsonplus, or terse
218 self.fio_job = fio_job
219 self.fio_pre_job = fio_pre_job
220 self.fio_pre_success = fio_pre_success if fio_pre_success else success
221 self.output_format = output_format
222 self.precon_failed = False
223 self.json_data = None
224 self.fio_output = "{0}.output".format(os.path.basename(self.fio_job))
227 "--output-format={0}".format(self.output_format),
228 "--output={0}".format(self.fio_output),
231 FioExeTest.__init__(self, fio_path, self.fio_args, success)
233 def setup(self, artifact_root, testnum):
234 """Setup instance variables for fio job test."""
236 super(FioJobTest, self).setup(artifact_root, testnum)
238 self.command_file = os.path.join(
240 "{0}.command".format(os.path.basename(self.fio_job)))
241 self.stdout_file = os.path.join(
243 "{0}.stdout".format(os.path.basename(self.fio_job)))
244 self.stderr_file = os.path.join(
246 "{0}.stderr".format(os.path.basename(self.fio_job)))
247 self.exitcode_file = os.path.join(
249 "{0}.exitcode".format(os.path.basename(self.fio_job)))
251 def run_pre_job(self):
252 """Run fio job precondition step."""
254 precon = FioJobTest(self.exe_path, self.fio_pre_job,
255 self.fio_pre_success,
256 output_format=self.output_format)
257 precon.setup(self.artifact_root, self.testnum)
259 precon.check_result()
260 self.precon_failed = not precon.passed
261 self.failure_reason = precon.failure_reason
264 """Run fio job test."""
269 if not self.precon_failed:
270 super(FioJobTest, self).run()
272 logging.debug("Test %d: precondition step failed", self.testnum)
275 def get_file(cls, filename):
276 """Safely read a file."""
281 with open(filename, "r") as output_file:
282 file_data = output_file.read()
286 return file_data, success
288 def check_result(self):
289 """Check fio job results."""
291 if self.precon_failed:
293 self.failure_reason = "{0} precondition step failed,".format(self.failure_reason)
296 super(FioJobTest, self).check_result()
301 if 'json' not in self.output_format:
304 file_data, success = self.get_file(os.path.join(self.test_dir, self.fio_output))
306 self.failure_reason = "{0} unable to open output file,".format(self.failure_reason)
311 # Sometimes fio informational messages are included at the top of the
312 # JSON output, especially under Windows. Try to decode output as JSON
313 # data, lopping off up to the first four lines
315 lines = file_data.splitlines()
317 file_data = '\n'.join(lines[i:])
319 self.json_data = json.loads(file_data)
320 except json.JSONDecodeError:
323 logging.debug("Test %d: skipped %d lines decoding JSON data", self.testnum, i)
326 self.failure_reason = "{0} unable to decode JSON data,".format(self.failure_reason)
330 class FioJobTest_t0005(FioJobTest):
331 """Test consists of fio test job t0005
332 Confirm that read['io_kbytes'] == write['io_kbytes'] == 102400"""
334 def check_result(self):
335 super(FioJobTest_t0005, self).check_result()
340 if self.json_data['jobs'][0]['read']['io_kbytes'] != 102400:
341 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
343 if self.json_data['jobs'][0]['write']['io_kbytes'] != 102400:
344 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
348 class FioJobTest_t0006(FioJobTest):
349 """Test consists of fio test job t0006
350 Confirm that read['io_kbytes'] ~ 2*write['io_kbytes']"""
352 def check_result(self):
353 super(FioJobTest_t0006, self).check_result()
358 ratio = self.json_data['jobs'][0]['read']['io_kbytes'] \
359 / self.json_data['jobs'][0]['write']['io_kbytes']
360 logging.debug("Test %d: ratio: %f", self.testnum, ratio)
361 if ratio < 1.99 or ratio > 2.01:
362 self.failure_reason = "{0} read/write ratio mismatch,".format(self.failure_reason)
366 class FioJobTest_t0007(FioJobTest):
367 """Test consists of fio test job t0007
368 Confirm that read['io_kbytes'] = 87040"""
370 def check_result(self):
371 super(FioJobTest_t0007, self).check_result()
376 if self.json_data['jobs'][0]['read']['io_kbytes'] != 87040:
377 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
381 class FioJobTest_t0008(FioJobTest):
382 """Test consists of fio test job t0008
383 Confirm that read['io_kbytes'] = 32768 and that
384 write['io_kbytes'] ~ 16568
386 I did runs with fio-ae2fafc8 and saw write['io_kbytes'] values of
387 16585, 16588. With two runs of fio-3.16 I obtained 16568"""
389 def check_result(self):
390 super(FioJobTest_t0008, self).check_result()
395 ratio = self.json_data['jobs'][0]['write']['io_kbytes'] / 16568
396 logging.debug("Test %d: ratio: %f", self.testnum, ratio)
398 if ratio < 0.99 or ratio > 1.01:
399 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
401 if self.json_data['jobs'][0]['read']['io_kbytes'] != 32768:
402 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
406 class FioJobTest_t0009(FioJobTest):
407 """Test consists of fio test job t0009
408 Confirm that runtime >= 60s"""
410 def check_result(self):
411 super(FioJobTest_t0009, self).check_result()
416 logging.debug('Test %d: elapsed: %d', self.testnum, self.json_data['jobs'][0]['elapsed'])
418 if self.json_data['jobs'][0]['elapsed'] < 60:
419 self.failure_reason = "{0} elapsed time mismatch,".format(self.failure_reason)
423 class FioJobTest_t0012(FioJobTest):
424 """Test consists of fio test job t0012
425 Confirm ratios of job iops are 1:5:10
426 job1,job2,job3 respectively"""
428 def check_result(self):
429 super(FioJobTest_t0012, self).check_result()
436 file_data, success = self.get_file(os.path.join(self.test_dir, "{0}_iops.{1}.log".format(os.path.basename(self.fio_job), i)))
439 self.failure_reason = "{0} unable to open output file,".format(self.failure_reason)
443 iops_files.append(file_data.splitlines())
445 # there are 9 samples for job1 and job2, 4 samples for job3
450 iops1 = iops1 + float(iops_files[0][i].split(',')[1])
451 iops2 = iops2 + float(iops_files[1][i].split(',')[1])
452 iops3 = iops3 + float(iops_files[2][i].split(',')[1])
457 "sample {0}: job1 iops={1} job2 iops={2} job3 iops={3} job3/job2={4:.3f} job3/job1={5:.3f}".format(
458 i, iops1, iops2, iops3, ratio1, ratio2
462 # test job1 and job2 succeeded to recalibrate
463 if ratio1 < 1 or ratio1 > 3 or ratio2 < 7 or ratio2 > 13:
464 self.failure_reason = "{0} iops ratio mismatch iops1={1} iops2={2} iops3={3} expected r1~2 r2~10 got r1={4:.3f} r2={5:.3f},".format(
465 self.failure_reason, iops1, iops2, iops3, ratio1, ratio2
471 class FioJobTest_t0014(FioJobTest):
472 """Test consists of fio test job t0014
473 Confirm that job1_iops / job2_iops ~ 1:2 for entire duration
474 and that job1_iops / job3_iops ~ 1:3 for first half of duration.
476 The test is about making sure the flow feature can
477 re-calibrate the activity dynamically"""
479 def check_result(self):
480 super(FioJobTest_t0014, self).check_result()
487 file_data, success = self.get_file(os.path.join(self.test_dir, "{0}_iops.{1}.log".format(os.path.basename(self.fio_job), i)))
490 self.failure_reason = "{0} unable to open output file,".format(self.failure_reason)
494 iops_files.append(file_data.splitlines())
496 # there are 9 samples for job1 and job2, 4 samples for job3
502 iops3 = iops3 + float(iops_files[2][i].split(',')[1])
504 ratio1 = iops1 / iops2
505 ratio2 = iops1 / iops3
508 if ratio1 < 0.43 or ratio1 > 0.57 or ratio2 < 0.21 or ratio2 > 0.45:
509 self.failure_reason = "{0} iops ratio mismatch iops1={1} iops2={2} iops3={3}\
510 expected r1~0.5 r2~0.33 got r1={4:.3f} r2={5:.3f},".format(
511 self.failure_reason, iops1, iops2, iops3, ratio1, ratio2
515 iops1 = iops1 + float(iops_files[0][i].split(',')[1])
516 iops2 = iops2 + float(iops_files[1][i].split(',')[1])
521 "sample {0}: job1 iops={1} job2 iops={2} job3 iops={3} job1/job2={4:.3f} job1/job3={5:.3f}".format(
522 i, iops1, iops2, iops3, ratio1, ratio2
526 # test job1 and job2 succeeded to recalibrate
527 if ratio1 < 0.43 or ratio1 > 0.57:
528 self.failure_reason = "{0} iops ratio mismatch iops1={1} iops2={2} expected ratio~0.5 got ratio={3:.3f},".format(
529 self.failure_reason, iops1, iops2, ratio1
535 class FioJobTest_iops_rate(FioJobTest):
536 """Test consists of fio test job t0009
537 Confirm that job0 iops == 1000
538 and that job1_iops / job0_iops ~ 8
539 With two runs of fio-3.16 I observed a ratio of 8.3"""
541 def check_result(self):
542 super(FioJobTest_iops_rate, self).check_result()
547 iops1 = self.json_data['jobs'][0]['read']['iops']
548 iops2 = self.json_data['jobs'][1]['read']['iops']
549 ratio = iops2 / iops1
550 logging.debug("Test %d: iops1: %f", self.testnum, iops1)
551 logging.debug("Test %d: ratio: %f", self.testnum, ratio)
553 if iops1 < 950 or iops1 > 1050:
554 self.failure_reason = "{0} iops value mismatch,".format(self.failure_reason)
557 if ratio < 6 or ratio > 10:
558 self.failure_reason = "{0} iops ratio mismatch,".format(self.failure_reason)
562 class Requirements(object):
563 """Requirements consists of multiple run environment characteristics.
564 These are to determine if a particular test can be run"""
576 def __init__(self, fio_root):
577 Requirements._not_macos = platform.system() != "Darwin"
578 Requirements._not_windows = platform.system() != "Windows"
579 Requirements._linux = platform.system() == "Linux"
581 if Requirements._linux:
582 config_file = os.path.join(fio_root, "config-host.h")
583 contents, success = FioJobTest.get_file(config_file)
585 print("Unable to open {0} to check requirements".format(config_file))
586 Requirements._zbd = True
588 Requirements._zbd = "CONFIG_HAS_BLKZONED" in contents
589 Requirements._libaio = "CONFIG_LIBAIO" in contents
591 Requirements._root = (os.geteuid() == 0)
592 if Requirements._zbd and Requirements._root:
594 subprocess.run(["modprobe", "null_blk"],
595 stdout=subprocess.PIPE,
596 stderr=subprocess.PIPE)
597 if os.path.exists("/sys/module/null_blk/parameters/zoned"):
598 Requirements._zoned_nullb = True
602 if platform.system() == "Windows":
603 utest_exe = "unittest.exe"
605 utest_exe = "unittest"
606 unittest_path = os.path.join(fio_root, "unittests", utest_exe)
607 Requirements._unittests = os.path.exists(unittest_path)
609 Requirements._cpucount4 = multiprocessing.cpu_count() >= 4
611 req_list = [Requirements.linux,
615 Requirements.zoned_nullb,
616 Requirements.not_macos,
617 Requirements.not_windows,
618 Requirements.unittests,
619 Requirements.cpucount4]
622 logging.debug("Requirements: Requirement '%s' met? %s", desc, value)
626 """Are we running on Linux?"""
627 return Requirements._linux, "Linux required"
631 """Is libaio available?"""
632 return Requirements._libaio, "libaio required"
636 """Is ZBD support available?"""
637 return Requirements._zbd, "Zoned block device support required"
641 """Are we running as root?"""
642 return Requirements._root, "root required"
645 def zoned_nullb(cls):
646 """Are zoned null block devices available?"""
647 return Requirements._zoned_nullb, "Zoned null block device support required"
651 """Are we running on a platform other than macOS?"""
652 return Requirements._not_macos, "platform other than macOS required"
655 def not_windows(cls):
656 """Are we running on a platform other than Windws?"""
657 return Requirements._not_windows, "platform other than Windows required"
661 """Were unittests built?"""
662 return Requirements._unittests, "Unittests support required"
666 """Do we have at least 4 CPUs?"""
667 return Requirements._cpucount4, "4+ CPUs required"
672 'stderr_empty': True,
676 'zero_return': False,
677 'stderr_empty': False,
682 'stderr_empty': False,
688 'test_class': FioJobTest,
689 'job': 't0001-52c58027.fio',
690 'success': SUCCESS_DEFAULT,
697 'test_class': FioJobTest,
698 'job': 't0002-13af05ae-post.fio',
699 'success': SUCCESS_DEFAULT,
700 'pre_job': 't0002-13af05ae-pre.fio',
702 'requirements': [Requirements.linux, Requirements.libaio],
706 'test_class': FioJobTest,
707 'job': 't0003-0ae2c6e1-post.fio',
708 'success': SUCCESS_NONZERO,
709 'pre_job': 't0003-0ae2c6e1-pre.fio',
710 'pre_success': SUCCESS_DEFAULT,
711 'requirements': [Requirements.linux, Requirements.libaio],
715 'test_class': FioJobTest,
716 'job': 't0004-8a99fdf6.fio',
717 'success': SUCCESS_DEFAULT,
720 'requirements': [Requirements.linux, Requirements.libaio],
724 'test_class': FioJobTest_t0005,
725 'job': 't0005-f7078f7b.fio',
726 'success': SUCCESS_DEFAULT,
729 'output_format': 'json',
730 'requirements': [Requirements.not_windows],
734 'test_class': FioJobTest_t0006,
735 'job': 't0006-82af2a7c.fio',
736 'success': SUCCESS_DEFAULT,
739 'output_format': 'json',
740 'requirements': [Requirements.linux, Requirements.libaio],
744 'test_class': FioJobTest_t0007,
745 'job': 't0007-37cf9e3c.fio',
746 'success': SUCCESS_DEFAULT,
749 'output_format': 'json',
754 'test_class': FioJobTest_t0008,
755 'job': 't0008-ae2fafc8.fio',
756 'success': SUCCESS_DEFAULT,
759 'output_format': 'json',
764 'test_class': FioJobTest_t0009,
765 'job': 't0009-f8b0bd10.fio',
766 'success': SUCCESS_DEFAULT,
769 'output_format': 'json',
770 'requirements': [Requirements.not_macos,
771 Requirements.cpucount4],
772 # mac os does not support CPU affinity
776 'test_class': FioJobTest,
777 'job': 't0010-b7aae4ba.fio',
778 'success': SUCCESS_DEFAULT,
785 'test_class': FioJobTest_iops_rate,
786 'job': 't0011-5d2788d5.fio',
787 'success': SUCCESS_DEFAULT,
790 'output_format': 'json',
795 'test_class': FioJobTest_t0012,
797 'success': SUCCESS_DEFAULT,
800 'output_format': 'json',
805 'test_class': FioJobTest,
807 'success': SUCCESS_DEFAULT,
810 'output_format': 'json',
815 'test_class': FioJobTest_t0014,
817 'success': SUCCESS_DEFAULT,
820 'output_format': 'json',
825 'test_class': FioExeTest,
828 'success': SUCCESS_DEFAULT,
833 'test_class': FioExeTest,
836 'success': SUCCESS_DEFAULT,
841 'test_class': FioExeTest,
842 'exe': 't/lfsr-test',
843 'parameters': ['0xFFFFFF', '0', '0', 'verify'],
844 'success': SUCCESS_STDERR,
849 'test_class': FioExeTest,
850 'exe': 't/readonly.py',
851 'parameters': ['-f', '{fio_path}'],
852 'success': SUCCESS_DEFAULT,
857 'test_class': FioExeTest,
858 'exe': 't/steadystate_tests.py',
859 'parameters': ['{fio_path}'],
860 'success': SUCCESS_DEFAULT,
865 'test_class': FioExeTest,
868 'success': SUCCESS_STDERR,
873 'test_class': FioExeTest,
874 'exe': 't/strided.py',
875 'parameters': ['{fio_path}'],
876 'success': SUCCESS_DEFAULT,
881 'test_class': FioExeTest,
882 'exe': 't/zbd/run-tests-against-nullb',
883 'parameters': ['-s', '1'],
884 'success': SUCCESS_DEFAULT,
885 'requirements': [Requirements.linux, Requirements.zbd,
890 'test_class': FioExeTest,
891 'exe': 't/zbd/run-tests-against-nullb',
892 'parameters': ['-s', '2'],
893 'success': SUCCESS_DEFAULT,
894 'requirements': [Requirements.linux, Requirements.zbd,
895 Requirements.root, Requirements.zoned_nullb],
899 'test_class': FioExeTest,
900 'exe': 'unittests/unittest',
902 'success': SUCCESS_DEFAULT,
903 'requirements': [Requirements.unittests],
907 'test_class': FioExeTest,
908 'exe': 't/latency_percentiles.py',
909 'parameters': ['-f', '{fio_path}'],
910 'success': SUCCESS_DEFAULT,
915 'test_class': FioExeTest,
916 'exe': 't/jsonplus2csv_test.py',
917 'parameters': ['-f', '{fio_path}'],
918 'success': SUCCESS_DEFAULT,
925 """Parse command-line arguments."""
927 parser = argparse.ArgumentParser()
928 parser.add_argument('-r', '--fio-root',
929 help='fio root path')
930 parser.add_argument('-f', '--fio',
931 help='path to fio executable (e.g., ./fio)')
932 parser.add_argument('-a', '--artifact-root',
933 help='artifact root directory')
934 parser.add_argument('-s', '--skip', nargs='+', type=int,
935 help='list of test(s) to skip')
936 parser.add_argument('-o', '--run-only', nargs='+', type=int,
937 help='list of test(s) to run, skipping all others')
938 parser.add_argument('-d', '--debug', action='store_true',
939 help='provide debug output')
940 parser.add_argument('-k', '--skip-req', action='store_true',
941 help='skip requirements checking')
942 parser.add_argument('-p', '--pass-through', action='append',
943 help='pass-through an argument to an executable test')
944 args = parser.parse_args()
954 logging.basicConfig(level=logging.DEBUG)
956 logging.basicConfig(level=logging.INFO)
959 if args.pass_through:
960 for arg in args.pass_through:
962 print("Invalid --pass-through argument '%s'" % arg)
963 print("Syntax for --pass-through is TESTNUMBER:ARGUMENT")
965 split = arg.split(":", 1)
966 pass_through[int(split[0])] = split[1]
967 logging.debug("Pass-through arguments: %s", pass_through)
970 fio_root = args.fio_root
972 fio_root = str(Path(__file__).absolute().parent.parent)
973 print("fio root is %s" % fio_root)
978 if platform.system() == "Windows":
982 fio_path = os.path.join(fio_root, fio_exe)
983 print("fio path is %s" % fio_path)
984 if not shutil.which(fio_path):
985 print("Warning: fio executable not found")
987 artifact_root = args.artifact_root if args.artifact_root else \
988 "fio-test-{0}".format(time.strftime("%Y%m%d-%H%M%S"))
989 os.mkdir(artifact_root)
990 print("Artifact directory is %s" % artifact_root)
992 if not args.skip_req:
993 req = Requirements(fio_root)
999 for config in TEST_LIST:
1000 if (args.skip and config['test_id'] in args.skip) or \
1001 (args.run_only and config['test_id'] not in args.run_only):
1002 skipped = skipped + 1
1003 print("Test {0} SKIPPED (User request)".format(config['test_id']))
1006 if issubclass(config['test_class'], FioJobTest):
1007 if config['pre_job']:
1008 fio_pre_job = os.path.join(fio_root, 't', 'jobs',
1012 if config['pre_success']:
1013 fio_pre_success = config['pre_success']
1015 fio_pre_success = None
1016 if 'output_format' in config:
1017 output_format = config['output_format']
1019 output_format = 'normal'
1020 test = config['test_class'](
1022 os.path.join(fio_root, 't', 'jobs', config['job']),
1024 fio_pre_job=fio_pre_job,
1025 fio_pre_success=fio_pre_success,
1026 output_format=output_format)
1027 desc = config['job']
1028 elif issubclass(config['test_class'], FioExeTest):
1029 exe_path = os.path.join(fio_root, config['exe'])
1030 if config['parameters']:
1031 parameters = [p.format(fio_path=fio_path) for p in config['parameters']]
1034 if Path(exe_path).suffix == '.py' and platform.system() == "Windows":
1035 parameters.insert(0, exe_path)
1036 exe_path = "python.exe"
1037 if config['test_id'] in pass_through:
1038 parameters += pass_through[config['test_id']].split()
1039 test = config['test_class'](exe_path, parameters,
1041 desc = config['exe']
1043 print("Test {0} FAILED: unable to process test config".format(config['test_id']))
1047 if not args.skip_req:
1049 for req in config['requirements']:
1050 reqs_met, reason = req()
1051 logging.debug("Test %d: Requirement '%s' met? %s", config['test_id'], reason,
1056 print("Test {0} SKIPPED ({1}) {2}".format(config['test_id'], reason, desc))
1057 skipped = skipped + 1
1060 test.setup(artifact_root, config['test_id'])
1067 result = "FAILED: {0}".format(test.failure_reason)
1069 contents, _ = FioJobTest.get_file(test.stderr_file)
1070 logging.debug("Test %d: stderr:\n%s", config['test_id'], contents)
1071 contents, _ = FioJobTest.get_file(test.stdout_file)
1072 logging.debug("Test %d: stdout:\n%s", config['test_id'], contents)
1073 print("Test {0} {1} {2}".format(config['test_id'], result, desc))
1075 print("{0} test(s) passed, {1} failed, {2} skipped".format(passed, failed, skipped))
1080 if __name__ == '__main__':