2 # SPDX-License-Identifier: GPL-2.0-only
4 # Copyright (c) 2019 Western Digital Corporation or its affiliates.
9 # Automate running of fio tests
12 # python3 run-fio-tests.py [-r fio-root] [-f fio-path] [-a artifact-root]
13 # [--skip # # #...] [--run-only # # #...]
17 # # git clone git://git.kernel.dk/fio.git
20 # # python3 t/run-fio-tests.py
24 # - Python 3.5 (subprocess.run)
25 # - Linux (libaio ioengine, zbd tests, etc)
26 # - The artifact directory must be on a file system that accepts 512-byte IO
27 # (t0002, t0003, t0004).
28 # - The artifact directory needs to be on an SSD. Otherwise tests that carry
29 # out file-based IO will trigger a timeout (t0006).
31 # - SciPy (steadystate_tests.py)
32 # - libzbc (zbd tests)
33 # - root privileges (zbd test)
34 # - kernel 4.19 or later for zoned null block devices (zbd tests)
35 # - CUnit support (unittests)
40 # TODO run multiple tests simultaneously
41 # TODO Add sgunmap tests (requires SAS SSD)
54 import multiprocessing
55 from pathlib import Path
56 from statsmodels.sandbox.stats.runs import runstest_1samp
60 """Base for all fio tests."""
62 def __init__(self, exe_path, parameters, success):
63 self.exe_path = exe_path
64 self.parameters = parameters
65 self.success = success
67 self.artifact_root = None
71 self.failure_reason = ''
72 self.command_file = None
73 self.stdout_file = None
74 self.stderr_file = None
75 self.exitcode_file = None
77 def setup(self, artifact_root, testnum):
78 """Setup instance variables for test."""
80 self.artifact_root = artifact_root
81 self.testnum = testnum
82 self.test_dir = os.path.join(artifact_root, f"{testnum:04d}")
83 if not os.path.exists(self.test_dir):
84 os.mkdir(self.test_dir)
86 self.command_file = os.path.join(
88 f"{os.path.basename(self.exe_path)}.command")
89 self.stdout_file = os.path.join(
91 f"{os.path.basename(self.exe_path)}.stdout")
92 self.stderr_file = os.path.join(
94 f"{os.path.basename(self.exe_path)}.stderr")
95 self.exitcode_file = os.path.join(
97 f"{os.path.basename(self.exe_path)}.exitcode")
102 raise NotImplementedError()
104 def check_result(self):
105 """Check test results."""
107 raise NotImplementedError()
110 class FioExeTest(FioTest):
111 """Test consists of an executable binary or script"""
113 def __init__(self, exe_path, parameters, success):
114 """Construct a FioExeTest which is a FioTest consisting of an
115 executable binary or script.
117 exe_path: location of executable binary or script
118 parameters: list of parameters for executable
119 success: Definition of test success
122 FioTest.__init__(self, exe_path, parameters, success)
125 """Execute the binary or script described by this instance."""
127 command = [self.exe_path] + self.parameters
128 command_file = open(self.command_file, "w+")
129 command_file.write(f"{command}\n")
132 stdout_file = open(self.stdout_file, "w+")
133 stderr_file = open(self.stderr_file, "w+")
134 exitcode_file = open(self.exitcode_file, "w+")
137 # Avoid using subprocess.run() here because when a timeout occurs,
138 # fio will be stopped with SIGKILL. This does not give fio a
139 # chance to clean up and means that child processes may continue
140 # running and submitting IO.
141 proc = subprocess.Popen(command,
145 universal_newlines=True)
146 proc.communicate(timeout=self.success['timeout'])
147 exitcode_file.write(f'{proc.returncode}\n')
148 logging.debug("Test %d: return code: %d", self.testnum, proc.returncode)
149 self.output['proc'] = proc
150 except subprocess.TimeoutExpired:
154 self.output['failure'] = 'timeout'
160 self.output['failure'] = 'exception'
161 self.output['exc_info'] = sys.exc_info()
165 exitcode_file.close()
167 def check_result(self):
168 """Check results of test run."""
170 if 'proc' not in self.output:
171 if self.output['failure'] == 'timeout':
172 self.failure_reason = f"{self.failure_reason} timeout,"
174 assert self.output['failure'] == 'exception'
175 self.failure_reason = '{0} exception: {1}, {2}'.format(
176 self.failure_reason, self.output['exc_info'][0],
177 self.output['exc_info'][1])
182 if 'zero_return' in self.success:
183 if self.success['zero_return']:
184 if self.output['proc'].returncode != 0:
186 self.failure_reason = f"{self.failure_reason} non-zero return code,"
188 if self.output['proc'].returncode == 0:
189 self.failure_reason = f"{self.failure_reason} zero return code,"
192 stderr_size = os.path.getsize(self.stderr_file)
193 if 'stderr_empty' in self.success:
194 if self.success['stderr_empty']:
196 self.failure_reason = f"{self.failure_reason} stderr not empty,"
200 self.failure_reason = f"{self.failure_reason} stderr empty,"
204 class FioJobTest(FioExeTest):
205 """Test consists of a fio job"""
207 def __init__(self, fio_path, fio_job, success, fio_pre_job=None,
208 fio_pre_success=None, output_format="normal"):
209 """Construct a FioJobTest which is a FioExeTest consisting of a
210 single fio job file with an optional setup step.
212 fio_path: location of fio executable
213 fio_job: location of fio job file
214 success: Definition of test success
215 fio_pre_job: fio job for preconditioning
216 fio_pre_success: Definition of test success for fio precon job
217 output_format: normal (default), json, jsonplus, or terse
220 self.fio_job = fio_job
221 self.fio_pre_job = fio_pre_job
222 self.fio_pre_success = fio_pre_success if fio_pre_success else success
223 self.output_format = output_format
224 self.precon_failed = False
225 self.json_data = None
226 self.fio_output = f"{os.path.basename(self.fio_job)}.output"
229 f"--output-format={self.output_format}",
230 f"--output={self.fio_output}",
233 FioExeTest.__init__(self, fio_path, self.fio_args, success)
235 def setup(self, artifact_root, testnum):
236 """Setup instance variables for fio job test."""
238 super().setup(artifact_root, testnum)
240 self.command_file = os.path.join(
242 f"{os.path.basename(self.fio_job)}.command")
243 self.stdout_file = os.path.join(
245 f"{os.path.basename(self.fio_job)}.stdout")
246 self.stderr_file = os.path.join(
248 f"{os.path.basename(self.fio_job)}.stderr")
249 self.exitcode_file = os.path.join(
251 f"{os.path.basename(self.fio_job)}.exitcode")
253 def run_pre_job(self):
254 """Run fio job precondition step."""
256 precon = FioJobTest(self.exe_path, self.fio_pre_job,
257 self.fio_pre_success,
258 output_format=self.output_format)
259 precon.setup(self.artifact_root, self.testnum)
261 precon.check_result()
262 self.precon_failed = not precon.passed
263 self.failure_reason = precon.failure_reason
266 """Run fio job test."""
271 if not self.precon_failed:
274 logging.debug("Test %d: precondition step failed", self.testnum)
277 def get_file(cls, filename):
278 """Safely read a file."""
283 with open(filename, "r") as output_file:
284 file_data = output_file.read()
288 return file_data, success
290 def get_file_fail(self, filename):
291 """Safely read a file and fail the test upon error."""
295 with open(filename, "r") as output_file:
296 file_data = output_file.read()
298 self.failure_reason += f" unable to read file {filename}"
303 def check_result(self):
304 """Check fio job results."""
306 if self.precon_failed:
308 self.failure_reason = f"{self.failure_reason} precondition step failed,"
311 super().check_result()
316 if 'json' not in self.output_format:
319 file_data = self.get_file_fail(os.path.join(self.test_dir, self.fio_output))
324 # Sometimes fio informational messages are included at the top of the
325 # JSON output, especially under Windows. Try to decode output as JSON
326 # data, skipping everything until the first {
328 lines = file_data.splitlines()
329 file_data = '\n'.join(lines[lines.index("{"):])
331 self.json_data = json.loads(file_data)
332 except json.JSONDecodeError:
333 self.failure_reason = f"{self.failure_reason} unable to decode JSON data,"
337 class FioJobTest_t0005(FioJobTest):
338 """Test consists of fio test job t0005
339 Confirm that read['io_kbytes'] == write['io_kbytes'] == 102400"""
341 def check_result(self):
342 super().check_result()
347 if self.json_data['jobs'][0]['read']['io_kbytes'] != 102400:
348 self.failure_reason = f"{self.failure_reason} bytes read mismatch,"
350 if self.json_data['jobs'][0]['write']['io_kbytes'] != 102400:
351 self.failure_reason = f"{self.failure_reason} bytes written mismatch,"
355 class FioJobTest_t0006(FioJobTest):
356 """Test consists of fio test job t0006
357 Confirm that read['io_kbytes'] ~ 2*write['io_kbytes']"""
359 def check_result(self):
360 super().check_result()
365 ratio = self.json_data['jobs'][0]['read']['io_kbytes'] \
366 / self.json_data['jobs'][0]['write']['io_kbytes']
367 logging.debug("Test %d: ratio: %f", self.testnum, ratio)
368 if ratio < 1.99 or ratio > 2.01:
369 self.failure_reason = f"{self.failure_reason} read/write ratio mismatch,"
373 class FioJobTest_t0007(FioJobTest):
374 """Test consists of fio test job t0007
375 Confirm that read['io_kbytes'] = 87040"""
377 def check_result(self):
378 super().check_result()
383 if self.json_data['jobs'][0]['read']['io_kbytes'] != 87040:
384 self.failure_reason = f"{self.failure_reason} bytes read mismatch,"
388 class FioJobTest_t0008(FioJobTest):
389 """Test consists of fio test job t0008
390 Confirm that read['io_kbytes'] = 32768 and that
391 write['io_kbytes'] ~ 16384
393 This is a 50/50 seq read/write workload. Since fio flips a coin to
394 determine whether to issue a read or a write, total bytes written will not
395 be exactly 16384K. But total bytes read will be exactly 32768K because
396 reads will include the initial phase as well as the verify phase where all
397 the blocks originally written will be read."""
399 def check_result(self):
400 super().check_result()
405 ratio = self.json_data['jobs'][0]['write']['io_kbytes'] / 16384
406 logging.debug("Test %d: ratio: %f", self.testnum, ratio)
408 if ratio < 0.97 or ratio > 1.03:
409 self.failure_reason = f"{self.failure_reason} bytes written mismatch,"
411 if self.json_data['jobs'][0]['read']['io_kbytes'] != 32768:
412 self.failure_reason = f"{self.failure_reason} bytes read mismatch,"
416 class FioJobTest_t0009(FioJobTest):
417 """Test consists of fio test job t0009
418 Confirm that runtime >= 60s"""
420 def check_result(self):
421 super().check_result()
426 logging.debug('Test %d: elapsed: %d', self.testnum, self.json_data['jobs'][0]['elapsed'])
428 if self.json_data['jobs'][0]['elapsed'] < 60:
429 self.failure_reason = f"{self.failure_reason} elapsed time mismatch,"
433 class FioJobTest_t0012(FioJobTest):
434 """Test consists of fio test job t0012
435 Confirm ratios of job iops are 1:5:10
436 job1,job2,job3 respectively"""
438 def check_result(self):
439 super().check_result()
445 for i in range(1, 4):
446 filename = os.path.join(self.test_dir, "{0}_iops.{1}.log".format(os.path.basename(
448 file_data = self.get_file_fail(filename)
452 iops_files.append(file_data.splitlines())
454 # there are 9 samples for job1 and job2, 4 samples for job3
459 iops1 = iops1 + float(iops_files[0][i].split(',')[1])
460 iops2 = iops2 + float(iops_files[1][i].split(',')[1])
461 iops3 = iops3 + float(iops_files[2][i].split(',')[1])
465 logging.debug("sample {0}: job1 iops={1} job2 iops={2} job3 iops={3} " \
466 "job3/job2={4:.3f} job3/job1={5:.3f}".format(i, iops1, iops2, iops3, ratio1,
469 # test job1 and job2 succeeded to recalibrate
470 if ratio1 < 1 or ratio1 > 3 or ratio2 < 7 or ratio2 > 13:
471 self.failure_reason += " iops ratio mismatch iops1={0} iops2={1} iops3={2} " \
472 "expected r1~2 r2~10 got r1={3:.3f} r2={4:.3f},".format(iops1, iops2, iops3,
478 class FioJobTest_t0014(FioJobTest):
479 """Test consists of fio test job t0014
480 Confirm that job1_iops / job2_iops ~ 1:2 for entire duration
481 and that job1_iops / job3_iops ~ 1:3 for first half of duration.
483 The test is about making sure the flow feature can
484 re-calibrate the activity dynamically"""
486 def check_result(self):
487 super().check_result()
493 for i in range(1, 4):
494 filename = os.path.join(self.test_dir, "{0}_iops.{1}.log".format(os.path.basename(
496 file_data = self.get_file_fail(filename)
500 iops_files.append(file_data.splitlines())
502 # there are 9 samples for job1 and job2, 4 samples for job3
508 iops3 = iops3 + float(iops_files[2][i].split(',')[1])
510 ratio1 = iops1 / iops2
511 ratio2 = iops1 / iops3
514 if ratio1 < 0.43 or ratio1 > 0.57 or ratio2 < 0.21 or ratio2 > 0.45:
515 self.failure_reason += " iops ratio mismatch iops1={0} iops2={1} iops3={2} " \
516 "expected r1~0.5 r2~0.33 got r1={3:.3f} r2={4:.3f},".format(
517 iops1, iops2, iops3, ratio1, ratio2)
520 iops1 = iops1 + float(iops_files[0][i].split(',')[1])
521 iops2 = iops2 + float(iops_files[1][i].split(',')[1])
525 logging.debug("sample {0}: job1 iops={1} job2 iops={2} job3 iops={3} " \
526 "job1/job2={4:.3f} job1/job3={5:.3f}".format(i, iops1, iops2, iops3,
529 # test job1 and job2 succeeded to recalibrate
530 if ratio1 < 0.43 or ratio1 > 0.57:
531 self.failure_reason += " iops ratio mismatch iops1={0} iops2={1} expected ratio~0.5 " \
532 "got ratio={2:.3f},".format(iops1, iops2, ratio1)
537 class FioJobTest_t0015(FioJobTest):
538 """Test consists of fio test jobs t0015 and t0016
539 Confirm that mean(slat) + mean(clat) = mean(tlat)"""
541 def check_result(self):
542 super().check_result()
547 slat = self.json_data['jobs'][0]['read']['slat_ns']['mean']
548 clat = self.json_data['jobs'][0]['read']['clat_ns']['mean']
549 tlat = self.json_data['jobs'][0]['read']['lat_ns']['mean']
550 logging.debug('Test %d: slat %f, clat %f, tlat %f', self.testnum, slat, clat, tlat)
552 if abs(slat + clat - tlat) > 1:
553 self.failure_reason = "{0} slat {1} + clat {2} = {3} != tlat {4},".format(
554 self.failure_reason, slat, clat, slat+clat, tlat)
558 class FioJobTest_t0019(FioJobTest):
559 """Test consists of fio test job t0019
560 Confirm that all offsets were touched sequentially"""
562 def check_result(self):
563 super().check_result()
565 bw_log_filename = os.path.join(self.test_dir, "test_bw.log")
566 file_data = self.get_file_fail(bw_log_filename)
570 log_lines = file_data.split('\n')
573 for line in log_lines:
574 if len(line.strip()) == 0:
576 cur = int(line.split(',')[4])
577 if cur - prev != 4096:
579 self.failure_reason = f"offsets {prev}, {cur} not sequential"
585 self.failure_reason = f"unexpected last offset {cur}"
588 class FioJobTest_t0020(FioJobTest):
589 """Test consists of fio test jobs t0020 and t0021
590 Confirm that almost all offsets were touched non-sequentially"""
592 def check_result(self):
593 super().check_result()
595 bw_log_filename = os.path.join(self.test_dir, "test_bw.log")
596 file_data = self.get_file_fail(bw_log_filename)
600 log_lines = file_data.split('\n')
604 prev = int(log_lines[0].split(',')[4])
605 for line in log_lines[1:]:
606 offsets.append(prev/4096)
607 if len(line.strip()) == 0:
609 cur = int(line.split(',')[4])
612 if len(offsets) != 256:
614 self.failure_reason += f" number of offsets is {len(offsets)} instead of 256"
619 self.failure_reason += f" missing offset {i * 4096}"
621 (_, p) = runstest_1samp(list(offsets))
624 self.failure_reason += f" runs test failed with p = {p}"
627 class FioJobTest_t0022(FioJobTest):
628 """Test consists of fio test job t0022"""
630 def check_result(self):
631 super().check_result()
633 bw_log_filename = os.path.join(self.test_dir, "test_bw.log")
634 file_data = self.get_file_fail(bw_log_filename)
638 log_lines = file_data.split('\n')
645 prev = int(log_lines[0].split(',')[4])
646 for line in log_lines[1:]:
648 if len(line.strip()) == 0:
650 cur = int(line.split(',')[4])
655 # 10 is an arbitrary threshold
658 self.failure_reason = f"too many ({seq_count}) consecutive offsets"
660 if len(offsets) == filesize/bs:
662 self.failure_reason += " no duplicate offsets found with norandommap=1"
665 class FioJobTest_t0023(FioJobTest):
666 """Test consists of fio test job t0023 randtrimwrite test."""
668 def check_trimwrite(self, filename):
669 """Make sure that trims are followed by writes of the same size at the same offset."""
671 bw_log_filename = os.path.join(self.test_dir, filename)
672 file_data = self.get_file_fail(bw_log_filename)
676 log_lines = file_data.split('\n')
679 for line in log_lines:
680 if len(line.strip()) == 0:
682 vals = line.split(',')
685 offset = int(vals[4])
689 self.failure_reason += " {0}: write not preceeded by trim: {1}".format(
690 bw_log_filename, line)
693 if ddir != 1: # pylint: disable=no-else-break
695 self.failure_reason += " {0}: trim not preceeded by write: {1}".format(
696 bw_log_filename, line)
701 self.failure_reason += " {0}: block size does not match: {1}".format(
702 bw_log_filename, line)
705 if prev_offset != offset:
707 self.failure_reason += " {0}: offset does not match: {1}".format(
708 bw_log_filename, line)
716 def check_all_offsets(self, filename, sectorsize, filesize):
717 """Make sure all offsets were touched."""
719 file_data = self.get_file_fail(os.path.join(self.test_dir, filename))
723 log_lines = file_data.split('\n')
727 for line in log_lines:
728 if len(line.strip()) == 0:
730 vals = line.split(',')
732 offset = int(vals[4])
733 if offset % sectorsize != 0:
735 self.failure_reason += " {0}: offset {1} not a multiple of sector size {2}".format(
736 filename, offset, sectorsize)
738 if bs % sectorsize != 0:
740 self.failure_reason += " {0}: block size {1} not a multiple of sector size " \
741 "{2}".format(filename, bs, sectorsize)
743 for i in range(int(bs/sectorsize)):
744 offsets.add(offset/sectorsize + i)
746 if len(offsets) != filesize/sectorsize:
748 self.failure_reason += " {0}: only {1} offsets touched; expected {2}".format(
749 filename, len(offsets), filesize/sectorsize)
751 logging.debug("%s: %d sectors touched", filename, len(offsets))
754 def check_result(self):
755 super().check_result()
759 self.check_trimwrite("basic_bw.log")
760 self.check_trimwrite("bs_bw.log")
761 self.check_trimwrite("bsrange_bw.log")
762 self.check_trimwrite("bssplit_bw.log")
763 self.check_trimwrite("basic_no_rm_bw.log")
764 self.check_trimwrite("bs_no_rm_bw.log")
765 self.check_trimwrite("bsrange_no_rm_bw.log")
766 self.check_trimwrite("bssplit_no_rm_bw.log")
768 self.check_all_offsets("basic_bw.log", 4096, filesize)
769 self.check_all_offsets("bs_bw.log", 8192, filesize)
770 self.check_all_offsets("bsrange_bw.log", 512, filesize)
771 self.check_all_offsets("bssplit_bw.log", 512, filesize)
774 class FioJobTest_t0024(FioJobTest_t0023):
775 """Test consists of fio test job t0024 trimwrite test."""
777 def check_result(self):
778 # call FioJobTest_t0023's parent to skip checks done by t0023
779 super(FioJobTest_t0023, self).check_result()
783 self.check_trimwrite("basic_bw.log")
784 self.check_trimwrite("bs_bw.log")
785 self.check_trimwrite("bsrange_bw.log")
786 self.check_trimwrite("bssplit_bw.log")
788 self.check_all_offsets("basic_bw.log", 4096, filesize)
789 self.check_all_offsets("bs_bw.log", 8192, filesize)
790 self.check_all_offsets("bsrange_bw.log", 512, filesize)
791 self.check_all_offsets("bssplit_bw.log", 512, filesize)
794 class FioJobTest_t0025(FioJobTest):
795 """Test experimental verify read backs written data pattern."""
796 def check_result(self):
797 super().check_result()
802 if self.json_data['jobs'][0]['read']['io_kbytes'] != 128:
805 class FioJobTest_t0027(FioJobTest):
806 def setup(self, *args, **kws):
807 super().setup(*args, **kws)
808 self.pattern_file = os.path.join(self.test_dir, "t0027.pattern")
809 self.output_file = os.path.join(self.test_dir, "t0027file")
810 self.pattern = os.urandom(16 << 10)
811 with open(self.pattern_file, "wb") as f:
812 f.write(self.pattern)
814 def check_result(self):
815 super().check_result()
820 with open(self.output_file, "rb") as f:
823 if data != self.pattern:
826 class FioJobTest_iops_rate(FioJobTest):
827 """Test consists of fio test job t0011
828 Confirm that job0 iops == 1000
829 and that job1_iops / job0_iops ~ 8
830 With two runs of fio-3.16 I observed a ratio of 8.3"""
832 def check_result(self):
833 super().check_result()
838 iops1 = self.json_data['jobs'][0]['read']['iops']
839 logging.debug("Test %d: iops1: %f", self.testnum, iops1)
840 iops2 = self.json_data['jobs'][1]['read']['iops']
841 logging.debug("Test %d: iops2: %f", self.testnum, iops2)
842 ratio = iops2 / iops1
843 logging.debug("Test %d: ratio: %f", self.testnum, ratio)
845 if iops1 < 950 or iops1 > 1050:
846 self.failure_reason = f"{self.failure_reason} iops value mismatch,"
849 if ratio < 6 or ratio > 10:
850 self.failure_reason = f"{self.failure_reason} iops ratio mismatch,"
854 class Requirements():
855 """Requirements consists of multiple run environment characteristics.
856 These are to determine if a particular test can be run"""
870 def __init__(self, fio_root, args):
871 Requirements._not_macos = platform.system() != "Darwin"
872 Requirements._not_windows = platform.system() != "Windows"
873 Requirements._linux = platform.system() == "Linux"
875 if Requirements._linux:
876 config_file = os.path.join(fio_root, "config-host.h")
877 contents, success = FioJobTest.get_file(config_file)
879 print(f"Unable to open {config_file} to check requirements")
880 Requirements._zbd = True
882 Requirements._zbd = "CONFIG_HAS_BLKZONED" in contents
883 Requirements._libaio = "CONFIG_LIBAIO" in contents
885 contents, success = FioJobTest.get_file("/proc/kallsyms")
887 print("Unable to open '/proc/kallsyms' to probe for io_uring support")
889 Requirements._io_uring = "io_uring_setup" in contents
891 Requirements._root = os.geteuid() == 0
892 if Requirements._zbd and Requirements._root:
894 subprocess.run(["modprobe", "null_blk"],
895 stdout=subprocess.PIPE,
896 stderr=subprocess.PIPE)
897 if os.path.exists("/sys/module/null_blk/parameters/zoned"):
898 Requirements._zoned_nullb = True
902 if platform.system() == "Windows":
903 utest_exe = "unittest.exe"
905 utest_exe = "unittest"
906 unittest_path = os.path.join(fio_root, "unittests", utest_exe)
907 Requirements._unittests = os.path.exists(unittest_path)
909 Requirements._cpucount4 = multiprocessing.cpu_count() >= 4
910 Requirements._nvmecdev = args.nvmecdev
915 Requirements.io_uring,
918 Requirements.zoned_nullb,
919 Requirements.not_macos,
920 Requirements.not_windows,
921 Requirements.unittests,
922 Requirements.cpucount4,
923 Requirements.nvmecdev,
927 logging.debug("Requirements: Requirement '%s' met? %s", desc, value)
931 """Are we running on Linux?"""
932 return Requirements._linux, "Linux required"
936 """Is libaio available?"""
937 return Requirements._libaio, "libaio required"
941 """Is io_uring available?"""
942 return Requirements._io_uring, "io_uring required"
946 """Is ZBD support available?"""
947 return Requirements._zbd, "Zoned block device support required"
951 """Are we running as root?"""
952 return Requirements._root, "root required"
955 def zoned_nullb(cls):
956 """Are zoned null block devices available?"""
957 return Requirements._zoned_nullb, "Zoned null block device support required"
961 """Are we running on a platform other than macOS?"""
962 return Requirements._not_macos, "platform other than macOS required"
965 def not_windows(cls):
966 """Are we running on a platform other than Windws?"""
967 return Requirements._not_windows, "platform other than Windows required"
971 """Were unittests built?"""
972 return Requirements._unittests, "Unittests support required"
976 """Do we have at least 4 CPUs?"""
977 return Requirements._cpucount4, "4+ CPUs required"
981 """Do we have an NVMe character device to test?"""
982 return Requirements._nvmecdev, "NVMe character device test target required"
987 'stderr_empty': True,
991 'zero_return': False,
992 'stderr_empty': False,
997 'stderr_empty': False,
1003 'test_class': FioJobTest,
1004 'job': 't0001-52c58027.fio',
1005 'success': SUCCESS_DEFAULT,
1007 'pre_success': None,
1012 'test_class': FioJobTest,
1013 'job': 't0002-13af05ae-post.fio',
1014 'success': SUCCESS_DEFAULT,
1015 'pre_job': 't0002-13af05ae-pre.fio',
1016 'pre_success': None,
1017 'requirements': [Requirements.linux, Requirements.libaio],
1021 'test_class': FioJobTest,
1022 'job': 't0003-0ae2c6e1-post.fio',
1023 'success': SUCCESS_NONZERO,
1024 'pre_job': 't0003-0ae2c6e1-pre.fio',
1025 'pre_success': SUCCESS_DEFAULT,
1026 'requirements': [Requirements.linux, Requirements.libaio],
1030 'test_class': FioJobTest,
1031 'job': 't0004-8a99fdf6.fio',
1032 'success': SUCCESS_DEFAULT,
1034 'pre_success': None,
1035 'requirements': [Requirements.linux, Requirements.libaio],
1039 'test_class': FioJobTest_t0005,
1040 'job': 't0005-f7078f7b.fio',
1041 'success': SUCCESS_DEFAULT,
1043 'pre_success': None,
1044 'output_format': 'json',
1045 'requirements': [Requirements.not_windows],
1049 'test_class': FioJobTest_t0006,
1050 'job': 't0006-82af2a7c.fio',
1051 'success': SUCCESS_DEFAULT,
1053 'pre_success': None,
1054 'output_format': 'json',
1055 'requirements': [Requirements.linux, Requirements.libaio],
1059 'test_class': FioJobTest_t0007,
1060 'job': 't0007-37cf9e3c.fio',
1061 'success': SUCCESS_DEFAULT,
1063 'pre_success': None,
1064 'output_format': 'json',
1069 'test_class': FioJobTest_t0008,
1070 'job': 't0008-ae2fafc8.fio',
1071 'success': SUCCESS_DEFAULT,
1073 'pre_success': None,
1074 'output_format': 'json',
1079 'test_class': FioJobTest_t0009,
1080 'job': 't0009-f8b0bd10.fio',
1081 'success': SUCCESS_DEFAULT,
1083 'pre_success': None,
1084 'output_format': 'json',
1085 'requirements': [Requirements.not_macos,
1086 Requirements.cpucount4],
1087 # mac os does not support CPU affinity
1091 'test_class': FioJobTest,
1092 'job': 't0010-b7aae4ba.fio',
1093 'success': SUCCESS_DEFAULT,
1095 'pre_success': None,
1100 'test_class': FioJobTest_iops_rate,
1101 'job': 't0011-5d2788d5.fio',
1102 'success': SUCCESS_DEFAULT,
1104 'pre_success': None,
1105 'output_format': 'json',
1110 'test_class': FioJobTest_t0012,
1112 'success': SUCCESS_DEFAULT,
1114 'pre_success': None,
1115 'output_format': 'json',
1120 'test_class': FioJobTest,
1122 'success': SUCCESS_DEFAULT,
1124 'pre_success': None,
1125 'output_format': 'json',
1130 'test_class': FioJobTest_t0014,
1132 'success': SUCCESS_DEFAULT,
1134 'pre_success': None,
1135 'output_format': 'json',
1140 'test_class': FioJobTest_t0015,
1141 'job': 't0015-e78980ff.fio',
1142 'success': SUCCESS_DEFAULT,
1144 'pre_success': None,
1145 'output_format': 'json',
1146 'requirements': [Requirements.linux, Requirements.libaio],
1150 'test_class': FioJobTest_t0015,
1151 'job': 't0016-d54ae22.fio',
1152 'success': SUCCESS_DEFAULT,
1154 'pre_success': None,
1155 'output_format': 'json',
1160 'test_class': FioJobTest_t0015,
1162 'success': SUCCESS_DEFAULT,
1164 'pre_success': None,
1165 'output_format': 'json',
1166 'requirements': [Requirements.not_windows],
1170 'test_class': FioJobTest,
1172 'success': SUCCESS_DEFAULT,
1174 'pre_success': None,
1175 'requirements': [Requirements.linux, Requirements.io_uring],
1179 'test_class': FioJobTest_t0019,
1181 'success': SUCCESS_DEFAULT,
1183 'pre_success': None,
1188 'test_class': FioJobTest_t0020,
1190 'success': SUCCESS_DEFAULT,
1192 'pre_success': None,
1197 'test_class': FioJobTest_t0020,
1199 'success': SUCCESS_DEFAULT,
1201 'pre_success': None,
1206 'test_class': FioJobTest_t0022,
1208 'success': SUCCESS_DEFAULT,
1210 'pre_success': None,
1215 'test_class': FioJobTest_t0023,
1217 'success': SUCCESS_DEFAULT,
1219 'pre_success': None,
1224 'test_class': FioJobTest_t0024,
1226 'success': SUCCESS_DEFAULT,
1228 'pre_success': None,
1233 'test_class': FioJobTest_t0025,
1235 'success': SUCCESS_DEFAULT,
1237 'pre_success': None,
1238 'output_format': 'json',
1243 'test_class': FioJobTest,
1245 'success': SUCCESS_DEFAULT,
1247 'pre_success': None,
1248 'requirements': [Requirements.not_windows],
1252 'test_class': FioJobTest_t0027,
1254 'success': SUCCESS_DEFAULT,
1256 'pre_success': None,
1261 'test_class': FioJobTest,
1262 'job': 't0028-c6cade16.fio',
1263 'success': SUCCESS_DEFAULT,
1265 'pre_success': None,
1270 'test_class': FioExeTest,
1273 'success': SUCCESS_DEFAULT,
1278 'test_class': FioExeTest,
1281 'success': SUCCESS_DEFAULT,
1286 'test_class': FioExeTest,
1287 'exe': 't/lfsr-test',
1288 'parameters': ['0xFFFFFF', '0', '0', 'verify'],
1289 'success': SUCCESS_STDERR,
1294 'test_class': FioExeTest,
1295 'exe': 't/readonly.py',
1296 'parameters': ['-f', '{fio_path}'],
1297 'success': SUCCESS_DEFAULT,
1302 'test_class': FioExeTest,
1303 'exe': 't/steadystate_tests.py',
1304 'parameters': ['{fio_path}'],
1305 'success': SUCCESS_DEFAULT,
1310 'test_class': FioExeTest,
1313 'success': SUCCESS_STDERR,
1318 'test_class': FioExeTest,
1319 'exe': 't/strided.py',
1320 'parameters': ['{fio_path}'],
1321 'success': SUCCESS_DEFAULT,
1326 'test_class': FioExeTest,
1327 'exe': 't/zbd/run-tests-against-nullb',
1328 'parameters': ['-s', '1'],
1329 'success': SUCCESS_DEFAULT,
1330 'requirements': [Requirements.linux, Requirements.zbd,
1335 'test_class': FioExeTest,
1336 'exe': 't/zbd/run-tests-against-nullb',
1337 'parameters': ['-s', '2'],
1338 'success': SUCCESS_DEFAULT,
1339 'requirements': [Requirements.linux, Requirements.zbd,
1340 Requirements.root, Requirements.zoned_nullb],
1344 'test_class': FioExeTest,
1345 'exe': 'unittests/unittest',
1347 'success': SUCCESS_DEFAULT,
1348 'requirements': [Requirements.unittests],
1352 'test_class': FioExeTest,
1353 'exe': 't/latency_percentiles.py',
1354 'parameters': ['-f', '{fio_path}'],
1355 'success': SUCCESS_DEFAULT,
1360 'test_class': FioExeTest,
1361 'exe': 't/jsonplus2csv_test.py',
1362 'parameters': ['-f', '{fio_path}'],
1363 'success': SUCCESS_DEFAULT,
1368 'test_class': FioExeTest,
1369 'exe': 't/log_compression.py',
1370 'parameters': ['-f', '{fio_path}'],
1371 'success': SUCCESS_DEFAULT,
1376 'test_class': FioExeTest,
1377 'exe': 't/random_seed.py',
1378 'parameters': ['-f', '{fio_path}'],
1379 'success': SUCCESS_DEFAULT,
1384 'test_class': FioExeTest,
1385 'exe': 't/nvmept.py',
1386 'parameters': ['-f', '{fio_path}', '--dut', '{nvmecdev}'],
1387 'success': SUCCESS_DEFAULT,
1388 'requirements': [Requirements.linux, Requirements.nvmecdev],
1394 """Parse command-line arguments."""
1396 parser = argparse.ArgumentParser()
1397 parser.add_argument('-r', '--fio-root',
1398 help='fio root path')
1399 parser.add_argument('-f', '--fio',
1400 help='path to fio executable (e.g., ./fio)')
1401 parser.add_argument('-a', '--artifact-root',
1402 help='artifact root directory')
1403 parser.add_argument('-s', '--skip', nargs='+', type=int,
1404 help='list of test(s) to skip')
1405 parser.add_argument('-o', '--run-only', nargs='+', type=int,
1406 help='list of test(s) to run, skipping all others')
1407 parser.add_argument('-d', '--debug', action='store_true',
1408 help='provide debug output')
1409 parser.add_argument('-k', '--skip-req', action='store_true',
1410 help='skip requirements checking')
1411 parser.add_argument('-p', '--pass-through', action='append',
1412 help='pass-through an argument to an executable test')
1413 parser.add_argument('--nvmecdev', action='store', default=None,
1414 help='NVMe character device for **DESTRUCTIVE** testing (e.g., /dev/ng0n1)')
1415 args = parser.parse_args()
1425 logging.basicConfig(level=logging.DEBUG)
1427 logging.basicConfig(level=logging.INFO)
1430 if args.pass_through:
1431 for arg in args.pass_through:
1433 print(f"Invalid --pass-through argument '{arg}'")
1434 print("Syntax for --pass-through is TESTNUMBER:ARGUMENT")
1436 split = arg.split(":", 1)
1437 pass_through[int(split[0])] = split[1]
1438 logging.debug("Pass-through arguments: %s", pass_through)
1441 fio_root = args.fio_root
1443 fio_root = str(Path(__file__).absolute().parent.parent)
1444 print(f"fio root is {fio_root}")
1449 if platform.system() == "Windows":
1453 fio_path = os.path.join(fio_root, fio_exe)
1454 print(f"fio path is {fio_path}")
1455 if not shutil.which(fio_path):
1456 print("Warning: fio executable not found")
1458 artifact_root = args.artifact_root if args.artifact_root else \
1459 f"fio-test-{time.strftime('%Y%m%d-%H%M%S')}"
1460 os.mkdir(artifact_root)
1461 print(f"Artifact directory is {artifact_root}")
1463 if not args.skip_req:
1464 req = Requirements(fio_root, args)
1470 for config in TEST_LIST:
1471 if (args.skip and config['test_id'] in args.skip) or \
1472 (args.run_only and config['test_id'] not in args.run_only):
1473 skipped = skipped + 1
1474 print(f"Test {config['test_id']} SKIPPED (User request)")
1477 if issubclass(config['test_class'], FioJobTest):
1478 if config['pre_job']:
1479 fio_pre_job = os.path.join(fio_root, 't', 'jobs',
1483 if config['pre_success']:
1484 fio_pre_success = config['pre_success']
1486 fio_pre_success = None
1487 if 'output_format' in config:
1488 output_format = config['output_format']
1490 output_format = 'normal'
1491 test = config['test_class'](
1493 os.path.join(fio_root, 't', 'jobs', config['job']),
1495 fio_pre_job=fio_pre_job,
1496 fio_pre_success=fio_pre_success,
1497 output_format=output_format)
1498 desc = config['job']
1499 elif issubclass(config['test_class'], FioExeTest):
1500 exe_path = os.path.join(fio_root, config['exe'])
1501 if config['parameters']:
1502 parameters = [p.format(fio_path=fio_path, nvmecdev=args.nvmecdev)
1503 for p in config['parameters']]
1506 if Path(exe_path).suffix == '.py' and platform.system() == "Windows":
1507 parameters.insert(0, exe_path)
1508 exe_path = "python.exe"
1509 if config['test_id'] in pass_through:
1510 parameters += pass_through[config['test_id']].split()
1511 test = config['test_class'](exe_path, parameters,
1513 desc = config['exe']
1515 print(f"Test {config['test_id']} FAILED: unable to process test config")
1519 if not args.skip_req:
1521 for req in config['requirements']:
1522 reqs_met, reason = req()
1523 logging.debug("Test %d: Requirement '%s' met? %s", config['test_id'], reason,
1528 print(f"Test {config['test_id']} SKIPPED ({reason}) {desc}")
1529 skipped = skipped + 1
1533 test.setup(artifact_root, config['test_id'])
1536 except KeyboardInterrupt:
1538 except Exception as e:
1540 test.failure_reason += str(e)
1541 logging.debug("Test %d exception:\n%s\n", config['test_id'], traceback.format_exc())
1546 result = f"FAILED: {test.failure_reason}"
1548 contents, _ = FioJobTest.get_file(test.stderr_file)
1549 logging.debug("Test %d: stderr:\n%s", config['test_id'], contents)
1550 contents, _ = FioJobTest.get_file(test.stdout_file)
1551 logging.debug("Test %d: stdout:\n%s", config['test_id'], contents)
1552 print(f"Test {config['test_id']} {result} {desc}")
1554 print(f"{passed} test(s) passed, {failed} failed, {skipped} skipped")
1559 if __name__ == '__main__':