2 # SPDX-License-Identifier: GPL-2.0-only
4 # Copyright (c) 2019 Western Digital Corporation or its affiliates.
9 # Automate running of fio tests
12 # python3 run-fio-tests.py [-r fio-root] [-f fio-path] [-a artifact-root]
13 # [--skip # # #...] [--run-only # # #...]
17 # # git clone git://git.kernel.dk/fio.git
20 # # python3 t/run-fio-tests.py
24 # - Python 3.5 (subprocess.run)
25 # - Linux (libaio ioengine, zbd tests, etc)
26 # - The artifact directory must be on a file system that accepts 512-byte IO
27 # (t0002, t0003, t0004).
28 # - The artifact directory needs to be on an SSD. Otherwise tests that carry
29 # out file-based IO will trigger a timeout (t0006).
31 # - SciPy (steadystate_tests.py)
32 # - libzbc (zbd tests)
33 # - root privileges (zbd test)
34 # - kernel 4.19 or later for zoned null block devices (zbd tests)
35 # - CUnit support (unittests)
40 # TODO run multiple tests simultaneously
41 # TODO Add sgunmap tests (requires SAS SSD)
54 import multiprocessing
55 from pathlib import Path
59 """Base for all fio tests."""
61 def __init__(self, exe_path, parameters, success):
62 self.exe_path = exe_path
63 self.parameters = parameters
64 self.success = success
66 self.artifact_root = None
70 self.failure_reason = ''
71 self.command_file = None
72 self.stdout_file = None
73 self.stderr_file = None
74 self.exitcode_file = None
76 def setup(self, artifact_root, testnum):
77 """Setup instance variables for test."""
79 self.artifact_root = artifact_root
80 self.testnum = testnum
81 self.test_dir = os.path.join(artifact_root, "{:04d}".format(testnum))
82 if not os.path.exists(self.test_dir):
83 os.mkdir(self.test_dir)
85 self.command_file = os.path.join(
87 "{0}.command".format(os.path.basename(self.exe_path)))
88 self.stdout_file = os.path.join(
90 "{0}.stdout".format(os.path.basename(self.exe_path)))
91 self.stderr_file = os.path.join(
93 "{0}.stderr".format(os.path.basename(self.exe_path)))
94 self.exitcode_file = os.path.join(
96 "{0}.exitcode".format(os.path.basename(self.exe_path)))
101 raise NotImplementedError()
103 def check_result(self):
104 """Check test results."""
106 raise NotImplementedError()
109 class FioExeTest(FioTest):
110 """Test consists of an executable binary or script"""
112 def __init__(self, exe_path, parameters, success):
113 """Construct a FioExeTest which is a FioTest consisting of an
114 executable binary or script.
116 exe_path: location of executable binary or script
117 parameters: list of parameters for executable
118 success: Definition of test success
121 FioTest.__init__(self, exe_path, parameters, success)
124 """Execute the binary or script described by this instance."""
126 command = [self.exe_path] + self.parameters
127 command_file = open(self.command_file, "w+")
128 command_file.write("%s\n" % command)
131 stdout_file = open(self.stdout_file, "w+")
132 stderr_file = open(self.stderr_file, "w+")
133 exitcode_file = open(self.exitcode_file, "w+")
136 # Avoid using subprocess.run() here because when a timeout occurs,
137 # fio will be stopped with SIGKILL. This does not give fio a
138 # chance to clean up and means that child processes may continue
139 # running and submitting IO.
140 proc = subprocess.Popen(command,
144 universal_newlines=True)
145 proc.communicate(timeout=self.success['timeout'])
146 exitcode_file.write('{0}\n'.format(proc.returncode))
147 logging.debug("Test %d: return code: %d", self.testnum, proc.returncode)
148 self.output['proc'] = proc
149 except subprocess.TimeoutExpired:
153 self.output['failure'] = 'timeout'
159 self.output['failure'] = 'exception'
160 self.output['exc_info'] = sys.exc_info()
164 exitcode_file.close()
166 def check_result(self):
167 """Check results of test run."""
169 if 'proc' not in self.output:
170 if self.output['failure'] == 'timeout':
171 self.failure_reason = "{0} timeout,".format(self.failure_reason)
173 assert self.output['failure'] == 'exception'
174 self.failure_reason = '{0} exception: {1}, {2}'.format(
175 self.failure_reason, self.output['exc_info'][0],
176 self.output['exc_info'][1])
181 if 'zero_return' in self.success:
182 if self.success['zero_return']:
183 if self.output['proc'].returncode != 0:
185 self.failure_reason = "{0} non-zero return code,".format(self.failure_reason)
187 if self.output['proc'].returncode == 0:
188 self.failure_reason = "{0} zero return code,".format(self.failure_reason)
191 stderr_size = os.path.getsize(self.stderr_file)
192 if 'stderr_empty' in self.success:
193 if self.success['stderr_empty']:
195 self.failure_reason = "{0} stderr not empty,".format(self.failure_reason)
199 self.failure_reason = "{0} stderr empty,".format(self.failure_reason)
203 class FioJobTest(FioExeTest):
204 """Test consists of a fio job"""
206 def __init__(self, fio_path, fio_job, success, fio_pre_job=None,
207 fio_pre_success=None, output_format="normal"):
208 """Construct a FioJobTest which is a FioExeTest consisting of a
209 single fio job file with an optional setup step.
211 fio_path: location of fio executable
212 fio_job: location of fio job file
213 success: Definition of test success
214 fio_pre_job: fio job for preconditioning
215 fio_pre_success: Definition of test success for fio precon job
216 output_format: normal (default), json, jsonplus, or terse
219 self.fio_job = fio_job
220 self.fio_pre_job = fio_pre_job
221 self.fio_pre_success = fio_pre_success if fio_pre_success else success
222 self.output_format = output_format
223 self.precon_failed = False
224 self.json_data = None
225 self.fio_output = "{0}.output".format(os.path.basename(self.fio_job))
228 "--output-format={0}".format(self.output_format),
229 "--output={0}".format(self.fio_output),
232 FioExeTest.__init__(self, fio_path, self.fio_args, success)
234 def setup(self, artifact_root, testnum):
235 """Setup instance variables for fio job test."""
237 super(FioJobTest, self).setup(artifact_root, testnum)
239 self.command_file = os.path.join(
241 "{0}.command".format(os.path.basename(self.fio_job)))
242 self.stdout_file = os.path.join(
244 "{0}.stdout".format(os.path.basename(self.fio_job)))
245 self.stderr_file = os.path.join(
247 "{0}.stderr".format(os.path.basename(self.fio_job)))
248 self.exitcode_file = os.path.join(
250 "{0}.exitcode".format(os.path.basename(self.fio_job)))
252 def run_pre_job(self):
253 """Run fio job precondition step."""
255 precon = FioJobTest(self.exe_path, self.fio_pre_job,
256 self.fio_pre_success,
257 output_format=self.output_format)
258 precon.setup(self.artifact_root, self.testnum)
260 precon.check_result()
261 self.precon_failed = not precon.passed
262 self.failure_reason = precon.failure_reason
265 """Run fio job test."""
270 if not self.precon_failed:
271 super(FioJobTest, self).run()
273 logging.debug("Test %d: precondition step failed", self.testnum)
276 def get_file(cls, filename):
277 """Safely read a file."""
282 with open(filename, "r") as output_file:
283 file_data = output_file.read()
287 return file_data, success
289 def get_file_fail(self, filename):
290 """Safely read a file and fail the test upon error."""
294 with open(filename, "r") as output_file:
295 file_data = output_file.read()
297 self.failure_reason += " unable to read file {0}".format(filename)
302 def check_result(self):
303 """Check fio job results."""
305 if self.precon_failed:
307 self.failure_reason = "{0} precondition step failed,".format(self.failure_reason)
310 super(FioJobTest, self).check_result()
315 if 'json' not in self.output_format:
318 file_data = self.get_file_fail(os.path.join(self.test_dir, self.fio_output))
323 # Sometimes fio informational messages are included at the top of the
324 # JSON output, especially under Windows. Try to decode output as JSON
325 # data, skipping everything until the first {
327 lines = file_data.splitlines()
328 file_data = '\n'.join(lines[lines.index("{"):])
330 self.json_data = json.loads(file_data)
331 except json.JSONDecodeError:
332 self.failure_reason = "{0} unable to decode JSON data,".format(self.failure_reason)
336 class FioJobTest_t0005(FioJobTest):
337 """Test consists of fio test job t0005
338 Confirm that read['io_kbytes'] == write['io_kbytes'] == 102400"""
340 def check_result(self):
341 super(FioJobTest_t0005, self).check_result()
346 if self.json_data['jobs'][0]['read']['io_kbytes'] != 102400:
347 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
349 if self.json_data['jobs'][0]['write']['io_kbytes'] != 102400:
350 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
354 class FioJobTest_t0006(FioJobTest):
355 """Test consists of fio test job t0006
356 Confirm that read['io_kbytes'] ~ 2*write['io_kbytes']"""
358 def check_result(self):
359 super(FioJobTest_t0006, self).check_result()
364 ratio = self.json_data['jobs'][0]['read']['io_kbytes'] \
365 / self.json_data['jobs'][0]['write']['io_kbytes']
366 logging.debug("Test %d: ratio: %f", self.testnum, ratio)
367 if ratio < 1.99 or ratio > 2.01:
368 self.failure_reason = "{0} read/write ratio mismatch,".format(self.failure_reason)
372 class FioJobTest_t0007(FioJobTest):
373 """Test consists of fio test job t0007
374 Confirm that read['io_kbytes'] = 87040"""
376 def check_result(self):
377 super(FioJobTest_t0007, self).check_result()
382 if self.json_data['jobs'][0]['read']['io_kbytes'] != 87040:
383 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
387 class FioJobTest_t0008(FioJobTest):
388 """Test consists of fio test job t0008
389 Confirm that read['io_kbytes'] = 32768 and that
390 write['io_kbytes'] ~ 16384
392 This is a 50/50 seq read/write workload. Since fio flips a coin to
393 determine whether to issue a read or a write, total bytes written will not
394 be exactly 16384K. But total bytes read will be exactly 32768K because
395 reads will include the initial phase as well as the verify phase where all
396 the blocks originally written will be read."""
398 def check_result(self):
399 super(FioJobTest_t0008, self).check_result()
404 ratio = self.json_data['jobs'][0]['write']['io_kbytes'] / 16384
405 logging.debug("Test %d: ratio: %f", self.testnum, ratio)
407 if ratio < 0.97 or ratio > 1.03:
408 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
410 if self.json_data['jobs'][0]['read']['io_kbytes'] != 32768:
411 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
415 class FioJobTest_t0009(FioJobTest):
416 """Test consists of fio test job t0009
417 Confirm that runtime >= 60s"""
419 def check_result(self):
420 super(FioJobTest_t0009, self).check_result()
425 logging.debug('Test %d: elapsed: %d', self.testnum, self.json_data['jobs'][0]['elapsed'])
427 if self.json_data['jobs'][0]['elapsed'] < 60:
428 self.failure_reason = "{0} elapsed time mismatch,".format(self.failure_reason)
432 class FioJobTest_t0012(FioJobTest):
433 """Test consists of fio test job t0012
434 Confirm ratios of job iops are 1:5:10
435 job1,job2,job3 respectively"""
437 def check_result(self):
438 super(FioJobTest_t0012, self).check_result()
444 for i in range(1, 4):
445 filename = os.path.join(self.test_dir, "{0}_iops.{1}.log".format(os.path.basename(
447 file_data = self.get_file_fail(filename)
451 iops_files.append(file_data.splitlines())
453 # there are 9 samples for job1 and job2, 4 samples for job3
458 iops1 = iops1 + float(iops_files[0][i].split(',')[1])
459 iops2 = iops2 + float(iops_files[1][i].split(',')[1])
460 iops3 = iops3 + float(iops_files[2][i].split(',')[1])
464 logging.debug("sample {0}: job1 iops={1} job2 iops={2} job3 iops={3} " \
465 "job3/job2={4:.3f} job3/job1={5:.3f}".format(i, iops1, iops2, iops3, ratio1,
468 # test job1 and job2 succeeded to recalibrate
469 if ratio1 < 1 or ratio1 > 3 or ratio2 < 7 or ratio2 > 13:
470 self.failure_reason += " iops ratio mismatch iops1={0} iops2={1} iops3={2} " \
471 "expected r1~2 r2~10 got r1={3:.3f} r2={4:.3f},".format(iops1, iops2, iops3,
477 class FioJobTest_t0014(FioJobTest):
478 """Test consists of fio test job t0014
479 Confirm that job1_iops / job2_iops ~ 1:2 for entire duration
480 and that job1_iops / job3_iops ~ 1:3 for first half of duration.
482 The test is about making sure the flow feature can
483 re-calibrate the activity dynamically"""
485 def check_result(self):
486 super(FioJobTest_t0014, self).check_result()
492 for i in range(1, 4):
493 filename = os.path.join(self.test_dir, "{0}_iops.{1}.log".format(os.path.basename(
495 file_data = self.get_file_fail(filename)
499 iops_files.append(file_data.splitlines())
501 # there are 9 samples for job1 and job2, 4 samples for job3
507 iops3 = iops3 + float(iops_files[2][i].split(',')[1])
509 ratio1 = iops1 / iops2
510 ratio2 = iops1 / iops3
513 if ratio1 < 0.43 or ratio1 > 0.57 or ratio2 < 0.21 or ratio2 > 0.45:
514 self.failure_reason += " iops ratio mismatch iops1={0} iops2={1} iops3={2} " \
515 "expected r1~0.5 r2~0.33 got r1={3:.3f} r2={4:.3f},".format(
516 iops1, iops2, iops3, ratio1, ratio2)
519 iops1 = iops1 + float(iops_files[0][i].split(',')[1])
520 iops2 = iops2 + float(iops_files[1][i].split(',')[1])
524 logging.debug("sample {0}: job1 iops={1} job2 iops={2} job3 iops={3} " \
525 "job1/job2={4:.3f} job1/job3={5:.3f}".format(i, iops1, iops2, iops3,
528 # test job1 and job2 succeeded to recalibrate
529 if ratio1 < 0.43 or ratio1 > 0.57:
530 self.failure_reason += " iops ratio mismatch iops1={0} iops2={1} expected ratio~0.5 " \
531 "got ratio={2:.3f},".format(iops1, iops2, ratio1)
536 class FioJobTest_t0015(FioJobTest):
537 """Test consists of fio test jobs t0015 and t0016
538 Confirm that mean(slat) + mean(clat) = mean(tlat)"""
540 def check_result(self):
541 super(FioJobTest_t0015, self).check_result()
546 slat = self.json_data['jobs'][0]['read']['slat_ns']['mean']
547 clat = self.json_data['jobs'][0]['read']['clat_ns']['mean']
548 tlat = self.json_data['jobs'][0]['read']['lat_ns']['mean']
549 logging.debug('Test %d: slat %f, clat %f, tlat %f', self.testnum, slat, clat, tlat)
551 if abs(slat + clat - tlat) > 1:
552 self.failure_reason = "{0} slat {1} + clat {2} = {3} != tlat {4},".format(
553 self.failure_reason, slat, clat, slat+clat, tlat)
557 class FioJobTest_t0019(FioJobTest):
558 """Test consists of fio test job t0019
559 Confirm that all offsets were touched sequentially"""
561 def check_result(self):
562 super(FioJobTest_t0019, self).check_result()
564 bw_log_filename = os.path.join(self.test_dir, "test_bw.log")
565 file_data = self.get_file_fail(bw_log_filename)
569 log_lines = file_data.split('\n')
572 for line in log_lines:
573 if len(line.strip()) == 0:
575 cur = int(line.split(',')[4])
576 if cur - prev != 4096:
578 self.failure_reason = "offsets {0}, {1} not sequential".format(prev, cur)
584 self.failure_reason = "unexpected last offset {0}".format(cur)
587 class FioJobTest_t0020(FioJobTest):
588 """Test consists of fio test jobs t0020 and t0021
589 Confirm that almost all offsets were touched non-sequentially"""
591 def check_result(self):
592 super(FioJobTest_t0020, self).check_result()
594 bw_log_filename = os.path.join(self.test_dir, "test_bw.log")
595 file_data = self.get_file_fail(bw_log_filename)
599 log_lines = file_data.split('\n')
604 prev = int(log_lines[0].split(',')[4])
605 for line in log_lines[1:]:
606 offsets.add(prev/4096)
607 if len(line.strip()) == 0:
609 cur = int(line.split(',')[4])
610 if cur - prev == 4096:
614 # 10 is an arbitrary threshold
617 self.failure_reason = "too many ({0}) consecutive offsets".format(seq_count)
619 if len(offsets) != 256:
621 self.failure_reason += " number of offsets is {0} instead of 256".format(len(offsets))
626 self.failure_reason += " missing offset {0}".format(i*4096)
629 class FioJobTest_t0022(FioJobTest):
630 """Test consists of fio test job t0022"""
632 def check_result(self):
633 super(FioJobTest_t0022, self).check_result()
635 bw_log_filename = os.path.join(self.test_dir, "test_bw.log")
636 file_data = self.get_file_fail(bw_log_filename)
640 log_lines = file_data.split('\n')
647 prev = int(log_lines[0].split(',')[4])
648 for line in log_lines[1:]:
650 if len(line.strip()) == 0:
652 cur = int(line.split(',')[4])
657 # 10 is an arbitrary threshold
660 self.failure_reason = "too many ({0}) consecutive offsets".format(seq_count)
662 if len(offsets) == filesize/bs:
664 self.failure_reason += " no duplicate offsets found with norandommap=1"
667 class FioJobTest_t0023(FioJobTest):
668 """Test consists of fio test job t0023 randtrimwrite test."""
670 def check_trimwrite(self, filename):
671 """Make sure that trims are followed by writes of the same size at the same offset."""
673 bw_log_filename = os.path.join(self.test_dir, filename)
674 file_data = self.get_file_fail(bw_log_filename)
678 log_lines = file_data.split('\n')
681 for line in log_lines:
682 if len(line.strip()) == 0:
684 vals = line.split(',')
687 offset = int(vals[4])
691 self.failure_reason += " {0}: write not preceeded by trim: {1}".format(
692 bw_log_filename, line)
697 self.failure_reason += " {0}: trim not preceeded by write: {1}".format(
698 bw_log_filename, line)
703 self.failure_reason += " {0}: block size does not match: {1}".format(
704 bw_log_filename, line)
706 if prev_offset != offset:
708 self.failure_reason += " {0}: offset does not match: {1}".format(
709 bw_log_filename, line)
716 def check_all_offsets(self, filename, sectorsize, filesize):
717 """Make sure all offsets were touched."""
719 file_data = self.get_file_fail(os.path.join(self.test_dir, filename))
723 log_lines = file_data.split('\n')
727 for line in log_lines:
728 if len(line.strip()) == 0:
730 vals = line.split(',')
732 offset = int(vals[4])
733 if offset % sectorsize != 0:
735 self.failure_reason += " {0}: offset {1} not a multiple of sector size {2}".format(
736 filename, offset, sectorsize)
738 if bs % sectorsize != 0:
740 self.failure_reason += " {0}: block size {1} not a multiple of sector size " \
741 "{2}".format(filename, bs, sectorsize)
743 for i in range(int(bs/sectorsize)):
744 offsets.add(offset/sectorsize + i)
746 if len(offsets) != filesize/sectorsize:
748 self.failure_reason += " {0}: only {1} offsets touched; expected {2}".format(
749 filename, len(offsets), filesize/sectorsize)
751 logging.debug("%s: %d sectors touched", filename, len(offsets))
754 def check_result(self):
755 super(FioJobTest_t0023, self).check_result()
759 self.check_trimwrite("basic_bw.log")
760 self.check_trimwrite("bs_bw.log")
761 self.check_trimwrite("bsrange_bw.log")
762 self.check_trimwrite("bssplit_bw.log")
763 self.check_trimwrite("basic_no_rm_bw.log")
764 self.check_trimwrite("bs_no_rm_bw.log")
765 self.check_trimwrite("bsrange_no_rm_bw.log")
766 self.check_trimwrite("bssplit_no_rm_bw.log")
768 self.check_all_offsets("basic_bw.log", 4096, filesize)
769 self.check_all_offsets("bs_bw.log", 8192, filesize)
770 self.check_all_offsets("bsrange_bw.log", 512, filesize)
771 self.check_all_offsets("bssplit_bw.log", 512, filesize)
774 class FioJobTest_t0024(FioJobTest_t0023):
775 """Test consists of fio test job t0024 trimwrite test."""
777 def check_result(self):
778 # call FioJobTest_t0023's parent to skip checks done by t0023
779 super(FioJobTest_t0023, self).check_result()
783 self.check_trimwrite("basic_bw.log")
784 self.check_trimwrite("bs_bw.log")
785 self.check_trimwrite("bsrange_bw.log")
786 self.check_trimwrite("bssplit_bw.log")
788 self.check_all_offsets("basic_bw.log", 4096, filesize)
789 self.check_all_offsets("bs_bw.log", 8192, filesize)
790 self.check_all_offsets("bsrange_bw.log", 512, filesize)
791 self.check_all_offsets("bssplit_bw.log", 512, filesize)
794 class FioJobTest_t0025(FioJobTest):
795 """Test experimental verify read backs written data pattern."""
796 def check_result(self):
797 super(FioJobTest_t0025, self).check_result()
802 if self.json_data['jobs'][0]['read']['io_kbytes'] != 128:
805 class FioJobTest_t0027(FioJobTest):
806 def setup(self, *args, **kws):
807 super(FioJobTest_t0027, self).setup(*args, **kws)
808 self.pattern_file = os.path.join(self.test_dir, "t0027.pattern")
809 self.output_file = os.path.join(self.test_dir, "t0027file")
810 self.pattern = os.urandom(16 << 10)
811 with open(self.pattern_file, "wb") as f:
812 f.write(self.pattern)
814 def check_result(self):
815 super(FioJobTest_t0027, self).check_result()
820 with open(self.output_file, "rb") as f:
823 if data != self.pattern:
826 class FioJobTest_iops_rate(FioJobTest):
827 """Test consists of fio test job t0009
828 Confirm that job0 iops == 1000
829 and that job1_iops / job0_iops ~ 8
830 With two runs of fio-3.16 I observed a ratio of 8.3"""
832 def check_result(self):
833 super(FioJobTest_iops_rate, self).check_result()
838 iops1 = self.json_data['jobs'][0]['read']['iops']
839 logging.debug("Test %d: iops1: %f", self.testnum, iops1)
840 iops2 = self.json_data['jobs'][1]['read']['iops']
841 logging.debug("Test %d: iops2: %f", self.testnum, iops2)
842 ratio = iops2 / iops1
843 logging.debug("Test %d: ratio: %f", self.testnum, ratio)
845 if iops1 < 950 or iops1 > 1050:
846 self.failure_reason = "{0} iops value mismatch,".format(self.failure_reason)
849 if ratio < 6 or ratio > 10:
850 self.failure_reason = "{0} iops ratio mismatch,".format(self.failure_reason)
854 class Requirements():
855 """Requirements consists of multiple run environment characteristics.
856 These are to determine if a particular test can be run"""
869 def __init__(self, fio_root):
870 Requirements._not_macos = platform.system() != "Darwin"
871 Requirements._not_windows = platform.system() != "Windows"
872 Requirements._linux = platform.system() == "Linux"
874 if Requirements._linux:
875 config_file = os.path.join(fio_root, "config-host.h")
876 contents, success = FioJobTest.get_file(config_file)
878 print("Unable to open {0} to check requirements".format(config_file))
879 Requirements._zbd = True
881 Requirements._zbd = "CONFIG_HAS_BLKZONED" in contents
882 Requirements._libaio = "CONFIG_LIBAIO" in contents
884 contents, success = FioJobTest.get_file("/proc/kallsyms")
886 print("Unable to open '/proc/kallsyms' to probe for io_uring support")
888 Requirements._io_uring = "io_uring_setup" in contents
890 Requirements._root = (os.geteuid() == 0)
891 if Requirements._zbd and Requirements._root:
893 subprocess.run(["modprobe", "null_blk"],
894 stdout=subprocess.PIPE,
895 stderr=subprocess.PIPE)
896 if os.path.exists("/sys/module/null_blk/parameters/zoned"):
897 Requirements._zoned_nullb = True
901 if platform.system() == "Windows":
902 utest_exe = "unittest.exe"
904 utest_exe = "unittest"
905 unittest_path = os.path.join(fio_root, "unittests", utest_exe)
906 Requirements._unittests = os.path.exists(unittest_path)
908 Requirements._cpucount4 = multiprocessing.cpu_count() >= 4
910 req_list = [Requirements.linux,
912 Requirements.io_uring,
915 Requirements.zoned_nullb,
916 Requirements.not_macos,
917 Requirements.not_windows,
918 Requirements.unittests,
919 Requirements.cpucount4]
922 logging.debug("Requirements: Requirement '%s' met? %s", desc, value)
926 """Are we running on Linux?"""
927 return Requirements._linux, "Linux required"
931 """Is libaio available?"""
932 return Requirements._libaio, "libaio required"
936 """Is io_uring available?"""
937 return Requirements._io_uring, "io_uring required"
941 """Is ZBD support available?"""
942 return Requirements._zbd, "Zoned block device support required"
946 """Are we running as root?"""
947 return Requirements._root, "root required"
950 def zoned_nullb(cls):
951 """Are zoned null block devices available?"""
952 return Requirements._zoned_nullb, "Zoned null block device support required"
956 """Are we running on a platform other than macOS?"""
957 return Requirements._not_macos, "platform other than macOS required"
960 def not_windows(cls):
961 """Are we running on a platform other than Windws?"""
962 return Requirements._not_windows, "platform other than Windows required"
966 """Were unittests built?"""
967 return Requirements._unittests, "Unittests support required"
971 """Do we have at least 4 CPUs?"""
972 return Requirements._cpucount4, "4+ CPUs required"
977 'stderr_empty': True,
981 'zero_return': False,
982 'stderr_empty': False,
987 'stderr_empty': False,
993 'test_class': FioJobTest,
994 'job': 't0001-52c58027.fio',
995 'success': SUCCESS_DEFAULT,
1002 'test_class': FioJobTest,
1003 'job': 't0002-13af05ae-post.fio',
1004 'success': SUCCESS_DEFAULT,
1005 'pre_job': 't0002-13af05ae-pre.fio',
1006 'pre_success': None,
1007 'requirements': [Requirements.linux, Requirements.libaio],
1011 'test_class': FioJobTest,
1012 'job': 't0003-0ae2c6e1-post.fio',
1013 'success': SUCCESS_NONZERO,
1014 'pre_job': 't0003-0ae2c6e1-pre.fio',
1015 'pre_success': SUCCESS_DEFAULT,
1016 'requirements': [Requirements.linux, Requirements.libaio],
1020 'test_class': FioJobTest,
1021 'job': 't0004-8a99fdf6.fio',
1022 'success': SUCCESS_DEFAULT,
1024 'pre_success': None,
1025 'requirements': [Requirements.linux, Requirements.libaio],
1029 'test_class': FioJobTest_t0005,
1030 'job': 't0005-f7078f7b.fio',
1031 'success': SUCCESS_DEFAULT,
1033 'pre_success': None,
1034 'output_format': 'json',
1035 'requirements': [Requirements.not_windows],
1039 'test_class': FioJobTest_t0006,
1040 'job': 't0006-82af2a7c.fio',
1041 'success': SUCCESS_DEFAULT,
1043 'pre_success': None,
1044 'output_format': 'json',
1045 'requirements': [Requirements.linux, Requirements.libaio],
1049 'test_class': FioJobTest_t0007,
1050 'job': 't0007-37cf9e3c.fio',
1051 'success': SUCCESS_DEFAULT,
1053 'pre_success': None,
1054 'output_format': 'json',
1059 'test_class': FioJobTest_t0008,
1060 'job': 't0008-ae2fafc8.fio',
1061 'success': SUCCESS_DEFAULT,
1063 'pre_success': None,
1064 'output_format': 'json',
1069 'test_class': FioJobTest_t0009,
1070 'job': 't0009-f8b0bd10.fio',
1071 'success': SUCCESS_DEFAULT,
1073 'pre_success': None,
1074 'output_format': 'json',
1075 'requirements': [Requirements.not_macos,
1076 Requirements.cpucount4],
1077 # mac os does not support CPU affinity
1081 'test_class': FioJobTest,
1082 'job': 't0010-b7aae4ba.fio',
1083 'success': SUCCESS_DEFAULT,
1085 'pre_success': None,
1090 'test_class': FioJobTest_iops_rate,
1091 'job': 't0011-5d2788d5.fio',
1092 'success': SUCCESS_DEFAULT,
1094 'pre_success': None,
1095 'output_format': 'json',
1100 'test_class': FioJobTest_t0012,
1102 'success': SUCCESS_DEFAULT,
1104 'pre_success': None,
1105 'output_format': 'json',
1110 'test_class': FioJobTest,
1112 'success': SUCCESS_DEFAULT,
1114 'pre_success': None,
1115 'output_format': 'json',
1120 'test_class': FioJobTest_t0014,
1122 'success': SUCCESS_DEFAULT,
1124 'pre_success': None,
1125 'output_format': 'json',
1130 'test_class': FioJobTest_t0015,
1131 'job': 't0015-e78980ff.fio',
1132 'success': SUCCESS_DEFAULT,
1134 'pre_success': None,
1135 'output_format': 'json',
1136 'requirements': [Requirements.linux, Requirements.libaio],
1140 'test_class': FioJobTest_t0015,
1141 'job': 't0016-d54ae22.fio',
1142 'success': SUCCESS_DEFAULT,
1144 'pre_success': None,
1145 'output_format': 'json',
1150 'test_class': FioJobTest_t0015,
1152 'success': SUCCESS_DEFAULT,
1154 'pre_success': None,
1155 'output_format': 'json',
1156 'requirements': [Requirements.not_windows],
1160 'test_class': FioJobTest,
1162 'success': SUCCESS_DEFAULT,
1164 'pre_success': None,
1165 'requirements': [Requirements.linux, Requirements.io_uring],
1169 'test_class': FioJobTest_t0019,
1171 'success': SUCCESS_DEFAULT,
1173 'pre_success': None,
1178 'test_class': FioJobTest_t0020,
1180 'success': SUCCESS_DEFAULT,
1182 'pre_success': None,
1187 'test_class': FioJobTest_t0020,
1189 'success': SUCCESS_DEFAULT,
1191 'pre_success': None,
1196 'test_class': FioJobTest_t0022,
1198 'success': SUCCESS_DEFAULT,
1200 'pre_success': None,
1205 'test_class': FioJobTest_t0023,
1207 'success': SUCCESS_DEFAULT,
1209 'pre_success': None,
1214 'test_class': FioJobTest_t0024,
1216 'success': SUCCESS_DEFAULT,
1218 'pre_success': None,
1223 'test_class': FioJobTest_t0025,
1225 'success': SUCCESS_DEFAULT,
1227 'pre_success': None,
1228 'output_format': 'json',
1233 'test_class': FioJobTest,
1235 'success': SUCCESS_DEFAULT,
1237 'pre_success': None,
1238 'requirements': [Requirements.not_windows],
1242 'test_class': FioJobTest_t0027,
1244 'success': SUCCESS_DEFAULT,
1246 'pre_success': None,
1251 'test_class': FioJobTest,
1252 'job': 't0028-c6cade16.fio',
1253 'success': SUCCESS_DEFAULT,
1255 'pre_success': None,
1260 'test_class': FioExeTest,
1263 'success': SUCCESS_DEFAULT,
1268 'test_class': FioExeTest,
1271 'success': SUCCESS_DEFAULT,
1276 'test_class': FioExeTest,
1277 'exe': 't/lfsr-test',
1278 'parameters': ['0xFFFFFF', '0', '0', 'verify'],
1279 'success': SUCCESS_STDERR,
1284 'test_class': FioExeTest,
1285 'exe': 't/readonly.py',
1286 'parameters': ['-f', '{fio_path}'],
1287 'success': SUCCESS_DEFAULT,
1292 'test_class': FioExeTest,
1293 'exe': 't/steadystate_tests.py',
1294 'parameters': ['{fio_path}'],
1295 'success': SUCCESS_DEFAULT,
1300 'test_class': FioExeTest,
1303 'success': SUCCESS_STDERR,
1308 'test_class': FioExeTest,
1309 'exe': 't/strided.py',
1310 'parameters': ['{fio_path}'],
1311 'success': SUCCESS_DEFAULT,
1316 'test_class': FioExeTest,
1317 'exe': 't/zbd/run-tests-against-nullb',
1318 'parameters': ['-s', '1'],
1319 'success': SUCCESS_DEFAULT,
1320 'requirements': [Requirements.linux, Requirements.zbd,
1325 'test_class': FioExeTest,
1326 'exe': 't/zbd/run-tests-against-nullb',
1327 'parameters': ['-s', '2'],
1328 'success': SUCCESS_DEFAULT,
1329 'requirements': [Requirements.linux, Requirements.zbd,
1330 Requirements.root, Requirements.zoned_nullb],
1334 'test_class': FioExeTest,
1335 'exe': 'unittests/unittest',
1337 'success': SUCCESS_DEFAULT,
1338 'requirements': [Requirements.unittests],
1342 'test_class': FioExeTest,
1343 'exe': 't/latency_percentiles.py',
1344 'parameters': ['-f', '{fio_path}'],
1345 'success': SUCCESS_DEFAULT,
1350 'test_class': FioExeTest,
1351 'exe': 't/jsonplus2csv_test.py',
1352 'parameters': ['-f', '{fio_path}'],
1353 'success': SUCCESS_DEFAULT,
1358 'test_class': FioExeTest,
1359 'exe': 't/log_compression.py',
1360 'parameters': ['-f', '{fio_path}'],
1361 'success': SUCCESS_DEFAULT,
1368 """Parse command-line arguments."""
1370 parser = argparse.ArgumentParser()
1371 parser.add_argument('-r', '--fio-root',
1372 help='fio root path')
1373 parser.add_argument('-f', '--fio',
1374 help='path to fio executable (e.g., ./fio)')
1375 parser.add_argument('-a', '--artifact-root',
1376 help='artifact root directory')
1377 parser.add_argument('-s', '--skip', nargs='+', type=int,
1378 help='list of test(s) to skip')
1379 parser.add_argument('-o', '--run-only', nargs='+', type=int,
1380 help='list of test(s) to run, skipping all others')
1381 parser.add_argument('-d', '--debug', action='store_true',
1382 help='provide debug output')
1383 parser.add_argument('-k', '--skip-req', action='store_true',
1384 help='skip requirements checking')
1385 parser.add_argument('-p', '--pass-through', action='append',
1386 help='pass-through an argument to an executable test')
1387 args = parser.parse_args()
1397 logging.basicConfig(level=logging.DEBUG)
1399 logging.basicConfig(level=logging.INFO)
1402 if args.pass_through:
1403 for arg in args.pass_through:
1405 print("Invalid --pass-through argument '%s'" % arg)
1406 print("Syntax for --pass-through is TESTNUMBER:ARGUMENT")
1408 split = arg.split(":", 1)
1409 pass_through[int(split[0])] = split[1]
1410 logging.debug("Pass-through arguments: %s", pass_through)
1413 fio_root = args.fio_root
1415 fio_root = str(Path(__file__).absolute().parent.parent)
1416 print("fio root is %s" % fio_root)
1421 if platform.system() == "Windows":
1425 fio_path = os.path.join(fio_root, fio_exe)
1426 print("fio path is %s" % fio_path)
1427 if not shutil.which(fio_path):
1428 print("Warning: fio executable not found")
1430 artifact_root = args.artifact_root if args.artifact_root else \
1431 "fio-test-{0}".format(time.strftime("%Y%m%d-%H%M%S"))
1432 os.mkdir(artifact_root)
1433 print("Artifact directory is %s" % artifact_root)
1435 if not args.skip_req:
1436 req = Requirements(fio_root)
1442 for config in TEST_LIST:
1443 if (args.skip and config['test_id'] in args.skip) or \
1444 (args.run_only and config['test_id'] not in args.run_only):
1445 skipped = skipped + 1
1446 print("Test {0} SKIPPED (User request)".format(config['test_id']))
1449 if issubclass(config['test_class'], FioJobTest):
1450 if config['pre_job']:
1451 fio_pre_job = os.path.join(fio_root, 't', 'jobs',
1455 if config['pre_success']:
1456 fio_pre_success = config['pre_success']
1458 fio_pre_success = None
1459 if 'output_format' in config:
1460 output_format = config['output_format']
1462 output_format = 'normal'
1463 test = config['test_class'](
1465 os.path.join(fio_root, 't', 'jobs', config['job']),
1467 fio_pre_job=fio_pre_job,
1468 fio_pre_success=fio_pre_success,
1469 output_format=output_format)
1470 desc = config['job']
1471 elif issubclass(config['test_class'], FioExeTest):
1472 exe_path = os.path.join(fio_root, config['exe'])
1473 if config['parameters']:
1474 parameters = [p.format(fio_path=fio_path) for p in config['parameters']]
1477 if Path(exe_path).suffix == '.py' and platform.system() == "Windows":
1478 parameters.insert(0, exe_path)
1479 exe_path = "python.exe"
1480 if config['test_id'] in pass_through:
1481 parameters += pass_through[config['test_id']].split()
1482 test = config['test_class'](exe_path, parameters,
1484 desc = config['exe']
1486 print("Test {0} FAILED: unable to process test config".format(config['test_id']))
1490 if not args.skip_req:
1492 for req in config['requirements']:
1493 reqs_met, reason = req()
1494 logging.debug("Test %d: Requirement '%s' met? %s", config['test_id'], reason,
1499 print("Test {0} SKIPPED ({1}) {2}".format(config['test_id'], reason, desc))
1500 skipped = skipped + 1
1504 test.setup(artifact_root, config['test_id'])
1507 except KeyboardInterrupt:
1509 except Exception as e:
1511 test.failure_reason += str(e)
1512 logging.debug("Test %d exception:\n%s\n", config['test_id'], traceback.format_exc())
1517 result = "FAILED: {0}".format(test.failure_reason)
1519 contents, _ = FioJobTest.get_file(test.stderr_file)
1520 logging.debug("Test %d: stderr:\n%s", config['test_id'], contents)
1521 contents, _ = FioJobTest.get_file(test.stdout_file)
1522 logging.debug("Test %d: stdout:\n%s", config['test_id'], contents)
1523 print("Test {0} {1} {2}".format(config['test_id'], result, desc))
1525 print("{0} test(s) passed, {1} failed, {2} skipped".format(passed, failed, skipped))
1530 if __name__ == '__main__':