2 # SPDX-License-Identifier: GPL-2.0-only
4 # Copyright (c) 2019 Western Digital Corporation or its affiliates.
9 # Automate running of fio tests
12 # python3 run-fio-tests.py [-r fio-root] [-f fio-path] [-a artifact-root]
13 # [--skip # # #...] [--run-only # # #...]
17 # # git clone git://git.kernel.dk/fio.git
20 # # python3 t/run-fio-tests.py
24 # - Python 3.5 (subprocess.run)
25 # - Linux (libaio ioengine, zbd tests, etc)
26 # - The artifact directory must be on a file system that accepts 512-byte IO
27 # (t0002, t0003, t0004).
28 # - The artifact directory needs to be on an SSD. Otherwise tests that carry
29 # out file-based IO will trigger a timeout (t0006).
31 # - SciPy (steadystate_tests.py)
32 # - libzbc (zbd tests)
33 # - root privileges (zbd test)
34 # - kernel 4.19 or later for zoned null block devices (zbd tests)
35 # - CUnit support (unittests)
40 # TODO run multiple tests simultaneously
41 # TODO Add sgunmap tests (requires SAS SSD)
54 import multiprocessing
55 from pathlib import Path
56 from statsmodels.sandbox.stats.runs import runstest_1samp
60 """Base for all fio tests."""
62 def __init__(self, exe_path, parameters, success):
63 self.exe_path = exe_path
64 self.parameters = parameters
65 self.success = success
67 self.artifact_root = None
71 self.failure_reason = ''
72 self.command_file = None
73 self.stdout_file = None
74 self.stderr_file = None
75 self.exitcode_file = None
77 def setup(self, artifact_root, testnum):
78 """Setup instance variables for test."""
80 self.artifact_root = artifact_root
81 self.testnum = testnum
82 self.test_dir = os.path.join(artifact_root, "{:04d}".format(testnum))
83 if not os.path.exists(self.test_dir):
84 os.mkdir(self.test_dir)
86 self.command_file = os.path.join(
88 "{0}.command".format(os.path.basename(self.exe_path)))
89 self.stdout_file = os.path.join(
91 "{0}.stdout".format(os.path.basename(self.exe_path)))
92 self.stderr_file = os.path.join(
94 "{0}.stderr".format(os.path.basename(self.exe_path)))
95 self.exitcode_file = os.path.join(
97 "{0}.exitcode".format(os.path.basename(self.exe_path)))
102 raise NotImplementedError()
104 def check_result(self):
105 """Check test results."""
107 raise NotImplementedError()
110 class FioExeTest(FioTest):
111 """Test consists of an executable binary or script"""
113 def __init__(self, exe_path, parameters, success):
114 """Construct a FioExeTest which is a FioTest consisting of an
115 executable binary or script.
117 exe_path: location of executable binary or script
118 parameters: list of parameters for executable
119 success: Definition of test success
122 FioTest.__init__(self, exe_path, parameters, success)
125 """Execute the binary or script described by this instance."""
127 command = [self.exe_path] + self.parameters
128 command_file = open(self.command_file, "w+")
129 command_file.write("%s\n" % command)
132 stdout_file = open(self.stdout_file, "w+")
133 stderr_file = open(self.stderr_file, "w+")
134 exitcode_file = open(self.exitcode_file, "w+")
137 # Avoid using subprocess.run() here because when a timeout occurs,
138 # fio will be stopped with SIGKILL. This does not give fio a
139 # chance to clean up and means that child processes may continue
140 # running and submitting IO.
141 proc = subprocess.Popen(command,
145 universal_newlines=True)
146 proc.communicate(timeout=self.success['timeout'])
147 exitcode_file.write('{0}\n'.format(proc.returncode))
148 logging.debug("Test %d: return code: %d", self.testnum, proc.returncode)
149 self.output['proc'] = proc
150 except subprocess.TimeoutExpired:
154 self.output['failure'] = 'timeout'
160 self.output['failure'] = 'exception'
161 self.output['exc_info'] = sys.exc_info()
165 exitcode_file.close()
167 def check_result(self):
168 """Check results of test run."""
170 if 'proc' not in self.output:
171 if self.output['failure'] == 'timeout':
172 self.failure_reason = "{0} timeout,".format(self.failure_reason)
174 assert self.output['failure'] == 'exception'
175 self.failure_reason = '{0} exception: {1}, {2}'.format(
176 self.failure_reason, self.output['exc_info'][0],
177 self.output['exc_info'][1])
182 if 'zero_return' in self.success:
183 if self.success['zero_return']:
184 if self.output['proc'].returncode != 0:
186 self.failure_reason = "{0} non-zero return code,".format(self.failure_reason)
188 if self.output['proc'].returncode == 0:
189 self.failure_reason = "{0} zero return code,".format(self.failure_reason)
192 stderr_size = os.path.getsize(self.stderr_file)
193 if 'stderr_empty' in self.success:
194 if self.success['stderr_empty']:
196 self.failure_reason = "{0} stderr not empty,".format(self.failure_reason)
200 self.failure_reason = "{0} stderr empty,".format(self.failure_reason)
204 class FioJobTest(FioExeTest):
205 """Test consists of a fio job"""
207 def __init__(self, fio_path, fio_job, success, fio_pre_job=None,
208 fio_pre_success=None, output_format="normal"):
209 """Construct a FioJobTest which is a FioExeTest consisting of a
210 single fio job file with an optional setup step.
212 fio_path: location of fio executable
213 fio_job: location of fio job file
214 success: Definition of test success
215 fio_pre_job: fio job for preconditioning
216 fio_pre_success: Definition of test success for fio precon job
217 output_format: normal (default), json, jsonplus, or terse
220 self.fio_job = fio_job
221 self.fio_pre_job = fio_pre_job
222 self.fio_pre_success = fio_pre_success if fio_pre_success else success
223 self.output_format = output_format
224 self.precon_failed = False
225 self.json_data = None
226 self.fio_output = "{0}.output".format(os.path.basename(self.fio_job))
229 "--output-format={0}".format(self.output_format),
230 "--output={0}".format(self.fio_output),
233 FioExeTest.__init__(self, fio_path, self.fio_args, success)
235 def setup(self, artifact_root, testnum):
236 """Setup instance variables for fio job test."""
238 super(FioJobTest, self).setup(artifact_root, testnum)
240 self.command_file = os.path.join(
242 "{0}.command".format(os.path.basename(self.fio_job)))
243 self.stdout_file = os.path.join(
245 "{0}.stdout".format(os.path.basename(self.fio_job)))
246 self.stderr_file = os.path.join(
248 "{0}.stderr".format(os.path.basename(self.fio_job)))
249 self.exitcode_file = os.path.join(
251 "{0}.exitcode".format(os.path.basename(self.fio_job)))
253 def run_pre_job(self):
254 """Run fio job precondition step."""
256 precon = FioJobTest(self.exe_path, self.fio_pre_job,
257 self.fio_pre_success,
258 output_format=self.output_format)
259 precon.setup(self.artifact_root, self.testnum)
261 precon.check_result()
262 self.precon_failed = not precon.passed
263 self.failure_reason = precon.failure_reason
266 """Run fio job test."""
271 if not self.precon_failed:
272 super(FioJobTest, self).run()
274 logging.debug("Test %d: precondition step failed", self.testnum)
277 def get_file(cls, filename):
278 """Safely read a file."""
283 with open(filename, "r") as output_file:
284 file_data = output_file.read()
288 return file_data, success
290 def get_file_fail(self, filename):
291 """Safely read a file and fail the test upon error."""
295 with open(filename, "r") as output_file:
296 file_data = output_file.read()
298 self.failure_reason += " unable to read file {0}".format(filename)
303 def check_result(self):
304 """Check fio job results."""
306 if self.precon_failed:
308 self.failure_reason = "{0} precondition step failed,".format(self.failure_reason)
311 super(FioJobTest, self).check_result()
316 if 'json' not in self.output_format:
319 file_data = self.get_file_fail(os.path.join(self.test_dir, self.fio_output))
324 # Sometimes fio informational messages are included at the top of the
325 # JSON output, especially under Windows. Try to decode output as JSON
326 # data, skipping everything until the first {
328 lines = file_data.splitlines()
329 file_data = '\n'.join(lines[lines.index("{"):])
331 self.json_data = json.loads(file_data)
332 except json.JSONDecodeError:
333 self.failure_reason = "{0} unable to decode JSON data,".format(self.failure_reason)
337 class FioJobTest_t0005(FioJobTest):
338 """Test consists of fio test job t0005
339 Confirm that read['io_kbytes'] == write['io_kbytes'] == 102400"""
341 def check_result(self):
342 super(FioJobTest_t0005, self).check_result()
347 if self.json_data['jobs'][0]['read']['io_kbytes'] != 102400:
348 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
350 if self.json_data['jobs'][0]['write']['io_kbytes'] != 102400:
351 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
355 class FioJobTest_t0006(FioJobTest):
356 """Test consists of fio test job t0006
357 Confirm that read['io_kbytes'] ~ 2*write['io_kbytes']"""
359 def check_result(self):
360 super(FioJobTest_t0006, self).check_result()
365 ratio = self.json_data['jobs'][0]['read']['io_kbytes'] \
366 / self.json_data['jobs'][0]['write']['io_kbytes']
367 logging.debug("Test %d: ratio: %f", self.testnum, ratio)
368 if ratio < 1.99 or ratio > 2.01:
369 self.failure_reason = "{0} read/write ratio mismatch,".format(self.failure_reason)
373 class FioJobTest_t0007(FioJobTest):
374 """Test consists of fio test job t0007
375 Confirm that read['io_kbytes'] = 87040"""
377 def check_result(self):
378 super(FioJobTest_t0007, self).check_result()
383 if self.json_data['jobs'][0]['read']['io_kbytes'] != 87040:
384 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
388 class FioJobTest_t0008(FioJobTest):
389 """Test consists of fio test job t0008
390 Confirm that read['io_kbytes'] = 32768 and that
391 write['io_kbytes'] ~ 16384
393 This is a 50/50 seq read/write workload. Since fio flips a coin to
394 determine whether to issue a read or a write, total bytes written will not
395 be exactly 16384K. But total bytes read will be exactly 32768K because
396 reads will include the initial phase as well as the verify phase where all
397 the blocks originally written will be read."""
399 def check_result(self):
400 super(FioJobTest_t0008, self).check_result()
405 ratio = self.json_data['jobs'][0]['write']['io_kbytes'] / 16384
406 logging.debug("Test %d: ratio: %f", self.testnum, ratio)
408 if ratio < 0.97 or ratio > 1.03:
409 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
411 if self.json_data['jobs'][0]['read']['io_kbytes'] != 32768:
412 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
416 class FioJobTest_t0009(FioJobTest):
417 """Test consists of fio test job t0009
418 Confirm that runtime >= 60s"""
420 def check_result(self):
421 super(FioJobTest_t0009, self).check_result()
426 logging.debug('Test %d: elapsed: %d', self.testnum, self.json_data['jobs'][0]['elapsed'])
428 if self.json_data['jobs'][0]['elapsed'] < 60:
429 self.failure_reason = "{0} elapsed time mismatch,".format(self.failure_reason)
433 class FioJobTest_t0012(FioJobTest):
434 """Test consists of fio test job t0012
435 Confirm ratios of job iops are 1:5:10
436 job1,job2,job3 respectively"""
438 def check_result(self):
439 super(FioJobTest_t0012, self).check_result()
445 for i in range(1, 4):
446 filename = os.path.join(self.test_dir, "{0}_iops.{1}.log".format(os.path.basename(
448 file_data = self.get_file_fail(filename)
452 iops_files.append(file_data.splitlines())
454 # there are 9 samples for job1 and job2, 4 samples for job3
459 iops1 = iops1 + float(iops_files[0][i].split(',')[1])
460 iops2 = iops2 + float(iops_files[1][i].split(',')[1])
461 iops3 = iops3 + float(iops_files[2][i].split(',')[1])
465 logging.debug("sample {0}: job1 iops={1} job2 iops={2} job3 iops={3} " \
466 "job3/job2={4:.3f} job3/job1={5:.3f}".format(i, iops1, iops2, iops3, ratio1,
469 # test job1 and job2 succeeded to recalibrate
470 if ratio1 < 1 or ratio1 > 3 or ratio2 < 7 or ratio2 > 13:
471 self.failure_reason += " iops ratio mismatch iops1={0} iops2={1} iops3={2} " \
472 "expected r1~2 r2~10 got r1={3:.3f} r2={4:.3f},".format(iops1, iops2, iops3,
478 class FioJobTest_t0014(FioJobTest):
479 """Test consists of fio test job t0014
480 Confirm that job1_iops / job2_iops ~ 1:2 for entire duration
481 and that job1_iops / job3_iops ~ 1:3 for first half of duration.
483 The test is about making sure the flow feature can
484 re-calibrate the activity dynamically"""
486 def check_result(self):
487 super(FioJobTest_t0014, self).check_result()
493 for i in range(1, 4):
494 filename = os.path.join(self.test_dir, "{0}_iops.{1}.log".format(os.path.basename(
496 file_data = self.get_file_fail(filename)
500 iops_files.append(file_data.splitlines())
502 # there are 9 samples for job1 and job2, 4 samples for job3
508 iops3 = iops3 + float(iops_files[2][i].split(',')[1])
510 ratio1 = iops1 / iops2
511 ratio2 = iops1 / iops3
514 if ratio1 < 0.43 or ratio1 > 0.57 or ratio2 < 0.21 or ratio2 > 0.45:
515 self.failure_reason += " iops ratio mismatch iops1={0} iops2={1} iops3={2} " \
516 "expected r1~0.5 r2~0.33 got r1={3:.3f} r2={4:.3f},".format(
517 iops1, iops2, iops3, ratio1, ratio2)
520 iops1 = iops1 + float(iops_files[0][i].split(',')[1])
521 iops2 = iops2 + float(iops_files[1][i].split(',')[1])
525 logging.debug("sample {0}: job1 iops={1} job2 iops={2} job3 iops={3} " \
526 "job1/job2={4:.3f} job1/job3={5:.3f}".format(i, iops1, iops2, iops3,
529 # test job1 and job2 succeeded to recalibrate
530 if ratio1 < 0.43 or ratio1 > 0.57:
531 self.failure_reason += " iops ratio mismatch iops1={0} iops2={1} expected ratio~0.5 " \
532 "got ratio={2:.3f},".format(iops1, iops2, ratio1)
537 class FioJobTest_t0015(FioJobTest):
538 """Test consists of fio test jobs t0015 and t0016
539 Confirm that mean(slat) + mean(clat) = mean(tlat)"""
541 def check_result(self):
542 super(FioJobTest_t0015, self).check_result()
547 slat = self.json_data['jobs'][0]['read']['slat_ns']['mean']
548 clat = self.json_data['jobs'][0]['read']['clat_ns']['mean']
549 tlat = self.json_data['jobs'][0]['read']['lat_ns']['mean']
550 logging.debug('Test %d: slat %f, clat %f, tlat %f', self.testnum, slat, clat, tlat)
552 if abs(slat + clat - tlat) > 1:
553 self.failure_reason = "{0} slat {1} + clat {2} = {3} != tlat {4},".format(
554 self.failure_reason, slat, clat, slat+clat, tlat)
558 class FioJobTest_t0019(FioJobTest):
559 """Test consists of fio test job t0019
560 Confirm that all offsets were touched sequentially"""
562 def check_result(self):
563 super(FioJobTest_t0019, self).check_result()
565 bw_log_filename = os.path.join(self.test_dir, "test_bw.log")
566 file_data = self.get_file_fail(bw_log_filename)
570 log_lines = file_data.split('\n')
573 for line in log_lines:
574 if len(line.strip()) == 0:
576 cur = int(line.split(',')[4])
577 if cur - prev != 4096:
579 self.failure_reason = "offsets {0}, {1} not sequential".format(prev, cur)
585 self.failure_reason = "unexpected last offset {0}".format(cur)
588 class FioJobTest_t0020(FioJobTest):
589 """Test consists of fio test jobs t0020 and t0021
590 Confirm that almost all offsets were touched non-sequentially"""
592 def check_result(self):
593 super(FioJobTest_t0020, self).check_result()
595 bw_log_filename = os.path.join(self.test_dir, "test_bw.log")
596 file_data = self.get_file_fail(bw_log_filename)
600 log_lines = file_data.split('\n')
604 prev = int(log_lines[0].split(',')[4])
605 for line in log_lines[1:]:
606 offsets.append(prev/4096)
607 if len(line.strip()) == 0:
609 cur = int(line.split(',')[4])
612 if len(offsets) != 256:
614 self.failure_reason += " number of offsets is {0} instead of 256".format(len(offsets))
619 self.failure_reason += " missing offset {0}".format(i*4096)
621 (z, p) = runstest_1samp(list(offsets))
624 self.failure_reason += f" runs test failed with p = {p}"
627 class FioJobTest_t0022(FioJobTest):
628 """Test consists of fio test job t0022"""
630 def check_result(self):
631 super(FioJobTest_t0022, self).check_result()
633 bw_log_filename = os.path.join(self.test_dir, "test_bw.log")
634 file_data = self.get_file_fail(bw_log_filename)
638 log_lines = file_data.split('\n')
645 prev = int(log_lines[0].split(',')[4])
646 for line in log_lines[1:]:
648 if len(line.strip()) == 0:
650 cur = int(line.split(',')[4])
655 # 10 is an arbitrary threshold
658 self.failure_reason = "too many ({0}) consecutive offsets".format(seq_count)
660 if len(offsets) == filesize/bs:
662 self.failure_reason += " no duplicate offsets found with norandommap=1"
665 class FioJobTest_t0023(FioJobTest):
666 """Test consists of fio test job t0023 randtrimwrite test."""
668 def check_trimwrite(self, filename):
669 """Make sure that trims are followed by writes of the same size at the same offset."""
671 bw_log_filename = os.path.join(self.test_dir, filename)
672 file_data = self.get_file_fail(bw_log_filename)
676 log_lines = file_data.split('\n')
679 for line in log_lines:
680 if len(line.strip()) == 0:
682 vals = line.split(',')
685 offset = int(vals[4])
689 self.failure_reason += " {0}: write not preceeded by trim: {1}".format(
690 bw_log_filename, line)
695 self.failure_reason += " {0}: trim not preceeded by write: {1}".format(
696 bw_log_filename, line)
701 self.failure_reason += " {0}: block size does not match: {1}".format(
702 bw_log_filename, line)
704 if prev_offset != offset:
706 self.failure_reason += " {0}: offset does not match: {1}".format(
707 bw_log_filename, line)
714 def check_all_offsets(self, filename, sectorsize, filesize):
715 """Make sure all offsets were touched."""
717 file_data = self.get_file_fail(os.path.join(self.test_dir, filename))
721 log_lines = file_data.split('\n')
725 for line in log_lines:
726 if len(line.strip()) == 0:
728 vals = line.split(',')
730 offset = int(vals[4])
731 if offset % sectorsize != 0:
733 self.failure_reason += " {0}: offset {1} not a multiple of sector size {2}".format(
734 filename, offset, sectorsize)
736 if bs % sectorsize != 0:
738 self.failure_reason += " {0}: block size {1} not a multiple of sector size " \
739 "{2}".format(filename, bs, sectorsize)
741 for i in range(int(bs/sectorsize)):
742 offsets.add(offset/sectorsize + i)
744 if len(offsets) != filesize/sectorsize:
746 self.failure_reason += " {0}: only {1} offsets touched; expected {2}".format(
747 filename, len(offsets), filesize/sectorsize)
749 logging.debug("%s: %d sectors touched", filename, len(offsets))
752 def check_result(self):
753 super(FioJobTest_t0023, self).check_result()
757 self.check_trimwrite("basic_bw.log")
758 self.check_trimwrite("bs_bw.log")
759 self.check_trimwrite("bsrange_bw.log")
760 self.check_trimwrite("bssplit_bw.log")
761 self.check_trimwrite("basic_no_rm_bw.log")
762 self.check_trimwrite("bs_no_rm_bw.log")
763 self.check_trimwrite("bsrange_no_rm_bw.log")
764 self.check_trimwrite("bssplit_no_rm_bw.log")
766 self.check_all_offsets("basic_bw.log", 4096, filesize)
767 self.check_all_offsets("bs_bw.log", 8192, filesize)
768 self.check_all_offsets("bsrange_bw.log", 512, filesize)
769 self.check_all_offsets("bssplit_bw.log", 512, filesize)
772 class FioJobTest_t0024(FioJobTest_t0023):
773 """Test consists of fio test job t0024 trimwrite test."""
775 def check_result(self):
776 # call FioJobTest_t0023's parent to skip checks done by t0023
777 super(FioJobTest_t0023, self).check_result()
781 self.check_trimwrite("basic_bw.log")
782 self.check_trimwrite("bs_bw.log")
783 self.check_trimwrite("bsrange_bw.log")
784 self.check_trimwrite("bssplit_bw.log")
786 self.check_all_offsets("basic_bw.log", 4096, filesize)
787 self.check_all_offsets("bs_bw.log", 8192, filesize)
788 self.check_all_offsets("bsrange_bw.log", 512, filesize)
789 self.check_all_offsets("bssplit_bw.log", 512, filesize)
792 class FioJobTest_t0025(FioJobTest):
793 """Test experimental verify read backs written data pattern."""
794 def check_result(self):
795 super(FioJobTest_t0025, self).check_result()
800 if self.json_data['jobs'][0]['read']['io_kbytes'] != 128:
803 class FioJobTest_t0027(FioJobTest):
804 def setup(self, *args, **kws):
805 super(FioJobTest_t0027, self).setup(*args, **kws)
806 self.pattern_file = os.path.join(self.test_dir, "t0027.pattern")
807 self.output_file = os.path.join(self.test_dir, "t0027file")
808 self.pattern = os.urandom(16 << 10)
809 with open(self.pattern_file, "wb") as f:
810 f.write(self.pattern)
812 def check_result(self):
813 super(FioJobTest_t0027, self).check_result()
818 with open(self.output_file, "rb") as f:
821 if data != self.pattern:
824 class FioJobTest_iops_rate(FioJobTest):
825 """Test consists of fio test job t0011
826 Confirm that job0 iops == 1000
827 and that job1_iops / job0_iops ~ 8
828 With two runs of fio-3.16 I observed a ratio of 8.3"""
830 def check_result(self):
831 super(FioJobTest_iops_rate, self).check_result()
836 iops1 = self.json_data['jobs'][0]['read']['iops']
837 logging.debug("Test %d: iops1: %f", self.testnum, iops1)
838 iops2 = self.json_data['jobs'][1]['read']['iops']
839 logging.debug("Test %d: iops2: %f", self.testnum, iops2)
840 ratio = iops2 / iops1
841 logging.debug("Test %d: ratio: %f", self.testnum, ratio)
843 if iops1 < 950 or iops1 > 1050:
844 self.failure_reason = "{0} iops value mismatch,".format(self.failure_reason)
847 if ratio < 6 or ratio > 10:
848 self.failure_reason = "{0} iops ratio mismatch,".format(self.failure_reason)
852 class Requirements():
853 """Requirements consists of multiple run environment characteristics.
854 These are to determine if a particular test can be run"""
867 def __init__(self, fio_root):
868 Requirements._not_macos = platform.system() != "Darwin"
869 Requirements._not_windows = platform.system() != "Windows"
870 Requirements._linux = platform.system() == "Linux"
872 if Requirements._linux:
873 config_file = os.path.join(fio_root, "config-host.h")
874 contents, success = FioJobTest.get_file(config_file)
876 print("Unable to open {0} to check requirements".format(config_file))
877 Requirements._zbd = True
879 Requirements._zbd = "CONFIG_HAS_BLKZONED" in contents
880 Requirements._libaio = "CONFIG_LIBAIO" in contents
882 contents, success = FioJobTest.get_file("/proc/kallsyms")
884 print("Unable to open '/proc/kallsyms' to probe for io_uring support")
886 Requirements._io_uring = "io_uring_setup" in contents
888 Requirements._root = (os.geteuid() == 0)
889 if Requirements._zbd and Requirements._root:
891 subprocess.run(["modprobe", "null_blk"],
892 stdout=subprocess.PIPE,
893 stderr=subprocess.PIPE)
894 if os.path.exists("/sys/module/null_blk/parameters/zoned"):
895 Requirements._zoned_nullb = True
899 if platform.system() == "Windows":
900 utest_exe = "unittest.exe"
902 utest_exe = "unittest"
903 unittest_path = os.path.join(fio_root, "unittests", utest_exe)
904 Requirements._unittests = os.path.exists(unittest_path)
906 Requirements._cpucount4 = multiprocessing.cpu_count() >= 4
908 req_list = [Requirements.linux,
910 Requirements.io_uring,
913 Requirements.zoned_nullb,
914 Requirements.not_macos,
915 Requirements.not_windows,
916 Requirements.unittests,
917 Requirements.cpucount4]
920 logging.debug("Requirements: Requirement '%s' met? %s", desc, value)
924 """Are we running on Linux?"""
925 return Requirements._linux, "Linux required"
929 """Is libaio available?"""
930 return Requirements._libaio, "libaio required"
934 """Is io_uring available?"""
935 return Requirements._io_uring, "io_uring required"
939 """Is ZBD support available?"""
940 return Requirements._zbd, "Zoned block device support required"
944 """Are we running as root?"""
945 return Requirements._root, "root required"
948 def zoned_nullb(cls):
949 """Are zoned null block devices available?"""
950 return Requirements._zoned_nullb, "Zoned null block device support required"
954 """Are we running on a platform other than macOS?"""
955 return Requirements._not_macos, "platform other than macOS required"
958 def not_windows(cls):
959 """Are we running on a platform other than Windws?"""
960 return Requirements._not_windows, "platform other than Windows required"
964 """Were unittests built?"""
965 return Requirements._unittests, "Unittests support required"
969 """Do we have at least 4 CPUs?"""
970 return Requirements._cpucount4, "4+ CPUs required"
975 'stderr_empty': True,
979 'zero_return': False,
980 'stderr_empty': False,
985 'stderr_empty': False,
991 'test_class': FioJobTest,
992 'job': 't0001-52c58027.fio',
993 'success': SUCCESS_DEFAULT,
1000 'test_class': FioJobTest,
1001 'job': 't0002-13af05ae-post.fio',
1002 'success': SUCCESS_DEFAULT,
1003 'pre_job': 't0002-13af05ae-pre.fio',
1004 'pre_success': None,
1005 'requirements': [Requirements.linux, Requirements.libaio],
1009 'test_class': FioJobTest,
1010 'job': 't0003-0ae2c6e1-post.fio',
1011 'success': SUCCESS_NONZERO,
1012 'pre_job': 't0003-0ae2c6e1-pre.fio',
1013 'pre_success': SUCCESS_DEFAULT,
1014 'requirements': [Requirements.linux, Requirements.libaio],
1018 'test_class': FioJobTest,
1019 'job': 't0004-8a99fdf6.fio',
1020 'success': SUCCESS_DEFAULT,
1022 'pre_success': None,
1023 'requirements': [Requirements.linux, Requirements.libaio],
1027 'test_class': FioJobTest_t0005,
1028 'job': 't0005-f7078f7b.fio',
1029 'success': SUCCESS_DEFAULT,
1031 'pre_success': None,
1032 'output_format': 'json',
1033 'requirements': [Requirements.not_windows],
1037 'test_class': FioJobTest_t0006,
1038 'job': 't0006-82af2a7c.fio',
1039 'success': SUCCESS_DEFAULT,
1041 'pre_success': None,
1042 'output_format': 'json',
1043 'requirements': [Requirements.linux, Requirements.libaio],
1047 'test_class': FioJobTest_t0007,
1048 'job': 't0007-37cf9e3c.fio',
1049 'success': SUCCESS_DEFAULT,
1051 'pre_success': None,
1052 'output_format': 'json',
1057 'test_class': FioJobTest_t0008,
1058 'job': 't0008-ae2fafc8.fio',
1059 'success': SUCCESS_DEFAULT,
1061 'pre_success': None,
1062 'output_format': 'json',
1067 'test_class': FioJobTest_t0009,
1068 'job': 't0009-f8b0bd10.fio',
1069 'success': SUCCESS_DEFAULT,
1071 'pre_success': None,
1072 'output_format': 'json',
1073 'requirements': [Requirements.not_macos,
1074 Requirements.cpucount4],
1075 # mac os does not support CPU affinity
1079 'test_class': FioJobTest,
1080 'job': 't0010-b7aae4ba.fio',
1081 'success': SUCCESS_DEFAULT,
1083 'pre_success': None,
1088 'test_class': FioJobTest_iops_rate,
1089 'job': 't0011-5d2788d5.fio',
1090 'success': SUCCESS_DEFAULT,
1092 'pre_success': None,
1093 'output_format': 'json',
1098 'test_class': FioJobTest_t0012,
1100 'success': SUCCESS_DEFAULT,
1102 'pre_success': None,
1103 'output_format': 'json',
1108 'test_class': FioJobTest,
1110 'success': SUCCESS_DEFAULT,
1112 'pre_success': None,
1113 'output_format': 'json',
1118 'test_class': FioJobTest_t0014,
1120 'success': SUCCESS_DEFAULT,
1122 'pre_success': None,
1123 'output_format': 'json',
1128 'test_class': FioJobTest_t0015,
1129 'job': 't0015-e78980ff.fio',
1130 'success': SUCCESS_DEFAULT,
1132 'pre_success': None,
1133 'output_format': 'json',
1134 'requirements': [Requirements.linux, Requirements.libaio],
1138 'test_class': FioJobTest_t0015,
1139 'job': 't0016-d54ae22.fio',
1140 'success': SUCCESS_DEFAULT,
1142 'pre_success': None,
1143 'output_format': 'json',
1148 'test_class': FioJobTest_t0015,
1150 'success': SUCCESS_DEFAULT,
1152 'pre_success': None,
1153 'output_format': 'json',
1154 'requirements': [Requirements.not_windows],
1158 'test_class': FioJobTest,
1160 'success': SUCCESS_DEFAULT,
1162 'pre_success': None,
1163 'requirements': [Requirements.linux, Requirements.io_uring],
1167 'test_class': FioJobTest_t0019,
1169 'success': SUCCESS_DEFAULT,
1171 'pre_success': None,
1176 'test_class': FioJobTest_t0020,
1178 'success': SUCCESS_DEFAULT,
1180 'pre_success': None,
1185 'test_class': FioJobTest_t0020,
1187 'success': SUCCESS_DEFAULT,
1189 'pre_success': None,
1194 'test_class': FioJobTest_t0022,
1196 'success': SUCCESS_DEFAULT,
1198 'pre_success': None,
1203 'test_class': FioJobTest_t0023,
1205 'success': SUCCESS_DEFAULT,
1207 'pre_success': None,
1212 'test_class': FioJobTest_t0024,
1214 'success': SUCCESS_DEFAULT,
1216 'pre_success': None,
1221 'test_class': FioJobTest_t0025,
1223 'success': SUCCESS_DEFAULT,
1225 'pre_success': None,
1226 'output_format': 'json',
1231 'test_class': FioJobTest,
1233 'success': SUCCESS_DEFAULT,
1235 'pre_success': None,
1236 'requirements': [Requirements.not_windows],
1240 'test_class': FioJobTest_t0027,
1242 'success': SUCCESS_DEFAULT,
1244 'pre_success': None,
1249 'test_class': FioJobTest,
1250 'job': 't0028-c6cade16.fio',
1251 'success': SUCCESS_DEFAULT,
1253 'pre_success': None,
1258 'test_class': FioExeTest,
1261 'success': SUCCESS_DEFAULT,
1266 'test_class': FioExeTest,
1269 'success': SUCCESS_DEFAULT,
1274 'test_class': FioExeTest,
1275 'exe': 't/lfsr-test',
1276 'parameters': ['0xFFFFFF', '0', '0', 'verify'],
1277 'success': SUCCESS_STDERR,
1282 'test_class': FioExeTest,
1283 'exe': 't/readonly.py',
1284 'parameters': ['-f', '{fio_path}'],
1285 'success': SUCCESS_DEFAULT,
1290 'test_class': FioExeTest,
1291 'exe': 't/steadystate_tests.py',
1292 'parameters': ['{fio_path}'],
1293 'success': SUCCESS_DEFAULT,
1298 'test_class': FioExeTest,
1301 'success': SUCCESS_STDERR,
1306 'test_class': FioExeTest,
1307 'exe': 't/strided.py',
1308 'parameters': ['{fio_path}'],
1309 'success': SUCCESS_DEFAULT,
1314 'test_class': FioExeTest,
1315 'exe': 't/zbd/run-tests-against-nullb',
1316 'parameters': ['-s', '1'],
1317 'success': SUCCESS_DEFAULT,
1318 'requirements': [Requirements.linux, Requirements.zbd,
1323 'test_class': FioExeTest,
1324 'exe': 't/zbd/run-tests-against-nullb',
1325 'parameters': ['-s', '2'],
1326 'success': SUCCESS_DEFAULT,
1327 'requirements': [Requirements.linux, Requirements.zbd,
1328 Requirements.root, Requirements.zoned_nullb],
1332 'test_class': FioExeTest,
1333 'exe': 'unittests/unittest',
1335 'success': SUCCESS_DEFAULT,
1336 'requirements': [Requirements.unittests],
1340 'test_class': FioExeTest,
1341 'exe': 't/latency_percentiles.py',
1342 'parameters': ['-f', '{fio_path}'],
1343 'success': SUCCESS_DEFAULT,
1348 'test_class': FioExeTest,
1349 'exe': 't/jsonplus2csv_test.py',
1350 'parameters': ['-f', '{fio_path}'],
1351 'success': SUCCESS_DEFAULT,
1356 'test_class': FioExeTest,
1357 'exe': 't/log_compression.py',
1358 'parameters': ['-f', '{fio_path}'],
1359 'success': SUCCESS_DEFAULT,
1364 'test_class': FioExeTest,
1365 'exe': 't/random_seed.py',
1366 'parameters': ['-f', '{fio_path}'],
1367 'success': SUCCESS_DEFAULT,
1374 """Parse command-line arguments."""
1376 parser = argparse.ArgumentParser()
1377 parser.add_argument('-r', '--fio-root',
1378 help='fio root path')
1379 parser.add_argument('-f', '--fio',
1380 help='path to fio executable (e.g., ./fio)')
1381 parser.add_argument('-a', '--artifact-root',
1382 help='artifact root directory')
1383 parser.add_argument('-s', '--skip', nargs='+', type=int,
1384 help='list of test(s) to skip')
1385 parser.add_argument('-o', '--run-only', nargs='+', type=int,
1386 help='list of test(s) to run, skipping all others')
1387 parser.add_argument('-d', '--debug', action='store_true',
1388 help='provide debug output')
1389 parser.add_argument('-k', '--skip-req', action='store_true',
1390 help='skip requirements checking')
1391 parser.add_argument('-p', '--pass-through', action='append',
1392 help='pass-through an argument to an executable test')
1393 args = parser.parse_args()
1403 logging.basicConfig(level=logging.DEBUG)
1405 logging.basicConfig(level=logging.INFO)
1408 if args.pass_through:
1409 for arg in args.pass_through:
1411 print("Invalid --pass-through argument '%s'" % arg)
1412 print("Syntax for --pass-through is TESTNUMBER:ARGUMENT")
1414 split = arg.split(":", 1)
1415 pass_through[int(split[0])] = split[1]
1416 logging.debug("Pass-through arguments: %s", pass_through)
1419 fio_root = args.fio_root
1421 fio_root = str(Path(__file__).absolute().parent.parent)
1422 print("fio root is %s" % fio_root)
1427 if platform.system() == "Windows":
1431 fio_path = os.path.join(fio_root, fio_exe)
1432 print("fio path is %s" % fio_path)
1433 if not shutil.which(fio_path):
1434 print("Warning: fio executable not found")
1436 artifact_root = args.artifact_root if args.artifact_root else \
1437 "fio-test-{0}".format(time.strftime("%Y%m%d-%H%M%S"))
1438 os.mkdir(artifact_root)
1439 print("Artifact directory is %s" % artifact_root)
1441 if not args.skip_req:
1442 req = Requirements(fio_root)
1448 for config in TEST_LIST:
1449 if (args.skip and config['test_id'] in args.skip) or \
1450 (args.run_only and config['test_id'] not in args.run_only):
1451 skipped = skipped + 1
1452 print("Test {0} SKIPPED (User request)".format(config['test_id']))
1455 if issubclass(config['test_class'], FioJobTest):
1456 if config['pre_job']:
1457 fio_pre_job = os.path.join(fio_root, 't', 'jobs',
1461 if config['pre_success']:
1462 fio_pre_success = config['pre_success']
1464 fio_pre_success = None
1465 if 'output_format' in config:
1466 output_format = config['output_format']
1468 output_format = 'normal'
1469 test = config['test_class'](
1471 os.path.join(fio_root, 't', 'jobs', config['job']),
1473 fio_pre_job=fio_pre_job,
1474 fio_pre_success=fio_pre_success,
1475 output_format=output_format)
1476 desc = config['job']
1477 elif issubclass(config['test_class'], FioExeTest):
1478 exe_path = os.path.join(fio_root, config['exe'])
1479 if config['parameters']:
1480 parameters = [p.format(fio_path=fio_path) for p in config['parameters']]
1483 if Path(exe_path).suffix == '.py' and platform.system() == "Windows":
1484 parameters.insert(0, exe_path)
1485 exe_path = "python.exe"
1486 if config['test_id'] in pass_through:
1487 parameters += pass_through[config['test_id']].split()
1488 test = config['test_class'](exe_path, parameters,
1490 desc = config['exe']
1492 print("Test {0} FAILED: unable to process test config".format(config['test_id']))
1496 if not args.skip_req:
1498 for req in config['requirements']:
1499 reqs_met, reason = req()
1500 logging.debug("Test %d: Requirement '%s' met? %s", config['test_id'], reason,
1505 print("Test {0} SKIPPED ({1}) {2}".format(config['test_id'], reason, desc))
1506 skipped = skipped + 1
1510 test.setup(artifact_root, config['test_id'])
1513 except KeyboardInterrupt:
1515 except Exception as e:
1517 test.failure_reason += str(e)
1518 logging.debug("Test %d exception:\n%s\n", config['test_id'], traceback.format_exc())
1523 result = "FAILED: {0}".format(test.failure_reason)
1525 contents, _ = FioJobTest.get_file(test.stderr_file)
1526 logging.debug("Test %d: stderr:\n%s", config['test_id'], contents)
1527 contents, _ = FioJobTest.get_file(test.stdout_file)
1528 logging.debug("Test %d: stdout:\n%s", config['test_id'], contents)
1529 print("Test {0} {1} {2}".format(config['test_id'], result, desc))
1531 print("{0} test(s) passed, {1} failed, {2} skipped".format(passed, failed, skipped))
1536 if __name__ == '__main__':