X-Git-Url: https://git.kernel.dk/?a=blobdiff_plain;ds=sidebyside;f=t%2Frun-fio-tests.py;h=225806134e59a6ea3d181500e1f7c1680717363c;hb=refs%2Fheads%2Fmaster;hp=7e0df7ed2e9d53f954d5675d3223f363a9d705d1;hpb=c994fa62425fbc1e5f62115c58829067bcc1e28f;p=fio.git diff --git a/t/run-fio-tests.py b/t/run-fio-tests.py index 7e0df7ed..22580613 100755 --- a/t/run-fio-tests.py +++ b/t/run-fio-tests.py @@ -43,320 +43,40 @@ import os import sys -import json import time import shutil import logging import argparse -import platform -import traceback -import subprocess -import multiprocessing from pathlib import Path +from statsmodels.sandbox.stats.runs import runstest_1samp +from fiotestlib import FioExeTest, FioJobFileTest, run_fio_tests +from fiotestcommon import * -class FioTest(): - """Base for all fio tests.""" - - def __init__(self, exe_path, parameters, success): - self.exe_path = exe_path - self.parameters = parameters - self.success = success - self.output = {} - self.artifact_root = None - self.testnum = None - self.test_dir = None - self.passed = True - self.failure_reason = '' - self.command_file = None - self.stdout_file = None - self.stderr_file = None - self.exitcode_file = None - - def setup(self, artifact_root, testnum): - """Setup instance variables for test.""" - - self.artifact_root = artifact_root - self.testnum = testnum - self.test_dir = os.path.join(artifact_root, "{:04d}".format(testnum)) - if not os.path.exists(self.test_dir): - os.mkdir(self.test_dir) - - self.command_file = os.path.join( - self.test_dir, - "{0}.command".format(os.path.basename(self.exe_path))) - self.stdout_file = os.path.join( - self.test_dir, - "{0}.stdout".format(os.path.basename(self.exe_path))) - self.stderr_file = os.path.join( - self.test_dir, - "{0}.stderr".format(os.path.basename(self.exe_path))) - self.exitcode_file = os.path.join( - self.test_dir, - "{0}.exitcode".format(os.path.basename(self.exe_path))) - - def run(self): - """Run the test.""" - - raise NotImplementedError() - - def check_result(self): - """Check test results.""" - - raise NotImplementedError() - - -class FioExeTest(FioTest): - """Test consists of an executable binary or script""" - - def __init__(self, exe_path, parameters, success): - """Construct a FioExeTest which is a FioTest consisting of an - executable binary or script. - - exe_path: location of executable binary or script - parameters: list of parameters for executable - success: Definition of test success - """ - - FioTest.__init__(self, exe_path, parameters, success) - - def run(self): - """Execute the binary or script described by this instance.""" - - command = [self.exe_path] + self.parameters - command_file = open(self.command_file, "w+") - command_file.write("%s\n" % command) - command_file.close() - - stdout_file = open(self.stdout_file, "w+") - stderr_file = open(self.stderr_file, "w+") - exitcode_file = open(self.exitcode_file, "w+") - try: - proc = None - # Avoid using subprocess.run() here because when a timeout occurs, - # fio will be stopped with SIGKILL. This does not give fio a - # chance to clean up and means that child processes may continue - # running and submitting IO. - proc = subprocess.Popen(command, - stdout=stdout_file, - stderr=stderr_file, - cwd=self.test_dir, - universal_newlines=True) - proc.communicate(timeout=self.success['timeout']) - exitcode_file.write('{0}\n'.format(proc.returncode)) - logging.debug("Test %d: return code: %d", self.testnum, proc.returncode) - self.output['proc'] = proc - except subprocess.TimeoutExpired: - proc.terminate() - proc.communicate() - assert proc.poll() - self.output['failure'] = 'timeout' - except Exception: - if proc: - if not proc.poll(): - proc.terminate() - proc.communicate() - self.output['failure'] = 'exception' - self.output['exc_info'] = sys.exc_info() - finally: - stdout_file.close() - stderr_file.close() - exitcode_file.close() - - def check_result(self): - """Check results of test run.""" - - if 'proc' not in self.output: - if self.output['failure'] == 'timeout': - self.failure_reason = "{0} timeout,".format(self.failure_reason) - else: - assert self.output['failure'] == 'exception' - self.failure_reason = '{0} exception: {1}, {2}'.format( - self.failure_reason, self.output['exc_info'][0], - self.output['exc_info'][1]) - - self.passed = False - return - - if 'zero_return' in self.success: - if self.success['zero_return']: - if self.output['proc'].returncode != 0: - self.passed = False - self.failure_reason = "{0} non-zero return code,".format(self.failure_reason) - else: - if self.output['proc'].returncode == 0: - self.failure_reason = "{0} zero return code,".format(self.failure_reason) - self.passed = False - - stderr_size = os.path.getsize(self.stderr_file) - if 'stderr_empty' in self.success: - if self.success['stderr_empty']: - if stderr_size != 0: - self.failure_reason = "{0} stderr not empty,".format(self.failure_reason) - self.passed = False - else: - if stderr_size == 0: - self.failure_reason = "{0} stderr empty,".format(self.failure_reason) - self.passed = False - - -class FioJobTest(FioExeTest): - """Test consists of a fio job""" - - def __init__(self, fio_path, fio_job, success, fio_pre_job=None, - fio_pre_success=None, output_format="normal"): - """Construct a FioJobTest which is a FioExeTest consisting of a - single fio job file with an optional setup step. - - fio_path: location of fio executable - fio_job: location of fio job file - success: Definition of test success - fio_pre_job: fio job for preconditioning - fio_pre_success: Definition of test success for fio precon job - output_format: normal (default), json, jsonplus, or terse - """ - - self.fio_job = fio_job - self.fio_pre_job = fio_pre_job - self.fio_pre_success = fio_pre_success if fio_pre_success else success - self.output_format = output_format - self.precon_failed = False - self.json_data = None - self.fio_output = "{0}.output".format(os.path.basename(self.fio_job)) - self.fio_args = [ - "--max-jobs=16", - "--output-format={0}".format(self.output_format), - "--output={0}".format(self.fio_output), - self.fio_job, - ] - FioExeTest.__init__(self, fio_path, self.fio_args, success) - - def setup(self, artifact_root, testnum): - """Setup instance variables for fio job test.""" - - super(FioJobTest, self).setup(artifact_root, testnum) - - self.command_file = os.path.join( - self.test_dir, - "{0}.command".format(os.path.basename(self.fio_job))) - self.stdout_file = os.path.join( - self.test_dir, - "{0}.stdout".format(os.path.basename(self.fio_job))) - self.stderr_file = os.path.join( - self.test_dir, - "{0}.stderr".format(os.path.basename(self.fio_job))) - self.exitcode_file = os.path.join( - self.test_dir, - "{0}.exitcode".format(os.path.basename(self.fio_job))) - - def run_pre_job(self): - """Run fio job precondition step.""" - - precon = FioJobTest(self.exe_path, self.fio_pre_job, - self.fio_pre_success, - output_format=self.output_format) - precon.setup(self.artifact_root, self.testnum) - precon.run() - precon.check_result() - self.precon_failed = not precon.passed - self.failure_reason = precon.failure_reason - - def run(self): - """Run fio job test.""" - - if self.fio_pre_job: - self.run_pre_job() - - if not self.precon_failed: - super(FioJobTest, self).run() - else: - logging.debug("Test %d: precondition step failed", self.testnum) - - @classmethod - def get_file(cls, filename): - """Safely read a file.""" - file_data = '' - success = True - - try: - with open(filename, "r") as output_file: - file_data = output_file.read() - except OSError: - success = False - - return file_data, success - - def get_file_fail(self, filename): - """Safely read a file and fail the test upon error.""" - file_data = None - - try: - with open(filename, "r") as output_file: - file_data = output_file.read() - except OSError: - self.failure_reason += " unable to read file {0}".format(filename) - self.passed = False - - return file_data - - def check_result(self): - """Check fio job results.""" - - if self.precon_failed: - self.passed = False - self.failure_reason = "{0} precondition step failed,".format(self.failure_reason) - return - - super(FioJobTest, self).check_result() - - if not self.passed: - return - - if 'json' not in self.output_format: - return - - file_data = self.get_file_fail(os.path.join(self.test_dir, self.fio_output)) - if not file_data: - return - - # - # Sometimes fio informational messages are included at the top of the - # JSON output, especially under Windows. Try to decode output as JSON - # data, skipping everything until the first { - # - lines = file_data.splitlines() - file_data = '\n'.join(lines[lines.index("{"):]) - try: - self.json_data = json.loads(file_data) - except json.JSONDecodeError: - self.failure_reason = "{0} unable to decode JSON data,".format(self.failure_reason) - self.passed = False - - -class FioJobTest_t0005(FioJobTest): +class FioJobFileTest_t0005(FioJobFileTest): """Test consists of fio test job t0005 Confirm that read['io_kbytes'] == write['io_kbytes'] == 102400""" def check_result(self): - super(FioJobTest_t0005, self).check_result() + super().check_result() if not self.passed: return if self.json_data['jobs'][0]['read']['io_kbytes'] != 102400: - self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason) + self.failure_reason = f"{self.failure_reason} bytes read mismatch," self.passed = False if self.json_data['jobs'][0]['write']['io_kbytes'] != 102400: - self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason) + self.failure_reason = f"{self.failure_reason} bytes written mismatch," self.passed = False -class FioJobTest_t0006(FioJobTest): +class FioJobFileTest_t0006(FioJobFileTest): """Test consists of fio test job t0006 Confirm that read['io_kbytes'] ~ 2*write['io_kbytes']""" def check_result(self): - super(FioJobTest_t0006, self).check_result() + super().check_result() if not self.passed: return @@ -365,26 +85,26 @@ class FioJobTest_t0006(FioJobTest): / self.json_data['jobs'][0]['write']['io_kbytes'] logging.debug("Test %d: ratio: %f", self.testnum, ratio) if ratio < 1.99 or ratio > 2.01: - self.failure_reason = "{0} read/write ratio mismatch,".format(self.failure_reason) + self.failure_reason = f"{self.failure_reason} read/write ratio mismatch," self.passed = False -class FioJobTest_t0007(FioJobTest): +class FioJobFileTest_t0007(FioJobFileTest): """Test consists of fio test job t0007 Confirm that read['io_kbytes'] = 87040""" def check_result(self): - super(FioJobTest_t0007, self).check_result() + super().check_result() if not self.passed: return if self.json_data['jobs'][0]['read']['io_kbytes'] != 87040: - self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason) + self.failure_reason = f"{self.failure_reason} bytes read mismatch," self.passed = False -class FioJobTest_t0008(FioJobTest): +class FioJobFileTest_t0008(FioJobFileTest): """Test consists of fio test job t0008 Confirm that read['io_kbytes'] = 32768 and that write['io_kbytes'] ~ 16384 @@ -396,7 +116,7 @@ class FioJobTest_t0008(FioJobTest): the blocks originally written will be read.""" def check_result(self): - super(FioJobTest_t0008, self).check_result() + super().check_result() if not self.passed: return @@ -405,19 +125,19 @@ class FioJobTest_t0008(FioJobTest): logging.debug("Test %d: ratio: %f", self.testnum, ratio) if ratio < 0.97 or ratio > 1.03: - self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason) + self.failure_reason = f"{self.failure_reason} bytes written mismatch," self.passed = False if self.json_data['jobs'][0]['read']['io_kbytes'] != 32768: - self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason) + self.failure_reason = f"{self.failure_reason} bytes read mismatch," self.passed = False -class FioJobTest_t0009(FioJobTest): +class FioJobFileTest_t0009(FioJobFileTest): """Test consists of fio test job t0009 Confirm that runtime >= 60s""" def check_result(self): - super(FioJobTest_t0009, self).check_result() + super().check_result() if not self.passed: return @@ -425,24 +145,24 @@ class FioJobTest_t0009(FioJobTest): logging.debug('Test %d: elapsed: %d', self.testnum, self.json_data['jobs'][0]['elapsed']) if self.json_data['jobs'][0]['elapsed'] < 60: - self.failure_reason = "{0} elapsed time mismatch,".format(self.failure_reason) + self.failure_reason = f"{self.failure_reason} elapsed time mismatch," self.passed = False -class FioJobTest_t0012(FioJobTest): +class FioJobFileTest_t0012(FioJobFileTest): """Test consists of fio test job t0012 Confirm ratios of job iops are 1:5:10 job1,job2,job3 respectively""" def check_result(self): - super(FioJobTest_t0012, self).check_result() + super().check_result() if not self.passed: return iops_files = [] for i in range(1, 4): - filename = os.path.join(self.test_dir, "{0}_iops.{1}.log".format(os.path.basename( + filename = os.path.join(self.paths['test_dir'], "{0}_iops.{1}.log".format(os.path.basename( self.fio_job), i)) file_data = self.get_file_fail(filename) if not file_data: @@ -474,7 +194,7 @@ class FioJobTest_t0012(FioJobTest): return -class FioJobTest_t0014(FioJobTest): +class FioJobFileTest_t0014(FioJobFileTest): """Test consists of fio test job t0014 Confirm that job1_iops / job2_iops ~ 1:2 for entire duration and that job1_iops / job3_iops ~ 1:3 for first half of duration. @@ -483,14 +203,14 @@ class FioJobTest_t0014(FioJobTest): re-calibrate the activity dynamically""" def check_result(self): - super(FioJobTest_t0014, self).check_result() + super().check_result() if not self.passed: return iops_files = [] for i in range(1, 4): - filename = os.path.join(self.test_dir, "{0}_iops.{1}.log".format(os.path.basename( + filename = os.path.join(self.paths['test_dir'], "{0}_iops.{1}.log".format(os.path.basename( self.fio_job), i)) file_data = self.get_file_fail(filename) if not file_data: @@ -533,12 +253,12 @@ class FioJobTest_t0014(FioJobTest): return -class FioJobTest_t0015(FioJobTest): +class FioJobFileTest_t0015(FioJobFileTest): """Test consists of fio test jobs t0015 and t0016 Confirm that mean(slat) + mean(clat) = mean(tlat)""" def check_result(self): - super(FioJobTest_t0015, self).check_result() + super().check_result() if not self.passed: return @@ -554,14 +274,14 @@ class FioJobTest_t0015(FioJobTest): self.passed = False -class FioJobTest_t0019(FioJobTest): +class FioJobFileTest_t0019(FioJobFileTest): """Test consists of fio test job t0019 Confirm that all offsets were touched sequentially""" def check_result(self): - super(FioJobTest_t0019, self).check_result() + super().check_result() - bw_log_filename = os.path.join(self.test_dir, "test_bw.log") + bw_log_filename = os.path.join(self.paths['test_dir'], "test_bw.log") file_data = self.get_file_fail(bw_log_filename) if not file_data: return @@ -575,64 +295,61 @@ class FioJobTest_t0019(FioJobTest): cur = int(line.split(',')[4]) if cur - prev != 4096: self.passed = False - self.failure_reason = "offsets {0}, {1} not sequential".format(prev, cur) + self.failure_reason = f"offsets {prev}, {cur} not sequential" return prev = cur if cur/4096 != 255: self.passed = False - self.failure_reason = "unexpected last offset {0}".format(cur) + self.failure_reason = f"unexpected last offset {cur}" -class FioJobTest_t0020(FioJobTest): +class FioJobFileTest_t0020(FioJobFileTest): """Test consists of fio test jobs t0020 and t0021 Confirm that almost all offsets were touched non-sequentially""" def check_result(self): - super(FioJobTest_t0020, self).check_result() + super().check_result() - bw_log_filename = os.path.join(self.test_dir, "test_bw.log") + bw_log_filename = os.path.join(self.paths['test_dir'], "test_bw.log") file_data = self.get_file_fail(bw_log_filename) if not file_data: return log_lines = file_data.split('\n') - seq_count = 0 - offsets = set() + offsets = [] prev = int(log_lines[0].split(',')[4]) for line in log_lines[1:]: - offsets.add(prev/4096) + offsets.append(prev/4096) if len(line.strip()) == 0: continue cur = int(line.split(',')[4]) - if cur - prev == 4096: - seq_count += 1 prev = cur - # 10 is an arbitrary threshold - if seq_count > 10: - self.passed = False - self.failure_reason = "too many ({0}) consecutive offsets".format(seq_count) - if len(offsets) != 256: self.passed = False - self.failure_reason += " number of offsets is {0} instead of 256".format(len(offsets)) + self.failure_reason += f" number of offsets is {len(offsets)} instead of 256" for i in range(256): if not i in offsets: self.passed = False - self.failure_reason += " missing offset {0}".format(i*4096) + self.failure_reason += f" missing offset {i * 4096}" + + (_, p) = runstest_1samp(list(offsets)) + if p < 0.05: + self.passed = False + self.failure_reason += f" runs test failed with p = {p}" -class FioJobTest_t0022(FioJobTest): +class FioJobFileTest_t0022(FioJobFileTest): """Test consists of fio test job t0022""" def check_result(self): - super(FioJobTest_t0022, self).check_result() + super().check_result() - bw_log_filename = os.path.join(self.test_dir, "test_bw.log") + bw_log_filename = os.path.join(self.paths['test_dir'], "test_bw.log") file_data = self.get_file_fail(bw_log_filename) if not file_data: return @@ -657,20 +374,20 @@ class FioJobTest_t0022(FioJobTest): # 10 is an arbitrary threshold if seq_count > 10: self.passed = False - self.failure_reason = "too many ({0}) consecutive offsets".format(seq_count) + self.failure_reason = f"too many ({seq_count}) consecutive offsets" if len(offsets) == filesize/bs: self.passed = False self.failure_reason += " no duplicate offsets found with norandommap=1" -class FioJobTest_t0023(FioJobTest): +class FioJobFileTest_t0023(FioJobFileTest): """Test consists of fio test job t0023 randtrimwrite test.""" def check_trimwrite(self, filename): """Make sure that trims are followed by writes of the same size at the same offset.""" - bw_log_filename = os.path.join(self.test_dir, filename) + bw_log_filename = os.path.join(self.paths['test_dir'], filename) file_data = self.get_file_fail(bw_log_filename) if not file_data: return @@ -692,7 +409,7 @@ class FioJobTest_t0023(FioJobTest): bw_log_filename, line) break else: - if ddir != 1: + if ddir != 1: # pylint: disable=no-else-break self.passed = False self.failure_reason += " {0}: trim not preceeded by write: {1}".format( bw_log_filename, line) @@ -703,11 +420,13 @@ class FioJobTest_t0023(FioJobTest): self.failure_reason += " {0}: block size does not match: {1}".format( bw_log_filename, line) break + if prev_offset != offset: self.passed = False self.failure_reason += " {0}: offset does not match: {1}".format( bw_log_filename, line) break + prev_ddir = ddir prev_bs = bs prev_offset = offset @@ -716,7 +435,7 @@ class FioJobTest_t0023(FioJobTest): def check_all_offsets(self, filename, sectorsize, filesize): """Make sure all offsets were touched.""" - file_data = self.get_file_fail(os.path.join(self.test_dir, filename)) + file_data = self.get_file_fail(os.path.join(self.paths['test_dir'], filename)) if not file_data: return @@ -752,7 +471,7 @@ class FioJobTest_t0023(FioJobTest): def check_result(self): - super(FioJobTest_t0023, self).check_result() + super().check_result() filesize = 1024*1024 @@ -771,12 +490,12 @@ class FioJobTest_t0023(FioJobTest): self.check_all_offsets("bssplit_bw.log", 512, filesize) -class FioJobTest_t0024(FioJobTest_t0023): +class FioJobFileTest_t0024(FioJobFileTest_t0023): """Test consists of fio test job t0024 trimwrite test.""" def check_result(self): - # call FioJobTest_t0023's parent to skip checks done by t0023 - super(FioJobTest_t0023, self).check_result() + # call FioJobFileTest_t0023's parent to skip checks done by t0023 + super(FioJobFileTest_t0023, self).check_result() filesize = 1024*1024 @@ -791,10 +510,10 @@ class FioJobTest_t0024(FioJobTest_t0023): self.check_all_offsets("bssplit_bw.log", 512, filesize) -class FioJobTest_t0025(FioJobTest): +class FioJobFileTest_t0025(FioJobFileTest): """Test experimental verify read backs written data pattern.""" def check_result(self): - super(FioJobTest_t0025, self).check_result() + super().check_result() if not self.passed: return @@ -802,17 +521,17 @@ class FioJobTest_t0025(FioJobTest): if self.json_data['jobs'][0]['read']['io_kbytes'] != 128: self.passed = False -class FioJobTest_t0027(FioJobTest): +class FioJobFileTest_t0027(FioJobFileTest): def setup(self, *args, **kws): - super(FioJobTest_t0027, self).setup(*args, **kws) - self.pattern_file = os.path.join(self.test_dir, "t0027.pattern") - self.output_file = os.path.join(self.test_dir, "t0027file") + super().setup(*args, **kws) + self.pattern_file = os.path.join(self.paths['test_dir'], "t0027.pattern") + self.output_file = os.path.join(self.paths['test_dir'], "t0027file") self.pattern = os.urandom(16 << 10) with open(self.pattern_file, "wb") as f: f.write(self.pattern) def check_result(self): - super(FioJobTest_t0027, self).check_result() + super().check_result() if not self.passed: return @@ -823,14 +542,25 @@ class FioJobTest_t0027(FioJobTest): if data != self.pattern: self.passed = False -class FioJobTest_iops_rate(FioJobTest): - """Test consists of fio test job t0009 +class FioJobFileTest_t0029(FioJobFileTest): + """Test loops option works with read-verify workload.""" + def check_result(self): + super().check_result() + + if not self.passed: + return + + if self.json_data['jobs'][1]['read']['io_kbytes'] != 8: + self.passed = False + +class FioJobFileTest_iops_rate(FioJobFileTest): + """Test consists of fio test job t0011 Confirm that job0 iops == 1000 and that job1_iops / job0_iops ~ 8 With two runs of fio-3.16 I observed a ratio of 8.3""" def check_result(self): - super(FioJobTest_iops_rate, self).check_result() + super().check_result() if not self.passed: return @@ -843,154 +573,18 @@ class FioJobTest_iops_rate(FioJobTest): logging.debug("Test %d: ratio: %f", self.testnum, ratio) if iops1 < 950 or iops1 > 1050: - self.failure_reason = "{0} iops value mismatch,".format(self.failure_reason) + self.failure_reason = f"{self.failure_reason} iops value mismatch," self.passed = False if ratio < 6 or ratio > 10: - self.failure_reason = "{0} iops ratio mismatch,".format(self.failure_reason) + self.failure_reason = f"{self.failure_reason} iops ratio mismatch," self.passed = False -class Requirements(): - """Requirements consists of multiple run environment characteristics. - These are to determine if a particular test can be run""" - - _linux = False - _libaio = False - _io_uring = False - _zbd = False - _root = False - _zoned_nullb = False - _not_macos = False - _not_windows = False - _unittests = False - _cpucount4 = False - - def __init__(self, fio_root): - Requirements._not_macos = platform.system() != "Darwin" - Requirements._not_windows = platform.system() != "Windows" - Requirements._linux = platform.system() == "Linux" - - if Requirements._linux: - config_file = os.path.join(fio_root, "config-host.h") - contents, success = FioJobTest.get_file(config_file) - if not success: - print("Unable to open {0} to check requirements".format(config_file)) - Requirements._zbd = True - else: - Requirements._zbd = "CONFIG_HAS_BLKZONED" in contents - Requirements._libaio = "CONFIG_LIBAIO" in contents - - contents, success = FioJobTest.get_file("/proc/kallsyms") - if not success: - print("Unable to open '/proc/kallsyms' to probe for io_uring support") - else: - Requirements._io_uring = "io_uring_setup" in contents - - Requirements._root = (os.geteuid() == 0) - if Requirements._zbd and Requirements._root: - try: - subprocess.run(["modprobe", "null_blk"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - if os.path.exists("/sys/module/null_blk/parameters/zoned"): - Requirements._zoned_nullb = True - except Exception: - pass - - if platform.system() == "Windows": - utest_exe = "unittest.exe" - else: - utest_exe = "unittest" - unittest_path = os.path.join(fio_root, "unittests", utest_exe) - Requirements._unittests = os.path.exists(unittest_path) - - Requirements._cpucount4 = multiprocessing.cpu_count() >= 4 - - req_list = [Requirements.linux, - Requirements.libaio, - Requirements.io_uring, - Requirements.zbd, - Requirements.root, - Requirements.zoned_nullb, - Requirements.not_macos, - Requirements.not_windows, - Requirements.unittests, - Requirements.cpucount4] - for req in req_list: - value, desc = req() - logging.debug("Requirements: Requirement '%s' met? %s", desc, value) - - @classmethod - def linux(cls): - """Are we running on Linux?""" - return Requirements._linux, "Linux required" - - @classmethod - def libaio(cls): - """Is libaio available?""" - return Requirements._libaio, "libaio required" - - @classmethod - def io_uring(cls): - """Is io_uring available?""" - return Requirements._io_uring, "io_uring required" - - @classmethod - def zbd(cls): - """Is ZBD support available?""" - return Requirements._zbd, "Zoned block device support required" - - @classmethod - def root(cls): - """Are we running as root?""" - return Requirements._root, "root required" - - @classmethod - def zoned_nullb(cls): - """Are zoned null block devices available?""" - return Requirements._zoned_nullb, "Zoned null block device support required" - - @classmethod - def not_macos(cls): - """Are we running on a platform other than macOS?""" - return Requirements._not_macos, "platform other than macOS required" - - @classmethod - def not_windows(cls): - """Are we running on a platform other than Windws?""" - return Requirements._not_windows, "platform other than Windows required" - - @classmethod - def unittests(cls): - """Were unittests built?""" - return Requirements._unittests, "Unittests support required" - - @classmethod - def cpucount4(cls): - """Do we have at least 4 CPUs?""" - return Requirements._cpucount4, "4+ CPUs required" - - -SUCCESS_DEFAULT = { - 'zero_return': True, - 'stderr_empty': True, - 'timeout': 600, - } -SUCCESS_NONZERO = { - 'zero_return': False, - 'stderr_empty': False, - 'timeout': 600, - } -SUCCESS_STDERR = { - 'zero_return': True, - 'stderr_empty': False, - 'timeout': 600, - } TEST_LIST = [ { 'test_id': 1, - 'test_class': FioJobTest, + 'test_class': FioJobFileTest, 'job': 't0001-52c58027.fio', 'success': SUCCESS_DEFAULT, 'pre_job': None, @@ -999,7 +593,7 @@ TEST_LIST = [ }, { 'test_id': 2, - 'test_class': FioJobTest, + 'test_class': FioJobFileTest, 'job': 't0002-13af05ae-post.fio', 'success': SUCCESS_DEFAULT, 'pre_job': 't0002-13af05ae-pre.fio', @@ -1008,7 +602,7 @@ TEST_LIST = [ }, { 'test_id': 3, - 'test_class': FioJobTest, + 'test_class': FioJobFileTest, 'job': 't0003-0ae2c6e1-post.fio', 'success': SUCCESS_NONZERO, 'pre_job': 't0003-0ae2c6e1-pre.fio', @@ -1017,7 +611,7 @@ TEST_LIST = [ }, { 'test_id': 4, - 'test_class': FioJobTest, + 'test_class': FioJobFileTest, 'job': 't0004-8a99fdf6.fio', 'success': SUCCESS_DEFAULT, 'pre_job': None, @@ -1026,7 +620,7 @@ TEST_LIST = [ }, { 'test_id': 5, - 'test_class': FioJobTest_t0005, + 'test_class': FioJobFileTest_t0005, 'job': 't0005-f7078f7b.fio', 'success': SUCCESS_DEFAULT, 'pre_job': None, @@ -1036,7 +630,7 @@ TEST_LIST = [ }, { 'test_id': 6, - 'test_class': FioJobTest_t0006, + 'test_class': FioJobFileTest_t0006, 'job': 't0006-82af2a7c.fio', 'success': SUCCESS_DEFAULT, 'pre_job': None, @@ -1046,7 +640,7 @@ TEST_LIST = [ }, { 'test_id': 7, - 'test_class': FioJobTest_t0007, + 'test_class': FioJobFileTest_t0007, 'job': 't0007-37cf9e3c.fio', 'success': SUCCESS_DEFAULT, 'pre_job': None, @@ -1056,7 +650,7 @@ TEST_LIST = [ }, { 'test_id': 8, - 'test_class': FioJobTest_t0008, + 'test_class': FioJobFileTest_t0008, 'job': 't0008-ae2fafc8.fio', 'success': SUCCESS_DEFAULT, 'pre_job': None, @@ -1066,7 +660,7 @@ TEST_LIST = [ }, { 'test_id': 9, - 'test_class': FioJobTest_t0009, + 'test_class': FioJobFileTest_t0009, 'job': 't0009-f8b0bd10.fio', 'success': SUCCESS_DEFAULT, 'pre_job': None, @@ -1078,7 +672,7 @@ TEST_LIST = [ }, { 'test_id': 10, - 'test_class': FioJobTest, + 'test_class': FioJobFileTest, 'job': 't0010-b7aae4ba.fio', 'success': SUCCESS_DEFAULT, 'pre_job': None, @@ -1087,7 +681,7 @@ TEST_LIST = [ }, { 'test_id': 11, - 'test_class': FioJobTest_iops_rate, + 'test_class': FioJobFileTest_iops_rate, 'job': 't0011-5d2788d5.fio', 'success': SUCCESS_DEFAULT, 'pre_job': None, @@ -1097,7 +691,7 @@ TEST_LIST = [ }, { 'test_id': 12, - 'test_class': FioJobTest_t0012, + 'test_class': FioJobFileTest_t0012, 'job': 't0012.fio', 'success': SUCCESS_DEFAULT, 'pre_job': None, @@ -1107,7 +701,7 @@ TEST_LIST = [ }, { 'test_id': 13, - 'test_class': FioJobTest, + 'test_class': FioJobFileTest, 'job': 't0013.fio', 'success': SUCCESS_DEFAULT, 'pre_job': None, @@ -1117,7 +711,7 @@ TEST_LIST = [ }, { 'test_id': 14, - 'test_class': FioJobTest_t0014, + 'test_class': FioJobFileTest_t0014, 'job': 't0014.fio', 'success': SUCCESS_DEFAULT, 'pre_job': None, @@ -1127,8 +721,8 @@ TEST_LIST = [ }, { 'test_id': 15, - 'test_class': FioJobTest_t0015, - 'job': 't0015-e78980ff.fio', + 'test_class': FioJobFileTest_t0015, + 'job': 't0015-4e7e7898.fio', 'success': SUCCESS_DEFAULT, 'pre_job': None, 'pre_success': None, @@ -1137,7 +731,7 @@ TEST_LIST = [ }, { 'test_id': 16, - 'test_class': FioJobTest_t0015, + 'test_class': FioJobFileTest_t0015, 'job': 't0016-d54ae22.fio', 'success': SUCCESS_DEFAULT, 'pre_job': None, @@ -1147,7 +741,7 @@ TEST_LIST = [ }, { 'test_id': 17, - 'test_class': FioJobTest_t0015, + 'test_class': FioJobFileTest_t0015, 'job': 't0017.fio', 'success': SUCCESS_DEFAULT, 'pre_job': None, @@ -1157,7 +751,7 @@ TEST_LIST = [ }, { 'test_id': 18, - 'test_class': FioJobTest, + 'test_class': FioJobFileTest, 'job': 't0018.fio', 'success': SUCCESS_DEFAULT, 'pre_job': None, @@ -1166,7 +760,7 @@ TEST_LIST = [ }, { 'test_id': 19, - 'test_class': FioJobTest_t0019, + 'test_class': FioJobFileTest_t0019, 'job': 't0019.fio', 'success': SUCCESS_DEFAULT, 'pre_job': None, @@ -1175,7 +769,7 @@ TEST_LIST = [ }, { 'test_id': 20, - 'test_class': FioJobTest_t0020, + 'test_class': FioJobFileTest_t0020, 'job': 't0020.fio', 'success': SUCCESS_DEFAULT, 'pre_job': None, @@ -1184,7 +778,7 @@ TEST_LIST = [ }, { 'test_id': 21, - 'test_class': FioJobTest_t0020, + 'test_class': FioJobFileTest_t0020, 'job': 't0021.fio', 'success': SUCCESS_DEFAULT, 'pre_job': None, @@ -1193,7 +787,7 @@ TEST_LIST = [ }, { 'test_id': 22, - 'test_class': FioJobTest_t0022, + 'test_class': FioJobFileTest_t0022, 'job': 't0022.fio', 'success': SUCCESS_DEFAULT, 'pre_job': None, @@ -1202,7 +796,7 @@ TEST_LIST = [ }, { 'test_id': 23, - 'test_class': FioJobTest_t0023, + 'test_class': FioJobFileTest_t0023, 'job': 't0023.fio', 'success': SUCCESS_DEFAULT, 'pre_job': None, @@ -1211,7 +805,7 @@ TEST_LIST = [ }, { 'test_id': 24, - 'test_class': FioJobTest_t0024, + 'test_class': FioJobFileTest_t0024, 'job': 't0024.fio', 'success': SUCCESS_DEFAULT, 'pre_job': None, @@ -1220,7 +814,7 @@ TEST_LIST = [ }, { 'test_id': 25, - 'test_class': FioJobTest_t0025, + 'test_class': FioJobFileTest_t0025, 'job': 't0025.fio', 'success': SUCCESS_DEFAULT, 'pre_job': None, @@ -1230,7 +824,7 @@ TEST_LIST = [ }, { 'test_id': 26, - 'test_class': FioJobTest, + 'test_class': FioJobFileTest, 'job': 't0026.fio', 'success': SUCCESS_DEFAULT, 'pre_job': None, @@ -1239,7 +833,7 @@ TEST_LIST = [ }, { 'test_id': 27, - 'test_class': FioJobTest_t0027, + 'test_class': FioJobFileTest_t0027, 'job': 't0027.fio', 'success': SUCCESS_DEFAULT, 'pre_job': None, @@ -1248,13 +842,42 @@ TEST_LIST = [ }, { 'test_id': 28, - 'test_class': FioJobTest, + 'test_class': FioJobFileTest, 'job': 't0028-c6cade16.fio', 'success': SUCCESS_DEFAULT, 'pre_job': None, 'pre_success': None, 'requirements': [], }, + { + 'test_id': 29, + 'test_class': FioJobFileTest_t0029, + 'job': 't0029.fio', + 'success': SUCCESS_DEFAULT, + 'pre_job': None, + 'pre_success': None, + 'output_format': 'json', + 'requirements': [], + }, + { + 'test_id': 30, + 'test_class': FioJobFileTest, + 'job': 't0030.fio', + 'success': SUCCESS_DEFAULT, + 'pre_job': None, + 'pre_success': None, + 'parameters': ['--bandwidth-log'], + 'requirements': [], + }, + { + 'test_id': 31, + 'test_class': FioJobFileTest, + 'job': 't0031.fio', + 'success': SUCCESS_DEFAULT, + 'pre_job': 't0031-pre.fio', + 'pre_success': SUCCESS_DEFAULT, + 'requirements': [Requirements.linux, Requirements.libaio], + }, { 'test_id': 1000, 'test_class': FioExeTest, @@ -1307,7 +930,7 @@ TEST_LIST = [ 'test_id': 1006, 'test_class': FioExeTest, 'exe': 't/strided.py', - 'parameters': ['{fio_path}'], + 'parameters': ['--fio', '{fio_path}'], 'success': SUCCESS_DEFAULT, 'requirements': [], }, @@ -1369,6 +992,22 @@ TEST_LIST = [ 'success': SUCCESS_DEFAULT, 'requirements': [], }, + { + 'test_id': 1014, + 'test_class': FioExeTest, + 'exe': 't/nvmept.py', + 'parameters': ['-f', '{fio_path}', '--dut', '{nvmecdev}'], + 'success': SUCCESS_DEFAULT, + 'requirements': [Requirements.linux, Requirements.nvmecdev], + }, + { + 'test_id': 1015, + 'test_class': FioExeTest, + 'exe': 't/nvmept_trim.py', + 'parameters': ['-f', '{fio_path}', '--dut', '{nvmecdev}'], + 'success': SUCCESS_DEFAULT, + 'requirements': [Requirements.linux, Requirements.nvmecdev], + }, ] @@ -1392,6 +1031,8 @@ def parse_args(): help='skip requirements checking') parser.add_argument('-p', '--pass-through', action='append', help='pass-through an argument to an executable test') + parser.add_argument('--nvmecdev', action='store', default=None, + help='NVMe character device for **DESTRUCTIVE** testing (e.g., /dev/ng0n1)') args = parser.parse_args() return args @@ -1410,7 +1051,7 @@ def main(): if args.pass_through: for arg in args.pass_through: if not ':' in arg: - print("Invalid --pass-through argument '%s'" % arg) + print(f"Invalid --pass-through argument '{arg}'") print("Syntax for --pass-through is TESTNUMBER:ARGUMENT") return split = arg.split(":", 1) @@ -1421,7 +1062,7 @@ def main(): fio_root = args.fio_root else: fio_root = str(Path(__file__).absolute().parent.parent) - print("fio root is %s" % fio_root) + print(f"fio root is {fio_root}") if args.fio: fio_path = args.fio @@ -1431,107 +1072,25 @@ def main(): else: fio_exe = "fio" fio_path = os.path.join(fio_root, fio_exe) - print("fio path is %s" % fio_path) + print(f"fio path is {fio_path}") if not shutil.which(fio_path): print("Warning: fio executable not found") artifact_root = args.artifact_root if args.artifact_root else \ - "fio-test-{0}".format(time.strftime("%Y%m%d-%H%M%S")) + f"fio-test-{time.strftime('%Y%m%d-%H%M%S')}" os.mkdir(artifact_root) - print("Artifact directory is %s" % artifact_root) + print(f"Artifact directory is {artifact_root}") if not args.skip_req: - req = Requirements(fio_root) - - passed = 0 - failed = 0 - skipped = 0 - - for config in TEST_LIST: - if (args.skip and config['test_id'] in args.skip) or \ - (args.run_only and config['test_id'] not in args.run_only): - skipped = skipped + 1 - print("Test {0} SKIPPED (User request)".format(config['test_id'])) - continue - - if issubclass(config['test_class'], FioJobTest): - if config['pre_job']: - fio_pre_job = os.path.join(fio_root, 't', 'jobs', - config['pre_job']) - else: - fio_pre_job = None - if config['pre_success']: - fio_pre_success = config['pre_success'] - else: - fio_pre_success = None - if 'output_format' in config: - output_format = config['output_format'] - else: - output_format = 'normal' - test = config['test_class']( - fio_path, - os.path.join(fio_root, 't', 'jobs', config['job']), - config['success'], - fio_pre_job=fio_pre_job, - fio_pre_success=fio_pre_success, - output_format=output_format) - desc = config['job'] - elif issubclass(config['test_class'], FioExeTest): - exe_path = os.path.join(fio_root, config['exe']) - if config['parameters']: - parameters = [p.format(fio_path=fio_path) for p in config['parameters']] - else: - parameters = [] - if Path(exe_path).suffix == '.py' and platform.system() == "Windows": - parameters.insert(0, exe_path) - exe_path = "python.exe" - if config['test_id'] in pass_through: - parameters += pass_through[config['test_id']].split() - test = config['test_class'](exe_path, parameters, - config['success']) - desc = config['exe'] - else: - print("Test {0} FAILED: unable to process test config".format(config['test_id'])) - failed = failed + 1 - continue - - if not args.skip_req: - reqs_met = True - for req in config['requirements']: - reqs_met, reason = req() - logging.debug("Test %d: Requirement '%s' met? %s", config['test_id'], reason, - reqs_met) - if not reqs_met: - break - if not reqs_met: - print("Test {0} SKIPPED ({1}) {2}".format(config['test_id'], reason, desc)) - skipped = skipped + 1 - continue - - try: - test.setup(artifact_root, config['test_id']) - test.run() - test.check_result() - except KeyboardInterrupt: - break - except Exception as e: - test.passed = False - test.failure_reason += str(e) - logging.debug("Test %d exception:\n%s\n", config['test_id'], traceback.format_exc()) - if test.passed: - result = "PASSED" - passed = passed + 1 - else: - result = "FAILED: {0}".format(test.failure_reason) - failed = failed + 1 - contents, _ = FioJobTest.get_file(test.stderr_file) - logging.debug("Test %d: stderr:\n%s", config['test_id'], contents) - contents, _ = FioJobTest.get_file(test.stdout_file) - logging.debug("Test %d: stdout:\n%s", config['test_id'], contents) - print("Test {0} {1} {2}".format(config['test_id'], result, desc)) - - print("{0} test(s) passed, {1} failed, {2} skipped".format(passed, failed, skipped)) - + Requirements(fio_root, args) + + test_env = { + 'fio_path': fio_path, + 'fio_root': fio_root, + 'artifact_root': artifact_root, + 'pass_through': pass_through, + } + _, failed, _ = run_fio_tests(TEST_LIST, test_env, args) sys.exit(failed)