--- /dev/null
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (c) 2019 Western Digital Corporation or its affiliates.
+#
+"""
+# run-fio-tests.py
+#
+# Automate running of fio tests
+#
+# USAGE
+# python3 run-fio-tests.py [-r fio-root] [-f fio-path] [-a artifact-root]
+# [--skip # # #...] [--run-only # # #...]
+#
+#
+# EXAMPLE
+# # git clone [fio-repository]
+# # cd fio
+# # make -j
+# # python3 t/run-fio-tests.py
+#
+#
+# REQUIREMENTS
+# - Python 3
+# - Linux (libaio ioengine, zbd tests, etc)
+# - The artifact directory must be on a file system that accepts 512-byte IO
+# (t0002, t0003, t0004).
+# - The artifact directory needs to be on an SSD. Otherwise tests that carry
+# out file-based IO will trigger a timeout (t0006).
+# - 4 CPUs (t0009)
+# - SciPy (steadystate_tests.py)
+# - libzbc (zbd tests)
+# - root privileges (zbd test)
+# - kernel 4.19 or later for zoned null block devices (zbd tests)
+# - CUnit support (unittests)
+#
+"""
+
+#
+# TODO run multiple tests simultaneously
+# TODO Add sgunmap tests (requires SAS SSD)
+# TODO automatically detect dependencies and skip tests accordingly
+#
+
+import os
+import sys
+import json
+import time
+import logging
+import argparse
+import subprocess
+from pathlib import Path
+
+
+class FioTest(object):
+ """Base for all fio tests."""
+
+ def __init__(self, exe_path, parameters, success):
+ self.exe_path = exe_path
+ self.parameters = parameters
+ self.success = success
+ self.output = {}
+ self.artifact_root = None
+ self.testnum = None
+ self.test_dir = None
+ self.passed = True
+ self.failure_reason = ''
+
+ def setup(self, artifact_root, testnum):
+ self.artifact_root = artifact_root
+ self.testnum = testnum
+ self.test_dir = os.path.join(artifact_root, "{:04d}".format(testnum))
+ if not os.path.exists(self.test_dir):
+ os.mkdir(self.test_dir)
+
+ self.command_file = os.path.join(
+ self.test_dir,
+ "{0}.command".format(os.path.basename(self.exe_path)))
+ self.stdout_file = os.path.join(
+ self.test_dir,
+ "{0}.stdout".format(os.path.basename(self.exe_path)))
+ self.stderr_file = os.path.join(
+ self.test_dir,
+ "{0}.stderr".format(os.path.basename(self.exe_path)))
+ self.exticode_file = os.path.join(
+ self.test_dir,
+ "{0}.exitcode".format(os.path.basename(self.exe_path)))
+
+ def run(self):
+ raise NotImplementedError()
+
+ def check_result(self):
+ raise NotImplementedError()
+
+
+class FioExeTest(FioTest):
+ """Test consists of an executable binary or script"""
+
+ def __init__(self, exe_path, parameters, success):
+ """Construct a FioExeTest which is a FioTest consisting of an
+ executable binary or script.
+
+ exe_path: location of executable binary or script
+ parameters: list of parameters for executable
+ success: Definition of test success
+ """
+
+ FioTest.__init__(self, exe_path, parameters, success)
+
+ def setup(self, artifact_root, testnum):
+ super(FioExeTest, self).setup(artifact_root, testnum)
+
+ def run(self):
+ if self.parameters:
+ command = [self.exe_path] + self.parameters
+ else:
+ command = [self.exe_path]
+ command_file = open(self.command_file, "w+")
+ command_file.write("%s\n" % command)
+ command_file.close()
+
+ stdout_file = open(self.stdout_file, "w+")
+ stderr_file = open(self.stderr_file, "w+")
+ exticode_file = open(self.exticode_file, "w+")
+ try:
+ # Avoid using subprocess.run() here because when a timeout occurs,
+ # fio will be stopped with SIGKILL. This does not give fio a
+ # chance to clean up and means that child processes may continue
+ # running and submitting IO.
+ proc = subprocess.Popen(command,
+ stdout=stdout_file,
+ stderr=stderr_file,
+ cwd=self.test_dir,
+ universal_newlines=True)
+ proc.communicate(timeout=self.success['timeout'])
+ exticode_file.write('{0}\n'.format(proc.returncode))
+ logging.debug("return code: %d" % proc.returncode)
+ self.output['proc'] = proc
+ except subprocess.TimeoutExpired:
+ proc.terminate()
+ proc.communicate()
+ assert proc.poll()
+ self.output['failure'] = 'timeout'
+ except Exception:
+ if not proc.poll():
+ proc.terminate()
+ proc.communicate()
+ self.output['failure'] = 'exception'
+ self.output['exc_info'] = sys.exc_info()
+ finally:
+ stdout_file.close()
+ stderr_file.close()
+ exticode_file.close()
+
+ def check_result(self):
+ if 'proc' not in self.output:
+ if self.output['failure'] == 'timeout':
+ self.failure_reason = "{0} timeout,".format(self.failure_reason)
+ else:
+ assert self.output['failure'] == 'exception'
+ self.failure_reason = '{0} exception: {1}, {2}'.format(
+ self.failure_reason, self.output['exc_info'][0],
+ self.output['exc_info'][1])
+
+ self.passed = False
+ return
+
+ if 'zero_return' in self.success:
+ if self.success['zero_return']:
+ if self.output['proc'].returncode != 0:
+ self.passed = False
+ self.failure_reason = "{0} non-zero return code,".format(self.failure_reason)
+ else:
+ if self.output['proc'].returncode == 0:
+ self.failure_reason = "{0} zero return code,".format(self.failure_reason)
+ self.passed = False
+
+ if 'stderr_empty' in self.success:
+ stderr_size = os.path.getsize(self.stderr_file)
+ if self.success['stderr_empty']:
+ if stderr_size != 0:
+ self.failure_reason = "{0} stderr not empty,".format(self.failure_reason)
+ self.passed = False
+ else:
+ if stderr_size == 0:
+ self.failure_reason = "{0} stderr empty,".format(self.failure_reason)
+ self.passed = False
+
+
+class FioJobTest(FioExeTest):
+ """Test consists of a fio job"""
+
+ def __init__(self, fio_path, fio_job, success, fio_pre_job=None,
+ fio_pre_success=None, output_format="normal"):
+ """Construct a FioJobTest which is a FioExeTest consisting of a
+ single fio job file with an optional setup step.
+
+ fio_path: location of fio executable
+ fio_job: location of fio job file
+ success: Definition of test success
+ fio_pre_job: fio job for preconditioning
+ fio_pre_success: Definition of test success for fio precon job
+ output_format: normal (default), json, jsonplus, or terse
+ """
+
+ self.fio_job = fio_job
+ self.fio_pre_job = fio_pre_job
+ self.fio_pre_success = fio_pre_success if fio_pre_success else success
+ self.output_format = output_format
+ self.precon_failed = False
+ self.json_data = None
+ self.fio_output = "{0}.output".format(os.path.basename(self.fio_job))
+ self.fio_args = [
+ "--output-format={0}".format(self.output_format),
+ "--output={0}".format(self.fio_output),
+ self.fio_job,
+ ]
+ FioExeTest.__init__(self, fio_path, self.fio_args, success)
+
+ def setup(self, artifact_root, testnum):
+ super(FioJobTest, self).setup(artifact_root, testnum)
+
+ self.command_file = os.path.join(
+ self.test_dir,
+ "{0}.command".format(os.path.basename(self.fio_job)))
+ self.stdout_file = os.path.join(
+ self.test_dir,
+ "{0}.stdout".format(os.path.basename(self.fio_job)))
+ self.stderr_file = os.path.join(
+ self.test_dir,
+ "{0}.stderr".format(os.path.basename(self.fio_job)))
+ self.exticode_file = os.path.join(
+ self.test_dir,
+ "{0}.exitcode".format(os.path.basename(self.fio_job)))
+
+ def run_pre_job(self):
+ precon = FioJobTest(self.exe_path, self.fio_pre_job,
+ self.fio_pre_success,
+ output_format=self.output_format)
+ precon.setup(self.artifact_root, self.testnum)
+ precon.run()
+ precon.check_result()
+ self.precon_failed = not precon.passed
+ self.failure_reason = precon.failure_reason
+
+ def run(self):
+ if self.fio_pre_job:
+ self.run_pre_job()
+
+ if not self.precon_failed:
+ super(FioJobTest, self).run()
+ else:
+ logging.debug("precondition step failed")
+
+ def check_result(self):
+ if self.precon_failed:
+ self.passed = False
+ self.failure_reason = "{0} precondition step failed,".format(self.failure_reason)
+ return
+
+ super(FioJobTest, self).check_result()
+
+ if 'json' in self.output_format:
+ output_file = open(os.path.join(self.test_dir, self.fio_output), "r")
+ file_data = output_file.read()
+ output_file.close()
+ try:
+ self.json_data = json.loads(file_data)
+ except json.JSONDecodeError:
+ self.failure_reason = "{0} unable to decode JSON data,".format(self.failure_reason)
+ self.passed = False
+
+
+class FioJobTest_t0005(FioJobTest):
+ """Test consists of fio test job t0005
+ Confirm that read['io_kbytes'] == write['io_kbytes'] == 102400"""
+
+ def check_result(self):
+ super(FioJobTest_t0005, self).check_result()
+
+ if not self.passed:
+ return
+
+ if self.json_data['jobs'][0]['read']['io_kbytes'] != 102400:
+ self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
+ self.passed = False
+ if self.json_data['jobs'][0]['write']['io_kbytes'] != 102400:
+ self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
+ self.passed = False
+
+
+class FioJobTest_t0006(FioJobTest):
+ """Test consists of fio test job t0006
+ Confirm that read['io_kbytes'] ~ 2*write['io_kbytes']"""
+
+ def check_result(self):
+ super(FioJobTest_t0006, self).check_result()
+
+ if not self.passed:
+ return
+
+ ratio = self.json_data['jobs'][0]['read']['io_kbytes'] \
+ / self.json_data['jobs'][0]['write']['io_kbytes']
+ logging.debug("ratio: %f" % ratio)
+ if ratio < 1.99 or ratio > 2.01:
+ self.failure_reason = "{0} read/write ratio mismatch,".format(self.failure_reason)
+ self.passed = False
+
+
+class FioJobTest_t0007(FioJobTest):
+ """Test consists of fio test job t0007
+ Confirm that read['io_kbytes'] = 87040"""
+
+ def check_result(self):
+ super(FioJobTest_t0007, self).check_result()
+
+ if not self.passed:
+ return
+
+ if self.json_data['jobs'][0]['read']['io_kbytes'] != 87040:
+ self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
+ self.passed = False
+
+
+class FioJobTest_t0008(FioJobTest):
+ """Test consists of fio test job t0008
+ Confirm that read['io_kbytes'] = 32768 and that
+ write['io_kbytes'] ~ 16568
+
+ I did runs with fio-ae2fafc8 and saw write['io_kbytes'] values of
+ 16585, 16588. With two runs of fio-3.16 I obtained 16568"""
+
+ def check_result(self):
+ super(FioJobTest_t0008, self).check_result()
+
+ if not self.passed:
+ return
+
+ ratio = self.json_data['jobs'][0]['write']['io_kbytes'] / 16568
+ logging.debug("ratio: %f" % ratio)
+
+ if ratio < 0.99 or ratio > 1.01:
+ self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
+ self.passed = False
+ if self.json_data['jobs'][0]['read']['io_kbytes'] != 32768:
+ self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
+ self.passed = False
+
+
+class FioJobTest_t0009(FioJobTest):
+ """Test consists of fio test job t0009
+ Confirm that runtime >= 60s"""
+
+ def check_result(self):
+ super(FioJobTest_t0009, self).check_result()
+
+ if not self.passed:
+ return
+
+ logging.debug('elapsed: %d' % self.json_data['jobs'][0]['elapsed'])
+
+ if self.json_data['jobs'][0]['elapsed'] < 60:
+ self.failure_reason = "{0} elapsed time mismatch,".format(self.failure_reason)
+ self.passed = False
+
+
+class FioJobTest_t0011(FioJobTest):
+ """Test consists of fio test job t0009
+ Confirm that job0 iops == 1000
+ and that job1_iops / job0_iops ~ 8
+ With two runs of fio-3.16 I observed a ratio of 8.3"""
+
+ def check_result(self):
+ super(FioJobTest_t0011, self).check_result()
+
+ if not self.passed:
+ return
+
+ iops1 = self.json_data['jobs'][0]['read']['iops']
+ iops2 = self.json_data['jobs'][1]['read']['iops']
+ ratio = iops2 / iops1
+ logging.debug("ratio: %f" % ratio)
+
+ if iops1 < 999 or iops1 > 1001:
+ self.failure_reason = "{0} iops value mismatch,".format(self.failure_reason)
+ self.passed = False
+
+ if ratio < 7 or ratio > 9:
+ self.failure_reason = "{0} iops ratio mismatch,".format(self.failure_reason)
+ self.passed = False
+
+
+SUCCESS_DEFAULT = {
+ 'zero_return': True,
+ 'stderr_empty': True,
+ 'timeout': 300,
+ }
+SUCCESS_NONZERO = {
+ 'zero_return': False,
+ 'stderr_empty': False,
+ 'timeout': 300,
+ }
+SUCCESS_STDERR = {
+ 'zero_return': True,
+ 'stderr_empty': False,
+ 'timeout': 300,
+ }
+TEST_LIST = [
+ {
+ 'test_id': 1,
+ 'test_class': FioJobTest,
+ 'job': 't0001-52c58027.fio',
+ 'success': SUCCESS_DEFAULT,
+ 'pre_job': None,
+ 'pre_success': None,
+ },
+ {
+ 'test_id': 2,
+ 'test_class': FioJobTest,
+ 'job': 't0002-13af05ae-post.fio',
+ 'success': SUCCESS_DEFAULT,
+ 'pre_job': 't0002-13af05ae-pre.fio',
+ 'pre_success': None,
+ },
+ {
+ 'test_id': 3,
+ 'test_class': FioJobTest,
+ 'job': 't0003-0ae2c6e1-post.fio',
+ 'success': SUCCESS_NONZERO,
+ 'pre_job': 't0003-0ae2c6e1-pre.fio',
+ 'pre_success': SUCCESS_DEFAULT,
+ },
+ {
+ 'test_id': 4,
+ 'test_class': FioJobTest,
+ 'job': 't0004-8a99fdf6.fio',
+ 'success': SUCCESS_DEFAULT,
+ 'pre_job': None,
+ 'pre_success': None,
+ },
+ {
+ 'test_id': 5,
+ 'test_class': FioJobTest_t0005,
+ 'job': 't0005-f7078f7b.fio',
+ 'success': SUCCESS_DEFAULT,
+ 'pre_job': None,
+ 'pre_success': None,
+ 'output_format': 'json',
+ },
+ {
+ 'test_id': 6,
+ 'test_class': FioJobTest_t0006,
+ 'job': 't0006-82af2a7c.fio',
+ 'success': SUCCESS_DEFAULT,
+ 'pre_job': None,
+ 'pre_success': None,
+ 'output_format': 'json',
+ },
+ {
+ 'test_id': 7,
+ 'test_class': FioJobTest_t0007,
+ 'job': 't0007-37cf9e3c.fio',
+ 'success': SUCCESS_DEFAULT,
+ 'pre_job': None,
+ 'pre_success': None,
+ 'output_format': 'json',
+ },
+ {
+ 'test_id': 8,
+ 'test_class': FioJobTest_t0008,
+ 'job': 't0008-ae2fafc8.fio',
+ 'success': SUCCESS_DEFAULT,
+ 'pre_job': None,
+ 'pre_success': None,
+ 'output_format': 'json',
+ },
+ {
+ 'test_id': 9,
+ 'test_class': FioJobTest_t0009,
+ 'job': 't0009-f8b0bd10.fio',
+ 'success': SUCCESS_DEFAULT,
+ 'pre_job': None,
+ 'pre_success': None,
+ 'output_format': 'json',
+ },
+ {
+ 'test_id': 10,
+ 'test_class': FioJobTest,
+ 'job': 't0010-b7aae4ba.fio',
+ 'success': SUCCESS_DEFAULT,
+ 'pre_job': None,
+ 'pre_success': None,
+ },
+ {
+ 'test_id': 11,
+ 'test_class': FioJobTest_t0011,
+ 'job': 't0011-5d2788d5.fio',
+ 'success': SUCCESS_DEFAULT,
+ 'pre_job': None,
+ 'pre_success': None,
+ 'output_format': 'json',
+ },
+ {
+ 'test_id': 1000,
+ 'test_class': FioExeTest,
+ 'exe': 't/axmap',
+ 'parameters': None,
+ 'success': SUCCESS_DEFAULT,
+ },
+ {
+ 'test_id': 1001,
+ 'test_class': FioExeTest,
+ 'exe': 't/ieee754',
+ 'parameters': None,
+ 'success': SUCCESS_DEFAULT,
+ },
+ {
+ 'test_id': 1002,
+ 'test_class': FioExeTest,
+ 'exe': 't/lfsr-test',
+ 'parameters': ['0xFFFFFF', '0', '0', 'verify'],
+ 'success': SUCCESS_STDERR,
+ },
+ {
+ 'test_id': 1003,
+ 'test_class': FioExeTest,
+ 'exe': 't/readonly.py',
+ 'parameters': ['-f', '{fio_path}'],
+ 'success': SUCCESS_DEFAULT,
+ },
+ {
+ 'test_id': 1004,
+ 'test_class': FioExeTest,
+ 'exe': 't/steadystate_tests.py',
+ 'parameters': ['{fio_path}'],
+ 'success': SUCCESS_DEFAULT,
+ },
+ {
+ 'test_id': 1005,
+ 'test_class': FioExeTest,
+ 'exe': 't/stest',
+ 'parameters': None,
+ 'success': SUCCESS_STDERR,
+ },
+ {
+ 'test_id': 1006,
+ 'test_class': FioExeTest,
+ 'exe': 't/strided.py',
+ 'parameters': ['{fio_path}'],
+ 'success': SUCCESS_DEFAULT,
+ },
+ {
+ 'test_id': 1007,
+ 'test_class': FioExeTest,
+ 'exe': 't/zbd/run-tests-against-regular-nullb',
+ 'parameters': None,
+ 'success': SUCCESS_DEFAULT,
+ },
+ {
+ 'test_id': 1008,
+ 'test_class': FioExeTest,
+ 'exe': 't/zbd/run-tests-against-zoned-nullb',
+ 'parameters': None,
+ 'success': SUCCESS_DEFAULT,
+ },
+ {
+ 'test_id': 1009,
+ 'test_class': FioExeTest,
+ 'exe': 'unittests/unittest',
+ 'parameters': None,
+ 'success': SUCCESS_DEFAULT,
+ },
+]
+
+
+def parse_args():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-r', '--fio-root',
+ help='fio root path')
+ parser.add_argument('-f', '--fio',
+ help='path to fio executable (e.g., ./fio)')
+ parser.add_argument('-a', '--artifact-root',
+ help='artifact root directory')
+ parser.add_argument('-s', '--skip', nargs='+', type=int,
+ help='list of test(s) to skip')
+ parser.add_argument('-o', '--run-only', nargs='+', type=int,
+ help='list of test(s) to run, skipping all others')
+ args = parser.parse_args()
+
+ return args
+
+
+def main():
+ logging.basicConfig(level=logging.INFO)
+
+ args = parse_args()
+ if args.fio_root:
+ fio_root = args.fio_root
+ else:
+ fio_root = Path(__file__).absolute().parent.parent
+ logging.debug("fio_root: %s" % fio_root)
+
+ if args.fio:
+ fio_path = args.fio
+ else:
+ fio_path = os.path.join(fio_root, "fio")
+ logging.debug("fio_path: %s" % fio_path)
+
+ artifact_root = args.artifact_root if args.artifact_root else \
+ "fio-test-{0}".format(time.strftime("%Y%m%d-%H%M%S"))
+ os.mkdir(artifact_root)
+ print("Artifact directory is %s" % artifact_root)
+
+ passed = 0
+ failed = 0
+ skipped = 0
+
+ for config in TEST_LIST:
+ if (args.skip and config['test_id'] in args.skip) or \
+ (args.run_only and config['test_id'] not in args.run_only):
+ skipped = skipped + 1
+ print("Test {0} SKIPPED".format(config['test_id']))
+ continue
+
+ if issubclass(config['test_class'], FioJobTest):
+ if config['pre_job']:
+ fio_pre_job = os.path.join(fio_root, 't', 'jobs',
+ config['pre_job'])
+ else:
+ fio_pre_job = None
+ if config['pre_success']:
+ fio_pre_success = config['pre_success']
+ else:
+ fio_pre_success = None
+ if 'output_format' in config:
+ output_format = config['output_format']
+ else:
+ output_format = 'normal'
+ test = config['test_class'](
+ fio_path,
+ os.path.join(fio_root, 't', 'jobs', config['job']),
+ config['success'],
+ fio_pre_job=fio_pre_job,
+ fio_pre_success=fio_pre_success,
+ output_format=output_format)
+ elif issubclass(config['test_class'], FioExeTest):
+ exe_path = os.path.join(fio_root, config['exe'])
+ if config['parameters']:
+ parameters = [p.format(fio_path=fio_path) for p in config['parameters']]
+ else:
+ parameters = None
+ test = config['test_class'](exe_path, parameters,
+ config['success'])
+ else:
+ print("Test {0} FAILED: unable to process test config".format(config['test_id']))
+ failed = failed + 1
+ continue
+
+ test.setup(artifact_root, config['test_id'])
+ test.run()
+ test.check_result()
+ if test.passed:
+ result = "PASSED"
+ passed = passed + 1
+ else:
+ result = "FAILED: {0}".format(test.failure_reason)
+ failed = failed + 1
+ print("Test {0} {1}".format(config['test_id'], result))
+
+ print("{0} test(s) passed, {1} failed, {2} skipped".format(passed, failed, skipped))
+
+ sys.exit(failed)
+
+
+if __name__ == '__main__':
+ main()
projects / fio.git / commitdiff