[fio.git] / t / fiotestlib.py

#!/usr/bin/env python3
"""
fiotestlib.py

This library contains FioTest objects that provide convenient means to run
different sorts of fio tests.

It also contains a test runner that runs an array of dictionary objects
describing fio tests.
"""

import os
import sys
import json
import locale
import logging
import platform
import traceback
import subprocess
from pathlib import Path
from fiotestcommon import get_file


class FioTest():
    """Base for all fio tests."""

    def __init__(self, exe_path, parameters, success):
        self.exe_path = exe_path
        self.parameters = parameters
        self.success = success
        self.output = {}
        self.artifact_root = None
        self.testnum = None
        self.test_dir = None
        self.passed = True
        self.failure_reason = ''
        self.command_file = None
        self.stdout_file = None
        self.stderr_file = None
        self.exitcode_file = None

    def setup(self, artifact_root, testnum):
        """Setup instance variables for test."""

        self.artifact_root = artifact_root
        self.testnum = testnum
        self.test_dir = os.path.join(artifact_root, f"{testnum:04d}")
        if not os.path.exists(self.test_dir):
            os.mkdir(self.test_dir)

        self.command_file = os.path.join( self.test_dir,
                                         f"{os.path.basename(self.exe_path)}.command")
        self.stdout_file = os.path.join( self.test_dir,
                                        f"{os.path.basename(self.exe_path)}.stdout")
        self.stderr_file = os.path.join( self.test_dir,
                                        f"{os.path.basename(self.exe_path)}.stderr")
        self.exitcode_file = os.path.join( self.test_dir,
                                          f"{os.path.basename(self.exe_path)}.exitcode")

    def run(self):
        """Run the test."""

        raise NotImplementedError()

    def check_result(self):
        """Check test results."""

        raise NotImplementedError()


class FioExeTest(FioTest):
    """Test consists of an executable binary or script"""

    def __init__(self, exe_path, parameters, success):
        """Construct a FioExeTest which is a FioTest consisting of an
        executable binary or script.

        exe_path:       location of executable binary or script
        parameters:     list of parameters for executable
        success:        Definition of test success
        """

        FioTest.__init__(self, exe_path, parameters, success)

    def run(self):
        """Execute the binary or script described by this instance."""

        command = [self.exe_path] + self.parameters
        command_file = open(self.command_file, "w+",
                            encoding=locale.getpreferredencoding())
        command_file.write(f"{command}\n")
        command_file.close()

        stdout_file = open(self.stdout_file, "w+",
                           encoding=locale.getpreferredencoding())
        stderr_file = open(self.stderr_file, "w+",
                           encoding=locale.getpreferredencoding())
        exitcode_file = open(self.exitcode_file, "w+",
                             encoding=locale.getpreferredencoding())
        try:
            proc = None
            # Avoid using subprocess.run() here because when a timeout occurs,
            # fio will be stopped with SIGKILL. This does not give fio a
            # chance to clean up and means that child processes may continue
            # running and submitting IO.
            proc = subprocess.Popen(command,
                                    stdout=stdout_file,
                                    stderr=stderr_file,
                                    cwd=self.test_dir,
                                    universal_newlines=True)
            proc.communicate(timeout=self.success['timeout'])
            exitcode_file.write(f'{proc.returncode}\n')
            logging.debug("Test %d: return code: %d", self.testnum, proc.returncode)
            self.output['proc'] = proc
        except subprocess.TimeoutExpired:
            proc.terminate()
            proc.communicate()
            assert proc.poll()
            self.output['failure'] = 'timeout'
        except Exception:
            if proc:
                if not proc.poll():
                    proc.terminate()
                    proc.communicate()
            self.output['failure'] = 'exception'
            self.output['exc_info'] = sys.exc_info()
        finally:
            stdout_file.close()
            stderr_file.close()
            exitcode_file.close()

    def check_result(self):
        """Check results of test run."""

        if 'proc' not in self.output:
            if self.output['failure'] == 'timeout':
                self.failure_reason = f"{self.failure_reason} timeout,"
            else:
                assert self.output['failure'] == 'exception'
                self.failure_reason = '{0} exception: {1}, {2}'.format(
                    self.failure_reason, self.output['exc_info'][0],
                    self.output['exc_info'][1])

            self.passed = False
            return

        if 'zero_return' in self.success:
            if self.success['zero_return']:
                if self.output['proc'].returncode != 0:
                    self.passed = False
                    self.failure_reason = f"{self.failure_reason} non-zero return code,"
            else:
                if self.output['proc'].returncode == 0:
                    self.failure_reason = f"{self.failure_reason} zero return code,"
                    self.passed = False

        stderr_size = os.path.getsize(self.stderr_file)
        if 'stderr_empty' in self.success:
            if self.success['stderr_empty']:
                if stderr_size != 0:
                    self.failure_reason = f"{self.failure_reason} stderr not empty,"
                    self.passed = False
            else:
                if stderr_size == 0:
                    self.failure_reason = f"{self.failure_reason} stderr empty,"
                    self.passed = False


class FioJobFileTest(FioExeTest):
    """Test consists of a fio job"""

    def __init__(self, fio_path, fio_job, success, fio_pre_job=None,
                 fio_pre_success=None, output_format="normal"):
        """Construct a FioJobFileTest which is a FioExeTest consisting of a
        single fio job file with an optional setup step.

        fio_path:           location of fio executable
        fio_job:            location of fio job file
        success:            Definition of test success
        fio_pre_job:        fio job for preconditioning
        fio_pre_success:    Definition of test success for fio precon job
        output_format:      normal (default), json, jsonplus, or terse
        """

        self.fio_job = fio_job
        self.fio_pre_job = fio_pre_job
        self.fio_pre_success = fio_pre_success if fio_pre_success else success
        self.output_format = output_format
        self.precon_failed = False
        self.json_data = None
        self.fio_output = f"{os.path.basename(self.fio_job)}.output"
        self.fio_args = [
            "--max-jobs=16",
            f"--output-format={self.output_format}",
            f"--output={self.fio_output}",
            self.fio_job,
            ]
        FioExeTest.__init__(self, fio_path, self.fio_args, success)

    def setup(self, artifact_root, testnum):
        """Setup instance variables for fio job test."""

        super().setup(artifact_root, testnum)

        self.command_file = os.path.join(self.test_dir,
                                         f"{os.path.basename(self.fio_job)}.command")
        self.stdout_file = os.path.join(self.test_dir,
                                        f"{os.path.basename(self.fio_job)}.stdout")
        self.stderr_file = os.path.join(self.test_dir,
                                        f"{os.path.basename(self.fio_job)}.stderr")
        self.exitcode_file = os.path.join(self.test_dir,
                                          f"{os.path.basename(self.fio_job)}.exitcode")

    def run_pre_job(self):
        """Run fio job precondition step."""

        precon = FioJobFileTest(self.exe_path, self.fio_pre_job,
                            self.fio_pre_success,
                            output_format=self.output_format)
        precon.setup(self.artifact_root, self.testnum)
        precon.run()
        precon.check_result()
        self.precon_failed = not precon.passed
        self.failure_reason = precon.failure_reason

    def run(self):
        """Run fio job test."""

        if self.fio_pre_job:
            self.run_pre_job()

        if not self.precon_failed:
            super().run()
        else:
            logging.debug("Test %d: precondition step failed", self.testnum)

    def get_file_fail(self, filename):
        """Safely read a file and fail the test upon error."""
        file_data = None

        try:
            with open(filename, "r", encoding=locale.getpreferredencoding()) as output_file:
                file_data = output_file.read()
        except OSError:
            self.failure_reason += f" unable to read file {filename}"
            self.passed = False

        return file_data

    def check_result(self):
        """Check fio job results."""

        if self.precon_failed:
            self.passed = False
            self.failure_reason = f"{self.failure_reason} precondition step failed,"
            return

        super().check_result()

        if not self.passed:
            return

        if 'json' not in self.output_format:
            return

        file_data = self.get_file_fail(os.path.join(self.test_dir, self.fio_output))
        if not file_data:
            return

        #
        # Sometimes fio informational messages are included at the top of the
        # JSON output, especially under Windows. Try to decode output as JSON
        # data, skipping everything until the first {
        #
        lines = file_data.splitlines()
        file_data = '\n'.join(lines[lines.index("{"):])
        try:
            self.json_data = json.loads(file_data)
        except json.JSONDecodeError:
            self.failure_reason = f"{self.failure_reason} unable to decode JSON data,"
            self.passed = False


def run_fio_tests(test_list, test_env, args):
    """
    Run tests as specified in test_list.
    """

    passed = 0
    failed = 0
    skipped = 0

    for config in test_list:
        if (args.skip and config['test_id'] in args.skip) or \
           (args.run_only and config['test_id'] not in args.run_only):
            skipped = skipped + 1
            print(f"Test {config['test_id']} SKIPPED (User request)")
            continue

        if issubclass(config['test_class'], FioJobFileTest):
            if config['pre_job']:
                fio_pre_job = os.path.join(test_env['fio_root'], 't', 'jobs',
                                           config['pre_job'])
            else:
                fio_pre_job = None
            if config['pre_success']:
                fio_pre_success = config['pre_success']
            else:
                fio_pre_success = None
            if 'output_format' in config:
                output_format = config['output_format']
            else:
                output_format = 'normal'
            test = config['test_class'](
                test_env['fio_path'],
                os.path.join(test_env['fio_root'], 't', 'jobs', config['job']),
                config['success'],
                fio_pre_job=fio_pre_job,
                fio_pre_success=fio_pre_success,
                output_format=output_format)
            desc = config['job']
        elif issubclass(config['test_class'], FioExeTest):
            exe_path = os.path.join(test_env['fio_root'], config['exe'])
            if config['parameters']:
                parameters = [p.format(fio_path=test_env['fio_path'], nvmecdev=args.nvmecdev)
                              for p in config['parameters']]
            else:
                parameters = []
            if Path(exe_path).suffix == '.py' and platform.system() == "Windows":
                parameters.insert(0, exe_path)
                exe_path = "python.exe"
            if config['test_id'] in test_env['pass_through']:
                parameters += test_env['pass_through'][config['test_id']].split()
            test = config['test_class'](exe_path, parameters,
                                        config['success'])
            desc = config['exe']
        else:
            print(f"Test {config['test_id']} FAILED: unable to process test config")
            failed = failed + 1
            continue

        if not args.skip_req:
            reqs_met = True
            for req in config['requirements']:
                reqs_met, reason = req()
                logging.debug("Test %d: Requirement '%s' met? %s", config['test_id'], reason,
                              reqs_met)
                if not reqs_met:
                    break
            if not reqs_met:
                print(f"Test {config['test_id']} SKIPPED ({reason}) {desc}")
                skipped = skipped + 1
                continue

        try:
            test.setup(test_env['artifact_root'], config['test_id'])
            test.run()
            test.check_result()
        except KeyboardInterrupt:
            break
        except Exception as e:
            test.passed = False
            test.failure_reason += str(e)
            logging.debug("Test %d exception:\n%s\n", config['test_id'], traceback.format_exc())
        if test.passed:
            result = "PASSED"
            passed = passed + 1
        else:
            result = f"FAILED: {test.failure_reason}"
            failed = failed + 1
            contents, _ = get_file(test.stderr_file)
            logging.debug("Test %d: stderr:\n%s", config['test_id'], contents)
            contents, _ = get_file(test.stdout_file)
            logging.debug("Test %d: stdout:\n%s", config['test_id'], contents)
        print(f"Test {config['test_id']} {result} {desc}")

    print(f"{passed} test(s) passed, {failed} failed, {skipped} skipped")

    return passed, failed, skipped
Commit	Line	Data
fb551941 VF	1	#!/usr/bin/env python3
	2	"""
	3	fiotestlib.py
	4
	5	This library contains FioTest objects that provide convenient means to run
	6	different sorts of fio tests.
	7
	8	It also contains a test runner that runs an array of dictionary objects
	9	describing fio tests.
	10	"""
	11
	12	import os
	13	import sys
	14	import json
	15	import locale
	16	import logging
	17	import platform
	18	import traceback
	19	import subprocess
	20	from pathlib import Path
ecd6d30f	21	from fiotestcommon import get_file
fb551941 VF	22
	23
	24	class FioTest():
	25	"""Base for all fio tests."""
	26
	27	def __init__(self, exe_path, parameters, success):
	28	self.exe_path = exe_path
	29	self.parameters = parameters
	30	self.success = success
	31	self.output = {}
	32	self.artifact_root = None
	33	self.testnum = None
	34	self.test_dir = None
	35	self.passed = True
	36	self.failure_reason = ''
	37	self.command_file = None
	38	self.stdout_file = None
	39	self.stderr_file = None
	40	self.exitcode_file = None
	41
	42	def setup(self, artifact_root, testnum):
	43	"""Setup instance variables for test."""
	44
	45	self.artifact_root = artifact_root
	46	self.testnum = testnum
	47	self.test_dir = os.path.join(artifact_root, f"{testnum:04d}")
	48	if not os.path.exists(self.test_dir):
	49	os.mkdir(self.test_dir)
	50
	51	self.command_file = os.path.join( self.test_dir,
	52	f"{os.path.basename(self.exe_path)}.command")
	53	self.stdout_file = os.path.join( self.test_dir,
	54	f"{os.path.basename(self.exe_path)}.stdout")
	55	self.stderr_file = os.path.join( self.test_dir,
	56	f"{os.path.basename(self.exe_path)}.stderr")
	57	self.exitcode_file = os.path.join( self.test_dir,
	58	f"{os.path.basename(self.exe_path)}.exitcode")
	59
	60	def run(self):
	61	"""Run the test."""
	62
	63	raise NotImplementedError()
	64
	65	def check_result(self):
	66	"""Check test results."""
	67
	68	raise NotImplementedError()
	69
	70
	71	class FioExeTest(FioTest):
	72	"""Test consists of an executable binary or script"""
	73
	74	def __init__(self, exe_path, parameters, success):
	75	"""Construct a FioExeTest which is a FioTest consisting of an
	76	executable binary or script.
	77
	78	exe_path: location of executable binary or script
	79	parameters: list of parameters for executable
	80	success: Definition of test success
	81	"""
	82
	83	FioTest.__init__(self, exe_path, parameters, success)
	84
	85	def run(self):
86	"""Execute the binary or script described by this instance."""
87
88	command = [self.exe_path] + self.parameters
89	command_file = open(self.command_file, "w+",
90	encoding=locale.getpreferredencoding())
91	command_file.write(f"{command}\n")
92	command_file.close()
93
94	stdout_file = open(self.stdout_file, "w+",
95	encoding=locale.getpreferredencoding())
96	stderr_file = open(self.stderr_file, "w+",
97	encoding=locale.getpreferredencoding())
98	exitcode_file = open(self.exitcode_file, "w+",
99	encoding=locale.getpreferredencoding())
100	try:
101	proc = None
102	# Avoid using subprocess.run() here because when a timeout occurs,
103	# fio will be stopped with SIGKILL. This does not give fio a
104	# chance to clean up and means that child processes may continue
105	# running and submitting IO.
106	proc = subprocess.Popen(command,
107	stdout=stdout_file,
108	stderr=stderr_file,
109	cwd=self.test_dir,
110	universal_newlines=True)
111	proc.communicate(timeout=self.success['timeout'])
112	exitcode_file.write(f'{proc.returncode}\n')
113	logging.debug("Test %d: return code: %d", self.testnum, proc.returncode)
114	self.output['proc'] = proc
115	except subprocess.TimeoutExpired:
116	proc.terminate()
117	proc.communicate()
118	assert proc.poll()
119	self.output['failure'] = 'timeout'
120	except Exception:
121	if proc:
122	if not proc.poll():
123	proc.terminate()
124	proc.communicate()
125	self.output['failure'] = 'exception'
126	self.output['exc_info'] = sys.exc_info()
127	finally:
128	stdout_file.close()
129	stderr_file.close()
130	exitcode_file.close()
131
132	def check_result(self):
133	"""Check results of test run."""
134
135	if 'proc' not in self.output:
136	if self.output['failure'] == 'timeout':
137	self.failure_reason = f"{self.failure_reason} timeout,"
138	else:
139	assert self.output['failure'] == 'exception'
140	self.failure_reason = '{0} exception: {1}, {2}'.format(
141	self.failure_reason, self.output['exc_info'][0],
142	self.output['exc_info'][1])
143
144	self.passed = False
145	return
146
147	if 'zero_return' in self.success:
148	if self.success['zero_return']:
149	if self.output['proc'].returncode != 0:
150	self.passed = False
151	self.failure_reason = f"{self.failure_reason} non-zero return code,"
152	else:
153	if self.output['proc'].returncode == 0:
154	self.failure_reason = f"{self.failure_reason} zero return code,"
155	self.passed = False
156
157	stderr_size = os.path.getsize(self.stderr_file)
158	if 'stderr_empty' in self.success:
159	if self.success['stderr_empty']:
160	if stderr_size != 0:
161	self.failure_reason = f"{self.failure_reason} stderr not empty,"
162	self.passed = False
163	else:
164	if stderr_size == 0:
165	self.failure_reason = f"{self.failure_reason} stderr empty,"
166	self.passed = False
167
168
0dc6e911	169	class FioJobFileTest(FioExeTest):
fb551941 VF	170	"""Test consists of a fio job"""
	171
	172	def __init__(self, fio_path, fio_job, success, fio_pre_job=None,
	173	fio_pre_success=None, output_format="normal"):
0dc6e911	174	"""Construct a FioJobFileTest which is a FioExeTest consisting of a
fb551941 VF	175	single fio job file with an optional setup step.
	176
	177	fio_path: location of fio executable
	178	fio_job: location of fio job file
	179	success: Definition of test success
	180	fio_pre_job: fio job for preconditioning
	181	fio_pre_success: Definition of test success for fio precon job
	182	output_format: normal (default), json, jsonplus, or terse
	183	"""
	184
	185	self.fio_job = fio_job
	186	self.fio_pre_job = fio_pre_job
	187	self.fio_pre_success = fio_pre_success if fio_pre_success else success
	188	self.output_format = output_format
	189	self.precon_failed = False
	190	self.json_data = None
	191	self.fio_output = f"{os.path.basename(self.fio_job)}.output"
	192	self.fio_args = [
	193	"--max-jobs=16",
	194	f"--output-format={self.output_format}",
	195	f"--output={self.fio_output}",
	196	self.fio_job,
	197	]
	198	FioExeTest.__init__(self, fio_path, self.fio_args, success)
	199
	200	def setup(self, artifact_root, testnum):
	201	"""Setup instance variables for fio job test."""
	202
	203	super().setup(artifact_root, testnum)
	204
	205	self.command_file = os.path.join(self.test_dir,
	206	f"{os.path.basename(self.fio_job)}.command")
	207	self.stdout_file = os.path.join(self.test_dir,
	208	f"{os.path.basename(self.fio_job)}.stdout")
	209	self.stderr_file = os.path.join(self.test_dir,
	210	f"{os.path.basename(self.fio_job)}.stderr")
	211	self.exitcode_file = os.path.join(self.test_dir,
	212	f"{os.path.basename(self.fio_job)}.exitcode")
	213
	214	def run_pre_job(self):
	215	"""Run fio job precondition step."""
	216
0dc6e911	217	precon = FioJobFileTest(self.exe_path, self.fio_pre_job,
fb551941 VF	218	self.fio_pre_success,
	219	output_format=self.output_format)
	220	precon.setup(self.artifact_root, self.testnum)
	221	precon.run()
	222	precon.check_result()
	223	self.precon_failed = not precon.passed
	224	self.failure_reason = precon.failure_reason
	225
	226	def run(self):
	227	"""Run fio job test."""
	228
	229	if self.fio_pre_job:
	230	self.run_pre_job()
	231
	232	if not self.precon_failed:
	233	super().run()
	234	else:
	235	logging.debug("Test %d: precondition step failed", self.testnum)
	236
fb551941 VF	237	def get_file_fail(self, filename):
	238	"""Safely read a file and fail the test upon error."""
	239	file_data = None
	240
	241	try:
	242	with open(filename, "r", encoding=locale.getpreferredencoding()) as output_file:
	243	file_data = output_file.read()
	244	except OSError:
	245	self.failure_reason += f" unable to read file {filename}"
	246	self.passed = False
	247
	248	return file_data
	249
	250	def check_result(self):
	251	"""Check fio job results."""
	252
	253	if self.precon_failed:
	254	self.passed = False
	255	self.failure_reason = f"{self.failure_reason} precondition step failed,"
	256	return
	257
	258	super().check_result()
	259
	260	if not self.passed:
	261	return
	262
	263	if 'json' not in self.output_format:
	264	return
	265
	266	file_data = self.get_file_fail(os.path.join(self.test_dir, self.fio_output))
	267	if not file_data:
	268	return
	269
	270	#
	271	# Sometimes fio informational messages are included at the top of the
	272	# JSON output, especially under Windows. Try to decode output as JSON
	273	# data, skipping everything until the first {
	274	#
	275	lines = file_data.splitlines()
	276	file_data = '\n'.join(lines[lines.index("{"):])
	277	try:
	278	self.json_data = json.loads(file_data)
	279	except json.JSONDecodeError:
	280	self.failure_reason = f"{self.failure_reason} unable to decode JSON data,"
	281	self.passed = False
	282
	283
	284	def run_fio_tests(test_list, test_env, args):
	285	"""
	286	Run tests as specified in test_list.
	287	"""
	288
	289	passed = 0
	290	failed = 0
	291	skipped = 0
	292
	293	for config in test_list:
	294	if (args.skip and config['test_id'] in args.skip) or \
	295	(args.run_only and config['test_id'] not in args.run_only):
	296	skipped = skipped + 1
	297	print(f"Test {config['test_id']} SKIPPED (User request)")
	298	continue
	299
0dc6e911	300	if issubclass(config['test_class'], FioJobFileTest):
fb551941 VF	301	if config['pre_job']:
	302	fio_pre_job = os.path.join(test_env['fio_root'], 't', 'jobs',
	303	config['pre_job'])
	304	else:
	305	fio_pre_job = None
	306	if config['pre_success']:
	307	fio_pre_success = config['pre_success']
	308	else:
	309	fio_pre_success = None
	310	if 'output_format' in config:
	311	output_format = config['output_format']
	312	else:
	313	output_format = 'normal'
	314	test = config['test_class'](
	315	test_env['fio_path'],
	316	os.path.join(test_env['fio_root'], 't', 'jobs', config['job']),
	317	config['success'],
	318	fio_pre_job=fio_pre_job,
	319	fio_pre_success=fio_pre_success,
	320	output_format=output_format)
	321	desc = config['job']
	322	elif issubclass(config['test_class'], FioExeTest):
	323	exe_path = os.path.join(test_env['fio_root'], config['exe'])
	324	if config['parameters']:
	325	parameters = [p.format(fio_path=test_env['fio_path'], nvmecdev=args.nvmecdev)
	326	for p in config['parameters']]
	327	else:
	328	parameters = []
	329	if Path(exe_path).suffix == '.py' and platform.system() == "Windows":
	330	parameters.insert(0, exe_path)
	331	exe_path = "python.exe"
	332	if config['test_id'] in test_env['pass_through']:
	333	parameters += test_env['pass_through'][config['test_id']].split()
	334	test = config['test_class'](exe_path, parameters,
	335	config['success'])
	336	desc = config['exe']
	337	else:
	338	print(f"Test {config['test_id']} FAILED: unable to process test config")
	339	failed = failed + 1
	340	continue
	341
	342	if not args.skip_req:
	343	reqs_met = True
	344	for req in config['requirements']:
	345	reqs_met, reason = req()
	346	logging.debug("Test %d: Requirement '%s' met? %s", config['test_id'], reason,
	347	reqs_met)
	348	if not reqs_met:
	349	break
	350	if not reqs_met:
	351	print(f"Test {config['test_id']} SKIPPED ({reason}) {desc}")
	352	skipped = skipped + 1
	353	continue
	354
	355	try:
	356	test.setup(test_env['artifact_root'], config['test_id'])
	357	test.run()
	358	test.check_result()
	359	except KeyboardInterrupt:
	360	break
	361	except Exception as e:
	362	test.passed = False
	363	test.failure_reason += str(e)
	364	logging.debug("Test %d exception:\n%s\n", config['test_id'], traceback.format_exc())
365	if test.passed:
366	result = "PASSED"
367	passed = passed + 1
368	else:
369	result = f"FAILED: {test.failure_reason}"
370	failed = failed + 1
ecd6d30f	371	contents, _ = get_file(test.stderr_file)
fb551941	372	logging.debug("Test %d: stderr:\n%s", config['test_id'], contents)
ecd6d30f	373	contents, _ = get_file(test.stdout_file)
fb551941 VF	374	logging.debug("Test %d: stdout:\n%s", config['test_id'], contents)
	375	print(f"Test {config['test_id']} {result} {desc}")
	376
	377	print(f"{passed} test(s) passed, {failed} failed, {skipped} skipped")
	378
	379	return passed, failed, skipped